2023-09-23T17:52:33.642 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-09-23T17:52:33.643 DEBUG:teuthology.run:Teuthology command: teuthology --description upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} --archive /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401774 --name teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi --owner scheduled_teuthology@teuthology --verbose -- /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401774/orig.config.yaml 2023-09-23T17:52:33.799 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-23T17:52:33.907 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401774 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' first_in_suite: false job_id: '7401774' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic import/export cli tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: ubuntu os_version: '20.04' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 kernel: hwe: true workunit: branch: reef sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 1860 sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 targets: smithi098.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM= smithi173.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM= tasks: - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-09-23_14:23:10 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-09-23T17:52:33.908 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa; will attempt to use it 2023-09-23T17:52:33.909 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks 2023-09-23T17:52:33.909 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-09-23T17:52:33.911 INFO:teuthology.task.internal:Checking packages... 2023-09-23T17:52:33.932 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash 'bb0daeff97f020f686596fca6eba20c310e26fb7' 2023-09-23T17:52:33.933 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T17:52:33.933 INFO:teuthology.packaging:ref: None 2023-09-23T17:52:33.933 INFO:teuthology.packaging:tag: None 2023-09-23T17:52:33.933 INFO:teuthology.packaging:branch: reef 2023-09-23T17:52:33.933 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T17:52:33.933 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=reef 2023-09-23T17:52:34.179 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-460-gbb0daeff-1focal 2023-09-23T17:52:34.180 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-09-23T17:52:34.187 INFO:teuthology.task.internal:no buildpackages task found 2023-09-23T17:52:34.187 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-09-23T17:52:34.254 INFO:teuthology.task.internal:Saving configuration 2023-09-23T17:52:34.276 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-09-23T17:52:34.282 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T17:52:34.305 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi098.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401774', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 17:46:41.669238', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGETOy3xONfbx7kUFCNiUtRLNfkcrcEsWzPdq6Np/6Bw'} 2023-09-23T17:52:34.331 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi173.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401774', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 17:46:41.671199', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGETOy3xONfbx7kUFCNiUtRLNfkcrcEsWzPdq6Np/6Bw'} 2023-09-23T17:52:34.331 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-09-23T17:52:34.337 INFO:teuthology.task.internal:roles: ubuntu@smithi098.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-09-23T17:52:34.338 INFO:teuthology.task.internal:roles: ubuntu@smithi173.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-09-23T17:52:34.338 INFO:teuthology.run_tasks:Running task console_log... 2023-09-23T17:52:34.426 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f212e033160>, signals=[15]) 2023-09-23T17:52:34.426 INFO:teuthology.run_tasks:Running task internal.connect... 2023-09-23T17:52:34.432 INFO:teuthology.task.internal:Opening connections... 2023-09-23T17:52:34.432 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi098.front.sepia.ceph.com 2023-09-23T17:52:34.434 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi098.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T17:52:34.482 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi173.front.sepia.ceph.com 2023-09-23T17:52:34.482 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi173.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T17:52:34.527 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-09-23T17:52:34.533 DEBUG:teuthology.orchestra.run.smithi098:> uname -m 2023-09-23T17:52:34.562 INFO:teuthology.orchestra.run.smithi098.stdout:x86_64 2023-09-23T17:52:34.562 DEBUG:teuthology.orchestra.run.smithi098:> cat /etc/os-release 2023-09-23T17:52:34.609 INFO:teuthology.orchestra.run.smithi098.stdout:NAME="Ubuntu" 2023-09-23T17:52:34.609 INFO:teuthology.orchestra.run.smithi098.stdout:VERSION="20.04.4 LTS (Focal Fossa)" 2023-09-23T17:52:34.609 INFO:teuthology.orchestra.run.smithi098.stdout:ID=ubuntu 2023-09-23T17:52:34.609 INFO:teuthology.orchestra.run.smithi098.stdout:ID_LIKE=debian 2023-09-23T17:52:34.609 INFO:teuthology.orchestra.run.smithi098.stdout:PRETTY_NAME="Ubuntu 20.04.4 LTS" 2023-09-23T17:52:34.610 INFO:teuthology.orchestra.run.smithi098.stdout:VERSION_ID="20.04" 2023-09-23T17:52:34.610 INFO:teuthology.orchestra.run.smithi098.stdout:HOME_URL="https://www.ubuntu.com/" 2023-09-23T17:52:34.610 INFO:teuthology.orchestra.run.smithi098.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2023-09-23T17:52:34.610 INFO:teuthology.orchestra.run.smithi098.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2023-09-23T17:52:34.610 INFO:teuthology.orchestra.run.smithi098.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2023-09-23T17:52:34.610 INFO:teuthology.orchestra.run.smithi098.stdout:VERSION_CODENAME=focal 2023-09-23T17:52:34.610 INFO:teuthology.orchestra.run.smithi098.stdout:UBUNTU_CODENAME=focal 2023-09-23T17:52:34.611 INFO:teuthology.lock.ops:Updating smithi098.front.sepia.ceph.com on lock server 2023-09-23T17:52:34.634 DEBUG:teuthology.orchestra.run.smithi173:> uname -m 2023-09-23T17:52:34.641 INFO:teuthology.orchestra.run.smithi173.stdout:x86_64 2023-09-23T17:52:34.641 DEBUG:teuthology.orchestra.run.smithi173:> cat /etc/os-release 2023-09-23T17:52:34.647 INFO:teuthology.orchestra.run.smithi173.stdout:NAME="Ubuntu" 2023-09-23T17:52:34.647 INFO:teuthology.orchestra.run.smithi173.stdout:VERSION="20.04.4 LTS (Focal Fossa)" 2023-09-23T17:52:34.648 INFO:teuthology.orchestra.run.smithi173.stdout:ID=ubuntu 2023-09-23T17:52:34.648 INFO:teuthology.orchestra.run.smithi173.stdout:ID_LIKE=debian 2023-09-23T17:52:34.648 INFO:teuthology.orchestra.run.smithi173.stdout:PRETTY_NAME="Ubuntu 20.04.4 LTS" 2023-09-23T17:52:34.648 INFO:teuthology.orchestra.run.smithi173.stdout:VERSION_ID="20.04" 2023-09-23T17:52:34.648 INFO:teuthology.orchestra.run.smithi173.stdout:HOME_URL="https://www.ubuntu.com/" 2023-09-23T17:52:34.648 INFO:teuthology.orchestra.run.smithi173.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2023-09-23T17:52:34.648 INFO:teuthology.orchestra.run.smithi173.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2023-09-23T17:52:34.648 INFO:teuthology.orchestra.run.smithi173.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2023-09-23T17:52:34.648 INFO:teuthology.orchestra.run.smithi173.stdout:VERSION_CODENAME=focal 2023-09-23T17:52:34.648 INFO:teuthology.orchestra.run.smithi173.stdout:UBUNTU_CODENAME=focal 2023-09-23T17:52:34.649 INFO:teuthology.lock.ops:Updating smithi173.front.sepia.ceph.com on lock server 2023-09-23T17:52:34.668 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-09-23T17:52:34.677 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-09-23T17:52:34.696 INFO:teuthology.task.internal:Checking for old test directory... 2023-09-23T17:52:34.696 DEBUG:teuthology.orchestra.run.smithi098:> test '!' -e /home/ubuntu/cephtest 2023-09-23T17:52:34.698 DEBUG:teuthology.orchestra.run.smithi173:> test '!' -e /home/ubuntu/cephtest 2023-09-23T17:52:34.704 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-09-23T17:52:34.721 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-09-23T17:52:34.721 DEBUG:teuthology.orchestra.run.smithi098:> test -z $(ls -A /var/lib/ceph) 2023-09-23T17:52:34.745 DEBUG:teuthology.orchestra.run.smithi173:> test -z $(ls -A /var/lib/ceph) 2023-09-23T17:52:34.763 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-09-23T17:52:34.815 INFO:teuthology.run_tasks:Running task kernel... 2023-09-23T17:52:34.828 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-09-23T17:52:34.828 INFO:teuthology.task.kernel:normalize config orig: {'hwe': True} 2023-09-23T17:52:34.829 DEBUG:teuthology.task.kernel:normalized overrides {'mon.a': {'hwe': True}, 'mon.c': {'hwe': True}, 'mgr.y': {'hwe': True}, 'osd.0': {'hwe': True}, 'osd.1': {'hwe': True}, 'osd.2': {'hwe': True}, 'osd.3': {'hwe': True}, 'client.0': {'hwe': True}, 'node-exporter.a': {'hwe': True}, 'alertmanager.a': {'hwe': True}, 'mon.b': {'hwe': True}, 'mgr.x': {'hwe': True}, 'osd.4': {'hwe': True}, 'osd.5': {'hwe': True}, 'osd.6': {'hwe': True}, 'osd.7': {'hwe': True}, 'client.1': {'hwe': True}, 'prometheus.a': {'hwe': True}, 'grafana.a': {'hwe': True}, 'node-exporter.b': {'hwe': True}} 2023-09-23T17:52:34.829 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro', 'hwe': True}, 'mon.b': {'kdb': True, 'sha1': 'distro', 'hwe': True}}, timeout 300 2023-09-23T17:52:34.829 DEBUG:teuthology.orchestra.run.smithi098:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-23T17:52:34.829 DEBUG:teuthology.orchestra.run.smithi173:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-23T17:52:34.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T17:52:34.835 DEBUG:teuthology.orchestra.run.smithi098:> uname -r 2023-09-23T17:52:34.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T17:52:34.836 DEBUG:teuthology.orchestra.run.smithi173:> uname -r 2023-09-23T17:52:34.842 INFO:teuthology.orchestra.run.smithi173.stdout:5.4.0-124-generic 2023-09-23T17:52:34.842 INFO:teuthology.task.kernel:Running kernel on smithi173: 5.4.0-124-generic 2023-09-23T17:52:34.842 DEBUG:teuthology.orchestra.run.smithi173:> sudo apt-get clean 2023-09-23T17:52:34.881 INFO:teuthology.orchestra.run.smithi098.stdout:5.4.0-124-generic 2023-09-23T17:52:34.881 INFO:teuthology.task.kernel:Running kernel on smithi098: 5.4.0-124-generic 2023-09-23T17:52:34.881 DEBUG:teuthology.orchestra.run.smithi098:> sudo apt-get clean 2023-09-23T17:52:34.926 DEBUG:teuthology.orchestra.run.smithi173:> sudo apt-get update 2023-09-23T17:52:34.998 DEBUG:teuthology.orchestra.run.smithi098:> sudo apt-get update 2023-09-23T17:52:35.116 INFO:teuthology.orchestra.run.smithi173.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-09-23T17:52:35.122 INFO:teuthology.orchestra.run.smithi173.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] 2023-09-23T17:52:35.180 INFO:teuthology.orchestra.run.smithi173.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2023-09-23T17:52:35.222 INFO:teuthology.orchestra.run.smithi173.stdout:Get:4 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2023-09-23T17:52:35.265 INFO:teuthology.orchestra.run.smithi098.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-09-23T17:52:35.265 INFO:teuthology.orchestra.run.smithi098.stdout:Get:2 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2023-09-23T17:52:35.333 INFO:teuthology.orchestra.run.smithi098.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] 2023-09-23T17:52:35.484 INFO:teuthology.orchestra.run.smithi173.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [2,852 kB] 2023-09-23T17:52:35.595 INFO:teuthology.orchestra.run.smithi173.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main i386 Packages [884 kB] 2023-09-23T17:52:35.610 INFO:teuthology.orchestra.run.smithi173.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main Translation-en [468 kB] 2023-09-23T17:52:35.614 INFO:teuthology.orchestra.run.smithi173.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 c-n-f Metadata [17.1 kB] 2023-09-23T17:52:35.614 INFO:teuthology.orchestra.run.smithi173.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 Packages [2,327 kB] 2023-09-23T17:52:35.631 INFO:teuthology.orchestra.run.smithi173.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/restricted i386 Packages [33.7 kB] 2023-09-23T17:52:35.631 INFO:teuthology.orchestra.run.smithi173.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/restricted Translation-en [325 kB] 2023-09-23T17:52:35.635 INFO:teuthology.orchestra.run.smithi173.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 c-n-f Metadata [576 B] 2023-09-23T17:52:35.635 INFO:teuthology.orchestra.run.smithi173.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal-updates/universe i386 Packages [748 kB] 2023-09-23T17:52:35.640 INFO:teuthology.orchestra.run.smithi173.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [1,117 kB] 2023-09-23T17:52:35.649 INFO:teuthology.orchestra.run.smithi173.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal-updates/universe Translation-en [267 kB] 2023-09-23T17:52:35.651 INFO:teuthology.orchestra.run.smithi173.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 c-n-f Metadata [25.6 kB] 2023-09-23T17:52:35.652 INFO:teuthology.orchestra.run.smithi173.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal-updates/multiverse i386 Packages [8,436 B] 2023-09-23T17:52:35.652 INFO:teuthology.orchestra.run.smithi173.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 Packages [25.8 kB] 2023-09-23T17:52:35.660 INFO:teuthology.orchestra.run.smithi173.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal-updates/multiverse Translation-en [7,484 B] 2023-09-23T17:52:35.660 INFO:teuthology.orchestra.run.smithi173.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 c-n-f Metadata [620 B] 2023-09-23T17:52:35.661 INFO:teuthology.orchestra.run.smithi173.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal-backports/main i386 Packages [36.1 kB] 2023-09-23T17:52:35.661 INFO:teuthology.orchestra.run.smithi173.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 Packages [45.7 kB] 2023-09-23T17:52:35.662 INFO:teuthology.orchestra.run.smithi173.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 c-n-f Metadata [1,420 B] 2023-09-23T17:52:35.675 INFO:teuthology.orchestra.run.smithi098.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2023-09-23T17:52:35.677 INFO:teuthology.orchestra.run.smithi173.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 Packages [25.0 kB] 2023-09-23T17:52:35.677 INFO:teuthology.orchestra.run.smithi173.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal-backports/universe i386 Packages [13.8 kB] 2023-09-23T17:52:35.677 INFO:teuthology.orchestra.run.smithi173.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 c-n-f Metadata [880 B] 2023-09-23T17:52:35.756 INFO:teuthology.orchestra.run.smithi098.stdout:Get:5 http://security.ubuntu.com/ubuntu focal-security/main i386 Packages [652 kB] 2023-09-23T17:52:35.789 INFO:teuthology.orchestra.run.smithi173.stdout:Get:27 http://security.ubuntu.com/ubuntu focal-security/main i386 Packages [652 kB] 2023-09-23T17:52:35.898 INFO:teuthology.orchestra.run.smithi098.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main i386 Packages [884 kB] 2023-09-23T17:52:36.110 INFO:teuthology.orchestra.run.smithi098.stdout:Get:7 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [2,464 kB] 2023-09-23T17:52:36.113 INFO:teuthology.orchestra.run.smithi173.stdout:Get:28 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [2,464 kB] 2023-09-23T17:52:36.227 INFO:teuthology.orchestra.run.smithi098.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [2,852 kB] 2023-09-23T17:52:36.298 INFO:teuthology.orchestra.run.smithi098.stdout:Get:9 http://security.ubuntu.com/ubuntu focal-security/main Translation-en [385 kB] 2023-09-23T17:52:36.301 INFO:teuthology.orchestra.run.smithi173.stdout:Get:29 http://security.ubuntu.com/ubuntu focal-security/main Translation-en [385 kB] 2023-09-23T17:52:36.311 INFO:teuthology.orchestra.run.smithi098.stdout:Get:10 http://security.ubuntu.com/ubuntu focal-security/main amd64 c-n-f Metadata [13.2 kB] 2023-09-23T17:52:36.312 INFO:teuthology.orchestra.run.smithi098.stdout:Get:11 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 Packages [2,210 kB] 2023-09-23T17:52:36.315 INFO:teuthology.orchestra.run.smithi173.stdout:Get:30 http://security.ubuntu.com/ubuntu focal-security/main amd64 c-n-f Metadata [13.2 kB] 2023-09-23T17:52:36.315 INFO:teuthology.orchestra.run.smithi173.stdout:Get:31 http://security.ubuntu.com/ubuntu focal-security/restricted i386 Packages [32.3 kB] 2023-09-23T17:52:36.316 INFO:teuthology.orchestra.run.smithi173.stdout:Get:32 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 Packages [2,210 kB] 2023-09-23T17:52:36.379 INFO:teuthology.orchestra.run.smithi098.stdout:Get:12 http://security.ubuntu.com/ubuntu focal-security/restricted i386 Packages [32.3 kB] 2023-09-23T17:52:36.380 INFO:teuthology.orchestra.run.smithi098.stdout:Get:13 http://security.ubuntu.com/ubuntu focal-security/restricted Translation-en [309 kB] 2023-09-23T17:52:36.383 INFO:teuthology.orchestra.run.smithi173.stdout:Get:33 http://security.ubuntu.com/ubuntu focal-security/restricted Translation-en [309 kB] 2023-09-23T17:52:36.386 INFO:teuthology.orchestra.run.smithi098.stdout:Get:14 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 c-n-f Metadata [580 B] 2023-09-23T17:52:36.386 INFO:teuthology.orchestra.run.smithi098.stdout:Get:15 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [886 kB] 2023-09-23T17:52:36.389 INFO:teuthology.orchestra.run.smithi173.stdout:Get:34 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 c-n-f Metadata [580 B] 2023-09-23T17:52:36.390 INFO:teuthology.orchestra.run.smithi173.stdout:Get:35 http://security.ubuntu.com/ubuntu focal-security/universe i386 Packages [615 kB] 2023-09-23T17:52:36.401 INFO:teuthology.orchestra.run.smithi173.stdout:Get:36 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [886 kB] 2023-09-23T17:52:36.401 INFO:teuthology.orchestra.run.smithi098.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal-updates/main Translation-en [468 kB] 2023-09-23T17:52:36.405 INFO:teuthology.orchestra.run.smithi098.stdout:Get:17 http://security.ubuntu.com/ubuntu focal-security/universe i386 Packages [615 kB] 2023-09-23T17:52:36.414 INFO:teuthology.orchestra.run.smithi098.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 c-n-f Metadata [17.1 kB] 2023-09-23T17:52:36.414 INFO:teuthology.orchestra.run.smithi098.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 Packages [2,327 kB] 2023-09-23T17:52:36.483 INFO:teuthology.orchestra.run.smithi098.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal-updates/restricted i386 Packages [33.7 kB] 2023-09-23T17:52:36.484 INFO:teuthology.orchestra.run.smithi098.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal-updates/restricted Translation-en [325 kB] 2023-09-23T17:52:36.489 INFO:teuthology.orchestra.run.smithi098.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 c-n-f Metadata [576 B] 2023-09-23T17:52:36.596 INFO:teuthology.orchestra.run.smithi098.stdout:Get:23 http://security.ubuntu.com/ubuntu focal-security/universe Translation-en [185 kB] 2023-09-23T17:52:36.601 INFO:teuthology.orchestra.run.smithi173.stdout:Get:37 http://security.ubuntu.com/ubuntu focal-security/universe Translation-en [185 kB] 2023-09-23T17:52:36.653 INFO:teuthology.orchestra.run.smithi098.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal-updates/universe i386 Packages [748 kB] 2023-09-23T17:52:37.012 INFO:teuthology.orchestra.run.smithi098.stdout:Get:25 http://security.ubuntu.com/ubuntu focal-security/universe amd64 c-n-f Metadata [19.1 kB] 2023-09-23T17:52:37.018 INFO:teuthology.orchestra.run.smithi098.stdout:Get:26 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 Packages [23.6 kB] 2023-09-23T17:52:37.019 INFO:teuthology.orchestra.run.smithi173.stdout:Get:38 http://security.ubuntu.com/ubuntu focal-security/universe amd64 c-n-f Metadata [19.1 kB] 2023-09-23T17:52:37.025 INFO:teuthology.orchestra.run.smithi173.stdout:Get:39 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 Packages [23.6 kB] 2023-09-23T17:52:37.026 INFO:teuthology.orchestra.run.smithi098.stdout:Get:27 http://security.ubuntu.com/ubuntu focal-security/multiverse i386 Packages [7,196 B] 2023-09-23T17:52:37.029 INFO:teuthology.orchestra.run.smithi098.stdout:Get:28 http://security.ubuntu.com/ubuntu focal-security/multiverse Translation-en [5,504 B] 2023-09-23T17:52:37.031 INFO:teuthology.orchestra.run.smithi098.stdout:Get:29 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 c-n-f Metadata [548 B] 2023-09-23T17:52:37.033 INFO:teuthology.orchestra.run.smithi173.stdout:Get:40 http://security.ubuntu.com/ubuntu focal-security/multiverse i386 Packages [7,196 B] 2023-09-23T17:52:37.035 INFO:teuthology.orchestra.run.smithi173.stdout:Get:41 http://security.ubuntu.com/ubuntu focal-security/multiverse Translation-en [5,504 B] 2023-09-23T17:52:37.038 INFO:teuthology.orchestra.run.smithi173.stdout:Get:42 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 c-n-f Metadata [548 B] 2023-09-23T17:52:37.238 INFO:teuthology.orchestra.run.smithi098.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [1,117 kB] 2023-09-23T17:52:37.347 INFO:teuthology.orchestra.run.smithi098.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal-updates/universe Translation-en [267 kB] 2023-09-23T17:52:37.368 INFO:teuthology.orchestra.run.smithi098.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 c-n-f Metadata [25.6 kB] 2023-09-23T17:52:37.370 INFO:teuthology.orchestra.run.smithi098.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 Packages [25.8 kB] 2023-09-23T17:52:37.371 INFO:teuthology.orchestra.run.smithi098.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal-updates/multiverse i386 Packages [8,436 B] 2023-09-23T17:52:37.372 INFO:teuthology.orchestra.run.smithi098.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal-updates/multiverse Translation-en [7,484 B] 2023-09-23T17:52:37.372 INFO:teuthology.orchestra.run.smithi098.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 c-n-f Metadata [620 B] 2023-09-23T17:52:37.372 INFO:teuthology.orchestra.run.smithi098.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal-backports/main i386 Packages [36.1 kB] 2023-09-23T17:52:37.375 INFO:teuthology.orchestra.run.smithi098.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 Packages [45.7 kB] 2023-09-23T17:52:37.378 INFO:teuthology.orchestra.run.smithi098.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 c-n-f Metadata [1,420 B] 2023-09-23T17:52:37.542 INFO:teuthology.orchestra.run.smithi098.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 Packages [25.0 kB] 2023-09-23T17:52:37.725 INFO:teuthology.orchestra.run.smithi098.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal-backports/universe i386 Packages [13.8 kB] 2023-09-23T17:52:37.747 INFO:teuthology.orchestra.run.smithi098.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 c-n-f Metadata [880 B] 2023-09-23T17:52:39.513 INFO:teuthology.orchestra.run.smithi173.stdout:Fetched 17.4 MB in 3s (6,872 kB/s) 2023-09-23T17:52:39.985 INFO:teuthology.orchestra.run.smithi098.stdout:Fetched 17.4 MB in 3s (5,764 kB/s) 2023-09-23T17:52:40.977 INFO:teuthology.orchestra.run.smithi173.stdout:Reading package lists... 2023-09-23T17:52:41.002 DEBUG:teuthology.orchestra.run.smithi173:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-09-23T17:52:41.068 INFO:teuthology.orchestra.run.smithi173.stdout:Reading package lists... 2023-09-23T17:52:41.253 INFO:teuthology.orchestra.run.smithi173.stdout:Building dependency tree... 2023-09-23T17:52:41.254 INFO:teuthology.orchestra.run.smithi173.stdout:Reading state information... 2023-09-23T17:52:41.309 INFO:teuthology.orchestra.run.smithi098.stdout:Reading package lists... 2023-09-23T17:52:41.333 DEBUG:teuthology.orchestra.run.smithi098:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-09-23T17:52:41.400 INFO:teuthology.orchestra.run.smithi098.stdout:Reading package lists... 2023-09-23T17:52:41.418 INFO:teuthology.orchestra.run.smithi173.stdout:The following additional packages will be installed: 2023-09-23T17:52:41.418 INFO:teuthology.orchestra.run.smithi173.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-09-23T17:52:41.419 INFO:teuthology.orchestra.run.smithi173.stdout: libusbmuxd6 linux-image-5.15.0-84-generic linux-modules-5.15.0-84-generic 2023-09-23T17:52:41.419 INFO:teuthology.orchestra.run.smithi173.stdout: linux-modules-extra-5.15.0-84-generic thermald upower usbmuxd 2023-09-23T17:52:41.420 INFO:teuthology.orchestra.run.smithi173.stdout:Suggested packages: 2023-09-23T17:52:41.420 INFO:teuthology.orchestra.run.smithi173.stdout: libusbmuxd-tools fdutils linux-doc | linux-hwe-5.15-source-5.15.0 2023-09-23T17:52:41.420 INFO:teuthology.orchestra.run.smithi173.stdout: linux-hwe-5.15-tools linux-headers-5.15.0-84-generic 2023-09-23T17:52:41.473 INFO:teuthology.orchestra.run.smithi173.stdout:The following NEW packages will be installed: 2023-09-23T17:52:41.473 INFO:teuthology.orchestra.run.smithi173.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-09-23T17:52:41.473 INFO:teuthology.orchestra.run.smithi173.stdout: libusbmuxd6 linux-image-5.15.0-84-generic linux-image-generic-hwe-20.04 2023-09-23T17:52:41.474 INFO:teuthology.orchestra.run.smithi173.stdout: linux-modules-5.15.0-84-generic linux-modules-extra-5.15.0-84-generic 2023-09-23T17:52:41.474 INFO:teuthology.orchestra.run.smithi173.stdout: thermald upower usbmuxd 2023-09-23T17:52:41.602 INFO:teuthology.orchestra.run.smithi098.stdout:Building dependency tree... 2023-09-23T17:52:41.602 INFO:teuthology.orchestra.run.smithi098.stdout:Reading state information... 2023-09-23T17:52:41.657 INFO:teuthology.orchestra.run.smithi173.stdout:0 upgraded, 13 newly installed, 0 to remove and 266 not upgraded. 2023-09-23T17:52:41.657 INFO:teuthology.orchestra.run.smithi173.stdout:Need to get 90.9 MB of archives. 2023-09-23T17:52:41.657 INFO:teuthology.orchestra.run.smithi173.stdout:After this operation, 497 MB of additional disk space will be used. 2023-09-23T17:52:41.658 INFO:teuthology.orchestra.run.smithi173.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 libdbus-glib-1-2 amd64 0.110-5fakssync1 [59.1 kB] 2023-09-23T17:52:41.771 INFO:teuthology.orchestra.run.smithi098.stdout:The following additional packages will be installed: 2023-09-23T17:52:41.771 INFO:teuthology.orchestra.run.smithi098.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-09-23T17:52:41.771 INFO:teuthology.orchestra.run.smithi098.stdout: libusbmuxd6 linux-image-5.15.0-84-generic linux-modules-5.15.0-84-generic 2023-09-23T17:52:41.771 INFO:teuthology.orchestra.run.smithi098.stdout: linux-modules-extra-5.15.0-84-generic thermald upower usbmuxd 2023-09-23T17:52:41.772 INFO:teuthology.orchestra.run.smithi098.stdout:Suggested packages: 2023-09-23T17:52:41.772 INFO:teuthology.orchestra.run.smithi098.stdout: libusbmuxd-tools fdutils linux-doc | linux-hwe-5.15-source-5.15.0 2023-09-23T17:52:41.772 INFO:teuthology.orchestra.run.smithi098.stdout: linux-hwe-5.15-tools linux-headers-5.15.0-84-generic 2023-09-23T17:52:41.824 INFO:teuthology.orchestra.run.smithi098.stdout:The following NEW packages will be installed: 2023-09-23T17:52:41.825 INFO:teuthology.orchestra.run.smithi098.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-09-23T17:52:41.825 INFO:teuthology.orchestra.run.smithi098.stdout: libusbmuxd6 linux-image-5.15.0-84-generic linux-image-generic-hwe-20.04 2023-09-23T17:52:41.825 INFO:teuthology.orchestra.run.smithi098.stdout: linux-modules-5.15.0-84-generic linux-modules-extra-5.15.0-84-generic 2023-09-23T17:52:41.825 INFO:teuthology.orchestra.run.smithi098.stdout: thermald upower usbmuxd 2023-09-23T17:52:41.930 INFO:teuthology.orchestra.run.smithi173.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 libplist3 amd64 2.1.0-4build2 [31.6 kB] 2023-09-23T17:52:41.959 INFO:teuthology.orchestra.run.smithi173.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 libusbmuxd6 amd64 2.0.1-2 [19.1 kB] 2023-09-23T17:52:41.977 INFO:teuthology.orchestra.run.smithi173.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 libimobiledevice6 amd64 1.2.1~git20191129.9f79242-1build1 [65.2 kB] 2023-09-23T17:52:42.007 INFO:teuthology.orchestra.run.smithi098.stdout:0 upgraded, 13 newly installed, 0 to remove and 266 not upgraded. 2023-09-23T17:52:42.007 INFO:teuthology.orchestra.run.smithi098.stdout:Need to get 90.9 MB of archives. 2023-09-23T17:52:42.007 INFO:teuthology.orchestra.run.smithi098.stdout:After this operation, 497 MB of additional disk space will be used. 2023-09-23T17:52:42.007 INFO:teuthology.orchestra.run.smithi098.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 libdbus-glib-1-2 amd64 0.110-5fakssync1 [59.1 kB] 2023-09-23T17:52:42.051 INFO:teuthology.orchestra.run.smithi173.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 libupower-glib3 amd64 0.99.11-1build2 [43.2 kB] 2023-09-23T17:52:42.087 INFO:teuthology.orchestra.run.smithi173.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-5.15.0-84-generic amd64 5.15.0-84.93~20.04.1 [20.9 MB] 2023-09-23T17:52:42.281 INFO:teuthology.orchestra.run.smithi098.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 libplist3 amd64 2.1.0-4build2 [31.6 kB] 2023-09-23T17:52:42.310 INFO:teuthology.orchestra.run.smithi098.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 libusbmuxd6 amd64 2.0.1-2 [19.1 kB] 2023-09-23T17:52:42.328 INFO:teuthology.orchestra.run.smithi098.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 libimobiledevice6 amd64 1.2.1~git20191129.9f79242-1build1 [65.2 kB] 2023-09-23T17:52:42.400 INFO:teuthology.orchestra.run.smithi098.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 libupower-glib3 amd64 0.99.11-1build2 [43.2 kB] 2023-09-23T17:52:42.436 INFO:teuthology.orchestra.run.smithi098.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-5.15.0-84-generic amd64 5.15.0-84.93~20.04.1 [20.9 MB] 2023-09-23T17:52:42.959 INFO:teuthology.orchestra.run.smithi173.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-5.15.0-84-generic amd64 5.15.0-84.93~20.04.1 [11.4 MB] 2023-09-23T17:52:43.267 INFO:teuthology.orchestra.run.smithi173.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-extra-5.15.0-84-generic amd64 5.15.0-84.93~20.04.1 [58.0 MB] 2023-09-23T17:52:43.400 INFO:teuthology.orchestra.run.smithi098.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-5.15.0-84-generic amd64 5.15.0-84.93~20.04.1 [11.4 MB] 2023-09-23T17:52:43.684 INFO:teuthology.orchestra.run.smithi098.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-extra-5.15.0-84-generic amd64 5.15.0-84.93~20.04.1 [58.0 MB] 2023-09-23T17:52:45.241 INFO:teuthology.orchestra.run.smithi173.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-generic-hwe-20.04 amd64 5.15.0.84.93~20.04.42 [2,692 B] 2023-09-23T17:52:45.248 INFO:teuthology.orchestra.run.smithi173.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libevdev2 amd64 1.9.0+dfsg-1ubuntu0.2 [31.6 kB] 2023-09-23T17:52:45.254 INFO:teuthology.orchestra.run.smithi173.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 thermald amd64 1.9.1-1ubuntu0.6 [233 kB] 2023-09-23T17:52:45.426 INFO:teuthology.orchestra.run.smithi173.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 upower amd64 0.99.11-1build2 [104 kB] 2023-09-23T17:52:45.582 INFO:teuthology.orchestra.run.smithi098.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-generic-hwe-20.04 amd64 5.15.0.84.93~20.04.42 [2,692 B] 2023-09-23T17:52:45.663 INFO:teuthology.orchestra.run.smithi098.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libevdev2 amd64 1.9.0+dfsg-1ubuntu0.2 [31.6 kB] 2023-09-23T17:52:45.664 INFO:teuthology.orchestra.run.smithi098.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 thermald amd64 1.9.1-1ubuntu0.6 [233 kB] 2023-09-23T17:52:45.770 INFO:teuthology.orchestra.run.smithi173.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 usbmuxd amd64 1.1.1~git20191130.9af2b12-1 [38.4 kB] 2023-09-23T17:52:45.833 INFO:teuthology.orchestra.run.smithi098.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 upower amd64 0.99.11-1build2 [104 kB] 2023-09-23T17:52:46.177 INFO:teuthology.orchestra.run.smithi098.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 usbmuxd amd64 1.1.1~git20191130.9af2b12-1 [38.4 kB] 2023-09-23T17:52:46.454 INFO:teuthology.orchestra.run.smithi173.stdout:Fetched 90.9 MB in 4s (21.1 MB/s) 2023-09-23T17:52:46.671 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libdbus-glib-1-2:amd64. 2023-09-23T17:52:46.893 INFO:teuthology.orchestra.run.smithi098.stdout:Fetched 90.9 MB in 4s (20.8 MB/s) 2023-09-23T17:52:47.101 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package libdbus-glib-1-2:amd64. 2023-09-23T17:52:48.403 INFO:teuthology.orchestra.run.smithi173.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 104373 files and directories currently installed.) 2023-09-23T17:52:48.408 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../00-libdbus-glib-1-2_0.110-5fakssync1_amd64.deb ... 2023-09-23T17:52:48.594 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-09-23T17:52:48.832 INFO:teuthology.orchestra.run.smithi098.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 104373 files and directories currently installed.) 2023-09-23T17:52:48.837 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../00-libdbus-glib-1-2_0.110-5fakssync1_amd64.deb ... 2023-09-23T17:52:48.965 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-09-23T17:52:48.972 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libplist3:amd64. 2023-09-23T17:52:48.985 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../01-libplist3_2.1.0-4build2_amd64.deb ... 2023-09-23T17:52:49.022 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libplist3:amd64 (2.1.0-4build2) ... 2023-09-23T17:52:49.337 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package libplist3:amd64. 2023-09-23T17:52:49.351 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../01-libplist3_2.1.0-4build2_amd64.deb ... 2023-09-23T17:52:49.359 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libusbmuxd6:amd64. 2023-09-23T17:52:49.372 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../02-libusbmuxd6_2.0.1-2_amd64.deb ... 2023-09-23T17:52:49.395 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking libplist3:amd64 (2.1.0-4build2) ... 2023-09-23T17:52:49.409 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libusbmuxd6:amd64 (2.0.1-2) ... 2023-09-23T17:52:49.732 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package libusbmuxd6:amd64. 2023-09-23T17:52:49.745 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../02-libusbmuxd6_2.0.1-2_amd64.deb ... 2023-09-23T17:52:49.771 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libimobiledevice6:amd64. 2023-09-23T17:52:49.781 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking libusbmuxd6:amd64 (2.0.1-2) ... 2023-09-23T17:52:49.784 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../03-libimobiledevice6_1.2.1~git20191129.9f79242-1build1_amd64.deb ... 2023-09-23T17:52:49.829 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-09-23T17:52:50.126 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package libimobiledevice6:amd64. 2023-09-23T17:52:50.140 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../03-libimobiledevice6_1.2.1~git20191129.9f79242-1build1_amd64.deb ... 2023-09-23T17:52:50.182 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libupower-glib3:amd64. 2023-09-23T17:52:50.193 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-09-23T17:52:50.196 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../04-libupower-glib3_0.99.11-1build2_amd64.deb ... 2023-09-23T17:52:50.241 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libupower-glib3:amd64 (0.99.11-1build2) ... 2023-09-23T17:52:50.538 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package libupower-glib3:amd64. 2023-09-23T17:52:50.553 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../04-libupower-glib3_0.99.11-1build2_amd64.deb ... 2023-09-23T17:52:50.568 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package linux-modules-5.15.0-84-generic. 2023-09-23T17:52:50.581 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../05-linux-modules-5.15.0-84-generic_5.15.0-84.93~20.04.1_amd64.deb ... 2023-09-23T17:52:50.596 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking libupower-glib3:amd64 (0.99.11-1build2) ... 2023-09-23T17:52:50.618 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking linux-modules-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T17:52:50.899 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package linux-modules-5.15.0-84-generic. 2023-09-23T17:52:50.912 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../05-linux-modules-5.15.0-84-generic_5.15.0-84.93~20.04.1_amd64.deb ... 2023-09-23T17:52:50.974 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking linux-modules-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T17:52:53.186 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package linux-image-5.15.0-84-generic. 2023-09-23T17:52:53.202 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../06-linux-image-5.15.0-84-generic_5.15.0-84.93~20.04.1_amd64.deb ... 2023-09-23T17:52:53.305 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking linux-image-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T17:52:53.476 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package linux-image-5.15.0-84-generic. 2023-09-23T17:52:53.490 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../06-linux-image-5.15.0-84-generic_5.15.0-84.93~20.04.1_amd64.deb ... 2023-09-23T17:52:53.588 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking linux-image-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T17:52:53.817 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-84-generic. 2023-09-23T17:52:53.833 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../07-linux-modules-extra-5.15.0-84-generic_5.15.0-84.93~20.04.1_amd64.deb ... 2023-09-23T17:52:53.875 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking linux-modules-extra-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T17:52:54.136 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-84-generic. 2023-09-23T17:52:54.153 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../07-linux-modules-extra-5.15.0-84-generic_5.15.0-84.93~20.04.1_amd64.deb ... 2023-09-23T17:52:54.194 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking linux-modules-extra-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T17:53:00.947 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package linux-image-generic-hwe-20.04. 2023-09-23T17:53:00.949 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package linux-image-generic-hwe-20.04. 2023-09-23T17:53:00.963 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../08-linux-image-generic-hwe-20.04_5.15.0.84.93~20.04.42_amd64.deb ... 2023-09-23T17:53:00.966 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../08-linux-image-generic-hwe-20.04_5.15.0.84.93~20.04.42_amd64.deb ... 2023-09-23T17:53:01.005 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking linux-image-generic-hwe-20.04 (5.15.0.84.93~20.04.42) ... 2023-09-23T17:53:01.007 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking linux-image-generic-hwe-20.04 (5.15.0.84.93~20.04.42) ... 2023-09-23T17:53:01.249 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package libevdev2:amd64. 2023-09-23T17:53:01.266 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../09-libevdev2_1.9.0+dfsg-1ubuntu0.2_amd64.deb ... 2023-09-23T17:53:01.299 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-09-23T17:53:01.343 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libevdev2:amd64. 2023-09-23T17:53:01.360 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../09-libevdev2_1.9.0+dfsg-1ubuntu0.2_amd64.deb ... 2023-09-23T17:53:01.402 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-09-23T17:53:01.694 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package thermald. 2023-09-23T17:53:01.711 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../10-thermald_1.9.1-1ubuntu0.6_amd64.deb ... 2023-09-23T17:53:01.721 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package thermald. 2023-09-23T17:53:01.738 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../10-thermald_1.9.1-1ubuntu0.6_amd64.deb ... 2023-09-23T17:53:01.752 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking thermald (1.9.1-1ubuntu0.6) ... 2023-09-23T17:53:01.771 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking thermald (1.9.1-1ubuntu0.6) ... 2023-09-23T17:53:02.326 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package upower. 2023-09-23T17:53:02.343 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../11-upower_0.99.11-1build2_amd64.deb ... 2023-09-23T17:53:02.404 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking upower (0.99.11-1build2) ... 2023-09-23T17:53:02.814 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package usbmuxd. 2023-09-23T17:53:02.828 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../12-usbmuxd_1.1.1~git20191130.9af2b12-1_amd64.deb ... 2023-09-23T17:53:02.872 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-09-23T17:53:02.907 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package upower. 2023-09-23T17:53:02.921 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../11-upower_0.99.11-1build2_amd64.deb ... 2023-09-23T17:53:02.981 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking upower (0.99.11-1build2) ... 2023-09-23T17:53:03.197 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libplist3:amd64 (2.1.0-4build2) ... 2023-09-23T17:53:03.301 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-09-23T17:53:03.395 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package usbmuxd. 2023-09-23T17:53:03.402 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libusbmuxd6:amd64 (2.0.1-2) ... 2023-09-23T17:53:03.409 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../12-usbmuxd_1.1.1~git20191130.9af2b12-1_amd64.deb ... 2023-09-23T17:53:03.454 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-09-23T17:53:03.503 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libupower-glib3:amd64 (0.99.11-1build2) ... 2023-09-23T17:53:03.654 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-09-23T17:53:03.778 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up libplist3:amd64 (2.1.0-4build2) ... 2023-09-23T17:53:03.780 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-09-23T17:53:03.874 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-09-23T17:53:03.906 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up upower (0.99.11-1build2) ... 2023-09-23T17:53:04.000 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up libusbmuxd6:amd64 (2.0.1-2) ... 2023-09-23T17:53:04.126 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up libupower-glib3:amd64 (0.99.11-1build2) ... 2023-09-23T17:53:04.261 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-09-23T17:53:04.387 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-09-23T17:53:04.497 INFO:teuthology.orchestra.run.smithi173.stdout:upower.service is a disabled or a static unit, not starting it. 2023-09-23T17:53:04.513 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up upower (0.99.11-1build2) ... 2023-09-23T17:53:04.552 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-09-23T17:53:04.723 INFO:teuthology.orchestra.run.smithi173.stdout:Warning: The home dir /var/lib/usbmux you specified can't be accessed: No such file or directory 2023-09-23T17:53:04.734 INFO:teuthology.orchestra.run.smithi173.stdout:Adding system user `usbmux' (UID 114) ... 2023-09-23T17:53:04.734 INFO:teuthology.orchestra.run.smithi173.stdout:Adding new user `usbmux' (UID 114) with group `plugdev' ... 2023-09-23T17:53:05.055 INFO:teuthology.orchestra.run.smithi098.stdout:upower.service is a disabled or a static unit, not starting it. 2023-09-23T17:53:05.109 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-09-23T17:53:05.269 INFO:teuthology.orchestra.run.smithi173.stdout:Not creating home directory `/var/lib/usbmux'. 2023-09-23T17:53:05.309 INFO:teuthology.orchestra.run.smithi098.stdout:Warning: The home dir /var/lib/usbmux you specified can't be accessed: No such file or directory 2023-09-23T17:53:05.323 INFO:teuthology.orchestra.run.smithi098.stdout:Adding system user `usbmux' (UID 114) ... 2023-09-23T17:53:05.324 INFO:teuthology.orchestra.run.smithi098.stdout:Adding new user `usbmux' (UID 114) with group `plugdev' ... 2023-09-23T17:53:05.474 INFO:teuthology.orchestra.run.smithi173.stdout:usbmuxd.service is a disabled or a static unit, not starting it. 2023-09-23T17:53:05.516 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up thermald (1.9.1-1ubuntu0.6) ... 2023-09-23T17:53:05.850 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/dbus-org.freedesktop.thermald.service -> /lib/systemd/system/thermald.service. 2023-09-23T17:53:05.850 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/thermald.service -> /lib/systemd/system/thermald.service. 2023-09-23T17:53:05.868 INFO:teuthology.orchestra.run.smithi098.stdout:Not creating home directory `/var/lib/usbmux'. 2023-09-23T17:53:06.088 INFO:teuthology.orchestra.run.smithi098.stdout:usbmuxd.service is a disabled or a static unit, not starting it. 2023-09-23T17:53:06.131 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up thermald (1.9.1-1ubuntu0.6) ... 2023-09-23T17:53:06.288 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up linux-image-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T17:53:06.443 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/dbus-org.freedesktop.thermald.service -> /lib/systemd/system/thermald.service. 2023-09-23T17:53:06.443 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/thermald.service -> /lib/systemd/system/thermald.service. 2023-09-23T17:53:06.913 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up linux-image-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T17:53:07.604 INFO:teuthology.orchestra.run.smithi173.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-84-generic 2023-09-23T17:53:07.604 INFO:teuthology.orchestra.run.smithi173.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-84-generic 2023-09-23T17:53:07.772 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up linux-modules-extra-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T17:53:08.329 INFO:teuthology.orchestra.run.smithi098.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-84-generic 2023-09-23T17:53:08.329 INFO:teuthology.orchestra.run.smithi098.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-84-generic 2023-09-23T17:53:08.588 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up linux-modules-extra-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T17:53:09.157 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up linux-image-generic-hwe-20.04 (5.15.0.84.93~20.04.42) ... 2023-09-23T17:53:09.299 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up linux-modules-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T17:53:09.964 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up linux-image-generic-hwe-20.04 (5.15.0.84.93~20.04.42) ... 2023-09-23T17:53:10.090 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up linux-modules-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T17:53:10.684 INFO:teuthology.orchestra.run.smithi173.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-09-23T17:53:11.458 INFO:teuthology.orchestra.run.smithi098.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-09-23T17:53:11.714 INFO:teuthology.orchestra.run.smithi098.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-09-23T17:53:11.732 INFO:teuthology.orchestra.run.smithi173.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-09-23T17:53:12.450 INFO:teuthology.orchestra.run.smithi098.stdout:Processing triggers for dbus (1.12.16-2ubuntu2.2) ... 2023-09-23T17:53:12.542 INFO:teuthology.orchestra.run.smithi098.stdout:Processing triggers for linux-image-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T17:53:12.616 INFO:teuthology.orchestra.run.smithi098.stdout:/etc/kernel/postinst.d/initramfs-tools: 2023-09-23T17:53:12.617 INFO:teuthology.orchestra.run.smithi098.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-84-generic 2023-09-23T17:53:13.226 INFO:teuthology.orchestra.run.smithi173.stdout:Processing triggers for dbus (1.12.16-2ubuntu2.2) ... 2023-09-23T17:53:13.335 INFO:teuthology.orchestra.run.smithi173.stdout:Processing triggers for linux-image-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-09-23T17:53:13.411 INFO:teuthology.orchestra.run.smithi173.stdout:/etc/kernel/postinst.d/initramfs-tools: 2023-09-23T17:53:13.411 INFO:teuthology.orchestra.run.smithi173.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-84-generic 2023-09-23T17:53:37.922 INFO:teuthology.orchestra.run.smithi098.stdout:/etc/kernel/postinst.d/zz-update-grub: 2023-09-23T17:53:37.923 INFO:teuthology.orchestra.run.smithi098.stdout:Sourcing file `/etc/default/grub' 2023-09-23T17:53:37.945 INFO:teuthology.orchestra.run.smithi098.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-09-23T17:53:37.973 INFO:teuthology.orchestra.run.smithi098.stdout:Generating grub configuration file ... 2023-09-23T17:53:38.244 INFO:teuthology.orchestra.run.smithi098.stdout:Found linux image: /boot/vmlinuz-5.15.0-84-generic 2023-09-23T17:53:38.271 INFO:teuthology.orchestra.run.smithi098.stdout:Found initrd image: /boot/initrd.img-5.15.0-84-generic 2023-09-23T17:53:38.666 INFO:teuthology.orchestra.run.smithi098.stdout:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-09-23T17:53:38.678 INFO:teuthology.orchestra.run.smithi098.stdout:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-09-23T17:53:38.769 INFO:teuthology.orchestra.run.smithi173.stdout:/etc/kernel/postinst.d/zz-update-grub: 2023-09-23T17:53:38.770 INFO:teuthology.orchestra.run.smithi173.stdout:Sourcing file `/etc/default/grub' 2023-09-23T17:53:38.781 INFO:teuthology.orchestra.run.smithi173.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-09-23T17:53:38.809 INFO:teuthology.orchestra.run.smithi173.stdout:Generating grub configuration file ... 2023-09-23T17:53:39.153 INFO:teuthology.orchestra.run.smithi173.stdout:Found linux image: /boot/vmlinuz-5.15.0-84-generic 2023-09-23T17:53:39.177 INFO:teuthology.orchestra.run.smithi173.stdout:Found initrd image: /boot/initrd.img-5.15.0-84-generic 2023-09-23T17:53:39.554 INFO:teuthology.orchestra.run.smithi173.stdout:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-09-23T17:53:39.565 INFO:teuthology.orchestra.run.smithi173.stdout:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-09-23T17:53:40.673 INFO:teuthology.orchestra.run.smithi098.stdout:done 2023-09-23T17:53:41.068 DEBUG:teuthology.orchestra.run.smithi098:> dpkg -s linux-image-generic-hwe-20.04 2023-09-23T17:53:41.119 INFO:teuthology.orchestra.run.smithi098.stdout:Package: linux-image-generic-hwe-20.04 2023-09-23T17:53:41.119 INFO:teuthology.orchestra.run.smithi098.stdout:Status: install ok installed 2023-09-23T17:53:41.119 INFO:teuthology.orchestra.run.smithi098.stdout:Priority: optional 2023-09-23T17:53:41.119 INFO:teuthology.orchestra.run.smithi098.stdout:Section: kernel 2023-09-23T17:53:41.119 INFO:teuthology.orchestra.run.smithi098.stdout:Installed-Size: 20 2023-09-23T17:53:41.120 INFO:teuthology.orchestra.run.smithi098.stdout:Maintainer: Ubuntu Kernel Team 2023-09-23T17:53:41.120 INFO:teuthology.orchestra.run.smithi098.stdout:Architecture: amd64 2023-09-23T17:53:41.120 INFO:teuthology.orchestra.run.smithi098.stdout:Source: linux-meta-hwe-5.15 2023-09-23T17:53:41.120 INFO:teuthology.orchestra.run.smithi098.stdout:Version: 5.15.0.84.93~20.04.42 2023-09-23T17:53:41.120 INFO:teuthology.orchestra.run.smithi098.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-84), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-09-23T17:53:41.120 INFO:teuthology.orchestra.run.smithi098.stdout:Depends: linux-image-5.15.0-84-generic, linux-modules-extra-5.15.0-84-generic, linux-firmware, intel-microcode, amd64-microcode 2023-09-23T17:53:41.120 INFO:teuthology.orchestra.run.smithi098.stdout:Recommends: thermald 2023-09-23T17:53:41.120 INFO:teuthology.orchestra.run.smithi098.stdout:Description: Generic Linux kernel image 2023-09-23T17:53:41.120 INFO:teuthology.orchestra.run.smithi098.stdout: This package will always depend on the latest generic kernel image 2023-09-23T17:53:41.120 INFO:teuthology.orchestra.run.smithi098.stdout: available. 2023-09-23T17:53:41.120 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.4.0-124-generic Expected: 5.15.0-84-generic 2023-09-23T17:53:41.121 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-09-23T17:53:41.121 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-09-23T17:53:41.121 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2023-09-23T17:53:41.121 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi098.front.sepia.ceph.com, path=None, version=distro) 2023-09-23T17:53:41.121 DEBUG:teuthology.orchestra.run.smithi098:> sudo apt-get clean 2023-09-23T17:53:41.213 DEBUG:teuthology.orchestra.run.smithi098:> sudo apt-get update 2023-09-23T17:53:41.493 INFO:teuthology.orchestra.run.smithi098.stdout:Hit:1 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-09-23T17:53:41.494 INFO:teuthology.orchestra.run.smithi098.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal InRelease 2023-09-23T17:53:41.565 INFO:teuthology.orchestra.run.smithi098.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-09-23T17:53:41.585 INFO:teuthology.orchestra.run.smithi173.stdout:done 2023-09-23T17:53:41.647 INFO:teuthology.orchestra.run.smithi098.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-09-23T17:53:42.025 DEBUG:teuthology.orchestra.run.smithi173:> dpkg -s linux-image-generic-hwe-20.04 2023-09-23T17:53:42.074 INFO:teuthology.orchestra.run.smithi173.stdout:Package: linux-image-generic-hwe-20.04 2023-09-23T17:53:42.074 INFO:teuthology.orchestra.run.smithi173.stdout:Status: install ok installed 2023-09-23T17:53:42.075 INFO:teuthology.orchestra.run.smithi173.stdout:Priority: optional 2023-09-23T17:53:42.075 INFO:teuthology.orchestra.run.smithi173.stdout:Section: kernel 2023-09-23T17:53:42.075 INFO:teuthology.orchestra.run.smithi173.stdout:Installed-Size: 20 2023-09-23T17:53:42.075 INFO:teuthology.orchestra.run.smithi173.stdout:Maintainer: Ubuntu Kernel Team 2023-09-23T17:53:42.075 INFO:teuthology.orchestra.run.smithi173.stdout:Architecture: amd64 2023-09-23T17:53:42.075 INFO:teuthology.orchestra.run.smithi173.stdout:Source: linux-meta-hwe-5.15 2023-09-23T17:53:42.075 INFO:teuthology.orchestra.run.smithi173.stdout:Version: 5.15.0.84.93~20.04.42 2023-09-23T17:53:42.075 INFO:teuthology.orchestra.run.smithi173.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-84), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-09-23T17:53:42.075 INFO:teuthology.orchestra.run.smithi173.stdout:Depends: linux-image-5.15.0-84-generic, linux-modules-extra-5.15.0-84-generic, linux-firmware, intel-microcode, amd64-microcode 2023-09-23T17:53:42.075 INFO:teuthology.orchestra.run.smithi173.stdout:Recommends: thermald 2023-09-23T17:53:42.075 INFO:teuthology.orchestra.run.smithi173.stdout:Description: Generic Linux kernel image 2023-09-23T17:53:42.075 INFO:teuthology.orchestra.run.smithi173.stdout: This package will always depend on the latest generic kernel image 2023-09-23T17:53:42.076 INFO:teuthology.orchestra.run.smithi173.stdout: available. 2023-09-23T17:53:42.076 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.4.0-124-generic Expected: 5.15.0-84-generic 2023-09-23T17:53:42.076 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-09-23T17:53:42.076 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-09-23T17:53:42.076 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2023-09-23T17:53:42.076 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi173.front.sepia.ceph.com, path=None, version=distro) 2023-09-23T17:53:42.076 DEBUG:teuthology.orchestra.run.smithi173:> sudo apt-get clean 2023-09-23T17:53:42.157 DEBUG:teuthology.orchestra.run.smithi173:> sudo apt-get update 2023-09-23T17:53:42.300 INFO:teuthology.orchestra.run.smithi173.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-09-23T17:53:42.307 INFO:teuthology.orchestra.run.smithi173.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-09-23T17:53:42.324 INFO:teuthology.orchestra.run.smithi173.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-09-23T17:53:42.415 INFO:teuthology.orchestra.run.smithi173.stdout:Hit:4 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-09-23T17:53:43.231 INFO:teuthology.orchestra.run.smithi098.stdout:Reading package lists... 2023-09-23T17:53:43.254 DEBUG:teuthology.orchestra.run.smithi098:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-09-23T17:53:43.321 INFO:teuthology.orchestra.run.smithi098.stdout:Reading package lists... 2023-09-23T17:53:43.521 INFO:teuthology.orchestra.run.smithi098.stdout:Building dependency tree... 2023-09-23T17:53:43.522 INFO:teuthology.orchestra.run.smithi098.stdout:Reading state information... 2023-09-23T17:53:43.733 INFO:teuthology.orchestra.run.smithi098.stdout:linux-image-generic-hwe-20.04 is already the newest version (5.15.0.84.93~20.04.42). 2023-09-23T17:53:43.734 INFO:teuthology.orchestra.run.smithi098.stdout:0 upgraded, 0 newly installed, 0 to remove and 266 not upgraded. 2023-09-23T17:53:43.735 DEBUG:teuthology.orchestra.run.smithi098:> dpkg -s linux-image-generic-hwe-20.04 2023-09-23T17:53:43.756 INFO:teuthology.orchestra.run.smithi098.stdout:Package: linux-image-generic-hwe-20.04 2023-09-23T17:53:43.756 INFO:teuthology.orchestra.run.smithi098.stdout:Status: install ok installed 2023-09-23T17:53:43.756 INFO:teuthology.orchestra.run.smithi098.stdout:Priority: optional 2023-09-23T17:53:43.756 INFO:teuthology.orchestra.run.smithi098.stdout:Section: kernel 2023-09-23T17:53:43.756 INFO:teuthology.orchestra.run.smithi098.stdout:Installed-Size: 20 2023-09-23T17:53:43.757 INFO:teuthology.orchestra.run.smithi098.stdout:Maintainer: Ubuntu Kernel Team 2023-09-23T17:53:43.757 INFO:teuthology.orchestra.run.smithi098.stdout:Architecture: amd64 2023-09-23T17:53:43.757 INFO:teuthology.orchestra.run.smithi098.stdout:Source: linux-meta-hwe-5.15 2023-09-23T17:53:43.757 INFO:teuthology.orchestra.run.smithi098.stdout:Version: 5.15.0.84.93~20.04.42 2023-09-23T17:53:43.757 INFO:teuthology.orchestra.run.smithi098.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-84), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-09-23T17:53:43.757 INFO:teuthology.orchestra.run.smithi098.stdout:Depends: linux-image-5.15.0-84-generic, linux-modules-extra-5.15.0-84-generic, linux-firmware, intel-microcode, amd64-microcode 2023-09-23T17:53:43.757 INFO:teuthology.orchestra.run.smithi098.stdout:Recommends: thermald 2023-09-23T17:53:43.757 INFO:teuthology.orchestra.run.smithi098.stdout:Description: Generic Linux kernel image 2023-09-23T17:53:43.757 INFO:teuthology.orchestra.run.smithi098.stdout: This package will always depend on the latest generic kernel image 2023-09-23T17:53:43.757 INFO:teuthology.orchestra.run.smithi098.stdout: available. 2023-09-23T17:53:43.758 DEBUG:teuthology.orchestra.run.smithi098:> mktemp 2023-09-23T17:53:43.763 INFO:teuthology.orchestra.run.smithi098.stdout:/tmp/tmp.f3lBy8qbM4 2023-09-23T17:53:43.763 DEBUG:teuthology.orchestra.run.smithi098:> sudo cp /boot/grub/grub.cfg /tmp/tmp.f3lBy8qbM4 2023-09-23T17:53:43.819 DEBUG:teuthology.orchestra.run.smithi098:> sudo chmod 0666 /tmp/tmp.f3lBy8qbM4 2023-09-23T17:53:43.862 DEBUG:teuthology.orchestra.remote:smithi098:/tmp/tmp.f3lBy8qbM4 is 9KB 2023-09-23T17:53:43.876 DEBUG:teuthology.orchestra.run.smithi098:> rm -fr /tmp/tmp.f3lBy8qbM4 2023-09-23T17:53:43.882 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2023-09-23T17:53:43.939 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-23T17:53:43.939 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd of=/etc/grub.d/01_ceph_kernel 2023-09-23T17:53:43.939 DEBUG:teuthology.orchestra.run.smithi098:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2023-09-23T17:53:43.969 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-84-generic 2023-09-23T17:53:43.970 DEBUG:teuthology.orchestra.run.smithi098:> sudo update-grub 2023-09-23T17:53:44.047 INFO:teuthology.orchestra.run.smithi173.stdout:Reading package lists... 2023-09-23T17:53:44.072 DEBUG:teuthology.orchestra.run.smithi173:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-09-23T17:53:44.142 INFO:teuthology.orchestra.run.smithi173.stdout:Reading package lists... 2023-09-23T17:53:44.320 INFO:teuthology.orchestra.run.smithi173.stdout:Building dependency tree... 2023-09-23T17:53:44.321 INFO:teuthology.orchestra.run.smithi173.stdout:Reading state information... 2023-09-23T17:53:44.529 INFO:teuthology.orchestra.run.smithi173.stdout:linux-image-generic-hwe-20.04 is already the newest version (5.15.0.84.93~20.04.42). 2023-09-23T17:53:44.529 INFO:teuthology.orchestra.run.smithi173.stdout:0 upgraded, 0 newly installed, 0 to remove and 266 not upgraded. 2023-09-23T17:53:44.530 DEBUG:teuthology.orchestra.run.smithi173:> dpkg -s linux-image-generic-hwe-20.04 2023-09-23T17:53:44.549 INFO:teuthology.orchestra.run.smithi173.stdout:Package: linux-image-generic-hwe-20.04 2023-09-23T17:53:44.549 INFO:teuthology.orchestra.run.smithi173.stdout:Status: install ok installed 2023-09-23T17:53:44.549 INFO:teuthology.orchestra.run.smithi173.stdout:Priority: optional 2023-09-23T17:53:44.549 INFO:teuthology.orchestra.run.smithi173.stdout:Section: kernel 2023-09-23T17:53:44.549 INFO:teuthology.orchestra.run.smithi173.stdout:Installed-Size: 20 2023-09-23T17:53:44.549 INFO:teuthology.orchestra.run.smithi173.stdout:Maintainer: Ubuntu Kernel Team 2023-09-23T17:53:44.549 INFO:teuthology.orchestra.run.smithi173.stdout:Architecture: amd64 2023-09-23T17:53:44.549 INFO:teuthology.orchestra.run.smithi173.stdout:Source: linux-meta-hwe-5.15 2023-09-23T17:53:44.549 INFO:teuthology.orchestra.run.smithi173.stdout:Version: 5.15.0.84.93~20.04.42 2023-09-23T17:53:44.549 INFO:teuthology.orchestra.run.smithi173.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-84), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-09-23T17:53:44.550 INFO:teuthology.orchestra.run.smithi173.stdout:Depends: linux-image-5.15.0-84-generic, linux-modules-extra-5.15.0-84-generic, linux-firmware, intel-microcode, amd64-microcode 2023-09-23T17:53:44.550 INFO:teuthology.orchestra.run.smithi173.stdout:Recommends: thermald 2023-09-23T17:53:44.550 INFO:teuthology.orchestra.run.smithi173.stdout:Description: Generic Linux kernel image 2023-09-23T17:53:44.550 INFO:teuthology.orchestra.run.smithi173.stdout: This package will always depend on the latest generic kernel image 2023-09-23T17:53:44.550 INFO:teuthology.orchestra.run.smithi173.stdout: available. 2023-09-23T17:53:44.550 DEBUG:teuthology.orchestra.run.smithi173:> mktemp 2023-09-23T17:53:44.597 INFO:teuthology.orchestra.run.smithi173.stdout:/tmp/tmp.QYCrTzEV0t 2023-09-23T17:53:44.597 DEBUG:teuthology.orchestra.run.smithi173:> sudo cp /boot/grub/grub.cfg /tmp/tmp.QYCrTzEV0t 2023-09-23T17:53:44.611 DEBUG:teuthology.orchestra.run.smithi173:> sudo chmod 0666 /tmp/tmp.QYCrTzEV0t 2023-09-23T17:53:44.695 DEBUG:teuthology.orchestra.remote:smithi173:/tmp/tmp.QYCrTzEV0t is 9KB 2023-09-23T17:53:44.710 DEBUG:teuthology.orchestra.run.smithi173:> rm -fr /tmp/tmp.QYCrTzEV0t 2023-09-23T17:53:44.715 DEBUG:teuthology.orchestra.run.smithi173:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2023-09-23T17:53:44.726 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-09-23T17:53:44.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd of=/etc/grub.d/01_ceph_kernel 2023-09-23T17:53:44.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2023-09-23T17:53:44.801 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-84-generic 2023-09-23T17:53:44.801 DEBUG:teuthology.orchestra.run.smithi173:> sudo update-grub 2023-09-23T17:53:45.278 INFO:teuthology.orchestra.run.smithi098.stderr:Sourcing file `/etc/default/grub' 2023-09-23T17:53:45.286 INFO:teuthology.orchestra.run.smithi098.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-09-23T17:53:45.296 INFO:teuthology.orchestra.run.smithi098.stderr:Generating grub configuration file ... 2023-09-23T17:53:45.563 INFO:teuthology.orchestra.run.smithi098.stderr:Found linux image: /boot/vmlinuz-5.15.0-84-generic 2023-09-23T17:53:45.575 INFO:teuthology.orchestra.run.smithi098.stderr:Found initrd image: /boot/initrd.img-5.15.0-84-generic 2023-09-23T17:53:45.934 INFO:teuthology.orchestra.run.smithi098.stderr:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-09-23T17:53:45.945 INFO:teuthology.orchestra.run.smithi098.stderr:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-09-23T17:53:46.215 INFO:teuthology.orchestra.run.smithi173.stderr:Sourcing file `/etc/default/grub' 2023-09-23T17:53:46.221 INFO:teuthology.orchestra.run.smithi173.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-09-23T17:53:46.237 INFO:teuthology.orchestra.run.smithi173.stderr:Generating grub configuration file ... 2023-09-23T17:53:46.477 INFO:teuthology.orchestra.run.smithi173.stderr:Found linux image: /boot/vmlinuz-5.15.0-84-generic 2023-09-23T17:53:46.488 INFO:teuthology.orchestra.run.smithi173.stderr:Found initrd image: /boot/initrd.img-5.15.0-84-generic 2023-09-23T17:53:46.844 INFO:teuthology.orchestra.run.smithi173.stderr:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-09-23T17:53:46.855 INFO:teuthology.orchestra.run.smithi173.stderr:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-09-23T17:53:46.922 INFO:teuthology.orchestra.run.smithi098.stderr:done 2023-09-23T17:53:46.923 DEBUG:teuthology.orchestra.run.smithi098:> sudo shutdown -r now 2023-09-23T17:53:47.844 INFO:teuthology.orchestra.run.smithi173.stderr:done 2023-09-23T17:53:47.846 DEBUG:teuthology.orchestra.run.smithi173:> sudo shutdown -r now 2023-09-23T17:54:16.940 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2023-09-23T17:54:16.941 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi098.front.sepia.ceph.com' 2023-09-23T17:54:16.941 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi098.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T17:54:17.849 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2023-09-23T17:54:17.849 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi173.front.sepia.ceph.com' 2023-09-23T17:54:17.850 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi173.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T17:54:35.258 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.98 2023-09-23T17:54:36.282 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.173 2023-09-23T17:54:44.268 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi098.front.sepia.ceph.com' 2023-09-23T17:54:44.268 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi098.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T17:54:45.284 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi173.front.sepia.ceph.com' 2023-09-23T17:54:45.284 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi173.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T17:54:47.326 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.98 2023-09-23T17:54:48.346 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.173 2023-09-23T17:54:59.339 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi098.front.sepia.ceph.com' 2023-09-23T17:54:59.339 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi098.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T17:55:00.348 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi173.front.sepia.ceph.com' 2023-09-23T17:55:00.349 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi173.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T17:55:18.778 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.173 2023-09-23T17:55:31.041 DEBUG:teuthology.orchestra.run.smithi098:> true 2023-09-23T17:55:31.580 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi098.front.sepia.ceph.com' 2023-09-23T17:55:31.580 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.15.0-84-generic"... 2023-09-23T17:55:31.581 DEBUG:teuthology.orchestra.run.smithi098:> uname -r 2023-09-23T17:55:31.629 INFO:teuthology.orchestra.run.smithi098.stdout:5.15.0-84-generic 2023-09-23T17:55:31.629 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-84-generic vs 5.15.0-84-generic 2023-09-23T17:55:31.629 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-09-23T17:55:31.630 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2023-09-23T17:55:32.630 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-09-23T17:55:32.631 DEBUG:teuthology.orchestra.run.smithi098:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-23T17:55:32.710 INFO:teuthology.orchestra.run.smithi098.stdout:ttyS1 2023-09-23T17:55:32.730 DEBUG:teuthology.parallel:result is None 2023-09-23T17:55:33.780 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi173.front.sepia.ceph.com' 2023-09-23T17:55:33.780 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi173.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T17:55:34.170 DEBUG:teuthology.orchestra.run.smithi173:> true 2023-09-23T17:55:34.837 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi173.front.sepia.ceph.com' 2023-09-23T17:55:34.837 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.15.0-84-generic"... 2023-09-23T17:55:34.837 DEBUG:teuthology.orchestra.run.smithi173:> uname -r 2023-09-23T17:55:34.884 INFO:teuthology.orchestra.run.smithi173.stdout:5.15.0-84-generic 2023-09-23T17:55:34.884 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-84-generic vs 5.15.0-84-generic 2023-09-23T17:55:34.884 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-09-23T17:55:34.884 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2023-09-23T17:55:35.885 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-09-23T17:55:35.885 DEBUG:teuthology.orchestra.run.smithi173:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-23T17:55:35.962 INFO:teuthology.orchestra.run.smithi173.stdout:ttyS1 2023-09-23T17:55:35.986 DEBUG:teuthology.parallel:result is None 2023-09-23T17:55:35.987 INFO:teuthology.run_tasks:Running task internal.base... 2023-09-23T17:55:35.994 INFO:teuthology.task.internal:Creating test directory... 2023-09-23T17:55:35.995 DEBUG:teuthology.orchestra.run.smithi098:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-23T17:55:35.997 DEBUG:teuthology.orchestra.run.smithi173:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-23T17:55:36.004 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-09-23T17:55:36.051 INFO:teuthology.run_tasks:Running task internal.archive... 2023-09-23T17:55:36.121 INFO:teuthology.task.internal:Creating archive directory... 2023-09-23T17:55:36.122 DEBUG:teuthology.orchestra.run.smithi098:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-23T17:55:36.124 DEBUG:teuthology.orchestra.run.smithi173:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-23T17:55:36.226 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-09-23T17:55:36.232 INFO:teuthology.task.internal:Enabling coredump saving... 2023-09-23T17:55:36.233 DEBUG:teuthology.orchestra.run.smithi098:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-23T17:55:36.235 DEBUG:teuthology.orchestra.run.smithi173:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-23T17:55:36.251 INFO:teuthology.orchestra.run.smithi098.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T17:55:36.257 INFO:teuthology.orchestra.run.smithi098.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T17:55:36.265 INFO:teuthology.orchestra.run.smithi173.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T17:55:36.277 INFO:teuthology.orchestra.run.smithi173.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T17:55:36.279 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-09-23T17:55:36.285 INFO:teuthology.task.internal:Configuring sudo... 2023-09-23T17:55:36.285 DEBUG:teuthology.orchestra.run.smithi098:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-23T17:55:36.301 DEBUG:teuthology.orchestra.run.smithi173:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-23T17:55:36.336 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-09-23T17:55:36.345 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-09-23T17:55:36.345 DEBUG:teuthology.orchestra.run.smithi098:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-23T17:55:36.361 DEBUG:teuthology.orchestra.run.smithi173:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-23T17:55:36.386 DEBUG:teuthology.orchestra.run.smithi098:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T17:55:36.414 DEBUG:teuthology.orchestra.run.smithi098:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T17:55:36.462 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-23T17:55:36.462 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-23T17:55:36.529 DEBUG:teuthology.orchestra.run.smithi173:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T17:55:36.536 DEBUG:teuthology.orchestra.run.smithi173:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T17:55:36.591 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-09-23T17:55:36.591 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-23T17:55:36.657 DEBUG:teuthology.orchestra.run.smithi098:> sudo service rsyslog restart 2023-09-23T17:55:36.659 DEBUG:teuthology.orchestra.run.smithi173:> sudo service rsyslog restart 2023-09-23T17:55:36.746 INFO:teuthology.run_tasks:Running task internal.timer... 2023-09-23T17:55:36.753 INFO:teuthology.task.internal:Starting timer... 2023-09-23T17:55:36.753 INFO:teuthology.run_tasks:Running task pcp... 2023-09-23T17:55:36.790 INFO:teuthology.run_tasks:Running task selinux... 2023-09-23T17:55:36.822 DEBUG:teuthology.task.selinux:Excluding smithi098: OS 'ubuntu' does not support SELinux 2023-09-23T17:55:36.822 DEBUG:teuthology.task.selinux:Excluding smithi173: OS 'ubuntu' does not support SELinux 2023-09-23T17:55:36.822 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-09-23T17:55:36.822 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2023-09-23T17:55:36.822 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-09-23T17:55:36.822 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-09-23T17:55:36.856 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-09-23T17:55:36.887 INFO:teuthology.repo_utils:Fetching main from origin 2023-09-23T17:55:37.010 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-09-23T17:55:37.034 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-09-23T17:55:37.035 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi098.front.sepia.ceph.com,smithi173.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-09-23T18:01:12.851 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi098.front.sepia.ceph.com'), Remote(name='ubuntu@smithi173.front.sepia.ceph.com')] 2023-09-23T18:01:12.852 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi098.front.sepia.ceph.com' 2023-09-23T18:01:12.852 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi098.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T18:01:12.944 DEBUG:teuthology.orchestra.run.smithi098:> true 2023-09-23T18:01:13.064 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi098.front.sepia.ceph.com' 2023-09-23T18:01:13.064 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi173.front.sepia.ceph.com' 2023-09-23T18:01:13.064 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi173.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T18:01:13.144 DEBUG:teuthology.orchestra.run.smithi173:> true 2023-09-23T18:01:13.263 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi173.front.sepia.ceph.com' 2023-09-23T18:01:13.264 INFO:teuthology.run_tasks:Running task clock... 2023-09-23T18:01:13.277 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-09-23T18:01:13.278 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-23T18:01:13.278 DEBUG:teuthology.orchestra.run.smithi098:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T18:01:13.280 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-23T18:01:13.281 DEBUG:teuthology.orchestra.run.smithi173:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T18:01:13.311 INFO:teuthology.orchestra.run.smithi098.stdout:23 Sep 18:01:13 ntpd[22547]: ntpd 4.2.8p12@1.3728-o (1): Starting 2023-09-23T18:01:13.312 INFO:teuthology.orchestra.run.smithi098.stdout:23 Sep 18:01:13 ntpd[22547]: Command line: ntpd -gq 2023-09-23T18:01:13.312 INFO:teuthology.orchestra.run.smithi098.stdout:23 Sep 18:01:13 ntpd[22547]: proto: precision = 0.093 usec (-23) 2023-09-23T18:01:13.312 INFO:teuthology.orchestra.run.smithi098.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-09-23T18:01:13.313 INFO:teuthology.orchestra.run.smithi098.stdout:23 Sep 18:01:13 ntpd[22547]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-09-23T18:01:13.313 INFO:teuthology.orchestra.run.smithi098.stderr:restrict ::: KOD does nothing without LIMITED. 2023-09-23T18:01:13.314 INFO:teuthology.orchestra.run.smithi098.stdout:23 Sep 18:01:13 ntpd[22547]: restrict ::: KOD does nothing without LIMITED. 2023-09-23T18:01:13.314 INFO:teuthology.orchestra.run.smithi098.stdout:23 Sep 18:01:13 ntpd[22547]: Listen and drop on 0 v6wildcard [::]:123 2023-09-23T18:01:13.314 INFO:teuthology.orchestra.run.smithi098.stdout:23 Sep 18:01:13 ntpd[22547]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2023-09-23T18:01:13.314 INFO:teuthology.orchestra.run.smithi098.stdout:23 Sep 18:01:13 ntpd[22547]: Listen normally on 2 lo 127.0.0.1:123 2023-09-23T18:01:13.315 INFO:teuthology.orchestra.run.smithi098.stdout:23 Sep 18:01:13 ntpd[22547]: Listen normally on 3 enp3s0f1 172.21.15.98:123 2023-09-23T18:01:13.315 INFO:teuthology.orchestra.run.smithi098.stdout:23 Sep 18:01:13 ntpd[22547]: Listen normally on 4 lo [::1]:123 2023-09-23T18:01:13.315 INFO:teuthology.orchestra.run.smithi098.stdout:23 Sep 18:01:13 ntpd[22547]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe8f:d26b%5]:123 2023-09-23T18:01:13.315 INFO:teuthology.orchestra.run.smithi098.stdout:23 Sep 18:01:13 ntpd[22547]: Listening on routing socket on fd #22 for interface updates 2023-09-23T18:01:13.338 INFO:teuthology.orchestra.run.smithi173.stdout:23 Sep 18:01:13 ntpd[22523]: ntpd 4.2.8p12@1.3728-o (1): Starting 2023-09-23T18:01:13.338 INFO:teuthology.orchestra.run.smithi173.stdout:23 Sep 18:01:13 ntpd[22523]: Command line: ntpd -gq 2023-09-23T18:01:13.338 INFO:teuthology.orchestra.run.smithi173.stdout:23 Sep 18:01:13 ntpd[22523]: proto: precision = 0.052 usec (-24) 2023-09-23T18:01:13.339 INFO:teuthology.orchestra.run.smithi173.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-09-23T18:01:13.339 INFO:teuthology.orchestra.run.smithi173.stdout:23 Sep 18:01:13 ntpd[22523]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-09-23T18:01:13.339 INFO:teuthology.orchestra.run.smithi173.stderr:restrict ::: KOD does nothing without LIMITED. 2023-09-23T18:01:13.340 INFO:teuthology.orchestra.run.smithi173.stdout:23 Sep 18:01:13 ntpd[22523]: restrict ::: KOD does nothing without LIMITED. 2023-09-23T18:01:13.340 INFO:teuthology.orchestra.run.smithi173.stdout:23 Sep 18:01:13 ntpd[22523]: Listen and drop on 0 v6wildcard [::]:123 2023-09-23T18:01:13.340 INFO:teuthology.orchestra.run.smithi173.stdout:23 Sep 18:01:13 ntpd[22523]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2023-09-23T18:01:13.341 INFO:teuthology.orchestra.run.smithi173.stdout:23 Sep 18:01:13 ntpd[22523]: Listen normally on 2 lo 127.0.0.1:123 2023-09-23T18:01:13.341 INFO:teuthology.orchestra.run.smithi173.stdout:23 Sep 18:01:13 ntpd[22523]: Listen normally on 3 enp3s0f1 172.21.15.173:123 2023-09-23T18:01:13.341 INFO:teuthology.orchestra.run.smithi173.stdout:23 Sep 18:01:13 ntpd[22523]: Listen normally on 4 lo [::1]:123 2023-09-23T18:01:13.341 INFO:teuthology.orchestra.run.smithi173.stdout:23 Sep 18:01:13 ntpd[22523]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe88:7259%5]:123 2023-09-23T18:01:13.341 INFO:teuthology.orchestra.run.smithi173.stdout:23 Sep 18:01:13 ntpd[22523]: Listening on routing socket on fd #22 for interface updates 2023-09-23T18:01:14.338 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:14 ntpd[22523]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:14.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:14 ntpd[22523]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:15.312 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:15 ntpd[22547]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:15.312 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:15 ntpd[22547]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:15.312 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:15 ntpd[22547]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:15.312 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:15 ntpd[22547]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:15.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:15 ntpd[22547]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:01:15.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:15 ntpd[22547]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:01:15.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:15 ntpd[22523]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:15.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:15 ntpd[22523]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:15.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:15 ntpd[22523]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:15.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:15 ntpd[22523]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:15.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:15 ntpd[22523]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:01:15.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:15 ntpd[22523]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:01:16.312 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:16 ntpd[22547]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:16.312 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:16 ntpd[22547]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:16.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:16 ntpd[22523]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:16.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:16 ntpd[22523]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:16.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:16 ntpd[22523]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:16.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:16 ntpd[22523]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:16.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:16 ntpd[22523]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:01:16.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:16 ntpd[22523]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:01:17.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:17 ntpd[22547]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:17.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:17 ntpd[22547]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:17.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:17 ntpd[22547]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:17.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:17 ntpd[22547]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:17.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:17 ntpd[22547]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:17.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:17 ntpd[22547]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:17.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:17 ntpd[22547]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:01:17.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:17 ntpd[22547]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:01:17.314 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:17 ntpd[22547]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:17.314 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:17 ntpd[22547]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:17.314 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:17 ntpd[22547]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:01:17.314 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:17 ntpd[22547]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:01:17.338 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:17 ntpd[22523]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:17.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:17 ntpd[22523]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:17.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:17 ntpd[22523]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:17.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:17 ntpd[22523]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:17.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:17 ntpd[22523]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:01:17.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:17 ntpd[22523]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:01:18.312 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:18 ntpd[22547]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:18.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:18 ntpd[22547]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:18.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:18 ntpd[22547]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:18.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:18 ntpd[22547]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:18.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:18 ntpd[22547]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:01:18.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:18 ntpd[22547]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:01:18.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:18 ntpd[22523]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:18.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:18 ntpd[22523]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:18.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:18 ntpd[22523]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:18.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:18 ntpd[22523]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:18.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:18 ntpd[22523]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:18.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:18 ntpd[22523]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:18.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:18 ntpd[22523]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:01:18.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:18 ntpd[22523]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:01:18.340 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:18 ntpd[22523]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:18.340 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:18 ntpd[22523]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:18.340 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:18 ntpd[22523]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:01:18.340 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:18 ntpd[22523]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:01:19.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:19 ntpd[22547]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:19.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:19 ntpd[22547]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:19.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:19 ntpd[22547]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:19.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:19 ntpd[22547]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:19.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:19 ntpd[22547]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:19.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:19 ntpd[22547]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:19.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:19 ntpd[22547]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:01:19.314 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:19 ntpd[22547]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:01:19.314 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:19 ntpd[22547]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:19.314 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:19 ntpd[22547]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:19.314 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:19 ntpd[22547]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:01:19.314 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:19 ntpd[22547]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:01:19.338 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:19 ntpd[22523]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:19.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:19 ntpd[22523]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:19.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:19 ntpd[22523]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:19.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:19 ntpd[22523]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:19.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:19 ntpd[22523]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:01:19.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:19 ntpd[22523]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:01:20.312 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:20 ntpd[22547]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:20.312 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:20 ntpd[22547]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:20.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:20 ntpd[22547]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:20.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:20 ntpd[22547]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:20.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:20 ntpd[22547]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:01:20.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:20 ntpd[22547]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:01:20.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:20 ntpd[22523]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:20.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:20 ntpd[22523]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:20.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:20 ntpd[22523]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:20.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:20 ntpd[22523]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:20.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:20 ntpd[22523]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:20.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:20 ntpd[22523]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:20.340 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:20 ntpd[22523]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:01:20.340 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:20 ntpd[22523]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:01:20.340 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:20 ntpd[22523]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:20.340 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:20 ntpd[22523]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:20.340 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:20 ntpd[22523]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:01:20.340 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:20 ntpd[22523]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:01:21.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:21 ntpd[22547]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:21.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:21 ntpd[22547]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:21.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:21 ntpd[22547]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:21.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:21 ntpd[22547]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:21.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:21 ntpd[22547]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:21.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:21 ntpd[22547]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:21.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:21 ntpd[22547]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:01:21.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:21 ntpd[22547]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:01:21.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:21 ntpd[22547]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:21.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:21 ntpd[22547]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:21.313 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:21 ntpd[22547]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:01:21.314 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:21 ntpd[22547]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:01:21.314 INFO:teuthology.orchestra.run.smithi098.stdout:23 Sep 18:01:21 ntpd[22547]: ntpd: time slew +0.002813 s 2023-09-23T18:01:21.314 INFO:teuthology.orchestra.run.smithi098.stdout:ntpd: time slew +0.002813s 2023-09-23T18:01:21.314 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:21 ntpd[22547]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2023-09-23T18:01:21.314 INFO:teuthology.orchestra.run.smithi098.stderr:23 Sep 18:01:21 ntpd[22547]: can't open /var/log/ntpstats/loopstats.20230923: Permission denied 2023-09-23T18:01:21.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:21 ntpd[22523]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:21.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:21 ntpd[22523]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:21.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:21 ntpd[22523]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-09-23T18:01:21.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:21 ntpd[22523]: can't open /var/log/ntpstats/rawstats.20230923: Permission denied 2023-09-23T18:01:21.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:21 ntpd[22523]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-09-23T18:01:21.339 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:21 ntpd[22523]: can't open /var/log/ntpstats/peerstats.20230923: Permission denied 2023-09-23T18:01:21.340 INFO:teuthology.orchestra.run.smithi173.stdout:23 Sep 18:01:21 ntpd[22523]: ntpd: time slew -0.000725 s 2023-09-23T18:01:21.340 INFO:teuthology.orchestra.run.smithi173.stdout:ntpd: time slew -0.000725s 2023-09-23T18:01:21.340 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:21 ntpd[22523]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2023-09-23T18:01:21.340 INFO:teuthology.orchestra.run.smithi173.stderr:23 Sep 18:01:21 ntpd[22523]: can't open /var/log/ntpstats/loopstats.20230923: Permission denied 2023-09-23T18:01:21.374 INFO:teuthology.orchestra.run.smithi098.stdout: remote refid st t when poll reach delay offset jitter 2023-09-23T18:01:21.374 INFO:teuthology.orchestra.run.smithi098.stdout:============================================================================== 2023-09-23T18:01:21.374 INFO:teuthology.orchestra.run.smithi098.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-09-23T18:01:21.374 INFO:teuthology.orchestra.run.smithi098.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-09-23T18:01:21.374 INFO:teuthology.orchestra.run.smithi098.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-09-23T18:01:21.375 INFO:teuthology.orchestra.run.smithi098.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-09-23T18:01:21.391 INFO:teuthology.orchestra.run.smithi173.stdout: remote refid st t when poll reach delay offset jitter 2023-09-23T18:01:21.391 INFO:teuthology.orchestra.run.smithi173.stdout:============================================================================== 2023-09-23T18:01:21.391 INFO:teuthology.orchestra.run.smithi173.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-09-23T18:01:21.391 INFO:teuthology.orchestra.run.smithi173.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-09-23T18:01:21.391 INFO:teuthology.orchestra.run.smithi173.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-09-23T18:01:21.391 INFO:teuthology.orchestra.run.smithi173.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-09-23T18:01:21.392 INFO:teuthology.run_tasks:Running task install... 2023-09-23T18:01:21.401 DEBUG:teuthology.task.install:project ceph 2023-09-23T18:01:21.401 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}} 2023-09-23T18:01:21.401 DEBUG:teuthology.task.install:config {'branch': 'pacific', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'} 2023-09-23T18:01:21.401 INFO:teuthology.task.install:Using flavor: default 2023-09-23T18:01:21.406 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-09-23T18:01:21.407 INFO:teuthology.task.install:extra packages: [] 2023-09-23T18:01:21.407 DEBUG:teuthology.orchestra.run.smithi098:> sudo apt-key list | grep Ceph 2023-09-23T18:01:21.407 DEBUG:teuthology.orchestra.run.smithi173:> sudo apt-key list | grep Ceph 2023-09-23T18:01:21.510 INFO:teuthology.orchestra.run.smithi098.stderr:Warning: apt-key output should not be parsed (stdout is not a terminal) 2023-09-23T18:01:21.526 INFO:teuthology.orchestra.run.smithi173.stderr:Warning: apt-key output should not be parsed (stdout is not a terminal) 2023-09-23T18:01:21.567 INFO:teuthology.orchestra.run.smithi098.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2023-09-23T18:01:21.567 INFO:teuthology.orchestra.run.smithi098.stdout:uid [ unknown] Ceph.com (release key) 2023-09-23T18:01:21.568 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2023-09-23T18:01:21.568 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T18:01:21.568 INFO:teuthology.packaging:ref: None 2023-09-23T18:01:21.568 INFO:teuthology.packaging:tag: None 2023-09-23T18:01:21.568 INFO:teuthology.packaging:branch: pacific 2023-09-23T18:01:21.568 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T18:01:21.569 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-09-23T18:01:21.582 INFO:teuthology.orchestra.run.smithi173.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2023-09-23T18:01:21.583 INFO:teuthology.orchestra.run.smithi173.stdout:uid [ unknown] Ceph.com (release key) 2023-09-23T18:01:21.583 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2023-09-23T18:01:21.583 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T18:01:21.583 INFO:teuthology.packaging:ref: None 2023-09-23T18:01:21.584 INFO:teuthology.packaging:tag: None 2023-09-23T18:01:21.584 INFO:teuthology.packaging:branch: pacific 2023-09-23T18:01:21.584 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T18:01:21.584 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-09-23T18:01:21.959 INFO:teuthology.task.install.deb:Pulling from https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default/ 2023-09-23T18:01:21.959 INFO:teuthology.task.install.deb:Package version is 16.2.14-37-g2e1f1015-1focal 2023-09-23T18:01:21.966 INFO:teuthology.task.install.deb:Pulling from https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default/ 2023-09-23T18:01:21.967 INFO:teuthology.task.install.deb:Package version is 16.2.14-37-g2e1f1015-1focal 2023-09-23T18:01:22.088 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-09-23T18:01:22.088 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2023-09-23T18:01:22.104 DEBUG:teuthology.orchestra.run.smithi173:> sudo apt-get update 2023-09-23T18:01:22.191 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-23T18:01:22.191 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2023-09-23T18:01:22.207 DEBUG:teuthology.orchestra.run.smithi098:> sudo apt-get update 2023-09-23T18:01:22.347 INFO:teuthology.orchestra.run.smithi173.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-09-23T18:01:22.347 INFO:teuthology.orchestra.run.smithi173.stdout:Ign:2 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal InRelease 2023-09-23T18:01:22.364 INFO:teuthology.orchestra.run.smithi173.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-09-23T18:01:22.368 INFO:teuthology.orchestra.run.smithi173.stdout:Get:4 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal Release [7,680 B] 2023-09-23T18:01:22.390 INFO:teuthology.orchestra.run.smithi173.stdout:Ign:5 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal Release.gpg 2023-09-23T18:01:22.395 INFO:teuthology.orchestra.run.smithi173.stdout:Hit:6 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-09-23T18:01:22.411 INFO:teuthology.orchestra.run.smithi173.stdout:Get:7 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main i386 Packages [3,104 B] 2023-09-23T18:01:22.425 INFO:teuthology.orchestra.run.smithi098.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-09-23T18:01:22.427 INFO:teuthology.orchestra.run.smithi173.stdout:Hit:8 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-09-23T18:01:22.443 INFO:teuthology.orchestra.run.smithi098.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-09-23T18:01:22.448 INFO:teuthology.orchestra.run.smithi173.stdout:Get:9 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 Packages [17.2 kB] 2023-09-23T18:01:22.450 INFO:teuthology.orchestra.run.smithi098.stdout:Ign:3 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal InRelease 2023-09-23T18:01:22.471 INFO:teuthology.orchestra.run.smithi098.stdout:Get:4 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal Release [7,680 B] 2023-09-23T18:01:22.474 INFO:teuthology.orchestra.run.smithi098.stdout:Hit:5 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-09-23T18:01:22.494 INFO:teuthology.orchestra.run.smithi098.stdout:Ign:6 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal Release.gpg 2023-09-23T18:01:22.515 INFO:teuthology.orchestra.run.smithi098.stdout:Get:7 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 Packages [17.2 kB] 2023-09-23T18:01:22.536 INFO:teuthology.orchestra.run.smithi098.stdout:Hit:8 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-09-23T18:01:22.557 INFO:teuthology.orchestra.run.smithi098.stdout:Get:9 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main i386 Packages [3,104 B] 2023-09-23T18:01:22.970 INFO:teuthology.orchestra.run.smithi173.stdout:Fetched 28.0 kB in 1s (42.2 kB/s) 2023-09-23T18:01:23.063 INFO:teuthology.orchestra.run.smithi098.stdout:Fetched 28.0 kB in 1s (43.9 kB/s) 2023-09-23T18:01:24.011 INFO:teuthology.orchestra.run.smithi173.stdout:Reading package lists... 2023-09-23T18:01:24.038 DEBUG:teuthology.orchestra.run.smithi173:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=16.2.14-37-g2e1f1015-1focal cephadm=16.2.14-37-g2e1f1015-1focal ceph-mds=16.2.14-37-g2e1f1015-1focal ceph-mgr=16.2.14-37-g2e1f1015-1focal ceph-common=16.2.14-37-g2e1f1015-1focal ceph-fuse=16.2.14-37-g2e1f1015-1focal ceph-test=16.2.14-37-g2e1f1015-1focal radosgw=16.2.14-37-g2e1f1015-1focal python3-rados=16.2.14-37-g2e1f1015-1focal python3-rgw=16.2.14-37-g2e1f1015-1focal python3-cephfs=16.2.14-37-g2e1f1015-1focal python3-rbd=16.2.14-37-g2e1f1015-1focal libcephfs2=16.2.14-37-g2e1f1015-1focal libcephfs-dev=16.2.14-37-g2e1f1015-1focal librados2=16.2.14-37-g2e1f1015-1focal librbd1=16.2.14-37-g2e1f1015-1focal rbd-fuse=16.2.14-37-g2e1f1015-1focal 2023-09-23T18:01:24.105 INFO:teuthology.orchestra.run.smithi098.stdout:Reading package lists... 2023-09-23T18:01:24.106 INFO:teuthology.orchestra.run.smithi173.stdout:Reading package lists... 2023-09-23T18:01:24.130 DEBUG:teuthology.orchestra.run.smithi098:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=16.2.14-37-g2e1f1015-1focal cephadm=16.2.14-37-g2e1f1015-1focal ceph-mds=16.2.14-37-g2e1f1015-1focal ceph-mgr=16.2.14-37-g2e1f1015-1focal ceph-common=16.2.14-37-g2e1f1015-1focal ceph-fuse=16.2.14-37-g2e1f1015-1focal ceph-test=16.2.14-37-g2e1f1015-1focal radosgw=16.2.14-37-g2e1f1015-1focal python3-rados=16.2.14-37-g2e1f1015-1focal python3-rgw=16.2.14-37-g2e1f1015-1focal python3-cephfs=16.2.14-37-g2e1f1015-1focal python3-rbd=16.2.14-37-g2e1f1015-1focal libcephfs2=16.2.14-37-g2e1f1015-1focal libcephfs-dev=16.2.14-37-g2e1f1015-1focal librados2=16.2.14-37-g2e1f1015-1focal librbd1=16.2.14-37-g2e1f1015-1focal rbd-fuse=16.2.14-37-g2e1f1015-1focal 2023-09-23T18:01:24.203 INFO:teuthology.orchestra.run.smithi098.stdout:Reading package lists... 2023-09-23T18:01:24.285 INFO:teuthology.orchestra.run.smithi173.stdout:Building dependency tree... 2023-09-23T18:01:24.286 INFO:teuthology.orchestra.run.smithi173.stdout:Reading state information... 2023-09-23T18:01:24.388 INFO:teuthology.orchestra.run.smithi098.stdout:Building dependency tree... 2023-09-23T18:01:24.388 INFO:teuthology.orchestra.run.smithi098.stdout:Reading state information... 2023-09-23T18:01:24.451 INFO:teuthology.orchestra.run.smithi173.stdout:The following packages were automatically installed and are no longer required: 2023-09-23T18:01:24.452 INFO:teuthology.orchestra.run.smithi173.stdout: libboost-iostreams1.71.0 libboost-thread1.71.0 2023-09-23T18:01:24.452 INFO:teuthology.orchestra.run.smithi173.stdout:Use 'sudo apt autoremove' to remove them. 2023-09-23T18:01:24.452 INFO:teuthology.orchestra.run.smithi173.stdout:The following additional packages will be installed: 2023-09-23T18:01:24.452 INFO:teuthology.orchestra.run.smithi173.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2023-09-23T18:01:24.453 INFO:teuthology.orchestra.run.smithi173.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq libjaeger 2023-09-23T18:01:24.453 INFO:teuthology.orchestra.run.smithi173.stdout: libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 libleveldb1d 2023-09-23T18:01:24.453 INFO:teuthology.orchestra.run.smithi173.stdout: liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 libradosstriper1 2023-09-23T18:01:24.453 INFO:teuthology.orchestra.run.smithi173.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli python-pastedeploy-tpl 2023-09-23T18:01:24.453 INFO:teuthology.orchestra.run.smithi173.stdout: python3-atomicwrites python3-attr python3-bcrypt python3-blinker python3-bs4 2023-09-23T18:01:24.453 INFO:teuthology.orchestra.run.smithi173.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common 2023-09-23T18:01:24.453 INFO:teuthology.orchestra.run.smithi173.stdout: python3-cherrypy3 python3-cryptography python3-decorator python3-google-auth 2023-09-23T18:01:24.453 INFO:teuthology.orchestra.run.smithi173.stdout: python3-html5lib python3-jinja2 python3-joblib python3-jwt 2023-09-23T18:01:24.453 INFO:teuthology.orchestra.run.smithi173.stdout: python3-kubernetes python3-logutils python3-lxml python3-mako 2023-09-23T18:01:24.453 INFO:teuthology.orchestra.run.smithi173.stdout: python3-markupsafe python3-oauthlib python3-openssl python3-packaging 2023-09-23T18:01:24.454 INFO:teuthology.orchestra.run.smithi173.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2023-09-23T18:01:24.454 INFO:teuthology.orchestra.run.smithi173.stdout: python3-pluggy python3-prettytable python3-psutil python3-py python3-pyasn1 2023-09-23T18:01:24.454 INFO:teuthology.orchestra.run.smithi173.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-repoze.lru 2023-09-23T18:01:24.454 INFO:teuthology.orchestra.run.smithi173.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-scipy 2023-09-23T18:01:24.454 INFO:teuthology.orchestra.run.smithi173.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2023-09-23T18:01:24.454 INFO:teuthology.orchestra.run.smithi173.stdout: python3-sklearn python3-sklearn-lib python3-soupsieve python3-tempita 2023-09-23T18:01:24.454 INFO:teuthology.orchestra.run.smithi173.stdout: python3-waitress python3-wcwidth python3-webencodings python3-webob 2023-09-23T18:01:24.454 INFO:teuthology.orchestra.run.smithi173.stdout: python3-websocket python3-webtest python3-werkzeug socat xmlstarlet 2023-09-23T18:01:24.454 INFO:teuthology.orchestra.run.smithi173.stdout:Suggested packages: 2023-09-23T18:01:24.455 INFO:teuthology.orchestra.run.smithi173.stdout: python3-influxdb python-attr-doc python-blinker-doc python-cryptography-doc 2023-09-23T18:01:24.455 INFO:teuthology.orchestra.run.smithi173.stdout: python3-cryptography-vectors python3-genshi python-jinja2-doc python3-crypto 2023-09-23T18:01:24.455 INFO:teuthology.orchestra.run.smithi173.stdout: python3-lxml-dbg python-lxml-doc python3-beaker python-mako-doc 2023-09-23T18:01:24.455 INFO:teuthology.orchestra.run.smithi173.stdout: python-openssl-doc python3-openssl-dbg httpd-wsgi libapache2-mod-python 2023-09-23T18:01:24.455 INFO:teuthology.orchestra.run.smithi173.stdout: libapache2-mod-scgi libjs-mochikit python-pecan-doc python-psutil-doc 2023-09-23T18:01:24.455 INFO:teuthology.orchestra.run.smithi173.stdout: subversion python-pyinotify-doc python-scipy-doc python3-dap 2023-09-23T18:01:24.455 INFO:teuthology.orchestra.run.smithi173.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2023-09-23T18:01:24.455 INFO:teuthology.orchestra.run.smithi173.stdout: python-webtest-doc python-werkzeug-doc python3-termcolor python3-watchdog 2023-09-23T18:01:24.455 INFO:teuthology.orchestra.run.smithi173.stdout:Recommended packages: 2023-09-23T18:01:24.455 INFO:teuthology.orchestra.run.smithi173.stdout: btrfs-tools 2023-09-23T18:01:24.560 INFO:teuthology.orchestra.run.smithi098.stdout:The following packages were automatically installed and are no longer required: 2023-09-23T18:01:24.561 INFO:teuthology.orchestra.run.smithi098.stdout: libboost-iostreams1.71.0 libboost-thread1.71.0 2023-09-23T18:01:24.561 INFO:teuthology.orchestra.run.smithi098.stdout:Use 'sudo apt autoremove' to remove them. 2023-09-23T18:01:24.561 INFO:teuthology.orchestra.run.smithi098.stdout:The following additional packages will be installed: 2023-09-23T18:01:24.561 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2023-09-23T18:01:24.562 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq libjaeger 2023-09-23T18:01:24.562 INFO:teuthology.orchestra.run.smithi098.stdout: libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 libleveldb1d 2023-09-23T18:01:24.562 INFO:teuthology.orchestra.run.smithi098.stdout: liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 libradosstriper1 2023-09-23T18:01:24.562 INFO:teuthology.orchestra.run.smithi098.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli python-pastedeploy-tpl 2023-09-23T18:01:24.562 INFO:teuthology.orchestra.run.smithi098.stdout: python3-atomicwrites python3-attr python3-bcrypt python3-blinker python3-bs4 2023-09-23T18:01:24.562 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common 2023-09-23T18:01:24.563 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cherrypy3 python3-cryptography python3-decorator python3-google-auth 2023-09-23T18:01:24.563 INFO:teuthology.orchestra.run.smithi098.stdout: python3-html5lib python3-jinja2 python3-joblib python3-jwt 2023-09-23T18:01:24.563 INFO:teuthology.orchestra.run.smithi098.stdout: python3-kubernetes python3-logutils python3-lxml python3-mako 2023-09-23T18:01:24.563 INFO:teuthology.orchestra.run.smithi098.stdout: python3-markupsafe python3-oauthlib python3-openssl python3-packaging 2023-09-23T18:01:24.563 INFO:teuthology.orchestra.run.smithi098.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2023-09-23T18:01:24.563 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pluggy python3-prettytable python3-psutil python3-py python3-pyasn1 2023-09-23T18:01:24.563 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-repoze.lru 2023-09-23T18:01:24.563 INFO:teuthology.orchestra.run.smithi098.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-scipy 2023-09-23T18:01:24.563 INFO:teuthology.orchestra.run.smithi098.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2023-09-23T18:01:24.563 INFO:teuthology.orchestra.run.smithi098.stdout: python3-sklearn python3-sklearn-lib python3-soupsieve python3-tempita 2023-09-23T18:01:24.563 INFO:teuthology.orchestra.run.smithi098.stdout: python3-waitress python3-wcwidth python3-webencodings python3-webob 2023-09-23T18:01:24.563 INFO:teuthology.orchestra.run.smithi098.stdout: python3-websocket python3-webtest python3-werkzeug socat xmlstarlet 2023-09-23T18:01:24.564 INFO:teuthology.orchestra.run.smithi098.stdout:Suggested packages: 2023-09-23T18:01:24.564 INFO:teuthology.orchestra.run.smithi098.stdout: python3-influxdb python-attr-doc python-blinker-doc python-cryptography-doc 2023-09-23T18:01:24.564 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cryptography-vectors python3-genshi python-jinja2-doc python3-crypto 2023-09-23T18:01:24.564 INFO:teuthology.orchestra.run.smithi098.stdout: python3-lxml-dbg python-lxml-doc python3-beaker python-mako-doc 2023-09-23T18:01:24.564 INFO:teuthology.orchestra.run.smithi098.stdout: python-openssl-doc python3-openssl-dbg httpd-wsgi libapache2-mod-python 2023-09-23T18:01:24.564 INFO:teuthology.orchestra.run.smithi098.stdout: libapache2-mod-scgi libjs-mochikit python-pecan-doc python-psutil-doc 2023-09-23T18:01:24.564 INFO:teuthology.orchestra.run.smithi098.stdout: subversion python-pyinotify-doc python-scipy-doc python3-dap 2023-09-23T18:01:24.564 INFO:teuthology.orchestra.run.smithi098.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2023-09-23T18:01:24.564 INFO:teuthology.orchestra.run.smithi098.stdout: python-webtest-doc python-werkzeug-doc python3-termcolor python3-watchdog 2023-09-23T18:01:24.564 INFO:teuthology.orchestra.run.smithi098.stdout:Recommended packages: 2023-09-23T18:01:24.565 INFO:teuthology.orchestra.run.smithi098.stdout: btrfs-tools 2023-09-23T18:01:24.570 INFO:teuthology.orchestra.run.smithi173.stdout:The following NEW packages will be installed: 2023-09-23T18:01:24.570 INFO:teuthology.orchestra.run.smithi173.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2023-09-23T18:01:24.570 INFO:teuthology.orchestra.run.smithi173.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2023-09-23T18:01:24.570 INFO:teuthology.orchestra.run.smithi173.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test cephadm jq libcephfs-dev 2023-09-23T18:01:24.570 INFO:teuthology.orchestra.run.smithi173.stdout: libcephfs2 libjaeger libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 2023-09-23T18:01:24.570 INFO:teuthology.orchestra.run.smithi173.stdout: libleveldb1d liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 2023-09-23T18:01:24.571 INFO:teuthology.orchestra.run.smithi173.stdout: libradosstriper1 librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli 2023-09-23T18:01:24.571 INFO:teuthology.orchestra.run.smithi173.stdout: python-pastedeploy-tpl python3-atomicwrites python3-attr python3-bcrypt 2023-09-23T18:01:24.571 INFO:teuthology.orchestra.run.smithi173.stdout: python3-blinker python3-bs4 python3-cachetools python3-ceph-argparse 2023-09-23T18:01:24.571 INFO:teuthology.orchestra.run.smithi173.stdout: python3-ceph-common python3-cephfs python3-cherrypy3 python3-cryptography 2023-09-23T18:01:24.571 INFO:teuthology.orchestra.run.smithi173.stdout: python3-decorator python3-google-auth python3-html5lib python3-jinja2 2023-09-23T18:01:24.571 INFO:teuthology.orchestra.run.smithi173.stdout: python3-joblib python3-jwt python3-kubernetes python3-logutils python3-lxml 2023-09-23T18:01:24.571 INFO:teuthology.orchestra.run.smithi173.stdout: python3-mako python3-markupsafe python3-oauthlib python3-openssl 2023-09-23T18:01:24.571 INFO:teuthology.orchestra.run.smithi173.stdout: python3-packaging python3-paste python3-pastedeploy python3-pastescript 2023-09-23T18:01:24.571 INFO:teuthology.orchestra.run.smithi173.stdout: python3-pecan python3-pluggy python3-prettytable python3-psutil python3-py 2023-09-23T18:01:24.572 INFO:teuthology.orchestra.run.smithi173.stdout: python3-pyasn1 python3-pyasn1-modules python3-pyinotify python3-pytest 2023-09-23T18:01:24.572 INFO:teuthology.orchestra.run.smithi173.stdout: python3-rados python3-rbd python3-repoze.lru python3-requests-oauthlib 2023-09-23T18:01:24.572 INFO:teuthology.orchestra.run.smithi173.stdout: python3-rgw python3-routes python3-rsa python3-scipy python3-simplegeneric 2023-09-23T18:01:24.572 INFO:teuthology.orchestra.run.smithi173.stdout: python3-simplejson python3-singledispatch python3-sklearn 2023-09-23T18:01:24.572 INFO:teuthology.orchestra.run.smithi173.stdout: python3-sklearn-lib python3-soupsieve python3-tempita python3-waitress 2023-09-23T18:01:24.572 INFO:teuthology.orchestra.run.smithi173.stdout: python3-wcwidth python3-webencodings python3-webob python3-websocket 2023-09-23T18:01:24.572 INFO:teuthology.orchestra.run.smithi173.stdout: python3-webtest python3-werkzeug radosgw rbd-fuse socat xmlstarlet 2023-09-23T18:01:24.572 INFO:teuthology.orchestra.run.smithi173.stdout:The following packages will be upgraded: 2023-09-23T18:01:24.573 INFO:teuthology.orchestra.run.smithi173.stdout: librados2 librbd1 2023-09-23T18:01:24.645 INFO:teuthology.orchestra.run.smithi173.stdout:2 upgraded, 98 newly installed, 0 to remove and 248 not upgraded. 2023-09-23T18:01:24.646 INFO:teuthology.orchestra.run.smithi173.stdout:Need to get 174 MB of archives. 2023-09-23T18:01:24.646 INFO:teuthology.orchestra.run.smithi173.stdout:After this operation, 804 MB of additional disk space will be used. 2023-09-23T18:01:24.646 INFO:teuthology.orchestra.run.smithi173.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 liblttng-ust0 amd64 2.11.0-1 [160 kB] 2023-09-23T18:01:24.672 INFO:teuthology.orchestra.run.smithi098.stdout:The following NEW packages will be installed: 2023-09-23T18:01:24.672 INFO:teuthology.orchestra.run.smithi098.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2023-09-23T18:01:24.672 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2023-09-23T18:01:24.672 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test cephadm jq libcephfs-dev 2023-09-23T18:01:24.673 INFO:teuthology.orchestra.run.smithi098.stdout: libcephfs2 libjaeger libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 2023-09-23T18:01:24.673 INFO:teuthology.orchestra.run.smithi098.stdout: libleveldb1d liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 2023-09-23T18:01:24.673 INFO:teuthology.orchestra.run.smithi098.stdout: libradosstriper1 librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli 2023-09-23T18:01:24.673 INFO:teuthology.orchestra.run.smithi098.stdout: python-pastedeploy-tpl python3-atomicwrites python3-attr python3-bcrypt 2023-09-23T18:01:24.674 INFO:teuthology.orchestra.run.smithi098.stdout: python3-blinker python3-bs4 python3-cachetools python3-ceph-argparse 2023-09-23T18:01:24.674 INFO:teuthology.orchestra.run.smithi098.stdout: python3-ceph-common python3-cephfs python3-cherrypy3 python3-cryptography 2023-09-23T18:01:24.674 INFO:teuthology.orchestra.run.smithi098.stdout: python3-decorator python3-google-auth python3-html5lib python3-jinja2 2023-09-23T18:01:24.674 INFO:teuthology.orchestra.run.smithi098.stdout: python3-joblib python3-jwt python3-kubernetes python3-logutils python3-lxml 2023-09-23T18:01:24.674 INFO:teuthology.orchestra.run.smithi098.stdout: python3-mako python3-markupsafe python3-oauthlib python3-openssl 2023-09-23T18:01:24.674 INFO:teuthology.orchestra.run.smithi098.stdout: python3-packaging python3-paste python3-pastedeploy python3-pastescript 2023-09-23T18:01:24.674 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pecan python3-pluggy python3-prettytable python3-psutil python3-py 2023-09-23T18:01:24.674 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pyasn1 python3-pyasn1-modules python3-pyinotify python3-pytest 2023-09-23T18:01:24.674 INFO:teuthology.orchestra.run.smithi098.stdout: python3-rados python3-rbd python3-repoze.lru python3-requests-oauthlib 2023-09-23T18:01:24.674 INFO:teuthology.orchestra.run.smithi098.stdout: python3-rgw python3-routes python3-rsa python3-scipy python3-simplegeneric 2023-09-23T18:01:24.674 INFO:teuthology.orchestra.run.smithi098.stdout: python3-simplejson python3-singledispatch python3-sklearn 2023-09-23T18:01:24.675 INFO:teuthology.orchestra.run.smithi098.stdout: python3-sklearn-lib python3-soupsieve python3-tempita python3-waitress 2023-09-23T18:01:24.675 INFO:teuthology.orchestra.run.smithi098.stdout: python3-wcwidth python3-webencodings python3-webob python3-websocket 2023-09-23T18:01:24.675 INFO:teuthology.orchestra.run.smithi098.stdout: python3-webtest python3-werkzeug radosgw rbd-fuse socat xmlstarlet 2023-09-23T18:01:24.675 INFO:teuthology.orchestra.run.smithi098.stdout:The following packages will be upgraded: 2023-09-23T18:01:24.675 INFO:teuthology.orchestra.run.smithi098.stdout: librados2 librbd1 2023-09-23T18:01:24.777 INFO:teuthology.orchestra.run.smithi173.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-prettytable all 0.7.2-5 [20.1 kB] 2023-09-23T18:01:24.777 INFO:teuthology.orchestra.run.smithi173.stdout:Get:3 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 libjaeger amd64 16.2.14-37-g2e1f1015-1focal [3,952 B] 2023-09-23T18:01:24.778 INFO:teuthology.orchestra.run.smithi173.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 liblua5.3-0 amd64 5.3.3-1.1ubuntu2 [116 kB] 2023-09-23T18:01:24.798 INFO:teuthology.orchestra.run.smithi173.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 librabbitmq4 amd64 0.10.0-1 [35.8 kB] 2023-09-23T18:01:24.810 INFO:teuthology.orchestra.run.smithi173.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal/universe amd64 librdkafka1 amd64 1.2.1-1ubuntu1 [410 kB] 2023-09-23T18:01:24.859 INFO:teuthology.orchestra.run.smithi173.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal/main amd64 libleveldb1d amd64 1.22-3ubuntu2 [139 kB] 2023-09-23T18:01:24.870 INFO:teuthology.orchestra.run.smithi098.stdout:2 upgraded, 98 newly installed, 0 to remove and 248 not upgraded. 2023-09-23T18:01:24.871 INFO:teuthology.orchestra.run.smithi098.stdout:Need to get 174 MB of archives. 2023-09-23T18:01:24.871 INFO:teuthology.orchestra.run.smithi098.stdout:After this operation, 804 MB of additional disk space will be used. 2023-09-23T18:01:24.871 INFO:teuthology.orchestra.run.smithi098.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 liblttng-ust0 amd64 2.11.0-1 [160 kB] 2023-09-23T18:01:24.872 INFO:teuthology.orchestra.run.smithi173.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal/main amd64 liboath0 amd64 2.6.1-1.3 [44.7 kB] 2023-09-23T18:01:24.874 INFO:teuthology.orchestra.run.smithi173.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-cryptography amd64 2.8-3ubuntu0.1 [211 kB] 2023-09-23T18:01:24.876 INFO:teuthology.orchestra.run.smithi173.stdout:Get:10 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 librbd1 amd64 16.2.14-37-g2e1f1015-1focal [3,385 kB] 2023-09-23T18:01:24.886 INFO:teuthology.orchestra.run.smithi173.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-openssl all 19.0.0-1build1 [43.3 kB] 2023-09-23T18:01:24.888 INFO:teuthology.orchestra.run.smithi173.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bcrypt amd64 3.1.7-2ubuntu1 [30.4 kB] 2023-09-23T18:01:24.889 INFO:teuthology.orchestra.run.smithi173.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cherrypy3 all 8.9.1-6 [160 kB] 2023-09-23T18:01:24.896 INFO:teuthology.orchestra.run.smithi173.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-jwt all 1.7.1-2ubuntu2.1 [18.0 kB] 2023-09-23T18:01:24.896 INFO:teuthology.orchestra.run.smithi173.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-logutils all 0.3.3-7 [17.3 kB] 2023-09-23T18:01:24.898 INFO:teuthology.orchestra.run.smithi173.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-markupsafe amd64 1.1.0-1build2 [13.9 kB] 2023-09-23T18:01:24.902 INFO:teuthology.orchestra.run.smithi173.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-mako all 1.1.0+ds1-1ubuntu2.1 [59.2 kB] 2023-09-23T18:01:24.910 INFO:teuthology.orchestra.run.smithi098.stdout:Get:2 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 libjaeger amd64 16.2.14-37-g2e1f1015-1focal [3,952 B] 2023-09-23T18:01:24.930 INFO:teuthology.orchestra.run.smithi173.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2023-09-23T18:01:24.931 INFO:teuthology.orchestra.run.smithi173.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-singledispatch all 3.4.0.3-2build1 [7,128 B] 2023-09-23T18:01:24.932 INFO:teuthology.orchestra.run.smithi173.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webob all 1:1.8.5-2 [84.4 kB] 2023-09-23T18:01:24.934 INFO:teuthology.orchestra.run.smithi173.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-soupsieve all 1.9.5+dfsg-1 [29.1 kB] 2023-09-23T18:01:24.935 INFO:teuthology.orchestra.run.smithi173.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bs4 all 4.8.2-1 [83.0 kB] 2023-09-23T18:01:24.937 INFO:teuthology.orchestra.run.smithi173.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-waitress all 1.4.1-1ubuntu0.1 [81.4 kB] 2023-09-23T18:01:24.940 INFO:teuthology.orchestra.run.smithi173.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-tempita all 0.5.2-6 [14.8 kB] 2023-09-23T18:01:24.941 INFO:teuthology.orchestra.run.smithi173.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-paste all 3.4.0+dfsg1-1 [456 kB] 2023-09-23T18:01:24.950 INFO:teuthology.orchestra.run.smithi173.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal/main amd64 python-pastedeploy-tpl all 2.1.0-1 [5,016 B] 2023-09-23T18:01:24.969 INFO:teuthology.orchestra.run.smithi173.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastedeploy all 2.1.0-1 [26.5 kB] 2023-09-23T18:01:24.970 INFO:teuthology.orchestra.run.smithi173.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webtest all 2.0.32-2 [28.2 kB] 2023-09-23T18:01:24.971 INFO:teuthology.orchestra.run.smithi173.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pecan all 1.3.3-3ubuntu1 [85.8 kB] 2023-09-23T18:01:24.973 INFO:teuthology.orchestra.run.smithi173.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-werkzeug all 0.16.1+dfsg1-2ubuntu0.1 [183 kB] 2023-09-23T18:01:25.000 INFO:teuthology.orchestra.run.smithi173.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jinja2 all 2.10.1-2 [95.5 kB] 2023-09-23T18:01:25.003 INFO:teuthology.orchestra.run.smithi173.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2023-09-23T18:01:25.004 INFO:teuthology.orchestra.run.smithi173.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libjs-underscore all 1.9.1~dfsg-1ubuntu0.20.04.1 [99.5 kB] 2023-09-23T18:01:25.006 INFO:teuthology.orchestra.run.smithi173.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal/main amd64 libjs-sphinxdoc all 1.8.5-7ubuntu3 [97.1 kB] 2023-09-23T18:01:25.008 INFO:teuthology.orchestra.run.smithi173.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-routes all 2.4.1-2 [88.1 kB] 2023-09-23T18:01:25.010 INFO:teuthology.orchestra.run.smithi173.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-decorator all 4.4.2-0ubuntu1 [10.3 kB] 2023-09-23T18:01:25.015 INFO:teuthology.orchestra.run.smithi098.stdout:Get:3 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 librbd1 amd64 16.2.14-37-g2e1f1015-1focal [3,385 kB] 2023-09-23T18:01:25.033 INFO:teuthology.orchestra.run.smithi173.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal/universe amd64 liblbfgsb0 amd64 3.0+dfsg.3-7build1 [29.5 kB] 2023-09-23T18:01:25.034 INFO:teuthology.orchestra.run.smithi173.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-scipy amd64 1.3.3-3build1 [11.1 MB] 2023-09-23T18:01:25.267 INFO:teuthology.orchestra.run.smithi098.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-prettytable all 0.7.2-5 [20.1 kB] 2023-09-23T18:01:25.274 INFO:teuthology.orchestra.run.smithi098.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 liblua5.3-0 amd64 5.3.3-1.1ubuntu2 [116 kB] 2023-09-23T18:01:25.292 INFO:teuthology.orchestra.run.smithi173.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn-lib amd64 0.22.2.post1+dfsg-5 [2,358 kB] 2023-09-23T18:01:25.318 INFO:teuthology.orchestra.run.smithi098.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal/main amd64 librabbitmq4 amd64 0.10.0-1 [35.8 kB] 2023-09-23T18:01:25.320 INFO:teuthology.orchestra.run.smithi173.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-joblib all 0.14.0-3 [192 kB] 2023-09-23T18:01:25.324 INFO:teuthology.orchestra.run.smithi173.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn all 0.22.2.post1+dfsg-5 [1,773 kB] 2023-09-23T18:01:25.331 INFO:teuthology.orchestra.run.smithi098.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal/universe amd64 librdkafka1 amd64 1.2.1-1ubuntu1 [410 kB] 2023-09-23T18:01:25.361 INFO:teuthology.orchestra.run.smithi173.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cachetools all 4.0.0-1 [9,820 B] 2023-09-23T18:01:25.361 INFO:teuthology.orchestra.run.smithi173.stdout:Get:43 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1 all 0.4.2-3build1 [46.7 kB] 2023-09-23T18:01:25.362 INFO:teuthology.orchestra.run.smithi173.stdout:Get:44 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1-modules all 0.2.1-0.2build1 [32.9 kB] 2023-09-23T18:01:25.362 INFO:teuthology.orchestra.run.smithi173.stdout:Get:45 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-rsa all 4.0-3 [24.7 kB] 2023-09-23T18:01:25.363 INFO:teuthology.orchestra.run.smithi173.stdout:Get:46 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-google-auth all 1.5.1-2 [35.6 kB] 2023-09-23T18:01:25.363 INFO:teuthology.orchestra.run.smithi173.stdout:Get:47 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-blinker all 1.4+dfsg1-0.3ubuntu1 [13.2 kB] 2023-09-23T18:01:25.364 INFO:teuthology.orchestra.run.smithi173.stdout:Get:48 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-oauthlib all 3.1.0-1ubuntu2 [84.8 kB] 2023-09-23T18:01:25.386 INFO:teuthology.orchestra.run.smithi173.stdout:Get:49 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-requests-oauthlib all 1.0.0-1.1build1 [16.8 kB] 2023-09-23T18:01:25.387 INFO:teuthology.orchestra.run.smithi173.stdout:Get:50 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-websocket all 0.53.0-2ubuntu1 [32.3 kB] 2023-09-23T18:01:25.419 INFO:teuthology.orchestra.run.smithi173.stdout:Get:51 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-kubernetes all 7.0.0~a1-2 [271 kB] 2023-09-23T18:01:25.424 INFO:teuthology.orchestra.run.smithi173.stdout:Get:52 http://archive.ubuntu.com/ubuntu focal/universe amd64 libonig5 amd64 6.9.4-1 [142 kB] 2023-09-23T18:01:25.427 INFO:teuthology.orchestra.run.smithi173.stdout:Get:53 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libjq1 amd64 1.6-1ubuntu0.20.04.1 [121 kB] 2023-09-23T18:01:25.428 INFO:teuthology.orchestra.run.smithi173.stdout:Get:54 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 jq amd64 1.6-1ubuntu0.20.04.1 [50.2 kB] 2023-09-23T18:01:25.429 INFO:teuthology.orchestra.run.smithi173.stdout:Get:55 http://archive.ubuntu.com/ubuntu focal/main amd64 socat amd64 1.7.3.3-2 [323 kB] 2023-09-23T18:01:25.432 INFO:teuthology.orchestra.run.smithi173.stdout:Get:56 http://archive.ubuntu.com/ubuntu focal/universe amd64 xmlstarlet amd64 1.6.1-2build1 [267 kB] 2023-09-23T18:01:25.435 INFO:teuthology.orchestra.run.smithi173.stdout:Get:57 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 nvme-cli amd64 1.9-1ubuntu0.1 [283 kB] 2023-09-23T18:01:25.439 INFO:teuthology.orchestra.run.smithi173.stdout:Get:58 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-atomicwrites all 1.1.5-2build1 [7,016 B] 2023-09-23T18:01:25.458 INFO:teuthology.orchestra.run.smithi098.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal/main amd64 libleveldb1d amd64 1.22-3ubuntu2 [139 kB] 2023-09-23T18:01:25.459 INFO:teuthology.orchestra.run.smithi173.stdout:Get:59 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-attr all 19.3.0-2 [33.9 kB] 2023-09-23T18:01:25.461 INFO:teuthology.orchestra.run.smithi173.stdout:Get:60 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webencodings all 0.5.1-1ubuntu1 [11.0 kB] 2023-09-23T18:01:25.481 INFO:teuthology.orchestra.run.smithi098.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal/main amd64 liboath0 amd64 2.6.1-1.3 [44.7 kB] 2023-09-23T18:01:25.488 INFO:teuthology.orchestra.run.smithi098.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-cryptography amd64 2.8-3ubuntu0.1 [211 kB] 2023-09-23T18:01:25.492 INFO:teuthology.orchestra.run.smithi173.stdout:Get:61 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-html5lib all 1.0.1-2 [84.3 kB] 2023-09-23T18:01:25.494 INFO:teuthology.orchestra.run.smithi173.stdout:Get:62 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-lxml amd64 4.5.0-1ubuntu0.5 [1,384 kB] 2023-09-23T18:01:25.514 INFO:teuthology.orchestra.run.smithi173.stdout:Get:63 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-packaging all 20.3-1 [26.8 kB] 2023-09-23T18:01:25.514 INFO:teuthology.orchestra.run.smithi173.stdout:Get:64 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2023-09-23T18:01:25.516 INFO:teuthology.orchestra.run.smithi173.stdout:Get:65 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pluggy all 0.13.0-2 [18.4 kB] 2023-09-23T18:01:25.516 INFO:teuthology.orchestra.run.smithi173.stdout:Get:66 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-psutil amd64 5.5.1-1ubuntu4 [141 kB] 2023-09-23T18:01:25.518 INFO:teuthology.orchestra.run.smithi173.stdout:Get:67 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python3-py all 1.8.1-1ubuntu0.1 [65.6 kB] 2023-09-23T18:01:25.518 INFO:teuthology.orchestra.run.smithi098.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-openssl all 19.0.0-1build1 [43.3 kB] 2023-09-23T18:01:25.519 INFO:teuthology.orchestra.run.smithi173.stdout:Get:68 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyinotify all 0.9.6-1.2ubuntu1 [24.8 kB] 2023-09-23T18:01:25.524 INFO:teuthology.orchestra.run.smithi098.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bcrypt amd64 3.1.7-2ubuntu1 [30.4 kB] 2023-09-23T18:01:25.526 INFO:teuthology.orchestra.run.smithi098.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cherrypy3 all 8.9.1-6 [160 kB] 2023-09-23T18:01:25.536 INFO:teuthology.orchestra.run.smithi173.stdout:Get:69 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-wcwidth all 0.1.8+dfsg1-3 [17.4 kB] 2023-09-23T18:01:25.536 INFO:teuthology.orchestra.run.smithi173.stdout:Get:70 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pytest all 4.6.9-1 [178 kB] 2023-09-23T18:01:25.569 INFO:teuthology.orchestra.run.smithi173.stdout:Get:71 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplejson amd64 3.16.0-2ubuntu2 [50.8 kB] 2023-09-23T18:01:25.705 INFO:teuthology.orchestra.run.smithi098.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-jwt all 1.7.1-2ubuntu2.1 [18.0 kB] 2023-09-23T18:01:25.873 INFO:teuthology.orchestra.run.smithi098.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-logutils all 0.3.3-7 [17.3 kB] 2023-09-23T18:01:25.902 INFO:teuthology.orchestra.run.smithi098.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-markupsafe amd64 1.1.0-1build2 [13.9 kB] 2023-09-23T18:01:25.926 INFO:teuthology.orchestra.run.smithi098.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-mako all 1.1.0+ds1-1ubuntu2.1 [59.2 kB] 2023-09-23T18:01:26.043 INFO:teuthology.orchestra.run.smithi098.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2023-09-23T18:01:26.055 INFO:teuthology.orchestra.run.smithi098.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-singledispatch all 3.4.0.3-2build1 [7,128 B] 2023-09-23T18:01:26.060 INFO:teuthology.orchestra.run.smithi098.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webob all 1:1.8.5-2 [84.4 kB] 2023-09-23T18:01:26.090 INFO:teuthology.orchestra.run.smithi173.stdout:Get:72 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 librados2 amd64 16.2.14-37-g2e1f1015-1focal [3,391 kB] 2023-09-23T18:01:26.127 INFO:teuthology.orchestra.run.smithi098.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-soupsieve all 1.9.5+dfsg-1 [29.1 kB] 2023-09-23T18:01:26.143 INFO:teuthology.orchestra.run.smithi098.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bs4 all 4.8.2-1 [83.0 kB] 2023-09-23T18:01:26.178 INFO:teuthology.orchestra.run.smithi098.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-waitress all 1.4.1-1ubuntu0.1 [81.4 kB] 2023-09-23T18:01:26.179 INFO:teuthology.orchestra.run.smithi098.stdout:Get:24 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 librados2 amd64 16.2.14-37-g2e1f1015-1focal [3,391 kB] 2023-09-23T18:01:26.204 INFO:teuthology.orchestra.run.smithi098.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-tempita all 0.5.2-6 [14.8 kB] 2023-09-23T18:01:26.372 INFO:teuthology.orchestra.run.smithi098.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-paste all 3.4.0+dfsg1-1 [456 kB] 2023-09-23T18:01:26.642 INFO:teuthology.orchestra.run.smithi173.stdout:Get:73 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 libcephfs2 amd64 16.2.14-37-g2e1f1015-1focal [729 kB] 2023-09-23T18:01:26.744 INFO:teuthology.orchestra.run.smithi173.stdout:Get:74 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 python3-rados amd64 16.2.14-37-g2e1f1015-1focal [413 kB] 2023-09-23T18:01:26.770 INFO:teuthology.orchestra.run.smithi098.stdout:Get:27 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 libcephfs2 amd64 16.2.14-37-g2e1f1015-1focal [729 kB] 2023-09-23T18:01:26.797 INFO:teuthology.orchestra.run.smithi173.stdout:Get:75 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-argparse all 16.2.14-37-g2e1f1015-1focal [22.2 kB] 2023-09-23T18:01:26.798 INFO:teuthology.orchestra.run.smithi173.stdout:Get:76 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 python3-cephfs amd64 16.2.14-37-g2e1f1015-1focal [212 kB] 2023-09-23T18:01:26.808 INFO:teuthology.orchestra.run.smithi173.stdout:Get:77 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-common all 16.2.14-37-g2e1f1015-1focal [38.4 kB] 2023-09-23T18:01:26.809 INFO:teuthology.orchestra.run.smithi173.stdout:Get:78 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 python3-rbd amd64 16.2.14-37-g2e1f1015-1focal [386 kB] 2023-09-23T18:01:26.833 INFO:teuthology.orchestra.run.smithi098.stdout:Get:28 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 python3-rados amd64 16.2.14-37-g2e1f1015-1focal [413 kB] 2023-09-23T18:01:26.860 INFO:teuthology.orchestra.run.smithi173.stdout:Get:79 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 librgw2 amd64 16.2.14-37-g2e1f1015-1focal [3,689 kB] 2023-09-23T18:01:26.888 INFO:teuthology.orchestra.run.smithi098.stdout:Get:29 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-argparse all 16.2.14-37-g2e1f1015-1focal [22.2 kB] 2023-09-23T18:01:26.889 INFO:teuthology.orchestra.run.smithi098.stdout:Get:30 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 python3-cephfs amd64 16.2.14-37-g2e1f1015-1focal [212 kB] 2023-09-23T18:01:26.897 INFO:teuthology.orchestra.run.smithi098.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal/main amd64 python-pastedeploy-tpl all 2.1.0-1 [5,016 B] 2023-09-23T18:01:26.898 INFO:teuthology.orchestra.run.smithi098.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastedeploy all 2.1.0-1 [26.5 kB] 2023-09-23T18:01:26.901 INFO:teuthology.orchestra.run.smithi098.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webtest all 2.0.32-2 [28.2 kB] 2023-09-23T18:01:26.905 INFO:teuthology.orchestra.run.smithi098.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pecan all 1.3.3-3ubuntu1 [85.8 kB] 2023-09-23T18:01:26.916 INFO:teuthology.orchestra.run.smithi098.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-werkzeug all 0.16.1+dfsg1-2ubuntu0.1 [183 kB] 2023-09-23T18:01:26.935 INFO:teuthology.orchestra.run.smithi098.stdout:Get:36 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-common all 16.2.14-37-g2e1f1015-1focal [38.4 kB] 2023-09-23T18:01:26.937 INFO:teuthology.orchestra.run.smithi098.stdout:Get:37 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 python3-rbd amd64 16.2.14-37-g2e1f1015-1focal [386 kB] 2023-09-23T18:01:26.939 INFO:teuthology.orchestra.run.smithi098.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jinja2 all 2.10.1-2 [95.5 kB] 2023-09-23T18:01:26.954 INFO:teuthology.orchestra.run.smithi098.stdout:Get:39 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 librgw2 amd64 16.2.14-37-g2e1f1015-1focal [3,689 kB] 2023-09-23T18:01:26.954 INFO:teuthology.orchestra.run.smithi098.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2023-09-23T18:01:26.954 INFO:teuthology.orchestra.run.smithi098.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libjs-underscore all 1.9.1~dfsg-1ubuntu0.20.04.1 [99.5 kB] 2023-09-23T18:01:26.988 INFO:teuthology.orchestra.run.smithi098.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal/main amd64 libjs-sphinxdoc all 1.8.5-7ubuntu3 [97.1 kB] 2023-09-23T18:01:27.000 INFO:teuthology.orchestra.run.smithi098.stdout:Get:43 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-routes all 2.4.1-2 [88.1 kB] 2023-09-23T18:01:27.175 INFO:teuthology.orchestra.run.smithi098.stdout:Get:44 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-decorator all 4.4.2-0ubuntu1 [10.3 kB] 2023-09-23T18:01:27.241 INFO:teuthology.orchestra.run.smithi173.stdout:Get:80 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 python3-rgw amd64 16.2.14-37-g2e1f1015-1focal [116 kB] 2023-09-23T18:01:27.246 INFO:teuthology.orchestra.run.smithi173.stdout:Get:81 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 libradosstriper1 amd64 16.2.14-37-g2e1f1015-1focal [448 kB] 2023-09-23T18:01:27.258 INFO:teuthology.orchestra.run.smithi098.stdout:Get:45 http://archive.ubuntu.com/ubuntu focal/universe amd64 liblbfgsb0 amd64 3.0+dfsg.3-7build1 [29.5 kB] 2023-09-23T18:01:27.264 INFO:teuthology.orchestra.run.smithi173.stdout:Get:82 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-common amd64 16.2.14-37-g2e1f1015-1focal [22.6 MB] 2023-09-23T18:01:27.337 INFO:teuthology.orchestra.run.smithi098.stdout:Get:46 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 python3-rgw amd64 16.2.14-37-g2e1f1015-1focal [116 kB] 2023-09-23T18:01:27.341 INFO:teuthology.orchestra.run.smithi098.stdout:Get:47 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 libradosstriper1 amd64 16.2.14-37-g2e1f1015-1focal [448 kB] 2023-09-23T18:01:27.342 INFO:teuthology.orchestra.run.smithi098.stdout:Get:48 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-scipy amd64 1.3.3-3build1 [11.1 MB] 2023-09-23T18:01:27.355 INFO:teuthology.orchestra.run.smithi098.stdout:Get:49 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-common amd64 16.2.14-37-g2e1f1015-1focal [22.6 MB] 2023-09-23T18:01:28.152 INFO:teuthology.orchestra.run.smithi098.stdout:Get:50 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn-lib amd64 0.22.2.post1+dfsg-5 [2,358 kB] 2023-09-23T18:01:28.168 INFO:teuthology.orchestra.run.smithi098.stdout:Get:51 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-joblib all 0.14.0-3 [192 kB] 2023-09-23T18:01:28.169 INFO:teuthology.orchestra.run.smithi098.stdout:Get:52 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn all 0.22.2.post1+dfsg-5 [1,773 kB] 2023-09-23T18:01:28.241 INFO:teuthology.orchestra.run.smithi098.stdout:Get:53 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cachetools all 4.0.0-1 [9,820 B] 2023-09-23T18:01:28.242 INFO:teuthology.orchestra.run.smithi098.stdout:Get:54 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1 all 0.4.2-3build1 [46.7 kB] 2023-09-23T18:01:28.243 INFO:teuthology.orchestra.run.smithi098.stdout:Get:55 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1-modules all 0.2.1-0.2build1 [32.9 kB] 2023-09-23T18:01:28.243 INFO:teuthology.orchestra.run.smithi098.stdout:Get:56 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-rsa all 4.0-3 [24.7 kB] 2023-09-23T18:01:28.244 INFO:teuthology.orchestra.run.smithi098.stdout:Get:57 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-google-auth all 1.5.1-2 [35.6 kB] 2023-09-23T18:01:28.409 INFO:teuthology.orchestra.run.smithi098.stdout:Get:58 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-blinker all 1.4+dfsg1-0.3ubuntu1 [13.2 kB] 2023-09-23T18:01:28.492 INFO:teuthology.orchestra.run.smithi098.stdout:Get:59 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-oauthlib all 3.1.0-1ubuntu2 [84.8 kB] 2023-09-23T18:01:28.673 INFO:teuthology.orchestra.run.smithi098.stdout:Get:60 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-requests-oauthlib all 1.0.0-1.1build1 [16.8 kB] 2023-09-23T18:01:28.689 INFO:teuthology.orchestra.run.smithi098.stdout:Get:61 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-websocket all 0.53.0-2ubuntu1 [32.3 kB] 2023-09-23T18:01:28.721 INFO:teuthology.orchestra.run.smithi098.stdout:Get:62 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-kubernetes all 7.0.0~a1-2 [271 kB] 2023-09-23T18:01:28.851 INFO:teuthology.orchestra.run.smithi173.stdout:Get:83 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-base amd64 16.2.14-37-g2e1f1015-1focal [5,913 kB] 2023-09-23T18:01:28.852 INFO:teuthology.orchestra.run.smithi098.stdout:Get:63 http://archive.ubuntu.com/ubuntu focal/universe amd64 libonig5 amd64 6.9.4-1 [142 kB] 2023-09-23T18:01:28.889 INFO:teuthology.orchestra.run.smithi098.stdout:Get:64 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libjq1 amd64 1.6-1ubuntu0.20.04.1 [121 kB] 2023-09-23T18:01:28.912 INFO:teuthology.orchestra.run.smithi098.stdout:Get:65 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 jq amd64 1.6-1ubuntu0.20.04.1 [50.2 kB] 2023-09-23T18:01:28.920 INFO:teuthology.orchestra.run.smithi098.stdout:Get:66 http://archive.ubuntu.com/ubuntu focal/main amd64 socat amd64 1.7.3.3-2 [323 kB] 2023-09-23T18:01:28.998 INFO:teuthology.orchestra.run.smithi098.stdout:Get:67 http://archive.ubuntu.com/ubuntu focal/universe amd64 xmlstarlet amd64 1.6.1-2build1 [267 kB] 2023-09-23T18:01:29.040 INFO:teuthology.orchestra.run.smithi098.stdout:Get:68 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 nvme-cli amd64 1.9-1ubuntu0.1 [283 kB] 2023-09-23T18:01:29.041 INFO:teuthology.orchestra.run.smithi098.stdout:Get:69 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-base amd64 16.2.14-37-g2e1f1015-1focal [5,913 kB] 2023-09-23T18:01:29.142 INFO:teuthology.orchestra.run.smithi173.stdout:Get:84 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-modules-core all 16.2.14-37-g2e1f1015-1focal [204 kB] 2023-09-23T18:01:29.143 INFO:teuthology.orchestra.run.smithi173.stdout:Get:85 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 libsqlite3-mod-ceph amd64 16.2.14-37-g2e1f1015-1focal [126 kB] 2023-09-23T18:01:29.145 INFO:teuthology.orchestra.run.smithi173.stdout:Get:86 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr amd64 16.2.14-37-g2e1f1015-1focal [1,536 kB] 2023-09-23T18:01:29.207 INFO:teuthology.orchestra.run.smithi098.stdout:Get:70 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-atomicwrites all 1.1.5-2build1 [7,016 B] 2023-09-23T18:01:29.226 INFO:teuthology.orchestra.run.smithi173.stdout:Get:87 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mon amd64 16.2.14-37-g2e1f1015-1focal [6,995 kB] 2023-09-23T18:01:29.290 INFO:teuthology.orchestra.run.smithi098.stdout:Get:71 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-attr all 19.3.0-2 [33.9 kB] 2023-09-23T18:01:29.334 INFO:teuthology.orchestra.run.smithi098.stdout:Get:72 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-modules-core all 16.2.14-37-g2e1f1015-1focal [204 kB] 2023-09-23T18:01:29.339 INFO:teuthology.orchestra.run.smithi098.stdout:Get:73 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 libsqlite3-mod-ceph amd64 16.2.14-37-g2e1f1015-1focal [126 kB] 2023-09-23T18:01:29.343 INFO:teuthology.orchestra.run.smithi098.stdout:Get:74 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr amd64 16.2.14-37-g2e1f1015-1focal [1,536 kB] 2023-09-23T18:01:29.391 INFO:teuthology.orchestra.run.smithi098.stdout:Get:75 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webencodings all 0.5.1-1ubuntu1 [11.0 kB] 2023-09-23T18:01:29.414 INFO:teuthology.orchestra.run.smithi098.stdout:Get:76 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-html5lib all 1.0.1-2 [84.3 kB] 2023-09-23T18:01:29.434 INFO:teuthology.orchestra.run.smithi098.stdout:Get:77 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mon amd64 16.2.14-37-g2e1f1015-1focal [6,995 kB] 2023-09-23T18:01:29.531 INFO:teuthology.orchestra.run.smithi098.stdout:Get:78 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-lxml amd64 4.5.0-1ubuntu0.5 [1,384 kB] 2023-09-23T18:01:29.661 INFO:teuthology.orchestra.run.smithi173.stdout:Get:88 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-osd amd64 16.2.14-37-g2e1f1015-1focal [26.2 MB] 2023-09-23T18:01:29.817 INFO:teuthology.orchestra.run.smithi098.stdout:Get:79 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-packaging all 20.3-1 [26.8 kB] 2023-09-23T18:01:29.819 INFO:teuthology.orchestra.run.smithi098.stdout:Get:80 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2023-09-23T18:01:29.823 INFO:teuthology.orchestra.run.smithi098.stdout:Get:81 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pluggy all 0.13.0-2 [18.4 kB] 2023-09-23T18:01:29.824 INFO:teuthology.orchestra.run.smithi098.stdout:Get:82 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-psutil amd64 5.5.1-1ubuntu4 [141 kB] 2023-09-23T18:01:29.834 INFO:teuthology.orchestra.run.smithi098.stdout:Get:83 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python3-py all 1.8.1-1ubuntu0.1 [65.6 kB] 2023-09-23T18:01:29.839 INFO:teuthology.orchestra.run.smithi098.stdout:Get:84 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyinotify all 0.9.6-1.2ubuntu1 [24.8 kB] 2023-09-23T18:01:29.978 INFO:teuthology.orchestra.run.smithi098.stdout:Get:85 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-osd amd64 16.2.14-37-g2e1f1015-1focal [26.2 MB] 2023-09-23T18:01:30.004 INFO:teuthology.orchestra.run.smithi098.stdout:Get:86 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-wcwidth all 0.1.8+dfsg1-3 [17.4 kB] 2023-09-23T18:01:30.171 INFO:teuthology.orchestra.run.smithi098.stdout:Get:87 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pytest all 4.6.9-1 [178 kB] 2023-09-23T18:01:30.424 INFO:teuthology.orchestra.run.smithi098.stdout:Get:88 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplejson amd64 3.16.0-2ubuntu2 [50.8 kB] 2023-09-23T18:01:31.431 INFO:teuthology.orchestra.run.smithi173.stdout:Get:89 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph amd64 16.2.14-37-g2e1f1015-1focal [4,048 B] 2023-09-23T18:01:31.431 INFO:teuthology.orchestra.run.smithi173.stdout:Get:90 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-fuse amd64 16.2.14-37-g2e1f1015-1focal [834 kB] 2023-09-23T18:01:31.437 INFO:teuthology.orchestra.run.smithi173.stdout:Get:91 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mds amd64 16.2.14-37-g2e1f1015-1focal [2,363 kB] 2023-09-23T18:01:31.525 INFO:teuthology.orchestra.run.smithi173.stdout:Get:92 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 cephadm amd64 16.2.14-37-g2e1f1015-1focal [76.4 kB] 2023-09-23T18:01:31.543 INFO:teuthology.orchestra.run.smithi173.stdout:Get:93 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-cephadm all 16.2.14-37-g2e1f1015-1focal [100 kB] 2023-09-23T18:01:31.546 INFO:teuthology.orchestra.run.smithi173.stdout:Get:94 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-dashboard all 16.2.14-37-g2e1f1015-1focal [2,823 kB] 2023-09-23T18:01:31.725 INFO:teuthology.orchestra.run.smithi173.stdout:Get:95 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-diskprediction-local all 16.2.14-37-g2e1f1015-1focal [7,891 kB] 2023-09-23T18:01:31.808 INFO:teuthology.orchestra.run.smithi098.stdout:Get:89 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph amd64 16.2.14-37-g2e1f1015-1focal [4,048 B] 2023-09-23T18:01:31.808 INFO:teuthology.orchestra.run.smithi098.stdout:Get:90 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-fuse amd64 16.2.14-37-g2e1f1015-1focal [834 kB] 2023-09-23T18:01:31.814 INFO:teuthology.orchestra.run.smithi098.stdout:Get:91 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mds amd64 16.2.14-37-g2e1f1015-1focal [2,363 kB] 2023-09-23T18:01:31.955 INFO:teuthology.orchestra.run.smithi098.stdout:Get:92 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 cephadm amd64 16.2.14-37-g2e1f1015-1focal [76.4 kB] 2023-09-23T18:01:31.958 INFO:teuthology.orchestra.run.smithi098.stdout:Get:93 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-cephadm all 16.2.14-37-g2e1f1015-1focal [100 kB] 2023-09-23T18:01:31.970 INFO:teuthology.orchestra.run.smithi098.stdout:Get:94 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-dashboard all 16.2.14-37-g2e1f1015-1focal [2,823 kB] 2023-09-23T18:01:32.160 INFO:teuthology.orchestra.run.smithi098.stdout:Get:95 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-diskprediction-local all 16.2.14-37-g2e1f1015-1focal [7,891 kB] 2023-09-23T18:01:32.204 INFO:teuthology.orchestra.run.smithi173.stdout:Get:96 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-k8sevents all 16.2.14-37-g2e1f1015-1focal [4,164 B] 2023-09-23T18:01:32.214 INFO:teuthology.orchestra.run.smithi173.stdout:Get:97 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-test amd64 16.2.14-37-g2e1f1015-1focal [49.3 MB] 2023-09-23T18:01:32.684 INFO:teuthology.orchestra.run.smithi098.stdout:Get:96 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-k8sevents all 16.2.14-37-g2e1f1015-1focal [4,164 B] 2023-09-23T18:01:32.684 INFO:teuthology.orchestra.run.smithi098.stdout:Get:97 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 ceph-test amd64 16.2.14-37-g2e1f1015-1focal [49.3 MB] 2023-09-23T18:01:35.534 INFO:teuthology.orchestra.run.smithi173.stdout:Get:98 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 libcephfs-dev amd64 16.2.14-37-g2e1f1015-1focal [21.5 kB] 2023-09-23T18:01:35.534 INFO:teuthology.orchestra.run.smithi173.stdout:Get:99 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 radosgw amd64 16.2.14-37-g2e1f1015-1focal [11.3 MB] 2023-09-23T18:01:36.079 INFO:teuthology.orchestra.run.smithi173.stdout:Get:100 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 rbd-fuse amd64 16.2.14-37-g2e1f1015-1focal [78.6 kB] 2023-09-23T18:01:36.278 INFO:teuthology.orchestra.run.smithi098.stdout:Get:98 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 libcephfs-dev amd64 16.2.14-37-g2e1f1015-1focal [21.5 kB] 2023-09-23T18:01:36.278 INFO:teuthology.orchestra.run.smithi098.stdout:Get:99 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 radosgw amd64 16.2.14-37-g2e1f1015-1focal [11.3 MB] 2023-09-23T18:01:36.612 INFO:teuthology.orchestra.run.smithi098.stdout:Get:100 https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/ubuntu/focal/flavors/default focal/main amd64 rbd-fuse amd64 16.2.14-37-g2e1f1015-1focal [78.6 kB] 2023-09-23T18:01:36.792 INFO:teuthology.orchestra.run.smithi173.stdout:Fetched 174 MB in 12s (15.0 MB/s) 2023-09-23T18:01:36.887 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libjaeger. 2023-09-23T18:01:36.933 INFO:teuthology.orchestra.run.smithi173.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 112648 files and directories currently installed.) 2023-09-23T18:01:36.937 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../00-libjaeger_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:36.977 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libjaeger (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:37.265 INFO:teuthology.orchestra.run.smithi098.stdout:Fetched 174 MB in 12s (14.5 MB/s) 2023-09-23T18:01:37.289 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package liblttng-ust0:amd64. 2023-09-23T18:01:37.303 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../01-liblttng-ust0_2.11.0-1_amd64.deb ... 2023-09-23T18:01:37.359 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package libjaeger. 2023-09-23T18:01:37.389 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking liblttng-ust0:amd64 (2.11.0-1) ... 2023-09-23T18:01:37.408 INFO:teuthology.orchestra.run.smithi098.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 112648 files and directories currently installed.) 2023-09-23T18:01:37.412 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../00-libjaeger_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:37.449 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking libjaeger (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:37.740 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../02-librbd1_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:37.743 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package liblttng-ust0:amd64. 2023-09-23T18:01:37.756 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../01-liblttng-ust0_2.11.0-1_amd64.deb ... 2023-09-23T18:01:37.835 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking liblttng-ust0:amd64 (2.11.0-1) ... 2023-09-23T18:01:37.868 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking librbd1 (16.2.14-37-g2e1f1015-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-09-23T18:01:38.154 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../02-librbd1_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:38.281 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking librbd1 (16.2.14-37-g2e1f1015-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-09-23T18:01:38.536 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../03-librados2_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:38.665 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking librados2 (16.2.14-37-g2e1f1015-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-09-23T18:01:38.965 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../03-librados2_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:39.118 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking librados2 (16.2.14-37-g2e1f1015-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-09-23T18:01:39.336 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libcephfs2. 2023-09-23T18:01:39.351 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../04-libcephfs2_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:39.394 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libcephfs2 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:39.722 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-rados. 2023-09-23T18:01:39.735 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../05-python3-rados_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:39.780 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-rados (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:39.806 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package libcephfs2. 2023-09-23T18:01:39.821 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../04-libcephfs2_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:39.864 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking libcephfs2 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:40.133 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-ceph-argparse. 2023-09-23T18:01:40.147 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../06-python3-ceph-argparse_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:01:40.191 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-ceph-argparse (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:40.191 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-rados. 2023-09-23T18:01:40.204 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../05-python3-rados_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:40.241 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-rados (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:40.503 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-cephfs. 2023-09-23T18:01:40.518 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../07-python3-cephfs_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:40.561 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-cephfs (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:40.653 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-ceph-argparse. 2023-09-23T18:01:40.666 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../06-python3-ceph-argparse_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:01:40.711 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-ceph-argparse (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:40.889 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-ceph-common. 2023-09-23T18:01:40.902 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../08-python3-ceph-common_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:01:40.939 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-ceph-common (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:41.039 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-cephfs. 2023-09-23T18:01:41.054 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../07-python3-cephfs_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:41.097 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-cephfs (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:41.259 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-prettytable. 2023-09-23T18:01:41.273 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../09-python3-prettytable_0.7.2-5_all.deb ... 2023-09-23T18:01:41.317 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-prettytable (0.7.2-5) ... 2023-09-23T18:01:41.463 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-ceph-common. 2023-09-23T18:01:41.477 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../08-python3-ceph-common_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:01:41.633 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-ceph-common (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:41.837 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-rbd. 2023-09-23T18:01:41.850 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../10-python3-rbd_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:41.887 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-rbd (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:41.987 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-prettytable. 2023-09-23T18:01:42.000 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../09-python3-prettytable_0.7.2-5_all.deb ... 2023-09-23T18:01:42.045 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-prettytable (0.7.2-5) ... 2023-09-23T18:01:42.240 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package liblua5.3-0:amd64. 2023-09-23T18:01:42.253 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../11-liblua5.3-0_5.3.3-1.1ubuntu2_amd64.deb ... 2023-09-23T18:01:42.298 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-09-23T18:01:42.347 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-rbd. 2023-09-23T18:01:42.361 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../10-python3-rbd_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:42.406 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-rbd (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:42.640 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package librabbitmq4:amd64. 2023-09-23T18:01:42.655 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../12-librabbitmq4_0.10.0-1_amd64.deb ... 2023-09-23T18:01:42.699 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking librabbitmq4:amd64 (0.10.0-1) ... 2023-09-23T18:01:42.768 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package liblua5.3-0:amd64. 2023-09-23T18:01:42.781 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../11-liblua5.3-0_5.3.3-1.1ubuntu2_amd64.deb ... 2023-09-23T18:01:42.817 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-09-23T18:01:43.041 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package librdkafka1:amd64. 2023-09-23T18:01:43.055 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../13-librdkafka1_1.2.1-1ubuntu1_amd64.deb ... 2023-09-23T18:01:43.116 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-09-23T18:01:43.213 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package librabbitmq4:amd64. 2023-09-23T18:01:43.229 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../12-librabbitmq4_0.10.0-1_amd64.deb ... 2023-09-23T18:01:43.280 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking librabbitmq4:amd64 (0.10.0-1) ... 2023-09-23T18:01:43.463 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package librgw2. 2023-09-23T18:01:43.478 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../14-librgw2_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:43.516 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking librgw2 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:43.683 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package librdkafka1:amd64. 2023-09-23T18:01:43.696 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../13-librdkafka1_1.2.1-1ubuntu1_amd64.deb ... 2023-09-23T18:01:43.734 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-09-23T18:01:44.109 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package librgw2. 2023-09-23T18:01:44.111 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-rgw. 2023-09-23T18:01:44.122 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../14-librgw2_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:44.127 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../15-python3-rgw_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:44.166 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking librgw2 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:44.169 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-rgw (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:44.498 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libleveldb1d:amd64. 2023-09-23T18:01:44.512 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../16-libleveldb1d_1.22-3ubuntu2_amd64.deb ... 2023-09-23T18:01:44.556 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-09-23T18:01:44.794 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-rgw. 2023-09-23T18:01:44.807 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../15-python3-rgw_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:44.861 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-rgw (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:44.895 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package liboath0. 2023-09-23T18:01:44.909 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../17-liboath0_2.6.1-1.3_amd64.deb ... 2023-09-23T18:01:44.953 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking liboath0 (2.6.1-1.3) ... 2023-09-23T18:01:45.139 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package libleveldb1d:amd64. 2023-09-23T18:01:45.154 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../16-libleveldb1d_1.22-3ubuntu2_amd64.deb ... 2023-09-23T18:01:45.189 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-09-23T18:01:45.256 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libradosstriper1. 2023-09-23T18:01:45.270 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../18-libradosstriper1_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:45.314 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libradosstriper1 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:45.534 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package liboath0. 2023-09-23T18:01:45.549 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../17-liboath0_2.6.1-1.3_amd64.deb ... 2023-09-23T18:01:45.592 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking liboath0 (2.6.1-1.3) ... 2023-09-23T18:01:45.709 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-common. 2023-09-23T18:01:45.722 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../19-ceph-common_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:45.767 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-common (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:45.895 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package libradosstriper1. 2023-09-23T18:01:45.910 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../18-libradosstriper1_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:45.953 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking libradosstriper1 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:46.340 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package ceph-common. 2023-09-23T18:01:46.354 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../19-ceph-common_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:46.389 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking ceph-common (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:48.073 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-base. 2023-09-23T18:01:48.087 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../20-ceph-base_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:48.141 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-base (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:48.671 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package ceph-base. 2023-09-23T18:01:48.684 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../20-ceph-base_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:48.739 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking ceph-base (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:49.052 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-cryptography. 2023-09-23T18:01:49.066 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../21-python3-cryptography_2.8-3ubuntu0.1_amd64.deb ... 2023-09-23T18:01:49.110 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-cryptography (2.8-3ubuntu0.1) ... 2023-09-23T18:01:49.464 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-openssl. 2023-09-23T18:01:49.478 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../22-python3-openssl_19.0.0-1build1_all.deb ... 2023-09-23T18:01:49.522 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-openssl (19.0.0-1build1) ... 2023-09-23T18:01:49.734 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-cryptography. 2023-09-23T18:01:49.749 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../21-python3-cryptography_2.8-3ubuntu0.1_amd64.deb ... 2023-09-23T18:01:49.783 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2023-09-23T18:01:49.792 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-cryptography (2.8-3ubuntu0.1) ... 2023-09-23T18:01:49.796 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../23-ceph-mgr-modules-core_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:01:49.833 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-mgr-modules-core (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:50.187 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-openssl. 2023-09-23T18:01:50.196 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-bcrypt. 2023-09-23T18:01:50.200 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../22-python3-openssl_19.0.0-1build1_all.deb ... 2023-09-23T18:01:50.211 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../24-python3-bcrypt_3.1.7-2ubuntu1_amd64.deb ... 2023-09-23T18:01:50.237 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-openssl (19.0.0-1build1) ... 2023-09-23T18:01:50.254 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-bcrypt (3.1.7-2ubuntu1) ... 2023-09-23T18:01:50.506 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2023-09-23T18:01:50.519 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../23-ceph-mgr-modules-core_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:01:50.556 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking ceph-mgr-modules-core (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:50.558 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-cherrypy3. 2023-09-23T18:01:50.572 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../25-python3-cherrypy3_8.9.1-6_all.deb ... 2023-09-23T18:01:50.616 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-cherrypy3 (8.9.1-6) ... 2023-09-23T18:01:50.929 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-bcrypt. 2023-09-23T18:01:50.943 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../24-python3-bcrypt_3.1.7-2ubuntu1_amd64.deb ... 2023-09-23T18:01:50.944 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-jwt. 2023-09-23T18:01:50.958 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../26-python3-jwt_1.7.1-2ubuntu2.1_all.deb ... 2023-09-23T18:01:50.987 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-bcrypt (3.1.7-2ubuntu1) ... 2023-09-23T18:01:51.003 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-jwt (1.7.1-2ubuntu2.1) ... 2023-09-23T18:01:51.290 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-cherrypy3. 2023-09-23T18:01:51.303 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../25-python3-cherrypy3_8.9.1-6_all.deb ... 2023-09-23T18:01:51.314 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-logutils. 2023-09-23T18:01:51.328 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../27-python3-logutils_0.3.3-7_all.deb ... 2023-09-23T18:01:51.340 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-cherrypy3 (8.9.1-6) ... 2023-09-23T18:01:51.372 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-logutils (0.3.3-7) ... 2023-09-23T18:01:51.700 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-markupsafe. 2023-09-23T18:01:51.710 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-jwt. 2023-09-23T18:01:51.713 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../28-python3-markupsafe_1.1.0-1build2_amd64.deb ... 2023-09-23T18:01:51.724 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../26-python3-jwt_1.7.1-2ubuntu2.1_all.deb ... 2023-09-23T18:01:51.750 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-markupsafe (1.1.0-1build2) ... 2023-09-23T18:01:51.768 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-jwt (1.7.1-2ubuntu2.1) ... 2023-09-23T18:01:52.053 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-mako. 2023-09-23T18:01:52.066 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../29-python3-mako_1.1.0+ds1-1ubuntu2.1_all.deb ... 2023-09-23T18:01:52.097 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-logutils. 2023-09-23T18:01:52.102 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-09-23T18:01:52.110 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../27-python3-logutils_0.3.3-7_all.deb ... 2023-09-23T18:01:52.146 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-logutils (0.3.3-7) ... 2023-09-23T18:01:52.439 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-simplegeneric. 2023-09-23T18:01:52.449 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-markupsafe. 2023-09-23T18:01:52.453 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../30-python3-simplegeneric_0.8.1-3_all.deb ... 2023-09-23T18:01:52.465 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../28-python3-markupsafe_1.1.0-1build2_amd64.deb ... 2023-09-23T18:01:52.497 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2023-09-23T18:01:52.507 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-markupsafe (1.1.0-1build2) ... 2023-09-23T18:01:52.792 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-singledispatch. 2023-09-23T18:01:52.800 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../31-python3-singledispatch_3.4.0.3-2build1_all.deb ... 2023-09-23T18:01:52.835 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-mako. 2023-09-23T18:01:52.842 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-singledispatch (3.4.0.3-2build1) ... 2023-09-23T18:01:52.850 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../29-python3-mako_1.1.0+ds1-1ubuntu2.1_all.deb ... 2023-09-23T18:01:52.894 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-09-23T18:01:53.144 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-webob. 2023-09-23T18:01:53.152 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../32-python3-webob_1%3a1.8.5-2_all.deb ... 2023-09-23T18:01:53.194 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-webob (1:1.8.5-2) ... 2023-09-23T18:01:53.222 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-simplegeneric. 2023-09-23T18:01:53.230 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../30-python3-simplegeneric_0.8.1-3_all.deb ... 2023-09-23T18:01:53.272 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2023-09-23T18:01:53.506 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-soupsieve. 2023-09-23T18:01:53.519 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../33-python3-soupsieve_1.9.5+dfsg-1_all.deb ... 2023-09-23T18:01:53.556 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-soupsieve (1.9.5+dfsg-1) ... 2023-09-23T18:01:53.575 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-singledispatch. 2023-09-23T18:01:53.588 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../31-python3-singledispatch_3.4.0.3-2build1_all.deb ... 2023-09-23T18:01:53.633 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-singledispatch (3.4.0.3-2build1) ... 2023-09-23T18:01:53.859 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-bs4. 2023-09-23T18:01:53.874 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../34-python3-bs4_4.8.2-1_all.deb ... 2023-09-23T18:01:53.916 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-bs4 (4.8.2-1) ... 2023-09-23T18:01:53.928 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-webob. 2023-09-23T18:01:53.942 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../32-python3-webob_1%3a1.8.5-2_all.deb ... 2023-09-23T18:01:53.986 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-webob (1:1.8.5-2) ... 2023-09-23T18:01:54.253 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-waitress. 2023-09-23T18:01:54.267 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../35-python3-waitress_1.4.1-1ubuntu0.1_all.deb ... 2023-09-23T18:01:54.314 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-waitress (1.4.1-1ubuntu0.1) ... 2023-09-23T18:01:54.322 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-soupsieve. 2023-09-23T18:01:54.336 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../33-python3-soupsieve_1.9.5+dfsg-1_all.deb ... 2023-09-23T18:01:54.381 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-soupsieve (1.9.5+dfsg-1) ... 2023-09-23T18:01:54.639 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-tempita. 2023-09-23T18:01:54.648 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../36-python3-tempita_0.5.2-6_all.deb ... 2023-09-23T18:01:54.667 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-bs4. 2023-09-23T18:01:54.681 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../34-python3-bs4_4.8.2-1_all.deb ... 2023-09-23T18:01:54.689 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-tempita (0.5.2-6) ... 2023-09-23T18:01:54.725 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-bs4 (4.8.2-1) ... 2023-09-23T18:01:54.967 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-paste. 2023-09-23T18:01:54.982 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../37-python3-paste_3.4.0+dfsg1-1_all.deb ... 2023-09-23T18:01:55.017 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-paste (3.4.0+dfsg1-1) ... 2023-09-23T18:01:55.086 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-waitress. 2023-09-23T18:01:55.100 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../35-python3-waitress_1.4.1-1ubuntu0.1_all.deb ... 2023-09-23T18:01:55.156 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-waitress (1.4.1-1ubuntu0.1) ... 2023-09-23T18:01:55.337 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2023-09-23T18:01:55.350 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../38-python-pastedeploy-tpl_2.1.0-1_all.deb ... 2023-09-23T18:01:55.395 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python-pastedeploy-tpl (2.1.0-1) ... 2023-09-23T18:01:55.465 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-tempita. 2023-09-23T18:01:55.478 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../36-python3-tempita_0.5.2-6_all.deb ... 2023-09-23T18:01:55.523 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-tempita (0.5.2-6) ... 2023-09-23T18:01:55.723 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-pastedeploy. 2023-09-23T18:01:55.738 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../39-python3-pastedeploy_2.1.0-1_all.deb ... 2023-09-23T18:01:55.781 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-pastedeploy (2.1.0-1) ... 2023-09-23T18:01:55.826 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-paste. 2023-09-23T18:01:55.841 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../37-python3-paste_3.4.0+dfsg1-1_all.deb ... 2023-09-23T18:01:55.884 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-paste (3.4.0+dfsg1-1) ... 2023-09-23T18:01:56.084 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-webtest. 2023-09-23T18:01:56.097 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../40-python3-webtest_2.0.32-2_all.deb ... 2023-09-23T18:01:56.142 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-webtest (2.0.32-2) ... 2023-09-23T18:01:56.204 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2023-09-23T18:01:56.217 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../38-python-pastedeploy-tpl_2.1.0-1_all.deb ... 2023-09-23T18:01:56.254 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python-pastedeploy-tpl (2.1.0-1) ... 2023-09-23T18:01:56.436 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-pecan. 2023-09-23T18:01:56.451 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../41-python3-pecan_1.3.3-3ubuntu1_all.deb ... 2023-09-23T18:01:56.494 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-pecan (1.3.3-3ubuntu1) ... 2023-09-23T18:01:56.549 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-pastedeploy. 2023-09-23T18:01:56.561 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../39-python3-pastedeploy_2.1.0-1_all.deb ... 2023-09-23T18:01:56.598 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-pastedeploy (2.1.0-1) ... 2023-09-23T18:01:56.849 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-werkzeug. 2023-09-23T18:01:56.863 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../42-python3-werkzeug_0.16.1+dfsg1-2ubuntu0.1_all.deb ... 2023-09-23T18:01:56.901 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-webtest. 2023-09-23T18:01:56.907 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-09-23T18:01:56.914 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../40-python3-webtest_2.0.32-2_all.deb ... 2023-09-23T18:01:56.959 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-webtest (2.0.32-2) ... 2023-09-23T18:01:58.077 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2023-09-23T18:01:58.090 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../43-libsqlite3-mod-ceph_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:58.136 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libsqlite3-mod-ceph (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:58.463 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-pecan. 2023-09-23T18:01:58.476 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../41-python3-pecan_1.3.3-3ubuntu1_all.deb ... 2023-09-23T18:01:58.480 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-mgr. 2023-09-23T18:01:58.495 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../44-ceph-mgr_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:58.521 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-pecan (1.3.3-3ubuntu1) ... 2023-09-23T18:01:58.538 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-mgr (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:58.859 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-werkzeug. 2023-09-23T18:01:58.867 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../42-python3-werkzeug_0.16.1+dfsg1-2ubuntu0.1_all.deb ... 2023-09-23T18:01:58.908 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-09-23T18:01:59.016 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-mon. 2023-09-23T18:01:59.031 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../45-ceph-mon_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:59.074 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-mon (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:59.312 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2023-09-23T18:01:59.321 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../43-libsqlite3-mod-ceph_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:59.362 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking libsqlite3-mod-ceph (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:01:59.707 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package ceph-mgr. 2023-09-23T18:01:59.720 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../44-ceph-mgr_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:01:59.765 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking ceph-mgr (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:00.186 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-osd. 2023-09-23T18:02:00.202 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../46-ceph-osd_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:02:00.245 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-osd (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:00.260 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package ceph-mon. 2023-09-23T18:02:00.273 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../45-ceph-mon_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:02:00.318 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking ceph-mon (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:01.364 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package ceph-osd. 2023-09-23T18:02:01.379 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../46-ceph-osd_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:02:01.422 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking ceph-osd (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:02.850 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph. 2023-09-23T18:02:02.864 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../47-ceph_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:02:02.908 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:03.219 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-fuse. 2023-09-23T18:02:03.234 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../48-ceph-fuse_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:02:03.277 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-fuse (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:03.689 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-mds. 2023-09-23T18:02:03.702 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../49-ceph-mds_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:02:03.747 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-mds (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:04.019 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package ceph. 2023-09-23T18:02:04.032 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../47-ceph_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:02:04.077 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking ceph (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:04.267 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package cephadm. 2023-09-23T18:02:04.282 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../50-cephadm_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:02:04.341 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking cephadm (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:04.372 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package ceph-fuse. 2023-09-23T18:02:04.386 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../48-ceph-fuse_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:02:04.421 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking ceph-fuse (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:04.602 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-jinja2. 2023-09-23T18:02:04.618 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../51-python3-jinja2_2.10.1-2_all.deb ... 2023-09-23T18:02:04.660 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-jinja2 (2.10.1-2) ... 2023-09-23T18:02:04.842 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package ceph-mds. 2023-09-23T18:02:04.857 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../49-ceph-mds_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:02:04.900 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking ceph-mds (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:04.980 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2023-09-23T18:02:04.993 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../52-ceph-mgr-cephadm_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:02:05.030 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-mgr-cephadm (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:05.358 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-repoze.lru. 2023-09-23T18:02:05.371 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../53-python3-repoze.lru_0.7-2_all.deb ... 2023-09-23T18:02:05.408 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2023-09-23T18:02:05.419 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package cephadm. 2023-09-23T18:02:05.428 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../50-cephadm_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:02:05.469 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking cephadm (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:05.660 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libjs-underscore. 2023-09-23T18:02:05.675 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../54-libjs-underscore_1.9.1~dfsg-1ubuntu0.20.04.1_all.deb ... 2023-09-23T18:02:05.719 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-09-23T18:02:05.797 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-jinja2. 2023-09-23T18:02:05.806 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../51-python3-jinja2_2.10.1-2_all.deb ... 2023-09-23T18:02:05.847 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-jinja2 (2.10.1-2) ... 2023-09-23T18:02:05.996 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libjs-sphinxdoc. 2023-09-23T18:02:06.011 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../55-libjs-sphinxdoc_1.8.5-7ubuntu3_all.deb ... 2023-09-23T18:02:06.054 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-09-23T18:02:06.167 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2023-09-23T18:02:06.180 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../52-ceph-mgr-cephadm_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:02:06.225 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking ceph-mgr-cephadm (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:06.365 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-routes. 2023-09-23T18:02:06.378 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../56-python3-routes_2.4.1-2_all.deb ... 2023-09-23T18:02:06.415 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-routes (2.4.1-2) ... 2023-09-23T18:02:06.570 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-repoze.lru. 2023-09-23T18:02:06.585 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../53-python3-repoze.lru_0.7-2_all.deb ... 2023-09-23T18:02:06.628 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2023-09-23T18:02:06.743 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2023-09-23T18:02:06.758 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../57-ceph-mgr-dashboard_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:02:06.801 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-mgr-dashboard (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:06.872 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package libjs-underscore. 2023-09-23T18:02:06.886 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../54-libjs-underscore_1.9.1~dfsg-1ubuntu0.20.04.1_all.deb ... 2023-09-23T18:02:06.930 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-09-23T18:02:07.200 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package libjs-sphinxdoc. 2023-09-23T18:02:07.214 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../55-libjs-sphinxdoc_1.8.5-7ubuntu3_all.deb ... 2023-09-23T18:02:07.258 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-09-23T18:02:07.569 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-routes. 2023-09-23T18:02:07.582 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../56-python3-routes_2.4.1-2_all.deb ... 2023-09-23T18:02:07.619 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-routes (2.4.1-2) ... 2023-09-23T18:02:07.784 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-decorator. 2023-09-23T18:02:07.797 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../58-python3-decorator_4.4.2-0ubuntu1_all.deb ... 2023-09-23T18:02:07.842 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-decorator (4.4.2-0ubuntu1) ... 2023-09-23T18:02:07.980 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2023-09-23T18:02:07.994 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../57-ceph-mgr-dashboard_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:02:08.038 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking ceph-mgr-dashboard (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:08.219 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package liblbfgsb0:amd64. 2023-09-23T18:02:08.233 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../59-liblbfgsb0_3.0+dfsg.3-7build1_amd64.deb ... 2023-09-23T18:02:08.277 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-09-23T18:02:08.589 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-scipy. 2023-09-23T18:02:08.602 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../60-python3-scipy_1.3.3-3build1_amd64.deb ... 2023-09-23T18:02:08.647 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-scipy (1.3.3-3build1) ... 2023-09-23T18:02:09.097 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-decorator. 2023-09-23T18:02:09.111 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../58-python3-decorator_4.4.2-0ubuntu1_all.deb ... 2023-09-23T18:02:09.155 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-decorator (4.4.2-0ubuntu1) ... 2023-09-23T18:02:09.458 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package liblbfgsb0:amd64. 2023-09-23T18:02:09.473 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../59-liblbfgsb0_3.0+dfsg.3-7build1_amd64.deb ... 2023-09-23T18:02:09.516 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-09-23T18:02:09.802 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-scipy. 2023-09-23T18:02:09.817 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../60-python3-scipy_1.3.3-3build1_amd64.deb ... 2023-09-23T18:02:09.860 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-scipy (1.3.3-3build1) ... 2023-09-23T18:02:09.897 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-sklearn-lib. 2023-09-23T18:02:09.912 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../61-python3-sklearn-lib_0.22.2.post1+dfsg-5_amd64.deb ... 2023-09-23T18:02:09.955 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-09-23T18:02:10.467 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-joblib. 2023-09-23T18:02:10.480 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../62-python3-joblib_0.14.0-3_all.deb ... 2023-09-23T18:02:10.525 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-joblib (0.14.0-3) ... 2023-09-23T18:02:10.861 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-sklearn. 2023-09-23T18:02:10.875 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../63-python3-sklearn_0.22.2.post1+dfsg-5_all.deb ... 2023-09-23T18:02:10.919 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-09-23T18:02:11.194 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-sklearn-lib. 2023-09-23T18:02:11.210 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../61-python3-sklearn-lib_0.22.2.post1+dfsg-5_amd64.deb ... 2023-09-23T18:02:11.252 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-09-23T18:02:11.594 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2023-09-23T18:02:11.604 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../64-ceph-mgr-diskprediction-local_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:02:11.644 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-mgr-diskprediction-local (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:11.800 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-joblib. 2023-09-23T18:02:11.813 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../62-python3-joblib_0.14.0-3_all.deb ... 2023-09-23T18:02:11.857 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-joblib (0.14.0-3) ... 2023-09-23T18:02:12.194 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-sklearn. 2023-09-23T18:02:12.210 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../63-python3-sklearn_0.22.2.post1+dfsg-5_all.deb ... 2023-09-23T18:02:12.252 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-09-23T18:02:12.806 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-cachetools. 2023-09-23T18:02:12.815 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../65-python3-cachetools_4.0.0-1_all.deb ... 2023-09-23T18:02:12.855 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-cachetools (4.0.0-1) ... 2023-09-23T18:02:12.986 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2023-09-23T18:02:13.002 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../64-ceph-mgr-diskprediction-local_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:02:13.044 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking ceph-mgr-diskprediction-local (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:13.525 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-pyasn1. 2023-09-23T18:02:13.539 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../66-python3-pyasn1_0.4.2-3build1_all.deb ... 2023-09-23T18:02:13.583 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-pyasn1 (0.4.2-3build1) ... 2023-09-23T18:02:13.911 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-pyasn1-modules. 2023-09-23T18:02:13.925 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../67-python3-pyasn1-modules_0.2.1-0.2build1_all.deb ... 2023-09-23T18:02:13.969 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-09-23T18:02:14.172 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-cachetools. 2023-09-23T18:02:14.187 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../65-python3-cachetools_4.0.0-1_all.deb ... 2023-09-23T18:02:14.230 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-cachetools (4.0.0-1) ... 2023-09-23T18:02:14.264 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-rsa. 2023-09-23T18:02:14.279 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../68-python3-rsa_4.0-3_all.deb ... 2023-09-23T18:02:14.322 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-rsa (4.0-3) ... 2023-09-23T18:02:14.567 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-pyasn1. 2023-09-23T18:02:14.582 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../66-python3-pyasn1_0.4.2-3build1_all.deb ... 2023-09-23T18:02:14.625 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-pyasn1 (0.4.2-3build1) ... 2023-09-23T18:02:14.634 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-google-auth. 2023-09-23T18:02:14.647 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../69-python3-google-auth_1.5.1-2_all.deb ... 2023-09-23T18:02:14.692 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-google-auth (1.5.1-2) ... 2023-09-23T18:02:14.945 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-pyasn1-modules. 2023-09-23T18:02:14.959 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../67-python3-pyasn1-modules_0.2.1-0.2build1_all.deb ... 2023-09-23T18:02:15.003 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-09-23T18:02:15.011 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-blinker. 2023-09-23T18:02:15.025 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../70-python3-blinker_1.4+dfsg1-0.3ubuntu1_all.deb ... 2023-09-23T18:02:15.069 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-09-23T18:02:15.314 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-rsa. 2023-09-23T18:02:15.330 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../68-python3-rsa_4.0-3_all.deb ... 2023-09-23T18:02:15.372 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-rsa (4.0-3) ... 2023-09-23T18:02:15.372 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-oauthlib. 2023-09-23T18:02:15.386 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../71-python3-oauthlib_3.1.0-1ubuntu2_all.deb ... 2023-09-23T18:02:15.431 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-oauthlib (3.1.0-1ubuntu2) ... 2023-09-23T18:02:15.667 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-google-auth. 2023-09-23T18:02:15.682 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../69-python3-google-auth_1.5.1-2_all.deb ... 2023-09-23T18:02:15.725 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-google-auth (1.5.1-2) ... 2023-09-23T18:02:15.759 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-requests-oauthlib. 2023-09-23T18:02:15.775 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../72-python3-requests-oauthlib_1.0.0-1.1build1_all.deb ... 2023-09-23T18:02:15.817 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-09-23T18:02:16.045 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-blinker. 2023-09-23T18:02:16.058 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../70-python3-blinker_1.4+dfsg1-0.3ubuntu1_all.deb ... 2023-09-23T18:02:16.103 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-09-23T18:02:16.111 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-websocket. 2023-09-23T18:02:16.126 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../73-python3-websocket_0.53.0-2ubuntu1_all.deb ... 2023-09-23T18:02:16.187 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-websocket (0.53.0-2ubuntu1) ... 2023-09-23T18:02:16.397 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-oauthlib. 2023-09-23T18:02:16.411 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../71-python3-oauthlib_3.1.0-1ubuntu2_all.deb ... 2023-09-23T18:02:16.455 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-oauthlib (3.1.0-1ubuntu2) ... 2023-09-23T18:02:16.497 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-kubernetes. 2023-09-23T18:02:16.511 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../74-python3-kubernetes_7.0.0~a1-2_all.deb ... 2023-09-23T18:02:16.556 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-kubernetes (7.0.0~a1-2) ... 2023-09-23T18:02:16.793 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-requests-oauthlib. 2023-09-23T18:02:16.806 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../72-python3-requests-oauthlib_1.0.0-1.1build1_all.deb ... 2023-09-23T18:02:16.851 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-09-23T18:02:17.086 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2023-09-23T18:02:17.100 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../75-ceph-mgr-k8sevents_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:02:17.143 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-mgr-k8sevents (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:17.170 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-websocket. 2023-09-23T18:02:17.186 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../73-python3-websocket_0.53.0-2ubuntu1_all.deb ... 2023-09-23T18:02:17.228 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-websocket (0.53.0-2ubuntu1) ... 2023-09-23T18:02:17.454 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libonig5:amd64. 2023-09-23T18:02:17.471 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../76-libonig5_6.9.4-1_amd64.deb ... 2023-09-23T18:02:17.513 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libonig5:amd64 (6.9.4-1) ... 2023-09-23T18:02:17.515 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-kubernetes. 2023-09-23T18:02:17.530 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../74-python3-kubernetes_7.0.0~a1-2_all.deb ... 2023-09-23T18:02:17.573 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-kubernetes (7.0.0~a1-2) ... 2023-09-23T18:02:17.849 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libjq1:amd64. 2023-09-23T18:02:17.863 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../77-libjq1_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-09-23T18:02:17.907 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-09-23T18:02:18.136 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2023-09-23T18:02:18.150 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../75-ceph-mgr-k8sevents_16.2.14-37-g2e1f1015-1focal_all.deb ... 2023-09-23T18:02:18.168 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package jq. 2023-09-23T18:02:18.183 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../78-jq_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-09-23T18:02:18.194 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking ceph-mgr-k8sevents (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:18.226 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking jq (1.6-1ubuntu0.20.04.1) ... 2023-09-23T18:02:18.461 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package socat. 2023-09-23T18:02:18.476 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../79-socat_1.7.3.3-2_amd64.deb ... 2023-09-23T18:02:18.513 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package libonig5:amd64. 2023-09-23T18:02:18.519 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking socat (1.7.3.3-2) ... 2023-09-23T18:02:18.527 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../76-libonig5_6.9.4-1_amd64.deb ... 2023-09-23T18:02:18.571 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking libonig5:amd64 (6.9.4-1) ... 2023-09-23T18:02:18.798 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package xmlstarlet. 2023-09-23T18:02:18.811 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../80-xmlstarlet_1.6.1-2build1_amd64.deb ... 2023-09-23T18:02:18.856 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking xmlstarlet (1.6.1-2build1) ... 2023-09-23T18:02:18.941 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package libjq1:amd64. 2023-09-23T18:02:18.955 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../77-libjq1_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-09-23T18:02:18.999 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-09-23T18:02:19.269 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package jq. 2023-09-23T18:02:19.279 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-test. 2023-09-23T18:02:19.283 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../78-jq_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-09-23T18:02:19.293 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../81-ceph-test_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:02:19.327 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking jq (1.6-1ubuntu0.20.04.1) ... 2023-09-23T18:02:19.337 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-test (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:19.579 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package socat. 2023-09-23T18:02:19.595 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../79-socat_1.7.3.3-2_amd64.deb ... 2023-09-23T18:02:19.637 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking socat (1.7.3.3-2) ... 2023-09-23T18:02:19.941 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package xmlstarlet. 2023-09-23T18:02:19.954 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../80-xmlstarlet_1.6.1-2build1_amd64.deb ... 2023-09-23T18:02:19.999 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking xmlstarlet (1.6.1-2build1) ... 2023-09-23T18:02:20.431 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package ceph-test. 2023-09-23T18:02:20.445 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../81-ceph-test_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:02:20.488 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking ceph-test (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:23.566 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libcephfs-dev. 2023-09-23T18:02:23.580 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../82-libcephfs-dev_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:02:23.623 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libcephfs-dev (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:24.027 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package nvme-cli. 2023-09-23T18:02:24.043 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../83-nvme-cli_1.9-1ubuntu0.1_amd64.deb ... 2023-09-23T18:02:24.101 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking nvme-cli (1.9-1ubuntu0.1) ... 2023-09-23T18:02:24.488 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-atomicwrites. 2023-09-23T18:02:24.503 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../84-python3-atomicwrites_1.1.5-2build1_all.deb ... 2023-09-23T18:02:24.548 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-atomicwrites (1.1.5-2build1) ... 2023-09-23T18:02:24.726 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package libcephfs-dev. 2023-09-23T18:02:24.742 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../82-libcephfs-dev_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:02:24.784 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking libcephfs-dev (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:24.843 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-attr. 2023-09-23T18:02:24.859 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../85-python3-attr_19.3.0-2_all.deb ... 2023-09-23T18:02:24.901 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-attr (19.3.0-2) ... 2023-09-23T18:02:25.171 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package nvme-cli. 2023-09-23T18:02:25.186 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../83-nvme-cli_1.9-1ubuntu0.1_amd64.deb ... 2023-09-23T18:02:25.229 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-webencodings. 2023-09-23T18:02:25.244 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../86-python3-webencodings_0.5.1-1ubuntu1_all.deb ... 2023-09-23T18:02:25.244 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking nvme-cli (1.9-1ubuntu0.1) ... 2023-09-23T18:02:25.287 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-webencodings (0.5.1-1ubuntu1) ... 2023-09-23T18:02:25.573 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-html5lib. 2023-09-23T18:02:25.588 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../87-python3-html5lib_1.0.1-2_all.deb ... 2023-09-23T18:02:25.617 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-atomicwrites. 2023-09-23T18:02:25.631 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../84-python3-atomicwrites_1.1.5-2build1_all.deb ... 2023-09-23T18:02:25.631 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-html5lib (1.0.1-2) ... 2023-09-23T18:02:25.675 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-atomicwrites (1.1.5-2build1) ... 2023-09-23T18:02:25.979 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-attr. 2023-09-23T18:02:25.985 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-lxml:amd64. 2023-09-23T18:02:25.994 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../85-python3-attr_19.3.0-2_all.deb ... 2023-09-23T18:02:25.998 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../88-python3-lxml_4.5.0-1ubuntu0.5_amd64.deb ... 2023-09-23T18:02:26.036 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-attr (19.3.0-2) ... 2023-09-23T18:02:26.042 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-09-23T18:02:26.381 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-webencodings. 2023-09-23T18:02:26.395 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../86-python3-webencodings_0.5.1-1ubuntu1_all.deb ... 2023-09-23T18:02:26.439 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-webencodings (0.5.1-1ubuntu1) ... 2023-09-23T18:02:26.504 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-packaging. 2023-09-23T18:02:26.519 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../89-python3-packaging_20.3-1_all.deb ... 2023-09-23T18:02:26.562 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-packaging (20.3-1) ... 2023-09-23T18:02:26.750 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-html5lib. 2023-09-23T18:02:26.766 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../87-python3-html5lib_1.0.1-2_all.deb ... 2023-09-23T18:02:26.808 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-html5lib (1.0.1-2) ... 2023-09-23T18:02:26.865 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-pastescript. 2023-09-23T18:02:26.880 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../90-python3-pastescript_2.0.2-4_all.deb ... 2023-09-23T18:02:26.923 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2023-09-23T18:02:27.162 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-lxml:amd64. 2023-09-23T18:02:27.175 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../88-python3-lxml_4.5.0-1ubuntu0.5_amd64.deb ... 2023-09-23T18:02:27.219 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-09-23T18:02:27.243 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-pluggy. 2023-09-23T18:02:27.259 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../91-python3-pluggy_0.13.0-2_all.deb ... 2023-09-23T18:02:27.302 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-pluggy (0.13.0-2) ... 2023-09-23T18:02:27.596 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-psutil. 2023-09-23T18:02:27.611 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../92-python3-psutil_5.5.1-1ubuntu4_amd64.deb ... 2023-09-23T18:02:27.654 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-psutil (5.5.1-1ubuntu4) ... 2023-09-23T18:02:27.715 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-packaging. 2023-09-23T18:02:27.724 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../89-python3-packaging_20.3-1_all.deb ... 2023-09-23T18:02:27.764 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-packaging (20.3-1) ... 2023-09-23T18:02:27.986 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-py. 2023-09-23T18:02:28.000 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../93-python3-py_1.8.1-1ubuntu0.1_all.deb ... 2023-09-23T18:02:28.044 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-py (1.8.1-1ubuntu0.1) ... 2023-09-23T18:02:28.067 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-pastescript. 2023-09-23T18:02:28.076 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../90-python3-pastescript_2.0.2-4_all.deb ... 2023-09-23T18:02:28.118 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2023-09-23T18:02:28.364 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-pyinotify. 2023-09-23T18:02:28.378 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../94-python3-pyinotify_0.9.6-1.2ubuntu1_all.deb ... 2023-09-23T18:02:28.422 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-09-23T18:02:28.467 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-pluggy. 2023-09-23T18:02:28.476 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../91-python3-pluggy_0.13.0-2_all.deb ... 2023-09-23T18:02:28.516 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-pluggy (0.13.0-2) ... 2023-09-23T18:02:28.748 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-wcwidth. 2023-09-23T18:02:28.762 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../95-python3-wcwidth_0.1.8+dfsg1-3_all.deb ... 2023-09-23T18:02:28.798 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-wcwidth (0.1.8+dfsg1-3) ... 2023-09-23T18:02:29.084 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-pytest. 2023-09-23T18:02:29.098 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../96-python3-pytest_4.6.9-1_all.deb ... 2023-09-23T18:02:29.119 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-psutil. 2023-09-23T18:02:29.128 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../92-python3-psutil_5.5.1-1ubuntu4_amd64.deb ... 2023-09-23T18:02:29.142 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-pytest (4.6.9-1) ... 2023-09-23T18:02:29.169 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-psutil (5.5.1-1ubuntu4) ... 2023-09-23T18:02:29.487 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-simplejson. 2023-09-23T18:02:29.497 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-py. 2023-09-23T18:02:29.501 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../97-python3-simplejson_3.16.0-2ubuntu2_amd64.deb ... 2023-09-23T18:02:29.513 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../93-python3-py_1.8.1-1ubuntu0.1_all.deb ... 2023-09-23T18:02:29.546 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-simplejson (3.16.0-2ubuntu2) ... 2023-09-23T18:02:29.555 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-py (1.8.1-1ubuntu0.1) ... 2023-09-23T18:02:29.907 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-pyinotify. 2023-09-23T18:02:29.923 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../94-python3-pyinotify_0.9.6-1.2ubuntu1_all.deb ... 2023-09-23T18:02:29.965 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-09-23T18:02:29.982 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package radosgw. 2023-09-23T18:02:29.996 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../98-radosgw_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:02:30.040 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking radosgw (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:30.277 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-wcwidth. 2023-09-23T18:02:30.290 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../95-python3-wcwidth_0.1.8+dfsg1-3_all.deb ... 2023-09-23T18:02:30.335 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-wcwidth (0.1.8+dfsg1-3) ... 2023-09-23T18:02:30.638 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-pytest. 2023-09-23T18:02:30.652 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../96-python3-pytest_4.6.9-1_all.deb ... 2023-09-23T18:02:30.696 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-pytest (4.6.9-1) ... 2023-09-23T18:02:31.017 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package python3-simplejson. 2023-09-23T18:02:31.031 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../97-python3-simplejson_3.16.0-2ubuntu2_amd64.deb ... 2023-09-23T18:02:31.083 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking python3-simplejson (3.16.0-2ubuntu2) ... 2023-09-23T18:02:31.211 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package rbd-fuse. 2023-09-23T18:02:31.225 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../99-rbd-fuse_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:02:31.269 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking rbd-fuse (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:31.487 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package radosgw. 2023-09-23T18:02:31.502 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../98-radosgw_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:02:31.545 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking radosgw (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:31.559 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-attr (19.3.0-2) ... 2023-09-23T18:02:31.848 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-09-23T18:02:31.983 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up librabbitmq4:amd64 (0.10.0-1) ... 2023-09-23T18:02:32.109 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up nvme-cli (1.9-1ubuntu0.1) ... 2023-09-23T18:02:32.302 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up cephadm (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:32.706 INFO:teuthology.orchestra.run.smithi098.stdout:Selecting previously unselected package rbd-fuse. 2023-09-23T18:02:32.722 INFO:teuthology.orchestra.run.smithi098.stdout:Preparing to unpack .../99-rbd-fuse_16.2.14-37-g2e1f1015-1focal_amd64.deb ... 2023-09-23T18:02:32.765 INFO:teuthology.orchestra.run.smithi098.stdout:Unpacking rbd-fuse (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:32.922 INFO:teuthology.orchestra.run.smithi173.stdout:Adding system user cephadm....done 2023-09-23T18:02:33.074 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-attr (19.3.0-2) ... 2023-09-23T18:02:33.091 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-waitress (1.4.1-1ubuntu0.1) ... 2023-09-23T18:02:33.370 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-09-23T18:02:33.451 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-repoze.lru (0.7-2) ... 2023-09-23T18:02:33.495 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up librabbitmq4:amd64 (0.10.0-1) ... 2023-09-23T18:02:33.605 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up nvme-cli (1.9-1ubuntu0.1) ... 2023-09-23T18:02:33.744 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up liboath0 (2.6.1-1.3) ... 2023-09-23T18:02:33.815 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up cephadm (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:33.870 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-py (1.8.1-1ubuntu0.1) ... 2023-09-23T18:02:34.196 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-joblib (0.14.0-3) ... 2023-09-23T18:02:34.451 INFO:teuthology.orchestra.run.smithi098.stdout:Adding system user cephadm....done 2023-09-23T18:02:34.581 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-jwt (1.7.1-2ubuntu2.1) ... 2023-09-23T18:02:34.604 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-waitress (1.4.1-1ubuntu0.1) ... 2023-09-23T18:02:34.857 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-cachetools (4.0.0-1) ... 2023-09-23T18:02:34.939 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-repoze.lru (0.7-2) ... 2023-09-23T18:02:35.150 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-bcrypt (3.1.7-2ubuntu1) ... 2023-09-23T18:02:35.231 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up liboath0 (2.6.1-1.3) ... 2023-09-23T18:02:35.357 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-py (1.8.1-1ubuntu0.1) ... 2023-09-23T18:02:35.442 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-09-23T18:02:35.684 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-joblib (0.14.0-3) ... 2023-09-23T18:02:35.727 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-ceph-argparse (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:36.020 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-09-23T18:02:36.078 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-jwt (1.7.1-2ubuntu2.1) ... 2023-09-23T18:02:36.145 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-markupsafe (1.1.0-1build2) ... 2023-09-23T18:02:36.379 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-cachetools (4.0.0-1) ... 2023-09-23T18:02:36.421 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-webencodings (0.5.1-1ubuntu1) ... 2023-09-23T18:02:36.647 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-bcrypt (3.1.7-2ubuntu1) ... 2023-09-23T18:02:36.697 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-psutil (5.5.1-1ubuntu4) ... 2023-09-23T18:02:36.923 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-09-23T18:02:37.074 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-atomicwrites (1.1.5-2build1) ... 2023-09-23T18:02:37.207 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-ceph-argparse (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:37.351 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-simplejson (3.16.0-2ubuntu2) ... 2023-09-23T18:02:37.510 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-09-23T18:02:37.636 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-markupsafe (1.1.0-1build2) ... 2023-09-23T18:02:37.686 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-decorator (4.4.2-0ubuntu1) ... 2023-09-23T18:02:37.904 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-webencodings (0.5.1-1ubuntu1) ... 2023-09-23T18:02:37.970 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libjaeger (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:38.096 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-jinja2 (2.10.1-2) ... 2023-09-23T18:02:38.188 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-psutil (5.5.1-1ubuntu4) ... 2023-09-23T18:02:38.422 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-tempita (0.5.2-6) ... 2023-09-23T18:02:38.556 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-atomicwrites (1.1.5-2build1) ... 2023-09-23T18:02:38.706 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-packaging (20.3-1) ... 2023-09-23T18:02:38.832 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-simplejson (3.16.0-2ubuntu2) ... 2023-09-23T18:02:39.008 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python-pastedeploy-tpl (2.1.0-1) ... 2023-09-23T18:02:39.133 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-wcwidth (0.1.8+dfsg1-3) ... 2023-09-23T18:02:39.183 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-decorator (4.4.2-0ubuntu1) ... 2023-09-23T18:02:39.426 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-paste (3.4.0+dfsg1-1) ... 2023-09-23T18:02:39.509 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up libjaeger (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:39.635 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-jinja2 (2.10.1-2) ... 2023-09-23T18:02:39.819 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-09-23T18:02:39.970 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-tempita (0.5.2-6) ... 2023-09-23T18:02:40.196 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-cryptography (2.8-3ubuntu0.1) ... 2023-09-23T18:02:40.271 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-packaging (20.3-1) ... 2023-09-23T18:02:40.555 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python-pastedeploy-tpl (2.1.0-1) ... 2023-09-23T18:02:40.556 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up socat (1.7.3.3-2) ... 2023-09-23T18:02:40.681 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-wcwidth (0.1.8+dfsg1-3) ... 2023-09-23T18:02:40.682 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-ceph-common (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:40.974 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-paste (3.4.0+dfsg1-1) ... 2023-09-23T18:02:41.008 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-html5lib (1.0.1-2) ... 2023-09-23T18:02:41.351 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-09-23T18:02:41.376 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-09-23T18:02:41.477 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up xmlstarlet (1.6.1-2build1) ... 2023-09-23T18:02:41.603 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-pluggy (0.13.0-2) ... 2023-09-23T18:02:41.736 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-cryptography (2.8-3ubuntu0.1) ... 2023-09-23T18:02:41.879 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-09-23T18:02:42.087 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up socat (1.7.3.3-2) ... 2023-09-23T18:02:42.180 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-09-23T18:02:42.213 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-ceph-common (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:42.306 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-pyasn1 (0.4.2-3build1) ... 2023-09-23T18:02:42.514 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-html5lib (1.0.1-2) ... 2023-09-23T18:02:42.633 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-singledispatch (3.4.0.3-2build1) ... 2023-09-23T18:02:42.849 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-09-23T18:02:42.909 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-logutils (0.3.3-7) ... 2023-09-23T18:02:42.975 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up xmlstarlet (1.6.1-2build1) ... 2023-09-23T18:02:43.101 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-pluggy (0.13.0-2) ... 2023-09-23T18:02:43.193 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up liblttng-ust0:amd64 (2.11.0-1) ... 2023-09-23T18:02:43.319 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2023-09-23T18:02:43.394 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-09-23T18:02:43.603 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-prettytable (0.7.2-5) ... 2023-09-23T18:02:43.687 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-09-23T18:02:43.813 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-pyasn1 (0.4.2-3build1) ... 2023-09-23T18:02:43.888 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-soupsieve (1.9.5+dfsg-1) ... 2023-09-23T18:02:44.131 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-singledispatch (3.4.0.3-2build1) ... 2023-09-23T18:02:44.524 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-logutils (0.3.3-7) ... 2023-09-23T18:02:44.531 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-websocket (0.53.0-2ubuntu1) ... 2023-09-23T18:02:44.817 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up liblttng-ust0:amd64 (2.11.0-1) ... 2023-09-23T18:02:44.961 INFO:teuthology.orchestra.run.smithi173.stdout:update-alternatives: using /usr/bin/python3-wsdump to provide /usr/bin/wsdump (wsdump) in auto mode 2023-09-23T18:02:44.968 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2023-09-23T18:02:45.225 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libonig5:amd64 (6.9.4-1) ... 2023-09-23T18:02:45.261 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-prettytable (0.7.2-5) ... 2023-09-23T18:02:45.351 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-09-23T18:02:45.553 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-soupsieve (1.9.5+dfsg-1) ... 2023-09-23T18:02:45.669 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-cherrypy3 (8.9.1-6) ... 2023-09-23T18:02:45.838 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-websocket (0.53.0-2ubuntu1) ... 2023-09-23T18:02:46.028 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-09-23T18:02:46.146 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-09-23T18:02:46.265 INFO:teuthology.orchestra.run.smithi098.stdout:update-alternatives: using /usr/bin/python3-wsdump to provide /usr/bin/wsdump (wsdump) in auto mode 2023-09-23T18:02:46.272 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-webob (1:1.8.5-2) ... 2023-09-23T18:02:46.499 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up libonig5:amd64 (6.9.4-1) ... 2023-09-23T18:02:46.606 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-09-23T18:02:46.625 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-09-23T18:02:46.899 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up librados2 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:46.943 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-cherrypy3 (8.9.1-6) ... 2023-09-23T18:02:47.026 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-scipy (1.3.3-3build1) ... 2023-09-23T18:02:47.303 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-09-23T18:02:47.429 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-09-23T18:02:47.538 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-webob (1:1.8.5-2) ... 2023-09-23T18:02:47.856 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-09-23T18:02:48.141 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up librados2 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:48.267 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-scipy (1.3.3-3build1) ... 2023-09-23T18:02:48.498 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up librgw2 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:48.649 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libsqlite3-mod-ceph (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:48.775 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-09-23T18:02:48.901 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-pytest (4.6.9-1) ... 2023-09-23T18:02:49.295 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-openssl (19.0.0-1build1) ... 2023-09-23T18:02:49.597 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-pastedeploy (2.1.0-1) ... 2023-09-23T18:02:49.748 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up librgw2 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:49.874 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up libsqlite3-mod-ceph (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:49.889 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-bs4 (4.8.2-1) ... 2023-09-23T18:02:50.009 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-09-23T18:02:50.143 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-pytest (4.6.9-1) ... 2023-09-23T18:02:50.206 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libcephfs2 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:50.333 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-oauthlib (3.1.0-1ubuntu2) ... 2023-09-23T18:02:50.520 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-openssl (19.0.0-1build1) ... 2023-09-23T18:02:50.651 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libradosstriper1 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:50.777 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-09-23T18:02:50.812 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-pastedeploy (2.1.0-1) ... 2023-09-23T18:02:51.098 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-09-23T18:02:51.105 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-bs4 (4.8.2-1) ... 2023-09-23T18:02:51.443 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up libcephfs2 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:51.569 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-oauthlib (3.1.0-1ubuntu2) ... 2023-09-23T18:02:51.905 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up libradosstriper1 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:52.030 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-09-23T18:02:52.204 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up librbd1 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:52.330 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-09-23T18:02:52.348 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-09-23T18:02:52.458 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up jq (1.6-1ubuntu0.20.04.1) ... 2023-09-23T18:02:52.581 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-rsa (4.0-3) ... 2023-09-23T18:02:52.875 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-webtest (2.0.32-2) ... 2023-09-23T18:02:53.167 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-mgr-modules-core (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:53.293 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-fuse (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:53.404 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up librbd1 (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:53.518 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-09-23T18:02:53.518 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-09-23T18:02:53.530 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-09-23T18:02:53.673 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up jq (1.6-1ubuntu0.20.04.1) ... 2023-09-23T18:02:53.791 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-rsa (4.0-3) ... 2023-09-23T18:02:53.961 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libcephfs-dev (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:54.086 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-webtest (2.0.32-2) ... 2023-09-23T18:02:54.089 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-09-23T18:02:54.379 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up ceph-mgr-modules-core (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:54.381 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-pastescript (2.0.2-4) ... 2023-09-23T18:02:54.505 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up ceph-fuse (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:54.701 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-pecan (1.3.3-3ubuntu1) ... 2023-09-23T18:02:54.729 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-09-23T18:02:54.802 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-09-23T18:02:55.051 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-rados (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:55.172 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-rbd (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:55.181 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up libcephfs-dev (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:55.297 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up rbd-fuse (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:55.307 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-09-23T18:02:55.423 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-routes (2.4.1-2) ... 2023-09-23T18:02:55.596 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-pastescript (2.0.2-4) ... 2023-09-23T18:02:55.734 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-google-auth (1.5.1-2) ... 2023-09-23T18:02:55.914 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-pecan (1.3.3-3ubuntu1) ... 2023-09-23T18:02:56.122 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-rgw (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:56.246 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-cephfs (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:56.267 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-rados (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:56.418 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-rbd (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:56.542 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-kubernetes (7.0.0~a1-2) ... 2023-09-23T18:02:56.544 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up rbd-fuse (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:56.672 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-routes (2.4.1-2) ... 2023-09-23T18:02:56.971 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-google-auth (1.5.1-2) ... 2023-09-23T18:02:57.369 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-rgw (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:57.470 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-common (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:57.494 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-cephfs (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:57.788 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up python3-kubernetes (7.0.0~a1-2) ... 2023-09-23T18:02:58.014 INFO:teuthology.orchestra.run.smithi173.stdout:Adding group ceph....done 2023-09-23T18:02:58.382 INFO:teuthology.orchestra.run.smithi173.stdout:Adding system user ceph....done 2023-09-23T18:02:58.541 INFO:teuthology.orchestra.run.smithi173.stdout:Setting system user ceph properties....done 2023-09-23T18:02:58.558 INFO:teuthology.orchestra.run.smithi173.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2023-09-23T18:02:58.699 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2023-09-23T18:02:58.733 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up ceph-common (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:59.019 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2023-09-23T18:02:59.285 INFO:teuthology.orchestra.run.smithi098.stdout:Adding group ceph....done 2023-09-23T18:02:59.471 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-test (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:59.597 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up radosgw (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:02:59.671 INFO:teuthology.orchestra.run.smithi098.stdout:Adding system user ceph....done 2023-09-23T18:02:59.897 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-09-23T18:02:59.897 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-09-23T18:03:00.139 INFO:teuthology.orchestra.run.smithi098.stdout:Setting system user ceph properties....done 2023-09-23T18:03:00.159 INFO:teuthology.orchestra.run.smithi098.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2023-09-23T18:03:00.311 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2023-09-23T18:03:00.524 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-base (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:03:00.627 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2023-09-23T18:03:00.899 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2023-09-23T18:03:01.093 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up ceph-test (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:03:01.219 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up radosgw (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:03:01.342 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-mds (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:03:01.500 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-09-23T18:03:01.501 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-09-23T18:03:01.565 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-09-23T18:03:01.565 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-09-23T18:03:02.011 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-mgr (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:03:02.163 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up ceph-base (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:03:02.226 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-09-23T18:03:02.226 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-09-23T18:03:02.540 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2023-09-23T18:03:02.670 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-osd (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:03:02.990 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up ceph-mds (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:03:03.170 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-09-23T18:03:03.170 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-09-23T18:03:03.209 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-09-23T18:03:03.209 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-09-23T18:03:03.638 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-mgr-k8sevents (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:03:03.649 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up ceph-mgr (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:03:03.764 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-mgr-diskprediction-local (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:03:03.866 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-09-23T18:03:03.867 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-09-23T18:03:03.907 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-mon (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:03:04.123 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-09-23T18:03:04.124 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-09-23T18:03:04.318 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up ceph-osd (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:03:04.567 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-mgr-cephadm (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:03:04.710 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:03:04.794 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-09-23T18:03:04.795 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-09-23T18:03:04.864 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-mgr-dashboard (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:03:05.008 INFO:teuthology.orchestra.run.smithi173.stdout:Processing triggers for systemd (245.4-4ubuntu3.17) ... 2023-09-23T18:03:05.254 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up ceph-mgr-k8sevents (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:03:05.292 INFO:teuthology.orchestra.run.smithi173.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-09-23T18:03:05.382 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up ceph-mgr-diskprediction-local (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:03:05.525 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up ceph-mon (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:03:05.757 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-09-23T18:03:05.757 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-09-23T18:03:06.243 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up ceph-mgr-cephadm (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:03:06.386 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up ceph (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:03:06.512 INFO:teuthology.orchestra.run.smithi098.stdout:Setting up ceph-mgr-dashboard (16.2.14-37-g2e1f1015-1focal) ... 2023-09-23T18:03:06.664 INFO:teuthology.orchestra.run.smithi098.stdout:Processing triggers for systemd (245.4-4ubuntu3.17) ... 2023-09-23T18:03:06.948 INFO:teuthology.orchestra.run.smithi098.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-09-23T18:03:08.270 INFO:teuthology.orchestra.run.smithi173.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-09-23T18:03:08.734 INFO:teuthology.orchestra.run.smithi173.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2023-09-23T18:03:08.738 DEBUG:teuthology.parallel:result is None 2023-09-23T18:03:09.921 INFO:teuthology.orchestra.run.smithi098.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-09-23T18:03:10.380 INFO:teuthology.orchestra.run.smithi098.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2023-09-23T18:03:10.383 DEBUG:teuthology.parallel:result is None 2023-09-23T18:03:10.383 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T18:03:10.383 INFO:teuthology.packaging:ref: None 2023-09-23T18:03:10.383 INFO:teuthology.packaging:tag: None 2023-09-23T18:03:10.383 INFO:teuthology.packaging:branch: pacific 2023-09-23T18:03:10.383 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T18:03:10.384 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-09-23T18:03:10.619 DEBUG:teuthology.orchestra.run.smithi098:> dpkg-query -W -f '${Version}' ceph 2023-09-23T18:03:10.643 INFO:teuthology.orchestra.run.smithi098.stdout:16.2.14-37-g2e1f1015-1focal 2023-09-23T18:03:10.643 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-37-g2e1f1015-1focal 2023-09-23T18:03:10.643 INFO:teuthology.task.install:The correct ceph version 16.2.14-37-g2e1f1015-1focal is installed. 2023-09-23T18:03:10.644 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T18:03:10.644 INFO:teuthology.packaging:ref: None 2023-09-23T18:03:10.644 INFO:teuthology.packaging:tag: None 2023-09-23T18:03:10.644 INFO:teuthology.packaging:branch: pacific 2023-09-23T18:03:10.644 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T18:03:10.644 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-09-23T18:03:10.874 DEBUG:teuthology.orchestra.run.smithi173:> dpkg-query -W -f '${Version}' ceph 2023-09-23T18:03:10.897 INFO:teuthology.orchestra.run.smithi173.stdout:16.2.14-37-g2e1f1015-1focal 2023-09-23T18:03:10.897 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-37-g2e1f1015-1focal 2023-09-23T18:03:10.898 INFO:teuthology.task.install:The correct ceph version 16.2.14-37-g2e1f1015-1focal is installed. 2023-09-23T18:03:10.898 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-09-23T18:03:10.898 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-23T18:03:10.898 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-23T18:03:10.914 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-09-23T18:03:10.914 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-23T18:03:10.955 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-09-23T18:03:10.956 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-23T18:03:10.956 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd of=/usr/bin/daemon-helper 2023-09-23T18:03:10.977 DEBUG:teuthology.orchestra.run.smithi098:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-23T18:03:11.032 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-09-23T18:03:11.032 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd of=/usr/bin/daemon-helper 2023-09-23T18:03:11.047 DEBUG:teuthology.orchestra.run.smithi173:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-23T18:03:11.105 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-09-23T18:03:11.105 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-23T18:03:11.105 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-23T18:03:11.121 DEBUG:teuthology.orchestra.run.smithi098:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-23T18:03:11.181 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-09-23T18:03:11.181 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-23T18:03:11.197 DEBUG:teuthology.orchestra.run.smithi173:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-23T18:03:11.252 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-09-23T18:03:11.252 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-23T18:03:11.253 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd of=/usr/bin/stdin-killer 2023-09-23T18:03:11.269 DEBUG:teuthology.orchestra.run.smithi098:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-23T18:03:11.324 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-09-23T18:03:11.325 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd of=/usr/bin/stdin-killer 2023-09-23T18:03:11.339 DEBUG:teuthology.orchestra.run.smithi173:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-23T18:03:11.395 INFO:teuthology.run_tasks:Running task cephadm... 2023-09-23T18:03:11.467 INFO:tasks.cephadm:Config: {'cephadm_branch': 'pacific', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:pacific', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'} 2023-09-23T18:03:11.468 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:pacific 2023-09-23T18:03:11.468 INFO:tasks.cephadm:Cluster fsid is 756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:03:11.468 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-09-23T18:03:11.468 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.98', 'mon.c': '[v2:172.21.15.98:3301,v1:172.21.15.98:6790]', 'mon.b': '172.21.15.173'} 2023-09-23T18:03:11.469 INFO:tasks.cephadm:First mon is mon.a on smithi098 2023-09-23T18:03:11.469 INFO:tasks.cephadm:First mgr is y 2023-09-23T18:03:11.469 INFO:tasks.cephadm:Normalizing hostnames... 2023-09-23T18:03:11.469 DEBUG:teuthology.orchestra.run.smithi098:> sudo hostname $(hostname -s) 2023-09-23T18:03:11.485 DEBUG:teuthology.orchestra.run.smithi173:> sudo hostname $(hostname -s) 2023-09-23T18:03:11.501 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref pacific)... 2023-09-23T18:03:11.502 DEBUG:teuthology.orchestra.run.smithi098:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-09-23T18:03:11.889 INFO:teuthology.orchestra.run.smithi098.stdout:-rw-rw-r-- 1 ubuntu ubuntu 350873 Sep 23 18:03 /home/ubuntu/cephtest/cephadm 2023-09-23T18:03:11.889 DEBUG:teuthology.orchestra.run.smithi173:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-09-23T18:03:12.080 INFO:teuthology.orchestra.run.smithi173.stdout:-rw-rw-r-- 1 ubuntu ubuntu 350873 Sep 23 18:03 /home/ubuntu/cephtest/cephadm 2023-09-23T18:03:12.080 DEBUG:teuthology.orchestra.run.smithi098:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-09-23T18:03:12.090 DEBUG:teuthology.orchestra.run.smithi173:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-09-23T18:03:12.108 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:pacific on all hosts... 2023-09-23T18:03:12.108 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-09-23T18:03:12.139 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-09-23T18:03:12.400 INFO:teuthology.orchestra.run.smithi098.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-09-23T18:03:12.407 INFO:teuthology.orchestra.run.smithi173.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-09-23T18:03:53.418 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:03:53.441 INFO:teuthology.orchestra.run.smithi173.stdout: "ceph_version": "ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)", 2023-09-23T18:03:53.441 INFO:teuthology.orchestra.run.smithi173.stdout: "image_id": "1e6a756174a8dbc0d5ced5aeb4272e8f04dc2045a9196dffb3df976d1caca13e", 2023-09-23T18:03:53.441 INFO:teuthology.orchestra.run.smithi173.stdout: "repo_digests": [ 2023-09-23T18:03:53.442 INFO:teuthology.orchestra.run.smithi173.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:396076557f19bac7d00d097a6db258821cfaec622cd332710d7957388aedf72b" 2023-09-23T18:03:53.442 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:03:53.442 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:03:56.340 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:03:56.358 INFO:teuthology.orchestra.run.smithi098.stdout: "ceph_version": "ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)", 2023-09-23T18:03:56.358 INFO:teuthology.orchestra.run.smithi098.stdout: "image_id": "1e6a756174a8dbc0d5ced5aeb4272e8f04dc2045a9196dffb3df976d1caca13e", 2023-09-23T18:03:56.358 INFO:teuthology.orchestra.run.smithi098.stdout: "repo_digests": [ 2023-09-23T18:03:56.358 INFO:teuthology.orchestra.run.smithi098.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:396076557f19bac7d00d097a6db258821cfaec622cd332710d7957388aedf72b" 2023-09-23T18:03:56.358 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:03:56.359 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:03:56.371 DEBUG:teuthology.orchestra.run.smithi098:> sudo mkdir -p /etc/ceph 2023-09-23T18:03:56.528 DEBUG:teuthology.orchestra.run.smithi173:> sudo mkdir -p /etc/ceph 2023-09-23T18:03:56.646 DEBUG:teuthology.orchestra.run.smithi098:> sudo chmod 777 /etc/ceph 2023-09-23T18:03:56.822 DEBUG:teuthology.orchestra.run.smithi173:> sudo chmod 777 /etc/ceph 2023-09-23T18:03:56.921 INFO:tasks.cephadm:Writing seed config... 2023-09-23T18:03:57.217 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-09-23T18:03:57.217 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-09-23T18:03:57.217 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-09-23T18:03:57.217 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-09-23T18:03:57.217 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-09-23T18:03:57.218 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2023-09-23T18:03:57.218 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-09-23T18:03:57.218 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-09-23T18:03:57.218 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-09-23T18:03:57.218 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-09-23T18:03:57.218 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-09-23T18:03:57.219 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-23T18:03:57.219 DEBUG:teuthology.orchestra.run.smithi098:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-09-23T18:03:57.226 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 756c96d4-5a3b-11ee-8db4-212e2dc638e7 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-09-23T18:03:57.314 DEBUG:teuthology.orchestra.run.smithi098:mon.a> sudo journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mon.a.service 2023-09-23T18:03:57.316 DEBUG:teuthology.orchestra.run.smithi098:mgr.y> sudo journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mgr.y.service 2023-09-23T18:03:57.359 INFO:tasks.cephadm:Bootstrapping... 2023-09-23T18:03:58.102 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific -v bootstrap --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.98 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-09-23T18:03:58.103 INFO:journalctl@ceph.mgr.y.smithi098.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:03:58.103 INFO:journalctl@ceph.mon.a.smithi098.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:03:58.321 INFO:teuthology.orchestra.run.smithi098.stderr:-------------------------------------------------------------------------------- 2023-09-23T18:03:58.393 INFO:teuthology.orchestra.run.smithi098.stderr:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:pacific', '-v', 'bootstrap', '--fsid', '756c96d4-5a3b-11ee-8db4-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.98', '--skip-admin-label'] 2023-09-23T18:03:58.394 INFO:teuthology.orchestra.run.smithi098.stderr:Verifying podman|docker is present... 2023-09-23T18:03:58.394 INFO:teuthology.orchestra.run.smithi098.stderr:Verifying lvm2 is present... 2023-09-23T18:03:58.523 INFO:teuthology.orchestra.run.smithi098.stderr:Verifying time synchronization is in place... 2023-09-23T18:03:58.528 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-09-23T18:03:59.446 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: inactive 2023-09-23T18:03:59.447 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: Failed to get unit file state for chronyd.service: No such file or directory 2023-09-23T18:03:59.447 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: inactive 2023-09-23T18:03:59.702 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: masked 2023-09-23T18:03:59.702 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: inactive 2023-09-23T18:03:59.702 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: Failed to get unit file state for ntpd.service: No such file or directory 2023-09-23T18:03:59.702 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: inactive 2023-09-23T18:03:59.702 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: enabled 2023-09-23T18:03:59.702 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: active 2023-09-23T18:03:59.702 INFO:teuthology.orchestra.run.smithi098.stderr:Unit ntp.service is enabled and running 2023-09-23T18:03:59.703 INFO:teuthology.orchestra.run.smithi098.stderr:Repeating the final host check... 2023-09-23T18:03:59.703 INFO:teuthology.orchestra.run.smithi098.stderr:docker (/usr/bin/docker) is present 2023-09-23T18:03:59.703 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl is present 2023-09-23T18:03:59.703 INFO:teuthology.orchestra.run.smithi098.stderr:lvcreate is present 2023-09-23T18:03:59.703 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-09-23T18:03:59.704 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: inactive 2023-09-23T18:03:59.704 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: Failed to get unit file state for chronyd.service: No such file or directory 2023-09-23T18:03:59.916 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: inactive 2023-09-23T18:03:59.917 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: masked 2023-09-23T18:03:59.917 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: inactive 2023-09-23T18:03:59.917 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: Failed to get unit file state for ntpd.service: No such file or directory 2023-09-23T18:03:59.917 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: inactive 2023-09-23T18:03:59.917 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: enabled 2023-09-23T18:03:59.917 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: active 2023-09-23T18:03:59.917 INFO:teuthology.orchestra.run.smithi098.stderr:Unit ntp.service is enabled and running 2023-09-23T18:03:59.917 INFO:teuthology.orchestra.run.smithi098.stderr:Host looks OK 2023-09-23T18:03:59.917 INFO:teuthology.orchestra.run.smithi098.stderr:Cluster fsid: 756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:03:59.918 INFO:teuthology.orchestra.run.smithi098.stderr:Acquiring lock 139712820117456 on /run/cephadm/756c96d4-5a3b-11ee-8db4-212e2dc638e7.lock 2023-09-23T18:03:59.918 INFO:teuthology.orchestra.run.smithi098.stderr:Lock 139712820117456 acquired on /run/cephadm/756c96d4-5a3b-11ee-8db4-212e2dc638e7.lock 2023-09-23T18:03:59.918 INFO:teuthology.orchestra.run.smithi098.stderr:Verifying IP 172.21.15.98 port 3300 ... 2023-09-23T18:03:59.918 INFO:teuthology.orchestra.run.smithi098.stderr:Verifying IP 172.21.15.98 port 6789 ... 2023-09-23T18:03:59.918 INFO:teuthology.orchestra.run.smithi098.stderr:Base mon IP(s) is [172.21.15.98:3300, 172.21.15.98:6789], mon addrv is [v2:172.21.15.98:3300,v1:172.21.15.98:6789] 2023-09-23T18:03:59.918 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/sbin/ip: default via 172.21.15.254 dev enp3s0f1 2023-09-23T18:03:59.918 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.98 metric 100 2023-09-23T18:03:59.918 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/sbin/ip: 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown 2023-09-23T18:03:59.918 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.98 2023-09-23T18:03:59.919 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/sbin/ip: 172.21.15.254 dev enp3s0f1 proto dhcp scope link src 172.21.15.98 metric 100 2023-09-23T18:03:59.919 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2023-09-23T18:03:59.919 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2023-09-23T18:03:59.919 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 expires 1795sec pref medium 2023-09-23T18:03:59.919 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-09-23T18:03:59.919 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/sbin/ip: inet6 ::1/128 scope host 2023-09-23T18:03:59.919 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/sbin/ip: valid_lft forever preferred_lft forever 2023-09-23T18:03:59.919 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2023-09-23T18:03:59.919 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/sbin/ip: inet6 fe80::ec4:7aff:fe8f:d26b/64 scope link 2023-09-23T18:03:59.919 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/sbin/ip: valid_lft forever preferred_lft forever 2023-09-23T18:03:59.919 INFO:teuthology.orchestra.run.smithi098.stderr:Mon IP `172.21.15.98` is in CIDR network `172.21.0.0/20` 2023-09-23T18:03:59.920 INFO:teuthology.orchestra.run.smithi098.stderr:Mon IP `172.21.15.98` is in CIDR network `172.21.0.0/20` 2023-09-23T18:03:59.920 INFO:teuthology.orchestra.run.smithi098.stderr:Mon IP `172.21.15.98` is in CIDR network `172.21.15.254/32` 2023-09-23T18:03:59.920 INFO:teuthology.orchestra.run.smithi098.stderr:Mon IP `172.21.15.98` is in CIDR network `172.21.15.254/32` 2023-09-23T18:03:59.920 INFO:teuthology.orchestra.run.smithi098.stderr:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20', '172.21.15.254/32', '172.21.15.254/32'] 2023-09-23T18:04:00.166 INFO:teuthology.orchestra.run.smithi098.stderr:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-09-23T18:04:00.166 INFO:teuthology.orchestra.run.smithi098.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-09-23T18:04:00.168 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/docker: pacific: Pulling from ceph-ci/ceph 2023-09-23T18:04:00.168 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/docker: Digest: sha256:396076557f19bac7d00d097a6db258821cfaec622cd332710d7957388aedf72b 2023-09-23T18:04:00.168 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/docker: Status: Image is up to date for quay.ceph.io/ceph-ci/ceph:pacific 2023-09-23T18:04:00.168 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/docker: quay.ceph.io/ceph-ci/ceph:pacific 2023-09-23T18:04:00.168 INFO:teuthology.orchestra.run.smithi098.stderr:ceph: ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable) 2023-09-23T18:04:00.402 INFO:teuthology.orchestra.run.smithi098.stderr:Ceph version: ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable) 2023-09-23T18:04:00.575 INFO:teuthology.orchestra.run.smithi098.stderr:Extracting ceph user uid/gid from container image... 2023-09-23T18:04:01.203 INFO:teuthology.orchestra.run.smithi098.stderr:stat: 167 167 2023-09-23T18:04:01.822 INFO:teuthology.orchestra.run.smithi098.stderr:Creating initial keys... 2023-09-23T18:04:02.679 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-authtool: AQCSKA9lOQJzKBAA6KnQ/qAjSHio5UiaoD7W4Q== 2023-09-23T18:04:04.189 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-authtool: AQCUKA9lkJU9CxAA2OelMaSGPTpnaJ7z72vNYA== 2023-09-23T18:04:05.621 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-authtool: AQCVKA9ln1D3JBAAMhjvcFt+EeqAsPinszHOAg== 2023-09-23T18:04:07.625 INFO:teuthology.orchestra.run.smithi098.stderr:Creating initial monmap... 2023-09-23T18:04:07.625 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2023-09-23T18:04:07.625 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:04:07.625 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-09-23T18:04:07.751 INFO:teuthology.orchestra.run.smithi098.stderr:monmaptool for a [v2:172.21.15.98:3300,v1:172.21.15.98:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-09-23T18:04:07.806 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/monmaptool: set fsid to 756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:04:07.806 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-09-23T18:04:07.806 INFO:teuthology.orchestra.run.smithi098.stderr: 2023-09-23T18:04:07.806 INFO:teuthology.orchestra.run.smithi098.stderr:Creating mon... 2023-09-23T18:04:08.571 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.566+0000 7fd74da00880 0 set uid:gid to 167:167 (ceph:ceph) 2023-09-23T18:04:08.747 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 1 imported monmap: 2023-09-23T18:04:10.031 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: epoch 0 2023-09-23T18:04:10.031 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:04:10.033 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: last_changed 2023-09-23T18:04:07.137241+0000 2023-09-23T18:04:10.056 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: created 2023-09-23T18:04:07.137241+0000 2023-09-23T18:04:10.196 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: min_mon_release 0 (unknown) 2023-09-23T18:04:10.196 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: election_strategy: 1 2023-09-23T18:04:10.196 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: 0: [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] mon.a 2023-09-23T18:04:10.196 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: 2023-09-23T18:04:10.196 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 0 /usr/bin/ceph-mon: set fsid to 756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:04:10.196 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: RocksDB version: 6.8.1 2023-09-23T18:04:10.196 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: 2023-09-23T18:04:10.197 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T18:04:10.197 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Compile date Sep 21 2023 2023-09-23T18:04:10.197 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: DB SUMMARY 2023-09-23T18:04:10.197 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: 2023-09-23T18:04:10.198 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2023-09-23T18:04:10.198 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: 2023-09-23T18:04:10.198 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2023-09-23T18:04:10.198 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: 2023-09-23T18:04:10.198 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.error_if_exists: 0 2023-09-23T18:04:10.198 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.create_if_missing: 1 2023-09-23T18:04:10.198 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.paranoid_checks: 1 2023-09-23T18:04:10.198 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.env: 0x558eb6dc1080 2023-09-23T18:04:10.199 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.fs: Posix File System 2023-09-23T18:04:10.199 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.info_log: 0x558eb8f89aa0 2023-09-23T18:04:10.199 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.max_file_opening_threads: 16 2023-09-23T18:04:10.199 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.statistics: (nil) 2023-09-23T18:04:10.199 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.use_fsync: 0 2023-09-23T18:04:10.199 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.max_log_file_size: 0 2023-09-23T18:04:10.199 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T18:04:10.199 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T18:04:10.199 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.keep_log_file_num: 1000 2023-09-23T18:04:10.578 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.recycle_log_file_num: 0 2023-09-23T18:04:10.578 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.allow_fallocate: 1 2023-09-23T18:04:10.578 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.allow_mmap_reads: 0 2023-09-23T18:04:10.578 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.allow_mmap_writes: 0 2023-09-23T18:04:10.578 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.use_direct_reads: 0 2023-09-23T18:04:10.578 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T18:04:10.578 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.create_missing_column_families: 0 2023-09-23T18:04:10.578 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.db_log_dir: 2023-09-23T18:04:10.579 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-09-23T18:04:10.579 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T18:04:10.579 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.max_subcompactions: 1 2023-09-23T18:04:10.579 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.max_background_flushes: -1 2023-09-23T18:04:10.579 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T18:04:10.579 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T18:04:10.579 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T18:04:10.579 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T18:04:10.579 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T18:04:10.579 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.advise_random_on_open: 1 2023-09-23T18:04:10.579 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.db_write_buffer_size: 0 2023-09-23T18:04:10.580 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.write_buffer_manager: 0x558eb8f91890 2023-09-23T18:04:10.580 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T18:04:10.580 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T18:04:10.580 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T18:04:10.580 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T18:04:10.580 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.rate_limiter: (nil) 2023-09-23T18:04:10.580 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T18:04:10.580 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.wal_recovery_mode: 2 2023-09-23T18:04:10.581 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.enable_thread_tracking: 0 2023-09-23T18:04:10.581 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.enable_pipelined_write: 0 2023-09-23T18:04:10.581 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.unordered_write: 0 2023-09-23T18:04:10.581 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T18:04:10.581 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T18:04:10.581 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T18:04:10.664 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T18:04:10.664 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.row_cache: None 2023-09-23T18:04:10.664 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.wal_filter: None 2023-09-23T18:04:10.664 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T18:04:10.665 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.allow_ingest_behind: 0 2023-09-23T18:04:10.665 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.preserve_deletes: 0 2023-09-23T18:04:10.665 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.two_write_queues: 0 2023-09-23T18:04:10.666 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.manual_wal_flush: 0 2023-09-23T18:04:10.666 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.atomic_flush: 0 2023-09-23T18:04:10.666 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T18:04:10.666 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T18:04:10.667 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T18:04:10.667 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.log_readahead_size: 0 2023-09-23T18:04:10.667 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-09-23T18:04:10.667 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.max_background_jobs: 2 2023-09-23T18:04:10.667 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.max_background_compactions: -1 2023-09-23T18:04:10.668 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T18:04:10.668 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T18:04:10.668 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T18:04:10.668 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.max_total_wal_size: 0 2023-09-23T18:04:10.668 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T18:04:10.668 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T18:04:10.669 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T18:04:10.669 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T18:04:10.669 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.max_open_files: -1 2023-09-23T18:04:10.921 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.bytes_per_sync: 0 2023-09-23T18:04:10.921 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T18:04:10.922 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T18:04:10.922 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.compaction_readahead_size: 0 2023-09-23T18:04:10.922 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Compression algorithms supported: 2023-09-23T18:04:10.922 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T18:04:10.922 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: kZSTD supported: 0 2023-09-23T18:04:10.923 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: kXpressCompression supported: 0 2023-09-23T18:04:10.923 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: kLZ4HCCompression supported: 1 2023-09-23T18:04:10.923 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: kLZ4Compression supported: 1 2023-09-23T18:04:10.923 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: kBZip2Compression supported: 0 2023-09-23T18:04:10.923 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: kZlibCompression supported: 1 2023-09-23T18:04:10.923 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: kSnappyCompression supported: 1 2023-09-23T18:04:10.924 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T18:04:10.924 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: [db_impl/db_impl_open.cc:273] Creating manifest 1 2023-09-23T18:04:10.924 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: 2023-09-23T18:04:10.924 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2023-09-23T18:04:10.925 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: 2023-09-23T18:04:10.925 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-09-23T18:04:10.925 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: 2023-09-23T18:04:10.925 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T18:04:10.925 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.merge_operator: 2023-09-23T18:04:10.925 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.compaction_filter: None 2023-09-23T18:04:10.926 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.compaction_filter_factory: None 2023-09-23T18:04:10.926 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T18:04:10.926 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T18:04:10.926 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558eb8e9fd20) 2023-09-23T18:04:10.926 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks: 1 2023-09-23T18:04:10.927 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T18:04:10.927 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T18:04:10.927 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: pin_top_level_index_and_filter: 1 2023-09-23T18:04:10.927 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: index_type: 0 2023-09-23T18:04:10.927 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: data_block_index_type: 0 2023-09-23T18:04:10.928 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: index_shortening: 1 2023-09-23T18:04:10.928 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: data_block_hash_table_util_ratio: 0.750000 2023-09-23T18:04:10.928 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: hash_index_allow_collision: 1 2023-09-23T18:04:10.928 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: checksum: 1 2023-09-23T18:04:10.928 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: no_block_cache: 0 2023-09-23T18:04:10.928 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: block_cache: 0x558eb8ed6d10 2023-09-23T18:04:10.929 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: block_cache_name: BinnedLRUCache 2023-09-23T18:04:10.929 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: block_cache_options: 2023-09-23T18:04:10.929 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: capacity : 536870912 2023-09-23T18:04:10.929 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: num_shard_bits : 4 2023-09-23T18:04:10.929 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: strict_capacity_limit : 0 2023-09-23T18:04:10.929 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: high_pri_pool_ratio: 0.000 2023-09-23T18:04:11.203 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: block_cache_compressed: (nil) 2023-09-23T18:04:11.203 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: persistent_cache: (nil) 2023-09-23T18:04:11.203 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: block_size: 4096 2023-09-23T18:04:11.203 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: block_size_deviation: 10 2023-09-23T18:04:11.203 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: block_restart_interval: 16 2023-09-23T18:04:11.204 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: index_block_restart_interval: 1 2023-09-23T18:04:11.204 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: metadata_block_size: 4096 2023-09-23T18:04:11.204 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: partition_filters: 0 2023-09-23T18:04:11.204 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: use_delta_encoding: 1 2023-09-23T18:04:11.204 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T18:04:11.204 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: whole_key_filtering: 1 2023-09-23T18:04:11.204 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: verify_compression: 0 2023-09-23T18:04:11.204 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: read_amp_bytes_per_bit: 0 2023-09-23T18:04:11.204 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: format_version: 2 2023-09-23T18:04:11.204 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: enable_index_compression: 1 2023-09-23T18:04:11.204 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: block_align: 0 2023-09-23T18:04:11.204 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: 2023-09-23T18:04:11.205 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.write_buffer_size: 33554432 2023-09-23T18:04:11.205 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.max_write_buffer_number: 2 2023-09-23T18:04:11.205 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.compression: NoCompression 2023-09-23T18:04:11.205 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.bottommost_compression: Disabled 2023-09-23T18:04:11.205 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.prefix_extractor: nullptr 2023-09-23T18:04:11.205 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T18:04:11.205 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.num_levels: 7 2023-09-23T18:04:11.205 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T18:04:11.205 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T18:04:11.205 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T18:04:11.205 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T18:04:11.205 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T18:04:11.206 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T18:04:11.206 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T18:04:11.206 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T18:04:11.206 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T18:04:11.206 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T18:04:11.206 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.compression_opts.level: 32767 2023-09-23T18:04:11.206 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.compression_opts.strategy: 0 2023-09-23T18:04:11.206 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T18:04:11.206 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T18:04:11.207 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.compression_opts.enabled: false 2023-09-23T18:04:11.207 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T18:04:11.207 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T18:04:11.207 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T18:04:11.207 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.target_file_size_base: 67108864 2023-09-23T18:04:11.207 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T18:04:11.207 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T18:04:11.207 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T18:04:11.207 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T18:04:11.207 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T18:04:11.207 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T18:04:11.207 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T18:04:11.208 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T18:04:11.208 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T18:04:11.208 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T18:04:11.208 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T18:04:11.208 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T18:04:11.208 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T18:04:11.208 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.arena_block_size: 4194304 2023-09-23T18:04:11.208 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T18:04:11.208 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T18:04:11.208 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T18:04:11.208 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.disable_auto_compactions: 0 2023-09-23T18:04:11.208 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T18:04:11.208 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T18:04:11.209 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T18:04:11.209 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T18:04:11.209 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T18:04:11.209 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T18:04:11.209 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T18:04:11.209 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T18:04:11.209 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T18:04:11.209 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T18:04:11.209 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.table_properties_collectors: 2023-09-23T18:04:11.209 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.inplace_update_support: 0 2023-09-23T18:04:11.209 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T18:04:11.209 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T18:04:11.682 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T18:04:11.682 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T18:04:11.682 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.bloom_locality: 0 2023-09-23T18:04:11.683 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.max_successive_merges: 0 2023-09-23T18:04:11.683 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T18:04:11.683 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.paranoid_file_checks: 0 2023-09-23T18:04:11.683 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.force_consistency_checks: 0 2023-09-23T18:04:11.683 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.report_bg_io_stats: 0 2023-09-23T18:04:11.683 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.ttl: 2592000 2023-09-23T18:04:11.683 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T18:04:11.683 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T18:04:11.683 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: 2023-09-23T18:04:11.683 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.570+0000 7fd74da00880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-09-23T18:04:11.683 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: 2023-09-23T18:04:11.684 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.574+0000 7fd74da00880 4 rocksdb: DB pointer 0x558eb8f9f800 2023-09-23T18:04:11.684 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.574+0000 7fd736889700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-09-23T18:04:11.684 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.574+0000 7fd736889700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-09-23T18:04:11.684 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: ** DB Stats ** 2023-09-23T18:04:11.684 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T18:04:11.684 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T18:04:11.685 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T18:04:11.685 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T18:04:11.685 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T18:04:11.685 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T18:04:11.685 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T18:04:11.685 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: 2023-09-23T18:04:11.685 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-09-23T18:04:11.685 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T18:04:11.685 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T18:04:14.133 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T18:04:14.133 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T18:04:14.133 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: 2023-09-23T18:04:14.133 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-09-23T18:04:14.134 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T18:04:14.134 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T18:04:14.134 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T18:04:14.134 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T18:04:14.134 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T18:04:14.134 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T18:04:14.134 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T18:04:14.517 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-09-23T18:04:14.517 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T18:04:14.518 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T18:04:14.518 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T18:04:14.518 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: 2023-09-23T18:04:14.518 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-09-23T18:04:14.518 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: 2023-09-23T18:04:14.518 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-09-23T18:04:14.518 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T18:04:14.518 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T18:04:14.518 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T18:04:14.518 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T18:04:14.519 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: 2023-09-23T18:04:14.519 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-09-23T18:04:14.519 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T18:04:14.519 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T18:04:14.519 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T18:04:14.519 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T18:04:14.519 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T18:04:14.519 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T18:04:14.519 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T18:04:14.519 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-09-23T18:04:14.519 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T18:04:14.519 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T18:04:14.520 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T18:04:14.520 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: 2023-09-23T18:04:14.520 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-09-23T18:04:14.520 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: 2023-09-23T18:04:14.520 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.574+0000 7fd74da00880 4 rocksdb: [db_impl/db_impl.cc:397] Shutdown: canceling all background work 2023-09-23T18:04:14.520 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.574+0000 7fd74da00880 4 rocksdb: [db_impl/db_impl.cc:573] Shutdown complete 2023-09-23T18:04:14.520 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph-mon: debug 2023-09-23T18:04:08.574+0000 7fd74da00880 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2023-09-23T18:04:14.520 INFO:teuthology.orchestra.run.smithi098.stderr:create mon.a on 2023-09-23T18:04:14.520 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-09-23T18:04:14.520 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /etc/systemd/system/ceph.target. 2023-09-23T18:04:14.520 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7.target -> /etc/systemd/system/ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7.target. 2023-09-23T18:04:14.521 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7.target -> /etc/systemd/system/ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7.target. 2023-09-23T18:04:14.521 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: Failed to reset failed state of unit ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mon.a.service: Unit ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mon.a.service not loaded. 2023-09-23T18:04:14.521 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: Created symlink /etc/systemd/system/ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7.target.wants/ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mon.a.service -> /etc/systemd/system/ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@.service. 2023-09-23T18:04:14.522 INFO:teuthology.orchestra.run.smithi098.stderr:firewalld does not appear to be present 2023-09-23T18:04:14.856 INFO:teuthology.orchestra.run.smithi098.stderr:Not possible to enable service . firewalld.service is not available 2023-09-23T18:04:14.857 INFO:teuthology.orchestra.run.smithi098.stderr:Waiting for mon to start... 2023-09-23T18:04:14.857 INFO:teuthology.orchestra.run.smithi098.stderr:Waiting for mon... 2023-09-23T18:04:14.857 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:11 smithi098 bash[26795]: cluster 2023-09-23T18:04:11.554752+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T18:04:14.859 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: cluster: 2023-09-23T18:04:14.859 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: id: 756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:04:15.150 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: health: HEALTH_OK 2023-09-23T18:04:15.150 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: 2023-09-23T18:04:15.150 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: services: 2023-09-23T18:04:15.150 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.202486s) 2023-09-23T18:04:15.150 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: mgr: no daemons active 2023-09-23T18:04:15.150 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2023-09-23T18:04:15.150 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: 2023-09-23T18:04:15.150 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: data: 2023-09-23T18:04:15.150 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2023-09-23T18:04:15.150 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2023-09-23T18:04:15.150 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2023-09-23T18:04:15.151 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: pgs: 2023-09-23T18:04:15.151 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: 2023-09-23T18:04:15.151 INFO:teuthology.orchestra.run.smithi098.stderr:mon is available 2023-09-23T18:04:15.151 INFO:teuthology.orchestra.run.smithi098.stderr:Assimilating anything we can from ceph.conf... 2023-09-23T18:04:15.151 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:12 smithi098 bash[26795]: cluster 2023-09-23T18:04:11.579150+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T18:04:15.151 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:12 smithi098 bash[26795]: cluster 2023-09-23T18:04:11.579528+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0]} removed_ranks: {} 2023-09-23T18:04:15.151 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:12 smithi098 bash[26795]: cluster 2023-09-23T18:04:11.583151+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap 2023-09-23T18:04:15.152 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:12 smithi098 bash[26795]: cluster 2023-09-23T18:04:11.587067+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-09-23T18:04:15.152 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:12 smithi098 bash[26795]: cluster 2023-09-23T18:04:11.587564+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2023-09-23T18:04:15.152 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:12 smithi098 bash[26795]: audit 2023-09-23T18:04:11.781576+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.98:0/1619643286' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-09-23T18:04:15.153 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: 2023-09-23T18:04:15.154 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: [global] 2023-09-23T18:04:15.154 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: fsid = 756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:04:15.154 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: log_to_journald = false 2023-09-23T18:04:15.154 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.98:3300,v1:172.21.15.98:6789] 2023-09-23T18:04:15.154 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-09-23T18:04:15.154 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-09-23T18:04:15.474 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-09-23T18:04:15.474 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-09-23T18:04:15.474 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: 2023-09-23T18:04:15.474 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: [mgr] 2023-09-23T18:04:15.474 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-09-23T18:04:15.474 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: 2023-09-23T18:04:15.474 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: [osd] 2023-09-23T18:04:15.475 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-09-23T18:04:15.475 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-09-23T18:04:15.475 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:13 smithi098 bash[26795]: audit 2023-09-23T18:04:13.518517+0000 mon.a (mon.0) 8 : audit [INF] from='client.? 172.21.15.98:0/467678908' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-09-23T18:04:15.475 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:13 smithi098 bash[26795]: audit 2023-09-23T18:04:13.525133+0000 mon.a (mon.0) 9 : audit [INF] from='client.? 172.21.15.98:0/467678908' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-09-23T18:04:15.475 INFO:teuthology.orchestra.run.smithi098.stderr:Generating new minimal ceph.conf... 2023-09-23T18:04:15.690 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:15 smithi098 bash[26795]: audit 2023-09-23T18:04:15.203041+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.98:0/3223383417' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:04:17.352 INFO:teuthology.orchestra.run.smithi098.stderr:Restarting the monitor... 2023-09-23T18:04:17.562 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:15 smithi098 systemd[1]: Stopping Ceph mon.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T18:04:17.562 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:15 smithi098 bash[27192]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-mon.a 2023-09-23T18:04:17.562 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:16 smithi098 bash[26795]: debug 2023-09-23T18:04:16.033+0000 7fc7f0c12700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-09-23T18:04:17.562 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:16 smithi098 bash[26795]: debug 2023-09-23T18:04:16.033+0000 7fc7f0c12700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-09-23T18:04:17.562 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:16 smithi098 bash[27229]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-mon-a 2023-09-23T18:04:17.562 INFO:teuthology.orchestra.run.smithi098.stderr:Setting mon public_network to 172.21.15.254/32,172.21.0.0/20 2023-09-23T18:04:17.563 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:16 smithi098 bash[27309]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-mon-a 2023-09-23T18:04:17.563 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:16 smithi098 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mon.a.service: Succeeded. 2023-09-23T18:04:17.563 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:16 smithi098 systemd[1]: Stopped Ceph mon.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:04:17.563 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:16 smithi098 systemd[1]: Started Ceph mon.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:04:18.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.949+0000 7fe07f4db880 0 set uid:gid to 167:167 (ceph:ceph) 2023-09-23T18:04:19.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.949+0000 7fe07f4db880 0 ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable), process ceph-mon, pid 7 2023-09-23T18:04:19.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.949+0000 7fe07f4db880 0 pidfile_write: ignore empty --pid-file 2023-09-23T18:04:19.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 0 load: jerasure load: lrc load: isa 2023-09-23T18:04:19.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: RocksDB version: 6.8.1 2023-09-23T18:04:19.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T18:04:19.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Compile date Sep 21 2023 2023-09-23T18:04:19.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: DB SUMMARY 2023-09-23T18:04:19.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: CURRENT file: CURRENT 2023-09-23T18:04:19.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: IDENTITY file: IDENTITY 2023-09-23T18:04:19.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 131 Bytes 2023-09-23T18:04:19.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000004.sst 2023-09-23T18:04:19.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000006.log size: 86697 ; 2023-09-23T18:04:19.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.error_if_exists: 0 2023-09-23T18:04:19.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.create_if_missing: 0 2023-09-23T18:04:19.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.paranoid_checks: 1 2023-09-23T18:04:19.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.env: 0x5618dfdc1080 2023-09-23T18:04:19.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.fs: Posix File System 2023-09-23T18:04:19.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.info_log: 0x5618e1a9da80 2023-09-23T18:04:19.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.max_file_opening_threads: 16 2023-09-23T18:04:19.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.statistics: (nil) 2023-09-23T18:04:19.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.use_fsync: 0 2023-09-23T18:04:19.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.max_log_file_size: 0 2023-09-23T18:04:19.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T18:04:19.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T18:04:19.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.keep_log_file_num: 1000 2023-09-23T18:04:19.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.recycle_log_file_num: 0 2023-09-23T18:04:19.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.allow_fallocate: 1 2023-09-23T18:04:19.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.allow_mmap_reads: 0 2023-09-23T18:04:19.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.allow_mmap_writes: 0 2023-09-23T18:04:19.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.use_direct_reads: 0 2023-09-23T18:04:19.447 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T18:04:19.447 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.create_missing_column_families: 0 2023-09-23T18:04:19.449 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.db_log_dir: 2023-09-23T18:04:19.449 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-09-23T18:04:19.449 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T18:04:19.449 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.max_subcompactions: 1 2023-09-23T18:04:19.449 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.max_background_flushes: -1 2023-09-23T18:04:19.449 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T18:04:19.449 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T18:04:19.720 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T18:04:19.720 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T18:04:19.720 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T18:04:19.720 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.advise_random_on_open: 1 2023-09-23T18:04:19.720 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.db_write_buffer_size: 0 2023-09-23T18:04:19.721 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.write_buffer_manager: 0x5618e279a330 2023-09-23T18:04:19.721 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T18:04:19.721 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T18:04:19.721 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T18:04:19.721 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T18:04:19.721 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.rate_limiter: (nil) 2023-09-23T18:04:19.721 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T18:04:19.721 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.wal_recovery_mode: 2 2023-09-23T18:04:19.721 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.enable_thread_tracking: 0 2023-09-23T18:04:19.721 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.enable_pipelined_write: 0 2023-09-23T18:04:19.721 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.unordered_write: 0 2023-09-23T18:04:19.721 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T18:04:19.723 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T18:04:19.724 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T18:04:19.724 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T18:04:19.724 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.row_cache: None 2023-09-23T18:04:19.724 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.wal_filter: None 2023-09-23T18:04:19.724 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T18:04:19.724 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.allow_ingest_behind: 0 2023-09-23T18:04:19.724 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.preserve_deletes: 0 2023-09-23T18:04:19.724 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.two_write_queues: 0 2023-09-23T18:04:19.724 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.manual_wal_flush: 0 2023-09-23T18:04:19.727 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.atomic_flush: 0 2023-09-23T18:04:19.727 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T18:04:19.727 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T18:04:19.728 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T18:04:19.728 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.log_readahead_size: 0 2023-09-23T18:04:19.728 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-09-23T18:04:19.728 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.max_background_jobs: 2 2023-09-23T18:04:19.728 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.max_background_compactions: -1 2023-09-23T18:04:19.989 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T18:04:19.989 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T18:04:19.991 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T18:04:19.992 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.max_total_wal_size: 0 2023-09-23T18:04:19.992 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T18:04:19.992 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T18:04:19.994 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T18:04:20.254 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T18:04:20.254 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.max_open_files: -1 2023-09-23T18:04:20.254 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.bytes_per_sync: 0 2023-09-23T18:04:20.254 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T18:04:20.255 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T18:04:20.255 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.compaction_readahead_size: 0 2023-09-23T18:04:20.255 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Compression algorithms supported: 2023-09-23T18:04:20.255 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T18:04:20.255 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: kZSTD supported: 0 2023-09-23T18:04:20.255 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: kXpressCompression supported: 0 2023-09-23T18:04:20.255 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: kLZ4HCCompression supported: 1 2023-09-23T18:04:20.255 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: kLZ4Compression supported: 1 2023-09-23T18:04:20.255 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: kBZip2Compression supported: 0 2023-09-23T18:04:20.255 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: kZlibCompression supported: 1 2023-09-23T18:04:20.256 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: kSnappyCompression supported: 1 2023-09-23T18:04:20.257 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T18:04:20.257 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 2023-09-23T18:04:20.257 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-09-23T18:04:20.257 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T18:04:20.258 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.merge_operator: 2023-09-23T18:04:20.258 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.compaction_filter: None 2023-09-23T18:04:20.258 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.compaction_filter_factory: None 2023-09-23T18:04:20.258 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T18:04:20.258 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T18:04:20.258 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5618e1a4bd18) 2023-09-23T18:04:20.258 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: cache_index_and_filter_blocks: 1 2023-09-23T18:04:20.259 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T18:04:20.259 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T18:04:20.259 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: pin_top_level_index_and_filter: 1 2023-09-23T18:04:20.259 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: index_type: 0 2023-09-23T18:04:20.259 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: data_block_index_type: 0 2023-09-23T18:04:20.259 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: index_shortening: 1 2023-09-23T18:04:20.259 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: data_block_hash_table_util_ratio: 0.750000 2023-09-23T18:04:20.532 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: hash_index_allow_collision: 1 2023-09-23T18:04:20.532 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: checksum: 1 2023-09-23T18:04:20.532 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: no_block_cache: 0 2023-09-23T18:04:20.532 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: block_cache: 0x5618e1a82f10 2023-09-23T18:04:20.758 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: block_cache_name: BinnedLRUCache 2023-09-23T18:04:20.758 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: block_cache_options: 2023-09-23T18:04:20.758 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: capacity : 536870912 2023-09-23T18:04:20.758 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: num_shard_bits : 4 2023-09-23T18:04:20.758 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: strict_capacity_limit : 0 2023-09-23T18:04:20.758 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: high_pri_pool_ratio: 0.000 2023-09-23T18:04:20.759 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: block_cache_compressed: (nil) 2023-09-23T18:04:20.759 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: persistent_cache: (nil) 2023-09-23T18:04:20.759 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: block_size: 4096 2023-09-23T18:04:20.759 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: block_size_deviation: 10 2023-09-23T18:04:20.759 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: block_restart_interval: 16 2023-09-23T18:04:20.759 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: index_block_restart_interval: 1 2023-09-23T18:04:20.759 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: metadata_block_size: 4096 2023-09-23T18:04:20.759 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: partition_filters: 0 2023-09-23T18:04:20.760 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: use_delta_encoding: 1 2023-09-23T18:04:20.760 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T18:04:20.760 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: whole_key_filtering: 1 2023-09-23T18:04:20.760 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: verify_compression: 0 2023-09-23T18:04:20.760 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: read_amp_bytes_per_bit: 0 2023-09-23T18:04:20.760 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: format_version: 2 2023-09-23T18:04:20.760 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: enable_index_compression: 1 2023-09-23T18:04:20.760 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: block_align: 0 2023-09-23T18:04:20.760 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.write_buffer_size: 33554432 2023-09-23T18:04:20.761 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.max_write_buffer_number: 2 2023-09-23T18:04:20.761 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.compression: NoCompression 2023-09-23T18:04:20.761 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.bottommost_compression: Disabled 2023-09-23T18:04:20.762 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.prefix_extractor: nullptr 2023-09-23T18:04:20.762 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T18:04:20.762 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.num_levels: 7 2023-09-23T18:04:20.762 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T18:04:20.762 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T18:04:20.762 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T18:04:20.763 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T18:04:20.763 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T18:04:20.763 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T18:04:20.763 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T18:04:20.763 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T18:04:20.763 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T18:04:20.763 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T18:04:20.763 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.compression_opts.level: 32767 2023-09-23T18:04:20.764 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.compression_opts.strategy: 0 2023-09-23T18:04:20.764 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T18:04:21.186 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T18:04:21.186 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.compression_opts.enabled: false 2023-09-23T18:04:21.186 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T18:04:21.187 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T18:04:21.187 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T18:04:21.187 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.target_file_size_base: 67108864 2023-09-23T18:04:21.187 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T18:04:21.187 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T18:04:21.187 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T18:04:21.187 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T18:04:21.188 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T18:04:21.188 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T18:04:21.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T18:04:21.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T18:04:21.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T18:04:21.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T18:04:21.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T18:04:21.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T18:04:21.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T18:04:21.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.arena_block_size: 4194304 2023-09-23T18:04:21.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T18:04:21.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T18:04:21.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T18:04:21.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.disable_auto_compactions: 0 2023-09-23T18:04:21.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T18:04:21.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T18:04:21.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T18:04:21.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T18:04:21.365 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T18:04:21.365 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T18:04:21.365 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T18:04:21.365 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T18:04:21.365 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T18:04:21.365 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T18:04:21.366 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.table_properties_collectors: 2023-09-23T18:04:21.366 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.inplace_update_support: 0 2023-09-23T18:04:21.366 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T18:04:21.366 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T18:04:21.366 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T18:04:21.366 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T18:04:21.366 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.bloom_locality: 0 2023-09-23T18:04:21.366 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.max_successive_merges: 0 2023-09-23T18:04:21.367 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T18:04:21.367 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.paranoid_file_checks: 0 2023-09-23T18:04:21.367 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.force_consistency_checks: 0 2023-09-23T18:04:21.367 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.report_bg_io_stats: 0 2023-09-23T18:04:21.367 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.ttl: 2592000 2023-09-23T18:04:21.367 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.953+0000 7fe07f4db880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T18:04:21.367 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.961+0000 7fe07f4db880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 5, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T18:04:21.367 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.961+0000 7fe07f4db880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 4 2023-09-23T18:04:21.367 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.961+0000 7fe07f4db880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695492257962723, "job": 1, "event": "recovery_started", "log_files": [6]} 2023-09-23T18:04:21.367 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.961+0000 7fe07f4db880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #6 mode 2 2023-09-23T18:04:21.367 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.961+0000 7fe07f4db880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T18:04:21.368 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.961+0000 7fe07f4db880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695492257964629, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 7, "file_size": 83297, "table_properties": {"data_size": 81570, "index_size": 332, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 581, "raw_key_size": 9829, "raw_average_key_size": 47, "raw_value_size": 75891, "raw_average_value_size": 368, "num_data_blocks": 11, "num_entries": 206, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695492257, "oldest_key_time": 3, "file_creation_time": 0}} 2023-09-23T18:04:21.368 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.961+0000 7fe07f4db880 4 rocksdb: [version_set.cc:3826] Creating manifest 8 2023-09-23T18:04:21.368 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.961+0000 7fe07f4db880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695492257965434, "job": 1, "event": "recovery_finished"} 2023-09-23T18:04:21.368 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.965+0000 7fe07f4db880 4 rocksdb: DB pointer 0x5618e1b4b800 2023-09-23T18:04:21.368 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.965+0000 7fe0662e3700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-09-23T18:04:21.368 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.965+0000 7fe0662e3700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-09-23T18:04:21.368 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: ** DB Stats ** 2023-09-23T18:04:21.368 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T18:04:21.368 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T18:04:21.368 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T18:04:21.368 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T18:04:21.618 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T18:04:21.618 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T18:04:21.618 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T18:04:21.618 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: ** Compaction Stats [default] ** 2023-09-23T18:04:21.618 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T18:04:21.619 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T18:04:21.619 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: L0 2/0 83.00 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 71.6 0.00 0.00 1 0.001 0 0 2023-09-23T18:04:21.619 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: Sum 2/0 83.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 71.6 0.00 0.00 1 0.001 0 0 2023-09-23T18:04:21.619 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 71.6 0.00 0.00 1 0.001 0 0 2023-09-23T18:04:21.619 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: ** Compaction Stats [default] ** 2023-09-23T18:04:21.619 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T18:04:21.619 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T18:04:21.619 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 71.6 0.00 0.00 1 0.001 0 0 2023-09-23T18:04:21.619 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T18:04:21.619 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T18:04:21.619 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T18:04:21.620 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T18:04:21.620 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T18:04:21.620 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: AddFile(Keys): cumulative 0, interval 0 2023-09-23T18:04:21.620 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: Cumulative compaction: 0.00 GB write, 8.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T18:04:21.620 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: Interval compaction: 0.00 GB write, 8.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T18:04:21.620 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T18:04:21.620 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: ** File Read Latency Histogram By Level [default] ** 2023-09-23T18:04:21.620 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: ** Compaction Stats [default] ** 2023-09-23T18:04:21.620 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T18:04:21.620 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T18:04:21.621 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: L0 2/0 83.00 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 71.6 0.00 0.00 1 0.001 0 0 2023-09-23T18:04:21.621 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: Sum 2/0 83.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 71.6 0.00 0.00 1 0.001 0 0 2023-09-23T18:04:21.621 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T18:04:21.621 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: ** Compaction Stats [default] ** 2023-09-23T18:04:21.621 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T18:04:21.621 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T18:04:21.621 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 71.6 0.00 0.00 1 0.001 0 0 2023-09-23T18:04:21.622 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T18:04:21.622 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T18:04:21.622 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T18:04:21.622 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T18:04:21.622 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T18:04:21.622 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: AddFile(Keys): cumulative 0, interval 0 2023-09-23T18:04:21.622 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: Cumulative compaction: 0.00 GB write, 8.42 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T18:04:21.622 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T18:04:21.622 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T18:04:21.622 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: ** File Read Latency Histogram By Level [default] ** 2023-09-23T18:04:21.622 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.965+0000 7fe07f4db880 0 starting mon.a rank 0 at public addrs [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] at bind addrs [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:04:21.909 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.969+0000 7fe07f4db880 1 mon.a@-1(???) e1 preinit fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:04:21.909 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.969+0000 7fe07f4db880 0 mon.a@-1(???).mds e1 new map 2023-09-23T18:04:21.909 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.969+0000 7fe07f4db880 0 mon.a@-1(???).mds e1 print_map 2023-09-23T18:04:21.909 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: e1 2023-09-23T18:04:21.910 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T18:04:21.910 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T18:04:21.910 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: legacy client fscid: -1 2023-09-23T18:04:21.910 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: 2023-09-23T18:04:21.910 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: No filesystems configured 2023-09-23T18:04:21.910 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.969+0000 7fe07f4db880 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T18:04:21.910 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.969+0000 7fe07f4db880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T18:04:21.910 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.969+0000 7fe07f4db880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T18:04:21.910 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.969+0000 7fe07f4db880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T18:04:21.910 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: audit 2023-09-23T18:04:15.203041+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.98:0/3223383417' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:04:21.910 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:17 smithi098 bash[27349]: debug 2023-09-23T18:04:17.969+0000 7fe07f4db880 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-09-23T18:04:21.910 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:18 smithi098 bash[27349]: cluster 2023-09-23T18:04:17.999249+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T18:04:21.911 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:18 smithi098 bash[27349]: cluster 2023-09-23T18:04:17.999367+0000 mon.a (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0]} removed_ranks: {} 2023-09-23T18:04:21.911 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:18 smithi098 bash[27349]: cluster 2023-09-23T18:04:18.000036+0000 mon.a (mon.0) 3 : cluster [DBG] fsmap 2023-09-23T18:04:21.911 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:18 smithi098 bash[27349]: cluster 2023-09-23T18:04:18.000070+0000 mon.a (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-09-23T18:04:21.911 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:18 smithi098 bash[27349]: cluster 2023-09-23T18:04:18.000516+0000 mon.a (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2023-09-23T18:04:21.912 INFO:teuthology.orchestra.run.smithi098.stderr:Wrote config to /etc/ceph/ceph.conf 2023-09-23T18:04:21.912 INFO:teuthology.orchestra.run.smithi098.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-09-23T18:04:21.913 INFO:teuthology.orchestra.run.smithi098.stderr:Creating mgr... 2023-09-23T18:04:21.913 INFO:teuthology.orchestra.run.smithi098.stderr:Verifying port 9283 ... 2023-09-23T18:04:21.913 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: Failed to reset failed state of unit ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mgr.y.service: Unit ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mgr.y.service not loaded. 2023-09-23T18:04:21.913 INFO:teuthology.orchestra.run.smithi098.stderr:systemctl: Created symlink /etc/systemd/system/ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7.target.wants/ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mgr.y.service -> /etc/systemd/system/ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@.service. 2023-09-23T18:04:21.913 INFO:teuthology.orchestra.run.smithi098.stderr:firewalld does not appear to be present 2023-09-23T18:04:21.913 INFO:teuthology.orchestra.run.smithi098.stderr:Not possible to enable service . firewalld.service is not available 2023-09-23T18:04:21.913 INFO:teuthology.orchestra.run.smithi098.stderr:firewalld does not appear to be present 2023-09-23T18:04:21.913 INFO:teuthology.orchestra.run.smithi098.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2023-09-23T18:04:22.265 INFO:teuthology.orchestra.run.smithi098.stderr:Waiting for mgr to start... 2023-09-23T18:04:22.268 INFO:teuthology.orchestra.run.smithi098.stderr:Waiting for mgr... 2023-09-23T18:04:22.269 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:19 smithi098 bash[27349]: audit 2023-09-23T18:04:18.228045+0000 mon.a (mon.0) 6 : audit [INF] from='client.? 172.21.15.98:0/1934897220' entity='client.admin' 2023-09-23T18:04:22.274 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: 2023-09-23T18:04:22.622 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: { 2023-09-23T18:04:22.622 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "fsid": "756c96d4-5a3b-11ee-8db4-212e2dc638e7", 2023-09-23T18:04:22.622 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "health": { 2023-09-23T18:04:22.622 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-09-23T18:04:22.622 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "checks": {}, 2023-09-23T18:04:22.622 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "mutes": [] 2023-09-23T18:04:22.622 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: }, 2023-09-23T18:04:22.623 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-09-23T18:04:22.623 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "quorum": [ 2023-09-23T18:04:22.623 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: 0 2023-09-23T18:04:22.623 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: ], 2023-09-23T18:04:22.623 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "quorum_names": [ 2023-09-23T18:04:22.623 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "a" 2023-09-23T18:04:22.623 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: ], 2023-09-23T18:04:22.623 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "quorum_age": 3, 2023-09-23T18:04:22.623 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "monmap": { 2023-09-23T18:04:22.623 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T18:04:22.623 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-09-23T18:04:22.623 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_mons": 1 2023-09-23T18:04:22.624 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: }, 2023-09-23T18:04:22.624 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "osdmap": { 2023-09-23T18:04:22.624 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T18:04:22.624 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_osds": 0, 2023-09-23T18:04:22.624 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-09-23T18:04:22.624 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-09-23T18:04:22.624 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-09-23T18:04:22.624 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-09-23T18:04:22.624 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-09-23T18:04:22.624 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: }, 2023-09-23T18:04:22.624 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "pgmap": { 2023-09-23T18:04:22.624 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-09-23T18:04:22.625 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-09-23T18:04:22.625 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_pools": 0, 2023-09-23T18:04:22.625 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_objects": 0, 2023-09-23T18:04:22.625 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-09-23T18:04:22.625 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-09-23T18:04:22.625 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-09-23T18:04:22.625 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "bytes_total": 0 2023-09-23T18:04:22.625 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: }, 2023-09-23T18:04:22.625 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "fsmap": { 2023-09-23T18:04:22.625 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T18:04:22.625 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "by_rank": [], 2023-09-23T18:04:22.626 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "up:standby": 0 2023-09-23T18:04:22.626 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: }, 2023-09-23T18:04:22.626 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "mgrmap": { 2023-09-23T18:04:22.626 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "available": false, 2023-09-23T18:04:22.626 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-09-23T18:04:22.626 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "modules": [ 2023-09-23T18:04:22.626 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "iostat", 2023-09-23T18:04:22.626 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "nfs", 2023-09-23T18:04:22.982 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "restful" 2023-09-23T18:04:22.982 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: ], 2023-09-23T18:04:22.982 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "services": {} 2023-09-23T18:04:22.982 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: }, 2023-09-23T18:04:22.982 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "servicemap": { 2023-09-23T18:04:22.982 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T18:04:22.982 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "modified": "2023-09-23T18:04:11.581374+0000", 2023-09-23T18:04:22.982 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "services": {} 2023-09-23T18:04:22.983 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: }, 2023-09-23T18:04:22.985 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "progress_events": {} 2023-09-23T18:04:22.985 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: } 2023-09-23T18:04:22.985 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:21 smithi098 bash[27349]: audit 2023-09-23T18:04:21.075852+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.98:0/3715991675' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T18:04:22.986 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:21 smithi098 bash[27618]: debug 2023-09-23T18:04:21.133+0000 7fb0905cb680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T18:04:22.986 INFO:teuthology.orchestra.run.smithi098.stderr:mgr not available, waiting (1/15)... 2023-09-23T18:04:22.986 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:21 smithi098 bash[27618]: debug 2023-09-23T18:04:21.609+0000 7fb0905cb680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T18:04:22.986 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:22 smithi098 bash[27618]: debug 2023-09-23T18:04:22.525+0000 7fb0905cb680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T18:04:22.986 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:22 smithi098 bash[27618]: debug 2023-09-23T18:04:22.621+0000 7fb0905cb680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T18:04:23.190 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:22 smithi098 bash[27618]: debug 2023-09-23T18:04:22.865+0000 7fb0905cb680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T18:04:24.190 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:24 smithi098 bash[27618]: debug 2023-09-23T18:04:24.021+0000 7fb0905cb680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T18:04:24.641 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:24 smithi098 bash[27618]: debug 2023-09-23T18:04:24.185+0000 7fb0905cb680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:04:25.085 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:24 smithi098 bash[27618]: debug 2023-09-23T18:04:24.285+0000 7fb0905cb680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T18:04:25.087 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:24 smithi098 bash[27618]: debug 2023-09-23T18:04:24.397+0000 7fb0905cb680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T18:04:25.087 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:24 smithi098 bash[27618]: debug 2023-09-23T18:04:24.637+0000 7fb0905cb680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T18:04:25.093 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: 2023-09-23T18:04:25.445 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: { 2023-09-23T18:04:25.445 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "fsid": "756c96d4-5a3b-11ee-8db4-212e2dc638e7", 2023-09-23T18:04:25.445 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "health": { 2023-09-23T18:04:25.445 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-09-23T18:04:25.445 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "checks": {}, 2023-09-23T18:04:25.445 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "mutes": [] 2023-09-23T18:04:25.446 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: }, 2023-09-23T18:04:25.447 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-09-23T18:04:25.447 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "quorum": [ 2023-09-23T18:04:25.447 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: 0 2023-09-23T18:04:25.447 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: ], 2023-09-23T18:04:25.447 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "quorum_names": [ 2023-09-23T18:04:25.447 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "a" 2023-09-23T18:04:25.447 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: ], 2023-09-23T18:04:25.447 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "quorum_age": 7, 2023-09-23T18:04:25.447 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "monmap": { 2023-09-23T18:04:25.448 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T18:04:25.448 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-09-23T18:04:25.448 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_mons": 1 2023-09-23T18:04:25.448 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: }, 2023-09-23T18:04:25.448 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "osdmap": { 2023-09-23T18:04:25.448 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T18:04:25.448 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_osds": 0, 2023-09-23T18:04:25.448 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-09-23T18:04:25.448 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-09-23T18:04:25.448 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-09-23T18:04:25.448 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-09-23T18:04:25.448 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-09-23T18:04:25.448 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: }, 2023-09-23T18:04:25.449 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "pgmap": { 2023-09-23T18:04:25.449 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-09-23T18:04:25.871 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-09-23T18:04:25.871 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_pools": 0, 2023-09-23T18:04:25.871 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_objects": 0, 2023-09-23T18:04:25.871 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-09-23T18:04:25.871 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-09-23T18:04:25.871 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-09-23T18:04:25.872 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "bytes_total": 0 2023-09-23T18:04:25.872 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: }, 2023-09-23T18:04:25.872 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "fsmap": { 2023-09-23T18:04:25.872 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T18:04:25.872 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "by_rank": [], 2023-09-23T18:04:25.872 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "up:standby": 0 2023-09-23T18:04:25.872 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: }, 2023-09-23T18:04:25.872 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "mgrmap": { 2023-09-23T18:04:25.872 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "available": false, 2023-09-23T18:04:25.872 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-09-23T18:04:25.872 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "modules": [ 2023-09-23T18:04:25.872 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "iostat", 2023-09-23T18:04:25.873 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "nfs", 2023-09-23T18:04:25.873 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "restful" 2023-09-23T18:04:25.873 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: ], 2023-09-23T18:04:25.873 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "services": {} 2023-09-23T18:04:25.873 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: }, 2023-09-23T18:04:25.873 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "servicemap": { 2023-09-23T18:04:25.873 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T18:04:25.873 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "modified": "2023-09-23T18:04:11.581374+0000", 2023-09-23T18:04:25.873 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "services": {} 2023-09-23T18:04:25.873 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: }, 2023-09-23T18:04:25.873 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "progress_events": {} 2023-09-23T18:04:25.873 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: } 2023-09-23T18:04:25.874 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:25 smithi098 bash[27349]: audit 2023-09-23T18:04:25.050353+0000 mon.a (mon.0) 8 : audit [DBG] from='client.? 172.21.15.98:0/469907485' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T18:04:25.874 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:25 smithi098 bash[27618]: debug 2023-09-23T18:04:25.005+0000 7fb0905cb680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T18:04:25.875 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:25 smithi098 bash[27618]: debug 2023-09-23T18:04:25.129+0000 7fb0905cb680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T18:04:25.875 INFO:teuthology.orchestra.run.smithi098.stderr:mgr not available, waiting (2/15)... 2023-09-23T18:04:26.440 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:26 smithi098 bash[27618]: debug 2023-09-23T18:04:26.045+0000 7fb0905cb680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T18:04:26.682 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:26 smithi098 bash[27618]: debug 2023-09-23T18:04:26.145+0000 7fb0905cb680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T18:04:26.682 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:26 smithi098 bash[27618]: debug 2023-09-23T18:04:26.265+0000 7fb0905cb680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T18:04:26.813 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:26 smithi098 bash[27618]: debug 2023-09-23T18:04:26.501+0000 7fb0905cb680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T18:04:27.146 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:26 smithi098 bash[27618]: debug 2023-09-23T18:04:26.809+0000 7fb0905cb680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T18:04:27.481 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:26 smithi098 bash[27618]: debug 2023-09-23T18:04:26.969+0000 7fb0905cb680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T18:04:27.481 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:27 smithi098 bash[27618]: debug 2023-09-23T18:04:27.141+0000 7fb0905cb680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:04:27.481 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:27 smithi098 bash[27618]: debug 2023-09-23T18:04:27.353+0000 7fb0905cb680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T18:04:27.719 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:27 smithi098 bash[27618]: debug 2023-09-23T18:04:27.461+0000 7fb0905cb680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T18:04:28.000 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:27 smithi098 bash[27349]: cluster 2023-09-23T18:04:27.484543+0000 mon.a (mon.0) 9 : cluster [INF] Activating manager daemon y 2023-09-23T18:04:28.000 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:27 smithi098 bash[27349]: cluster 2023-09-23T18:04:27.491311+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e2: y(active, starting, since 0.00698597s) 2023-09-23T18:04:28.000 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:27 smithi098 bash[27349]: audit 2023-09-23T18:04:27.493239+0000 mon.a (mon.0) 11 : audit [DBG] from='mgr.14100 172.21.15.98:0/2036612763' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T18:04:28.000 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:27 smithi098 bash[27349]: audit 2023-09-23T18:04:27.493610+0000 mon.a (mon.0) 12 : audit [DBG] from='mgr.14100 172.21.15.98:0/2036612763' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T18:04:28.000 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:27 smithi098 bash[27349]: audit 2023-09-23T18:04:27.493855+0000 mon.a (mon.0) 13 : audit [DBG] from='mgr.14100 172.21.15.98:0/2036612763' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T18:04:28.000 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:27 smithi098 bash[27349]: audit 2023-09-23T18:04:27.494113+0000 mon.a (mon.0) 14 : audit [DBG] from='mgr.14100 172.21.15.98:0/2036612763' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:04:28.000 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:27 smithi098 bash[27349]: audit 2023-09-23T18:04:27.494846+0000 mon.a (mon.0) 15 : audit [DBG] from='mgr.14100 172.21.15.98:0/2036612763' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T18:04:28.000 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:27 smithi098 bash[27349]: cluster 2023-09-23T18:04:27.498806+0000 mon.a (mon.0) 16 : cluster [INF] Manager daemon y is now available 2023-09-23T18:04:28.000 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:27 smithi098 bash[27349]: audit 2023-09-23T18:04:27.509420+0000 mon.a (mon.0) 17 : audit [INF] from='mgr.14100 172.21.15.98:0/2036612763' entity='mgr.y' 2023-09-23T18:04:28.001 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:27 smithi098 bash[27349]: audit 2023-09-23T18:04:27.520170+0000 mon.a (mon.0) 18 : audit [INF] from='mgr.14100 172.21.15.98:0/2036612763' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:04:28.001 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:27 smithi098 bash[27349]: audit 2023-09-23T18:04:27.523822+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.14100 172.21.15.98:0/2036612763' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T18:04:28.001 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:27 smithi098 bash[27349]: audit 2023-09-23T18:04:27.527962+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.14100 172.21.15.98:0/2036612763' entity='mgr.y' 2023-09-23T18:04:28.001 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:27 smithi098 bash[27349]: audit 2023-09-23T18:04:27.530964+0000 mon.a (mon.0) 21 : audit [INF] from='mgr.14100 172.21.15.98:0/2036612763' entity='mgr.y' 2023-09-23T18:04:29.139 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: 2023-09-23T18:04:29.566 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: { 2023-09-23T18:04:29.566 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "fsid": "756c96d4-5a3b-11ee-8db4-212e2dc638e7", 2023-09-23T18:04:29.567 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "health": { 2023-09-23T18:04:29.567 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-09-23T18:04:29.572 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "checks": {}, 2023-09-23T18:04:29.902 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "mutes": [] 2023-09-23T18:04:29.902 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: }, 2023-09-23T18:04:29.902 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-09-23T18:04:29.902 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "quorum": [ 2023-09-23T18:04:29.903 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: 0 2023-09-23T18:04:29.903 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: ], 2023-09-23T18:04:29.903 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "quorum_names": [ 2023-09-23T18:04:29.903 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "a" 2023-09-23T18:04:29.903 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: ], 2023-09-23T18:04:29.903 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "quorum_age": 11, 2023-09-23T18:04:29.903 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "monmap": { 2023-09-23T18:04:29.903 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T18:04:29.903 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-09-23T18:04:29.903 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_mons": 1 2023-09-23T18:04:29.903 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: }, 2023-09-23T18:04:29.903 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "osdmap": { 2023-09-23T18:04:29.904 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T18:04:29.904 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_osds": 0, 2023-09-23T18:04:29.904 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-09-23T18:04:29.904 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-09-23T18:04:29.904 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-09-23T18:04:29.904 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-09-23T18:04:29.904 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-09-23T18:04:29.904 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: }, 2023-09-23T18:04:29.904 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "pgmap": { 2023-09-23T18:04:29.904 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-09-23T18:04:29.904 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-09-23T18:04:29.904 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_pools": 0, 2023-09-23T18:04:29.904 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_objects": 0, 2023-09-23T18:04:29.905 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-09-23T18:04:29.905 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-09-23T18:04:29.906 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-09-23T18:04:29.906 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "bytes_total": 0 2023-09-23T18:04:29.906 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: }, 2023-09-23T18:04:29.906 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "fsmap": { 2023-09-23T18:04:29.907 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T18:04:29.907 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "by_rank": [], 2023-09-23T18:04:29.907 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "up:standby": 0 2023-09-23T18:04:29.907 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: }, 2023-09-23T18:04:29.907 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "mgrmap": { 2023-09-23T18:04:29.907 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "available": true, 2023-09-23T18:04:29.907 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-09-23T18:04:29.907 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "modules": [ 2023-09-23T18:04:29.907 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "iostat", 2023-09-23T18:04:29.907 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "nfs", 2023-09-23T18:04:29.907 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "restful" 2023-09-23T18:04:29.907 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: ], 2023-09-23T18:04:30.239 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "services": {} 2023-09-23T18:04:30.239 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: }, 2023-09-23T18:04:30.239 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "servicemap": { 2023-09-23T18:04:30.239 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T18:04:30.239 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "modified": "2023-09-23T18:04:11.581374+0000", 2023-09-23T18:04:30.239 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "services": {} 2023-09-23T18:04:30.239 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: }, 2023-09-23T18:04:30.239 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "progress_events": {} 2023-09-23T18:04:30.239 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: } 2023-09-23T18:04:30.240 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:29 smithi098 bash[27349]: cluster 2023-09-23T18:04:28.496326+0000 mon.a (mon.0) 22 : cluster [DBG] mgrmap e3: y(active, since 1.012s) 2023-09-23T18:04:30.240 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:29 smithi098 bash[27349]: audit 2023-09-23T18:04:29.136910+0000 mon.a (mon.0) 23 : audit [DBG] from='client.? 172.21.15.98:0/4242000916' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T18:04:30.240 INFO:teuthology.orchestra.run.smithi098.stderr:mgr is available 2023-09-23T18:04:31.149 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: 2023-09-23T18:04:31.878 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: [global] 2023-09-23T18:04:31.878 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: fsid = 756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:04:31.878 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: log_to_journald = false 2023-09-23T18:04:31.878 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-09-23T18:04:31.881 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-09-23T18:04:32.302 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-09-23T18:04:32.302 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-09-23T18:04:32.302 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: 2023-09-23T18:04:32.302 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: [mgr] 2023-09-23T18:04:32.303 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-09-23T18:04:32.303 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: 2023-09-23T18:04:32.303 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: [osd] 2023-09-23T18:04:32.303 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-09-23T18:04:32.303 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-09-23T18:04:32.303 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:31 smithi098 bash[27349]: cluster 2023-09-23T18:04:30.493712+0000 mon.a (mon.0) 24 : cluster [DBG] mgrmap e4: y(active, since 3s) 2023-09-23T18:04:32.303 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:31 smithi098 bash[27349]: audit 2023-09-23T18:04:31.146397+0000 mon.a (mon.0) 25 : audit [INF] from='client.? 172.21.15.98:0/2159969297' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-09-23T18:04:32.304 INFO:teuthology.orchestra.run.smithi098.stderr:Enabling cephadm module... 2023-09-23T18:04:34.246 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:34 smithi098 bash[27618]: ignoring --setuser ceph since I am not root 2023-09-23T18:04:36.482 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:34 smithi098 bash[27618]: ignoring --setgroup ceph since I am not root 2023-09-23T18:04:36.482 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:34 smithi098 bash[27618]: debug 2023-09-23T18:04:34.189+0000 7f013608a680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T18:04:36.482 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:33 smithi098 bash[27349]: audit 2023-09-23T18:04:33.253704+0000 mon.a (mon.0) 26 : audit [INF] from='client.? 172.21.15.98:0/4261373498' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-09-23T18:04:36.483 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:34 smithi098 bash[27618]: debug 2023-09-23T18:04:34.293+0000 7f013608a680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T18:04:36.483 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:34 smithi098 bash[27618]: debug 2023-09-23T18:04:34.757+0000 7f013608a680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T18:04:36.483 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:34 smithi098 bash[27349]: audit 2023-09-23T18:04:33.987080+0000 mon.a (mon.0) 27 : audit [INF] from='client.? 172.21.15.98:0/4261373498' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-09-23T18:04:36.483 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:34 smithi098 bash[27349]: cluster 2023-09-23T18:04:33.987172+0000 mon.a (mon.0) 28 : cluster [DBG] mgrmap e5: y(active, since 6s) 2023-09-23T18:04:36.484 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:35 smithi098 bash[27618]: debug 2023-09-23T18:04:35.681+0000 7f013608a680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T18:04:36.484 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:35 smithi098 bash[27618]: debug 2023-09-23T18:04:35.777+0000 7f013608a680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T18:04:36.485 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: { 2023-09-23T18:04:36.485 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "epoch": 5, 2023-09-23T18:04:36.485 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "available": true, 2023-09-23T18:04:36.485 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "active_name": "y", 2023-09-23T18:04:36.759 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_standby": 0 2023-09-23T18:04:36.759 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: } 2023-09-23T18:04:36.760 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:36 smithi098 bash[27349]: audit 2023-09-23T18:04:36.177610+0000 mon.a (mon.0) 29 : audit [DBG] from='client.? 172.21.15.98:0/900632570' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-09-23T18:04:36.760 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:36 smithi098 bash[27618]: debug 2023-09-23T18:04:36.033+0000 7f013608a680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T18:04:36.933 INFO:teuthology.orchestra.run.smithi098.stderr:Waiting for the mgr to restart... 2023-09-23T18:04:37.294 INFO:teuthology.orchestra.run.smithi098.stderr:Waiting for mgr epoch 5... 2023-09-23T18:04:37.339 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:37 smithi098 bash[27618]: debug 2023-09-23T18:04:37.169+0000 7f013608a680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T18:04:38.819 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:37 smithi098 bash[27618]: debug 2023-09-23T18:04:37.337+0000 7f013608a680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:04:38.819 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:37 smithi098 bash[27618]: debug 2023-09-23T18:04:37.429+0000 7f013608a680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T18:04:38.819 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:37 smithi098 bash[27618]: debug 2023-09-23T18:04:37.549+0000 7f013608a680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T18:04:38.820 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:37 smithi098 bash[27618]: debug 2023-09-23T18:04:37.773+0000 7f013608a680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T18:04:38.820 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:38 smithi098 bash[27618]: debug 2023-09-23T18:04:38.173+0000 7f013608a680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T18:04:38.820 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:38 smithi098 bash[27618]: debug 2023-09-23T18:04:38.301+0000 7f013608a680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T18:04:39.440 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:39 smithi098 bash[27618]: debug 2023-09-23T18:04:39.185+0000 7f013608a680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T18:04:39.686 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:39 smithi098 bash[27618]: debug 2023-09-23T18:04:39.293+0000 7f013608a680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T18:04:39.686 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:39 smithi098 bash[27618]: debug 2023-09-23T18:04:39.409+0000 7f013608a680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T18:04:39.924 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:39 smithi098 bash[27618]: debug 2023-09-23T18:04:39.629+0000 7f013608a680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T18:04:40.190 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:39 smithi098 bash[27618]: debug 2023-09-23T18:04:39.921+0000 7f013608a680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T18:04:40.418 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:40 smithi098 bash[27618]: debug 2023-09-23T18:04:40.081+0000 7f013608a680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T18:04:40.575 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:40 smithi098 bash[27618]: debug 2023-09-23T18:04:40.253+0000 7f013608a680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:04:41.012 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:40 smithi098 bash[27618]: debug 2023-09-23T18:04:40.469+0000 7f013608a680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T18:04:41.013 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:40 smithi098 bash[27618]: debug 2023-09-23T18:04:40.573+0000 7f013608a680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T18:04:41.014 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:40 smithi098 bash[27349]: cluster 2023-09-23T18:04:40.596788+0000 mon.a (mon.0) 30 : cluster [INF] Active manager daemon y restarted 2023-09-23T18:04:41.014 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:40 smithi098 bash[27349]: cluster 2023-09-23T18:04:40.597225+0000 mon.a (mon.0) 31 : cluster [INF] Activating manager daemon y 2023-09-23T18:04:41.014 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:40 smithi098 bash[27349]: cluster 2023-09-23T18:04:40.602818+0000 mon.a (mon.0) 32 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2023-09-23T18:04:41.014 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:40 smithi098 bash[27349]: cluster 2023-09-23T18:04:40.602986+0000 mon.a (mon.0) 33 : cluster [DBG] mgrmap e6: y(active, starting, since 0.00591679s) 2023-09-23T18:04:41.014 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:40 smithi098 bash[27349]: audit 2023-09-23T18:04:40.605250+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.14116 172.21.15.98:0/3637674121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:04:41.014 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:40 smithi098 bash[27349]: audit 2023-09-23T18:04:40.606192+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.14116 172.21.15.98:0/3637674121' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T18:04:41.014 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:40 smithi098 bash[27349]: audit 2023-09-23T18:04:40.607380+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.14116 172.21.15.98:0/3637674121' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T18:04:41.014 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:40 smithi098 bash[27349]: audit 2023-09-23T18:04:40.607687+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.14116 172.21.15.98:0/3637674121' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T18:04:41.014 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:40 smithi098 bash[27349]: audit 2023-09-23T18:04:40.607880+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.14116 172.21.15.98:0/3637674121' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T18:04:41.015 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:40 smithi098 bash[27349]: cluster 2023-09-23T18:04:40.610918+0000 mon.a (mon.0) 39 : cluster [INF] Manager daemon y is now available 2023-09-23T18:04:41.015 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:40 smithi098 bash[27349]: audit 2023-09-23T18:04:40.633199+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.14116 172.21.15.98:0/3637674121' entity='mgr.y' 2023-09-23T18:04:41.623 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: { 2023-09-23T18:04:41.842 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2023-09-23T18:04:41.842 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "initialized": true 2023-09-23T18:04:41.843 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: } 2023-09-23T18:04:41.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:41 smithi098 bash[27349]: cephadm 2023-09-23T18:04:40.628202+0000 mgr.y (mgr.14116) 1 : cephadm [INF] Found migration_current of "None". Setting to last migration. 2023-09-23T18:04:42.066 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:41 smithi098 bash[27349]: audit 2023-09-23T18:04:40.763359+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.14116 172.21.15.98:0/3637674121' entity='mgr.y' 2023-09-23T18:04:42.066 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:41 smithi098 bash[27349]: audit 2023-09-23T18:04:40.765413+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.14116 172.21.15.98:0/3637674121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:04:42.067 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:41 smithi098 bash[27349]: audit 2023-09-23T18:04:40.767528+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.14116 172.21.15.98:0/3637674121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:04:42.068 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:41 smithi098 bash[27349]: audit 2023-09-23T18:04:40.771367+0000 mon.a (mon.0) 44 : audit [DBG] from='mgr.14116 172.21.15.98:0/3637674121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:04:42.068 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:41 smithi098 bash[27349]: audit 2023-09-23T18:04:40.795686+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.14116 172.21.15.98:0/3637674121' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:04:42.068 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:41 smithi098 bash[27349]: audit 2023-09-23T18:04:40.799686+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.14116 172.21.15.98:0/3637674121' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T18:04:42.068 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:41 smithi098 bash[27349]: cluster 2023-09-23T18:04:41.607270+0000 mon.a (mon.0) 47 : cluster [DBG] mgrmap e7: y(active, since 1.0102s) 2023-09-23T18:04:42.399 INFO:teuthology.orchestra.run.smithi098.stderr:mgr epoch 5 is available 2023-09-23T18:04:42.571 INFO:teuthology.orchestra.run.smithi098.stderr:Setting orchestrator backend to cephadm... 2023-09-23T18:04:43.107 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:42 smithi098 bash[27349]: audit 2023-09-23T18:04:41.606696+0000 mgr.y (mgr.14116) 2 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-09-23T18:04:43.362 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:42 smithi098 bash[27349]: audit 2023-09-23T18:04:41.621651+0000 mgr.y (mgr.14116) 3 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-09-23T18:04:44.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:43 smithi098 bash[27349]: cluster 2023-09-23T18:04:42.766967+0000 mon.a (mon.0) 48 : cluster [DBG] mgrmap e8: y(active, since 2s) 2023-09-23T18:04:47.926 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:43 smithi098 bash[27349]: audit 2023-09-23T18:04:43.721870+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.14116 172.21.15.98:0/3637674121' entity='mgr.y' 2023-09-23T18:04:47.926 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:43 smithi098 bash[27349]: audit 2023-09-23T18:04:43.750607+0000 mon.a (mon.0) 50 : audit [DBG] from='mgr.14116 172.21.15.98:0/3637674121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:04:47.927 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:44 smithi098 bash[27349]: audit 2023-09-23T18:04:43.712429+0000 mgr.y (mgr.14116) 4 : audit [DBG] from='client.14126 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:04:47.928 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: value unchanged 2023-09-23T18:04:47.928 INFO:teuthology.orchestra.run.smithi098.stderr:Generating ssh key... 2023-09-23T18:04:47.928 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:46 smithi098 bash[27349]: audit 2023-09-23T18:04:45.723904+0000 mgr.y (mgr.14116) 5 : audit [DBG] from='client.14128 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:04:48.441 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:48 smithi098 bash[27618]: Generating public/private rsa key pair. 2023-09-23T18:04:49.368 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:48 smithi098 bash[27618]: Your identification has been saved in /tmp/tmpueo1lxnc/key. 2023-09-23T18:04:49.368 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:48 smithi098 bash[27618]: Your public key has been saved in /tmp/tmpueo1lxnc/key.pub. 2023-09-23T18:04:49.368 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:48 smithi098 bash[27618]: The key fingerprint is: 2023-09-23T18:04:49.368 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:48 smithi098 bash[27618]: SHA256:pDzoSQfHXQ4vgB9a7kRRynr7NUvmQJw0rVX7Yf6Mgxo ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:04:49.368 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:48 smithi098 bash[27618]: The key's randomart image is: 2023-09-23T18:04:49.368 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:48 smithi098 bash[27618]: +---[RSA 3072]----+ 2023-09-23T18:04:53.472 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:48 smithi098 bash[27618]: | .ooo . . | 2023-09-23T18:04:53.472 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:48 smithi098 bash[27618]: | .o++ * . . | 2023-09-23T18:04:53.473 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:48 smithi098 bash[27618]: | .*=.* = . o | 2023-09-23T18:04:53.473 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:48 smithi098 bash[27618]: | .*+= * + . | 2023-09-23T18:04:53.473 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:48 smithi098 bash[27618]: | +o* S o | 2023-09-23T18:04:53.473 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:48 smithi098 bash[27618]: | o +.+ . + | 2023-09-23T18:04:53.473 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:48 smithi098 bash[27618]: | o . . E . o o| 2023-09-23T18:04:53.473 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:48 smithi098 bash[27618]: | . * = . | 2023-09-23T18:04:53.473 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:48 smithi098 bash[27618]: | . + | 2023-09-23T18:04:53.473 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:48 smithi098 bash[27618]: +----[SHA256]-----+ 2023-09-23T18:04:53.474 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:48 smithi098 bash[27349]: audit 2023-09-23T18:04:47.851770+0000 mgr.y (mgr.14116) 6 : audit [DBG] from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:04:53.474 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:48 smithi098 bash[27349]: cephadm 2023-09-23T18:04:47.852191+0000 mgr.y (mgr.14116) 7 : cephadm [INF] Generating ssh key... 2023-09-23T18:04:53.474 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:48 smithi098 bash[27349]: audit 2023-09-23T18:04:48.049869+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.14116 172.21.15.98:0/3637674121' entity='mgr.y' 2023-09-23T18:04:53.474 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:48 smithi098 bash[27349]: audit 2023-09-23T18:04:48.053979+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.14116 172.21.15.98:0/3637674121' entity='mgr.y' 2023-09-23T18:04:53.474 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDiOC0Vauqm/LO3SfV494E8z+XRRKWV+Fw+p26C6RwaVgi9mczlKNwbhDp/NZe4POgHsJwh/iHYsIsjDumRhVrHYzqMD1e9ELyzoio9LjY3Wilh12YEU5FwIXgoV7ICL3vbHOLYzJ0jSDjgfREBYqn7K6E8HLOYmRdSBr2gRbd7hbOyJNDyH+9pQDyNQyINMkKmSz8HyyqxkJfmYrnXYFesHjn+Aj2UyLWt+tQQZ8X/fu26zzoLLDB7J1crL0sN6ExLIbxbti/hBe+ALSYJ9jSIoGuc2PxnTV3uDT3rY4amnZcitkG9XuGSz9OxxqIoBWsB/4uHKNmICp6TL29T5hz4m4xJMtxGR4Cjt7qOhm54oFu4DNBWKGOsx8vvPNvRqg16my9hVW+bwcosswaiQ52N7DHSHSmtraPXrOFAas8v/ZfOioHgpyAYcjpkS0vftfoGzQZ1GRkFQ4i4++W4HIMsqsRqvEwCu+Vf8oJX1AcjqX2ByRkgNkgJvZQYm7V+g0k= ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:04:53.475 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:50 smithi098 bash[27349]: audit 2023-09-23T18:04:49.949891+0000 mgr.y (mgr.14116) 8 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:04:53.475 INFO:teuthology.orchestra.run.smithi098.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-09-23T18:04:56.014 INFO:teuthology.orchestra.run.smithi098.stderr:Adding key to root@localhost authorized_keys... 2023-09-23T18:04:56.015 INFO:teuthology.orchestra.run.smithi098.stderr:Adding host smithi098... 2023-09-23T18:04:56.018 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:52 smithi098 bash[27349]: audit 2023-09-23T18:04:51.952263+0000 mgr.y (mgr.14116) 9 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi098", "addr": "172.21.15.98", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:04:57.671 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:04:52 smithi098 bash[27618]: Warning: Permanently added '172.21.15.98' (ECDSA) to the list of known hosts. 2023-09-23T18:04:57.671 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: Added host 'smithi098' with addr '172.21.15.98' 2023-09-23T18:04:57.672 INFO:teuthology.orchestra.run.smithi098.stderr:Deploying unmanaged mon service... 2023-09-23T18:04:57.672 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:53 smithi098 bash[27349]: audit 2023-09-23T18:04:52.782866+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.14116 172.21.15.98:0/3637674121' entity='mgr.y' 2023-09-23T18:04:57.950 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:53 smithi098 bash[27349]: cephadm 2023-09-23T18:04:52.783597+0000 mgr.y (mgr.14116) 10 : cephadm [INF] Added host smithi098 2023-09-23T18:04:57.950 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:53 smithi098 bash[27349]: audit 2023-09-23T18:04:52.797415+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.14116 172.21.15.98:0/3637674121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:04:57.951 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: Scheduled mon update... 2023-09-23T18:04:58.589 INFO:teuthology.orchestra.run.smithi098.stderr:Deploying unmanaged mgr service... 2023-09-23T18:04:58.589 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:56 smithi098 bash[27349]: audit 2023-09-23T18:04:55.697642+0000 mgr.y (mgr.14116) 11 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:04:58.589 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:56 smithi098 bash[27349]: cephadm 2023-09-23T18:04:55.699035+0000 mgr.y (mgr.14116) 12 : cephadm [INF] Saving service mon spec with placement count:5 2023-09-23T18:04:58.589 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:56 smithi098 bash[27349]: audit 2023-09-23T18:04:55.701355+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.14116 172.21.15.98:0/3637674121' entity='mgr.y' 2023-09-23T18:04:58.589 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:56 smithi098 bash[27349]: audit 2023-09-23T18:04:56.269451+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.14116 172.21.15.98:0/3637674121' entity='mgr.y' 2023-09-23T18:04:58.589 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:56 smithi098 bash[27349]: audit 2023-09-23T18:04:56.499093+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.14116 172.21.15.98:0/3637674121' entity='mgr.y' 2023-09-23T18:04:58.590 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: Scheduled mgr update... 2023-09-23T18:04:59.406 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:58 smithi098 bash[27349]: audit 2023-09-23T18:04:57.977603+0000 mgr.y (mgr.14116) 13 : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:04:59.881 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:58 smithi098 bash[27349]: cephadm 2023-09-23T18:04:57.979950+0000 mgr.y (mgr.14116) 14 : cephadm [INF] Saving service mgr spec with placement count:2 2023-09-23T18:04:59.881 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:04:58 smithi098 bash[27349]: audit 2023-09-23T18:04:57.984186+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.14116 172.21.15.98:0/3637674121' entity='mgr.y' 2023-09-23T18:05:01.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:01 smithi098 bash[27349]: audit 2023-09-23T18:05:00.033140+0000 mon.a (mon.0) 59 : audit [INF] from='client.? 172.21.15.98:0/3425310232' entity='client.admin' 2023-09-23T18:05:02.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:02 smithi098 bash[27349]: cluster 2023-09-23T18:05:00.609343+0000 mgr.y (mgr.14116) 15 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:05:02.210 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:02 smithi098 bash[27349]: audit 2023-09-23T18:05:01.535117+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.14116 172.21.15.98:0/3637674121' entity='mgr.y' 2023-09-23T18:05:02.733 INFO:teuthology.orchestra.run.smithi098.stderr:Enabling the dashboard module... 2023-09-23T18:05:03.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:03 smithi098 bash[27349]: audit 2023-09-23T18:05:02.060191+0000 mon.a (mon.0) 61 : audit [INF] from='client.? 172.21.15.98:0/4120949662' entity='client.admin' 2023-09-23T18:05:03.572 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:03 smithi098 bash[27349]: cluster 2023-09-23T18:05:02.609820+0000 mgr.y (mgr.14116) 16 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:05:04.466 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:04 smithi098 bash[27618]: ignoring --setuser ceph since I am not root 2023-09-23T18:05:06.773 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:04 smithi098 bash[27618]: ignoring --setgroup ceph since I am not root 2023-09-23T18:05:06.773 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:04 smithi098 bash[27618]: debug 2023-09-23T18:05:04.384+0000 7fa0c403a680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T18:05:06.774 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:04 smithi098 bash[27349]: audit 2023-09-23T18:05:04.133576+0000 mon.a (mon.0) 62 : audit [INF] from='client.? 172.21.15.98:0/395087533' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-09-23T18:05:06.774 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:04 smithi098 bash[27618]: debug 2023-09-23T18:05:04.488+0000 7fa0c403a680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T18:05:06.774 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:04 smithi098 bash[27618]: debug 2023-09-23T18:05:04.948+0000 7fa0c403a680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T18:05:06.775 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:05 smithi098 bash[27349]: audit 2023-09-23T18:05:04.191063+0000 mon.a (mon.0) 63 : audit [INF] from='client.? 172.21.15.98:0/395087533' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-09-23T18:05:06.775 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:05 smithi098 bash[27349]: cluster 2023-09-23T18:05:04.191192+0000 mon.a (mon.0) 64 : cluster [DBG] mgrmap e9: y(active, since 23s) 2023-09-23T18:05:06.775 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:05 smithi098 bash[27618]: debug 2023-09-23T18:05:05.904+0000 7fa0c403a680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T18:05:07.446 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:06 smithi098 bash[27618]: debug 2023-09-23T18:05:06.012+0000 7fa0c403a680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T18:05:07.447 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: { 2023-09-23T18:05:07.447 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "epoch": 9, 2023-09-23T18:05:07.447 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "available": true, 2023-09-23T18:05:07.448 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "active_name": "y", 2023-09-23T18:05:07.448 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "num_standby": 0 2023-09-23T18:05:07.448 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: } 2023-09-23T18:05:07.448 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:06 smithi098 bash[27618]: debug 2023-09-23T18:05:06.388+0000 7fa0c403a680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T18:05:07.448 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:06 smithi098 bash[27349]: audit 2023-09-23T18:05:06.595706+0000 mon.a (mon.0) 65 : audit [DBG] from='client.? 172.21.15.98:0/3094943033' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-09-23T18:05:07.449 INFO:teuthology.orchestra.run.smithi098.stderr:Waiting for the mgr to restart... 2023-09-23T18:05:08.800 INFO:teuthology.orchestra.run.smithi098.stderr:Waiting for mgr epoch 9... 2023-09-23T18:05:08.801 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:07 smithi098 bash[27618]: debug 2023-09-23T18:05:07.656+0000 7fa0c403a680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T18:05:08.801 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:07 smithi098 bash[27618]: debug 2023-09-23T18:05:07.824+0000 7fa0c403a680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:05:08.801 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:07 smithi098 bash[27618]: debug 2023-09-23T18:05:07.920+0000 7fa0c403a680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T18:05:08.801 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:08 smithi098 bash[27618]: debug 2023-09-23T18:05:08.028+0000 7fa0c403a680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T18:05:08.801 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:08 smithi098 bash[27618]: debug 2023-09-23T18:05:08.264+0000 7fa0c403a680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T18:05:08.940 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:08 smithi098 bash[27618]: debug 2023-09-23T18:05:08.624+0000 7fa0c403a680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T18:05:12.220 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:08 smithi098 bash[27618]: debug 2023-09-23T18:05:08.744+0000 7fa0c403a680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T18:05:12.222 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:09 smithi098 bash[27618]: debug 2023-09-23T18:05:09.636+0000 7fa0c403a680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T18:05:13.572 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:09 smithi098 bash[27618]: debug 2023-09-23T18:05:09.744+0000 7fa0c403a680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T18:05:13.572 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:09 smithi098 bash[27618]: debug 2023-09-23T18:05:09.860+0000 7fa0c403a680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T18:05:13.572 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:10 smithi098 bash[27618]: debug 2023-09-23T18:05:10.084+0000 7fa0c403a680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T18:05:13.572 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:10 smithi098 bash[27618]: debug 2023-09-23T18:05:10.376+0000 7fa0c403a680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T18:05:13.572 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:10 smithi098 bash[27618]: debug 2023-09-23T18:05:10.544+0000 7fa0c403a680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T18:05:15.987 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:10 smithi098 bash[27618]: debug 2023-09-23T18:05:10.712+0000 7fa0c403a680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:05:15.987 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:10 smithi098 bash[27618]: debug 2023-09-23T18:05:10.924+0000 7fa0c403a680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T18:05:15.987 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:11 smithi098 bash[27618]: debug 2023-09-23T18:05:11.036+0000 7fa0c403a680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T18:05:15.987 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:11 smithi098 bash[27618]: Warning: Permanently added '172.21.15.98' (ECDSA) to the list of known hosts. 2023-09-23T18:05:15.988 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:11 smithi098 bash[27349]: cluster 2023-09-23T18:05:11.060598+0000 mon.a (mon.0) 66 : cluster [INF] Active manager daemon y restarted 2023-09-23T18:05:15.988 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:11 smithi098 bash[27349]: cluster 2023-09-23T18:05:11.061013+0000 mon.a (mon.0) 67 : cluster [INF] Activating manager daemon y 2023-09-23T18:05:15.988 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:11 smithi098 bash[27349]: cluster 2023-09-23T18:05:11.068779+0000 mon.a (mon.0) 68 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2023-09-23T18:05:15.988 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:11 smithi098 bash[27349]: cluster 2023-09-23T18:05:11.068922+0000 mon.a (mon.0) 69 : cluster [DBG] mgrmap e10: y(active, starting, since 0.00807867s) 2023-09-23T18:05:15.988 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:11 smithi098 bash[27349]: audit 2023-09-23T18:05:11.070199+0000 mon.a (mon.0) 70 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:05:15.988 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:11 smithi098 bash[27349]: audit 2023-09-23T18:05:11.071169+0000 mon.a (mon.0) 71 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T18:05:15.988 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:11 smithi098 bash[27349]: audit 2023-09-23T18:05:11.072647+0000 mon.a (mon.0) 72 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T18:05:15.989 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:11 smithi098 bash[27349]: audit 2023-09-23T18:05:11.072894+0000 mon.a (mon.0) 73 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T18:05:15.989 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:11 smithi098 bash[27349]: audit 2023-09-23T18:05:11.073029+0000 mon.a (mon.0) 74 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T18:05:15.989 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:11 smithi098 bash[27349]: cluster 2023-09-23T18:05:11.076981+0000 mon.a (mon.0) 75 : cluster [INF] Manager daemon y is now available 2023-09-23T18:05:15.989 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: { 2023-09-23T18:05:16.456 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2023-09-23T18:05:16.456 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: "initialized": true 2023-09-23T18:05:16.456 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: } 2023-09-23T18:05:16.456 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:12 smithi098 bash[27349]: audit 2023-09-23T18:05:11.213120+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:05:16.457 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:12 smithi098 bash[27349]: audit 2023-09-23T18:05:11.218271+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:05:16.457 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:12 smithi098 bash[27349]: audit 2023-09-23T18:05:11.255618+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:05:16.457 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:12 smithi098 bash[27349]: audit 2023-09-23T18:05:11.260966+0000 mon.a (mon.0) 79 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T18:05:16.457 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:12 smithi098 bash[27349]: cluster 2023-09-23T18:05:12.070330+0000 mon.a (mon.0) 80 : cluster [DBG] mgrmap e11: y(active, since 1.00948s) 2023-09-23T18:05:16.457 INFO:teuthology.orchestra.run.smithi098.stderr:mgr epoch 9 is available 2023-09-23T18:05:16.457 INFO:teuthology.orchestra.run.smithi098.stderr:Generating a dashboard self-signed certificate... 2023-09-23T18:05:16.458 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:13 smithi098 bash[27349]: audit 2023-09-23T18:05:12.070387+0000 mgr.y (mgr.14146) 1 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-09-23T18:05:16.458 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:13 smithi098 bash[27349]: audit 2023-09-23T18:05:12.085914+0000 mgr.y (mgr.14146) 2 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-09-23T18:05:16.458 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: Self-signed certificate created 2023-09-23T18:05:16.458 INFO:teuthology.orchestra.run.smithi098.stderr:Creating initial admin user... 2023-09-23T18:05:16.459 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:14 smithi098 bash[27349]: cluster 2023-09-23T18:05:13.994161+0000 mon.a (mon.0) 81 : cluster [DBG] mgrmap e12: y(active, since 2s) 2023-09-23T18:05:16.459 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:14 smithi098 bash[27349]: audit 2023-09-23T18:05:14.189465+0000 mgr.y (mgr.14146) 3 : audit [DBG] from='client.14156 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:05:16.459 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:14 smithi098 bash[27349]: audit 2023-09-23T18:05:14.310686+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:05:16.459 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:14 smithi098 bash[27349]: audit 2023-09-23T18:05:14.316111+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:05:16.742 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$Y5aa42EU0M3rC9hhSQBStOVgw38zV5XNYnyou/k7BbWnsXKw5bYXu", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1695492316, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-09-23T18:05:17.439 INFO:teuthology.orchestra.run.smithi098.stderr:Fetching dashboard port number... 2023-09-23T18:05:20.381 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:17 smithi098 bash[27349]: audit 2023-09-23T18:05:16.521684+0000 mgr.y (mgr.14146) 4 : audit [DBG] from='client.14158 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:05:20.381 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:17 smithi098 bash[27349]: audit 2023-09-23T18:05:16.739709+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:05:20.381 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: 8443 2023-09-23T18:05:20.382 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:18 smithi098 bash[27349]: cluster 2023-09-23T18:05:17.746854+0000 mon.a (mon.0) 85 : cluster [DBG] mgrmap e13: y(active, since 6s) 2023-09-23T18:05:20.382 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:18 smithi098 bash[27349]: audit 2023-09-23T18:05:18.268684+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:05:20.383 INFO:teuthology.orchestra.run.smithi098.stderr:firewalld does not appear to be present 2023-09-23T18:05:20.383 INFO:teuthology.orchestra.run.smithi098.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2023-09-23T18:05:20.945 INFO:teuthology.orchestra.run.smithi098.stderr:Ceph Dashboard is now available at: 2023-09-23T18:05:20.945 INFO:teuthology.orchestra.run.smithi098.stderr: 2023-09-23T18:05:20.945 INFO:teuthology.orchestra.run.smithi098.stderr: URL: https://smithi098.front.sepia.ceph.com:8443/ 2023-09-23T18:05:20.945 INFO:teuthology.orchestra.run.smithi098.stderr: User: admin 2023-09-23T18:05:20.945 INFO:teuthology.orchestra.run.smithi098.stderr: Password: emfp8yec16 2023-09-23T18:05:20.945 INFO:teuthology.orchestra.run.smithi098.stderr: 2023-09-23T18:05:20.945 INFO:teuthology.orchestra.run.smithi098.stderr:Enabling autotune for osd_memory_target 2023-09-23T18:05:20.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:19 smithi098 bash[27349]: audit 2023-09-23T18:05:18.758384+0000 mon.a (mon.0) 87 : audit [DBG] from='client.? 172.21.15.98:0/905802331' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-09-23T18:05:23.076 INFO:teuthology.orchestra.run.smithi098.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2023-09-23T18:05:23.783 INFO:teuthology.orchestra.run.smithi098.stderr:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-09-23T18:05:23.784 INFO:teuthology.orchestra.run.smithi098.stderr: 2023-09-23T18:05:23.784 INFO:teuthology.orchestra.run.smithi098.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-09-23T18:05:23.784 INFO:teuthology.orchestra.run.smithi098.stderr: 2023-09-23T18:05:23.784 INFO:teuthology.orchestra.run.smithi098.stderr:Or, if you are only running a single cluster on this host: 2023-09-23T18:05:23.785 INFO:teuthology.orchestra.run.smithi098.stderr: 2023-09-23T18:05:23.785 INFO:teuthology.orchestra.run.smithi098.stderr: sudo /home/ubuntu/cephtest/cephadm shell 2023-09-23T18:05:23.785 INFO:teuthology.orchestra.run.smithi098.stderr: 2023-09-23T18:05:23.785 INFO:teuthology.orchestra.run.smithi098.stderr:Please consider enabling telemetry to help improve Ceph: 2023-09-23T18:05:23.786 INFO:teuthology.orchestra.run.smithi098.stderr: 2023-09-23T18:05:23.786 INFO:teuthology.orchestra.run.smithi098.stderr: ceph telemetry on 2023-09-23T18:05:23.786 INFO:teuthology.orchestra.run.smithi098.stderr: 2023-09-23T18:05:23.786 INFO:teuthology.orchestra.run.smithi098.stderr:For more information see: 2023-09-23T18:05:23.786 INFO:teuthology.orchestra.run.smithi098.stderr: 2023-09-23T18:05:23.786 INFO:teuthology.orchestra.run.smithi098.stderr: https://docs.ceph.com/en/pacific/mgr/telemetry/ 2023-09-23T18:05:23.786 INFO:teuthology.orchestra.run.smithi098.stderr: 2023-09-23T18:05:23.786 INFO:teuthology.orchestra.run.smithi098.stderr:Bootstrap complete. 2023-09-23T18:05:23.786 INFO:tasks.cephadm:Fetching config... 2023-09-23T18:05:23.787 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-23T18:05:23.787 DEBUG:teuthology.orchestra.run.smithi098:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-09-23T18:05:24.080 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-09-23T18:05:24.405 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-23T18:05:24.405 DEBUG:teuthology.orchestra.run.smithi098:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-09-23T18:05:24.406 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:24 smithi098 bash[27349]: audit 2023-09-23T18:05:23.070467+0000 mon.a (mon.0) 88 : audit [INF] from='client.? 172.21.15.98:0/3140304028' entity='client.admin' 2023-09-23T18:05:24.413 INFO:tasks.cephadm:Fetching mon keyring... 2023-09-23T18:05:24.992 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-23T18:05:24.992 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-09-23T18:05:25.002 INFO:tasks.cephadm:Fetching pub ssh key... 2023-09-23T18:05:25.238 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-23T18:05:25.238 DEBUG:teuthology.orchestra.run.smithi098:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-09-23T18:05:25.245 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-09-23T18:05:25.566 DEBUG:teuthology.orchestra.run.smithi098:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDiOC0Vauqm/LO3SfV494E8z+XRRKWV+Fw+p26C6RwaVgi9mczlKNwbhDp/NZe4POgHsJwh/iHYsIsjDumRhVrHYzqMD1e9ELyzoio9LjY3Wilh12YEU5FwIXgoV7ICL3vbHOLYzJ0jSDjgfREBYqn7K6E8HLOYmRdSBr2gRbd7hbOyJNDyH+9pQDyNQyINMkKmSz8HyyqxkJfmYrnXYFesHjn+Aj2UyLWt+tQQZ8X/fu26zzoLLDB7J1crL0sN6ExLIbxbti/hBe+ALSYJ9jSIoGuc2PxnTV3uDT3rY4amnZcitkG9XuGSz9OxxqIoBWsB/4uHKNmICp6TL29T5hz4m4xJMtxGR4Cjt7qOhm54oFu4DNBWKGOsx8vvPNvRqg16my9hVW+bwcosswaiQ52N7DHSHSmtraPXrOFAas8v/ZfOioHgpyAYcjpkS0vftfoGzQZ1GRkFQ4i4++W4HIMsqsRqvEwCu+Vf8oJX1AcjqX2ByRkgNkgJvZQYm7V+g0k= ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-09-23T18:05:25.592 INFO:teuthology.orchestra.run.smithi098.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDiOC0Vauqm/LO3SfV494E8z+XRRKWV+Fw+p26C6RwaVgi9mczlKNwbhDp/NZe4POgHsJwh/iHYsIsjDumRhVrHYzqMD1e9ELyzoio9LjY3Wilh12YEU5FwIXgoV7ICL3vbHOLYzJ0jSDjgfREBYqn7K6E8HLOYmRdSBr2gRbd7hbOyJNDyH+9pQDyNQyINMkKmSz8HyyqxkJfmYrnXYFesHjn+Aj2UyLWt+tQQZ8X/fu26zzoLLDB7J1crL0sN6ExLIbxbti/hBe+ALSYJ9jSIoGuc2PxnTV3uDT3rY4amnZcitkG9XuGSz9OxxqIoBWsB/4uHKNmICp6TL29T5hz4m4xJMtxGR4Cjt7qOhm54oFu4DNBWKGOsx8vvPNvRqg16my9hVW+bwcosswaiQ52N7DHSHSmtraPXrOFAas8v/ZfOioHgpyAYcjpkS0vftfoGzQZ1GRkFQ4i4++W4HIMsqsRqvEwCu+Vf8oJX1AcjqX2ByRkgNkgJvZQYm7V+g0k= ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:05:25.787 DEBUG:teuthology.orchestra.run.smithi173:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDiOC0Vauqm/LO3SfV494E8z+XRRKWV+Fw+p26C6RwaVgi9mczlKNwbhDp/NZe4POgHsJwh/iHYsIsjDumRhVrHYzqMD1e9ELyzoio9LjY3Wilh12YEU5FwIXgoV7ICL3vbHOLYzJ0jSDjgfREBYqn7K6E8HLOYmRdSBr2gRbd7hbOyJNDyH+9pQDyNQyINMkKmSz8HyyqxkJfmYrnXYFesHjn+Aj2UyLWt+tQQZ8X/fu26zzoLLDB7J1crL0sN6ExLIbxbti/hBe+ALSYJ9jSIoGuc2PxnTV3uDT3rY4amnZcitkG9XuGSz9OxxqIoBWsB/4uHKNmICp6TL29T5hz4m4xJMtxGR4Cjt7qOhm54oFu4DNBWKGOsx8vvPNvRqg16my9hVW+bwcosswaiQ52N7DHSHSmtraPXrOFAas8v/ZfOioHgpyAYcjpkS0vftfoGzQZ1GRkFQ4i4++W4HIMsqsRqvEwCu+Vf8oJX1AcjqX2ByRkgNkgJvZQYm7V+g0k= ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-09-23T18:05:25.815 INFO:teuthology.orchestra.run.smithi173.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDiOC0Vauqm/LO3SfV494E8z+XRRKWV+Fw+p26C6RwaVgi9mczlKNwbhDp/NZe4POgHsJwh/iHYsIsjDumRhVrHYzqMD1e9ELyzoio9LjY3Wilh12YEU5FwIXgoV7ICL3vbHOLYzJ0jSDjgfREBYqn7K6E8HLOYmRdSBr2gRbd7hbOyJNDyH+9pQDyNQyINMkKmSz8HyyqxkJfmYrnXYFesHjn+Aj2UyLWt+tQQZ8X/fu26zzoLLDB7J1crL0sN6ExLIbxbti/hBe+ALSYJ9jSIoGuc2PxnTV3uDT3rY4amnZcitkG9XuGSz9OxxqIoBWsB/4uHKNmICp6TL29T5hz4m4xJMtxGR4Cjt7qOhm54oFu4DNBWKGOsx8vvPNvRqg16my9hVW+bwcosswaiQ52N7DHSHSmtraPXrOFAas8v/ZfOioHgpyAYcjpkS0vftfoGzQZ1GRkFQ4i4++W4HIMsqsRqvEwCu+Vf8oJX1AcjqX2ByRkgNkgJvZQYm7V+g0k= ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:05:26.240 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-09-23T18:05:31.564 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-09-23T18:05:31.700 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-09-23T18:05:31.701 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:31 smithi098 bash[27349]: audit 2023-09-23T18:05:30.391969+0000 mon.a (mon.0) 89 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:05:31.701 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:31 smithi098 bash[27349]: audit 2023-09-23T18:05:30.686566+0000 mon.a (mon.0) 90 : audit [INF] from='client.? 172.21.15.98:0/2424068170' entity='client.admin' 2023-09-23T18:05:31.701 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:31 smithi098 bash[27349]: audit 2023-09-23T18:05:30.745575+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi098", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:05:31.703 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:31 smithi098 bash[27349]: audit 2023-09-23T18:05:30.751630+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:05:31.704 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:31 smithi098 bash[27349]: audit 2023-09-23T18:05:30.754128+0000 mon.a (mon.0) 93 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:05:31.704 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:31 smithi098 bash[27349]: audit 2023-09-23T18:05:30.758767+0000 mon.a (mon.0) 94 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:05:31.704 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:31 smithi098 bash[27349]: cluster 2023-09-23T18:05:31.074447+0000 mgr.y (mgr.14146) 5 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:05:34.368 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:33 smithi098 bash[27349]: cluster 2023-09-23T18:05:33.074892+0000 mgr.y (mgr.14146) 6 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:05:35.536 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi173 2023-09-23T18:05:35.713 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-09-23T18:05:35.713 DEBUG:teuthology.orchestra.run.smithi173:> dd of=/etc/ceph/ceph.conf 2023-09-23T18:05:35.721 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-09-23T18:05:35.899 DEBUG:teuthology.orchestra.run.smithi173:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:05:35.907 INFO:tasks.cephadm:Adding host smithi173 to orchestrator... 2023-09-23T18:05:36.252 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph orch host add smithi173 2023-09-23T18:05:36.253 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:35 smithi098 bash[27349]: audit 2023-09-23T18:05:34.774255+0000 mon.a (mon.0) 95 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:05:36.254 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:35 smithi098 bash[27349]: audit 2023-09-23T18:05:34.783611+0000 mon.a (mon.0) 96 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:05:36.254 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:35 smithi098 bash[27349]: audit 2023-09-23T18:05:34.786331+0000 mon.a (mon.0) 97 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:05:36.254 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:35 smithi098 bash[27349]: audit 2023-09-23T18:05:34.787440+0000 mon.a (mon.0) 98 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:05:36.254 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:35 smithi098 bash[27349]: cephadm 2023-09-23T18:05:34.789152+0000 mgr.y (mgr.14146) 8 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.conf 2023-09-23T18:05:36.254 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:35 smithi098 bash[27349]: cephadm 2023-09-23T18:05:34.807240+0000 mgr.y (mgr.14146) 9 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:05:36.255 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:35 smithi098 bash[27349]: audit 2023-09-23T18:05:34.845390+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:05:36.255 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:35 smithi098 bash[27349]: audit 2023-09-23T18:05:34.850131+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:05:36.255 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:35 smithi098 bash[27349]: cluster 2023-09-23T18:05:35.075298+0000 mgr.y (mgr.14146) 10 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:05:38.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:38 smithi098 bash[27349]: cluster 2023-09-23T18:05:37.075729+0000 mgr.y (mgr.14146) 11 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:05:39.940 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:39 smithi098 bash[27618]: Warning: Permanently added 'smithi173,172.21.15.173' (ECDSA) to the list of known hosts. 2023-09-23T18:05:40.258 INFO:teuthology.orchestra.run.smithi098.stdout:Added host 'smithi173' with addr '172.21.15.173' 2023-09-23T18:05:40.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:40 smithi098 bash[27349]: cluster 2023-09-23T18:05:39.076204+0000 mgr.y (mgr.14146) 12 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:05:40.466 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:05:40 smithi098 bash[27618]: Warning: Permanently added '172.21.15.173' (ECDSA) to the list of known hosts. 2023-09-23T18:05:41.033 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph orch host ls --format=json 2023-09-23T18:05:41.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:41 smithi098 bash[27349]: audit 2023-09-23T18:05:39.415633+0000 mgr.y (mgr.14146) 13 : audit [DBG] from='client.14170 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi173", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:05:41.489 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:41 smithi098 bash[27349]: audit 2023-09-23T18:05:40.255454+0000 mon.a (mon.0) 101 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:05:41.489 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:41 smithi098 bash[27349]: audit 2023-09-23T18:05:40.287602+0000 mon.a (mon.0) 102 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:05:41.490 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:41 smithi098 bash[27349]: audit 2023-09-23T18:05:40.848352+0000 mon.a (mon.0) 103 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:05:42.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:42 smithi098 bash[27349]: cephadm 2023-09-23T18:05:40.256236+0000 mgr.y (mgr.14146) 14 : cephadm [INF] Added host smithi173 2023-09-23T18:05:42.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:42 smithi098 bash[27349]: cluster 2023-09-23T18:05:41.076766+0000 mgr.y (mgr.14146) 15 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:05:42.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:42 smithi098 bash[27349]: audit 2023-09-23T18:05:42.111979+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:05:44.395 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:05:44.396 INFO:teuthology.orchestra.run.smithi098.stdout:[{"addr": "172.21.15.98", "hostname": "smithi098", "labels": [], "status": ""}, {"addr": "172.21.15.173", "hostname": "smithi173", "labels": [], "status": ""}] 2023-09-23T18:05:44.427 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:43 smithi098 bash[27349]: cluster 2023-09-23T18:05:43.077227+0000 mgr.y (mgr.14146) 16 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:05:45.143 INFO:tasks.cephadm:Setting crush tunables to default 2023-09-23T18:05:45.143 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd crush tunables default 2023-09-23T18:05:46.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:46 smithi098 bash[27349]: audit 2023-09-23T18:05:44.392882+0000 mgr.y (mgr.14146) 17 : audit [DBG] from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:05:46.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:46 smithi098 bash[27349]: cluster 2023-09-23T18:05:45.077697+0000 mgr.y (mgr.14146) 18 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:05:48.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:48 smithi098 bash[27349]: cluster 2023-09-23T18:05:47.078079+0000 mgr.y (mgr.14146) 19 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:05:48.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:48 smithi098 bash[27349]: audit 2023-09-23T18:05:47.373536+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:05:48.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:48 smithi098 bash[27349]: audit 2023-09-23T18:05:47.682454+0000 mon.a (mon.0) 106 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:05:48.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:48 smithi098 bash[27349]: audit 2023-09-23T18:05:47.683866+0000 mon.a (mon.0) 107 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi173", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:05:48.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:48 smithi098 bash[27349]: audit 2023-09-23T18:05:47.685528+0000 mon.a (mon.0) 108 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:05:48.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:48 smithi098 bash[27349]: audit 2023-09-23T18:05:47.686551+0000 mon.a (mon.0) 109 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:05:48.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:48 smithi098 bash[27349]: audit 2023-09-23T18:05:47.738207+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:05:48.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:48 smithi098 bash[27349]: audit 2023-09-23T18:05:47.743555+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:05:49.151 INFO:teuthology.orchestra.run.smithi098.stderr:adjusted tunables profile to default 2023-09-23T18:05:49.426 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:49 smithi098 bash[27349]: cephadm 2023-09-23T18:05:47.688163+0000 mgr.y (mgr.14146) 20 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-09-23T18:05:49.426 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:49 smithi098 bash[27349]: cephadm 2023-09-23T18:05:47.706631+0000 mgr.y (mgr.14146) 21 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:05:49.426 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:49 smithi098 bash[27349]: audit 2023-09-23T18:05:48.413466+0000 mon.a (mon.0) 112 : audit [INF] from='client.? 172.21.15.98:0/3635012488' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-09-23T18:05:49.879 INFO:tasks.cephadm:Adding mon.a on smithi098 2023-09-23T18:05:49.879 INFO:tasks.cephadm:Adding mon.c on smithi098 2023-09-23T18:05:49.879 INFO:tasks.cephadm:Adding mon.b on smithi173 2023-09-23T18:05:49.879 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph orch apply mon '3;smithi098:172.21.15.98=a;smithi098:[v2:172.21.15.98:3301,v1:172.21.15.98:6790]=c;smithi173:172.21.15.173=b' 2023-09-23T18:05:50.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:50 smithi098 bash[27349]: cluster 2023-09-23T18:05:49.078485+0000 mgr.y (mgr.14146) 22 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:05:50.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:50 smithi098 bash[27349]: audit 2023-09-23T18:05:49.144092+0000 mon.a (mon.0) 113 : audit [INF] from='client.? 172.21.15.98:0/3635012488' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-23T18:05:50.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:50 smithi098 bash[27349]: cluster 2023-09-23T18:05:49.144191+0000 mon.a (mon.0) 114 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-09-23T18:05:52.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:52 smithi098 bash[27349]: cluster 2023-09-23T18:05:51.078855+0000 mgr.y (mgr.14146) 23 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:05:52.816 INFO:teuthology.orchestra.run.smithi173.stdout:Scheduled mon update... 2023-09-23T18:05:53.778 DEBUG:teuthology.orchestra.run.smithi098:mon.c> sudo journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mon.c.service 2023-09-23T18:05:53.780 DEBUG:teuthology.orchestra.run.smithi173:mon.b> sudo journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mon.b.service 2023-09-23T18:05:53.784 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T18:05:53.785 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T18:05:54.042 INFO:journalctl@ceph.mon.c.smithi098.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:05:54.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:53 smithi098 bash[27349]: audit 2023-09-23T18:05:52.803223+0000 mgr.y (mgr.14146) 24 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi098:172.21.15.98=a;smithi098:[v2:172.21.15.98:3301,v1:172.21.15.98:6790]=c;smithi173:172.21.15.173=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:05:54.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:53 smithi098 bash[27349]: cephadm 2023-09-23T18:05:52.809309+0000 mgr.y (mgr.14146) 25 : cephadm [INF] Saving service mon spec with placement smithi098:172.21.15.98=a;smithi098:[v2:172.21.15.98:3301,v1:172.21.15.98:6790]=c;smithi173:172.21.15.173=b;count:3 2023-09-23T18:05:54.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:53 smithi098 bash[27349]: audit 2023-09-23T18:05:52.814249+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:05:54.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:53 smithi098 bash[27349]: audit 2023-09-23T18:05:52.821519+0000 mon.a (mon.0) 116 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:05:54.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:53 smithi098 bash[27349]: audit 2023-09-23T18:05:52.824675+0000 mon.a (mon.0) 117 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:05:54.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:53 smithi098 bash[27349]: audit 2023-09-23T18:05:52.826235+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:05:54.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:53 smithi098 bash[27349]: audit 2023-09-23T18:05:52.835613+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:05:54.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:53 smithi098 bash[27349]: audit 2023-09-23T18:05:52.838355+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:05:54.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:53 smithi098 bash[27349]: audit 2023-09-23T18:05:52.839687+0000 mon.a (mon.0) 121 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:05:54.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:53 smithi098 bash[27349]: cephadm 2023-09-23T18:05:52.840854+0000 mgr.y (mgr.14146) 26 : cephadm [INF] Deploying daemon mon.b on smithi173 2023-09-23T18:05:54.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:53 smithi098 bash[27349]: cluster 2023-09-23T18:05:53.079278+0000 mgr.y (mgr.14146) 27 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:05:54.864 INFO:journalctl@ceph.mon.b.smithi173.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:05:56.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:56 smithi098 bash[27349]: cluster 2023-09-23T18:05:55.079706+0000 mgr.y (mgr.14146) 28 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:05:58.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:58 smithi098 bash[27349]: cluster 2023-09-23T18:05:57.080121+0000 mgr.y (mgr.14146) 29 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:05:58.814 INFO:teuthology.orchestra.run.smithi173.stdout: 2023-09-23T18:05:58.814 INFO:teuthology.orchestra.run.smithi173.stdout:{"epoch":1,"fsid":"756c96d4-5a3b-11ee-8db4-212e2dc638e7","modified":"2023-09-23T18:04:07.137241Z","created":"2023-09-23T18:04:07.137241Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:3300","nonce":0},{"type":"v1","addr":"172.21.15.98:6789","nonce":0}]},"addr":"172.21.15.98:6789/0","public_addr":"172.21.15.98:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-09-23T18:05:58.818 INFO:teuthology.orchestra.run.smithi173.stderr:dumped monmap epoch 1 2023-09-23T18:05:59.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:05:59 smithi098 bash[27349]: audit 2023-09-23T18:05:58.812178+0000 mon.a (mon.0) 122 : audit [DBG] from='client.? 172.21.15.173:0/1623577909' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T18:06:00.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:00 smithi098 bash[27349]: cluster 2023-09-23T18:05:59.080560+0000 mgr.y (mgr.14146) 30 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:00.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:00 smithi098 bash[27349]: audit 2023-09-23T18:05:59.378882+0000 mon.a (mon.0) 123 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:00.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:00 smithi098 bash[27349]: audit 2023-09-23T18:05:59.380601+0000 mon.a (mon.0) 124 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:06:00.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:00 smithi098 bash[27349]: audit 2023-09-23T18:05:59.382244+0000 mon.a (mon.0) 125 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:00.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:00 smithi173 bash[27694]: debug 2023-09-23T18:06:00.404+0000 7f99183f5700 1 mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-09-23T18:06:00.916 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T18:06:00.916 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T18:06:04.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:04 smithi098 bash[33599]: debug 2023-09-23T18:06:04.479+0000 7f0ef7813700 1 mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-09-23T18:06:05.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:05 smithi098 bash[27349]: audit 2023-09-23T18:06:00.415195+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:06:05.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:05 smithi098 bash[27349]: audit 2023-09-23T18:06:00.415409+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:06:05.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:05 smithi098 bash[27349]: cluster 2023-09-23T18:06:00.415555+0000 mon.a (mon.0) 129 : cluster [INF] mon.a calling monitor election 2023-09-23T18:06:05.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:05 smithi098 bash[27349]: cluster 2023-09-23T18:06:01.081064+0000 mgr.y (mgr.14146) 32 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:05.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:05 smithi098 bash[27349]: audit 2023-09-23T18:06:01.411069+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:06:05.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:05 smithi098 bash[27349]: audit 2023-09-23T18:06:02.411208+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:06:05.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:05 smithi098 bash[27349]: cluster 2023-09-23T18:06:02.413749+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-09-23T18:06:05.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:05 smithi098 bash[27349]: cluster 2023-09-23T18:06:03.081458+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:05.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:05 smithi098 bash[27349]: audit 2023-09-23T18:06:03.410855+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:06:05.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:05 smithi098 bash[27349]: audit 2023-09-23T18:06:04.412092+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:06:05.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:05 smithi098 bash[27349]: audit 2023-09-23T18:06:04.486999+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:05.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:05 smithi098 bash[27349]: cluster 2023-09-23T18:06:05.081886+0000 mgr.y (mgr.14146) 34 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:05.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:05 smithi098 bash[27349]: audit 2023-09-23T18:06:05.411714+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:06:05.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:05 smithi098 bash[27349]: cluster 2023-09-23T18:06:05.457682+0000 mon.a (mon.0) 136 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T18:06:05.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:05 smithi098 bash[27349]: cluster 2023-09-23T18:06:05.464276+0000 mon.a (mon.0) 137 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0],b=[v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0]} removed_ranks: {} 2023-09-23T18:06:05.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:05 smithi098 bash[27349]: cluster 2023-09-23T18:06:05.483162+0000 mon.a (mon.0) 138 : cluster [DBG] fsmap 2023-09-23T18:06:05.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:05 smithi098 bash[27349]: cluster 2023-09-23T18:06:05.483220+0000 mon.a (mon.0) 139 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-09-23T18:06:05.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:05 smithi098 bash[27349]: cluster 2023-09-23T18:06:05.483551+0000 mon.a (mon.0) 140 : cluster [DBG] mgrmap e13: y(active, since 54s) 2023-09-23T18:06:05.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:05 smithi098 bash[27349]: cluster 2023-09-23T18:06:05.489474+0000 mon.a (mon.0) 141 : cluster [INF] overall HEALTH_OK 2023-09-23T18:06:05.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:05 smithi098 bash[27349]: audit 2023-09-23T18:06:05.490087+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:05.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:05 smithi098 bash[27349]: audit 2023-09-23T18:06:05.492976+0000 mon.a (mon.0) 143 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:05.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:05 smithi098 bash[27349]: audit 2023-09-23T18:06:05.494833+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:06:06.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:05 smithi173 bash[27694]: audit 2023-09-23T18:06:00.415195+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:06:06.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:05 smithi173 bash[27694]: audit 2023-09-23T18:06:00.415409+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:06:06.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:05 smithi173 bash[27694]: cluster 2023-09-23T18:06:00.415555+0000 mon.a (mon.0) 129 : cluster [INF] mon.a calling monitor election 2023-09-23T18:06:06.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:05 smithi173 bash[27694]: cluster 2023-09-23T18:06:01.081064+0000 mgr.y (mgr.14146) 32 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:06.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:05 smithi173 bash[27694]: audit 2023-09-23T18:06:01.411069+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:06:06.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:05 smithi173 bash[27694]: audit 2023-09-23T18:06:02.411208+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:06:06.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:05 smithi173 bash[27694]: cluster 2023-09-23T18:06:02.413749+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-09-23T18:06:06.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:05 smithi173 bash[27694]: cluster 2023-09-23T18:06:03.081458+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:06.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:05 smithi173 bash[27694]: audit 2023-09-23T18:06:03.410855+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:06:06.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:05 smithi173 bash[27694]: audit 2023-09-23T18:06:04.412092+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:06:06.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:05 smithi173 bash[27694]: audit 2023-09-23T18:06:04.486999+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:06.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:05 smithi173 bash[27694]: cluster 2023-09-23T18:06:05.081886+0000 mgr.y (mgr.14146) 34 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:06.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:05 smithi173 bash[27694]: audit 2023-09-23T18:06:05.411714+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:06:06.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:05 smithi173 bash[27694]: cluster 2023-09-23T18:06:05.457682+0000 mon.a (mon.0) 136 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T18:06:06.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:05 smithi173 bash[27694]: cluster 2023-09-23T18:06:05.464276+0000 mon.a (mon.0) 137 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0],b=[v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0]} removed_ranks: {} 2023-09-23T18:06:06.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:05 smithi173 bash[27694]: cluster 2023-09-23T18:06:05.483162+0000 mon.a (mon.0) 138 : cluster [DBG] fsmap 2023-09-23T18:06:06.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:05 smithi173 bash[27694]: cluster 2023-09-23T18:06:05.483220+0000 mon.a (mon.0) 139 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-09-23T18:06:06.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:05 smithi173 bash[27694]: cluster 2023-09-23T18:06:05.483551+0000 mon.a (mon.0) 140 : cluster [DBG] mgrmap e13: y(active, since 54s) 2023-09-23T18:06:06.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:05 smithi173 bash[27694]: cluster 2023-09-23T18:06:05.489474+0000 mon.a (mon.0) 141 : cluster [INF] overall HEALTH_OK 2023-09-23T18:06:06.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:05 smithi173 bash[27694]: audit 2023-09-23T18:06:05.490087+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:06.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:05 smithi173 bash[27694]: audit 2023-09-23T18:06:05.492976+0000 mon.a (mon.0) 143 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:06.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:05 smithi173 bash[27694]: audit 2023-09-23T18:06:05.494833+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:06:11.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: audit 2023-09-23T18:06:00.415195+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:06:11.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: audit 2023-09-23T18:06:00.415409+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:06:11.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: cluster 2023-09-23T18:06:00.415555+0000 mon.a (mon.0) 129 : cluster [INF] mon.a calling monitor election 2023-09-23T18:06:11.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: cluster 2023-09-23T18:06:01.081064+0000 mgr.y (mgr.14146) 32 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:11.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: audit 2023-09-23T18:06:01.411069+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:06:11.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: audit 2023-09-23T18:06:02.411208+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:06:11.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: cluster 2023-09-23T18:06:02.413749+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-09-23T18:06:11.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: cluster 2023-09-23T18:06:03.081458+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:11.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: audit 2023-09-23T18:06:03.410855+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:06:11.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: audit 2023-09-23T18:06:04.412092+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:06:11.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: audit 2023-09-23T18:06:04.486999+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:11.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: cluster 2023-09-23T18:06:05.081886+0000 mgr.y (mgr.14146) 34 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:11.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: audit 2023-09-23T18:06:05.411714+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:06:11.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: cluster 2023-09-23T18:06:05.457682+0000 mon.a (mon.0) 136 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T18:06:11.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: cluster 2023-09-23T18:06:05.464276+0000 mon.a (mon.0) 137 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0],b=[v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0]} removed_ranks: {} 2023-09-23T18:06:11.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: cluster 2023-09-23T18:06:05.483162+0000 mon.a (mon.0) 138 : cluster [DBG] fsmap 2023-09-23T18:06:11.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: cluster 2023-09-23T18:06:05.483220+0000 mon.a (mon.0) 139 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-09-23T18:06:11.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: cluster 2023-09-23T18:06:05.483551+0000 mon.a (mon.0) 140 : cluster [DBG] mgrmap e13: y(active, since 54s) 2023-09-23T18:06:11.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: cluster 2023-09-23T18:06:05.489474+0000 mon.a (mon.0) 141 : cluster [INF] overall HEALTH_OK 2023-09-23T18:06:11.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: audit 2023-09-23T18:06:05.490087+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:11.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: audit 2023-09-23T18:06:05.492976+0000 mon.a (mon.0) 143 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:11.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[27349]: audit 2023-09-23T18:06:06.493222+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:06:11.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[27349]: cluster 2023-09-23T18:06:06.493559+0000 mon.a (mon.0) 149 : cluster [INF] mon.a calling monitor election 2023-09-23T18:06:11.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[27349]: cluster 2023-09-23T18:06:06.499191+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-09-23T18:06:11.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[27349]: audit 2023-09-23T18:06:06.511794+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:06:11.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[27349]: audit 2023-09-23T18:06:06.512051+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:11.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[27349]: cluster 2023-09-23T18:06:07.082331+0000 mgr.y (mgr.14146) 35 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:11.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[27349]: audit 2023-09-23T18:06:07.487698+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:11.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[27349]: audit 2023-09-23T18:06:08.488093+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:11.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[27349]: cluster 2023-09-23T18:06:08.490352+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-09-23T18:06:11.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[27349]: cluster 2023-09-23T18:06:09.082761+0000 mgr.y (mgr.14146) 36 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:11.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[27349]: audit 2023-09-23T18:06:09.487973+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:11.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[27349]: audit 2023-09-23T18:06:10.488054+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:11.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[27349]: cluster 2023-09-23T18:06:11.083188+0000 mgr.y (mgr.14146) 37 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:11.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[27349]: audit 2023-09-23T18:06:11.488440+0000 mon.a (mon.0) 156 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:11.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[27349]: cluster 2023-09-23T18:06:11.533568+0000 mon.a (mon.0) 157 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T18:06:11.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[27349]: cluster 2023-09-23T18:06:11.539675+0000 mon.a (mon.0) 158 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0],b=[v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0],c=[v2:172.21.15.98:3301/0,v1:172.21.15.98:6790/0]} removed_ranks: {} 2023-09-23T18:06:11.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[27349]: cluster 2023-09-23T18:06:11.540272+0000 mon.a (mon.0) 159 : cluster [DBG] fsmap 2023-09-23T18:06:11.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[27349]: cluster 2023-09-23T18:06:11.540330+0000 mon.a (mon.0) 160 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-09-23T18:06:11.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[27349]: cluster 2023-09-23T18:06:11.540593+0000 mon.a (mon.0) 161 : cluster [DBG] mgrmap e13: y(active, since 60s) 2023-09-23T18:06:11.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[27349]: cluster 2023-09-23T18:06:11.563531+0000 mon.a (mon.0) 162 : cluster [INF] overall HEALTH_OK 2023-09-23T18:06:11.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[27349]: audit 2023-09-23T18:06:11.567652+0000 mon.a (mon.0) 163 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:11.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[27349]: audit 2023-09-23T18:06:11.570900+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:11.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: audit 2023-09-23T18:06:05.494833+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:06:11.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: audit 2023-09-23T18:06:06.493222+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:06:11.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: cluster 2023-09-23T18:06:06.493559+0000 mon.a (mon.0) 149 : cluster [INF] mon.a calling monitor election 2023-09-23T18:06:11.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: cluster 2023-09-23T18:06:06.499191+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-09-23T18:06:11.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: audit 2023-09-23T18:06:06.511794+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:06:11.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: audit 2023-09-23T18:06:06.512051+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:11.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: cluster 2023-09-23T18:06:07.082331+0000 mgr.y (mgr.14146) 35 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:11.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: audit 2023-09-23T18:06:07.487698+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:11.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: audit 2023-09-23T18:06:08.488093+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:11.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: cluster 2023-09-23T18:06:08.490352+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-09-23T18:06:11.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: cluster 2023-09-23T18:06:09.082761+0000 mgr.y (mgr.14146) 36 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:11.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: audit 2023-09-23T18:06:09.487973+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:11.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: audit 2023-09-23T18:06:10.488054+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:11.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: cluster 2023-09-23T18:06:11.083188+0000 mgr.y (mgr.14146) 37 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:11.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: audit 2023-09-23T18:06:11.488440+0000 mon.a (mon.0) 156 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:11.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: cluster 2023-09-23T18:06:11.533568+0000 mon.a (mon.0) 157 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T18:06:11.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: cluster 2023-09-23T18:06:11.539675+0000 mon.a (mon.0) 158 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0],b=[v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0],c=[v2:172.21.15.98:3301/0,v1:172.21.15.98:6790/0]} removed_ranks: {} 2023-09-23T18:06:11.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: cluster 2023-09-23T18:06:11.540272+0000 mon.a (mon.0) 159 : cluster [DBG] fsmap 2023-09-23T18:06:11.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: cluster 2023-09-23T18:06:11.540330+0000 mon.a (mon.0) 160 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-09-23T18:06:11.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: cluster 2023-09-23T18:06:11.540593+0000 mon.a (mon.0) 161 : cluster [DBG] mgrmap e13: y(active, since 60s) 2023-09-23T18:06:11.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: cluster 2023-09-23T18:06:11.563531+0000 mon.a (mon.0) 162 : cluster [INF] overall HEALTH_OK 2023-09-23T18:06:11.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: audit 2023-09-23T18:06:11.567652+0000 mon.a (mon.0) 163 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:11.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: audit 2023-09-23T18:06:11.570900+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:11.948 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: audit 2023-09-23T18:06:11.572026+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:11.948 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[33599]: audit 2023-09-23T18:06:11.573206+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:06:11.948 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[27349]: audit 2023-09-23T18:06:11.572026+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:11.948 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:11 smithi098 bash[27349]: audit 2023-09-23T18:06:11.573206+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:06:12.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:11 smithi173 bash[27694]: audit 2023-09-23T18:06:06.493222+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:06:12.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:11 smithi173 bash[27694]: cluster 2023-09-23T18:06:06.493559+0000 mon.a (mon.0) 149 : cluster [INF] mon.a calling monitor election 2023-09-23T18:06:12.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:11 smithi173 bash[27694]: cluster 2023-09-23T18:06:06.499191+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-09-23T18:06:12.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:11 smithi173 bash[27694]: audit 2023-09-23T18:06:06.511794+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:06:12.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:11 smithi173 bash[27694]: audit 2023-09-23T18:06:06.512051+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:12.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:11 smithi173 bash[27694]: cluster 2023-09-23T18:06:07.082331+0000 mgr.y (mgr.14146) 35 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:12.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:11 smithi173 bash[27694]: audit 2023-09-23T18:06:07.487698+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:12.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:11 smithi173 bash[27694]: audit 2023-09-23T18:06:08.488093+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:12.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:11 smithi173 bash[27694]: cluster 2023-09-23T18:06:08.490352+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-09-23T18:06:12.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:11 smithi173 bash[27694]: cluster 2023-09-23T18:06:09.082761+0000 mgr.y (mgr.14146) 36 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:12.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:11 smithi173 bash[27694]: audit 2023-09-23T18:06:09.487973+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:12.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:11 smithi173 bash[27694]: audit 2023-09-23T18:06:10.488054+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:12.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:11 smithi173 bash[27694]: cluster 2023-09-23T18:06:11.083188+0000 mgr.y (mgr.14146) 37 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:12.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:11 smithi173 bash[27694]: audit 2023-09-23T18:06:11.488440+0000 mon.a (mon.0) 156 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:12.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:11 smithi173 bash[27694]: cluster 2023-09-23T18:06:11.533568+0000 mon.a (mon.0) 157 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T18:06:12.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:11 smithi173 bash[27694]: cluster 2023-09-23T18:06:11.539675+0000 mon.a (mon.0) 158 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0],b=[v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0],c=[v2:172.21.15.98:3301/0,v1:172.21.15.98:6790/0]} removed_ranks: {} 2023-09-23T18:06:12.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:11 smithi173 bash[27694]: cluster 2023-09-23T18:06:11.540272+0000 mon.a (mon.0) 159 : cluster [DBG] fsmap 2023-09-23T18:06:12.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:11 smithi173 bash[27694]: cluster 2023-09-23T18:06:11.540330+0000 mon.a (mon.0) 160 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-09-23T18:06:12.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:11 smithi173 bash[27694]: cluster 2023-09-23T18:06:11.540593+0000 mon.a (mon.0) 161 : cluster [DBG] mgrmap e13: y(active, since 60s) 2023-09-23T18:06:12.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:11 smithi173 bash[27694]: cluster 2023-09-23T18:06:11.563531+0000 mon.a (mon.0) 162 : cluster [INF] overall HEALTH_OK 2023-09-23T18:06:12.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:11 smithi173 bash[27694]: audit 2023-09-23T18:06:11.567652+0000 mon.a (mon.0) 163 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:12.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:11 smithi173 bash[27694]: audit 2023-09-23T18:06:11.570900+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:12.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:11 smithi173 bash[27694]: audit 2023-09-23T18:06:11.572026+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:12.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:11 smithi173 bash[27694]: audit 2023-09-23T18:06:11.573206+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:06:12.869 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:12 smithi098 bash[33599]: cephadm 2023-09-23T18:06:11.574743+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.conf 2023-09-23T18:06:12.869 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:12 smithi098 bash[33599]: cephadm 2023-09-23T18:06:11.575221+0000 mgr.y (mgr.14146) 39 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-09-23T18:06:12.869 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:12 smithi098 bash[33599]: audit 2023-09-23T18:06:11.608330+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:12.870 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:12 smithi098 bash[33599]: audit 2023-09-23T18:06:11.746956+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:12.870 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:12 smithi098 bash[33599]: audit 2023-09-23T18:06:11.751997+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:12.870 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:12 smithi098 bash[33599]: cephadm 2023-09-23T18:06:11.754088+0000 mgr.y (mgr.14146) 40 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-09-23T18:06:12.870 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:12 smithi098 bash[33599]: audit 2023-09-23T18:06:11.754501+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:06:12.870 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:12 smithi098 bash[33599]: audit 2023-09-23T18:06:11.755973+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:06:12.870 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:12 smithi098 bash[33599]: audit 2023-09-23T18:06:11.757344+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:12.870 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:12 smithi098 bash[33599]: cephadm 2023-09-23T18:06:11.758493+0000 mgr.y (mgr.14146) 41 : cephadm [INF] Reconfiguring daemon mon.a on smithi098 2023-09-23T18:06:12.870 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:12 smithi098 bash[33599]: audit 2023-09-23T18:06:12.488523+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:12.870 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:12 smithi098 bash[27349]: cephadm 2023-09-23T18:06:11.574743+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.conf 2023-09-23T18:06:12.870 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:12 smithi098 bash[27349]: cephadm 2023-09-23T18:06:11.575221+0000 mgr.y (mgr.14146) 39 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-09-23T18:06:12.871 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:12 smithi098 bash[27349]: audit 2023-09-23T18:06:11.608330+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:12.871 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:12 smithi098 bash[27349]: audit 2023-09-23T18:06:11.746956+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:12.871 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:12 smithi098 bash[27349]: audit 2023-09-23T18:06:11.751997+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:12.871 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:12 smithi098 bash[27349]: cephadm 2023-09-23T18:06:11.754088+0000 mgr.y (mgr.14146) 40 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-09-23T18:06:12.871 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:12 smithi098 bash[27349]: audit 2023-09-23T18:06:11.754501+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:06:12.871 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:12 smithi098 bash[27349]: audit 2023-09-23T18:06:11.755973+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:06:12.871 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:12 smithi098 bash[27349]: audit 2023-09-23T18:06:11.757344+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:12.871 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:12 smithi098 bash[27349]: cephadm 2023-09-23T18:06:11.758493+0000 mgr.y (mgr.14146) 41 : cephadm [INF] Reconfiguring daemon mon.a on smithi098 2023-09-23T18:06:12.871 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:12 smithi098 bash[27349]: audit 2023-09-23T18:06:12.488523+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:13.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:12 smithi173 bash[27694]: cephadm 2023-09-23T18:06:11.574743+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.conf 2023-09-23T18:06:13.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:12 smithi173 bash[27694]: cephadm 2023-09-23T18:06:11.575221+0000 mgr.y (mgr.14146) 39 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-09-23T18:06:13.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:12 smithi173 bash[27694]: audit 2023-09-23T18:06:11.608330+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:13.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:12 smithi173 bash[27694]: audit 2023-09-23T18:06:11.746956+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:13.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:12 smithi173 bash[27694]: audit 2023-09-23T18:06:11.751997+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:13.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:12 smithi173 bash[27694]: cephadm 2023-09-23T18:06:11.754088+0000 mgr.y (mgr.14146) 40 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-09-23T18:06:13.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:12 smithi173 bash[27694]: audit 2023-09-23T18:06:11.754501+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:06:13.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:12 smithi173 bash[27694]: audit 2023-09-23T18:06:11.755973+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:06:13.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:12 smithi173 bash[27694]: audit 2023-09-23T18:06:11.757344+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:13.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:12 smithi173 bash[27694]: cephadm 2023-09-23T18:06:11.758493+0000 mgr.y (mgr.14146) 41 : cephadm [INF] Reconfiguring daemon mon.a on smithi098 2023-09-23T18:06:13.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:12 smithi173 bash[27694]: audit 2023-09-23T18:06:12.488523+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:06:13.115 INFO:teuthology.orchestra.run.smithi173.stdout: 2023-09-23T18:06:13.115 INFO:teuthology.orchestra.run.smithi173.stdout:{"epoch":3,"fsid":"756c96d4-5a3b-11ee-8db4-212e2dc638e7","modified":"2023-09-23T18:06:06.488382Z","created":"2023-09-23T18:04:07.137241Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:3300","nonce":0},{"type":"v1","addr":"172.21.15.98:6789","nonce":0}]},"addr":"172.21.15.98:6789/0","public_addr":"172.21.15.98:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:3300","nonce":0},{"type":"v1","addr":"172.21.15.173:6789","nonce":0}]},"addr":"172.21.15.173:6789/0","public_addr":"172.21.15.173:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:3301","nonce":0},{"type":"v1","addr":"172.21.15.98:6790","nonce":0}]},"addr":"172.21.15.98:6790/0","public_addr":"172.21.15.98:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2023-09-23T18:06:13.118 INFO:teuthology.orchestra.run.smithi173.stderr:dumped monmap epoch 3 2023-09-23T18:06:13.691 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:06:13 smithi098 bash[27618]: debug 2023-09-23T18:06:13.483+0000 7fa077840700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-09-23T18:06:13.879 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-09-23T18:06:13.879 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph config generate-minimal-conf 2023-09-23T18:06:14.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:14 smithi098 bash[27349]: cluster 2023-09-23T18:06:13.083628+0000 mgr.y (mgr.14146) 42 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:14.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:14 smithi098 bash[27349]: audit 2023-09-23T18:06:13.112501+0000 mon.a (mon.0) 174 : audit [DBG] from='client.? 172.21.15.173:0/3957125663' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T18:06:14.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:14 smithi098 bash[27349]: audit 2023-09-23T18:06:13.829774+0000 mon.a (mon.0) 175 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:14.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:14 smithi098 bash[27349]: audit 2023-09-23T18:06:13.831163+0000 mon.a (mon.0) 176 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:06:14.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:14 smithi098 bash[27349]: audit 2023-09-23T18:06:13.832285+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:06:14.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:14 smithi098 bash[27349]: audit 2023-09-23T18:06:13.833211+0000 mon.a (mon.0) 178 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:14.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:14 smithi098 bash[33599]: cluster 2023-09-23T18:06:13.083628+0000 mgr.y (mgr.14146) 42 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:14.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:14 smithi098 bash[33599]: audit 2023-09-23T18:06:13.112501+0000 mon.a (mon.0) 174 : audit [DBG] from='client.? 172.21.15.173:0/3957125663' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T18:06:14.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:14 smithi098 bash[33599]: audit 2023-09-23T18:06:13.829774+0000 mon.a (mon.0) 175 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:14.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:14 smithi098 bash[33599]: audit 2023-09-23T18:06:13.831163+0000 mon.a (mon.0) 176 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:06:14.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:14 smithi098 bash[33599]: audit 2023-09-23T18:06:13.832285+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:06:14.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:14 smithi098 bash[33599]: audit 2023-09-23T18:06:13.833211+0000 mon.a (mon.0) 178 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:14.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:14 smithi173 bash[27694]: cluster 2023-09-23T18:06:13.083628+0000 mgr.y (mgr.14146) 42 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:14.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:14 smithi173 bash[27694]: audit 2023-09-23T18:06:13.112501+0000 mon.a (mon.0) 174 : audit [DBG] from='client.? 172.21.15.173:0/3957125663' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T18:06:14.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:14 smithi173 bash[27694]: audit 2023-09-23T18:06:13.829774+0000 mon.a (mon.0) 175 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:14.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:14 smithi173 bash[27694]: audit 2023-09-23T18:06:13.831163+0000 mon.a (mon.0) 176 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:06:14.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:14 smithi173 bash[27694]: audit 2023-09-23T18:06:13.832285+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:06:14.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:14 smithi173 bash[27694]: audit 2023-09-23T18:06:13.833211+0000 mon.a (mon.0) 178 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:15.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:15 smithi173 bash[27694]: cephadm 2023-09-23T18:06:13.830796+0000 mgr.y (mgr.14146) 43 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-09-23T18:06:15.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:15 smithi173 bash[27694]: cephadm 2023-09-23T18:06:13.834108+0000 mgr.y (mgr.14146) 44 : cephadm [INF] Reconfiguring daemon mon.c on smithi098 2023-09-23T18:06:15.343 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:15 smithi098 bash[33599]: cephadm 2023-09-23T18:06:13.830796+0000 mgr.y (mgr.14146) 43 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-09-23T18:06:15.343 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:15 smithi098 bash[33599]: cephadm 2023-09-23T18:06:13.834108+0000 mgr.y (mgr.14146) 44 : cephadm [INF] Reconfiguring daemon mon.c on smithi098 2023-09-23T18:06:15.343 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:15 smithi098 bash[27349]: cephadm 2023-09-23T18:06:13.830796+0000 mgr.y (mgr.14146) 43 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-09-23T18:06:15.343 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:15 smithi098 bash[27349]: cephadm 2023-09-23T18:06:13.834108+0000 mgr.y (mgr.14146) 44 : cephadm [INF] Reconfiguring daemon mon.c on smithi098 2023-09-23T18:06:16.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:16 smithi173 bash[27694]: cluster 2023-09-23T18:06:15.084063+0000 mgr.y (mgr.14146) 45 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:16.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:16 smithi098 bash[33599]: cluster 2023-09-23T18:06:15.084063+0000 mgr.y (mgr.14146) 45 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:16.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:16 smithi098 bash[27349]: cluster 2023-09-23T18:06:15.084063+0000 mgr.y (mgr.14146) 45 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:17.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:17 smithi098 bash[27349]: audit 2023-09-23T18:06:16.638751+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:17.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:17 smithi098 bash[27349]: cephadm 2023-09-23T18:06:16.639952+0000 mgr.y (mgr.14146) 46 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-09-23T18:06:17.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:17 smithi098 bash[27349]: audit 2023-09-23T18:06:16.640598+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:06:17.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:17 smithi098 bash[27349]: audit 2023-09-23T18:06:16.641938+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:06:17.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:17 smithi098 bash[27349]: audit 2023-09-23T18:06:16.643231+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:17.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:17 smithi098 bash[27349]: cephadm 2023-09-23T18:06:16.644184+0000 mgr.y (mgr.14146) 47 : cephadm [INF] Reconfiguring daemon mon.b on smithi173 2023-09-23T18:06:17.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:17 smithi098 bash[27349]: cluster 2023-09-23T18:06:17.084445+0000 mgr.y (mgr.14146) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:17.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:17 smithi098 bash[33599]: audit 2023-09-23T18:06:16.638751+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:17.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:17 smithi098 bash[33599]: cephadm 2023-09-23T18:06:16.639952+0000 mgr.y (mgr.14146) 46 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-09-23T18:06:17.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:17 smithi098 bash[33599]: audit 2023-09-23T18:06:16.640598+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:06:17.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:17 smithi098 bash[33599]: audit 2023-09-23T18:06:16.641938+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:06:17.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:17 smithi098 bash[33599]: audit 2023-09-23T18:06:16.643231+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:17.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:17 smithi098 bash[33599]: cephadm 2023-09-23T18:06:16.644184+0000 mgr.y (mgr.14146) 47 : cephadm [INF] Reconfiguring daemon mon.b on smithi173 2023-09-23T18:06:17.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:17 smithi098 bash[33599]: cluster 2023-09-23T18:06:17.084445+0000 mgr.y (mgr.14146) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:17.847 INFO:teuthology.orchestra.run.smithi098.stdout:# minimal ceph.conf for 756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:06:17.847 INFO:teuthology.orchestra.run.smithi098.stdout:[global] 2023-09-23T18:06:17.848 INFO:teuthology.orchestra.run.smithi098.stdout: fsid = 756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:06:17.848 INFO:teuthology.orchestra.run.smithi098.stdout: mon_host = [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] [v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0] [v2:172.21.15.98:3301/0,v1:172.21.15.98:6790/0] 2023-09-23T18:06:18.021 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:17 smithi173 bash[27694]: audit 2023-09-23T18:06:16.638751+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:18.022 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:17 smithi173 bash[27694]: cephadm 2023-09-23T18:06:16.639952+0000 mgr.y (mgr.14146) 46 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-09-23T18:06:18.022 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:17 smithi173 bash[27694]: audit 2023-09-23T18:06:16.640598+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:06:18.022 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:17 smithi173 bash[27694]: audit 2023-09-23T18:06:16.641938+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:06:18.022 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:17 smithi173 bash[27694]: audit 2023-09-23T18:06:16.643231+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:18.022 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:17 smithi173 bash[27694]: cephadm 2023-09-23T18:06:16.644184+0000 mgr.y (mgr.14146) 47 : cephadm [INF] Reconfiguring daemon mon.b on smithi173 2023-09-23T18:06:18.022 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:17 smithi173 bash[27694]: cluster 2023-09-23T18:06:17.084445+0000 mgr.y (mgr.14146) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:18.578 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-09-23T18:06:18.578 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-23T18:06:18.578 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd of=/etc/ceph/ceph.conf 2023-09-23T18:06:18.590 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-23T18:06:18.590 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:06:18.645 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-09-23T18:06:18.645 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd of=/etc/ceph/ceph.conf 2023-09-23T18:06:18.660 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-09-23T18:06:18.661 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:06:18.718 INFO:tasks.cephadm:Adding mgr.y on smithi098 2023-09-23T18:06:18.718 INFO:tasks.cephadm:Adding mgr.x on smithi173 2023-09-23T18:06:18.719 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph orch apply mgr '2;smithi098=y;smithi173=x' 2023-09-23T18:06:18.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:18 smithi098 bash[27349]: audit 2023-09-23T18:06:17.845690+0000 mon.a (mon.0) 183 : audit [DBG] from='client.? 172.21.15.98:0/2765848761' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:18.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:18 smithi098 bash[27349]: audit 2023-09-23T18:06:18.429073+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:18.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:18 smithi098 bash[27349]: audit 2023-09-23T18:06:18.430472+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:06:18.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:18 smithi098 bash[27349]: audit 2023-09-23T18:06:18.433062+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:18.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:18 smithi098 bash[27349]: audit 2023-09-23T18:06:18.434327+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:06:18.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:18 smithi098 bash[27349]: audit 2023-09-23T18:06:18.439353+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:18.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:18 smithi098 bash[33599]: audit 2023-09-23T18:06:17.845690+0000 mon.a (mon.0) 183 : audit [DBG] from='client.? 172.21.15.98:0/2765848761' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:18.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:18 smithi098 bash[33599]: audit 2023-09-23T18:06:18.429073+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:18.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:18 smithi098 bash[33599]: audit 2023-09-23T18:06:18.430472+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:06:18.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:18 smithi098 bash[33599]: audit 2023-09-23T18:06:18.433062+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:18.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:18 smithi098 bash[33599]: audit 2023-09-23T18:06:18.434327+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:06:18.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:18 smithi098 bash[33599]: audit 2023-09-23T18:06:18.439353+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:18.956 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:18 smithi173 bash[27694]: audit 2023-09-23T18:06:17.845690+0000 mon.a (mon.0) 183 : audit [DBG] from='client.? 172.21.15.98:0/2765848761' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:18.956 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:18 smithi173 bash[27694]: audit 2023-09-23T18:06:18.429073+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:18.956 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:18 smithi173 bash[27694]: audit 2023-09-23T18:06:18.430472+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:06:18.957 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:18 smithi173 bash[27694]: audit 2023-09-23T18:06:18.433062+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:18.957 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:18 smithi173 bash[27694]: audit 2023-09-23T18:06:18.434327+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:06:18.957 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:18 smithi173 bash[27694]: audit 2023-09-23T18:06:18.439353+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:19.931 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:19 smithi173 bash[27694]: cluster 2023-09-23T18:06:19.084887+0000 mgr.y (mgr.14146) 49 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:19.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:19 smithi098 bash[27349]: cluster 2023-09-23T18:06:19.084887+0000 mgr.y (mgr.14146) 49 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:19.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:19 smithi098 bash[33599]: cluster 2023-09-23T18:06:19.084887+0000 mgr.y (mgr.14146) 49 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:21.611 INFO:teuthology.orchestra.run.smithi173.stdout:Scheduled mgr update... 2023-09-23T18:06:22.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:22 smithi098 bash[27349]: cluster 2023-09-23T18:06:21.085309+0000 mgr.y (mgr.14146) 50 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:22.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:22 smithi098 bash[27349]: audit 2023-09-23T18:06:21.608750+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:22.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:22 smithi098 bash[27349]: audit 2023-09-23T18:06:21.613176+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:06:22.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:22 smithi098 bash[27349]: audit 2023-09-23T18:06:21.615703+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:22.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:22 smithi098 bash[27349]: audit 2023-09-23T18:06:21.617012+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:06:22.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:22 smithi098 bash[27349]: audit 2023-09-23T18:06:21.623346+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:22.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:22 smithi098 bash[27349]: audit 2023-09-23T18:06:21.625378+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:06:22.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:22 smithi098 bash[27349]: audit 2023-09-23T18:06:21.632699+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T18:06:22.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:22 smithi098 bash[27349]: audit 2023-09-23T18:06:21.633904+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:06:22.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:22 smithi098 bash[27349]: audit 2023-09-23T18:06:21.635218+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:22.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:22 smithi098 bash[33599]: cluster 2023-09-23T18:06:21.085309+0000 mgr.y (mgr.14146) 50 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:22.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:22 smithi098 bash[33599]: audit 2023-09-23T18:06:21.608750+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:22.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:22 smithi098 bash[33599]: audit 2023-09-23T18:06:21.613176+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:06:22.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:22 smithi098 bash[33599]: audit 2023-09-23T18:06:21.615703+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:22.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:22 smithi098 bash[33599]: audit 2023-09-23T18:06:21.617012+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:06:22.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:22 smithi098 bash[33599]: audit 2023-09-23T18:06:21.623346+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:22.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:22 smithi098 bash[33599]: audit 2023-09-23T18:06:21.625378+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:06:22.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:22 smithi098 bash[33599]: audit 2023-09-23T18:06:21.632699+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T18:06:22.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:22 smithi098 bash[33599]: audit 2023-09-23T18:06:21.633904+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:06:22.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:22 smithi098 bash[33599]: audit 2023-09-23T18:06:21.635218+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:22.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:22 smithi173 bash[27694]: cluster 2023-09-23T18:06:21.085309+0000 mgr.y (mgr.14146) 50 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:22.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:22 smithi173 bash[27694]: audit 2023-09-23T18:06:21.608750+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:22.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:22 smithi173 bash[27694]: audit 2023-09-23T18:06:21.613176+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:06:22.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:22 smithi173 bash[27694]: audit 2023-09-23T18:06:21.615703+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:22.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:22 smithi173 bash[27694]: audit 2023-09-23T18:06:21.617012+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:06:22.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:22 smithi173 bash[27694]: audit 2023-09-23T18:06:21.623346+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:22.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:22 smithi173 bash[27694]: audit 2023-09-23T18:06:21.625378+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:06:22.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:22 smithi173 bash[27694]: audit 2023-09-23T18:06:21.632699+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T18:06:22.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:22 smithi173 bash[27694]: audit 2023-09-23T18:06:21.633904+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:06:22.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:22 smithi173 bash[27694]: audit 2023-09-23T18:06:21.635218+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:22.764 DEBUG:teuthology.orchestra.run.smithi173:mgr.x> sudo journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mgr.x.service 2023-09-23T18:06:22.766 INFO:tasks.cephadm:Deploying OSDs... 2023-09-23T18:06:22.766 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-23T18:06:22.766 DEBUG:teuthology.orchestra.run.smithi098:> dd if=/scratch_devs of=/dev/stdout 2023-09-23T18:06:22.774 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-23T18:06:22.774 DEBUG:teuthology.orchestra.run.smithi098:> stat /dev/vg_nvme/lv_1 2023-09-23T18:06:22.821 INFO:teuthology.orchestra.run.smithi098.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-23T18:06:22.821 INFO:teuthology.orchestra.run.smithi098.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T18:06:22.821 INFO:teuthology.orchestra.run.smithi098.stdout:Device: 5h/5d Inode: 490 Links: 1 2023-09-23T18:06:22.821 INFO:teuthology.orchestra.run.smithi098.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T18:06:22.822 INFO:teuthology.orchestra.run.smithi098.stdout:Access: 2023-09-23 17:59:43.609305368 +0000 2023-09-23T18:06:22.822 INFO:teuthology.orchestra.run.smithi098.stdout:Modify: 2023-09-23 17:59:43.285312840 +0000 2023-09-23T18:06:22.822 INFO:teuthology.orchestra.run.smithi098.stdout:Change: 2023-09-23 17:59:43.285312840 +0000 2023-09-23T18:06:22.822 INFO:teuthology.orchestra.run.smithi098.stdout: Birth: - 2023-09-23T18:06:22.822 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-23T18:06:22.876 INFO:teuthology.orchestra.run.smithi098.stderr:1+0 records in 2023-09-23T18:06:22.876 INFO:teuthology.orchestra.run.smithi098.stderr:1+0 records out 2023-09-23T18:06:22.876 INFO:teuthology.orchestra.run.smithi098.stderr:512 bytes copied, 0.000396466 s, 1.3 MB/s 2023-09-23T18:06:22.877 DEBUG:teuthology.orchestra.run.smithi098:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-23T18:06:22.927 DEBUG:teuthology.orchestra.run.smithi098:> stat /dev/vg_nvme/lv_2 2023-09-23T18:06:22.978 INFO:teuthology.orchestra.run.smithi098.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-23T18:06:22.978 INFO:teuthology.orchestra.run.smithi098.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T18:06:22.978 INFO:teuthology.orchestra.run.smithi098.stdout:Device: 5h/5d Inode: 495 Links: 1 2023-09-23T18:06:22.978 INFO:teuthology.orchestra.run.smithi098.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T18:06:22.978 INFO:teuthology.orchestra.run.smithi098.stdout:Access: 2023-09-23 17:59:44.149292915 +0000 2023-09-23T18:06:22.978 INFO:teuthology.orchestra.run.smithi098.stdout:Modify: 2023-09-23 17:59:43.845299926 +0000 2023-09-23T18:06:22.978 INFO:teuthology.orchestra.run.smithi098.stdout:Change: 2023-09-23 17:59:43.845299926 +0000 2023-09-23T18:06:22.978 INFO:teuthology.orchestra.run.smithi098.stdout: Birth: - 2023-09-23T18:06:22.979 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-23T18:06:23.031 INFO:teuthology.orchestra.run.smithi098.stderr:1+0 records in 2023-09-23T18:06:23.031 INFO:teuthology.orchestra.run.smithi098.stderr:1+0 records out 2023-09-23T18:06:23.031 INFO:teuthology.orchestra.run.smithi098.stderr:512 bytes copied, 0.000414596 s, 1.2 MB/s 2023-09-23T18:06:23.032 DEBUG:teuthology.orchestra.run.smithi098:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-23T18:06:23.083 DEBUG:teuthology.orchestra.run.smithi098:> stat /dev/vg_nvme/lv_3 2023-09-23T18:06:23.134 INFO:teuthology.orchestra.run.smithi098.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-23T18:06:23.134 INFO:teuthology.orchestra.run.smithi098.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T18:06:23.134 INFO:teuthology.orchestra.run.smithi098.stdout:Device: 5h/5d Inode: 502 Links: 1 2023-09-23T18:06:23.134 INFO:teuthology.orchestra.run.smithi098.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T18:06:23.134 INFO:teuthology.orchestra.run.smithi098.stdout:Access: 2023-09-23 17:59:44.737279355 +0000 2023-09-23T18:06:23.134 INFO:teuthology.orchestra.run.smithi098.stdout:Modify: 2023-09-23 17:59:44.413286828 +0000 2023-09-23T18:06:23.134 INFO:teuthology.orchestra.run.smithi098.stdout:Change: 2023-09-23 17:59:44.413286828 +0000 2023-09-23T18:06:23.134 INFO:teuthology.orchestra.run.smithi098.stdout: Birth: - 2023-09-23T18:06:23.135 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-23T18:06:23.169 INFO:journalctl@ceph.mgr.x.smithi173.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:06:23.187 INFO:teuthology.orchestra.run.smithi098.stderr:1+0 records in 2023-09-23T18:06:23.188 INFO:teuthology.orchestra.run.smithi098.stderr:1+0 records out 2023-09-23T18:06:23.188 INFO:teuthology.orchestra.run.smithi098.stderr:512 bytes copied, 0.000344156 s, 1.5 MB/s 2023-09-23T18:06:23.188 DEBUG:teuthology.orchestra.run.smithi098:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-23T18:06:23.243 DEBUG:teuthology.orchestra.run.smithi098:> stat /dev/vg_nvme/lv_4 2023-09-23T18:06:23.298 INFO:teuthology.orchestra.run.smithi098.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-23T18:06:23.298 INFO:teuthology.orchestra.run.smithi098.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T18:06:23.298 INFO:teuthology.orchestra.run.smithi098.stdout:Device: 5h/5d Inode: 508 Links: 1 2023-09-23T18:06:23.298 INFO:teuthology.orchestra.run.smithi098.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T18:06:23.298 INFO:teuthology.orchestra.run.smithi098.stdout:Access: 2023-09-23 17:59:45.253267456 +0000 2023-09-23T18:06:23.298 INFO:teuthology.orchestra.run.smithi098.stdout:Modify: 2023-09-23 17:59:44.945274558 +0000 2023-09-23T18:06:23.298 INFO:teuthology.orchestra.run.smithi098.stdout:Change: 2023-09-23 17:59:44.945274558 +0000 2023-09-23T18:06:23.298 INFO:teuthology.orchestra.run.smithi098.stdout: Birth: - 2023-09-23T18:06:23.299 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-23T18:06:23.351 INFO:teuthology.orchestra.run.smithi098.stderr:1+0 records in 2023-09-23T18:06:23.352 INFO:teuthology.orchestra.run.smithi098.stderr:1+0 records out 2023-09-23T18:06:23.352 INFO:teuthology.orchestra.run.smithi098.stderr:512 bytes copied, 0.000428949 s, 1.2 MB/s 2023-09-23T18:06:23.353 DEBUG:teuthology.orchestra.run.smithi098:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-23T18:06:23.399 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:23 smithi173 bash[27694]: audit 2023-09-23T18:06:21.598918+0000 mgr.y (mgr.14146) 51 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi098=y;smithi173=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:06:23.399 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:23 smithi173 bash[27694]: cephadm 2023-09-23T18:06:21.602446+0000 mgr.y (mgr.14146) 52 : cephadm [INF] Saving service mgr spec with placement smithi098=y;smithi173=x;count:2 2023-09-23T18:06:23.399 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:23 smithi173 bash[27694]: cephadm 2023-09-23T18:06:21.636246+0000 mgr.y (mgr.14146) 53 : cephadm [INF] Deploying daemon mgr.x on smithi173 2023-09-23T18:06:23.403 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-09-23T18:06:23.403 DEBUG:teuthology.orchestra.run.smithi173:> dd if=/scratch_devs of=/dev/stdout 2023-09-23T18:06:23.410 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-23T18:06:23.410 DEBUG:teuthology.orchestra.run.smithi173:> stat /dev/vg_nvme/lv_1 2023-09-23T18:06:23.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:23 smithi098 bash[27349]: audit 2023-09-23T18:06:21.598918+0000 mgr.y (mgr.14146) 51 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi098=y;smithi173=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:06:23.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:23 smithi098 bash[27349]: cephadm 2023-09-23T18:06:21.602446+0000 mgr.y (mgr.14146) 52 : cephadm [INF] Saving service mgr spec with placement smithi098=y;smithi173=x;count:2 2023-09-23T18:06:23.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:23 smithi098 bash[27349]: cephadm 2023-09-23T18:06:21.636246+0000 mgr.y (mgr.14146) 53 : cephadm [INF] Deploying daemon mgr.x on smithi173 2023-09-23T18:06:23.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:23 smithi098 bash[33599]: audit 2023-09-23T18:06:21.598918+0000 mgr.y (mgr.14146) 51 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi098=y;smithi173=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:06:23.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:23 smithi098 bash[33599]: cephadm 2023-09-23T18:06:21.602446+0000 mgr.y (mgr.14146) 52 : cephadm [INF] Saving service mgr spec with placement smithi098=y;smithi173=x;count:2 2023-09-23T18:06:23.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:23 smithi098 bash[33599]: cephadm 2023-09-23T18:06:21.636246+0000 mgr.y (mgr.14146) 53 : cephadm [INF] Deploying daemon mgr.x on smithi173 2023-09-23T18:06:23.458 INFO:teuthology.orchestra.run.smithi173.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-23T18:06:23.458 INFO:teuthology.orchestra.run.smithi173.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T18:06:23.458 INFO:teuthology.orchestra.run.smithi173.stdout:Device: 5h/5d Inode: 489 Links: 1 2023-09-23T18:06:23.458 INFO:teuthology.orchestra.run.smithi173.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T18:06:23.458 INFO:teuthology.orchestra.run.smithi173.stdout:Access: 2023-09-23 17:59:43.600840216 +0000 2023-09-23T18:06:23.459 INFO:teuthology.orchestra.run.smithi173.stdout:Modify: 2023-09-23 17:59:43.268847870 +0000 2023-09-23T18:06:23.459 INFO:teuthology.orchestra.run.smithi173.stdout:Change: 2023-09-23 17:59:43.268847870 +0000 2023-09-23T18:06:23.459 INFO:teuthology.orchestra.run.smithi173.stdout: Birth: - 2023-09-23T18:06:23.459 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-23T18:06:23.514 INFO:teuthology.orchestra.run.smithi173.stderr:1+0 records in 2023-09-23T18:06:23.514 INFO:teuthology.orchestra.run.smithi173.stderr:1+0 records out 2023-09-23T18:06:23.514 INFO:teuthology.orchestra.run.smithi173.stderr:512 bytes copied, 0.000407672 s, 1.3 MB/s 2023-09-23T18:06:23.517 DEBUG:teuthology.orchestra.run.smithi173:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-23T18:06:23.569 DEBUG:teuthology.orchestra.run.smithi173:> stat /dev/vg_nvme/lv_2 2023-09-23T18:06:23.619 INFO:teuthology.orchestra.run.smithi173.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-23T18:06:23.619 INFO:teuthology.orchestra.run.smithi173.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T18:06:23.619 INFO:teuthology.orchestra.run.smithi173.stdout:Device: 5h/5d Inode: 497 Links: 1 2023-09-23T18:06:23.619 INFO:teuthology.orchestra.run.smithi173.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T18:06:23.620 INFO:teuthology.orchestra.run.smithi173.stdout:Access: 2023-09-23 17:59:44.120828231 +0000 2023-09-23T18:06:23.620 INFO:teuthology.orchestra.run.smithi173.stdout:Modify: 2023-09-23 17:59:43.824835054 +0000 2023-09-23T18:06:23.620 INFO:teuthology.orchestra.run.smithi173.stdout:Change: 2023-09-23 17:59:43.824835054 +0000 2023-09-23T18:06:23.620 INFO:teuthology.orchestra.run.smithi173.stdout: Birth: - 2023-09-23T18:06:23.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-23T18:06:23.672 INFO:teuthology.orchestra.run.smithi173.stderr:1+0 records in 2023-09-23T18:06:23.672 INFO:teuthology.orchestra.run.smithi173.stderr:1+0 records out 2023-09-23T18:06:23.673 INFO:teuthology.orchestra.run.smithi173.stderr:512 bytes copied, 0.000286103 s, 1.8 MB/s 2023-09-23T18:06:23.673 DEBUG:teuthology.orchestra.run.smithi173:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-23T18:06:23.724 DEBUG:teuthology.orchestra.run.smithi173:> stat /dev/vg_nvme/lv_3 2023-09-23T18:06:23.768 INFO:teuthology.orchestra.run.smithi173.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-23T18:06:23.768 INFO:teuthology.orchestra.run.smithi173.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T18:06:23.769 INFO:teuthology.orchestra.run.smithi173.stdout:Device: 5h/5d Inode: 503 Links: 1 2023-09-23T18:06:23.769 INFO:teuthology.orchestra.run.smithi173.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T18:06:23.769 INFO:teuthology.orchestra.run.smithi173.stdout:Access: 2023-09-23 17:59:44.652815968 +0000 2023-09-23T18:06:23.769 INFO:teuthology.orchestra.run.smithi173.stdout:Modify: 2023-09-23 17:59:44.348822976 +0000 2023-09-23T18:06:23.769 INFO:teuthology.orchestra.run.smithi173.stdout:Change: 2023-09-23 17:59:44.348822976 +0000 2023-09-23T18:06:23.769 INFO:teuthology.orchestra.run.smithi173.stdout: Birth: - 2023-09-23T18:06:23.769 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-23T18:06:23.827 INFO:teuthology.orchestra.run.smithi173.stderr:1+0 records in 2023-09-23T18:06:23.827 INFO:teuthology.orchestra.run.smithi173.stderr:1+0 records out 2023-09-23T18:06:23.827 INFO:teuthology.orchestra.run.smithi173.stderr:512 bytes copied, 0.00038862 s, 1.3 MB/s 2023-09-23T18:06:23.828 DEBUG:teuthology.orchestra.run.smithi173:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-23T18:06:23.881 DEBUG:teuthology.orchestra.run.smithi173:> stat /dev/vg_nvme/lv_4 2023-09-23T18:06:23.931 INFO:teuthology.orchestra.run.smithi173.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-23T18:06:23.931 INFO:teuthology.orchestra.run.smithi173.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T18:06:23.931 INFO:teuthology.orchestra.run.smithi173.stdout:Device: 5h/5d Inode: 510 Links: 1 2023-09-23T18:06:23.931 INFO:teuthology.orchestra.run.smithi173.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T18:06:23.931 INFO:teuthology.orchestra.run.smithi173.stdout:Access: 2023-09-23 17:59:45.176803890 +0000 2023-09-23T18:06:23.932 INFO:teuthology.orchestra.run.smithi173.stdout:Modify: 2023-09-23 17:59:44.872810897 +0000 2023-09-23T18:06:23.932 INFO:teuthology.orchestra.run.smithi173.stdout:Change: 2023-09-23 17:59:44.872810897 +0000 2023-09-23T18:06:23.932 INFO:teuthology.orchestra.run.smithi173.stdout: Birth: - 2023-09-23T18:06:23.932 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-23T18:06:23.984 INFO:teuthology.orchestra.run.smithi173.stderr:1+0 records in 2023-09-23T18:06:23.984 INFO:teuthology.orchestra.run.smithi173.stderr:1+0 records out 2023-09-23T18:06:23.985 INFO:teuthology.orchestra.run.smithi173.stderr:512 bytes copied, 0.000514226 s, 996 kB/s 2023-09-23T18:06:23.986 DEBUG:teuthology.orchestra.run.smithi173:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-23T18:06:24.041 INFO:tasks.cephadm:Deploying osd.0 on smithi098 with /dev/vg_nvme/lv_4... 2023-09-23T18:06:24.041 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-09-23T18:06:24.238 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:24 smithi173 bash[27694]: cluster 2023-09-23T18:06:23.085753+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:24.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:24 smithi098 bash[27349]: cluster 2023-09-23T18:06:23.085753+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:24.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:24 smithi098 bash[33599]: cluster 2023-09-23T18:06:23.085753+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:24.518 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:06:24 smithi173 systemd[1]: Started Ceph mgr.x for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:06:25.526 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:06:25 smithi173 bash[28552]: debug 2023-09-23T18:06:25.408+0000 7f452a687700 1 -- 172.21.15.173:0/4147025171 <== mon.0 v2:172.21.15.98:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a80adae4e0 con 0x55a80aea2800 2023-09-23T18:06:25.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:25 smithi173 bash[27694]: audit 2023-09-23T18:06:24.489154+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:25.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:25 smithi173 bash[27694]: audit 2023-09-23T18:06:24.492922+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:06:25.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:25 smithi173 bash[27694]: cluster 2023-09-23T18:06:25.086224+0000 mgr.y (mgr.14146) 55 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:25.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:25 smithi098 bash[27349]: audit 2023-09-23T18:06:24.489154+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:25.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:25 smithi098 bash[27349]: audit 2023-09-23T18:06:24.492922+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:06:25.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:25 smithi098 bash[27349]: cluster 2023-09-23T18:06:25.086224+0000 mgr.y (mgr.14146) 55 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:25.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:25 smithi098 bash[33599]: audit 2023-09-23T18:06:24.489154+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:25.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:25 smithi098 bash[33599]: audit 2023-09-23T18:06:24.492922+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:06:25.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:25 smithi098 bash[33599]: cluster 2023-09-23T18:06:25.086224+0000 mgr.y (mgr.14146) 55 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:26.026 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:06:25 smithi173 bash[28552]: debug 2023-09-23T18:06:25.568+0000 7f453ac6c680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T18:06:26.026 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:06:25 smithi173 bash[28552]: debug 2023-09-23T18:06:25.676+0000 7f453ac6c680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T18:06:26.527 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:06:26 smithi173 bash[28552]: debug 2023-09-23T18:06:26.140+0000 7f453ac6c680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T18:06:27.407 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:27 smithi173 bash[27694]: audit 2023-09-23T18:06:26.263307+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:27.407 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:27 smithi173 bash[27694]: cluster 2023-09-23T18:06:27.086734+0000 mgr.y (mgr.14146) 56 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:27.408 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:06:27 smithi173 bash[28552]: debug 2023-09-23T18:06:27.056+0000 7f453ac6c680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T18:06:27.408 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:06:27 smithi173 bash[28552]: debug 2023-09-23T18:06:27.152+0000 7f453ac6c680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T18:06:27.556 INFO:teuthology.orchestra.run.smithi098.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-09-23T18:06:27.556 INFO:teuthology.orchestra.run.smithi098.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-09-23T18:06:27.556 INFO:teuthology.orchestra.run.smithi098.stderr: stderr: 10+0 records in 2023-09-23T18:06:27.556 INFO:teuthology.orchestra.run.smithi098.stderr:10+0 records out 2023-09-23T18:06:27.556 INFO:teuthology.orchestra.run.smithi098.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0140675 s, 745 MB/s 2023-09-23T18:06:27.556 INFO:teuthology.orchestra.run.smithi098.stderr:--> Zapping successful for: 2023-09-23T18:06:27.598 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:27 smithi098 bash[33599]: audit 2023-09-23T18:06:26.263307+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:27.598 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:27 smithi098 bash[33599]: cluster 2023-09-23T18:06:27.086734+0000 mgr.y (mgr.14146) 56 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:27.599 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:27 smithi098 bash[27349]: audit 2023-09-23T18:06:26.263307+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:27.599 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:27 smithi098 bash[27349]: cluster 2023-09-23T18:06:27.086734+0000 mgr.y (mgr.14146) 56 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:27.776 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:06:27 smithi173 bash[28552]: debug 2023-09-23T18:06:27.404+0000 7f453ac6c680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T18:06:28.307 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi098:vg_nvme/lv_4 2023-09-23T18:06:28.896 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:06:28 smithi173 bash[28552]: debug 2023-09-23T18:06:28.632+0000 7f453ac6c680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T18:06:28.896 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:06:28 smithi173 bash[28552]: debug 2023-09-23T18:06:28.800+0000 7f453ac6c680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:06:29.229 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:06:28 smithi173 bash[28552]: debug 2023-09-23T18:06:28.892+0000 7f453ac6c680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T18:06:29.229 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:06:29 smithi173 bash[28552]: debug 2023-09-23T18:06:29.004+0000 7f453ac6c680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T18:06:29.526 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:06:29 smithi173 bash[28552]: debug 2023-09-23T18:06:29.224+0000 7f453ac6c680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T18:06:30.026 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:06:29 smithi173 bash[28552]: debug 2023-09-23T18:06:29.608+0000 7f453ac6c680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T18:06:30.026 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:06:29 smithi173 bash[28552]: debug 2023-09-23T18:06:29.728+0000 7f453ac6c680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T18:06:30.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:30 smithi098 bash[27349]: cluster 2023-09-23T18:06:29.087190+0000 mgr.y (mgr.14146) 57 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:30.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:30 smithi098 bash[27349]: audit 2023-09-23T18:06:29.518503+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:30.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:30 smithi098 bash[27349]: audit 2023-09-23T18:06:29.519799+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:30.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:30 smithi098 bash[27349]: audit 2023-09-23T18:06:29.521525+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:06:30.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:30 smithi098 bash[27349]: audit 2023-09-23T18:06:29.529147+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:30.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:30 smithi098 bash[27349]: audit 2023-09-23T18:06:29.533509+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:06:30.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:30 smithi098 bash[27349]: audit 2023-09-23T18:06:29.534746+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:06:30.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:30 smithi098 bash[27349]: audit 2023-09-23T18:06:29.535875+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:30.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:30 smithi098 bash[33599]: cluster 2023-09-23T18:06:29.087190+0000 mgr.y (mgr.14146) 57 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:30.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:30 smithi098 bash[33599]: audit 2023-09-23T18:06:29.518503+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:30.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:30 smithi098 bash[33599]: audit 2023-09-23T18:06:29.519799+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:30.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:30 smithi098 bash[33599]: audit 2023-09-23T18:06:29.521525+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:06:30.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:30 smithi098 bash[33599]: audit 2023-09-23T18:06:29.529147+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:30.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:30 smithi098 bash[33599]: audit 2023-09-23T18:06:29.533509+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:06:30.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:30 smithi098 bash[33599]: audit 2023-09-23T18:06:29.534746+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:06:30.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:30 smithi098 bash[33599]: audit 2023-09-23T18:06:29.535875+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:30.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:30 smithi173 bash[27694]: cluster 2023-09-23T18:06:29.087190+0000 mgr.y (mgr.14146) 57 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:30.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:30 smithi173 bash[27694]: audit 2023-09-23T18:06:29.518503+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:30.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:30 smithi173 bash[27694]: audit 2023-09-23T18:06:29.519799+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:30.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:30 smithi173 bash[27694]: audit 2023-09-23T18:06:29.521525+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:06:30.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:30 smithi173 bash[27694]: audit 2023-09-23T18:06:29.529147+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:30.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:30 smithi173 bash[27694]: audit 2023-09-23T18:06:29.533509+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:06:30.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:30 smithi173 bash[27694]: audit 2023-09-23T18:06:29.534746+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:06:30.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:30 smithi173 bash[27694]: audit 2023-09-23T18:06:29.535875+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:31.026 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:06:30 smithi173 bash[28552]: debug 2023-09-23T18:06:30.588+0000 7f453ac6c680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T18:06:31.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:06:30 smithi173 bash[28552]: debug 2023-09-23T18:06:30.692+0000 7f453ac6c680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T18:06:31.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:06:30 smithi173 bash[28552]: debug 2023-09-23T18:06:30.808+0000 7f453ac6c680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T18:06:31.327 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:31 smithi173 bash[27694]: cephadm 2023-09-23T18:06:29.533035+0000 mgr.y (mgr.14146) 58 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-09-23T18:06:31.327 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:31 smithi173 bash[27694]: cephadm 2023-09-23T18:06:29.536747+0000 mgr.y (mgr.14146) 59 : cephadm [INF] Reconfiguring daemon mgr.y on smithi098 2023-09-23T18:06:31.327 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:06:31 smithi173 bash[28552]: debug 2023-09-23T18:06:31.032+0000 7f453ac6c680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T18:06:31.438 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:31 smithi098 bash[27349]: cephadm 2023-09-23T18:06:29.533035+0000 mgr.y (mgr.14146) 58 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-09-23T18:06:31.438 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:31 smithi098 bash[27349]: cephadm 2023-09-23T18:06:29.536747+0000 mgr.y (mgr.14146) 59 : cephadm [INF] Reconfiguring daemon mgr.y on smithi098 2023-09-23T18:06:31.438 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:31 smithi098 bash[33599]: cephadm 2023-09-23T18:06:29.533035+0000 mgr.y (mgr.14146) 58 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-09-23T18:06:31.438 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:31 smithi098 bash[33599]: cephadm 2023-09-23T18:06:29.536747+0000 mgr.y (mgr.14146) 59 : cephadm [INF] Reconfiguring daemon mgr.y on smithi098 2023-09-23T18:06:31.652 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:06:31 smithi173 bash[28552]: debug 2023-09-23T18:06:31.324+0000 7f453ac6c680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T18:06:31.652 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:06:31 smithi173 bash[28552]: debug 2023-09-23T18:06:31.480+0000 7f453ac6c680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T18:06:31.974 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:06:31 smithi173 bash[28552]: debug 2023-09-23T18:06:31.648+0000 7f453ac6c680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:06:31.974 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:06:31 smithi173 bash[28552]: debug 2023-09-23T18:06:31.864+0000 7f453ac6c680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T18:06:32.276 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:06:31 smithi173 bash[28552]: debug 2023-09-23T18:06:31.972+0000 7f453ac6c680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T18:06:32.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:32 smithi173 bash[27694]: cluster 2023-09-23T18:06:31.087613+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:32.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:32 smithi173 bash[27694]: audit 2023-09-23T18:06:31.961836+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:06:32.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:32 smithi173 bash[27694]: audit 2023-09-23T18:06:31.965016+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:06:32.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:32 smithi173 bash[27694]: audit 2023-09-23T18:06:31.965785+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:32.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:32 smithi173 bash[27694]: cluster 2023-09-23T18:06:31.995951+0000 mon.a (mon.0) 211 : cluster [DBG] Standby manager daemon x started 2023-09-23T18:06:32.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:32 smithi173 bash[27694]: audit 2023-09-23T18:06:31.999079+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.173:0/467188941' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T18:06:32.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:32 smithi173 bash[27694]: audit 2023-09-23T18:06:32.000142+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.173:0/467188941' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:06:32.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:32 smithi173 bash[27694]: audit 2023-09-23T18:06:32.002354+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.173:0/467188941' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T18:06:32.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:32 smithi173 bash[27694]: audit 2023-09-23T18:06:32.003026+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.173:0/467188941' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:06:32.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:32 smithi098 bash[27349]: cluster 2023-09-23T18:06:31.087613+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:32.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:32 smithi098 bash[27349]: audit 2023-09-23T18:06:31.961836+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:06:32.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:32 smithi098 bash[27349]: audit 2023-09-23T18:06:31.965016+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:06:32.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:32 smithi098 bash[27349]: audit 2023-09-23T18:06:31.965785+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:32.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:32 smithi098 bash[27349]: cluster 2023-09-23T18:06:31.995951+0000 mon.a (mon.0) 211 : cluster [DBG] Standby manager daemon x started 2023-09-23T18:06:32.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:32 smithi098 bash[27349]: audit 2023-09-23T18:06:31.999079+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.173:0/467188941' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T18:06:32.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:32 smithi098 bash[27349]: audit 2023-09-23T18:06:32.000142+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.173:0/467188941' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:06:32.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:32 smithi098 bash[27349]: audit 2023-09-23T18:06:32.002354+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.173:0/467188941' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T18:06:32.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:32 smithi098 bash[27349]: audit 2023-09-23T18:06:32.003026+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.173:0/467188941' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:06:32.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:32 smithi098 bash[33599]: cluster 2023-09-23T18:06:31.087613+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:32.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:32 smithi098 bash[33599]: audit 2023-09-23T18:06:31.961836+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:06:32.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:32 smithi098 bash[33599]: audit 2023-09-23T18:06:31.965016+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:06:32.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:32 smithi098 bash[33599]: audit 2023-09-23T18:06:31.965785+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:32.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:32 smithi098 bash[33599]: cluster 2023-09-23T18:06:31.995951+0000 mon.a (mon.0) 211 : cluster [DBG] Standby manager daemon x started 2023-09-23T18:06:32.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:32 smithi098 bash[33599]: audit 2023-09-23T18:06:31.999079+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.173:0/467188941' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T18:06:32.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:32 smithi098 bash[33599]: audit 2023-09-23T18:06:32.000142+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.173:0/467188941' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:06:32.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:32 smithi098 bash[33599]: audit 2023-09-23T18:06:32.002354+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.173:0/467188941' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T18:06:32.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:32 smithi098 bash[33599]: audit 2023-09-23T18:06:32.003026+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.173:0/467188941' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:06:33.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:33 smithi098 bash[27349]: audit 2023-09-23T18:06:31.954650+0000 mgr.y (mgr.14146) 61 : audit [DBG] from='client.24122 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi098:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:06:33.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:33 smithi098 bash[27349]: cluster 2023-09-23T18:06:32.157656+0000 mon.a (mon.0) 212 : cluster [DBG] mgrmap e14: y(active, since 81s), standbys: x 2023-09-23T18:06:33.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:33 smithi098 bash[27349]: audit 2023-09-23T18:06:32.158214+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T18:06:33.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:33 smithi098 bash[27349]: audit 2023-09-23T18:06:32.474054+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:33.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:33 smithi098 bash[27349]: audit 2023-09-23T18:06:32.475838+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:06:33.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:33 smithi098 bash[27349]: audit 2023-09-23T18:06:32.732907+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:33.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:33 smithi098 bash[27349]: audit 2023-09-23T18:06:32.734255+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:06:33.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:33 smithi098 bash[27349]: audit 2023-09-23T18:06:32.742698+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:33.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:33 smithi098 bash[33599]: audit 2023-09-23T18:06:31.954650+0000 mgr.y (mgr.14146) 61 : audit [DBG] from='client.24122 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi098:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:06:33.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:33 smithi098 bash[33599]: cluster 2023-09-23T18:06:32.157656+0000 mon.a (mon.0) 212 : cluster [DBG] mgrmap e14: y(active, since 81s), standbys: x 2023-09-23T18:06:33.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:33 smithi098 bash[33599]: audit 2023-09-23T18:06:32.158214+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T18:06:33.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:33 smithi098 bash[33599]: audit 2023-09-23T18:06:32.474054+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:33.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:33 smithi098 bash[33599]: audit 2023-09-23T18:06:32.475838+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:06:33.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:33 smithi098 bash[33599]: audit 2023-09-23T18:06:32.732907+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:33.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:33 smithi098 bash[33599]: audit 2023-09-23T18:06:32.734255+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:06:33.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:33 smithi098 bash[33599]: audit 2023-09-23T18:06:32.742698+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:33.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:33 smithi173 bash[27694]: audit 2023-09-23T18:06:31.954650+0000 mgr.y (mgr.14146) 61 : audit [DBG] from='client.24122 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi098:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:06:33.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:33 smithi173 bash[27694]: cluster 2023-09-23T18:06:32.157656+0000 mon.a (mon.0) 212 : cluster [DBG] mgrmap e14: y(active, since 81s), standbys: x 2023-09-23T18:06:33.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:33 smithi173 bash[27694]: audit 2023-09-23T18:06:32.158214+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T18:06:33.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:33 smithi173 bash[27694]: audit 2023-09-23T18:06:32.474054+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:33.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:33 smithi173 bash[27694]: audit 2023-09-23T18:06:32.475838+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:06:33.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:33 smithi173 bash[27694]: audit 2023-09-23T18:06:32.732907+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:33.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:33 smithi173 bash[27694]: audit 2023-09-23T18:06:32.734255+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:06:33.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:33 smithi173 bash[27694]: audit 2023-09-23T18:06:32.742698+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:34.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:34 smithi098 bash[27349]: cluster 2023-09-23T18:06:33.088052+0000 mgr.y (mgr.14146) 62 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:34.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:34 smithi098 bash[33599]: cluster 2023-09-23T18:06:33.088052+0000 mgr.y (mgr.14146) 62 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:34.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:34 smithi173 bash[27694]: cluster 2023-09-23T18:06:33.088052+0000 mgr.y (mgr.14146) 62 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:36.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:36 smithi098 bash[27349]: cluster 2023-09-23T18:06:35.088506+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:36.455 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:36 smithi098 bash[33599]: cluster 2023-09-23T18:06:35.088506+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:36.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:36 smithi173 bash[27694]: cluster 2023-09-23T18:06:35.088506+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:38.411 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:38 smithi098 bash[27349]: cluster 2023-09-23T18:06:37.088952+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:38.412 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:38 smithi098 bash[33599]: cluster 2023-09-23T18:06:37.088952+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:38.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:38 smithi173 bash[27694]: cluster 2023-09-23T18:06:37.088952+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:40.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:40 smithi098 bash[27349]: cluster 2023-09-23T18:06:39.089402+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:40.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:40 smithi098 bash[27349]: audit 2023-09-23T18:06:40.169486+0000 mon.a (mon.0) 219 : audit [INF] from='client.? 172.21.15.98:0/1171260126' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4f35f176-4d07-4157-aeb9-dd8f71da3a63"}]: dispatch 2023-09-23T18:06:40.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:40 smithi098 bash[33599]: cluster 2023-09-23T18:06:39.089402+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:40.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:40 smithi098 bash[33599]: audit 2023-09-23T18:06:40.169486+0000 mon.a (mon.0) 219 : audit [INF] from='client.? 172.21.15.98:0/1171260126' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4f35f176-4d07-4157-aeb9-dd8f71da3a63"}]: dispatch 2023-09-23T18:06:40.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:40 smithi173 bash[27694]: cluster 2023-09-23T18:06:39.089402+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:40.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:40 smithi173 bash[27694]: audit 2023-09-23T18:06:40.169486+0000 mon.a (mon.0) 219 : audit [INF] from='client.? 172.21.15.98:0/1171260126' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4f35f176-4d07-4157-aeb9-dd8f71da3a63"}]: dispatch 2023-09-23T18:06:41.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:41 smithi098 bash[27349]: audit 2023-09-23T18:06:40.178487+0000 mon.a (mon.0) 220 : audit [INF] from='client.? 172.21.15.98:0/1171260126' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4f35f176-4d07-4157-aeb9-dd8f71da3a63"}]': finished 2023-09-23T18:06:41.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:41 smithi098 bash[27349]: cluster 2023-09-23T18:06:40.178715+0000 mon.a (mon.0) 221 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-09-23T18:06:41.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:41 smithi098 bash[27349]: audit 2023-09-23T18:06:40.178974+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:06:41.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:41 smithi098 bash[27349]: audit 2023-09-23T18:06:40.873939+0000 mon.a (mon.0) 223 : audit [DBG] from='client.? 172.21.15.98:0/26871957' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:06:41.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:41 smithi098 bash[33599]: audit 2023-09-23T18:06:40.178487+0000 mon.a (mon.0) 220 : audit [INF] from='client.? 172.21.15.98:0/1171260126' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4f35f176-4d07-4157-aeb9-dd8f71da3a63"}]': finished 2023-09-23T18:06:41.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:41 smithi098 bash[33599]: cluster 2023-09-23T18:06:40.178715+0000 mon.a (mon.0) 221 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-09-23T18:06:41.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:41 smithi098 bash[33599]: audit 2023-09-23T18:06:40.178974+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:06:41.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:41 smithi098 bash[33599]: audit 2023-09-23T18:06:40.873939+0000 mon.a (mon.0) 223 : audit [DBG] from='client.? 172.21.15.98:0/26871957' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:06:41.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:41 smithi173 bash[27694]: audit 2023-09-23T18:06:40.178487+0000 mon.a (mon.0) 220 : audit [INF] from='client.? 172.21.15.98:0/1171260126' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4f35f176-4d07-4157-aeb9-dd8f71da3a63"}]': finished 2023-09-23T18:06:41.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:41 smithi173 bash[27694]: cluster 2023-09-23T18:06:40.178715+0000 mon.a (mon.0) 221 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-09-23T18:06:41.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:41 smithi173 bash[27694]: audit 2023-09-23T18:06:40.178974+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:06:41.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:41 smithi173 bash[27694]: audit 2023-09-23T18:06:40.873939+0000 mon.a (mon.0) 223 : audit [DBG] from='client.? 172.21.15.98:0/26871957' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:06:42.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:42 smithi173 bash[27694]: cluster 2023-09-23T18:06:41.089807+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:42.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:42 smithi098 bash[27349]: cluster 2023-09-23T18:06:41.089807+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:42.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:42 smithi098 bash[33599]: cluster 2023-09-23T18:06:41.089807+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:44.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:44 smithi173 bash[27694]: cluster 2023-09-23T18:06:43.090312+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:44.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:44 smithi098 bash[27349]: cluster 2023-09-23T18:06:43.090312+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:44.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:44 smithi098 bash[33599]: cluster 2023-09-23T18:06:43.090312+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:46.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:46 smithi098 bash[27349]: cluster 2023-09-23T18:06:45.090755+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:46.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:46 smithi098 bash[33599]: cluster 2023-09-23T18:06:45.090755+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:46.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:46 smithi173 bash[27694]: cluster 2023-09-23T18:06:45.090755+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:48.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:48 smithi098 bash[27349]: cluster 2023-09-23T18:06:47.091185+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:48.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:48 smithi098 bash[33599]: cluster 2023-09-23T18:06:47.091185+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:48.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:48 smithi173 bash[27694]: cluster 2023-09-23T18:06:47.091185+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:50.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:50 smithi098 bash[27349]: cluster 2023-09-23T18:06:49.091683+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:50.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:50 smithi098 bash[33599]: cluster 2023-09-23T18:06:49.091683+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:50.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:50 smithi173 bash[27694]: cluster 2023-09-23T18:06:49.091683+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:52.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:52 smithi098 bash[27349]: cluster 2023-09-23T18:06:51.092163+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:52.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:52 smithi098 bash[33599]: cluster 2023-09-23T18:06:51.092163+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:52.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:52 smithi173 bash[27694]: cluster 2023-09-23T18:06:51.092163+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:53.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:53 smithi098 bash[27349]: audit 2023-09-23T18:06:52.755541+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T18:06:53.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:53 smithi098 bash[27349]: audit 2023-09-23T18:06:52.756947+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:53.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:53 smithi098 bash[33599]: audit 2023-09-23T18:06:52.755541+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T18:06:53.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:53 smithi098 bash[33599]: audit 2023-09-23T18:06:52.756947+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:53.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:53 smithi173 bash[27694]: audit 2023-09-23T18:06:52.755541+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T18:06:53.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:53 smithi173 bash[27694]: audit 2023-09-23T18:06:52.756947+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:06:54.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:54 smithi098 bash[33599]: cephadm 2023-09-23T18:06:52.758053+0000 mgr.y (mgr.14146) 72 : cephadm [INF] Deploying daemon osd.0 on smithi098 2023-09-23T18:06:54.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:54 smithi098 bash[33599]: cluster 2023-09-23T18:06:53.092602+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:54.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:54 smithi098 bash[27349]: cephadm 2023-09-23T18:06:52.758053+0000 mgr.y (mgr.14146) 72 : cephadm [INF] Deploying daemon osd.0 on smithi098 2023-09-23T18:06:54.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:54 smithi098 bash[27349]: cluster 2023-09-23T18:06:53.092602+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:54.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:54 smithi173 bash[27694]: cephadm 2023-09-23T18:06:52.758053+0000 mgr.y (mgr.14146) 72 : cephadm [INF] Deploying daemon osd.0 on smithi098 2023-09-23T18:06:54.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:54 smithi173 bash[27694]: cluster 2023-09-23T18:06:53.092602+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:56.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:56 smithi098 bash[27349]: cluster 2023-09-23T18:06:55.093102+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:56.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:56 smithi098 bash[33599]: cluster 2023-09-23T18:06:55.093102+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:56.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:56 smithi173 bash[27694]: cluster 2023-09-23T18:06:55.093102+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:58.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:57 smithi098 bash[27349]: audit 2023-09-23T18:06:56.959458+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:58.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:57 smithi098 bash[27349]: audit 2023-09-23T18:06:56.974998+0000 mon.a (mon.0) 227 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:06:58.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:06:57 smithi098 bash[27349]: cluster 2023-09-23T18:06:57.093487+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:58.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:57 smithi098 bash[33599]: audit 2023-09-23T18:06:56.959458+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:58.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:57 smithi098 bash[33599]: audit 2023-09-23T18:06:56.974998+0000 mon.a (mon.0) 227 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:06:58.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:06:57 smithi098 bash[33599]: cluster 2023-09-23T18:06:57.093487+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:06:58.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:57 smithi173 bash[27694]: audit 2023-09-23T18:06:56.959458+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:06:58.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:57 smithi173 bash[27694]: audit 2023-09-23T18:06:56.974998+0000 mon.a (mon.0) 227 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:06:58.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:06:57 smithi173 bash[27694]: cluster 2023-09-23T18:06:57.093487+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:07:00.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:00 smithi098 bash[27349]: cluster 2023-09-23T18:06:59.094019+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:07:00.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:00 smithi098 bash[33599]: cluster 2023-09-23T18:06:59.094019+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:07:00.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:00 smithi173 bash[27694]: cluster 2023-09-23T18:06:59.094019+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:07:02.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:02 smithi098 bash[27349]: cluster 2023-09-23T18:07:01.094534+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:07:02.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:02 smithi098 bash[27349]: audit 2023-09-23T18:07:02.129307+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:02.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:02 smithi098 bash[27349]: audit 2023-09-23T18:07:02.130190+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:07:02.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:02 smithi098 bash[27349]: audit 2023-09-23T18:07:02.131112+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:07:02.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:02 smithi098 bash[27349]: audit 2023-09-23T18:07:02.139894+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:02.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:02 smithi098 bash[33599]: cluster 2023-09-23T18:07:01.094534+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:07:02.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:02 smithi098 bash[33599]: audit 2023-09-23T18:07:02.129307+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:02.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:02 smithi098 bash[33599]: audit 2023-09-23T18:07:02.130190+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:07:02.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:02 smithi098 bash[33599]: audit 2023-09-23T18:07:02.131112+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:07:02.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:02 smithi098 bash[33599]: audit 2023-09-23T18:07:02.139894+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:02.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:02 smithi173 bash[27694]: cluster 2023-09-23T18:07:01.094534+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:07:02.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:02 smithi173 bash[27694]: audit 2023-09-23T18:07:02.129307+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:02.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:02 smithi173 bash[27694]: audit 2023-09-23T18:07:02.130190+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:07:02.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:02 smithi173 bash[27694]: audit 2023-09-23T18:07:02.131112+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:07:02.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:02 smithi173 bash[27694]: audit 2023-09-23T18:07:02.139894+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:04.371 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:04 smithi098 bash[27349]: cluster 2023-09-23T18:07:03.094977+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:07:04.372 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:04 smithi098 bash[27349]: audit 2023-09-23T18:07:03.725056+0000 mon.c (mon.2) 2 : audit [INF] from='osd.0 [v2:172.21.15.98:6802/1147685383,v1:172.21.15.98:6803/1147685383]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T18:07:04.372 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:04 smithi098 bash[27349]: audit 2023-09-23T18:07:03.725500+0000 mon.a (mon.0) 232 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T18:07:04.372 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:04 smithi098 bash[33599]: cluster 2023-09-23T18:07:03.094977+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:07:04.372 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:04 smithi098 bash[33599]: audit 2023-09-23T18:07:03.725056+0000 mon.c (mon.2) 2 : audit [INF] from='osd.0 [v2:172.21.15.98:6802/1147685383,v1:172.21.15.98:6803/1147685383]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T18:07:04.372 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:04 smithi098 bash[33599]: audit 2023-09-23T18:07:03.725500+0000 mon.a (mon.0) 232 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T18:07:04.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:04 smithi173 bash[27694]: cluster 2023-09-23T18:07:03.094977+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:07:04.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:04 smithi173 bash[27694]: audit 2023-09-23T18:07:03.725056+0000 mon.c (mon.2) 2 : audit [INF] from='osd.0 [v2:172.21.15.98:6802/1147685383,v1:172.21.15.98:6803/1147685383]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T18:07:04.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:04 smithi173 bash[27694]: audit 2023-09-23T18:07:03.725500+0000 mon.a (mon.0) 232 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T18:07:05.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:05 smithi098 bash[33599]: audit 2023-09-23T18:07:04.025647+0000 mon.a (mon.0) 233 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T18:07:05.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:05 smithi098 bash[33599]: cluster 2023-09-23T18:07:04.025770+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-09-23T18:07:05.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:05 smithi098 bash[33599]: audit 2023-09-23T18:07:04.026061+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:07:05.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:05 smithi098 bash[33599]: audit 2023-09-23T18:07:04.027209+0000 mon.c (mon.2) 3 : audit [INF] from='osd.0 [v2:172.21.15.98:6802/1147685383,v1:172.21.15.98:6803/1147685383]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:07:05.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:05 smithi098 bash[33599]: audit 2023-09-23T18:07:04.027603+0000 mon.a (mon.0) 236 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:07:05.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:05 smithi098 bash[27349]: audit 2023-09-23T18:07:04.025647+0000 mon.a (mon.0) 233 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T18:07:05.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:05 smithi098 bash[27349]: cluster 2023-09-23T18:07:04.025770+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-09-23T18:07:05.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:05 smithi098 bash[27349]: audit 2023-09-23T18:07:04.026061+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:07:05.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:05 smithi098 bash[27349]: audit 2023-09-23T18:07:04.027209+0000 mon.c (mon.2) 3 : audit [INF] from='osd.0 [v2:172.21.15.98:6802/1147685383,v1:172.21.15.98:6803/1147685383]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:07:05.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:05 smithi098 bash[27349]: audit 2023-09-23T18:07:04.027603+0000 mon.a (mon.0) 236 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:07:05.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:05 smithi173 bash[27694]: audit 2023-09-23T18:07:04.025647+0000 mon.a (mon.0) 233 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T18:07:05.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:05 smithi173 bash[27694]: cluster 2023-09-23T18:07:04.025770+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-09-23T18:07:05.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:05 smithi173 bash[27694]: audit 2023-09-23T18:07:04.026061+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:07:05.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:05 smithi173 bash[27694]: audit 2023-09-23T18:07:04.027209+0000 mon.c (mon.2) 3 : audit [INF] from='osd.0 [v2:172.21.15.98:6802/1147685383,v1:172.21.15.98:6803/1147685383]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:07:05.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:05 smithi173 bash[27694]: audit 2023-09-23T18:07:04.027603+0000 mon.a (mon.0) 236 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:07:05.769 INFO:teuthology.orchestra.run.smithi098.stdout:Created osd(s) 0 on host 'smithi098' 2023-09-23T18:07:06.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:06 smithi098 bash[27349]: audit 2023-09-23T18:07:05.027349+0000 mon.a (mon.0) 237 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]': finished 2023-09-23T18:07:06.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:06 smithi098 bash[27349]: cluster 2023-09-23T18:07:05.027465+0000 mon.a (mon.0) 238 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-09-23T18:07:06.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:06 smithi098 bash[27349]: audit 2023-09-23T18:07:05.027923+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:07:06.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:06 smithi098 bash[27349]: audit 2023-09-23T18:07:05.037094+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:07:06.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:06 smithi098 bash[27349]: cluster 2023-09-23T18:07:05.095396+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:07:06.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:06 smithi098 bash[27349]: audit 2023-09-23T18:07:05.766090+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:06.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:06 smithi098 bash[27349]: audit 2023-09-23T18:07:05.767023+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:07:06.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:06 smithi098 bash[33599]: audit 2023-09-23T18:07:05.027349+0000 mon.a (mon.0) 237 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]': finished 2023-09-23T18:07:06.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:06 smithi098 bash[33599]: cluster 2023-09-23T18:07:05.027465+0000 mon.a (mon.0) 238 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-09-23T18:07:06.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:06 smithi098 bash[33599]: audit 2023-09-23T18:07:05.027923+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:07:06.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:06 smithi098 bash[33599]: audit 2023-09-23T18:07:05.037094+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:07:06.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:06 smithi098 bash[33599]: cluster 2023-09-23T18:07:05.095396+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:07:06.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:06 smithi098 bash[33599]: audit 2023-09-23T18:07:05.766090+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:06.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:06 smithi098 bash[33599]: audit 2023-09-23T18:07:05.767023+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:07:06.508 DEBUG:teuthology.orchestra.run.smithi098:osd.0> sudo journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.0.service 2023-09-23T18:07:06.511 INFO:tasks.cephadm:Deploying osd.1 on smithi098 with /dev/vg_nvme/lv_3... 2023-09-23T18:07:06.511 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-09-23T18:07:06.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:06 smithi173 bash[27694]: audit 2023-09-23T18:07:05.027349+0000 mon.a (mon.0) 237 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]': finished 2023-09-23T18:07:06.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:06 smithi173 bash[27694]: cluster 2023-09-23T18:07:05.027465+0000 mon.a (mon.0) 238 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-09-23T18:07:06.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:06 smithi173 bash[27694]: audit 2023-09-23T18:07:05.027923+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:07:06.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:06 smithi173 bash[27694]: audit 2023-09-23T18:07:05.037094+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:07:06.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:06 smithi173 bash[27694]: cluster 2023-09-23T18:07:05.095396+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T18:07:06.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:06 smithi173 bash[27694]: audit 2023-09-23T18:07:05.766090+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:06.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:06 smithi173 bash[27694]: audit 2023-09-23T18:07:05.767023+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:07:06.704 INFO:journalctl@ceph.osd.0.smithi098.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:07:07.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:07 smithi098 bash[27349]: cluster 2023-09-23T18:07:04.739137+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:07:07.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:07 smithi098 bash[27349]: cluster 2023-09-23T18:07:04.739238+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:07:07.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:07 smithi098 bash[27349]: audit 2023-09-23T18:07:06.034895+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:07:07.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:07 smithi098 bash[27349]: cluster 2023-09-23T18:07:06.047186+0000 mon.a (mon.0) 244 : cluster [INF] osd.0 [v2:172.21.15.98:6802/1147685383,v1:172.21.15.98:6803/1147685383] boot 2023-09-23T18:07:07.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:07 smithi098 bash[27349]: cluster 2023-09-23T18:07:06.047270+0000 mon.a (mon.0) 245 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-09-23T18:07:07.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:07 smithi098 bash[27349]: audit 2023-09-23T18:07:06.047449+0000 mon.a (mon.0) 246 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:07:07.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:07 smithi098 bash[33599]: cluster 2023-09-23T18:07:04.739137+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:07:07.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:07 smithi098 bash[33599]: cluster 2023-09-23T18:07:04.739238+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:07:07.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:07 smithi098 bash[33599]: audit 2023-09-23T18:07:06.034895+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:07:07.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:07 smithi098 bash[33599]: cluster 2023-09-23T18:07:06.047186+0000 mon.a (mon.0) 244 : cluster [INF] osd.0 [v2:172.21.15.98:6802/1147685383,v1:172.21.15.98:6803/1147685383] boot 2023-09-23T18:07:07.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:07 smithi098 bash[33599]: cluster 2023-09-23T18:07:06.047270+0000 mon.a (mon.0) 245 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-09-23T18:07:07.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:07 smithi098 bash[33599]: audit 2023-09-23T18:07:06.047449+0000 mon.a (mon.0) 246 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:07:07.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:07 smithi173 bash[27694]: cluster 2023-09-23T18:07:04.739137+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:07:07.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:07 smithi173 bash[27694]: cluster 2023-09-23T18:07:04.739238+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:07:07.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:07 smithi173 bash[27694]: audit 2023-09-23T18:07:06.034895+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:07:07.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:07 smithi173 bash[27694]: cluster 2023-09-23T18:07:06.047186+0000 mon.a (mon.0) 244 : cluster [INF] osd.0 [v2:172.21.15.98:6802/1147685383,v1:172.21.15.98:6803/1147685383] boot 2023-09-23T18:07:07.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:07 smithi173 bash[27694]: cluster 2023-09-23T18:07:06.047270+0000 mon.a (mon.0) 245 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-09-23T18:07:07.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:07 smithi173 bash[27694]: audit 2023-09-23T18:07:06.047449+0000 mon.a (mon.0) 246 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:07:08.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:08 smithi098 bash[27349]: cluster 2023-09-23T18:07:07.052350+0000 mon.a (mon.0) 247 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-09-23T18:07:08.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:08 smithi098 bash[27349]: cluster 2023-09-23T18:07:07.095955+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:08.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:08 smithi098 bash[33599]: cluster 2023-09-23T18:07:07.052350+0000 mon.a (mon.0) 247 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-09-23T18:07:08.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:08 smithi098 bash[33599]: cluster 2023-09-23T18:07:07.095955+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:08.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:08 smithi173 bash[27694]: cluster 2023-09-23T18:07:07.052350+0000 mon.a (mon.0) 247 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-09-23T18:07:08.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:08 smithi173 bash[27694]: cluster 2023-09-23T18:07:07.095955+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:10.007 INFO:teuthology.orchestra.run.smithi098.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-09-23T18:07:10.007 INFO:teuthology.orchestra.run.smithi098.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-09-23T18:07:10.007 INFO:teuthology.orchestra.run.smithi098.stderr: stderr: 10+0 records in 2023-09-23T18:07:10.007 INFO:teuthology.orchestra.run.smithi098.stderr:10+0 records out 2023-09-23T18:07:10.007 INFO:teuthology.orchestra.run.smithi098.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0170038 s, 617 MB/s 2023-09-23T18:07:10.007 INFO:teuthology.orchestra.run.smithi098.stderr:--> Zapping successful for: 2023-09-23T18:07:10.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:10 smithi098 bash[27349]: cluster 2023-09-23T18:07:09.096350+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:10.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:10 smithi098 bash[33599]: cluster 2023-09-23T18:07:09.096350+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:10.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:10 smithi173 bash[27694]: cluster 2023-09-23T18:07:09.096350+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:10.795 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi098:vg_nvme/lv_3 2023-09-23T18:07:12.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:12 smithi098 bash[27349]: cluster 2023-09-23T18:07:11.096599+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:12.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:12 smithi098 bash[33599]: cluster 2023-09-23T18:07:11.096599+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:12.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:12 smithi173 bash[27694]: cluster 2023-09-23T18:07:11.096599+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:14.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:14 smithi098 bash[27349]: cluster 2023-09-23T18:07:13.096976+0000 mgr.y (mgr.14146) 83 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:14.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:14 smithi098 bash[33599]: cluster 2023-09-23T18:07:13.096976+0000 mgr.y (mgr.14146) 83 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:14.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:14 smithi173 bash[27694]: cluster 2023-09-23T18:07:13.096976+0000 mgr.y (mgr.14146) 83 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:16.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:16 smithi098 bash[27349]: cluster 2023-09-23T18:07:15.097477+0000 mgr.y (mgr.14146) 84 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:16.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:16 smithi098 bash[27349]: audit 2023-09-23T18:07:15.562202+0000 mon.a (mon.0) 248 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:07:16.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:16 smithi098 bash[27349]: audit 2023-09-23T18:07:15.566849+0000 mon.a (mon.0) 249 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:07:16.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:16 smithi098 bash[27349]: audit 2023-09-23T18:07:15.567753+0000 mon.a (mon.0) 250 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:07:16.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:16 smithi098 bash[33599]: cluster 2023-09-23T18:07:15.097477+0000 mgr.y (mgr.14146) 84 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:16.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:16 smithi098 bash[33599]: audit 2023-09-23T18:07:15.562202+0000 mon.a (mon.0) 248 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:07:16.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:16 smithi098 bash[33599]: audit 2023-09-23T18:07:15.566849+0000 mon.a (mon.0) 249 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:07:16.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:16 smithi098 bash[33599]: audit 2023-09-23T18:07:15.567753+0000 mon.a (mon.0) 250 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:07:16.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:16 smithi173 bash[27694]: cluster 2023-09-23T18:07:15.097477+0000 mgr.y (mgr.14146) 84 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:16.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:16 smithi173 bash[27694]: audit 2023-09-23T18:07:15.562202+0000 mon.a (mon.0) 248 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:07:16.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:16 smithi173 bash[27694]: audit 2023-09-23T18:07:15.566849+0000 mon.a (mon.0) 249 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:07:16.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:16 smithi173 bash[27694]: audit 2023-09-23T18:07:15.567753+0000 mon.a (mon.0) 250 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:07:17.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:17 smithi098 bash[27349]: audit 2023-09-23T18:07:15.559347+0000 mgr.y (mgr.14146) 85 : audit [DBG] from='client.24149 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi098:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:07:17.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:17 smithi098 bash[27349]: audit 2023-09-23T18:07:16.798147+0000 mon.a (mon.0) 251 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:17.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:17 smithi098 bash[27349]: audit 2023-09-23T18:07:16.799535+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:07:17.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:17 smithi098 bash[27349]: audit 2023-09-23T18:07:16.807215+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:17.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:17 smithi098 bash[27349]: audit 2023-09-23T18:07:16.808812+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:07:17.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:17 smithi098 bash[27349]: audit 2023-09-23T18:07:16.810065+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:07:17.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:17 smithi098 bash[27349]: audit 2023-09-23T18:07:16.818408+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:17.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:17 smithi098 bash[33599]: audit 2023-09-23T18:07:15.559347+0000 mgr.y (mgr.14146) 85 : audit [DBG] from='client.24149 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi098:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:07:17.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:17 smithi098 bash[33599]: audit 2023-09-23T18:07:16.798147+0000 mon.a (mon.0) 251 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:17.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:17 smithi098 bash[33599]: audit 2023-09-23T18:07:16.799535+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:07:17.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:17 smithi098 bash[33599]: audit 2023-09-23T18:07:16.807215+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:17.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:17 smithi098 bash[33599]: audit 2023-09-23T18:07:16.808812+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:07:17.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:17 smithi098 bash[33599]: audit 2023-09-23T18:07:16.810065+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:07:17.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:17 smithi098 bash[33599]: audit 2023-09-23T18:07:16.818408+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:17.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:17 smithi173 bash[27694]: audit 2023-09-23T18:07:15.559347+0000 mgr.y (mgr.14146) 85 : audit [DBG] from='client.24149 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi098:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:07:17.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:17 smithi173 bash[27694]: audit 2023-09-23T18:07:16.798147+0000 mon.a (mon.0) 251 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:17.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:17 smithi173 bash[27694]: audit 2023-09-23T18:07:16.799535+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:07:17.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:17 smithi173 bash[27694]: audit 2023-09-23T18:07:16.807215+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:17.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:17 smithi173 bash[27694]: audit 2023-09-23T18:07:16.808812+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:07:17.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:17 smithi173 bash[27694]: audit 2023-09-23T18:07:16.810065+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:07:17.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:17 smithi173 bash[27694]: audit 2023-09-23T18:07:16.818408+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:18.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:18 smithi098 bash[27349]: cephadm 2023-09-23T18:07:16.789591+0000 mgr.y (mgr.14146) 86 : cephadm [INF] Detected new or changed devices on smithi098 2023-09-23T18:07:18.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:18 smithi098 bash[27349]: cephadm 2023-09-23T18:07:16.800373+0000 mgr.y (mgr.14146) 87 : cephadm [INF] Adjusting osd_memory_target on smithi098 to 16241M 2023-09-23T18:07:18.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:18 smithi098 bash[27349]: cluster 2023-09-23T18:07:17.097953+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:18.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:18 smithi098 bash[33599]: cephadm 2023-09-23T18:07:16.789591+0000 mgr.y (mgr.14146) 86 : cephadm [INF] Detected new or changed devices on smithi098 2023-09-23T18:07:18.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:18 smithi098 bash[33599]: cephadm 2023-09-23T18:07:16.800373+0000 mgr.y (mgr.14146) 87 : cephadm [INF] Adjusting osd_memory_target on smithi098 to 16241M 2023-09-23T18:07:18.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:18 smithi098 bash[33599]: cluster 2023-09-23T18:07:17.097953+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:18.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:18 smithi173 bash[27694]: cephadm 2023-09-23T18:07:16.789591+0000 mgr.y (mgr.14146) 86 : cephadm [INF] Detected new or changed devices on smithi098 2023-09-23T18:07:18.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:18 smithi173 bash[27694]: cephadm 2023-09-23T18:07:16.800373+0000 mgr.y (mgr.14146) 87 : cephadm [INF] Adjusting osd_memory_target on smithi098 to 16241M 2023-09-23T18:07:18.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:18 smithi173 bash[27694]: cluster 2023-09-23T18:07:17.097953+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:20.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:20 smithi098 bash[27349]: cluster 2023-09-23T18:07:19.098524+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:20.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:20 smithi098 bash[33599]: cluster 2023-09-23T18:07:19.098524+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:20.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:20 smithi173 bash[27694]: cluster 2023-09-23T18:07:19.098524+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:22.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:22 smithi098 bash[27349]: cluster 2023-09-23T18:07:21.098937+0000 mgr.y (mgr.14146) 90 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:22.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:22 smithi098 bash[33599]: cluster 2023-09-23T18:07:21.098937+0000 mgr.y (mgr.14146) 90 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:22.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:22 smithi173 bash[27694]: cluster 2023-09-23T18:07:21.098937+0000 mgr.y (mgr.14146) 90 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:24.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:24 smithi098 bash[33599]: cluster 2023-09-23T18:07:23.099378+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:24.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:24 smithi098 bash[33599]: audit 2023-09-23T18:07:23.789337+0000 mon.c (mon.2) 4 : audit [INF] from='client.? 172.21.15.98:0/661112536' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5d777d1f-3bd3-49ba-b0de-0ac532ee3934"}]: dispatch 2023-09-23T18:07:24.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:24 smithi098 bash[33599]: audit 2023-09-23T18:07:23.790113+0000 mon.a (mon.0) 257 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5d777d1f-3bd3-49ba-b0de-0ac532ee3934"}]: dispatch 2023-09-23T18:07:24.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:24 smithi098 bash[33599]: audit 2023-09-23T18:07:23.799264+0000 mon.a (mon.0) 258 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5d777d1f-3bd3-49ba-b0de-0ac532ee3934"}]': finished 2023-09-23T18:07:24.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:24 smithi098 bash[33599]: cluster 2023-09-23T18:07:23.799318+0000 mon.a (mon.0) 259 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-09-23T18:07:24.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:24 smithi098 bash[33599]: audit 2023-09-23T18:07:23.799580+0000 mon.a (mon.0) 260 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:07:24.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:24 smithi098 bash[27349]: cluster 2023-09-23T18:07:23.099378+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:24.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:24 smithi098 bash[27349]: audit 2023-09-23T18:07:23.789337+0000 mon.c (mon.2) 4 : audit [INF] from='client.? 172.21.15.98:0/661112536' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5d777d1f-3bd3-49ba-b0de-0ac532ee3934"}]: dispatch 2023-09-23T18:07:24.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:24 smithi098 bash[27349]: audit 2023-09-23T18:07:23.790113+0000 mon.a (mon.0) 257 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5d777d1f-3bd3-49ba-b0de-0ac532ee3934"}]: dispatch 2023-09-23T18:07:24.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:24 smithi098 bash[27349]: audit 2023-09-23T18:07:23.799264+0000 mon.a (mon.0) 258 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5d777d1f-3bd3-49ba-b0de-0ac532ee3934"}]': finished 2023-09-23T18:07:24.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:24 smithi098 bash[27349]: cluster 2023-09-23T18:07:23.799318+0000 mon.a (mon.0) 259 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-09-23T18:07:24.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:24 smithi098 bash[27349]: audit 2023-09-23T18:07:23.799580+0000 mon.a (mon.0) 260 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:07:24.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:24 smithi173 bash[27694]: cluster 2023-09-23T18:07:23.099378+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:24.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:24 smithi173 bash[27694]: audit 2023-09-23T18:07:23.789337+0000 mon.c (mon.2) 4 : audit [INF] from='client.? 172.21.15.98:0/661112536' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5d777d1f-3bd3-49ba-b0de-0ac532ee3934"}]: dispatch 2023-09-23T18:07:24.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:24 smithi173 bash[27694]: audit 2023-09-23T18:07:23.790113+0000 mon.a (mon.0) 257 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5d777d1f-3bd3-49ba-b0de-0ac532ee3934"}]: dispatch 2023-09-23T18:07:24.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:24 smithi173 bash[27694]: audit 2023-09-23T18:07:23.799264+0000 mon.a (mon.0) 258 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5d777d1f-3bd3-49ba-b0de-0ac532ee3934"}]': finished 2023-09-23T18:07:24.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:24 smithi173 bash[27694]: cluster 2023-09-23T18:07:23.799318+0000 mon.a (mon.0) 259 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-09-23T18:07:24.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:24 smithi173 bash[27694]: audit 2023-09-23T18:07:23.799580+0000 mon.a (mon.0) 260 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:07:25.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:25 smithi098 bash[27349]: audit 2023-09-23T18:07:24.543524+0000 mon.c (mon.2) 5 : audit [DBG] from='client.? 172.21.15.98:0/1658865774' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:07:25.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:25 smithi098 bash[33599]: audit 2023-09-23T18:07:24.543524+0000 mon.c (mon.2) 5 : audit [DBG] from='client.? 172.21.15.98:0/1658865774' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:07:25.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:25 smithi173 bash[27694]: audit 2023-09-23T18:07:24.543524+0000 mon.c (mon.2) 5 : audit [DBG] from='client.? 172.21.15.98:0/1658865774' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:07:26.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:26 smithi098 bash[27349]: cluster 2023-09-23T18:07:25.099865+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:26.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:26 smithi098 bash[33599]: cluster 2023-09-23T18:07:25.099865+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:26.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:26 smithi173 bash[27694]: cluster 2023-09-23T18:07:25.099865+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:28.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:28 smithi098 bash[27349]: cluster 2023-09-23T18:07:27.100368+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:28.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:28 smithi098 bash[33599]: cluster 2023-09-23T18:07:27.100368+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:28.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:28 smithi173 bash[27694]: cluster 2023-09-23T18:07:27.100368+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:30.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:30 smithi098 bash[27349]: cluster 2023-09-23T18:07:29.100853+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:30.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:30 smithi098 bash[33599]: cluster 2023-09-23T18:07:29.100853+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:30.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:30 smithi173 bash[27694]: cluster 2023-09-23T18:07:29.100853+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:32.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:32 smithi098 bash[27349]: cluster 2023-09-23T18:07:31.101306+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:32.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:32 smithi098 bash[33599]: cluster 2023-09-23T18:07:31.101306+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:32.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:32 smithi173 bash[27694]: cluster 2023-09-23T18:07:31.101306+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:34.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:34 smithi098 bash[33599]: cluster 2023-09-23T18:07:33.101893+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:34.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:34 smithi098 bash[27349]: cluster 2023-09-23T18:07:33.101893+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:34.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:34 smithi173 bash[27694]: cluster 2023-09-23T18:07:33.101893+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:36.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:36 smithi098 bash[27349]: cluster 2023-09-23T18:07:35.102453+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:36.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:36 smithi098 bash[33599]: cluster 2023-09-23T18:07:35.102453+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:36.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:36 smithi173 bash[27694]: cluster 2023-09-23T18:07:35.102453+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:37.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:37 smithi098 bash[27349]: audit 2023-09-23T18:07:36.497738+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T18:07:37.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:37 smithi098 bash[27349]: audit 2023-09-23T18:07:36.499320+0000 mon.a (mon.0) 262 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:07:37.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:37 smithi098 bash[33599]: audit 2023-09-23T18:07:36.497738+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T18:07:37.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:37 smithi098 bash[33599]: audit 2023-09-23T18:07:36.499320+0000 mon.a (mon.0) 262 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:07:37.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:37 smithi173 bash[27694]: audit 2023-09-23T18:07:36.497738+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T18:07:37.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:37 smithi173 bash[27694]: audit 2023-09-23T18:07:36.499320+0000 mon.a (mon.0) 262 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:07:38.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:38 smithi098 bash[27349]: cephadm 2023-09-23T18:07:36.500464+0000 mgr.y (mgr.14146) 98 : cephadm [INF] Deploying daemon osd.1 on smithi098 2023-09-23T18:07:38.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:38 smithi098 bash[27349]: cluster 2023-09-23T18:07:37.102904+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:38.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:38 smithi098 bash[33599]: cephadm 2023-09-23T18:07:36.500464+0000 mgr.y (mgr.14146) 98 : cephadm [INF] Deploying daemon osd.1 on smithi098 2023-09-23T18:07:38.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:38 smithi098 bash[33599]: cluster 2023-09-23T18:07:37.102904+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:38.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:38 smithi173 bash[27694]: cephadm 2023-09-23T18:07:36.500464+0000 mgr.y (mgr.14146) 98 : cephadm [INF] Deploying daemon osd.1 on smithi098 2023-09-23T18:07:38.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:38 smithi173 bash[27694]: cluster 2023-09-23T18:07:37.102904+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:40.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:40 smithi098 bash[27349]: cluster 2023-09-23T18:07:39.103302+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:40.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:40 smithi098 bash[33599]: cluster 2023-09-23T18:07:39.103302+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:40.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:40 smithi173 bash[27694]: cluster 2023-09-23T18:07:39.103302+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:42.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:41 smithi098 bash[27349]: audit 2023-09-23T18:07:40.891997+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:42.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:41 smithi098 bash[27349]: audit 2023-09-23T18:07:40.899006+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:07:42.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:41 smithi098 bash[27349]: cluster 2023-09-23T18:07:41.103657+0000 mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:42.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:41 smithi098 bash[33599]: audit 2023-09-23T18:07:40.891997+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:42.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:41 smithi098 bash[33599]: audit 2023-09-23T18:07:40.899006+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:07:42.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:41 smithi098 bash[33599]: cluster 2023-09-23T18:07:41.103657+0000 mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:42.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:41 smithi173 bash[27694]: audit 2023-09-23T18:07:40.891997+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:42.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:41 smithi173 bash[27694]: audit 2023-09-23T18:07:40.899006+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:07:42.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:41 smithi173 bash[27694]: cluster 2023-09-23T18:07:41.103657+0000 mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:44.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:44 smithi098 bash[27349]: cluster 2023-09-23T18:07:43.104000+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:44.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:44 smithi098 bash[33599]: cluster 2023-09-23T18:07:43.104000+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:44.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:44 smithi173 bash[27694]: cluster 2023-09-23T18:07:43.104000+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:46.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:46 smithi098 bash[27349]: cluster 2023-09-23T18:07:45.104583+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:46.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:46 smithi098 bash[33599]: cluster 2023-09-23T18:07:45.104583+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:46.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:46 smithi173 bash[27694]: cluster 2023-09-23T18:07:45.104583+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:47.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:47 smithi098 bash[33599]: audit 2023-09-23T18:07:46.157711+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:47.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:47 smithi098 bash[33599]: audit 2023-09-23T18:07:46.400329+0000 mon.a (mon.0) 266 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:07:47.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:47 smithi098 bash[33599]: audit 2023-09-23T18:07:46.401329+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:07:47.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:47 smithi098 bash[33599]: audit 2023-09-23T18:07:46.408828+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:47.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:47 smithi098 bash[27349]: audit 2023-09-23T18:07:46.157711+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:47.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:47 smithi098 bash[27349]: audit 2023-09-23T18:07:46.400329+0000 mon.a (mon.0) 266 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:07:47.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:47 smithi098 bash[27349]: audit 2023-09-23T18:07:46.401329+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:07:47.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:47 smithi098 bash[27349]: audit 2023-09-23T18:07:46.408828+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:47.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:47 smithi173 bash[27694]: audit 2023-09-23T18:07:46.157711+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:47.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:47 smithi173 bash[27694]: audit 2023-09-23T18:07:46.400329+0000 mon.a (mon.0) 266 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:07:47.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:47 smithi173 bash[27694]: audit 2023-09-23T18:07:46.401329+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:07:47.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:47 smithi173 bash[27694]: audit 2023-09-23T18:07:46.408828+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:48.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:48 smithi098 bash[27349]: cluster 2023-09-23T18:07:47.105009+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:48.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:48 smithi098 bash[27349]: audit 2023-09-23T18:07:47.595635+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.98:6810/3945616078,v1:172.21.15.98:6811/3945616078]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T18:07:48.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:48 smithi098 bash[27349]: audit 2023-09-23T18:07:47.596221+0000 mon.a (mon.0) 269 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T18:07:48.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:48 smithi098 bash[33599]: cluster 2023-09-23T18:07:47.105009+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:48.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:48 smithi098 bash[33599]: audit 2023-09-23T18:07:47.595635+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.98:6810/3945616078,v1:172.21.15.98:6811/3945616078]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T18:07:48.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:48 smithi098 bash[33599]: audit 2023-09-23T18:07:47.596221+0000 mon.a (mon.0) 269 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T18:07:48.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:48 smithi173 bash[27694]: cluster 2023-09-23T18:07:47.105009+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:48.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:48 smithi173 bash[27694]: audit 2023-09-23T18:07:47.595635+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.98:6810/3945616078,v1:172.21.15.98:6811/3945616078]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T18:07:48.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:48 smithi173 bash[27694]: audit 2023-09-23T18:07:47.596221+0000 mon.a (mon.0) 269 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T18:07:49.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:49 smithi098 bash[33599]: audit 2023-09-23T18:07:48.411964+0000 mon.a (mon.0) 270 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T18:07:49.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:49 smithi098 bash[33599]: cluster 2023-09-23T18:07:48.412063+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-09-23T18:07:49.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:49 smithi098 bash[33599]: audit 2023-09-23T18:07:48.412330+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:07:49.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:49 smithi098 bash[33599]: audit 2023-09-23T18:07:48.415335+0000 mon.c (mon.2) 7 : audit [INF] from='osd.1 [v2:172.21.15.98:6810/3945616078,v1:172.21.15.98:6811/3945616078]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:07:49.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:49 smithi098 bash[33599]: audit 2023-09-23T18:07:48.415978+0000 mon.a (mon.0) 273 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:07:49.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:49 smithi098 bash[33599]: cluster 2023-09-23T18:07:49.105444+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:49.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:49 smithi098 bash[27349]: audit 2023-09-23T18:07:48.411964+0000 mon.a (mon.0) 270 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T18:07:49.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:49 smithi098 bash[27349]: cluster 2023-09-23T18:07:48.412063+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-09-23T18:07:49.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:49 smithi098 bash[27349]: audit 2023-09-23T18:07:48.412330+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:07:49.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:49 smithi098 bash[27349]: audit 2023-09-23T18:07:48.415335+0000 mon.c (mon.2) 7 : audit [INF] from='osd.1 [v2:172.21.15.98:6810/3945616078,v1:172.21.15.98:6811/3945616078]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:07:49.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:49 smithi098 bash[27349]: audit 2023-09-23T18:07:48.415978+0000 mon.a (mon.0) 273 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:07:49.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:49 smithi098 bash[27349]: cluster 2023-09-23T18:07:49.105444+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:49.776 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:49 smithi173 bash[27694]: audit 2023-09-23T18:07:48.411964+0000 mon.a (mon.0) 270 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T18:07:49.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:49 smithi173 bash[27694]: cluster 2023-09-23T18:07:48.412063+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-09-23T18:07:49.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:49 smithi173 bash[27694]: audit 2023-09-23T18:07:48.412330+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:07:49.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:49 smithi173 bash[27694]: audit 2023-09-23T18:07:48.415335+0000 mon.c (mon.2) 7 : audit [INF] from='osd.1 [v2:172.21.15.98:6810/3945616078,v1:172.21.15.98:6811/3945616078]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:07:49.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:49 smithi173 bash[27694]: audit 2023-09-23T18:07:48.415978+0000 mon.a (mon.0) 273 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:07:49.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:49 smithi173 bash[27694]: cluster 2023-09-23T18:07:49.105444+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:50.120 INFO:teuthology.orchestra.run.smithi098.stdout:Created osd(s) 1 on host 'smithi098' 2023-09-23T18:07:50.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:50 smithi098 bash[27349]: audit 2023-09-23T18:07:49.416255+0000 mon.a (mon.0) 274 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]': finished 2023-09-23T18:07:50.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:50 smithi098 bash[27349]: cluster 2023-09-23T18:07:49.416558+0000 mon.a (mon.0) 275 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-09-23T18:07:50.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:50 smithi098 bash[27349]: audit 2023-09-23T18:07:49.417239+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:07:50.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:50 smithi098 bash[27349]: audit 2023-09-23T18:07:49.423888+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:07:50.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:50 smithi098 bash[27349]: audit 2023-09-23T18:07:50.116868+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:50.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:50 smithi098 bash[27349]: audit 2023-09-23T18:07:50.135324+0000 mon.a (mon.0) 279 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:07:50.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:50 smithi098 bash[33599]: audit 2023-09-23T18:07:49.416255+0000 mon.a (mon.0) 274 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]': finished 2023-09-23T18:07:50.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:50 smithi098 bash[33599]: cluster 2023-09-23T18:07:49.416558+0000 mon.a (mon.0) 275 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-09-23T18:07:50.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:50 smithi098 bash[33599]: audit 2023-09-23T18:07:49.417239+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:07:50.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:50 smithi098 bash[33599]: audit 2023-09-23T18:07:49.423888+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:07:50.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:50 smithi098 bash[33599]: audit 2023-09-23T18:07:50.116868+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:50.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:50 smithi098 bash[33599]: audit 2023-09-23T18:07:50.135324+0000 mon.a (mon.0) 279 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:07:50.776 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:50 smithi173 bash[27694]: audit 2023-09-23T18:07:49.416255+0000 mon.a (mon.0) 274 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]': finished 2023-09-23T18:07:50.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:50 smithi173 bash[27694]: cluster 2023-09-23T18:07:49.416558+0000 mon.a (mon.0) 275 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-09-23T18:07:50.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:50 smithi173 bash[27694]: audit 2023-09-23T18:07:49.417239+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:07:50.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:50 smithi173 bash[27694]: audit 2023-09-23T18:07:49.423888+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:07:50.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:50 smithi173 bash[27694]: audit 2023-09-23T18:07:50.116868+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:07:50.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:50 smithi173 bash[27694]: audit 2023-09-23T18:07:50.135324+0000 mon.a (mon.0) 279 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:07:50.794 DEBUG:teuthology.orchestra.run.smithi098:osd.1> sudo journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.1.service 2023-09-23T18:07:50.797 INFO:tasks.cephadm:Deploying osd.2 on smithi098 with /dev/vg_nvme/lv_2... 2023-09-23T18:07:50.797 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-09-23T18:07:51.015 INFO:journalctl@ceph.osd.1.smithi098.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:07:51.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:51 smithi173 bash[27694]: cluster 2023-09-23T18:07:48.583095+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:07:51.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:51 smithi173 bash[27694]: cluster 2023-09-23T18:07:48.583191+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:07:51.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:51 smithi173 bash[27694]: cluster 2023-09-23T18:07:50.427070+0000 mon.a (mon.0) 280 : cluster [INF] osd.1 [v2:172.21.15.98:6810/3945616078,v1:172.21.15.98:6811/3945616078] boot 2023-09-23T18:07:51.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:51 smithi173 bash[27694]: cluster 2023-09-23T18:07:50.427153+0000 mon.a (mon.0) 281 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-09-23T18:07:51.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:51 smithi173 bash[27694]: audit 2023-09-23T18:07:50.427897+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:07:51.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:51 smithi173 bash[27694]: cluster 2023-09-23T18:07:51.105932+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:51.873 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:51 smithi098 bash[27349]: cluster 2023-09-23T18:07:48.583095+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:07:51.873 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:51 smithi098 bash[27349]: cluster 2023-09-23T18:07:48.583191+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:07:51.873 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:51 smithi098 bash[27349]: cluster 2023-09-23T18:07:50.427070+0000 mon.a (mon.0) 280 : cluster [INF] osd.1 [v2:172.21.15.98:6810/3945616078,v1:172.21.15.98:6811/3945616078] boot 2023-09-23T18:07:51.873 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:51 smithi098 bash[27349]: cluster 2023-09-23T18:07:50.427153+0000 mon.a (mon.0) 281 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-09-23T18:07:51.873 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:51 smithi098 bash[27349]: audit 2023-09-23T18:07:50.427897+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:07:51.873 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:51 smithi098 bash[27349]: cluster 2023-09-23T18:07:51.105932+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:51.874 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:51 smithi098 bash[33599]: cluster 2023-09-23T18:07:48.583095+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:07:51.874 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:51 smithi098 bash[33599]: cluster 2023-09-23T18:07:48.583191+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:07:51.874 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:51 smithi098 bash[33599]: cluster 2023-09-23T18:07:50.427070+0000 mon.a (mon.0) 280 : cluster [INF] osd.1 [v2:172.21.15.98:6810/3945616078,v1:172.21.15.98:6811/3945616078] boot 2023-09-23T18:07:51.874 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:51 smithi098 bash[33599]: cluster 2023-09-23T18:07:50.427153+0000 mon.a (mon.0) 281 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-09-23T18:07:51.874 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:51 smithi098 bash[33599]: audit 2023-09-23T18:07:50.427897+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:07:51.874 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:51 smithi098 bash[33599]: cluster 2023-09-23T18:07:51.105932+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T18:07:52.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:52 smithi098 bash[27349]: cluster 2023-09-23T18:07:51.446280+0000 mon.a (mon.0) 283 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-09-23T18:07:52.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:52 smithi098 bash[33599]: cluster 2023-09-23T18:07:51.446280+0000 mon.a (mon.0) 283 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-09-23T18:07:52.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:52 smithi173 bash[27694]: cluster 2023-09-23T18:07:51.446280+0000 mon.a (mon.0) 283 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-09-23T18:07:54.408 INFO:teuthology.orchestra.run.smithi098.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-09-23T18:07:54.408 INFO:teuthology.orchestra.run.smithi098.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-09-23T18:07:54.408 INFO:teuthology.orchestra.run.smithi098.stderr: stderr: 10+0 records in 2023-09-23T18:07:54.408 INFO:teuthology.orchestra.run.smithi098.stderr:10+0 records out 2023-09-23T18:07:54.408 INFO:teuthology.orchestra.run.smithi098.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0155711 s, 673 MB/s 2023-09-23T18:07:54.409 INFO:teuthology.orchestra.run.smithi098.stderr:--> Zapping successful for: 2023-09-23T18:07:54.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:54 smithi098 bash[33599]: cluster 2023-09-23T18:07:53.106429+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:07:54.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:54 smithi098 bash[27349]: cluster 2023-09-23T18:07:53.106429+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:07:54.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:54 smithi173 bash[27694]: cluster 2023-09-23T18:07:53.106429+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:07:55.154 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi098:vg_nvme/lv_2 2023-09-23T18:07:56.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:56 smithi098 bash[27349]: cluster 2023-09-23T18:07:55.106974+0000 mgr.y (mgr.14146) 108 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:07:56.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:56 smithi098 bash[33599]: cluster 2023-09-23T18:07:55.106974+0000 mgr.y (mgr.14146) 108 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:07:56.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:56 smithi173 bash[27694]: cluster 2023-09-23T18:07:55.106974+0000 mgr.y (mgr.14146) 108 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:07:58.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:07:58 smithi098 bash[27349]: cluster 2023-09-23T18:07:57.107471+0000 mgr.y (mgr.14146) 109 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:07:58.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:07:58 smithi098 bash[33599]: cluster 2023-09-23T18:07:57.107471+0000 mgr.y (mgr.14146) 109 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:07:58.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:07:58 smithi173 bash[27694]: cluster 2023-09-23T18:07:57.107471+0000 mgr.y (mgr.14146) 109 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:00.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:00 smithi098 bash[27349]: cluster 2023-09-23T18:07:59.107979+0000 mgr.y (mgr.14146) 110 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:00.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:00 smithi098 bash[27349]: audit 2023-09-23T18:07:59.853927+0000 mon.a (mon.0) 284 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:08:00.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:00 smithi098 bash[27349]: audit 2023-09-23T18:07:59.857178+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:08:00.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:00 smithi098 bash[27349]: audit 2023-09-23T18:07:59.858273+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:08:00.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:00 smithi098 bash[33599]: cluster 2023-09-23T18:07:59.107979+0000 mgr.y (mgr.14146) 110 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:00.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:00 smithi098 bash[33599]: audit 2023-09-23T18:07:59.853927+0000 mon.a (mon.0) 284 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:08:00.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:00 smithi098 bash[33599]: audit 2023-09-23T18:07:59.857178+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:08:00.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:00 smithi098 bash[33599]: audit 2023-09-23T18:07:59.858273+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:08:00.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:00 smithi173 bash[27694]: cluster 2023-09-23T18:07:59.107979+0000 mgr.y (mgr.14146) 110 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:00.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:00 smithi173 bash[27694]: audit 2023-09-23T18:07:59.853927+0000 mon.a (mon.0) 284 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:08:00.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:00 smithi173 bash[27694]: audit 2023-09-23T18:07:59.857178+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:08:00.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:00 smithi173 bash[27694]: audit 2023-09-23T18:07:59.858273+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:08:01.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:01 smithi098 bash[27349]: audit 2023-09-23T18:07:59.850787+0000 mgr.y (mgr.14146) 111 : audit [DBG] from='client.14262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi098:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:08:01.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:01 smithi098 bash[33599]: audit 2023-09-23T18:07:59.850787+0000 mgr.y (mgr.14146) 111 : audit [DBG] from='client.14262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi098:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:08:01.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:01 smithi173 bash[27694]: audit 2023-09-23T18:07:59.850787+0000 mgr.y (mgr.14146) 111 : audit [DBG] from='client.14262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi098:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:08:02.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:02 smithi173 bash[27694]: cluster 2023-09-23T18:08:01.108339+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:02.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:02 smithi173 bash[27694]: cephadm 2023-09-23T18:08:01.253051+0000 mgr.y (mgr.14146) 113 : cephadm [INF] Detected new or changed devices on smithi098 2023-09-23T18:08:02.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:02 smithi173 bash[27694]: audit 2023-09-23T18:08:01.263454+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:02.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:02 smithi173 bash[27694]: audit 2023-09-23T18:08:01.264620+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:08:02.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:02 smithi173 bash[27694]: audit 2023-09-23T18:08:01.265551+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:08:02.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:02 smithi173 bash[27694]: cephadm 2023-09-23T18:08:01.266215+0000 mgr.y (mgr.14146) 114 : cephadm [INF] Adjusting osd_memory_target on smithi098 to 8120M 2023-09-23T18:08:02.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:02 smithi173 bash[27694]: audit 2023-09-23T18:08:01.271061+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:02.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:02 smithi173 bash[27694]: audit 2023-09-23T18:08:01.272616+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:08:02.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:02 smithi173 bash[27694]: audit 2023-09-23T18:08:01.273755+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:08:02.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:02 smithi173 bash[27694]: audit 2023-09-23T18:08:01.280481+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:02.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:02 smithi098 bash[27349]: cluster 2023-09-23T18:08:01.108339+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:02.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:02 smithi098 bash[27349]: cephadm 2023-09-23T18:08:01.253051+0000 mgr.y (mgr.14146) 113 : cephadm [INF] Detected new or changed devices on smithi098 2023-09-23T18:08:02.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:02 smithi098 bash[27349]: audit 2023-09-23T18:08:01.263454+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:02.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:02 smithi098 bash[27349]: audit 2023-09-23T18:08:01.264620+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:08:02.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:02 smithi098 bash[27349]: audit 2023-09-23T18:08:01.265551+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:08:02.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:02 smithi098 bash[27349]: cephadm 2023-09-23T18:08:01.266215+0000 mgr.y (mgr.14146) 114 : cephadm [INF] Adjusting osd_memory_target on smithi098 to 8120M 2023-09-23T18:08:02.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:02 smithi098 bash[27349]: audit 2023-09-23T18:08:01.271061+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:02.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:02 smithi098 bash[27349]: audit 2023-09-23T18:08:01.272616+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:08:02.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:02 smithi098 bash[27349]: audit 2023-09-23T18:08:01.273755+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:08:02.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:02 smithi098 bash[27349]: audit 2023-09-23T18:08:01.280481+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:02.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:02 smithi098 bash[33599]: cluster 2023-09-23T18:08:01.108339+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:02.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:02 smithi098 bash[33599]: cephadm 2023-09-23T18:08:01.253051+0000 mgr.y (mgr.14146) 113 : cephadm [INF] Detected new or changed devices on smithi098 2023-09-23T18:08:02.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:02 smithi098 bash[33599]: audit 2023-09-23T18:08:01.263454+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:02.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:02 smithi098 bash[33599]: audit 2023-09-23T18:08:01.264620+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:08:02.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:02 smithi098 bash[33599]: audit 2023-09-23T18:08:01.265551+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:08:02.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:02 smithi098 bash[33599]: cephadm 2023-09-23T18:08:01.266215+0000 mgr.y (mgr.14146) 114 : cephadm [INF] Adjusting osd_memory_target on smithi098 to 8120M 2023-09-23T18:08:02.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:02 smithi098 bash[33599]: audit 2023-09-23T18:08:01.271061+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:02.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:02 smithi098 bash[33599]: audit 2023-09-23T18:08:01.272616+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:08:02.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:02 smithi098 bash[33599]: audit 2023-09-23T18:08:01.273755+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:08:02.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:02 smithi098 bash[33599]: audit 2023-09-23T18:08:01.280481+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:04.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:04 smithi098 bash[27349]: cluster 2023-09-23T18:08:03.108842+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:04.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:04 smithi098 bash[33599]: cluster 2023-09-23T18:08:03.108842+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:04.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:04 smithi173 bash[27694]: cluster 2023-09-23T18:08:03.108842+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:06.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:06 smithi098 bash[27349]: cluster 2023-09-23T18:08:05.109226+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:06.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:06 smithi098 bash[33599]: cluster 2023-09-23T18:08:05.109226+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:06.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:06 smithi173 bash[27694]: cluster 2023-09-23T18:08:05.109226+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:08.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:08 smithi098 bash[33599]: cluster 2023-09-23T18:08:07.109712+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:08.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:08 smithi098 bash[27349]: cluster 2023-09-23T18:08:07.109712+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:08.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:08 smithi173 bash[27694]: cluster 2023-09-23T18:08:07.109712+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:09.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:09 smithi098 bash[27349]: audit 2023-09-23T18:08:08.325000+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.98:0/4081730457' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "63978288-ac36-4cf2-bbb7-46dc828e6ce5"}]: dispatch 2023-09-23T18:08:09.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:09 smithi098 bash[27349]: audit 2023-09-23T18:08:08.325694+0000 mon.a (mon.0) 294 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "63978288-ac36-4cf2-bbb7-46dc828e6ce5"}]: dispatch 2023-09-23T18:08:09.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:09 smithi098 bash[27349]: audit 2023-09-23T18:08:08.334097+0000 mon.a (mon.0) 295 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "63978288-ac36-4cf2-bbb7-46dc828e6ce5"}]': finished 2023-09-23T18:08:09.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:09 smithi098 bash[27349]: cluster 2023-09-23T18:08:08.334346+0000 mon.a (mon.0) 296 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-09-23T18:08:09.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:09 smithi098 bash[27349]: audit 2023-09-23T18:08:08.334537+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:08:09.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:09 smithi098 bash[27349]: audit 2023-09-23T18:08:09.036047+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.98:0/1376707798' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:08:09.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:09 smithi098 bash[33599]: audit 2023-09-23T18:08:08.325000+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.98:0/4081730457' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "63978288-ac36-4cf2-bbb7-46dc828e6ce5"}]: dispatch 2023-09-23T18:08:09.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:09 smithi098 bash[33599]: audit 2023-09-23T18:08:08.325694+0000 mon.a (mon.0) 294 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "63978288-ac36-4cf2-bbb7-46dc828e6ce5"}]: dispatch 2023-09-23T18:08:09.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:09 smithi098 bash[33599]: audit 2023-09-23T18:08:08.334097+0000 mon.a (mon.0) 295 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "63978288-ac36-4cf2-bbb7-46dc828e6ce5"}]': finished 2023-09-23T18:08:09.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:09 smithi098 bash[33599]: cluster 2023-09-23T18:08:08.334346+0000 mon.a (mon.0) 296 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-09-23T18:08:09.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:09 smithi098 bash[33599]: audit 2023-09-23T18:08:08.334537+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:08:09.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:09 smithi098 bash[33599]: audit 2023-09-23T18:08:09.036047+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.98:0/1376707798' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:08:09.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:09 smithi173 bash[27694]: audit 2023-09-23T18:08:08.325000+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.98:0/4081730457' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "63978288-ac36-4cf2-bbb7-46dc828e6ce5"}]: dispatch 2023-09-23T18:08:09.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:09 smithi173 bash[27694]: audit 2023-09-23T18:08:08.325694+0000 mon.a (mon.0) 294 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "63978288-ac36-4cf2-bbb7-46dc828e6ce5"}]: dispatch 2023-09-23T18:08:09.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:09 smithi173 bash[27694]: audit 2023-09-23T18:08:08.334097+0000 mon.a (mon.0) 295 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "63978288-ac36-4cf2-bbb7-46dc828e6ce5"}]': finished 2023-09-23T18:08:09.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:09 smithi173 bash[27694]: cluster 2023-09-23T18:08:08.334346+0000 mon.a (mon.0) 296 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-09-23T18:08:09.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:09 smithi173 bash[27694]: audit 2023-09-23T18:08:08.334537+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:08:09.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:09 smithi173 bash[27694]: audit 2023-09-23T18:08:09.036047+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.98:0/1376707798' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:08:10.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:10 smithi098 bash[27349]: cluster 2023-09-23T18:08:09.110171+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:10.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:10 smithi098 bash[33599]: cluster 2023-09-23T18:08:09.110171+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:10.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:10 smithi173 bash[27694]: cluster 2023-09-23T18:08:09.110171+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:12.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:12 smithi098 bash[27349]: cluster 2023-09-23T18:08:11.110638+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:12.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:12 smithi098 bash[33599]: cluster 2023-09-23T18:08:11.110638+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:12.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:12 smithi173 bash[27694]: cluster 2023-09-23T18:08:11.110638+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:14.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:14 smithi098 bash[27349]: cluster 2023-09-23T18:08:13.111088+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:14.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:14 smithi098 bash[33599]: cluster 2023-09-23T18:08:13.111088+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:14.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:14 smithi173 bash[27694]: cluster 2023-09-23T18:08:13.111088+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:16.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:16 smithi098 bash[27349]: cluster 2023-09-23T18:08:15.111510+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:16.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:16 smithi098 bash[33599]: cluster 2023-09-23T18:08:15.111510+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:16.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:16 smithi173 bash[27694]: cluster 2023-09-23T18:08:15.111510+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:18.365 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:18 smithi098 bash[33599]: cluster 2023-09-23T18:08:17.111990+0000 mgr.y (mgr.14146) 122 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:18.365 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:18 smithi098 bash[27349]: cluster 2023-09-23T18:08:17.111990+0000 mgr.y (mgr.14146) 122 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:18.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:18 smithi173 bash[27694]: cluster 2023-09-23T18:08:17.111990+0000 mgr.y (mgr.14146) 122 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:20.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:20 smithi098 bash[27349]: cluster 2023-09-23T18:08:19.112490+0000 mgr.y (mgr.14146) 123 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:20.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:20 smithi098 bash[33599]: cluster 2023-09-23T18:08:19.112490+0000 mgr.y (mgr.14146) 123 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:20.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:20 smithi173 bash[27694]: cluster 2023-09-23T18:08:19.112490+0000 mgr.y (mgr.14146) 123 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:21.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:21 smithi098 bash[33599]: audit 2023-09-23T18:08:20.958578+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T18:08:21.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:21 smithi098 bash[33599]: audit 2023-09-23T18:08:20.959908+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:08:21.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:21 smithi098 bash[27349]: audit 2023-09-23T18:08:20.958578+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T18:08:21.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:21 smithi098 bash[27349]: audit 2023-09-23T18:08:20.959908+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:08:21.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:21 smithi173 bash[27694]: audit 2023-09-23T18:08:20.958578+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T18:08:21.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:21 smithi173 bash[27694]: audit 2023-09-23T18:08:20.959908+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:08:22.432 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:22 smithi098 bash[27349]: cephadm 2023-09-23T18:08:20.960905+0000 mgr.y (mgr.14146) 124 : cephadm [INF] Deploying daemon osd.2 on smithi098 2023-09-23T18:08:22.432 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:22 smithi098 bash[27349]: cluster 2023-09-23T18:08:21.112959+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:22.434 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:22 smithi098 bash[33599]: cephadm 2023-09-23T18:08:20.960905+0000 mgr.y (mgr.14146) 124 : cephadm [INF] Deploying daemon osd.2 on smithi098 2023-09-23T18:08:22.434 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:22 smithi098 bash[33599]: cluster 2023-09-23T18:08:21.112959+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:22.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:22 smithi173 bash[27694]: cephadm 2023-09-23T18:08:20.960905+0000 mgr.y (mgr.14146) 124 : cephadm [INF] Deploying daemon osd.2 on smithi098 2023-09-23T18:08:22.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:22 smithi173 bash[27694]: cluster 2023-09-23T18:08:21.112959+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:24.306 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:24 smithi098 bash[27349]: cluster 2023-09-23T18:08:23.113439+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:24.309 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:24 smithi098 bash[33599]: cluster 2023-09-23T18:08:23.113439+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:24.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:24 smithi173 bash[27694]: cluster 2023-09-23T18:08:23.113439+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:26.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:26 smithi098 bash[27349]: cluster 2023-09-23T18:08:25.113837+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:26.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:26 smithi098 bash[27349]: audit 2023-09-23T18:08:25.379323+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:26.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:26 smithi098 bash[27349]: audit 2023-09-23T18:08:25.411340+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:08:26.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:26 smithi098 bash[33599]: cluster 2023-09-23T18:08:25.113837+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:26.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:26 smithi098 bash[33599]: audit 2023-09-23T18:08:25.379323+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:26.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:26 smithi098 bash[33599]: audit 2023-09-23T18:08:25.411340+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:08:26.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:26 smithi173 bash[27694]: cluster 2023-09-23T18:08:25.113837+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:26.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:26 smithi173 bash[27694]: audit 2023-09-23T18:08:25.379323+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:26.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:26 smithi173 bash[27694]: audit 2023-09-23T18:08:25.411340+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:08:28.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:28 smithi098 bash[27349]: cluster 2023-09-23T18:08:27.114445+0000 mgr.y (mgr.14146) 128 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:28.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:28 smithi098 bash[33599]: cluster 2023-09-23T18:08:27.114445+0000 mgr.y (mgr.14146) 128 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:28.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:28 smithi173 bash[27694]: cluster 2023-09-23T18:08:27.114445+0000 mgr.y (mgr.14146) 128 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:30.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:30 smithi098 bash[27349]: cluster 2023-09-23T18:08:29.115050+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v104: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:30.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:30 smithi098 bash[33599]: cluster 2023-09-23T18:08:29.115050+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v104: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:30.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:30 smithi173 bash[27694]: cluster 2023-09-23T18:08:29.115050+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v104: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:31.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:31 smithi098 bash[27349]: audit 2023-09-23T18:08:30.660787+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:31.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:31 smithi098 bash[27349]: audit 2023-09-23T18:08:30.661788+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:08:31.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:31 smithi098 bash[27349]: audit 2023-09-23T18:08:30.662840+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:08:31.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:31 smithi098 bash[27349]: audit 2023-09-23T18:08:30.670944+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:31.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:31 smithi098 bash[27349]: cluster 2023-09-23T18:08:31.115553+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v105: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:31.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:31 smithi098 bash[33599]: audit 2023-09-23T18:08:30.660787+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:31.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:31 smithi098 bash[33599]: audit 2023-09-23T18:08:30.661788+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:08:31.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:31 smithi098 bash[33599]: audit 2023-09-23T18:08:30.662840+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:08:31.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:31 smithi098 bash[33599]: audit 2023-09-23T18:08:30.670944+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:31.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:31 smithi098 bash[33599]: cluster 2023-09-23T18:08:31.115553+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v105: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:32.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:31 smithi173 bash[27694]: audit 2023-09-23T18:08:30.660787+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:32.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:31 smithi173 bash[27694]: audit 2023-09-23T18:08:30.661788+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:08:32.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:31 smithi173 bash[27694]: audit 2023-09-23T18:08:30.662840+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:08:32.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:31 smithi173 bash[27694]: audit 2023-09-23T18:08:30.670944+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:32.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:31 smithi173 bash[27694]: cluster 2023-09-23T18:08:31.115553+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v105: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:32.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:32 smithi098 bash[27349]: audit 2023-09-23T18:08:32.177321+0000 mon.a (mon.0) 306 : audit [INF] from='osd.2 [v2:172.21.15.98:6818/2338555672,v1:172.21.15.98:6819/2338555672]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T18:08:32.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:32 smithi098 bash[33599]: audit 2023-09-23T18:08:32.177321+0000 mon.a (mon.0) 306 : audit [INF] from='osd.2 [v2:172.21.15.98:6818/2338555672,v1:172.21.15.98:6819/2338555672]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T18:08:33.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:32 smithi173 bash[27694]: audit 2023-09-23T18:08:32.177321+0000 mon.a (mon.0) 306 : audit [INF] from='osd.2 [v2:172.21.15.98:6818/2338555672,v1:172.21.15.98:6819/2338555672]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T18:08:33.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:33 smithi098 bash[27349]: audit 2023-09-23T18:08:32.685702+0000 mon.a (mon.0) 307 : audit [INF] from='osd.2 [v2:172.21.15.98:6818/2338555672,v1:172.21.15.98:6819/2338555672]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T18:08:33.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:33 smithi098 bash[27349]: cluster 2023-09-23T18:08:32.685777+0000 mon.a (mon.0) 308 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-09-23T18:08:33.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:33 smithi098 bash[27349]: audit 2023-09-23T18:08:32.685929+0000 mon.a (mon.0) 309 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:08:33.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:33 smithi098 bash[27349]: audit 2023-09-23T18:08:32.686439+0000 mon.a (mon.0) 310 : audit [INF] from='osd.2 [v2:172.21.15.98:6818/2338555672,v1:172.21.15.98:6819/2338555672]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:08:33.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:33 smithi098 bash[27349]: cluster 2023-09-23T18:08:33.116079+0000 mgr.y (mgr.14146) 131 : cluster [DBG] pgmap v107: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:33.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:33 smithi098 bash[33599]: audit 2023-09-23T18:08:32.685702+0000 mon.a (mon.0) 307 : audit [INF] from='osd.2 [v2:172.21.15.98:6818/2338555672,v1:172.21.15.98:6819/2338555672]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T18:08:33.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:33 smithi098 bash[33599]: cluster 2023-09-23T18:08:32.685777+0000 mon.a (mon.0) 308 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-09-23T18:08:33.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:33 smithi098 bash[33599]: audit 2023-09-23T18:08:32.685929+0000 mon.a (mon.0) 309 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:08:33.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:33 smithi098 bash[33599]: audit 2023-09-23T18:08:32.686439+0000 mon.a (mon.0) 310 : audit [INF] from='osd.2 [v2:172.21.15.98:6818/2338555672,v1:172.21.15.98:6819/2338555672]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:08:33.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:33 smithi098 bash[33599]: cluster 2023-09-23T18:08:33.116079+0000 mgr.y (mgr.14146) 131 : cluster [DBG] pgmap v107: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:34.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:33 smithi173 bash[27694]: audit 2023-09-23T18:08:32.685702+0000 mon.a (mon.0) 307 : audit [INF] from='osd.2 [v2:172.21.15.98:6818/2338555672,v1:172.21.15.98:6819/2338555672]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T18:08:34.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:33 smithi173 bash[27694]: cluster 2023-09-23T18:08:32.685777+0000 mon.a (mon.0) 308 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-09-23T18:08:34.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:33 smithi173 bash[27694]: audit 2023-09-23T18:08:32.685929+0000 mon.a (mon.0) 309 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:08:34.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:33 smithi173 bash[27694]: audit 2023-09-23T18:08:32.686439+0000 mon.a (mon.0) 310 : audit [INF] from='osd.2 [v2:172.21.15.98:6818/2338555672,v1:172.21.15.98:6819/2338555672]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:08:34.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:33 smithi173 bash[27694]: cluster 2023-09-23T18:08:33.116079+0000 mgr.y (mgr.14146) 131 : cluster [DBG] pgmap v107: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:34.497 INFO:teuthology.orchestra.run.smithi098.stdout:Created osd(s) 2 on host 'smithi098' 2023-09-23T18:08:34.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:34 smithi098 bash[27349]: audit 2023-09-23T18:08:33.686245+0000 mon.a (mon.0) 311 : audit [INF] from='osd.2 [v2:172.21.15.98:6818/2338555672,v1:172.21.15.98:6819/2338555672]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]': finished 2023-09-23T18:08:34.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:34 smithi098 bash[27349]: cluster 2023-09-23T18:08:33.686343+0000 mon.a (mon.0) 312 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-09-23T18:08:34.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:34 smithi098 bash[27349]: audit 2023-09-23T18:08:33.686907+0000 mon.a (mon.0) 313 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:08:34.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:34 smithi098 bash[27349]: audit 2023-09-23T18:08:33.694197+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:08:34.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:34 smithi098 bash[27349]: audit 2023-09-23T18:08:34.493302+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:34.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:34 smithi098 bash[27349]: audit 2023-09-23T18:08:34.498098+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:08:34.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:34 smithi098 bash[33599]: audit 2023-09-23T18:08:33.686245+0000 mon.a (mon.0) 311 : audit [INF] from='osd.2 [v2:172.21.15.98:6818/2338555672,v1:172.21.15.98:6819/2338555672]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]': finished 2023-09-23T18:08:34.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:34 smithi098 bash[33599]: cluster 2023-09-23T18:08:33.686343+0000 mon.a (mon.0) 312 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-09-23T18:08:34.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:34 smithi098 bash[33599]: audit 2023-09-23T18:08:33.686907+0000 mon.a (mon.0) 313 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:08:34.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:34 smithi098 bash[33599]: audit 2023-09-23T18:08:33.694197+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:08:34.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:34 smithi098 bash[33599]: audit 2023-09-23T18:08:34.493302+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:34.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:34 smithi098 bash[33599]: audit 2023-09-23T18:08:34.498098+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:08:35.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:34 smithi173 bash[27694]: audit 2023-09-23T18:08:33.686245+0000 mon.a (mon.0) 311 : audit [INF] from='osd.2 [v2:172.21.15.98:6818/2338555672,v1:172.21.15.98:6819/2338555672]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]': finished 2023-09-23T18:08:35.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:34 smithi173 bash[27694]: cluster 2023-09-23T18:08:33.686343+0000 mon.a (mon.0) 312 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-09-23T18:08:35.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:34 smithi173 bash[27694]: audit 2023-09-23T18:08:33.686907+0000 mon.a (mon.0) 313 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:08:35.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:34 smithi173 bash[27694]: audit 2023-09-23T18:08:33.694197+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:08:35.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:34 smithi173 bash[27694]: audit 2023-09-23T18:08:34.493302+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:35.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:34 smithi173 bash[27694]: audit 2023-09-23T18:08:34.498098+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:08:35.213 DEBUG:teuthology.orchestra.run.smithi098:osd.2> sudo journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.2.service 2023-09-23T18:08:35.216 INFO:tasks.cephadm:Deploying osd.3 on smithi098 with /dev/vg_nvme/lv_1... 2023-09-23T18:08:35.216 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-09-23T18:08:35.354 INFO:journalctl@ceph.osd.2.smithi098.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:08:35.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:35 smithi098 bash[33599]: cluster 2023-09-23T18:08:33.173263+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:08:35.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:35 smithi098 bash[33599]: cluster 2023-09-23T18:08:33.173344+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:08:35.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:35 smithi098 bash[33599]: audit 2023-09-23T18:08:34.694385+0000 mon.a (mon.0) 317 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:08:35.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:35 smithi098 bash[33599]: cluster 2023-09-23T18:08:34.700246+0000 mon.a (mon.0) 318 : cluster [INF] osd.2 [v2:172.21.15.98:6818/2338555672,v1:172.21.15.98:6819/2338555672] boot 2023-09-23T18:08:35.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:35 smithi098 bash[33599]: cluster 2023-09-23T18:08:34.700297+0000 mon.a (mon.0) 319 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-09-23T18:08:35.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:35 smithi098 bash[33599]: audit 2023-09-23T18:08:34.700454+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:08:35.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:35 smithi098 bash[33599]: audit 2023-09-23T18:08:34.701015+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T18:08:35.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:35 smithi098 bash[33599]: cluster 2023-09-23T18:08:35.116383+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v110: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:35.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:35 smithi098 bash[27349]: cluster 2023-09-23T18:08:33.173263+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:08:35.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:35 smithi098 bash[27349]: cluster 2023-09-23T18:08:33.173344+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:08:35.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:35 smithi098 bash[27349]: audit 2023-09-23T18:08:34.694385+0000 mon.a (mon.0) 317 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:08:35.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:35 smithi098 bash[27349]: cluster 2023-09-23T18:08:34.700246+0000 mon.a (mon.0) 318 : cluster [INF] osd.2 [v2:172.21.15.98:6818/2338555672,v1:172.21.15.98:6819/2338555672] boot 2023-09-23T18:08:35.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:35 smithi098 bash[27349]: cluster 2023-09-23T18:08:34.700297+0000 mon.a (mon.0) 319 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-09-23T18:08:35.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:35 smithi098 bash[27349]: audit 2023-09-23T18:08:34.700454+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:08:35.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:35 smithi098 bash[27349]: audit 2023-09-23T18:08:34.701015+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T18:08:35.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:35 smithi098 bash[27349]: cluster 2023-09-23T18:08:35.116383+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v110: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:36.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:35 smithi173 bash[27694]: cluster 2023-09-23T18:08:33.173263+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:08:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:35 smithi173 bash[27694]: cluster 2023-09-23T18:08:33.173344+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:08:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:35 smithi173 bash[27694]: audit 2023-09-23T18:08:34.694385+0000 mon.a (mon.0) 317 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:08:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:35 smithi173 bash[27694]: cluster 2023-09-23T18:08:34.700246+0000 mon.a (mon.0) 318 : cluster [INF] osd.2 [v2:172.21.15.98:6818/2338555672,v1:172.21.15.98:6819/2338555672] boot 2023-09-23T18:08:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:35 smithi173 bash[27694]: cluster 2023-09-23T18:08:34.700297+0000 mon.a (mon.0) 319 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-09-23T18:08:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:35 smithi173 bash[27694]: audit 2023-09-23T18:08:34.700454+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:08:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:35 smithi173 bash[27694]: audit 2023-09-23T18:08:34.701015+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T18:08:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:35 smithi173 bash[27694]: cluster 2023-09-23T18:08:35.116383+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v110: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T18:08:37.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:36 smithi173 bash[27694]: audit 2023-09-23T18:08:35.711171+0000 mon.a (mon.0) 322 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T18:08:37.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:36 smithi173 bash[27694]: cluster 2023-09-23T18:08:35.711222+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-09-23T18:08:37.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:36 smithi173 bash[27694]: audit 2023-09-23T18:08:35.712145+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-09-23T18:08:37.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:36 smithi098 bash[27349]: audit 2023-09-23T18:08:35.711171+0000 mon.a (mon.0) 322 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T18:08:37.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:36 smithi098 bash[27349]: cluster 2023-09-23T18:08:35.711222+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-09-23T18:08:37.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:36 smithi098 bash[27349]: audit 2023-09-23T18:08:35.712145+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-09-23T18:08:37.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:36 smithi098 bash[33599]: audit 2023-09-23T18:08:35.711171+0000 mon.a (mon.0) 322 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T18:08:37.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:36 smithi098 bash[33599]: cluster 2023-09-23T18:08:35.711222+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-09-23T18:08:37.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:36 smithi098 bash[33599]: audit 2023-09-23T18:08:35.712145+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-09-23T18:08:38.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:37 smithi173 bash[27694]: audit 2023-09-23T18:08:36.714978+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-09-23T18:08:38.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:37 smithi173 bash[27694]: cluster 2023-09-23T18:08:36.715092+0000 mon.a (mon.0) 326 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-09-23T18:08:38.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:37 smithi173 bash[27694]: cluster 2023-09-23T18:08:37.116912+0000 mgr.y (mgr.14146) 133 : cluster [DBG] pgmap v113: 1 pgs: 1 creating+peering; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:38.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:37 smithi098 bash[27349]: audit 2023-09-23T18:08:36.714978+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-09-23T18:08:38.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:37 smithi098 bash[27349]: cluster 2023-09-23T18:08:36.715092+0000 mon.a (mon.0) 326 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-09-23T18:08:38.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:37 smithi098 bash[27349]: cluster 2023-09-23T18:08:37.116912+0000 mgr.y (mgr.14146) 133 : cluster [DBG] pgmap v113: 1 pgs: 1 creating+peering; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:38.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:37 smithi098 bash[33599]: audit 2023-09-23T18:08:36.714978+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-09-23T18:08:38.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:37 smithi098 bash[33599]: cluster 2023-09-23T18:08:36.715092+0000 mon.a (mon.0) 326 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-09-23T18:08:38.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:37 smithi098 bash[33599]: cluster 2023-09-23T18:08:37.116912+0000 mgr.y (mgr.14146) 133 : cluster [DBG] pgmap v113: 1 pgs: 1 creating+peering; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:38.897 INFO:teuthology.orchestra.run.smithi098.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-09-23T18:08:38.897 INFO:teuthology.orchestra.run.smithi098.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-09-23T18:08:38.897 INFO:teuthology.orchestra.run.smithi098.stderr: stderr: 10+0 records in 2023-09-23T18:08:38.897 INFO:teuthology.orchestra.run.smithi098.stderr:10+0 records out 2023-09-23T18:08:38.897 INFO:teuthology.orchestra.run.smithi098.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.021265 s, 493 MB/s 2023-09-23T18:08:38.897 INFO:teuthology.orchestra.run.smithi098.stderr:--> Zapping successful for: 2023-09-23T18:08:39.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:38 smithi173 bash[27694]: cluster 2023-09-23T18:08:37.730711+0000 mon.a (mon.0) 327 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-09-23T18:08:39.163 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:38 smithi098 bash[27349]: cluster 2023-09-23T18:08:37.730711+0000 mon.a (mon.0) 327 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-09-23T18:08:39.163 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:38 smithi098 bash[33599]: cluster 2023-09-23T18:08:37.730711+0000 mon.a (mon.0) 327 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-09-23T18:08:39.682 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi098:vg_nvme/lv_1 2023-09-23T18:08:39.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:39 smithi098 bash[27349]: cluster 2023-09-23T18:08:39.117485+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v115: 1 pgs: 1 creating+peering; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:39.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:39 smithi098 bash[33599]: cluster 2023-09-23T18:08:39.117485+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v115: 1 pgs: 1 creating+peering; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:40.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:39 smithi173 bash[27694]: cluster 2023-09-23T18:08:39.117485+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v115: 1 pgs: 1 creating+peering; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:42.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:42 smithi098 bash[33599]: cluster 2023-09-23T18:08:41.118000+0000 mgr.y (mgr.14146) 135 : cluster [DBG] pgmap v116: 1 pgs: 1 creating+peering; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:42.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:42 smithi098 bash[27349]: cluster 2023-09-23T18:08:41.118000+0000 mgr.y (mgr.14146) 135 : cluster [DBG] pgmap v116: 1 pgs: 1 creating+peering; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:42.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:42 smithi173 bash[27694]: cluster 2023-09-23T18:08:41.118000+0000 mgr.y (mgr.14146) 135 : cluster [DBG] pgmap v116: 1 pgs: 1 creating+peering; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:44.347 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:44 smithi098 bash[27349]: cluster 2023-09-23T18:08:43.118610+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:44.347 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:44 smithi098 bash[33599]: cluster 2023-09-23T18:08:43.118610+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:44.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:44 smithi173 bash[27694]: cluster 2023-09-23T18:08:43.118610+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:45.396 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:45 smithi098 bash[27349]: audit 2023-09-23T18:08:44.537058+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:08:45.396 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:45 smithi098 bash[27349]: audit 2023-09-23T18:08:44.541938+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:08:45.396 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:45 smithi098 bash[27349]: audit 2023-09-23T18:08:44.543024+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:08:45.396 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:45 smithi098 bash[33599]: audit 2023-09-23T18:08:44.537058+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:08:45.397 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:45 smithi098 bash[33599]: audit 2023-09-23T18:08:44.541938+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:08:45.397 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:45 smithi098 bash[33599]: audit 2023-09-23T18:08:44.543024+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:08:45.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:45 smithi173 bash[27694]: audit 2023-09-23T18:08:44.537058+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:08:45.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:45 smithi173 bash[27694]: audit 2023-09-23T18:08:44.541938+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:08:45.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:45 smithi173 bash[27694]: audit 2023-09-23T18:08:44.543024+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:08:46.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:46 smithi098 bash[33599]: audit 2023-09-23T18:08:44.534140+0000 mgr.y (mgr.14146) 137 : audit [DBG] from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi098:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:08:46.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:46 smithi098 bash[33599]: cluster 2023-09-23T18:08:45.119060+0000 mgr.y (mgr.14146) 138 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:46.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:46 smithi098 bash[27349]: audit 2023-09-23T18:08:44.534140+0000 mgr.y (mgr.14146) 137 : audit [DBG] from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi098:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:08:46.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:46 smithi098 bash[27349]: cluster 2023-09-23T18:08:45.119060+0000 mgr.y (mgr.14146) 138 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:46.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:46 smithi173 bash[27694]: audit 2023-09-23T18:08:44.534140+0000 mgr.y (mgr.14146) 137 : audit [DBG] from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi098:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:08:46.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:46 smithi173 bash[27694]: cluster 2023-09-23T18:08:45.119060+0000 mgr.y (mgr.14146) 138 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:47.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:47 smithi098 bash[33599]: cephadm 2023-09-23T18:08:46.128208+0000 mgr.y (mgr.14146) 139 : cephadm [INF] Detected new or changed devices on smithi098 2023-09-23T18:08:47.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:47 smithi098 bash[33599]: audit 2023-09-23T18:08:46.137642+0000 mon.a (mon.0) 331 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:47.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:47 smithi098 bash[33599]: audit 2023-09-23T18:08:46.139299+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:08:47.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:47 smithi098 bash[33599]: audit 2023-09-23T18:08:46.140831+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:08:47.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:47 smithi098 bash[33599]: audit 2023-09-23T18:08:46.142220+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:08:47.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:47 smithi098 bash[33599]: cephadm 2023-09-23T18:08:46.143314+0000 mgr.y (mgr.14146) 140 : cephadm [INF] Adjusting osd_memory_target on smithi098 to 5413M 2023-09-23T18:08:47.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:47 smithi098 bash[33599]: audit 2023-09-23T18:08:46.148230+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:47.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:47 smithi098 bash[33599]: audit 2023-09-23T18:08:46.149778+0000 mon.a (mon.0) 336 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:08:47.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:47 smithi098 bash[33599]: audit 2023-09-23T18:08:46.150879+0000 mon.a (mon.0) 337 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:08:47.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:47 smithi098 bash[33599]: audit 2023-09-23T18:08:46.157822+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:47.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:47 smithi098 bash[27349]: cephadm 2023-09-23T18:08:46.128208+0000 mgr.y (mgr.14146) 139 : cephadm [INF] Detected new or changed devices on smithi098 2023-09-23T18:08:47.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:47 smithi098 bash[27349]: audit 2023-09-23T18:08:46.137642+0000 mon.a (mon.0) 331 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:47.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:47 smithi098 bash[27349]: audit 2023-09-23T18:08:46.139299+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:08:47.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:47 smithi098 bash[27349]: audit 2023-09-23T18:08:46.140831+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:08:47.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:47 smithi098 bash[27349]: audit 2023-09-23T18:08:46.142220+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:08:47.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:47 smithi098 bash[27349]: cephadm 2023-09-23T18:08:46.143314+0000 mgr.y (mgr.14146) 140 : cephadm [INF] Adjusting osd_memory_target on smithi098 to 5413M 2023-09-23T18:08:47.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:47 smithi098 bash[27349]: audit 2023-09-23T18:08:46.148230+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:47.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:47 smithi098 bash[27349]: audit 2023-09-23T18:08:46.149778+0000 mon.a (mon.0) 336 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:08:47.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:47 smithi098 bash[27349]: audit 2023-09-23T18:08:46.150879+0000 mon.a (mon.0) 337 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:08:47.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:47 smithi098 bash[27349]: audit 2023-09-23T18:08:46.157822+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:47.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:47 smithi173 bash[27694]: cephadm 2023-09-23T18:08:46.128208+0000 mgr.y (mgr.14146) 139 : cephadm [INF] Detected new or changed devices on smithi098 2023-09-23T18:08:47.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:47 smithi173 bash[27694]: audit 2023-09-23T18:08:46.137642+0000 mon.a (mon.0) 331 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:47.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:47 smithi173 bash[27694]: audit 2023-09-23T18:08:46.139299+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:08:47.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:47 smithi173 bash[27694]: audit 2023-09-23T18:08:46.140831+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:08:47.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:47 smithi173 bash[27694]: audit 2023-09-23T18:08:46.142220+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:08:47.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:47 smithi173 bash[27694]: cephadm 2023-09-23T18:08:46.143314+0000 mgr.y (mgr.14146) 140 : cephadm [INF] Adjusting osd_memory_target on smithi098 to 5413M 2023-09-23T18:08:47.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:47 smithi173 bash[27694]: audit 2023-09-23T18:08:46.148230+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:47.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:47 smithi173 bash[27694]: audit 2023-09-23T18:08:46.149778+0000 mon.a (mon.0) 336 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:08:47.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:47 smithi173 bash[27694]: audit 2023-09-23T18:08:46.150879+0000 mon.a (mon.0) 337 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:08:47.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:47 smithi173 bash[27694]: audit 2023-09-23T18:08:46.157822+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:08:48.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:48 smithi098 bash[27349]: cluster 2023-09-23T18:08:47.119639+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:48.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:48 smithi098 bash[33599]: cluster 2023-09-23T18:08:47.119639+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:48.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:48 smithi173 bash[27694]: cluster 2023-09-23T18:08:47.119639+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:50.421 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:50 smithi098 bash[33599]: cluster 2023-09-23T18:08:49.120260+0000 mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:50.422 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:50 smithi098 bash[27349]: cluster 2023-09-23T18:08:49.120260+0000 mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:50.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:50 smithi173 bash[27694]: cluster 2023-09-23T18:08:49.120260+0000 mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:52.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:52 smithi098 bash[27349]: cluster 2023-09-23T18:08:51.120917+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:52.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:52 smithi098 bash[33599]: cluster 2023-09-23T18:08:51.120917+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:52.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:52 smithi173 bash[27694]: cluster 2023-09-23T18:08:51.120917+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:53.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:53 smithi098 bash[33599]: audit 2023-09-23T18:08:53.033484+0000 mon.a (mon.0) 339 : audit [INF] from='client.? 172.21.15.98:0/3118541067' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "48596d91-46da-4815-b810-d493bbf01dcc"}]: dispatch 2023-09-23T18:08:53.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:53 smithi098 bash[33599]: audit 2023-09-23T18:08:53.043110+0000 mon.a (mon.0) 340 : audit [INF] from='client.? 172.21.15.98:0/3118541067' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "48596d91-46da-4815-b810-d493bbf01dcc"}]': finished 2023-09-23T18:08:53.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:53 smithi098 bash[33599]: cluster 2023-09-23T18:08:53.043268+0000 mon.a (mon.0) 341 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-09-23T18:08:53.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:53 smithi098 bash[33599]: audit 2023-09-23T18:08:53.043550+0000 mon.a (mon.0) 342 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:08:53.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:53 smithi098 bash[27349]: audit 2023-09-23T18:08:53.033484+0000 mon.a (mon.0) 339 : audit [INF] from='client.? 172.21.15.98:0/3118541067' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "48596d91-46da-4815-b810-d493bbf01dcc"}]: dispatch 2023-09-23T18:08:53.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:53 smithi098 bash[27349]: audit 2023-09-23T18:08:53.043110+0000 mon.a (mon.0) 340 : audit [INF] from='client.? 172.21.15.98:0/3118541067' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "48596d91-46da-4815-b810-d493bbf01dcc"}]': finished 2023-09-23T18:08:53.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:53 smithi098 bash[27349]: cluster 2023-09-23T18:08:53.043268+0000 mon.a (mon.0) 341 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-09-23T18:08:53.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:53 smithi098 bash[27349]: audit 2023-09-23T18:08:53.043550+0000 mon.a (mon.0) 342 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:08:53.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:53 smithi173 bash[27694]: audit 2023-09-23T18:08:53.033484+0000 mon.a (mon.0) 339 : audit [INF] from='client.? 172.21.15.98:0/3118541067' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "48596d91-46da-4815-b810-d493bbf01dcc"}]: dispatch 2023-09-23T18:08:53.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:53 smithi173 bash[27694]: audit 2023-09-23T18:08:53.043110+0000 mon.a (mon.0) 340 : audit [INF] from='client.? 172.21.15.98:0/3118541067' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "48596d91-46da-4815-b810-d493bbf01dcc"}]': finished 2023-09-23T18:08:53.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:53 smithi173 bash[27694]: cluster 2023-09-23T18:08:53.043268+0000 mon.a (mon.0) 341 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-09-23T18:08:53.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:53 smithi173 bash[27694]: audit 2023-09-23T18:08:53.043550+0000 mon.a (mon.0) 342 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:08:54.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:54 smithi098 bash[33599]: cluster 2023-09-23T18:08:53.121540+0000 mgr.y (mgr.14146) 144 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:54.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:54 smithi098 bash[33599]: audit 2023-09-23T18:08:53.778377+0000 mon.a (mon.0) 343 : audit [DBG] from='client.? 172.21.15.98:0/3111762368' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:08:54.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:54 smithi098 bash[27349]: cluster 2023-09-23T18:08:53.121540+0000 mgr.y (mgr.14146) 144 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:54.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:54 smithi098 bash[27349]: audit 2023-09-23T18:08:53.778377+0000 mon.a (mon.0) 343 : audit [DBG] from='client.? 172.21.15.98:0/3111762368' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:08:54.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:54 smithi173 bash[27694]: cluster 2023-09-23T18:08:53.121540+0000 mgr.y (mgr.14146) 144 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:54.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:54 smithi173 bash[27694]: audit 2023-09-23T18:08:53.778377+0000 mon.a (mon.0) 343 : audit [DBG] from='client.? 172.21.15.98:0/3111762368' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:08:56.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:56 smithi098 bash[27349]: cluster 2023-09-23T18:08:55.122093+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:56.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:56 smithi098 bash[33599]: cluster 2023-09-23T18:08:55.122093+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:56.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:56 smithi173 bash[27694]: cluster 2023-09-23T18:08:55.122093+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:58.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:08:58 smithi098 bash[27349]: cluster 2023-09-23T18:08:57.122652+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:58.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:08:58 smithi098 bash[33599]: cluster 2023-09-23T18:08:57.122652+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:08:58.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:08:58 smithi173 bash[27694]: cluster 2023-09-23T18:08:57.122652+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:00.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:00 smithi098 bash[27349]: cluster 2023-09-23T18:08:59.123112+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:00.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:00 smithi098 bash[33599]: cluster 2023-09-23T18:08:59.123112+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:00.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:00 smithi173 bash[27694]: cluster 2023-09-23T18:08:59.123112+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:02.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:02 smithi173 bash[27694]: cluster 2023-09-23T18:09:01.123660+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:02.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:02 smithi098 bash[27349]: cluster 2023-09-23T18:09:01.123660+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:02.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:02 smithi098 bash[33599]: cluster 2023-09-23T18:09:01.123660+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:04.381 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:04 smithi098 bash[27349]: cluster 2023-09-23T18:09:03.124224+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:04.381 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:04 smithi098 bash[33599]: cluster 2023-09-23T18:09:03.124224+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:04.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:04 smithi173 bash[27694]: cluster 2023-09-23T18:09:03.124224+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:06.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:06 smithi098 bash[27349]: cluster 2023-09-23T18:09:05.124753+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:06.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:06 smithi098 bash[27349]: audit 2023-09-23T18:09:05.734138+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T18:09:06.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:06 smithi098 bash[27349]: audit 2023-09-23T18:09:05.735811+0000 mon.a (mon.0) 345 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:09:06.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:06 smithi098 bash[33599]: cluster 2023-09-23T18:09:05.124753+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:06.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:06 smithi098 bash[33599]: audit 2023-09-23T18:09:05.734138+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T18:09:06.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:06 smithi098 bash[33599]: audit 2023-09-23T18:09:05.735811+0000 mon.a (mon.0) 345 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:09:06.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:06 smithi173 bash[27694]: cluster 2023-09-23T18:09:05.124753+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:06.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:06 smithi173 bash[27694]: audit 2023-09-23T18:09:05.734138+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T18:09:06.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:06 smithi173 bash[27694]: audit 2023-09-23T18:09:05.735811+0000 mon.a (mon.0) 345 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:09:07.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:07 smithi098 bash[27349]: cephadm 2023-09-23T18:09:05.737176+0000 mgr.y (mgr.14146) 151 : cephadm [INF] Deploying daemon osd.3 on smithi098 2023-09-23T18:09:07.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:07 smithi098 bash[33599]: cephadm 2023-09-23T18:09:05.737176+0000 mgr.y (mgr.14146) 151 : cephadm [INF] Deploying daemon osd.3 on smithi098 2023-09-23T18:09:07.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:07 smithi173 bash[27694]: cephadm 2023-09-23T18:09:05.737176+0000 mgr.y (mgr.14146) 151 : cephadm [INF] Deploying daemon osd.3 on smithi098 2023-09-23T18:09:08.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:08 smithi173 bash[27694]: cluster 2023-09-23T18:09:07.125324+0000 mgr.y (mgr.14146) 152 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:08.556 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:08 smithi098 bash[27349]: cluster 2023-09-23T18:09:07.125324+0000 mgr.y (mgr.14146) 152 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:08.556 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:08 smithi098 bash[33599]: cluster 2023-09-23T18:09:07.125324+0000 mgr.y (mgr.14146) 152 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:10.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:10 smithi098 bash[27349]: cluster 2023-09-23T18:09:09.125900+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:10.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:10 smithi098 bash[33599]: cluster 2023-09-23T18:09:09.125900+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:10.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:10 smithi173 bash[27694]: cluster 2023-09-23T18:09:09.125900+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:11.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:11 smithi098 bash[33599]: audit 2023-09-23T18:09:10.227514+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:11.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:11 smithi098 bash[33599]: audit 2023-09-23T18:09:10.237096+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:09:11.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:11 smithi098 bash[27349]: audit 2023-09-23T18:09:10.227514+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:11.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:11 smithi098 bash[27349]: audit 2023-09-23T18:09:10.237096+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:09:11.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:11 smithi173 bash[27694]: audit 2023-09-23T18:09:10.227514+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:11.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:11 smithi173 bash[27694]: audit 2023-09-23T18:09:10.237096+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:09:12.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:12 smithi098 bash[27349]: cluster 2023-09-23T18:09:11.126455+0000 mgr.y (mgr.14146) 154 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:12.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:12 smithi098 bash[33599]: cluster 2023-09-23T18:09:11.126455+0000 mgr.y (mgr.14146) 154 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:12.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:12 smithi173 bash[27694]: cluster 2023-09-23T18:09:11.126455+0000 mgr.y (mgr.14146) 154 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:14.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:14 smithi098 bash[27349]: cluster 2023-09-23T18:09:13.127061+0000 mgr.y (mgr.14146) 155 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:14.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:14 smithi098 bash[33599]: cluster 2023-09-23T18:09:13.127061+0000 mgr.y (mgr.14146) 155 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:14.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:14 smithi173 bash[27694]: cluster 2023-09-23T18:09:13.127061+0000 mgr.y (mgr.14146) 155 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:16.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:16 smithi098 bash[27349]: cluster 2023-09-23T18:09:15.127723+0000 mgr.y (mgr.14146) 156 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:16.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:16 smithi098 bash[33599]: cluster 2023-09-23T18:09:15.127723+0000 mgr.y (mgr.14146) 156 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:16.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:16 smithi173 bash[27694]: cluster 2023-09-23T18:09:15.127723+0000 mgr.y (mgr.14146) 156 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:17.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:17 smithi098 bash[27349]: audit 2023-09-23T18:09:16.648814+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:17.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:17 smithi098 bash[27349]: audit 2023-09-23T18:09:16.938116+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:09:17.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:17 smithi098 bash[27349]: audit 2023-09-23T18:09:16.939017+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:09:17.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:17 smithi098 bash[27349]: audit 2023-09-23T18:09:16.947272+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:17.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:17 smithi098 bash[27349]: cluster 2023-09-23T18:09:17.128145+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:17.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:17 smithi098 bash[27349]: audit 2023-09-23T18:09:17.209169+0000 mon.c (mon.2) 10 : audit [INF] from='osd.3 [v2:172.21.15.98:6826/2634725099,v1:172.21.15.98:6827/2634725099]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T18:09:17.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:17 smithi098 bash[27349]: audit 2023-09-23T18:09:17.209636+0000 mon.a (mon.0) 352 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T18:09:17.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:17 smithi098 bash[33599]: audit 2023-09-23T18:09:16.648814+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:17.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:17 smithi098 bash[33599]: audit 2023-09-23T18:09:16.938116+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:09:17.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:17 smithi098 bash[33599]: audit 2023-09-23T18:09:16.939017+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:09:17.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:17 smithi098 bash[33599]: audit 2023-09-23T18:09:16.947272+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:17.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:17 smithi098 bash[33599]: cluster 2023-09-23T18:09:17.128145+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:17.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:17 smithi098 bash[33599]: audit 2023-09-23T18:09:17.209169+0000 mon.c (mon.2) 10 : audit [INF] from='osd.3 [v2:172.21.15.98:6826/2634725099,v1:172.21.15.98:6827/2634725099]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T18:09:17.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:17 smithi098 bash[33599]: audit 2023-09-23T18:09:17.209636+0000 mon.a (mon.0) 352 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T18:09:18.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:17 smithi173 bash[27694]: audit 2023-09-23T18:09:16.648814+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:18.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:17 smithi173 bash[27694]: audit 2023-09-23T18:09:16.938116+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:09:18.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:17 smithi173 bash[27694]: audit 2023-09-23T18:09:16.939017+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:09:18.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:17 smithi173 bash[27694]: audit 2023-09-23T18:09:16.947272+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:18.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:17 smithi173 bash[27694]: cluster 2023-09-23T18:09:17.128145+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:18.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:17 smithi173 bash[27694]: audit 2023-09-23T18:09:17.209169+0000 mon.c (mon.2) 10 : audit [INF] from='osd.3 [v2:172.21.15.98:6826/2634725099,v1:172.21.15.98:6827/2634725099]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T18:09:18.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:17 smithi173 bash[27694]: audit 2023-09-23T18:09:17.209636+0000 mon.a (mon.0) 352 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T18:09:19.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:18 smithi173 bash[27694]: audit 2023-09-23T18:09:17.960697+0000 mon.a (mon.0) 353 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T18:09:19.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:18 smithi173 bash[27694]: cluster 2023-09-23T18:09:17.960934+0000 mon.a (mon.0) 354 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-09-23T18:09:19.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:18 smithi173 bash[27694]: audit 2023-09-23T18:09:17.961101+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:09:19.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:18 smithi173 bash[27694]: audit 2023-09-23T18:09:17.961921+0000 mon.c (mon.2) 11 : audit [INF] from='osd.3 [v2:172.21.15.98:6826/2634725099,v1:172.21.15.98:6827/2634725099]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:09:19.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:18 smithi173 bash[27694]: audit 2023-09-23T18:09:17.962457+0000 mon.a (mon.0) 356 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:09:19.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:18 smithi098 bash[33599]: audit 2023-09-23T18:09:17.960697+0000 mon.a (mon.0) 353 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T18:09:19.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:18 smithi098 bash[33599]: cluster 2023-09-23T18:09:17.960934+0000 mon.a (mon.0) 354 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-09-23T18:09:19.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:18 smithi098 bash[33599]: audit 2023-09-23T18:09:17.961101+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:09:19.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:18 smithi098 bash[33599]: audit 2023-09-23T18:09:17.961921+0000 mon.c (mon.2) 11 : audit [INF] from='osd.3 [v2:172.21.15.98:6826/2634725099,v1:172.21.15.98:6827/2634725099]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:09:19.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:18 smithi098 bash[33599]: audit 2023-09-23T18:09:17.962457+0000 mon.a (mon.0) 356 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:09:19.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:18 smithi098 bash[27349]: audit 2023-09-23T18:09:17.960697+0000 mon.a (mon.0) 353 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T18:09:19.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:18 smithi098 bash[27349]: cluster 2023-09-23T18:09:17.960934+0000 mon.a (mon.0) 354 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-09-23T18:09:19.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:18 smithi098 bash[27349]: audit 2023-09-23T18:09:17.961101+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:09:19.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:18 smithi098 bash[27349]: audit 2023-09-23T18:09:17.961921+0000 mon.c (mon.2) 11 : audit [INF] from='osd.3 [v2:172.21.15.98:6826/2634725099,v1:172.21.15.98:6827/2634725099]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:09:19.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:18 smithi098 bash[27349]: audit 2023-09-23T18:09:17.962457+0000 mon.a (mon.0) 356 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:09:20.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:19 smithi098 bash[27349]: audit 2023-09-23T18:09:18.961081+0000 mon.a (mon.0) 357 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]': finished 2023-09-23T18:09:20.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:19 smithi098 bash[27349]: cluster 2023-09-23T18:09:18.961221+0000 mon.a (mon.0) 358 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-09-23T18:09:20.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:19 smithi098 bash[27349]: audit 2023-09-23T18:09:18.961970+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:09:20.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:19 smithi098 bash[27349]: audit 2023-09-23T18:09:18.975206+0000 mon.a (mon.0) 360 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:09:20.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:19 smithi098 bash[27349]: cluster 2023-09-23T18:09:19.128624+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:20.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:19 smithi098 bash[33599]: audit 2023-09-23T18:09:18.961081+0000 mon.a (mon.0) 357 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]': finished 2023-09-23T18:09:20.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:19 smithi098 bash[33599]: cluster 2023-09-23T18:09:18.961221+0000 mon.a (mon.0) 358 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-09-23T18:09:20.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:19 smithi098 bash[33599]: audit 2023-09-23T18:09:18.961970+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:09:20.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:19 smithi098 bash[33599]: audit 2023-09-23T18:09:18.975206+0000 mon.a (mon.0) 360 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:09:20.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:19 smithi098 bash[33599]: cluster 2023-09-23T18:09:19.128624+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:20.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:19 smithi173 bash[27694]: audit 2023-09-23T18:09:18.961081+0000 mon.a (mon.0) 357 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]': finished 2023-09-23T18:09:20.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:19 smithi173 bash[27694]: cluster 2023-09-23T18:09:18.961221+0000 mon.a (mon.0) 358 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-09-23T18:09:20.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:19 smithi173 bash[27694]: audit 2023-09-23T18:09:18.961970+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:09:20.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:19 smithi173 bash[27694]: audit 2023-09-23T18:09:18.975206+0000 mon.a (mon.0) 360 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:09:20.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:19 smithi173 bash[27694]: cluster 2023-09-23T18:09:19.128624+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T18:09:20.499 INFO:teuthology.orchestra.run.smithi098.stdout:Created osd(s) 3 on host 'smithi098' 2023-09-23T18:09:21.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:20 smithi098 bash[27349]: cluster 2023-09-23T18:09:18.218443+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:09:21.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:20 smithi098 bash[27349]: cluster 2023-09-23T18:09:18.218499+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:09:21.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:20 smithi098 bash[27349]: audit 2023-09-23T18:09:19.971824+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:09:21.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:20 smithi098 bash[27349]: cluster 2023-09-23T18:09:19.981838+0000 mon.a (mon.0) 362 : cluster [INF] osd.3 [v2:172.21.15.98:6826/2634725099,v1:172.21.15.98:6827/2634725099] boot 2023-09-23T18:09:21.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:20 smithi098 bash[27349]: cluster 2023-09-23T18:09:19.981898+0000 mon.a (mon.0) 363 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-09-23T18:09:21.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:20 smithi098 bash[27349]: audit 2023-09-23T18:09:19.982123+0000 mon.a (mon.0) 364 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:09:21.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:20 smithi098 bash[27349]: audit 2023-09-23T18:09:20.495277+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:21.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:20 smithi098 bash[27349]: audit 2023-09-23T18:09:20.523335+0000 mon.a (mon.0) 366 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:09:21.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:20 smithi098 bash[33599]: cluster 2023-09-23T18:09:18.218443+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:09:21.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:20 smithi098 bash[33599]: cluster 2023-09-23T18:09:18.218499+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:09:21.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:20 smithi098 bash[33599]: audit 2023-09-23T18:09:19.971824+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:09:21.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:20 smithi098 bash[33599]: cluster 2023-09-23T18:09:19.981838+0000 mon.a (mon.0) 362 : cluster [INF] osd.3 [v2:172.21.15.98:6826/2634725099,v1:172.21.15.98:6827/2634725099] boot 2023-09-23T18:09:21.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:20 smithi098 bash[33599]: cluster 2023-09-23T18:09:19.981898+0000 mon.a (mon.0) 363 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-09-23T18:09:21.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:20 smithi098 bash[33599]: audit 2023-09-23T18:09:19.982123+0000 mon.a (mon.0) 364 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:09:21.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:20 smithi098 bash[33599]: audit 2023-09-23T18:09:20.495277+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:21.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:20 smithi098 bash[33599]: audit 2023-09-23T18:09:20.523335+0000 mon.a (mon.0) 366 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:09:21.255 DEBUG:teuthology.orchestra.run.smithi098:osd.3> sudo journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.3.service 2023-09-23T18:09:21.258 INFO:tasks.cephadm:Deploying osd.4 on smithi173 with /dev/vg_nvme/lv_4... 2023-09-23T18:09:21.258 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-09-23T18:09:21.272 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:20 smithi173 bash[27694]: cluster 2023-09-23T18:09:18.218443+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:09:21.272 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:20 smithi173 bash[27694]: cluster 2023-09-23T18:09:18.218499+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:09:21.272 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:20 smithi173 bash[27694]: audit 2023-09-23T18:09:19.971824+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:09:21.273 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:20 smithi173 bash[27694]: cluster 2023-09-23T18:09:19.981838+0000 mon.a (mon.0) 362 : cluster [INF] osd.3 [v2:172.21.15.98:6826/2634725099,v1:172.21.15.98:6827/2634725099] boot 2023-09-23T18:09:21.273 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:20 smithi173 bash[27694]: cluster 2023-09-23T18:09:19.981898+0000 mon.a (mon.0) 363 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-09-23T18:09:21.273 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:20 smithi173 bash[27694]: audit 2023-09-23T18:09:19.982123+0000 mon.a (mon.0) 364 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:09:21.273 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:20 smithi173 bash[27694]: audit 2023-09-23T18:09:20.495277+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:21.273 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:20 smithi173 bash[27694]: audit 2023-09-23T18:09:20.523335+0000 mon.a (mon.0) 366 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:09:21.309 INFO:journalctl@ceph.osd.3.smithi098.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:09:22.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:21 smithi173 bash[27694]: cluster 2023-09-23T18:09:20.984527+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-09-23T18:09:22.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:21 smithi173 bash[27694]: cluster 2023-09-23T18:09:21.129119+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:22.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:21 smithi098 bash[27349]: cluster 2023-09-23T18:09:20.984527+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-09-23T18:09:22.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:21 smithi098 bash[27349]: cluster 2023-09-23T18:09:21.129119+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:22.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:21 smithi098 bash[33599]: cluster 2023-09-23T18:09:20.984527+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-09-23T18:09:22.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:21 smithi098 bash[33599]: cluster 2023-09-23T18:09:21.129119+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:24.428 INFO:teuthology.orchestra.run.smithi173.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-09-23T18:09:24.428 INFO:teuthology.orchestra.run.smithi173.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-09-23T18:09:24.429 INFO:teuthology.orchestra.run.smithi173.stderr: stderr: 10+0 records in 2023-09-23T18:09:24.429 INFO:teuthology.orchestra.run.smithi173.stderr:10+0 records out 2023-09-23T18:09:24.429 INFO:teuthology.orchestra.run.smithi173.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0155603 s, 674 MB/s 2023-09-23T18:09:24.429 INFO:teuthology.orchestra.run.smithi173.stderr:--> Zapping successful for: 2023-09-23T18:09:24.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:24 smithi098 bash[27349]: cluster 2023-09-23T18:09:23.129665+0000 mgr.y (mgr.14146) 160 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:24.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:24 smithi098 bash[33599]: cluster 2023-09-23T18:09:23.129665+0000 mgr.y (mgr.14146) 160 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:24.467 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:24 smithi173 bash[27694]: cluster 2023-09-23T18:09:23.129665+0000 mgr.y (mgr.14146) 160 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:25.033 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi173:vg_nvme/lv_4 2023-09-23T18:09:26.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:26 smithi173 bash[27694]: cluster 2023-09-23T18:09:25.130223+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:26.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:26 smithi098 bash[27349]: cluster 2023-09-23T18:09:25.130223+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:26.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:26 smithi098 bash[33599]: cluster 2023-09-23T18:09:25.130223+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:28.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:28 smithi098 bash[27349]: cluster 2023-09-23T18:09:27.130704+0000 mgr.y (mgr.14146) 162 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:28.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:28 smithi098 bash[27349]: audit 2023-09-23T18:09:27.820164+0000 mon.a (mon.0) 368 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:09:28.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:28 smithi098 bash[27349]: audit 2023-09-23T18:09:27.823803+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:09:28.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:28 smithi098 bash[27349]: audit 2023-09-23T18:09:27.825171+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:09:28.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:28 smithi098 bash[33599]: cluster 2023-09-23T18:09:27.130704+0000 mgr.y (mgr.14146) 162 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:28.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:28 smithi098 bash[33599]: audit 2023-09-23T18:09:27.820164+0000 mon.a (mon.0) 368 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:09:28.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:28 smithi098 bash[33599]: audit 2023-09-23T18:09:27.823803+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:09:28.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:28 smithi098 bash[33599]: audit 2023-09-23T18:09:27.825171+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:09:28.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:28 smithi173 bash[27694]: cluster 2023-09-23T18:09:27.130704+0000 mgr.y (mgr.14146) 162 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:28.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:28 smithi173 bash[27694]: audit 2023-09-23T18:09:27.820164+0000 mon.a (mon.0) 368 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:09:28.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:28 smithi173 bash[27694]: audit 2023-09-23T18:09:27.823803+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:09:28.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:28 smithi173 bash[27694]: audit 2023-09-23T18:09:27.825171+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:09:29.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:29 smithi098 bash[33599]: audit 2023-09-23T18:09:27.814203+0000 mgr.y (mgr.14146) 163 : audit [DBG] from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi173:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:09:29.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:29 smithi098 bash[27349]: audit 2023-09-23T18:09:27.814203+0000 mgr.y (mgr.14146) 163 : audit [DBG] from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi173:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:09:29.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:29 smithi173 bash[27694]: audit 2023-09-23T18:09:27.814203+0000 mgr.y (mgr.14146) 163 : audit [DBG] from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi173:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:09:30.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:30 smithi173 bash[27694]: cluster 2023-09-23T18:09:29.131236+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:30.533 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:30 smithi173 bash[27694]: audit 2023-09-23T18:09:29.839951+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:30.533 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:30 smithi173 bash[27694]: audit 2023-09-23T18:09:29.841503+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:09:30.533 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:30 smithi173 bash[27694]: audit 2023-09-23T18:09:29.842682+0000 mon.a (mon.0) 373 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:09:30.533 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:30 smithi173 bash[27694]: audit 2023-09-23T18:09:29.843747+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:09:30.533 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:30 smithi173 bash[27694]: audit 2023-09-23T18:09:29.844914+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:09:30.533 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:30 smithi173 bash[27694]: audit 2023-09-23T18:09:29.850710+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:30.533 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:30 smithi173 bash[27694]: audit 2023-09-23T18:09:29.852191+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:09:30.533 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:30 smithi173 bash[27694]: audit 2023-09-23T18:09:29.854686+0000 mon.a (mon.0) 378 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:09:30.533 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:30 smithi173 bash[27694]: audit 2023-09-23T18:09:29.860474+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:30.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:30 smithi098 bash[27349]: cluster 2023-09-23T18:09:29.131236+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:30.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:30 smithi098 bash[27349]: audit 2023-09-23T18:09:29.839951+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:30.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:30 smithi098 bash[27349]: audit 2023-09-23T18:09:29.841503+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:09:30.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:30 smithi098 bash[27349]: audit 2023-09-23T18:09:29.842682+0000 mon.a (mon.0) 373 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:09:30.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:30 smithi098 bash[27349]: audit 2023-09-23T18:09:29.843747+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:09:30.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:30 smithi098 bash[27349]: audit 2023-09-23T18:09:29.844914+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:09:30.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:30 smithi098 bash[27349]: audit 2023-09-23T18:09:29.850710+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:30.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:30 smithi098 bash[27349]: audit 2023-09-23T18:09:29.852191+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:09:30.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:30 smithi098 bash[27349]: audit 2023-09-23T18:09:29.854686+0000 mon.a (mon.0) 378 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:09:30.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:30 smithi098 bash[27349]: audit 2023-09-23T18:09:29.860474+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:30.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:30 smithi098 bash[33599]: cluster 2023-09-23T18:09:29.131236+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:30.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:30 smithi098 bash[33599]: audit 2023-09-23T18:09:29.839951+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:30.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:30 smithi098 bash[33599]: audit 2023-09-23T18:09:29.841503+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:09:30.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:30 smithi098 bash[33599]: audit 2023-09-23T18:09:29.842682+0000 mon.a (mon.0) 373 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:09:30.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:30 smithi098 bash[33599]: audit 2023-09-23T18:09:29.843747+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:09:30.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:30 smithi098 bash[33599]: audit 2023-09-23T18:09:29.844914+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:09:30.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:30 smithi098 bash[33599]: audit 2023-09-23T18:09:29.850710+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:30.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:30 smithi098 bash[33599]: audit 2023-09-23T18:09:29.852191+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:09:30.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:30 smithi098 bash[33599]: audit 2023-09-23T18:09:29.854686+0000 mon.a (mon.0) 378 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:09:30.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:30 smithi098 bash[33599]: audit 2023-09-23T18:09:29.860474+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:31.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:31 smithi173 bash[27694]: cephadm 2023-09-23T18:09:29.830398+0000 mgr.y (mgr.14146) 165 : cephadm [INF] Detected new or changed devices on smithi098 2023-09-23T18:09:31.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:31 smithi173 bash[27694]: cephadm 2023-09-23T18:09:29.845829+0000 mgr.y (mgr.14146) 166 : cephadm [INF] Adjusting osd_memory_target on smithi098 to 4060M 2023-09-23T18:09:31.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:31 smithi098 bash[27349]: cephadm 2023-09-23T18:09:29.830398+0000 mgr.y (mgr.14146) 165 : cephadm [INF] Detected new or changed devices on smithi098 2023-09-23T18:09:31.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:31 smithi098 bash[27349]: cephadm 2023-09-23T18:09:29.845829+0000 mgr.y (mgr.14146) 166 : cephadm [INF] Adjusting osd_memory_target on smithi098 to 4060M 2023-09-23T18:09:31.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:31 smithi098 bash[33599]: cephadm 2023-09-23T18:09:29.830398+0000 mgr.y (mgr.14146) 165 : cephadm [INF] Detected new or changed devices on smithi098 2023-09-23T18:09:31.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:31 smithi098 bash[33599]: cephadm 2023-09-23T18:09:29.845829+0000 mgr.y (mgr.14146) 166 : cephadm [INF] Adjusting osd_memory_target on smithi098 to 4060M 2023-09-23T18:09:32.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:32 smithi173 bash[27694]: cluster 2023-09-23T18:09:31.131810+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:32.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:32 smithi098 bash[27349]: cluster 2023-09-23T18:09:31.131810+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:32.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:32 smithi098 bash[33599]: cluster 2023-09-23T18:09:31.131810+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:34.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:34 smithi098 bash[27349]: cluster 2023-09-23T18:09:33.132365+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:34.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:34 smithi098 bash[33599]: cluster 2023-09-23T18:09:33.132365+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:34.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:34 smithi173 bash[27694]: cluster 2023-09-23T18:09:33.132365+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:36.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:36 smithi098 bash[27349]: cluster 2023-09-23T18:09:35.132908+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:36.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:36 smithi098 bash[27349]: audit 2023-09-23T18:09:35.921443+0000 mon.a (mon.0) 380 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a562afc6-ffe6-44bb-91a9-8a04018c3c6f"}]: dispatch 2023-09-23T18:09:36.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:36 smithi098 bash[27349]: audit 2023-09-23T18:09:35.923589+0000 mon.b (mon.1) 7 : audit [INF] from='client.? 172.21.15.173:0/3801162357' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a562afc6-ffe6-44bb-91a9-8a04018c3c6f"}]: dispatch 2023-09-23T18:09:36.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:36 smithi098 bash[27349]: audit 2023-09-23T18:09:35.931957+0000 mon.a (mon.0) 381 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a562afc6-ffe6-44bb-91a9-8a04018c3c6f"}]': finished 2023-09-23T18:09:36.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:36 smithi098 bash[27349]: cluster 2023-09-23T18:09:35.932137+0000 mon.a (mon.0) 382 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-09-23T18:09:36.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:36 smithi098 bash[27349]: audit 2023-09-23T18:09:35.932399+0000 mon.a (mon.0) 383 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:09:36.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:36 smithi098 bash[33599]: cluster 2023-09-23T18:09:35.132908+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:36.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:36 smithi098 bash[33599]: audit 2023-09-23T18:09:35.921443+0000 mon.a (mon.0) 380 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a562afc6-ffe6-44bb-91a9-8a04018c3c6f"}]: dispatch 2023-09-23T18:09:36.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:36 smithi098 bash[33599]: audit 2023-09-23T18:09:35.923589+0000 mon.b (mon.1) 7 : audit [INF] from='client.? 172.21.15.173:0/3801162357' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a562afc6-ffe6-44bb-91a9-8a04018c3c6f"}]: dispatch 2023-09-23T18:09:36.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:36 smithi098 bash[33599]: audit 2023-09-23T18:09:35.931957+0000 mon.a (mon.0) 381 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a562afc6-ffe6-44bb-91a9-8a04018c3c6f"}]': finished 2023-09-23T18:09:36.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:36 smithi098 bash[33599]: cluster 2023-09-23T18:09:35.932137+0000 mon.a (mon.0) 382 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-09-23T18:09:36.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:36 smithi098 bash[33599]: audit 2023-09-23T18:09:35.932399+0000 mon.a (mon.0) 383 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:09:36.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:36 smithi173 bash[27694]: cluster 2023-09-23T18:09:35.132908+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:36.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:36 smithi173 bash[27694]: audit 2023-09-23T18:09:35.921443+0000 mon.a (mon.0) 380 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a562afc6-ffe6-44bb-91a9-8a04018c3c6f"}]: dispatch 2023-09-23T18:09:36.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:36 smithi173 bash[27694]: audit 2023-09-23T18:09:35.923589+0000 mon.b (mon.1) 7 : audit [INF] from='client.? 172.21.15.173:0/3801162357' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a562afc6-ffe6-44bb-91a9-8a04018c3c6f"}]: dispatch 2023-09-23T18:09:36.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:36 smithi173 bash[27694]: audit 2023-09-23T18:09:35.931957+0000 mon.a (mon.0) 381 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a562afc6-ffe6-44bb-91a9-8a04018c3c6f"}]': finished 2023-09-23T18:09:36.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:36 smithi173 bash[27694]: cluster 2023-09-23T18:09:35.932137+0000 mon.a (mon.0) 382 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-09-23T18:09:36.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:36 smithi173 bash[27694]: audit 2023-09-23T18:09:35.932399+0000 mon.a (mon.0) 383 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:09:37.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:37 smithi173 bash[27694]: audit 2023-09-23T18:09:36.665987+0000 mon.b (mon.1) 8 : audit [DBG] from='client.? 172.21.15.173:0/1039600251' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:09:37.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:37 smithi098 bash[27349]: audit 2023-09-23T18:09:36.665987+0000 mon.b (mon.1) 8 : audit [DBG] from='client.? 172.21.15.173:0/1039600251' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:09:37.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:37 smithi098 bash[33599]: audit 2023-09-23T18:09:36.665987+0000 mon.b (mon.1) 8 : audit [DBG] from='client.? 172.21.15.173:0/1039600251' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:09:38.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:38 smithi173 bash[27694]: cluster 2023-09-23T18:09:37.133463+0000 mgr.y (mgr.14146) 170 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:38.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:38 smithi098 bash[27349]: cluster 2023-09-23T18:09:37.133463+0000 mgr.y (mgr.14146) 170 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:38.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:38 smithi098 bash[33599]: cluster 2023-09-23T18:09:37.133463+0000 mgr.y (mgr.14146) 170 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:40.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:40 smithi173 bash[27694]: cluster 2023-09-23T18:09:39.133976+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:40.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:40 smithi098 bash[27349]: cluster 2023-09-23T18:09:39.133976+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:40.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:40 smithi098 bash[33599]: cluster 2023-09-23T18:09:39.133976+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:42.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:42 smithi173 bash[27694]: cluster 2023-09-23T18:09:41.134525+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:42.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:42 smithi098 bash[27349]: cluster 2023-09-23T18:09:41.134525+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:42.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:42 smithi098 bash[33599]: cluster 2023-09-23T18:09:41.134525+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:44.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:44 smithi098 bash[33599]: cluster 2023-09-23T18:09:43.135018+0000 mgr.y (mgr.14146) 173 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:44.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:44 smithi098 bash[27349]: cluster 2023-09-23T18:09:43.135018+0000 mgr.y (mgr.14146) 173 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:44.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:44 smithi173 bash[27694]: cluster 2023-09-23T18:09:43.135018+0000 mgr.y (mgr.14146) 173 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:46.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:46 smithi173 bash[27694]: cluster 2023-09-23T18:09:45.135635+0000 mgr.y (mgr.14146) 174 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:46.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:46 smithi098 bash[27349]: cluster 2023-09-23T18:09:45.135635+0000 mgr.y (mgr.14146) 174 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:46.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:46 smithi098 bash[33599]: cluster 2023-09-23T18:09:45.135635+0000 mgr.y (mgr.14146) 174 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:48.473 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:48 smithi173 bash[27694]: cluster 2023-09-23T18:09:47.136312+0000 mgr.y (mgr.14146) 175 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:48.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:48 smithi098 bash[27349]: cluster 2023-09-23T18:09:47.136312+0000 mgr.y (mgr.14146) 175 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:48.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:48 smithi098 bash[33599]: cluster 2023-09-23T18:09:47.136312+0000 mgr.y (mgr.14146) 175 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:49.460 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:49 smithi173 bash[27694]: audit 2023-09-23T18:09:48.200194+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T18:09:49.460 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:49 smithi173 bash[27694]: audit 2023-09-23T18:09:48.201748+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:09:49.460 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:49 smithi173 bash[27694]: cephadm 2023-09-23T18:09:48.202915+0000 mgr.y (mgr.14146) 176 : cephadm [INF] Deploying daemon osd.4 on smithi173 2023-09-23T18:09:49.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:49 smithi098 bash[33599]: audit 2023-09-23T18:09:48.200194+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T18:09:49.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:49 smithi098 bash[33599]: audit 2023-09-23T18:09:48.201748+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:09:49.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:49 smithi098 bash[33599]: cephadm 2023-09-23T18:09:48.202915+0000 mgr.y (mgr.14146) 176 : cephadm [INF] Deploying daemon osd.4 on smithi173 2023-09-23T18:09:49.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:49 smithi098 bash[27349]: audit 2023-09-23T18:09:48.200194+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T18:09:49.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:49 smithi098 bash[27349]: audit 2023-09-23T18:09:48.201748+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:09:49.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:49 smithi098 bash[27349]: cephadm 2023-09-23T18:09:48.202915+0000 mgr.y (mgr.14146) 176 : cephadm [INF] Deploying daemon osd.4 on smithi173 2023-09-23T18:09:50.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:50 smithi173 bash[27694]: cluster 2023-09-23T18:09:49.136824+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:50.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:50 smithi098 bash[27349]: cluster 2023-09-23T18:09:49.136824+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:50.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:50 smithi098 bash[33599]: cluster 2023-09-23T18:09:49.136824+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:52.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:52 smithi173 bash[27694]: cluster 2023-09-23T18:09:51.137494+0000 mgr.y (mgr.14146) 178 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:52.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:52 smithi098 bash[27349]: cluster 2023-09-23T18:09:51.137494+0000 mgr.y (mgr.14146) 178 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:52.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:52 smithi098 bash[33599]: cluster 2023-09-23T18:09:51.137494+0000 mgr.y (mgr.14146) 178 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:53.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:53 smithi173 bash[27694]: audit 2023-09-23T18:09:52.240817+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:53.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:53 smithi173 bash[27694]: audit 2023-09-23T18:09:52.241302+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:09:53.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:53 smithi098 bash[27349]: audit 2023-09-23T18:09:52.240817+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:53.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:53 smithi098 bash[27349]: audit 2023-09-23T18:09:52.241302+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:09:53.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:53 smithi098 bash[33599]: audit 2023-09-23T18:09:52.240817+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:53.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:53 smithi098 bash[33599]: audit 2023-09-23T18:09:52.241302+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:09:54.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:54 smithi173 bash[27694]: cluster 2023-09-23T18:09:53.138094+0000 mgr.y (mgr.14146) 179 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:54.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:54 smithi098 bash[27349]: cluster 2023-09-23T18:09:53.138094+0000 mgr.y (mgr.14146) 179 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:54.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:54 smithi098 bash[33599]: cluster 2023-09-23T18:09:53.138094+0000 mgr.y (mgr.14146) 179 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:56.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:56 smithi173 bash[27694]: cluster 2023-09-23T18:09:55.138612+0000 mgr.y (mgr.14146) 180 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:56.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:56 smithi098 bash[27349]: cluster 2023-09-23T18:09:55.138612+0000 mgr.y (mgr.14146) 180 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:56.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:56 smithi098 bash[33599]: cluster 2023-09-23T18:09:55.138612+0000 mgr.y (mgr.14146) 180 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:58.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:58 smithi173 bash[27694]: cluster 2023-09-23T18:09:57.139165+0000 mgr.y (mgr.14146) 181 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:58.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:58 smithi173 bash[27694]: audit 2023-09-23T18:09:57.331174+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:58.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:58 smithi173 bash[27694]: audit 2023-09-23T18:09:57.606457+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:09:58.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:58 smithi173 bash[27694]: audit 2023-09-23T18:09:57.608363+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:09:58.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:58 smithi173 bash[27694]: audit 2023-09-23T18:09:57.619928+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:58.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:58 smithi098 bash[27349]: cluster 2023-09-23T18:09:57.139165+0000 mgr.y (mgr.14146) 181 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:58.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:58 smithi098 bash[27349]: audit 2023-09-23T18:09:57.331174+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:58.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:58 smithi098 bash[27349]: audit 2023-09-23T18:09:57.606457+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:09:58.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:58 smithi098 bash[27349]: audit 2023-09-23T18:09:57.608363+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:09:58.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:58 smithi098 bash[27349]: audit 2023-09-23T18:09:57.619928+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:58.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:58 smithi098 bash[33599]: cluster 2023-09-23T18:09:57.139165+0000 mgr.y (mgr.14146) 181 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:09:58.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:58 smithi098 bash[33599]: audit 2023-09-23T18:09:57.331174+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:58.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:58 smithi098 bash[33599]: audit 2023-09-23T18:09:57.606457+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:09:58.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:58 smithi098 bash[33599]: audit 2023-09-23T18:09:57.608363+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:09:58.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:58 smithi098 bash[33599]: audit 2023-09-23T18:09:57.619928+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:09:59.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:59 smithi173 bash[27694]: audit 2023-09-23T18:09:58.721329+0000 mon.a (mon.0) 392 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T18:09:59.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:09:59 smithi173 bash[27694]: audit 2023-09-23T18:09:58.724068+0000 mon.b (mon.1) 9 : audit [INF] from='osd.4 [v2:172.21.15.173:6800/165906554,v1:172.21.15.173:6801/165906554]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T18:09:59.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:59 smithi098 bash[27349]: audit 2023-09-23T18:09:58.721329+0000 mon.a (mon.0) 392 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T18:09:59.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:09:59 smithi098 bash[27349]: audit 2023-09-23T18:09:58.724068+0000 mon.b (mon.1) 9 : audit [INF] from='osd.4 [v2:172.21.15.173:6800/165906554,v1:172.21.15.173:6801/165906554]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T18:09:59.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:59 smithi098 bash[33599]: audit 2023-09-23T18:09:58.721329+0000 mon.a (mon.0) 392 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T18:09:59.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:09:59 smithi098 bash[33599]: audit 2023-09-23T18:09:58.724068+0000 mon.b (mon.1) 9 : audit [INF] from='osd.4 [v2:172.21.15.173:6800/165906554,v1:172.21.15.173:6801/165906554]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T18:10:00.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:00 smithi173 bash[27694]: cluster 2023-09-23T18:09:59.139687+0000 mgr.y (mgr.14146) 182 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:10:00.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:00 smithi173 bash[27694]: audit 2023-09-23T18:09:59.272513+0000 mon.a (mon.0) 393 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T18:10:00.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:00 smithi173 bash[27694]: cluster 2023-09-23T18:09:59.272589+0000 mon.a (mon.0) 394 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-09-23T18:10:00.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:00 smithi173 bash[27694]: audit 2023-09-23T18:09:59.272860+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:10:00.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:00 smithi173 bash[27694]: audit 2023-09-23T18:09:59.274018+0000 mon.a (mon.0) 396 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:10:00.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:00 smithi173 bash[27694]: audit 2023-09-23T18:09:59.277109+0000 mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.173:6800/165906554,v1:172.21.15.173:6801/165906554]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:10:00.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:00 smithi173 bash[27694]: cluster 2023-09-23T18:10:00.000152+0000 mon.a (mon.0) 397 : cluster [INF] overall HEALTH_OK 2023-09-23T18:10:00.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:00 smithi098 bash[27349]: cluster 2023-09-23T18:09:59.139687+0000 mgr.y (mgr.14146) 182 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:10:00.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:00 smithi098 bash[27349]: audit 2023-09-23T18:09:59.272513+0000 mon.a (mon.0) 393 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T18:10:00.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:00 smithi098 bash[27349]: cluster 2023-09-23T18:09:59.272589+0000 mon.a (mon.0) 394 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-09-23T18:10:00.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:00 smithi098 bash[27349]: audit 2023-09-23T18:09:59.272860+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:10:00.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:00 smithi098 bash[27349]: audit 2023-09-23T18:09:59.274018+0000 mon.a (mon.0) 396 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:10:00.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:00 smithi098 bash[27349]: audit 2023-09-23T18:09:59.277109+0000 mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.173:6800/165906554,v1:172.21.15.173:6801/165906554]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:10:00.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:00 smithi098 bash[27349]: cluster 2023-09-23T18:10:00.000152+0000 mon.a (mon.0) 397 : cluster [INF] overall HEALTH_OK 2023-09-23T18:10:00.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:00 smithi098 bash[33599]: cluster 2023-09-23T18:09:59.139687+0000 mgr.y (mgr.14146) 182 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:10:00.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:00 smithi098 bash[33599]: audit 2023-09-23T18:09:59.272513+0000 mon.a (mon.0) 393 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T18:10:00.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:00 smithi098 bash[33599]: cluster 2023-09-23T18:09:59.272589+0000 mon.a (mon.0) 394 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-09-23T18:10:00.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:00 smithi098 bash[33599]: audit 2023-09-23T18:09:59.272860+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:10:00.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:00 smithi098 bash[33599]: audit 2023-09-23T18:09:59.274018+0000 mon.a (mon.0) 396 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:10:00.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:00 smithi098 bash[33599]: audit 2023-09-23T18:09:59.277109+0000 mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.173:6800/165906554,v1:172.21.15.173:6801/165906554]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:10:00.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:00 smithi098 bash[33599]: cluster 2023-09-23T18:10:00.000152+0000 mon.a (mon.0) 397 : cluster [INF] overall HEALTH_OK 2023-09-23T18:10:00.789 INFO:teuthology.orchestra.run.smithi173.stdout:Created osd(s) 4 on host 'smithi173' 2023-09-23T18:10:01.429 DEBUG:teuthology.orchestra.run.smithi173:osd.4> sudo journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.4.service 2023-09-23T18:10:01.432 INFO:tasks.cephadm:Deploying osd.5 on smithi173 with /dev/vg_nvme/lv_3... 2023-09-23T18:10:01.432 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-09-23T18:10:01.635 INFO:journalctl@ceph.osd.4.smithi173.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:10:01.671 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:01 smithi173 bash[27694]: audit 2023-09-23T18:10:00.274874+0000 mon.a (mon.0) 398 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]': finished 2023-09-23T18:10:01.672 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:01 smithi173 bash[27694]: cluster 2023-09-23T18:10:00.275006+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-09-23T18:10:01.672 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:01 smithi173 bash[27694]: audit 2023-09-23T18:10:00.275851+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:10:01.672 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:01 smithi173 bash[27694]: audit 2023-09-23T18:10:00.281728+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:10:01.672 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:01 smithi173 bash[27694]: audit 2023-09-23T18:10:00.784482+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:01.672 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:01 smithi173 bash[27694]: audit 2023-09-23T18:10:00.796848+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:10:01.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:01 smithi098 bash[27349]: audit 2023-09-23T18:10:00.274874+0000 mon.a (mon.0) 398 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]': finished 2023-09-23T18:10:01.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:01 smithi098 bash[27349]: cluster 2023-09-23T18:10:00.275006+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-09-23T18:10:01.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:01 smithi098 bash[27349]: audit 2023-09-23T18:10:00.275851+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:10:01.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:01 smithi098 bash[27349]: audit 2023-09-23T18:10:00.281728+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:10:01.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:01 smithi098 bash[27349]: audit 2023-09-23T18:10:00.784482+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:01.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:01 smithi098 bash[27349]: audit 2023-09-23T18:10:00.796848+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:10:01.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:01 smithi098 bash[33599]: audit 2023-09-23T18:10:00.274874+0000 mon.a (mon.0) 398 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]': finished 2023-09-23T18:10:01.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:01 smithi098 bash[33599]: cluster 2023-09-23T18:10:00.275006+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-09-23T18:10:01.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:01 smithi098 bash[33599]: audit 2023-09-23T18:10:00.275851+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:10:01.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:01 smithi098 bash[33599]: audit 2023-09-23T18:10:00.281728+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:10:01.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:01 smithi098 bash[33599]: audit 2023-09-23T18:10:00.784482+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:01.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:01 smithi098 bash[33599]: audit 2023-09-23T18:10:00.796848+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:10:02.612 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:02 smithi173 bash[27694]: cluster 2023-09-23T18:09:59.764912+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:10:02.612 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:02 smithi173 bash[27694]: cluster 2023-09-23T18:09:59.765014+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:10:02.612 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:02 smithi173 bash[27694]: cluster 2023-09-23T18:10:01.140216+0000 mgr.y (mgr.14146) 183 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:10:02.612 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:02 smithi173 bash[27694]: audit 2023-09-23T18:10:01.287721+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:10:02.613 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:02 smithi173 bash[27694]: cluster 2023-09-23T18:10:01.294159+0000 mon.a (mon.0) 405 : cluster [INF] osd.4 [v2:172.21.15.173:6800/165906554,v1:172.21.15.173:6801/165906554] boot 2023-09-23T18:10:02.613 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:02 smithi173 bash[27694]: cluster 2023-09-23T18:10:01.294223+0000 mon.a (mon.0) 406 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-09-23T18:10:02.613 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:02 smithi173 bash[27694]: audit 2023-09-23T18:10:01.294493+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:10:02.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:02 smithi098 bash[27349]: cluster 2023-09-23T18:09:59.764912+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:10:02.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:02 smithi098 bash[27349]: cluster 2023-09-23T18:09:59.765014+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:10:02.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:02 smithi098 bash[27349]: cluster 2023-09-23T18:10:01.140216+0000 mgr.y (mgr.14146) 183 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:10:02.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:02 smithi098 bash[27349]: audit 2023-09-23T18:10:01.287721+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:10:02.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:02 smithi098 bash[27349]: cluster 2023-09-23T18:10:01.294159+0000 mon.a (mon.0) 405 : cluster [INF] osd.4 [v2:172.21.15.173:6800/165906554,v1:172.21.15.173:6801/165906554] boot 2023-09-23T18:10:02.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:02 smithi098 bash[27349]: cluster 2023-09-23T18:10:01.294223+0000 mon.a (mon.0) 406 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-09-23T18:10:02.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:02 smithi098 bash[27349]: audit 2023-09-23T18:10:01.294493+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:10:02.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:02 smithi098 bash[33599]: cluster 2023-09-23T18:09:59.764912+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:10:02.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:02 smithi098 bash[33599]: cluster 2023-09-23T18:09:59.765014+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:10:02.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:02 smithi098 bash[33599]: cluster 2023-09-23T18:10:01.140216+0000 mgr.y (mgr.14146) 183 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T18:10:02.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:02 smithi098 bash[33599]: audit 2023-09-23T18:10:01.287721+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:10:02.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:02 smithi098 bash[33599]: cluster 2023-09-23T18:10:01.294159+0000 mon.a (mon.0) 405 : cluster [INF] osd.4 [v2:172.21.15.173:6800/165906554,v1:172.21.15.173:6801/165906554] boot 2023-09-23T18:10:02.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:02 smithi098 bash[33599]: cluster 2023-09-23T18:10:01.294223+0000 mon.a (mon.0) 406 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-09-23T18:10:02.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:02 smithi098 bash[33599]: audit 2023-09-23T18:10:01.294493+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:10:03.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:03 smithi098 bash[27349]: cluster 2023-09-23T18:10:02.296475+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-09-23T18:10:03.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:03 smithi098 bash[33599]: cluster 2023-09-23T18:10:02.296475+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-09-23T18:10:03.776 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:03 smithi173 bash[27694]: cluster 2023-09-23T18:10:02.296475+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-09-23T18:10:04.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:04 smithi098 bash[27349]: cluster 2023-09-23T18:10:03.140770+0000 mgr.y (mgr.14146) 184 : cluster [DBG] pgmap v167: 1 pgs: 1 remapped+peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:04.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:04 smithi098 bash[27349]: cluster 2023-09-23T18:10:03.299876+0000 mon.a (mon.0) 409 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-09-23T18:10:04.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:04 smithi098 bash[33599]: cluster 2023-09-23T18:10:03.140770+0000 mgr.y (mgr.14146) 184 : cluster [DBG] pgmap v167: 1 pgs: 1 remapped+peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:04.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:04 smithi098 bash[33599]: cluster 2023-09-23T18:10:03.299876+0000 mon.a (mon.0) 409 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-09-23T18:10:04.722 INFO:teuthology.orchestra.run.smithi173.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-09-23T18:10:04.722 INFO:teuthology.orchestra.run.smithi173.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-09-23T18:10:04.722 INFO:teuthology.orchestra.run.smithi173.stderr: stderr: 10+0 records in 2023-09-23T18:10:04.723 INFO:teuthology.orchestra.run.smithi173.stderr:10+0 records out 2023-09-23T18:10:04.723 INFO:teuthology.orchestra.run.smithi173.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0165572 s, 633 MB/s 2023-09-23T18:10:04.723 INFO:teuthology.orchestra.run.smithi173.stderr:--> Zapping successful for: 2023-09-23T18:10:04.776 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:04 smithi173 bash[27694]: cluster 2023-09-23T18:10:03.140770+0000 mgr.y (mgr.14146) 184 : cluster [DBG] pgmap v167: 1 pgs: 1 remapped+peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:04.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:04 smithi173 bash[27694]: cluster 2023-09-23T18:10:03.299876+0000 mon.a (mon.0) 409 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-09-23T18:10:05.473 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi173:vg_nvme/lv_3 2023-09-23T18:10:05.661 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:05 smithi173 bash[27694]: cluster 2023-09-23T18:10:05.141338+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v169: 1 pgs: 1 remapped+peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:05.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:05 smithi098 bash[27349]: cluster 2023-09-23T18:10:05.141338+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v169: 1 pgs: 1 remapped+peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:05.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:05 smithi098 bash[33599]: cluster 2023-09-23T18:10:05.141338+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v169: 1 pgs: 1 remapped+peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:08.467 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:08 smithi173 bash[27694]: cluster 2023-09-23T18:10:07.141832+0000 mgr.y (mgr.14146) 186 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:08.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:08 smithi098 bash[27349]: cluster 2023-09-23T18:10:07.141832+0000 mgr.y (mgr.14146) 186 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:08.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:08 smithi098 bash[33599]: cluster 2023-09-23T18:10:07.141832+0000 mgr.y (mgr.14146) 186 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:10.503 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:10 smithi173 bash[27694]: cluster 2023-09-23T18:10:09.142364+0000 mgr.y (mgr.14146) 187 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:10.503 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:10 smithi173 bash[27694]: audit 2023-09-23T18:10:09.999116+0000 mon.a (mon.0) 410 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:10:10.503 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:10 smithi173 bash[27694]: audit 2023-09-23T18:10:10.005311+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:10:10.503 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:10 smithi173 bash[27694]: audit 2023-09-23T18:10:10.006707+0000 mon.a (mon.0) 412 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:10:10.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:10 smithi098 bash[27349]: cluster 2023-09-23T18:10:09.142364+0000 mgr.y (mgr.14146) 187 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:10.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:10 smithi098 bash[27349]: audit 2023-09-23T18:10:09.999116+0000 mon.a (mon.0) 410 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:10:10.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:10 smithi098 bash[27349]: audit 2023-09-23T18:10:10.005311+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:10:10.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:10 smithi098 bash[27349]: audit 2023-09-23T18:10:10.006707+0000 mon.a (mon.0) 412 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:10:10.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:10 smithi098 bash[33599]: cluster 2023-09-23T18:10:09.142364+0000 mgr.y (mgr.14146) 187 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:10.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:10 smithi098 bash[33599]: audit 2023-09-23T18:10:09.999116+0000 mon.a (mon.0) 410 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:10:10.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:10 smithi098 bash[33599]: audit 2023-09-23T18:10:10.005311+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:10:10.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:10 smithi098 bash[33599]: audit 2023-09-23T18:10:10.006707+0000 mon.a (mon.0) 412 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:10:11.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:11 smithi173 bash[27694]: audit 2023-09-23T18:10:09.992125+0000 mgr.y (mgr.14146) 188 : audit [DBG] from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi173:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:10:11.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:11 smithi173 bash[27694]: audit 2023-09-23T18:10:11.197026+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:11.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:11 smithi173 bash[27694]: audit 2023-09-23T18:10:11.198585+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:10:11.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:11 smithi098 bash[27349]: audit 2023-09-23T18:10:09.992125+0000 mgr.y (mgr.14146) 188 : audit [DBG] from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi173:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:10:11.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:11 smithi098 bash[27349]: audit 2023-09-23T18:10:11.197026+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:11.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:11 smithi098 bash[27349]: audit 2023-09-23T18:10:11.198585+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:10:11.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:11 smithi098 bash[33599]: audit 2023-09-23T18:10:09.992125+0000 mgr.y (mgr.14146) 188 : audit [DBG] from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi173:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:10:11.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:11 smithi098 bash[33599]: audit 2023-09-23T18:10:11.197026+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:11.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:11 smithi098 bash[33599]: audit 2023-09-23T18:10:11.198585+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:10:12.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:12 smithi173 bash[27694]: cluster 2023-09-23T18:10:11.142839+0000 mgr.y (mgr.14146) 189 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:12.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:12 smithi173 bash[27694]: cephadm 2023-09-23T18:10:11.188336+0000 mgr.y (mgr.14146) 190 : cephadm [INF] Detected new or changed devices on smithi173 2023-09-23T18:10:12.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:12 smithi173 bash[27694]: cephadm 2023-09-23T18:10:11.199649+0000 mgr.y (mgr.14146) 191 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 17265M 2023-09-23T18:10:12.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:12 smithi173 bash[27694]: audit 2023-09-23T18:10:11.204785+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:12.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:12 smithi173 bash[27694]: audit 2023-09-23T18:10:11.209411+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:10:12.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:12 smithi173 bash[27694]: audit 2023-09-23T18:10:11.210683+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:10:12.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:12 smithi173 bash[27694]: audit 2023-09-23T18:10:11.224043+0000 mon.a (mon.0) 418 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:12.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:12 smithi098 bash[27349]: cluster 2023-09-23T18:10:11.142839+0000 mgr.y (mgr.14146) 189 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:12.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:12 smithi098 bash[27349]: cephadm 2023-09-23T18:10:11.188336+0000 mgr.y (mgr.14146) 190 : cephadm [INF] Detected new or changed devices on smithi173 2023-09-23T18:10:12.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:12 smithi098 bash[27349]: cephadm 2023-09-23T18:10:11.199649+0000 mgr.y (mgr.14146) 191 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 17265M 2023-09-23T18:10:12.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:12 smithi098 bash[27349]: audit 2023-09-23T18:10:11.204785+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:12.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:12 smithi098 bash[27349]: audit 2023-09-23T18:10:11.209411+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:10:12.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:12 smithi098 bash[27349]: audit 2023-09-23T18:10:11.210683+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:10:12.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:12 smithi098 bash[27349]: audit 2023-09-23T18:10:11.224043+0000 mon.a (mon.0) 418 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:12.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:12 smithi098 bash[33599]: cluster 2023-09-23T18:10:11.142839+0000 mgr.y (mgr.14146) 189 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:12.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:12 smithi098 bash[33599]: cephadm 2023-09-23T18:10:11.188336+0000 mgr.y (mgr.14146) 190 : cephadm [INF] Detected new or changed devices on smithi173 2023-09-23T18:10:12.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:12 smithi098 bash[33599]: cephadm 2023-09-23T18:10:11.199649+0000 mgr.y (mgr.14146) 191 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 17265M 2023-09-23T18:10:12.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:12 smithi098 bash[33599]: audit 2023-09-23T18:10:11.204785+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:12.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:12 smithi098 bash[33599]: audit 2023-09-23T18:10:11.209411+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:10:12.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:12 smithi098 bash[33599]: audit 2023-09-23T18:10:11.210683+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:10:12.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:12 smithi098 bash[33599]: audit 2023-09-23T18:10:11.224043+0000 mon.a (mon.0) 418 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:14.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:14 smithi098 bash[33599]: cluster 2023-09-23T18:10:13.143400+0000 mgr.y (mgr.14146) 192 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:14.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:14 smithi098 bash[27349]: cluster 2023-09-23T18:10:13.143400+0000 mgr.y (mgr.14146) 192 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:14.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:14 smithi173 bash[27694]: cluster 2023-09-23T18:10:13.143400+0000 mgr.y (mgr.14146) 192 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:16.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:16 smithi173 bash[27694]: cluster 2023-09-23T18:10:15.143964+0000 mgr.y (mgr.14146) 193 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:16.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:16 smithi098 bash[27349]: cluster 2023-09-23T18:10:15.143964+0000 mgr.y (mgr.14146) 193 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:16.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:16 smithi098 bash[33599]: cluster 2023-09-23T18:10:15.143964+0000 mgr.y (mgr.14146) 193 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:18.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:18 smithi173 bash[27694]: cluster 2023-09-23T18:10:17.144516+0000 mgr.y (mgr.14146) 194 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:18.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:18 smithi173 bash[27694]: audit 2023-09-23T18:10:18.100818+0000 mon.a (mon.0) 419 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "add28146-8301-4b4c-9616-967e9df3a8d8"}]: dispatch 2023-09-23T18:10:18.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:18 smithi173 bash[27694]: audit 2023-09-23T18:10:18.103780+0000 mon.b (mon.1) 11 : audit [INF] from='client.? 172.21.15.173:0/4014612236' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "add28146-8301-4b4c-9616-967e9df3a8d8"}]: dispatch 2023-09-23T18:10:18.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:18 smithi173 bash[27694]: audit 2023-09-23T18:10:18.114022+0000 mon.a (mon.0) 420 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "add28146-8301-4b4c-9616-967e9df3a8d8"}]': finished 2023-09-23T18:10:18.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:18 smithi173 bash[27694]: cluster 2023-09-23T18:10:18.114179+0000 mon.a (mon.0) 421 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-09-23T18:10:18.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:18 smithi173 bash[27694]: audit 2023-09-23T18:10:18.114484+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:10:18.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:18 smithi098 bash[27349]: cluster 2023-09-23T18:10:17.144516+0000 mgr.y (mgr.14146) 194 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:18.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:18 smithi098 bash[27349]: audit 2023-09-23T18:10:18.100818+0000 mon.a (mon.0) 419 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "add28146-8301-4b4c-9616-967e9df3a8d8"}]: dispatch 2023-09-23T18:10:18.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:18 smithi098 bash[27349]: audit 2023-09-23T18:10:18.103780+0000 mon.b (mon.1) 11 : audit [INF] from='client.? 172.21.15.173:0/4014612236' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "add28146-8301-4b4c-9616-967e9df3a8d8"}]: dispatch 2023-09-23T18:10:18.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:18 smithi098 bash[27349]: audit 2023-09-23T18:10:18.114022+0000 mon.a (mon.0) 420 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "add28146-8301-4b4c-9616-967e9df3a8d8"}]': finished 2023-09-23T18:10:18.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:18 smithi098 bash[27349]: cluster 2023-09-23T18:10:18.114179+0000 mon.a (mon.0) 421 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-09-23T18:10:18.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:18 smithi098 bash[27349]: audit 2023-09-23T18:10:18.114484+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:10:18.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:18 smithi098 bash[33599]: cluster 2023-09-23T18:10:17.144516+0000 mgr.y (mgr.14146) 194 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:18.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:18 smithi098 bash[33599]: audit 2023-09-23T18:10:18.100818+0000 mon.a (mon.0) 419 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "add28146-8301-4b4c-9616-967e9df3a8d8"}]: dispatch 2023-09-23T18:10:18.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:18 smithi098 bash[33599]: audit 2023-09-23T18:10:18.103780+0000 mon.b (mon.1) 11 : audit [INF] from='client.? 172.21.15.173:0/4014612236' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "add28146-8301-4b4c-9616-967e9df3a8d8"}]: dispatch 2023-09-23T18:10:18.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:18 smithi098 bash[33599]: audit 2023-09-23T18:10:18.114022+0000 mon.a (mon.0) 420 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "add28146-8301-4b4c-9616-967e9df3a8d8"}]': finished 2023-09-23T18:10:18.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:18 smithi098 bash[33599]: cluster 2023-09-23T18:10:18.114179+0000 mon.a (mon.0) 421 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-09-23T18:10:18.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:18 smithi098 bash[33599]: audit 2023-09-23T18:10:18.114484+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:10:19.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:19 smithi173 bash[27694]: audit 2023-09-23T18:10:18.843480+0000 mon.a (mon.0) 423 : audit [DBG] from='client.? 172.21.15.173:0/179645753' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:10:19.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:19 smithi098 bash[27349]: audit 2023-09-23T18:10:18.843480+0000 mon.a (mon.0) 423 : audit [DBG] from='client.? 172.21.15.173:0/179645753' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:10:19.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:19 smithi098 bash[33599]: audit 2023-09-23T18:10:18.843480+0000 mon.a (mon.0) 423 : audit [DBG] from='client.? 172.21.15.173:0/179645753' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:10:20.525 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:20 smithi098 bash[33599]: cluster 2023-09-23T18:10:19.145140+0000 mgr.y (mgr.14146) 195 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:20.525 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:20 smithi098 bash[27349]: cluster 2023-09-23T18:10:19.145140+0000 mgr.y (mgr.14146) 195 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:20.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:20 smithi173 bash[27694]: cluster 2023-09-23T18:10:19.145140+0000 mgr.y (mgr.14146) 195 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:22.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:22 smithi173 bash[27694]: cluster 2023-09-23T18:10:21.145769+0000 mgr.y (mgr.14146) 196 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:22.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:22 smithi098 bash[27349]: cluster 2023-09-23T18:10:21.145769+0000 mgr.y (mgr.14146) 196 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:22.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:22 smithi098 bash[33599]: cluster 2023-09-23T18:10:21.145769+0000 mgr.y (mgr.14146) 196 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:24.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:24 smithi098 bash[27349]: cluster 2023-09-23T18:10:23.146472+0000 mgr.y (mgr.14146) 197 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:24.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:24 smithi098 bash[33599]: cluster 2023-09-23T18:10:23.146472+0000 mgr.y (mgr.14146) 197 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:24.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:24 smithi173 bash[27694]: cluster 2023-09-23T18:10:23.146472+0000 mgr.y (mgr.14146) 197 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:26.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:26 smithi173 bash[27694]: cluster 2023-09-23T18:10:25.147163+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:26.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:26 smithi098 bash[27349]: cluster 2023-09-23T18:10:25.147163+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:26.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:26 smithi098 bash[33599]: cluster 2023-09-23T18:10:25.147163+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:28.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:28 smithi173 bash[27694]: cluster 2023-09-23T18:10:27.147860+0000 mgr.y (mgr.14146) 199 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:28.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:28 smithi098 bash[27349]: cluster 2023-09-23T18:10:27.147860+0000 mgr.y (mgr.14146) 199 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:28.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:28 smithi098 bash[33599]: cluster 2023-09-23T18:10:27.147860+0000 mgr.y (mgr.14146) 199 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:30.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:30 smithi173 bash[27694]: cluster 2023-09-23T18:10:29.148518+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:30.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:30 smithi098 bash[27349]: cluster 2023-09-23T18:10:29.148518+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:30.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:30 smithi098 bash[33599]: cluster 2023-09-23T18:10:29.148518+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:31.517 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:31 smithi173 bash[27694]: audit 2023-09-23T18:10:30.541652+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T18:10:31.517 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:31 smithi173 bash[27694]: audit 2023-09-23T18:10:30.543550+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:10:31.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:31 smithi098 bash[27349]: audit 2023-09-23T18:10:30.541652+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T18:10:31.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:31 smithi098 bash[27349]: audit 2023-09-23T18:10:30.543550+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:10:31.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:31 smithi098 bash[33599]: audit 2023-09-23T18:10:30.541652+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T18:10:31.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:31 smithi098 bash[33599]: audit 2023-09-23T18:10:30.543550+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:10:32.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:32 smithi173 bash[27694]: cephadm 2023-09-23T18:10:30.544923+0000 mgr.y (mgr.14146) 201 : cephadm [INF] Deploying daemon osd.5 on smithi173 2023-09-23T18:10:32.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:32 smithi173 bash[27694]: cluster 2023-09-23T18:10:31.149223+0000 mgr.y (mgr.14146) 202 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:32.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:32 smithi098 bash[27349]: cephadm 2023-09-23T18:10:30.544923+0000 mgr.y (mgr.14146) 201 : cephadm [INF] Deploying daemon osd.5 on smithi173 2023-09-23T18:10:32.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:32 smithi098 bash[27349]: cluster 2023-09-23T18:10:31.149223+0000 mgr.y (mgr.14146) 202 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:32.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:32 smithi098 bash[33599]: cephadm 2023-09-23T18:10:30.544923+0000 mgr.y (mgr.14146) 201 : cephadm [INF] Deploying daemon osd.5 on smithi173 2023-09-23T18:10:32.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:32 smithi098 bash[33599]: cluster 2023-09-23T18:10:31.149223+0000 mgr.y (mgr.14146) 202 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:34.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:34 smithi098 bash[33599]: cluster 2023-09-23T18:10:33.149838+0000 mgr.y (mgr.14146) 203 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:34.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:34 smithi098 bash[27349]: cluster 2023-09-23T18:10:33.149838+0000 mgr.y (mgr.14146) 203 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:34.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:34 smithi173 bash[27694]: cluster 2023-09-23T18:10:33.149838+0000 mgr.y (mgr.14146) 203 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:36.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:35 smithi173 bash[27694]: audit 2023-09-23T18:10:34.777630+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:35 smithi173 bash[27694]: audit 2023-09-23T18:10:34.804545+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:10:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:35 smithi173 bash[27694]: cluster 2023-09-23T18:10:35.150429+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:36.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:35 smithi098 bash[27349]: audit 2023-09-23T18:10:34.777630+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:36.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:35 smithi098 bash[27349]: audit 2023-09-23T18:10:34.804545+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:10:36.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:35 smithi098 bash[27349]: cluster 2023-09-23T18:10:35.150429+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:36.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:35 smithi098 bash[33599]: audit 2023-09-23T18:10:34.777630+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:36.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:35 smithi098 bash[33599]: audit 2023-09-23T18:10:34.804545+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:10:36.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:35 smithi098 bash[33599]: cluster 2023-09-23T18:10:35.150429+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:38.476 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:38 smithi173 bash[27694]: cluster 2023-09-23T18:10:37.150914+0000 mgr.y (mgr.14146) 205 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:38.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:38 smithi098 bash[33599]: cluster 2023-09-23T18:10:37.150914+0000 mgr.y (mgr.14146) 205 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:38.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:38 smithi098 bash[27349]: cluster 2023-09-23T18:10:37.150914+0000 mgr.y (mgr.14146) 205 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:40.506 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:40 smithi173 bash[27694]: cluster 2023-09-23T18:10:39.151454+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:40.507 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:40 smithi173 bash[27694]: audit 2023-09-23T18:10:39.977090+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:40.507 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:40 smithi173 bash[27694]: audit 2023-09-23T18:10:39.978528+0000 mon.a (mon.0) 429 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:10:40.507 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:40 smithi173 bash[27694]: audit 2023-09-23T18:10:39.979835+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:10:40.507 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:40 smithi173 bash[27694]: audit 2023-09-23T18:10:39.988408+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:40.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:40 smithi098 bash[33599]: cluster 2023-09-23T18:10:39.151454+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:40.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:40 smithi098 bash[33599]: audit 2023-09-23T18:10:39.977090+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:40.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:40 smithi098 bash[33599]: audit 2023-09-23T18:10:39.978528+0000 mon.a (mon.0) 429 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:10:40.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:40 smithi098 bash[33599]: audit 2023-09-23T18:10:39.979835+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:10:40.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:40 smithi098 bash[33599]: audit 2023-09-23T18:10:39.988408+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:40.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:40 smithi098 bash[27349]: cluster 2023-09-23T18:10:39.151454+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:40.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:40 smithi098 bash[27349]: audit 2023-09-23T18:10:39.977090+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:40.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:40 smithi098 bash[27349]: audit 2023-09-23T18:10:39.978528+0000 mon.a (mon.0) 429 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:10:40.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:40 smithi098 bash[27349]: audit 2023-09-23T18:10:39.979835+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:10:40.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:40 smithi098 bash[27349]: audit 2023-09-23T18:10:39.988408+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:42.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:42 smithi173 bash[27694]: cluster 2023-09-23T18:10:41.151988+0000 mgr.y (mgr.14146) 207 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:42.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:42 smithi173 bash[27694]: audit 2023-09-23T18:10:41.464666+0000 mon.a (mon.0) 432 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T18:10:42.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:42 smithi173 bash[27694]: audit 2023-09-23T18:10:41.467747+0000 mon.b (mon.1) 12 : audit [INF] from='osd.5 [v2:172.21.15.173:6808/2894909344,v1:172.21.15.173:6809/2894909344]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T18:10:42.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:42 smithi098 bash[33599]: cluster 2023-09-23T18:10:41.151988+0000 mgr.y (mgr.14146) 207 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:42.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:42 smithi098 bash[33599]: audit 2023-09-23T18:10:41.464666+0000 mon.a (mon.0) 432 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T18:10:42.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:42 smithi098 bash[33599]: audit 2023-09-23T18:10:41.467747+0000 mon.b (mon.1) 12 : audit [INF] from='osd.5 [v2:172.21.15.173:6808/2894909344,v1:172.21.15.173:6809/2894909344]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T18:10:42.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:42 smithi098 bash[27349]: cluster 2023-09-23T18:10:41.151988+0000 mgr.y (mgr.14146) 207 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:42.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:42 smithi098 bash[27349]: audit 2023-09-23T18:10:41.464666+0000 mon.a (mon.0) 432 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T18:10:42.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:42 smithi098 bash[27349]: audit 2023-09-23T18:10:41.467747+0000 mon.b (mon.1) 12 : audit [INF] from='osd.5 [v2:172.21.15.173:6808/2894909344,v1:172.21.15.173:6809/2894909344]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T18:10:43.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:43 smithi173 bash[27694]: audit 2023-09-23T18:10:42.223731+0000 mon.a (mon.0) 433 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T18:10:43.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:43 smithi173 bash[27694]: cluster 2023-09-23T18:10:42.223880+0000 mon.a (mon.0) 434 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-09-23T18:10:43.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:43 smithi173 bash[27694]: audit 2023-09-23T18:10:42.224508+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:10:43.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:43 smithi173 bash[27694]: audit 2023-09-23T18:10:42.229156+0000 mon.b (mon.1) 13 : audit [INF] from='osd.5 [v2:172.21.15.173:6808/2894909344,v1:172.21.15.173:6809/2894909344]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:10:43.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:43 smithi173 bash[27694]: audit 2023-09-23T18:10:42.234387+0000 mon.a (mon.0) 436 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:10:43.542 INFO:teuthology.orchestra.run.smithi173.stdout:Created osd(s) 5 on host 'smithi173' 2023-09-23T18:10:43.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:43 smithi098 bash[27349]: audit 2023-09-23T18:10:42.223731+0000 mon.a (mon.0) 433 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T18:10:43.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:43 smithi098 bash[27349]: cluster 2023-09-23T18:10:42.223880+0000 mon.a (mon.0) 434 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-09-23T18:10:43.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:43 smithi098 bash[27349]: audit 2023-09-23T18:10:42.224508+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:10:43.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:43 smithi098 bash[27349]: audit 2023-09-23T18:10:42.229156+0000 mon.b (mon.1) 13 : audit [INF] from='osd.5 [v2:172.21.15.173:6808/2894909344,v1:172.21.15.173:6809/2894909344]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:10:43.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:43 smithi098 bash[27349]: audit 2023-09-23T18:10:42.234387+0000 mon.a (mon.0) 436 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:10:43.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:43 smithi098 bash[33599]: audit 2023-09-23T18:10:42.223731+0000 mon.a (mon.0) 433 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T18:10:43.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:43 smithi098 bash[33599]: cluster 2023-09-23T18:10:42.223880+0000 mon.a (mon.0) 434 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-09-23T18:10:43.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:43 smithi098 bash[33599]: audit 2023-09-23T18:10:42.224508+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:10:43.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:43 smithi098 bash[33599]: audit 2023-09-23T18:10:42.229156+0000 mon.b (mon.1) 13 : audit [INF] from='osd.5 [v2:172.21.15.173:6808/2894909344,v1:172.21.15.173:6809/2894909344]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:10:43.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:43 smithi098 bash[33599]: audit 2023-09-23T18:10:42.234387+0000 mon.a (mon.0) 436 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:10:44.225 DEBUG:teuthology.orchestra.run.smithi173:osd.5> sudo journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.5.service 2023-09-23T18:10:44.227 INFO:tasks.cephadm:Deploying osd.6 on smithi173 with /dev/vg_nvme/lv_2... 2023-09-23T18:10:44.227 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-09-23T18:10:44.397 INFO:journalctl@ceph.osd.5.smithi173.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:10:44.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:44 smithi173 bash[27694]: cluster 2023-09-23T18:10:43.152528+0000 mgr.y (mgr.14146) 208 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:44.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:44 smithi173 bash[27694]: audit 2023-09-23T18:10:43.239460+0000 mon.a (mon.0) 437 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]': finished 2023-09-23T18:10:44.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:44 smithi173 bash[27694]: cluster 2023-09-23T18:10:43.239606+0000 mon.a (mon.0) 438 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-09-23T18:10:44.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:44 smithi173 bash[27694]: audit 2023-09-23T18:10:43.240274+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:10:44.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:44 smithi173 bash[27694]: audit 2023-09-23T18:10:43.246544+0000 mon.a (mon.0) 440 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:10:44.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:44 smithi173 bash[27694]: audit 2023-09-23T18:10:43.537134+0000 mon.a (mon.0) 441 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:44.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:44 smithi173 bash[27694]: audit 2023-09-23T18:10:43.565813+0000 mon.a (mon.0) 442 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:10:44.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:44 smithi098 bash[27349]: cluster 2023-09-23T18:10:43.152528+0000 mgr.y (mgr.14146) 208 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:44.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:44 smithi098 bash[27349]: audit 2023-09-23T18:10:43.239460+0000 mon.a (mon.0) 437 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]': finished 2023-09-23T18:10:44.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:44 smithi098 bash[27349]: cluster 2023-09-23T18:10:43.239606+0000 mon.a (mon.0) 438 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-09-23T18:10:44.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:44 smithi098 bash[27349]: audit 2023-09-23T18:10:43.240274+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:10:44.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:44 smithi098 bash[27349]: audit 2023-09-23T18:10:43.246544+0000 mon.a (mon.0) 440 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:10:44.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:44 smithi098 bash[27349]: audit 2023-09-23T18:10:43.537134+0000 mon.a (mon.0) 441 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:44.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:44 smithi098 bash[27349]: audit 2023-09-23T18:10:43.565813+0000 mon.a (mon.0) 442 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:10:44.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:44 smithi098 bash[33599]: cluster 2023-09-23T18:10:43.152528+0000 mgr.y (mgr.14146) 208 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:44.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:44 smithi098 bash[33599]: audit 2023-09-23T18:10:43.239460+0000 mon.a (mon.0) 437 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]': finished 2023-09-23T18:10:44.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:44 smithi098 bash[33599]: cluster 2023-09-23T18:10:43.239606+0000 mon.a (mon.0) 438 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-09-23T18:10:44.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:44 smithi098 bash[33599]: audit 2023-09-23T18:10:43.240274+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:10:44.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:44 smithi098 bash[33599]: audit 2023-09-23T18:10:43.246544+0000 mon.a (mon.0) 440 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:10:44.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:44 smithi098 bash[33599]: audit 2023-09-23T18:10:43.537134+0000 mon.a (mon.0) 441 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:44.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:44 smithi098 bash[33599]: audit 2023-09-23T18:10:43.565813+0000 mon.a (mon.0) 442 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:10:45.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:45 smithi173 bash[27694]: cluster 2023-09-23T18:10:42.490596+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:10:45.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:45 smithi173 bash[27694]: cluster 2023-09-23T18:10:42.490684+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:10:45.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:45 smithi173 bash[27694]: cluster 2023-09-23T18:10:44.249240+0000 mon.a (mon.0) 443 : cluster [INF] osd.5 [v2:172.21.15.173:6808/2894909344,v1:172.21.15.173:6809/2894909344] boot 2023-09-23T18:10:45.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:45 smithi173 bash[27694]: cluster 2023-09-23T18:10:44.249337+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-09-23T18:10:45.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:45 smithi173 bash[27694]: audit 2023-09-23T18:10:44.249973+0000 mon.a (mon.0) 445 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:10:45.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:45 smithi098 bash[27349]: cluster 2023-09-23T18:10:42.490596+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:10:45.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:45 smithi098 bash[27349]: cluster 2023-09-23T18:10:42.490684+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:10:45.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:45 smithi098 bash[27349]: cluster 2023-09-23T18:10:44.249240+0000 mon.a (mon.0) 443 : cluster [INF] osd.5 [v2:172.21.15.173:6808/2894909344,v1:172.21.15.173:6809/2894909344] boot 2023-09-23T18:10:45.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:45 smithi098 bash[27349]: cluster 2023-09-23T18:10:44.249337+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-09-23T18:10:45.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:45 smithi098 bash[27349]: audit 2023-09-23T18:10:44.249973+0000 mon.a (mon.0) 445 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:10:45.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:45 smithi098 bash[33599]: cluster 2023-09-23T18:10:42.490596+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:10:45.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:45 smithi098 bash[33599]: cluster 2023-09-23T18:10:42.490684+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:10:45.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:45 smithi098 bash[33599]: cluster 2023-09-23T18:10:44.249240+0000 mon.a (mon.0) 443 : cluster [INF] osd.5 [v2:172.21.15.173:6808/2894909344,v1:172.21.15.173:6809/2894909344] boot 2023-09-23T18:10:45.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:45 smithi098 bash[33599]: cluster 2023-09-23T18:10:44.249337+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-09-23T18:10:45.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:45 smithi098 bash[33599]: audit 2023-09-23T18:10:44.249973+0000 mon.a (mon.0) 445 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:10:46.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:46 smithi173 bash[27694]: cluster 2023-09-23T18:10:45.153075+0000 mgr.y (mgr.14146) 209 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:46.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:46 smithi173 bash[27694]: cluster 2023-09-23T18:10:45.264178+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-09-23T18:10:46.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:46 smithi098 bash[27349]: cluster 2023-09-23T18:10:45.153075+0000 mgr.y (mgr.14146) 209 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:46.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:46 smithi098 bash[27349]: cluster 2023-09-23T18:10:45.264178+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-09-23T18:10:46.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:46 smithi098 bash[33599]: cluster 2023-09-23T18:10:45.153075+0000 mgr.y (mgr.14146) 209 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T18:10:46.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:46 smithi098 bash[33599]: cluster 2023-09-23T18:10:45.264178+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-09-23T18:10:47.331 INFO:teuthology.orchestra.run.smithi173.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-09-23T18:10:47.357 INFO:teuthology.orchestra.run.smithi173.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-09-23T18:10:47.357 INFO:teuthology.orchestra.run.smithi173.stderr: stderr: 10+0 records in 2023-09-23T18:10:47.357 INFO:teuthology.orchestra.run.smithi173.stderr:10+0 records out 2023-09-23T18:10:47.357 INFO:teuthology.orchestra.run.smithi173.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0173429 s, 605 MB/s 2023-09-23T18:10:47.357 INFO:teuthology.orchestra.run.smithi173.stderr:--> Zapping successful for: 2023-09-23T18:10:47.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:47 smithi173 bash[27694]: cluster 2023-09-23T18:10:46.266215+0000 mon.a (mon.0) 447 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-09-23T18:10:47.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:47 smithi098 bash[27349]: cluster 2023-09-23T18:10:46.266215+0000 mon.a (mon.0) 447 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-09-23T18:10:47.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:47 smithi098 bash[33599]: cluster 2023-09-23T18:10:46.266215+0000 mon.a (mon.0) 447 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-09-23T18:10:48.047 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi173:vg_nvme/lv_2 2023-09-23T18:10:48.504 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:48 smithi173 bash[27694]: cluster 2023-09-23T18:10:47.153714+0000 mgr.y (mgr.14146) 210 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:10:48.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:48 smithi098 bash[27349]: cluster 2023-09-23T18:10:47.153714+0000 mgr.y (mgr.14146) 210 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:10:48.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:48 smithi098 bash[33599]: cluster 2023-09-23T18:10:47.153714+0000 mgr.y (mgr.14146) 210 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:10:50.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:50 smithi098 bash[27349]: cluster 2023-09-23T18:10:49.154329+0000 mgr.y (mgr.14146) 211 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:10:50.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:50 smithi098 bash[33599]: cluster 2023-09-23T18:10:49.154329+0000 mgr.y (mgr.14146) 211 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:10:50.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:50 smithi173 bash[27694]: cluster 2023-09-23T18:10:49.154329+0000 mgr.y (mgr.14146) 211 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:10:52.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:52 smithi173 bash[27694]: cluster 2023-09-23T18:10:51.154894+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:10:52.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:52 smithi098 bash[27349]: cluster 2023-09-23T18:10:51.154894+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:10:52.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:52 smithi098 bash[33599]: cluster 2023-09-23T18:10:51.154894+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:10:53.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:53 smithi173 bash[27694]: audit 2023-09-23T18:10:52.671907+0000 mon.a (mon.0) 448 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:10:53.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:53 smithi173 bash[27694]: audit 2023-09-23T18:10:52.676892+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:10:53.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:53 smithi173 bash[27694]: audit 2023-09-23T18:10:52.678394+0000 mon.a (mon.0) 450 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:10:53.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:53 smithi098 bash[33599]: audit 2023-09-23T18:10:52.671907+0000 mon.a (mon.0) 448 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:10:53.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:53 smithi098 bash[33599]: audit 2023-09-23T18:10:52.676892+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:10:53.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:53 smithi098 bash[33599]: audit 2023-09-23T18:10:52.678394+0000 mon.a (mon.0) 450 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:10:53.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:53 smithi098 bash[27349]: audit 2023-09-23T18:10:52.671907+0000 mon.a (mon.0) 448 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:10:53.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:53 smithi098 bash[27349]: audit 2023-09-23T18:10:52.676892+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:10:53.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:53 smithi098 bash[27349]: audit 2023-09-23T18:10:52.678394+0000 mon.a (mon.0) 450 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:10:54.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:54 smithi098 bash[33599]: audit 2023-09-23T18:10:52.666759+0000 mgr.y (mgr.14146) 213 : audit [DBG] from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi173:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:10:54.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:54 smithi098 bash[33599]: cluster 2023-09-23T18:10:53.155540+0000 mgr.y (mgr.14146) 214 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:10:54.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:54 smithi098 bash[33599]: audit 2023-09-23T18:10:53.926464+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:54.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:54 smithi098 bash[33599]: audit 2023-09-23T18:10:53.928275+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:10:54.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:54 smithi098 bash[33599]: audit 2023-09-23T18:10:53.929659+0000 mon.a (mon.0) 453 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:10:54.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:54 smithi098 bash[33599]: audit 2023-09-23T18:10:53.937534+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:54.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:54 smithi098 bash[33599]: audit 2023-09-23T18:10:53.939204+0000 mon.a (mon.0) 455 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:10:54.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:54 smithi098 bash[33599]: audit 2023-09-23T18:10:53.940468+0000 mon.a (mon.0) 456 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:10:54.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:54 smithi098 bash[33599]: audit 2023-09-23T18:10:53.950378+0000 mon.a (mon.0) 457 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:54.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:54 smithi098 bash[27349]: audit 2023-09-23T18:10:52.666759+0000 mgr.y (mgr.14146) 213 : audit [DBG] from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi173:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:10:54.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:54 smithi098 bash[27349]: cluster 2023-09-23T18:10:53.155540+0000 mgr.y (mgr.14146) 214 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:10:54.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:54 smithi098 bash[27349]: audit 2023-09-23T18:10:53.926464+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:54.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:54 smithi098 bash[27349]: audit 2023-09-23T18:10:53.928275+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:10:54.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:54 smithi098 bash[27349]: audit 2023-09-23T18:10:53.929659+0000 mon.a (mon.0) 453 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:10:54.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:54 smithi098 bash[27349]: audit 2023-09-23T18:10:53.937534+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:54.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:54 smithi098 bash[27349]: audit 2023-09-23T18:10:53.939204+0000 mon.a (mon.0) 455 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:10:54.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:54 smithi098 bash[27349]: audit 2023-09-23T18:10:53.940468+0000 mon.a (mon.0) 456 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:10:54.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:54 smithi098 bash[27349]: audit 2023-09-23T18:10:53.950378+0000 mon.a (mon.0) 457 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:54.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:54 smithi173 bash[27694]: audit 2023-09-23T18:10:52.666759+0000 mgr.y (mgr.14146) 213 : audit [DBG] from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi173:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:10:54.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:54 smithi173 bash[27694]: cluster 2023-09-23T18:10:53.155540+0000 mgr.y (mgr.14146) 214 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:10:54.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:54 smithi173 bash[27694]: audit 2023-09-23T18:10:53.926464+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:54.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:54 smithi173 bash[27694]: audit 2023-09-23T18:10:53.928275+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:10:54.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:54 smithi173 bash[27694]: audit 2023-09-23T18:10:53.929659+0000 mon.a (mon.0) 453 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:10:54.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:54 smithi173 bash[27694]: audit 2023-09-23T18:10:53.937534+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:54.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:54 smithi173 bash[27694]: audit 2023-09-23T18:10:53.939204+0000 mon.a (mon.0) 455 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:10:54.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:54 smithi173 bash[27694]: audit 2023-09-23T18:10:53.940468+0000 mon.a (mon.0) 456 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:10:54.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:54 smithi173 bash[27694]: audit 2023-09-23T18:10:53.950378+0000 mon.a (mon.0) 457 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:10:55.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:55 smithi098 bash[33599]: cephadm 2023-09-23T18:10:53.916867+0000 mgr.y (mgr.14146) 215 : cephadm [INF] Detected new or changed devices on smithi173 2023-09-23T18:10:55.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:55 smithi098 bash[33599]: cephadm 2023-09-23T18:10:53.930723+0000 mgr.y (mgr.14146) 216 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 8632M 2023-09-23T18:10:55.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:55 smithi098 bash[27349]: cephadm 2023-09-23T18:10:53.916867+0000 mgr.y (mgr.14146) 215 : cephadm [INF] Detected new or changed devices on smithi173 2023-09-23T18:10:55.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:55 smithi098 bash[27349]: cephadm 2023-09-23T18:10:53.930723+0000 mgr.y (mgr.14146) 216 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 8632M 2023-09-23T18:10:55.776 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:55 smithi173 bash[27694]: cephadm 2023-09-23T18:10:53.916867+0000 mgr.y (mgr.14146) 215 : cephadm [INF] Detected new or changed devices on smithi173 2023-09-23T18:10:55.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:55 smithi173 bash[27694]: cephadm 2023-09-23T18:10:53.930723+0000 mgr.y (mgr.14146) 216 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 8632M 2023-09-23T18:10:56.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:56 smithi098 bash[27349]: cluster 2023-09-23T18:10:55.156136+0000 mgr.y (mgr.14146) 217 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:10:56.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:56 smithi098 bash[33599]: cluster 2023-09-23T18:10:55.156136+0000 mgr.y (mgr.14146) 217 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:10:56.776 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:56 smithi173 bash[27694]: cluster 2023-09-23T18:10:55.156136+0000 mgr.y (mgr.14146) 217 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:10:57.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:10:57 smithi098 bash[27349]: cluster 2023-09-23T18:10:57.156686+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:10:57.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:10:57 smithi098 bash[33599]: cluster 2023-09-23T18:10:57.156686+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:10:57.776 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:10:57 smithi173 bash[27694]: cluster 2023-09-23T18:10:57.156686+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:00.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:00 smithi173 bash[27694]: cluster 2023-09-23T18:10:59.157219+0000 mgr.y (mgr.14146) 219 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:00.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:00 smithi098 bash[27349]: cluster 2023-09-23T18:10:59.157219+0000 mgr.y (mgr.14146) 219 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:00.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:00 smithi098 bash[33599]: cluster 2023-09-23T18:10:59.157219+0000 mgr.y (mgr.14146) 219 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:01.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:01 smithi173 bash[27694]: audit 2023-09-23T18:11:00.590703+0000 mon.a (mon.0) 458 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f30854de-bc26-465d-8b0e-a9211ae2d9fa"}]: dispatch 2023-09-23T18:11:01.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:01 smithi173 bash[27694]: audit 2023-09-23T18:11:00.593936+0000 mon.b (mon.1) 14 : audit [INF] from='client.? 172.21.15.173:0/1004819427' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f30854de-bc26-465d-8b0e-a9211ae2d9fa"}]: dispatch 2023-09-23T18:11:01.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:01 smithi173 bash[27694]: audit 2023-09-23T18:11:00.602488+0000 mon.a (mon.0) 459 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f30854de-bc26-465d-8b0e-a9211ae2d9fa"}]': finished 2023-09-23T18:11:01.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:01 smithi173 bash[27694]: cluster 2023-09-23T18:11:00.602742+0000 mon.a (mon.0) 460 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-09-23T18:11:01.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:01 smithi173 bash[27694]: audit 2023-09-23T18:11:00.603009+0000 mon.a (mon.0) 461 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:11:01.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:01 smithi098 bash[27349]: audit 2023-09-23T18:11:00.590703+0000 mon.a (mon.0) 458 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f30854de-bc26-465d-8b0e-a9211ae2d9fa"}]: dispatch 2023-09-23T18:11:01.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:01 smithi098 bash[27349]: audit 2023-09-23T18:11:00.593936+0000 mon.b (mon.1) 14 : audit [INF] from='client.? 172.21.15.173:0/1004819427' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f30854de-bc26-465d-8b0e-a9211ae2d9fa"}]: dispatch 2023-09-23T18:11:01.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:01 smithi098 bash[27349]: audit 2023-09-23T18:11:00.602488+0000 mon.a (mon.0) 459 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f30854de-bc26-465d-8b0e-a9211ae2d9fa"}]': finished 2023-09-23T18:11:01.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:01 smithi098 bash[27349]: cluster 2023-09-23T18:11:00.602742+0000 mon.a (mon.0) 460 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-09-23T18:11:01.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:01 smithi098 bash[27349]: audit 2023-09-23T18:11:00.603009+0000 mon.a (mon.0) 461 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:11:01.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:01 smithi098 bash[33599]: audit 2023-09-23T18:11:00.590703+0000 mon.a (mon.0) 458 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f30854de-bc26-465d-8b0e-a9211ae2d9fa"}]: dispatch 2023-09-23T18:11:01.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:01 smithi098 bash[33599]: audit 2023-09-23T18:11:00.593936+0000 mon.b (mon.1) 14 : audit [INF] from='client.? 172.21.15.173:0/1004819427' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f30854de-bc26-465d-8b0e-a9211ae2d9fa"}]: dispatch 2023-09-23T18:11:01.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:01 smithi098 bash[33599]: audit 2023-09-23T18:11:00.602488+0000 mon.a (mon.0) 459 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f30854de-bc26-465d-8b0e-a9211ae2d9fa"}]': finished 2023-09-23T18:11:01.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:01 smithi098 bash[33599]: cluster 2023-09-23T18:11:00.602742+0000 mon.a (mon.0) 460 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-09-23T18:11:01.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:01 smithi098 bash[33599]: audit 2023-09-23T18:11:00.603009+0000 mon.a (mon.0) 461 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:11:02.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:02 smithi173 bash[27694]: cluster 2023-09-23T18:11:01.157735+0000 mgr.y (mgr.14146) 220 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:02.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:02 smithi173 bash[27694]: audit 2023-09-23T18:11:01.271002+0000 mon.b (mon.1) 15 : audit [DBG] from='client.? 172.21.15.173:0/412018844' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:11:02.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:02 smithi098 bash[27349]: cluster 2023-09-23T18:11:01.157735+0000 mgr.y (mgr.14146) 220 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:02.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:02 smithi098 bash[27349]: audit 2023-09-23T18:11:01.271002+0000 mon.b (mon.1) 15 : audit [DBG] from='client.? 172.21.15.173:0/412018844' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:11:02.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:02 smithi098 bash[33599]: cluster 2023-09-23T18:11:01.157735+0000 mgr.y (mgr.14146) 220 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:02.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:02 smithi098 bash[33599]: audit 2023-09-23T18:11:01.271002+0000 mon.b (mon.1) 15 : audit [DBG] from='client.? 172.21.15.173:0/412018844' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:11:04.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:04 smithi173 bash[27694]: cluster 2023-09-23T18:11:03.158341+0000 mgr.y (mgr.14146) 221 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:04.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:04 smithi098 bash[27349]: cluster 2023-09-23T18:11:03.158341+0000 mgr.y (mgr.14146) 221 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:04.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:04 smithi098 bash[33599]: cluster 2023-09-23T18:11:03.158341+0000 mgr.y (mgr.14146) 221 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:06.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:06 smithi173 bash[27694]: cluster 2023-09-23T18:11:05.158878+0000 mgr.y (mgr.14146) 222 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:06.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:06 smithi098 bash[27349]: cluster 2023-09-23T18:11:05.158878+0000 mgr.y (mgr.14146) 222 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:06.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:06 smithi098 bash[33599]: cluster 2023-09-23T18:11:05.158878+0000 mgr.y (mgr.14146) 222 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:08.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:08 smithi173 bash[27694]: cluster 2023-09-23T18:11:07.159459+0000 mgr.y (mgr.14146) 223 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:08.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:08 smithi098 bash[27349]: cluster 2023-09-23T18:11:07.159459+0000 mgr.y (mgr.14146) 223 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:08.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:08 smithi098 bash[33599]: cluster 2023-09-23T18:11:07.159459+0000 mgr.y (mgr.14146) 223 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:10.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:10 smithi173 bash[27694]: cluster 2023-09-23T18:11:09.160032+0000 mgr.y (mgr.14146) 224 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:10.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:10 smithi098 bash[27349]: cluster 2023-09-23T18:11:09.160032+0000 mgr.y (mgr.14146) 224 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:10.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:10 smithi098 bash[33599]: cluster 2023-09-23T18:11:09.160032+0000 mgr.y (mgr.14146) 224 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:12.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:12 smithi173 bash[27694]: cluster 2023-09-23T18:11:11.160586+0000 mgr.y (mgr.14146) 225 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:12.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:12 smithi098 bash[27349]: cluster 2023-09-23T18:11:11.160586+0000 mgr.y (mgr.14146) 225 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:12.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:12 smithi098 bash[33599]: cluster 2023-09-23T18:11:11.160586+0000 mgr.y (mgr.14146) 225 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:13.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:13 smithi173 bash[27694]: audit 2023-09-23T18:11:12.678519+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T18:11:13.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:13 smithi173 bash[27694]: audit 2023-09-23T18:11:12.680198+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:11:13.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:13 smithi098 bash[27349]: audit 2023-09-23T18:11:12.678519+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T18:11:13.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:13 smithi098 bash[27349]: audit 2023-09-23T18:11:12.680198+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:11:13.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:13 smithi098 bash[33599]: audit 2023-09-23T18:11:12.678519+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T18:11:13.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:13 smithi098 bash[33599]: audit 2023-09-23T18:11:12.680198+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:11:14.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:14 smithi173 bash[27694]: cephadm 2023-09-23T18:11:12.681677+0000 mgr.y (mgr.14146) 226 : cephadm [INF] Deploying daemon osd.6 on smithi173 2023-09-23T18:11:14.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:14 smithi173 bash[27694]: cluster 2023-09-23T18:11:13.161178+0000 mgr.y (mgr.14146) 227 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:14.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:14 smithi098 bash[27349]: cephadm 2023-09-23T18:11:12.681677+0000 mgr.y (mgr.14146) 226 : cephadm [INF] Deploying daemon osd.6 on smithi173 2023-09-23T18:11:14.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:14 smithi098 bash[27349]: cluster 2023-09-23T18:11:13.161178+0000 mgr.y (mgr.14146) 227 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:14.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:14 smithi098 bash[33599]: cephadm 2023-09-23T18:11:12.681677+0000 mgr.y (mgr.14146) 226 : cephadm [INF] Deploying daemon osd.6 on smithi173 2023-09-23T18:11:14.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:14 smithi098 bash[33599]: cluster 2023-09-23T18:11:13.161178+0000 mgr.y (mgr.14146) 227 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:16.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:16 smithi173 bash[27694]: cluster 2023-09-23T18:11:15.161673+0000 mgr.y (mgr.14146) 228 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:16.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:16 smithi098 bash[27349]: cluster 2023-09-23T18:11:15.161673+0000 mgr.y (mgr.14146) 228 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:16.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:16 smithi098 bash[33599]: cluster 2023-09-23T18:11:15.161673+0000 mgr.y (mgr.14146) 228 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:18.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:17 smithi173 bash[27694]: audit 2023-09-23T18:11:16.858360+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:11:18.478 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:17 smithi173 bash[27694]: audit 2023-09-23T18:11:16.883188+0000 mon.a (mon.0) 465 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:11:18.478 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:17 smithi173 bash[27694]: cluster 2023-09-23T18:11:17.162231+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:18.479 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:17 smithi098 bash[27349]: audit 2023-09-23T18:11:16.858360+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:11:18.479 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:17 smithi098 bash[27349]: audit 2023-09-23T18:11:16.883188+0000 mon.a (mon.0) 465 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:11:18.479 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:17 smithi098 bash[27349]: cluster 2023-09-23T18:11:17.162231+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:18.479 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:17 smithi098 bash[33599]: audit 2023-09-23T18:11:16.858360+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:11:18.479 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:17 smithi098 bash[33599]: audit 2023-09-23T18:11:16.883188+0000 mon.a (mon.0) 465 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:11:18.479 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:17 smithi098 bash[33599]: cluster 2023-09-23T18:11:17.162231+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:20.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:20 smithi173 bash[27694]: cluster 2023-09-23T18:11:19.162780+0000 mgr.y (mgr.14146) 230 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:20.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:20 smithi098 bash[27349]: cluster 2023-09-23T18:11:19.162780+0000 mgr.y (mgr.14146) 230 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:20.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:20 smithi098 bash[33599]: cluster 2023-09-23T18:11:19.162780+0000 mgr.y (mgr.14146) 230 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:22.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:22 smithi173 bash[27694]: cluster 2023-09-23T18:11:21.163341+0000 mgr.y (mgr.14146) 231 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:22.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:22 smithi173 bash[27694]: audit 2023-09-23T18:11:22.037720+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:11:22.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:22 smithi098 bash[27349]: cluster 2023-09-23T18:11:21.163341+0000 mgr.y (mgr.14146) 231 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:22.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:22 smithi098 bash[27349]: audit 2023-09-23T18:11:22.037720+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:11:22.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:22 smithi098 bash[33599]: cluster 2023-09-23T18:11:21.163341+0000 mgr.y (mgr.14146) 231 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:22.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:22 smithi098 bash[33599]: audit 2023-09-23T18:11:22.037720+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:11:23.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:23 smithi173 bash[27694]: audit 2023-09-23T18:11:22.307793+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:11:23.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:23 smithi173 bash[27694]: audit 2023-09-23T18:11:22.309574+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:11:23.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:23 smithi173 bash[27694]: audit 2023-09-23T18:11:22.319972+0000 mon.a (mon.0) 469 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:11:23.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:23 smithi098 bash[27349]: audit 2023-09-23T18:11:22.307793+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:11:23.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:23 smithi098 bash[27349]: audit 2023-09-23T18:11:22.309574+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:11:23.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:23 smithi098 bash[27349]: audit 2023-09-23T18:11:22.319972+0000 mon.a (mon.0) 469 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:11:23.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:23 smithi098 bash[33599]: audit 2023-09-23T18:11:22.307793+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:11:23.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:23 smithi098 bash[33599]: audit 2023-09-23T18:11:22.309574+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:11:23.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:23 smithi098 bash[33599]: audit 2023-09-23T18:11:22.319972+0000 mon.a (mon.0) 469 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:11:24.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:24 smithi173 bash[27694]: cluster 2023-09-23T18:11:23.163914+0000 mgr.y (mgr.14146) 232 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:24.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:24 smithi173 bash[27694]: audit 2023-09-23T18:11:23.312840+0000 mon.a (mon.0) 470 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T18:11:24.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:24 smithi173 bash[27694]: audit 2023-09-23T18:11:23.316455+0000 mon.b (mon.1) 16 : audit [INF] from='osd.6 [v2:172.21.15.173:6816/3550013548,v1:172.21.15.173:6817/3550013548]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T18:11:24.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:24 smithi098 bash[27349]: cluster 2023-09-23T18:11:23.163914+0000 mgr.y (mgr.14146) 232 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:24.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:24 smithi098 bash[27349]: audit 2023-09-23T18:11:23.312840+0000 mon.a (mon.0) 470 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T18:11:24.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:24 smithi098 bash[27349]: audit 2023-09-23T18:11:23.316455+0000 mon.b (mon.1) 16 : audit [INF] from='osd.6 [v2:172.21.15.173:6816/3550013548,v1:172.21.15.173:6817/3550013548]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T18:11:24.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:24 smithi098 bash[33599]: cluster 2023-09-23T18:11:23.163914+0000 mgr.y (mgr.14146) 232 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:24.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:24 smithi098 bash[33599]: audit 2023-09-23T18:11:23.312840+0000 mon.a (mon.0) 470 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T18:11:24.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:24 smithi098 bash[33599]: audit 2023-09-23T18:11:23.316455+0000 mon.b (mon.1) 16 : audit [INF] from='osd.6 [v2:172.21.15.173:6816/3550013548,v1:172.21.15.173:6817/3550013548]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T18:11:25.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:25 smithi173 bash[27694]: audit 2023-09-23T18:11:24.239152+0000 mon.a (mon.0) 471 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T18:11:25.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:25 smithi173 bash[27694]: cluster 2023-09-23T18:11:24.239359+0000 mon.a (mon.0) 472 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-09-23T18:11:25.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:25 smithi173 bash[27694]: audit 2023-09-23T18:11:24.240300+0000 mon.a (mon.0) 473 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:11:25.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:25 smithi173 bash[27694]: audit 2023-09-23T18:11:24.241158+0000 mon.a (mon.0) 474 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:11:25.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:25 smithi173 bash[27694]: audit 2023-09-23T18:11:24.244627+0000 mon.b (mon.1) 17 : audit [INF] from='osd.6 [v2:172.21.15.173:6816/3550013548,v1:172.21.15.173:6817/3550013548]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:11:25.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:25 smithi098 bash[33599]: audit 2023-09-23T18:11:24.239152+0000 mon.a (mon.0) 471 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T18:11:25.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:25 smithi098 bash[33599]: cluster 2023-09-23T18:11:24.239359+0000 mon.a (mon.0) 472 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-09-23T18:11:25.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:25 smithi098 bash[33599]: audit 2023-09-23T18:11:24.240300+0000 mon.a (mon.0) 473 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:11:25.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:25 smithi098 bash[33599]: audit 2023-09-23T18:11:24.241158+0000 mon.a (mon.0) 474 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:11:25.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:25 smithi098 bash[33599]: audit 2023-09-23T18:11:24.244627+0000 mon.b (mon.1) 17 : audit [INF] from='osd.6 [v2:172.21.15.173:6816/3550013548,v1:172.21.15.173:6817/3550013548]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:11:25.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:25 smithi098 bash[27349]: audit 2023-09-23T18:11:24.239152+0000 mon.a (mon.0) 471 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T18:11:25.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:25 smithi098 bash[27349]: cluster 2023-09-23T18:11:24.239359+0000 mon.a (mon.0) 472 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-09-23T18:11:25.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:25 smithi098 bash[27349]: audit 2023-09-23T18:11:24.240300+0000 mon.a (mon.0) 473 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:11:25.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:25 smithi098 bash[27349]: audit 2023-09-23T18:11:24.241158+0000 mon.a (mon.0) 474 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:11:25.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:25 smithi098 bash[27349]: audit 2023-09-23T18:11:24.244627+0000 mon.b (mon.1) 17 : audit [INF] from='osd.6 [v2:172.21.15.173:6816/3550013548,v1:172.21.15.173:6817/3550013548]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:11:25.924 INFO:teuthology.orchestra.run.smithi173.stdout:Created osd(s) 6 on host 'smithi173' 2023-09-23T18:11:26.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:26 smithi173 bash[27694]: cluster 2023-09-23T18:11:25.164479+0000 mgr.y (mgr.14146) 233 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:26.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:26 smithi173 bash[27694]: audit 2023-09-23T18:11:25.241532+0000 mon.a (mon.0) 475 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]': finished 2023-09-23T18:11:26.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:26 smithi173 bash[27694]: cluster 2023-09-23T18:11:25.241660+0000 mon.a (mon.0) 476 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-09-23T18:11:26.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:26 smithi173 bash[27694]: audit 2023-09-23T18:11:25.241849+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:11:26.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:26 smithi173 bash[27694]: audit 2023-09-23T18:11:25.251474+0000 mon.a (mon.0) 478 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:11:26.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:26 smithi173 bash[27694]: audit 2023-09-23T18:11:25.918814+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:11:26.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:26 smithi173 bash[27694]: audit 2023-09-23T18:11:25.948356+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:11:26.659 DEBUG:teuthology.orchestra.run.smithi173:osd.6> sudo journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.6.service 2023-09-23T18:11:26.661 INFO:tasks.cephadm:Deploying osd.7 on smithi173 with /dev/vg_nvme/lv_1... 2023-09-23T18:11:26.661 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-09-23T18:11:26.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:26 smithi098 bash[27349]: cluster 2023-09-23T18:11:25.164479+0000 mgr.y (mgr.14146) 233 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:26.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:26 smithi098 bash[27349]: audit 2023-09-23T18:11:25.241532+0000 mon.a (mon.0) 475 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]': finished 2023-09-23T18:11:26.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:26 smithi098 bash[27349]: cluster 2023-09-23T18:11:25.241660+0000 mon.a (mon.0) 476 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-09-23T18:11:26.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:26 smithi098 bash[27349]: audit 2023-09-23T18:11:25.241849+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:11:26.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:26 smithi098 bash[27349]: audit 2023-09-23T18:11:25.251474+0000 mon.a (mon.0) 478 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:11:26.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:26 smithi098 bash[27349]: audit 2023-09-23T18:11:25.918814+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:11:26.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:26 smithi098 bash[27349]: audit 2023-09-23T18:11:25.948356+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:11:26.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:26 smithi098 bash[33599]: cluster 2023-09-23T18:11:25.164479+0000 mgr.y (mgr.14146) 233 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:26.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:26 smithi098 bash[33599]: audit 2023-09-23T18:11:25.241532+0000 mon.a (mon.0) 475 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]': finished 2023-09-23T18:11:26.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:26 smithi098 bash[33599]: cluster 2023-09-23T18:11:25.241660+0000 mon.a (mon.0) 476 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-09-23T18:11:26.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:26 smithi098 bash[33599]: audit 2023-09-23T18:11:25.241849+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:11:26.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:26 smithi098 bash[33599]: audit 2023-09-23T18:11:25.251474+0000 mon.a (mon.0) 478 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:11:26.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:26 smithi098 bash[33599]: audit 2023-09-23T18:11:25.918814+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:11:26.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:26 smithi098 bash[33599]: audit 2023-09-23T18:11:25.948356+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:11:26.840 INFO:journalctl@ceph.osd.6.smithi173.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:11:27.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:27 smithi173 bash[27694]: cluster 2023-09-23T18:11:24.357835+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:11:27.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:27 smithi173 bash[27694]: cluster 2023-09-23T18:11:24.357916+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:11:27.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:27 smithi173 bash[27694]: cluster 2023-09-23T18:11:26.251893+0000 mon.a (mon.0) 481 : cluster [INF] osd.6 [v2:172.21.15.173:6816/3550013548,v1:172.21.15.173:6817/3550013548] boot 2023-09-23T18:11:27.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:27 smithi173 bash[27694]: cluster 2023-09-23T18:11:26.251966+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-09-23T18:11:27.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:27 smithi173 bash[27694]: audit 2023-09-23T18:11:26.252956+0000 mon.a (mon.0) 483 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:11:27.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:27 smithi098 bash[33599]: cluster 2023-09-23T18:11:24.357835+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:11:27.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:27 smithi098 bash[33599]: cluster 2023-09-23T18:11:24.357916+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:11:27.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:27 smithi098 bash[33599]: cluster 2023-09-23T18:11:26.251893+0000 mon.a (mon.0) 481 : cluster [INF] osd.6 [v2:172.21.15.173:6816/3550013548,v1:172.21.15.173:6817/3550013548] boot 2023-09-23T18:11:27.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:27 smithi098 bash[33599]: cluster 2023-09-23T18:11:26.251966+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-09-23T18:11:27.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:27 smithi098 bash[33599]: audit 2023-09-23T18:11:26.252956+0000 mon.a (mon.0) 483 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:11:27.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:27 smithi098 bash[27349]: cluster 2023-09-23T18:11:24.357835+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:11:27.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:27 smithi098 bash[27349]: cluster 2023-09-23T18:11:24.357916+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:11:27.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:27 smithi098 bash[27349]: cluster 2023-09-23T18:11:26.251893+0000 mon.a (mon.0) 481 : cluster [INF] osd.6 [v2:172.21.15.173:6816/3550013548,v1:172.21.15.173:6817/3550013548] boot 2023-09-23T18:11:27.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:27 smithi098 bash[27349]: cluster 2023-09-23T18:11:26.251966+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-09-23T18:11:27.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:27 smithi098 bash[27349]: audit 2023-09-23T18:11:26.252956+0000 mon.a (mon.0) 483 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:11:28.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:28 smithi173 bash[27694]: cluster 2023-09-23T18:11:27.165066+0000 mgr.y (mgr.14146) 234 : cluster [DBG] pgmap v220: 1 pgs: 1 remapped+peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:28.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:28 smithi173 bash[27694]: cluster 2023-09-23T18:11:27.255386+0000 mon.a (mon.0) 484 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:11:28.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:28 smithi173 bash[27694]: cluster 2023-09-23T18:11:27.267113+0000 mon.a (mon.0) 485 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-09-23T18:11:28.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:28 smithi098 bash[27349]: cluster 2023-09-23T18:11:27.165066+0000 mgr.y (mgr.14146) 234 : cluster [DBG] pgmap v220: 1 pgs: 1 remapped+peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:28.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:28 smithi098 bash[27349]: cluster 2023-09-23T18:11:27.255386+0000 mon.a (mon.0) 484 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:11:28.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:28 smithi098 bash[27349]: cluster 2023-09-23T18:11:27.267113+0000 mon.a (mon.0) 485 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-09-23T18:11:28.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:28 smithi098 bash[33599]: cluster 2023-09-23T18:11:27.165066+0000 mgr.y (mgr.14146) 234 : cluster [DBG] pgmap v220: 1 pgs: 1 remapped+peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T18:11:28.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:28 smithi098 bash[33599]: cluster 2023-09-23T18:11:27.255386+0000 mon.a (mon.0) 484 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:11:28.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:28 smithi098 bash[33599]: cluster 2023-09-23T18:11:27.267113+0000 mon.a (mon.0) 485 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-09-23T18:11:29.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:29 smithi098 bash[27349]: cluster 2023-09-23T18:11:28.269401+0000 mon.a (mon.0) 486 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-09-23T18:11:29.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:29 smithi098 bash[33599]: cluster 2023-09-23T18:11:28.269401+0000 mon.a (mon.0) 486 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-09-23T18:11:29.776 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:29 smithi173 bash[27694]: cluster 2023-09-23T18:11:28.269401+0000 mon.a (mon.0) 486 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-09-23T18:11:30.098 INFO:teuthology.orchestra.run.smithi173.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-09-23T18:11:30.098 INFO:teuthology.orchestra.run.smithi173.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-09-23T18:11:30.098 INFO:teuthology.orchestra.run.smithi173.stderr: stderr: 10+0 records in 2023-09-23T18:11:30.098 INFO:teuthology.orchestra.run.smithi173.stderr:10+0 records out 2023-09-23T18:11:30.098 INFO:teuthology.orchestra.run.smithi173.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0174196 s, 602 MB/s 2023-09-23T18:11:30.098 INFO:teuthology.orchestra.run.smithi173.stderr:--> Zapping successful for: 2023-09-23T18:11:30.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:30 smithi173 bash[27694]: cluster 2023-09-23T18:11:29.165683+0000 mgr.y (mgr.14146) 235 : cluster [DBG] pgmap v223: 1 pgs: 1 remapped+peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:30.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:30 smithi098 bash[27349]: cluster 2023-09-23T18:11:29.165683+0000 mgr.y (mgr.14146) 235 : cluster [DBG] pgmap v223: 1 pgs: 1 remapped+peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:30.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:30 smithi098 bash[33599]: cluster 2023-09-23T18:11:29.165683+0000 mgr.y (mgr.14146) 235 : cluster [DBG] pgmap v223: 1 pgs: 1 remapped+peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:30.844 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi173:vg_nvme/lv_1 2023-09-23T18:11:32.560 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:32 smithi173 bash[27694]: cluster 2023-09-23T18:11:31.166324+0000 mgr.y (mgr.14146) 236 : cluster [DBG] pgmap v224: 1 pgs: 1 remapped+peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:32.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:32 smithi098 bash[27349]: cluster 2023-09-23T18:11:31.166324+0000 mgr.y (mgr.14146) 236 : cluster [DBG] pgmap v224: 1 pgs: 1 remapped+peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:32.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:32 smithi098 bash[33599]: cluster 2023-09-23T18:11:31.166324+0000 mgr.y (mgr.14146) 236 : cluster [DBG] pgmap v224: 1 pgs: 1 remapped+peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:34.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:34 smithi098 bash[27349]: cluster 2023-09-23T18:11:33.166914+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:34.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:34 smithi098 bash[33599]: cluster 2023-09-23T18:11:33.166914+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:34.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:34 smithi173 bash[27694]: cluster 2023-09-23T18:11:33.166914+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:35.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:35 smithi098 bash[27349]: cluster 2023-09-23T18:11:34.092691+0000 mon.a (mon.0) 487 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:11:35.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:35 smithi098 bash[27349]: cluster 2023-09-23T18:11:34.092743+0000 mon.a (mon.0) 488 : cluster [INF] Cluster is now healthy 2023-09-23T18:11:35.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:35 smithi098 bash[33599]: cluster 2023-09-23T18:11:34.092691+0000 mon.a (mon.0) 487 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:11:35.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:35 smithi098 bash[33599]: cluster 2023-09-23T18:11:34.092743+0000 mon.a (mon.0) 488 : cluster [INF] Cluster is now healthy 2023-09-23T18:11:35.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:35 smithi173 bash[27694]: cluster 2023-09-23T18:11:34.092691+0000 mon.a (mon.0) 487 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:11:35.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:35 smithi173 bash[27694]: cluster 2023-09-23T18:11:34.092743+0000 mon.a (mon.0) 488 : cluster [INF] Cluster is now healthy 2023-09-23T18:11:36.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:36 smithi098 bash[27349]: cluster 2023-09-23T18:11:35.167462+0000 mgr.y (mgr.14146) 238 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:36.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:36 smithi098 bash[27349]: audit 2023-09-23T18:11:35.211138+0000 mgr.y (mgr.14146) 239 : audit [DBG] from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi173:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:11:36.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:36 smithi098 bash[27349]: audit 2023-09-23T18:11:35.217747+0000 mon.a (mon.0) 489 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:11:36.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:36 smithi098 bash[27349]: audit 2023-09-23T18:11:35.222275+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:11:36.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:36 smithi098 bash[27349]: audit 2023-09-23T18:11:35.223482+0000 mon.a (mon.0) 491 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:11:36.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:36 smithi098 bash[33599]: cluster 2023-09-23T18:11:35.167462+0000 mgr.y (mgr.14146) 238 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:36.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:36 smithi098 bash[33599]: audit 2023-09-23T18:11:35.211138+0000 mgr.y (mgr.14146) 239 : audit [DBG] from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi173:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:11:36.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:36 smithi098 bash[33599]: audit 2023-09-23T18:11:35.217747+0000 mon.a (mon.0) 489 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:11:36.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:36 smithi098 bash[33599]: audit 2023-09-23T18:11:35.222275+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:11:36.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:36 smithi098 bash[33599]: audit 2023-09-23T18:11:35.223482+0000 mon.a (mon.0) 491 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:11:36.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:36 smithi173 bash[27694]: cluster 2023-09-23T18:11:35.167462+0000 mgr.y (mgr.14146) 238 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:36.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:36 smithi173 bash[27694]: audit 2023-09-23T18:11:35.211138+0000 mgr.y (mgr.14146) 239 : audit [DBG] from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi173:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:11:36.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:36 smithi173 bash[27694]: audit 2023-09-23T18:11:35.217747+0000 mon.a (mon.0) 489 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T18:11:36.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:36 smithi173 bash[27694]: audit 2023-09-23T18:11:35.222275+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T18:11:36.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:36 smithi173 bash[27694]: audit 2023-09-23T18:11:35.223482+0000 mon.a (mon.0) 491 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:11:37.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:37 smithi098 bash[27349]: cephadm 2023-09-23T18:11:36.586430+0000 mgr.y (mgr.14146) 240 : cephadm [INF] Detected new or changed devices on smithi173 2023-09-23T18:11:37.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:37 smithi098 bash[27349]: audit 2023-09-23T18:11:36.597786+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:11:37.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:37 smithi098 bash[27349]: audit 2023-09-23T18:11:36.599699+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:11:37.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:37 smithi098 bash[27349]: audit 2023-09-23T18:11:36.601140+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:11:37.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:37 smithi098 bash[27349]: audit 2023-09-23T18:11:36.602616+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:11:37.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:37 smithi098 bash[27349]: cephadm 2023-09-23T18:11:36.603542+0000 mgr.y (mgr.14146) 241 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 5755M 2023-09-23T18:11:37.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:37 smithi098 bash[27349]: audit 2023-09-23T18:11:36.611380+0000 mon.a (mon.0) 496 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:11:37.947 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:37 smithi098 bash[27349]: audit 2023-09-23T18:11:36.613030+0000 mon.a (mon.0) 497 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:11:37.947 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:37 smithi098 bash[27349]: audit 2023-09-23T18:11:36.614228+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:11:37.947 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:37 smithi098 bash[27349]: audit 2023-09-23T18:11:36.622542+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:11:37.947 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:37 smithi098 bash[27349]: cluster 2023-09-23T18:11:37.168000+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:37.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:37 smithi098 bash[33599]: cephadm 2023-09-23T18:11:36.586430+0000 mgr.y (mgr.14146) 240 : cephadm [INF] Detected new or changed devices on smithi173 2023-09-23T18:11:37.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:37 smithi098 bash[33599]: audit 2023-09-23T18:11:36.597786+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:11:37.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:37 smithi098 bash[33599]: audit 2023-09-23T18:11:36.599699+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:11:37.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:37 smithi098 bash[33599]: audit 2023-09-23T18:11:36.601140+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:11:37.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:37 smithi098 bash[33599]: audit 2023-09-23T18:11:36.602616+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:11:37.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:37 smithi098 bash[33599]: cephadm 2023-09-23T18:11:36.603542+0000 mgr.y (mgr.14146) 241 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 5755M 2023-09-23T18:11:37.948 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:37 smithi098 bash[33599]: audit 2023-09-23T18:11:36.611380+0000 mon.a (mon.0) 496 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:11:37.948 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:37 smithi098 bash[33599]: audit 2023-09-23T18:11:36.613030+0000 mon.a (mon.0) 497 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:11:37.948 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:37 smithi098 bash[33599]: audit 2023-09-23T18:11:36.614228+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:11:37.948 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:37 smithi098 bash[33599]: audit 2023-09-23T18:11:36.622542+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:11:37.948 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:37 smithi098 bash[33599]: cluster 2023-09-23T18:11:37.168000+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:38.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:37 smithi173 bash[27694]: cephadm 2023-09-23T18:11:36.586430+0000 mgr.y (mgr.14146) 240 : cephadm [INF] Detected new or changed devices on smithi173 2023-09-23T18:11:38.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:37 smithi173 bash[27694]: audit 2023-09-23T18:11:36.597786+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:11:38.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:37 smithi173 bash[27694]: audit 2023-09-23T18:11:36.599699+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:11:38.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:37 smithi173 bash[27694]: audit 2023-09-23T18:11:36.601140+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:11:38.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:37 smithi173 bash[27694]: audit 2023-09-23T18:11:36.602616+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:11:38.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:37 smithi173 bash[27694]: cephadm 2023-09-23T18:11:36.603542+0000 mgr.y (mgr.14146) 241 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 5755M 2023-09-23T18:11:38.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:37 smithi173 bash[27694]: audit 2023-09-23T18:11:36.611380+0000 mon.a (mon.0) 496 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:11:38.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:37 smithi173 bash[27694]: audit 2023-09-23T18:11:36.613030+0000 mon.a (mon.0) 497 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:11:38.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:37 smithi173 bash[27694]: audit 2023-09-23T18:11:36.614228+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:11:38.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:37 smithi173 bash[27694]: audit 2023-09-23T18:11:36.622542+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:11:38.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:37 smithi173 bash[27694]: cluster 2023-09-23T18:11:37.168000+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:40.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:40 smithi173 bash[27694]: cluster 2023-09-23T18:11:39.168539+0000 mgr.y (mgr.14146) 243 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:40.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:40 smithi098 bash[27349]: cluster 2023-09-23T18:11:39.168539+0000 mgr.y (mgr.14146) 243 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:40.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:40 smithi098 bash[33599]: cluster 2023-09-23T18:11:39.168539+0000 mgr.y (mgr.14146) 243 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:42.508 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:42 smithi173 bash[27694]: cluster 2023-09-23T18:11:41.169118+0000 mgr.y (mgr.14146) 244 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:42.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:42 smithi098 bash[27349]: cluster 2023-09-23T18:11:41.169118+0000 mgr.y (mgr.14146) 244 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:42.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:42 smithi098 bash[33599]: cluster 2023-09-23T18:11:41.169118+0000 mgr.y (mgr.14146) 244 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:44.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:44 smithi098 bash[27349]: cluster 2023-09-23T18:11:43.169641+0000 mgr.y (mgr.14146) 245 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:44.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:44 smithi098 bash[27349]: audit 2023-09-23T18:11:43.418801+0000 mon.a (mon.0) 500 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f540de3-48f9-4bc8-b14a-9aadf838f16b"}]: dispatch 2023-09-23T18:11:44.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:44 smithi098 bash[27349]: audit 2023-09-23T18:11:43.422389+0000 mon.b (mon.1) 18 : audit [INF] from='client.? 172.21.15.173:0/73717073' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f540de3-48f9-4bc8-b14a-9aadf838f16b"}]: dispatch 2023-09-23T18:11:44.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:44 smithi098 bash[27349]: audit 2023-09-23T18:11:43.427040+0000 mon.a (mon.0) 501 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5f540de3-48f9-4bc8-b14a-9aadf838f16b"}]': finished 2023-09-23T18:11:44.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:44 smithi098 bash[27349]: cluster 2023-09-23T18:11:43.427204+0000 mon.a (mon.0) 502 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-09-23T18:11:44.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:44 smithi098 bash[27349]: audit 2023-09-23T18:11:43.427395+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:11:44.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:44 smithi098 bash[33599]: cluster 2023-09-23T18:11:43.169641+0000 mgr.y (mgr.14146) 245 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:44.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:44 smithi098 bash[33599]: audit 2023-09-23T18:11:43.418801+0000 mon.a (mon.0) 500 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f540de3-48f9-4bc8-b14a-9aadf838f16b"}]: dispatch 2023-09-23T18:11:44.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:44 smithi098 bash[33599]: audit 2023-09-23T18:11:43.422389+0000 mon.b (mon.1) 18 : audit [INF] from='client.? 172.21.15.173:0/73717073' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f540de3-48f9-4bc8-b14a-9aadf838f16b"}]: dispatch 2023-09-23T18:11:44.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:44 smithi098 bash[33599]: audit 2023-09-23T18:11:43.427040+0000 mon.a (mon.0) 501 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5f540de3-48f9-4bc8-b14a-9aadf838f16b"}]': finished 2023-09-23T18:11:44.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:44 smithi098 bash[33599]: cluster 2023-09-23T18:11:43.427204+0000 mon.a (mon.0) 502 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-09-23T18:11:44.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:44 smithi098 bash[33599]: audit 2023-09-23T18:11:43.427395+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:11:44.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:44 smithi173 bash[27694]: cluster 2023-09-23T18:11:43.169641+0000 mgr.y (mgr.14146) 245 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:44.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:44 smithi173 bash[27694]: audit 2023-09-23T18:11:43.418801+0000 mon.a (mon.0) 500 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f540de3-48f9-4bc8-b14a-9aadf838f16b"}]: dispatch 2023-09-23T18:11:44.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:44 smithi173 bash[27694]: audit 2023-09-23T18:11:43.422389+0000 mon.b (mon.1) 18 : audit [INF] from='client.? 172.21.15.173:0/73717073' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f540de3-48f9-4bc8-b14a-9aadf838f16b"}]: dispatch 2023-09-23T18:11:44.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:44 smithi173 bash[27694]: audit 2023-09-23T18:11:43.427040+0000 mon.a (mon.0) 501 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5f540de3-48f9-4bc8-b14a-9aadf838f16b"}]': finished 2023-09-23T18:11:44.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:44 smithi173 bash[27694]: cluster 2023-09-23T18:11:43.427204+0000 mon.a (mon.0) 502 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-09-23T18:11:44.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:44 smithi173 bash[27694]: audit 2023-09-23T18:11:43.427395+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:11:45.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:45 smithi098 bash[27349]: audit 2023-09-23T18:11:44.152367+0000 mon.b (mon.1) 19 : audit [DBG] from='client.? 172.21.15.173:0/2352157886' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:11:45.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:45 smithi098 bash[33599]: audit 2023-09-23T18:11:44.152367+0000 mon.b (mon.1) 19 : audit [DBG] from='client.? 172.21.15.173:0/2352157886' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:11:45.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:45 smithi173 bash[27694]: audit 2023-09-23T18:11:44.152367+0000 mon.b (mon.1) 19 : audit [DBG] from='client.? 172.21.15.173:0/2352157886' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T18:11:46.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:46 smithi098 bash[27349]: cluster 2023-09-23T18:11:45.170341+0000 mgr.y (mgr.14146) 246 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:46.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:46 smithi098 bash[33599]: cluster 2023-09-23T18:11:45.170341+0000 mgr.y (mgr.14146) 246 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:46.531 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:46 smithi173 bash[27694]: cluster 2023-09-23T18:11:45.170341+0000 mgr.y (mgr.14146) 246 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:48.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:48 smithi173 bash[27694]: cluster 2023-09-23T18:11:47.171046+0000 mgr.y (mgr.14146) 247 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:48.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:48 smithi098 bash[27349]: cluster 2023-09-23T18:11:47.171046+0000 mgr.y (mgr.14146) 247 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:48.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:48 smithi098 bash[33599]: cluster 2023-09-23T18:11:47.171046+0000 mgr.y (mgr.14146) 247 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:50.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:50 smithi173 bash[27694]: cluster 2023-09-23T18:11:49.171776+0000 mgr.y (mgr.14146) 248 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:50.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:50 smithi098 bash[27349]: cluster 2023-09-23T18:11:49.171776+0000 mgr.y (mgr.14146) 248 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:50.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:50 smithi098 bash[33599]: cluster 2023-09-23T18:11:49.171776+0000 mgr.y (mgr.14146) 248 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:52.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:52 smithi173 bash[27694]: cluster 2023-09-23T18:11:51.172356+0000 mgr.y (mgr.14146) 249 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:52.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:52 smithi098 bash[27349]: cluster 2023-09-23T18:11:51.172356+0000 mgr.y (mgr.14146) 249 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:52.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:52 smithi098 bash[33599]: cluster 2023-09-23T18:11:51.172356+0000 mgr.y (mgr.14146) 249 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:54.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:54 smithi173 bash[27694]: cluster 2023-09-23T18:11:53.173015+0000 mgr.y (mgr.14146) 250 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:54.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:54 smithi098 bash[27349]: cluster 2023-09-23T18:11:53.173015+0000 mgr.y (mgr.14146) 250 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:54.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:54 smithi098 bash[33599]: cluster 2023-09-23T18:11:53.173015+0000 mgr.y (mgr.14146) 250 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:56.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:56 smithi173 bash[27694]: cluster 2023-09-23T18:11:55.173577+0000 mgr.y (mgr.14146) 251 : cluster [DBG] pgmap v237: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:56.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:56 smithi173 bash[27694]: audit 2023-09-23T18:11:56.089448+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T18:11:56.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:56 smithi173 bash[27694]: audit 2023-09-23T18:11:56.091011+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:11:56.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:56 smithi098 bash[27349]: cluster 2023-09-23T18:11:55.173577+0000 mgr.y (mgr.14146) 251 : cluster [DBG] pgmap v237: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:56.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:56 smithi098 bash[27349]: audit 2023-09-23T18:11:56.089448+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T18:11:56.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:56 smithi098 bash[27349]: audit 2023-09-23T18:11:56.091011+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:11:56.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:56 smithi098 bash[33599]: cluster 2023-09-23T18:11:55.173577+0000 mgr.y (mgr.14146) 251 : cluster [DBG] pgmap v237: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:56.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:56 smithi098 bash[33599]: audit 2023-09-23T18:11:56.089448+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T18:11:56.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:56 smithi098 bash[33599]: audit 2023-09-23T18:11:56.091011+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:11:57.453 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:57 smithi173 bash[27694]: cephadm 2023-09-23T18:11:56.092528+0000 mgr.y (mgr.14146) 252 : cephadm [INF] Deploying daemon osd.7 on smithi173 2023-09-23T18:11:57.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:57 smithi098 bash[33599]: cephadm 2023-09-23T18:11:56.092528+0000 mgr.y (mgr.14146) 252 : cephadm [INF] Deploying daemon osd.7 on smithi173 2023-09-23T18:11:57.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:57 smithi098 bash[27349]: cephadm 2023-09-23T18:11:56.092528+0000 mgr.y (mgr.14146) 252 : cephadm [INF] Deploying daemon osd.7 on smithi173 2023-09-23T18:11:58.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:11:58 smithi173 bash[27694]: cluster 2023-09-23T18:11:57.174065+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v238: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:58.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:11:58 smithi098 bash[27349]: cluster 2023-09-23T18:11:57.174065+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v238: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:11:58.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:11:58 smithi098 bash[33599]: cluster 2023-09-23T18:11:57.174065+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v238: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:12:00.460 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:00 smithi173 bash[27694]: cluster 2023-09-23T18:11:59.174656+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:12:00.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:00 smithi098 bash[27349]: cluster 2023-09-23T18:11:59.174656+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:12:00.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:00 smithi098 bash[33599]: cluster 2023-09-23T18:11:59.174656+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:12:01.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:01 smithi098 bash[27349]: audit 2023-09-23T18:12:00.427860+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:01.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:01 smithi098 bash[27349]: audit 2023-09-23T18:12:00.457588+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:12:01.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:01 smithi098 bash[27349]: cluster 2023-09-23T18:12:01.175237+0000 mgr.y (mgr.14146) 255 : cluster [DBG] pgmap v240: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:12:01.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:01 smithi098 bash[33599]: audit 2023-09-23T18:12:00.427860+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:01.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:01 smithi098 bash[33599]: audit 2023-09-23T18:12:00.457588+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:12:01.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:01 smithi098 bash[33599]: cluster 2023-09-23T18:12:01.175237+0000 mgr.y (mgr.14146) 255 : cluster [DBG] pgmap v240: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:12:01.776 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:01 smithi173 bash[27694]: audit 2023-09-23T18:12:00.427860+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:01.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:01 smithi173 bash[27694]: audit 2023-09-23T18:12:00.457588+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:12:01.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:01 smithi173 bash[27694]: cluster 2023-09-23T18:12:01.175237+0000 mgr.y (mgr.14146) 255 : cluster [DBG] pgmap v240: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:12:04.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:04 smithi098 bash[27349]: cluster 2023-09-23T18:12:03.175843+0000 mgr.y (mgr.14146) 256 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:12:04.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:04 smithi098 bash[33599]: cluster 2023-09-23T18:12:03.175843+0000 mgr.y (mgr.14146) 256 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:12:04.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:04 smithi173 bash[27694]: cluster 2023-09-23T18:12:03.175843+0000 mgr.y (mgr.14146) 256 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:12:06.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:06 smithi173 bash[27694]: cluster 2023-09-23T18:12:05.176367+0000 mgr.y (mgr.14146) 257 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:12:06.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:06 smithi173 bash[27694]: audit 2023-09-23T18:12:05.654365+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:06.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:06 smithi173 bash[27694]: audit 2023-09-23T18:12:05.655978+0000 mon.a (mon.0) 509 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:12:06.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:06 smithi173 bash[27694]: audit 2023-09-23T18:12:05.657492+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:12:06.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:06 smithi173 bash[27694]: audit 2023-09-23T18:12:05.664062+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:06.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:06 smithi098 bash[33599]: cluster 2023-09-23T18:12:05.176367+0000 mgr.y (mgr.14146) 257 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:12:06.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:06 smithi098 bash[33599]: audit 2023-09-23T18:12:05.654365+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:06.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:06 smithi098 bash[33599]: audit 2023-09-23T18:12:05.655978+0000 mon.a (mon.0) 509 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:12:06.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:06 smithi098 bash[33599]: audit 2023-09-23T18:12:05.657492+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:12:06.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:06 smithi098 bash[33599]: audit 2023-09-23T18:12:05.664062+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:06.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:06 smithi098 bash[27349]: cluster 2023-09-23T18:12:05.176367+0000 mgr.y (mgr.14146) 257 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:12:06.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:06 smithi098 bash[27349]: audit 2023-09-23T18:12:05.654365+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:06.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:06 smithi098 bash[27349]: audit 2023-09-23T18:12:05.655978+0000 mon.a (mon.0) 509 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:12:06.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:06 smithi098 bash[27349]: audit 2023-09-23T18:12:05.657492+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:12:06.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:06 smithi098 bash[27349]: audit 2023-09-23T18:12:05.664062+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:07.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:07 smithi173 bash[27694]: audit 2023-09-23T18:12:07.140949+0000 mon.a (mon.0) 512 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T18:12:07.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:07 smithi173 bash[27694]: audit 2023-09-23T18:12:07.144624+0000 mon.b (mon.1) 20 : audit [INF] from='osd.7 [v2:172.21.15.173:6824/135608222,v1:172.21.15.173:6825/135608222]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T18:12:07.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:07 smithi098 bash[27349]: audit 2023-09-23T18:12:07.140949+0000 mon.a (mon.0) 512 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T18:12:07.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:07 smithi098 bash[27349]: audit 2023-09-23T18:12:07.144624+0000 mon.b (mon.1) 20 : audit [INF] from='osd.7 [v2:172.21.15.173:6824/135608222,v1:172.21.15.173:6825/135608222]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T18:12:07.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:07 smithi098 bash[33599]: audit 2023-09-23T18:12:07.140949+0000 mon.a (mon.0) 512 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T18:12:07.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:07 smithi098 bash[33599]: audit 2023-09-23T18:12:07.144624+0000 mon.b (mon.1) 20 : audit [INF] from='osd.7 [v2:172.21.15.173:6824/135608222,v1:172.21.15.173:6825/135608222]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T18:12:08.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:08 smithi173 bash[27694]: cluster 2023-09-23T18:12:07.176912+0000 mgr.y (mgr.14146) 258 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:12:08.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:08 smithi173 bash[27694]: audit 2023-09-23T18:12:07.244399+0000 mon.a (mon.0) 513 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T18:12:08.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:08 smithi173 bash[27694]: cluster 2023-09-23T18:12:07.244547+0000 mon.a (mon.0) 514 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-09-23T18:12:08.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:08 smithi173 bash[27694]: audit 2023-09-23T18:12:07.244708+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:12:08.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:08 smithi173 bash[27694]: audit 2023-09-23T18:12:07.245642+0000 mon.a (mon.0) 516 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:12:08.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:08 smithi173 bash[27694]: audit 2023-09-23T18:12:07.249927+0000 mon.b (mon.1) 21 : audit [INF] from='osd.7 [v2:172.21.15.173:6824/135608222,v1:172.21.15.173:6825/135608222]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:12:08.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:08 smithi098 bash[33599]: cluster 2023-09-23T18:12:07.176912+0000 mgr.y (mgr.14146) 258 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:12:08.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:08 smithi098 bash[33599]: audit 2023-09-23T18:12:07.244399+0000 mon.a (mon.0) 513 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T18:12:08.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:08 smithi098 bash[33599]: cluster 2023-09-23T18:12:07.244547+0000 mon.a (mon.0) 514 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-09-23T18:12:08.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:08 smithi098 bash[33599]: audit 2023-09-23T18:12:07.244708+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:12:08.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:08 smithi098 bash[33599]: audit 2023-09-23T18:12:07.245642+0000 mon.a (mon.0) 516 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:12:08.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:08 smithi098 bash[33599]: audit 2023-09-23T18:12:07.249927+0000 mon.b (mon.1) 21 : audit [INF] from='osd.7 [v2:172.21.15.173:6824/135608222,v1:172.21.15.173:6825/135608222]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:12:08.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:08 smithi098 bash[27349]: cluster 2023-09-23T18:12:07.176912+0000 mgr.y (mgr.14146) 258 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:12:08.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:08 smithi098 bash[27349]: audit 2023-09-23T18:12:07.244399+0000 mon.a (mon.0) 513 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T18:12:08.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:08 smithi098 bash[27349]: cluster 2023-09-23T18:12:07.244547+0000 mon.a (mon.0) 514 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-09-23T18:12:08.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:08 smithi098 bash[27349]: audit 2023-09-23T18:12:07.244708+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:12:08.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:08 smithi098 bash[27349]: audit 2023-09-23T18:12:07.245642+0000 mon.a (mon.0) 516 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:12:08.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:08 smithi098 bash[27349]: audit 2023-09-23T18:12:07.249927+0000 mon.b (mon.1) 21 : audit [INF] from='osd.7 [v2:172.21.15.173:6824/135608222,v1:172.21.15.173:6825/135608222]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:12:09.378 INFO:teuthology.orchestra.run.smithi173.stdout:Created osd(s) 7 on host 'smithi173' 2023-09-23T18:12:09.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:09 smithi173 bash[27694]: audit 2023-09-23T18:12:08.248396+0000 mon.a (mon.0) 517 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]': finished 2023-09-23T18:12:09.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:09 smithi173 bash[27694]: cluster 2023-09-23T18:12:08.248639+0000 mon.a (mon.0) 518 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-09-23T18:12:09.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:09 smithi173 bash[27694]: audit 2023-09-23T18:12:08.249760+0000 mon.a (mon.0) 519 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:12:09.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:09 smithi173 bash[27694]: audit 2023-09-23T18:12:08.254970+0000 mon.a (mon.0) 520 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:12:09.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:09 smithi098 bash[27349]: audit 2023-09-23T18:12:08.248396+0000 mon.a (mon.0) 517 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]': finished 2023-09-23T18:12:09.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:09 smithi098 bash[27349]: cluster 2023-09-23T18:12:08.248639+0000 mon.a (mon.0) 518 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-09-23T18:12:09.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:09 smithi098 bash[27349]: audit 2023-09-23T18:12:08.249760+0000 mon.a (mon.0) 519 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:12:09.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:09 smithi098 bash[27349]: audit 2023-09-23T18:12:08.254970+0000 mon.a (mon.0) 520 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:12:09.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:09 smithi098 bash[33599]: audit 2023-09-23T18:12:08.248396+0000 mon.a (mon.0) 517 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]': finished 2023-09-23T18:12:09.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:09 smithi098 bash[33599]: cluster 2023-09-23T18:12:08.248639+0000 mon.a (mon.0) 518 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-09-23T18:12:09.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:09 smithi098 bash[33599]: audit 2023-09-23T18:12:08.249760+0000 mon.a (mon.0) 519 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:12:09.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:09 smithi098 bash[33599]: audit 2023-09-23T18:12:08.254970+0000 mon.a (mon.0) 520 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:12:10.138 DEBUG:teuthology.orchestra.run.smithi173:osd.7> sudo journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.7.service 2023-09-23T18:12:10.141 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-09-23T18:12:10.141 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd stat -f json 2023-09-23T18:12:10.317 INFO:journalctl@ceph.osd.7.smithi173.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:12:10.517 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:10 smithi098 bash[33599]: cluster 2023-09-23T18:12:08.168812+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:12:10.517 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:10 smithi098 bash[33599]: cluster 2023-09-23T18:12:08.168896+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:12:10.517 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:10 smithi098 bash[33599]: cluster 2023-09-23T18:12:09.177438+0000 mgr.y (mgr.14146) 259 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:12:10.517 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:10 smithi098 bash[33599]: cluster 2023-09-23T18:12:09.252023+0000 mon.a (mon.0) 521 : cluster [INF] osd.7 [v2:172.21.15.173:6824/135608222,v1:172.21.15.173:6825/135608222] boot 2023-09-23T18:12:10.517 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:10 smithi098 bash[33599]: cluster 2023-09-23T18:12:09.252111+0000 mon.a (mon.0) 522 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-09-23T18:12:10.518 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:10 smithi098 bash[33599]: audit 2023-09-23T18:12:09.252346+0000 mon.a (mon.0) 523 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:12:10.518 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:10 smithi098 bash[33599]: audit 2023-09-23T18:12:09.372331+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:10.518 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:10 smithi098 bash[33599]: audit 2023-09-23T18:12:09.393741+0000 mon.a (mon.0) 525 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:12:10.518 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:10 smithi098 bash[33599]: cluster 2023-09-23T18:12:10.252938+0000 mon.a (mon.0) 526 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-09-23T18:12:10.518 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:10 smithi098 bash[27349]: cluster 2023-09-23T18:12:08.168812+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:12:10.518 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:10 smithi098 bash[27349]: cluster 2023-09-23T18:12:08.168896+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:12:10.518 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:10 smithi098 bash[27349]: cluster 2023-09-23T18:12:09.177438+0000 mgr.y (mgr.14146) 259 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:12:10.518 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:10 smithi098 bash[27349]: cluster 2023-09-23T18:12:09.252023+0000 mon.a (mon.0) 521 : cluster [INF] osd.7 [v2:172.21.15.173:6824/135608222,v1:172.21.15.173:6825/135608222] boot 2023-09-23T18:12:10.518 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:10 smithi098 bash[27349]: cluster 2023-09-23T18:12:09.252111+0000 mon.a (mon.0) 522 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-09-23T18:12:10.518 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:10 smithi098 bash[27349]: audit 2023-09-23T18:12:09.252346+0000 mon.a (mon.0) 523 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:12:10.519 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:10 smithi098 bash[27349]: audit 2023-09-23T18:12:09.372331+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:10.519 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:10 smithi098 bash[27349]: audit 2023-09-23T18:12:09.393741+0000 mon.a (mon.0) 525 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:12:10.519 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:10 smithi098 bash[27349]: cluster 2023-09-23T18:12:10.252938+0000 mon.a (mon.0) 526 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-09-23T18:12:10.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:10 smithi173 bash[27694]: cluster 2023-09-23T18:12:08.168812+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T18:12:10.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:10 smithi173 bash[27694]: cluster 2023-09-23T18:12:08.168896+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T18:12:10.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:10 smithi173 bash[27694]: cluster 2023-09-23T18:12:09.177438+0000 mgr.y (mgr.14146) 259 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:12:10.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:10 smithi173 bash[27694]: cluster 2023-09-23T18:12:09.252023+0000 mon.a (mon.0) 521 : cluster [INF] osd.7 [v2:172.21.15.173:6824/135608222,v1:172.21.15.173:6825/135608222] boot 2023-09-23T18:12:10.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:10 smithi173 bash[27694]: cluster 2023-09-23T18:12:09.252111+0000 mon.a (mon.0) 522 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-09-23T18:12:10.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:10 smithi173 bash[27694]: audit 2023-09-23T18:12:09.252346+0000 mon.a (mon.0) 523 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:12:10.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:10 smithi173 bash[27694]: audit 2023-09-23T18:12:09.372331+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:10.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:10 smithi173 bash[27694]: audit 2023-09-23T18:12:09.393741+0000 mon.a (mon.0) 525 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:12:10.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:10 smithi173 bash[27694]: cluster 2023-09-23T18:12:10.252938+0000 mon.a (mon.0) 526 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-09-23T18:12:12.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:12 smithi173 bash[27694]: cluster 2023-09-23T18:12:11.177977+0000 mgr.y (mgr.14146) 260 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:12.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:12 smithi173 bash[27694]: cluster 2023-09-23T18:12:11.253221+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-09-23T18:12:12.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:12 smithi098 bash[33599]: cluster 2023-09-23T18:12:11.177977+0000 mgr.y (mgr.14146) 260 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:12.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:12 smithi098 bash[33599]: cluster 2023-09-23T18:12:11.253221+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-09-23T18:12:12.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:12 smithi098 bash[27349]: cluster 2023-09-23T18:12:11.177977+0000 mgr.y (mgr.14146) 260 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:12.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:12 smithi098 bash[27349]: cluster 2023-09-23T18:12:11.253221+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-09-23T18:12:13.565 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:12:14.327 INFO:teuthology.orchestra.run.smithi098.stdout:{"epoch":50,"num_osds":8,"num_up_osds":8,"osd_up_since":1695492729,"num_in_osds":8,"osd_in_since":1695492703,"num_remapped_pgs":0} 2023-09-23T18:12:14.327 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T18:12:14.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:14 smithi098 bash[27349]: cluster 2023-09-23T18:12:13.178621+0000 mgr.y (mgr.14146) 261 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:14.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:14 smithi098 bash[27349]: audit 2023-09-23T18:12:13.560805+0000 mon.a (mon.0) 528 : audit [DBG] from='client.? 172.21.15.98:0/1632385622' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T18:12:14.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:14 smithi098 bash[33599]: cluster 2023-09-23T18:12:13.178621+0000 mgr.y (mgr.14146) 261 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:14.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:14 smithi098 bash[33599]: audit 2023-09-23T18:12:13.560805+0000 mon.a (mon.0) 528 : audit [DBG] from='client.? 172.21.15.98:0/1632385622' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T18:12:14.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:14 smithi173 bash[27694]: cluster 2023-09-23T18:12:13.178621+0000 mgr.y (mgr.14146) 261 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:14.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:14 smithi173 bash[27694]: audit 2023-09-23T18:12:13.560805+0000 mon.a (mon.0) 528 : audit [DBG] from='client.? 172.21.15.98:0/1632385622' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T18:12:16.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:16 smithi173 bash[27694]: cluster 2023-09-23T18:12:15.179103+0000 mgr.y (mgr.14146) 262 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:16.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:16 smithi098 bash[27349]: cluster 2023-09-23T18:12:15.179103+0000 mgr.y (mgr.14146) 262 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:16.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:16 smithi098 bash[33599]: cluster 2023-09-23T18:12:15.179103+0000 mgr.y (mgr.14146) 262 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:18.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:18 smithi173 bash[27694]: cluster 2023-09-23T18:12:17.179709+0000 mgr.y (mgr.14146) 263 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:18.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:18 smithi098 bash[27349]: cluster 2023-09-23T18:12:17.179709+0000 mgr.y (mgr.14146) 263 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:18.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:18 smithi098 bash[33599]: cluster 2023-09-23T18:12:17.179709+0000 mgr.y (mgr.14146) 263 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:19.086 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:19.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:19 smithi173 bash[27694]: cephadm 2023-09-23T18:12:18.481657+0000 mgr.y (mgr.14146) 264 : cephadm [INF] Detected new or changed devices on smithi173 2023-09-23T18:12:19.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:19 smithi173 bash[27694]: audit 2023-09-23T18:12:18.491324+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:19.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:19 smithi173 bash[27694]: audit 2023-09-23T18:12:18.493007+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:12:19.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:19 smithi173 bash[27694]: audit 2023-09-23T18:12:18.494238+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:12:19.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:19 smithi173 bash[27694]: audit 2023-09-23T18:12:18.495391+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:12:19.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:19 smithi173 bash[27694]: audit 2023-09-23T18:12:18.496373+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:12:19.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:19 smithi173 bash[27694]: cephadm 2023-09-23T18:12:18.497102+0000 mgr.y (mgr.14146) 265 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 4316M 2023-09-23T18:12:19.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:19 smithi173 bash[27694]: audit 2023-09-23T18:12:18.501719+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:19.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:19 smithi173 bash[27694]: audit 2023-09-23T18:12:18.503497+0000 mon.a (mon.0) 535 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:12:19.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:19 smithi173 bash[27694]: audit 2023-09-23T18:12:18.504835+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:12:19.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:19 smithi173 bash[27694]: audit 2023-09-23T18:12:18.510965+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:19.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:19 smithi173 bash[27694]: cluster 2023-09-23T18:12:19.180294+0000 mgr.y (mgr.14146) 266 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:19.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:19 smithi098 bash[27349]: cephadm 2023-09-23T18:12:18.481657+0000 mgr.y (mgr.14146) 264 : cephadm [INF] Detected new or changed devices on smithi173 2023-09-23T18:12:19.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:19 smithi098 bash[27349]: audit 2023-09-23T18:12:18.491324+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:19.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:19 smithi098 bash[27349]: audit 2023-09-23T18:12:18.493007+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:12:19.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:19 smithi098 bash[27349]: audit 2023-09-23T18:12:18.494238+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:12:19.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:19 smithi098 bash[27349]: audit 2023-09-23T18:12:18.495391+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:12:19.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:19 smithi098 bash[27349]: audit 2023-09-23T18:12:18.496373+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:12:19.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:19 smithi098 bash[27349]: cephadm 2023-09-23T18:12:18.497102+0000 mgr.y (mgr.14146) 265 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 4316M 2023-09-23T18:12:19.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:19 smithi098 bash[27349]: audit 2023-09-23T18:12:18.501719+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:19.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:19 smithi098 bash[27349]: audit 2023-09-23T18:12:18.503497+0000 mon.a (mon.0) 535 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:12:19.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:19 smithi098 bash[27349]: audit 2023-09-23T18:12:18.504835+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:12:19.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:19 smithi098 bash[27349]: audit 2023-09-23T18:12:18.510965+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:19.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:19 smithi098 bash[27349]: cluster 2023-09-23T18:12:19.180294+0000 mgr.y (mgr.14146) 266 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:19.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:19 smithi098 bash[33599]: cephadm 2023-09-23T18:12:18.481657+0000 mgr.y (mgr.14146) 264 : cephadm [INF] Detected new or changed devices on smithi173 2023-09-23T18:12:19.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:19 smithi098 bash[33599]: audit 2023-09-23T18:12:18.491324+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:19.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:19 smithi098 bash[33599]: audit 2023-09-23T18:12:18.493007+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:12:19.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:19 smithi098 bash[33599]: audit 2023-09-23T18:12:18.494238+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:12:19.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:19 smithi098 bash[33599]: audit 2023-09-23T18:12:18.495391+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:12:19.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:19 smithi098 bash[33599]: audit 2023-09-23T18:12:18.496373+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:12:19.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:19 smithi098 bash[33599]: cephadm 2023-09-23T18:12:18.497102+0000 mgr.y (mgr.14146) 265 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 4316M 2023-09-23T18:12:19.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:19 smithi098 bash[33599]: audit 2023-09-23T18:12:18.501719+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:19.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:19 smithi098 bash[33599]: audit 2023-09-23T18:12:18.503497+0000 mon.a (mon.0) 535 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:12:19.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:19 smithi098 bash[33599]: audit 2023-09-23T18:12:18.504835+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:12:19.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:19 smithi098 bash[33599]: audit 2023-09-23T18:12:18.510965+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:19.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:19 smithi098 bash[33599]: cluster 2023-09-23T18:12:19.180294+0000 mgr.y (mgr.14146) 266 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:22.213 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:12:22.213 INFO:teuthology.orchestra.run.smithi098.stdout:{"epoch":50,"fsid":"756c96d4-5a3b-11ee-8db4-212e2dc638e7","created":"2023-09-23T18:04:11.580781+0000","modified":"2023-09-23T18:12:11.246995+0000","last_up_change":"2023-09-23T18:12:09.242233+0000","last_in_change":"2023-09-23T18:11:43.419979+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-23T18:08:35.146924+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"4f35f176-4d07-4157-aeb9-dd8f71da3a63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6802","nonce":1147685383},{"type":"v1","addr":"172.21.15.98:6803","nonce":1147685383}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6804","nonce":1147685383},{"type":"v1","addr":"172.21.15.98:6805","nonce":1147685383}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6808","nonce":1147685383},{"type":"v1","addr":"172.21.15.98:6809","nonce":1147685383}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6806","nonce":1147685383},{"type":"v1","addr":"172.21.15.98:6807","nonce":1147685383}]},"public_addr":"172.21.15.98:6803/1147685383","cluster_addr":"172.21.15.98:6805/1147685383","heartbeat_back_addr":"172.21.15.98:6809/1147685383","heartbeat_front_addr":"172.21.15.98:6807/1147685383","state":["exists","up"]},{"osd":1,"uuid":"5d777d1f-3bd3-49ba-b0de-0ac532ee3934","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6810","nonce":3945616078},{"type":"v1","addr":"172.21.15.98:6811","nonce":3945616078}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6812","nonce":3945616078},{"type":"v1","addr":"172.21.15.98:6813","nonce":3945616078}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6816","nonce":3945616078},{"type":"v1","addr":"172.21.15.98:6817","nonce":3945616078}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6814","nonce":3945616078},{"type":"v1","addr":"172.21.15.98:6815","nonce":3945616078}]},"public_addr":"172.21.15.98:6811/3945616078","cluster_addr":"172.21.15.98:6813/3945616078","heartbeat_back_addr":"172.21.15.98:6817/3945616078","heartbeat_front_addr":"172.21.15.98:6815/3945616078","state":["exists","up"]},{"osd":2,"uuid":"63978288-ac36-4cf2-bbb7-46dc828e6ce5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6818","nonce":2338555672},{"type":"v1","addr":"172.21.15.98:6819","nonce":2338555672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6820","nonce":2338555672},{"type":"v1","addr":"172.21.15.98:6821","nonce":2338555672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6824","nonce":2338555672},{"type":"v1","addr":"172.21.15.98:6825","nonce":2338555672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6822","nonce":2338555672},{"type":"v1","addr":"172.21.15.98:6823","nonce":2338555672}]},"public_addr":"172.21.15.98:6819/2338555672","cluster_addr":"172.21.15.98:6821/2338555672","heartbeat_back_addr":"172.21.15.98:6825/2338555672","heartbeat_front_addr":"172.21.15.98:6823/2338555672","state":["exists","up"]},{"osd":3,"uuid":"48596d91-46da-4815-b810-d493bbf01dcc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6826","nonce":2634725099},{"type":"v1","addr":"172.21.15.98:6827","nonce":2634725099}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6828","nonce":2634725099},{"type":"v1","addr":"172.21.15.98:6829","nonce":2634725099}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6832","nonce":2634725099},{"type":"v1","addr":"172.21.15.98:6833","nonce":2634725099}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6830","nonce":2634725099},{"type":"v1","addr":"172.21.15.98:6831","nonce":2634725099}]},"public_addr":"172.21.15.98:6827/2634725099","cluster_addr":"172.21.15.98:6829/2634725099","heartbeat_back_addr":"172.21.15.98:6833/2634725099","heartbeat_front_addr":"172.21.15.98:6831/2634725099","state":["exists","up"]},{"osd":4,"uuid":"a562afc6-ffe6-44bb-91a9-8a04018c3c6f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6800","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6801","nonce":165906554}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6802","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6803","nonce":165906554}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6806","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6807","nonce":165906554}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6804","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6805","nonce":165906554}]},"public_addr":"172.21.15.173:6801/165906554","cluster_addr":"172.21.15.173:6803/165906554","heartbeat_back_addr":"172.21.15.173:6807/165906554","heartbeat_front_addr":"172.21.15.173:6805/165906554","state":["exists","up"]},{"osd":5,"uuid":"add28146-8301-4b4c-9616-967e9df3a8d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6808","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6809","nonce":2894909344}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6810","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6811","nonce":2894909344}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6814","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6815","nonce":2894909344}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6812","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6813","nonce":2894909344}]},"public_addr":"172.21.15.173:6809/2894909344","cluster_addr":"172.21.15.173:6811/2894909344","heartbeat_back_addr":"172.21.15.173:6815/2894909344","heartbeat_front_addr":"172.21.15.173:6813/2894909344","state":["exists","up"]},{"osd":6,"uuid":"f30854de-bc26-465d-8b0e-a9211ae2d9fa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6816","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6817","nonce":3550013548}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6818","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6819","nonce":3550013548}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6822","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6823","nonce":3550013548}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6820","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6821","nonce":3550013548}]},"public_addr":"172.21.15.173:6817/3550013548","cluster_addr":"172.21.15.173:6819/3550013548","heartbeat_back_addr":"172.21.15.173:6823/3550013548","heartbeat_front_addr":"172.21.15.173:6821/3550013548","state":["exists","up"]},{"osd":7,"uuid":"5f540de3-48f9-4bc8-b14a-9aadf838f16b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6824","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6825","nonce":135608222}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6826","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6827","nonce":135608222}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6830","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6831","nonce":135608222}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6828","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6829","nonce":135608222}]},"public_addr":"172.21.15.173:6825/135608222","cluster_addr":"172.21.15.173:6827/135608222","heartbeat_back_addr":"172.21.15.173:6831/135608222","heartbeat_front_addr":"172.21.15.173:6829/135608222","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:07:04.739243+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:07:48.583198+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:08:33.173348+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:09:18.218502+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:09:59.765019+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:10:42.490689+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:11:24.357920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:12:08.168900+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.98:0/3472030975":"2023-09-24T18:05:11.060631+0000","172.21.15.98:6800/1392949322":"2023-09-24T18:05:11.060631+0000","172.21.15.98:0/3976553993":"2023-09-24T18:04:40.596823+0000","172.21.15.98:0/3213365423":"2023-09-24T18:04:40.596823+0000","172.21.15.98:6801/1392949322":"2023-09-24T18:05:11.060631+0000","172.21.15.98:6800/3368447653":"2023-09-24T18:04:40.596823+0000","172.21.15.98:0/1603044865":"2023-09-24T18:05:11.060631+0000","172.21.15.98:6801/3368447653":"2023-09-24T18:04:40.596823+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T18:12:22.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:22 smithi173 bash[27694]: cluster 2023-09-23T18:12:21.180823+0000 mgr.y (mgr.14146) 267 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:22.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:22 smithi173 bash[27694]: audit 2023-09-23T18:12:22.206612+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.98:0/1934048810' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:12:22.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:22 smithi098 bash[33599]: cluster 2023-09-23T18:12:21.180823+0000 mgr.y (mgr.14146) 267 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:22.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:22 smithi098 bash[33599]: audit 2023-09-23T18:12:22.206612+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.98:0/1934048810' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:12:22.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:22 smithi098 bash[27349]: cluster 2023-09-23T18:12:21.180823+0000 mgr.y (mgr.14146) 267 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:22.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:22 smithi098 bash[27349]: audit 2023-09-23T18:12:22.206612+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.98:0/1934048810' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:12:22.959 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': 'device_health_metrics', 'create_time': '2023-09-23T18:08:35.146924+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr_devicehealth': {}}}] 2023-09-23T18:12:22.960 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool get device_health_metrics pg_num 2023-09-23T18:12:24.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:24 smithi098 bash[27349]: cluster 2023-09-23T18:12:23.181346+0000 mgr.y (mgr.14146) 268 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:24.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:24 smithi098 bash[33599]: cluster 2023-09-23T18:12:23.181346+0000 mgr.y (mgr.14146) 268 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:24.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:24 smithi173 bash[27694]: cluster 2023-09-23T18:12:23.181346+0000 mgr.y (mgr.14146) 268 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:26.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:26 smithi173 bash[27694]: cluster 2023-09-23T18:12:25.181846+0000 mgr.y (mgr.14146) 269 : cluster [DBG] pgmap v257: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:26.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:26 smithi098 bash[27349]: cluster 2023-09-23T18:12:25.181846+0000 mgr.y (mgr.14146) 269 : cluster [DBG] pgmap v257: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:26.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:26 smithi098 bash[33599]: cluster 2023-09-23T18:12:25.181846+0000 mgr.y (mgr.14146) 269 : cluster [DBG] pgmap v257: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:27.716 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:28.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:28 smithi173 bash[27694]: cluster 2023-09-23T18:12:27.182457+0000 mgr.y (mgr.14146) 270 : cluster [DBG] pgmap v258: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:28.654 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:28 smithi098 bash[27349]: cluster 2023-09-23T18:12:27.182457+0000 mgr.y (mgr.14146) 270 : cluster [DBG] pgmap v258: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:28.654 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:28 smithi098 bash[33599]: cluster 2023-09-23T18:12:27.182457+0000 mgr.y (mgr.14146) 270 : cluster [DBG] pgmap v258: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:30.503 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:30 smithi098 bash[33599]: cluster 2023-09-23T18:12:29.182908+0000 mgr.y (mgr.14146) 271 : cluster [DBG] pgmap v259: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:30.504 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:30 smithi098 bash[27349]: cluster 2023-09-23T18:12:29.182908+0000 mgr.y (mgr.14146) 271 : cluster [DBG] pgmap v259: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:30.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:30 smithi173 bash[27694]: cluster 2023-09-23T18:12:29.182908+0000 mgr.y (mgr.14146) 271 : cluster [DBG] pgmap v259: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:30.850 INFO:teuthology.orchestra.run.smithi098.stdout:pg_num: 1 2023-09-23T18:12:31.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:31 smithi098 bash[27349]: audit 2023-09-23T18:12:30.845283+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.98:0/508498597' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-09-23T18:12:31.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:31 smithi098 bash[33599]: audit 2023-09-23T18:12:30.845283+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.98:0/508498597' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-09-23T18:12:31.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:31 smithi173 bash[27694]: audit 2023-09-23T18:12:30.845283+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.98:0/508498597' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-09-23T18:12:31.591 INFO:tasks.cephadm:Adding prometheus.a on smithi173 2023-09-23T18:12:31.592 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph orch apply prometheus '1;smithi173=a' 2023-09-23T18:12:32.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:32 smithi173 bash[27694]: cluster 2023-09-23T18:12:31.183487+0000 mgr.y (mgr.14146) 272 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:32.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:32 smithi098 bash[27349]: cluster 2023-09-23T18:12:31.183487+0000 mgr.y (mgr.14146) 272 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:32.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:32 smithi098 bash[33599]: cluster 2023-09-23T18:12:31.183487+0000 mgr.y (mgr.14146) 272 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:34.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:34 smithi098 bash[27349]: cluster 2023-09-23T18:12:33.184063+0000 mgr.y (mgr.14146) 273 : cluster [DBG] pgmap v261: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:34.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:34 smithi098 bash[33599]: cluster 2023-09-23T18:12:33.184063+0000 mgr.y (mgr.14146) 273 : cluster [DBG] pgmap v261: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:34.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:34 smithi173 bash[27694]: cluster 2023-09-23T18:12:33.184063+0000 mgr.y (mgr.14146) 273 : cluster [DBG] pgmap v261: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:34.653 INFO:teuthology.orchestra.run.smithi173.stdout:Scheduled prometheus update... 2023-09-23T18:12:35.427 DEBUG:teuthology.orchestra.run.smithi173:prometheus.a> sudo journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@prometheus.a.service 2023-09-23T18:12:35.430 INFO:tasks.cephadm:Adding node-exporter.a on smithi098 2023-09-23T18:12:35.430 INFO:tasks.cephadm:Adding node-exporter.b on smithi173 2023-09-23T18:12:35.430 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi098=a;smithi173=b' 2023-09-23T18:12:35.528 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:12:35.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:35 smithi173 bash[27694]: audit 2023-09-23T18:12:34.637461+0000 mgr.y (mgr.14146) 274 : audit [DBG] from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi173=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:12:35.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:35 smithi173 bash[27694]: cephadm 2023-09-23T18:12:34.640821+0000 mgr.y (mgr.14146) 275 : cephadm [INF] Saving service prometheus spec with placement smithi173=a;count:1 2023-09-23T18:12:35.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:35 smithi173 bash[27694]: audit 2023-09-23T18:12:34.647063+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:35.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:35 smithi173 bash[27694]: audit 2023-09-23T18:12:34.671404+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:12:35.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:35 smithi173 bash[27694]: audit 2023-09-23T18:12:34.931725+0000 mon.a (mon.0) 540 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:12:35.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:35 smithi173 bash[27694]: audit 2023-09-23T18:12:34.933346+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:12:35.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:35 smithi173 bash[27694]: audit 2023-09-23T18:12:34.942076+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:35.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:35 smithi173 bash[27694]: audit 2023-09-23T18:12:34.951910+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T18:12:35.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:35 smithi173 bash[27694]: cluster 2023-09-23T18:12:35.184629+0000 mgr.y (mgr.14146) 276 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:35.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:35 smithi098 bash[33599]: audit 2023-09-23T18:12:34.637461+0000 mgr.y (mgr.14146) 274 : audit [DBG] from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi173=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:12:35.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:35 smithi098 bash[33599]: cephadm 2023-09-23T18:12:34.640821+0000 mgr.y (mgr.14146) 275 : cephadm [INF] Saving service prometheus spec with placement smithi173=a;count:1 2023-09-23T18:12:35.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:35 smithi098 bash[33599]: audit 2023-09-23T18:12:34.647063+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:35.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:35 smithi098 bash[33599]: audit 2023-09-23T18:12:34.671404+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:12:35.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:35 smithi098 bash[33599]: audit 2023-09-23T18:12:34.931725+0000 mon.a (mon.0) 540 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:12:35.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:35 smithi098 bash[33599]: audit 2023-09-23T18:12:34.933346+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:12:35.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:35 smithi098 bash[33599]: audit 2023-09-23T18:12:34.942076+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:35.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:35 smithi098 bash[33599]: audit 2023-09-23T18:12:34.951910+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T18:12:35.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:35 smithi098 bash[33599]: cluster 2023-09-23T18:12:35.184629+0000 mgr.y (mgr.14146) 276 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:35.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:35 smithi098 bash[27349]: audit 2023-09-23T18:12:34.637461+0000 mgr.y (mgr.14146) 274 : audit [DBG] from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi173=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:12:35.947 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:35 smithi098 bash[27349]: cephadm 2023-09-23T18:12:34.640821+0000 mgr.y (mgr.14146) 275 : cephadm [INF] Saving service prometheus spec with placement smithi173=a;count:1 2023-09-23T18:12:35.947 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:35 smithi098 bash[27349]: audit 2023-09-23T18:12:34.647063+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:35.947 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:35 smithi098 bash[27349]: audit 2023-09-23T18:12:34.671404+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:12:35.947 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:35 smithi098 bash[27349]: audit 2023-09-23T18:12:34.931725+0000 mon.a (mon.0) 540 : audit [DBG] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:12:35.947 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:35 smithi098 bash[27349]: audit 2023-09-23T18:12:34.933346+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:12:35.948 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:35 smithi098 bash[27349]: audit 2023-09-23T18:12:34.942076+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' 2023-09-23T18:12:35.948 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:35 smithi098 bash[27349]: audit 2023-09-23T18:12:34.951910+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T18:12:35.948 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:35 smithi098 bash[27349]: cluster 2023-09-23T18:12:35.184629+0000 mgr.y (mgr.14146) 276 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:36.267 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:36 smithi173 bash[28552]: ignoring --setuser ceph since I am not root 2023-09-23T18:12:36.267 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:36 smithi173 bash[28552]: ignoring --setgroup ceph since I am not root 2023-09-23T18:12:36.267 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:36 smithi173 bash[28552]: debug 2023-09-23T18:12:36.004+0000 7f81a0822700 1 -- 172.21.15.173:0/1638230677 <== mon.1 v2:172.21.15.173:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564a60de44e0 con 0x564a60ed8800 2023-09-23T18:12:36.267 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:36 smithi173 bash[28552]: debug 2023-09-23T18:12:36.156+0000 7f81b09e9680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T18:12:36.268 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:36 smithi098 bash[27618]: ignoring --setuser ceph since I am not root 2023-09-23T18:12:36.269 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:36 smithi098 bash[27618]: ignoring --setgroup ceph since I am not root 2023-09-23T18:12:36.269 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:36 smithi098 bash[27618]: debug 2023-09-23T18:12:35.999+0000 7fb4fc236700 1 -- 172.21.15.98:0/429112610 <== mon.2 v2:172.21.15.98:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d042d864e0 con 0x55d042e7b000 2023-09-23T18:12:36.269 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:36 smithi098 bash[27618]: debug 2023-09-23T18:12:36.151+0000 7fb50c3fd680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T18:12:36.526 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:36 smithi173 bash[28552]: debug 2023-09-23T18:12:36.264+0000 7f81b09e9680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T18:12:36.694 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:36 smithi098 bash[27618]: debug 2023-09-23T18:12:36.259+0000 7fb50c3fd680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T18:12:36.958 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:36 smithi173 bash[28552]: debug 2023-09-23T18:12:36.756+0000 7f81b09e9680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T18:12:37.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:36 smithi098 bash[27349]: audit 2023-09-23T18:12:35.958713+0000 mon.a (mon.0) 544 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T18:12:37.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:36 smithi098 bash[27349]: cluster 2023-09-23T18:12:35.958797+0000 mon.a (mon.0) 545 : cluster [DBG] mgrmap e15: y(active, since 7m), standbys: x 2023-09-23T18:12:37.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:36 smithi098 bash[27618]: debug 2023-09-23T18:12:36.739+0000 7fb50c3fd680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T18:12:37.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:36 smithi098 bash[33599]: audit 2023-09-23T18:12:35.958713+0000 mon.a (mon.0) 544 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T18:12:37.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:36 smithi098 bash[33599]: cluster 2023-09-23T18:12:35.958797+0000 mon.a (mon.0) 545 : cluster [DBG] mgrmap e15: y(active, since 7m), standbys: x 2023-09-23T18:12:37.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:36 smithi173 bash[27694]: audit 2023-09-23T18:12:35.958713+0000 mon.a (mon.0) 544 : audit [INF] from='mgr.14146 172.21.15.98:0/3858281433' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T18:12:37.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:36 smithi173 bash[27694]: cluster 2023-09-23T18:12:35.958797+0000 mon.a (mon.0) 545 : cluster [DBG] mgrmap e15: y(active, since 7m), standbys: x 2023-09-23T18:12:37.944 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:37 smithi098 bash[27618]: debug 2023-09-23T18:12:37.679+0000 7fb50c3fd680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T18:12:37.944 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:37 smithi098 bash[27618]: debug 2023-09-23T18:12:37.775+0000 7fb50c3fd680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T18:12:38.026 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:37 smithi173 bash[28552]: debug 2023-09-23T18:12:37.708+0000 7f81b09e9680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T18:12:38.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:37 smithi173 bash[28552]: debug 2023-09-23T18:12:37.808+0000 7f81b09e9680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T18:12:38.360 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:38 smithi173 bash[28552]: debug 2023-09-23T18:12:38.060+0000 7f81b09e9680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T18:12:38.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:38 smithi098 bash[27618]: debug 2023-09-23T18:12:38.023+0000 7fb50c3fd680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T18:12:39.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:39 smithi098 bash[27618]: debug 2023-09-23T18:12:39.163+0000 7fb50c3fd680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T18:12:39.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:39 smithi098 bash[27618]: debug 2023-09-23T18:12:39.335+0000 7fb50c3fd680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:12:39.517 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:39 smithi173 bash[28552]: debug 2023-09-23T18:12:39.252+0000 7f81b09e9680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T18:12:39.517 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:39 smithi173 bash[28552]: debug 2023-09-23T18:12:39.416+0000 7f81b09e9680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:12:39.776 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:39 smithi173 bash[28552]: debug 2023-09-23T18:12:39.512+0000 7f81b09e9680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T18:12:39.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:39 smithi173 bash[28552]: debug 2023-09-23T18:12:39.628+0000 7f81b09e9680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T18:12:39.777 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:39 smithi098 bash[27618]: debug 2023-09-23T18:12:39.435+0000 7fb50c3fd680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T18:12:39.777 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:39 smithi098 bash[27618]: debug 2023-09-23T18:12:39.547+0000 7fb50c3fd680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T18:12:40.140 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:39 smithi098 bash[27618]: debug 2023-09-23T18:12:39.767+0000 7fb50c3fd680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T18:12:40.223 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:39 smithi173 bash[28552]: debug 2023-09-23T18:12:39.852+0000 7f81b09e9680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T18:12:40.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:40 smithi098 bash[27618]: debug 2023-09-23T18:12:40.131+0000 7fb50c3fd680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T18:12:40.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:40 smithi098 bash[27618]: debug 2023-09-23T18:12:40.255+0000 7fb50c3fd680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T18:12:40.526 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:40 smithi173 bash[28552]: debug 2023-09-23T18:12:40.220+0000 7f81b09e9680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T18:12:40.527 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:40 smithi173 bash[28552]: debug 2023-09-23T18:12:40.340+0000 7f81b09e9680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T18:12:41.445 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:41 smithi098 bash[27618]: debug 2023-09-23T18:12:41.111+0000 7fb50c3fd680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T18:12:41.445 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:41 smithi098 bash[27618]: debug 2023-09-23T18:12:41.215+0000 7fb50c3fd680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T18:12:41.445 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:41 smithi098 bash[27618]: debug 2023-09-23T18:12:41.327+0000 7fb50c3fd680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T18:12:41.527 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:41 smithi173 bash[28552]: debug 2023-09-23T18:12:41.208+0000 7f81b09e9680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T18:12:41.527 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:41 smithi173 bash[28552]: debug 2023-09-23T18:12:41.316+0000 7f81b09e9680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T18:12:41.527 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:41 smithi173 bash[28552]: debug 2023-09-23T18:12:41.432+0000 7f81b09e9680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T18:12:41.846 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:41 smithi098 bash[27618]: debug 2023-09-23T18:12:41.551+0000 7fb50c3fd680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T18:12:41.946 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:41 smithi173 bash[28552]: debug 2023-09-23T18:12:41.648+0000 7f81b09e9680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T18:12:42.181 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:41 smithi098 bash[27618]: debug 2023-09-23T18:12:41.835+0000 7fb50c3fd680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T18:12:42.181 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27618]: debug 2023-09-23T18:12:42.003+0000 7fb50c3fd680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T18:12:42.276 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:41 smithi173 bash[28552]: debug 2023-09-23T18:12:41.944+0000 7f81b09e9680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T18:12:42.277 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[28552]: debug 2023-09-23T18:12:42.108+0000 7f81b09e9680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T18:12:42.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27618]: debug 2023-09-23T18:12:42.171+0000 7fb50c3fd680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:12:42.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27618]: debug 2023-09-23T18:12:42.391+0000 7fb50c3fd680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T18:12:42.587 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[28552]: debug 2023-09-23T18:12:42.280+0000 7f81b09e9680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:12:42.587 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[28552]: debug 2023-09-23T18:12:42.492+0000 7f81b09e9680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T18:12:42.795 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27349]: cluster 2023-09-23T18:12:42.525813+0000 mon.a (mon.0) 546 : cluster [INF] Active manager daemon y restarted 2023-09-23T18:12:42.795 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27349]: cluster 2023-09-23T18:12:42.526224+0000 mon.a (mon.0) 547 : cluster [INF] Activating manager daemon y 2023-09-23T18:12:42.795 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27349]: cluster 2023-09-23T18:12:42.538232+0000 mon.a (mon.0) 548 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-09-23T18:12:42.795 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27349]: cluster 2023-09-23T18:12:42.538835+0000 mon.a (mon.0) 549 : cluster [DBG] mgrmap e16: y(active, starting, since 0.0127383s), standbys: x 2023-09-23T18:12:42.795 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27349]: audit 2023-09-23T18:12:42.542532+0000 mon.c (mon.2) 14 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:12:42.795 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27349]: audit 2023-09-23T18:12:42.542856+0000 mon.c (mon.2) 15 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:12:42.795 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27349]: audit 2023-09-23T18:12:42.543161+0000 mon.c (mon.2) 16 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:12:42.795 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27349]: audit 2023-09-23T18:12:42.544537+0000 mon.c (mon.2) 17 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T18:12:42.795 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27349]: audit 2023-09-23T18:12:42.544917+0000 mon.c (mon.2) 18 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T18:12:42.795 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27349]: audit 2023-09-23T18:12:42.545190+0000 mon.c (mon.2) 19 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:12:42.796 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27349]: audit 2023-09-23T18:12:42.545459+0000 mon.c (mon.2) 20 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:12:42.796 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27349]: audit 2023-09-23T18:12:42.545814+0000 mon.c (mon.2) 21 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:12:42.796 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27349]: audit 2023-09-23T18:12:42.546174+0000 mon.c (mon.2) 22 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:12:42.796 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27349]: audit 2023-09-23T18:12:42.546523+0000 mon.c (mon.2) 23 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:12:42.796 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27349]: audit 2023-09-23T18:12:42.546855+0000 mon.c (mon.2) 24 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:12:42.796 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27349]: audit 2023-09-23T18:12:42.547188+0000 mon.c (mon.2) 25 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:12:42.796 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27349]: audit 2023-09-23T18:12:42.547520+0000 mon.c (mon.2) 26 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:12:42.796 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27349]: audit 2023-09-23T18:12:42.547860+0000 mon.c (mon.2) 27 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T18:12:42.796 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27349]: audit 2023-09-23T18:12:42.548123+0000 mon.c (mon.2) 28 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T18:12:42.796 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27349]: audit 2023-09-23T18:12:42.548777+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T18:12:42.797 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27618]: debug 2023-09-23T18:12:42.499+0000 7fb50c3fd680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T18:12:42.797 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27618]: [23/Sep/2023:18:12:42] ENGINE Bus STARTING 2023-09-23T18:12:42.797 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[33599]: cluster 2023-09-23T18:12:42.525813+0000 mon.a (mon.0) 546 : cluster [INF] Active manager daemon y restarted 2023-09-23T18:12:42.797 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[33599]: cluster 2023-09-23T18:12:42.526224+0000 mon.a (mon.0) 547 : cluster [INF] Activating manager daemon y 2023-09-23T18:12:42.797 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[33599]: cluster 2023-09-23T18:12:42.538232+0000 mon.a (mon.0) 548 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-09-23T18:12:42.797 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[33599]: cluster 2023-09-23T18:12:42.538835+0000 mon.a (mon.0) 549 : cluster [DBG] mgrmap e16: y(active, starting, since 0.0127383s), standbys: x 2023-09-23T18:12:42.798 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[33599]: audit 2023-09-23T18:12:42.542532+0000 mon.c (mon.2) 14 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:12:42.798 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[33599]: audit 2023-09-23T18:12:42.542856+0000 mon.c (mon.2) 15 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:12:42.798 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[33599]: audit 2023-09-23T18:12:42.543161+0000 mon.c (mon.2) 16 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:12:42.798 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[33599]: audit 2023-09-23T18:12:42.544537+0000 mon.c (mon.2) 17 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T18:12:42.798 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[33599]: audit 2023-09-23T18:12:42.544917+0000 mon.c (mon.2) 18 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T18:12:42.798 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[33599]: audit 2023-09-23T18:12:42.545190+0000 mon.c (mon.2) 19 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:12:42.799 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[33599]: audit 2023-09-23T18:12:42.545459+0000 mon.c (mon.2) 20 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:12:42.799 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[33599]: audit 2023-09-23T18:12:42.545814+0000 mon.c (mon.2) 21 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:12:42.800 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[33599]: audit 2023-09-23T18:12:42.546174+0000 mon.c (mon.2) 22 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:12:42.800 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[33599]: audit 2023-09-23T18:12:42.546523+0000 mon.c (mon.2) 23 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:12:42.800 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[33599]: audit 2023-09-23T18:12:42.546855+0000 mon.c (mon.2) 24 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:12:42.800 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[33599]: audit 2023-09-23T18:12:42.547188+0000 mon.c (mon.2) 25 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:12:42.800 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[33599]: audit 2023-09-23T18:12:42.547520+0000 mon.c (mon.2) 26 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:12:42.800 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[33599]: audit 2023-09-23T18:12:42.547860+0000 mon.c (mon.2) 27 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T18:12:42.800 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[33599]: audit 2023-09-23T18:12:42.548123+0000 mon.c (mon.2) 28 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T18:12:42.800 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[33599]: audit 2023-09-23T18:12:42.548777+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T18:12:42.857 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[28552]: debug 2023-09-23T18:12:42.604+0000 7f81b09e9680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T18:12:42.858 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[28552]: [23/Sep/2023:18:12:42] ENGINE Bus STARTING 2023-09-23T18:12:42.858 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[28552]: CherryPy Checker: 2023-09-23T18:12:42.858 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[28552]: The Application mounted at '' has an empty config. 2023-09-23T18:12:42.858 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[28552]: [23/Sep/2023:18:12:42] ENGINE Serving on http://:::9283 2023-09-23T18:12:42.858 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[28552]: [23/Sep/2023:18:12:42] ENGINE Bus STARTED 2023-09-23T18:12:42.858 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[27694]: cluster 2023-09-23T18:12:42.525813+0000 mon.a (mon.0) 546 : cluster [INF] Active manager daemon y restarted 2023-09-23T18:12:42.859 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[27694]: cluster 2023-09-23T18:12:42.526224+0000 mon.a (mon.0) 547 : cluster [INF] Activating manager daemon y 2023-09-23T18:12:42.859 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[27694]: cluster 2023-09-23T18:12:42.538232+0000 mon.a (mon.0) 548 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-09-23T18:12:42.859 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[27694]: cluster 2023-09-23T18:12:42.538835+0000 mon.a (mon.0) 549 : cluster [DBG] mgrmap e16: y(active, starting, since 0.0127383s), standbys: x 2023-09-23T18:12:42.859 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[27694]: audit 2023-09-23T18:12:42.542532+0000 mon.c (mon.2) 14 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:12:42.859 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[27694]: audit 2023-09-23T18:12:42.542856+0000 mon.c (mon.2) 15 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:12:42.859 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[27694]: audit 2023-09-23T18:12:42.543161+0000 mon.c (mon.2) 16 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:12:42.859 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[27694]: audit 2023-09-23T18:12:42.544537+0000 mon.c (mon.2) 17 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T18:12:42.859 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[27694]: audit 2023-09-23T18:12:42.544917+0000 mon.c (mon.2) 18 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T18:12:42.859 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[27694]: audit 2023-09-23T18:12:42.545190+0000 mon.c (mon.2) 19 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:12:42.859 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[27694]: audit 2023-09-23T18:12:42.545459+0000 mon.c (mon.2) 20 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:12:42.859 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[27694]: audit 2023-09-23T18:12:42.545814+0000 mon.c (mon.2) 21 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:12:42.860 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[27694]: audit 2023-09-23T18:12:42.546174+0000 mon.c (mon.2) 22 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:12:42.860 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[27694]: audit 2023-09-23T18:12:42.546523+0000 mon.c (mon.2) 23 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:12:42.860 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[27694]: audit 2023-09-23T18:12:42.546855+0000 mon.c (mon.2) 24 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:12:42.860 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[27694]: audit 2023-09-23T18:12:42.547188+0000 mon.c (mon.2) 25 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:12:42.860 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[27694]: audit 2023-09-23T18:12:42.547520+0000 mon.c (mon.2) 26 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:12:42.860 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[27694]: audit 2023-09-23T18:12:42.547860+0000 mon.c (mon.2) 27 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T18:12:42.860 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[27694]: audit 2023-09-23T18:12:42.548123+0000 mon.c (mon.2) 28 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T18:12:42.860 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:42 smithi173 bash[27694]: audit 2023-09-23T18:12:42.548777+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T18:12:43.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27618]: Warning: Permanently added '172.21.15.173' (ECDSA) to the list of known hosts. 2023-09-23T18:12:43.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27618]: Warning: Permanently added '172.21.15.98' (ECDSA) to the list of known hosts. 2023-09-23T18:12:43.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27618]: CherryPy Checker: 2023-09-23T18:12:43.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27618]: The Application mounted at '' has an empty config. 2023-09-23T18:12:43.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27618]: [23/Sep/2023:18:12:42] ENGINE Serving on http://:::9283 2023-09-23T18:12:43.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:12:42 smithi098 bash[27618]: [23/Sep/2023:18:12:42] ENGINE Bus STARTED 2023-09-23T18:12:43.565 INFO:teuthology.orchestra.run.smithi173.stdout:Scheduled node-exporter update... 2023-09-23T18:12:43.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[27349]: cluster 2023-09-23T18:12:42.580523+0000 mon.a (mon.0) 550 : cluster [INF] Manager daemon y is now available 2023-09-23T18:12:43.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[27349]: cluster 2023-09-23T18:12:42.624002+0000 mon.a (mon.0) 551 : cluster [DBG] Standby manager daemon x restarted 2023-09-23T18:12:43.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[27349]: cluster 2023-09-23T18:12:42.624201+0000 mon.a (mon.0) 552 : cluster [DBG] Standby manager daemon x started 2023-09-23T18:12:43.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[27349]: audit 2023-09-23T18:12:42.631479+0000 mon.b (mon.1) 22 : audit [DBG] from='mgr.? 172.21.15.173:0/2175748338' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T18:12:43.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[27349]: audit 2023-09-23T18:12:42.632682+0000 mon.b (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.173:0/2175748338' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:12:43.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[27349]: audit 2023-09-23T18:12:42.636245+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.173:0/2175748338' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T18:12:43.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[27349]: audit 2023-09-23T18:12:42.637053+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.173:0/2175748338' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:12:43.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[27349]: audit 2023-09-23T18:12:42.727284+0000 mon.c (mon.2) 30 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:12:43.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[27349]: audit 2023-09-23T18:12:42.730224+0000 mon.c (mon.2) 31 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:12:43.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[27349]: audit 2023-09-23T18:12:42.782848+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:43.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[27349]: audit 2023-09-23T18:12:42.802236+0000 mon.c (mon.2) 32 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:12:43.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[27349]: audit 2023-09-23T18:12:42.802669+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:12:43.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[27349]: audit 2023-09-23T18:12:42.812073+0000 mon.c (mon.2) 33 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T18:12:43.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[27349]: audit 2023-09-23T18:12:42.812554+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T18:12:43.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[27349]: cluster 2023-09-23T18:12:43.545525+0000 mon.a (mon.0) 556 : cluster [DBG] mgrmap e17: y(active, since 1.01943s), standbys: x 2023-09-23T18:12:43.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[27349]: audit 2023-09-23T18:12:43.559218+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:43.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[33599]: cluster 2023-09-23T18:12:42.580523+0000 mon.a (mon.0) 550 : cluster [INF] Manager daemon y is now available 2023-09-23T18:12:43.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[33599]: cluster 2023-09-23T18:12:42.624002+0000 mon.a (mon.0) 551 : cluster [DBG] Standby manager daemon x restarted 2023-09-23T18:12:43.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[33599]: cluster 2023-09-23T18:12:42.624201+0000 mon.a (mon.0) 552 : cluster [DBG] Standby manager daemon x started 2023-09-23T18:12:43.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[33599]: audit 2023-09-23T18:12:42.631479+0000 mon.b (mon.1) 22 : audit [DBG] from='mgr.? 172.21.15.173:0/2175748338' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T18:12:43.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[33599]: audit 2023-09-23T18:12:42.632682+0000 mon.b (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.173:0/2175748338' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:12:43.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[33599]: audit 2023-09-23T18:12:42.636245+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.173:0/2175748338' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T18:12:43.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[33599]: audit 2023-09-23T18:12:42.637053+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.173:0/2175748338' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:12:43.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[33599]: audit 2023-09-23T18:12:42.727284+0000 mon.c (mon.2) 30 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:12:43.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[33599]: audit 2023-09-23T18:12:42.730224+0000 mon.c (mon.2) 31 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:12:43.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[33599]: audit 2023-09-23T18:12:42.782848+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:43.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[33599]: audit 2023-09-23T18:12:42.802236+0000 mon.c (mon.2) 32 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:12:43.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[33599]: audit 2023-09-23T18:12:42.802669+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:12:43.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[33599]: audit 2023-09-23T18:12:42.812073+0000 mon.c (mon.2) 33 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T18:12:43.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[33599]: audit 2023-09-23T18:12:42.812554+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T18:12:43.948 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[33599]: cluster 2023-09-23T18:12:43.545525+0000 mon.a (mon.0) 556 : cluster [DBG] mgrmap e17: y(active, since 1.01943s), standbys: x 2023-09-23T18:12:43.948 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:43 smithi098 bash[33599]: audit 2023-09-23T18:12:43.559218+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:43 smithi173 bash[27694]: cluster 2023-09-23T18:12:42.580523+0000 mon.a (mon.0) 550 : cluster [INF] Manager daemon y is now available 2023-09-23T18:12:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:43 smithi173 bash[27694]: cluster 2023-09-23T18:12:42.624002+0000 mon.a (mon.0) 551 : cluster [DBG] Standby manager daemon x restarted 2023-09-23T18:12:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:43 smithi173 bash[27694]: cluster 2023-09-23T18:12:42.624201+0000 mon.a (mon.0) 552 : cluster [DBG] Standby manager daemon x started 2023-09-23T18:12:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:43 smithi173 bash[27694]: audit 2023-09-23T18:12:42.631479+0000 mon.b (mon.1) 22 : audit [DBG] from='mgr.? 172.21.15.173:0/2175748338' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T18:12:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:43 smithi173 bash[27694]: audit 2023-09-23T18:12:42.632682+0000 mon.b (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.173:0/2175748338' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:12:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:43 smithi173 bash[27694]: audit 2023-09-23T18:12:42.636245+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.173:0/2175748338' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T18:12:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:43 smithi173 bash[27694]: audit 2023-09-23T18:12:42.637053+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.173:0/2175748338' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:12:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:43 smithi173 bash[27694]: audit 2023-09-23T18:12:42.727284+0000 mon.c (mon.2) 30 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:12:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:43 smithi173 bash[27694]: audit 2023-09-23T18:12:42.730224+0000 mon.c (mon.2) 31 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:12:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:43 smithi173 bash[27694]: audit 2023-09-23T18:12:42.782848+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:43 smithi173 bash[27694]: audit 2023-09-23T18:12:42.802236+0000 mon.c (mon.2) 32 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:12:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:43 smithi173 bash[27694]: audit 2023-09-23T18:12:42.802669+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:12:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:43 smithi173 bash[27694]: audit 2023-09-23T18:12:42.812073+0000 mon.c (mon.2) 33 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T18:12:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:43 smithi173 bash[27694]: audit 2023-09-23T18:12:42.812554+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T18:12:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:43 smithi173 bash[27694]: cluster 2023-09-23T18:12:43.545525+0000 mon.a (mon.0) 556 : cluster [DBG] mgrmap e17: y(active, since 1.01943s), standbys: x 2023-09-23T18:12:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:43 smithi173 bash[27694]: audit 2023-09-23T18:12:43.559218+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:44.356 DEBUG:teuthology.orchestra.run.smithi098:node-exporter.a> sudo journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@node-exporter.a.service 2023-09-23T18:12:44.358 DEBUG:teuthology.orchestra.run.smithi173:node-exporter.b> sudo journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@node-exporter.b.service 2023-09-23T18:12:44.360 INFO:tasks.cephadm:Adding alertmanager.a on smithi098 2023-09-23T18:12:44.361 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi098=a' 2023-09-23T18:12:44.527 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:12:44.579 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:12:44.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:44 smithi098 bash[27349]: cephadm 2023-09-23T18:12:43.553900+0000 mgr.y (mgr.24344) 2 : cephadm [INF] Saving service node-exporter spec with placement smithi098=a;smithi173=b;count:2 2023-09-23T18:12:44.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:44 smithi098 bash[27349]: cluster 2023-09-23T18:12:43.554439+0000 mgr.y (mgr.24344) 3 : cluster [DBG] pgmap v3: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:44.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:44 smithi098 bash[33599]: cephadm 2023-09-23T18:12:43.553900+0000 mgr.y (mgr.24344) 2 : cephadm [INF] Saving service node-exporter spec with placement smithi098=a;smithi173=b;count:2 2023-09-23T18:12:44.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:44 smithi098 bash[33599]: cluster 2023-09-23T18:12:43.554439+0000 mgr.y (mgr.24344) 3 : cluster [DBG] pgmap v3: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:44.776 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:44 smithi173 bash[27694]: cephadm 2023-09-23T18:12:43.553900+0000 mgr.y (mgr.24344) 2 : cephadm [INF] Saving service node-exporter spec with placement smithi098=a;smithi173=b;count:2 2023-09-23T18:12:44.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:44 smithi173 bash[27694]: cluster 2023-09-23T18:12:43.554439+0000 mgr.y (mgr.24344) 3 : cluster [DBG] pgmap v3: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:45.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:45 smithi173 bash[27694]: cluster 2023-09-23T18:12:44.546031+0000 mgr.y (mgr.24344) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:45.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:45 smithi173 bash[27694]: cluster 2023-09-23T18:12:45.553207+0000 mon.a (mon.0) 558 : cluster [DBG] mgrmap e18: y(active, since 3s), standbys: x 2023-09-23T18:12:45.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:45 smithi098 bash[27349]: cluster 2023-09-23T18:12:44.546031+0000 mgr.y (mgr.24344) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:45.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:45 smithi098 bash[27349]: cluster 2023-09-23T18:12:45.553207+0000 mon.a (mon.0) 558 : cluster [DBG] mgrmap e18: y(active, since 3s), standbys: x 2023-09-23T18:12:45.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:45 smithi098 bash[33599]: cluster 2023-09-23T18:12:44.546031+0000 mgr.y (mgr.24344) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:45.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:45 smithi098 bash[33599]: cluster 2023-09-23T18:12:45.553207+0000 mon.a (mon.0) 558 : cluster [DBG] mgrmap e18: y(active, since 3s), standbys: x 2023-09-23T18:12:47.486 INFO:teuthology.orchestra.run.smithi173.stdout:Scheduled alertmanager update... 2023-09-23T18:12:47.776 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:47 smithi173 bash[27694]: cluster 2023-09-23T18:12:46.546662+0000 mgr.y (mgr.24344) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:47.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:47 smithi173 bash[27694]: audit 2023-09-23T18:12:47.480508+0000 mon.a (mon.0) 559 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:47.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:47 smithi098 bash[33599]: cluster 2023-09-23T18:12:46.546662+0000 mgr.y (mgr.24344) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:47.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:47 smithi098 bash[33599]: audit 2023-09-23T18:12:47.480508+0000 mon.a (mon.0) 559 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:47.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:47 smithi098 bash[27349]: cluster 2023-09-23T18:12:46.546662+0000 mgr.y (mgr.24344) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:47.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:47 smithi098 bash[27349]: audit 2023-09-23T18:12:47.480508+0000 mon.a (mon.0) 559 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:48.236 DEBUG:teuthology.orchestra.run.smithi098:alertmanager.a> sudo journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@alertmanager.a.service 2023-09-23T18:12:48.238 INFO:tasks.cephadm:Adding grafana.a on smithi173 2023-09-23T18:12:48.239 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph orch apply grafana '1;smithi173=a' 2023-09-23T18:12:48.436 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:12:48.776 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:48 smithi173 bash[27694]: audit 2023-09-23T18:12:47.470069+0000 mgr.y (mgr.24344) 6 : audit [DBG] from='client.24385 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi098=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:12:48.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:48 smithi173 bash[27694]: cephadm 2023-09-23T18:12:47.473386+0000 mgr.y (mgr.24344) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi098=a;count:1 2023-09-23T18:12:48.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:48 smithi173 bash[27694]: audit 2023-09-23T18:12:48.399169+0000 mon.a (mon.0) 560 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:48.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:48 smithi098 bash[27349]: audit 2023-09-23T18:12:47.470069+0000 mgr.y (mgr.24344) 6 : audit [DBG] from='client.24385 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi098=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:12:48.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:48 smithi098 bash[27349]: cephadm 2023-09-23T18:12:47.473386+0000 mgr.y (mgr.24344) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi098=a;count:1 2023-09-23T18:12:48.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:48 smithi098 bash[27349]: audit 2023-09-23T18:12:48.399169+0000 mon.a (mon.0) 560 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:48.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:48 smithi098 bash[33599]: audit 2023-09-23T18:12:47.470069+0000 mgr.y (mgr.24344) 6 : audit [DBG] from='client.24385 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi098=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:12:48.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:48 smithi098 bash[33599]: cephadm 2023-09-23T18:12:47.473386+0000 mgr.y (mgr.24344) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi098=a;count:1 2023-09-23T18:12:48.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:48 smithi098 bash[33599]: audit 2023-09-23T18:12:48.399169+0000 mon.a (mon.0) 560 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:49.604 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:49 smithi173 bash[27694]: cluster 2023-09-23T18:12:48.547133+0000 mgr.y (mgr.24344) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:49.604 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:49 smithi173 bash[27694]: audit 2023-09-23T18:12:49.103203+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:49.604 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:49 smithi173 bash[27694]: audit 2023-09-23T18:12:49.403605+0000 mon.c (mon.2) 34 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:12:49.604 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:49 smithi173 bash[27694]: audit 2023-09-23T18:12:49.405251+0000 mon.c (mon.2) 35 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:12:49.604 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:49 smithi173 bash[27694]: audit 2023-09-23T18:12:49.469196+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:49.604 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:49 smithi173 bash[27694]: audit 2023-09-23T18:12:49.475892+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:49.604 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:49 smithi173 bash[27694]: audit 2023-09-23T18:12:49.482663+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:49.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:49 smithi098 bash[27349]: cluster 2023-09-23T18:12:48.547133+0000 mgr.y (mgr.24344) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:49.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:49 smithi098 bash[27349]: audit 2023-09-23T18:12:49.103203+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:49.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:49 smithi098 bash[27349]: audit 2023-09-23T18:12:49.403605+0000 mon.c (mon.2) 34 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:12:49.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:49 smithi098 bash[27349]: audit 2023-09-23T18:12:49.405251+0000 mon.c (mon.2) 35 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:12:49.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:49 smithi098 bash[27349]: audit 2023-09-23T18:12:49.469196+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:49.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:49 smithi098 bash[27349]: audit 2023-09-23T18:12:49.475892+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:49.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:49 smithi098 bash[27349]: audit 2023-09-23T18:12:49.482663+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:49.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:49 smithi098 bash[33599]: cluster 2023-09-23T18:12:48.547133+0000 mgr.y (mgr.24344) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:49.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:49 smithi098 bash[33599]: audit 2023-09-23T18:12:49.103203+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:49.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:49 smithi098 bash[33599]: audit 2023-09-23T18:12:49.403605+0000 mon.c (mon.2) 34 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:12:49.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:49 smithi098 bash[33599]: audit 2023-09-23T18:12:49.405251+0000 mon.c (mon.2) 35 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:12:49.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:49 smithi098 bash[33599]: audit 2023-09-23T18:12:49.469196+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:49.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:49 smithi098 bash[33599]: audit 2023-09-23T18:12:49.475892+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:49.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:49 smithi098 bash[33599]: audit 2023-09-23T18:12:49.482663+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:50.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:50 smithi098 bash[27349]: cephadm 2023-09-23T18:12:49.407510+0000 mgr.y (mgr.24344) 9 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.conf 2023-09-23T18:12:50.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:50 smithi098 bash[27349]: cephadm 2023-09-23T18:12:49.407884+0000 mgr.y (mgr.24344) 10 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-09-23T18:12:50.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:50 smithi098 bash[27349]: cephadm 2023-09-23T18:12:49.433430+0000 mgr.y (mgr.24344) 11 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:12:50.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:50 smithi098 bash[27349]: cephadm 2023-09-23T18:12:49.435729+0000 mgr.y (mgr.24344) 12 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:12:50.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:50 smithi098 bash[27349]: cephadm 2023-09-23T18:12:49.508257+0000 mgr.y (mgr.24344) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi173 2023-09-23T18:12:50.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:50 smithi098 bash[33599]: cephadm 2023-09-23T18:12:49.407510+0000 mgr.y (mgr.24344) 9 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.conf 2023-09-23T18:12:50.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:50 smithi098 bash[33599]: cephadm 2023-09-23T18:12:49.407884+0000 mgr.y (mgr.24344) 10 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-09-23T18:12:50.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:50 smithi098 bash[33599]: cephadm 2023-09-23T18:12:49.433430+0000 mgr.y (mgr.24344) 11 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:12:50.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:50 smithi098 bash[33599]: cephadm 2023-09-23T18:12:49.435729+0000 mgr.y (mgr.24344) 12 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:12:50.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:50 smithi098 bash[33599]: cephadm 2023-09-23T18:12:49.508257+0000 mgr.y (mgr.24344) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi173 2023-09-23T18:12:51.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:50 smithi173 bash[27694]: cephadm 2023-09-23T18:12:49.407510+0000 mgr.y (mgr.24344) 9 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.conf 2023-09-23T18:12:51.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:50 smithi173 bash[27694]: cephadm 2023-09-23T18:12:49.407884+0000 mgr.y (mgr.24344) 10 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-09-23T18:12:51.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:50 smithi173 bash[27694]: cephadm 2023-09-23T18:12:49.433430+0000 mgr.y (mgr.24344) 11 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:12:51.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:50 smithi173 bash[27694]: cephadm 2023-09-23T18:12:49.435729+0000 mgr.y (mgr.24344) 12 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:12:51.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:50 smithi173 bash[27694]: cephadm 2023-09-23T18:12:49.508257+0000 mgr.y (mgr.24344) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi173 2023-09-23T18:12:51.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:51 smithi098 bash[33599]: cluster 2023-09-23T18:12:50.547761+0000 mgr.y (mgr.24344) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:51.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:51 smithi098 bash[27349]: cluster 2023-09-23T18:12:50.547761+0000 mgr.y (mgr.24344) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:52.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:51 smithi173 bash[27694]: cluster 2023-09-23T18:12:50.547761+0000 mgr.y (mgr.24344) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:52.047 INFO:teuthology.orchestra.run.smithi173.stdout:Scheduled grafana update... 2023-09-23T18:12:53.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:53 smithi098 bash[27349]: audit 2023-09-23T18:12:52.030728+0000 mgr.y (mgr.24344) 15 : audit [DBG] from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi173=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:12:53.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:53 smithi098 bash[27349]: cephadm 2023-09-23T18:12:52.034084+0000 mgr.y (mgr.24344) 16 : cephadm [INF] Saving service grafana spec with placement smithi173=a;count:1 2023-09-23T18:12:53.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:53 smithi098 bash[27349]: audit 2023-09-23T18:12:52.041604+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:53.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:53 smithi098 bash[27349]: cluster 2023-09-23T18:12:52.548349+0000 mgr.y (mgr.24344) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:53.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:53 smithi098 bash[33599]: audit 2023-09-23T18:12:52.030728+0000 mgr.y (mgr.24344) 15 : audit [DBG] from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi173=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:12:53.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:53 smithi098 bash[33599]: cephadm 2023-09-23T18:12:52.034084+0000 mgr.y (mgr.24344) 16 : cephadm [INF] Saving service grafana spec with placement smithi173=a;count:1 2023-09-23T18:12:53.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:53 smithi098 bash[33599]: audit 2023-09-23T18:12:52.041604+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:53.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:53 smithi098 bash[33599]: cluster 2023-09-23T18:12:52.548349+0000 mgr.y (mgr.24344) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:53.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:53 smithi173 bash[27694]: audit 2023-09-23T18:12:52.030728+0000 mgr.y (mgr.24344) 15 : audit [DBG] from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi173=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:12:53.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:53 smithi173 bash[27694]: cephadm 2023-09-23T18:12:52.034084+0000 mgr.y (mgr.24344) 16 : cephadm [INF] Saving service grafana spec with placement smithi173=a;count:1 2023-09-23T18:12:53.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:53 smithi173 bash[27694]: audit 2023-09-23T18:12:52.041604+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:12:53.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:53 smithi173 bash[27694]: cluster 2023-09-23T18:12:52.548349+0000 mgr.y (mgr.24344) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:54.865 DEBUG:teuthology.orchestra.run.smithi173:grafana.a> sudo journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@grafana.a.service 2023-09-23T18:12:54.867 INFO:tasks.cephadm:Setting up client nodes... 2023-09-23T18:12:54.868 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-09-23T18:12:55.080 INFO:journalctl@ceph.grafana.a.smithi173.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:12:55.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:55 smithi098 bash[33599]: cluster 2023-09-23T18:12:54.549062+0000 mgr.y (mgr.24344) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:55.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:55 smithi098 bash[27349]: cluster 2023-09-23T18:12:54.549062+0000 mgr.y (mgr.24344) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:56.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:55 smithi173 bash[27694]: cluster 2023-09-23T18:12:54.549062+0000 mgr.y (mgr.24344) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:57.776 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:57 smithi173 bash[27694]: cluster 2023-09-23T18:12:56.549625+0000 mgr.y (mgr.24344) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:57.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:57 smithi098 bash[27349]: cluster 2023-09-23T18:12:56.549625+0000 mgr.y (mgr.24344) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:57.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:57 smithi098 bash[33599]: cluster 2023-09-23T18:12:56.549625+0000 mgr.y (mgr.24344) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:58.151 INFO:teuthology.orchestra.run.smithi098.stdout:[client.0] 2023-09-23T18:12:58.151 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQCqKg9lLjs6CBAAEdaFq5p5n7OgSu8QKB66Lw== 2023-09-23T18:12:59.110 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-23T18:12:59.110 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-09-23T18:12:59.110 DEBUG:teuthology.orchestra.run.smithi098:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-09-23T18:12:59.134 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-09-23T18:12:59.211 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:59 smithi173 bash[27694]: audit 2023-09-23T18:12:58.137425+0000 mon.c (mon.2) 36 : audit [INF] from='client.? 172.21.15.98:0/3724277600' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T18:12:59.212 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:59 smithi173 bash[27694]: audit 2023-09-23T18:12:58.137880+0000 mon.a (mon.0) 566 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T18:12:59.212 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:59 smithi173 bash[27694]: audit 2023-09-23T18:12:58.145423+0000 mon.a (mon.0) 567 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T18:12:59.212 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:12:59 smithi173 bash[27694]: cluster 2023-09-23T18:12:58.550176+0000 mgr.y (mgr.24344) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:59.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:59 smithi098 bash[27349]: audit 2023-09-23T18:12:58.137425+0000 mon.c (mon.2) 36 : audit [INF] from='client.? 172.21.15.98:0/3724277600' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T18:12:59.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:59 smithi098 bash[27349]: audit 2023-09-23T18:12:58.137880+0000 mon.a (mon.0) 566 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T18:12:59.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:59 smithi098 bash[27349]: audit 2023-09-23T18:12:58.145423+0000 mon.a (mon.0) 567 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T18:12:59.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:12:59 smithi098 bash[27349]: cluster 2023-09-23T18:12:58.550176+0000 mgr.y (mgr.24344) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:12:59.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:59 smithi098 bash[33599]: audit 2023-09-23T18:12:58.137425+0000 mon.c (mon.2) 36 : audit [INF] from='client.? 172.21.15.98:0/3724277600' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T18:12:59.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:59 smithi098 bash[33599]: audit 2023-09-23T18:12:58.137880+0000 mon.a (mon.0) 566 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T18:12:59.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:59 smithi098 bash[33599]: audit 2023-09-23T18:12:58.145423+0000 mon.a (mon.0) 567 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T18:12:59.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:12:59 smithi098 bash[33599]: cluster 2023-09-23T18:12:58.550176+0000 mgr.y (mgr.24344) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:01.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:01 smithi173 bash[27694]: cluster 2023-09-23T18:13:00.550716+0000 mgr.y (mgr.24344) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:01.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:01 smithi098 bash[27349]: cluster 2023-09-23T18:13:00.550716+0000 mgr.y (mgr.24344) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:01.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:01 smithi098 bash[33599]: cluster 2023-09-23T18:13:00.550716+0000 mgr.y (mgr.24344) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:03.610 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:13:03 smithi173 systemd[1]: Started Ceph prometheus.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:13:03.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:03 smithi098 bash[33599]: cluster 2023-09-23T18:13:02.551258+0000 mgr.y (mgr.24344) 22 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:03.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:03 smithi098 bash[33599]: audit 2023-09-23T18:13:03.589945+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:03.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:03 smithi098 bash[27349]: cluster 2023-09-23T18:13:02.551258+0000 mgr.y (mgr.24344) 22 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:03.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:03 smithi098 bash[27349]: audit 2023-09-23T18:13:03.589945+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:03.979 INFO:teuthology.orchestra.run.smithi173.stdout:[client.1] 2023-09-23T18:13:03.979 INFO:teuthology.orchestra.run.smithi173.stdout: key = AQCvKg9l7nutORAAQNRApbwXhy8sRsUR3JTmWA== 2023-09-23T18:13:04.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:03 smithi173 bash[27694]: cluster 2023-09-23T18:13:02.551258+0000 mgr.y (mgr.24344) 22 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:04.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:03 smithi173 bash[27694]: audit 2023-09-23T18:13:03.589945+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:04.374 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:04 smithi098 systemd[1]: Started Ceph node-exporter.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:13:04.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:04 smithi098 bash[27349]: cephadm 2023-09-23T18:13:03.593968+0000 mgr.y (mgr.24344) 23 : cephadm [INF] Deploying daemon node-exporter.a on smithi098 2023-09-23T18:13:04.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:04 smithi098 bash[27349]: audit 2023-09-23T18:13:03.967370+0000 mon.a (mon.0) 569 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T18:13:04.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:04 smithi098 bash[27349]: audit 2023-09-23T18:13:03.971605+0000 mon.b (mon.1) 26 : audit [INF] from='client.? 172.21.15.173:0/4179818705' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T18:13:04.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:04 smithi098 bash[27349]: audit 2023-09-23T18:13:03.973728+0000 mon.a (mon.0) 570 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T18:13:04.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:04 smithi098 bash[27349]: audit 2023-09-23T18:13:04.348990+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:04.696 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:04 smithi098 bash[52322]: Unable to find image 'quay.io/prometheus/node-exporter:v1.3.1' locally 2023-09-23T18:13:04.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:04 smithi098 bash[33599]: cephadm 2023-09-23T18:13:03.593968+0000 mgr.y (mgr.24344) 23 : cephadm [INF] Deploying daemon node-exporter.a on smithi098 2023-09-23T18:13:04.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:04 smithi098 bash[33599]: audit 2023-09-23T18:13:03.967370+0000 mon.a (mon.0) 569 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T18:13:04.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:04 smithi098 bash[33599]: audit 2023-09-23T18:13:03.971605+0000 mon.b (mon.1) 26 : audit [INF] from='client.? 172.21.15.173:0/4179818705' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T18:13:04.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:04 smithi098 bash[33599]: audit 2023-09-23T18:13:03.973728+0000 mon.a (mon.0) 570 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T18:13:04.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:04 smithi098 bash[33599]: audit 2023-09-23T18:13:04.348990+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:04.836 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:04 smithi173 bash[27694]: cephadm 2023-09-23T18:13:03.593968+0000 mgr.y (mgr.24344) 23 : cephadm [INF] Deploying daemon node-exporter.a on smithi098 2023-09-23T18:13:04.836 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:04 smithi173 bash[27694]: audit 2023-09-23T18:13:03.967370+0000 mon.a (mon.0) 569 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T18:13:04.836 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:04 smithi173 bash[27694]: audit 2023-09-23T18:13:03.971605+0000 mon.b (mon.1) 26 : audit [INF] from='client.? 172.21.15.173:0/4179818705' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T18:13:04.836 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:04 smithi173 bash[27694]: audit 2023-09-23T18:13:03.973728+0000 mon.a (mon.0) 570 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T18:13:04.837 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:04 smithi173 bash[27694]: audit 2023-09-23T18:13:04.348990+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:05.044 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-09-23T18:13:05.044 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-09-23T18:13:05.044 DEBUG:teuthology.orchestra.run.smithi173:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-09-23T18:13:05.063 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-09-23T18:13:05.064 DEBUG:teuthology.orchestra.run.smithi098:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-09-23T18:13:05.111 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:05 smithi173 systemd[1]: Started Ceph node-exporter.b for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:13:05.179 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:04 smithi098 bash[52322]: v1.3.1: Pulling from prometheus/node-exporter 2023-09-23T18:13:05.180 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:05 smithi098 bash[52322]: aa2a8d90b84c: Pulling fs layer 2023-09-23T18:13:05.409 INFO:teuthology.misc.health.smithi098.stdout: 2023-09-23T18:13:05.409 INFO:teuthology.misc.health.smithi098.stdout:{"epoch":51,"fsid":"756c96d4-5a3b-11ee-8db4-212e2dc638e7","created":"2023-09-23T18:04:11.580781+0000","modified":"2023-09-23T18:12:42.525892+0000","last_up_change":"2023-09-23T18:12:09.242233+0000","last_in_change":"2023-09-23T18:11:43.419979+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-23T18:08:35.146924+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"4f35f176-4d07-4157-aeb9-dd8f71da3a63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6802","nonce":1147685383},{"type":"v1","addr":"172.21.15.98:6803","nonce":1147685383}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6804","nonce":1147685383},{"type":"v1","addr":"172.21.15.98:6805","nonce":1147685383}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6808","nonce":1147685383},{"type":"v1","addr":"172.21.15.98:6809","nonce":1147685383}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6806","nonce":1147685383},{"type":"v1","addr":"172.21.15.98:6807","nonce":1147685383}]},"public_addr":"172.21.15.98:6803/1147685383","cluster_addr":"172.21.15.98:6805/1147685383","heartbeat_back_addr":"172.21.15.98:6809/1147685383","heartbeat_front_addr":"172.21.15.98:6807/1147685383","state":["exists","up"]},{"osd":1,"uuid":"5d777d1f-3bd3-49ba-b0de-0ac532ee3934","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6810","nonce":3945616078},{"type":"v1","addr":"172.21.15.98:6811","nonce":3945616078}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6812","nonce":3945616078},{"type":"v1","addr":"172.21.15.98:6813","nonce":3945616078}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6816","nonce":3945616078},{"type":"v1","addr":"172.21.15.98:6817","nonce":3945616078}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6814","nonce":3945616078},{"type":"v1","addr":"172.21.15.98:6815","nonce":3945616078}]},"public_addr":"172.21.15.98:6811/3945616078","cluster_addr":"172.21.15.98:6813/3945616078","heartbeat_back_addr":"172.21.15.98:6817/3945616078","heartbeat_front_addr":"172.21.15.98:6815/3945616078","state":["exists","up"]},{"osd":2,"uuid":"63978288-ac36-4cf2-bbb7-46dc828e6ce5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6818","nonce":2338555672},{"type":"v1","addr":"172.21.15.98:6819","nonce":2338555672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6820","nonce":2338555672},{"type":"v1","addr":"172.21.15.98:6821","nonce":2338555672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6824","nonce":2338555672},{"type":"v1","addr":"172.21.15.98:6825","nonce":2338555672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6822","nonce":2338555672},{"type":"v1","addr":"172.21.15.98:6823","nonce":2338555672}]},"public_addr":"172.21.15.98:6819/2338555672","cluster_addr":"172.21.15.98:6821/2338555672","heartbeat_back_addr":"172.21.15.98:6825/2338555672","heartbeat_front_addr":"172.21.15.98:6823/2338555672","state":["exists","up"]},{"osd":3,"uuid":"48596d91-46da-4815-b810-d493bbf01dcc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6826","nonce":2634725099},{"type":"v1","addr":"172.21.15.98:6827","nonce":2634725099}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6828","nonce":2634725099},{"type":"v1","addr":"172.21.15.98:6829","nonce":2634725099}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6832","nonce":2634725099},{"type":"v1","addr":"172.21.15.98:6833","nonce":2634725099}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6830","nonce":2634725099},{"type":"v1","addr":"172.21.15.98:6831","nonce":2634725099}]},"public_addr":"172.21.15.98:6827/2634725099","cluster_addr":"172.21.15.98:6829/2634725099","heartbeat_back_addr":"172.21.15.98:6833/2634725099","heartbeat_front_addr":"172.21.15.98:6831/2634725099","state":["exists","up"]},{"osd":4,"uuid":"a562afc6-ffe6-44bb-91a9-8a04018c3c6f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6800","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6801","nonce":165906554}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6802","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6803","nonce":165906554}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6806","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6807","nonce":165906554}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6804","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6805","nonce":165906554}]},"public_addr":"172.21.15.173:6801/165906554","cluster_addr":"172.21.15.173:6803/165906554","heartbeat_back_addr":"172.21.15.173:6807/165906554","heartbeat_front_addr":"172.21.15.173:6805/165906554","state":["exists","up"]},{"osd":5,"uuid":"add28146-8301-4b4c-9616-967e9df3a8d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6808","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6809","nonce":2894909344}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6810","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6811","nonce":2894909344}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6814","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6815","nonce":2894909344}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6812","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6813","nonce":2894909344}]},"public_addr":"172.21.15.173:6809/2894909344","cluster_addr":"172.21.15.173:6811/2894909344","heartbeat_back_addr":"172.21.15.173:6815/2894909344","heartbeat_front_addr":"172.21.15.173:6813/2894909344","state":["exists","up"]},{"osd":6,"uuid":"f30854de-bc26-465d-8b0e-a9211ae2d9fa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6816","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6817","nonce":3550013548}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6818","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6819","nonce":3550013548}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6822","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6823","nonce":3550013548}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6820","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6821","nonce":3550013548}]},"public_addr":"172.21.15.173:6817/3550013548","cluster_addr":"172.21.15.173:6819/3550013548","heartbeat_back_addr":"172.21.15.173:6823/3550013548","heartbeat_front_addr":"172.21.15.173:6821/3550013548","state":["exists","up"]},{"osd":7,"uuid":"5f540de3-48f9-4bc8-b14a-9aadf838f16b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6824","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6825","nonce":135608222}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6826","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6827","nonce":135608222}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6830","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6831","nonce":135608222}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6828","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6829","nonce":135608222}]},"public_addr":"172.21.15.173:6825/135608222","cluster_addr":"172.21.15.173:6827/135608222","heartbeat_back_addr":"172.21.15.173:6831/135608222","heartbeat_front_addr":"172.21.15.173:6829/135608222","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:07:04.739243+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:07:48.583198+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:08:33.173348+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:09:18.218502+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:09:59.765019+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:10:42.490689+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:11:24.357920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:12:08.168900+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.98:6801/483469857":"2023-09-24T18:12:42.525837+0000","172.21.15.98:6800/483469857":"2023-09-24T18:12:42.525837+0000","172.21.15.98:0/1142988558":"2023-09-24T18:12:42.525837+0000","172.21.15.98:0/3472030975":"2023-09-24T18:05:11.060631+0000","172.21.15.98:6800/1392949322":"2023-09-24T18:05:11.060631+0000","172.21.15.98:0/3976553993":"2023-09-24T18:04:40.596823+0000","172.21.15.98:0/2194193535":"2023-09-24T18:12:42.525837+0000","172.21.15.98:0/3213365423":"2023-09-24T18:04:40.596823+0000","172.21.15.98:6801/1392949322":"2023-09-24T18:05:11.060631+0000","172.21.15.98:6800/3368447653":"2023-09-24T18:04:40.596823+0000","172.21.15.98:0/1603044865":"2023-09-24T18:05:11.060631+0000","172.21.15.98:6801/3368447653":"2023-09-24T18:04:40.596823+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T18:13:05.422 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-09-23T18:13:05.422 INFO:tasks.cephadm:Creating RBD pool 2023-09-23T18:13:05.422 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-09-23T18:13:05.444 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:05 smithi098 bash[52322]: b45d31ee2d7f: Pulling fs layer 2023-09-23T18:13:05.445 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:05 smithi098 bash[52322]: b5db1e299295: Pulling fs layer 2023-09-23T18:13:05.445 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:05 smithi098 bash[52322]: b45d31ee2d7f: Verifying Checksum 2023-09-23T18:13:05.445 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:05 smithi098 bash[52322]: b45d31ee2d7f: Download complete 2023-09-23T18:13:05.445 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:05 smithi098 bash[52322]: aa2a8d90b84c: Download complete 2023-09-23T18:13:05.445 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:05 smithi098 bash[52322]: b5db1e299295: Verifying Checksum 2023-09-23T18:13:05.445 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:05 smithi098 bash[52322]: b5db1e299295: Download complete 2023-09-23T18:13:05.526 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[47036]: Unable to find image 'quay.io/prometheus/node-exporter:v1.3.1' locally 2023-09-23T18:13:05.527 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[46883]: ts=2023-09-23T18:13:05.239Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-09-23T18:13:05.527 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[46883]: ts=2023-09-23T18:13:05.239Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-09-23T18:13:05.527 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[46883]: ts=2023-09-23T18:13:05.239Z caller=main.go:518 level=info host_details="(Linux 5.15.0-84-generic #93~20.04.1-Ubuntu SMP Wed Sep 6 16:15:40 UTC 2023 x86_64 smithi173 (none))" 2023-09-23T18:13:05.527 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[46883]: ts=2023-09-23T18:13:05.239Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T18:13:05.527 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[46883]: ts=2023-09-23T18:13:05.239Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T18:13:05.527 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[46883]: ts=2023-09-23T18:13:05.240Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T18:13:05.527 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[46883]: ts=2023-09-23T18:13:05.241Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-09-23T18:13:05.527 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[46883]: ts=2023-09-23T18:13:05.242Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-09-23T18:13:05.528 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[46883]: ts=2023-09-23T18:13:05.247Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T18:13:05.528 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[46883]: ts=2023-09-23T18:13:05.247Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.636µs 2023-09-23T18:13:05.528 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[46883]: ts=2023-09-23T18:13:05.247Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T18:13:05.528 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[46883]: ts=2023-09-23T18:13:05.247Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2023-09-23T18:13:05.528 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[46883]: ts=2023-09-23T18:13:05.247Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=38.081µs wal_replay_duration=284.618µs total_replay_duration=356.538µs 2023-09-23T18:13:05.528 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[46883]: ts=2023-09-23T18:13:05.249Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T18:13:05.528 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[46883]: ts=2023-09-23T18:13:05.249Z caller=main.go:947 level=info msg="TSDB started" 2023-09-23T18:13:05.528 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[46883]: ts=2023-09-23T18:13:05.249Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T18:13:05.528 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[46883]: ts=2023-09-23T18:13:05.269Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.700122ms db_storage=777ns remote_storage=1.875µs web_handler=386ns query_engine=743ns scrape=400.985µs scrape_sd=36.066µs notify=928ns notify_sd=1.739µs rules=19.957751ms 2023-09-23T18:13:05.528 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[46883]: ts=2023-09-23T18:13:05.269Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-09-23T18:13:05.908 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[27694]: cephadm 2023-09-23T18:13:04.351253+0000 mgr.y (mgr.24344) 24 : cephadm [INF] Deploying daemon node-exporter.b on smithi173 2023-09-23T18:13:05.908 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[27694]: cluster 2023-09-23T18:13:04.551845+0000 mgr.y (mgr.24344) 25 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:05.909 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[27694]: audit 2023-09-23T18:13:05.093462+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:05.909 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[27694]: audit 2023-09-23T18:13:05.403508+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.98:0/3543543108' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:13:05.909 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[47036]: v1.3.1: Pulling from prometheus/node-exporter 2023-09-23T18:13:05.944 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:05 smithi098 bash[52322]: aa2a8d90b84c: Pull complete 2023-09-23T18:13:05.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:05 smithi098 bash[27349]: cephadm 2023-09-23T18:13:04.351253+0000 mgr.y (mgr.24344) 24 : cephadm [INF] Deploying daemon node-exporter.b on smithi173 2023-09-23T18:13:05.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:05 smithi098 bash[27349]: cluster 2023-09-23T18:13:04.551845+0000 mgr.y (mgr.24344) 25 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:05.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:05 smithi098 bash[27349]: audit 2023-09-23T18:13:05.093462+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:05.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:05 smithi098 bash[27349]: audit 2023-09-23T18:13:05.403508+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.98:0/3543543108' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:13:05.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:05 smithi098 bash[33599]: cephadm 2023-09-23T18:13:04.351253+0000 mgr.y (mgr.24344) 24 : cephadm [INF] Deploying daemon node-exporter.b on smithi173 2023-09-23T18:13:05.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:05 smithi098 bash[33599]: cluster 2023-09-23T18:13:04.551845+0000 mgr.y (mgr.24344) 25 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:05.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:05 smithi098 bash[33599]: audit 2023-09-23T18:13:05.093462+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:05.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:05 smithi098 bash[33599]: audit 2023-09-23T18:13:05.403508+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.98:0/3543543108' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:13:06.178 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[47036]: aa2a8d90b84c: Pulling fs layer 2023-09-23T18:13:06.179 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[47036]: b45d31ee2d7f: Pulling fs layer 2023-09-23T18:13:06.179 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:05 smithi173 bash[47036]: b5db1e299295: Pulling fs layer 2023-09-23T18:13:06.179 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:06 smithi173 bash[47036]: aa2a8d90b84c: Verifying Checksum 2023-09-23T18:13:06.179 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:06 smithi173 bash[47036]: aa2a8d90b84c: Download complete 2023-09-23T18:13:06.179 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:06 smithi173 bash[47036]: b45d31ee2d7f: Verifying Checksum 2023-09-23T18:13:06.179 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:06 smithi173 bash[47036]: b45d31ee2d7f: Download complete 2023-09-23T18:13:06.526 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:06 smithi173 bash[47036]: b5db1e299295: Verifying Checksum 2023-09-23T18:13:06.527 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:06 smithi173 bash[47036]: b5db1e299295: Download complete 2023-09-23T18:13:06.527 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:06 smithi173 bash[47036]: aa2a8d90b84c: Pull complete 2023-09-23T18:13:06.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:06 smithi098 bash[27349]: cephadm 2023-09-23T18:13:05.106268+0000 mgr.y (mgr.24344) 26 : cephadm [INF] Deploying daemon alertmanager.a on smithi098 2023-09-23T18:13:06.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:06 smithi098 bash[33599]: cephadm 2023-09-23T18:13:05.106268+0000 mgr.y (mgr.24344) 26 : cephadm [INF] Deploying daemon alertmanager.a on smithi098 2023-09-23T18:13:06.945 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:06 smithi098 bash[52322]: b45d31ee2d7f: Pull complete 2023-09-23T18:13:07.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:06 smithi173 bash[27694]: cephadm 2023-09-23T18:13:05.106268+0000 mgr.y (mgr.24344) 26 : cephadm [INF] Deploying daemon alertmanager.a on smithi098 2023-09-23T18:13:07.027 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:06 smithi173 bash[47036]: b45d31ee2d7f: Pull complete 2023-09-23T18:13:07.527 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:07 smithi173 bash[47036]: b5db1e299295: Pull complete 2023-09-23T18:13:07.527 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:07 smithi173 bash[47036]: Digest: sha256:f2269e73124dd0f60a7d19a2ce1264d33d08a985aed0ee6b0b89d0be470592cd 2023-09-23T18:13:07.527 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:07 smithi173 bash[47036]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.3.1 2023-09-23T18:13:07.611 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:07 smithi098 bash[52322]: b5db1e299295: Pull complete 2023-09-23T18:13:07.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:07 smithi098 bash[27349]: cluster 2023-09-23T18:13:06.552212+0000 mgr.y (mgr.24344) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:07.945 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:07 smithi098 bash[52322]: Digest: sha256:f2269e73124dd0f60a7d19a2ce1264d33d08a985aed0ee6b0b89d0be470592cd 2023-09-23T18:13:07.945 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:07 smithi098 bash[52322]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.3.1 2023-09-23T18:13:07.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:07 smithi098 bash[33599]: cluster 2023-09-23T18:13:06.552212+0000 mgr.y (mgr.24344) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:08.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:07 smithi173 bash[27694]: cluster 2023-09-23T18:13:06.552212+0000 mgr.y (mgr.24344) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:09.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[27694]: audit 2023-09-23T18:13:07.775411+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:09.027 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.727Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-09-23T18:13:09.027 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.727Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-09-23T18:13:09.027 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.728Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-09-23T18:13:09.027 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.728Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T18:13:09.028 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.729Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-09-23T18:13:09.028 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.729Z caller=node_exporter.go:115 level=info collector=arp 2023-09-23T18:13:09.028 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.729Z caller=node_exporter.go:115 level=info collector=bcache 2023-09-23T18:13:09.028 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.729Z caller=node_exporter.go:115 level=info collector=bonding 2023-09-23T18:13:09.028 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.729Z caller=node_exporter.go:115 level=info collector=btrfs 2023-09-23T18:13:09.028 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.729Z caller=node_exporter.go:115 level=info collector=conntrack 2023-09-23T18:13:09.028 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.729Z caller=node_exporter.go:115 level=info collector=cpu 2023-09-23T18:13:09.028 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.729Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-09-23T18:13:09.028 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.729Z caller=node_exporter.go:115 level=info collector=diskstats 2023-09-23T18:13:09.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.729Z caller=node_exporter.go:115 level=info collector=dmi 2023-09-23T18:13:09.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.729Z caller=node_exporter.go:115 level=info collector=edac 2023-09-23T18:13:09.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=entropy 2023-09-23T18:13:09.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-09-23T18:13:09.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=filefd 2023-09-23T18:13:09.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=filesystem 2023-09-23T18:13:09.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=hwmon 2023-09-23T18:13:09.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=infiniband 2023-09-23T18:13:09.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=ipvs 2023-09-23T18:13:09.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=loadavg 2023-09-23T18:13:09.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=mdadm 2023-09-23T18:13:09.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=meminfo 2023-09-23T18:13:09.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=netclass 2023-09-23T18:13:09.030 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=netdev 2023-09-23T18:13:09.030 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=netstat 2023-09-23T18:13:09.030 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=nfs 2023-09-23T18:13:09.030 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=nfsd 2023-09-23T18:13:09.030 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=nvme 2023-09-23T18:13:09.030 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=os 2023-09-23T18:13:09.030 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-09-23T18:13:09.030 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=pressure 2023-09-23T18:13:09.031 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=rapl 2023-09-23T18:13:09.031 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=schedstat 2023-09-23T18:13:09.031 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=sockstat 2023-09-23T18:13:09.031 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=softnet 2023-09-23T18:13:09.031 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=stat 2023-09-23T18:13:09.031 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=tapestats 2023-09-23T18:13:09.031 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=textfile 2023-09-23T18:13:09.031 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-09-23T18:13:09.031 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=time 2023-09-23T18:13:09.031 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-09-23T18:13:09.031 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=uname 2023-09-23T18:13:09.031 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=vmstat 2023-09-23T18:13:09.032 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=xfs 2023-09-23T18:13:09.032 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:115 level=info collector=zfs 2023-09-23T18:13:09.032 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-09-23T18:13:09.032 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:13:08 smithi173 bash[47036]: ts=2023-09-23T18:13:08.730Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-09-23T18:13:09.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:08 smithi098 bash[27349]: audit 2023-09-23T18:13:07.775411+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:09.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:08 smithi098 bash[33599]: audit 2023-09-23T18:13:07.775411+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:09.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:09 smithi098 bash[27349]: cluster 2023-09-23T18:13:08.552779+0000 mgr.y (mgr.24344) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:09.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:09 smithi098 bash[33599]: cluster 2023-09-23T18:13:08.552779+0000 mgr.y (mgr.24344) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:10.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:09 smithi173 bash[27694]: cluster 2023-09-23T18:13:08.552779+0000 mgr.y (mgr.24344) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:11.195 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.059Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-09-23T18:13:11.195 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.059Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-09-23T18:13:11.195 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.060Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-09-23T18:13:11.195 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T18:13:11.195 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-09-23T18:13:11.195 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=arp 2023-09-23T18:13:11.196 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=bcache 2023-09-23T18:13:11.196 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=bonding 2023-09-23T18:13:11.196 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=btrfs 2023-09-23T18:13:11.196 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=conntrack 2023-09-23T18:13:11.196 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=cpu 2023-09-23T18:13:11.196 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-09-23T18:13:11.196 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=diskstats 2023-09-23T18:13:11.196 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=dmi 2023-09-23T18:13:11.196 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=edac 2023-09-23T18:13:11.196 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=entropy 2023-09-23T18:13:11.196 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-09-23T18:13:11.197 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=filefd 2023-09-23T18:13:11.197 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=filesystem 2023-09-23T18:13:11.197 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=hwmon 2023-09-23T18:13:11.197 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=infiniband 2023-09-23T18:13:11.197 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=ipvs 2023-09-23T18:13:11.197 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=loadavg 2023-09-23T18:13:11.197 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=mdadm 2023-09-23T18:13:11.197 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=meminfo 2023-09-23T18:13:11.197 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=netclass 2023-09-23T18:13:11.197 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=netdev 2023-09-23T18:13:11.197 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=netstat 2023-09-23T18:13:11.197 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=nfs 2023-09-23T18:13:11.197 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=nfsd 2023-09-23T18:13:11.198 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=nvme 2023-09-23T18:13:11.198 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=os 2023-09-23T18:13:11.198 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-09-23T18:13:11.198 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=pressure 2023-09-23T18:13:11.198 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=rapl 2023-09-23T18:13:11.198 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=schedstat 2023-09-23T18:13:11.198 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=sockstat 2023-09-23T18:13:11.198 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=softnet 2023-09-23T18:13:11.198 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=stat 2023-09-23T18:13:11.198 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=tapestats 2023-09-23T18:13:11.198 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=textfile 2023-09-23T18:13:11.198 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-09-23T18:13:11.199 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=time 2023-09-23T18:13:11.199 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-09-23T18:13:11.199 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=uname 2023-09-23T18:13:11.199 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=vmstat 2023-09-23T18:13:11.199 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=xfs 2023-09-23T18:13:11.199 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:115 level=info collector=zfs 2023-09-23T18:13:11.199 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.061Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-09-23T18:13:11.199 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[52322]: ts=2023-09-23T18:13:11.062Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-09-23T18:13:11.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[33599]: cluster 2023-09-23T18:13:10.553377+0000 mgr.y (mgr.24344) 29 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:11.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:11 smithi098 bash[27349]: cluster 2023-09-23T18:13:10.553377+0000 mgr.y (mgr.24344) 29 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:12.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:11 smithi173 bash[27694]: cluster 2023-09-23T18:13:10.553377+0000 mgr.y (mgr.24344) 29 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:12.799 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:12 smithi098 bash[27349]: audit 2023-09-23T18:13:11.990971+0000 mon.a (mon.0) 574 : audit [INF] from='client.? 172.21.15.98:0/2194176739' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T18:13:12.808 INFO:teuthology.orchestra.run.smithi098.stderr:pool 'rbd' created 2023-09-23T18:13:13.104 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:12 smithi098 bash[33599]: audit 2023-09-23T18:13:11.990971+0000 mon.a (mon.0) 574 : audit [INF] from='client.? 172.21.15.98:0/2194176739' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T18:13:13.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:12 smithi173 bash[27694]: audit 2023-09-23T18:13:11.990971+0000 mon.a (mon.0) 574 : audit [INF] from='client.? 172.21.15.98:0/2194176739' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T18:13:13.784 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-09-23T18:13:13.869 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:13 smithi098 bash[33599]: cluster 2023-09-23T18:13:12.553913+0000 mgr.y (mgr.24344) 30 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:13.870 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:13 smithi098 bash[33599]: audit 2023-09-23T18:13:12.796815+0000 mon.a (mon.0) 575 : audit [INF] from='client.? 172.21.15.98:0/2194176739' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T18:13:13.870 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:13 smithi098 bash[33599]: cluster 2023-09-23T18:13:12.796916+0000 mon.a (mon.0) 576 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-09-23T18:13:13.870 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:13 smithi098 bash[33599]: audit 2023-09-23T18:13:12.798977+0000 mon.c (mon.2) 38 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T18:13:13.870 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:13 smithi098 bash[33599]: audit 2023-09-23T18:13:12.799692+0000 mon.a (mon.0) 577 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T18:13:13.870 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:13 smithi098 bash[33599]: audit 2023-09-23T18:13:12.801159+0000 mon.c (mon.2) 39 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-09-23T18:13:13.871 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:13 smithi098 bash[33599]: audit 2023-09-23T18:13:12.801604+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-09-23T18:13:13.871 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:13 smithi098 bash[27349]: cluster 2023-09-23T18:13:12.553913+0000 mgr.y (mgr.24344) 30 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:13.871 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:13 smithi098 bash[27349]: audit 2023-09-23T18:13:12.796815+0000 mon.a (mon.0) 575 : audit [INF] from='client.? 172.21.15.98:0/2194176739' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T18:13:13.871 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:13 smithi098 bash[27349]: cluster 2023-09-23T18:13:12.796916+0000 mon.a (mon.0) 576 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-09-23T18:13:13.872 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:13 smithi098 bash[27349]: audit 2023-09-23T18:13:12.798977+0000 mon.c (mon.2) 38 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T18:13:13.872 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:13 smithi098 bash[27349]: audit 2023-09-23T18:13:12.799692+0000 mon.a (mon.0) 577 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T18:13:13.872 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:13 smithi098 bash[27349]: audit 2023-09-23T18:13:12.801159+0000 mon.c (mon.2) 39 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-09-23T18:13:13.872 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:13 smithi098 bash[27349]: audit 2023-09-23T18:13:12.801604+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-09-23T18:13:14.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:13 smithi173 bash[27694]: cluster 2023-09-23T18:13:12.553913+0000 mgr.y (mgr.24344) 30 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:14.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:13 smithi173 bash[27694]: audit 2023-09-23T18:13:12.796815+0000 mon.a (mon.0) 575 : audit [INF] from='client.? 172.21.15.98:0/2194176739' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T18:13:14.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:13 smithi173 bash[27694]: cluster 2023-09-23T18:13:12.796916+0000 mon.a (mon.0) 576 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-09-23T18:13:14.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:13 smithi173 bash[27694]: audit 2023-09-23T18:13:12.798977+0000 mon.c (mon.2) 38 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T18:13:14.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:13 smithi173 bash[27694]: audit 2023-09-23T18:13:12.799692+0000 mon.a (mon.0) 577 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T18:13:14.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:13 smithi173 bash[27694]: audit 2023-09-23T18:13:12.801159+0000 mon.c (mon.2) 39 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-09-23T18:13:14.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:13 smithi173 bash[27694]: audit 2023-09-23T18:13:12.801604+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-09-23T18:13:15.028 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:14 smithi098 bash[27349]: audit 2023-09-23T18:13:13.799443+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-09-23T18:13:15.028 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:14 smithi098 bash[27349]: cluster 2023-09-23T18:13:13.799646+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-09-23T18:13:15.028 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:14 smithi098 bash[33599]: audit 2023-09-23T18:13:13.799443+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-09-23T18:13:15.028 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:14 smithi098 bash[33599]: cluster 2023-09-23T18:13:13.799646+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-09-23T18:13:15.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:14 smithi173 bash[27694]: audit 2023-09-23T18:13:13.799443+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-09-23T18:13:15.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:14 smithi173 bash[27694]: cluster 2023-09-23T18:13:13.799646+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-09-23T18:13:15.776 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:13:15 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:13:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:13:15.937 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:15 smithi098 bash[27349]: cluster 2023-09-23T18:13:14.554427+0000 mgr.y (mgr.24344) 31 : cluster [DBG] pgmap v21: 9 pgs: 8 creating+peering, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:15.937 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:15 smithi098 bash[27349]: cluster 2023-09-23T18:13:14.816807+0000 mon.a (mon.0) 581 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-09-23T18:13:15.937 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:15 smithi098 bash[33599]: cluster 2023-09-23T18:13:14.554427+0000 mgr.y (mgr.24344) 31 : cluster [DBG] pgmap v21: 9 pgs: 8 creating+peering, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:15.937 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:15 smithi098 bash[33599]: cluster 2023-09-23T18:13:14.816807+0000 mon.a (mon.0) 581 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-09-23T18:13:16.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:15 smithi173 bash[27694]: cluster 2023-09-23T18:13:14.554427+0000 mgr.y (mgr.24344) 31 : cluster [DBG] pgmap v21: 9 pgs: 8 creating+peering, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:16.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:15 smithi173 bash[27694]: cluster 2023-09-23T18:13:14.816807+0000 mon.a (mon.0) 581 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-09-23T18:13:17.975 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:13:17 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:13:17] "GET /metrics HTTP/1.1" 200 154668 "" "Prometheus/2.33.4" 2023-09-23T18:13:17.975 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:17 smithi098 bash[33599]: cluster 2023-09-23T18:13:16.554926+0000 mgr.y (mgr.24344) 32 : cluster [DBG] pgmap v23: 9 pgs: 8 creating+peering, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:17.976 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:17 smithi098 bash[27349]: cluster 2023-09-23T18:13:16.554926+0000 mgr.y (mgr.24344) 32 : cluster [DBG] pgmap v23: 9 pgs: 8 creating+peering, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:18.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:17 smithi173 bash[27694]: cluster 2023-09-23T18:13:16.554926+0000 mgr.y (mgr.24344) 32 : cluster [DBG] pgmap v23: 9 pgs: 8 creating+peering, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:19.280 INFO:teuthology.orchestra.run.smithi098.stderr:enabled application 'rbd' on pool 'rbd' 2023-09-23T18:13:19.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:19 smithi173 bash[27694]: audit 2023-09-23T18:13:18.260972+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:19.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:19 smithi173 bash[27694]: audit 2023-09-23T18:13:18.264317+0000 mon.c (mon.2) 40 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:13:19.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:19 smithi173 bash[27694]: cluster 2023-09-23T18:13:18.555315+0000 mgr.y (mgr.24344) 33 : cluster [DBG] pgmap v24: 9 pgs: 8 active+clean, 1 creating+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:19.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:19 smithi173 bash[27694]: audit 2023-09-23T18:13:18.586691+0000 mon.a (mon.0) 583 : audit [INF] from='client.? 172.21.15.98:0/1337075733' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:13:19.542 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:19 smithi098 bash[27349]: audit 2023-09-23T18:13:18.260972+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:19.542 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:19 smithi098 bash[27349]: audit 2023-09-23T18:13:18.264317+0000 mon.c (mon.2) 40 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:13:19.542 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:19 smithi098 bash[27349]: cluster 2023-09-23T18:13:18.555315+0000 mgr.y (mgr.24344) 33 : cluster [DBG] pgmap v24: 9 pgs: 8 active+clean, 1 creating+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:19.542 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:19 smithi098 bash[27349]: audit 2023-09-23T18:13:18.586691+0000 mon.a (mon.0) 583 : audit [INF] from='client.? 172.21.15.98:0/1337075733' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:13:19.542 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:19 smithi098 bash[33599]: audit 2023-09-23T18:13:18.260972+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:19.543 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:19 smithi098 bash[33599]: audit 2023-09-23T18:13:18.264317+0000 mon.c (mon.2) 40 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:13:19.543 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:19 smithi098 bash[33599]: cluster 2023-09-23T18:13:18.555315+0000 mgr.y (mgr.24344) 33 : cluster [DBG] pgmap v24: 9 pgs: 8 active+clean, 1 creating+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:19.543 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:19 smithi098 bash[33599]: audit 2023-09-23T18:13:18.586691+0000 mon.a (mon.0) 583 : audit [INF] from='client.? 172.21.15.98:0/1337075733' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:13:19.945 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:13:19 smithi098 bash[53034]: level=info ts=2023-09-23T18:13:19.631Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-09-23T18:13:20.104 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T18:13:20.105 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T18:13:20.105 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T18:13:20.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:20 smithi098 bash[27349]: audit 2023-09-23T18:13:19.271040+0000 mon.a (mon.0) 584 : audit [INF] from='client.? 172.21.15.98:0/1337075733' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:13:20.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:20 smithi098 bash[27349]: cluster 2023-09-23T18:13:19.271160+0000 mon.a (mon.0) 585 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-09-23T18:13:20.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:20 smithi098 bash[33599]: audit 2023-09-23T18:13:19.271040+0000 mon.a (mon.0) 584 : audit [INF] from='client.? 172.21.15.98:0/1337075733' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:13:20.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:20 smithi098 bash[33599]: cluster 2023-09-23T18:13:19.271160+0000 mon.a (mon.0) 585 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-09-23T18:13:20.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:20 smithi173 bash[27694]: audit 2023-09-23T18:13:19.271040+0000 mon.a (mon.0) 584 : audit [INF] from='client.? 172.21.15.98:0/1337075733' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:13:20.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:20 smithi173 bash[27694]: cluster 2023-09-23T18:13:19.271160+0000 mon.a (mon.0) 585 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-09-23T18:13:21.585 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:21 smithi098 bash[33599]: cluster 2023-09-23T18:13:20.555863+0000 mgr.y (mgr.24344) 34 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:21.585 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:21 smithi098 bash[27349]: cluster 2023-09-23T18:13:20.555863+0000 mgr.y (mgr.24344) 34 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:21.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:21 smithi173 bash[27694]: cluster 2023-09-23T18:13:20.555863+0000 mgr.y (mgr.24344) 34 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:21.945 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:13:21 smithi098 bash[53034]: level=info ts=2023-09-23T18:13:21.579Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000138763s 2023-09-23T18:13:24.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:23 smithi098 bash[33599]: cluster 2023-09-23T18:13:22.556510+0000 mgr.y (mgr.24344) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:24.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:23 smithi098 bash[33599]: audit 2023-09-23T18:13:22.795381+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:24.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:23 smithi098 bash[27349]: cluster 2023-09-23T18:13:22.556510+0000 mgr.y (mgr.24344) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:24.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:23 smithi098 bash[27349]: audit 2023-09-23T18:13:22.795381+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:24.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:23 smithi173 bash[27694]: cluster 2023-09-23T18:13:22.556510+0000 mgr.y (mgr.24344) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:24.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:23 smithi173 bash[27694]: audit 2023-09-23T18:13:22.795381+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:24.643 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:24.920 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:24 smithi098 bash[27349]: audit 2023-09-23T18:13:23.908792+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:25.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:24 smithi098 bash[33599]: audit 2023-09-23T18:13:23.908792+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:25.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:24 smithi173 bash[27694]: audit 2023-09-23T18:13:23.908792+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:25.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:13:25 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:13:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:13:25.948 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:13:25 smithi098 bash[27618]: [23/Sep/2023:18:13:25] ENGINE Bus STOPPING 2023-09-23T18:13:25.948 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:13:25 smithi098 bash[27618]: [23/Sep/2023:18:13:25] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T18:13:25.948 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:13:25 smithi098 bash[27618]: [23/Sep/2023:18:13:25] ENGINE Bus STOPPED 2023-09-23T18:13:25.949 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:13:25 smithi098 bash[27618]: [23/Sep/2023:18:13:25] ENGINE Bus STARTING 2023-09-23T18:13:25.949 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:25 smithi098 bash[27349]: cluster 2023-09-23T18:13:24.557266+0000 mgr.y (mgr.24344) 36 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:25.949 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:25 smithi098 bash[27349]: audit 2023-09-23T18:13:25.490708+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:25.949 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:25 smithi098 bash[27349]: audit 2023-09-23T18:13:25.495078+0000 mon.c (mon.2) 41 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:13:25.949 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:25 smithi098 bash[27349]: audit 2023-09-23T18:13:25.496351+0000 mon.c (mon.2) 42 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:13:25.949 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:25 smithi098 bash[27349]: audit 2023-09-23T18:13:25.506030+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:25.949 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:25 smithi098 bash[27349]: audit 2023-09-23T18:13:25.780069+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:26.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:25 smithi173 bash[27694]: cluster 2023-09-23T18:13:24.557266+0000 mgr.y (mgr.24344) 36 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:26.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:25 smithi173 bash[27694]: audit 2023-09-23T18:13:25.490708+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:26.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:25 smithi173 bash[27694]: audit 2023-09-23T18:13:25.495078+0000 mon.c (mon.2) 41 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:13:26.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:25 smithi173 bash[27694]: audit 2023-09-23T18:13:25.496351+0000 mon.c (mon.2) 42 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:13:26.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:25 smithi173 bash[27694]: audit 2023-09-23T18:13:25.506030+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:26.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:25 smithi173 bash[27694]: audit 2023-09-23T18:13:25.780069+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:26.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:25 smithi173 bash[27694]: audit 2023-09-23T18:13:25.787499+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:26.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:25 smithi173 bash[27694]: audit 2023-09-23T18:13:25.795039+0000 mon.c (mon.2) 43 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T18:13:26.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:25 smithi173 bash[27694]: audit 2023-09-23T18:13:25.803622+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:26.450 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:25 smithi098 bash[27349]: audit 2023-09-23T18:13:25.787499+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:26.451 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:25 smithi098 bash[27349]: audit 2023-09-23T18:13:25.795039+0000 mon.c (mon.2) 43 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T18:13:26.451 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:25 smithi098 bash[27349]: audit 2023-09-23T18:13:25.803622+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:26.451 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:13:25 smithi098 bash[27618]: [23/Sep/2023:18:13:25] ENGINE Serving on http://:::9283 2023-09-23T18:13:26.451 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:13:25 smithi098 bash[27618]: [23/Sep/2023:18:13:25] ENGINE Bus STARTED 2023-09-23T18:13:26.451 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:25 smithi098 bash[33599]: cluster 2023-09-23T18:13:24.557266+0000 mgr.y (mgr.24344) 36 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:26.452 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:25 smithi098 bash[33599]: audit 2023-09-23T18:13:25.490708+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:26.452 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:25 smithi098 bash[33599]: audit 2023-09-23T18:13:25.495078+0000 mon.c (mon.2) 41 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:13:26.452 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:25 smithi098 bash[33599]: audit 2023-09-23T18:13:25.496351+0000 mon.c (mon.2) 42 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:13:26.452 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:25 smithi098 bash[33599]: audit 2023-09-23T18:13:25.506030+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:26.452 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:25 smithi098 bash[33599]: audit 2023-09-23T18:13:25.780069+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:26.452 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:25 smithi098 bash[33599]: audit 2023-09-23T18:13:25.787499+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:26.452 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:25 smithi098 bash[33599]: audit 2023-09-23T18:13:25.795039+0000 mon.c (mon.2) 43 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T18:13:26.452 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:25 smithi098 bash[33599]: audit 2023-09-23T18:13:25.803622+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:27.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:26 smithi098 bash[27349]: audit 2023-09-23T18:13:25.795683+0000 mgr.y (mgr.24344) 37 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T18:13:27.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:26 smithi098 bash[27349]: cephadm 2023-09-23T18:13:25.820682+0000 mgr.y (mgr.24344) 38 : cephadm [INF] Deploying daemon grafana.a on smithi173 2023-09-23T18:13:27.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:26 smithi098 bash[33599]: audit 2023-09-23T18:13:25.795683+0000 mgr.y (mgr.24344) 37 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T18:13:27.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:26 smithi098 bash[33599]: cephadm 2023-09-23T18:13:25.820682+0000 mgr.y (mgr.24344) 38 : cephadm [INF] Deploying daemon grafana.a on smithi173 2023-09-23T18:13:27.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:26 smithi173 bash[27694]: audit 2023-09-23T18:13:25.795683+0000 mgr.y (mgr.24344) 37 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T18:13:27.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:26 smithi173 bash[27694]: cephadm 2023-09-23T18:13:25.820682+0000 mgr.y (mgr.24344) 38 : cephadm [INF] Deploying daemon grafana.a on smithi173 2023-09-23T18:13:27.934 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:13:27 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:13:27] "GET /metrics HTTP/1.1" 200 154668 "" "Prometheus/2.33.4" 2023-09-23T18:13:28.036 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:13:28.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:27 smithi098 bash[33599]: cluster 2023-09-23T18:13:26.557876+0000 mgr.y (mgr.24344) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:28.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:27 smithi098 bash[27349]: cluster 2023-09-23T18:13:26.557876+0000 mgr.y (mgr.24344) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:28.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:27 smithi173 bash[27694]: cluster 2023-09-23T18:13:26.557876+0000 mgr.y (mgr.24344) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:28.809 INFO:teuthology.orchestra.run.smithi098.stdout:{"epoch":18,"active_gid":24344,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6800","nonce":3706591260},{"type":"v1","addr":"172.21.15.98:6801","nonce":3706591260}]},"active_addr":"172.21.15.98:6801/3706591260","active_change":"2023-09-23T18:12:42.526079+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24358,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.98:8443/","prometheus":"http://172.21.15.98:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.98:0","nonce":4085572202}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.98:0","nonce":2098141849}]}]} 2023-09-23T18:13:28.813 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T18:13:28.813 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T18:13:28.814 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T18:13:29.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:28 smithi098 bash[27349]: audit 2023-09-23T18:13:28.019524+0000 mon.a (mon.0) 593 : audit [DBG] from='client.? 172.21.15.98:0/4290211807' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T18:13:29.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:28 smithi098 bash[33599]: audit 2023-09-23T18:13:28.019524+0000 mon.a (mon.0) 593 : audit [DBG] from='client.? 172.21.15.98:0/4290211807' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T18:13:29.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:28 smithi173 bash[27694]: audit 2023-09-23T18:13:28.019524+0000 mon.a (mon.0) 593 : audit [DBG] from='client.? 172.21.15.98:0/4290211807' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T18:13:29.939 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:13:29 smithi098 bash[53034]: level=info ts=2023-09-23T18:13:29.581Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002139656s 2023-09-23T18:13:30.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:29 smithi098 bash[27349]: cluster 2023-09-23T18:13:28.558503+0000 mgr.y (mgr.24344) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:30.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:29 smithi098 bash[33599]: cluster 2023-09-23T18:13:28.558503+0000 mgr.y (mgr.24344) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:30.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:29 smithi173 bash[27694]: cluster 2023-09-23T18:13:28.558503+0000 mgr.y (mgr.24344) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:32.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:31 smithi098 bash[27349]: cluster 2023-09-23T18:13:30.559037+0000 mgr.y (mgr.24344) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:32.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:31 smithi098 bash[33599]: cluster 2023-09-23T18:13:30.559037+0000 mgr.y (mgr.24344) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:32.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:31 smithi173 bash[27694]: cluster 2023-09-23T18:13:30.559037+0000 mgr.y (mgr.24344) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:33.585 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:33.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:33 smithi098 bash[27349]: cluster 2023-09-23T18:13:32.559661+0000 mgr.y (mgr.24344) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:33.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:33 smithi098 bash[27349]: audit 2023-09-23T18:13:32.810893+0000 mon.a (mon.0) 594 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:33.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:33 smithi098 bash[33599]: cluster 2023-09-23T18:13:32.559661+0000 mgr.y (mgr.24344) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:33.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:33 smithi098 bash[33599]: audit 2023-09-23T18:13:32.810893+0000 mon.a (mon.0) 594 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:34.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:33 smithi173 bash[27694]: cluster 2023-09-23T18:13:32.559661+0000 mgr.y (mgr.24344) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:34.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:33 smithi173 bash[27694]: audit 2023-09-23T18:13:32.810893+0000 mon.a (mon.0) 594 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:35.776 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:13:35 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:13:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:13:35.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:35 smithi098 bash[27349]: cluster 2023-09-23T18:13:34.560349+0000 mgr.y (mgr.24344) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:35.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:35 smithi098 bash[33599]: cluster 2023-09-23T18:13:34.560349+0000 mgr.y (mgr.24344) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:36.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:35 smithi173 bash[27694]: cluster 2023-09-23T18:13:34.560349+0000 mgr.y (mgr.24344) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:36.757 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:13:36.758 INFO:teuthology.orchestra.run.smithi098.stdout:{"epoch":55,"fsid":"756c96d4-5a3b-11ee-8db4-212e2dc638e7","created":"2023-09-23T18:04:11.580781+0000","modified":"2023-09-23T18:13:19.258837+0000","last_up_change":"2023-09-23T18:12:09.242233+0000","last_in_change":"2023-09-23T18:11:43.419979+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-23T18:08:35.146924+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T18:13:12.421802+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"4f35f176-4d07-4157-aeb9-dd8f71da3a63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6802","nonce":1147685383},{"type":"v1","addr":"172.21.15.98:6803","nonce":1147685383}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6804","nonce":1147685383},{"type":"v1","addr":"172.21.15.98:6805","nonce":1147685383}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6808","nonce":1147685383},{"type":"v1","addr":"172.21.15.98:6809","nonce":1147685383}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6806","nonce":1147685383},{"type":"v1","addr":"172.21.15.98:6807","nonce":1147685383}]},"public_addr":"172.21.15.98:6803/1147685383","cluster_addr":"172.21.15.98:6805/1147685383","heartbeat_back_addr":"172.21.15.98:6809/1147685383","heartbeat_front_addr":"172.21.15.98:6807/1147685383","state":["exists","up"]},{"osd":1,"uuid":"5d777d1f-3bd3-49ba-b0de-0ac532ee3934","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6810","nonce":3945616078},{"type":"v1","addr":"172.21.15.98:6811","nonce":3945616078}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6812","nonce":3945616078},{"type":"v1","addr":"172.21.15.98:6813","nonce":3945616078}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6816","nonce":3945616078},{"type":"v1","addr":"172.21.15.98:6817","nonce":3945616078}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6814","nonce":3945616078},{"type":"v1","addr":"172.21.15.98:6815","nonce":3945616078}]},"public_addr":"172.21.15.98:6811/3945616078","cluster_addr":"172.21.15.98:6813/3945616078","heartbeat_back_addr":"172.21.15.98:6817/3945616078","heartbeat_front_addr":"172.21.15.98:6815/3945616078","state":["exists","up"]},{"osd":2,"uuid":"63978288-ac36-4cf2-bbb7-46dc828e6ce5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6818","nonce":2338555672},{"type":"v1","addr":"172.21.15.98:6819","nonce":2338555672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6820","nonce":2338555672},{"type":"v1","addr":"172.21.15.98:6821","nonce":2338555672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6824","nonce":2338555672},{"type":"v1","addr":"172.21.15.98:6825","nonce":2338555672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6822","nonce":2338555672},{"type":"v1","addr":"172.21.15.98:6823","nonce":2338555672}]},"public_addr":"172.21.15.98:6819/2338555672","cluster_addr":"172.21.15.98:6821/2338555672","heartbeat_back_addr":"172.21.15.98:6825/2338555672","heartbeat_front_addr":"172.21.15.98:6823/2338555672","state":["exists","up"]},{"osd":3,"uuid":"48596d91-46da-4815-b810-d493bbf01dcc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6826","nonce":2634725099},{"type":"v1","addr":"172.21.15.98:6827","nonce":2634725099}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6828","nonce":2634725099},{"type":"v1","addr":"172.21.15.98:6829","nonce":2634725099}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6832","nonce":2634725099},{"type":"v1","addr":"172.21.15.98:6833","nonce":2634725099}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6830","nonce":2634725099},{"type":"v1","addr":"172.21.15.98:6831","nonce":2634725099}]},"public_addr":"172.21.15.98:6827/2634725099","cluster_addr":"172.21.15.98:6829/2634725099","heartbeat_back_addr":"172.21.15.98:6833/2634725099","heartbeat_front_addr":"172.21.15.98:6831/2634725099","state":["exists","up"]},{"osd":4,"uuid":"a562afc6-ffe6-44bb-91a9-8a04018c3c6f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6800","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6801","nonce":165906554}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6802","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6803","nonce":165906554}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6806","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6807","nonce":165906554}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6804","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6805","nonce":165906554}]},"public_addr":"172.21.15.173:6801/165906554","cluster_addr":"172.21.15.173:6803/165906554","heartbeat_back_addr":"172.21.15.173:6807/165906554","heartbeat_front_addr":"172.21.15.173:6805/165906554","state":["exists","up"]},{"osd":5,"uuid":"add28146-8301-4b4c-9616-967e9df3a8d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6808","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6809","nonce":2894909344}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6810","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6811","nonce":2894909344}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6814","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6815","nonce":2894909344}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6812","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6813","nonce":2894909344}]},"public_addr":"172.21.15.173:6809/2894909344","cluster_addr":"172.21.15.173:6811/2894909344","heartbeat_back_addr":"172.21.15.173:6815/2894909344","heartbeat_front_addr":"172.21.15.173:6813/2894909344","state":["exists","up"]},{"osd":6,"uuid":"f30854de-bc26-465d-8b0e-a9211ae2d9fa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6816","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6817","nonce":3550013548}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6818","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6819","nonce":3550013548}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6822","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6823","nonce":3550013548}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6820","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6821","nonce":3550013548}]},"public_addr":"172.21.15.173:6817/3550013548","cluster_addr":"172.21.15.173:6819/3550013548","heartbeat_back_addr":"172.21.15.173:6823/3550013548","heartbeat_front_addr":"172.21.15.173:6821/3550013548","state":["exists","up"]},{"osd":7,"uuid":"5f540de3-48f9-4bc8-b14a-9aadf838f16b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6824","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6825","nonce":135608222}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6826","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6827","nonce":135608222}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6830","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6831","nonce":135608222}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6828","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6829","nonce":135608222}]},"public_addr":"172.21.15.173:6825/135608222","cluster_addr":"172.21.15.173:6827/135608222","heartbeat_back_addr":"172.21.15.173:6831/135608222","heartbeat_front_addr":"172.21.15.173:6829/135608222","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:07:04.739243+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:07:48.583198+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:08:33.173348+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:09:18.218502+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:09:59.765019+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:10:42.490689+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:11:24.357920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:12:08.168900+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.98:6801/483469857":"2023-09-24T18:12:42.525837+0000","172.21.15.98:6800/483469857":"2023-09-24T18:12:42.525837+0000","172.21.15.98:0/1142988558":"2023-09-24T18:12:42.525837+0000","172.21.15.98:0/3472030975":"2023-09-24T18:05:11.060631+0000","172.21.15.98:6800/1392949322":"2023-09-24T18:05:11.060631+0000","172.21.15.98:0/3976553993":"2023-09-24T18:04:40.596823+0000","172.21.15.98:0/2194193535":"2023-09-24T18:12:42.525837+0000","172.21.15.98:0/3213365423":"2023-09-24T18:04:40.596823+0000","172.21.15.98:6801/1392949322":"2023-09-24T18:05:11.060631+0000","172.21.15.98:6800/3368447653":"2023-09-24T18:04:40.596823+0000","172.21.15.98:0/1603044865":"2023-09-24T18:05:11.060631+0000","172.21.15.98:6801/3368447653":"2023-09-24T18:04:40.596823+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T18:13:37.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:36 smithi098 bash[27349]: audit 2023-09-23T18:13:36.750524+0000 mon.a (mon.0) 595 : audit [DBG] from='client.? 172.21.15.98:0/3929427931' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:13:37.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:36 smithi098 bash[33599]: audit 2023-09-23T18:13:36.750524+0000 mon.a (mon.0) 595 : audit [DBG] from='client.? 172.21.15.98:0/3929427931' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:13:37.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:36 smithi173 bash[27694]: audit 2023-09-23T18:13:36.750524+0000 mon.a (mon.0) 595 : audit [DBG] from='client.? 172.21.15.98:0/3929427931' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:13:37.555 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T18:13:37.555 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T18:13:37.827 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:13:37 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:13:37] "GET /metrics HTTP/1.1" 200 155657 "" "Prometheus/2.33.4" 2023-09-23T18:13:38.136 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:37 smithi173 bash[27694]: cluster 2023-09-23T18:13:36.560804+0000 mgr.y (mgr.24344) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:38.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:37 smithi098 bash[27349]: cluster 2023-09-23T18:13:36.560804+0000 mgr.y (mgr.24344) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:38.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:37 smithi098 bash[33599]: cluster 2023-09-23T18:13:36.560804+0000 mgr.y (mgr.24344) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:39.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:39 smithi098 bash[27349]: cluster 2023-09-23T18:13:38.561278+0000 mgr.y (mgr.24344) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:39.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:39 smithi098 bash[33599]: cluster 2023-09-23T18:13:38.561278+0000 mgr.y (mgr.24344) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:39.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:39 smithi173 bash[27694]: cluster 2023-09-23T18:13:38.561278+0000 mgr.y (mgr.24344) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:41.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:41 smithi098 bash[27349]: cluster 2023-09-23T18:13:40.561889+0000 mgr.y (mgr.24344) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:41.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:41 smithi098 bash[33599]: cluster 2023-09-23T18:13:40.561889+0000 mgr.y (mgr.24344) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:42.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:41 smithi173 bash[27694]: cluster 2023-09-23T18:13:40.561889+0000 mgr.y (mgr.24344) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:42.323 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:43.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:43 smithi098 bash[27349]: cluster 2023-09-23T18:13:42.562527+0000 mgr.y (mgr.24344) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:43.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:43 smithi098 bash[33599]: cluster 2023-09-23T18:13:42.562527+0000 mgr.y (mgr.24344) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:44.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:43 smithi173 bash[27694]: cluster 2023-09-23T18:13:42.562527+0000 mgr.y (mgr.24344) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:45.398 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:13:45.398 INFO:teuthology.orchestra.run.smithi098.stdout:{"epoch":55,"fsid":"756c96d4-5a3b-11ee-8db4-212e2dc638e7","created":"2023-09-23T18:04:11.580781+0000","modified":"2023-09-23T18:13:19.258837+0000","last_up_change":"2023-09-23T18:12:09.242233+0000","last_in_change":"2023-09-23T18:11:43.419979+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-23T18:08:35.146924+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T18:13:12.421802+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"4f35f176-4d07-4157-aeb9-dd8f71da3a63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6802","nonce":1147685383},{"type":"v1","addr":"172.21.15.98:6803","nonce":1147685383}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6804","nonce":1147685383},{"type":"v1","addr":"172.21.15.98:6805","nonce":1147685383}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6808","nonce":1147685383},{"type":"v1","addr":"172.21.15.98:6809","nonce":1147685383}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6806","nonce":1147685383},{"type":"v1","addr":"172.21.15.98:6807","nonce":1147685383}]},"public_addr":"172.21.15.98:6803/1147685383","cluster_addr":"172.21.15.98:6805/1147685383","heartbeat_back_addr":"172.21.15.98:6809/1147685383","heartbeat_front_addr":"172.21.15.98:6807/1147685383","state":["exists","up"]},{"osd":1,"uuid":"5d777d1f-3bd3-49ba-b0de-0ac532ee3934","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6810","nonce":3945616078},{"type":"v1","addr":"172.21.15.98:6811","nonce":3945616078}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6812","nonce":3945616078},{"type":"v1","addr":"172.21.15.98:6813","nonce":3945616078}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6816","nonce":3945616078},{"type":"v1","addr":"172.21.15.98:6817","nonce":3945616078}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6814","nonce":3945616078},{"type":"v1","addr":"172.21.15.98:6815","nonce":3945616078}]},"public_addr":"172.21.15.98:6811/3945616078","cluster_addr":"172.21.15.98:6813/3945616078","heartbeat_back_addr":"172.21.15.98:6817/3945616078","heartbeat_front_addr":"172.21.15.98:6815/3945616078","state":["exists","up"]},{"osd":2,"uuid":"63978288-ac36-4cf2-bbb7-46dc828e6ce5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6818","nonce":2338555672},{"type":"v1","addr":"172.21.15.98:6819","nonce":2338555672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6820","nonce":2338555672},{"type":"v1","addr":"172.21.15.98:6821","nonce":2338555672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6824","nonce":2338555672},{"type":"v1","addr":"172.21.15.98:6825","nonce":2338555672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6822","nonce":2338555672},{"type":"v1","addr":"172.21.15.98:6823","nonce":2338555672}]},"public_addr":"172.21.15.98:6819/2338555672","cluster_addr":"172.21.15.98:6821/2338555672","heartbeat_back_addr":"172.21.15.98:6825/2338555672","heartbeat_front_addr":"172.21.15.98:6823/2338555672","state":["exists","up"]},{"osd":3,"uuid":"48596d91-46da-4815-b810-d493bbf01dcc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6826","nonce":2634725099},{"type":"v1","addr":"172.21.15.98:6827","nonce":2634725099}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6828","nonce":2634725099},{"type":"v1","addr":"172.21.15.98:6829","nonce":2634725099}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6832","nonce":2634725099},{"type":"v1","addr":"172.21.15.98:6833","nonce":2634725099}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6830","nonce":2634725099},{"type":"v1","addr":"172.21.15.98:6831","nonce":2634725099}]},"public_addr":"172.21.15.98:6827/2634725099","cluster_addr":"172.21.15.98:6829/2634725099","heartbeat_back_addr":"172.21.15.98:6833/2634725099","heartbeat_front_addr":"172.21.15.98:6831/2634725099","state":["exists","up"]},{"osd":4,"uuid":"a562afc6-ffe6-44bb-91a9-8a04018c3c6f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6800","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6801","nonce":165906554}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6802","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6803","nonce":165906554}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6806","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6807","nonce":165906554}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6804","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6805","nonce":165906554}]},"public_addr":"172.21.15.173:6801/165906554","cluster_addr":"172.21.15.173:6803/165906554","heartbeat_back_addr":"172.21.15.173:6807/165906554","heartbeat_front_addr":"172.21.15.173:6805/165906554","state":["exists","up"]},{"osd":5,"uuid":"add28146-8301-4b4c-9616-967e9df3a8d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6808","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6809","nonce":2894909344}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6810","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6811","nonce":2894909344}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6814","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6815","nonce":2894909344}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6812","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6813","nonce":2894909344}]},"public_addr":"172.21.15.173:6809/2894909344","cluster_addr":"172.21.15.173:6811/2894909344","heartbeat_back_addr":"172.21.15.173:6815/2894909344","heartbeat_front_addr":"172.21.15.173:6813/2894909344","state":["exists","up"]},{"osd":6,"uuid":"f30854de-bc26-465d-8b0e-a9211ae2d9fa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6816","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6817","nonce":3550013548}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6818","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6819","nonce":3550013548}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6822","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6823","nonce":3550013548}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6820","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6821","nonce":3550013548}]},"public_addr":"172.21.15.173:6817/3550013548","cluster_addr":"172.21.15.173:6819/3550013548","heartbeat_back_addr":"172.21.15.173:6823/3550013548","heartbeat_front_addr":"172.21.15.173:6821/3550013548","state":["exists","up"]},{"osd":7,"uuid":"5f540de3-48f9-4bc8-b14a-9aadf838f16b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6824","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6825","nonce":135608222}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6826","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6827","nonce":135608222}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6830","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6831","nonce":135608222}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6828","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6829","nonce":135608222}]},"public_addr":"172.21.15.173:6825/135608222","cluster_addr":"172.21.15.173:6827/135608222","heartbeat_back_addr":"172.21.15.173:6831/135608222","heartbeat_front_addr":"172.21.15.173:6829/135608222","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:07:04.739243+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:07:48.583198+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:08:33.173348+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:09:18.218502+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:09:59.765019+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:10:42.490689+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:11:24.357920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:12:08.168900+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.98:6801/483469857":"2023-09-24T18:12:42.525837+0000","172.21.15.98:6800/483469857":"2023-09-24T18:12:42.525837+0000","172.21.15.98:0/1142988558":"2023-09-24T18:12:42.525837+0000","172.21.15.98:0/3472030975":"2023-09-24T18:05:11.060631+0000","172.21.15.98:6800/1392949322":"2023-09-24T18:05:11.060631+0000","172.21.15.98:0/3976553993":"2023-09-24T18:04:40.596823+0000","172.21.15.98:0/2194193535":"2023-09-24T18:12:42.525837+0000","172.21.15.98:0/3213365423":"2023-09-24T18:04:40.596823+0000","172.21.15.98:6801/1392949322":"2023-09-24T18:05:11.060631+0000","172.21.15.98:6800/3368447653":"2023-09-24T18:04:40.596823+0000","172.21.15.98:0/1603044865":"2023-09-24T18:05:11.060631+0000","172.21.15.98:6801/3368447653":"2023-09-24T18:04:40.596823+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T18:13:45.526 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:13:45 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:13:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:13:45.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:45 smithi098 bash[27349]: cluster 2023-09-23T18:13:44.562996+0000 mgr.y (mgr.24344) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:45.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:45 smithi098 bash[27349]: audit 2023-09-23T18:13:45.391105+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.98:0/1475697983' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:13:45.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:45 smithi098 bash[33599]: cluster 2023-09-23T18:13:44.562996+0000 mgr.y (mgr.24344) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:45.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:45 smithi098 bash[33599]: audit 2023-09-23T18:13:45.391105+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.98:0/1475697983' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:13:46.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:45 smithi173 bash[27694]: cluster 2023-09-23T18:13:44.562996+0000 mgr.y (mgr.24344) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:46.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:45 smithi173 bash[27694]: audit 2023-09-23T18:13:45.391105+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.98:0/1475697983' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:13:46.214 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T18:13:46.215 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T18:13:46.215 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T18:13:46.216 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T18:13:46.216 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T18:13:46.216 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T18:13:46.217 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T18:13:46.217 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T18:13:47.650 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:47 smithi173 bash[27694]: cluster 2023-09-23T18:13:46.563470+0000 mgr.y (mgr.24344) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:47.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:47 smithi098 bash[27349]: cluster 2023-09-23T18:13:46.563470+0000 mgr.y (mgr.24344) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:47.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:47 smithi098 bash[33599]: cluster 2023-09-23T18:13:46.563470+0000 mgr.y (mgr.24344) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:47.945 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:13:47 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:13:47] "GET /metrics HTTP/1.1" 200 155654 "" "Prometheus/2.33.4" 2023-09-23T18:13:48.027 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:47 smithi173 systemd[1]: Started Ceph grafana.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:13:49.077 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:48 smithi173 bash[27694]: audit 2023-09-23T18:13:47.918736+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:49.077 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:48 smithi173 bash[27694]: audit 2023-09-23T18:13:47.921734+0000 mon.c (mon.2) 45 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:13:49.077 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:48 smithi173 bash[47708]: t=2023-09-23T18:13:48+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-09-23T18:13:49.078 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:48 smithi173 bash[47708]: t=2023-09-23T18:13:48+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-09-23T18:13:49.078 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:48 smithi173 bash[47708]: t=2023-09-23T18:13:48+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-09-23T18:13:49.078 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:48 smithi173 bash[47708]: t=2023-09-23T18:13:48+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-09-23T18:13:49.078 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:48 smithi173 bash[47708]: t=2023-09-23T18:13:48+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-09-23T18:13:49.078 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:48 smithi173 bash[47708]: t=2023-09-23T18:13:48+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-09-23T18:13:49.078 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:48 smithi173 bash[47708]: t=2023-09-23T18:13:48+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-09-23T18:13:49.078 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:48 smithi173 bash[47708]: t=2023-09-23T18:13:48+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-09-23T18:13:49.078 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:48 smithi173 bash[47708]: t=2023-09-23T18:13:48+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-09-23T18:13:49.078 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:48 smithi173 bash[47708]: t=2023-09-23T18:13:48+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-09-23T18:13:49.078 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:48 smithi173 bash[47708]: t=2023-09-23T18:13:48+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-09-23T18:13:49.078 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:48 smithi173 bash[47708]: t=2023-09-23T18:13:48+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-09-23T18:13:49.079 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:48 smithi173 bash[47708]: t=2023-09-23T18:13:48+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-09-23T18:13:49.079 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:48 smithi173 bash[47708]: t=2023-09-23T18:13:48+0000 lvl=info msg="App mode production" logger=settings 2023-09-23T18:13:49.079 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:48 smithi173 bash[47708]: t=2023-09-23T18:13:48+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-09-23T18:13:49.079 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:48 smithi173 bash[47708]: t=2023-09-23T18:13:48+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-09-23T18:13:49.079 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:48 smithi173 bash[47708]: t=2023-09-23T18:13:48+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-09-23T18:13:49.079 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:48 smithi173 bash[47708]: t=2023-09-23T18:13:48+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2023-09-23T18:13:49.079 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2023-09-23T18:13:49.079 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2023-09-23T18:13:49.079 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2023-09-23T18:13:49.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:48 smithi098 bash[27349]: audit 2023-09-23T18:13:47.918736+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:49.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:48 smithi098 bash[27349]: audit 2023-09-23T18:13:47.921734+0000 mon.c (mon.2) 45 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:13:49.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:48 smithi098 bash[33599]: audit 2023-09-23T18:13:47.918736+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:49.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:48 smithi098 bash[33599]: audit 2023-09-23T18:13:47.921734+0000 mon.c (mon.2) 45 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:13:49.338 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2023-09-23T18:13:49.338 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2023-09-23T18:13:49.338 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2023-09-23T18:13:49.338 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2023-09-23T18:13:49.339 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2023-09-23T18:13:49.339 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2023-09-23T18:13:49.339 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-09-23T18:13:49.339 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2023-09-23T18:13:49.599 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2023-09-23T18:13:49.600 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2023-09-23T18:13:49.600 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2023-09-23T18:13:49.600 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2023-09-23T18:13:49.600 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2023-09-23T18:13:49.600 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2023-09-23T18:13:49.601 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2023-09-23T18:13:49.852 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2023-09-23T18:13:49.852 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2023-09-23T18:13:49.852 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2023-09-23T18:13:49.852 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2023-09-23T18:13:49.852 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2023-09-23T18:13:49.852 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2023-09-23T18:13:49.852 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2023-09-23T18:13:49.852 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2023-09-23T18:13:49.852 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2023-09-23T18:13:50.112 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[27694]: cluster 2023-09-23T18:13:48.564009+0000 mgr.y (mgr.24344) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:50.112 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2023-09-23T18:13:50.113 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-09-23T18:13:50.113 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:49 smithi173 bash[47708]: t=2023-09-23T18:13:49+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2023-09-23T18:13:50.113 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2023-09-23T18:13:50.113 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2023-09-23T18:13:50.113 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2023-09-23T18:13:50.113 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2023-09-23T18:13:50.113 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2023-09-23T18:13:50.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:49 smithi098 bash[27349]: cluster 2023-09-23T18:13:48.564009+0000 mgr.y (mgr.24344) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:50.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:49 smithi098 bash[33599]: cluster 2023-09-23T18:13:48.564009+0000 mgr.y (mgr.24344) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:50.390 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2023-09-23T18:13:50.390 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2023-09-23T18:13:50.390 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2023-09-23T18:13:50.390 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2023-09-23T18:13:50.390 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2023-09-23T18:13:50.390 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2023-09-23T18:13:50.391 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2023-09-23T18:13:50.391 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2023-09-23T18:13:50.659 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2023-09-23T18:13:50.659 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2023-09-23T18:13:50.659 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2023-09-23T18:13:50.659 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2023-09-23T18:13:50.660 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2023-09-23T18:13:50.660 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2023-09-23T18:13:50.660 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2023-09-23T18:13:50.660 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2023-09-23T18:13:50.911 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2023-09-23T18:13:50.912 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2023-09-23T18:13:50.912 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-09-23T18:13:50.912 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2023-09-23T18:13:50.912 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2023-09-23T18:13:50.912 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2023-09-23T18:13:50.912 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2023-09-23T18:13:51.118 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:51.119 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:51.119 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:51.123 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:51.123 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:51.123 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:51.124 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:51.129 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:51.172 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2023-09-23T18:13:51.173 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2023-09-23T18:13:51.173 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:50 smithi173 bash[47708]: t=2023-09-23T18:13:50+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2023-09-23T18:13:51.173 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2023-09-23T18:13:51.173 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2023-09-23T18:13:51.173 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2023-09-23T18:13:51.173 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2023-09-23T18:13:51.173 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2023-09-23T18:13:51.173 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2023-09-23T18:13:51.425 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2023-09-23T18:13:51.425 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2023-09-23T18:13:51.425 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2023-09-23T18:13:51.425 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2023-09-23T18:13:51.426 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2023-09-23T18:13:51.426 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2023-09-23T18:13:51.426 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2023-09-23T18:13:51.686 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2023-09-23T18:13:51.686 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2023-09-23T18:13:51.686 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2023-09-23T18:13:51.686 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2023-09-23T18:13:51.687 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2023-09-23T18:13:51.687 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2023-09-23T18:13:51.687 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-09-23T18:13:51.687 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2023-09-23T18:13:51.687 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-09-23T18:13:51.957 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[27694]: cluster 2023-09-23T18:13:50.564563+0000 mgr.y (mgr.24344) 51 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:51.958 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-09-23T18:13:51.958 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2023-09-23T18:13:51.958 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2023-09-23T18:13:51.958 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2023-09-23T18:13:51.958 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2023-09-23T18:13:51.958 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2023-09-23T18:13:51.958 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2023-09-23T18:13:51.958 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2023-09-23T18:13:51.959 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2023-09-23T18:13:52.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:51 smithi098 bash[27349]: cluster 2023-09-23T18:13:50.564563+0000 mgr.y (mgr.24344) 51 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:52.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:51 smithi098 bash[33599]: cluster 2023-09-23T18:13:50.564563+0000 mgr.y (mgr.24344) 51 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:52.209 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:51 smithi173 bash[47708]: t=2023-09-23T18:13:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2023-09-23T18:13:52.210 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2023-09-23T18:13:52.210 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2023-09-23T18:13:52.210 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2023-09-23T18:13:52.210 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2023-09-23T18:13:52.210 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2023-09-23T18:13:52.210 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2023-09-23T18:13:52.469 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2023-09-23T18:13:52.470 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-09-23T18:13:52.470 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2023-09-23T18:13:52.470 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2023-09-23T18:13:52.470 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2023-09-23T18:13:52.470 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2023-09-23T18:13:52.470 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2023-09-23T18:13:52.470 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2023-09-23T18:13:52.470 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2023-09-23T18:13:52.470 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2023-09-23T18:13:52.746 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2023-09-23T18:13:52.747 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2023-09-23T18:13:52.747 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2023-09-23T18:13:52.747 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2023-09-23T18:13:52.747 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2023-09-23T18:13:52.747 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2023-09-23T18:13:52.747 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2023-09-23T18:13:53.027 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2023-09-23T18:13:53.027 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2023-09-23T18:13:53.027 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2023-09-23T18:13:53.027 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2023-09-23T18:13:53.027 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2023-09-23T18:13:53.027 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:52 smithi173 bash[47708]: t=2023-09-23T18:13:52+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2023-09-23T18:13:53.329 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2023-09-23T18:13:53.329 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2023-09-23T18:13:53.329 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2023-09-23T18:13:53.329 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2023-09-23T18:13:53.329 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2023-09-23T18:13:53.330 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2023-09-23T18:13:53.330 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2023-09-23T18:13:53.330 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2023-09-23T18:13:53.590 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2023-09-23T18:13:53.590 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2023-09-23T18:13:53.590 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2023-09-23T18:13:53.590 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2023-09-23T18:13:53.590 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-09-23T18:13:53.590 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2023-09-23T18:13:53.590 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2023-09-23T18:13:53.843 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2023-09-23T18:13:53.844 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2023-09-23T18:13:53.844 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2023-09-23T18:13:53.844 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-09-23T18:13:53.844 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2023-09-23T18:13:53.844 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2023-09-23T18:13:53.901 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:53 smithi098 bash[33599]: cluster 2023-09-23T18:13:52.565050+0000 mgr.y (mgr.24344) 52 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:53.902 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:53 smithi098 bash[33599]: audit 2023-09-23T18:13:52.838101+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:53.902 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:53 smithi098 bash[33599]: audit 2023-09-23T18:13:52.908585+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:53.902 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:53 smithi098 bash[33599]: audit 2023-09-23T18:13:53.677036+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:53.902 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:53 smithi098 bash[27349]: cluster 2023-09-23T18:13:52.565050+0000 mgr.y (mgr.24344) 52 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:53.902 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:53 smithi098 bash[27349]: audit 2023-09-23T18:13:52.838101+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:53.902 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:53 smithi098 bash[27349]: audit 2023-09-23T18:13:52.908585+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:53.902 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:53 smithi098 bash[27349]: audit 2023-09-23T18:13:53.677036+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:54.111 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[27694]: cluster 2023-09-23T18:13:52.565050+0000 mgr.y (mgr.24344) 52 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:54.111 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[27694]: audit 2023-09-23T18:13:52.838101+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:54.112 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[27694]: audit 2023-09-23T18:13:52.908585+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:54.112 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[27694]: audit 2023-09-23T18:13:53.677036+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:54.112 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2023-09-23T18:13:54.112 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2023-09-23T18:13:54.112 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-09-23T18:13:54.112 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2023-09-23T18:13:54.113 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:53 smithi173 bash[47708]: t=2023-09-23T18:13:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2023-09-23T18:13:54.113 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2023-09-23T18:13:54.113 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2023-09-23T18:13:54.113 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2023-09-23T18:13:54.113 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2023-09-23T18:13:54.364 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2023-09-23T18:13:54.364 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2023-09-23T18:13:54.364 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2023-09-23T18:13:54.364 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2023-09-23T18:13:54.364 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2023-09-23T18:13:54.364 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2023-09-23T18:13:54.364 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2023-09-23T18:13:54.364 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2023-09-23T18:13:54.364 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2023-09-23T18:13:54.616 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2023-09-23T18:13:54.616 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2023-09-23T18:13:54.616 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2023-09-23T18:13:54.616 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2023-09-23T18:13:54.616 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2023-09-23T18:13:54.616 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2023-09-23T18:13:54.616 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-09-23T18:13:54.616 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-09-23T18:13:54.885 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-09-23T18:13:54.885 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2023-09-23T18:13:54.885 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-09-23T18:13:54.885 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2023-09-23T18:13:54.886 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2023-09-23T18:13:54.886 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2023-09-23T18:13:54.886 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2023-09-23T18:13:54.886 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[27694]: audit 2023-09-23T18:13:53.942002+0000 mon.c (mon.2) 46 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:13:54.886 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[27694]: audit 2023-09-23T18:13:53.943053+0000 mon.c (mon.2) 47 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:13:54.886 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[27694]: audit 2023-09-23T18:13:53.949116+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:54.886 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[27694]: cephadm 2023-09-23T18:13:53.953255+0000 mgr.y (mgr.24344) 53 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T18:13:54.886 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[27694]: cephadm 2023-09-23T18:13:53.957219+0000 mgr.y (mgr.24344) 54 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi098 2023-09-23T18:13:55.163 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2023-09-23T18:13:55.163 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2023-09-23T18:13:55.163 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2023-09-23T18:13:55.163 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:54 smithi173 bash[47708]: t=2023-09-23T18:13:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2023-09-23T18:13:55.163 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2023-09-23T18:13:55.163 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2023-09-23T18:13:55.163 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2023-09-23T18:13:55.163 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2023-09-23T18:13:55.164 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2023-09-23T18:13:55.164 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2023-09-23T18:13:55.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:54 smithi098 bash[27349]: audit 2023-09-23T18:13:53.942002+0000 mon.c (mon.2) 46 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:13:55.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:54 smithi098 bash[27349]: audit 2023-09-23T18:13:53.943053+0000 mon.c (mon.2) 47 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:13:55.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:54 smithi098 bash[27349]: audit 2023-09-23T18:13:53.949116+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:55.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:54 smithi098 bash[27349]: cephadm 2023-09-23T18:13:53.953255+0000 mgr.y (mgr.24344) 53 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T18:13:55.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:54 smithi098 bash[27349]: cephadm 2023-09-23T18:13:53.957219+0000 mgr.y (mgr.24344) 54 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi098 2023-09-23T18:13:55.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:54 smithi098 bash[33599]: audit 2023-09-23T18:13:53.942002+0000 mon.c (mon.2) 46 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:13:55.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:54 smithi098 bash[33599]: audit 2023-09-23T18:13:53.943053+0000 mon.c (mon.2) 47 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:13:55.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:54 smithi098 bash[33599]: audit 2023-09-23T18:13:53.949116+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:13:55.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:54 smithi098 bash[33599]: cephadm 2023-09-23T18:13:53.953255+0000 mgr.y (mgr.24344) 53 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T18:13:55.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:54 smithi098 bash[33599]: cephadm 2023-09-23T18:13:53.957219+0000 mgr.y (mgr.24344) 54 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi098 2023-09-23T18:13:55.431 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:13:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:13:55.432 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-09-23T18:13:55.432 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2023-09-23T18:13:55.432 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2023-09-23T18:13:55.432 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2023-09-23T18:13:55.432 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2023-09-23T18:13:55.432 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2023-09-23T18:13:55.433 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2023-09-23T18:13:55.433 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2023-09-23T18:13:55.684 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2023-09-23T18:13:55.684 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2023-09-23T18:13:55.684 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2023-09-23T18:13:55.684 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2023-09-23T18:13:55.684 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2023-09-23T18:13:55.684 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2023-09-23T18:13:55.684 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2023-09-23T18:13:55.957 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2023-09-23T18:13:55.957 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2023-09-23T18:13:55.957 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2023-09-23T18:13:55.957 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2023-09-23T18:13:55.957 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2023-09-23T18:13:55.958 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2023-09-23T18:13:55.958 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2023-09-23T18:13:55.958 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2023-09-23T18:13:55.958 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-09-23T18:13:55.958 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-09-23T18:13:56.247 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[27694]: cluster 2023-09-23T18:13:54.565480+0000 mgr.y (mgr.24344) 55 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:56.272 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2023-09-23T18:13:56.272 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:55 smithi173 bash[47708]: t=2023-09-23T18:13:55+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-09-23T18:13:56.273 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2023-09-23T18:13:56.273 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2023-09-23T18:13:56.273 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2023-09-23T18:13:56.273 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2023-09-23T18:13:56.273 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2023-09-23T18:13:56.273 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2023-09-23T18:13:56.273 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2023-09-23T18:13:56.274 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2023-09-23T18:13:56.274 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:55 smithi098 bash[27349]: cluster 2023-09-23T18:13:54.565480+0000 mgr.y (mgr.24344) 55 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:56.275 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:55 smithi098 bash[33599]: cluster 2023-09-23T18:13:54.565480+0000 mgr.y (mgr.24344) 55 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:56.499 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2023-09-23T18:13:56.500 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2023-09-23T18:13:56.500 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2023-09-23T18:13:56.500 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2023-09-23T18:13:56.500 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2023-09-23T18:13:56.501 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-09-23T18:13:56.501 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-09-23T18:13:56.501 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2023-09-23T18:13:56.760 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2023-09-23T18:13:56.761 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2023-09-23T18:13:56.761 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2023-09-23T18:13:56.761 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2023-09-23T18:13:56.761 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2023-09-23T18:13:56.761 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-09-23T18:13:56.761 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2023-09-23T18:13:56.761 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2023-09-23T18:13:56.761 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2023-09-23T18:13:57.021 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2023-09-23T18:13:57.021 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2023-09-23T18:13:57.022 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2023-09-23T18:13:57.022 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2023-09-23T18:13:57.022 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2023-09-23T18:13:57.022 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:56 smithi173 bash[47708]: t=2023-09-23T18:13:56+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2023-09-23T18:13:57.022 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2023-09-23T18:13:57.277 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2023-09-23T18:13:57.277 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2023-09-23T18:13:57.277 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2023-09-23T18:13:57.277 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2023-09-23T18:13:57.277 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2023-09-23T18:13:57.277 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2023-09-23T18:13:57.277 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2023-09-23T18:13:57.543 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2023-09-23T18:13:57.543 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2023-09-23T18:13:57.543 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2023-09-23T18:13:57.543 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2023-09-23T18:13:57.543 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2023-09-23T18:13:57.543 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2023-09-23T18:13:57.543 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2023-09-23T18:13:57.543 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2023-09-23T18:13:57.544 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2023-09-23T18:13:57.544 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2023-09-23T18:13:57.794 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2023-09-23T18:13:57.795 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-09-23T18:13:57.795 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2023-09-23T18:13:57.795 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2023-09-23T18:13:57.795 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2023-09-23T18:13:57.795 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2023-09-23T18:13:57.795 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2023-09-23T18:13:57.795 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-09-23T18:13:58.047 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[27694]: cluster 2023-09-23T18:13:56.565859+0000 mgr.y (mgr.24344) 56 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:58.048 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2023-09-23T18:13:58.048 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2023-09-23T18:13:58.048 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2023-09-23T18:13:58.048 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2023-09-23T18:13:58.048 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2023-09-23T18:13:58.048 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:57 smithi173 bash[47708]: t=2023-09-23T18:13:57+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2023-09-23T18:13:58.048 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2023-09-23T18:13:58.049 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2023-09-23T18:13:58.107 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:13:57 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:13:57] "GET /metrics HTTP/1.1" 200 155654 "" "Prometheus/2.33.4" 2023-09-23T18:13:58.108 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:57 smithi098 bash[33599]: cluster 2023-09-23T18:13:56.565859+0000 mgr.y (mgr.24344) 56 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:58.108 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:57 smithi098 bash[27349]: cluster 2023-09-23T18:13:56.565859+0000 mgr.y (mgr.24344) 56 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:58.308 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2023-09-23T18:13:58.308 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2023-09-23T18:13:58.308 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2023-09-23T18:13:58.308 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2023-09-23T18:13:58.308 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2023-09-23T18:13:58.308 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2023-09-23T18:13:58.308 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2023-09-23T18:13:58.308 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2023-09-23T18:13:58.560 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2023-09-23T18:13:58.560 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2023-09-23T18:13:58.560 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2023-09-23T18:13:58.560 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2023-09-23T18:13:58.560 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2023-09-23T18:13:58.560 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2023-09-23T18:13:58.560 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2023-09-23T18:13:58.560 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2023-09-23T18:13:58.561 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2023-09-23T18:13:58.821 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2023-09-23T18:13:58.821 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2023-09-23T18:13:58.821 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2023-09-23T18:13:58.821 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2023-09-23T18:13:58.821 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2023-09-23T18:13:58.822 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-09-23T18:13:58.822 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-09-23T18:13:58.822 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2023-09-23T18:13:59.107 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2023-09-23T18:13:59.107 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2023-09-23T18:13:59.107 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-09-23T18:13:59.107 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2023-09-23T18:13:59.107 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:58 smithi173 bash[47708]: t=2023-09-23T18:13:58+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2023-09-23T18:13:59.107 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-09-23T18:13:59.107 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2023-09-23T18:13:59.108 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2023-09-23T18:13:59.108 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2023-09-23T18:13:59.360 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[27694]: cluster 2023-09-23T18:13:58.566222+0000 mgr.y (mgr.24344) 57 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:59.360 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-09-23T18:13:59.360 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2023-09-23T18:13:59.361 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2023-09-23T18:13:59.361 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2023-09-23T18:13:59.361 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2023-09-23T18:13:59.361 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-09-23T18:13:59.361 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2023-09-23T18:13:59.361 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2023-09-23T18:13:59.361 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2023-09-23T18:13:59.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:13:59 smithi098 bash[27349]: cluster 2023-09-23T18:13:58.566222+0000 mgr.y (mgr.24344) 57 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:59.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:13:59 smithi098 bash[33599]: cluster 2023-09-23T18:13:58.566222+0000 mgr.y (mgr.24344) 57 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:13:59.628 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2023-09-23T18:13:59.629 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2023-09-23T18:13:59.629 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-09-23T18:13:59.629 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2023-09-23T18:13:59.629 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2023-09-23T18:13:59.629 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-09-23T18:13:59.630 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2023-09-23T18:13:59.630 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-09-23T18:13:59.630 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-09-23T18:13:59.889 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2023-09-23T18:13:59.889 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2023-09-23T18:13:59.889 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2023-09-23T18:13:59.890 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2023-09-23T18:13:59.890 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2023-09-23T18:13:59.890 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2023-09-23T18:13:59.890 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-09-23T18:13:59.890 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2023-09-23T18:13:59.890 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2023-09-23T18:14:00.159 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2023-09-23T18:14:00.159 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2023-09-23T18:14:00.159 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2023-09-23T18:14:00.159 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:13:59 smithi173 bash[47708]: t=2023-09-23T18:13:59+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2023-09-23T18:14:00.159 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2023-09-23T18:14:00.159 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2023-09-23T18:14:00.159 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2023-09-23T18:14:00.159 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2023-09-23T18:14:00.160 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2023-09-23T18:14:00.160 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2023-09-23T18:14:00.428 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2023-09-23T18:14:00.428 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2023-09-23T18:14:00.428 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2023-09-23T18:14:00.428 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2023-09-23T18:14:00.428 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2023-09-23T18:14:00.428 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2023-09-23T18:14:00.428 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2023-09-23T18:14:00.429 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2023-09-23T18:14:00.429 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2023-09-23T18:14:00.688 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2023-09-23T18:14:00.688 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2023-09-23T18:14:00.688 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2023-09-23T18:14:00.688 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2023-09-23T18:14:00.689 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2023-09-23T18:14:00.689 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2023-09-23T18:14:00.689 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2023-09-23T18:14:00.948 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2023-09-23T18:14:00.949 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2023-09-23T18:14:00.949 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2023-09-23T18:14:00.949 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2023-09-23T18:14:00.949 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2023-09-23T18:14:00.949 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2023-09-23T18:14:00.949 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2023-09-23T18:14:00.950 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2023-09-23T18:14:00.950 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2023-09-23T18:14:01.217 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:00 smithi173 bash[47708]: t=2023-09-23T18:14:00+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2023-09-23T18:14:01.217 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:01 smithi173 bash[47708]: t=2023-09-23T18:14:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2023-09-23T18:14:01.217 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:01 smithi173 bash[47708]: t=2023-09-23T18:14:01+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2023-09-23T18:14:01.217 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:01 smithi173 bash[47708]: t=2023-09-23T18:14:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2023-09-23T18:14:01.218 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:01 smithi173 bash[47708]: t=2023-09-23T18:14:01+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2023-09-23T18:14:01.218 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:01 smithi173 bash[47708]: t=2023-09-23T18:14:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2023-09-23T18:14:01.218 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:01 smithi173 bash[47708]: t=2023-09-23T18:14:01+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=12.179268232s 2023-09-23T18:14:01.218 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:01 smithi173 bash[47708]: t=2023-09-23T18:14:01+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-09-23T18:14:01.487 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:01 smithi173 bash[47708]: t=2023-09-23T18:14:01+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-09-23T18:14:01.488 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:01 smithi173 bash[47708]: t=2023-09-23T18:14:01+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-09-23T18:14:01.488 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:01 smithi173 bash[47708]: t=2023-09-23T18:14:01+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-09-23T18:14:01.488 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:01 smithi173 bash[47708]: t=2023-09-23T18:14:01+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-09-23T18:14:01.488 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:01 smithi173 bash[47708]: t=2023-09-23T18:14:01+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-09-23T18:14:01.488 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:01 smithi173 bash[47708]: t=2023-09-23T18:14:01+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-09-23T18:14:01.488 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:01 smithi173 bash[47708]: t=2023-09-23T18:14:01+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-09-23T18:14:01.488 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:01 smithi173 bash[47708]: t=2023-09-23T18:14:01+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-09-23T18:14:01.488 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:01 smithi173 bash[47708]: t=2023-09-23T18:14:01+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-09-23T18:14:01.488 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:14:01 smithi173 bash[47708]: t=2023-09-23T18:14:01+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-09-23T18:14:01.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:01 smithi173 bash[27694]: cluster 2023-09-23T18:14:00.566674+0000 mgr.y (mgr.24344) 58 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:01.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:01 smithi098 bash[27349]: cluster 2023-09-23T18:14:00.566674+0000 mgr.y (mgr.24344) 58 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:01.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:01 smithi098 bash[33599]: cluster 2023-09-23T18:14:00.566674+0000 mgr.y (mgr.24344) 58 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:03.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:03 smithi098 bash[33599]: cluster 2023-09-23T18:14:02.567007+0000 mgr.y (mgr.24344) 59 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:03.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:03 smithi098 bash[27349]: cluster 2023-09-23T18:14:02.567007+0000 mgr.y (mgr.24344) 59 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:04.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:03 smithi173 bash[27694]: cluster 2023-09-23T18:14:02.567007+0000 mgr.y (mgr.24344) 59 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:05.637 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:14:05 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:14:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:14:05.755 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:05 smithi098 bash[27349]: cluster 2023-09-23T18:14:04.567580+0000 mgr.y (mgr.24344) 60 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:05.756 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:05 smithi098 bash[33599]: cluster 2023-09-23T18:14:04.567580+0000 mgr.y (mgr.24344) 60 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:06.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:05 smithi173 bash[27694]: cluster 2023-09-23T18:14:04.567580+0000 mgr.y (mgr.24344) 60 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:07.786 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:14:07 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:14:07] "GET /metrics HTTP/1.1" 200 155662 "" "Prometheus/2.33.4" 2023-09-23T18:14:07.787 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:07 smithi098 bash[33599]: cluster 2023-09-23T18:14:06.568047+0000 mgr.y (mgr.24344) 61 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:07.787 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:07 smithi098 bash[27349]: cluster 2023-09-23T18:14:06.568047+0000 mgr.y (mgr.24344) 61 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:08.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:07 smithi173 bash[27694]: cluster 2023-09-23T18:14:06.568047+0000 mgr.y (mgr.24344) 61 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:09.272 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:09 smithi098 bash[27349]: cluster 2023-09-23T18:14:08.568359+0000 mgr.y (mgr.24344) 62 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:09.272 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:09 smithi098 bash[33599]: cluster 2023-09-23T18:14:08.568359+0000 mgr.y (mgr.24344) 62 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:09.273 INFO:teuthology.orchestra.run.smithi098.stdout:55834574925 2023-09-23T18:14:09.273 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T18:14:09.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:09 smithi173 bash[27694]: cluster 2023-09-23T18:14:08.568359+0000 mgr.y (mgr.24344) 62 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:10.043 INFO:teuthology.orchestra.run.smithi098.stdout:77309411395 2023-09-23T18:14:10.043 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T18:14:10.045 INFO:teuthology.orchestra.run.smithi098.stdout:34359738454 2023-09-23T18:14:10.045 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T18:14:11.568 INFO:teuthology.orchestra.run.smithi098.stdout:180388626465 2023-09-23T18:14:11.568 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T18:14:11.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:11 smithi098 bash[33599]: cluster 2023-09-23T18:14:10.568820+0000 mgr.y (mgr.24344) 63 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:11.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:11 smithi098 bash[27349]: cluster 2023-09-23T18:14:10.568820+0000 mgr.y (mgr.24344) 63 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:12.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:11 smithi173 bash[27694]: cluster 2023-09-23T18:14:10.568820+0000 mgr.y (mgr.24344) 63 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:12.060 INFO:teuthology.orchestra.run.smithi098.stdout:154618822699 2023-09-23T18:14:12.060 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T18:14:13.284 INFO:teuthology.orchestra.run.smithi098.stdout:107374182459 2023-09-23T18:14:13.284 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T18:14:13.419 INFO:teuthology.orchestra.run.smithi098.stdout:206158430234 2023-09-23T18:14:13.420 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T18:14:13.433 INFO:teuthology.orchestra.run.smithi098.stdout:128849018932 2023-09-23T18:14:13.433 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T18:14:13.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:13 smithi098 bash[27349]: cluster 2023-09-23T18:14:12.569372+0000 mgr.y (mgr.24344) 64 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:13.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:13 smithi098 bash[33599]: cluster 2023-09-23T18:14:12.569372+0000 mgr.y (mgr.24344) 64 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:14.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:13 smithi173 bash[27694]: cluster 2023-09-23T18:14:12.569372+0000 mgr.y (mgr.24344) 64 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:14.695 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:14:14 smithi098 systemd[1]: Stopping Ceph alertmanager.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T18:14:14.695 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:14:14 smithi098 bash[56448]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-alertmanager.a 2023-09-23T18:14:14.696 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:14:14 smithi098 bash[53034]: level=info ts=2023-09-23T18:14:14.478Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-09-23T18:14:15.309 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:14:15 smithi098 bash[56483]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-alertmanager-a 2023-09-23T18:14:15.309 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:14:15 smithi098 bash[56594]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-alertmanager-a 2023-09-23T18:14:15.309 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:14:15 smithi098 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T18:14:15.309 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:14:15 smithi098 systemd[1]: Stopped Ceph alertmanager.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:14:15.309 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:14:15 smithi098 systemd[1]: Started Ceph alertmanager.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:14:15.639 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:14:15 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:14:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:14:15.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:15 smithi173 bash[27694]: cluster 2023-09-23T18:14:14.569859+0000 mgr.y (mgr.24344) 65 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:15.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:15 smithi173 bash[27694]: audit 2023-09-23T18:14:15.271927+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:15.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:15 smithi098 bash[33599]: cluster 2023-09-23T18:14:14.569859+0000 mgr.y (mgr.24344) 65 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:15.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:15 smithi098 bash[33599]: audit 2023-09-23T18:14:15.271927+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:15.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:15 smithi098 bash[27349]: cluster 2023-09-23T18:14:14.569859+0000 mgr.y (mgr.24344) 65 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:15.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:15 smithi098 bash[27349]: audit 2023-09-23T18:14:15.271927+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:16.446 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:14:16 smithi098 bash[56623]: level=info ts=2023-09-23T18:14:16.342Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-09-23T18:14:16.446 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:14:16 smithi098 bash[56623]: level=info ts=2023-09-23T18:14:16.342Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-09-23T18:14:16.446 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:14:16 smithi098 bash[56623]: level=info ts=2023-09-23T18:14:16.345Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.98 port=9094 2023-09-23T18:14:16.446 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:14:16 smithi098 bash[56623]: level=info ts=2023-09-23T18:14:16.346Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T18:14:16.446 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:14:16 smithi098 bash[56623]: level=info ts=2023-09-23T18:14:16.397Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T18:14:16.446 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:14:16 smithi098 bash[56623]: level=info ts=2023-09-23T18:14:16.397Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T18:14:16.446 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:14:16 smithi098 bash[56623]: level=info ts=2023-09-23T18:14:16.402Z caller=main.go:518 msg=Listening address=:9093 2023-09-23T18:14:16.446 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:14:16 smithi098 bash[56623]: level=info ts=2023-09-23T18:14:16.402Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-09-23T18:14:16.902 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:16 smithi173 bash[27694]: cephadm 2023-09-23T18:14:15.275455+0000 mgr.y (mgr.24344) 66 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T18:14:16.902 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:16 smithi173 bash[27694]: cephadm 2023-09-23T18:14:15.280344+0000 mgr.y (mgr.24344) 67 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi173 2023-09-23T18:14:16.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:16 smithi098 bash[27349]: cephadm 2023-09-23T18:14:15.275455+0000 mgr.y (mgr.24344) 66 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T18:14:16.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:16 smithi098 bash[27349]: cephadm 2023-09-23T18:14:15.280344+0000 mgr.y (mgr.24344) 67 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi173 2023-09-23T18:14:16.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:16 smithi098 bash[33599]: cephadm 2023-09-23T18:14:15.275455+0000 mgr.y (mgr.24344) 66 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T18:14:16.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:16 smithi098 bash[33599]: cephadm 2023-09-23T18:14:15.280344+0000 mgr.y (mgr.24344) 67 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi173 2023-09-23T18:14:17.945 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:14:17 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:14:17] "GET /metrics HTTP/1.1" 200 155657 "" "Prometheus/2.33.4" 2023-09-23T18:14:17.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:17 smithi098 bash[27349]: cluster 2023-09-23T18:14:16.570405+0000 mgr.y (mgr.24344) 68 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:17.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:17 smithi098 bash[33599]: cluster 2023-09-23T18:14:16.570405+0000 mgr.y (mgr.24344) 68 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:18.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:17 smithi173 bash[27694]: cluster 2023-09-23T18:14:16.570405+0000 mgr.y (mgr.24344) 68 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:18.047 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:18.048 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:18.048 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:18.055 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:18.055 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:18.058 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:18.059 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:18.061 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:18.695 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:14:18 smithi098 bash[56623]: level=info ts=2023-09-23T18:14:18.347Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000807869s 2023-09-23T18:14:19.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:19 smithi173 bash[27694]: cluster 2023-09-23T18:14:18.571014+0000 mgr.y (mgr.24344) 69 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:19.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:19 smithi098 bash[27349]: cluster 2023-09-23T18:14:18.571014+0000 mgr.y (mgr.24344) 69 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:19.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:19 smithi098 bash[33599]: cluster 2023-09-23T18:14:18.571014+0000 mgr.y (mgr.24344) 69 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:20.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:19 smithi173 systemd[1]: Stopping Ceph prometheus.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T18:14:20.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:19 smithi173 bash[48293]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-prometheus.a 2023-09-23T18:14:20.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:19 smithi173 bash[46883]: ts=2023-09-23T18:14:19.914Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T18:14:20.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:19 smithi173 bash[46883]: ts=2023-09-23T18:14:19.914Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-09-23T18:14:20.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:19 smithi173 bash[46883]: ts=2023-09-23T18:14:19.914Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-09-23T18:14:20.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:19 smithi173 bash[46883]: ts=2023-09-23T18:14:19.914Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-09-23T18:14:20.028 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:19 smithi173 bash[46883]: ts=2023-09-23T18:14:19.914Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-09-23T18:14:20.028 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:19 smithi173 bash[46883]: ts=2023-09-23T18:14:19.914Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-09-23T18:14:20.028 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:19 smithi173 bash[46883]: ts=2023-09-23T18:14:19.914Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T18:14:20.028 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:19 smithi173 bash[46883]: ts=2023-09-23T18:14:19.914Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-09-23T18:14:20.028 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:19 smithi173 bash[46883]: ts=2023-09-23T18:14:19.914Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T18:14:20.028 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:19 smithi173 bash[46883]: ts=2023-09-23T18:14:19.915Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T18:14:20.028 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:19 smithi173 bash[46883]: ts=2023-09-23T18:14:19.915Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-09-23T18:14:20.028 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:19 smithi173 bash[46883]: ts=2023-09-23T18:14:19.915Z caller=main.go:1066 level=info msg="See you next time!" 2023-09-23T18:14:20.686 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:20 smithi173 bash[48327]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-prometheus-a 2023-09-23T18:14:20.686 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:20 smithi173 bash[48391]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-prometheus-a 2023-09-23T18:14:20.687 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:20 smithi173 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T18:14:20.687 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:20 smithi173 systemd[1]: Stopped Ceph prometheus.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:14:20.687 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:20 smithi173 systemd[1]: Started Ceph prometheus.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:14:20.945 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:14:20 smithi098 bash[27618]: [23/Sep/2023:18:14:20] ENGINE Bus STOPPING 2023-09-23T18:14:20.945 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:14:20 smithi098 bash[27618]: [23/Sep/2023:18:14:20] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T18:14:20.945 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:14:20 smithi098 bash[27618]: [23/Sep/2023:18:14:20] ENGINE Bus STOPPED 2023-09-23T18:14:20.945 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:14:20 smithi098 bash[27618]: [23/Sep/2023:18:14:20] ENGINE Bus STARTING 2023-09-23T18:14:20.946 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:14:20 smithi098 bash[27618]: [23/Sep/2023:18:14:20] ENGINE Serving on http://:::9283 2023-09-23T18:14:20.946 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:14:20 smithi098 bash[27618]: [23/Sep/2023:18:14:20] ENGINE Bus STARTED 2023-09-23T18:14:20.946 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:14:20 smithi098 bash[27618]: [23/Sep/2023:18:14:20] ENGINE Bus STOPPING 2023-09-23T18:14:20.946 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:14:20 smithi098 bash[27618]: [23/Sep/2023:18:14:20] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T18:14:20.946 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:14:20 smithi098 bash[27618]: [23/Sep/2023:18:14:20] ENGINE Bus STOPPED 2023-09-23T18:14:20.946 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:14:20 smithi098 bash[27618]: [23/Sep/2023:18:14:20] ENGINE Bus STARTING 2023-09-23T18:14:20.946 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:14:20 smithi098 bash[27618]: [23/Sep/2023:18:14:20] ENGINE Serving on http://:::9283 2023-09-23T18:14:20.946 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:14:20 smithi098 bash[27618]: [23/Sep/2023:18:14:20] ENGINE Bus STARTED 2023-09-23T18:14:20.946 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:14:20 smithi098 bash[27618]: [23/Sep/2023:18:14:20] ENGINE Bus STOPPING 2023-09-23T18:14:20.946 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:14:20 smithi098 bash[27618]: [23/Sep/2023:18:14:20] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T18:14:20.946 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:14:20 smithi098 bash[27618]: [23/Sep/2023:18:14:20] ENGINE Bus STOPPED 2023-09-23T18:14:20.947 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:14:20 smithi098 bash[27618]: [23/Sep/2023:18:14:20] ENGINE Bus STARTING 2023-09-23T18:14:21.413 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[27618]: [23/Sep/2023:18:14:21] ENGINE Serving on http://:::9283 2023-09-23T18:14:21.413 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[27618]: [23/Sep/2023:18:14:21] ENGINE Bus STARTED 2023-09-23T18:14:21.736 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[27349]: cluster 2023-09-23T18:14:20.571683+0000 mgr.y (mgr.24344) 70 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:21.736 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[27349]: audit 2023-09-23T18:14:20.651341+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:21.736 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[27349]: audit 2023-09-23T18:14:20.656339+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T18:14:21.736 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[27349]: audit 2023-09-23T18:14:20.656826+0000 mgr.y (mgr.24344) 71 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T18:14:21.736 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[27349]: audit 2023-09-23T18:14:20.658109+0000 mon.c (mon.2) 49 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi098.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T18:14:21.895 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[27694]: cluster 2023-09-23T18:14:20.571683+0000 mgr.y (mgr.24344) 70 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:21.895 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[27694]: audit 2023-09-23T18:14:20.651341+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:21.895 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[27694]: audit 2023-09-23T18:14:20.656339+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T18:14:21.895 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[27694]: audit 2023-09-23T18:14:20.656826+0000 mgr.y (mgr.24344) 71 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T18:14:21.895 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[27694]: audit 2023-09-23T18:14:20.658109+0000 mon.c (mon.2) 49 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi098.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T18:14:21.895 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[27694]: audit 2023-09-23T18:14:20.658491+0000 mgr.y (mgr.24344) 72 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi098.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T18:14:21.895 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[27694]: audit 2023-09-23T18:14:20.664030+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:21.896 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[27694]: audit 2023-09-23T18:14:20.674467+0000 mon.c (mon.2) 50 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T18:14:21.896 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[27694]: audit 2023-09-23T18:14:20.675041+0000 mgr.y (mgr.24344) 73 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T18:14:21.896 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[27694]: audit 2023-09-23T18:14:20.679758+0000 mon.c (mon.2) 51 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi173.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T18:14:21.896 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[27694]: audit 2023-09-23T18:14:20.680295+0000 mgr.y (mgr.24344) 74 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi173.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T18:14:21.896 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[27694]: audit 2023-09-23T18:14:20.687720+0000 mon.a (mon.0) 604 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:21.896 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[27694]: audit 2023-09-23T18:14:20.695526+0000 mon.c (mon.2) 52 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T18:14:21.896 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[27694]: audit 2023-09-23T18:14:20.696160+0000 mgr.y (mgr.24344) 75 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T18:14:21.896 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[27694]: audit 2023-09-23T18:14:20.702453+0000 mon.c (mon.2) 53 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi173.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T18:14:21.896 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[27694]: audit 2023-09-23T18:14:20.703118+0000 mgr.y (mgr.24344) 76 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi173.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T18:14:21.896 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[27694]: audit 2023-09-23T18:14:20.708887+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:21.896 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[27694]: audit 2023-09-23T18:14:20.714122+0000 mon.c (mon.2) 54 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:14:21.897 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[48419]: ts=2023-09-23T18:14:21.757Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-09-23T18:14:21.897 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[48419]: ts=2023-09-23T18:14:21.757Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-09-23T18:14:21.897 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[48419]: ts=2023-09-23T18:14:21.757Z caller=main.go:518 level=info host_details="(Linux 5.15.0-84-generic #93~20.04.1-Ubuntu SMP Wed Sep 6 16:15:40 UTC 2023 x86_64 smithi173 (none))" 2023-09-23T18:14:21.897 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[48419]: ts=2023-09-23T18:14:21.757Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T18:14:21.897 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[48419]: ts=2023-09-23T18:14:21.757Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T18:14:21.897 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[48419]: ts=2023-09-23T18:14:21.760Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T18:14:21.897 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[48419]: ts=2023-09-23T18:14:21.761Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-09-23T18:14:21.897 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[48419]: ts=2023-09-23T18:14:21.761Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-09-23T18:14:21.897 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[48419]: ts=2023-09-23T18:14:21.766Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T18:14:21.897 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[48419]: ts=2023-09-23T18:14:21.766Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.102µs 2023-09-23T18:14:21.897 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[48419]: ts=2023-09-23T18:14:21.766Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T18:14:21.897 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[48419]: ts=2023-09-23T18:14:21.894Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-09-23T18:14:21.898 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[48419]: ts=2023-09-23T18:14:21.894Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-09-23T18:14:21.898 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[48419]: ts=2023-09-23T18:14:21.894Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=31.648µs wal_replay_duration=128.757034ms total_replay_duration=128.803323ms 2023-09-23T18:14:21.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[27349]: audit 2023-09-23T18:14:20.658491+0000 mgr.y (mgr.24344) 72 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi098.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T18:14:21.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[27349]: audit 2023-09-23T18:14:20.664030+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:21.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[27349]: audit 2023-09-23T18:14:20.674467+0000 mon.c (mon.2) 50 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T18:14:21.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[27349]: audit 2023-09-23T18:14:20.675041+0000 mgr.y (mgr.24344) 73 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T18:14:21.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[27349]: audit 2023-09-23T18:14:20.679758+0000 mon.c (mon.2) 51 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi173.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T18:14:21.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[27349]: audit 2023-09-23T18:14:20.680295+0000 mgr.y (mgr.24344) 74 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi173.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T18:14:21.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[27349]: audit 2023-09-23T18:14:20.687720+0000 mon.a (mon.0) 604 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:21.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[27349]: audit 2023-09-23T18:14:20.695526+0000 mon.c (mon.2) 52 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T18:14:21.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[27349]: audit 2023-09-23T18:14:20.696160+0000 mgr.y (mgr.24344) 75 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T18:14:21.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[27349]: audit 2023-09-23T18:14:20.702453+0000 mon.c (mon.2) 53 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi173.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T18:14:21.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[27349]: audit 2023-09-23T18:14:20.703118+0000 mgr.y (mgr.24344) 76 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi173.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T18:14:21.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[27349]: audit 2023-09-23T18:14:20.708887+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:21.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[27349]: audit 2023-09-23T18:14:20.714122+0000 mon.c (mon.2) 54 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:14:21.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[33599]: cluster 2023-09-23T18:14:20.571683+0000 mgr.y (mgr.24344) 70 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:21.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[33599]: audit 2023-09-23T18:14:20.651341+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:21.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[33599]: audit 2023-09-23T18:14:20.656339+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T18:14:21.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[33599]: audit 2023-09-23T18:14:20.656826+0000 mgr.y (mgr.24344) 71 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T18:14:21.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[33599]: audit 2023-09-23T18:14:20.658109+0000 mon.c (mon.2) 49 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi098.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T18:14:21.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[33599]: audit 2023-09-23T18:14:20.658491+0000 mgr.y (mgr.24344) 72 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi098.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T18:14:21.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[33599]: audit 2023-09-23T18:14:20.664030+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:21.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[33599]: audit 2023-09-23T18:14:20.674467+0000 mon.c (mon.2) 50 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T18:14:21.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[33599]: audit 2023-09-23T18:14:20.675041+0000 mgr.y (mgr.24344) 73 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T18:14:21.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[33599]: audit 2023-09-23T18:14:20.679758+0000 mon.c (mon.2) 51 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi173.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T18:14:21.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[33599]: audit 2023-09-23T18:14:20.680295+0000 mgr.y (mgr.24344) 74 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi173.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T18:14:21.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[33599]: audit 2023-09-23T18:14:20.687720+0000 mon.a (mon.0) 604 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:21.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[33599]: audit 2023-09-23T18:14:20.695526+0000 mon.c (mon.2) 52 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T18:14:21.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[33599]: audit 2023-09-23T18:14:20.696160+0000 mgr.y (mgr.24344) 75 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T18:14:21.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[33599]: audit 2023-09-23T18:14:20.702453+0000 mon.c (mon.2) 53 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi173.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T18:14:21.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[33599]: audit 2023-09-23T18:14:20.703118+0000 mgr.y (mgr.24344) 76 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi173.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T18:14:21.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[33599]: audit 2023-09-23T18:14:20.708887+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:21.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:21 smithi098 bash[33599]: audit 2023-09-23T18:14:20.714122+0000 mon.c (mon.2) 54 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:14:22.277 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[48419]: ts=2023-09-23T18:14:21.898Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T18:14:22.277 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[48419]: ts=2023-09-23T18:14:21.898Z caller=main.go:947 level=info msg="TSDB started" 2023-09-23T18:14:22.277 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[48419]: ts=2023-09-23T18:14:21.898Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T18:14:22.277 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[48419]: ts=2023-09-23T18:14:21.926Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=28.277227ms db_storage=1.209µs remote_storage=2.048µs web_handler=477ns query_engine=1.057µs scrape=356.847µs scrape_sd=47.754µs notify=33.621µs notify_sd=19.183µs rules=27.341243ms 2023-09-23T18:14:22.277 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:14:21 smithi173 bash[48419]: ts=2023-09-23T18:14:21.926Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-09-23T18:14:23.929 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:23 smithi098 bash[27349]: cluster 2023-09-23T18:14:22.572302+0000 mgr.y (mgr.24344) 77 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:23.930 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:23 smithi098 bash[33599]: cluster 2023-09-23T18:14:22.572302+0000 mgr.y (mgr.24344) 77 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:24.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:23 smithi173 bash[27694]: cluster 2023-09-23T18:14:22.572302+0000 mgr.y (mgr.24344) 77 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:25.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:25 smithi098 bash[27349]: cluster 2023-09-23T18:14:24.572932+0000 mgr.y (mgr.24344) 78 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:25.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:25 smithi098 bash[33599]: cluster 2023-09-23T18:14:24.572932+0000 mgr.y (mgr.24344) 78 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:26.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:25 smithi173 bash[27694]: cluster 2023-09-23T18:14:24.572932+0000 mgr.y (mgr.24344) 78 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:26.617 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:14:26 smithi098 bash[56623]: level=info ts=2023-09-23T18:14:26.350Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003717361s 2023-09-23T18:14:27.945 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:14:27 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:14:27] "GET /metrics HTTP/1.1" 200 155657 "" "Prometheus/2.33.4" 2023-09-23T18:14:27.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:27 smithi098 bash[27349]: cluster 2023-09-23T18:14:26.573363+0000 mgr.y (mgr.24344) 79 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:27.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:27 smithi098 bash[27349]: audit 2023-09-23T18:14:26.588396+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:27.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:27 smithi098 bash[33599]: cluster 2023-09-23T18:14:26.573363+0000 mgr.y (mgr.24344) 79 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:27.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:27 smithi098 bash[33599]: audit 2023-09-23T18:14:26.588396+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:28.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:27 smithi173 bash[27694]: cluster 2023-09-23T18:14:26.573363+0000 mgr.y (mgr.24344) 79 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:28.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:27 smithi173 bash[27694]: audit 2023-09-23T18:14:26.588396+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:29.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:29 smithi098 bash[27349]: cluster 2023-09-23T18:14:28.573725+0000 mgr.y (mgr.24344) 80 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:29.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:29 smithi098 bash[33599]: cluster 2023-09-23T18:14:28.573725+0000 mgr.y (mgr.24344) 80 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:29.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:29 smithi173 bash[27694]: cluster 2023-09-23T18:14:28.573725+0000 mgr.y (mgr.24344) 80 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:29.576 INFO:teuthology.orchestra.run.smithi098.stdout:77309411400 2023-09-23T18:14:30.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:30 smithi098 bash[33599]: audit 2023-09-23T18:14:29.570886+0000 mon.a (mon.0) 607 : audit [DBG] from='client.? 172.21.15.98:0/2387779313' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T18:14:30.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:30 smithi098 bash[27349]: audit 2023-09-23T18:14:29.570886+0000 mon.a (mon.0) 607 : audit [DBG] from='client.? 172.21.15.98:0/2387779313' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T18:14:30.463 INFO:teuthology.orchestra.run.smithi098.stdout:107374182463 2023-09-23T18:14:30.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:30 smithi173 bash[27694]: audit 2023-09-23T18:14:29.570886+0000 mon.a (mon.0) 607 : audit [DBG] from='client.? 172.21.15.98:0/2387779313' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T18:14:31.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:31 smithi098 bash[27349]: audit 2023-09-23T18:14:30.457900+0000 mon.a (mon.0) 608 : audit [DBG] from='client.? 172.21.15.98:0/3593990549' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T18:14:31.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:31 smithi098 bash[27349]: cluster 2023-09-23T18:14:30.574238+0000 mgr.y (mgr.24344) 81 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:31.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:31 smithi098 bash[33599]: audit 2023-09-23T18:14:30.457900+0000 mon.a (mon.0) 608 : audit [DBG] from='client.? 172.21.15.98:0/3593990549' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T18:14:31.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:31 smithi098 bash[33599]: cluster 2023-09-23T18:14:30.574238+0000 mgr.y (mgr.24344) 81 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:31.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:31 smithi173 bash[27694]: audit 2023-09-23T18:14:30.457900+0000 mon.a (mon.0) 608 : audit [DBG] from='client.? 172.21.15.98:0/3593990549' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T18:14:31.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:31 smithi173 bash[27694]: cluster 2023-09-23T18:14:30.574238+0000 mgr.y (mgr.24344) 81 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:32.411 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411395 got 77309411400 for osd.2 2023-09-23T18:14:32.411 DEBUG:teuthology.parallel:result is None 2023-09-23T18:14:33.144 INFO:teuthology.orchestra.run.smithi098.stdout:154618822704 2023-09-23T18:14:33.711 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:33 smithi098 bash[33599]: cluster 2023-09-23T18:14:32.574785+0000 mgr.y (mgr.24344) 82 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:33.711 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:33 smithi098 bash[33599]: audit 2023-09-23T18:14:33.139183+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.98:0/612728843' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T18:14:33.712 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:33 smithi098 bash[27349]: cluster 2023-09-23T18:14:32.574785+0000 mgr.y (mgr.24344) 82 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:33.712 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:33 smithi098 bash[27349]: audit 2023-09-23T18:14:33.139183+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.98:0/612728843' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T18:14:33.972 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182459 got 107374182463 for osd.3 2023-09-23T18:14:33.972 DEBUG:teuthology.parallel:result is None 2023-09-23T18:14:34.025 INFO:teuthology.orchestra.run.smithi098.stdout:128849018936 2023-09-23T18:14:34.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:33 smithi173 bash[27694]: cluster 2023-09-23T18:14:32.574785+0000 mgr.y (mgr.24344) 82 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:34.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:33 smithi173 bash[27694]: audit 2023-09-23T18:14:33.139183+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.98:0/612728843' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T18:14:34.460 INFO:teuthology.orchestra.run.smithi098.stdout:206158430238 2023-09-23T18:14:34.859 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:34 smithi098 bash[33599]: audit 2023-09-23T18:14:34.019761+0000 mon.a (mon.0) 609 : audit [DBG] from='client.? 172.21.15.98:0/990945593' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T18:14:34.859 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:34 smithi098 bash[33599]: audit 2023-09-23T18:14:34.455044+0000 mon.c (mon.2) 56 : audit [DBG] from='client.? 172.21.15.98:0/3619896154' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T18:14:34.859 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:34 smithi098 bash[27349]: audit 2023-09-23T18:14:34.019761+0000 mon.a (mon.0) 609 : audit [DBG] from='client.? 172.21.15.98:0/990945593' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T18:14:34.860 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:34 smithi098 bash[27349]: audit 2023-09-23T18:14:34.455044+0000 mon.c (mon.2) 56 : audit [DBG] from='client.? 172.21.15.98:0/3619896154' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T18:14:35.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:34 smithi173 bash[27694]: audit 2023-09-23T18:14:34.019761+0000 mon.a (mon.0) 609 : audit [DBG] from='client.? 172.21.15.98:0/990945593' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T18:14:35.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:34 smithi173 bash[27694]: audit 2023-09-23T18:14:34.455044+0000 mon.c (mon.2) 56 : audit [DBG] from='client.? 172.21.15.98:0/3619896154' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T18:14:35.091 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822699 got 154618822704 for osd.5 2023-09-23T18:14:35.091 DEBUG:teuthology.parallel:result is None 2023-09-23T18:14:35.319 INFO:teuthology.orchestra.run.smithi098.stdout:34359738460 2023-09-23T18:14:35.359 INFO:teuthology.orchestra.run.smithi098.stdout:55834574931 2023-09-23T18:14:35.378 INFO:teuthology.orchestra.run.smithi098.stdout:180388626471 2023-09-23T18:14:35.647 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:14:35 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:14:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:14:35.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:35 smithi098 bash[33599]: cluster 2023-09-23T18:14:34.575389+0000 mgr.y (mgr.24344) 83 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:35.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:35 smithi098 bash[33599]: audit 2023-09-23T18:14:35.313543+0000 mon.a (mon.0) 610 : audit [DBG] from='client.? 172.21.15.98:0/784174266' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T18:14:35.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:35 smithi098 bash[33599]: audit 2023-09-23T18:14:35.353293+0000 mon.a (mon.0) 611 : audit [DBG] from='client.? 172.21.15.98:0/497874805' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T18:14:35.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:35 smithi098 bash[33599]: audit 2023-09-23T18:14:35.372616+0000 mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.98:0/2167742683' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T18:14:35.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:35 smithi098 bash[27349]: cluster 2023-09-23T18:14:34.575389+0000 mgr.y (mgr.24344) 83 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:35.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:35 smithi098 bash[27349]: audit 2023-09-23T18:14:35.313543+0000 mon.a (mon.0) 610 : audit [DBG] from='client.? 172.21.15.98:0/784174266' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T18:14:35.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:35 smithi098 bash[27349]: audit 2023-09-23T18:14:35.353293+0000 mon.a (mon.0) 611 : audit [DBG] from='client.? 172.21.15.98:0/497874805' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T18:14:35.947 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:35 smithi098 bash[27349]: audit 2023-09-23T18:14:35.372616+0000 mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.98:0/2167742683' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T18:14:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:35 smithi173 bash[27694]: cluster 2023-09-23T18:14:34.575389+0000 mgr.y (mgr.24344) 83 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:35 smithi173 bash[27694]: audit 2023-09-23T18:14:35.313543+0000 mon.a (mon.0) 610 : audit [DBG] from='client.? 172.21.15.98:0/784174266' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T18:14:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:35 smithi173 bash[27694]: audit 2023-09-23T18:14:35.353293+0000 mon.a (mon.0) 611 : audit [DBG] from='client.? 172.21.15.98:0/497874805' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T18:14:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:35 smithi173 bash[27694]: audit 2023-09-23T18:14:35.372616+0000 mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.98:0/2167742683' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T18:14:36.683 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018932 got 128849018936 for osd.4 2023-09-23T18:14:36.683 DEBUG:teuthology.parallel:result is None 2023-09-23T18:14:36.828 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430234 got 206158430238 for osd.7 2023-09-23T18:14:36.828 DEBUG:teuthology.parallel:result is None 2023-09-23T18:14:36.982 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574925 got 55834574931 for osd.1 2023-09-23T18:14:36.983 DEBUG:teuthology.parallel:result is None 2023-09-23T18:14:37.670 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626465 got 180388626471 for osd.6 2023-09-23T18:14:37.670 DEBUG:teuthology.parallel:result is None 2023-09-23T18:14:37.680 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738454 got 34359738460 for osd.0 2023-09-23T18:14:37.680 DEBUG:teuthology.parallel:result is None 2023-09-23T18:14:37.680 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T18:14:37.680 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:14:37.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:37 smithi098 bash[27349]: cluster 2023-09-23T18:14:36.575823+0000 mgr.y (mgr.24344) 84 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:37.945 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:14:37 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:14:37] "GET /metrics HTTP/1.1" 200 155660 "" "Prometheus/2.33.4" 2023-09-23T18:14:37.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:37 smithi098 bash[33599]: cluster 2023-09-23T18:14:36.575823+0000 mgr.y (mgr.24344) 84 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:38.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:37 smithi173 bash[27694]: cluster 2023-09-23T18:14:36.575823+0000 mgr.y (mgr.24344) 84 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:40.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:39 smithi173 bash[27694]: cluster 2023-09-23T18:14:38.576401+0000 mgr.y (mgr.24344) 85 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:40.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:39 smithi173 bash[27694]: audit 2023-09-23T18:14:38.981518+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:40.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:39 smithi173 bash[27694]: audit 2023-09-23T18:14:38.983578+0000 mon.c (mon.2) 58 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:14:40.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:39 smithi173 bash[27694]: audit 2023-09-23T18:14:38.984921+0000 mon.c (mon.2) 59 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:14:40.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:39 smithi173 bash[27694]: audit 2023-09-23T18:14:38.992906+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:40.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:39 smithi173 bash[27694]: audit 2023-09-23T18:14:38.999357+0000 mon.c (mon.2) 60 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:14:40.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:39 smithi098 bash[27349]: cluster 2023-09-23T18:14:38.576401+0000 mgr.y (mgr.24344) 85 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:40.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:39 smithi098 bash[27349]: audit 2023-09-23T18:14:38.981518+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:40.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:39 smithi098 bash[27349]: audit 2023-09-23T18:14:38.983578+0000 mon.c (mon.2) 58 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:14:40.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:39 smithi098 bash[27349]: audit 2023-09-23T18:14:38.984921+0000 mon.c (mon.2) 59 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:14:40.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:39 smithi098 bash[27349]: audit 2023-09-23T18:14:38.992906+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:40.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:39 smithi098 bash[27349]: audit 2023-09-23T18:14:38.999357+0000 mon.c (mon.2) 60 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:14:40.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:39 smithi098 bash[33599]: cluster 2023-09-23T18:14:38.576401+0000 mgr.y (mgr.24344) 85 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:40.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:39 smithi098 bash[33599]: audit 2023-09-23T18:14:38.981518+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:40.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:39 smithi098 bash[33599]: audit 2023-09-23T18:14:38.983578+0000 mon.c (mon.2) 58 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:14:40.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:39 smithi098 bash[33599]: audit 2023-09-23T18:14:38.984921+0000 mon.c (mon.2) 59 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:14:40.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:39 smithi098 bash[33599]: audit 2023-09-23T18:14:38.992906+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:40.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:39 smithi098 bash[33599]: audit 2023-09-23T18:14:38.999357+0000 mon.c (mon.2) 60 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:14:42.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:41 smithi173 bash[27694]: cluster 2023-09-23T18:14:40.576971+0000 mgr.y (mgr.24344) 86 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:42.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:41 smithi098 bash[27349]: cluster 2023-09-23T18:14:40.576971+0000 mgr.y (mgr.24344) 86 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:42.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:41 smithi098 bash[33599]: cluster 2023-09-23T18:14:40.576971+0000 mgr.y (mgr.24344) 86 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:42.493 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:44.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:43 smithi098 bash[33599]: cluster 2023-09-23T18:14:42.577561+0000 mgr.y (mgr.24344) 87 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:44.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:43 smithi098 bash[27349]: cluster 2023-09-23T18:14:42.577561+0000 mgr.y (mgr.24344) 87 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:44.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:43 smithi173 bash[27694]: cluster 2023-09-23T18:14:42.577561+0000 mgr.y (mgr.24344) 87 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:45.723 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:14:45.725 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-23T18:14:45.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:14:45 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:14:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:14:46.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:45 smithi098 bash[27349]: cluster 2023-09-23T18:14:44.578308+0000 mgr.y (mgr.24344) 88 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:46.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:45 smithi098 bash[27349]: audit 2023-09-23T18:14:44.858618+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:46.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:45 smithi098 bash[27349]: audit 2023-09-23T18:14:44.860505+0000 mon.c (mon.2) 61 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:14:46.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:45 smithi098 bash[27349]: audit 2023-09-23T18:14:44.862243+0000 mon.c (mon.2) 62 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:14:46.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:45 smithi098 bash[27349]: audit 2023-09-23T18:14:44.869441+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:46.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:45 smithi098 bash[33599]: cluster 2023-09-23T18:14:44.578308+0000 mgr.y (mgr.24344) 88 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:46.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:45 smithi098 bash[33599]: audit 2023-09-23T18:14:44.858618+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:46.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:45 smithi098 bash[33599]: audit 2023-09-23T18:14:44.860505+0000 mon.c (mon.2) 61 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:14:46.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:45 smithi098 bash[33599]: audit 2023-09-23T18:14:44.862243+0000 mon.c (mon.2) 62 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:14:46.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:45 smithi098 bash[33599]: audit 2023-09-23T18:14:44.869441+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:46.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:45 smithi173 bash[27694]: cluster 2023-09-23T18:14:44.578308+0000 mgr.y (mgr.24344) 88 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:46.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:45 smithi173 bash[27694]: audit 2023-09-23T18:14:44.858618+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:46.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:45 smithi173 bash[27694]: audit 2023-09-23T18:14:44.860505+0000 mon.c (mon.2) 61 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:14:46.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:45 smithi173 bash[27694]: audit 2023-09-23T18:14:44.862243+0000 mon.c (mon.2) 62 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:14:46.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:45 smithi173 bash[27694]: audit 2023-09-23T18:14:44.869441+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:14:46.482 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":68,"stamp":"2023-09-23T18:14:44.577933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2380976,"kb_used_data":4208,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580240,"statfs":{"total":767960285184,"available":765522165760,"internally_reserved":0,"allocated":4308992,"data_stored":1762860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T18:13:15.286677+0000","last_change":"2023-09-23T18:13:13.809723+0000","last_active":"2023-09-23T18:13:15.286677+0000","last_peered":"2023-09-23T18:13:15.286677+0000","last_clean":"2023-09-23T18:13:15.286677+0000","last_became_active":"2023-09-23T18:13:13.809032+0000","last_became_peered":"2023-09-23T18:13:13.809032+0000","last_unstale":"2023-09-23T18:13:15.286677+0000","last_undegraded":"2023-09-23T18:13:15.286677+0000","last_fullsized":"2023-09-23T18:13:15.286677+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T18:13:15.280297+0000","last_change":"2023-09-23T18:13:13.929843+0000","last_active":"2023-09-23T18:13:15.280297+0000","last_peered":"2023-09-23T18:13:15.280297+0000","last_clean":"2023-09-23T18:13:15.280297+0000","last_became_active":"2023-09-23T18:13:13.929407+0000","last_became_peered":"2023-09-23T18:13:13.929407+0000","last_unstale":"2023-09-23T18:13:15.280297+0000","last_undegraded":"2023-09-23T18:13:15.280297+0000","last_fullsized":"2023-09-23T18:13:15.280297+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T18:13:15.286650+0000","last_change":"2023-09-23T18:13:13.935373+0000","last_active":"2023-09-23T18:13:15.286650+0000","last_peered":"2023-09-23T18:13:15.286650+0000","last_clean":"2023-09-23T18:13:15.286650+0000","last_became_active":"2023-09-23T18:13:13.934631+0000","last_became_peered":"2023-09-23T18:13:13.934631+0000","last_unstale":"2023-09-23T18:13:15.286650+0000","last_undegraded":"2023-09-23T18:13:15.286650+0000","last_fullsized":"2023-09-23T18:13:15.286650+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T18:13:15.286545+0000","last_change":"2023-09-23T18:13:13.933648+0000","last_active":"2023-09-23T18:13:15.286545+0000","last_peered":"2023-09-23T18:13:15.286545+0000","last_clean":"2023-09-23T18:13:15.286545+0000","last_became_active":"2023-09-23T18:13:13.932371+0000","last_became_peered":"2023-09-23T18:13:13.932371+0000","last_unstale":"2023-09-23T18:13:15.286545+0000","last_undegraded":"2023-09-23T18:13:15.286545+0000","last_fullsized":"2023-09-23T18:13:15.286545+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T18:13:15.286662+0000","last_change":"2023-09-23T18:12:11.949514+0000","last_active":"2023-09-23T18:13:15.286662+0000","last_peered":"2023-09-23T18:13:15.286662+0000","last_clean":"2023-09-23T18:13:15.286662+0000","last_became_active":"2023-09-23T18:12:11.948568+0000","last_became_peered":"2023-09-23T18:12:11.948568+0000","last_unstale":"2023-09-23T18:13:15.286662+0000","last_undegraded":"2023-09-23T18:13:15.286662+0000","last_fullsized":"2023-09-23T18:13:15.286662+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_clean_scrub_stamp":"2023-09-23T18:08:35.701374+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T18:13:15.286206+0000","last_change":"2023-09-23T18:13:13.810157+0000","last_active":"2023-09-23T18:13:15.286206+0000","last_peered":"2023-09-23T18:13:15.286206+0000","last_clean":"2023-09-23T18:13:15.286206+0000","last_became_active":"2023-09-23T18:13:13.809729+0000","last_became_peered":"2023-09-23T18:13:13.809729+0000","last_unstale":"2023-09-23T18:13:15.286206+0000","last_undegraded":"2023-09-23T18:13:15.286206+0000","last_fullsized":"2023-09-23T18:13:15.286206+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T18:13:14.815836+0000","last_change":"2023-09-23T18:13:13.927539+0000","last_active":"2023-09-23T18:13:14.815836+0000","last_peered":"2023-09-23T18:13:14.815836+0000","last_clean":"2023-09-23T18:13:14.815836+0000","last_became_active":"2023-09-23T18:13:13.927170+0000","last_became_peered":"2023-09-23T18:13:13.927170+0000","last_unstale":"2023-09-23T18:13:14.815836+0000","last_undegraded":"2023-09-23T18:13:14.815836+0000","last_fullsized":"2023-09-23T18:13:14.815836+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T18:13:15.286186+0000","last_change":"2023-09-23T18:13:13.814684+0000","last_active":"2023-09-23T18:13:15.286186+0000","last_peered":"2023-09-23T18:13:15.286186+0000","last_clean":"2023-09-23T18:13:15.286186+0000","last_became_active":"2023-09-23T18:13:13.814218+0000","last_became_peered":"2023-09-23T18:13:13.814218+0000","last_unstale":"2023-09-23T18:13:15.286186+0000","last_undegraded":"2023-09-23T18:13:15.286186+0000","last_fullsized":"2023-09-23T18:13:15.286186+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T18:13:15.280152+0000","last_change":"2023-09-23T18:13:13.928048+0000","last_active":"2023-09-23T18:13:15.280152+0000","last_peered":"2023-09-23T18:13:15.280152+0000","last_clean":"2023-09-23T18:13:15.280152+0000","last_became_active":"2023-09-23T18:13:13.927311+0000","last_became_peered":"2023-09-23T18:13:13.927311+0000","last_unstale":"2023-09-23T18:13:15.280152+0000","last_undegraded":"2023-09-23T18:13:15.280152+0000","last_fullsized":"2023-09-23T18:13:15.280152+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430241,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:14:11 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":0.88600000000000001,"15min":0.89000000000000001},"min":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.1819999999999999,"5min":1.1819999999999999,"15min":1.1819999999999999},"last":1.0920000000000001},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.92200000000000004,"15min":0.92300000000000004},"min":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.23,"5min":1.3180000000000001,"15min":1.3180000000000001},"last":0.75900000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:14:11 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":0.86599999999999999,"15min":0.876},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":1.083,"5min":1.268,"15min":1.268},"last":0.82599999999999996},{"interface":"front","average":{"1min":0.84699999999999998,"5min":0.82299999999999995,"15min":0.81899999999999995},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.1100000000000001,"5min":1.286,"15min":1.286},"last":0.95499999999999996}]},{"osd":2,"last update":"Sat Sep 23 18:14:11 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":0.94899999999999995,"15min":0.96099999999999997},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.194,"5min":1.304,"15min":1.304},"last":1.3660000000000001},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.89000000000000001,"15min":0.88900000000000001},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.258,"5min":1.258,"15min":1.258},"last":0.92400000000000004}]},{"osd":3,"last update":"Sat Sep 23 18:14:11 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":0.96399999999999997,"15min":0.96599999999999997},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.3360000000000001,"5min":1.3560000000000001,"15min":1.3560000000000001},"last":1.323},{"interface":"front","average":{"1min":0.92900000000000005,"5min":0.93700000000000006,"15min":0.93799999999999994},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.1479999999999999,"5min":1.2250000000000001,"15min":1.2250000000000001},"last":1.0089999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:14:11 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":0.89000000000000001,"15min":0.89200000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.5249999999999999,"5min":1.5249999999999999,"15min":1.5249999999999999},"last":1.0469999999999999},{"interface":"front","average":{"1min":0.91500000000000004,"5min":0.94699999999999995,"15min":0.95299999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.4099999999999999,"5min":1.427,"15min":1.427},"last":1.2609999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:14:11 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":0.91900000000000004,"15min":0.91500000000000004},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.3839999999999999,"5min":1.3839999999999999,"15min":1.3839999999999999},"last":1.169},{"interface":"front","average":{"1min":0.99199999999999999,"5min":0.96299999999999997,"15min":0.95799999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.3400000000000001,"5min":1.379,"15min":1.379},"last":1.405}]},{"osd":6,"last update":"Sat Sep 23 18:14:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":0.97699999999999998,"15min":0.97099999999999997},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.5229999999999999,"5min":1.5229999999999999,"15min":1.5229999999999999},"last":1.2170000000000001},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.1020000000000001,"15min":1.1080000000000001},"min":{"1min":0.54400000000000004,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.4590000000000001,"5min":1.554,"15min":1.554},"last":1.4330000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626473,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":0.85299999999999998,"15min":0.85699999999999998},"min":{"1min":0.58899999999999997,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.1910000000000001,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.93600000000000005},{"interface":"front","average":{"1min":0.80300000000000005,"5min":0.871,"15min":0.90500000000000003},"min":{"1min":0.60699999999999998,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.1839999999999999,"5min":1.268,"15min":1.268},"last":0.81799999999999995}]},{"osd":1,"last update":"Sat Sep 23 18:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":0.88800000000000001,"15min":0.90000000000000002},"min":{"1min":0.55700000000000005,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.2370000000000001,"5min":1.2370000000000001,"15min":1.2370000000000001},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.85899999999999999,"5min":0.89700000000000002,"15min":0.92400000000000004},"min":{"1min":0.68600000000000005,"5min":0.58999999999999997,"15min":0.58999999999999997},"max":{"1min":1.1399999999999999,"5min":1.726,"15min":1.726},"last":1.028}]},{"osd":2,"last update":"Sat Sep 23 18:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":0.871,"15min":0.88500000000000001},"min":{"1min":0.58199999999999996,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.256,"5min":1.256,"15min":1.256},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.90700000000000003,"15min":0.90100000000000002},"min":{"1min":0.66600000000000004,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.538,"5min":1.538,"15min":1.538},"last":0.95999999999999996}]},{"osd":3,"last update":"Sat Sep 23 18:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":0.91300000000000003,"15min":0.93600000000000005},"min":{"1min":0.69599999999999995,"5min":0.65800000000000003,"15min":0.65800000000000003},"max":{"1min":1.141,"5min":1.141,"15min":1.141},"last":0.89700000000000002},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.98799999999999999,"15min":1.01},"min":{"1min":0.71299999999999997,"5min":0.66800000000000004,"15min":0.66800000000000004},"max":{"1min":1.3049999999999999,"5min":1.7949999999999999,"15min":1.7949999999999999},"last":0.85599999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":0.84599999999999997,"15min":0.83899999999999997},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.4790000000000001,"5min":1.4790000000000001,"15min":1.4790000000000001},"last":1.048},{"interface":"front","average":{"1min":0.95899999999999996,"5min":0.90100000000000002,"15min":0.89900000000000002},"min":{"1min":0.57199999999999995,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":1.3029999999999999,"5min":1.3029999999999999,"15min":1.3029999999999999},"last":1.0069999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":0.91400000000000003,"15min":0.89000000000000001},"min":{"1min":0.51300000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.6160000000000001,"5min":1.6160000000000001,"15min":1.6160000000000001},"last":1.099},{"interface":"front","average":{"1min":1.0009999999999999,"5min":0.97799999999999998,"15min":0.98399999999999999},"min":{"1min":0.50900000000000001,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.4550000000000001,"5min":1.7649999999999999,"15min":1.7649999999999999},"last":0.80100000000000005}]},{"osd":7,"last update":"Sat Sep 23 18:14:10 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.028,"15min":1.024},"min":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.351,"5min":1.403,"15min":1.403},"last":0.98799999999999999},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.0580000000000001,"15min":1.054},"min":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"max":{"1min":1.5760000000000001,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":1.1399999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574933,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.66500000000000004,"15min":0.56299999999999994},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.34599999999999997},"max":{"1min":1.0269999999999999,"5min":1.641,"15min":1.641},"last":0.93100000000000005},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.69599999999999995,"15min":0.57599999999999996},"min":{"1min":0.443,"5min":0.34599999999999997,"15min":0.32500000000000001},"max":{"1min":0.97099999999999997,"5min":1.6859999999999999,"15min":1.6859999999999999},"last":0.80600000000000005}]},{"osd":2,"last update":"Sat Sep 23 18:13:43 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.72699999999999998,"15min":0.67300000000000004},"min":{"1min":0.48399999999999999,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.3009999999999999,"5min":1.49,"15min":1.49},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.77000000000000002,"5min":0.76900000000000002,"15min":0.67100000000000004},"min":{"1min":0.45700000000000002,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.0489999999999999,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":0.54700000000000004}]},{"osd":3,"last update":"Sat Sep 23 18:14:29 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.79500000000000004,"15min":0.79000000000000004},"min":{"1min":0.48499999999999999,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.0820000000000001,"5min":1.613,"15min":1.613},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.82399999999999995,"5min":0.84299999999999997,"15min":0.80700000000000005},"min":{"1min":0.51300000000000001,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":2.4940000000000002,"5min":2.4940000000000002,"15min":2.4940000000000002},"last":0.71299999999999997}]},{"osd":4,"last update":"Sat Sep 23 18:14:13 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.92000000000000004,"15min":0.879},"min":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"max":{"1min":1.262,"5min":1.8260000000000001,"15min":1.8260000000000001},"last":0.78700000000000003},{"interface":"front","average":{"1min":1.006,"5min":0.95599999999999996,"15min":0.89400000000000002},"min":{"1min":0.69599999999999995,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.2829999999999999,"5min":1.601,"15min":1.601},"last":0.84899999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:13:48 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":0.95899999999999996,"15min":0.95199999999999996},"min":{"1min":0.64200000000000002,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.2450000000000001,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":0.97799999999999998},{"interface":"front","average":{"1min":0.99399999999999999,"5min":0.98599999999999999,"15min":0.96199999999999997},"min":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.3799999999999999,"5min":1.73,"15min":1.73},"last":0.83299999999999996}]},{"osd":6,"last update":"Sat Sep 23 18:14:33 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.1339999999999999,"15min":1.1619999999999999},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.3220000000000001,"5min":1.51,"15min":1.51},"last":1.0049999999999999},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.081,"15min":1.1120000000000001},"min":{"1min":0.64300000000000002,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.2030000000000001,"5min":1.556,"15min":1.556},"last":1.0469999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:14:13 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.1160000000000001,"15min":1.127},"min":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.3180000000000001,"5min":1.7669999999999999,"15min":1.7669999999999999},"last":1.1100000000000001},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.1479999999999999,"15min":1.149},"min":{"1min":0.86799999999999999,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.329,"5min":1.53,"15min":1.53},"last":1.0700000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738462,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:14:02 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":0.67900000000000005,"15min":0.58699999999999997},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.373},"max":{"1min":0.98699999999999999,"5min":1.016,"15min":1.016},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.71299999999999997,"5min":0.70699999999999996,"15min":0.61099999999999999},"min":{"1min":0.51300000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.115,"5min":1.115,"15min":1.115},"last":0.82999999999999996}]},{"osd":2,"last update":"Sat Sep 23 18:13:53 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.72399999999999998,"15min":0.65600000000000003},"min":{"1min":0.372,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.1040000000000001,"5min":1.1040000000000001,"15min":1.1040000000000001},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.79200000000000004,"5min":0.74099999999999999,"15min":0.66300000000000003},"min":{"1min":0.58499999999999996,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.93700000000000006,"5min":1.012,"15min":1.012},"last":0.89100000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.746,"15min":0.72399999999999998},"min":{"1min":0.42799999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.1180000000000001,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.747,"5min":0.79000000000000004,"15min":0.74399999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.0169999999999999,"5min":1.0529999999999999,"15min":1.0529999999999999},"last":0.59999999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:14:10 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.90100000000000002,"15min":0.90400000000000003},"min":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"max":{"1min":1.2290000000000001,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":1.097},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.95799999999999996,"15min":0.93400000000000005},"min":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.2549999999999999,"5min":1.294,"15min":1.294},"last":0.92900000000000005}]},{"osd":5,"last update":"Sat Sep 23 18:13:53 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":1.0009999999999999,"15min":1.0069999999999999},"min":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.351,"5min":1.444,"15min":1.444},"last":0.99399999999999999},{"interface":"front","average":{"1min":1.038,"5min":0.995,"15min":0.98999999999999999},"min":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.2629999999999999,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.85999999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:14:31 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.004,"15min":0.98399999999999999},"min":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"max":{"1min":1.4550000000000001,"5min":1.4550000000000001,"15min":1.4550000000000001},"last":0.96999999999999997},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1.137,"15min":1.1519999999999999},"min":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"max":{"1min":1.482,"5min":1.5109999999999999,"15min":1.5109999999999999},"last":1.0680000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:14:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":1.1200000000000001,"15min":1.1259999999999999},"min":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.5249999999999999,"5min":1.5249999999999999,"15min":1.5249999999999999},"last":1.133},{"interface":"front","average":{"1min":1.119,"5min":1.056,"15min":1.046},"min":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"max":{"1min":1.5920000000000001,"5min":1.5920000000000001,"15min":1.5920000000000001},"last":1.034}]}]},{"osd":2,"up_from":18,"seq":77309411403,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:13:51 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":0.66000000000000003,"15min":0.60999999999999999},"min":{"1min":0.35999999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":2.23,"5min":2.23,"15min":2.23},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.81299999999999994,"5min":0.71399999999999997,"15min":0.60599999999999998},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.4420000000000002,"5min":2.4420000000000002,"15min":2.4420000000000002},"last":0.82499999999999996}]},{"osd":1,"last update":"Sat Sep 23 18:13:51 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":0.75700000000000001,"15min":0.66700000000000004},"min":{"1min":0.501,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.5149999999999999,"5min":1.5149999999999999,"15min":1.5149999999999999},"last":0.80000000000000004},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.85599999999999998,"15min":0.749},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.9399999999999999,"5min":1.9399999999999999,"15min":1.9399999999999999},"last":1.1479999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:14:32 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.77100000000000002,"15min":0.755},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":2.012,"5min":2.012,"15min":2.012},"last":0.99199999999999999},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.73699999999999999,"15min":0.73399999999999999},"min":{"1min":0.39300000000000002,"5min":0.373,"15min":0.373},"max":{"1min":1.171,"5min":1.522,"15min":1.522},"last":0.92900000000000005}]},{"osd":4,"last update":"Sat Sep 23 18:14:10 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.91400000000000003,"15min":0.88600000000000001},"min":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"max":{"1min":1.5629999999999999,"5min":1.5629999999999999,"15min":1.5629999999999999},"last":1.038},{"interface":"front","average":{"1min":0.92000000000000004,"5min":0.91800000000000004,"15min":0.89400000000000002},"min":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.391,"5min":1.6160000000000001,"15min":1.6160000000000001},"last":0.95699999999999996}]},{"osd":5,"last update":"Sat Sep 23 18:13:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.0309999999999999,"15min":1.0209999999999999},"min":{"1min":0.752,"5min":0.66400000000000003,"15min":0.66400000000000003},"max":{"1min":1.7589999999999999,"5min":1.7589999999999999,"15min":1.7589999999999999},"last":1.2110000000000001},{"interface":"front","average":{"1min":1,"5min":0.997,"15min":0.99299999999999999},"min":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.6950000000000001,"5min":1.6950000000000001,"15min":1.6950000000000001},"last":1.1779999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:14:32 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":1.0189999999999999,"15min":1.0369999999999999},"min":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":1.3,"5min":1.5209999999999999,"15min":1.5209999999999999},"last":1.099},{"interface":"front","average":{"1min":0.97299999999999998,"5min":1.111,"15min":1.149},"min":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.615,"5min":1.615,"15min":1.615},"last":1.355}]},{"osd":7,"last update":"Sat Sep 23 18:14:14 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":1.0469999999999999,"15min":1.056},"min":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.4590000000000001,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":1.2350000000000001},{"interface":"front","average":{"1min":1.054,"5min":1.147,"15min":1.163},"min":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.4330000000000001,"5min":1.6870000000000001,"15min":1.6870000000000001},"last":1.3260000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182466,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.64600000000000002,"15min":0.621},"min":{"1min":0.38600000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.80200000000000005,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.68500000000000005,"15min":0.70499999999999996},"min":{"1min":0.38600000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.97399999999999998,"5min":1.135,"15min":1.135},"last":0.622}]},{"osd":1,"last update":"Sat Sep 23 18:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.72899999999999998,"15min":0.74299999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.0660000000000001,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.94699999999999995},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.68400000000000005,"15min":0.67800000000000005},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.85799999999999998,"5min":1.2989999999999999,"15min":1.2989999999999999},"last":0.65800000000000003}]},{"osd":2,"last update":"Sat Sep 23 18:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.80200000000000005,"15min":0.81200000000000006},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.167,"5min":1.1819999999999999,"15min":1.1819999999999999},"last":0.86199999999999999},{"interface":"front","average":{"1min":0.70199999999999996,"5min":0.78100000000000003,"15min":0.77300000000000002},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.1910000000000001,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.91100000000000003}]},{"osd":4,"last update":"Sat Sep 23 18:14:09 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.91700000000000004,"15min":0.90700000000000003},"min":{"1min":0.68200000000000005,"5min":0.622,"15min":0.622},"max":{"1min":1.7410000000000001,"5min":1.7410000000000001,"15min":1.7410000000000001},"last":0.98099999999999998},{"interface":"front","average":{"1min":0.92900000000000005,"5min":0.93899999999999995,"15min":0.92100000000000004},"min":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.4159999999999999,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":1.1779999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:13:50 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":0.98199999999999998,"15min":0.97899999999999998},"min":{"1min":0.628,"5min":0.628,"15min":0.628},"max":{"1min":1.327,"5min":1.361,"15min":1.361},"last":1.101},{"interface":"front","average":{"1min":0.92200000000000004,"5min":0.96599999999999997,"15min":0.97099999999999997},"min":{"1min":0.71399999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"max":{"1min":1.1739999999999999,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":1.214}]},{"osd":6,"last update":"Sat Sep 23 18:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":1.05,"15min":1.0820000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.3440000000000001,"5min":1.512,"15min":1.512},"last":1.252},{"interface":"front","average":{"1min":0.88700000000000001,"5min":1.032,"15min":1.054},"min":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.1240000000000001,"5min":1.6699999999999999,"15min":1.6699999999999999},"last":1.292}]},{"osd":7,"last update":"Sat Sep 23 18:14:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.052,"15min":1.056},"min":{"1min":0.72799999999999998,"5min":0.67400000000000004,"15min":0.67400000000000004},"max":{"1min":1.595,"5min":1.595,"15min":1.595},"last":1.351},{"interface":"front","average":{"1min":1.0429999999999999,"5min":1.0669999999999999,"15min":1.071},"min":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.429,"5min":1.454,"15min":1.454},"last":1.319}]}]},{"osd":4,"up_from":30,"seq":128849018939,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:14:07 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.82299999999999995,"15min":0.83899999999999997},"min":{"1min":0.47699999999999998,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.0700000000000001,"5min":1.163,"15min":1.163},"last":0.81999999999999995},{"interface":"front","average":{"1min":0.82599999999999996,"5min":0.83999999999999997,"15min":0.82199999999999995},"min":{"1min":0.623,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.1299999999999999,"5min":1.177,"15min":1.177},"last":0.92400000000000004}]},{"osd":1,"last update":"Sat Sep 23 18:14:07 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.84599999999999997,"15min":0.85199999999999998},"min":{"1min":0.622,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.304,"5min":1.304,"15min":1.304},"last":0.94699999999999995},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.83899999999999997,"15min":0.83099999999999996},"min":{"1min":0.54500000000000004,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.254,"5min":1.254,"15min":1.254},"last":0.89400000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:14:07 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":0.85499999999999998,"15min":0.86599999999999999},"min":{"1min":0.65800000000000003,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":0.98999999999999999,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":1.0189999999999999},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.90300000000000002,"15min":0.89900000000000002},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.099,"5min":1.264,"15min":1.264},"last":0.75900000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:14:07 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.878,"15min":0.83799999999999997},"min":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":1.288,"5min":1.3169999999999999,"15min":1.3169999999999999},"last":0.83699999999999997},{"interface":"front","average":{"1min":0.92000000000000004,"5min":0.90400000000000003,"15min":0.92800000000000005},"min":{"1min":0.58899999999999997,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.3720000000000001,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":0.90400000000000003}]},{"osd":5,"last update":"Sat Sep 23 18:13:53 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.75900000000000001,"15min":0.73899999999999999},"min":{"1min":0.46000000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.1539999999999999,"5min":1.254,"15min":1.254},"last":1.0389999999999999},{"interface":"front","average":{"1min":0.82399999999999995,"5min":0.80600000000000005,"15min":0.80200000000000005},"min":{"1min":0.44500000000000001,"5min":0.443,"15min":0.443},"max":{"1min":1.1779999999999999,"5min":1.1899999999999999,"15min":1.1899999999999999},"last":0.95899999999999996}]},{"osd":6,"last update":"Sat Sep 23 18:14:32 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.93799999999999994,"15min":0.94099999999999995},"min":{"1min":0.55300000000000005,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.256,"5min":1.256,"15min":1.256},"last":0.998},{"interface":"front","average":{"1min":0.91500000000000004,"5min":0.89400000000000002,"15min":0.89000000000000001},"min":{"1min":0.63800000000000001,"5min":0.434,"15min":0.434},"max":{"1min":1.181,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":0.97499999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:14:14 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":0.94699999999999995,"15min":0.93700000000000006},"min":{"1min":0.58399999999999996,"5min":0.53400000000000003,"15min":0.53400000000000003},"max":{"1min":1.466,"5min":1.7010000000000001,"15min":1.7010000000000001},"last":0.85799999999999998},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.006,"15min":0.99399999999999999},"min":{"1min":0.78400000000000003,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.3580000000000001,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":1.0780000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822707,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297616,"kb_used_data":520,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447536,"statfs":{"total":95995035648,"available":95690276864,"internally_reserved":0,"allocated":532480,"data_stored":219885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":0.88400000000000001,"15min":0.89700000000000002},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.052,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":0.97999999999999998},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.86199999999999999,"15min":0.86499999999999999},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.2490000000000001,"5min":1.3480000000000001,"15min":1.3480000000000001},"last":0.70499999999999996}]},{"osd":1,"last update":"Sat Sep 23 18:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":0.88,"15min":0.88},"min":{"1min":0.48999999999999999,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.3,"5min":1.387,"15min":1.387},"last":0.626},{"interface":"front","average":{"1min":0.90500000000000003,"5min":0.90400000000000003,"15min":0.90500000000000003},"min":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.181,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":0.64800000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":0.876,"15min":0.879},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.194,"5min":1.194,"15min":1.194},"last":0.80700000000000005},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.94599999999999995,"15min":0.95999999999999996},"min":{"1min":0.63700000000000001,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.218,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":0.75700000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.90900000000000003,"15min":0.92800000000000005},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.125,"5min":1.333,"15min":1.333},"last":0.94199999999999995},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.97899999999999998,"15min":0.97799999999999998},"min":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.333,"5min":2.0259999999999998,"15min":2.0259999999999998},"last":1.044}]},{"osd":4,"last update":"Sat Sep 23 18:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":0.82799999999999996,"15min":0.81899999999999995},"min":{"1min":0.50700000000000001,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.3879999999999999,"5min":1.7509999999999999,"15min":1.7509999999999999},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.83399999999999996,"5min":0.82399999999999995,"15min":0.81999999999999995},"min":{"1min":0.54000000000000004,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.095,"5min":1.256,"15min":1.256},"last":1.024}]},{"osd":6,"last update":"Sat Sep 23 18:14:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":0.93100000000000005,"15min":0.91300000000000003},"min":{"1min":0.73299999999999998,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.325,"5min":1.8460000000000001,"15min":1.8460000000000001},"last":0.84499999999999997},{"interface":"front","average":{"1min":1.075,"5min":0.98799999999999999,"15min":0.97199999999999998},"min":{"1min":0.76200000000000001,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.4299999999999999,"5min":1.4299999999999999,"15min":1.4299999999999999},"last":1.0760000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:14:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":0.90800000000000003,"15min":0.88400000000000001},"min":{"1min":0.746,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.4139999999999999,"5min":1.4139999999999999,"15min":1.4139999999999999},"last":0.89700000000000002},{"interface":"front","average":{"1min":1.0700000000000001,"5min":0.96399999999999997,"15min":0.94599999999999995},"min":{"1min":0.70099999999999996,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.468,"5min":1.468,"15min":1.468},"last":0.91500000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:14:46.484 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:14:47.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:46 smithi098 bash[27349]: audit 2023-09-23T18:14:45.708845+0000 mgr.y (mgr.24344) 89 : audit [DBG] from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:14:47.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:46 smithi098 bash[33599]: audit 2023-09-23T18:14:45.708845+0000 mgr.y (mgr.24344) 89 : audit [DBG] from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:14:47.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:46 smithi173 bash[27694]: audit 2023-09-23T18:14:45.708845+0000 mgr.y (mgr.24344) 89 : audit [DBG] from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:14:48.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:47 smithi098 bash[27349]: cluster 2023-09-23T18:14:46.578809+0000 mgr.y (mgr.24344) 90 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:48.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:14:47 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:14:47] "GET /metrics HTTP/1.1" 200 155660 "" "Prometheus/2.33.4" 2023-09-23T18:14:48.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:47 smithi098 bash[33599]: cluster 2023-09-23T18:14:46.578809+0000 mgr.y (mgr.24344) 90 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:48.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:47 smithi173 bash[27694]: cluster 2023-09-23T18:14:46.578809+0000 mgr.y (mgr.24344) 90 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:49.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:49 smithi098 bash[27349]: cluster 2023-09-23T18:14:48.579474+0000 mgr.y (mgr.24344) 91 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:49.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:49 smithi098 bash[33599]: cluster 2023-09-23T18:14:48.579474+0000 mgr.y (mgr.24344) 91 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:49.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:49 smithi173 bash[27694]: cluster 2023-09-23T18:14:48.579474+0000 mgr.y (mgr.24344) 91 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:51.257 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:51.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:51 smithi098 bash[27349]: cluster 2023-09-23T18:14:50.580047+0000 mgr.y (mgr.24344) 92 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:51.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:51 smithi098 bash[33599]: cluster 2023-09-23T18:14:50.580047+0000 mgr.y (mgr.24344) 92 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:52.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:51 smithi173 bash[27694]: cluster 2023-09-23T18:14:50.580047+0000 mgr.y (mgr.24344) 92 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:53.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:53 smithi098 bash[33599]: cluster 2023-09-23T18:14:52.580590+0000 mgr.y (mgr.24344) 93 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:53.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:53 smithi098 bash[27349]: cluster 2023-09-23T18:14:52.580590+0000 mgr.y (mgr.24344) 93 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:54.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:53 smithi173 bash[27694]: cluster 2023-09-23T18:14:52.580590+0000 mgr.y (mgr.24344) 93 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:54.437 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:14:54.442 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-23T18:14:55.279 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":72,"stamp":"2023-09-23T18:14:52.580342+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2380976,"kb_used_data":4208,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580240,"statfs":{"total":767960285184,"available":765522165760,"internally_reserved":0,"allocated":4308992,"data_stored":1762860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T18:13:15.286677+0000","last_change":"2023-09-23T18:13:13.809723+0000","last_active":"2023-09-23T18:13:15.286677+0000","last_peered":"2023-09-23T18:13:15.286677+0000","last_clean":"2023-09-23T18:13:15.286677+0000","last_became_active":"2023-09-23T18:13:13.809032+0000","last_became_peered":"2023-09-23T18:13:13.809032+0000","last_unstale":"2023-09-23T18:13:15.286677+0000","last_undegraded":"2023-09-23T18:13:15.286677+0000","last_fullsized":"2023-09-23T18:13:15.286677+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T18:13:15.280297+0000","last_change":"2023-09-23T18:13:13.929843+0000","last_active":"2023-09-23T18:13:15.280297+0000","last_peered":"2023-09-23T18:13:15.280297+0000","last_clean":"2023-09-23T18:13:15.280297+0000","last_became_active":"2023-09-23T18:13:13.929407+0000","last_became_peered":"2023-09-23T18:13:13.929407+0000","last_unstale":"2023-09-23T18:13:15.280297+0000","last_undegraded":"2023-09-23T18:13:15.280297+0000","last_fullsized":"2023-09-23T18:13:15.280297+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T18:13:15.286650+0000","last_change":"2023-09-23T18:13:13.935373+0000","last_active":"2023-09-23T18:13:15.286650+0000","last_peered":"2023-09-23T18:13:15.286650+0000","last_clean":"2023-09-23T18:13:15.286650+0000","last_became_active":"2023-09-23T18:13:13.934631+0000","last_became_peered":"2023-09-23T18:13:13.934631+0000","last_unstale":"2023-09-23T18:13:15.286650+0000","last_undegraded":"2023-09-23T18:13:15.286650+0000","last_fullsized":"2023-09-23T18:13:15.286650+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T18:13:15.286545+0000","last_change":"2023-09-23T18:13:13.933648+0000","last_active":"2023-09-23T18:13:15.286545+0000","last_peered":"2023-09-23T18:13:15.286545+0000","last_clean":"2023-09-23T18:13:15.286545+0000","last_became_active":"2023-09-23T18:13:13.932371+0000","last_became_peered":"2023-09-23T18:13:13.932371+0000","last_unstale":"2023-09-23T18:13:15.286545+0000","last_undegraded":"2023-09-23T18:13:15.286545+0000","last_fullsized":"2023-09-23T18:13:15.286545+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T18:13:15.286662+0000","last_change":"2023-09-23T18:12:11.949514+0000","last_active":"2023-09-23T18:13:15.286662+0000","last_peered":"2023-09-23T18:13:15.286662+0000","last_clean":"2023-09-23T18:13:15.286662+0000","last_became_active":"2023-09-23T18:12:11.948568+0000","last_became_peered":"2023-09-23T18:12:11.948568+0000","last_unstale":"2023-09-23T18:13:15.286662+0000","last_undegraded":"2023-09-23T18:13:15.286662+0000","last_fullsized":"2023-09-23T18:13:15.286662+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_clean_scrub_stamp":"2023-09-23T18:08:35.701374+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T18:13:15.286206+0000","last_change":"2023-09-23T18:13:13.810157+0000","last_active":"2023-09-23T18:13:15.286206+0000","last_peered":"2023-09-23T18:13:15.286206+0000","last_clean":"2023-09-23T18:13:15.286206+0000","last_became_active":"2023-09-23T18:13:13.809729+0000","last_became_peered":"2023-09-23T18:13:13.809729+0000","last_unstale":"2023-09-23T18:13:15.286206+0000","last_undegraded":"2023-09-23T18:13:15.286206+0000","last_fullsized":"2023-09-23T18:13:15.286206+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T18:13:14.815836+0000","last_change":"2023-09-23T18:13:13.927539+0000","last_active":"2023-09-23T18:13:14.815836+0000","last_peered":"2023-09-23T18:13:14.815836+0000","last_clean":"2023-09-23T18:13:14.815836+0000","last_became_active":"2023-09-23T18:13:13.927170+0000","last_became_peered":"2023-09-23T18:13:13.927170+0000","last_unstale":"2023-09-23T18:13:14.815836+0000","last_undegraded":"2023-09-23T18:13:14.815836+0000","last_fullsized":"2023-09-23T18:13:14.815836+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T18:13:15.286186+0000","last_change":"2023-09-23T18:13:13.814684+0000","last_active":"2023-09-23T18:13:15.286186+0000","last_peered":"2023-09-23T18:13:15.286186+0000","last_clean":"2023-09-23T18:13:15.286186+0000","last_became_active":"2023-09-23T18:13:13.814218+0000","last_became_peered":"2023-09-23T18:13:13.814218+0000","last_unstale":"2023-09-23T18:13:15.286186+0000","last_undegraded":"2023-09-23T18:13:15.286186+0000","last_fullsized":"2023-09-23T18:13:15.286186+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T18:13:15.280152+0000","last_change":"2023-09-23T18:13:13.928048+0000","last_active":"2023-09-23T18:13:15.280152+0000","last_peered":"2023-09-23T18:13:15.280152+0000","last_clean":"2023-09-23T18:13:15.280152+0000","last_became_active":"2023-09-23T18:13:13.927311+0000","last_became_peered":"2023-09-23T18:13:13.927311+0000","last_unstale":"2023-09-23T18:13:15.280152+0000","last_undegraded":"2023-09-23T18:13:15.280152+0000","last_fullsized":"2023-09-23T18:13:15.280152+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430242,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:14:11 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":0.88600000000000001,"15min":0.89000000000000001},"min":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.1819999999999999,"5min":1.1819999999999999,"15min":1.1819999999999999},"last":0.83899999999999997},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.92200000000000004,"15min":0.92300000000000004},"min":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.23,"5min":1.3180000000000001,"15min":1.3180000000000001},"last":1.0720000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:14:11 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":0.86599999999999999,"15min":0.876},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":1.083,"5min":1.268,"15min":1.268},"last":0.86799999999999999},{"interface":"front","average":{"1min":0.84699999999999998,"5min":0.82299999999999995,"15min":0.81899999999999995},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.1100000000000001,"5min":1.286,"15min":1.286},"last":1.2130000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:14:11 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":0.94899999999999995,"15min":0.96099999999999997},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.194,"5min":1.304,"15min":1.304},"last":0.92500000000000004},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.89000000000000001,"15min":0.88900000000000001},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.258,"5min":1.258,"15min":1.258},"last":1.097}]},{"osd":3,"last update":"Sat Sep 23 18:14:11 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":0.96399999999999997,"15min":0.96599999999999997},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.3360000000000001,"5min":1.3560000000000001,"15min":1.3560000000000001},"last":0.99299999999999999},{"interface":"front","average":{"1min":0.92900000000000005,"5min":0.93700000000000006,"15min":0.93799999999999994},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.1479999999999999,"5min":1.2250000000000001,"15min":1.2250000000000001},"last":1.016}]},{"osd":4,"last update":"Sat Sep 23 18:14:11 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":0.89000000000000001,"15min":0.89200000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.5249999999999999,"5min":1.5249999999999999,"15min":1.5249999999999999},"last":1.2829999999999999},{"interface":"front","average":{"1min":0.91500000000000004,"5min":0.94699999999999995,"15min":0.95299999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.4099999999999999,"5min":1.427,"15min":1.427},"last":1.175}]},{"osd":5,"last update":"Sat Sep 23 18:14:11 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":0.91900000000000004,"15min":0.91500000000000004},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.3839999999999999,"5min":1.3839999999999999,"15min":1.3839999999999999},"last":0.95199999999999996},{"interface":"front","average":{"1min":0.99199999999999999,"5min":0.96299999999999997,"15min":0.95799999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.3400000000000001,"5min":1.379,"15min":1.379},"last":1.3620000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:14:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":0.97699999999999998,"15min":0.97099999999999997},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.5229999999999999,"5min":1.5229999999999999,"15min":1.5229999999999999},"last":1.323},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.1020000000000001,"15min":1.1080000000000001},"min":{"1min":0.54400000000000004,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.4590000000000001,"5min":1.554,"15min":1.554},"last":1.244}]}]},{"osd":6,"up_from":42,"seq":180388626474,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":0.85299999999999998,"15min":0.85699999999999998},"min":{"1min":0.58899999999999997,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.1910000000000001,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.81100000000000005},{"interface":"front","average":{"1min":0.80300000000000005,"5min":0.871,"15min":0.90500000000000003},"min":{"1min":0.60699999999999998,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.1839999999999999,"5min":1.268,"15min":1.268},"last":0.67100000000000004}]},{"osd":1,"last update":"Sat Sep 23 18:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":0.88800000000000001,"15min":0.90000000000000002},"min":{"1min":0.55700000000000005,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.2370000000000001,"5min":1.2370000000000001,"15min":1.2370000000000001},"last":1.008},{"interface":"front","average":{"1min":0.85899999999999999,"5min":0.89700000000000002,"15min":0.92400000000000004},"min":{"1min":0.68600000000000005,"5min":0.58999999999999997,"15min":0.58999999999999997},"max":{"1min":1.1399999999999999,"5min":1.726,"15min":1.726},"last":0.99199999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":0.871,"15min":0.88500000000000001},"min":{"1min":0.58199999999999996,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.256,"5min":1.256,"15min":1.256},"last":0.94699999999999995},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.90700000000000003,"15min":0.90100000000000002},"min":{"1min":0.66600000000000004,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.538,"5min":1.538,"15min":1.538},"last":1.054}]},{"osd":3,"last update":"Sat Sep 23 18:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":0.91300000000000003,"15min":0.93600000000000005},"min":{"1min":0.69599999999999995,"5min":0.65800000000000003,"15min":0.65800000000000003},"max":{"1min":1.141,"5min":1.141,"15min":1.141},"last":0.92300000000000004},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.98799999999999999,"15min":1.01},"min":{"1min":0.71299999999999997,"5min":0.66800000000000004,"15min":0.66800000000000004},"max":{"1min":1.3049999999999999,"5min":1.7949999999999999,"15min":1.7949999999999999},"last":0.89800000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":0.84599999999999997,"15min":0.83899999999999997},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.4790000000000001,"5min":1.4790000000000001,"15min":1.4790000000000001},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.95899999999999996,"5min":0.90100000000000002,"15min":0.89900000000000002},"min":{"1min":0.57199999999999995,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":1.3029999999999999,"5min":1.3029999999999999,"15min":1.3029999999999999},"last":1.091}]},{"osd":5,"last update":"Sat Sep 23 18:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":0.91400000000000003,"15min":0.89000000000000001},"min":{"1min":0.51300000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.6160000000000001,"5min":1.6160000000000001,"15min":1.6160000000000001},"last":0.95699999999999996},{"interface":"front","average":{"1min":1.0009999999999999,"5min":0.97799999999999998,"15min":0.98399999999999999},"min":{"1min":0.50900000000000001,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.4550000000000001,"5min":1.7649999999999999,"15min":1.7649999999999999},"last":1.1459999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:14:10 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.028,"15min":1.024},"min":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.351,"5min":1.403,"15min":1.403},"last":1.0309999999999999},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.0580000000000001,"15min":1.054},"min":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"max":{"1min":1.5760000000000001,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":0.86099999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574934,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.66500000000000004,"15min":0.56299999999999994},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.34599999999999997},"max":{"1min":1.0269999999999999,"5min":1.641,"15min":1.641},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.69599999999999995,"15min":0.57599999999999996},"min":{"1min":0.443,"5min":0.34599999999999997,"15min":0.32500000000000001},"max":{"1min":0.97099999999999997,"5min":1.6859999999999999,"15min":1.6859999999999999},"last":0.61699999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:14:44 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.71599999999999997,"15min":0.67000000000000004},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.79300000000000004,"5min":1.49,"15min":1.49},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.71299999999999997,"5min":0.78700000000000003,"15min":0.67700000000000005},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.40699999999999997},"max":{"1min":2.3290000000000002,"5min":2.3290000000000002,"15min":2.3290000000000002},"last":0.68500000000000005}]},{"osd":3,"last update":"Sat Sep 23 18:14:29 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.79500000000000004,"15min":0.79000000000000004},"min":{"1min":0.48499999999999999,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.0820000000000001,"5min":1.613,"15min":1.613},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.82399999999999995,"5min":0.84299999999999997,"15min":0.80700000000000005},"min":{"1min":0.51300000000000001,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":2.4940000000000002,"5min":2.4940000000000002,"15min":2.4940000000000002},"last":0.73199999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:14:13 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.92000000000000004,"15min":0.879},"min":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"max":{"1min":1.262,"5min":1.8260000000000001,"15min":1.8260000000000001},"last":0.86399999999999999},{"interface":"front","average":{"1min":1.006,"5min":0.95599999999999996,"15min":0.89400000000000002},"min":{"1min":0.69599999999999995,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.2829999999999999,"5min":1.601,"15min":1.601},"last":0.94099999999999995}]},{"osd":5,"last update":"Sat Sep 23 18:13:48 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":0.95899999999999996,"15min":0.95199999999999996},"min":{"1min":0.64200000000000002,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.2450000000000001,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":0.88100000000000001},{"interface":"front","average":{"1min":0.99399999999999999,"5min":0.98599999999999999,"15min":0.96199999999999997},"min":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.3799999999999999,"5min":1.73,"15min":1.73},"last":0.82399999999999995}]},{"osd":6,"last update":"Sat Sep 23 18:14:33 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.1339999999999999,"15min":1.1619999999999999},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.3220000000000001,"5min":1.51,"15min":1.51},"last":1.0780000000000001},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.081,"15min":1.1120000000000001},"min":{"1min":0.64300000000000002,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.2030000000000001,"5min":1.556,"15min":1.556},"last":0.96399999999999997}]},{"osd":7,"last update":"Sat Sep 23 18:14:13 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.1160000000000001,"15min":1.127},"min":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.3180000000000001,"5min":1.7669999999999999,"15min":1.7669999999999999},"last":1.038},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.1479999999999999,"15min":1.149},"min":{"1min":0.86799999999999999,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.329,"5min":1.53,"15min":1.53},"last":1.02}]}]},{"osd":0,"up_from":8,"seq":34359738463,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:14:02 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":0.67900000000000005,"15min":0.58699999999999997},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.373},"max":{"1min":0.98699999999999999,"5min":1.016,"15min":1.016},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.71299999999999997,"5min":0.70699999999999996,"15min":0.61099999999999999},"min":{"1min":0.51300000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.115,"5min":1.115,"15min":1.115},"last":0.59199999999999997}]},{"osd":2,"last update":"Sat Sep 23 18:13:53 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.72399999999999998,"15min":0.65600000000000003},"min":{"1min":0.372,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.1040000000000001,"5min":1.1040000000000001,"15min":1.1040000000000001},"last":0.622},{"interface":"front","average":{"1min":0.79200000000000004,"5min":0.74099999999999999,"15min":0.66300000000000003},"min":{"1min":0.58499999999999996,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.93700000000000006,"5min":1.012,"15min":1.012},"last":0.83099999999999996}]},{"osd":3,"last update":"Sat Sep 23 18:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.746,"15min":0.72399999999999998},"min":{"1min":0.42799999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.1180000000000001,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":0.91300000000000003},{"interface":"front","average":{"1min":0.747,"5min":0.79000000000000004,"15min":0.74399999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.0169999999999999,"5min":1.0529999999999999,"15min":1.0529999999999999},"last":0.73999999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:14:10 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.90100000000000002,"15min":0.90400000000000003},"min":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"max":{"1min":1.2290000000000001,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":0.89600000000000002},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.95799999999999996,"15min":0.93400000000000005},"min":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.2549999999999999,"5min":1.294,"15min":1.294},"last":1.129}]},{"osd":5,"last update":"Sat Sep 23 18:13:53 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":1.0009999999999999,"15min":1.0069999999999999},"min":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.351,"5min":1.444,"15min":1.444},"last":1.246},{"interface":"front","average":{"1min":1.038,"5min":0.995,"15min":0.98999999999999999},"min":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.2629999999999999,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.80000000000000004}]},{"osd":6,"last update":"Sat Sep 23 18:14:31 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.004,"15min":0.98399999999999999},"min":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"max":{"1min":1.4550000000000001,"5min":1.4550000000000001,"15min":1.4550000000000001},"last":1.294},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1.137,"15min":1.1519999999999999},"min":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"max":{"1min":1.482,"5min":1.5109999999999999,"15min":1.5109999999999999},"last":1.3959999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:14:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":1.1200000000000001,"15min":1.1259999999999999},"min":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.5249999999999999,"5min":1.5249999999999999,"15min":1.5249999999999999},"last":1.323},{"interface":"front","average":{"1min":1.119,"5min":1.056,"15min":1.046},"min":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"max":{"1min":1.5920000000000001,"5min":1.5920000000000001,"15min":1.5920000000000001},"last":1.1679999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411404,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:13:51 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":0.66000000000000003,"15min":0.60999999999999999},"min":{"1min":0.35999999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":2.23,"5min":2.23,"15min":2.23},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.81299999999999994,"5min":0.71399999999999997,"15min":0.60599999999999998},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.4420000000000002,"5min":2.4420000000000002,"15min":2.4420000000000002},"last":0.60499999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:13:51 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":0.75700000000000001,"15min":0.66700000000000004},"min":{"1min":0.501,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.5149999999999999,"5min":1.5149999999999999,"15min":1.5149999999999999},"last":0.627},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.85599999999999998,"15min":0.749},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.9399999999999999,"5min":1.9399999999999999,"15min":1.9399999999999999},"last":0.55000000000000004}]},{"osd":3,"last update":"Sat Sep 23 18:14:32 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.77100000000000002,"15min":0.755},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":2.012,"5min":2.012,"15min":2.012},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.73699999999999999,"15min":0.73399999999999999},"min":{"1min":0.39300000000000002,"5min":0.373,"15min":0.373},"max":{"1min":1.171,"5min":1.522,"15min":1.522},"last":0.68000000000000005}]},{"osd":4,"last update":"Sat Sep 23 18:14:10 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.91400000000000003,"15min":0.88600000000000001},"min":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"max":{"1min":1.5629999999999999,"5min":1.5629999999999999,"15min":1.5629999999999999},"last":0.84199999999999997},{"interface":"front","average":{"1min":0.92000000000000004,"5min":0.91800000000000004,"15min":0.89400000000000002},"min":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.391,"5min":1.6160000000000001,"15min":1.6160000000000001},"last":0.86499999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:13:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.0309999999999999,"15min":1.0209999999999999},"min":{"1min":0.752,"5min":0.66400000000000003,"15min":0.66400000000000003},"max":{"1min":1.7589999999999999,"5min":1.7589999999999999,"15min":1.7589999999999999},"last":1.071},{"interface":"front","average":{"1min":1,"5min":0.997,"15min":0.99299999999999999},"min":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.6950000000000001,"5min":1.6950000000000001,"15min":1.6950000000000001},"last":0.90600000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:14:32 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":1.0189999999999999,"15min":1.0369999999999999},"min":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":1.3,"5min":1.5209999999999999,"15min":1.5209999999999999},"last":0.78400000000000003},{"interface":"front","average":{"1min":0.97299999999999998,"5min":1.111,"15min":1.149},"min":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.615,"5min":1.615,"15min":1.615},"last":1.0269999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:14:14 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":1.0469999999999999,"15min":1.056},"min":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.4590000000000001,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":1.0409999999999999},{"interface":"front","average":{"1min":1.054,"5min":1.147,"15min":1.163},"min":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.4330000000000001,"5min":1.6870000000000001,"15min":1.6870000000000001},"last":0.92400000000000004}]}]},{"osd":3,"up_from":25,"seq":107374182467,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.64600000000000002,"15min":0.621},"min":{"1min":0.38600000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.80200000000000005,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.77500000000000002},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.68500000000000005,"15min":0.70499999999999996},"min":{"1min":0.38600000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.97399999999999998,"5min":1.135,"15min":1.135},"last":0.81000000000000005}]},{"osd":1,"last update":"Sat Sep 23 18:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.72899999999999998,"15min":0.74299999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.0660000000000001,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.91900000000000004},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.68400000000000005,"15min":0.67800000000000005},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.85799999999999998,"5min":1.2989999999999999,"15min":1.2989999999999999},"last":0.85099999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.80200000000000005,"15min":0.81200000000000006},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.167,"5min":1.1819999999999999,"15min":1.1819999999999999},"last":0.97499999999999998},{"interface":"front","average":{"1min":0.70199999999999996,"5min":0.78100000000000003,"15min":0.77300000000000002},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.1910000000000001,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":1.026}]},{"osd":4,"last update":"Sat Sep 23 18:14:09 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.91700000000000004,"15min":0.90700000000000003},"min":{"1min":0.68200000000000005,"5min":0.622,"15min":0.622},"max":{"1min":1.7410000000000001,"5min":1.7410000000000001,"15min":1.7410000000000001},"last":1.1819999999999999},{"interface":"front","average":{"1min":0.92900000000000005,"5min":0.93899999999999995,"15min":0.92100000000000004},"min":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.4159999999999999,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":1.0660000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:13:50 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":0.98199999999999998,"15min":0.97899999999999998},"min":{"1min":0.628,"5min":0.628,"15min":0.628},"max":{"1min":1.327,"5min":1.361,"15min":1.361},"last":1.1180000000000001},{"interface":"front","average":{"1min":0.92200000000000004,"5min":0.96599999999999997,"15min":0.97099999999999997},"min":{"1min":0.71399999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"max":{"1min":1.1739999999999999,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":1.333}]},{"osd":6,"last update":"Sat Sep 23 18:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":1.05,"15min":1.0820000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.3440000000000001,"5min":1.512,"15min":1.512},"last":1.167},{"interface":"front","average":{"1min":0.88700000000000001,"5min":1.032,"15min":1.054},"min":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.1240000000000001,"5min":1.6699999999999999,"15min":1.6699999999999999},"last":1.304}]},{"osd":7,"last update":"Sat Sep 23 18:14:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.052,"15min":1.056},"min":{"1min":0.72799999999999998,"5min":0.67400000000000004,"15min":0.67400000000000004},"max":{"1min":1.595,"5min":1.595,"15min":1.595},"last":1.389},{"interface":"front","average":{"1min":1.0429999999999999,"5min":1.0669999999999999,"15min":1.071},"min":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.429,"5min":1.454,"15min":1.454},"last":1.2130000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018940,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:14:07 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.82299999999999995,"15min":0.83899999999999997},"min":{"1min":0.47699999999999998,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.0700000000000001,"5min":1.163,"15min":1.163},"last":0.79800000000000004},{"interface":"front","average":{"1min":0.82599999999999996,"5min":0.83999999999999997,"15min":0.82199999999999995},"min":{"1min":0.623,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.1299999999999999,"5min":1.177,"15min":1.177},"last":1.026}]},{"osd":1,"last update":"Sat Sep 23 18:14:07 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.84599999999999997,"15min":0.85199999999999998},"min":{"1min":0.622,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.304,"5min":1.304,"15min":1.304},"last":0.82799999999999996},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.83899999999999997,"15min":0.83099999999999996},"min":{"1min":0.54500000000000004,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.254,"5min":1.254,"15min":1.254},"last":0.89500000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:14:07 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":0.85499999999999998,"15min":0.86599999999999999},"min":{"1min":0.65800000000000003,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":0.98999999999999999,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":1.0569999999999999},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.90300000000000002,"15min":0.89900000000000002},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.099,"5min":1.264,"15min":1.264},"last":1.1919999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:14:07 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.878,"15min":0.83799999999999997},"min":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":1.288,"5min":1.3169999999999999,"15min":1.3169999999999999},"last":0.95199999999999996},{"interface":"front","average":{"1min":0.92000000000000004,"5min":0.90400000000000003,"15min":0.92800000000000005},"min":{"1min":0.58899999999999997,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.3720000000000001,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":1.1639999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:13:53 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.75900000000000001,"15min":0.73899999999999999},"min":{"1min":0.46000000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.1539999999999999,"5min":1.254,"15min":1.254},"last":0.73999999999999999},{"interface":"front","average":{"1min":0.82399999999999995,"5min":0.80600000000000005,"15min":0.80200000000000005},"min":{"1min":0.44500000000000001,"5min":0.443,"15min":0.443},"max":{"1min":1.1779999999999999,"5min":1.1899999999999999,"15min":1.1899999999999999},"last":0.97199999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:14:32 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.93799999999999994,"15min":0.94099999999999995},"min":{"1min":0.55300000000000005,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.256,"5min":1.256,"15min":1.256},"last":0.85599999999999998},{"interface":"front","average":{"1min":0.91500000000000004,"5min":0.89400000000000002,"15min":0.89000000000000001},"min":{"1min":0.63800000000000001,"5min":0.434,"15min":0.434},"max":{"1min":1.181,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":0.92100000000000004}]},{"osd":7,"last update":"Sat Sep 23 18:14:14 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":0.94699999999999995,"15min":0.93700000000000006},"min":{"1min":0.58399999999999996,"5min":0.53400000000000003,"15min":0.53400000000000003},"max":{"1min":1.466,"5min":1.7010000000000001,"15min":1.7010000000000001},"last":1.083},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.006,"15min":0.99399999999999999},"min":{"1min":0.78400000000000003,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.3580000000000001,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":1.1100000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822708,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297616,"kb_used_data":520,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447536,"statfs":{"total":95995035648,"available":95690276864,"internally_reserved":0,"allocated":532480,"data_stored":219885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":0.88400000000000001,"15min":0.89700000000000002},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.052,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":0.85399999999999998},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.86199999999999999,"15min":0.86499999999999999},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.2490000000000001,"5min":1.3480000000000001,"15min":1.3480000000000001},"last":0.61299999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":0.88,"15min":0.88},"min":{"1min":0.48999999999999999,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.3,"5min":1.387,"15min":1.387},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.90500000000000003,"5min":0.90400000000000003,"15min":0.90500000000000003},"min":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.181,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":0.77400000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":0.876,"15min":0.879},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.194,"5min":1.194,"15min":1.194},"last":1.165},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.94599999999999995,"15min":0.95999999999999996},"min":{"1min":0.63700000000000001,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.218,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":0.97099999999999997}]},{"osd":3,"last update":"Sat Sep 23 18:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.90900000000000003,"15min":0.92800000000000005},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.125,"5min":1.333,"15min":1.333},"last":1.079},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.97899999999999998,"15min":0.97799999999999998},"min":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.333,"5min":2.0259999999999998,"15min":2.0259999999999998},"last":1.0629999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":0.82799999999999996,"15min":0.81899999999999995},"min":{"1min":0.50700000000000001,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.3879999999999999,"5min":1.7509999999999999,"15min":1.7509999999999999},"last":0.90100000000000002},{"interface":"front","average":{"1min":0.83399999999999996,"5min":0.82399999999999995,"15min":0.81999999999999995},"min":{"1min":0.54000000000000004,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.095,"5min":1.256,"15min":1.256},"last":1.234}]},{"osd":6,"last update":"Sat Sep 23 18:14:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":0.93100000000000005,"15min":0.91300000000000003},"min":{"1min":0.73299999999999998,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.325,"5min":1.8460000000000001,"15min":1.8460000000000001},"last":0.995},{"interface":"front","average":{"1min":1.075,"5min":0.98799999999999999,"15min":0.97199999999999998},"min":{"1min":0.76200000000000001,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.4299999999999999,"5min":1.4299999999999999,"15min":1.4299999999999999},"last":1.298}]},{"osd":7,"last update":"Sat Sep 23 18:14:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":0.90800000000000003,"15min":0.88400000000000001},"min":{"1min":0.746,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.4139999999999999,"5min":1.4139999999999999,"15min":1.4139999999999999},"last":1.022},{"interface":"front","average":{"1min":1.0700000000000001,"5min":0.96399999999999997,"15min":0.94599999999999995},"min":{"1min":0.70099999999999996,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.468,"5min":1.468,"15min":1.468},"last":1.044}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:14:55.282 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T18:14:55.282 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T18:14:55.282 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T18:14:55.282 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T18:14:55.653 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:14:55 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:14:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:14:55.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:55 smithi098 bash[27349]: audit 2023-09-23T18:14:54.421832+0000 mgr.y (mgr.24344) 94 : audit [DBG] from='client.14625 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:14:55.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:55 smithi098 bash[27349]: cluster 2023-09-23T18:14:54.581275+0000 mgr.y (mgr.24344) 95 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:55.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:55 smithi098 bash[33599]: audit 2023-09-23T18:14:54.421832+0000 mgr.y (mgr.24344) 94 : audit [DBG] from='client.14625 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:14:55.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:55 smithi098 bash[33599]: cluster 2023-09-23T18:14:54.581275+0000 mgr.y (mgr.24344) 95 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:56.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:55 smithi173 bash[27694]: audit 2023-09-23T18:14:54.421832+0000 mgr.y (mgr.24344) 94 : audit [DBG] from='client.14625 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:14:56.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:55 smithi173 bash[27694]: cluster 2023-09-23T18:14:54.581275+0000 mgr.y (mgr.24344) 95 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:57.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:57 smithi098 bash[27349]: cluster 2023-09-23T18:14:56.581869+0000 mgr.y (mgr.24344) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:57.946 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:14:57 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:14:57] "GET /metrics HTTP/1.1" 200 155660 "" "Prometheus/2.33.4" 2023-09-23T18:14:57.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:57 smithi098 bash[33599]: cluster 2023-09-23T18:14:56.581869+0000 mgr.y (mgr.24344) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:58.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:57 smithi173 bash[27694]: cluster 2023-09-23T18:14:56.581869+0000 mgr.y (mgr.24344) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:59.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:14:59 smithi098 bash[27349]: cluster 2023-09-23T18:14:58.582539+0000 mgr.y (mgr.24344) 97 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:59.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:14:59 smithi098 bash[33599]: cluster 2023-09-23T18:14:58.582539+0000 mgr.y (mgr.24344) 97 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:14:59.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:14:59 smithi173 bash[27694]: cluster 2023-09-23T18:14:58.582539+0000 mgr.y (mgr.24344) 97 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:00.065 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:01.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:01 smithi173 bash[27694]: cluster 2023-09-23T18:15:00.582886+0000 mgr.y (mgr.24344) 98 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:01.949 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:01 smithi098 bash[27349]: cluster 2023-09-23T18:15:00.582886+0000 mgr.y (mgr.24344) 98 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:01.950 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:01 smithi098 bash[33599]: cluster 2023-09-23T18:15:00.582886+0000 mgr.y (mgr.24344) 98 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:03.339 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:15:03.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:03 smithi098 bash[27349]: cluster 2023-09-23T18:15:02.583415+0000 mgr.y (mgr.24344) 99 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:03.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:03 smithi098 bash[27349]: audit 2023-09-23T18:15:03.333127+0000 mon.c (mon.2) 63 : audit [DBG] from='client.? 172.21.15.98:0/4107715714' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T18:15:03.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:03 smithi098 bash[33599]: cluster 2023-09-23T18:15:02.583415+0000 mgr.y (mgr.24344) 99 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:03.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:03 smithi098 bash[33599]: audit 2023-09-23T18:15:03.333127+0000 mon.c (mon.2) 63 : audit [DBG] from='client.? 172.21.15.98:0/4107715714' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T18:15:04.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:03 smithi173 bash[27694]: cluster 2023-09-23T18:15:02.583415+0000 mgr.y (mgr.24344) 99 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:04.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:03 smithi173 bash[27694]: audit 2023-09-23T18:15:03.333127+0000 mon.c (mon.2) 63 : audit [DBG] from='client.? 172.21.15.98:0/4107715714' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T18:15:04.078 INFO:teuthology.orchestra.run.smithi098.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T18:15:04.079 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T18:15:04.079 INFO:tasks.cephadm:Setup complete, yielding 2023-09-23T18:15:04.079 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-09-23T18:15:04.093 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi098.front.sepia.ceph.com 2023-09-23T18:15:04.093 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-09-23T18:15:05.654 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:15:05 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:15:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:15:05.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:05 smithi098 bash[27349]: cluster 2023-09-23T18:15:04.583875+0000 mgr.y (mgr.24344) 100 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:05.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:05 smithi098 bash[33599]: cluster 2023-09-23T18:15:04.583875+0000 mgr.y (mgr.24344) 100 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:06.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:05 smithi173 bash[27694]: cluster 2023-09-23T18:15:04.583875+0000 mgr.y (mgr.24344) 100 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:08.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:15:07 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:15:07] "GET /metrics HTTP/1.1" 200 155679 "" "Prometheus/2.33.4" 2023-09-23T18:15:08.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:07 smithi098 bash[27349]: cluster 2023-09-23T18:15:06.584487+0000 mgr.y (mgr.24344) 101 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:08.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:07 smithi098 bash[27349]: audit 2023-09-23T18:15:07.373617+0000 mon.c (mon.2) 64 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T18:15:08.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:07 smithi098 bash[33599]: cluster 2023-09-23T18:15:06.584487+0000 mgr.y (mgr.24344) 101 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:08.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:07 smithi098 bash[33599]: audit 2023-09-23T18:15:07.373617+0000 mon.c (mon.2) 64 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T18:15:08.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:07 smithi173 bash[27694]: cluster 2023-09-23T18:15:06.584487+0000 mgr.y (mgr.24344) 101 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:08.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:07 smithi173 bash[27694]: audit 2023-09-23T18:15:07.373617+0000 mon.c (mon.2) 64 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T18:15:09.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:08 smithi098 bash[33599]: audit 2023-09-23T18:15:07.372873+0000 mgr.y (mgr.24344) 102 : audit [DBG] from='client.14637 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:15:09.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:08 smithi098 bash[33599]: audit 2023-09-23T18:15:07.374234+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T18:15:09.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:08 smithi098 bash[33599]: audit 2023-09-23T18:15:07.879912+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T18:15:09.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:08 smithi098 bash[33599]: cluster 2023-09-23T18:15:07.880040+0000 mon.a (mon.0) 618 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-09-23T18:15:09.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:08 smithi098 bash[33599]: audit 2023-09-23T18:15:07.884161+0000 mon.c (mon.2) 65 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T18:15:09.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:08 smithi098 bash[33599]: audit 2023-09-23T18:15:07.884852+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T18:15:09.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:08 smithi098 bash[27349]: audit 2023-09-23T18:15:07.372873+0000 mgr.y (mgr.24344) 102 : audit [DBG] from='client.14637 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:15:09.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:08 smithi098 bash[27349]: audit 2023-09-23T18:15:07.374234+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T18:15:09.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:08 smithi098 bash[27349]: audit 2023-09-23T18:15:07.879912+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T18:15:09.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:08 smithi098 bash[27349]: cluster 2023-09-23T18:15:07.880040+0000 mon.a (mon.0) 618 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-09-23T18:15:09.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:08 smithi098 bash[27349]: audit 2023-09-23T18:15:07.884161+0000 mon.c (mon.2) 65 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T18:15:09.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:08 smithi098 bash[27349]: audit 2023-09-23T18:15:07.884852+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T18:15:09.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:08 smithi098 bash[27349]: debug 2023-09-23T18:15:08.884+0000 7fe068ae8700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T18:15:09.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:08 smithi173 bash[27694]: audit 2023-09-23T18:15:07.372873+0000 mgr.y (mgr.24344) 102 : audit [DBG] from='client.14637 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:15:09.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:08 smithi173 bash[27694]: audit 2023-09-23T18:15:07.374234+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T18:15:09.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:08 smithi173 bash[27694]: audit 2023-09-23T18:15:07.879912+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T18:15:09.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:08 smithi173 bash[27694]: cluster 2023-09-23T18:15:07.880040+0000 mon.a (mon.0) 618 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-09-23T18:15:09.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:08 smithi173 bash[27694]: audit 2023-09-23T18:15:07.884161+0000 mon.c (mon.2) 65 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T18:15:09.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:08 smithi173 bash[27694]: audit 2023-09-23T18:15:07.884852+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T18:15:09.835 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-09-23T18:15:10.020 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[33599]: cluster 2023-09-23T18:15:08.585275+0000 mgr.y (mgr.24344) 103 : cluster [DBG] pgmap v81: 41 pgs: 32 creating+peering, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:10.020 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[33599]: audit 2023-09-23T18:15:08.878338+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T18:15:10.020 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[33599]: cluster 2023-09-23T18:15:08.878482+0000 mon.a (mon.0) 621 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-09-23T18:15:10.020 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[33599]: audit 2023-09-23T18:15:08.888682+0000 mon.c (mon.2) 66 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T18:15:10.020 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[33599]: audit 2023-09-23T18:15:08.889246+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T18:15:10.021 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[33599]: cluster 2023-09-23T18:15:08.890178+0000 mon.a (mon.0) 623 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T18:15:10.021 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[33599]: cluster 2023-09-23T18:15:08.890215+0000 mon.a (mon.0) 624 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T18:15:10.021 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[33599]: cluster 2023-09-23T18:15:08.899312+0000 mon.a (mon.0) 625 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-09-23T18:15:10.021 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[33599]: audit 2023-09-23T18:15:08.899979+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T18:15:10.021 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[33599]: cluster 2023-09-23T18:15:08.900085+0000 mon.a (mon.0) 627 : cluster [DBG] fsmap foo:0 2023-09-23T18:15:10.021 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[33599]: cephadm 2023-09-23T18:15:08.906624+0000 mgr.y (mgr.24344) 104 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-09-23T18:15:10.021 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[33599]: audit 2023-09-23T18:15:08.918560+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:10.021 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[33599]: audit 2023-09-23T18:15:08.952348+0000 mon.c (mon.2) 67 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:15:10.021 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[33599]: audit 2023-09-23T18:15:09.207797+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:10.021 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[33599]: audit 2023-09-23T18:15:09.449372+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:15:10.021 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[33599]: audit 2023-09-23T18:15:09.450378+0000 mon.c (mon.2) 69 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:15:10.021 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[33599]: audit 2023-09-23T18:15:09.456503+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:10.021 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[33599]: audit 2023-09-23T18:15:09.466979+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:10.022 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[33599]: audit 2023-09-23T18:15:09.468140+0000 mon.c (mon.2) 70 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi098.lowjqo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T18:15:10.022 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[33599]: audit 2023-09-23T18:15:09.468498+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi098.lowjqo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T18:15:10.022 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[33599]: audit 2023-09-23T18:15:09.473126+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi098.lowjqo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T18:15:10.022 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[33599]: audit 2023-09-23T18:15:09.474098+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:15:10.022 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[27349]: cluster 2023-09-23T18:15:08.585275+0000 mgr.y (mgr.24344) 103 : cluster [DBG] pgmap v81: 41 pgs: 32 creating+peering, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:10.022 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[27349]: audit 2023-09-23T18:15:08.878338+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T18:15:10.022 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[27349]: cluster 2023-09-23T18:15:08.878482+0000 mon.a (mon.0) 621 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-09-23T18:15:10.022 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[27349]: audit 2023-09-23T18:15:08.888682+0000 mon.c (mon.2) 66 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T18:15:10.023 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[27349]: audit 2023-09-23T18:15:08.889246+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T18:15:10.023 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[27349]: cluster 2023-09-23T18:15:08.890178+0000 mon.a (mon.0) 623 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T18:15:10.023 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[27349]: cluster 2023-09-23T18:15:08.890215+0000 mon.a (mon.0) 624 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T18:15:10.023 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[27349]: cluster 2023-09-23T18:15:08.899312+0000 mon.a (mon.0) 625 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-09-23T18:15:10.023 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[27349]: audit 2023-09-23T18:15:08.899979+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T18:15:10.023 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[27349]: cluster 2023-09-23T18:15:08.900085+0000 mon.a (mon.0) 627 : cluster [DBG] fsmap foo:0 2023-09-23T18:15:10.023 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[27349]: cephadm 2023-09-23T18:15:08.906624+0000 mgr.y (mgr.24344) 104 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-09-23T18:15:10.023 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[27349]: audit 2023-09-23T18:15:08.918560+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:10.023 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[27349]: audit 2023-09-23T18:15:08.952348+0000 mon.c (mon.2) 67 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:15:10.023 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[27349]: audit 2023-09-23T18:15:09.207797+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:10.023 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[27349]: audit 2023-09-23T18:15:09.449372+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:15:10.023 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[27349]: audit 2023-09-23T18:15:09.450378+0000 mon.c (mon.2) 69 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:15:10.024 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[27349]: audit 2023-09-23T18:15:09.456503+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:10.024 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[27349]: audit 2023-09-23T18:15:09.466979+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:10.024 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[27349]: audit 2023-09-23T18:15:09.468140+0000 mon.c (mon.2) 70 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi098.lowjqo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T18:15:10.024 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[27349]: audit 2023-09-23T18:15:09.468498+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi098.lowjqo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T18:15:10.024 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[27349]: audit 2023-09-23T18:15:09.473126+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi098.lowjqo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T18:15:10.024 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:09 smithi098 bash[27349]: audit 2023-09-23T18:15:09.474098+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:15:10.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:09 smithi173 bash[27694]: cluster 2023-09-23T18:15:08.585275+0000 mgr.y (mgr.24344) 103 : cluster [DBG] pgmap v81: 41 pgs: 32 creating+peering, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:10.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:09 smithi173 bash[27694]: audit 2023-09-23T18:15:08.878338+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T18:15:10.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:09 smithi173 bash[27694]: cluster 2023-09-23T18:15:08.878482+0000 mon.a (mon.0) 621 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-09-23T18:15:10.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:09 smithi173 bash[27694]: audit 2023-09-23T18:15:08.888682+0000 mon.c (mon.2) 66 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T18:15:10.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:09 smithi173 bash[27694]: audit 2023-09-23T18:15:08.889246+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T18:15:10.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:09 smithi173 bash[27694]: cluster 2023-09-23T18:15:08.890178+0000 mon.a (mon.0) 623 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T18:15:10.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:09 smithi173 bash[27694]: cluster 2023-09-23T18:15:08.890215+0000 mon.a (mon.0) 624 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T18:15:10.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:09 smithi173 bash[27694]: cluster 2023-09-23T18:15:08.899312+0000 mon.a (mon.0) 625 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-09-23T18:15:10.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:09 smithi173 bash[27694]: audit 2023-09-23T18:15:08.899979+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T18:15:10.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:09 smithi173 bash[27694]: cluster 2023-09-23T18:15:08.900085+0000 mon.a (mon.0) 627 : cluster [DBG] fsmap foo:0 2023-09-23T18:15:10.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:09 smithi173 bash[27694]: cephadm 2023-09-23T18:15:08.906624+0000 mgr.y (mgr.24344) 104 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-09-23T18:15:10.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:09 smithi173 bash[27694]: audit 2023-09-23T18:15:08.918560+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:10.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:09 smithi173 bash[27694]: audit 2023-09-23T18:15:08.952348+0000 mon.c (mon.2) 67 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:15:10.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:09 smithi173 bash[27694]: audit 2023-09-23T18:15:09.207797+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:10.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:09 smithi173 bash[27694]: audit 2023-09-23T18:15:09.449372+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:15:10.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:09 smithi173 bash[27694]: audit 2023-09-23T18:15:09.450378+0000 mon.c (mon.2) 69 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:15:10.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:09 smithi173 bash[27694]: audit 2023-09-23T18:15:09.456503+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:10.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:09 smithi173 bash[27694]: audit 2023-09-23T18:15:09.466979+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:10.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:09 smithi173 bash[27694]: audit 2023-09-23T18:15:09.468140+0000 mon.c (mon.2) 70 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi098.lowjqo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T18:15:10.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:09 smithi173 bash[27694]: audit 2023-09-23T18:15:09.468498+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi098.lowjqo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T18:15:10.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:09 smithi173 bash[27694]: audit 2023-09-23T18:15:09.473126+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi098.lowjqo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T18:15:10.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:09 smithi173 bash[27694]: audit 2023-09-23T18:15:09.474098+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:15:11.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:10 smithi098 bash[27349]: cephadm 2023-09-23T18:15:09.474907+0000 mgr.y (mgr.24344) 105 : cephadm [INF] Deploying daemon mds.foo.smithi098.lowjqo on smithi098 2023-09-23T18:15:11.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:10 smithi098 bash[27349]: cluster 2023-09-23T18:15:09.902134+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-09-23T18:15:11.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:10 smithi098 bash[33599]: cephadm 2023-09-23T18:15:09.474907+0000 mgr.y (mgr.24344) 105 : cephadm [INF] Deploying daemon mds.foo.smithi098.lowjqo on smithi098 2023-09-23T18:15:11.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:10 smithi098 bash[33599]: cluster 2023-09-23T18:15:09.902134+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-09-23T18:15:11.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:10 smithi173 bash[27694]: cephadm 2023-09-23T18:15:09.474907+0000 mgr.y (mgr.24344) 105 : cephadm [INF] Deploying daemon mds.foo.smithi098.lowjqo on smithi098 2023-09-23T18:15:11.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:10 smithi173 bash[27694]: cluster 2023-09-23T18:15:09.902134+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-09-23T18:15:12.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:11 smithi098 bash[27349]: cluster 2023-09-23T18:15:10.585860+0000 mgr.y (mgr.24344) 106 : cluster [DBG] pgmap v85: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:12.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:11 smithi098 bash[27349]: cluster 2023-09-23T18:15:10.907150+0000 mon.a (mon.0) 635 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-09-23T18:15:12.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:11 smithi098 bash[33599]: cluster 2023-09-23T18:15:10.585860+0000 mgr.y (mgr.24344) 106 : cluster [DBG] pgmap v85: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:12.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:11 smithi098 bash[33599]: cluster 2023-09-23T18:15:10.907150+0000 mon.a (mon.0) 635 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-09-23T18:15:12.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:11 smithi173 bash[27694]: cluster 2023-09-23T18:15:10.585860+0000 mgr.y (mgr.24344) 106 : cluster [DBG] pgmap v85: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:12.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:11 smithi173 bash[27694]: cluster 2023-09-23T18:15:10.907150+0000 mon.a (mon.0) 635 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-09-23T18:15:14.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:13 smithi098 bash[33599]: cluster 2023-09-23T18:15:12.586544+0000 mgr.y (mgr.24344) 107 : cluster [DBG] pgmap v87: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:14.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:13 smithi098 bash[33599]: audit 2023-09-23T18:15:12.798836+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:14.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:13 smithi098 bash[33599]: audit 2023-09-23T18:15:13.266375+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:14.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:13 smithi098 bash[33599]: audit 2023-09-23T18:15:13.268306+0000 mon.c (mon.2) 72 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi173.qjefje", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T18:15:14.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:13 smithi098 bash[33599]: audit 2023-09-23T18:15:13.268693+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi173.qjefje", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T18:15:14.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:13 smithi098 bash[33599]: audit 2023-09-23T18:15:13.275369+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi173.qjefje", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T18:15:14.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:13 smithi098 bash[33599]: audit 2023-09-23T18:15:13.276673+0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:15:14.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:13 smithi098 bash[27349]: cluster 2023-09-23T18:15:12.586544+0000 mgr.y (mgr.24344) 107 : cluster [DBG] pgmap v87: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:14.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:13 smithi098 bash[27349]: audit 2023-09-23T18:15:12.798836+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:14.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:13 smithi098 bash[27349]: audit 2023-09-23T18:15:13.266375+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:14.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:13 smithi098 bash[27349]: audit 2023-09-23T18:15:13.268306+0000 mon.c (mon.2) 72 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi173.qjefje", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T18:15:14.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:13 smithi098 bash[27349]: audit 2023-09-23T18:15:13.268693+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi173.qjefje", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T18:15:14.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:13 smithi098 bash[27349]: audit 2023-09-23T18:15:13.275369+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi173.qjefje", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T18:15:14.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:13 smithi098 bash[27349]: audit 2023-09-23T18:15:13.276673+0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:15:14.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:13 smithi173 bash[27694]: cluster 2023-09-23T18:15:12.586544+0000 mgr.y (mgr.24344) 107 : cluster [DBG] pgmap v87: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:14.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:13 smithi173 bash[27694]: audit 2023-09-23T18:15:12.798836+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:14.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:13 smithi173 bash[27694]: audit 2023-09-23T18:15:13.266375+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:14.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:13 smithi173 bash[27694]: audit 2023-09-23T18:15:13.268306+0000 mon.c (mon.2) 72 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi173.qjefje", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T18:15:14.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:13 smithi173 bash[27694]: audit 2023-09-23T18:15:13.268693+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi173.qjefje", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T18:15:14.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:13 smithi173 bash[27694]: audit 2023-09-23T18:15:13.275369+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi173.qjefje", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T18:15:14.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:13 smithi173 bash[27694]: audit 2023-09-23T18:15:13.276673+0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:15:14.886 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:14 smithi098 bash[27349]: cephadm 2023-09-23T18:15:13.277607+0000 mgr.y (mgr.24344) 108 : cephadm [INF] Deploying daemon mds.foo.smithi173.qjefje on smithi173 2023-09-23T18:15:14.886 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:14 smithi098 bash[33599]: cephadm 2023-09-23T18:15:13.277607+0000 mgr.y (mgr.24344) 108 : cephadm [INF] Deploying daemon mds.foo.smithi173.qjefje on smithi173 2023-09-23T18:15:14.895 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:14 smithi173 bash[27694]: cephadm 2023-09-23T18:15:13.277607+0000 mgr.y (mgr.24344) 108 : cephadm [INF] Deploying daemon mds.foo.smithi173.qjefje on smithi173 2023-09-23T18:15:15.763 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-09-23T18:15:15.776 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:15:15 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:15:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:15:16.072 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:15 smithi173 bash[27694]: cluster 2023-09-23T18:15:14.587257+0000 mgr.y (mgr.24344) 109 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:16.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:15 smithi098 bash[27349]: cluster 2023-09-23T18:15:14.587257+0000 mgr.y (mgr.24344) 109 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:16.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:15 smithi098 bash[33599]: cluster 2023-09-23T18:15:14.587257+0000 mgr.y (mgr.24344) 109 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:17.079 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:16 smithi098 bash[33599]: cluster 2023-09-23T18:15:15.813072+0000 mon.a (mon.0) 640 : cluster [DBG] mds.? [v2:172.21.15.98:6834/2708276357,v1:172.21.15.98:6835/2708276357] up:boot 2023-09-23T18:15:17.079 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:16 smithi098 bash[33599]: cluster 2023-09-23T18:15:15.813191+0000 mon.a (mon.0) 641 : cluster [INF] daemon mds.foo.smithi098.lowjqo assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T18:15:17.079 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:16 smithi098 bash[33599]: cluster 2023-09-23T18:15:15.813338+0000 mon.a (mon.0) 642 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T18:15:17.079 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:16 smithi098 bash[33599]: cluster 2023-09-23T18:15:15.813382+0000 mon.a (mon.0) 643 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T18:15:17.079 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:16 smithi098 bash[33599]: cluster 2023-09-23T18:15:15.813403+0000 mon.a (mon.0) 644 : cluster [INF] Cluster is now healthy 2023-09-23T18:15:17.079 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:16 smithi098 bash[33599]: cluster 2023-09-23T18:15:15.813897+0000 mon.a (mon.0) 645 : cluster [DBG] fsmap foo:0 1 up:standby 2023-09-23T18:15:17.079 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:16 smithi098 bash[33599]: audit 2023-09-23T18:15:15.815050+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi098.lowjqo"}]: dispatch 2023-09-23T18:15:17.079 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:16 smithi098 bash[33599]: cluster 2023-09-23T18:15:15.821804+0000 mon.a (mon.0) 646 : cluster [DBG] fsmap foo:1 {0=foo.smithi098.lowjqo=up:creating} 2023-09-23T18:15:17.079 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:16 smithi098 bash[33599]: cluster 2023-09-23T18:15:15.837821+0000 mon.a (mon.0) 647 : cluster [INF] daemon mds.foo.smithi098.lowjqo is now active in filesystem foo as rank 0 2023-09-23T18:15:17.079 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:16 smithi098 bash[33599]: audit 2023-09-23T18:15:16.104862+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:17.080 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:16 smithi098 bash[33599]: audit 2023-09-23T18:15:16.108817+0000 mon.c (mon.2) 75 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:15:17.080 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:16 smithi098 bash[27349]: cluster 2023-09-23T18:15:15.813072+0000 mon.a (mon.0) 640 : cluster [DBG] mds.? [v2:172.21.15.98:6834/2708276357,v1:172.21.15.98:6835/2708276357] up:boot 2023-09-23T18:15:17.080 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:16 smithi098 bash[27349]: cluster 2023-09-23T18:15:15.813191+0000 mon.a (mon.0) 641 : cluster [INF] daemon mds.foo.smithi098.lowjqo assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T18:15:17.080 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:16 smithi098 bash[27349]: cluster 2023-09-23T18:15:15.813338+0000 mon.a (mon.0) 642 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T18:15:17.080 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:16 smithi098 bash[27349]: cluster 2023-09-23T18:15:15.813382+0000 mon.a (mon.0) 643 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T18:15:17.080 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:16 smithi098 bash[27349]: cluster 2023-09-23T18:15:15.813403+0000 mon.a (mon.0) 644 : cluster [INF] Cluster is now healthy 2023-09-23T18:15:17.080 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:16 smithi098 bash[27349]: cluster 2023-09-23T18:15:15.813897+0000 mon.a (mon.0) 645 : cluster [DBG] fsmap foo:0 1 up:standby 2023-09-23T18:15:17.080 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:16 smithi098 bash[27349]: audit 2023-09-23T18:15:15.815050+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi098.lowjqo"}]: dispatch 2023-09-23T18:15:17.080 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:16 smithi098 bash[27349]: cluster 2023-09-23T18:15:15.821804+0000 mon.a (mon.0) 646 : cluster [DBG] fsmap foo:1 {0=foo.smithi098.lowjqo=up:creating} 2023-09-23T18:15:17.080 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:16 smithi098 bash[27349]: cluster 2023-09-23T18:15:15.837821+0000 mon.a (mon.0) 647 : cluster [INF] daemon mds.foo.smithi098.lowjqo is now active in filesystem foo as rank 0 2023-09-23T18:15:17.081 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:16 smithi098 bash[27349]: audit 2023-09-23T18:15:16.104862+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:17.081 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:16 smithi098 bash[27349]: audit 2023-09-23T18:15:16.108817+0000 mon.c (mon.2) 75 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:15:17.143 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:16 smithi173 bash[27694]: cluster 2023-09-23T18:15:15.813072+0000 mon.a (mon.0) 640 : cluster [DBG] mds.? [v2:172.21.15.98:6834/2708276357,v1:172.21.15.98:6835/2708276357] up:boot 2023-09-23T18:15:17.143 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:16 smithi173 bash[27694]: cluster 2023-09-23T18:15:15.813191+0000 mon.a (mon.0) 641 : cluster [INF] daemon mds.foo.smithi098.lowjqo assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T18:15:17.143 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:16 smithi173 bash[27694]: cluster 2023-09-23T18:15:15.813338+0000 mon.a (mon.0) 642 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T18:15:17.143 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:16 smithi173 bash[27694]: cluster 2023-09-23T18:15:15.813382+0000 mon.a (mon.0) 643 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T18:15:17.143 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:16 smithi173 bash[27694]: cluster 2023-09-23T18:15:15.813403+0000 mon.a (mon.0) 644 : cluster [INF] Cluster is now healthy 2023-09-23T18:15:17.144 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:16 smithi173 bash[27694]: cluster 2023-09-23T18:15:15.813897+0000 mon.a (mon.0) 645 : cluster [DBG] fsmap foo:0 1 up:standby 2023-09-23T18:15:17.144 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:16 smithi173 bash[27694]: audit 2023-09-23T18:15:15.815050+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi098.lowjqo"}]: dispatch 2023-09-23T18:15:17.144 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:16 smithi173 bash[27694]: cluster 2023-09-23T18:15:15.821804+0000 mon.a (mon.0) 646 : cluster [DBG] fsmap foo:1 {0=foo.smithi098.lowjqo=up:creating} 2023-09-23T18:15:17.144 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:16 smithi173 bash[27694]: cluster 2023-09-23T18:15:15.837821+0000 mon.a (mon.0) 647 : cluster [INF] daemon mds.foo.smithi098.lowjqo is now active in filesystem foo as rank 0 2023-09-23T18:15:17.144 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:16 smithi173 bash[27694]: audit 2023-09-23T18:15:16.104862+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:17.144 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:16 smithi173 bash[27694]: audit 2023-09-23T18:15:16.108817+0000 mon.c (mon.2) 75 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:15:18.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:15:17 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:15:17] "GET /metrics HTTP/1.1" 200 160701 "" "Prometheus/2.33.4" 2023-09-23T18:15:18.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:17 smithi098 bash[27349]: cluster 2023-09-23T18:15:16.587654+0000 mgr.y (mgr.24344) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:18.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:17 smithi098 bash[27349]: cluster 2023-09-23T18:15:16.828039+0000 mon.a (mon.0) 649 : cluster [DBG] mds.? [v2:172.21.15.98:6834/2708276357,v1:172.21.15.98:6835/2708276357] up:active 2023-09-23T18:15:18.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:17 smithi098 bash[27349]: cluster 2023-09-23T18:15:16.828100+0000 mon.a (mon.0) 650 : cluster [DBG] fsmap foo:1 {0=foo.smithi098.lowjqo=up:active} 2023-09-23T18:15:18.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:17 smithi098 bash[33599]: cluster 2023-09-23T18:15:16.587654+0000 mgr.y (mgr.24344) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:18.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:17 smithi098 bash[33599]: cluster 2023-09-23T18:15:16.828039+0000 mon.a (mon.0) 649 : cluster [DBG] mds.? [v2:172.21.15.98:6834/2708276357,v1:172.21.15.98:6835/2708276357] up:active 2023-09-23T18:15:18.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:17 smithi098 bash[33599]: cluster 2023-09-23T18:15:16.828100+0000 mon.a (mon.0) 650 : cluster [DBG] fsmap foo:1 {0=foo.smithi098.lowjqo=up:active} 2023-09-23T18:15:18.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:17 smithi173 bash[27694]: cluster 2023-09-23T18:15:16.587654+0000 mgr.y (mgr.24344) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:18.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:17 smithi173 bash[27694]: cluster 2023-09-23T18:15:16.828039+0000 mon.a (mon.0) 649 : cluster [DBG] mds.? [v2:172.21.15.98:6834/2708276357,v1:172.21.15.98:6835/2708276357] up:active 2023-09-23T18:15:18.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:17 smithi173 bash[27694]: cluster 2023-09-23T18:15:16.828100+0000 mon.a (mon.0) 650 : cluster [DBG] fsmap foo:1 {0=foo.smithi098.lowjqo=up:active} 2023-09-23T18:15:19.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:18 smithi098 bash[27349]: audit 2023-09-23T18:15:17.887460+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:19.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:18 smithi098 bash[27349]: audit 2023-09-23T18:15:18.108428+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi173.qjefje"}]: dispatch 2023-09-23T18:15:19.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:18 smithi098 bash[27349]: cluster 2023-09-23T18:15:18.109616+0000 mon.a (mon.0) 652 : cluster [DBG] mds.? [v2:172.21.15.173:6832/621117609,v1:172.21.15.173:6833/621117609] up:boot 2023-09-23T18:15:19.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:18 smithi098 bash[27349]: cluster 2023-09-23T18:15:18.110162+0000 mon.a (mon.0) 653 : cluster [DBG] fsmap foo:1 {0=foo.smithi098.lowjqo=up:active} 1 up:standby 2023-09-23T18:15:19.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:18 smithi098 bash[27349]: cluster 2023-09-23T18:15:18.113386+0000 mon.a (mon.0) 654 : cluster [DBG] fsmap foo:1 {0=foo.smithi098.lowjqo=up:active} 1 up:standby 2023-09-23T18:15:19.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:18 smithi098 bash[33599]: audit 2023-09-23T18:15:17.887460+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:19.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:18 smithi098 bash[33599]: audit 2023-09-23T18:15:18.108428+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi173.qjefje"}]: dispatch 2023-09-23T18:15:19.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:18 smithi098 bash[33599]: cluster 2023-09-23T18:15:18.109616+0000 mon.a (mon.0) 652 : cluster [DBG] mds.? [v2:172.21.15.173:6832/621117609,v1:172.21.15.173:6833/621117609] up:boot 2023-09-23T18:15:19.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:18 smithi098 bash[33599]: cluster 2023-09-23T18:15:18.110162+0000 mon.a (mon.0) 653 : cluster [DBG] fsmap foo:1 {0=foo.smithi098.lowjqo=up:active} 1 up:standby 2023-09-23T18:15:19.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:18 smithi098 bash[33599]: cluster 2023-09-23T18:15:18.113386+0000 mon.a (mon.0) 654 : cluster [DBG] fsmap foo:1 {0=foo.smithi098.lowjqo=up:active} 1 up:standby 2023-09-23T18:15:19.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:18 smithi173 bash[27694]: audit 2023-09-23T18:15:17.887460+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:19.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:18 smithi173 bash[27694]: audit 2023-09-23T18:15:18.108428+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi173.qjefje"}]: dispatch 2023-09-23T18:15:19.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:18 smithi173 bash[27694]: cluster 2023-09-23T18:15:18.109616+0000 mon.a (mon.0) 652 : cluster [DBG] mds.? [v2:172.21.15.173:6832/621117609,v1:172.21.15.173:6833/621117609] up:boot 2023-09-23T18:15:19.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:18 smithi173 bash[27694]: cluster 2023-09-23T18:15:18.110162+0000 mon.a (mon.0) 653 : cluster [DBG] fsmap foo:1 {0=foo.smithi098.lowjqo=up:active} 1 up:standby 2023-09-23T18:15:19.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:18 smithi173 bash[27694]: cluster 2023-09-23T18:15:18.113386+0000 mon.a (mon.0) 654 : cluster [DBG] fsmap foo:1 {0=foo.smithi098.lowjqo=up:active} 1 up:standby 2023-09-23T18:15:19.925 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-09-23T18:15:19.934 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T18:15:19.934 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T18:15:19.934 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T18:15:20.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:19 smithi098 bash[27349]: cluster 2023-09-23T18:15:18.588615+0000 mgr.y (mgr.24344) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:20.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:19 smithi098 bash[33599]: cluster 2023-09-23T18:15:18.588615+0000 mgr.y (mgr.24344) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:20.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:19 smithi173 bash[27694]: cluster 2023-09-23T18:15:18.588615+0000 mgr.y (mgr.24344) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:22.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:21 smithi098 bash[33599]: cluster 2023-09-23T18:15:20.589292+0000 mgr.y (mgr.24344) 112 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:22.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:21 smithi098 bash[27349]: cluster 2023-09-23T18:15:20.589292+0000 mgr.y (mgr.24344) 112 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:22.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:21 smithi173 bash[27694]: cluster 2023-09-23T18:15:20.589292+0000 mgr.y (mgr.24344) 112 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:23.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:23 smithi098 bash[33599]: audit 2023-09-23T18:15:22.052404+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:23.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:23 smithi098 bash[33599]: audit 2023-09-23T18:15:22.378654+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:23.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:23 smithi098 bash[33599]: audit 2023-09-23T18:15:22.383424+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:15:23.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:23 smithi098 bash[33599]: audit 2023-09-23T18:15:22.384863+0000 mon.c (mon.2) 78 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:15:23.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:23 smithi098 bash[33599]: audit 2023-09-23T18:15:22.391473+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:23.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:23 smithi098 bash[33599]: audit 2023-09-23T18:15:22.398987+0000 mon.c (mon.2) 79 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:15:23.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:23 smithi098 bash[33599]: audit 2023-09-23T18:15:22.400960+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:15:23.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:23 smithi098 bash[33599]: audit 2023-09-23T18:15:22.402113+0000 mon.c (mon.2) 81 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:15:23.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:23 smithi098 bash[33599]: audit 2023-09-23T18:15:22.409811+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:23.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:23 smithi098 bash[33599]: cluster 2023-09-23T18:15:22.589942+0000 mgr.y (mgr.24344) 113 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:23.447 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:23 smithi098 bash[27349]: audit 2023-09-23T18:15:22.052404+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:23.447 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:23 smithi098 bash[27349]: audit 2023-09-23T18:15:22.378654+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:23.447 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:23 smithi098 bash[27349]: audit 2023-09-23T18:15:22.383424+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:15:23.448 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:23 smithi098 bash[27349]: audit 2023-09-23T18:15:22.384863+0000 mon.c (mon.2) 78 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:15:23.448 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:23 smithi098 bash[27349]: audit 2023-09-23T18:15:22.391473+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:23.448 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:23 smithi098 bash[27349]: audit 2023-09-23T18:15:22.398987+0000 mon.c (mon.2) 79 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:15:23.448 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:23 smithi098 bash[27349]: audit 2023-09-23T18:15:22.400960+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:15:23.448 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:23 smithi098 bash[27349]: audit 2023-09-23T18:15:22.402113+0000 mon.c (mon.2) 81 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:15:23.448 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:23 smithi098 bash[27349]: audit 2023-09-23T18:15:22.409811+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:23.448 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:23 smithi098 bash[27349]: cluster 2023-09-23T18:15:22.589942+0000 mgr.y (mgr.24344) 113 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:23.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:23 smithi173 bash[27694]: audit 2023-09-23T18:15:22.052404+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:23.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:23 smithi173 bash[27694]: audit 2023-09-23T18:15:22.378654+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:23.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:23 smithi173 bash[27694]: audit 2023-09-23T18:15:22.383424+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:15:23.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:23 smithi173 bash[27694]: audit 2023-09-23T18:15:22.384863+0000 mon.c (mon.2) 78 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:15:23.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:23 smithi173 bash[27694]: audit 2023-09-23T18:15:22.391473+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:23.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:23 smithi173 bash[27694]: audit 2023-09-23T18:15:22.398987+0000 mon.c (mon.2) 79 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:15:23.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:23 smithi173 bash[27694]: audit 2023-09-23T18:15:22.400960+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:15:23.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:23 smithi173 bash[27694]: audit 2023-09-23T18:15:22.402113+0000 mon.c (mon.2) 81 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:15:23.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:23 smithi173 bash[27694]: audit 2023-09-23T18:15:22.409811+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:23.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:23 smithi173 bash[27694]: cluster 2023-09-23T18:15:22.589942+0000 mgr.y (mgr.24344) 113 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:24.474 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:25.653 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:15:25 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:15:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:15:25.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:25 smithi098 bash[27349]: cluster 2023-09-23T18:15:24.590903+0000 mgr.y (mgr.24344) 114 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T18:15:25.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:25 smithi098 bash[33599]: cluster 2023-09-23T18:15:24.590903+0000 mgr.y (mgr.24344) 114 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T18:15:26.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:25 smithi173 bash[27694]: cluster 2023-09-23T18:15:24.590903+0000 mgr.y (mgr.24344) 114 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T18:15:27.557 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:15:27.945 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:15:27 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:15:27] "GET /metrics HTTP/1.1" 200 160701 "" "Prometheus/2.33.4" 2023-09-23T18:15:27.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:27 smithi098 bash[33599]: cluster 2023-09-23T18:15:26.591451+0000 mgr.y (mgr.24344) 115 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T18:15:27.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:27 smithi098 bash[33599]: audit 2023-09-23T18:15:27.542544+0000 mon.a (mon.0) 659 : audit [DBG] from='client.? 172.21.15.98:0/2352182402' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T18:15:27.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:27 smithi098 bash[27349]: cluster 2023-09-23T18:15:26.591451+0000 mgr.y (mgr.24344) 115 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T18:15:27.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:27 smithi098 bash[27349]: audit 2023-09-23T18:15:27.542544+0000 mon.a (mon.0) 659 : audit [DBG] from='client.? 172.21.15.98:0/2352182402' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T18:15:28.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:27 smithi173 bash[27694]: cluster 2023-09-23T18:15:26.591451+0000 mgr.y (mgr.24344) 115 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T18:15:28.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:27 smithi173 bash[27694]: audit 2023-09-23T18:15:27.542544+0000 mon.a (mon.0) 659 : audit [DBG] from='client.? 172.21.15.98:0/2352182402' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T18:15:28.371 INFO:teuthology.orchestra.run.smithi098.stdout:{"epoch":18,"active_gid":24344,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6800","nonce":3706591260},{"type":"v1","addr":"172.21.15.98:6801","nonce":3706591260}]},"active_addr":"172.21.15.98:6801/3706591260","active_change":"2023-09-23T18:12:42.526079+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24358,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.98:8443/","prometheus":"http://172.21.15.98:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.98:0","nonce":4085572202}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.98:0","nonce":2098141849}]}]} 2023-09-23T18:15:28.376 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T18:15:28.376 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T18:15:28.377 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T18:15:29.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:28 smithi098 bash[27349]: audit 2023-09-23T18:15:27.799060+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:29.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:28 smithi098 bash[27349]: audit 2023-09-23T18:15:27.909038+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:29.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:28 smithi098 bash[33599]: audit 2023-09-23T18:15:27.799060+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:29.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:28 smithi098 bash[33599]: audit 2023-09-23T18:15:27.909038+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:29.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:28 smithi173 bash[27694]: audit 2023-09-23T18:15:27.799060+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:29.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:28 smithi173 bash[27694]: audit 2023-09-23T18:15:27.909038+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:15:30.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:29 smithi098 bash[27349]: cluster 2023-09-23T18:15:28.592563+0000 mgr.y (mgr.24344) 116 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T18:15:30.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:29 smithi098 bash[33599]: cluster 2023-09-23T18:15:28.592563+0000 mgr.y (mgr.24344) 116 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T18:15:30.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:29 smithi173 bash[27694]: cluster 2023-09-23T18:15:28.592563+0000 mgr.y (mgr.24344) 116 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T18:15:32.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:31 smithi098 bash[27349]: cluster 2023-09-23T18:15:30.593227+0000 mgr.y (mgr.24344) 117 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T18:15:32.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:31 smithi098 bash[33599]: cluster 2023-09-23T18:15:30.593227+0000 mgr.y (mgr.24344) 117 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T18:15:32.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:31 smithi173 bash[27694]: cluster 2023-09-23T18:15:30.593227+0000 mgr.y (mgr.24344) 117 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T18:15:33.155 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:34.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:33 smithi098 bash[33599]: cluster 2023-09-23T18:15:32.593925+0000 mgr.y (mgr.24344) 118 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T18:15:34.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:33 smithi098 bash[27349]: cluster 2023-09-23T18:15:32.593925+0000 mgr.y (mgr.24344) 118 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T18:15:34.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:33 smithi173 bash[27694]: cluster 2023-09-23T18:15:32.593925+0000 mgr.y (mgr.24344) 118 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T18:15:35.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:15:35 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:15:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:15:36.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:35 smithi098 bash[27349]: cluster 2023-09-23T18:15:34.595173+0000 mgr.y (mgr.24344) 119 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T18:15:36.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:35 smithi098 bash[33599]: cluster 2023-09-23T18:15:34.595173+0000 mgr.y (mgr.24344) 119 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T18:15:36.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:35 smithi173 bash[27694]: cluster 2023-09-23T18:15:34.595173+0000 mgr.y (mgr.24344) 119 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T18:15:36.373 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:15:36.373 INFO:teuthology.orchestra.run.smithi098.stdout:{"epoch":60,"fsid":"756c96d4-5a3b-11ee-8db4-212e2dc638e7","created":"2023-09-23T18:04:11.580781+0000","modified":"2023-09-23T18:15:10.896711+0000","last_up_change":"2023-09-23T18:12:09.242233+0000","last_in_change":"2023-09-23T18:11:43.419979+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-23T18:08:35.146924+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T18:13:12.421802+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T18:15:07.820408+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T18:15:08.333052+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"4f35f176-4d07-4157-aeb9-dd8f71da3a63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6802","nonce":1147685383},{"type":"v1","addr":"172.21.15.98:6803","nonce":1147685383}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6804","nonce":1147685383},{"type":"v1","addr":"172.21.15.98:6805","nonce":1147685383}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6808","nonce":1147685383},{"type":"v1","addr":"172.21.15.98:6809","nonce":1147685383}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6806","nonce":1147685383},{"type":"v1","addr":"172.21.15.98:6807","nonce":1147685383}]},"public_addr":"172.21.15.98:6803/1147685383","cluster_addr":"172.21.15.98:6805/1147685383","heartbeat_back_addr":"172.21.15.98:6809/1147685383","heartbeat_front_addr":"172.21.15.98:6807/1147685383","state":["exists","up"]},{"osd":1,"uuid":"5d777d1f-3bd3-49ba-b0de-0ac532ee3934","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6810","nonce":3945616078},{"type":"v1","addr":"172.21.15.98:6811","nonce":3945616078}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6812","nonce":3945616078},{"type":"v1","addr":"172.21.15.98:6813","nonce":3945616078}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6816","nonce":3945616078},{"type":"v1","addr":"172.21.15.98:6817","nonce":3945616078}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6814","nonce":3945616078},{"type":"v1","addr":"172.21.15.98:6815","nonce":3945616078}]},"public_addr":"172.21.15.98:6811/3945616078","cluster_addr":"172.21.15.98:6813/3945616078","heartbeat_back_addr":"172.21.15.98:6817/3945616078","heartbeat_front_addr":"172.21.15.98:6815/3945616078","state":["exists","up"]},{"osd":2,"uuid":"63978288-ac36-4cf2-bbb7-46dc828e6ce5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6818","nonce":2338555672},{"type":"v1","addr":"172.21.15.98:6819","nonce":2338555672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6820","nonce":2338555672},{"type":"v1","addr":"172.21.15.98:6821","nonce":2338555672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6824","nonce":2338555672},{"type":"v1","addr":"172.21.15.98:6825","nonce":2338555672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6822","nonce":2338555672},{"type":"v1","addr":"172.21.15.98:6823","nonce":2338555672}]},"public_addr":"172.21.15.98:6819/2338555672","cluster_addr":"172.21.15.98:6821/2338555672","heartbeat_back_addr":"172.21.15.98:6825/2338555672","heartbeat_front_addr":"172.21.15.98:6823/2338555672","state":["exists","up"]},{"osd":3,"uuid":"48596d91-46da-4815-b810-d493bbf01dcc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6826","nonce":2634725099},{"type":"v1","addr":"172.21.15.98:6827","nonce":2634725099}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6828","nonce":2634725099},{"type":"v1","addr":"172.21.15.98:6829","nonce":2634725099}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6832","nonce":2634725099},{"type":"v1","addr":"172.21.15.98:6833","nonce":2634725099}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6830","nonce":2634725099},{"type":"v1","addr":"172.21.15.98:6831","nonce":2634725099}]},"public_addr":"172.21.15.98:6827/2634725099","cluster_addr":"172.21.15.98:6829/2634725099","heartbeat_back_addr":"172.21.15.98:6833/2634725099","heartbeat_front_addr":"172.21.15.98:6831/2634725099","state":["exists","up"]},{"osd":4,"uuid":"a562afc6-ffe6-44bb-91a9-8a04018c3c6f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6800","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6801","nonce":165906554}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6802","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6803","nonce":165906554}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6806","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6807","nonce":165906554}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6804","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6805","nonce":165906554}]},"public_addr":"172.21.15.173:6801/165906554","cluster_addr":"172.21.15.173:6803/165906554","heartbeat_back_addr":"172.21.15.173:6807/165906554","heartbeat_front_addr":"172.21.15.173:6805/165906554","state":["exists","up"]},{"osd":5,"uuid":"add28146-8301-4b4c-9616-967e9df3a8d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6808","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6809","nonce":2894909344}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6810","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6811","nonce":2894909344}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6814","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6815","nonce":2894909344}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6812","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6813","nonce":2894909344}]},"public_addr":"172.21.15.173:6809/2894909344","cluster_addr":"172.21.15.173:6811/2894909344","heartbeat_back_addr":"172.21.15.173:6815/2894909344","heartbeat_front_addr":"172.21.15.173:6813/2894909344","state":["exists","up"]},{"osd":6,"uuid":"f30854de-bc26-465d-8b0e-a9211ae2d9fa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6816","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6817","nonce":3550013548}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6818","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6819","nonce":3550013548}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6822","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6823","nonce":3550013548}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6820","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6821","nonce":3550013548}]},"public_addr":"172.21.15.173:6817/3550013548","cluster_addr":"172.21.15.173:6819/3550013548","heartbeat_back_addr":"172.21.15.173:6823/3550013548","heartbeat_front_addr":"172.21.15.173:6821/3550013548","state":["exists","up"]},{"osd":7,"uuid":"5f540de3-48f9-4bc8-b14a-9aadf838f16b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6824","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6825","nonce":135608222}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6826","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6827","nonce":135608222}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6830","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6831","nonce":135608222}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6828","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6829","nonce":135608222}]},"public_addr":"172.21.15.173:6825/135608222","cluster_addr":"172.21.15.173:6827/135608222","heartbeat_back_addr":"172.21.15.173:6831/135608222","heartbeat_front_addr":"172.21.15.173:6829/135608222","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:07:04.739243+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:07:48.583198+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:08:33.173348+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:09:18.218502+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:09:59.765019+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:10:42.490689+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:11:24.357920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:12:08.168900+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.98:6801/483469857":"2023-09-24T18:12:42.525837+0000","172.21.15.98:6800/483469857":"2023-09-24T18:12:42.525837+0000","172.21.15.98:0/1142988558":"2023-09-24T18:12:42.525837+0000","172.21.15.98:0/3472030975":"2023-09-24T18:05:11.060631+0000","172.21.15.98:6800/1392949322":"2023-09-24T18:05:11.060631+0000","172.21.15.98:0/3976553993":"2023-09-24T18:04:40.596823+0000","172.21.15.98:0/2194193535":"2023-09-24T18:12:42.525837+0000","172.21.15.98:0/3213365423":"2023-09-24T18:04:40.596823+0000","172.21.15.98:6801/1392949322":"2023-09-24T18:05:11.060631+0000","172.21.15.98:6800/3368447653":"2023-09-24T18:04:40.596823+0000","172.21.15.98:0/1603044865":"2023-09-24T18:05:11.060631+0000","172.21.15.98:6801/3368447653":"2023-09-24T18:04:40.596823+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T18:15:36.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:36 smithi098 bash[33599]: audit 2023-09-23T18:15:36.365541+0000 mon.c (mon.2) 82 : audit [DBG] from='client.? 172.21.15.98:0/4164615590' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:15:36.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:36 smithi098 bash[27349]: audit 2023-09-23T18:15:36.365541+0000 mon.c (mon.2) 82 : audit [DBG] from='client.? 172.21.15.98:0/4164615590' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:15:37.234 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T18:15:37.234 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T18:15:37.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:36 smithi173 bash[27694]: audit 2023-09-23T18:15:36.365541+0000 mon.c (mon.2) 82 : audit [DBG] from='client.? 172.21.15.98:0/4164615590' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:15:38.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:37 smithi098 bash[27349]: cluster 2023-09-23T18:15:36.595640+0000 mgr.y (mgr.24344) 120 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:38.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:37 smithi098 bash[33599]: cluster 2023-09-23T18:15:36.595640+0000 mgr.y (mgr.24344) 120 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:38.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:15:37 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:15:37] "GET /metrics HTTP/1.1" 200 191497 "" "Prometheus/2.33.4" 2023-09-23T18:15:38.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:37 smithi173 bash[27694]: cluster 2023-09-23T18:15:36.595640+0000 mgr.y (mgr.24344) 120 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:39.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:39 smithi098 bash[27349]: cluster 2023-09-23T18:15:38.596738+0000 mgr.y (mgr.24344) 121 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:39.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:39 smithi098 bash[33599]: cluster 2023-09-23T18:15:38.596738+0000 mgr.y (mgr.24344) 121 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:39.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:39 smithi173 bash[27694]: cluster 2023-09-23T18:15:38.596738+0000 mgr.y (mgr.24344) 121 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:41.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:41 smithi098 bash[27349]: cluster 2023-09-23T18:15:40.597438+0000 mgr.y (mgr.24344) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:41.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:41 smithi098 bash[33599]: cluster 2023-09-23T18:15:40.597438+0000 mgr.y (mgr.24344) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:42.011 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:42.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:41 smithi173 bash[27694]: cluster 2023-09-23T18:15:40.597438+0000 mgr.y (mgr.24344) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:42.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:42 smithi098 bash[27349]: audit 2023-09-23T18:15:42.596515+0000 mon.c (mon.2) 83 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-09-23T18:15:42.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:42 smithi098 bash[27349]: audit 2023-09-23T18:15:42.597220+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-09-23T18:15:42.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:42 smithi098 bash[27349]: audit 2023-09-23T18:15:42.597220+0000 mon.c (mon.2) 84 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-09-23T18:15:42.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:42 smithi098 bash[27349]: audit 2023-09-23T18:15:42.597690+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-09-23T18:15:42.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:42 smithi098 bash[27349]: audit 2023-09-23T18:15:42.597856+0000 mon.c (mon.2) 85 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2023-09-23T18:15:42.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:42 smithi098 bash[27349]: audit 2023-09-23T18:15:42.598417+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2023-09-23T18:15:42.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:42 smithi098 bash[33599]: audit 2023-09-23T18:15:42.596515+0000 mon.c (mon.2) 83 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-09-23T18:15:42.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:42 smithi098 bash[33599]: audit 2023-09-23T18:15:42.597220+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-09-23T18:15:42.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:42 smithi098 bash[33599]: audit 2023-09-23T18:15:42.597220+0000 mon.c (mon.2) 84 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-09-23T18:15:42.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:42 smithi098 bash[33599]: audit 2023-09-23T18:15:42.597690+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-09-23T18:15:42.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:42 smithi098 bash[33599]: audit 2023-09-23T18:15:42.597856+0000 mon.c (mon.2) 85 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2023-09-23T18:15:42.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:42 smithi098 bash[33599]: audit 2023-09-23T18:15:42.598417+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2023-09-23T18:15:43.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:42 smithi173 bash[27694]: audit 2023-09-23T18:15:42.596515+0000 mon.c (mon.2) 83 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-09-23T18:15:43.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:42 smithi173 bash[27694]: audit 2023-09-23T18:15:42.597220+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-09-23T18:15:43.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:42 smithi173 bash[27694]: audit 2023-09-23T18:15:42.597220+0000 mon.c (mon.2) 84 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-09-23T18:15:43.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:42 smithi173 bash[27694]: audit 2023-09-23T18:15:42.597690+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-09-23T18:15:43.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:42 smithi173 bash[27694]: audit 2023-09-23T18:15:42.597856+0000 mon.c (mon.2) 85 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2023-09-23T18:15:43.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:42 smithi173 bash[27694]: audit 2023-09-23T18:15:42.598417+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2023-09-23T18:15:43.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:43 smithi098 bash[27349]: cluster 2023-09-23T18:15:42.598143+0000 mgr.y (mgr.24344) 123 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:43.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:43 smithi098 bash[27349]: audit 2023-09-23T18:15:42.667624+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-09-23T18:15:43.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:43 smithi098 bash[27349]: audit 2023-09-23T18:15:42.667742+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2023-09-23T18:15:43.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:43 smithi098 bash[27349]: audit 2023-09-23T18:15:42.667840+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2023-09-23T18:15:43.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:43 smithi098 bash[27349]: cluster 2023-09-23T18:15:42.667944+0000 mon.a (mon.0) 668 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-09-23T18:15:43.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:43 smithi098 bash[33599]: cluster 2023-09-23T18:15:42.598143+0000 mgr.y (mgr.24344) 123 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:43.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:43 smithi098 bash[33599]: audit 2023-09-23T18:15:42.667624+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-09-23T18:15:43.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:43 smithi098 bash[33599]: audit 2023-09-23T18:15:42.667742+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2023-09-23T18:15:43.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:43 smithi098 bash[33599]: audit 2023-09-23T18:15:42.667840+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2023-09-23T18:15:43.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:43 smithi098 bash[33599]: cluster 2023-09-23T18:15:42.667944+0000 mon.a (mon.0) 668 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-09-23T18:15:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:43 smithi173 bash[27694]: cluster 2023-09-23T18:15:42.598143+0000 mgr.y (mgr.24344) 123 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:43 smithi173 bash[27694]: audit 2023-09-23T18:15:42.667624+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-09-23T18:15:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:43 smithi173 bash[27694]: audit 2023-09-23T18:15:42.667742+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2023-09-23T18:15:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:43 smithi173 bash[27694]: audit 2023-09-23T18:15:42.667840+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2023-09-23T18:15:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:43 smithi173 bash[27694]: cluster 2023-09-23T18:15:42.667944+0000 mon.a (mon.0) 668 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-09-23T18:15:44.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:44 smithi098 bash[27349]: cluster 2023-09-23T18:15:43.667462+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-09-23T18:15:44.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:44 smithi098 bash[33599]: cluster 2023-09-23T18:15:43.667462+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-09-23T18:15:45.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:44 smithi173 bash[27694]: cluster 2023-09-23T18:15:43.667462+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-09-23T18:15:45.214 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:15:45.214 INFO:teuthology.orchestra.run.smithi098.stdout:{"epoch":62,"fsid":"756c96d4-5a3b-11ee-8db4-212e2dc638e7","created":"2023-09-23T18:04:11.580781+0000","modified":"2023-09-23T18:15:43.661860+0000","last_up_change":"2023-09-23T18:12:09.242233+0000","last_in_change":"2023-09-23T18:11:43.419979+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-23T18:08:35.146924+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T18:13:12.421802+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T18:15:07.820408+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T18:15:08.333052+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"4f35f176-4d07-4157-aeb9-dd8f71da3a63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6802","nonce":1147685383},{"type":"v1","addr":"172.21.15.98:6803","nonce":1147685383}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6804","nonce":1147685383},{"type":"v1","addr":"172.21.15.98:6805","nonce":1147685383}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6808","nonce":1147685383},{"type":"v1","addr":"172.21.15.98:6809","nonce":1147685383}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6806","nonce":1147685383},{"type":"v1","addr":"172.21.15.98:6807","nonce":1147685383}]},"public_addr":"172.21.15.98:6803/1147685383","cluster_addr":"172.21.15.98:6805/1147685383","heartbeat_back_addr":"172.21.15.98:6809/1147685383","heartbeat_front_addr":"172.21.15.98:6807/1147685383","state":["exists","up"]},{"osd":1,"uuid":"5d777d1f-3bd3-49ba-b0de-0ac532ee3934","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6810","nonce":3945616078},{"type":"v1","addr":"172.21.15.98:6811","nonce":3945616078}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6812","nonce":3945616078},{"type":"v1","addr":"172.21.15.98:6813","nonce":3945616078}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6816","nonce":3945616078},{"type":"v1","addr":"172.21.15.98:6817","nonce":3945616078}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6814","nonce":3945616078},{"type":"v1","addr":"172.21.15.98:6815","nonce":3945616078}]},"public_addr":"172.21.15.98:6811/3945616078","cluster_addr":"172.21.15.98:6813/3945616078","heartbeat_back_addr":"172.21.15.98:6817/3945616078","heartbeat_front_addr":"172.21.15.98:6815/3945616078","state":["exists","up"]},{"osd":2,"uuid":"63978288-ac36-4cf2-bbb7-46dc828e6ce5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6818","nonce":2338555672},{"type":"v1","addr":"172.21.15.98:6819","nonce":2338555672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6820","nonce":2338555672},{"type":"v1","addr":"172.21.15.98:6821","nonce":2338555672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6824","nonce":2338555672},{"type":"v1","addr":"172.21.15.98:6825","nonce":2338555672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6822","nonce":2338555672},{"type":"v1","addr":"172.21.15.98:6823","nonce":2338555672}]},"public_addr":"172.21.15.98:6819/2338555672","cluster_addr":"172.21.15.98:6821/2338555672","heartbeat_back_addr":"172.21.15.98:6825/2338555672","heartbeat_front_addr":"172.21.15.98:6823/2338555672","state":["exists","up"]},{"osd":3,"uuid":"48596d91-46da-4815-b810-d493bbf01dcc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6826","nonce":2634725099},{"type":"v1","addr":"172.21.15.98:6827","nonce":2634725099}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6828","nonce":2634725099},{"type":"v1","addr":"172.21.15.98:6829","nonce":2634725099}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6832","nonce":2634725099},{"type":"v1","addr":"172.21.15.98:6833","nonce":2634725099}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6830","nonce":2634725099},{"type":"v1","addr":"172.21.15.98:6831","nonce":2634725099}]},"public_addr":"172.21.15.98:6827/2634725099","cluster_addr":"172.21.15.98:6829/2634725099","heartbeat_back_addr":"172.21.15.98:6833/2634725099","heartbeat_front_addr":"172.21.15.98:6831/2634725099","state":["exists","up"]},{"osd":4,"uuid":"a562afc6-ffe6-44bb-91a9-8a04018c3c6f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6800","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6801","nonce":165906554}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6802","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6803","nonce":165906554}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6806","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6807","nonce":165906554}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6804","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6805","nonce":165906554}]},"public_addr":"172.21.15.173:6801/165906554","cluster_addr":"172.21.15.173:6803/165906554","heartbeat_back_addr":"172.21.15.173:6807/165906554","heartbeat_front_addr":"172.21.15.173:6805/165906554","state":["exists","up"]},{"osd":5,"uuid":"add28146-8301-4b4c-9616-967e9df3a8d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6808","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6809","nonce":2894909344}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6810","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6811","nonce":2894909344}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6814","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6815","nonce":2894909344}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6812","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6813","nonce":2894909344}]},"public_addr":"172.21.15.173:6809/2894909344","cluster_addr":"172.21.15.173:6811/2894909344","heartbeat_back_addr":"172.21.15.173:6815/2894909344","heartbeat_front_addr":"172.21.15.173:6813/2894909344","state":["exists","up"]},{"osd":6,"uuid":"f30854de-bc26-465d-8b0e-a9211ae2d9fa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6816","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6817","nonce":3550013548}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6818","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6819","nonce":3550013548}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6822","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6823","nonce":3550013548}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6820","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6821","nonce":3550013548}]},"public_addr":"172.21.15.173:6817/3550013548","cluster_addr":"172.21.15.173:6819/3550013548","heartbeat_back_addr":"172.21.15.173:6823/3550013548","heartbeat_front_addr":"172.21.15.173:6821/3550013548","state":["exists","up"]},{"osd":7,"uuid":"5f540de3-48f9-4bc8-b14a-9aadf838f16b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6824","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6825","nonce":135608222}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6826","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6827","nonce":135608222}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6830","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6831","nonce":135608222}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6828","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6829","nonce":135608222}]},"public_addr":"172.21.15.173:6825/135608222","cluster_addr":"172.21.15.173:6827/135608222","heartbeat_back_addr":"172.21.15.173:6831/135608222","heartbeat_front_addr":"172.21.15.173:6829/135608222","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:07:04.739243+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:07:48.583198+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:08:33.173348+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:09:18.218502+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:09:59.765019+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:10:42.490689+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:11:24.357920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:12:08.168900+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":5}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.98:6801/483469857":"2023-09-24T18:12:42.525837+0000","172.21.15.98:6800/483469857":"2023-09-24T18:12:42.525837+0000","172.21.15.98:0/1142988558":"2023-09-24T18:12:42.525837+0000","172.21.15.98:0/3472030975":"2023-09-24T18:05:11.060631+0000","172.21.15.98:6800/1392949322":"2023-09-24T18:05:11.060631+0000","172.21.15.98:0/3976553993":"2023-09-24T18:04:40.596823+0000","172.21.15.98:0/2194193535":"2023-09-24T18:12:42.525837+0000","172.21.15.98:0/3213365423":"2023-09-24T18:04:40.596823+0000","172.21.15.98:6801/1392949322":"2023-09-24T18:05:11.060631+0000","172.21.15.98:6800/3368447653":"2023-09-24T18:04:40.596823+0000","172.21.15.98:0/1603044865":"2023-09-24T18:05:11.060631+0000","172.21.15.98:6801/3368447653":"2023-09-24T18:04:40.596823+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T18:15:45.694 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:15:45 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:15:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:15:45.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:45 smithi098 bash[33599]: cluster 2023-09-23T18:15:44.598778+0000 mgr.y (mgr.24344) 124 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:45.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:45 smithi098 bash[33599]: audit 2023-09-23T18:15:45.206647+0000 mon.c (mon.2) 86 : audit [DBG] from='client.? 172.21.15.98:0/904111834' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:15:45.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:45 smithi098 bash[27349]: cluster 2023-09-23T18:15:44.598778+0000 mgr.y (mgr.24344) 124 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:45.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:45 smithi098 bash[27349]: audit 2023-09-23T18:15:45.206647+0000 mon.c (mon.2) 86 : audit [DBG] from='client.? 172.21.15.98:0/904111834' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:15:45.990 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T18:15:45.990 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T18:15:45.991 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T18:15:45.991 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T18:15:45.992 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T18:15:45.992 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T18:15:45.992 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T18:15:45.993 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T18:15:46.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:45 smithi173 bash[27694]: cluster 2023-09-23T18:15:44.598778+0000 mgr.y (mgr.24344) 124 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:46.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:45 smithi173 bash[27694]: audit 2023-09-23T18:15:45.206647+0000 mon.c (mon.2) 86 : audit [DBG] from='client.? 172.21.15.98:0/904111834' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:15:48.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:47 smithi173 bash[27694]: cluster 2023-09-23T18:15:46.599486+0000 mgr.y (mgr.24344) 125 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:48.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:47 smithi098 bash[33599]: cluster 2023-09-23T18:15:46.599486+0000 mgr.y (mgr.24344) 125 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:48.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:47 smithi098 bash[27349]: cluster 2023-09-23T18:15:46.599486+0000 mgr.y (mgr.24344) 125 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:48.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:15:47 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:15:47] "GET /metrics HTTP/1.1" 200 191498 "" "Prometheus/2.33.4" 2023-09-23T18:15:49.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:49 smithi098 bash[33599]: cluster 2023-09-23T18:15:48.600564+0000 mgr.y (mgr.24344) 126 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:49.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:49 smithi098 bash[27349]: cluster 2023-09-23T18:15:48.600564+0000 mgr.y (mgr.24344) 126 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:49.776 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:49 smithi173 bash[27694]: cluster 2023-09-23T18:15:48.600564+0000 mgr.y (mgr.24344) 126 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:50.875 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:50.876 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:50.876 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:50.877 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:50.877 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:50.882 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:50.882 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:50.882 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:51.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:51 smithi098 bash[27349]: cluster 2023-09-23T18:15:50.601219+0000 mgr.y (mgr.24344) 127 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:51.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:51 smithi098 bash[33599]: cluster 2023-09-23T18:15:50.601219+0000 mgr.y (mgr.24344) 127 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:52.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:51 smithi173 bash[27694]: cluster 2023-09-23T18:15:50.601219+0000 mgr.y (mgr.24344) 127 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:53.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:53 smithi098 bash[33599]: cluster 2023-09-23T18:15:52.601824+0000 mgr.y (mgr.24344) 128 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:53.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:53 smithi098 bash[27349]: cluster 2023-09-23T18:15:52.601824+0000 mgr.y (mgr.24344) 128 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:54.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:53 smithi173 bash[27694]: cluster 2023-09-23T18:15:52.601824+0000 mgr.y (mgr.24344) 128 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:55.672 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:15:55 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:15:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:15:55.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:55 smithi098 bash[33599]: cluster 2023-09-23T18:15:54.602902+0000 mgr.y (mgr.24344) 129 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:55.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:55 smithi098 bash[27349]: cluster 2023-09-23T18:15:54.602902+0000 mgr.y (mgr.24344) 129 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:56.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:55 smithi173 bash[27694]: cluster 2023-09-23T18:15:54.602902+0000 mgr.y (mgr.24344) 129 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:57.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:57 smithi098 bash[27349]: cluster 2023-09-23T18:15:56.603320+0000 mgr.y (mgr.24344) 130 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:57.945 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:15:57 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:15:57] "GET /metrics HTTP/1.1" 200 191498 "" "Prometheus/2.33.4" 2023-09-23T18:15:57.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:57 smithi098 bash[33599]: cluster 2023-09-23T18:15:56.603320+0000 mgr.y (mgr.24344) 130 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:58.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:57 smithi173 bash[27694]: cluster 2023-09-23T18:15:56.603320+0000 mgr.y (mgr.24344) 130 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:59.611 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:15:59 smithi098 bash[27349]: cluster 2023-09-23T18:15:58.604307+0000 mgr.y (mgr.24344) 131 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:59.611 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:15:59 smithi098 bash[33599]: cluster 2023-09-23T18:15:58.604307+0000 mgr.y (mgr.24344) 131 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:15:59.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:15:59 smithi173 bash[27694]: cluster 2023-09-23T18:15:58.604307+0000 mgr.y (mgr.24344) 131 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:01.897 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:01 smithi098 bash[27349]: cluster 2023-09-23T18:16:00.604886+0000 mgr.y (mgr.24344) 132 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:01.898 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:01 smithi098 bash[33599]: cluster 2023-09-23T18:16:00.604886+0000 mgr.y (mgr.24344) 132 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:02.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:01 smithi173 bash[27694]: cluster 2023-09-23T18:16:00.604886+0000 mgr.y (mgr.24344) 132 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:03.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:03 smithi098 bash[27349]: cluster 2023-09-23T18:16:02.605589+0000 mgr.y (mgr.24344) 133 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:03.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:03 smithi098 bash[33599]: cluster 2023-09-23T18:16:02.605589+0000 mgr.y (mgr.24344) 133 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:04.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:03 smithi173 bash[27694]: cluster 2023-09-23T18:16:02.605589+0000 mgr.y (mgr.24344) 133 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:04.333 INFO:teuthology.orchestra.run.smithi098.stdout:154618822723 2023-09-23T18:16:04.333 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T18:16:05.223 INFO:teuthology.orchestra.run.smithi098.stdout:34359738478 2023-09-23T18:16:05.223 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T18:16:05.682 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:16:05 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:16:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:16:05.896 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:05 smithi098 bash[33599]: cluster 2023-09-23T18:16:04.606722+0000 mgr.y (mgr.24344) 134 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:05.896 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:05 smithi098 bash[27349]: cluster 2023-09-23T18:16:04.606722+0000 mgr.y (mgr.24344) 134 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:06.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:05 smithi173 bash[27694]: cluster 2023-09-23T18:16:04.606722+0000 mgr.y (mgr.24344) 134 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:07.783 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:07 smithi098 bash[27349]: cluster 2023-09-23T18:16:06.607332+0000 mgr.y (mgr.24344) 135 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:07.783 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:07 smithi098 bash[33599]: cluster 2023-09-23T18:16:06.607332+0000 mgr.y (mgr.24344) 135 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:07.944 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:16:07 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:16:07] "GET /metrics HTTP/1.1" 200 191516 "" "Prometheus/2.33.4" 2023-09-23T18:16:08.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:07 smithi173 bash[27694]: cluster 2023-09-23T18:16:06.607332+0000 mgr.y (mgr.24344) 135 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:08.230 INFO:teuthology.orchestra.run.smithi098.stdout:107374182483 2023-09-23T18:16:08.231 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T18:16:08.345 INFO:teuthology.orchestra.run.smithi098.stdout:180388626490 2023-09-23T18:16:08.346 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T18:16:08.716 INFO:teuthology.orchestra.run.smithi098.stdout:206158430258 2023-09-23T18:16:08.716 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T18:16:09.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:09 smithi098 bash[33599]: cluster 2023-09-23T18:16:08.607833+0000 mgr.y (mgr.24344) 136 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:09.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:09 smithi098 bash[27349]: cluster 2023-09-23T18:16:08.607833+0000 mgr.y (mgr.24344) 136 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:09.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:09 smithi173 bash[27694]: cluster 2023-09-23T18:16:08.607833+0000 mgr.y (mgr.24344) 136 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:10.050 INFO:teuthology.orchestra.run.smithi098.stdout:77309411420 2023-09-23T18:16:10.050 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T18:16:10.193 INFO:teuthology.orchestra.run.smithi098.stdout:55834574950 2023-09-23T18:16:10.193 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T18:16:10.278 INFO:teuthology.orchestra.run.smithi098.stdout:128849018956 2023-09-23T18:16:10.278 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T18:16:11.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:11 smithi098 bash[27349]: cluster 2023-09-23T18:16:10.608420+0000 mgr.y (mgr.24344) 137 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:11.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:11 smithi098 bash[33599]: cluster 2023-09-23T18:16:10.608420+0000 mgr.y (mgr.24344) 137 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:12.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:11 smithi173 bash[27694]: cluster 2023-09-23T18:16:10.608420+0000 mgr.y (mgr.24344) 137 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:13.009 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:13.009 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:13.010 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:13.011 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:13.011 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:13.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:13 smithi098 bash[27349]: cluster 2023-09-23T18:16:12.609049+0000 mgr.y (mgr.24344) 138 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:13.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:13 smithi098 bash[33599]: cluster 2023-09-23T18:16:12.609049+0000 mgr.y (mgr.24344) 138 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:14.018 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:14.018 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:14.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:13 smithi173 bash[27694]: cluster 2023-09-23T18:16:12.609049+0000 mgr.y (mgr.24344) 138 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:15.031 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:15.682 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:16:15 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:16:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:16:15.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:15 smithi098 bash[33599]: cluster 2023-09-23T18:16:14.609666+0000 mgr.y (mgr.24344) 139 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:15.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:15 smithi098 bash[27349]: cluster 2023-09-23T18:16:14.609666+0000 mgr.y (mgr.24344) 139 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:16.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:15 smithi173 bash[27694]: cluster 2023-09-23T18:16:14.609666+0000 mgr.y (mgr.24344) 139 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:17.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:17 smithi098 bash[27349]: cluster 2023-09-23T18:16:16.610262+0000 mgr.y (mgr.24344) 140 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:17.944 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:16:17 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:16:17] "GET /metrics HTTP/1.1" 200 191514 "" "Prometheus/2.33.4" 2023-09-23T18:16:17.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:17 smithi098 bash[33599]: cluster 2023-09-23T18:16:16.610262+0000 mgr.y (mgr.24344) 140 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:18.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:17 smithi173 bash[27694]: cluster 2023-09-23T18:16:16.610262+0000 mgr.y (mgr.24344) 140 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:19.756 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:19 smithi098 bash[33599]: cluster 2023-09-23T18:16:18.610983+0000 mgr.y (mgr.24344) 141 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:19.756 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:19 smithi098 bash[27349]: cluster 2023-09-23T18:16:18.610983+0000 mgr.y (mgr.24344) 141 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:19.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:19 smithi173 bash[27694]: cluster 2023-09-23T18:16:18.610983+0000 mgr.y (mgr.24344) 141 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:21.925 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:21 smithi098 bash[27349]: cluster 2023-09-23T18:16:20.611426+0000 mgr.y (mgr.24344) 142 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:21.925 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:21 smithi098 bash[33599]: cluster 2023-09-23T18:16:20.611426+0000 mgr.y (mgr.24344) 142 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:22.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:21 smithi173 bash[27694]: cluster 2023-09-23T18:16:20.611426+0000 mgr.y (mgr.24344) 142 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:22.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:22 smithi098 bash[27349]: audit 2023-09-23T18:16:22.417602+0000 mon.c (mon.2) 87 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:16:22.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:22 smithi098 bash[33599]: audit 2023-09-23T18:16:22.417602+0000 mon.c (mon.2) 87 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:16:23.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:22 smithi173 bash[27694]: audit 2023-09-23T18:16:22.417602+0000 mon.c (mon.2) 87 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:16:23.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:23 smithi098 bash[33599]: cluster 2023-09-23T18:16:22.611798+0000 mgr.y (mgr.24344) 143 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:23.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:23 smithi098 bash[33599]: audit 2023-09-23T18:16:22.686981+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:16:23.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:23 smithi098 bash[33599]: audit 2023-09-23T18:16:22.968666+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:16:23.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:23 smithi098 bash[33599]: audit 2023-09-23T18:16:22.969941+0000 mon.c (mon.2) 89 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:16:23.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:23 smithi098 bash[33599]: audit 2023-09-23T18:16:22.977618+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:16:23.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:23 smithi098 bash[27349]: cluster 2023-09-23T18:16:22.611798+0000 mgr.y (mgr.24344) 143 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:23.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:23 smithi098 bash[27349]: audit 2023-09-23T18:16:22.686981+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:16:23.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:23 smithi098 bash[27349]: audit 2023-09-23T18:16:22.968666+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:16:23.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:23 smithi098 bash[27349]: audit 2023-09-23T18:16:22.969941+0000 mon.c (mon.2) 89 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:16:23.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:23 smithi098 bash[27349]: audit 2023-09-23T18:16:22.977618+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:16:24.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:23 smithi173 bash[27694]: cluster 2023-09-23T18:16:22.611798+0000 mgr.y (mgr.24344) 143 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:24.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:23 smithi173 bash[27694]: audit 2023-09-23T18:16:22.686981+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:16:24.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:23 smithi173 bash[27694]: audit 2023-09-23T18:16:22.968666+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:16:24.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:23 smithi173 bash[27694]: audit 2023-09-23T18:16:22.969941+0000 mon.c (mon.2) 89 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:16:24.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:23 smithi173 bash[27694]: audit 2023-09-23T18:16:22.977618+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:16:24.561 INFO:teuthology.orchestra.run.smithi098.stdout:34359738482 2023-09-23T18:16:25.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:24 smithi173 bash[27694]: audit 2023-09-23T18:16:24.556130+0000 mon.a (mon.0) 672 : audit [DBG] from='client.? 172.21.15.98:0/2993120994' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T18:16:25.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:24 smithi098 bash[27349]: audit 2023-09-23T18:16:24.556130+0000 mon.a (mon.0) 672 : audit [DBG] from='client.? 172.21.15.98:0/2993120994' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T18:16:25.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:24 smithi098 bash[33599]: audit 2023-09-23T18:16:24.556130+0000 mon.a (mon.0) 672 : audit [DBG] from='client.? 172.21.15.98:0/2993120994' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T18:16:25.703 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:16:25 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:16:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:16:26.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:25 smithi173 bash[27694]: cluster 2023-09-23T18:16:24.612807+0000 mgr.y (mgr.24344) 144 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:26.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:25 smithi098 bash[27349]: cluster 2023-09-23T18:16:24.612807+0000 mgr.y (mgr.24344) 144 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:26.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:25 smithi098 bash[33599]: cluster 2023-09-23T18:16:24.612807+0000 mgr.y (mgr.24344) 144 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:26.668 INFO:teuthology.orchestra.run.smithi098.stdout:154618822728 2023-09-23T18:16:27.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:26 smithi173 bash[27694]: audit 2023-09-23T18:16:26.663387+0000 mon.a (mon.0) 673 : audit [DBG] from='client.? 172.21.15.98:0/4006983069' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T18:16:27.167 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:26 smithi098 bash[27349]: audit 2023-09-23T18:16:26.663387+0000 mon.a (mon.0) 673 : audit [DBG] from='client.? 172.21.15.98:0/4006983069' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T18:16:27.168 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:26 smithi098 bash[33599]: audit 2023-09-23T18:16:26.663387+0000 mon.a (mon.0) 673 : audit [DBG] from='client.? 172.21.15.98:0/4006983069' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T18:16:28.014 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:16:27 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:16:27] "GET /metrics HTTP/1.1" 200 191514 "" "Prometheus/2.33.4" 2023-09-23T18:16:28.015 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:27 smithi098 bash[33599]: cluster 2023-09-23T18:16:26.613258+0000 mgr.y (mgr.24344) 145 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:28.015 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:27 smithi098 bash[27349]: cluster 2023-09-23T18:16:26.613258+0000 mgr.y (mgr.24344) 145 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:28.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:27 smithi173 bash[27694]: cluster 2023-09-23T18:16:26.613258+0000 mgr.y (mgr.24344) 145 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:28.337 INFO:teuthology.orchestra.run.smithi098.stdout:107374182487 2023-09-23T18:16:28.925 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738478 got 34359738482 for osd.0 2023-09-23T18:16:28.925 DEBUG:teuthology.parallel:result is None 2023-09-23T18:16:29.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:28 smithi173 bash[27694]: audit 2023-09-23T18:16:28.332508+0000 mon.a (mon.0) 674 : audit [DBG] from='client.? 172.21.15.98:0/1111231176' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T18:16:29.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:28 smithi098 bash[27349]: audit 2023-09-23T18:16:28.332508+0000 mon.a (mon.0) 674 : audit [DBG] from='client.? 172.21.15.98:0/1111231176' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T18:16:29.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:28 smithi098 bash[33599]: audit 2023-09-23T18:16:28.332508+0000 mon.a (mon.0) 674 : audit [DBG] from='client.? 172.21.15.98:0/1111231176' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T18:16:29.895 INFO:teuthology.orchestra.run.smithi098.stdout:77309411425 2023-09-23T18:16:29.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:29 smithi098 bash[27349]: cluster 2023-09-23T18:16:28.614349+0000 mgr.y (mgr.24344) 146 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:29.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:29 smithi098 bash[33599]: cluster 2023-09-23T18:16:28.614349+0000 mgr.y (mgr.24344) 146 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:30.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:29 smithi173 bash[27694]: cluster 2023-09-23T18:16:28.614349+0000 mgr.y (mgr.24344) 146 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:30.571 INFO:teuthology.orchestra.run.smithi098.stdout:206158430263 2023-09-23T18:16:31.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:30 smithi173 bash[27694]: audit 2023-09-23T18:16:29.890255+0000 mon.c (mon.2) 90 : audit [DBG] from='client.? 172.21.15.98:0/3527378907' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T18:16:31.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:30 smithi173 bash[27694]: audit 2023-09-23T18:16:30.567013+0000 mon.a (mon.0) 675 : audit [DBG] from='client.? 172.21.15.98:0/1649831083' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T18:16:31.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:30 smithi098 bash[27349]: audit 2023-09-23T18:16:29.890255+0000 mon.c (mon.2) 90 : audit [DBG] from='client.? 172.21.15.98:0/3527378907' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T18:16:31.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:30 smithi098 bash[27349]: audit 2023-09-23T18:16:30.567013+0000 mon.a (mon.0) 675 : audit [DBG] from='client.? 172.21.15.98:0/1649831083' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T18:16:31.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:30 smithi098 bash[33599]: audit 2023-09-23T18:16:29.890255+0000 mon.c (mon.2) 90 : audit [DBG] from='client.? 172.21.15.98:0/3527378907' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T18:16:31.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:30 smithi098 bash[33599]: audit 2023-09-23T18:16:30.567013+0000 mon.a (mon.0) 675 : audit [DBG] from='client.? 172.21.15.98:0/1649831083' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T18:16:31.261 INFO:teuthology.orchestra.run.smithi098.stdout:128849018961 2023-09-23T18:16:31.312 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822723 got 154618822728 for osd.5 2023-09-23T18:16:31.312 DEBUG:teuthology.parallel:result is None 2023-09-23T18:16:31.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:31 smithi098 bash[27349]: cluster 2023-09-23T18:16:30.614928+0000 mgr.y (mgr.24344) 147 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:31.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:31 smithi098 bash[27349]: audit 2023-09-23T18:16:31.256027+0000 mon.c (mon.2) 91 : audit [DBG] from='client.? 172.21.15.98:0/219382221' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T18:16:31.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:31 smithi098 bash[33599]: cluster 2023-09-23T18:16:30.614928+0000 mgr.y (mgr.24344) 147 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:31.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:31 smithi098 bash[33599]: audit 2023-09-23T18:16:31.256027+0000 mon.c (mon.2) 91 : audit [DBG] from='client.? 172.21.15.98:0/219382221' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T18:16:31.951 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182483 got 107374182487 for osd.3 2023-09-23T18:16:31.951 DEBUG:teuthology.parallel:result is None 2023-09-23T18:16:32.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:31 smithi173 bash[27694]: cluster 2023-09-23T18:16:30.614928+0000 mgr.y (mgr.24344) 147 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:32.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:31 smithi173 bash[27694]: audit 2023-09-23T18:16:31.256027+0000 mon.c (mon.2) 91 : audit [DBG] from='client.? 172.21.15.98:0/219382221' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T18:16:33.098 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411420 got 77309411425 for osd.2 2023-09-23T18:16:33.098 DEBUG:teuthology.parallel:result is None 2023-09-23T18:16:33.143 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430258 got 206158430263 for osd.7 2023-09-23T18:16:33.143 DEBUG:teuthology.parallel:result is None 2023-09-23T18:16:33.463 INFO:teuthology.orchestra.run.smithi098.stdout:55834574955 2023-09-23T18:16:33.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:33 smithi098 bash[27349]: cluster 2023-09-23T18:16:32.615558+0000 mgr.y (mgr.24344) 148 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:33.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:33 smithi098 bash[27349]: audit 2023-09-23T18:16:33.458874+0000 mon.c (mon.2) 92 : audit [DBG] from='client.? 172.21.15.98:0/1798659675' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T18:16:33.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:33 smithi098 bash[33599]: cluster 2023-09-23T18:16:32.615558+0000 mgr.y (mgr.24344) 148 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:33.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:33 smithi098 bash[33599]: audit 2023-09-23T18:16:33.458874+0000 mon.c (mon.2) 92 : audit [DBG] from='client.? 172.21.15.98:0/1798659675' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T18:16:34.018 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018956 got 128849018961 for osd.4 2023-09-23T18:16:34.019 DEBUG:teuthology.parallel:result is None 2023-09-23T18:16:34.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:33 smithi173 bash[27694]: cluster 2023-09-23T18:16:32.615558+0000 mgr.y (mgr.24344) 148 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:34.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:33 smithi173 bash[27694]: audit 2023-09-23T18:16:33.458874+0000 mon.c (mon.2) 92 : audit [DBG] from='client.? 172.21.15.98:0/1798659675' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T18:16:34.277 INFO:teuthology.orchestra.run.smithi098.stdout:180388626495 2023-09-23T18:16:34.971 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574950 got 55834574955 for osd.1 2023-09-23T18:16:34.971 DEBUG:teuthology.parallel:result is None 2023-09-23T18:16:35.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:34 smithi173 bash[27694]: audit 2023-09-23T18:16:34.272644+0000 mon.a (mon.0) 676 : audit [DBG] from='client.? 172.21.15.98:0/3185231243' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T18:16:35.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:34 smithi098 bash[33599]: audit 2023-09-23T18:16:34.272644+0000 mon.a (mon.0) 676 : audit [DBG] from='client.? 172.21.15.98:0/3185231243' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T18:16:35.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:34 smithi098 bash[27349]: audit 2023-09-23T18:16:34.272644+0000 mon.a (mon.0) 676 : audit [DBG] from='client.? 172.21.15.98:0/3185231243' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T18:16:35.370 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626490 got 180388626495 for osd.6 2023-09-23T18:16:35.370 DEBUG:teuthology.parallel:result is None 2023-09-23T18:16:35.370 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T18:16:35.370 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:16:35.742 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:16:35 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:16:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:16:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:35 smithi173 bash[27694]: cluster 2023-09-23T18:16:34.616385+0000 mgr.y (mgr.24344) 149 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:36.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:35 smithi098 bash[33599]: cluster 2023-09-23T18:16:34.616385+0000 mgr.y (mgr.24344) 149 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:36.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:35 smithi098 bash[27349]: cluster 2023-09-23T18:16:34.616385+0000 mgr.y (mgr.24344) 149 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:38.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:37 smithi173 bash[27694]: cluster 2023-09-23T18:16:36.616997+0000 mgr.y (mgr.24344) 150 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:38.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:16:37 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:16:37] "GET /metrics HTTP/1.1" 200 191528 "" "Prometheus/2.33.4" 2023-09-23T18:16:38.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:37 smithi098 bash[33599]: cluster 2023-09-23T18:16:36.616997+0000 mgr.y (mgr.24344) 150 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:38.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:37 smithi098 bash[27349]: cluster 2023-09-23T18:16:36.616997+0000 mgr.y (mgr.24344) 150 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:39.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:39 smithi098 bash[27349]: cluster 2023-09-23T18:16:38.617811+0000 mgr.y (mgr.24344) 151 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:39.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:39 smithi098 bash[33599]: cluster 2023-09-23T18:16:38.617811+0000 mgr.y (mgr.24344) 151 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:39.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:39 smithi173 bash[27694]: cluster 2023-09-23T18:16:38.617811+0000 mgr.y (mgr.24344) 151 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:40.148 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:41.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:41 smithi098 bash[27349]: cluster 2023-09-23T18:16:40.618516+0000 mgr.y (mgr.24344) 152 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:41.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:41 smithi098 bash[33599]: cluster 2023-09-23T18:16:40.618516+0000 mgr.y (mgr.24344) 152 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:42.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:41 smithi173 bash[27694]: cluster 2023-09-23T18:16:40.618516+0000 mgr.y (mgr.24344) 152 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:43.440 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:16:43.442 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-23T18:16:43.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:43 smithi098 bash[33599]: cluster 2023-09-23T18:16:42.619176+0000 mgr.y (mgr.24344) 153 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:43.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:43 smithi098 bash[27349]: cluster 2023-09-23T18:16:42.619176+0000 mgr.y (mgr.24344) 153 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:43 smithi173 bash[27694]: cluster 2023-09-23T18:16:42.619176+0000 mgr.y (mgr.24344) 153 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:44.249 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":134,"stamp":"2023-09-23T18:16:42.618826+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381744,"kb_used_data":4976,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579472,"statfs":{"total":767960285184,"available":765521379328,"internally_reserved":0,"allocated":5095424,"data_stored":2165242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004210"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T18:15:43.682873+0000","last_change":"2023-09-23T18:15:43.682873+0000","last_active":"2023-09-23T18:15:43.682873+0000","last_peered":"2023-09-23T18:15:43.682873+0000","last_clean":"2023-09-23T18:15:43.682873+0000","last_became_active":"2023-09-23T18:15:43.682465+0000","last_became_peered":"2023-09-23T18:15:43.682465+0000","last_unstale":"2023-09-23T18:15:43.682873+0000","last_undegraded":"2023-09-23T18:15:43.682873+0000","last_fullsized":"2023-09-23T18:15:43.682873+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.667341+0000","last_change":"2023-09-23T18:15:08.898901+0000","last_active":"2023-09-23T18:15:42.667341+0000","last_peered":"2023-09-23T18:15:42.667341+0000","last_clean":"2023-09-23T18:15:42.667341+0000","last_became_active":"2023-09-23T18:15:08.898541+0000","last_became_peered":"2023-09-23T18:15:08.898541+0000","last_unstale":"2023-09-23T18:15:42.667341+0000","last_undegraded":"2023-09-23T18:15:42.667341+0000","last_fullsized":"2023-09-23T18:15:42.667341+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:12.307786+0000","last_change":"2023-09-23T18:15:09.151602+0000","last_active":"2023-09-23T18:15:12.307786+0000","last_peered":"2023-09-23T18:15:12.307786+0000","last_clean":"2023-09-23T18:15:12.307786+0000","last_became_active":"2023-09-23T18:15:09.151075+0000","last_became_peered":"2023-09-23T18:15:09.151075+0000","last_unstale":"2023-09-23T18:15:12.307786+0000","last_undegraded":"2023-09-23T18:15:12.307786+0000","last_fullsized":"2023-09-23T18:15:12.307786+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:15.832836+0000","last_change":"2023-09-23T18:15:08.895527+0000","last_active":"2023-09-23T18:15:15.832836+0000","last_peered":"2023-09-23T18:15:15.832836+0000","last_clean":"2023-09-23T18:15:15.832836+0000","last_became_active":"2023-09-23T18:15:08.895035+0000","last_became_peered":"2023-09-23T18:15:08.895035+0000","last_unstale":"2023-09-23T18:15:15.832836+0000","last_undegraded":"2023-09-23T18:15:15.832836+0000","last_fullsized":"2023-09-23T18:15:15.832836+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.675522+0000","last_change":"2023-09-23T18:15:09.157800+0000","last_active":"2023-09-23T18:15:42.675522+0000","last_peered":"2023-09-23T18:15:42.675522+0000","last_clean":"2023-09-23T18:15:42.675522+0000","last_became_active":"2023-09-23T18:15:09.156786+0000","last_became_peered":"2023-09-23T18:15:09.156786+0000","last_unstale":"2023-09-23T18:15:42.675522+0000","last_undegraded":"2023-09-23T18:15:42.675522+0000","last_fullsized":"2023-09-23T18:15:42.675522+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.671844+0000","last_change":"2023-09-23T18:15:08.902664+0000","last_active":"2023-09-23T18:15:42.671844+0000","last_peered":"2023-09-23T18:15:42.671844+0000","last_clean":"2023-09-23T18:15:42.671844+0000","last_became_active":"2023-09-23T18:15:08.901252+0000","last_became_peered":"2023-09-23T18:15:08.901252+0000","last_unstale":"2023-09-23T18:15:42.671844+0000","last_undegraded":"2023-09-23T18:15:42.671844+0000","last_fullsized":"2023-09-23T18:15:42.671844+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.676890+0000","last_change":"2023-09-23T18:15:08.925664+0000","last_active":"2023-09-23T18:15:42.676890+0000","last_peered":"2023-09-23T18:15:42.676890+0000","last_clean":"2023-09-23T18:15:42.676890+0000","last_became_active":"2023-09-23T18:15:08.925278+0000","last_became_peered":"2023-09-23T18:15:08.925278+0000","last_unstale":"2023-09-23T18:15:42.676890+0000","last_undegraded":"2023-09-23T18:15:42.676890+0000","last_fullsized":"2023-09-23T18:15:42.676890+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.671303+0000","last_change":"2023-09-23T18:15:08.910412+0000","last_active":"2023-09-23T18:15:42.671303+0000","last_peered":"2023-09-23T18:15:42.671303+0000","last_clean":"2023-09-23T18:15:42.671303+0000","last_became_active":"2023-09-23T18:15:08.910060+0000","last_became_peered":"2023-09-23T18:15:08.910060+0000","last_unstale":"2023-09-23T18:15:42.671303+0000","last_undegraded":"2023-09-23T18:15:42.671303+0000","last_fullsized":"2023-09-23T18:15:42.671303+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.665247+0000","last_change":"2023-09-23T18:15:08.914431+0000","last_active":"2023-09-23T18:15:42.665247+0000","last_peered":"2023-09-23T18:15:42.665247+0000","last_clean":"2023-09-23T18:15:42.665247+0000","last_became_active":"2023-09-23T18:15:08.913999+0000","last_became_peered":"2023-09-23T18:15:08.913999+0000","last_unstale":"2023-09-23T18:15:42.665247+0000","last_undegraded":"2023-09-23T18:15:42.665247+0000","last_fullsized":"2023-09-23T18:15:42.665247+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.666617+0000","last_change":"2023-09-23T18:15:08.897663+0000","last_active":"2023-09-23T18:15:42.666617+0000","last_peered":"2023-09-23T18:15:42.666617+0000","last_clean":"2023-09-23T18:15:42.666617+0000","last_became_active":"2023-09-23T18:15:08.897322+0000","last_became_peered":"2023-09-23T18:15:08.897322+0000","last_unstale":"2023-09-23T18:15:42.666617+0000","last_undegraded":"2023-09-23T18:15:42.666617+0000","last_fullsized":"2023-09-23T18:15:42.666617+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.675436+0000","last_change":"2023-09-23T18:15:09.125623+0000","last_active":"2023-09-23T18:15:42.675436+0000","last_peered":"2023-09-23T18:15:42.675436+0000","last_clean":"2023-09-23T18:15:42.675436+0000","last_became_active":"2023-09-23T18:15:09.125018+0000","last_became_peered":"2023-09-23T18:15:09.125018+0000","last_unstale":"2023-09-23T18:15:42.675436+0000","last_undegraded":"2023-09-23T18:15:42.675436+0000","last_fullsized":"2023-09-23T18:15:42.675436+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.666998+0000","last_change":"2023-09-23T18:15:08.894175+0000","last_active":"2023-09-23T18:15:42.666998+0000","last_peered":"2023-09-23T18:15:42.666998+0000","last_clean":"2023-09-23T18:15:42.666998+0000","last_became_active":"2023-09-23T18:15:08.893863+0000","last_became_peered":"2023-09-23T18:15:08.893863+0000","last_unstale":"2023-09-23T18:15:42.666998+0000","last_undegraded":"2023-09-23T18:15:42.666998+0000","last_fullsized":"2023-09-23T18:15:42.666998+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.678447+0000","last_change":"2023-09-23T18:13:13.809723+0000","last_active":"2023-09-23T18:15:42.678447+0000","last_peered":"2023-09-23T18:15:42.678447+0000","last_clean":"2023-09-23T18:15:42.678447+0000","last_became_active":"2023-09-23T18:13:13.809032+0000","last_became_peered":"2023-09-23T18:13:13.809032+0000","last_unstale":"2023-09-23T18:15:42.678447+0000","last_undegraded":"2023-09-23T18:15:42.678447+0000","last_fullsized":"2023-09-23T18:15:42.678447+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:12.305630+0000","last_change":"2023-09-23T18:15:09.151438+0000","last_active":"2023-09-23T18:15:12.305630+0000","last_peered":"2023-09-23T18:15:12.305630+0000","last_clean":"2023-09-23T18:15:12.305630+0000","last_became_active":"2023-09-23T18:15:09.150944+0000","last_became_peered":"2023-09-23T18:15:09.150944+0000","last_unstale":"2023-09-23T18:15:12.305630+0000","last_undegraded":"2023-09-23T18:15:12.305630+0000","last_fullsized":"2023-09-23T18:15:12.305630+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.670439+0000","last_change":"2023-09-23T18:13:13.929843+0000","last_active":"2023-09-23T18:15:42.670439+0000","last_peered":"2023-09-23T18:15:42.670439+0000","last_clean":"2023-09-23T18:15:42.670439+0000","last_became_active":"2023-09-23T18:13:13.929407+0000","last_became_peered":"2023-09-23T18:13:13.929407+0000","last_unstale":"2023-09-23T18:15:42.670439+0000","last_undegraded":"2023-09-23T18:15:42.670439+0000","last_fullsized":"2023-09-23T18:15:42.670439+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:15.836832+0000","last_change":"2023-09-23T18:15:08.901488+0000","last_active":"2023-09-23T18:15:15.836832+0000","last_peered":"2023-09-23T18:15:15.836832+0000","last_clean":"2023-09-23T18:15:15.836832+0000","last_became_active":"2023-09-23T18:15:08.900679+0000","last_became_peered":"2023-09-23T18:15:08.900679+0000","last_unstale":"2023-09-23T18:15:15.836832+0000","last_undegraded":"2023-09-23T18:15:15.836832+0000","last_fullsized":"2023-09-23T18:15:15.836832+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.666499+0000","last_change":"2023-09-23T18:15:10.322054+0000","last_active":"2023-09-23T18:15:42.666499+0000","last_peered":"2023-09-23T18:15:42.666499+0000","last_clean":"2023-09-23T18:15:42.666499+0000","last_became_active":"2023-09-23T18:15:10.321376+0000","last_became_peered":"2023-09-23T18:15:10.321376+0000","last_unstale":"2023-09-23T18:15:42.666499+0000","last_undegraded":"2023-09-23T18:15:42.666499+0000","last_fullsized":"2023-09-23T18:15:42.666499+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.670095+0000","last_change":"2023-09-23T18:15:08.899677+0000","last_active":"2023-09-23T18:15:42.670095+0000","last_peered":"2023-09-23T18:15:42.670095+0000","last_clean":"2023-09-23T18:15:42.670095+0000","last_became_active":"2023-09-23T18:15:08.899416+0000","last_became_peered":"2023-09-23T18:15:08.899416+0000","last_unstale":"2023-09-23T18:15:42.670095+0000","last_undegraded":"2023-09-23T18:15:42.670095+0000","last_fullsized":"2023-09-23T18:15:42.670095+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.683279+0000","last_change":"2023-09-23T18:13:13.935373+0000","last_active":"2023-09-23T18:15:42.683279+0000","last_peered":"2023-09-23T18:15:42.683279+0000","last_clean":"2023-09-23T18:15:42.683279+0000","last_became_active":"2023-09-23T18:13:13.934631+0000","last_became_peered":"2023-09-23T18:13:13.934631+0000","last_unstale":"2023-09-23T18:15:42.683279+0000","last_undegraded":"2023-09-23T18:15:42.683279+0000","last_fullsized":"2023-09-23T18:15:42.683279+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.667100+0000","last_change":"2023-09-23T18:15:10.103026+0000","last_active":"2023-09-23T18:15:42.667100+0000","last_peered":"2023-09-23T18:15:42.667100+0000","last_clean":"2023-09-23T18:15:42.667100+0000","last_became_active":"2023-09-23T18:15:10.101995+0000","last_became_peered":"2023-09-23T18:15:10.101995+0000","last_unstale":"2023-09-23T18:15:42.667100+0000","last_undegraded":"2023-09-23T18:15:42.667100+0000","last_fullsized":"2023-09-23T18:15:42.667100+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.666397+0000","last_change":"2023-09-23T18:15:08.895627+0000","last_active":"2023-09-23T18:15:42.666397+0000","last_peered":"2023-09-23T18:15:42.666397+0000","last_clean":"2023-09-23T18:15:42.666397+0000","last_became_active":"2023-09-23T18:15:08.894363+0000","last_became_peered":"2023-09-23T18:15:08.894363+0000","last_unstale":"2023-09-23T18:15:42.666397+0000","last_undegraded":"2023-09-23T18:15:42.666397+0000","last_fullsized":"2023-09-23T18:15:42.666397+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.682899+0000","last_change":"2023-09-23T18:13:13.933648+0000","last_active":"2023-09-23T18:15:42.682899+0000","last_peered":"2023-09-23T18:15:42.682899+0000","last_clean":"2023-09-23T18:15:42.682899+0000","last_became_active":"2023-09-23T18:13:13.932371+0000","last_became_peered":"2023-09-23T18:13:13.932371+0000","last_unstale":"2023-09-23T18:15:42.682899+0000","last_undegraded":"2023-09-23T18:15:42.682899+0000","last_fullsized":"2023-09-23T18:15:42.682899+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.678376+0000","last_change":"2023-09-23T18:15:08.921629+0000","last_active":"2023-09-23T18:15:42.678376+0000","last_peered":"2023-09-23T18:15:42.678376+0000","last_clean":"2023-09-23T18:15:42.678376+0000","last_became_active":"2023-09-23T18:15:08.921283+0000","last_became_peered":"2023-09-23T18:15:08.921283+0000","last_unstale":"2023-09-23T18:15:42.678376+0000","last_undegraded":"2023-09-23T18:15:42.678376+0000","last_fullsized":"2023-09-23T18:15:42.678376+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:12.307943+0000","last_change":"2023-09-23T18:15:08.893175+0000","last_active":"2023-09-23T18:15:12.307943+0000","last_peered":"2023-09-23T18:15:12.307943+0000","last_clean":"2023-09-23T18:15:12.307943+0000","last_became_active":"2023-09-23T18:15:08.892794+0000","last_became_peered":"2023-09-23T18:15:08.892794+0000","last_unstale":"2023-09-23T18:15:12.307943+0000","last_undegraded":"2023-09-23T18:15:12.307943+0000","last_fullsized":"2023-09-23T18:15:12.307943+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.682818+0000","last_change":"2023-09-23T18:12:11.949514+0000","last_active":"2023-09-23T18:15:42.682818+0000","last_peered":"2023-09-23T18:15:42.682818+0000","last_clean":"2023-09-23T18:15:42.682818+0000","last_became_active":"2023-09-23T18:12:11.948568+0000","last_became_peered":"2023-09-23T18:12:11.948568+0000","last_unstale":"2023-09-23T18:15:42.682818+0000","last_undegraded":"2023-09-23T18:15:42.682818+0000","last_fullsized":"2023-09-23T18:15:42.682818+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_clean_scrub_stamp":"2023-09-23T18:08:35.701374+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.671456+0000","last_change":"2023-09-23T18:13:13.810157+0000","last_active":"2023-09-23T18:15:42.671456+0000","last_peered":"2023-09-23T18:15:42.671456+0000","last_clean":"2023-09-23T18:15:42.671456+0000","last_became_active":"2023-09-23T18:13:13.809729+0000","last_became_peered":"2023-09-23T18:13:13.809729+0000","last_unstale":"2023-09-23T18:15:42.671456+0000","last_undegraded":"2023-09-23T18:15:42.671456+0000","last_fullsized":"2023-09-23T18:15:42.671456+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.670411+0000","last_change":"2023-09-23T18:15:09.151709+0000","last_active":"2023-09-23T18:15:42.670411+0000","last_peered":"2023-09-23T18:15:42.670411+0000","last_clean":"2023-09-23T18:15:42.670411+0000","last_became_active":"2023-09-23T18:15:09.151271+0000","last_became_peered":"2023-09-23T18:15:09.151271+0000","last_unstale":"2023-09-23T18:15:42.670411+0000","last_undegraded":"2023-09-23T18:15:42.670411+0000","last_fullsized":"2023-09-23T18:15:42.670411+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.669617+0000","last_change":"2023-09-23T18:15:08.898724+0000","last_active":"2023-09-23T18:15:42.669617+0000","last_peered":"2023-09-23T18:15:42.669617+0000","last_clean":"2023-09-23T18:15:42.669617+0000","last_became_active":"2023-09-23T18:15:08.898437+0000","last_became_peered":"2023-09-23T18:15:08.898437+0000","last_unstale":"2023-09-23T18:15:42.669617+0000","last_undegraded":"2023-09-23T18:15:42.669617+0000","last_fullsized":"2023-09-23T18:15:42.669617+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.664776+0000","last_change":"2023-09-23T18:13:13.927539+0000","last_active":"2023-09-23T18:15:42.664776+0000","last_peered":"2023-09-23T18:15:42.664776+0000","last_clean":"2023-09-23T18:15:42.664776+0000","last_became_active":"2023-09-23T18:13:13.927170+0000","last_became_peered":"2023-09-23T18:13:13.927170+0000","last_unstale":"2023-09-23T18:15:42.664776+0000","last_undegraded":"2023-09-23T18:15:42.664776+0000","last_fullsized":"2023-09-23T18:15:42.664776+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.670031+0000","last_change":"2023-09-23T18:15:08.915348+0000","last_active":"2023-09-23T18:15:42.670031+0000","last_peered":"2023-09-23T18:15:42.670031+0000","last_clean":"2023-09-23T18:15:42.670031+0000","last_became_active":"2023-09-23T18:15:08.914894+0000","last_became_peered":"2023-09-23T18:15:08.914894+0000","last_unstale":"2023-09-23T18:15:42.670031+0000","last_undegraded":"2023-09-23T18:15:42.670031+0000","last_fullsized":"2023-09-23T18:15:42.670031+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.676811+0000","last_change":"2023-09-23T18:15:08.916431+0000","last_active":"2023-09-23T18:15:42.676811+0000","last_peered":"2023-09-23T18:15:42.676811+0000","last_clean":"2023-09-23T18:15:42.676811+0000","last_became_active":"2023-09-23T18:15:08.914203+0000","last_became_peered":"2023-09-23T18:15:08.914203+0000","last_unstale":"2023-09-23T18:15:42.676811+0000","last_undegraded":"2023-09-23T18:15:42.676811+0000","last_fullsized":"2023-09-23T18:15:42.676811+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.671133+0000","last_change":"2023-09-23T18:13:13.814684+0000","last_active":"2023-09-23T18:15:42.671133+0000","last_peered":"2023-09-23T18:15:42.671133+0000","last_clean":"2023-09-23T18:15:42.671133+0000","last_became_active":"2023-09-23T18:13:13.814218+0000","last_became_peered":"2023-09-23T18:13:13.814218+0000","last_unstale":"2023-09-23T18:15:42.671133+0000","last_undegraded":"2023-09-23T18:15:42.671133+0000","last_fullsized":"2023-09-23T18:15:42.671133+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.669777+0000","last_change":"2023-09-23T18:15:09.120394+0000","last_active":"2023-09-23T18:15:42.669777+0000","last_peered":"2023-09-23T18:15:42.669777+0000","last_clean":"2023-09-23T18:15:42.669777+0000","last_became_active":"2023-09-23T18:15:09.119449+0000","last_became_peered":"2023-09-23T18:15:09.119449+0000","last_unstale":"2023-09-23T18:15:42.669777+0000","last_undegraded":"2023-09-23T18:15:42.669777+0000","last_fullsized":"2023-09-23T18:15:42.669777+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.671711+0000","last_change":"2023-09-23T18:15:08.897989+0000","last_active":"2023-09-23T18:15:42.671711+0000","last_peered":"2023-09-23T18:15:42.671711+0000","last_clean":"2023-09-23T18:15:42.671711+0000","last_became_active":"2023-09-23T18:15:08.897552+0000","last_became_peered":"2023-09-23T18:15:08.897552+0000","last_unstale":"2023-09-23T18:15:42.671711+0000","last_undegraded":"2023-09-23T18:15:42.671711+0000","last_fullsized":"2023-09-23T18:15:42.671711+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.669982+0000","last_change":"2023-09-23T18:13:13.928048+0000","last_active":"2023-09-23T18:15:42.669982+0000","last_peered":"2023-09-23T18:15:42.669982+0000","last_clean":"2023-09-23T18:15:42.669982+0000","last_became_active":"2023-09-23T18:13:13.927311+0000","last_became_peered":"2023-09-23T18:13:13.927311+0000","last_unstale":"2023-09-23T18:15:42.669982+0000","last_undegraded":"2023-09-23T18:15:42.669982+0000","last_fullsized":"2023-09-23T18:15:42.669982+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.670481+0000","last_change":"2023-09-23T18:15:08.916141+0000","last_active":"2023-09-23T18:15:42.670481+0000","last_peered":"2023-09-23T18:15:42.670481+0000","last_clean":"2023-09-23T18:15:42.670481+0000","last_became_active":"2023-09-23T18:15:08.915803+0000","last_became_peered":"2023-09-23T18:15:08.915803+0000","last_unstale":"2023-09-23T18:15:42.670481+0000","last_undegraded":"2023-09-23T18:15:42.670481+0000","last_fullsized":"2023-09-23T18:15:42.670481+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:12.305728+0000","last_change":"2023-09-23T18:15:08.903482+0000","last_active":"2023-09-23T18:15:12.305728+0000","last_peered":"2023-09-23T18:15:12.305728+0000","last_clean":"2023-09-23T18:15:12.305728+0000","last_became_active":"2023-09-23T18:15:08.897800+0000","last_became_peered":"2023-09-23T18:15:08.897800+0000","last_unstale":"2023-09-23T18:15:12.305728+0000","last_undegraded":"2023-09-23T18:15:12.305728+0000","last_fullsized":"2023-09-23T18:15:12.305728+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.676004+0000","last_change":"2023-09-23T18:15:08.920582+0000","last_active":"2023-09-23T18:15:42.676004+0000","last_peered":"2023-09-23T18:15:42.676004+0000","last_clean":"2023-09-23T18:15:42.676004+0000","last_became_active":"2023-09-23T18:15:08.920214+0000","last_became_peered":"2023-09-23T18:15:08.920214+0000","last_unstale":"2023-09-23T18:15:42.676004+0000","last_undegraded":"2023-09-23T18:15:42.676004+0000","last_fullsized":"2023-09-23T18:15:42.676004+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.675519+0000","last_change":"2023-09-23T18:15:08.911348+0000","last_active":"2023-09-23T18:15:42.675519+0000","last_peered":"2023-09-23T18:15:42.675519+0000","last_clean":"2023-09-23T18:15:42.675519+0000","last_became_active":"2023-09-23T18:15:08.910442+0000","last_became_peered":"2023-09-23T18:15:08.910442+0000","last_unstale":"2023-09-23T18:15:42.675519+0000","last_undegraded":"2023-09-23T18:15:42.675519+0000","last_fullsized":"2023-09-23T18:15:42.675519+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.675862+0000","last_change":"2023-09-23T18:15:08.919838+0000","last_active":"2023-09-23T18:15:42.675862+0000","last_peered":"2023-09-23T18:15:42.675862+0000","last_clean":"2023-09-23T18:15:42.675862+0000","last_became_active":"2023-09-23T18:15:08.919523+0000","last_became_peered":"2023-09-23T18:15:08.919523+0000","last_unstale":"2023-09-23T18:15:42.675862+0000","last_undegraded":"2023-09-23T18:15:42.675862+0000","last_fullsized":"2023-09-23T18:15:42.675862+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.679121+0000","last_change":"2023-09-23T18:15:08.903211+0000","last_active":"2023-09-23T18:15:42.679121+0000","last_peered":"2023-09-23T18:15:42.679121+0000","last_clean":"2023-09-23T18:15:42.679121+0000","last_became_active":"2023-09-23T18:15:08.902886+0000","last_became_peered":"2023-09-23T18:15:08.902886+0000","last_unstale":"2023-09-23T18:15:42.679121+0000","last_undegraded":"2023-09-23T18:15:42.679121+0000","last_fullsized":"2023-09-23T18:15:42.679121+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.675718+0000","last_change":"2023-09-23T18:15:08.925430+0000","last_active":"2023-09-23T18:15:42.675718+0000","last_peered":"2023-09-23T18:15:42.675718+0000","last_clean":"2023-09-23T18:15:42.675718+0000","last_became_active":"2023-09-23T18:15:08.925105+0000","last_became_peered":"2023-09-23T18:15:08.925105+0000","last_unstale":"2023-09-23T18:15:42.675718+0000","last_undegraded":"2023-09-23T18:15:42.675718+0000","last_fullsized":"2023-09-23T18:15:42.675718+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:15.830151+0000","last_change":"2023-09-23T18:15:08.900021+0000","last_active":"2023-09-23T18:15:15.830151+0000","last_peered":"2023-09-23T18:15:15.830151+0000","last_clean":"2023-09-23T18:15:15.830151+0000","last_became_active":"2023-09-23T18:15:08.897530+0000","last_became_peered":"2023-09-23T18:15:08.897530+0000","last_unstale":"2023-09-23T18:15:15.830151+0000","last_undegraded":"2023-09-23T18:15:15.830151+0000","last_fullsized":"2023-09-23T18:15:15.830151+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T18:15:43.675185+0000","last_change":"2023-09-23T18:15:43.675185+0000","last_active":"2023-09-23T18:15:43.675185+0000","last_peered":"2023-09-23T18:15:43.675185+0000","last_clean":"2023-09-23T18:15:43.675185+0000","last_became_active":"2023-09-23T18:15:43.674701+0000","last_became_peered":"2023-09-23T18:15:43.674701+0000","last_unstale":"2023-09-23T18:15:43.675185+0000","last_undegraded":"2023-09-23T18:15:43.675185+0000","last_fullsized":"2023-09-23T18:15:43.675185+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.671185+0000","last_change":"2023-09-23T18:15:08.906344+0000","last_active":"2023-09-23T18:15:42.671185+0000","last_peered":"2023-09-23T18:15:42.671185+0000","last_clean":"2023-09-23T18:15:42.671185+0000","last_became_active":"2023-09-23T18:15:08.905995+0000","last_became_peered":"2023-09-23T18:15:08.905995+0000","last_unstale":"2023-09-23T18:15:42.671185+0000","last_undegraded":"2023-09-23T18:15:42.671185+0000","last_fullsized":"2023-09-23T18:15:42.671185+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T18:15:43.679220+0000","last_change":"2023-09-23T18:15:43.679220+0000","last_active":"2023-09-23T18:15:43.679220+0000","last_peered":"2023-09-23T18:15:43.679220+0000","last_clean":"2023-09-23T18:15:43.679220+0000","last_became_active":"2023-09-23T18:15:43.678833+0000","last_became_peered":"2023-09-23T18:15:43.678833+0000","last_unstale":"2023-09-23T18:15:43.679220+0000","last_undegraded":"2023-09-23T18:15:43.679220+0000","last_fullsized":"2023-09-23T18:15:43.679220+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.683398+0000","last_change":"2023-09-23T18:15:08.908376+0000","last_active":"2023-09-23T18:15:42.683398+0000","last_peered":"2023-09-23T18:15:42.683398+0000","last_clean":"2023-09-23T18:15:42.683398+0000","last_became_active":"2023-09-23T18:15:08.907752+0000","last_became_peered":"2023-09-23T18:15:08.907752+0000","last_unstale":"2023-09-23T18:15:42.683398+0000","last_undegraded":"2023-09-23T18:15:42.683398+0000","last_fullsized":"2023-09-23T18:15:42.683398+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.675309+0000","last_change":"2023-09-23T18:15:08.927602+0000","last_active":"2023-09-23T18:15:42.675309+0000","last_peered":"2023-09-23T18:15:42.675309+0000","last_clean":"2023-09-23T18:15:42.675309+0000","last_became_active":"2023-09-23T18:15:08.927345+0000","last_became_peered":"2023-09-23T18:15:08.927345+0000","last_unstale":"2023-09-23T18:15:42.675309+0000","last_undegraded":"2023-09-23T18:15:42.675309+0000","last_fullsized":"2023-09-23T18:15:42.675309+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.683280+0000","last_change":"2023-09-23T18:15:08.913714+0000","last_active":"2023-09-23T18:15:42.683280+0000","last_peered":"2023-09-23T18:15:42.683280+0000","last_clean":"2023-09-23T18:15:42.683280+0000","last_became_active":"2023-09-23T18:15:08.913358+0000","last_became_peered":"2023-09-23T18:15:08.913358+0000","last_unstale":"2023-09-23T18:15:42.683280+0000","last_undegraded":"2023-09-23T18:15:42.683280+0000","last_fullsized":"2023-09-23T18:15:42.683280+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.670960+0000","last_change":"2023-09-23T18:15:10.327919+0000","last_active":"2023-09-23T18:15:42.670960+0000","last_peered":"2023-09-23T18:15:42.670960+0000","last_clean":"2023-09-23T18:15:42.670960+0000","last_became_active":"2023-09-23T18:15:10.327052+0000","last_became_peered":"2023-09-23T18:15:10.327052+0000","last_unstale":"2023-09-23T18:15:42.670960+0000","last_undegraded":"2023-09-23T18:15:42.670960+0000","last_fullsized":"2023-09-23T18:15:42.670960+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.683316+0000","last_change":"2023-09-23T18:15:08.914726+0000","last_active":"2023-09-23T18:15:42.683316+0000","last_peered":"2023-09-23T18:15:42.683316+0000","last_clean":"2023-09-23T18:15:42.683316+0000","last_became_active":"2023-09-23T18:15:08.914386+0000","last_became_peered":"2023-09-23T18:15:08.914386+0000","last_unstale":"2023-09-23T18:15:42.683316+0000","last_undegraded":"2023-09-23T18:15:42.683316+0000","last_fullsized":"2023-09-23T18:15:42.683316+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:12.308339+0000","last_change":"2023-09-23T18:15:08.917055+0000","last_active":"2023-09-23T18:15:12.308339+0000","last_peered":"2023-09-23T18:15:12.308339+0000","last_clean":"2023-09-23T18:15:12.308339+0000","last_became_active":"2023-09-23T18:15:08.916758+0000","last_became_peered":"2023-09-23T18:15:08.916758+0000","last_unstale":"2023-09-23T18:15:12.308339+0000","last_undegraded":"2023-09-23T18:15:12.308339+0000","last_fullsized":"2023-09-23T18:15:12.308339+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.678852+0000","last_change":"2023-09-23T18:15:08.907176+0000","last_active":"2023-09-23T18:15:42.678852+0000","last_peered":"2023-09-23T18:15:42.678852+0000","last_clean":"2023-09-23T18:15:42.678852+0000","last_became_active":"2023-09-23T18:15:08.906773+0000","last_became_peered":"2023-09-23T18:15:08.906773+0000","last_unstale":"2023-09-23T18:15:42.678852+0000","last_undegraded":"2023-09-23T18:15:42.678852+0000","last_fullsized":"2023-09-23T18:15:42.678852+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.666825+0000","last_change":"2023-09-23T18:15:10.325214+0000","last_active":"2023-09-23T18:15:42.666825+0000","last_peered":"2023-09-23T18:15:42.666825+0000","last_clean":"2023-09-23T18:15:42.666825+0000","last_became_active":"2023-09-23T18:15:10.324440+0000","last_became_peered":"2023-09-23T18:15:10.324440+0000","last_unstale":"2023-09-23T18:15:42.666825+0000","last_undegraded":"2023-09-23T18:15:42.666825+0000","last_fullsized":"2023-09-23T18:15:42.666825+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.682826+0000","last_change":"2023-09-23T18:15:08.916658+0000","last_active":"2023-09-23T18:15:42.682826+0000","last_peered":"2023-09-23T18:15:42.682826+0000","last_clean":"2023-09-23T18:15:42.682826+0000","last_became_active":"2023-09-23T18:15:08.916301+0000","last_became_peered":"2023-09-23T18:15:08.916301+0000","last_unstale":"2023-09-23T18:15:42.682826+0000","last_undegraded":"2023-09-23T18:15:42.682826+0000","last_fullsized":"2023-09-23T18:15:42.682826+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.671576+0000","last_change":"2023-09-23T18:15:10.327408+0000","last_active":"2023-09-23T18:15:42.671576+0000","last_peered":"2023-09-23T18:15:42.671576+0000","last_clean":"2023-09-23T18:15:42.671576+0000","last_became_active":"2023-09-23T18:15:10.326853+0000","last_became_peered":"2023-09-23T18:15:10.326853+0000","last_unstale":"2023-09-23T18:15:42.671576+0000","last_undegraded":"2023-09-23T18:15:42.671576+0000","last_fullsized":"2023-09-23T18:15:42.671576+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.666678+0000","last_change":"2023-09-23T18:15:08.900373+0000","last_active":"2023-09-23T18:15:42.666678+0000","last_peered":"2023-09-23T18:15:42.666678+0000","last_clean":"2023-09-23T18:15:42.666678+0000","last_became_active":"2023-09-23T18:15:08.900036+0000","last_became_peered":"2023-09-23T18:15:08.900036+0000","last_unstale":"2023-09-23T18:15:42.666678+0000","last_undegraded":"2023-09-23T18:15:42.666678+0000","last_fullsized":"2023-09-23T18:15:42.666678+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:12.308030+0000","last_change":"2023-09-23T18:15:08.916527+0000","last_active":"2023-09-23T18:15:12.308030+0000","last_peered":"2023-09-23T18:15:12.308030+0000","last_clean":"2023-09-23T18:15:12.308030+0000","last_became_active":"2023-09-23T18:15:08.915280+0000","last_became_peered":"2023-09-23T18:15:08.915280+0000","last_unstale":"2023-09-23T18:15:12.308030+0000","last_undegraded":"2023-09-23T18:15:12.308030+0000","last_fullsized":"2023-09-23T18:15:12.308030+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.683273+0000","last_change":"2023-09-23T18:15:08.913813+0000","last_active":"2023-09-23T18:15:42.683273+0000","last_peered":"2023-09-23T18:15:42.683273+0000","last_clean":"2023-09-23T18:15:42.683273+0000","last_became_active":"2023-09-23T18:15:08.913386+0000","last_became_peered":"2023-09-23T18:15:08.913386+0000","last_unstale":"2023-09-23T18:15:42.683273+0000","last_undegraded":"2023-09-23T18:15:42.683273+0000","last_fullsized":"2023-09-23T18:15:42.683273+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.676728+0000","last_change":"2023-09-23T18:15:08.920940+0000","last_active":"2023-09-23T18:15:42.676728+0000","last_peered":"2023-09-23T18:15:42.676728+0000","last_clean":"2023-09-23T18:15:42.676728+0000","last_became_active":"2023-09-23T18:15:08.920341+0000","last_became_peered":"2023-09-23T18:15:08.920341+0000","last_unstale":"2023-09-23T18:15:42.676728+0000","last_undegraded":"2023-09-23T18:15:42.676728+0000","last_fullsized":"2023-09-23T18:15:42.676728+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":20,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.675996+0000","last_change":"2023-09-23T18:15:08.909339+0000","last_active":"2023-09-23T18:15:42.675996+0000","last_peered":"2023-09-23T18:15:42.675996+0000","last_clean":"2023-09-23T18:15:42.675996+0000","last_became_active":"2023-09-23T18:15:08.908912+0000","last_became_peered":"2023-09-23T18:15:08.908912+0000","last_unstale":"2023-09-23T18:15:42.675996+0000","last_undegraded":"2023-09-23T18:15:42.675996+0000","last_fullsized":"2023-09-23T18:15:42.675996+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.669658+0000","last_change":"2023-09-23T18:15:08.912014+0000","last_active":"2023-09-23T18:15:42.669658+0000","last_peered":"2023-09-23T18:15:42.669658+0000","last_clean":"2023-09-23T18:15:42.669658+0000","last_became_active":"2023-09-23T18:15:08.911675+0000","last_became_peered":"2023-09-23T18:15:08.911675+0000","last_unstale":"2023-09-23T18:15:42.669658+0000","last_undegraded":"2023-09-23T18:15:42.669658+0000","last_fullsized":"2023-09-23T18:15:42.669658+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.683277+0000","last_change":"2023-09-23T18:15:08.908837+0000","last_active":"2023-09-23T18:15:42.683277+0000","last_peered":"2023-09-23T18:15:42.683277+0000","last_clean":"2023-09-23T18:15:42.683277+0000","last_became_active":"2023-09-23T18:15:08.908482+0000","last_became_peered":"2023-09-23T18:15:08.908482+0000","last_unstale":"2023-09-23T18:15:42.683277+0000","last_undegraded":"2023-09-23T18:15:42.683277+0000","last_fullsized":"2023-09-23T18:15:42.683277+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:12.307685+0000","last_change":"2023-09-23T18:15:08.917123+0000","last_active":"2023-09-23T18:15:12.307685+0000","last_peered":"2023-09-23T18:15:12.307685+0000","last_clean":"2023-09-23T18:15:12.307685+0000","last_became_active":"2023-09-23T18:15:08.916702+0000","last_became_peered":"2023-09-23T18:15:08.916702+0000","last_unstale":"2023-09-23T18:15:12.307685+0000","last_undegraded":"2023-09-23T18:15:12.307685+0000","last_fullsized":"2023-09-23T18:15:12.307685+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.671400+0000","last_change":"2023-09-23T18:15:08.905115+0000","last_active":"2023-09-23T18:15:42.671400+0000","last_peered":"2023-09-23T18:15:42.671400+0000","last_clean":"2023-09-23T18:15:42.671400+0000","last_became_active":"2023-09-23T18:15:08.904751+0000","last_became_peered":"2023-09-23T18:15:08.904751+0000","last_unstale":"2023-09-23T18:15:42.671400+0000","last_undegraded":"2023-09-23T18:15:42.671400+0000","last_fullsized":"2023-09-23T18:15:42.671400+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:12.305825+0000","last_change":"2023-09-23T18:15:08.916784+0000","last_active":"2023-09-23T18:15:12.305825+0000","last_peered":"2023-09-23T18:15:12.305825+0000","last_clean":"2023-09-23T18:15:12.305825+0000","last_became_active":"2023-09-23T18:15:08.915458+0000","last_became_peered":"2023-09-23T18:15:08.915458+0000","last_unstale":"2023-09-23T18:15:12.305825+0000","last_undegraded":"2023-09-23T18:15:12.305825+0000","last_fullsized":"2023-09-23T18:15:12.305825+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.667333+0000","last_change":"2023-09-23T18:15:08.904819+0000","last_active":"2023-09-23T18:15:42.667333+0000","last_peered":"2023-09-23T18:15:42.667333+0000","last_clean":"2023-09-23T18:15:42.667333+0000","last_became_active":"2023-09-23T18:15:08.904416+0000","last_became_peered":"2023-09-23T18:15:08.904416+0000","last_unstale":"2023-09-23T18:15:42.667333+0000","last_undegraded":"2023-09-23T18:15:42.667333+0000","last_fullsized":"2023-09-23T18:15:42.667333+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.678232+0000","last_change":"2023-09-23T18:15:09.125581+0000","last_active":"2023-09-23T18:15:42.678232+0000","last_peered":"2023-09-23T18:15:42.678232+0000","last_clean":"2023-09-23T18:15:42.678232+0000","last_became_active":"2023-09-23T18:15:09.125165+0000","last_became_peered":"2023-09-23T18:15:09.125165+0000","last_unstale":"2023-09-23T18:15:42.678232+0000","last_undegraded":"2023-09-23T18:15:42.678232+0000","last_fullsized":"2023-09-23T18:15:42.678232+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:12.305913+0000","last_change":"2023-09-23T18:15:08.896913+0000","last_active":"2023-09-23T18:15:12.305913+0000","last_peered":"2023-09-23T18:15:12.305913+0000","last_clean":"2023-09-23T18:15:12.305913+0000","last_became_active":"2023-09-23T18:15:08.896610+0000","last_became_peered":"2023-09-23T18:15:08.896610+0000","last_unstale":"2023-09-23T18:15:12.305913+0000","last_undegraded":"2023-09-23T18:15:12.305913+0000","last_fullsized":"2023-09-23T18:15:12.305913+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.666906+0000","last_change":"2023-09-23T18:15:10.324426+0000","last_active":"2023-09-23T18:15:42.666906+0000","last_peered":"2023-09-23T18:15:42.666906+0000","last_clean":"2023-09-23T18:15:42.666906+0000","last_became_active":"2023-09-23T18:15:10.324092+0000","last_became_peered":"2023-09-23T18:15:10.324092+0000","last_unstale":"2023-09-23T18:15:42.666906+0000","last_undegraded":"2023-09-23T18:15:42.666906+0000","last_fullsized":"2023-09-23T18:15:42.666906+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.669471+0000","last_change":"2023-09-23T18:15:08.893438+0000","last_active":"2023-09-23T18:15:42.669471+0000","last_peered":"2023-09-23T18:15:42.669471+0000","last_clean":"2023-09-23T18:15:42.669471+0000","last_became_active":"2023-09-23T18:15:08.893078+0000","last_became_peered":"2023-09-23T18:15:08.893078+0000","last_unstale":"2023-09-23T18:15:42.669471+0000","last_undegraded":"2023-09-23T18:15:42.669471+0000","last_fullsized":"2023-09-23T18:15:42.669471+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:12.308186+0000","last_change":"2023-09-23T18:15:08.922234+0000","last_active":"2023-09-23T18:15:12.308186+0000","last_peered":"2023-09-23T18:15:12.308186+0000","last_clean":"2023-09-23T18:15:12.308186+0000","last_became_active":"2023-09-23T18:15:08.921890+0000","last_became_peered":"2023-09-23T18:15:08.921890+0000","last_unstale":"2023-09-23T18:15:12.308186+0000","last_undegraded":"2023-09-23T18:15:12.308186+0000","last_fullsized":"2023-09-23T18:15:12.308186+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.675795+0000","last_change":"2023-09-23T18:15:08.900528+0000","last_active":"2023-09-23T18:15:42.675795+0000","last_peered":"2023-09-23T18:15:42.675795+0000","last_clean":"2023-09-23T18:15:42.675795+0000","last_became_active":"2023-09-23T18:15:08.900179+0000","last_became_peered":"2023-09-23T18:15:08.900179+0000","last_unstale":"2023-09-23T18:15:42.675795+0000","last_undegraded":"2023-09-23T18:15:42.675795+0000","last_fullsized":"2023-09-23T18:15:42.675795+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430265,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297736,"kb_used_data":640,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447416,"statfs":{"total":95995035648,"available":95690153984,"internally_reserved":0,"allocated":655360,"data_stored":271700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.89400000000000002,"15min":0.89300000000000002},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.198,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":1.169},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.93200000000000005,"15min":0.92600000000000005},"min":{"1min":0.56599999999999995,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.292,"5min":1.411,"15min":1.411},"last":1.0069999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.89100000000000001,"15min":0.88400000000000001},"min":{"1min":0.63600000000000001,"5min":0.499,"15min":0.499},"max":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.753},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.84999999999999998,"15min":0.82799999999999996},"min":{"1min":0.66000000000000003,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.22,"5min":1.286,"15min":1.286},"last":0.96699999999999997}]},{"osd":2,"last update":"Sat Sep 23 18:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":0.95399999999999996,"15min":0.96299999999999997},"min":{"1min":0.60799999999999998,"5min":0.311,"15min":0.311},"max":{"1min":1.2769999999999999,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.96199999999999997,"5min":0.92000000000000004,"15min":0.89900000000000002},"min":{"1min":0.71399999999999997,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.24,"5min":1.278,"15min":1.278},"last":0.82299999999999995}]},{"osd":3,"last update":"Sat Sep 23 18:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.96799999999999997,"15min":0.96799999999999997},"min":{"1min":0.55900000000000005,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.208,"5min":1.3560000000000001,"15min":1.3560000000000001},"last":1.095},{"interface":"front","average":{"1min":0.99199999999999999,"5min":0.96299999999999997,"15min":0.94699999999999995},"min":{"1min":0.76200000000000001,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.363,"5min":1.363,"15min":1.363},"last":0.90900000000000003}]},{"osd":4,"last update":"Sat Sep 23 18:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.90400000000000003,"15min":0.89700000000000002},"min":{"1min":0.73899999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.2969999999999999,"5min":1.5249999999999999,"15min":1.5249999999999999},"last":1.0680000000000001},{"interface":"front","average":{"1min":1.0740000000000001,"5min":0.995,"15min":0.96899999999999997},"min":{"1min":0.54300000000000004,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.3999999999999999,"5min":1.427,"15min":1.427},"last":1.254}]},{"osd":5,"last update":"Sat Sep 23 18:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":0.95999999999999996,"15min":0.92900000000000005},"min":{"1min":0.64800000000000002,"5min":0.442,"15min":0.442},"max":{"1min":1.369,"5min":1.4470000000000001,"15min":1.4470000000000001},"last":1.0429999999999999},{"interface":"front","average":{"1min":0.995,"5min":0.99199999999999999,"15min":0.96799999999999997},"min":{"1min":0.49199999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.417,"5min":1.417,"15min":1.417},"last":1.1200000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:16:12 2023","interfaces":[{"interface":"back","average":{"1min":1.073,"5min":1.0209999999999999,"15min":0.98499999999999999},"min":{"1min":0.54000000000000004,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.4379999999999999,"5min":1.5229999999999999,"15min":1.5229999999999999},"last":1.1859999999999999},{"interface":"front","average":{"1min":1.161,"5min":1.137,"15min":1.119},"min":{"1min":0.71899999999999997,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.4610000000000001,"5min":1.554,"15min":1.554},"last":1.2110000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626497,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297720,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447432,"statfs":{"total":95995035648,"available":95690170368,"internally_reserved":0,"allocated":638976,"data_stored":271239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:16:35 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":0.83599999999999997,"15min":0.85199999999999998},"min":{"1min":0.61899999999999999,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.0569999999999999,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.91700000000000004,"5min":0.85099999999999998,"15min":0.89900000000000002},"min":{"1min":0.66700000000000004,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.1399999999999999,"5min":1.268,"15min":1.268},"last":1.1399999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:16:35 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.89300000000000002},"min":{"1min":0.65900000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.137,"5min":1.2370000000000001,"15min":1.2370000000000001},"last":0.995},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.88400000000000001,"15min":0.92000000000000004},"min":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.151,"5min":1.726,"15min":1.726},"last":0.88600000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:16:35 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.88800000000000001,"15min":0.89000000000000001},"min":{"1min":0.61399999999999999,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.117,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":1.044},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.91500000000000004,"15min":0.90400000000000003},"min":{"1min":0.67400000000000004,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.208,"5min":1.538,"15min":1.538},"last":1.1970000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:16:35 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.90900000000000003,"15min":0.93500000000000005},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.097,"5min":1.264,"15min":1.264},"last":1.0940000000000001},{"interface":"front","average":{"1min":0.997,"5min":0.97899999999999998,"15min":1.0069999999999999},"min":{"1min":0.755,"5min":0.66800000000000004,"15min":0.66800000000000004},"max":{"1min":1.2310000000000001,"5min":1.7949999999999999,"15min":1.7949999999999999},"last":1.109}]},{"osd":4,"last update":"Sat Sep 23 18:16:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":0.90600000000000003,"15min":0.85899999999999999},"min":{"1min":0.78400000000000003,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.292,"5min":1.4790000000000001,"15min":1.4790000000000001},"last":0.92700000000000005},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.93999999999999995,"15min":0.91200000000000003},"min":{"1min":0.76800000000000002,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":1.2529999999999999,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":1.2529999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:16:35 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.0009999999999999,"15min":0.91900000000000004},"min":{"1min":0.69799999999999995,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.2490000000000001,"5min":1.6160000000000001,"15min":1.6160000000000001},"last":1.073},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.016,"15min":0.996},"min":{"1min":0.88200000000000001,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.3069999999999999,"5min":1.7649999999999999,"15min":1.7649999999999999},"last":1.1739999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:16:14 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.0469999999999999,"15min":1.0309999999999999},"min":{"1min":0.94799999999999995,"5min":0.58899999999999997,"15min":0.58899999999999997},"max":{"1min":1.3049999999999999,"5min":1.403,"15min":1.403},"last":1.224},{"interface":"front","average":{"1min":1.2370000000000001,"5min":1.105,"15min":1.069},"min":{"1min":0.91900000000000004,"5min":0.69299999999999995,"15min":0.69299999999999995},"max":{"1min":1.4059999999999999,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":1.2849999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574957,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":269948,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:16:01 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.69299999999999995,"15min":0.58399999999999996},"min":{"1min":0.504,"5min":0.38500000000000001,"15min":0.34599999999999997},"max":{"1min":1.1359999999999999,"5min":1.641,"15min":1.641},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.69999999999999996,"15min":0.59499999999999997},"min":{"1min":0.39700000000000002,"5min":0.35799999999999998,"15min":0.32500000000000001},"max":{"1min":0.91900000000000004,"5min":1.6859999999999999,"15min":1.6859999999999999},"last":0.71199999999999997}]},{"osd":2,"last update":"Sat Sep 23 18:15:47 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":0.71399999999999997,"15min":0.67400000000000004},"min":{"1min":0.33600000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.0509999999999999,"5min":1.49,"15min":1.49},"last":0.83199999999999996},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.80100000000000005,"15min":0.68899999999999995},"min":{"1min":0.47799999999999998,"5min":0.43099999999999999,"15min":0.40699999999999997},"max":{"1min":1.2210000000000001,"5min":2.3290000000000002,"15min":2.3290000000000002},"last":0.86899999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:16:34 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.752,"15min":0.77700000000000002},"min":{"1min":0.41599999999999998,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.95999999999999996,"5min":1.613,"15min":1.613},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.83099999999999996,"15min":0.80500000000000005},"min":{"1min":0.43099999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.0549999999999999,"5min":2.4940000000000002,"15min":2.4940000000000002},"last":1.052}]},{"osd":4,"last update":"Sat Sep 23 18:16:16 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.93200000000000005,"15min":0.88300000000000001},"min":{"1min":0.65100000000000002,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.1930000000000001,"5min":1.8260000000000001,"15min":1.8260000000000001},"last":0.89300000000000002},{"interface":"front","average":{"1min":0.98499999999999999,"5min":0.998,"15min":0.90800000000000003},"min":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.3700000000000001,"5min":1.601,"15min":1.601},"last":1.1240000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:15:53 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.94099999999999995,"15min":0.94599999999999995},"min":{"1min":0.67400000000000004,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.3899999999999999,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":1.2},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.97799999999999998,"15min":0.95899999999999996},"min":{"1min":0.77100000000000002,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.24,"5min":1.73,"15min":1.73},"last":0.92900000000000005}]},{"osd":6,"last update":"Sat Sep 23 18:16:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.0960000000000001,"15min":1.1499999999999999},"min":{"1min":0.80200000000000005,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.421,"5min":1.51,"15min":1.51},"last":1.28},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.0429999999999999,"15min":1.099},"min":{"1min":0.63500000000000001,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.4399999999999999,"5min":1.556,"15min":1.556},"last":1.0900000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:16:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.077,"15min":1.1140000000000001},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.5309999999999999,"5min":1.7669999999999999,"15min":1.7669999999999999},"last":1.1539999999999999},{"interface":"front","average":{"1min":1.139,"5min":1.1299999999999999,"15min":1.143},"min":{"1min":0.88600000000000001,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.3140000000000001,"5min":1.53,"15min":1.53},"last":1.325}]}]},{"osd":0,"up_from":8,"seq":34359738486,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:16:07 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":0.69299999999999995,"15min":0.60399999999999998},"min":{"1min":0.37,"5min":0.317,"15min":0.317},"max":{"1min":1.268,"5min":1.268,"15min":1.268},"last":0.89400000000000002},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.71599999999999997,"15min":0.626},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.34799999999999998},"max":{"1min":1.1739999999999999,"5min":1.1739999999999999,"15min":1.1739999999999999},"last":0.78000000000000003}]},{"osd":2,"last update":"Sat Sep 23 18:15:58 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.72299999999999998,"15min":0.66100000000000003},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":1.018,"5min":1.1040000000000001,"15min":1.1040000000000001},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.70599999999999996,"5min":0.753,"15min":0.67500000000000004},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.0609999999999999,"5min":1.0609999999999999,"15min":1.0609999999999999},"last":0.73299999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:16:33 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.74299999999999999,"15min":0.72599999999999998},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.93700000000000006,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":0.98699999999999999},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.76200000000000001,"15min":0.74299999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.874,"5min":1.0529999999999999,"15min":1.0529999999999999},"last":0.84999999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:16:13 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":0.92800000000000005,"15min":0.91300000000000003},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.405,"5min":1.407,"15min":1.407},"last":1.05},{"interface":"front","average":{"1min":1.0029999999999999,"5min":0.98199999999999998,"15min":0.94199999999999995},"min":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.5800000000000001,"5min":1.5800000000000001,"15min":1.5800000000000001},"last":1.2390000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:15:58 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.0009999999999999,"15min":1.0069999999999999},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.448,"5min":1.448,"15min":1.448},"last":1.403},{"interface":"front","average":{"1min":1.026,"5min":0.997,"15min":0.99099999999999999},"min":{"1min":0.66100000000000003,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.3069999999999999,"5min":1.419,"15min":1.419},"last":1.2549999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:16:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":1.0569999999999999,"15min":1.0009999999999999},"min":{"1min":0.76100000000000001,"5min":0.70899999999999996,"15min":0.70899999999999996},"max":{"1min":1.502,"5min":3.2309999999999999,"15min":3.2309999999999999},"last":1.2729999999999999},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.133,"15min":1.151},"min":{"1min":0.77800000000000002,"5min":0.72599999999999998,"15min":0.72599999999999998},"max":{"1min":1.2989999999999999,"5min":3.2090000000000001,"15min":3.2090000000000001},"last":1.4399999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:16:13 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":1.121,"15min":1.127},"min":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"max":{"1min":1.528,"5min":1.538,"15min":1.538},"last":1.359},{"interface":"front","average":{"1min":1.0609999999999999,"5min":1.0640000000000001,"15min":1.048},"min":{"1min":0.72699999999999998,"5min":0.71699999999999997,"15min":0.71699999999999997},"max":{"1min":1.4139999999999999,"5min":1.5920000000000001,"15min":1.5920000000000001},"last":1.349}]}]},{"osd":2,"up_from":18,"seq":77309411427,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:15:56 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.67000000000000004,"15min":0.61599999999999999},"min":{"1min":0.29299999999999998,"5min":0.247,"15min":0.247},"max":{"1min":0.93999999999999995,"5min":2.23,"15min":2.23},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.72599999999999998,"15min":0.61699999999999999},"min":{"1min":0.375,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.96799999999999997,"5min":2.4420000000000002,"15min":2.4420000000000002},"last":0.872}]},{"osd":1,"last update":"Sat Sep 23 18:15:56 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.77300000000000002,"15min":0.68100000000000005},"min":{"1min":0.41599999999999998,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.96499999999999997,"5min":1.5580000000000001,"15min":1.5580000000000001},"last":0.82599999999999996},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.86399999999999999,"15min":0.76100000000000001},"min":{"1min":0.45200000000000001,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.0429999999999999,"5min":1.9399999999999999,"15min":1.9399999999999999},"last":0.74399999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:16:32 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.754,"15min":0.754},"min":{"1min":0.46800000000000003,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.036,"5min":2.012,"15min":2.012},"last":1.0620000000000001},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.73199999999999998,"15min":0.73499999999999999},"min":{"1min":0.47599999999999998,"5min":0.373,"15min":0.373},"max":{"1min":0.96199999999999997,"5min":1.522,"15min":1.522},"last":0.77500000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:16:16 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.93500000000000005,"15min":0.89300000000000002},"min":{"1min":0.72999999999999998,"5min":0.61099999999999999,"15min":0.61099999999999999},"max":{"1min":1.228,"5min":1.5629999999999999,"15min":1.5629999999999999},"last":0.97899999999999998},{"interface":"front","average":{"1min":0.98599999999999999,"5min":0.95799999999999996,"15min":0.90700000000000003},"min":{"1min":0.628,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.1970000000000001,"5min":1.6160000000000001,"15min":1.6160000000000001},"last":0.93500000000000005}]},{"osd":5,"last update":"Sat Sep 23 18:15:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"min":{"1min":0.69699999999999995,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.355,"5min":1.7589999999999999,"15min":1.7589999999999999},"last":1.008},{"interface":"front","average":{"1min":0.999,"5min":0.997,"15min":0.99299999999999999},"min":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.3200000000000001,"5min":1.6950000000000001,"15min":1.6950000000000001},"last":0.90900000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:16:32 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.0389999999999999,"15min":1.044},"min":{"1min":0.80700000000000005,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":1.514,"5min":1.5209999999999999,"15min":1.5209999999999999},"last":1.036},{"interface":"front","average":{"1min":1.044,"5min":1.0740000000000001,"15min":1.1359999999999999},"min":{"1min":0.66900000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.383,"5min":1.615,"15min":1.615},"last":1.107}]},{"osd":7,"last update":"Sat Sep 23 18:16:16 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.0549999999999999,"15min":1.0589999999999999},"min":{"1min":0.62,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.55,"5min":1.55,"15min":1.55},"last":1.1679999999999999},{"interface":"front","average":{"1min":1.1379999999999999,"5min":1.1299999999999999,"15min":1.157},"min":{"1min":0.78900000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.484,"5min":1.6870000000000001,"15min":1.6870000000000001},"last":1.1499999999999999}]}]},{"osd":3,"up_from":25,"seq":107374182490,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":269723,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:16:29 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.63,"15min":0.621},"min":{"1min":0.33900000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.90700000000000003,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.67700000000000005,"15min":0.70099999999999996},"min":{"1min":0.40200000000000002,"5min":0.25,"15min":0.25},"max":{"1min":2.5920000000000001,"5min":2.5920000000000001,"15min":2.5920000000000001},"last":0.76200000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:16:29 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.70799999999999996,"15min":0.73799999999999999},"min":{"1min":0.45400000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.91000000000000003,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":1.0169999999999999},{"interface":"front","average":{"1min":0.73699999999999999,"5min":0.67900000000000005,"15min":0.68100000000000005},"min":{"1min":0.42599999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.379,"5min":2.379,"15min":2.379},"last":0.80700000000000005}]},{"osd":2,"last update":"Sat Sep 23 18:16:29 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":0.76500000000000001,"15min":0.80300000000000005},"min":{"1min":0.497,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.0409999999999999,"5min":1.167,"15min":1.1819999999999999},"last":0.90200000000000002},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.75800000000000001,"15min":0.76700000000000002},"min":{"1min":0.499,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.0289999999999999,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":1.0429999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:16:13 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.92800000000000005,"15min":0.91100000000000003},"min":{"1min":0.66700000000000004,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.476,"5min":1.7410000000000001,"15min":1.7410000000000001},"last":1.1399999999999999},{"interface":"front","average":{"1min":1.0589999999999999,"5min":0.97399999999999998,"15min":0.93300000000000005},"min":{"1min":0.72099999999999997,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.413,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":1.077}]},{"osd":5,"last update":"Sat Sep 23 18:15:54 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":0.97299999999999998,"15min":0.97599999999999998},"min":{"1min":0.755,"5min":0.58899999999999997,"15min":0.58899999999999997},"max":{"1min":1.2270000000000001,"5min":1.361,"15min":1.361},"last":1.123},{"interface":"front","average":{"1min":0.95599999999999996,"5min":0.95499999999999996,"15min":0.96699999999999997},"min":{"1min":0.67500000000000004,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":1.4450000000000001,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":1.0049999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:16:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.028,"15min":1.0740000000000001},"min":{"1min":0.77400000000000002,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.351,"5min":1.512,"15min":1.512},"last":1.2509999999999999},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.02,"15min":1.05},"min":{"1min":0.66100000000000003,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.579,"5min":1.6699999999999999,"15min":1.6699999999999999},"last":1.381}]},{"osd":7,"last update":"Sat Sep 23 18:16:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.048,"15min":1.054},"min":{"1min":0.61399999999999999,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.514,"5min":1.595,"15min":1.595},"last":1.3100000000000001},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.0600000000000001,"15min":1.0680000000000001},"min":{"1min":0.67700000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.4299999999999999,"5min":1.454,"15min":1.454},"last":1.21}]}]},{"osd":4,"up_from":30,"seq":128849018963,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297732,"kb_used_data":636,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447420,"statfs":{"total":95995035648,"available":95690158080,"internally_reserved":0,"allocated":651264,"data_stored":271630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.84399999999999997,"15min":0.84599999999999997},"min":{"1min":0.54000000000000004,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.143,"5min":1.3029999999999999,"15min":1.3029999999999999},"last":0.89200000000000002},{"interface":"front","average":{"1min":0.96099999999999997,"5min":0.88600000000000001,"15min":0.83699999999999997},"min":{"1min":0.57999999999999996,"5min":0.49199999999999999,"15min":0.46300000000000002},"max":{"1min":1.3149999999999999,"5min":1.3149999999999999,"15min":1.3149999999999999},"last":1.3740000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":0.84799999999999998,"15min":0.85199999999999998},"min":{"1min":0.64600000000000002,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.0940000000000001,"5min":1.304,"15min":1.304},"last":0.99199999999999999},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.86099999999999999,"15min":0.83899999999999997},"min":{"1min":0.55000000000000004,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.1759999999999999,"5min":1.254,"15min":1.254},"last":0.79800000000000004}]},{"osd":2,"last update":"Sat Sep 23 18:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.88200000000000001,"15min":0.875},"min":{"1min":0.71199999999999997,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.1759999999999999,"5min":1.2789999999999999,"15min":1.2789999999999999},"last":0.94599999999999995},{"interface":"front","average":{"1min":1.046,"5min":0.92600000000000005,"15min":0.90700000000000003},"min":{"1min":0.67000000000000004,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":2.1789999999999998,"5min":2.1789999999999998,"15min":2.1789999999999998},"last":1.3129999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":0.93999999999999995,"15min":0.85899999999999999},"min":{"1min":0.67100000000000004,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.732,"5min":1.732,"15min":1.732},"last":0.86499999999999999},{"interface":"front","average":{"1min":0.88100000000000001,"5min":0.879,"15min":0.92000000000000004},"min":{"1min":0.69199999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.1120000000000001,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":1.0580000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:16:00 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":0.84999999999999998,"15min":0.77000000000000002},"min":{"1min":0.70999999999999996,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.2110000000000001,"5min":1.254,"15min":1.254},"last":0.76400000000000001},{"interface":"front","average":{"1min":0.91800000000000004,"5min":0.876,"15min":0.82599999999999996},"min":{"1min":0.67500000000000004,"5min":0.443,"15min":0.443},"max":{"1min":1.1180000000000001,"5min":1.333,"15min":1.333},"last":1.26}]},{"osd":6,"last update":"Sat Sep 23 18:16:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":0.97999999999999998,"15min":0.95499999999999996},"min":{"1min":0.61599999999999999,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.3700000000000001,"5min":1.3700000000000001,"15min":1.3700000000000001},"last":1.1830000000000001},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.93500000000000005,"15min":0.90300000000000002},"min":{"1min":0.50700000000000001,"5min":0.434,"15min":0.434},"max":{"1min":1.2529999999999999,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":0.91300000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":0.996,"15min":0.95399999999999996},"min":{"1min":0.81100000000000005,"5min":0.53400000000000003,"15min":0.53400000000000003},"max":{"1min":1.341,"5min":1.7010000000000001,"15min":1.7010000000000001},"last":1.2949999999999999},{"interface":"front","average":{"1min":1.204,"5min":1.0840000000000001,"15min":1.02},"min":{"1min":0.99299999999999999,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.379,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":1.4239999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822731,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":269942,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:15:51 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.879,"15min":0.89500000000000002},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":1.2769999999999999,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":1.1830000000000001},{"interface":"front","average":{"1min":0.878,"5min":0.86199999999999999,"15min":0.86499999999999999},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":1.1890000000000001,"5min":1.3480000000000001,"15min":1.3480000000000001},"last":0.99399999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:15:51 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":0.874,"15min":0.878},"min":{"1min":0.625,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.3340000000000001,"5min":1.387,"15min":1.387},"last":0.84299999999999997},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.90400000000000003,"15min":0.90500000000000003},"min":{"1min":0.68600000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.101,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":1.194}]},{"osd":2,"last update":"Sat Sep 23 18:15:51 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":0.89400000000000002,"15min":0.88500000000000001},"min":{"1min":0.63700000000000001,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.266,"5min":1.266,"15min":1.266},"last":1.0609999999999999},{"interface":"front","average":{"1min":0.99199999999999999,"5min":0.93300000000000005,"15min":0.95599999999999996},"min":{"1min":0.56999999999999995,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.2669999999999999,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":1.038}]},{"osd":3,"last update":"Sat Sep 23 18:15:51 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":0.91400000000000003,"15min":0.93000000000000005},"min":{"1min":0.76900000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.206,"5min":1.333,"15min":1.333},"last":0.879},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.97999999999999998,"15min":0.97799999999999998},"min":{"1min":0.64800000000000002,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.284,"5min":2.0259999999999998,"15min":2.0259999999999998},"last":0.93500000000000005}]},{"osd":4,"last update":"Sat Sep 23 18:15:51 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.875,"15min":0.83499999999999996},"min":{"1min":0.63200000000000001,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.256,"5min":1.7509999999999999,"15min":1.7509999999999999},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.96599999999999997,"5min":0.88800000000000001,"15min":0.84099999999999997},"min":{"1min":0.66300000000000003,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.163,"5min":1.278,"15min":1.278},"last":0.751}]},{"osd":6,"last update":"Sat Sep 23 18:16:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":0.97799999999999998,"15min":0.92900000000000005},"min":{"1min":0.65900000000000003,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.1930000000000001,"5min":1.8460000000000001,"15min":1.8460000000000001},"last":0.90200000000000002},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.0409999999999999,"15min":0.98999999999999999},"min":{"1min":0.73699999999999999,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.306,"5min":1.4299999999999999,"15min":1.4299999999999999},"last":0.96599999999999997}]},{"osd":7,"last update":"Sat Sep 23 18:16:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":0.98599999999999999,"15min":0.91000000000000003},"min":{"1min":0.69599999999999995,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.343,"5min":1.472,"15min":1.472},"last":1.1339999999999999},{"interface":"front","average":{"1min":1.113,"5min":1.036,"15min":0.96999999999999997},"min":{"1min":0.875,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.28,"5min":1.468,"15min":1.468},"last":1.0900000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:16:44.254 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:16:44.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:44 smithi098 bash[27349]: audit 2023-09-23T18:16:43.416193+0000 mgr.y (mgr.24344) 154 : audit [DBG] from='client.14778 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:16:44.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:44 smithi098 bash[33599]: audit 2023-09-23T18:16:43.416193+0000 mgr.y (mgr.24344) 154 : audit [DBG] from='client.14778 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:16:45.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:44 smithi173 bash[27694]: audit 2023-09-23T18:16:43.416193+0000 mgr.y (mgr.24344) 154 : audit [DBG] from='client.14778 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:16:45.700 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:16:45 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:16:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:16:46.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:45 smithi173 bash[27694]: cluster 2023-09-23T18:16:44.620403+0000 mgr.y (mgr.24344) 155 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:46.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:45 smithi098 bash[27349]: cluster 2023-09-23T18:16:44.620403+0000 mgr.y (mgr.24344) 155 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:46.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:45 smithi098 bash[33599]: cluster 2023-09-23T18:16:44.620403+0000 mgr.y (mgr.24344) 155 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:48.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:47 smithi173 bash[27694]: cluster 2023-09-23T18:16:46.621192+0000 mgr.y (mgr.24344) 156 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:48.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:47 smithi098 bash[27349]: cluster 2023-09-23T18:16:46.621192+0000 mgr.y (mgr.24344) 156 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:48.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:16:47 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:16:47] "GET /metrics HTTP/1.1" 200 191534 "" "Prometheus/2.33.4" 2023-09-23T18:16:48.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:47 smithi098 bash[33599]: cluster 2023-09-23T18:16:46.621192+0000 mgr.y (mgr.24344) 156 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:49.031 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:49.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:49 smithi098 bash[27349]: cluster 2023-09-23T18:16:48.622195+0000 mgr.y (mgr.24344) 157 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:49.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:49 smithi098 bash[33599]: cluster 2023-09-23T18:16:48.622195+0000 mgr.y (mgr.24344) 157 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:49.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:49 smithi173 bash[27694]: cluster 2023-09-23T18:16:48.622195+0000 mgr.y (mgr.24344) 157 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:51.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:51 smithi098 bash[27349]: cluster 2023-09-23T18:16:50.622827+0000 mgr.y (mgr.24344) 158 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:51.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:51 smithi098 bash[33599]: cluster 2023-09-23T18:16:50.622827+0000 mgr.y (mgr.24344) 158 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:52.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:51 smithi173 bash[27694]: cluster 2023-09-23T18:16:50.622827+0000 mgr.y (mgr.24344) 158 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:52.150 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:16:52.153 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-23T18:16:52.940 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":138,"stamp":"2023-09-23T18:16:50.622478+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381744,"kb_used_data":4976,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579472,"statfs":{"total":767960285184,"available":765521379328,"internally_reserved":0,"allocated":5095424,"data_stored":2165242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005246"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T18:15:43.682873+0000","last_change":"2023-09-23T18:15:43.682873+0000","last_active":"2023-09-23T18:15:43.682873+0000","last_peered":"2023-09-23T18:15:43.682873+0000","last_clean":"2023-09-23T18:15:43.682873+0000","last_became_active":"2023-09-23T18:15:43.682465+0000","last_became_peered":"2023-09-23T18:15:43.682465+0000","last_unstale":"2023-09-23T18:15:43.682873+0000","last_undegraded":"2023-09-23T18:15:43.682873+0000","last_fullsized":"2023-09-23T18:15:43.682873+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.667341+0000","last_change":"2023-09-23T18:15:08.898901+0000","last_active":"2023-09-23T18:15:42.667341+0000","last_peered":"2023-09-23T18:15:42.667341+0000","last_clean":"2023-09-23T18:15:42.667341+0000","last_became_active":"2023-09-23T18:15:08.898541+0000","last_became_peered":"2023-09-23T18:15:08.898541+0000","last_unstale":"2023-09-23T18:15:42.667341+0000","last_undegraded":"2023-09-23T18:15:42.667341+0000","last_fullsized":"2023-09-23T18:15:42.667341+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:12.307786+0000","last_change":"2023-09-23T18:15:09.151602+0000","last_active":"2023-09-23T18:15:12.307786+0000","last_peered":"2023-09-23T18:15:12.307786+0000","last_clean":"2023-09-23T18:15:12.307786+0000","last_became_active":"2023-09-23T18:15:09.151075+0000","last_became_peered":"2023-09-23T18:15:09.151075+0000","last_unstale":"2023-09-23T18:15:12.307786+0000","last_undegraded":"2023-09-23T18:15:12.307786+0000","last_fullsized":"2023-09-23T18:15:12.307786+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:15.832836+0000","last_change":"2023-09-23T18:15:08.895527+0000","last_active":"2023-09-23T18:15:15.832836+0000","last_peered":"2023-09-23T18:15:15.832836+0000","last_clean":"2023-09-23T18:15:15.832836+0000","last_became_active":"2023-09-23T18:15:08.895035+0000","last_became_peered":"2023-09-23T18:15:08.895035+0000","last_unstale":"2023-09-23T18:15:15.832836+0000","last_undegraded":"2023-09-23T18:15:15.832836+0000","last_fullsized":"2023-09-23T18:15:15.832836+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.675522+0000","last_change":"2023-09-23T18:15:09.157800+0000","last_active":"2023-09-23T18:15:42.675522+0000","last_peered":"2023-09-23T18:15:42.675522+0000","last_clean":"2023-09-23T18:15:42.675522+0000","last_became_active":"2023-09-23T18:15:09.156786+0000","last_became_peered":"2023-09-23T18:15:09.156786+0000","last_unstale":"2023-09-23T18:15:42.675522+0000","last_undegraded":"2023-09-23T18:15:42.675522+0000","last_fullsized":"2023-09-23T18:15:42.675522+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.671844+0000","last_change":"2023-09-23T18:15:08.902664+0000","last_active":"2023-09-23T18:15:42.671844+0000","last_peered":"2023-09-23T18:15:42.671844+0000","last_clean":"2023-09-23T18:15:42.671844+0000","last_became_active":"2023-09-23T18:15:08.901252+0000","last_became_peered":"2023-09-23T18:15:08.901252+0000","last_unstale":"2023-09-23T18:15:42.671844+0000","last_undegraded":"2023-09-23T18:15:42.671844+0000","last_fullsized":"2023-09-23T18:15:42.671844+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.676890+0000","last_change":"2023-09-23T18:15:08.925664+0000","last_active":"2023-09-23T18:15:42.676890+0000","last_peered":"2023-09-23T18:15:42.676890+0000","last_clean":"2023-09-23T18:15:42.676890+0000","last_became_active":"2023-09-23T18:15:08.925278+0000","last_became_peered":"2023-09-23T18:15:08.925278+0000","last_unstale":"2023-09-23T18:15:42.676890+0000","last_undegraded":"2023-09-23T18:15:42.676890+0000","last_fullsized":"2023-09-23T18:15:42.676890+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.671303+0000","last_change":"2023-09-23T18:15:08.910412+0000","last_active":"2023-09-23T18:15:42.671303+0000","last_peered":"2023-09-23T18:15:42.671303+0000","last_clean":"2023-09-23T18:15:42.671303+0000","last_became_active":"2023-09-23T18:15:08.910060+0000","last_became_peered":"2023-09-23T18:15:08.910060+0000","last_unstale":"2023-09-23T18:15:42.671303+0000","last_undegraded":"2023-09-23T18:15:42.671303+0000","last_fullsized":"2023-09-23T18:15:42.671303+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.665247+0000","last_change":"2023-09-23T18:15:08.914431+0000","last_active":"2023-09-23T18:15:42.665247+0000","last_peered":"2023-09-23T18:15:42.665247+0000","last_clean":"2023-09-23T18:15:42.665247+0000","last_became_active":"2023-09-23T18:15:08.913999+0000","last_became_peered":"2023-09-23T18:15:08.913999+0000","last_unstale":"2023-09-23T18:15:42.665247+0000","last_undegraded":"2023-09-23T18:15:42.665247+0000","last_fullsized":"2023-09-23T18:15:42.665247+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.666617+0000","last_change":"2023-09-23T18:15:08.897663+0000","last_active":"2023-09-23T18:15:42.666617+0000","last_peered":"2023-09-23T18:15:42.666617+0000","last_clean":"2023-09-23T18:15:42.666617+0000","last_became_active":"2023-09-23T18:15:08.897322+0000","last_became_peered":"2023-09-23T18:15:08.897322+0000","last_unstale":"2023-09-23T18:15:42.666617+0000","last_undegraded":"2023-09-23T18:15:42.666617+0000","last_fullsized":"2023-09-23T18:15:42.666617+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.675436+0000","last_change":"2023-09-23T18:15:09.125623+0000","last_active":"2023-09-23T18:15:42.675436+0000","last_peered":"2023-09-23T18:15:42.675436+0000","last_clean":"2023-09-23T18:15:42.675436+0000","last_became_active":"2023-09-23T18:15:09.125018+0000","last_became_peered":"2023-09-23T18:15:09.125018+0000","last_unstale":"2023-09-23T18:15:42.675436+0000","last_undegraded":"2023-09-23T18:15:42.675436+0000","last_fullsized":"2023-09-23T18:15:42.675436+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.666998+0000","last_change":"2023-09-23T18:15:08.894175+0000","last_active":"2023-09-23T18:15:42.666998+0000","last_peered":"2023-09-23T18:15:42.666998+0000","last_clean":"2023-09-23T18:15:42.666998+0000","last_became_active":"2023-09-23T18:15:08.893863+0000","last_became_peered":"2023-09-23T18:15:08.893863+0000","last_unstale":"2023-09-23T18:15:42.666998+0000","last_undegraded":"2023-09-23T18:15:42.666998+0000","last_fullsized":"2023-09-23T18:15:42.666998+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.678447+0000","last_change":"2023-09-23T18:13:13.809723+0000","last_active":"2023-09-23T18:15:42.678447+0000","last_peered":"2023-09-23T18:15:42.678447+0000","last_clean":"2023-09-23T18:15:42.678447+0000","last_became_active":"2023-09-23T18:13:13.809032+0000","last_became_peered":"2023-09-23T18:13:13.809032+0000","last_unstale":"2023-09-23T18:15:42.678447+0000","last_undegraded":"2023-09-23T18:15:42.678447+0000","last_fullsized":"2023-09-23T18:15:42.678447+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:12.305630+0000","last_change":"2023-09-23T18:15:09.151438+0000","last_active":"2023-09-23T18:15:12.305630+0000","last_peered":"2023-09-23T18:15:12.305630+0000","last_clean":"2023-09-23T18:15:12.305630+0000","last_became_active":"2023-09-23T18:15:09.150944+0000","last_became_peered":"2023-09-23T18:15:09.150944+0000","last_unstale":"2023-09-23T18:15:12.305630+0000","last_undegraded":"2023-09-23T18:15:12.305630+0000","last_fullsized":"2023-09-23T18:15:12.305630+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.670439+0000","last_change":"2023-09-23T18:13:13.929843+0000","last_active":"2023-09-23T18:15:42.670439+0000","last_peered":"2023-09-23T18:15:42.670439+0000","last_clean":"2023-09-23T18:15:42.670439+0000","last_became_active":"2023-09-23T18:13:13.929407+0000","last_became_peered":"2023-09-23T18:13:13.929407+0000","last_unstale":"2023-09-23T18:15:42.670439+0000","last_undegraded":"2023-09-23T18:15:42.670439+0000","last_fullsized":"2023-09-23T18:15:42.670439+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:15.836832+0000","last_change":"2023-09-23T18:15:08.901488+0000","last_active":"2023-09-23T18:15:15.836832+0000","last_peered":"2023-09-23T18:15:15.836832+0000","last_clean":"2023-09-23T18:15:15.836832+0000","last_became_active":"2023-09-23T18:15:08.900679+0000","last_became_peered":"2023-09-23T18:15:08.900679+0000","last_unstale":"2023-09-23T18:15:15.836832+0000","last_undegraded":"2023-09-23T18:15:15.836832+0000","last_fullsized":"2023-09-23T18:15:15.836832+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.666499+0000","last_change":"2023-09-23T18:15:10.322054+0000","last_active":"2023-09-23T18:15:42.666499+0000","last_peered":"2023-09-23T18:15:42.666499+0000","last_clean":"2023-09-23T18:15:42.666499+0000","last_became_active":"2023-09-23T18:15:10.321376+0000","last_became_peered":"2023-09-23T18:15:10.321376+0000","last_unstale":"2023-09-23T18:15:42.666499+0000","last_undegraded":"2023-09-23T18:15:42.666499+0000","last_fullsized":"2023-09-23T18:15:42.666499+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.670095+0000","last_change":"2023-09-23T18:15:08.899677+0000","last_active":"2023-09-23T18:15:42.670095+0000","last_peered":"2023-09-23T18:15:42.670095+0000","last_clean":"2023-09-23T18:15:42.670095+0000","last_became_active":"2023-09-23T18:15:08.899416+0000","last_became_peered":"2023-09-23T18:15:08.899416+0000","last_unstale":"2023-09-23T18:15:42.670095+0000","last_undegraded":"2023-09-23T18:15:42.670095+0000","last_fullsized":"2023-09-23T18:15:42.670095+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.683279+0000","last_change":"2023-09-23T18:13:13.935373+0000","last_active":"2023-09-23T18:15:42.683279+0000","last_peered":"2023-09-23T18:15:42.683279+0000","last_clean":"2023-09-23T18:15:42.683279+0000","last_became_active":"2023-09-23T18:13:13.934631+0000","last_became_peered":"2023-09-23T18:13:13.934631+0000","last_unstale":"2023-09-23T18:15:42.683279+0000","last_undegraded":"2023-09-23T18:15:42.683279+0000","last_fullsized":"2023-09-23T18:15:42.683279+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.667100+0000","last_change":"2023-09-23T18:15:10.103026+0000","last_active":"2023-09-23T18:15:42.667100+0000","last_peered":"2023-09-23T18:15:42.667100+0000","last_clean":"2023-09-23T18:15:42.667100+0000","last_became_active":"2023-09-23T18:15:10.101995+0000","last_became_peered":"2023-09-23T18:15:10.101995+0000","last_unstale":"2023-09-23T18:15:42.667100+0000","last_undegraded":"2023-09-23T18:15:42.667100+0000","last_fullsized":"2023-09-23T18:15:42.667100+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.666397+0000","last_change":"2023-09-23T18:15:08.895627+0000","last_active":"2023-09-23T18:15:42.666397+0000","last_peered":"2023-09-23T18:15:42.666397+0000","last_clean":"2023-09-23T18:15:42.666397+0000","last_became_active":"2023-09-23T18:15:08.894363+0000","last_became_peered":"2023-09-23T18:15:08.894363+0000","last_unstale":"2023-09-23T18:15:42.666397+0000","last_undegraded":"2023-09-23T18:15:42.666397+0000","last_fullsized":"2023-09-23T18:15:42.666397+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.682899+0000","last_change":"2023-09-23T18:13:13.933648+0000","last_active":"2023-09-23T18:15:42.682899+0000","last_peered":"2023-09-23T18:15:42.682899+0000","last_clean":"2023-09-23T18:15:42.682899+0000","last_became_active":"2023-09-23T18:13:13.932371+0000","last_became_peered":"2023-09-23T18:13:13.932371+0000","last_unstale":"2023-09-23T18:15:42.682899+0000","last_undegraded":"2023-09-23T18:15:42.682899+0000","last_fullsized":"2023-09-23T18:15:42.682899+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.678376+0000","last_change":"2023-09-23T18:15:08.921629+0000","last_active":"2023-09-23T18:15:42.678376+0000","last_peered":"2023-09-23T18:15:42.678376+0000","last_clean":"2023-09-23T18:15:42.678376+0000","last_became_active":"2023-09-23T18:15:08.921283+0000","last_became_peered":"2023-09-23T18:15:08.921283+0000","last_unstale":"2023-09-23T18:15:42.678376+0000","last_undegraded":"2023-09-23T18:15:42.678376+0000","last_fullsized":"2023-09-23T18:15:42.678376+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:12.307943+0000","last_change":"2023-09-23T18:15:08.893175+0000","last_active":"2023-09-23T18:15:12.307943+0000","last_peered":"2023-09-23T18:15:12.307943+0000","last_clean":"2023-09-23T18:15:12.307943+0000","last_became_active":"2023-09-23T18:15:08.892794+0000","last_became_peered":"2023-09-23T18:15:08.892794+0000","last_unstale":"2023-09-23T18:15:12.307943+0000","last_undegraded":"2023-09-23T18:15:12.307943+0000","last_fullsized":"2023-09-23T18:15:12.307943+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.682818+0000","last_change":"2023-09-23T18:12:11.949514+0000","last_active":"2023-09-23T18:15:42.682818+0000","last_peered":"2023-09-23T18:15:42.682818+0000","last_clean":"2023-09-23T18:15:42.682818+0000","last_became_active":"2023-09-23T18:12:11.948568+0000","last_became_peered":"2023-09-23T18:12:11.948568+0000","last_unstale":"2023-09-23T18:15:42.682818+0000","last_undegraded":"2023-09-23T18:15:42.682818+0000","last_fullsized":"2023-09-23T18:15:42.682818+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_clean_scrub_stamp":"2023-09-23T18:08:35.701374+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.671456+0000","last_change":"2023-09-23T18:13:13.810157+0000","last_active":"2023-09-23T18:15:42.671456+0000","last_peered":"2023-09-23T18:15:42.671456+0000","last_clean":"2023-09-23T18:15:42.671456+0000","last_became_active":"2023-09-23T18:13:13.809729+0000","last_became_peered":"2023-09-23T18:13:13.809729+0000","last_unstale":"2023-09-23T18:15:42.671456+0000","last_undegraded":"2023-09-23T18:15:42.671456+0000","last_fullsized":"2023-09-23T18:15:42.671456+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.670411+0000","last_change":"2023-09-23T18:15:09.151709+0000","last_active":"2023-09-23T18:15:42.670411+0000","last_peered":"2023-09-23T18:15:42.670411+0000","last_clean":"2023-09-23T18:15:42.670411+0000","last_became_active":"2023-09-23T18:15:09.151271+0000","last_became_peered":"2023-09-23T18:15:09.151271+0000","last_unstale":"2023-09-23T18:15:42.670411+0000","last_undegraded":"2023-09-23T18:15:42.670411+0000","last_fullsized":"2023-09-23T18:15:42.670411+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.669617+0000","last_change":"2023-09-23T18:15:08.898724+0000","last_active":"2023-09-23T18:15:42.669617+0000","last_peered":"2023-09-23T18:15:42.669617+0000","last_clean":"2023-09-23T18:15:42.669617+0000","last_became_active":"2023-09-23T18:15:08.898437+0000","last_became_peered":"2023-09-23T18:15:08.898437+0000","last_unstale":"2023-09-23T18:15:42.669617+0000","last_undegraded":"2023-09-23T18:15:42.669617+0000","last_fullsized":"2023-09-23T18:15:42.669617+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.664776+0000","last_change":"2023-09-23T18:13:13.927539+0000","last_active":"2023-09-23T18:15:42.664776+0000","last_peered":"2023-09-23T18:15:42.664776+0000","last_clean":"2023-09-23T18:15:42.664776+0000","last_became_active":"2023-09-23T18:13:13.927170+0000","last_became_peered":"2023-09-23T18:13:13.927170+0000","last_unstale":"2023-09-23T18:15:42.664776+0000","last_undegraded":"2023-09-23T18:15:42.664776+0000","last_fullsized":"2023-09-23T18:15:42.664776+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.670031+0000","last_change":"2023-09-23T18:15:08.915348+0000","last_active":"2023-09-23T18:15:42.670031+0000","last_peered":"2023-09-23T18:15:42.670031+0000","last_clean":"2023-09-23T18:15:42.670031+0000","last_became_active":"2023-09-23T18:15:08.914894+0000","last_became_peered":"2023-09-23T18:15:08.914894+0000","last_unstale":"2023-09-23T18:15:42.670031+0000","last_undegraded":"2023-09-23T18:15:42.670031+0000","last_fullsized":"2023-09-23T18:15:42.670031+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.676811+0000","last_change":"2023-09-23T18:15:08.916431+0000","last_active":"2023-09-23T18:15:42.676811+0000","last_peered":"2023-09-23T18:15:42.676811+0000","last_clean":"2023-09-23T18:15:42.676811+0000","last_became_active":"2023-09-23T18:15:08.914203+0000","last_became_peered":"2023-09-23T18:15:08.914203+0000","last_unstale":"2023-09-23T18:15:42.676811+0000","last_undegraded":"2023-09-23T18:15:42.676811+0000","last_fullsized":"2023-09-23T18:15:42.676811+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.671133+0000","last_change":"2023-09-23T18:13:13.814684+0000","last_active":"2023-09-23T18:15:42.671133+0000","last_peered":"2023-09-23T18:15:42.671133+0000","last_clean":"2023-09-23T18:15:42.671133+0000","last_became_active":"2023-09-23T18:13:13.814218+0000","last_became_peered":"2023-09-23T18:13:13.814218+0000","last_unstale":"2023-09-23T18:15:42.671133+0000","last_undegraded":"2023-09-23T18:15:42.671133+0000","last_fullsized":"2023-09-23T18:15:42.671133+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.669777+0000","last_change":"2023-09-23T18:15:09.120394+0000","last_active":"2023-09-23T18:15:42.669777+0000","last_peered":"2023-09-23T18:15:42.669777+0000","last_clean":"2023-09-23T18:15:42.669777+0000","last_became_active":"2023-09-23T18:15:09.119449+0000","last_became_peered":"2023-09-23T18:15:09.119449+0000","last_unstale":"2023-09-23T18:15:42.669777+0000","last_undegraded":"2023-09-23T18:15:42.669777+0000","last_fullsized":"2023-09-23T18:15:42.669777+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.671711+0000","last_change":"2023-09-23T18:15:08.897989+0000","last_active":"2023-09-23T18:15:42.671711+0000","last_peered":"2023-09-23T18:15:42.671711+0000","last_clean":"2023-09-23T18:15:42.671711+0000","last_became_active":"2023-09-23T18:15:08.897552+0000","last_became_peered":"2023-09-23T18:15:08.897552+0000","last_unstale":"2023-09-23T18:15:42.671711+0000","last_undegraded":"2023-09-23T18:15:42.671711+0000","last_fullsized":"2023-09-23T18:15:42.671711+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.669982+0000","last_change":"2023-09-23T18:13:13.928048+0000","last_active":"2023-09-23T18:15:42.669982+0000","last_peered":"2023-09-23T18:15:42.669982+0000","last_clean":"2023-09-23T18:15:42.669982+0000","last_became_active":"2023-09-23T18:13:13.927311+0000","last_became_peered":"2023-09-23T18:13:13.927311+0000","last_unstale":"2023-09-23T18:15:42.669982+0000","last_undegraded":"2023-09-23T18:15:42.669982+0000","last_fullsized":"2023-09-23T18:15:42.669982+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.670481+0000","last_change":"2023-09-23T18:15:08.916141+0000","last_active":"2023-09-23T18:15:42.670481+0000","last_peered":"2023-09-23T18:15:42.670481+0000","last_clean":"2023-09-23T18:15:42.670481+0000","last_became_active":"2023-09-23T18:15:08.915803+0000","last_became_peered":"2023-09-23T18:15:08.915803+0000","last_unstale":"2023-09-23T18:15:42.670481+0000","last_undegraded":"2023-09-23T18:15:42.670481+0000","last_fullsized":"2023-09-23T18:15:42.670481+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:12.305728+0000","last_change":"2023-09-23T18:15:08.903482+0000","last_active":"2023-09-23T18:15:12.305728+0000","last_peered":"2023-09-23T18:15:12.305728+0000","last_clean":"2023-09-23T18:15:12.305728+0000","last_became_active":"2023-09-23T18:15:08.897800+0000","last_became_peered":"2023-09-23T18:15:08.897800+0000","last_unstale":"2023-09-23T18:15:12.305728+0000","last_undegraded":"2023-09-23T18:15:12.305728+0000","last_fullsized":"2023-09-23T18:15:12.305728+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.676004+0000","last_change":"2023-09-23T18:15:08.920582+0000","last_active":"2023-09-23T18:15:42.676004+0000","last_peered":"2023-09-23T18:15:42.676004+0000","last_clean":"2023-09-23T18:15:42.676004+0000","last_became_active":"2023-09-23T18:15:08.920214+0000","last_became_peered":"2023-09-23T18:15:08.920214+0000","last_unstale":"2023-09-23T18:15:42.676004+0000","last_undegraded":"2023-09-23T18:15:42.676004+0000","last_fullsized":"2023-09-23T18:15:42.676004+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.675519+0000","last_change":"2023-09-23T18:15:08.911348+0000","last_active":"2023-09-23T18:15:42.675519+0000","last_peered":"2023-09-23T18:15:42.675519+0000","last_clean":"2023-09-23T18:15:42.675519+0000","last_became_active":"2023-09-23T18:15:08.910442+0000","last_became_peered":"2023-09-23T18:15:08.910442+0000","last_unstale":"2023-09-23T18:15:42.675519+0000","last_undegraded":"2023-09-23T18:15:42.675519+0000","last_fullsized":"2023-09-23T18:15:42.675519+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.675862+0000","last_change":"2023-09-23T18:15:08.919838+0000","last_active":"2023-09-23T18:15:42.675862+0000","last_peered":"2023-09-23T18:15:42.675862+0000","last_clean":"2023-09-23T18:15:42.675862+0000","last_became_active":"2023-09-23T18:15:08.919523+0000","last_became_peered":"2023-09-23T18:15:08.919523+0000","last_unstale":"2023-09-23T18:15:42.675862+0000","last_undegraded":"2023-09-23T18:15:42.675862+0000","last_fullsized":"2023-09-23T18:15:42.675862+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.679121+0000","last_change":"2023-09-23T18:15:08.903211+0000","last_active":"2023-09-23T18:15:42.679121+0000","last_peered":"2023-09-23T18:15:42.679121+0000","last_clean":"2023-09-23T18:15:42.679121+0000","last_became_active":"2023-09-23T18:15:08.902886+0000","last_became_peered":"2023-09-23T18:15:08.902886+0000","last_unstale":"2023-09-23T18:15:42.679121+0000","last_undegraded":"2023-09-23T18:15:42.679121+0000","last_fullsized":"2023-09-23T18:15:42.679121+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.675718+0000","last_change":"2023-09-23T18:15:08.925430+0000","last_active":"2023-09-23T18:15:42.675718+0000","last_peered":"2023-09-23T18:15:42.675718+0000","last_clean":"2023-09-23T18:15:42.675718+0000","last_became_active":"2023-09-23T18:15:08.925105+0000","last_became_peered":"2023-09-23T18:15:08.925105+0000","last_unstale":"2023-09-23T18:15:42.675718+0000","last_undegraded":"2023-09-23T18:15:42.675718+0000","last_fullsized":"2023-09-23T18:15:42.675718+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:15.830151+0000","last_change":"2023-09-23T18:15:08.900021+0000","last_active":"2023-09-23T18:15:15.830151+0000","last_peered":"2023-09-23T18:15:15.830151+0000","last_clean":"2023-09-23T18:15:15.830151+0000","last_became_active":"2023-09-23T18:15:08.897530+0000","last_became_peered":"2023-09-23T18:15:08.897530+0000","last_unstale":"2023-09-23T18:15:15.830151+0000","last_undegraded":"2023-09-23T18:15:15.830151+0000","last_fullsized":"2023-09-23T18:15:15.830151+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T18:15:43.675185+0000","last_change":"2023-09-23T18:15:43.675185+0000","last_active":"2023-09-23T18:15:43.675185+0000","last_peered":"2023-09-23T18:15:43.675185+0000","last_clean":"2023-09-23T18:15:43.675185+0000","last_became_active":"2023-09-23T18:15:43.674701+0000","last_became_peered":"2023-09-23T18:15:43.674701+0000","last_unstale":"2023-09-23T18:15:43.675185+0000","last_undegraded":"2023-09-23T18:15:43.675185+0000","last_fullsized":"2023-09-23T18:15:43.675185+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.671185+0000","last_change":"2023-09-23T18:15:08.906344+0000","last_active":"2023-09-23T18:15:42.671185+0000","last_peered":"2023-09-23T18:15:42.671185+0000","last_clean":"2023-09-23T18:15:42.671185+0000","last_became_active":"2023-09-23T18:15:08.905995+0000","last_became_peered":"2023-09-23T18:15:08.905995+0000","last_unstale":"2023-09-23T18:15:42.671185+0000","last_undegraded":"2023-09-23T18:15:42.671185+0000","last_fullsized":"2023-09-23T18:15:42.671185+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T18:15:43.679220+0000","last_change":"2023-09-23T18:15:43.679220+0000","last_active":"2023-09-23T18:15:43.679220+0000","last_peered":"2023-09-23T18:15:43.679220+0000","last_clean":"2023-09-23T18:15:43.679220+0000","last_became_active":"2023-09-23T18:15:43.678833+0000","last_became_peered":"2023-09-23T18:15:43.678833+0000","last_unstale":"2023-09-23T18:15:43.679220+0000","last_undegraded":"2023-09-23T18:15:43.679220+0000","last_fullsized":"2023-09-23T18:15:43.679220+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.683398+0000","last_change":"2023-09-23T18:15:08.908376+0000","last_active":"2023-09-23T18:15:42.683398+0000","last_peered":"2023-09-23T18:15:42.683398+0000","last_clean":"2023-09-23T18:15:42.683398+0000","last_became_active":"2023-09-23T18:15:08.907752+0000","last_became_peered":"2023-09-23T18:15:08.907752+0000","last_unstale":"2023-09-23T18:15:42.683398+0000","last_undegraded":"2023-09-23T18:15:42.683398+0000","last_fullsized":"2023-09-23T18:15:42.683398+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.675309+0000","last_change":"2023-09-23T18:15:08.927602+0000","last_active":"2023-09-23T18:15:42.675309+0000","last_peered":"2023-09-23T18:15:42.675309+0000","last_clean":"2023-09-23T18:15:42.675309+0000","last_became_active":"2023-09-23T18:15:08.927345+0000","last_became_peered":"2023-09-23T18:15:08.927345+0000","last_unstale":"2023-09-23T18:15:42.675309+0000","last_undegraded":"2023-09-23T18:15:42.675309+0000","last_fullsized":"2023-09-23T18:15:42.675309+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.683280+0000","last_change":"2023-09-23T18:15:08.913714+0000","last_active":"2023-09-23T18:15:42.683280+0000","last_peered":"2023-09-23T18:15:42.683280+0000","last_clean":"2023-09-23T18:15:42.683280+0000","last_became_active":"2023-09-23T18:15:08.913358+0000","last_became_peered":"2023-09-23T18:15:08.913358+0000","last_unstale":"2023-09-23T18:15:42.683280+0000","last_undegraded":"2023-09-23T18:15:42.683280+0000","last_fullsized":"2023-09-23T18:15:42.683280+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.670960+0000","last_change":"2023-09-23T18:15:10.327919+0000","last_active":"2023-09-23T18:15:42.670960+0000","last_peered":"2023-09-23T18:15:42.670960+0000","last_clean":"2023-09-23T18:15:42.670960+0000","last_became_active":"2023-09-23T18:15:10.327052+0000","last_became_peered":"2023-09-23T18:15:10.327052+0000","last_unstale":"2023-09-23T18:15:42.670960+0000","last_undegraded":"2023-09-23T18:15:42.670960+0000","last_fullsized":"2023-09-23T18:15:42.670960+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.683316+0000","last_change":"2023-09-23T18:15:08.914726+0000","last_active":"2023-09-23T18:15:42.683316+0000","last_peered":"2023-09-23T18:15:42.683316+0000","last_clean":"2023-09-23T18:15:42.683316+0000","last_became_active":"2023-09-23T18:15:08.914386+0000","last_became_peered":"2023-09-23T18:15:08.914386+0000","last_unstale":"2023-09-23T18:15:42.683316+0000","last_undegraded":"2023-09-23T18:15:42.683316+0000","last_fullsized":"2023-09-23T18:15:42.683316+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:12.308339+0000","last_change":"2023-09-23T18:15:08.917055+0000","last_active":"2023-09-23T18:15:12.308339+0000","last_peered":"2023-09-23T18:15:12.308339+0000","last_clean":"2023-09-23T18:15:12.308339+0000","last_became_active":"2023-09-23T18:15:08.916758+0000","last_became_peered":"2023-09-23T18:15:08.916758+0000","last_unstale":"2023-09-23T18:15:12.308339+0000","last_undegraded":"2023-09-23T18:15:12.308339+0000","last_fullsized":"2023-09-23T18:15:12.308339+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.678852+0000","last_change":"2023-09-23T18:15:08.907176+0000","last_active":"2023-09-23T18:15:42.678852+0000","last_peered":"2023-09-23T18:15:42.678852+0000","last_clean":"2023-09-23T18:15:42.678852+0000","last_became_active":"2023-09-23T18:15:08.906773+0000","last_became_peered":"2023-09-23T18:15:08.906773+0000","last_unstale":"2023-09-23T18:15:42.678852+0000","last_undegraded":"2023-09-23T18:15:42.678852+0000","last_fullsized":"2023-09-23T18:15:42.678852+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.666825+0000","last_change":"2023-09-23T18:15:10.325214+0000","last_active":"2023-09-23T18:15:42.666825+0000","last_peered":"2023-09-23T18:15:42.666825+0000","last_clean":"2023-09-23T18:15:42.666825+0000","last_became_active":"2023-09-23T18:15:10.324440+0000","last_became_peered":"2023-09-23T18:15:10.324440+0000","last_unstale":"2023-09-23T18:15:42.666825+0000","last_undegraded":"2023-09-23T18:15:42.666825+0000","last_fullsized":"2023-09-23T18:15:42.666825+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.682826+0000","last_change":"2023-09-23T18:15:08.916658+0000","last_active":"2023-09-23T18:15:42.682826+0000","last_peered":"2023-09-23T18:15:42.682826+0000","last_clean":"2023-09-23T18:15:42.682826+0000","last_became_active":"2023-09-23T18:15:08.916301+0000","last_became_peered":"2023-09-23T18:15:08.916301+0000","last_unstale":"2023-09-23T18:15:42.682826+0000","last_undegraded":"2023-09-23T18:15:42.682826+0000","last_fullsized":"2023-09-23T18:15:42.682826+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.671576+0000","last_change":"2023-09-23T18:15:10.327408+0000","last_active":"2023-09-23T18:15:42.671576+0000","last_peered":"2023-09-23T18:15:42.671576+0000","last_clean":"2023-09-23T18:15:42.671576+0000","last_became_active":"2023-09-23T18:15:10.326853+0000","last_became_peered":"2023-09-23T18:15:10.326853+0000","last_unstale":"2023-09-23T18:15:42.671576+0000","last_undegraded":"2023-09-23T18:15:42.671576+0000","last_fullsized":"2023-09-23T18:15:42.671576+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.666678+0000","last_change":"2023-09-23T18:15:08.900373+0000","last_active":"2023-09-23T18:15:42.666678+0000","last_peered":"2023-09-23T18:15:42.666678+0000","last_clean":"2023-09-23T18:15:42.666678+0000","last_became_active":"2023-09-23T18:15:08.900036+0000","last_became_peered":"2023-09-23T18:15:08.900036+0000","last_unstale":"2023-09-23T18:15:42.666678+0000","last_undegraded":"2023-09-23T18:15:42.666678+0000","last_fullsized":"2023-09-23T18:15:42.666678+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:12.308030+0000","last_change":"2023-09-23T18:15:08.916527+0000","last_active":"2023-09-23T18:15:12.308030+0000","last_peered":"2023-09-23T18:15:12.308030+0000","last_clean":"2023-09-23T18:15:12.308030+0000","last_became_active":"2023-09-23T18:15:08.915280+0000","last_became_peered":"2023-09-23T18:15:08.915280+0000","last_unstale":"2023-09-23T18:15:12.308030+0000","last_undegraded":"2023-09-23T18:15:12.308030+0000","last_fullsized":"2023-09-23T18:15:12.308030+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.683273+0000","last_change":"2023-09-23T18:15:08.913813+0000","last_active":"2023-09-23T18:15:42.683273+0000","last_peered":"2023-09-23T18:15:42.683273+0000","last_clean":"2023-09-23T18:15:42.683273+0000","last_became_active":"2023-09-23T18:15:08.913386+0000","last_became_peered":"2023-09-23T18:15:08.913386+0000","last_unstale":"2023-09-23T18:15:42.683273+0000","last_undegraded":"2023-09-23T18:15:42.683273+0000","last_fullsized":"2023-09-23T18:15:42.683273+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.676728+0000","last_change":"2023-09-23T18:15:08.920940+0000","last_active":"2023-09-23T18:15:42.676728+0000","last_peered":"2023-09-23T18:15:42.676728+0000","last_clean":"2023-09-23T18:15:42.676728+0000","last_became_active":"2023-09-23T18:15:08.920341+0000","last_became_peered":"2023-09-23T18:15:08.920341+0000","last_unstale":"2023-09-23T18:15:42.676728+0000","last_undegraded":"2023-09-23T18:15:42.676728+0000","last_fullsized":"2023-09-23T18:15:42.676728+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":20,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.675996+0000","last_change":"2023-09-23T18:15:08.909339+0000","last_active":"2023-09-23T18:15:42.675996+0000","last_peered":"2023-09-23T18:15:42.675996+0000","last_clean":"2023-09-23T18:15:42.675996+0000","last_became_active":"2023-09-23T18:15:08.908912+0000","last_became_peered":"2023-09-23T18:15:08.908912+0000","last_unstale":"2023-09-23T18:15:42.675996+0000","last_undegraded":"2023-09-23T18:15:42.675996+0000","last_fullsized":"2023-09-23T18:15:42.675996+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.669658+0000","last_change":"2023-09-23T18:15:08.912014+0000","last_active":"2023-09-23T18:15:42.669658+0000","last_peered":"2023-09-23T18:15:42.669658+0000","last_clean":"2023-09-23T18:15:42.669658+0000","last_became_active":"2023-09-23T18:15:08.911675+0000","last_became_peered":"2023-09-23T18:15:08.911675+0000","last_unstale":"2023-09-23T18:15:42.669658+0000","last_undegraded":"2023-09-23T18:15:42.669658+0000","last_fullsized":"2023-09-23T18:15:42.669658+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.683277+0000","last_change":"2023-09-23T18:15:08.908837+0000","last_active":"2023-09-23T18:15:42.683277+0000","last_peered":"2023-09-23T18:15:42.683277+0000","last_clean":"2023-09-23T18:15:42.683277+0000","last_became_active":"2023-09-23T18:15:08.908482+0000","last_became_peered":"2023-09-23T18:15:08.908482+0000","last_unstale":"2023-09-23T18:15:42.683277+0000","last_undegraded":"2023-09-23T18:15:42.683277+0000","last_fullsized":"2023-09-23T18:15:42.683277+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:12.307685+0000","last_change":"2023-09-23T18:15:08.917123+0000","last_active":"2023-09-23T18:15:12.307685+0000","last_peered":"2023-09-23T18:15:12.307685+0000","last_clean":"2023-09-23T18:15:12.307685+0000","last_became_active":"2023-09-23T18:15:08.916702+0000","last_became_peered":"2023-09-23T18:15:08.916702+0000","last_unstale":"2023-09-23T18:15:12.307685+0000","last_undegraded":"2023-09-23T18:15:12.307685+0000","last_fullsized":"2023-09-23T18:15:12.307685+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.671400+0000","last_change":"2023-09-23T18:15:08.905115+0000","last_active":"2023-09-23T18:15:42.671400+0000","last_peered":"2023-09-23T18:15:42.671400+0000","last_clean":"2023-09-23T18:15:42.671400+0000","last_became_active":"2023-09-23T18:15:08.904751+0000","last_became_peered":"2023-09-23T18:15:08.904751+0000","last_unstale":"2023-09-23T18:15:42.671400+0000","last_undegraded":"2023-09-23T18:15:42.671400+0000","last_fullsized":"2023-09-23T18:15:42.671400+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:12.305825+0000","last_change":"2023-09-23T18:15:08.916784+0000","last_active":"2023-09-23T18:15:12.305825+0000","last_peered":"2023-09-23T18:15:12.305825+0000","last_clean":"2023-09-23T18:15:12.305825+0000","last_became_active":"2023-09-23T18:15:08.915458+0000","last_became_peered":"2023-09-23T18:15:08.915458+0000","last_unstale":"2023-09-23T18:15:12.305825+0000","last_undegraded":"2023-09-23T18:15:12.305825+0000","last_fullsized":"2023-09-23T18:15:12.305825+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.667333+0000","last_change":"2023-09-23T18:15:08.904819+0000","last_active":"2023-09-23T18:15:42.667333+0000","last_peered":"2023-09-23T18:15:42.667333+0000","last_clean":"2023-09-23T18:15:42.667333+0000","last_became_active":"2023-09-23T18:15:08.904416+0000","last_became_peered":"2023-09-23T18:15:08.904416+0000","last_unstale":"2023-09-23T18:15:42.667333+0000","last_undegraded":"2023-09-23T18:15:42.667333+0000","last_fullsized":"2023-09-23T18:15:42.667333+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.678232+0000","last_change":"2023-09-23T18:15:09.125581+0000","last_active":"2023-09-23T18:15:42.678232+0000","last_peered":"2023-09-23T18:15:42.678232+0000","last_clean":"2023-09-23T18:15:42.678232+0000","last_became_active":"2023-09-23T18:15:09.125165+0000","last_became_peered":"2023-09-23T18:15:09.125165+0000","last_unstale":"2023-09-23T18:15:42.678232+0000","last_undegraded":"2023-09-23T18:15:42.678232+0000","last_fullsized":"2023-09-23T18:15:42.678232+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:12.305913+0000","last_change":"2023-09-23T18:15:08.896913+0000","last_active":"2023-09-23T18:15:12.305913+0000","last_peered":"2023-09-23T18:15:12.305913+0000","last_clean":"2023-09-23T18:15:12.305913+0000","last_became_active":"2023-09-23T18:15:08.896610+0000","last_became_peered":"2023-09-23T18:15:08.896610+0000","last_unstale":"2023-09-23T18:15:12.305913+0000","last_undegraded":"2023-09-23T18:15:12.305913+0000","last_fullsized":"2023-09-23T18:15:12.305913+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.666906+0000","last_change":"2023-09-23T18:15:10.324426+0000","last_active":"2023-09-23T18:15:42.666906+0000","last_peered":"2023-09-23T18:15:42.666906+0000","last_clean":"2023-09-23T18:15:42.666906+0000","last_became_active":"2023-09-23T18:15:10.324092+0000","last_became_peered":"2023-09-23T18:15:10.324092+0000","last_unstale":"2023-09-23T18:15:42.666906+0000","last_undegraded":"2023-09-23T18:15:42.666906+0000","last_fullsized":"2023-09-23T18:15:42.666906+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.669471+0000","last_change":"2023-09-23T18:15:08.893438+0000","last_active":"2023-09-23T18:15:42.669471+0000","last_peered":"2023-09-23T18:15:42.669471+0000","last_clean":"2023-09-23T18:15:42.669471+0000","last_became_active":"2023-09-23T18:15:08.893078+0000","last_became_peered":"2023-09-23T18:15:08.893078+0000","last_unstale":"2023-09-23T18:15:42.669471+0000","last_undegraded":"2023-09-23T18:15:42.669471+0000","last_fullsized":"2023-09-23T18:15:42.669471+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T18:15:12.308186+0000","last_change":"2023-09-23T18:15:08.922234+0000","last_active":"2023-09-23T18:15:12.308186+0000","last_peered":"2023-09-23T18:15:12.308186+0000","last_clean":"2023-09-23T18:15:12.308186+0000","last_became_active":"2023-09-23T18:15:08.921890+0000","last_became_peered":"2023-09-23T18:15:08.921890+0000","last_unstale":"2023-09-23T18:15:12.308186+0000","last_undegraded":"2023-09-23T18:15:12.308186+0000","last_fullsized":"2023-09-23T18:15:12.308186+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T18:15:42.675795+0000","last_change":"2023-09-23T18:15:08.900528+0000","last_active":"2023-09-23T18:15:42.675795+0000","last_peered":"2023-09-23T18:15:42.675795+0000","last_clean":"2023-09-23T18:15:42.675795+0000","last_became_active":"2023-09-23T18:15:08.900179+0000","last_became_peered":"2023-09-23T18:15:08.900179+0000","last_unstale":"2023-09-23T18:15:42.675795+0000","last_undegraded":"2023-09-23T18:15:42.675795+0000","last_fullsized":"2023-09-23T18:15:42.675795+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430267,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297736,"kb_used_data":640,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447416,"statfs":{"total":95995035648,"available":95690153984,"internally_reserved":0,"allocated":655360,"data_stored":271700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.89400000000000002,"15min":0.89300000000000002},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.198,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.93200000000000005,"15min":0.92600000000000005},"min":{"1min":0.56599999999999995,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.292,"5min":1.411,"15min":1.411},"last":0.60799999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.89100000000000001,"15min":0.88400000000000001},"min":{"1min":0.63600000000000001,"5min":0.499,"15min":0.499},"max":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.84999999999999998,"15min":0.82799999999999996},"min":{"1min":0.66000000000000003,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.22,"5min":1.286,"15min":1.286},"last":0.87}]},{"osd":2,"last update":"Sat Sep 23 18:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":0.95399999999999996,"15min":0.96299999999999997},"min":{"1min":0.60799999999999998,"5min":0.311,"15min":0.311},"max":{"1min":1.2769999999999999,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":0.78400000000000003},{"interface":"front","average":{"1min":0.96199999999999997,"5min":0.92000000000000004,"15min":0.89900000000000002},"min":{"1min":0.71399999999999997,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.24,"5min":1.278,"15min":1.278},"last":0.66100000000000003}]},{"osd":3,"last update":"Sat Sep 23 18:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.96799999999999997,"15min":0.96799999999999997},"min":{"1min":0.55900000000000005,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.208,"5min":1.3560000000000001,"15min":1.3560000000000001},"last":0.90300000000000002},{"interface":"front","average":{"1min":0.99199999999999999,"5min":0.96299999999999997,"15min":0.94699999999999995},"min":{"1min":0.76200000000000001,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.363,"5min":1.363,"15min":1.363},"last":0.82399999999999995}]},{"osd":4,"last update":"Sat Sep 23 18:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.90400000000000003,"15min":0.89700000000000002},"min":{"1min":0.73899999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.2969999999999999,"5min":1.5249999999999999,"15min":1.5249999999999999},"last":0.94599999999999995},{"interface":"front","average":{"1min":1.0740000000000001,"5min":0.995,"15min":0.96899999999999997},"min":{"1min":0.54300000000000004,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.3999999999999999,"5min":1.427,"15min":1.427},"last":0.84199999999999997}]},{"osd":5,"last update":"Sat Sep 23 18:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":0.95999999999999996,"15min":0.92900000000000005},"min":{"1min":0.64800000000000002,"5min":0.442,"15min":0.442},"max":{"1min":1.369,"5min":1.4470000000000001,"15min":1.4470000000000001},"last":0.92800000000000005},{"interface":"front","average":{"1min":0.995,"5min":0.99199999999999999,"15min":0.96799999999999997},"min":{"1min":0.49199999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.417,"5min":1.417,"15min":1.417},"last":1}]},{"osd":6,"last update":"Sat Sep 23 18:16:12 2023","interfaces":[{"interface":"back","average":{"1min":1.073,"5min":1.0209999999999999,"15min":0.98499999999999999},"min":{"1min":0.54000000000000004,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.4379999999999999,"5min":1.5229999999999999,"15min":1.5229999999999999},"last":1.04},{"interface":"front","average":{"1min":1.161,"5min":1.137,"15min":1.119},"min":{"1min":0.71899999999999997,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.4610000000000001,"5min":1.554,"15min":1.554},"last":0.96399999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626499,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297720,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447432,"statfs":{"total":95995035648,"available":95690170368,"internally_reserved":0,"allocated":638976,"data_stored":271239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:16:35 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":0.83599999999999997,"15min":0.85199999999999998},"min":{"1min":0.61899999999999999,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.0569999999999999,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.82699999999999996},{"interface":"front","average":{"1min":0.91700000000000004,"5min":0.85099999999999998,"15min":0.89900000000000002},"min":{"1min":0.66700000000000004,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.1399999999999999,"5min":1.268,"15min":1.268},"last":0.70599999999999996}]},{"osd":1,"last update":"Sat Sep 23 18:16:35 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.89300000000000002},"min":{"1min":0.65900000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.137,"5min":1.2370000000000001,"15min":1.2370000000000001},"last":0.98199999999999998},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.88400000000000001,"15min":0.92000000000000004},"min":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.151,"5min":1.726,"15min":1.726},"last":0.72999999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:16:35 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.88800000000000001,"15min":0.89000000000000001},"min":{"1min":0.61399999999999999,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.117,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":1.0469999999999999},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.91500000000000004,"15min":0.90400000000000003},"min":{"1min":0.67400000000000004,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.208,"5min":1.538,"15min":1.538},"last":0.88200000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:16:35 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.90900000000000003,"15min":0.93500000000000005},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.097,"5min":1.264,"15min":1.264},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.997,"5min":0.97899999999999998,"15min":1.0069999999999999},"min":{"1min":0.755,"5min":0.66800000000000004,"15min":0.66800000000000004},"max":{"1min":1.2310000000000001,"5min":1.7949999999999999,"15min":1.7949999999999999},"last":1.1339999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:16:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":0.90600000000000003,"15min":0.85899999999999999},"min":{"1min":0.78400000000000003,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.292,"5min":1.4790000000000001,"15min":1.4790000000000001},"last":1.0089999999999999},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.93999999999999995,"15min":0.91200000000000003},"min":{"1min":0.76800000000000002,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":1.2529999999999999,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":0.92300000000000004}]},{"osd":5,"last update":"Sat Sep 23 18:16:35 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.0009999999999999,"15min":0.91900000000000004},"min":{"1min":0.69799999999999995,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.2490000000000001,"5min":1.6160000000000001,"15min":1.6160000000000001},"last":1.105},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.016,"15min":0.996},"min":{"1min":0.88200000000000001,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.3069999999999999,"5min":1.7649999999999999,"15min":1.7649999999999999},"last":1.071}]},{"osd":7,"last update":"Sat Sep 23 18:16:14 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.0469999999999999,"15min":1.0309999999999999},"min":{"1min":0.94799999999999995,"5min":0.58899999999999997,"15min":0.58899999999999997},"max":{"1min":1.3049999999999999,"5min":1.403,"15min":1.403},"last":0.93799999999999994},{"interface":"front","average":{"1min":1.2370000000000001,"5min":1.105,"15min":1.069},"min":{"1min":0.91900000000000004,"5min":0.69299999999999995,"15min":0.69299999999999995},"max":{"1min":1.4059999999999999,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":1.1850000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574959,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":269948,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:16:01 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.69299999999999995,"15min":0.58399999999999996},"min":{"1min":0.504,"5min":0.38500000000000001,"15min":0.34599999999999997},"max":{"1min":1.1359999999999999,"5min":1.641,"15min":1.641},"last":0.877},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.69999999999999996,"15min":0.59499999999999997},"min":{"1min":0.39700000000000002,"5min":0.35799999999999998,"15min":0.32500000000000001},"max":{"1min":0.91900000000000004,"5min":1.6859999999999999,"15min":1.6859999999999999},"last":0.751}]},{"osd":2,"last update":"Sat Sep 23 18:16:47 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":0.70999999999999996,"15min":0.67700000000000005},"min":{"1min":0.41099999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":2.1560000000000001,"5min":2.1560000000000001,"15min":2.1560000000000001},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.77800000000000002,"15min":0.69599999999999995},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.1990000000000001,"5min":2.3290000000000002,"15min":2.3290000000000002},"last":1.016}]},{"osd":3,"last update":"Sat Sep 23 18:16:34 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.752,"15min":0.77700000000000002},"min":{"1min":0.41599999999999998,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.95999999999999996,"5min":1.613,"15min":1.613},"last":0.93999999999999995},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.83099999999999996,"15min":0.80500000000000005},"min":{"1min":0.43099999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.0549999999999999,"5min":2.4940000000000002,"15min":2.4940000000000002},"last":0.68400000000000005}]},{"osd":4,"last update":"Sat Sep 23 18:16:16 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.93200000000000005,"15min":0.88300000000000001},"min":{"1min":0.65100000000000002,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.1930000000000001,"5min":1.8260000000000001,"15min":1.8260000000000001},"last":1.167},{"interface":"front","average":{"1min":0.98499999999999999,"5min":0.998,"15min":0.90800000000000003},"min":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.3700000000000001,"5min":1.601,"15min":1.601},"last":1.095}]},{"osd":5,"last update":"Sat Sep 23 18:15:53 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.94099999999999995,"15min":0.94599999999999995},"min":{"1min":0.67400000000000004,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.3899999999999999,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":1.0409999999999999},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.97799999999999998,"15min":0.95899999999999996},"min":{"1min":0.77100000000000002,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.24,"5min":1.73,"15min":1.73},"last":1.0680000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:16:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.0960000000000001,"15min":1.1499999999999999},"min":{"1min":0.80200000000000005,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.421,"5min":1.51,"15min":1.51},"last":1.143},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.0429999999999999,"15min":1.099},"min":{"1min":0.63500000000000001,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.4399999999999999,"5min":1.556,"15min":1.556},"last":1.224}]},{"osd":7,"last update":"Sat Sep 23 18:16:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.077,"15min":1.1140000000000001},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.5309999999999999,"5min":1.7669999999999999,"15min":1.7669999999999999},"last":1.119},{"interface":"front","average":{"1min":1.139,"5min":1.1299999999999999,"15min":1.143},"min":{"1min":0.88600000000000001,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.3140000000000001,"5min":1.53,"15min":1.53},"last":1.2549999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738488,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:16:07 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":0.69299999999999995,"15min":0.60399999999999998},"min":{"1min":0.37,"5min":0.317,"15min":0.317},"max":{"1min":1.268,"5min":1.268,"15min":1.268},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.71599999999999997,"15min":0.626},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.34799999999999998},"max":{"1min":1.1739999999999999,"5min":1.1739999999999999,"15min":1.1739999999999999},"last":0.72099999999999997}]},{"osd":2,"last update":"Sat Sep 23 18:15:58 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.72299999999999998,"15min":0.66100000000000003},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":1.018,"5min":1.1040000000000001,"15min":1.1040000000000001},"last":0.78600000000000003},{"interface":"front","average":{"1min":0.70599999999999996,"5min":0.753,"15min":0.67500000000000004},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.0609999999999999,"5min":1.0609999999999999,"15min":1.0609999999999999},"last":0.77000000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:16:33 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.74299999999999999,"15min":0.72599999999999998},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.93700000000000006,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":0.88300000000000001},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.76200000000000001,"15min":0.74299999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.874,"5min":1.0529999999999999,"15min":1.0529999999999999},"last":0.91000000000000003}]},{"osd":4,"last update":"Sat Sep 23 18:16:13 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":0.92800000000000005,"15min":0.91300000000000003},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.405,"5min":1.407,"15min":1.407},"last":0.82399999999999995},{"interface":"front","average":{"1min":1.0029999999999999,"5min":0.98199999999999998,"15min":0.94199999999999995},"min":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.5800000000000001,"5min":1.5800000000000001,"15min":1.5800000000000001},"last":1.0529999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:15:58 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.0009999999999999,"15min":1.0069999999999999},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.448,"5min":1.448,"15min":1.448},"last":0.95299999999999996},{"interface":"front","average":{"1min":1.026,"5min":0.997,"15min":0.99099999999999999},"min":{"1min":0.66100000000000003,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.3069999999999999,"5min":1.419,"15min":1.419},"last":1.2669999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:16:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":1.0569999999999999,"15min":1.0009999999999999},"min":{"1min":0.76100000000000001,"5min":0.70899999999999996,"15min":0.70899999999999996},"max":{"1min":1.502,"5min":3.2309999999999999,"15min":3.2309999999999999},"last":0.86799999999999999},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.133,"15min":1.151},"min":{"1min":0.77800000000000002,"5min":0.72599999999999998,"15min":0.72599999999999998},"max":{"1min":1.2989999999999999,"5min":3.2090000000000001,"15min":3.2090000000000001},"last":1.107}]},{"osd":7,"last update":"Sat Sep 23 18:16:13 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":1.121,"15min":1.127},"min":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"max":{"1min":1.528,"5min":1.538,"15min":1.538},"last":1.1519999999999999},{"interface":"front","average":{"1min":1.0609999999999999,"5min":1.0640000000000001,"15min":1.048},"min":{"1min":0.72699999999999998,"5min":0.71699999999999997,"15min":0.71699999999999997},"max":{"1min":1.4139999999999999,"5min":1.5920000000000001,"15min":1.5920000000000001},"last":1.3109999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411429,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:15:56 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.67000000000000004,"15min":0.61599999999999999},"min":{"1min":0.29299999999999998,"5min":0.247,"15min":0.247},"max":{"1min":0.93999999999999995,"5min":2.23,"15min":2.23},"last":0.84399999999999997},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.72599999999999998,"15min":0.61699999999999999},"min":{"1min":0.375,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.96799999999999997,"5min":2.4420000000000002,"15min":2.4420000000000002},"last":0.97599999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:15:56 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.77300000000000002,"15min":0.68100000000000005},"min":{"1min":0.41599999999999998,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.96499999999999997,"5min":1.5580000000000001,"15min":1.5580000000000001},"last":0.94899999999999995},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.86399999999999999,"15min":0.76100000000000001},"min":{"1min":0.45200000000000001,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.0429999999999999,"5min":1.9399999999999999,"15min":1.9399999999999999},"last":1.155}]},{"osd":3,"last update":"Sat Sep 23 18:16:32 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.754,"15min":0.754},"min":{"1min":0.46800000000000003,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.036,"5min":2.012,"15min":2.012},"last":0.77500000000000002},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.73199999999999998,"15min":0.73499999999999999},"min":{"1min":0.47599999999999998,"5min":0.373,"15min":0.373},"max":{"1min":0.96199999999999997,"5min":1.522,"15min":1.522},"last":0.88800000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:16:16 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.93500000000000005,"15min":0.89300000000000002},"min":{"1min":0.72999999999999998,"5min":0.61099999999999999,"15min":0.61099999999999999},"max":{"1min":1.228,"5min":1.5629999999999999,"15min":1.5629999999999999},"last":0.81399999999999995},{"interface":"front","average":{"1min":0.98599999999999999,"5min":0.95799999999999996,"15min":0.90700000000000003},"min":{"1min":0.628,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.1970000000000001,"5min":1.6160000000000001,"15min":1.6160000000000001},"last":1.014}]},{"osd":5,"last update":"Sat Sep 23 18:15:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"min":{"1min":0.69699999999999995,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.355,"5min":1.7589999999999999,"15min":1.7589999999999999},"last":1.25},{"interface":"front","average":{"1min":0.999,"5min":0.997,"15min":0.99299999999999999},"min":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.3200000000000001,"5min":1.6950000000000001,"15min":1.6950000000000001},"last":1.0449999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:16:32 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.0389999999999999,"15min":1.044},"min":{"1min":0.80700000000000005,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":1.514,"5min":1.5209999999999999,"15min":1.5209999999999999},"last":1.1259999999999999},{"interface":"front","average":{"1min":1.044,"5min":1.0740000000000001,"15min":1.1359999999999999},"min":{"1min":0.66900000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.383,"5min":1.615,"15min":1.615},"last":1.0669999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:16:16 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.0549999999999999,"15min":1.0589999999999999},"min":{"1min":0.62,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.55,"5min":1.55,"15min":1.55},"last":1.1839999999999999},{"interface":"front","average":{"1min":1.1379999999999999,"5min":1.1299999999999999,"15min":1.157},"min":{"1min":0.78900000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.484,"5min":1.6870000000000001,"15min":1.6870000000000001},"last":1.226}]}]},{"osd":3,"up_from":25,"seq":107374182492,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":269723,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:16:29 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.63,"15min":0.621},"min":{"1min":0.33900000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.90700000000000003,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.67700000000000005,"15min":0.70099999999999996},"min":{"1min":0.40200000000000002,"5min":0.25,"15min":0.25},"max":{"1min":2.5920000000000001,"5min":2.5920000000000001,"15min":2.5920000000000001},"last":0.70099999999999996}]},{"osd":1,"last update":"Sat Sep 23 18:16:29 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.70799999999999996,"15min":0.73799999999999999},"min":{"1min":0.45400000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.91000000000000003,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.94499999999999995},{"interface":"front","average":{"1min":0.73699999999999999,"5min":0.67900000000000005,"15min":0.68100000000000005},"min":{"1min":0.42599999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.379,"5min":2.379,"15min":2.379},"last":0.79100000000000004}]},{"osd":2,"last update":"Sat Sep 23 18:16:29 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":0.76500000000000001,"15min":0.80300000000000005},"min":{"1min":0.497,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.0409999999999999,"5min":1.167,"15min":1.1819999999999999},"last":1.0940000000000001},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.75800000000000001,"15min":0.76700000000000002},"min":{"1min":0.499,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.0289999999999999,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.93799999999999994}]},{"osd":4,"last update":"Sat Sep 23 18:16:13 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.92800000000000005,"15min":0.91100000000000003},"min":{"1min":0.66700000000000004,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.476,"5min":1.7410000000000001,"15min":1.7410000000000001},"last":0.88200000000000001},{"interface":"front","average":{"1min":1.0589999999999999,"5min":0.97399999999999998,"15min":0.93300000000000005},"min":{"1min":0.72099999999999997,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.413,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":1.0309999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:15:54 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":0.97299999999999998,"15min":0.97599999999999998},"min":{"1min":0.755,"5min":0.58899999999999997,"15min":0.58899999999999997},"max":{"1min":1.2270000000000001,"5min":1.361,"15min":1.361},"last":1.081},{"interface":"front","average":{"1min":0.95599999999999996,"5min":0.95499999999999996,"15min":0.96699999999999997},"min":{"1min":0.67500000000000004,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":1.4450000000000001,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":1.0009999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:16:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.028,"15min":1.0740000000000001},"min":{"1min":0.77400000000000002,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.351,"5min":1.512,"15min":1.512},"last":1.1439999999999999},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.02,"15min":1.05},"min":{"1min":0.66100000000000003,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.579,"5min":1.6699999999999999,"15min":1.6699999999999999},"last":1.1200000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:16:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.048,"15min":1.054},"min":{"1min":0.61399999999999999,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.514,"5min":1.595,"15min":1.595},"last":1.052},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.0600000000000001,"15min":1.0680000000000001},"min":{"1min":0.67700000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.4299999999999999,"5min":1.454,"15min":1.454},"last":1.206}]}]},{"osd":4,"up_from":30,"seq":128849018965,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297732,"kb_used_data":636,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447420,"statfs":{"total":95995035648,"available":95690158080,"internally_reserved":0,"allocated":651264,"data_stored":271630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.84399999999999997,"15min":0.84599999999999997},"min":{"1min":0.54000000000000004,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.143,"5min":1.3029999999999999,"15min":1.3029999999999999},"last":1.0840000000000001},{"interface":"front","average":{"1min":0.96099999999999997,"5min":0.88600000000000001,"15min":0.83699999999999997},"min":{"1min":0.57999999999999996,"5min":0.49199999999999999,"15min":0.46300000000000002},"max":{"1min":1.3149999999999999,"5min":1.3149999999999999,"15min":1.3149999999999999},"last":0.88300000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":0.84799999999999998,"15min":0.85199999999999998},"min":{"1min":0.64600000000000002,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.0940000000000001,"5min":1.304,"15min":1.304},"last":1.048},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.86099999999999999,"15min":0.83899999999999997},"min":{"1min":0.55000000000000004,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.1759999999999999,"5min":1.254,"15min":1.254},"last":0.91900000000000004}]},{"osd":2,"last update":"Sat Sep 23 18:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.88200000000000001,"15min":0.875},"min":{"1min":0.71199999999999997,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.1759999999999999,"5min":1.2789999999999999,"15min":1.2789999999999999},"last":0.82299999999999995},{"interface":"front","average":{"1min":1.046,"5min":0.92600000000000005,"15min":0.90700000000000003},"min":{"1min":0.67000000000000004,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":2.1789999999999998,"5min":2.1789999999999998,"15min":2.1789999999999998},"last":1.3580000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":0.93999999999999995,"15min":0.85899999999999999},"min":{"1min":0.67100000000000004,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.732,"5min":1.732,"15min":1.732},"last":0.98399999999999999},{"interface":"front","average":{"1min":0.88100000000000001,"5min":0.879,"15min":0.92000000000000004},"min":{"1min":0.69199999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.1120000000000001,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":1.202}]},{"osd":5,"last update":"Sat Sep 23 18:16:00 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":0.84999999999999998,"15min":0.77000000000000002},"min":{"1min":0.70999999999999996,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.2110000000000001,"5min":1.254,"15min":1.254},"last":1.141},{"interface":"front","average":{"1min":0.91800000000000004,"5min":0.876,"15min":0.82599999999999996},"min":{"1min":0.67500000000000004,"5min":0.443,"15min":0.443},"max":{"1min":1.1180000000000001,"5min":1.333,"15min":1.333},"last":1.018}]},{"osd":6,"last update":"Sat Sep 23 18:16:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":0.97999999999999998,"15min":0.95499999999999996},"min":{"1min":0.61599999999999999,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.3700000000000001,"5min":1.3700000000000001,"15min":1.3700000000000001},"last":1.276},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.93500000000000005,"15min":0.90300000000000002},"min":{"1min":0.50700000000000001,"5min":0.434,"15min":0.434},"max":{"1min":1.2529999999999999,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":1.262}]},{"osd":7,"last update":"Sat Sep 23 18:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":0.996,"15min":0.95399999999999996},"min":{"1min":0.81100000000000005,"5min":0.53400000000000003,"15min":0.53400000000000003},"max":{"1min":1.341,"5min":1.7010000000000001,"15min":1.7010000000000001},"last":1.3049999999999999},{"interface":"front","average":{"1min":1.204,"5min":1.0840000000000001,"15min":1.02},"min":{"1min":0.99299999999999999,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.379,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":1.4379999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822733,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":269942,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:15:51 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.879,"15min":0.89500000000000002},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":1.2769999999999999,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.878,"5min":0.86199999999999999,"15min":0.86499999999999999},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":1.1890000000000001,"5min":1.3480000000000001,"15min":1.3480000000000001},"last":0.995}]},{"osd":1,"last update":"Sat Sep 23 18:15:51 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":0.874,"15min":0.878},"min":{"1min":0.625,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.3340000000000001,"5min":1.387,"15min":1.387},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.90400000000000003,"15min":0.90500000000000003},"min":{"1min":0.68600000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.101,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":0.95399999999999996}]},{"osd":2,"last update":"Sat Sep 23 18:15:51 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":0.89400000000000002,"15min":0.88500000000000001},"min":{"1min":0.63700000000000001,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.266,"5min":1.266,"15min":1.266},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.99199999999999999,"5min":0.93300000000000005,"15min":0.95599999999999996},"min":{"1min":0.56999999999999995,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.2669999999999999,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":0.92100000000000004}]},{"osd":3,"last update":"Sat Sep 23 18:15:51 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":0.91400000000000003,"15min":0.93000000000000005},"min":{"1min":0.76900000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.206,"5min":1.333,"15min":1.333},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.97999999999999998,"15min":0.97799999999999998},"min":{"1min":0.64800000000000002,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.284,"5min":2.0259999999999998,"15min":2.0259999999999998},"last":0.86599999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:15:51 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.875,"15min":0.83499999999999996},"min":{"1min":0.63200000000000001,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.256,"5min":1.7509999999999999,"15min":1.7509999999999999},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.96599999999999997,"5min":0.88800000000000001,"15min":0.84099999999999997},"min":{"1min":0.66300000000000003,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.163,"5min":1.278,"15min":1.278},"last":0.65100000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:16:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":0.97799999999999998,"15min":0.92900000000000005},"min":{"1min":0.65900000000000003,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.1930000000000001,"5min":1.8460000000000001,"15min":1.8460000000000001},"last":0.90800000000000003},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.0409999999999999,"15min":0.98999999999999999},"min":{"1min":0.73699999999999999,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.306,"5min":1.4299999999999999,"15min":1.4299999999999999},"last":0.61299999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:16:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":0.98599999999999999,"15min":0.91000000000000003},"min":{"1min":0.69599999999999995,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.343,"5min":1.472,"15min":1.472},"last":0.58999999999999997},{"interface":"front","average":{"1min":1.113,"5min":1.036,"15min":0.96999999999999997},"min":{"1min":0.875,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.28,"5min":1.468,"15min":1.468},"last":0.72399999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:16:52.945 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T18:16:52.945 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T18:16:52.945 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T18:16:52.946 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T18:16:53.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:53 smithi098 bash[27349]: audit 2023-09-23T18:16:52.126809+0000 mgr.y (mgr.24344) 159 : audit [DBG] from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:16:53.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:53 smithi098 bash[27349]: cluster 2023-09-23T18:16:52.623495+0000 mgr.y (mgr.24344) 160 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:53.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:53 smithi098 bash[33599]: audit 2023-09-23T18:16:52.126809+0000 mgr.y (mgr.24344) 159 : audit [DBG] from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:16:53.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:53 smithi098 bash[33599]: cluster 2023-09-23T18:16:52.623495+0000 mgr.y (mgr.24344) 160 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:54.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:53 smithi173 bash[27694]: audit 2023-09-23T18:16:52.126809+0000 mgr.y (mgr.24344) 159 : audit [DBG] from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:16:54.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:53 smithi173 bash[27694]: cluster 2023-09-23T18:16:52.623495+0000 mgr.y (mgr.24344) 160 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:55.693 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:16:55 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:16:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:16:55.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:55 smithi098 bash[27349]: cluster 2023-09-23T18:16:54.624556+0000 mgr.y (mgr.24344) 161 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:55.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:55 smithi098 bash[33599]: cluster 2023-09-23T18:16:54.624556+0000 mgr.y (mgr.24344) 161 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:56.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:55 smithi173 bash[27694]: cluster 2023-09-23T18:16:54.624556+0000 mgr.y (mgr.24344) 161 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:57.761 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:58.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:57 smithi173 bash[27694]: cluster 2023-09-23T18:16:56.625134+0000 mgr.y (mgr.24344) 162 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:58.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:16:57 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:16:57] "GET /metrics HTTP/1.1" 200 191534 "" "Prometheus/2.33.4" 2023-09-23T18:16:58.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:57 smithi098 bash[27349]: cluster 2023-09-23T18:16:56.625134+0000 mgr.y (mgr.24344) 162 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:58.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:57 smithi098 bash[33599]: cluster 2023-09-23T18:16:56.625134+0000 mgr.y (mgr.24344) 162 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:59.639 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:16:59 smithi098 bash[33599]: cluster 2023-09-23T18:16:58.626174+0000 mgr.y (mgr.24344) 163 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:59.639 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:16:59 smithi098 bash[27349]: cluster 2023-09-23T18:16:58.626174+0000 mgr.y (mgr.24344) 163 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:16:59.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:16:59 smithi173 bash[27694]: cluster 2023-09-23T18:16:58.626174+0000 mgr.y (mgr.24344) 163 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:00.931 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:17:01.729 INFO:teuthology.orchestra.run.smithi098.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T18:17:01.730 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T18:17:01.730 INFO:teuthology.run_tasks:Running task print... 2023-09-23T18:17:01.931 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-09-23T18:17:01.932 INFO:teuthology.run_tasks:Running task parallel... 2023-09-23T18:17:01.960 INFO:teuthology.task.parallel:starting parallel... 2023-09-23T18:17:01.960 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T18:17:01.961 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T18:17:01.966 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-09-23T18:17:01.967 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T18:17:01.967 DEBUG:teuthology.orchestra.run.smithi098:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T18:17:01.968 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T18:17:01.968 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-09-23T18:17:01.969 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi098.front.sepia.ceph.com 2023-09-23T18:17:01.969 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-09-23T18:17:01.970 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:01 smithi098 bash[27349]: cluster 2023-09-23T18:17:00.626799+0000 mgr.y (mgr.24344) 164 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:01.971 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:01 smithi098 bash[27349]: audit 2023-09-23T18:17:00.926220+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.98:0/656842' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T18:17:01.971 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:01 smithi098 bash[33599]: cluster 2023-09-23T18:17:00.626799+0000 mgr.y (mgr.24344) 164 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:01.971 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:01 smithi098 bash[33599]: audit 2023-09-23T18:17:00.926220+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.98:0/656842' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T18:17:01.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:17:01.984 INFO:teuthology.orchestra.run.smithi098.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T18:17:01.984 DEBUG:teuthology.orchestra.run.smithi098:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T18:17:02.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:01 smithi173 bash[27694]: cluster 2023-09-23T18:17:00.626799+0000 mgr.y (mgr.24344) 164 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:02.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:01 smithi173 bash[27694]: audit 2023-09-23T18:17:00.926220+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.98:0/656842' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T18:17:02.034 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T18:17:02.034 DEBUG:teuthology.orchestra.run.smithi098:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T18:17:02.086 INFO:tasks.workunit:timeout=3h 2023-09-23T18:17:02.087 INFO:tasks.workunit:cleanup=True 2023-09-23T18:17:02.087 DEBUG:teuthology.orchestra.run.smithi098:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T18:17:02.144 INFO:tasks.workunit.client.0.smithi098.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T18:17:03.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:03 smithi098 bash[27349]: cluster 2023-09-23T18:17:02.627461+0000 mgr.y (mgr.24344) 165 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:03.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:03 smithi098 bash[33599]: cluster 2023-09-23T18:17:02.627461+0000 mgr.y (mgr.24344) 165 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:04.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:03 smithi173 bash[27694]: cluster 2023-09-23T18:17:02.627461+0000 mgr.y (mgr.24344) 165 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:05.613 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:17:05 smithi098 bash[27618]: [23/Sep/2023:18:17:05] ENGINE Bus STOPPING 2023-09-23T18:17:05.613 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:17:05 smithi098 bash[27618]: [23/Sep/2023:18:17:05] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T18:17:05.613 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:17:05 smithi098 bash[27618]: [23/Sep/2023:18:17:05] ENGINE Bus STOPPED 2023-09-23T18:17:05.613 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:17:05 smithi098 bash[27618]: [23/Sep/2023:18:17:05] ENGINE Bus STARTING 2023-09-23T18:17:05.613 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:17:05 smithi098 bash[27618]: [23/Sep/2023:18:17:05] ENGINE Serving on http://:::9283 2023-09-23T18:17:05.613 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:17:05 smithi098 bash[27618]: [23/Sep/2023:18:17:05] ENGINE Bus STARTED 2023-09-23T18:17:05.698 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:17:05 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:17:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:17:05.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:05 smithi098 bash[27349]: cluster 2023-09-23T18:17:04.628426+0000 mgr.y (mgr.24344) 166 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:05.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:05 smithi098 bash[27349]: audit 2023-09-23T18:17:05.347767+0000 mon.a (mon.0) 677 : audit [INF] from='client.? 172.21.15.98:0/1070702611' entity='client.admin' 2023-09-23T18:17:05.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:05 smithi098 bash[27349]: audit 2023-09-23T18:17:05.404161+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:17:05.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:05 smithi098 bash[33599]: cluster 2023-09-23T18:17:04.628426+0000 mgr.y (mgr.24344) 166 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:05.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:05 smithi098 bash[33599]: audit 2023-09-23T18:17:05.347767+0000 mon.a (mon.0) 677 : audit [INF] from='client.? 172.21.15.98:0/1070702611' entity='client.admin' 2023-09-23T18:17:05.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:05 smithi098 bash[33599]: audit 2023-09-23T18:17:05.404161+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:17:06.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:05 smithi173 bash[27694]: cluster 2023-09-23T18:17:04.628426+0000 mgr.y (mgr.24344) 166 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:06.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:05 smithi173 bash[27694]: audit 2023-09-23T18:17:05.347767+0000 mon.a (mon.0) 677 : audit [INF] from='client.? 172.21.15.98:0/1070702611' entity='client.admin' 2023-09-23T18:17:06.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:05 smithi173 bash[27694]: audit 2023-09-23T18:17:05.404161+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:17:06.618 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph config set global log_to_journald false --force' 2023-09-23T18:17:06.939 DEBUG:teuthology.orchestra.run.smithi098:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T18:17:07.002 INFO:tasks.workunit.client.0.smithi098.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T18:17:07.006 INFO:tasks.workunit.client.0.smithi098.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T18:17:07.006 INFO:tasks.workunit.client.0.smithi098.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T18:17:08.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:07 smithi173 bash[27694]: cluster 2023-09-23T18:17:06.628995+0000 mgr.y (mgr.24344) 167 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:08.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:07 smithi098 bash[33599]: cluster 2023-09-23T18:17:06.628995+0000 mgr.y (mgr.24344) 167 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:08.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:07 smithi098 bash[27349]: cluster 2023-09-23T18:17:06.628995+0000 mgr.y (mgr.24344) 167 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:08.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:17:07 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:17:07] "GET /metrics HTTP/1.1" 200 191532 "" "Prometheus/2.33.4" 2023-09-23T18:17:09.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:09 smithi098 bash[27349]: cluster 2023-09-23T18:17:08.629630+0000 mgr.y (mgr.24344) 168 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:09.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:09 smithi098 bash[33599]: cluster 2023-09-23T18:17:08.629630+0000 mgr.y (mgr.24344) 168 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:09.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:09 smithi173 bash[27694]: cluster 2023-09-23T18:17:08.629630+0000 mgr.y (mgr.24344) 168 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:12.275 INFO:tasks.workunit.client.0.smithi098.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T18:17:12.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:11 smithi173 bash[27694]: cluster 2023-09-23T18:17:10.630013+0000 mgr.y (mgr.24344) 169 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:12.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:11 smithi173 bash[27694]: audit 2023-09-23T18:17:10.987711+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:17:12.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:11 smithi173 bash[27694]: audit 2023-09-23T18:17:11.293001+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:17:12.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:11 smithi173 bash[27694]: audit 2023-09-23T18:17:11.295362+0000 mon.c (mon.2) 95 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:17:12.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:11 smithi173 bash[27694]: audit 2023-09-23T18:17:11.296615+0000 mon.c (mon.2) 96 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:17:12.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:11 smithi173 bash[27694]: audit 2023-09-23T18:17:11.303855+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:17:12.443 INFO:tasks.workunit.client.0.smithi098.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T18:17:12.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:11 smithi098 bash[27349]: cluster 2023-09-23T18:17:10.630013+0000 mgr.y (mgr.24344) 169 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:12.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:11 smithi098 bash[27349]: audit 2023-09-23T18:17:10.987711+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:17:12.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:11 smithi098 bash[27349]: audit 2023-09-23T18:17:11.293001+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:17:12.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:11 smithi098 bash[27349]: audit 2023-09-23T18:17:11.295362+0000 mon.c (mon.2) 95 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:17:12.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:11 smithi098 bash[27349]: audit 2023-09-23T18:17:11.296615+0000 mon.c (mon.2) 96 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:17:12.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:11 smithi098 bash[27349]: audit 2023-09-23T18:17:11.303855+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:17:12.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:11 smithi098 bash[33599]: cluster 2023-09-23T18:17:10.630013+0000 mgr.y (mgr.24344) 169 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:12.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:11 smithi098 bash[33599]: audit 2023-09-23T18:17:10.987711+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:17:12.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:11 smithi098 bash[33599]: audit 2023-09-23T18:17:11.293001+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:17:12.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:11 smithi098 bash[33599]: audit 2023-09-23T18:17:11.295362+0000 mon.c (mon.2) 95 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:17:12.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:11 smithi098 bash[33599]: audit 2023-09-23T18:17:11.296615+0000 mon.c (mon.2) 96 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:17:12.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:11 smithi098 bash[33599]: audit 2023-09-23T18:17:11.303855+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:17:12.501 INFO:tasks.workunit.client.0.smithi098.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T18:17:12.503 INFO:tasks.workunit.client.0.smithi098.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T18:17:12.503 INFO:tasks.workunit.client.0.smithi098.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T18:17:12.552 INFO:tasks.workunit.client.0.smithi098.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T18:17:12.556 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-23T18:17:12.556 DEBUG:teuthology.orchestra.run.smithi098:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T18:17:12.563 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-09-23T18:17:12.564 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-09-23T18:17:12.564 DEBUG:teuthology.orchestra.run.smithi098:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-09-23T18:17:12.615 INFO:tasks.workunit.client.0.smithi098.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-09-23T18:17:12.616 INFO:tasks.workunit.client.0.smithi098.stderr:+ TMPDIR=/tmp/rbd_import_export_65731 2023-09-23T18:17:12.616 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm -rf /tmp/rbd_import_export_65731 2023-09-23T18:17:12.616 INFO:tasks.workunit.client.0.smithi098.stderr:+ mkdir /tmp/rbd_import_export_65731 2023-09-23T18:17:12.617 INFO:tasks.workunit.client.0.smithi098.stderr:+ trap rm -rf /tmp/rbd_import_export_65731 INT TERM EXIT 2023-09-23T18:17:12.617 INFO:tasks.workunit.client.0.smithi098.stderr:+ mkdir foo.65731 2023-09-23T18:17:12.618 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import foo.65731 foo.dir 2023-09-23T18:17:12.669 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: cannot import a directory 2023-09-23T18:17:12.669 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 0% complete...failed. 2023-09-23T18:17:12.669 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: import failed: (21) Is a directory 2023-09-23T18:17:12.672 INFO:tasks.workunit.client.0.smithi098.stderr:+ true 2023-09-23T18:17:12.672 INFO:tasks.workunit.client.0.smithi098.stderr:+ rmdir foo.65731 2023-09-23T18:17:12.684 INFO:tasks.workunit.client.0.smithi098.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_65731/img bs=1k count=1 seek=10 2023-09-23T18:17:12.685 INFO:tasks.workunit.client.0.smithi098.stderr:1+0 records in 2023-09-23T18:17:12.685 INFO:tasks.workunit.client.0.smithi098.stderr:1+0 records out 2023-09-23T18:17:12.685 INFO:tasks.workunit.client.0.smithi098.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 0.000195945 s, 5.2 MB/s 2023-09-23T18:17:12.685 INFO:tasks.workunit.client.0.smithi098.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_65731/img bs=1k count=10 seek=100 2023-09-23T18:17:12.686 INFO:tasks.workunit.client.0.smithi098.stderr:10+0 records in 2023-09-23T18:17:12.686 INFO:tasks.workunit.client.0.smithi098.stderr:10+0 records out 2023-09-23T18:17:12.686 INFO:tasks.workunit.client.0.smithi098.stderr:10240 bytes (10 kB, 10 KiB) copied, 0.0001073 s, 95.4 MB/s 2023-09-23T18:17:12.686 INFO:tasks.workunit.client.0.smithi098.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_65731/img bs=1k count=100 seek=1000 2023-09-23T18:17:12.688 INFO:tasks.workunit.client.0.smithi098.stderr:70+1 records in 2023-09-23T18:17:12.688 INFO:tasks.workunit.client.0.smithi098.stderr:70+1 records out 2023-09-23T18:17:12.688 INFO:tasks.workunit.client.0.smithi098.stderr:72056 bytes (72 kB, 70 KiB) copied, 0.000618698 s, 116 MB/s 2023-09-23T18:17:12.688 INFO:tasks.workunit.client.0.smithi098.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_65731/img bs=1k seek=10000 2023-09-23T18:17:12.689 INFO:tasks.workunit.client.0.smithi098.stderr:138+1 records in 2023-09-23T18:17:12.689 INFO:tasks.workunit.client.0.smithi098.stderr:138+1 records out 2023-09-23T18:17:12.689 INFO:tasks.workunit.client.0.smithi098.stderr:142144 bytes (142 kB, 139 KiB) copied, 0.000469883 s, 303 MB/s 2023-09-23T18:17:12.689 INFO:tasks.workunit.client.0.smithi098.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_65731/img bs=1k seek=100000 2023-09-23T18:17:12.691 INFO:tasks.workunit.client.0.smithi098.stderr:74+1 records in 2023-09-23T18:17:12.691 INFO:tasks.workunit.client.0.smithi098.stderr:74+1 records out 2023-09-23T18:17:12.691 INFO:tasks.workunit.client.0.smithi098.stderr:76160 bytes (76 kB, 74 KiB) copied, 0.000656063 s, 116 MB/s 2023-09-23T18:17:12.691 INFO:tasks.workunit.client.0.smithi098.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_65731/img bs=1k seek=1000000 2023-09-23T18:17:12.693 INFO:tasks.workunit.client.0.smithi098.stderr:194+1 records in 2023-09-23T18:17:12.693 INFO:tasks.workunit.client.0.smithi098.stderr:194+1 records out 2023-09-23T18:17:12.693 INFO:tasks.workunit.client.0.smithi098.stderr:199136 bytes (199 kB, 194 KiB) copied, 0.000659487 s, 302 MB/s 2023-09-23T18:17:12.693 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm testimg 2023-09-23T18:17:12.743 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 0% complete...failed. 2023-09-23T18:17:12.743 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T18:17:12.747 INFO:tasks.workunit.client.0.smithi098.stderr:+ true 2023-09-23T18:17:12.747 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --new-format /tmp/rbd_import_export_65731/img testimg 2023-09-23T18:17:13.245 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:17:12 smithi098 bash[27618]: [23/Sep/2023:18:17:12] ENGINE Bus STOPPING 2023-09-23T18:17:13.245 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:17:12 smithi098 bash[27618]: [23/Sep/2023:18:17:12] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T18:17:13.245 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:17:12 smithi098 bash[27618]: [23/Sep/2023:18:17:12] ENGINE Bus STOPPED 2023-09-23T18:17:13.245 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:17:12 smithi098 bash[27618]: [23/Sep/2023:18:17:12] ENGINE Bus STARTING 2023-09-23T18:17:13.245 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:17:13 smithi098 bash[27618]: [23/Sep/2023:18:17:13] ENGINE Serving on http://:::9283 2023-09-23T18:17:13.246 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:17:13 smithi098 bash[27618]: [23/Sep/2023:18:17:13] ENGINE Bus STARTED 2023-09-23T18:17:13.850 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-09-23T18:17:14.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:13 smithi098 bash[27349]: cluster 2023-09-23T18:17:12.630420+0000 mgr.y (mgr.24344) 170 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:14.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:13 smithi098 bash[27349]: audit 2023-09-23T18:17:12.965778+0000 mon.a (mon.0) 681 : audit [INF] from='client.? 172.21.15.98:0/637604204' entity='client.admin' 2023-09-23T18:17:14.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:13 smithi098 bash[27349]: audit 2023-09-23T18:17:12.985190+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:17:14.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:13 smithi098 bash[27349]: audit 2023-09-23T18:17:12.991249+0000 mon.c (mon.2) 98 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:17:14.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:13 smithi098 bash[27349]: cluster 2023-09-23T18:17:12.996318+0000 mon.a (mon.0) 682 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-09-23T18:17:14.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:13 smithi098 bash[27349]: audit 2023-09-23T18:17:12.999750+0000 mon.c (mon.2) 99 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:17:14.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:13 smithi098 bash[27349]: audit 2023-09-23T18:17:13.007355+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:17:14.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:13 smithi098 bash[33599]: cluster 2023-09-23T18:17:12.630420+0000 mgr.y (mgr.24344) 170 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:14.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:13 smithi098 bash[33599]: audit 2023-09-23T18:17:12.965778+0000 mon.a (mon.0) 681 : audit [INF] from='client.? 172.21.15.98:0/637604204' entity='client.admin' 2023-09-23T18:17:14.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:13 smithi098 bash[33599]: audit 2023-09-23T18:17:12.985190+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:17:14.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:13 smithi098 bash[33599]: audit 2023-09-23T18:17:12.991249+0000 mon.c (mon.2) 98 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:17:14.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:13 smithi098 bash[33599]: cluster 2023-09-23T18:17:12.996318+0000 mon.a (mon.0) 682 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-09-23T18:17:14.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:13 smithi098 bash[33599]: audit 2023-09-23T18:17:12.999750+0000 mon.c (mon.2) 99 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:17:14.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:13 smithi098 bash[33599]: audit 2023-09-23T18:17:13.007355+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:17:14.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:13 smithi173 bash[27694]: cluster 2023-09-23T18:17:12.630420+0000 mgr.y (mgr.24344) 170 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:14.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:13 smithi173 bash[27694]: audit 2023-09-23T18:17:12.965778+0000 mon.a (mon.0) 681 : audit [INF] from='client.? 172.21.15.98:0/637604204' entity='client.admin' 2023-09-23T18:17:14.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:13 smithi173 bash[27694]: audit 2023-09-23T18:17:12.985190+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:17:14.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:13 smithi173 bash[27694]: audit 2023-09-23T18:17:12.991249+0000 mon.c (mon.2) 98 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:17:14.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:13 smithi173 bash[27694]: cluster 2023-09-23T18:17:12.996318+0000 mon.a (mon.0) 682 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-09-23T18:17:14.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:13 smithi173 bash[27694]: audit 2023-09-23T18:17:12.999750+0000 mon.c (mon.2) 99 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:17:14.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:13 smithi173 bash[27694]: audit 2023-09-23T18:17:13.007355+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:17:14.624 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-09-23T18:17:14.625 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T18:17:14.629 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export testimg /tmp/rbd_import_export_65731/img2 2023-09-23T18:17:14.927 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T18:17:14.931 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export testimg - 2023-09-23T18:17:15.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:15 smithi173 bash[27694]: cluster 2023-09-23T18:17:14.001250+0000 mon.a (mon.0) 684 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-09-23T18:17:15.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:15 smithi098 bash[33599]: cluster 2023-09-23T18:17:14.001250+0000 mon.a (mon.0) 684 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-09-23T18:17:15.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:15 smithi098 bash[27349]: cluster 2023-09-23T18:17:14.001250+0000 mon.a (mon.0) 684 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-09-23T18:17:15.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:17:15 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:17:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:17:15.792 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T18:17:15.795 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm testimg 2023-09-23T18:17:15.939 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T18:17:15.942 INFO:tasks.workunit.client.0.smithi098.stderr:+ cmp /tmp/rbd_import_export_65731/img /tmp/rbd_import_export_65731/img2 2023-09-23T18:17:16.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:16 smithi173 bash[27694]: cluster 2023-09-23T18:17:14.630742+0000 mgr.y (mgr.24344) 171 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2023-09-23T18:17:16.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:16 smithi098 bash[27349]: cluster 2023-09-23T18:17:14.630742+0000 mgr.y (mgr.24344) 171 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2023-09-23T18:17:16.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:16 smithi098 bash[33599]: cluster 2023-09-23T18:17:14.630742+0000 mgr.y (mgr.24344) 171 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2023-09-23T18:17:16.656 INFO:tasks.workunit.client.0.smithi098.stderr:+ cmp /tmp/rbd_import_export_65731/img /tmp/rbd_import_export_65731/img3 2023-09-23T18:17:17.157 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_65731/img2 /tmp/rbd_import_export_65731/img3 2023-09-23T18:17:18.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:18 smithi098 bash[27349]: cluster 2023-09-23T18:17:16.631177+0000 mgr.y (mgr.24344) 172 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2023-09-23T18:17:18.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:17:17 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:17:17] "GET /metrics HTTP/1.1" 200 191529 "" "Prometheus/2.33.4" 2023-09-23T18:17:18.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:18 smithi098 bash[33599]: cluster 2023-09-23T18:17:16.631177+0000 mgr.y (mgr.24344) 172 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2023-09-23T18:17:18.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:18 smithi173 bash[27694]: cluster 2023-09-23T18:17:16.631177+0000 mgr.y (mgr.24344) 172 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2023-09-23T18:17:19.463 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --new-format - testimg 2023-09-23T18:17:19.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:19 smithi098 bash[27349]: cluster 2023-09-23T18:17:18.631802+0000 mgr.y (mgr.24344) 173 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 151 KiB/s rd, 65 KiB/s wr, 38 op/s 2023-09-23T18:17:19.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:19 smithi098 bash[33599]: cluster 2023-09-23T18:17:18.631802+0000 mgr.y (mgr.24344) 173 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 151 KiB/s rd, 65 KiB/s wr, 38 op/s 2023-09-23T18:17:19.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:19 smithi173 bash[27694]: cluster 2023-09-23T18:17:18.631802+0000 mgr.y (mgr.24344) 173 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 151 KiB/s rd, 65 KiB/s wr, 38 op/s 2023-09-23T18:17:19.981 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 100% complete...done. 2023-09-23T18:17:19.982 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T18:17:19.986 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export testimg /tmp/rbd_import_export_65731/img2 2023-09-23T18:17:20.351 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T18:17:20.355 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export testimg - 2023-09-23T18:17:21.228 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T18:17:21.234 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm testimg 2023-09-23T18:17:21.372 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T18:17:21.377 INFO:tasks.workunit.client.0.smithi098.stderr:+ cmp /tmp/rbd_import_export_65731/img /tmp/rbd_import_export_65731/img2 2023-09-23T18:17:21.475 INFO:teuthology.orchestra.run.smithi098.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T18:17:21.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:21 smithi098 bash[27349]: cluster 2023-09-23T18:17:20.632403+0000 mgr.y (mgr.24344) 174 : cluster [DBG] pgmap v155: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 151 KiB/s rd, 65 KiB/s wr, 38 op/s 2023-09-23T18:17:21.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:21 smithi098 bash[27349]: audit 2023-09-23T18:17:21.468150+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:17:21.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:21 smithi098 bash[27349]: audit 2023-09-23T18:17:21.497230+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:17:21.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:21 smithi098 bash[27349]: audit 2023-09-23T18:17:21.499921+0000 mon.c (mon.2) 101 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:17:21.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:21 smithi098 bash[27349]: audit 2023-09-23T18:17:21.501027+0000 mon.c (mon.2) 102 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:17:21.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:21 smithi098 bash[27349]: audit 2023-09-23T18:17:21.507695+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:17:21.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:21 smithi098 bash[33599]: cluster 2023-09-23T18:17:20.632403+0000 mgr.y (mgr.24344) 174 : cluster [DBG] pgmap v155: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 151 KiB/s rd, 65 KiB/s wr, 38 op/s 2023-09-23T18:17:21.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:21 smithi098 bash[33599]: audit 2023-09-23T18:17:21.468150+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:17:21.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:21 smithi098 bash[33599]: audit 2023-09-23T18:17:21.497230+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:17:21.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:21 smithi098 bash[33599]: audit 2023-09-23T18:17:21.499921+0000 mon.c (mon.2) 101 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:17:21.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:21 smithi098 bash[33599]: audit 2023-09-23T18:17:21.501027+0000 mon.c (mon.2) 102 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:17:21.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:21 smithi098 bash[33599]: audit 2023-09-23T18:17:21.507695+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:17:22.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:21 smithi173 bash[27694]: cluster 2023-09-23T18:17:20.632403+0000 mgr.y (mgr.24344) 174 : cluster [DBG] pgmap v155: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 151 KiB/s rd, 65 KiB/s wr, 38 op/s 2023-09-23T18:17:22.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:21 smithi173 bash[27694]: audit 2023-09-23T18:17:21.468150+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:17:22.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:21 smithi173 bash[27694]: audit 2023-09-23T18:17:21.497230+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:17:22.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:21 smithi173 bash[27694]: audit 2023-09-23T18:17:21.499921+0000 mon.c (mon.2) 101 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:17:22.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:21 smithi173 bash[27694]: audit 2023-09-23T18:17:21.501027+0000 mon.c (mon.2) 102 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:17:22.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:21 smithi173 bash[27694]: audit 2023-09-23T18:17:21.507695+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:17:22.125 INFO:tasks.workunit.client.0.smithi098.stderr:+ cmp /tmp/rbd_import_export_65731/img /tmp/rbd_import_export_65731/img3 2023-09-23T18:17:22.633 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_65731/img /tmp/rbd_import_export_65731/img2 /tmp/rbd_import_export_65731/img3 2023-09-23T18:17:23.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:22 smithi173 bash[27694]: audit 2023-09-23T18:17:21.459742+0000 mgr.y (mgr.24344) 175 : audit [DBG] from='client.14862 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:17:23.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:22 smithi173 bash[27694]: cephadm 2023-09-23T18:17:21.460912+0000 mgr.y (mgr.24344) 176 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T18:17:23.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:22 smithi173 bash[27694]: cephadm 2023-09-23T18:17:21.515795+0000 mgr.y (mgr.24344) 177 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T18:17:23.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:22 smithi098 bash[33599]: audit 2023-09-23T18:17:21.459742+0000 mgr.y (mgr.24344) 175 : audit [DBG] from='client.14862 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:17:23.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:22 smithi098 bash[33599]: cephadm 2023-09-23T18:17:21.460912+0000 mgr.y (mgr.24344) 176 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T18:17:23.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:22 smithi098 bash[33599]: cephadm 2023-09-23T18:17:21.515795+0000 mgr.y (mgr.24344) 177 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T18:17:23.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:22 smithi098 bash[27349]: audit 2023-09-23T18:17:21.459742+0000 mgr.y (mgr.24344) 175 : audit [DBG] from='client.14862 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:17:23.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:22 smithi098 bash[27349]: cephadm 2023-09-23T18:17:21.460912+0000 mgr.y (mgr.24344) 176 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T18:17:23.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:22 smithi098 bash[27349]: cephadm 2023-09-23T18:17:21.515795+0000 mgr.y (mgr.24344) 177 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T18:17:24.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:23 smithi173 bash[27694]: cluster 2023-09-23T18:17:22.633016+0000 mgr.y (mgr.24344) 178 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 54 KiB/s wr, 31 op/s 2023-09-23T18:17:24.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:23 smithi173 bash[27694]: cluster 2023-09-23T18:17:23.381063+0000 mon.a (mon.0) 687 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-09-23T18:17:24.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:23 smithi098 bash[27349]: cluster 2023-09-23T18:17:22.633016+0000 mgr.y (mgr.24344) 178 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 54 KiB/s wr, 31 op/s 2023-09-23T18:17:24.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:23 smithi098 bash[27349]: cluster 2023-09-23T18:17:23.381063+0000 mon.a (mon.0) 687 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-09-23T18:17:24.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:23 smithi098 bash[33599]: cluster 2023-09-23T18:17:22.633016+0000 mgr.y (mgr.24344) 178 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 54 KiB/s wr, 31 op/s 2023-09-23T18:17:24.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:23 smithi098 bash[33599]: cluster 2023-09-23T18:17:23.381063+0000 mon.a (mon.0) 687 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-09-23T18:17:25.716 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:17:25 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:17:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:17:26.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:25 smithi173 bash[27694]: cluster 2023-09-23T18:17:24.633664+0000 mgr.y (mgr.24344) 179 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 256 KiB/s rd, 105 KiB/s wr, 80 op/s 2023-09-23T18:17:26.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:25 smithi098 bash[27349]: cluster 2023-09-23T18:17:24.633664+0000 mgr.y (mgr.24344) 179 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 256 KiB/s rd, 105 KiB/s wr, 80 op/s 2023-09-23T18:17:26.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:25 smithi098 bash[33599]: cluster 2023-09-23T18:17:24.633664+0000 mgr.y (mgr.24344) 179 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 256 KiB/s rd, 105 KiB/s wr, 80 op/s 2023-09-23T18:17:28.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:27 smithi173 bash[27694]: cluster 2023-09-23T18:17:26.634174+0000 mgr.y (mgr.24344) 180 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 256 KiB/s rd, 105 KiB/s wr, 80 op/s 2023-09-23T18:17:28.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:27 smithi098 bash[27349]: cluster 2023-09-23T18:17:26.634174+0000 mgr.y (mgr.24344) 180 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 256 KiB/s rd, 105 KiB/s wr, 80 op/s 2023-09-23T18:17:28.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:17:27 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:17:27] "GET /metrics HTTP/1.1" 200 191529 "" "Prometheus/2.33.4" 2023-09-23T18:17:28.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:27 smithi098 bash[33599]: cluster 2023-09-23T18:17:26.634174+0000 mgr.y (mgr.24344) 180 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 256 KiB/s rd, 105 KiB/s wr, 80 op/s 2023-09-23T18:17:28.337 INFO:tasks.workunit.client.0.smithi098.stderr:+ + grep -q export-format 2023-09-23T18:17:28.338 INFO:tasks.workunit.client.0.smithi098.stderr:rbd help export 2023-09-23T18:17:28.356 INFO:tasks.workunit.client.0.smithi098.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_65731/img bs=1k count=10 seek=100 2023-09-23T18:17:28.357 INFO:tasks.workunit.client.0.smithi098.stderr:10+0 records in 2023-09-23T18:17:28.357 INFO:tasks.workunit.client.0.smithi098.stderr:10+0 records out 2023-09-23T18:17:28.357 INFO:tasks.workunit.client.0.smithi098.stderr:10240 bytes (10 kB, 10 KiB) copied, 0.000118999 s, 86.1 MB/s 2023-09-23T18:17:28.357 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --new-format /tmp/rbd_import_export_65731/img testimg 2023-09-23T18:17:28.426 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 100% complete...done. 2023-09-23T18:17:28.427 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T18:17:28.429 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd snap create testimg@snap 2023-09-23T18:17:29.373 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T18:17:29.408 INFO:tasks.workunit.client.0.smithi098.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T18:17:29.419 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd image-meta set testimg key1 value1 2023-09-23T18:17:29.492 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd image-meta list testimg 2023-09-23T18:17:29.545 INFO:tasks.workunit.client.0.smithi098.stderr:+ IMAGEMETA_BEFORE=There is 1 metadatum on this image: 2023-09-23T18:17:29.545 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-23T18:17:29.545 INFO:tasks.workunit.client.0.smithi098.stderr:Key Value 2023-09-23T18:17:29.545 INFO:tasks.workunit.client.0.smithi098.stderr:key1 value1 2023-09-23T18:17:29.545 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_65731/img_v2 2023-09-23T18:17:29.597 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T18:17:29.604 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_65731/img_v2 testimg_import 2023-09-23T18:17:29.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:29 smithi098 bash[27349]: cluster 2023-09-23T18:17:28.635242+0000 mgr.y (mgr.24344) 181 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 142 KiB/s rd, 55 KiB/s wr, 59 op/s 2023-09-23T18:17:29.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:29 smithi098 bash[33599]: cluster 2023-09-23T18:17:28.635242+0000 mgr.y (mgr.24344) 181 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 142 KiB/s rd, 55 KiB/s wr, 59 op/s 2023-09-23T18:17:29.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:29 smithi173 bash[27694]: cluster 2023-09-23T18:17:28.635242+0000 mgr.y (mgr.24344) 181 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 142 KiB/s rd, 55 KiB/s wr, 59 op/s 2023-09-23T18:17:30.425 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T18:17:30.430 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info testimg_import 2023-09-23T18:17:30.472 INFO:tasks.workunit.client.0.smithi098.stdout:rbd image 'testimg_import': 2023-09-23T18:17:30.472 INFO:tasks.workunit.client.0.smithi098.stdout: size 110 KiB in 1 objects 2023-09-23T18:17:30.473 INFO:tasks.workunit.client.0.smithi098.stdout: order 22 (4 MiB objects) 2023-09-23T18:17:30.473 INFO:tasks.workunit.client.0.smithi098.stdout: snapshot_count: 1 2023-09-23T18:17:30.473 INFO:tasks.workunit.client.0.smithi098.stdout: id: 60e3eab998f 2023-09-23T18:17:30.473 INFO:tasks.workunit.client.0.smithi098.stdout: block_name_prefix: rbd_data.60e3eab998f 2023-09-23T18:17:30.473 INFO:tasks.workunit.client.0.smithi098.stdout: format: 2 2023-09-23T18:17:30.473 INFO:tasks.workunit.client.0.smithi098.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T18:17:30.473 INFO:tasks.workunit.client.0.smithi098.stdout: op_features: 2023-09-23T18:17:30.473 INFO:tasks.workunit.client.0.smithi098.stdout: flags: 2023-09-23T18:17:30.473 INFO:tasks.workunit.client.0.smithi098.stdout: create_timestamp: Sat Sep 23 18:17:29 2023 2023-09-23T18:17:30.473 INFO:tasks.workunit.client.0.smithi098.stdout: access_timestamp: Sat Sep 23 18:17:29 2023 2023-09-23T18:17:30.473 INFO:tasks.workunit.client.0.smithi098.stdout: modify_timestamp: Sat Sep 23 18:17:29 2023 2023-09-23T18:17:30.477 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info testimg_import@snap 2023-09-23T18:17:30.519 INFO:tasks.workunit.client.0.smithi098.stdout:rbd image 'testimg_import': 2023-09-23T18:17:30.520 INFO:tasks.workunit.client.0.smithi098.stdout: size 110 KiB in 1 objects 2023-09-23T18:17:30.520 INFO:tasks.workunit.client.0.smithi098.stdout: order 22 (4 MiB objects) 2023-09-23T18:17:30.520 INFO:tasks.workunit.client.0.smithi098.stdout: snapshot_count: 1 2023-09-23T18:17:30.520 INFO:tasks.workunit.client.0.smithi098.stdout: id: 60e3eab998f 2023-09-23T18:17:30.520 INFO:tasks.workunit.client.0.smithi098.stdout: block_name_prefix: rbd_data.60e3eab998f 2023-09-23T18:17:30.520 INFO:tasks.workunit.client.0.smithi098.stdout: format: 2 2023-09-23T18:17:30.520 INFO:tasks.workunit.client.0.smithi098.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T18:17:30.520 INFO:tasks.workunit.client.0.smithi098.stdout: op_features: 2023-09-23T18:17:30.520 INFO:tasks.workunit.client.0.smithi098.stdout: flags: 2023-09-23T18:17:30.520 INFO:tasks.workunit.client.0.smithi098.stdout: create_timestamp: Sat Sep 23 18:17:29 2023 2023-09-23T18:17:30.520 INFO:tasks.workunit.client.0.smithi098.stdout: access_timestamp: Sat Sep 23 18:17:29 2023 2023-09-23T18:17:30.520 INFO:tasks.workunit.client.0.smithi098.stdout: modify_timestamp: Sat Sep 23 18:17:29 2023 2023-09-23T18:17:30.521 INFO:tasks.workunit.client.0.smithi098.stdout: protected: False 2023-09-23T18:17:30.524 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd image-meta list testimg_import 2023-09-23T18:17:30.576 INFO:tasks.workunit.client.0.smithi098.stderr:+ IMAGEMETA_AFTER=There is 1 metadatum on this image: 2023-09-23T18:17:30.576 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-23T18:17:30.576 INFO:tasks.workunit.client.0.smithi098.stderr:Key Value 2023-09-23T18:17:30.576 INFO:tasks.workunit.client.0.smithi098.stderr:key1 value1 2023-09-23T18:17:30.576 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ There is 1 metadatum on this image: 2023-09-23T18:17:30.576 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-23T18:17:30.577 INFO:tasks.workunit.client.0.smithi098.stderr:Key Value 2023-09-23T18:17:30.577 INFO:tasks.workunit.client.0.smithi098.stderr:key1 value1 = There is 1 metadatum on this image: 2023-09-23T18:17:30.577 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-23T18:17:30.577 INFO:tasks.workunit.client.0.smithi098.stderr:Key Value 2023-09-23T18:17:30.577 INFO:tasks.workunit.client.0.smithi098.stderr:key1 value1 ] 2023-09-23T18:17:30.577 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export testimg_import /tmp/rbd_import_export_65731/img_import 2023-09-23T18:17:30.622 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 100% complete...done. 2023-09-23T18:17:30.626 INFO:tasks.workunit.client.0.smithi098.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_65731/img /tmp/rbd_import_export_65731/img_import 2023-09-23T18:17:30.626 INFO:tasks.workunit.client.0.smithi098.stderr:+ cmp -l /tmp/rbd_import_export_65731/img /tmp/rbd_import_export_65731/img_import 2023-09-23T18:17:30.627 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_65731/img --format %b 2023-09-23T18:17:30.628 INFO:tasks.workunit.client.0.smithi098.stderr:+ origsize=24 2023-09-23T18:17:30.628 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_65731/img_import --format %b 2023-09-23T18:17:30.630 INFO:tasks.workunit.client.0.smithi098.stderr:+ exportsize=224 2023-09-23T18:17:30.630 INFO:tasks.workunit.client.0.smithi098.stderr:+ difference=200 2023-09-23T18:17:30.630 INFO:tasks.workunit.client.0.smithi098.stderr:+ difference=200 2023-09-23T18:17:30.630 INFO:tasks.workunit.client.0.smithi098.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-09-23T18:17:30.630 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_65731/img_snap 2023-09-23T18:17:30.674 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 100% complete...done. 2023-09-23T18:17:30.679 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_65731/img_snap_import 2023-09-23T18:17:30.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:30 smithi098 bash[33599]: cluster 2023-09-23T18:17:29.391469+0000 mon.a (mon.0) 688 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-09-23T18:17:30.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:30 smithi098 bash[27349]: cluster 2023-09-23T18:17:29.391469+0000 mon.a (mon.0) 688 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-09-23T18:17:30.722 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 100% complete...done. 2023-09-23T18:17:30.726 INFO:tasks.workunit.client.0.smithi098.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_65731/img_snap /tmp/rbd_import_export_65731/img_snap_import 2023-09-23T18:17:30.726 INFO:tasks.workunit.client.0.smithi098.stderr:+ cmp -l /tmp/rbd_import_export_65731/img_snap /tmp/rbd_import_export_65731/img_snap_import 2023-09-23T18:17:30.727 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_65731/img_snap --format %b 2023-09-23T18:17:30.729 INFO:tasks.workunit.client.0.smithi098.stderr:+ origsize=224 2023-09-23T18:17:30.729 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_65731/img_snap_import --format %b 2023-09-23T18:17:30.730 INFO:tasks.workunit.client.0.smithi098.stderr:+ exportsize=224 2023-09-23T18:17:30.730 INFO:tasks.workunit.client.0.smithi098.stderr:+ difference=0 2023-09-23T18:17:30.730 INFO:tasks.workunit.client.0.smithi098.stderr:+ difference=0 2023-09-23T18:17:30.730 INFO:tasks.workunit.client.0.smithi098.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T18:17:30.730 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_65731/img_v2 2023-09-23T18:17:30.731 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_65731/img_import 2023-09-23T18:17:30.731 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_65731/img_snap 2023-09-23T18:17:30.732 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_65731/img_snap_import 2023-09-23T18:17:30.733 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd snap rm testimg_import@snap 2023-09-23T18:17:30.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:30 smithi173 bash[27694]: cluster 2023-09-23T18:17:29.391469+0000 mon.a (mon.0) 688 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-09-23T18:17:31.418 INFO:tasks.workunit.client.0.smithi098.stderr: Removing snap: 100% complete...done. 2023-09-23T18:17:31.430 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd remove testimg_import 2023-09-23T18:17:31.558 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 100% complete...done. 2023-09-23T18:17:31.562 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd snap rm testimg@snap 2023-09-23T18:17:31.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:31 smithi098 bash[27349]: cluster 2023-09-23T18:17:30.399233+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-09-23T18:17:31.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:31 smithi098 bash[27349]: cluster 2023-09-23T18:17:30.635772+0000 mgr.y (mgr.24344) 182 : cluster [DBG] pgmap v163: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 196 KiB/s rd, 76 KiB/s wr, 81 op/s 2023-09-23T18:17:31.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:31 smithi098 bash[33599]: cluster 2023-09-23T18:17:30.399233+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-09-23T18:17:31.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:31 smithi098 bash[33599]: cluster 2023-09-23T18:17:30.635772+0000 mgr.y (mgr.24344) 182 : cluster [DBG] pgmap v163: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 196 KiB/s rd, 76 KiB/s wr, 81 op/s 2023-09-23T18:17:31.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:31 smithi173 bash[27694]: cluster 2023-09-23T18:17:30.399233+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-09-23T18:17:31.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:31 smithi173 bash[27694]: cluster 2023-09-23T18:17:30.635772+0000 mgr.y (mgr.24344) 182 : cluster [DBG] pgmap v163: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 196 KiB/s rd, 76 KiB/s wr, 81 op/s 2023-09-23T18:17:32.428 INFO:tasks.workunit.client.0.smithi098.stderr: Removing snap: 100% complete...done. 2023-09-23T18:17:32.438 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm testimg 2023-09-23T18:17:32.574 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 100% complete...done. 2023-09-23T18:17:32.582 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --order 20 /tmp/rbd_import_export_65731/img testimg 2023-09-23T18:17:32.664 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 100% complete...done. 2023-09-23T18:17:32.665 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T18:17:32.667 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_65731/img_v2 2023-09-23T18:17:32.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:32 smithi098 bash[33599]: cluster 2023-09-23T18:17:31.406814+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-09-23T18:17:32.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:32 smithi098 bash[27349]: cluster 2023-09-23T18:17:31.406814+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-09-23T18:17:32.723 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T18:17:32.727 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_65731/img_v2 testimg_import 2023-09-23T18:17:32.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:32 smithi173 bash[27694]: cluster 2023-09-23T18:17:31.406814+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-09-23T18:17:32.816 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T18:17:32.822 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info testimg_import 2023-09-23T18:17:32.822 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep order 2023-09-23T18:17:32.823 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk {print $2} 2023-09-23T18:17:32.823 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep 20 2023-09-23T18:17:32.869 INFO:tasks.workunit.client.0.smithi098.stdout:20 2023-09-23T18:17:32.869 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_65731/img_v2 2023-09-23T18:17:32.869 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd remove testimg_import 2023-09-23T18:17:33.012 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 100% complete...done. 2023-09-23T18:17:33.017 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd remove testimg 2023-09-23T18:17:33.177 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 100% complete...done. 2023-09-23T18:17:33.177 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_65731/img testimg 2023-09-23T18:17:33.234 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 100% complete...done. 2023-09-23T18:17:33.238 INFO:tasks.workunit.client.0.smithi098.stderr:+ + rbd info testimg 2023-09-23T18:17:33.238 INFO:tasks.workunit.client.0.smithi098.stderr:grep features 2023-09-23T18:17:33.288 INFO:tasks.workunit.client.0.smithi098.stderr:+ FEATURES_BEFORE= features: layering 2023-09-23T18:17:33.288 INFO:tasks.workunit.client.0.smithi098.stderr: op_features: 2023-09-23T18:17:33.289 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_65731/img_v2 2023-09-23T18:17:33.335 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T18:17:33.340 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_65731/img_v2 testimg_import 2023-09-23T18:17:33.421 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T18:17:33.426 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info testimg_import 2023-09-23T18:17:33.427 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep features 2023-09-23T18:17:33.478 INFO:tasks.workunit.client.0.smithi098.stderr:+ FEATURES_AFTER= features: layering 2023-09-23T18:17:33.478 INFO:tasks.workunit.client.0.smithi098.stderr: op_features: 2023-09-23T18:17:33.479 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ features: layering 2023-09-23T18:17:33.479 INFO:tasks.workunit.client.0.smithi098.stderr: op_features: != features: layering 2023-09-23T18:17:33.479 INFO:tasks.workunit.client.0.smithi098.stderr: op_features: ] 2023-09-23T18:17:33.479 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_65731/img_v2 2023-09-23T18:17:33.479 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd remove testimg_import 2023-09-23T18:17:33.481 INFO:teuthology.orchestra.run.smithi098.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T18:17:33.481 INFO:teuthology.orchestra.run.smithi098.stdout:alertmanager.a smithi098 *:9093,9094 running (3m) 22s ago 4m 13.0M - 0.23.0 ba2b418f427c 8915ff19e07f 2023-09-23T18:17:33.481 INFO:teuthology.orchestra.run.smithi098.stdout:grafana.a smithi173 *:3000 running (3m) 22s ago 3m 43.6M - 8.3.5 dad864ee21e9 721421fe67ad 2023-09-23T18:17:33.481 INFO:teuthology.orchestra.run.smithi098.stdout:mds.foo.smithi098.lowjqo smithi098 running (2m) 22s ago 2m 13.1M - 16.2.14-37-g2e1f1015 1e6a756174a8 e1555281d4fd 2023-09-23T18:17:33.481 INFO:teuthology.orchestra.run.smithi098.stdout:mds.foo.smithi173.qjefje smithi173 running (2m) 22s ago 2m 11.5M - 16.2.14-37-g2e1f1015 1e6a756174a8 fc5513981852 2023-09-23T18:17:33.481 INFO:teuthology.orchestra.run.smithi098.stdout:mgr.x smithi173 *:8443 running (11m) 22s ago 11m 375M - 16.2.14-37-g2e1f1015 1e6a756174a8 064fb393b852 2023-09-23T18:17:33.481 INFO:teuthology.orchestra.run.smithi098.stdout:mgr.y smithi098 *:9283 running (13m) 22s ago 13m 427M - 16.2.14-37-g2e1f1015 1e6a756174a8 18927b226cb1 2023-09-23T18:17:33.482 INFO:teuthology.orchestra.run.smithi098.stdout:mon.a smithi098 running (13m) 22s ago 13m 50.4M 2048M 16.2.14-37-g2e1f1015 1e6a756174a8 35c9bdb63747 2023-09-23T18:17:33.482 INFO:teuthology.orchestra.run.smithi098.stdout:mon.b smithi173 running (11m) 22s ago 11m 44.6M 2048M 16.2.14-37-g2e1f1015 1e6a756174a8 0c6482ba189f 2023-09-23T18:17:33.482 INFO:teuthology.orchestra.run.smithi098.stdout:mon.c smithi098 running (11m) 22s ago 11m 45.2M 2048M 16.2.14-37-g2e1f1015 1e6a756174a8 5d5fc8fcca20 2023-09-23T18:17:33.482 INFO:teuthology.orchestra.run.smithi098.stdout:node-exporter.a smithi098 *:9100 running (4m) 22s ago 4m 10.2M - 1.3.1 1dbe0e931976 693f31cef03b 2023-09-23T18:17:33.482 INFO:teuthology.orchestra.run.smithi098.stdout:node-exporter.b smithi173 *:9100 running (4m) 22s ago 4m 9627k - 1.3.1 1dbe0e931976 2b51cc6dc9d7 2023-09-23T18:17:33.482 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0 smithi098 running (10m) 22s ago 10m 38.0M 4060M 16.2.14-37-g2e1f1015 1e6a756174a8 5900bdd3fa79 2023-09-23T18:17:33.482 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1 smithi098 running (9m) 22s ago 9m 38.4M 4060M 16.2.14-37-g2e1f1015 1e6a756174a8 7fd00c3be01c 2023-09-23T18:17:33.482 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2 smithi098 running (9m) 22s ago 9m 39.5M 4060M 16.2.14-37-g2e1f1015 1e6a756174a8 587769f25c85 2023-09-23T18:17:33.482 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3 smithi098 running (8m) 22s ago 8m 39.4M 4060M 16.2.14-37-g2e1f1015 1e6a756174a8 af525819942d 2023-09-23T18:17:33.482 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4 smithi173 running (7m) 22s ago 7m 38.7M 4316M 16.2.14-37-g2e1f1015 1e6a756174a8 0a479b881bee 2023-09-23T18:17:33.482 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5 smithi173 running (6m) 22s ago 6m 37.7M 4316M 16.2.14-37-g2e1f1015 1e6a756174a8 501903da3aa9 2023-09-23T18:17:33.482 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6 smithi173 running (6m) 22s ago 6m 38.3M 4316M 16.2.14-37-g2e1f1015 1e6a756174a8 b37149ed4271 2023-09-23T18:17:33.483 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7 smithi173 running (5m) 22s ago 5m 38.7M 4316M 16.2.14-37-g2e1f1015 1e6a756174a8 81723c51be2b 2023-09-23T18:17:33.483 INFO:teuthology.orchestra.run.smithi098.stdout:prometheus.a smithi173 *:9095 running (3m) 22s ago 4m 50.8M - 2.33.4 514e6a882f6e bd8e76c2d80a 2023-09-23T18:17:33.590 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 100% complete...done. 2023-09-23T18:17:33.594 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd remove testimg 2023-09-23T18:17:33.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:33 smithi098 bash[27349]: cluster 2023-09-23T18:17:32.416642+0000 mon.a (mon.0) 691 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-09-23T18:17:33.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:33 smithi098 bash[27349]: cluster 2023-09-23T18:17:32.636299+0000 mgr.y (mgr.24344) 183 : cluster [DBG] pgmap v166: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:33.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:33 smithi098 bash[33599]: cluster 2023-09-23T18:17:32.416642+0000 mon.a (mon.0) 691 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-09-23T18:17:33.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:33 smithi098 bash[33599]: cluster 2023-09-23T18:17:32.636299+0000 mgr.y (mgr.24344) 183 : cluster [DBG] pgmap v166: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:33.697 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 100% complete...done. 2023-09-23T18:17:33.702 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_65731/img testimg 2023-09-23T18:17:33.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:33 smithi173 bash[27694]: cluster 2023-09-23T18:17:32.416642+0000 mon.a (mon.0) 691 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-09-23T18:17:33.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:33 smithi173 bash[27694]: cluster 2023-09-23T18:17:32.636299+0000 mgr.y (mgr.24344) 183 : cluster [DBG] pgmap v166: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:33.789 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 100% complete...done. 2023-09-23T18:17:33.794 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_65731/img_v2 2023-09-23T18:17:33.848 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T18:17:33.852 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_65731/img_v2 testimg_import 2023-09-23T18:17:33.943 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T18:17:33.948 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info testimg_import 2023-09-23T18:17:33.948 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep stripe unit 2023-09-23T18:17:33.948 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep -Ei (4 KiB|4096) 2023-09-23T18:17:33.999 INFO:tasks.workunit.client.0.smithi098.stdout: stripe unit: 4 KiB 2023-09-23T18:17:34.000 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info testimg_import 2023-09-23T18:17:34.000 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep stripe count 2023-09-23T18:17:34.001 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk {print $3} 2023-09-23T18:17:34.001 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep 1000 2023-09-23T18:17:34.052 INFO:tasks.workunit.client.0.smithi098.stdout:1000 2023-09-23T18:17:34.052 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_65731/img_v2 2023-09-23T18:17:34.212 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd remove testimg_import 2023-09-23T18:17:34.360 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T18:17:34.365 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd remove testimg 2023-09-23T18:17:34.503 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T18:17:34.507 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_65731/img testimg 2023-09-23T18:17:34.575 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for minority of mons to upgrade' 2023-09-23T18:17:34.587 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 100% complete...done. 2023-09-23T18:17:34.592 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd snap create testimg@snap1 2023-09-23T18:17:34.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:34 smithi098 bash[33599]: audit 2023-09-23T18:17:33.461312+0000 mgr.y (mgr.24344) 184 : audit [DBG] from='client.15012 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:17:34.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:34 smithi098 bash[27349]: audit 2023-09-23T18:17:33.461312+0000 mgr.y (mgr.24344) 184 : audit [DBG] from='client.15012 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:17:34.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:34 smithi173 bash[27694]: audit 2023-09-23T18:17:33.461312+0000 mgr.y (mgr.24344) 184 : audit [DBG] from='client.15012 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:17:35.452 INFO:tasks.workunit.client.0.smithi098.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T18:17:35.463 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd snap create testimg@snap2 2023-09-23T18:17:35.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:35 smithi098 bash[33599]: cluster 2023-09-23T18:17:34.637089+0000 mgr.y (mgr.24344) 185 : cluster [DBG] pgmap v167: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 257 KiB/s rd, 25 KiB/s wr, 329 op/s 2023-09-23T18:17:35.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:35 smithi098 bash[27349]: cluster 2023-09-23T18:17:34.637089+0000 mgr.y (mgr.24344) 185 : cluster [DBG] pgmap v167: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 257 KiB/s rd, 25 KiB/s wr, 329 op/s 2023-09-23T18:17:35.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:35 smithi173 bash[27694]: cluster 2023-09-23T18:17:34.637089+0000 mgr.y (mgr.24344) 185 : cluster [DBG] pgmap v167: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 257 KiB/s rd, 25 KiB/s wr, 329 op/s 2023-09-23T18:17:35.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:17:35 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:17:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:17:36.470 INFO:tasks.workunit.client.0.smithi098.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T18:17:36.480 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd snap protect testimg@snap2 2023-09-23T18:17:36.539 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_65731/snap_protect 2023-09-23T18:17:36.592 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T18:17:36.596 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_65731/snap_protect testimg_import 2023-09-23T18:17:36.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:36 smithi098 bash[33599]: cluster 2023-09-23T18:17:35.434760+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-09-23T18:17:36.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:36 smithi098 bash[27349]: cluster 2023-09-23T18:17:35.434760+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-09-23T18:17:36.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:36 smithi173 bash[27694]: cluster 2023-09-23T18:17:35.434760+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-09-23T18:17:37.774 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:37 smithi098 bash[27349]: cluster 2023-09-23T18:17:36.449018+0000 mon.a (mon.0) 693 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-09-23T18:17:37.774 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:37 smithi098 bash[27349]: cluster 2023-09-23T18:17:36.637488+0000 mgr.y (mgr.24344) 186 : cluster [DBG] pgmap v170: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 258 KiB/s rd, 25 KiB/s wr, 330 op/s 2023-09-23T18:17:37.774 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:37 smithi098 bash[33599]: cluster 2023-09-23T18:17:36.449018+0000 mon.a (mon.0) 693 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-09-23T18:17:37.774 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:37 smithi098 bash[33599]: cluster 2023-09-23T18:17:36.637488+0000 mgr.y (mgr.24344) 186 : cluster [DBG] pgmap v170: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 258 KiB/s rd, 25 KiB/s wr, 330 op/s 2023-09-23T18:17:37.776 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:37 smithi173 bash[27694]: cluster 2023-09-23T18:17:36.449018+0000 mon.a (mon.0) 693 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-09-23T18:17:37.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:37 smithi173 bash[27694]: cluster 2023-09-23T18:17:36.637488+0000 mgr.y (mgr.24344) 186 : cluster [DBG] pgmap v170: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 258 KiB/s rd, 25 KiB/s wr, 330 op/s 2023-09-23T18:17:38.083 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:17:37 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:17:37] "GET /metrics HTTP/1.1" 200 192041 "" "Prometheus/2.33.4" 2023-09-23T18:17:38.122 INFO:teuthology.orchestra.run.smithi098.stdout:wait for minority of mons to upgrade 2023-09-23T18:17:38.420 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T18:17:38.424 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info testimg_import@snap1 2023-09-23T18:17:38.424 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep protected: False 2023-09-23T18:17:38.480 INFO:tasks.workunit.client.0.smithi098.stdout: protected: False 2023-09-23T18:17:38.481 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info testimg_import@snap2 2023-09-23T18:17:38.481 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep protected: True 2023-09-23T18:17:38.534 INFO:tasks.workunit.client.0.smithi098.stdout: protected: True 2023-09-23T18:17:38.534 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_65731/snap_protect 2023-09-23T18:17:38.535 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd snap unprotect testimg@snap2 2023-09-23T18:17:38.596 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd snap unprotect testimg_import@snap2 2023-09-23T18:17:38.658 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd snap purge testimg 2023-09-23T18:17:38.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:38 smithi098 bash[33599]: cluster 2023-09-23T18:17:37.453601+0000 mon.a (mon.0) 694 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-09-23T18:17:38.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:38 smithi098 bash[33599]: cluster 2023-09-23T18:17:38.383233+0000 mon.a (mon.0) 695 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-09-23T18:17:38.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:38 smithi098 bash[27349]: cluster 2023-09-23T18:17:37.453601+0000 mon.a (mon.0) 694 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-09-23T18:17:38.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:38 smithi098 bash[27349]: cluster 2023-09-23T18:17:38.383233+0000 mon.a (mon.0) 695 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-09-23T18:17:38.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:38 smithi173 bash[27694]: cluster 2023-09-23T18:17:37.453601+0000 mon.a (mon.0) 694 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-09-23T18:17:38.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:38 smithi173 bash[27694]: cluster 2023-09-23T18:17:38.383233+0000 mon.a (mon.0) 695 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-09-23T18:17:39.064 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-09-23T18:17:39.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:39 smithi173 bash[27694]: cluster 2023-09-23T18:17:38.638103+0000 mgr.y (mgr.24344) 187 : cluster [DBG] pgmap v173: 73 pgs: 73 active+clean; 223 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 247 KiB/s rd, 28 KiB/s wr, 333 op/s 2023-09-23T18:17:39.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:39 smithi098 bash[27349]: cluster 2023-09-23T18:17:38.638103+0000 mgr.y (mgr.24344) 187 : cluster [DBG] pgmap v173: 73 pgs: 73 active+clean; 223 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 247 KiB/s rd, 28 KiB/s wr, 333 op/s 2023-09-23T18:17:39.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:39 smithi098 bash[33599]: cluster 2023-09-23T18:17:38.638103+0000 mgr.y (mgr.24344) 187 : cluster [DBG] pgmap v173: 73 pgs: 73 active+clean; 223 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 247 KiB/s rd, 28 KiB/s wr, 333 op/s 2023-09-23T18:17:40.480 INFO:tasks.workunit.client.0.smithi098.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T18:17:40.489 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd snap purge testimg_import 2023-09-23T18:17:40.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:40 smithi098 bash[33599]: cluster 2023-09-23T18:17:39.462629+0000 mon.a (mon.0) 696 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-09-23T18:17:40.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:40 smithi098 bash[27349]: cluster 2023-09-23T18:17:39.462629+0000 mon.a (mon.0) 696 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-09-23T18:17:40.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:40 smithi173 bash[27694]: cluster 2023-09-23T18:17:39.462629+0000 mon.a (mon.0) 696 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-09-23T18:17:41.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:41 smithi173 bash[27694]: cluster 2023-09-23T18:17:40.469062+0000 mon.a (mon.0) 697 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-09-23T18:17:41.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:41 smithi173 bash[27694]: cluster 2023-09-23T18:17:40.638629+0000 mgr.y (mgr.24344) 188 : cluster [DBG] pgmap v176: 73 pgs: 73 active+clean; 223 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 247 KiB/s rd, 28 KiB/s wr, 333 op/s 2023-09-23T18:17:41.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:41 smithi098 bash[33599]: cluster 2023-09-23T18:17:40.469062+0000 mon.a (mon.0) 697 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-09-23T18:17:41.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:41 smithi098 bash[33599]: cluster 2023-09-23T18:17:40.638629+0000 mgr.y (mgr.24344) 188 : cluster [DBG] pgmap v176: 73 pgs: 73 active+clean; 223 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 247 KiB/s rd, 28 KiB/s wr, 333 op/s 2023-09-23T18:17:41.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:41 smithi098 bash[27349]: cluster 2023-09-23T18:17:40.469062+0000 mon.a (mon.0) 697 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-09-23T18:17:41.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:41 smithi098 bash[27349]: cluster 2023-09-23T18:17:40.638629+0000 mgr.y (mgr.24344) 188 : cluster [DBG] pgmap v176: 73 pgs: 73 active+clean; 223 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 247 KiB/s rd, 28 KiB/s wr, 333 op/s 2023-09-23T18:17:42.504 INFO:tasks.workunit.client.0.smithi098.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T18:17:42.514 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd remove testimg 2023-09-23T18:17:42.655 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 100% complete...done. 2023-09-23T18:17:42.659 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd remove testimg_import 2023-09-23T18:17:42.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:42 smithi173 bash[27694]: cluster 2023-09-23T18:17:41.487766+0000 mon.a (mon.0) 698 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-09-23T18:17:42.792 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 100% complete...done. 2023-09-23T18:17:42.797 INFO:tasks.workunit.client.0.smithi098.stderr:+ tiered=0 2023-09-23T18:17:42.797 INFO:tasks.workunit.client.0.smithi098.stderr:+ ceph osd dump 2023-09-23T18:17:42.798 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep 'rbd' 2023-09-23T18:17:42.798 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep ^pool 2023-09-23T18:17:42.798 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep tier 2023-09-23T18:17:42.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:42 smithi098 bash[33599]: cluster 2023-09-23T18:17:41.487766+0000 mon.a (mon.0) 698 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-09-23T18:17:42.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:42 smithi098 bash[27349]: cluster 2023-09-23T18:17:41.487766+0000 mon.a (mon.0) 698 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-09-23T18:17:43.249 INFO:tasks.workunit.client.0.smithi098.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_65731/sparse1 2023-09-23T18:17:43.254 INFO:tasks.workunit.client.0.smithi098.stderr:1+0 records in 2023-09-23T18:17:43.254 INFO:tasks.workunit.client.0.smithi098.stderr:1+0 records out 2023-09-23T18:17:43.255 INFO:tasks.workunit.client.0.smithi098.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00442293 s, 237 MB/s 2023-09-23T18:17:43.255 INFO:tasks.workunit.client.0.smithi098.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_65731/sparse2 2023-09-23T18:17:43.263 INFO:tasks.workunit.client.0.smithi098.stderr:1+0 records in 2023-09-23T18:17:43.263 INFO:tasks.workunit.client.0.smithi098.stderr:1+0 records out 2023-09-23T18:17:43.263 INFO:tasks.workunit.client.0.smithi098.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00617712 s, 170 MB/s 2023-09-23T18:17:43.263 INFO:tasks.workunit.client.0.smithi098.stderr:+ truncate /tmp/rbd_import_export_65731/sparse2 -s 2M 2023-09-23T18:17:43.540 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm sparse1 2023-09-23T18:17:43.588 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 0% complete...failed. 2023-09-23T18:17:43.588 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T18:17:43.593 INFO:tasks.workunit.client.0.smithi098.stderr:+ true 2023-09-23T18:17:43.593 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_65731/sparse1 2023-09-23T18:17:43.677 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T18:17:43.679 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T18:17:43.679 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T18:17:43.682 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd ls -l 2023-09-23T18:17:43.682 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep sparse1 2023-09-23T18:17:43.682 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep -Ei (2 MiB|2048k) 2023-09-23T18:17:43.732 INFO:tasks.workunit.client.0.smithi098.stdout:sparse1 2 MiB 2 2023-09-23T18:17:43.733 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects sparse1 2023-09-23T18:17:43.734 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse1 2023-09-23T18:17:43.734 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse1 2023-09-23T18:17:43.734 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep block_name_prefix 2023-09-23T18:17:43.734 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk {print $NF;} 2023-09-23T18:17:43.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:43 smithi173 bash[27694]: cluster 2023-09-23T18:17:42.495444+0000 mon.a (mon.0) 699 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-09-23T18:17:43.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:43 smithi173 bash[27694]: cluster 2023-09-23T18:17:42.639049+0000 mgr.y (mgr.24344) 189 : cluster [DBG] pgmap v179: 73 pgs: 73 active+clean; 223 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:43.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:43 smithi173 bash[27694]: audit 2023-09-23T18:17:43.097623+0000 mon.c (mon.2) 103 : audit [DBG] from='client.? 172.21.15.98:0/3309119614' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T18:17:43.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:43 smithi173 bash[27694]: audit 2023-09-23T18:17:43.228112+0000 mon.c (mon.2) 104 : audit [DBG] from='client.? 172.21.15.98:0/3309119614' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T18:17:43.788 INFO:tasks.workunit.client.0.smithi098.stderr:+ prefix=rbd_data.6203c8ddaef4 2023-09-23T18:17:43.789 INFO:tasks.workunit.client.0.smithi098.stderr:+ + grep rbd_data.6203c8ddaef4 2023-09-23T18:17:43.790 INFO:tasks.workunit.client.0.smithi098.stderr:get_image_data_pool sparse1 2023-09-23T18:17:43.790 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse1 2023-09-23T18:17:43.790 INFO:tasks.workunit.client.0.smithi098.stderr:+ + sed -e s/rbd_data.6203c8ddaef4\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-09-23T18:17:43.790 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse1 2023-09-23T18:17:43.790 INFO:tasks.workunit.client.0.smithi098.stderr:sort -u 2023-09-23T18:17:43.790 INFO:tasks.workunit.client.0.smithi098.stderr:+ + grep data_pool: 2023-09-23T18:17:43.790 INFO:tasks.workunit.client.0.smithi098.stderr:awk -F: { print $NF } 2023-09-23T18:17:43.839 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool= 2023-09-23T18:17:43.839 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ -z ] 2023-09-23T18:17:43.839 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool=rbd 2023-09-23T18:17:43.840 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo rbd 2023-09-23T18:17:43.840 INFO:tasks.workunit.client.0.smithi098.stderr:+ rados ls -p rbd 2023-09-23T18:17:43.882 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects=1 2023-09-23T18:17:43.882 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo 1 2023-09-23T18:17:43.882 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-09-23T18:17:43.882 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export sparse1 /tmp/rbd_import_export_65731/sparse1.out 2023-09-23T18:17:43.929 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T18:17:43.933 INFO:tasks.workunit.client.0.smithi098.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_65731/sparse1 /tmp/rbd_import_export_65731/sparse1.out 2023-09-23T18:17:43.934 INFO:tasks.workunit.client.0.smithi098.stderr:+ cmp -l /tmp/rbd_import_export_65731/sparse1 /tmp/rbd_import_export_65731/sparse1.out 2023-09-23T18:17:43.937 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_65731/sparse1 --format %b 2023-09-23T18:17:43.938 INFO:tasks.workunit.client.0.smithi098.stderr:+ origsize=2048 2023-09-23T18:17:43.938 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_65731/sparse1.out --format %b 2023-09-23T18:17:43.939 INFO:tasks.workunit.client.0.smithi098.stderr:+ exportsize=2048 2023-09-23T18:17:43.940 INFO:tasks.workunit.client.0.smithi098.stderr:+ difference=0 2023-09-23T18:17:43.940 INFO:tasks.workunit.client.0.smithi098.stderr:+ difference=0 2023-09-23T18:17:43.940 INFO:tasks.workunit.client.0.smithi098.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T18:17:43.940 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_65731/sparse1.out 2023-09-23T18:17:43.940 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm sparse1 2023-09-23T18:17:43.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:43 smithi098 bash[27349]: cluster 2023-09-23T18:17:42.495444+0000 mon.a (mon.0) 699 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-09-23T18:17:43.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:43 smithi098 bash[27349]: cluster 2023-09-23T18:17:42.639049+0000 mgr.y (mgr.24344) 189 : cluster [DBG] pgmap v179: 73 pgs: 73 active+clean; 223 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:43.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:43 smithi098 bash[27349]: audit 2023-09-23T18:17:43.097623+0000 mon.c (mon.2) 103 : audit [DBG] from='client.? 172.21.15.98:0/3309119614' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T18:17:43.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:43 smithi098 bash[27349]: audit 2023-09-23T18:17:43.228112+0000 mon.c (mon.2) 104 : audit [DBG] from='client.? 172.21.15.98:0/3309119614' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T18:17:43.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:43 smithi098 bash[33599]: cluster 2023-09-23T18:17:42.495444+0000 mon.a (mon.0) 699 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-09-23T18:17:43.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:43 smithi098 bash[33599]: cluster 2023-09-23T18:17:42.639049+0000 mgr.y (mgr.24344) 189 : cluster [DBG] pgmap v179: 73 pgs: 73 active+clean; 223 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:17:43.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:43 smithi098 bash[33599]: audit 2023-09-23T18:17:43.097623+0000 mon.c (mon.2) 103 : audit [DBG] from='client.? 172.21.15.98:0/3309119614' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T18:17:43.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:43 smithi098 bash[33599]: audit 2023-09-23T18:17:43.228112+0000 mon.c (mon.2) 104 : audit [DBG] from='client.? 172.21.15.98:0/3309119614' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T18:17:44.081 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T18:17:44.085 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm sparse2 2023-09-23T18:17:44.141 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 0% complete...failed. 2023-09-23T18:17:44.142 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T18:17:44.145 INFO:tasks.workunit.client.0.smithi098.stderr:+ true 2023-09-23T18:17:44.146 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_65731/sparse2 2023-09-23T18:17:44.233 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T18:17:44.234 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T18:17:44.234 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T18:17:44.237 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd ls -l 2023-09-23T18:17:44.237 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep sparse2 2023-09-23T18:17:44.237 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep -Ei (2 MiB|2048k) 2023-09-23T18:17:44.285 INFO:tasks.workunit.client.0.smithi098.stdout:sparse2 2 MiB 2 2023-09-23T18:17:44.287 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects sparse2 2023-09-23T18:17:44.287 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse2 2023-09-23T18:17:44.287 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse2 2023-09-23T18:17:44.287 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep block_name_prefix 2023-09-23T18:17:44.287 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk {print $NF;} 2023-09-23T18:17:44.340 INFO:tasks.workunit.client.0.smithi098.stderr:+ prefix=rbd_data.6236d1a88b89 2023-09-23T18:17:44.341 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep rbd_data.6236d1a88b89 2023-09-23T18:17:44.341 INFO:tasks.workunit.client.0.smithi098.stderr:+ sed -e s/rbd_data.6236d1a88b89\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-09-23T18:17:44.341 INFO:tasks.workunit.client.0.smithi098.stderr:+ get_image_data_pool sparse2 2023-09-23T18:17:44.341 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse2 2023-09-23T18:17:44.341 INFO:tasks.workunit.client.0.smithi098.stderr:+ sort -u 2023-09-23T18:17:44.341 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse2 2023-09-23T18:17:44.341 INFO:tasks.workunit.client.0.smithi098.stderr:+ + grep data_pool: 2023-09-23T18:17:44.341 INFO:tasks.workunit.client.0.smithi098.stderr:awk -F: { print $NF } 2023-09-23T18:17:44.392 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool= 2023-09-23T18:17:44.393 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ -z ] 2023-09-23T18:17:44.393 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool=rbd 2023-09-23T18:17:44.393 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo rbd 2023-09-23T18:17:44.393 INFO:tasks.workunit.client.0.smithi098.stderr:+ rados ls -p rbd 2023-09-23T18:17:44.437 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects=0 2023-09-23T18:17:44.438 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo 0 2023-09-23T18:17:44.438 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2023-09-23T18:17:44.438 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export sparse2 /tmp/rbd_import_export_65731/sparse2.out 2023-09-23T18:17:44.489 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T18:17:44.493 INFO:tasks.workunit.client.0.smithi098.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_65731/sparse2 /tmp/rbd_import_export_65731/sparse2.out 2023-09-23T18:17:44.493 INFO:tasks.workunit.client.0.smithi098.stderr:+ cmp -l /tmp/rbd_import_export_65731/sparse2 /tmp/rbd_import_export_65731/sparse2.out 2023-09-23T18:17:44.496 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_65731/sparse2 --format %b 2023-09-23T18:17:44.497 INFO:tasks.workunit.client.0.smithi098.stderr:+ origsize=2048 2023-09-23T18:17:44.497 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_65731/sparse2.out --format %b 2023-09-23T18:17:44.498 INFO:tasks.workunit.client.0.smithi098.stderr:+ exportsize=2048 2023-09-23T18:17:44.499 INFO:tasks.workunit.client.0.smithi098.stderr:+ difference=0 2023-09-23T18:17:44.499 INFO:tasks.workunit.client.0.smithi098.stderr:+ difference=0 2023-09-23T18:17:44.499 INFO:tasks.workunit.client.0.smithi098.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T18:17:44.499 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_65731/sparse2.out 2023-09-23T18:17:44.499 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm sparse2 2023-09-23T18:17:44.637 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T18:17:44.641 INFO:tasks.workunit.client.0.smithi098.stderr:+ truncate /tmp/rbd_import_export_65731/sparse1 -s 10M 2023-09-23T18:17:44.642 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --new-format --order 20 - sparse1 2023-09-23T18:17:44.786 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 100% complete...done. 2023-09-23T18:17:44.787 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T18:17:44.788 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T18:17:44.791 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd ls -l 2023-09-23T18:17:44.791 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep sparse1 2023-09-23T18:17:44.791 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep -Ei (10 MiB|10240k) 2023-09-23T18:17:44.848 INFO:tasks.workunit.client.0.smithi098.stdout:sparse1 10 MiB 2 2023-09-23T18:17:44.849 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects sparse1 2023-09-23T18:17:44.849 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse1 2023-09-23T18:17:44.849 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse1 2023-09-23T18:17:44.849 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep block_name_prefix 2023-09-23T18:17:44.849 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk {print $NF;} 2023-09-23T18:17:44.907 INFO:tasks.workunit.client.0.smithi098.stderr:+ prefix=rbd_data.3bb55a624fbc 2023-09-23T18:17:44.907 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep rbd_data.3bb55a624fbc 2023-09-23T18:17:44.907 INFO:tasks.workunit.client.0.smithi098.stderr:+ get_image_data_pool sparse1 2023-09-23T18:17:44.907 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse1 2023-09-23T18:17:44.907 INFO:tasks.workunit.client.0.smithi098.stderr:+ sed -e s/rbd_data.3bb55a624fbc\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-09-23T18:17:44.907 INFO:tasks.workunit.client.0.smithi098.stderr:+ sort -u 2023-09-23T18:17:44.907 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd+ grep data_pool: 2023-09-23T18:17:44.907 INFO:tasks.workunit.client.0.smithi098.stderr: info sparse1 2023-09-23T18:17:44.908 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk -F: { print $NF } 2023-09-23T18:17:44.958 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool= 2023-09-23T18:17:44.958 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ -z ] 2023-09-23T18:17:44.958 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool=rbd 2023-09-23T18:17:44.958 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo rbd 2023-09-23T18:17:44.958 INFO:tasks.workunit.client.0.smithi098.stderr:+ rados ls -p rbd 2023-09-23T18:17:45.005 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects=1 2023-09-23T18:17:45.005 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo 1 2023-09-23T18:17:45.006 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-09-23T18:17:45.006 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export sparse1 /tmp/rbd_import_export_65731/sparse1.out 2023-09-23T18:17:45.059 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-09-23T18:17:45.063 INFO:tasks.workunit.client.0.smithi098.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_65731/sparse1 /tmp/rbd_import_export_65731/sparse1.out 2023-09-23T18:17:45.063 INFO:tasks.workunit.client.0.smithi098.stderr:+ cmp -l /tmp/rbd_import_export_65731/sparse1 /tmp/rbd_import_export_65731/sparse1.out 2023-09-23T18:17:45.073 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_65731/sparse1 --format %b 2023-09-23T18:17:45.074 INFO:tasks.workunit.client.0.smithi098.stderr:+ origsize=2048 2023-09-23T18:17:45.074 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_65731/sparse1.out --format %b 2023-09-23T18:17:45.074 INFO:tasks.workunit.client.0.smithi098.stderr:+ exportsize=2048 2023-09-23T18:17:45.075 INFO:tasks.workunit.client.0.smithi098.stderr:+ difference=0 2023-09-23T18:17:45.075 INFO:tasks.workunit.client.0.smithi098.stderr:+ difference=0 2023-09-23T18:17:45.075 INFO:tasks.workunit.client.0.smithi098.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T18:17:45.075 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_65731/sparse1.out 2023-09-23T18:17:45.077 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm sparse1 2023-09-23T18:17:45.220 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T18:17:45.224 INFO:tasks.workunit.client.0.smithi098.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_65731/sparse2 oflag=append conv=notrunc 2023-09-23T18:17:45.235 INFO:tasks.workunit.client.0.smithi098.stderr:1+0 records in 2023-09-23T18:17:45.235 INFO:tasks.workunit.client.0.smithi098.stderr:1+0 records out 2023-09-23T18:17:45.235 INFO:tasks.workunit.client.0.smithi098.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.00973841 s, 215 MB/s 2023-09-23T18:17:45.235 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --new-format --order 20 - sparse2 2023-09-23T18:17:45.355 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 100% complete...done. 2023-09-23T18:17:45.356 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T18:17:45.357 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T18:17:45.360 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd ls -l 2023-09-23T18:17:45.360 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep sparse2 2023-09-23T18:17:45.360 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep -Ei (4 MiB|4096k) 2023-09-23T18:17:45.417 INFO:tasks.workunit.client.0.smithi098.stdout:sparse2 4 MiB 2 2023-09-23T18:17:45.418 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects sparse2 2023-09-23T18:17:45.418 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse2 2023-09-23T18:17:45.418 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse2 2023-09-23T18:17:45.418 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep block_name_prefix 2023-09-23T18:17:45.419 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk {print $NF;} 2023-09-23T18:17:45.464 INFO:tasks.workunit.client.0.smithi098.stderr:+ prefix=rbd_data.3be8820a3f4b 2023-09-23T18:17:45.464 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep rbd_data.3be8820a3f4b 2023-09-23T18:17:45.465 INFO:tasks.workunit.client.0.smithi098.stderr:+ + get_image_data_pool sparse2 2023-09-23T18:17:45.465 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse2 2023-09-23T18:17:45.465 INFO:tasks.workunit.client.0.smithi098.stderr:+ sort -u 2023-09-23T18:17:45.466 INFO:tasks.workunit.client.0.smithi098.stderr:sed -e s/rbd_data.3be8820a3f4b\.// -e s/^0*\([0-9a-f]\)/\1/+ 2023-09-23T18:17:45.466 INFO:tasks.workunit.client.0.smithi098.stderr:rbd+ grep data_pool: 2023-09-23T18:17:45.466 INFO:tasks.workunit.client.0.smithi098.stderr: info sparse2 2023-09-23T18:17:45.466 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk -F: { print $NF } 2023-09-23T18:17:45.514 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool= 2023-09-23T18:17:45.514 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ -z ] 2023-09-23T18:17:45.514 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool=rbd 2023-09-23T18:17:45.514 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo rbd 2023-09-23T18:17:45.514 INFO:tasks.workunit.client.0.smithi098.stderr:+ rados ls -p rbd 2023-09-23T18:17:45.560 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects=0 2023-09-23T18:17:45.560 INFO:tasks.workunit.client.0.smithi098.stderr:2 2023-09-23T18:17:45.560 INFO:tasks.workunit.client.0.smithi098.stderr:3 2023-09-23T18:17:45.560 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo 0 2 3 2023-09-23T18:17:45.560 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ 0 -eq 1 -o 0 2 3 = 0 2 3 ] 2023-09-23T18:17:45.560 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export sparse2 /tmp/rbd_import_export_65731/sparse2.out 2023-09-23T18:17:45.616 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-09-23T18:17:45.621 INFO:tasks.workunit.client.0.smithi098.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_65731/sparse2 /tmp/rbd_import_export_65731/sparse2.out 2023-09-23T18:17:45.621 INFO:tasks.workunit.client.0.smithi098.stderr:+ cmp -l /tmp/rbd_import_export_65731/sparse2 /tmp/rbd_import_export_65731/sparse2.out 2023-09-23T18:17:45.627 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_65731/sparse2 --format %b 2023-09-23T18:17:45.628 INFO:tasks.workunit.client.0.smithi098.stderr:+ origsize=6144 2023-09-23T18:17:45.628 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_65731/sparse2.out --format %b 2023-09-23T18:17:45.629 INFO:tasks.workunit.client.0.smithi098.stderr:+ exportsize=6144 2023-09-23T18:17:45.629 INFO:tasks.workunit.client.0.smithi098.stderr:+ difference=0 2023-09-23T18:17:45.629 INFO:tasks.workunit.client.0.smithi098.stderr:+ difference=0 2023-09-23T18:17:45.629 INFO:tasks.workunit.client.0.smithi098.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T18:17:45.629 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_65731/sparse2.out 2023-09-23T18:17:45.630 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm sparse2 2023-09-23T18:17:45.702 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:17:45 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:17:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:17:45.776 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-09-23T18:17:45.780 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo partially-sparse file imports to partially-sparse image 2023-09-23T18:17:45.780 INFO:tasks.workunit.client.0.smithi098.stdout:partially-sparse file imports to partially-sparse image 2023-09-23T18:17:45.781 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_65731/sparse1 sparse 2023-09-23T18:17:45.873 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-09-23T18:17:45.874 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T18:17:45.874 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T18:17:45.878 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects sparse 2023-09-23T18:17:45.878 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse 2023-09-23T18:17:45.879 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse 2023-09-23T18:17:45.879 INFO:tasks.workunit.client.0.smithi098.stderr:+ + grep block_name_prefix 2023-09-23T18:17:45.879 INFO:tasks.workunit.client.0.smithi098.stderr:awk {print $NF;} 2023-09-23T18:17:45.929 INFO:tasks.workunit.client.0.smithi098.stderr:+ prefix=rbd_data.3c1519434d93 2023-09-23T18:17:45.931 INFO:tasks.workunit.client.0.smithi098.stderr:+ + + sed -e s/rbd_data.3c1519434d93\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-09-23T18:17:45.931 INFO:tasks.workunit.client.0.smithi098.stderr:get_image_data_pool sparse 2023-09-23T18:17:45.931 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse 2023-09-23T18:17:45.931 INFO:tasks.workunit.client.0.smithi098.stderr:+ sort -u 2023-09-23T18:17:45.931 INFO:tasks.workunit.client.0.smithi098.stderr:grep rbd_data.3c1519434d93 2023-09-23T18:17:45.931 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info+ sparsegrep data_pool: 2023-09-23T18:17:45.931 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-23T18:17:45.931 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk -F: { print $NF } 2023-09-23T18:17:45.979 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool= 2023-09-23T18:17:45.979 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ -z ] 2023-09-23T18:17:45.979 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool=rbd 2023-09-23T18:17:45.979 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo rbd 2023-09-23T18:17:45.979 INFO:tasks.workunit.client.0.smithi098.stderr:+ rados ls -p rbd 2023-09-23T18:17:46.023 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects=1 2023-09-23T18:17:46.023 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo 1 2023-09-23T18:17:46.023 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-09-23T18:17:46.023 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm sparse 2023-09-23T18:17:46.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:45 smithi173 bash[27694]: cluster 2023-09-23T18:17:44.639838+0000 mgr.y (mgr.24344) 190 : cluster [DBG] pgmap v180: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 7.3 KiB/s wr, 115 op/s 2023-09-23T18:17:46.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:45 smithi173 bash[27694]: audit 2023-09-23T18:17:45.313289+0000 mon.b (mon.1) 27 : audit [DBG] from='client.? 172.21.15.98:0/4245032731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:17:46.169 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T18:17:46.174 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo zeros import through stdin to sparse image 2023-09-23T18:17:46.175 INFO:tasks.workunit.client.0.smithi098.stdout:zeros import through stdin to sparse image 2023-09-23T18:17:46.175 INFO:tasks.workunit.client.0.smithi098.stderr:+ dd if=/dev/zero bs=1M count=4 2023-09-23T18:17:46.175 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --new-format - sparse 2023-09-23T18:17:46.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:45 smithi098 bash[27349]: cluster 2023-09-23T18:17:44.639838+0000 mgr.y (mgr.24344) 190 : cluster [DBG] pgmap v180: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 7.3 KiB/s wr, 115 op/s 2023-09-23T18:17:46.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:45 smithi098 bash[27349]: audit 2023-09-23T18:17:45.313289+0000 mon.b (mon.1) 27 : audit [DBG] from='client.? 172.21.15.98:0/4245032731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:17:46.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:45 smithi098 bash[33599]: cluster 2023-09-23T18:17:44.639838+0000 mgr.y (mgr.24344) 190 : cluster [DBG] pgmap v180: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 7.3 KiB/s wr, 115 op/s 2023-09-23T18:17:46.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:45 smithi098 bash[33599]: audit 2023-09-23T18:17:45.313289+0000 mon.b (mon.1) 27 : audit [DBG] from='client.? 172.21.15.98:0/4245032731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:17:46.247 INFO:tasks.workunit.client.0.smithi098.stderr:4+0 records in 2023-09-23T18:17:46.248 INFO:tasks.workunit.client.0.smithi098.stderr:4+0 records out 2023-09-23T18:17:46.248 INFO:tasks.workunit.client.0.smithi098.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.072185 s, 58.1 MB/s 2023-09-23T18:17:46.267 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 100% complete...done. 2023-09-23T18:17:46.268 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T18:17:46.273 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects sparse 2023-09-23T18:17:46.274 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse 2023-09-23T18:17:46.274 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse 2023-09-23T18:17:46.274 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep block_name_prefix 2023-09-23T18:17:46.274 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk {print $NF;} 2023-09-23T18:17:46.333 INFO:tasks.workunit.client.0.smithi098.stderr:+ prefix=rbd_data.62e4e39c58b2 2023-09-23T18:17:46.335 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep rbd_data.62e4e39c58b2 2023-09-23T18:17:46.335 INFO:tasks.workunit.client.0.smithi098.stderr:+ get_image_data_pool sparse 2023-09-23T18:17:46.335 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse 2023-09-23T18:17:46.335 INFO:tasks.workunit.client.0.smithi098.stderr:+ sed -e s/rbd_data.62e4e39c58b2\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-09-23T18:17:46.335 INFO:tasks.workunit.client.0.smithi098.stderr:+ sort -u 2023-09-23T18:17:46.335 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd+ info sparse 2023-09-23T18:17:46.335 INFO:tasks.workunit.client.0.smithi098.stderr:grep data_pool: 2023-09-23T18:17:46.335 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk -F: { print $NF } 2023-09-23T18:17:46.385 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool= 2023-09-23T18:17:46.385 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ -z ] 2023-09-23T18:17:46.385 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool=rbd 2023-09-23T18:17:46.385 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo rbd 2023-09-23T18:17:46.385 INFO:tasks.workunit.client.0.smithi098.stderr:+ rados ls -p rbd 2023-09-23T18:17:46.429 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects= 2023-09-23T18:17:46.429 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo 2023-09-23T18:17:46.429 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ 0 -eq 1 -o = ] 2023-09-23T18:17:46.429 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm sparse 2023-09-23T18:17:46.576 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 100% complete...done. 2023-09-23T18:17:46.580 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo zeros export to sparse file 2023-09-23T18:17:46.581 INFO:tasks.workunit.client.0.smithi098.stdout:zeros export to sparse file 2023-09-23T18:17:46.581 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd create --new-format sparse --size 4 2023-09-23T18:17:46.639 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T18:17:46.643 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse 2023-09-23T18:17:46.643 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep block_name_prefix 2023-09-23T18:17:46.643 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk {print $NF;} 2023-09-23T18:17:46.696 INFO:tasks.workunit.client.0.smithi098.stderr:+ prefix=rbd_data.3c57607d763c 2023-09-23T18:17:46.696 INFO:tasks.workunit.client.0.smithi098.stderr:+ dd if=/dev/zero bs=4M count=1 2023-09-23T18:17:46.697 INFO:tasks.workunit.client.0.smithi098.stderr:+ get_image_data_pool sparse 2023-09-23T18:17:46.697 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse 2023-09-23T18:17:46.697 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info+ sparse 2023-09-23T18:17:46.697 INFO:tasks.workunit.client.0.smithi098.stderr:grep data_pool: 2023-09-23T18:17:46.698 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk -F: { print $NF } 2023-09-23T18:17:46.750 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool= 2023-09-23T18:17:46.750 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ -z ] 2023-09-23T18:17:46.750 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool=rbd 2023-09-23T18:17:46.750 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo rbd 2023-09-23T18:17:46.750 INFO:tasks.workunit.client.0.smithi098.stderr:+ rados -p rbd put rbd_data.3c57607d763c.000000000000 - 2023-09-23T18:17:46.787 INFO:tasks.workunit.client.0.smithi098.stderr:1+0 records in 2023-09-23T18:17:46.787 INFO:tasks.workunit.client.0.smithi098.stderr:1+0 records out 2023-09-23T18:17:46.787 INFO:tasks.workunit.client.0.smithi098.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0900382 s, 46.6 MB/s 2023-09-23T18:17:46.813 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects sparse 2023-09-23T18:17:46.814 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse 2023-09-23T18:17:46.814 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse 2023-09-23T18:17:46.814 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep block_name_prefix 2023-09-23T18:17:46.814 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk {print $NF;} 2023-09-23T18:17:46.865 INFO:tasks.workunit.client.0.smithi098.stderr:+ prefix=rbd_data.3c57607d763c 2023-09-23T18:17:46.866 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep rbd_data.3c57607d763c 2023-09-23T18:17:46.867 INFO:tasks.workunit.client.0.smithi098.stderr:+ get_image_data_pool sparse 2023-09-23T18:17:46.867 INFO:tasks.workunit.client.0.smithi098.stderr:+ + sedimage=sparse 2023-09-23T18:17:46.867 INFO:tasks.workunit.client.0.smithi098.stderr: -e s/rbd_data.3c57607d763c\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-09-23T18:17:46.867 INFO:tasks.workunit.client.0.smithi098.stderr:+ sort -u 2023-09-23T18:17:46.867 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse 2023-09-23T18:17:46.867 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep data_pool: 2023-09-23T18:17:46.867 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk -F: { print $NF } 2023-09-23T18:17:46.923 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool= 2023-09-23T18:17:46.923 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ -z ] 2023-09-23T18:17:46.923 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool=rbd 2023-09-23T18:17:46.923 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo rbd 2023-09-23T18:17:46.923 INFO:tasks.workunit.client.0.smithi098.stderr:+ rados ls -p rbd 2023-09-23T18:17:46.970 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects=0 2023-09-23T18:17:46.970 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo 0 2023-09-23T18:17:46.970 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2023-09-23T18:17:46.970 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_65731/sparse 2023-09-23T18:17:46.970 INFO:tasks.workunit.client.0.smithi098.stderr:rm: cannot remove '/tmp/rbd_import_export_65731/sparse': No such file or directory 2023-09-23T18:17:46.970 INFO:tasks.workunit.client.0.smithi098.stderr:+ true 2023-09-23T18:17:46.971 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export sparse /tmp/rbd_import_export_65731/sparse 2023-09-23T18:17:47.021 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 100% complete...done. 2023-09-23T18:17:47.026 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_65731/sparse --format=%b 2023-09-23T18:17:47.027 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ 0 = 0 ] 2023-09-23T18:17:47.027 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm sparse 2023-09-23T18:17:47.169 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 100% complete...done. 2023-09-23T18:17:47.174 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_65731/sparse /tmp/rbd_import_export_65731/sparse1 /tmp/rbd_import_export_65731/sparse2 /tmp/rbd_import_export_65731/sparse3 2023-09-23T18:17:47.176 INFO:tasks.workunit.client.0.smithi098.stderr:rm: cannot remove '/tmp/rbd_import_export_65731/sparse3': No such file or directory 2023-09-23T18:17:47.176 INFO:tasks.workunit.client.0.smithi098.stderr:+ true 2023-09-23T18:17:47.177 INFO:tasks.workunit.client.0.smithi098.stdout:OK 2023-09-23T18:17:47.178 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo OK 2023-09-23T18:17:47.178 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm -rf /tmp/rbd_import_export_65731 2023-09-23T18:17:47.178 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T18:17:47.178 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T18:17:47.188 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-09-23T18:17:47.188 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T18:17:47.494 DEBUG:teuthology.parallel:result is None 2023-09-23T18:17:47.495 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T18:17:47.505 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T18:17:47.506 DEBUG:teuthology.orchestra.run.smithi098:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T18:17:47.550 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T18:17:47.550 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T18:17:47.551 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-09-23T18:17:47.551 DEBUG:teuthology.parallel:result is None 2023-09-23T18:17:47.773 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:47 smithi098 bash[27349]: cluster 2023-09-23T18:17:46.640288+0000 mgr.y (mgr.24344) 191 : cluster [DBG] pgmap v181: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 70 KiB/s rd, 6.2 KiB/s wr, 96 op/s 2023-09-23T18:17:47.774 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:47 smithi098 bash[33599]: cluster 2023-09-23T18:17:46.640288+0000 mgr.y (mgr.24344) 191 : cluster [DBG] pgmap v181: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 70 KiB/s rd, 6.2 KiB/s wr, 96 op/s 2023-09-23T18:17:48.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:47 smithi173 bash[27694]: cluster 2023-09-23T18:17:46.640288+0000 mgr.y (mgr.24344) 191 : cluster [DBG] pgmap v181: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 70 KiB/s rd, 6.2 KiB/s wr, 96 op/s 2023-09-23T18:17:48.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:17:47 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:17:47] "GET /metrics HTTP/1.1" 200 192174 "" "Prometheus/2.33.4" 2023-09-23T18:17:49.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:48 smithi173 bash[27694]: audit 2023-09-23T18:17:47.792460+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.98:0/3809373478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:17:49.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:48 smithi173 bash[27694]: cluster 2023-09-23T18:17:48.385711+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-09-23T18:17:49.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:48 smithi098 bash[27349]: audit 2023-09-23T18:17:47.792460+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.98:0/3809373478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:17:49.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:48 smithi098 bash[27349]: cluster 2023-09-23T18:17:48.385711+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-09-23T18:17:49.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:48 smithi098 bash[33599]: audit 2023-09-23T18:17:47.792460+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.98:0/3809373478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:17:49.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:48 smithi098 bash[33599]: cluster 2023-09-23T18:17:48.385711+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-09-23T18:17:50.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:49 smithi173 bash[27694]: cluster 2023-09-23T18:17:48.641034+0000 mgr.y (mgr.24344) 192 : cluster [DBG] pgmap v183: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.6 MiB/s wr, 402 op/s 2023-09-23T18:17:50.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:49 smithi098 bash[27349]: cluster 2023-09-23T18:17:48.641034+0000 mgr.y (mgr.24344) 192 : cluster [DBG] pgmap v183: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.6 MiB/s wr, 402 op/s 2023-09-23T18:17:50.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:49 smithi098 bash[33599]: cluster 2023-09-23T18:17:48.641034+0000 mgr.y (mgr.24344) 192 : cluster [DBG] pgmap v183: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.6 MiB/s wr, 402 op/s 2023-09-23T18:17:51.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:50 smithi173 bash[27694]: audit 2023-09-23T18:17:50.292683+0000 mon.a (mon.0) 701 : audit [DBG] from='client.? 172.21.15.98:0/2103175897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:17:51.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:50 smithi098 bash[27349]: audit 2023-09-23T18:17:50.292683+0000 mon.a (mon.0) 701 : audit [DBG] from='client.? 172.21.15.98:0/2103175897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:17:51.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:50 smithi098 bash[33599]: audit 2023-09-23T18:17:50.292683+0000 mon.a (mon.0) 701 : audit [DBG] from='client.? 172.21.15.98:0/2103175897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:17:52.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:51 smithi173 bash[27694]: cluster 2023-09-23T18:17:50.641543+0000 mgr.y (mgr.24344) 193 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1017 KiB/s rd, 1.4 MiB/s wr, 353 op/s 2023-09-23T18:17:52.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:51 smithi098 bash[27349]: cluster 2023-09-23T18:17:50.641543+0000 mgr.y (mgr.24344) 193 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1017 KiB/s rd, 1.4 MiB/s wr, 353 op/s 2023-09-23T18:17:52.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:51 smithi098 bash[33599]: cluster 2023-09-23T18:17:50.641543+0000 mgr.y (mgr.24344) 193 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1017 KiB/s rd, 1.4 MiB/s wr, 353 op/s 2023-09-23T18:17:54.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:53 smithi173 bash[27694]: cluster 2023-09-23T18:17:52.642036+0000 mgr.y (mgr.24344) 194 : cluster [DBG] pgmap v185: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 828 KiB/s rd, 1.1 MiB/s wr, 287 op/s 2023-09-23T18:17:54.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:53 smithi173 bash[27694]: audit 2023-09-23T18:17:52.760510+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.98:0/2643469537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:17:54.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:53 smithi098 bash[27349]: cluster 2023-09-23T18:17:52.642036+0000 mgr.y (mgr.24344) 194 : cluster [DBG] pgmap v185: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 828 KiB/s rd, 1.1 MiB/s wr, 287 op/s 2023-09-23T18:17:54.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:53 smithi098 bash[27349]: audit 2023-09-23T18:17:52.760510+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.98:0/2643469537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:17:54.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:53 smithi098 bash[33599]: cluster 2023-09-23T18:17:52.642036+0000 mgr.y (mgr.24344) 194 : cluster [DBG] pgmap v185: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 828 KiB/s rd, 1.1 MiB/s wr, 287 op/s 2023-09-23T18:17:54.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:53 smithi098 bash[33599]: audit 2023-09-23T18:17:52.760510+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.98:0/2643469537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:17:55.732 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:17:55 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:17:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:17:56.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:55 smithi173 bash[27694]: cluster 2023-09-23T18:17:54.643152+0000 mgr.y (mgr.24344) 195 : cluster [DBG] pgmap v186: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 227 op/s 2023-09-23T18:17:56.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:55 smithi173 bash[27694]: audit 2023-09-23T18:17:55.163077+0000 mon.a (mon.0) 702 : audit [DBG] from='client.? 172.21.15.98:0/3084558769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:17:56.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:55 smithi098 bash[33599]: cluster 2023-09-23T18:17:54.643152+0000 mgr.y (mgr.24344) 195 : cluster [DBG] pgmap v186: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 227 op/s 2023-09-23T18:17:56.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:55 smithi098 bash[33599]: audit 2023-09-23T18:17:55.163077+0000 mon.a (mon.0) 702 : audit [DBG] from='client.? 172.21.15.98:0/3084558769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:17:56.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:55 smithi098 bash[27349]: cluster 2023-09-23T18:17:54.643152+0000 mgr.y (mgr.24344) 195 : cluster [DBG] pgmap v186: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 227 op/s 2023-09-23T18:17:56.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:55 smithi098 bash[27349]: audit 2023-09-23T18:17:55.163077+0000 mon.a (mon.0) 702 : audit [DBG] from='client.? 172.21.15.98:0/3084558769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:17:58.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:57 smithi173 bash[27694]: cluster 2023-09-23T18:17:56.643796+0000 mgr.y (mgr.24344) 196 : cluster [DBG] pgmap v187: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 227 op/s 2023-09-23T18:17:58.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:57 smithi173 bash[27694]: audit 2023-09-23T18:17:57.563686+0000 mon.a (mon.0) 703 : audit [DBG] from='client.? 172.21.15.98:0/2348446287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:17:58.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:17:57 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:17:57] "GET /metrics HTTP/1.1" 200 192174 "" "Prometheus/2.33.4" 2023-09-23T18:17:58.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:57 smithi098 bash[33599]: cluster 2023-09-23T18:17:56.643796+0000 mgr.y (mgr.24344) 196 : cluster [DBG] pgmap v187: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 227 op/s 2023-09-23T18:17:58.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:57 smithi098 bash[33599]: audit 2023-09-23T18:17:57.563686+0000 mon.a (mon.0) 703 : audit [DBG] from='client.? 172.21.15.98:0/2348446287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:17:58.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:57 smithi098 bash[27349]: cluster 2023-09-23T18:17:56.643796+0000 mgr.y (mgr.24344) 196 : cluster [DBG] pgmap v187: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 227 op/s 2023-09-23T18:17:58.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:57 smithi098 bash[27349]: audit 2023-09-23T18:17:57.563686+0000 mon.a (mon.0) 703 : audit [DBG] from='client.? 172.21.15.98:0/2348446287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:17:59.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:17:59 smithi098 bash[27349]: cluster 2023-09-23T18:17:58.644778+0000 mgr.y (mgr.24344) 197 : cluster [DBG] pgmap v188: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 766 KiB/s rd, 1.1 MiB/s wr, 222 op/s 2023-09-23T18:17:59.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:17:59 smithi098 bash[33599]: cluster 2023-09-23T18:17:58.644778+0000 mgr.y (mgr.24344) 197 : cluster [DBG] pgmap v188: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 766 KiB/s rd, 1.1 MiB/s wr, 222 op/s 2023-09-23T18:17:59.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:17:59 smithi173 bash[27694]: cluster 2023-09-23T18:17:58.644778+0000 mgr.y (mgr.24344) 197 : cluster [DBG] pgmap v188: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 766 KiB/s rd, 1.1 MiB/s wr, 222 op/s 2023-09-23T18:18:00.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:00 smithi098 bash[27349]: audit 2023-09-23T18:17:59.967462+0000 mon.a (mon.0) 704 : audit [DBG] from='client.? 172.21.15.98:0/2377757725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:00.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:00 smithi098 bash[33599]: audit 2023-09-23T18:17:59.967462+0000 mon.a (mon.0) 704 : audit [DBG] from='client.? 172.21.15.98:0/2377757725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:00.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:00 smithi173 bash[27694]: audit 2023-09-23T18:17:59.967462+0000 mon.a (mon.0) 704 : audit [DBG] from='client.? 172.21.15.98:0/2377757725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:01.586 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:01 smithi098 bash[33599]: cluster 2023-09-23T18:18:00.645262+0000 mgr.y (mgr.24344) 198 : cluster [DBG] pgmap v189: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:01.586 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:01 smithi098 bash[27349]: cluster 2023-09-23T18:18:00.645262+0000 mgr.y (mgr.24344) 198 : cluster [DBG] pgmap v189: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:01.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:01 smithi173 bash[27694]: cluster 2023-09-23T18:18:00.645262+0000 mgr.y (mgr.24344) 198 : cluster [DBG] pgmap v189: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:03.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:03 smithi098 bash[27349]: audit 2023-09-23T18:18:02.358472+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:18:03.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:03 smithi098 bash[27349]: cephadm 2023-09-23T18:18:02.361087+0000 mgr.y (mgr.24344) 199 : cephadm [INF] Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T18:18:03.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:03 smithi098 bash[27349]: cephadm 2023-09-23T18:18:02.361208+0000 mgr.y (mgr.24344) 200 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T18:18:03.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:03 smithi098 bash[27349]: audit 2023-09-23T18:18:02.362673+0000 mon.c (mon.2) 107 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:18:03.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:03 smithi098 bash[27349]: audit 2023-09-23T18:18:02.369171+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:18:03.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:03 smithi098 bash[27349]: audit 2023-09-23T18:18:02.369411+0000 mon.a (mon.0) 707 : audit [DBG] from='client.? 172.21.15.98:0/814327660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:03.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:03 smithi098 bash[27349]: cephadm 2023-09-23T18:18:02.370356+0000 mgr.y (mgr.24344) 201 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T18:18:03.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:03 smithi098 bash[27349]: cephadm 2023-09-23T18:18:02.590466+0000 mgr.y (mgr.24344) 202 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi173 2023-09-23T18:18:03.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:03 smithi098 bash[27349]: cluster 2023-09-23T18:18:02.645848+0000 mgr.y (mgr.24344) 203 : cluster [DBG] pgmap v190: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:03.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:03 smithi098 bash[33599]: audit 2023-09-23T18:18:02.358472+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:18:03.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:03 smithi098 bash[33599]: cephadm 2023-09-23T18:18:02.361087+0000 mgr.y (mgr.24344) 199 : cephadm [INF] Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T18:18:03.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:03 smithi098 bash[33599]: cephadm 2023-09-23T18:18:02.361208+0000 mgr.y (mgr.24344) 200 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T18:18:03.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:03 smithi098 bash[33599]: audit 2023-09-23T18:18:02.362673+0000 mon.c (mon.2) 107 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:18:03.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:03 smithi098 bash[33599]: audit 2023-09-23T18:18:02.369171+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:18:03.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:03 smithi098 bash[33599]: audit 2023-09-23T18:18:02.369411+0000 mon.a (mon.0) 707 : audit [DBG] from='client.? 172.21.15.98:0/814327660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:03.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:03 smithi098 bash[33599]: cephadm 2023-09-23T18:18:02.370356+0000 mgr.y (mgr.24344) 201 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T18:18:03.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:03 smithi098 bash[33599]: cephadm 2023-09-23T18:18:02.590466+0000 mgr.y (mgr.24344) 202 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi173 2023-09-23T18:18:03.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:03 smithi098 bash[33599]: cluster 2023-09-23T18:18:02.645848+0000 mgr.y (mgr.24344) 203 : cluster [DBG] pgmap v190: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:03.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:03 smithi173 bash[27694]: audit 2023-09-23T18:18:02.358472+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:18:03.840 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:03 smithi173 bash[27694]: cephadm 2023-09-23T18:18:02.361087+0000 mgr.y (mgr.24344) 199 : cephadm [INF] Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T18:18:03.840 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:03 smithi173 bash[27694]: cephadm 2023-09-23T18:18:02.361208+0000 mgr.y (mgr.24344) 200 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T18:18:03.840 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:03 smithi173 bash[27694]: audit 2023-09-23T18:18:02.362673+0000 mon.c (mon.2) 107 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:18:03.840 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:03 smithi173 bash[27694]: audit 2023-09-23T18:18:02.369171+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:18:03.840 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:03 smithi173 bash[27694]: audit 2023-09-23T18:18:02.369411+0000 mon.a (mon.0) 707 : audit [DBG] from='client.? 172.21.15.98:0/814327660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:03.840 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:03 smithi173 bash[27694]: cephadm 2023-09-23T18:18:02.370356+0000 mgr.y (mgr.24344) 201 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T18:18:03.841 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:03 smithi173 bash[27694]: cephadm 2023-09-23T18:18:02.590466+0000 mgr.y (mgr.24344) 202 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi173 2023-09-23T18:18:03.841 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:03 smithi173 bash[27694]: cluster 2023-09-23T18:18:02.645848+0000 mgr.y (mgr.24344) 203 : cluster [DBG] pgmap v190: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:05.710 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:05 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:18:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:18:06.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:05 smithi173 bash[27694]: cluster 2023-09-23T18:18:04.646923+0000 mgr.y (mgr.24344) 204 : cluster [DBG] pgmap v191: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:06.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:05 smithi173 bash[27694]: audit 2023-09-23T18:18:04.775263+0000 mon.c (mon.2) 108 : audit [DBG] from='client.? 172.21.15.98:0/497639383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:06.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:05 smithi098 bash[27349]: cluster 2023-09-23T18:18:04.646923+0000 mgr.y (mgr.24344) 204 : cluster [DBG] pgmap v191: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:06.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:05 smithi098 bash[27349]: audit 2023-09-23T18:18:04.775263+0000 mon.c (mon.2) 108 : audit [DBG] from='client.? 172.21.15.98:0/497639383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:06.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:05 smithi098 bash[33599]: cluster 2023-09-23T18:18:04.646923+0000 mgr.y (mgr.24344) 204 : cluster [DBG] pgmap v191: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:06.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:05 smithi098 bash[33599]: audit 2023-09-23T18:18:04.775263+0000 mon.c (mon.2) 108 : audit [DBG] from='client.? 172.21.15.98:0/497639383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:08.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:07 smithi173 bash[27694]: cluster 2023-09-23T18:18:06.647548+0000 mgr.y (mgr.24344) 205 : cluster [DBG] pgmap v192: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:08.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:07 smithi173 bash[27694]: audit 2023-09-23T18:18:07.176678+0000 mon.a (mon.0) 708 : audit [DBG] from='client.? 172.21.15.98:0/3747454635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:08.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:07 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:18:07] "GET /metrics HTTP/1.1" 200 192222 "" "Prometheus/2.33.4" 2023-09-23T18:18:08.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:07 smithi098 bash[27349]: cluster 2023-09-23T18:18:06.647548+0000 mgr.y (mgr.24344) 205 : cluster [DBG] pgmap v192: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:08.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:07 smithi098 bash[27349]: audit 2023-09-23T18:18:07.176678+0000 mon.a (mon.0) 708 : audit [DBG] from='client.? 172.21.15.98:0/3747454635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:08.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:07 smithi098 bash[33599]: cluster 2023-09-23T18:18:06.647548+0000 mgr.y (mgr.24344) 205 : cluster [DBG] pgmap v192: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:08.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:07 smithi098 bash[33599]: audit 2023-09-23T18:18:07.176678+0000 mon.a (mon.0) 708 : audit [DBG] from='client.? 172.21.15.98:0/3747454635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:09.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:09 smithi098 bash[27349]: cluster 2023-09-23T18:18:08.648653+0000 mgr.y (mgr.24344) 206 : cluster [DBG] pgmap v193: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:09.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:09 smithi098 bash[33599]: cluster 2023-09-23T18:18:08.648653+0000 mgr.y (mgr.24344) 206 : cluster [DBG] pgmap v193: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:09.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:09 smithi173 bash[27694]: cluster 2023-09-23T18:18:08.648653+0000 mgr.y (mgr.24344) 206 : cluster [DBG] pgmap v193: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:10.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:10 smithi098 bash[27349]: audit 2023-09-23T18:18:09.582732+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.98:0/2129302219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:10.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:10 smithi098 bash[33599]: audit 2023-09-23T18:18:09.582732+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.98:0/2129302219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:10.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:10 smithi173 bash[27694]: audit 2023-09-23T18:18:09.582732+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.98:0/2129302219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:11.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:11 smithi098 bash[27349]: cluster 2023-09-23T18:18:10.649439+0000 mgr.y (mgr.24344) 207 : cluster [DBG] pgmap v194: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:11.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:11 smithi098 bash[33599]: cluster 2023-09-23T18:18:10.649439+0000 mgr.y (mgr.24344) 207 : cluster [DBG] pgmap v194: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:11.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:11 smithi173 bash[27694]: cluster 2023-09-23T18:18:10.649439+0000 mgr.y (mgr.24344) 207 : cluster [DBG] pgmap v194: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:12.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:12 smithi098 bash[33599]: audit 2023-09-23T18:18:11.986949+0000 mon.a (mon.0) 709 : audit [DBG] from='client.? 172.21.15.98:0/819359847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:12.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:12 smithi098 bash[27349]: audit 2023-09-23T18:18:11.986949+0000 mon.a (mon.0) 709 : audit [DBG] from='client.? 172.21.15.98:0/819359847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:12.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:12 smithi173 bash[27694]: audit 2023-09-23T18:18:11.986949+0000 mon.a (mon.0) 709 : audit [DBG] from='client.? 172.21.15.98:0/819359847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:13.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:13 smithi098 bash[27349]: cluster 2023-09-23T18:18:12.650119+0000 mgr.y (mgr.24344) 208 : cluster [DBG] pgmap v195: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:13.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:13 smithi098 bash[33599]: cluster 2023-09-23T18:18:12.650119+0000 mgr.y (mgr.24344) 208 : cluster [DBG] pgmap v195: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:13.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:13 smithi173 bash[27694]: cluster 2023-09-23T18:18:12.650119+0000 mgr.y (mgr.24344) 208 : cluster [DBG] pgmap v195: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:14.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:14 smithi098 bash[27349]: audit 2023-09-23T18:18:14.392509+0000 mon.a (mon.0) 710 : audit [DBG] from='client.? 172.21.15.98:0/2022109989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:14.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:14 smithi098 bash[33599]: audit 2023-09-23T18:18:14.392509+0000 mon.a (mon.0) 710 : audit [DBG] from='client.? 172.21.15.98:0/2022109989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:14.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:14 smithi173 bash[27694]: audit 2023-09-23T18:18:14.392509+0000 mon.a (mon.0) 710 : audit [DBG] from='client.? 172.21.15.98:0/2022109989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:15.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:15 smithi098 bash[27349]: cluster 2023-09-23T18:18:14.651179+0000 mgr.y (mgr.24344) 209 : cluster [DBG] pgmap v196: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:15.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:15 smithi098 bash[33599]: cluster 2023-09-23T18:18:14.651179+0000 mgr.y (mgr.24344) 209 : cluster [DBG] pgmap v196: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:15.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:15 smithi173 bash[27694]: cluster 2023-09-23T18:18:14.651179+0000 mgr.y (mgr.24344) 209 : cluster [DBG] pgmap v196: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:15.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:15 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:18:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:18:18.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:17 smithi173 bash[27694]: cluster 2023-09-23T18:18:16.651922+0000 mgr.y (mgr.24344) 210 : cluster [DBG] pgmap v197: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:18.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:17 smithi173 bash[27694]: audit 2023-09-23T18:18:16.796209+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.98:0/1328703046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:18.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:17 smithi098 bash[33599]: cluster 2023-09-23T18:18:16.651922+0000 mgr.y (mgr.24344) 210 : cluster [DBG] pgmap v197: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:18.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:17 smithi098 bash[33599]: audit 2023-09-23T18:18:16.796209+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.98:0/1328703046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:18.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:17 smithi098 bash[27349]: cluster 2023-09-23T18:18:16.651922+0000 mgr.y (mgr.24344) 210 : cluster [DBG] pgmap v197: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:18.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:17 smithi098 bash[27349]: audit 2023-09-23T18:18:16.796209+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.98:0/1328703046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:18.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:17 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:18:17] "GET /metrics HTTP/1.1" 200 192220 "" "Prometheus/2.33.4" 2023-09-23T18:18:19.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:19 smithi098 bash[27349]: cluster 2023-09-23T18:18:18.652942+0000 mgr.y (mgr.24344) 211 : cluster [DBG] pgmap v198: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:19.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:19 smithi098 bash[27349]: audit 2023-09-23T18:18:19.200586+0000 mon.c (mon.2) 111 : audit [DBG] from='client.? 172.21.15.98:0/1992704284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:19.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:19 smithi098 bash[33599]: cluster 2023-09-23T18:18:18.652942+0000 mgr.y (mgr.24344) 211 : cluster [DBG] pgmap v198: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:19.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:19 smithi098 bash[33599]: audit 2023-09-23T18:18:19.200586+0000 mon.c (mon.2) 111 : audit [DBG] from='client.? 172.21.15.98:0/1992704284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:19.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:19 smithi173 bash[27694]: cluster 2023-09-23T18:18:18.652942+0000 mgr.y (mgr.24344) 211 : cluster [DBG] pgmap v198: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:19.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:19 smithi173 bash[27694]: audit 2023-09-23T18:18:19.200586+0000 mon.c (mon.2) 111 : audit [DBG] from='client.? 172.21.15.98:0/1992704284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:22.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:21 smithi173 bash[27694]: cluster 2023-09-23T18:18:20.653542+0000 mgr.y (mgr.24344) 212 : cluster [DBG] pgmap v199: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:22.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:21 smithi173 bash[27694]: audit 2023-09-23T18:18:21.622453+0000 mon.b (mon.1) 28 : audit [DBG] from='client.? 172.21.15.98:0/3475621878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:22.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:21 smithi098 bash[27349]: cluster 2023-09-23T18:18:20.653542+0000 mgr.y (mgr.24344) 212 : cluster [DBG] pgmap v199: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:22.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:21 smithi098 bash[27349]: audit 2023-09-23T18:18:21.622453+0000 mon.b (mon.1) 28 : audit [DBG] from='client.? 172.21.15.98:0/3475621878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:22.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:21 smithi098 bash[33599]: cluster 2023-09-23T18:18:20.653542+0000 mgr.y (mgr.24344) 212 : cluster [DBG] pgmap v199: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:22.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:21 smithi098 bash[33599]: audit 2023-09-23T18:18:21.622453+0000 mon.b (mon.1) 28 : audit [DBG] from='client.? 172.21.15.98:0/3475621878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:24.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:23 smithi173 bash[27694]: cluster 2023-09-23T18:18:22.654207+0000 mgr.y (mgr.24344) 213 : cluster [DBG] pgmap v200: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:24.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:23 smithi098 bash[27349]: cluster 2023-09-23T18:18:22.654207+0000 mgr.y (mgr.24344) 213 : cluster [DBG] pgmap v200: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:24.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:23 smithi098 bash[33599]: cluster 2023-09-23T18:18:22.654207+0000 mgr.y (mgr.24344) 213 : cluster [DBG] pgmap v200: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:25.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:24 smithi173 bash[27694]: audit 2023-09-23T18:18:24.033334+0000 mon.a (mon.0) 711 : audit [DBG] from='client.? 172.21.15.98:0/2516260192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:25.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:24 smithi098 bash[27349]: audit 2023-09-23T18:18:24.033334+0000 mon.a (mon.0) 711 : audit [DBG] from='client.? 172.21.15.98:0/2516260192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:25.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:24 smithi098 bash[33599]: audit 2023-09-23T18:18:24.033334+0000 mon.a (mon.0) 711 : audit [DBG] from='client.? 172.21.15.98:0/2516260192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:25.729 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:25 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:18:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:18:26.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:25 smithi173 bash[27694]: cluster 2023-09-23T18:18:24.655340+0000 mgr.y (mgr.24344) 214 : cluster [DBG] pgmap v201: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:26.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:25 smithi098 bash[33599]: cluster 2023-09-23T18:18:24.655340+0000 mgr.y (mgr.24344) 214 : cluster [DBG] pgmap v201: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:26.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:25 smithi098 bash[27349]: cluster 2023-09-23T18:18:24.655340+0000 mgr.y (mgr.24344) 214 : cluster [DBG] pgmap v201: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:27.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:26 smithi173 bash[27694]: audit 2023-09-23T18:18:26.439433+0000 mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.98:0/2259397321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:27.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:26 smithi098 bash[27349]: audit 2023-09-23T18:18:26.439433+0000 mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.98:0/2259397321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:27.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:26 smithi098 bash[33599]: audit 2023-09-23T18:18:26.439433+0000 mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.98:0/2259397321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:28.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:27 smithi173 bash[27694]: cluster 2023-09-23T18:18:26.655979+0000 mgr.y (mgr.24344) 215 : cluster [DBG] pgmap v202: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:28.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:27 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:18:27] "GET /metrics HTTP/1.1" 200 192220 "" "Prometheus/2.33.4" 2023-09-23T18:18:28.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:27 smithi098 bash[33599]: cluster 2023-09-23T18:18:26.655979+0000 mgr.y (mgr.24344) 215 : cluster [DBG] pgmap v202: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:28.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:27 smithi098 bash[27349]: cluster 2023-09-23T18:18:26.655979+0000 mgr.y (mgr.24344) 215 : cluster [DBG] pgmap v202: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:29.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:29 smithi098 bash[27349]: cluster 2023-09-23T18:18:28.657143+0000 mgr.y (mgr.24344) 216 : cluster [DBG] pgmap v203: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:29.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:29 smithi098 bash[27349]: audit 2023-09-23T18:18:28.847485+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.98:0/3461806646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:29.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:29 smithi098 bash[33599]: cluster 2023-09-23T18:18:28.657143+0000 mgr.y (mgr.24344) 216 : cluster [DBG] pgmap v203: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:29.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:29 smithi098 bash[33599]: audit 2023-09-23T18:18:28.847485+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.98:0/3461806646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:29.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:29 smithi173 bash[27694]: cluster 2023-09-23T18:18:28.657143+0000 mgr.y (mgr.24344) 216 : cluster [DBG] pgmap v203: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:29.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:29 smithi173 bash[27694]: audit 2023-09-23T18:18:28.847485+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.98:0/3461806646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:32.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:31 smithi173 bash[27694]: cluster 2023-09-23T18:18:30.657896+0000 mgr.y (mgr.24344) 217 : cluster [DBG] pgmap v204: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:32.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:31 smithi173 bash[27694]: audit 2023-09-23T18:18:31.257635+0000 mon.c (mon.2) 114 : audit [DBG] from='client.? 172.21.15.98:0/2640415589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:32.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:31 smithi098 bash[27349]: cluster 2023-09-23T18:18:30.657896+0000 mgr.y (mgr.24344) 217 : cluster [DBG] pgmap v204: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:32.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:31 smithi098 bash[27349]: audit 2023-09-23T18:18:31.257635+0000 mon.c (mon.2) 114 : audit [DBG] from='client.? 172.21.15.98:0/2640415589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:32.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:31 smithi098 bash[33599]: cluster 2023-09-23T18:18:30.657896+0000 mgr.y (mgr.24344) 217 : cluster [DBG] pgmap v204: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:32.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:31 smithi098 bash[33599]: audit 2023-09-23T18:18:31.257635+0000 mon.c (mon.2) 114 : audit [DBG] from='client.? 172.21.15.98:0/2640415589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:34.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:33 smithi173 bash[27694]: cluster 2023-09-23T18:18:32.658650+0000 mgr.y (mgr.24344) 218 : cluster [DBG] pgmap v205: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:34.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:33 smithi173 bash[27694]: audit 2023-09-23T18:18:33.663329+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.98:0/3263157057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:34.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:33 smithi098 bash[27349]: cluster 2023-09-23T18:18:32.658650+0000 mgr.y (mgr.24344) 218 : cluster [DBG] pgmap v205: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:34.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:33 smithi098 bash[27349]: audit 2023-09-23T18:18:33.663329+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.98:0/3263157057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:34.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:33 smithi098 bash[33599]: cluster 2023-09-23T18:18:32.658650+0000 mgr.y (mgr.24344) 218 : cluster [DBG] pgmap v205: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:34.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:33 smithi098 bash[33599]: audit 2023-09-23T18:18:33.663329+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.98:0/3263157057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:35.731 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:35 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:18:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:18:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:35 smithi173 bash[27694]: cluster 2023-09-23T18:18:34.659722+0000 mgr.y (mgr.24344) 219 : cluster [DBG] pgmap v206: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:36.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:35 smithi098 bash[27349]: cluster 2023-09-23T18:18:34.659722+0000 mgr.y (mgr.24344) 219 : cluster [DBG] pgmap v206: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:36.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:35 smithi098 bash[33599]: cluster 2023-09-23T18:18:34.659722+0000 mgr.y (mgr.24344) 219 : cluster [DBG] pgmap v206: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:37.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:36 smithi173 bash[27694]: audit 2023-09-23T18:18:36.085808+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.98:0/1042677511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:37.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:36 smithi098 bash[27349]: audit 2023-09-23T18:18:36.085808+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.98:0/1042677511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:37.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:36 smithi098 bash[33599]: audit 2023-09-23T18:18:36.085808+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.98:0/1042677511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:38.053 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:37 smithi173 bash[27694]: cluster 2023-09-23T18:18:36.660358+0000 mgr.y (mgr.24344) 220 : cluster [DBG] pgmap v207: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:38.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:37 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:18:37] "GET /metrics HTTP/1.1" 200 192220 "" "Prometheus/2.33.4" 2023-09-23T18:18:38.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:37 smithi098 bash[27349]: cluster 2023-09-23T18:18:36.660358+0000 mgr.y (mgr.24344) 220 : cluster [DBG] pgmap v207: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:38.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:37 smithi098 bash[33599]: cluster 2023-09-23T18:18:36.660358+0000 mgr.y (mgr.24344) 220 : cluster [DBG] pgmap v207: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:39.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:39 smithi098 bash[27349]: audit 2023-09-23T18:18:38.502178+0000 mon.a (mon.0) 712 : audit [DBG] from='client.? 172.21.15.98:0/3669609429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:39.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:39 smithi098 bash[27349]: cluster 2023-09-23T18:18:38.661420+0000 mgr.y (mgr.24344) 221 : cluster [DBG] pgmap v208: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:39.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:39 smithi098 bash[33599]: audit 2023-09-23T18:18:38.502178+0000 mon.a (mon.0) 712 : audit [DBG] from='client.? 172.21.15.98:0/3669609429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:39.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:39 smithi098 bash[33599]: cluster 2023-09-23T18:18:38.661420+0000 mgr.y (mgr.24344) 221 : cluster [DBG] pgmap v208: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:39.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:39 smithi173 bash[27694]: audit 2023-09-23T18:18:38.502178+0000 mon.a (mon.0) 712 : audit [DBG] from='client.? 172.21.15.98:0/3669609429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:39.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:39 smithi173 bash[27694]: cluster 2023-09-23T18:18:38.661420+0000 mgr.y (mgr.24344) 221 : cluster [DBG] pgmap v208: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:42.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:41 smithi173 bash[27694]: cluster 2023-09-23T18:18:40.661978+0000 mgr.y (mgr.24344) 222 : cluster [DBG] pgmap v209: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:42.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:41 smithi173 bash[27694]: audit 2023-09-23T18:18:40.918243+0000 mon.c (mon.2) 117 : audit [DBG] from='client.? 172.21.15.98:0/3580073498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:42.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:41 smithi098 bash[27349]: cluster 2023-09-23T18:18:40.661978+0000 mgr.y (mgr.24344) 222 : cluster [DBG] pgmap v209: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:42.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:41 smithi098 bash[27349]: audit 2023-09-23T18:18:40.918243+0000 mon.c (mon.2) 117 : audit [DBG] from='client.? 172.21.15.98:0/3580073498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:42.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:41 smithi098 bash[33599]: cluster 2023-09-23T18:18:40.661978+0000 mgr.y (mgr.24344) 222 : cluster [DBG] pgmap v209: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:42.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:41 smithi098 bash[33599]: audit 2023-09-23T18:18:40.918243+0000 mon.c (mon.2) 117 : audit [DBG] from='client.? 172.21.15.98:0/3580073498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:43 smithi173 bash[27694]: cluster 2023-09-23T18:18:42.662610+0000 mgr.y (mgr.24344) 223 : cluster [DBG] pgmap v210: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:43 smithi173 bash[27694]: audit 2023-09-23T18:18:43.322933+0000 mon.c (mon.2) 118 : audit [DBG] from='client.? 172.21.15.98:0/4283218161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:43 smithi173 bash[27694]: audit 2023-09-23T18:18:43.716465+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:18:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:43 smithi173 bash[27694]: audit 2023-09-23T18:18:43.718755+0000 mon.c (mon.2) 119 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:18:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:43 smithi173 bash[27694]: audit 2023-09-23T18:18:43.719221+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:18:44.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:43 smithi098 bash[27349]: cluster 2023-09-23T18:18:42.662610+0000 mgr.y (mgr.24344) 223 : cluster [DBG] pgmap v210: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:44.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:43 smithi098 bash[27349]: audit 2023-09-23T18:18:43.322933+0000 mon.c (mon.2) 118 : audit [DBG] from='client.? 172.21.15.98:0/4283218161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:44.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:43 smithi098 bash[27349]: audit 2023-09-23T18:18:43.716465+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:18:44.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:43 smithi098 bash[27349]: audit 2023-09-23T18:18:43.718755+0000 mon.c (mon.2) 119 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:18:44.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:43 smithi098 bash[27349]: audit 2023-09-23T18:18:43.719221+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:18:44.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:43 smithi098 bash[33599]: cluster 2023-09-23T18:18:42.662610+0000 mgr.y (mgr.24344) 223 : cluster [DBG] pgmap v210: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:44.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:43 smithi098 bash[33599]: audit 2023-09-23T18:18:43.322933+0000 mon.c (mon.2) 118 : audit [DBG] from='client.? 172.21.15.98:0/4283218161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:44.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:43 smithi098 bash[33599]: audit 2023-09-23T18:18:43.716465+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:18:44.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:43 smithi098 bash[33599]: audit 2023-09-23T18:18:43.718755+0000 mon.c (mon.2) 119 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:18:44.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:43 smithi098 bash[33599]: audit 2023-09-23T18:18:43.719221+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:18:45.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:44 smithi173 bash[27694]: cephadm 2023-09-23T18:18:43.705773+0000 mgr.y (mgr.24344) 224 : cephadm [INF] Upgrade: Updating mgr.x 2023-09-23T18:18:45.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:44 smithi173 bash[27694]: audit 2023-09-23T18:18:43.720605+0000 mon.c (mon.2) 120 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:18:45.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:44 smithi173 bash[27694]: audit 2023-09-23T18:18:43.729132+0000 mon.c (mon.2) 121 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:18:45.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:44 smithi173 bash[27694]: cephadm 2023-09-23T18:18:43.730170+0000 mgr.y (mgr.24344) 225 : cephadm [INF] Deploying daemon mgr.x on smithi173 2023-09-23T18:18:45.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:44 smithi098 bash[33599]: cephadm 2023-09-23T18:18:43.705773+0000 mgr.y (mgr.24344) 224 : cephadm [INF] Upgrade: Updating mgr.x 2023-09-23T18:18:45.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:44 smithi098 bash[33599]: audit 2023-09-23T18:18:43.720605+0000 mon.c (mon.2) 120 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:18:45.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:44 smithi098 bash[33599]: audit 2023-09-23T18:18:43.729132+0000 mon.c (mon.2) 121 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:18:45.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:44 smithi098 bash[33599]: cephadm 2023-09-23T18:18:43.730170+0000 mgr.y (mgr.24344) 225 : cephadm [INF] Deploying daemon mgr.x on smithi173 2023-09-23T18:18:45.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:44 smithi098 bash[27349]: cephadm 2023-09-23T18:18:43.705773+0000 mgr.y (mgr.24344) 224 : cephadm [INF] Upgrade: Updating mgr.x 2023-09-23T18:18:45.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:44 smithi098 bash[27349]: audit 2023-09-23T18:18:43.720605+0000 mon.c (mon.2) 120 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:18:45.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:44 smithi098 bash[27349]: audit 2023-09-23T18:18:43.729132+0000 mon.c (mon.2) 121 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:18:45.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:44 smithi098 bash[27349]: cephadm 2023-09-23T18:18:43.730170+0000 mgr.y (mgr.24344) 225 : cephadm [INF] Deploying daemon mgr.x on smithi173 2023-09-23T18:18:45.740 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:45 smithi173 bash[28552]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:18:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:18:46.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:45 smithi173 bash[27694]: cluster 2023-09-23T18:18:44.663799+0000 mgr.y (mgr.24344) 226 : cluster [DBG] pgmap v211: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:46.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:45 smithi098 bash[27349]: cluster 2023-09-23T18:18:44.663799+0000 mgr.y (mgr.24344) 226 : cluster [DBG] pgmap v211: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:46.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:45 smithi098 bash[33599]: cluster 2023-09-23T18:18:44.663799+0000 mgr.y (mgr.24344) 226 : cluster [DBG] pgmap v211: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:46.523 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:46 smithi173 systemd[1]: Stopping Ceph mgr.x for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T18:18:46.523 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:46 smithi173 bash[50452]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-mgr.x 2023-09-23T18:18:46.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:46 smithi173 bash[27694]: audit 2023-09-23T18:18:45.739081+0000 mon.a (mon.0) 715 : audit [DBG] from='client.? 172.21.15.98:0/906012049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:47.090 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:46 smithi173 bash[50487]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-mgr-x 2023-09-23T18:18:47.090 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:47 smithi173 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-09-23T18:18:47.090 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:47 smithi173 bash[50562]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-mgr-x 2023-09-23T18:18:47.090 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:47 smithi173 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-09-23T18:18:47.090 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:47 smithi173 systemd[1]: Stopped Ceph mgr.x for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:18:47.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:46 smithi098 bash[33599]: audit 2023-09-23T18:18:45.739081+0000 mon.a (mon.0) 715 : audit [DBG] from='client.? 172.21.15.98:0/906012049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:47.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:46 smithi098 bash[27349]: audit 2023-09-23T18:18:45.739081+0000 mon.a (mon.0) 715 : audit [DBG] from='client.? 172.21.15.98:0/906012049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:47.389 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:47 smithi173 systemd[1]: Started Ceph mgr.x for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:18:48.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:47 smithi173 bash[27694]: cluster 2023-09-23T18:18:46.664489+0000 mgr.y (mgr.24344) 227 : cluster [DBG] pgmap v212: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:48.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:47 smithi173 bash[27694]: audit 2023-09-23T18:18:47.356186+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:18:48.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:47 smithi173 bash[27694]: audit 2023-09-23T18:18:47.358387+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:18:48.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:47 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:18:47] "GET /metrics HTTP/1.1" 200 192212 "" "Prometheus/2.33.4" 2023-09-23T18:18:48.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:47 smithi098 bash[33599]: cluster 2023-09-23T18:18:46.664489+0000 mgr.y (mgr.24344) 227 : cluster [DBG] pgmap v212: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:48.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:47 smithi098 bash[33599]: audit 2023-09-23T18:18:47.356186+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:18:48.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:47 smithi098 bash[33599]: audit 2023-09-23T18:18:47.358387+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:18:48.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:47 smithi098 bash[27349]: cluster 2023-09-23T18:18:46.664489+0000 mgr.y (mgr.24344) 227 : cluster [DBG] pgmap v212: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:48.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:47 smithi098 bash[27349]: audit 2023-09-23T18:18:47.356186+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:18:48.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:47 smithi098 bash[27349]: audit 2023-09-23T18:18:47.358387+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:18:48.665 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:48 smithi173 bash[50615]: debug 2023-09-23T18:18:48.496+0000 7fec75664700 1 -- 172.21.15.173:0/1796656285 <== mon.1 v2:172.21.15.173:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b9269c11e0 con 0x55b927740400 2023-09-23T18:18:49.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:48 smithi173 bash[27694]: audit 2023-09-23T18:18:48.130771+0000 mon.a (mon.0) 717 : audit [DBG] from='client.? 172.21.15.98:0/1768967184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:49.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:48 smithi173 bash[50615]: debug 2023-09-23T18:18:48.660+0000 7fec7f7b1200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T18:18:49.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:48 smithi098 bash[33599]: audit 2023-09-23T18:18:48.130771+0000 mon.a (mon.0) 717 : audit [DBG] from='client.? 172.21.15.98:0/1768967184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:49.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:48 smithi098 bash[27349]: audit 2023-09-23T18:18:48.130771+0000 mon.a (mon.0) 717 : audit [DBG] from='client.? 172.21.15.98:0/1768967184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:50.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:49 smithi173 bash[27694]: cluster 2023-09-23T18:18:48.665514+0000 mgr.y (mgr.24344) 228 : cluster [DBG] pgmap v213: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:50.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:49 smithi173 bash[50615]: debug 2023-09-23T18:18:49.572+0000 7fec7f7b1200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T18:18:50.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:49 smithi173 bash[50615]: debug 2023-09-23T18:18:49.676+0000 7fec7f7b1200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T18:18:50.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:49 smithi173 bash[50615]: debug 2023-09-23T18:18:49.779+0000 7fec7f7b1200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T18:18:50.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:49 smithi098 bash[27349]: cluster 2023-09-23T18:18:48.665514+0000 mgr.y (mgr.24344) 228 : cluster [DBG] pgmap v213: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:50.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:49 smithi098 bash[33599]: cluster 2023-09-23T18:18:48.665514+0000 mgr.y (mgr.24344) 228 : cluster [DBG] pgmap v213: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:50.420 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:50 smithi173 bash[50615]: debug 2023-09-23T18:18:50.047+0000 7fec7f7b1200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:18:50.759 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:50 smithi173 bash[50615]: debug 2023-09-23T18:18:50.415+0000 7fec7f7b1200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T18:18:50.759 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:50 smithi173 bash[50615]: debug 2023-09-23T18:18:50.591+0000 7fec7f7b1200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T18:18:51.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:50 smithi173 bash[27694]: audit 2023-09-23T18:18:50.525047+0000 mon.a (mon.0) 718 : audit [DBG] from='client.? 172.21.15.98:0/1972145768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:51.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:50 smithi173 bash[50615]: debug 2023-09-23T18:18:50.831+0000 7fec7f7b1200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T18:18:51.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:50 smithi098 bash[27349]: audit 2023-09-23T18:18:50.525047+0000 mon.a (mon.0) 718 : audit [DBG] from='client.? 172.21.15.98:0/1972145768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:51.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:50 smithi098 bash[33599]: audit 2023-09-23T18:18:50.525047+0000 mon.a (mon.0) 718 : audit [DBG] from='client.? 172.21.15.98:0/1972145768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:52.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:51 smithi173 bash[27694]: cluster 2023-09-23T18:18:50.666222+0000 mgr.y (mgr.24344) 229 : cluster [DBG] pgmap v214: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:52.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:51 smithi173 bash[50615]: debug 2023-09-23T18:18:51.895+0000 7fec7f7b1200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T18:18:52.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:52 smithi173 bash[50615]: debug 2023-09-23T18:18:52.011+0000 7fec7f7b1200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T18:18:52.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:51 smithi098 bash[27349]: cluster 2023-09-23T18:18:50.666222+0000 mgr.y (mgr.24344) 229 : cluster [DBG] pgmap v214: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:52.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:51 smithi098 bash[33599]: cluster 2023-09-23T18:18:50.666222+0000 mgr.y (mgr.24344) 229 : cluster [DBG] pgmap v214: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:52.527 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:52 smithi173 bash[50615]: debug 2023-09-23T18:18:52.131+0000 7fec7f7b1200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T18:18:53.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:52 smithi173 bash[50615]: debug 2023-09-23T18:18:52.647+0000 7fec7f7b1200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T18:18:53.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:52 smithi173 bash[50615]: debug 2023-09-23T18:18:52.895+0000 7fec7f7b1200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T18:18:53.771 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:53 smithi173 bash[50615]: debug 2023-09-23T18:18:53.399+0000 7fec7f7b1200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T18:18:54.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:53 smithi173 bash[50615]: debug 2023-09-23T18:18:53.939+0000 7fec7f7b1200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:18:54.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:53 smithi173 bash[27694]: cluster 2023-09-23T18:18:52.666840+0000 mgr.y (mgr.24344) 230 : cluster [DBG] pgmap v215: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:54.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:53 smithi173 bash[27694]: audit 2023-09-23T18:18:52.994841+0000 mon.a (mon.0) 719 : audit [DBG] from='client.? 172.21.15.98:0/1075530154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:54.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:53 smithi173 bash[27694]: audit 2023-09-23T18:18:53.084097+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:18:54.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:53 smithi173 bash[27694]: audit 2023-09-23T18:18:53.427903+0000 mon.a (mon.0) 721 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:18:54.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:53 smithi173 bash[27694]: audit 2023-09-23T18:18:53.683562+0000 mon.c (mon.2) 123 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:18:54.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:53 smithi173 bash[27694]: audit 2023-09-23T18:18:53.685667+0000 mon.c (mon.2) 124 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:18:54.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:53 smithi173 bash[27694]: audit 2023-09-23T18:18:53.694268+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:18:54.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:53 smithi173 bash[27694]: audit 2023-09-23T18:18:53.703123+0000 mon.c (mon.2) 125 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:18:54.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:53 smithi173 bash[27694]: audit 2023-09-23T18:18:53.711841+0000 mon.c (mon.2) 126 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T18:18:54.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:53 smithi173 bash[27694]: audit 2023-09-23T18:18:53.712464+0000 mon.a (mon.0) 723 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T18:18:54.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:53 smithi173 bash[27694]: cluster 2023-09-23T18:18:53.719647+0000 mon.a (mon.0) 724 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-09-23T18:18:54.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:53 smithi173 bash[27694]: audit 2023-09-23T18:18:53.720058+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T18:18:54.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:53 smithi173 bash[27694]: cluster 2023-09-23T18:18:53.720143+0000 mon.a (mon.0) 726 : cluster [DBG] mgrmap e19: x(active, starting, since 0.00697178s) 2023-09-23T18:18:54.047 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[27618]: debug 2023-09-23T18:18:53.718+0000 7fb4f9a31700 -1 mgr handle_mgr_map I was active but no longer am 2023-09-23T18:18:54.047 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[27618]: ignoring --setuser ceph since I am not root 2023-09-23T18:18:54.047 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[27618]: ignoring --setgroup ceph since I am not root 2023-09-23T18:18:54.048 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[27618]: debug 2023-09-23T18:18:53.774+0000 7f7292126700 1 -- 172.21.15.98:0/2234833206 <== mon.2 v2:172.21.15.98:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55bea9c584e0 con 0x55bea9d4c800 2023-09-23T18:18:54.048 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[27618]: debug 2023-09-23T18:18:53.930+0000 7f72a22ed680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T18:18:54.048 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[33599]: cluster 2023-09-23T18:18:52.666840+0000 mgr.y (mgr.24344) 230 : cluster [DBG] pgmap v215: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:54.048 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[33599]: audit 2023-09-23T18:18:52.994841+0000 mon.a (mon.0) 719 : audit [DBG] from='client.? 172.21.15.98:0/1075530154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:54.048 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[33599]: audit 2023-09-23T18:18:53.084097+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:18:54.048 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[33599]: audit 2023-09-23T18:18:53.427903+0000 mon.a (mon.0) 721 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:18:54.048 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[33599]: audit 2023-09-23T18:18:53.683562+0000 mon.c (mon.2) 123 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:18:54.048 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[33599]: audit 2023-09-23T18:18:53.685667+0000 mon.c (mon.2) 124 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:18:54.049 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[33599]: audit 2023-09-23T18:18:53.694268+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:18:54.049 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[33599]: audit 2023-09-23T18:18:53.703123+0000 mon.c (mon.2) 125 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:18:54.049 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[33599]: audit 2023-09-23T18:18:53.711841+0000 mon.c (mon.2) 126 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T18:18:54.049 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[33599]: audit 2023-09-23T18:18:53.712464+0000 mon.a (mon.0) 723 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T18:18:54.049 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[33599]: cluster 2023-09-23T18:18:53.719647+0000 mon.a (mon.0) 724 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-09-23T18:18:54.049 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[33599]: audit 2023-09-23T18:18:53.720058+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T18:18:54.049 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[33599]: cluster 2023-09-23T18:18:53.720143+0000 mon.a (mon.0) 726 : cluster [DBG] mgrmap e19: x(active, starting, since 0.00697178s) 2023-09-23T18:18:54.049 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[27349]: cluster 2023-09-23T18:18:52.666840+0000 mgr.y (mgr.24344) 230 : cluster [DBG] pgmap v215: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:54.049 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[27349]: audit 2023-09-23T18:18:52.994841+0000 mon.a (mon.0) 719 : audit [DBG] from='client.? 172.21.15.98:0/1075530154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:54.050 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[27349]: audit 2023-09-23T18:18:53.084097+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:18:54.050 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[27349]: audit 2023-09-23T18:18:53.427903+0000 mon.a (mon.0) 721 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:18:54.050 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[27349]: audit 2023-09-23T18:18:53.683562+0000 mon.c (mon.2) 123 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:18:54.050 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[27349]: audit 2023-09-23T18:18:53.685667+0000 mon.c (mon.2) 124 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:18:54.050 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[27349]: audit 2023-09-23T18:18:53.694268+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.24344 ' entity='mgr.y' 2023-09-23T18:18:54.050 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[27349]: audit 2023-09-23T18:18:53.703123+0000 mon.c (mon.2) 125 : audit [DBG] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:18:54.050 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[27349]: audit 2023-09-23T18:18:53.711841+0000 mon.c (mon.2) 126 : audit [INF] from='mgr.24344 172.21.15.98:0/3656930816' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T18:18:54.050 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[27349]: audit 2023-09-23T18:18:53.712464+0000 mon.a (mon.0) 723 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T18:18:54.050 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[27349]: cluster 2023-09-23T18:18:53.719647+0000 mon.a (mon.0) 724 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-09-23T18:18:54.050 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[27349]: audit 2023-09-23T18:18:53.720058+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T18:18:54.050 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:53 smithi098 bash[27349]: cluster 2023-09-23T18:18:53.720143+0000 mon.a (mon.0) 726 : cluster [DBG] mgrmap e19: x(active, starting, since 0.00697178s) 2023-09-23T18:18:54.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:54 smithi098 bash[27618]: debug 2023-09-23T18:18:54.038+0000 7f72a22ed680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T18:18:54.944 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:54 smithi098 bash[27618]: debug 2023-09-23T18:18:54.510+0000 7f72a22ed680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T18:18:55.001 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:54 smithi173 bash[50615]: debug 2023-09-23T18:18:54.531+0000 7fec7f7b1200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T18:18:55.001 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:54 smithi173 bash[50615]: debug 2023-09-23T18:18:54.647+0000 7fec7f7b1200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T18:18:55.001 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:54 smithi173 bash[50615]: debug 2023-09-23T18:18:54.995+0000 7fec7f7b1200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T18:18:55.263 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:55 smithi173 bash[50615]: debug 2023-09-23T18:18:55.111+0000 7fec7f7b1200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T18:18:55.264 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:55 smithi173 bash[50615]: debug 2023-09-23T18:18:55.259+0000 7fec7f7b1200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T18:18:55.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:55 smithi173 bash[27694]: audit 2023-09-23T18:18:55.423039+0000 mon.a (mon.0) 727 : audit [DBG] from='client.? 172.21.15.98:0/3940441368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:55.527 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:55 smithi173 bash[50615]: debug 2023-09-23T18:18:55.375+0000 7fec7f7b1200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T18:18:55.527 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:55 smithi173 bash[50615]: debug 2023-09-23T18:18:55.507+0000 7fec7f7b1200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T18:18:55.802 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:55 smithi098 bash[27349]: audit 2023-09-23T18:18:55.423039+0000 mon.a (mon.0) 727 : audit [DBG] from='client.? 172.21.15.98:0/3940441368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:55.803 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:55 smithi098 bash[27618]: debug 2023-09-23T18:18:55.438+0000 7f72a22ed680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T18:18:55.803 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:55 smithi098 bash[27618]: debug 2023-09-23T18:18:55.542+0000 7f72a22ed680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T18:18:55.803 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:55 smithi098 bash[33599]: audit 2023-09-23T18:18:55.423039+0000 mon.a (mon.0) 727 : audit [DBG] from='client.? 172.21.15.98:0/3940441368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:56.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:55 smithi173 bash[50615]: debug 2023-09-23T18:18:55.703+0000 7fec7f7b1200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T18:18:56.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:55 smithi173 bash[50615]: debug 2023-09-23T18:18:55.831+0000 7fec7f7b1200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T18:18:56.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:55 smithi098 bash[27618]: debug 2023-09-23T18:18:55.790+0000 7f72a22ed680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T18:18:57.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:56 smithi098 bash[27618]: debug 2023-09-23T18:18:56.926+0000 7f72a22ed680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T18:18:57.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27618]: debug 2023-09-23T18:18:57.094+0000 7f72a22ed680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:18:57.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: cluster 2023-09-23T18:18:57.082519+0000 mon.a (mon.0) 728 : cluster [INF] Active manager daemon x restarted 2023-09-23T18:18:57.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: cluster 2023-09-23T18:18:57.083186+0000 mon.a (mon.0) 729 : cluster [INF] Activating manager daemon x 2023-09-23T18:18:57.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: audit 2023-09-23T18:18:57.086957+0000 mon.c (mon.2) 127 : audit [DBG] from='mgr.? 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T18:18:57.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: cluster 2023-09-23T18:18:57.093607+0000 mon.a (mon.0) 730 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2023-09-23T18:18:57.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: cluster 2023-09-23T18:18:57.093888+0000 mon.a (mon.0) 731 : cluster [DBG] mgrmap e20: x(active, starting, since 0.0109522s) 2023-09-23T18:18:57.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: audit 2023-09-23T18:18:57.100804+0000 mon.b (mon.1) 29 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T18:18:57.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: audit 2023-09-23T18:18:57.101011+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:18:57.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: audit 2023-09-23T18:18:57.101246+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:18:57.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: audit 2023-09-23T18:18:57.101490+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:18:57.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: audit 2023-09-23T18:18:57.101988+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi173.qjefje"}]: dispatch 2023-09-23T18:18:57.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: audit 2023-09-23T18:18:57.102348+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi098.lowjqo"}]: dispatch 2023-09-23T18:18:57.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: audit 2023-09-23T18:18:57.102787+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T18:18:57.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: audit 2023-09-23T18:18:57.103109+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:18:57.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: audit 2023-09-23T18:18:57.103419+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:18:57.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: audit 2023-09-23T18:18:57.103609+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:18:57.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: audit 2023-09-23T18:18:57.103808+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:18:57.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: audit 2023-09-23T18:18:57.104001+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:18:57.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: audit 2023-09-23T18:18:57.104196+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:18:57.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: audit 2023-09-23T18:18:57.104381+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:18:57.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: audit 2023-09-23T18:18:57.104587+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:18:57.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: audit 2023-09-23T18:18:57.105801+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T18:18:57.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: audit 2023-09-23T18:18:57.106145+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T18:18:57.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: audit 2023-09-23T18:18:57.106640+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T18:18:57.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: audit 2023-09-23T18:18:57.106894+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:18:57.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: audit 2023-09-23T18:18:57.107929+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T18:18:57.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27349]: audit 2023-09-23T18:18:57.108786+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:18:57.197 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: cluster 2023-09-23T18:18:57.082519+0000 mon.a (mon.0) 728 : cluster [INF] Active manager daemon x restarted 2023-09-23T18:18:57.198 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: cluster 2023-09-23T18:18:57.083186+0000 mon.a (mon.0) 729 : cluster [INF] Activating manager daemon x 2023-09-23T18:18:57.198 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: audit 2023-09-23T18:18:57.086957+0000 mon.c (mon.2) 127 : audit [DBG] from='mgr.? 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T18:18:57.198 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: cluster 2023-09-23T18:18:57.093607+0000 mon.a (mon.0) 730 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2023-09-23T18:18:57.198 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: cluster 2023-09-23T18:18:57.093888+0000 mon.a (mon.0) 731 : cluster [DBG] mgrmap e20: x(active, starting, since 0.0109522s) 2023-09-23T18:18:57.198 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: audit 2023-09-23T18:18:57.100804+0000 mon.b (mon.1) 29 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T18:18:57.198 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: audit 2023-09-23T18:18:57.101011+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:18:57.198 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: audit 2023-09-23T18:18:57.101246+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:18:57.198 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: audit 2023-09-23T18:18:57.101490+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:18:57.198 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: audit 2023-09-23T18:18:57.101988+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi173.qjefje"}]: dispatch 2023-09-23T18:18:57.198 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: audit 2023-09-23T18:18:57.102348+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi098.lowjqo"}]: dispatch 2023-09-23T18:18:57.198 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: audit 2023-09-23T18:18:57.102787+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T18:18:57.199 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: audit 2023-09-23T18:18:57.103109+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:18:57.199 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: audit 2023-09-23T18:18:57.103419+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:18:57.199 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: audit 2023-09-23T18:18:57.103609+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:18:57.199 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: audit 2023-09-23T18:18:57.103808+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:18:57.199 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: audit 2023-09-23T18:18:57.104001+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:18:57.199 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: audit 2023-09-23T18:18:57.104196+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:18:57.199 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: audit 2023-09-23T18:18:57.104381+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:18:57.199 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: audit 2023-09-23T18:18:57.104587+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:18:57.199 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: audit 2023-09-23T18:18:57.105801+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T18:18:57.199 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: audit 2023-09-23T18:18:57.106145+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T18:18:57.199 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: audit 2023-09-23T18:18:57.106640+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T18:18:57.199 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: audit 2023-09-23T18:18:57.106894+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:18:57.200 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: audit 2023-09-23T18:18:57.107929+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T18:18:57.200 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[33599]: audit 2023-09-23T18:18:57.108786+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:18:57.368 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: cluster 2023-09-23T18:18:57.082519+0000 mon.a (mon.0) 728 : cluster [INF] Active manager daemon x restarted 2023-09-23T18:18:57.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: cluster 2023-09-23T18:18:57.083186+0000 mon.a (mon.0) 729 : cluster [INF] Activating manager daemon x 2023-09-23T18:18:57.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: audit 2023-09-23T18:18:57.086957+0000 mon.c (mon.2) 127 : audit [DBG] from='mgr.? 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T18:18:57.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: cluster 2023-09-23T18:18:57.093607+0000 mon.a (mon.0) 730 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2023-09-23T18:18:57.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: cluster 2023-09-23T18:18:57.093888+0000 mon.a (mon.0) 731 : cluster [DBG] mgrmap e20: x(active, starting, since 0.0109522s) 2023-09-23T18:18:57.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: audit 2023-09-23T18:18:57.100804+0000 mon.b (mon.1) 29 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T18:18:57.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: audit 2023-09-23T18:18:57.101011+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:18:57.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: audit 2023-09-23T18:18:57.101246+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:18:57.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: audit 2023-09-23T18:18:57.101490+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:18:57.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: audit 2023-09-23T18:18:57.101988+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi173.qjefje"}]: dispatch 2023-09-23T18:18:57.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: audit 2023-09-23T18:18:57.102348+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi098.lowjqo"}]: dispatch 2023-09-23T18:18:57.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: audit 2023-09-23T18:18:57.102787+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T18:18:57.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: audit 2023-09-23T18:18:57.103109+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:18:57.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: audit 2023-09-23T18:18:57.103419+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:18:57.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: audit 2023-09-23T18:18:57.103609+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:18:57.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: audit 2023-09-23T18:18:57.103808+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:18:57.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: audit 2023-09-23T18:18:57.104001+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:18:57.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: audit 2023-09-23T18:18:57.104196+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:18:57.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: audit 2023-09-23T18:18:57.104381+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:18:57.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: audit 2023-09-23T18:18:57.104587+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:18:57.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: audit 2023-09-23T18:18:57.105801+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T18:18:57.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: audit 2023-09-23T18:18:57.106145+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T18:18:57.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: audit 2023-09-23T18:18:57.106640+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T18:18:57.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: audit 2023-09-23T18:18:57.106894+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:18:57.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: audit 2023-09-23T18:18:57.107929+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T18:18:57.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[27694]: audit 2023-09-23T18:18:57.108786+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:18:57.371 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[50615]: debug 2023-09-23T18:18:57.079+0000 7fec7f7b1200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T18:18:57.371 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[50615]: [23/Sep/2023:18:18:57] ENGINE Bus STARTING 2023-09-23T18:18:57.371 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[50615]: CherryPy Checker: 2023-09-23T18:18:57.371 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[50615]: The Application mounted at '' has an empty config. 2023-09-23T18:18:57.371 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[50615]: [23/Sep/2023:18:18:57] ENGINE Serving on http://:::9283 2023-09-23T18:18:57.371 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[50615]: [23/Sep/2023:18:18:57] ENGINE Bus STARTED 2023-09-23T18:18:57.371 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[50615]: [23/Sep/2023:18:18:57] ENGINE Bus STOPPING 2023-09-23T18:18:57.371 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[50615]: [23/Sep/2023:18:18:57] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T18:18:57.372 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[50615]: [23/Sep/2023:18:18:57] ENGINE Bus STOPPED 2023-09-23T18:18:57.372 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[50615]: [23/Sep/2023:18:18:57] ENGINE Bus STARTING 2023-09-23T18:18:57.485 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27618]: debug 2023-09-23T18:18:57.202+0000 7f72a22ed680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T18:18:57.485 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27618]: debug 2023-09-23T18:18:57.318+0000 7f72a22ed680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T18:18:57.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[50615]: debug 2023-09-23T18:18:57.463+0000 7febddc4e700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:18:57.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[50615]: debug 2023-09-23T18:18:57.463+0000 7febddc4e700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:18:57.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[50615]: debug 2023-09-23T18:18:57.463+0000 7febddc4e700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:18:57.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[50615]: debug 2023-09-23T18:18:57.463+0000 7febddc4e700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:18:57.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[50615]: debug 2023-09-23T18:18:57.463+0000 7febddc4e700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:18:57.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[50615]: debug 2023-09-23T18:18:57.463+0000 7febdf451700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:18:57.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[50615]: debug 2023-09-23T18:18:57.463+0000 7febdf451700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:18:57.778 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[50615]: debug 2023-09-23T18:18:57.463+0000 7febdf451700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:18:57.778 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[50615]: debug 2023-09-23T18:18:57.463+0000 7febdf451700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:18:57.778 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[50615]: debug 2023-09-23T18:18:57.463+0000 7febdf451700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:18:57.778 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[50615]: [23/Sep/2023:18:18:57] ENGINE Serving on http://:::9283 2023-09-23T18:18:57.778 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:18:57 smithi173 bash[50615]: [23/Sep/2023:18:18:57] ENGINE Bus STARTED 2023-09-23T18:18:57.944 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27618]: debug 2023-09-23T18:18:57.554+0000 7f72a22ed680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T18:18:58.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:57 smithi098 bash[27618]: debug 2023-09-23T18:18:57.958+0000 7f72a22ed680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T18:18:58.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[27618]: debug 2023-09-23T18:18:58.090+0000 7f72a22ed680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T18:18:58.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[27349]: cluster 2023-09-23T18:18:57.268517+0000 mon.a (mon.0) 732 : cluster [INF] Manager daemon x is now available 2023-09-23T18:18:58.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[27349]: audit 2023-09-23T18:18:57.307341+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:18:58.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[27349]: audit 2023-09-23T18:18:57.325820+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:18:58.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[27349]: audit 2023-09-23T18:18:57.332278+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:18:58.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[27349]: audit 2023-09-23T18:18:57.341921+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:18:58.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[27349]: audit 2023-09-23T18:18:57.345457+0000 mon.b (mon.1) 52 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:18:58.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[27349]: audit 2023-09-23T18:18:57.366605+0000 mon.b (mon.1) 53 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:18:58.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[27349]: audit 2023-09-23T18:18:57.371467+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:18:58.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[27349]: audit 2023-09-23T18:18:57.375419+0000 mon.b (mon.1) 54 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:18:58.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[27349]: audit 2023-09-23T18:18:57.410662+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T18:18:58.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[27349]: audit 2023-09-23T18:18:57.414478+0000 mon.b (mon.1) 55 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T18:18:58.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[27349]: audit 2023-09-23T18:18:57.419648+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:18:58.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[27349]: audit 2023-09-23T18:18:57.600875+0000 mon.a (mon.0) 738 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:18:58.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[27349]: audit 2023-09-23T18:18:57.625929+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:18:58.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[27349]: audit 2023-09-23T18:18:57.839089+0000 mon.a (mon.0) 740 : audit [DBG] from='client.? 172.21.15.98:0/180842261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:58.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[33599]: cluster 2023-09-23T18:18:57.268517+0000 mon.a (mon.0) 732 : cluster [INF] Manager daemon x is now available 2023-09-23T18:18:58.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[33599]: audit 2023-09-23T18:18:57.307341+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:18:58.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[33599]: audit 2023-09-23T18:18:57.325820+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:18:58.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[33599]: audit 2023-09-23T18:18:57.332278+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:18:58.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[33599]: audit 2023-09-23T18:18:57.341921+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:18:58.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[33599]: audit 2023-09-23T18:18:57.345457+0000 mon.b (mon.1) 52 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:18:58.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[33599]: audit 2023-09-23T18:18:57.366605+0000 mon.b (mon.1) 53 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:18:58.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[33599]: audit 2023-09-23T18:18:57.371467+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:18:58.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[33599]: audit 2023-09-23T18:18:57.375419+0000 mon.b (mon.1) 54 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:18:58.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[33599]: audit 2023-09-23T18:18:57.410662+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T18:18:58.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[33599]: audit 2023-09-23T18:18:57.414478+0000 mon.b (mon.1) 55 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T18:18:58.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[33599]: audit 2023-09-23T18:18:57.419648+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:18:58.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[33599]: audit 2023-09-23T18:18:57.600875+0000 mon.a (mon.0) 738 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:18:58.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[33599]: audit 2023-09-23T18:18:57.625929+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:18:58.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:58 smithi098 bash[33599]: audit 2023-09-23T18:18:57.839089+0000 mon.a (mon.0) 740 : audit [DBG] from='client.? 172.21.15.98:0/180842261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:58.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:58 smithi173 bash[27694]: cluster 2023-09-23T18:18:57.268517+0000 mon.a (mon.0) 732 : cluster [INF] Manager daemon x is now available 2023-09-23T18:18:58.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:58 smithi173 bash[27694]: audit 2023-09-23T18:18:57.307341+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:18:58.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:58 smithi173 bash[27694]: audit 2023-09-23T18:18:57.325820+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:18:58.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:58 smithi173 bash[27694]: audit 2023-09-23T18:18:57.332278+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:18:58.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:58 smithi173 bash[27694]: audit 2023-09-23T18:18:57.341921+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:18:58.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:58 smithi173 bash[27694]: audit 2023-09-23T18:18:57.345457+0000 mon.b (mon.1) 52 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:18:58.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:58 smithi173 bash[27694]: audit 2023-09-23T18:18:57.366605+0000 mon.b (mon.1) 53 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:18:58.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:58 smithi173 bash[27694]: audit 2023-09-23T18:18:57.371467+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:18:58.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:58 smithi173 bash[27694]: audit 2023-09-23T18:18:57.375419+0000 mon.b (mon.1) 54 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:18:58.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:58 smithi173 bash[27694]: audit 2023-09-23T18:18:57.410662+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T18:18:58.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:58 smithi173 bash[27694]: audit 2023-09-23T18:18:57.414478+0000 mon.b (mon.1) 55 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T18:18:58.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:58 smithi173 bash[27694]: audit 2023-09-23T18:18:57.419648+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:18:58.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:58 smithi173 bash[27694]: audit 2023-09-23T18:18:57.600875+0000 mon.a (mon.0) 738 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:18:58.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:58 smithi173 bash[27694]: audit 2023-09-23T18:18:57.625929+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:18:58.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:58 smithi173 bash[27694]: audit 2023-09-23T18:18:57.839089+0000 mon.a (mon.0) 740 : audit [DBG] from='client.? 172.21.15.98:0/180842261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:18:59.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:59 smithi098 bash[27618]: debug 2023-09-23T18:18:58.994+0000 7f72a22ed680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T18:18:59.445 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:59 smithi098 bash[27618]: debug 2023-09-23T18:18:59.102+0000 7f72a22ed680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T18:18:59.445 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:59 smithi098 bash[27618]: debug 2023-09-23T18:18:59.218+0000 7f72a22ed680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T18:18:59.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:59 smithi098 bash[27349]: cephadm 2023-09-23T18:18:57.854002+0000 mgr.x (mgr.25562) 1 : cephadm [INF] Deploying cephadm binary to smithi098 2023-09-23T18:18:59.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:59 smithi098 bash[27349]: cephadm 2023-09-23T18:18:57.857213+0000 mgr.x (mgr.25562) 2 : cephadm [INF] Deploying cephadm binary to smithi173 2023-09-23T18:18:59.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:59 smithi098 bash[27349]: cluster 2023-09-23T18:18:58.159428+0000 mon.a (mon.0) 741 : cluster [DBG] mgrmap e21: x(active, since 1.07649s) 2023-09-23T18:18:59.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:59 smithi098 bash[27349]: cluster 2023-09-23T18:18:58.192113+0000 mgr.x (mgr.25562) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:59.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:59 smithi098 bash[27349]: audit 2023-09-23T18:18:58.310885+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:18:59.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:59 smithi098 bash[27349]: cluster 2023-09-23T18:18:58.311022+0000 mon.a (mon.0) 743 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2023-09-23T18:18:59.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:59 smithi098 bash[27349]: audit 2023-09-23T18:18:58.317732+0000 mon.a (mon.0) 744 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:18:59.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:59 smithi098 bash[27349]: audit 2023-09-23T18:18:58.321761+0000 mon.b (mon.1) 56 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:18:59.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:18:59 smithi098 bash[27349]: cephadm 2023-09-23T18:18:58.409096+0000 mgr.x (mgr.25562) 4 : cephadm [INF] [23/Sep/2023:18:18:58] ENGINE Bus STARTING 2023-09-23T18:18:59.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:59 smithi098 bash[33599]: cephadm 2023-09-23T18:18:57.854002+0000 mgr.x (mgr.25562) 1 : cephadm [INF] Deploying cephadm binary to smithi098 2023-09-23T18:18:59.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:59 smithi098 bash[33599]: cephadm 2023-09-23T18:18:57.857213+0000 mgr.x (mgr.25562) 2 : cephadm [INF] Deploying cephadm binary to smithi173 2023-09-23T18:18:59.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:59 smithi098 bash[33599]: cluster 2023-09-23T18:18:58.159428+0000 mon.a (mon.0) 741 : cluster [DBG] mgrmap e21: x(active, since 1.07649s) 2023-09-23T18:18:59.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:59 smithi098 bash[33599]: cluster 2023-09-23T18:18:58.192113+0000 mgr.x (mgr.25562) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:59.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:59 smithi098 bash[33599]: audit 2023-09-23T18:18:58.310885+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:18:59.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:59 smithi098 bash[33599]: cluster 2023-09-23T18:18:58.311022+0000 mon.a (mon.0) 743 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2023-09-23T18:18:59.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:59 smithi098 bash[33599]: audit 2023-09-23T18:18:58.317732+0000 mon.a (mon.0) 744 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:18:59.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:59 smithi098 bash[33599]: audit 2023-09-23T18:18:58.321761+0000 mon.b (mon.1) 56 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:18:59.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:18:59 smithi098 bash[33599]: cephadm 2023-09-23T18:18:58.409096+0000 mgr.x (mgr.25562) 4 : cephadm [INF] [23/Sep/2023:18:18:58] ENGINE Bus STARTING 2023-09-23T18:18:59.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:59 smithi173 bash[27694]: cephadm 2023-09-23T18:18:57.854002+0000 mgr.x (mgr.25562) 1 : cephadm [INF] Deploying cephadm binary to smithi098 2023-09-23T18:18:59.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:59 smithi173 bash[27694]: cephadm 2023-09-23T18:18:57.857213+0000 mgr.x (mgr.25562) 2 : cephadm [INF] Deploying cephadm binary to smithi173 2023-09-23T18:18:59.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:59 smithi173 bash[27694]: cluster 2023-09-23T18:18:58.159428+0000 mon.a (mon.0) 741 : cluster [DBG] mgrmap e21: x(active, since 1.07649s) 2023-09-23T18:18:59.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:59 smithi173 bash[27694]: cluster 2023-09-23T18:18:58.192113+0000 mgr.x (mgr.25562) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:18:59.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:59 smithi173 bash[27694]: audit 2023-09-23T18:18:58.310885+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:18:59.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:59 smithi173 bash[27694]: cluster 2023-09-23T18:18:58.311022+0000 mon.a (mon.0) 743 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2023-09-23T18:18:59.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:59 smithi173 bash[27694]: audit 2023-09-23T18:18:58.317732+0000 mon.a (mon.0) 744 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:18:59.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:59 smithi173 bash[27694]: audit 2023-09-23T18:18:58.321761+0000 mon.b (mon.1) 56 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:18:59.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:18:59 smithi173 bash[27694]: cephadm 2023-09-23T18:18:58.409096+0000 mgr.x (mgr.25562) 4 : cephadm [INF] [23/Sep/2023:18:18:58] ENGINE Bus STARTING 2023-09-23T18:18:59.760 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:59 smithi098 bash[27618]: debug 2023-09-23T18:18:59.454+0000 7f72a22ed680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T18:19:00.162 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:59 smithi098 bash[27618]: debug 2023-09-23T18:18:59.750+0000 7f72a22ed680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T18:19:00.162 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:18:59 smithi098 bash[27618]: debug 2023-09-23T18:18:59.922+0000 7f72a22ed680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T18:19:00.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:00 smithi098 bash[33599]: cephadm 2023-09-23T18:18:58.642104+0000 mgr.x (mgr.25562) 5 : cephadm [INF] [23/Sep/2023:18:18:58] ENGINE Serving on https://172.21.15.173:7150 2023-09-23T18:19:00.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:00 smithi098 bash[33599]: cephadm 2023-09-23T18:18:58.755221+0000 mgr.x (mgr.25562) 6 : cephadm [INF] [23/Sep/2023:18:18:58] ENGINE Serving on http://172.21.15.173:8765 2023-09-23T18:19:00.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:00 smithi098 bash[33599]: cephadm 2023-09-23T18:18:58.756204+0000 mgr.x (mgr.25562) 7 : cephadm [INF] [23/Sep/2023:18:18:58] ENGINE Bus STARTED 2023-09-23T18:19:00.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:00 smithi098 bash[33599]: cluster 2023-09-23T18:18:59.106047+0000 mgr.x (mgr.25562) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:00.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:00 smithi098 bash[33599]: audit 2023-09-23T18:18:59.313725+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:19:00.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:00 smithi098 bash[33599]: cluster 2023-09-23T18:18:59.313892+0000 mon.a (mon.0) 746 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-09-23T18:19:00.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:00 smithi098 bash[33599]: cluster 2023-09-23T18:18:59.411079+0000 mon.a (mon.0) 747 : cluster [DBG] mgrmap e22: x(active, since 2s) 2023-09-23T18:19:00.445 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:19:00 smithi098 bash[27618]: debug 2023-09-23T18:19:00.150+0000 7f72a22ed680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:19:00.445 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:19:00 smithi098 bash[27618]: debug 2023-09-23T18:19:00.386+0000 7f72a22ed680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T18:19:00.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:00 smithi098 bash[27349]: cephadm 2023-09-23T18:18:58.642104+0000 mgr.x (mgr.25562) 5 : cephadm [INF] [23/Sep/2023:18:18:58] ENGINE Serving on https://172.21.15.173:7150 2023-09-23T18:19:00.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:00 smithi098 bash[27349]: cephadm 2023-09-23T18:18:58.755221+0000 mgr.x (mgr.25562) 6 : cephadm [INF] [23/Sep/2023:18:18:58] ENGINE Serving on http://172.21.15.173:8765 2023-09-23T18:19:00.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:00 smithi098 bash[27349]: cephadm 2023-09-23T18:18:58.756204+0000 mgr.x (mgr.25562) 7 : cephadm [INF] [23/Sep/2023:18:18:58] ENGINE Bus STARTED 2023-09-23T18:19:00.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:00 smithi098 bash[27349]: cluster 2023-09-23T18:18:59.106047+0000 mgr.x (mgr.25562) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:00.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:00 smithi098 bash[27349]: audit 2023-09-23T18:18:59.313725+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:19:00.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:00 smithi098 bash[27349]: cluster 2023-09-23T18:18:59.313892+0000 mon.a (mon.0) 746 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-09-23T18:19:00.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:00 smithi098 bash[27349]: cluster 2023-09-23T18:18:59.411079+0000 mon.a (mon.0) 747 : cluster [DBG] mgrmap e22: x(active, since 2s) 2023-09-23T18:19:00.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:00 smithi173 bash[27694]: cephadm 2023-09-23T18:18:58.642104+0000 mgr.x (mgr.25562) 5 : cephadm [INF] [23/Sep/2023:18:18:58] ENGINE Serving on https://172.21.15.173:7150 2023-09-23T18:19:00.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:00 smithi173 bash[27694]: cephadm 2023-09-23T18:18:58.755221+0000 mgr.x (mgr.25562) 6 : cephadm [INF] [23/Sep/2023:18:18:58] ENGINE Serving on http://172.21.15.173:8765 2023-09-23T18:19:00.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:00 smithi173 bash[27694]: cephadm 2023-09-23T18:18:58.756204+0000 mgr.x (mgr.25562) 7 : cephadm [INF] [23/Sep/2023:18:18:58] ENGINE Bus STARTED 2023-09-23T18:19:00.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:00 smithi173 bash[27694]: cluster 2023-09-23T18:18:59.106047+0000 mgr.x (mgr.25562) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:00.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:00 smithi173 bash[27694]: audit 2023-09-23T18:18:59.313725+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:19:00.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:00 smithi173 bash[27694]: cluster 2023-09-23T18:18:59.313892+0000 mon.a (mon.0) 746 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-09-23T18:19:00.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:00 smithi173 bash[27694]: cluster 2023-09-23T18:18:59.411079+0000 mon.a (mon.0) 747 : cluster [DBG] mgrmap e22: x(active, since 2s) 2023-09-23T18:19:00.944 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:19:00 smithi098 bash[27618]: debug 2023-09-23T18:19:00.506+0000 7f72a22ed680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T18:19:00.944 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:19:00 smithi098 bash[27618]: [23/Sep/2023:18:19:00] ENGINE Bus STARTING 2023-09-23T18:19:00.945 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:19:00 smithi098 bash[27618]: CherryPy Checker: 2023-09-23T18:19:00.945 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:19:00 smithi098 bash[27618]: The Application mounted at '' has an empty config. 2023-09-23T18:19:00.945 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:19:00 smithi098 bash[27618]: [23/Sep/2023:18:19:00] ENGINE Serving on http://:::9283 2023-09-23T18:19:00.945 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:19:00 smithi098 bash[27618]: [23/Sep/2023:18:19:00] ENGINE Bus STARTED 2023-09-23T18:19:01.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:01 smithi098 bash[27349]: audit 2023-09-23T18:19:00.333053+0000 mon.a (mon.0) 748 : audit [DBG] from='client.? 172.21.15.98:0/3845079211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:01.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:01 smithi098 bash[27349]: cluster 2023-09-23T18:19:00.533051+0000 mon.a (mon.0) 749 : cluster [DBG] Standby manager daemon y started 2023-09-23T18:19:01.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:01 smithi098 bash[27349]: audit 2023-09-23T18:19:00.534254+0000 mon.c (mon.2) 128 : audit [DBG] from='mgr.? 172.21.15.98:0/1672589040' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T18:19:01.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:01 smithi098 bash[27349]: audit 2023-09-23T18:19:00.540454+0000 mon.c (mon.2) 129 : audit [DBG] from='mgr.? 172.21.15.98:0/1672589040' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:19:01.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:01 smithi098 bash[27349]: audit 2023-09-23T18:19:00.544792+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:01.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:01 smithi098 bash[27349]: audit 2023-09-23T18:19:00.545070+0000 mon.c (mon.2) 130 : audit [DBG] from='mgr.? 172.21.15.98:0/1672589040' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T18:19:01.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:01 smithi098 bash[27349]: audit 2023-09-23T18:19:00.545543+0000 mon.c (mon.2) 131 : audit [DBG] from='mgr.? 172.21.15.98:0/1672589040' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:19:01.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:01 smithi098 bash[33599]: audit 2023-09-23T18:19:00.333053+0000 mon.a (mon.0) 748 : audit [DBG] from='client.? 172.21.15.98:0/3845079211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:01.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:01 smithi098 bash[33599]: cluster 2023-09-23T18:19:00.533051+0000 mon.a (mon.0) 749 : cluster [DBG] Standby manager daemon y started 2023-09-23T18:19:01.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:01 smithi098 bash[33599]: audit 2023-09-23T18:19:00.534254+0000 mon.c (mon.2) 128 : audit [DBG] from='mgr.? 172.21.15.98:0/1672589040' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T18:19:01.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:01 smithi098 bash[33599]: audit 2023-09-23T18:19:00.540454+0000 mon.c (mon.2) 129 : audit [DBG] from='mgr.? 172.21.15.98:0/1672589040' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:19:01.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:01 smithi098 bash[33599]: audit 2023-09-23T18:19:00.544792+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:01.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:01 smithi098 bash[33599]: audit 2023-09-23T18:19:00.545070+0000 mon.c (mon.2) 130 : audit [DBG] from='mgr.? 172.21.15.98:0/1672589040' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T18:19:01.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:01 smithi098 bash[33599]: audit 2023-09-23T18:19:00.545543+0000 mon.c (mon.2) 131 : audit [DBG] from='mgr.? 172.21.15.98:0/1672589040' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:19:01.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:01 smithi173 bash[27694]: audit 2023-09-23T18:19:00.333053+0000 mon.a (mon.0) 748 : audit [DBG] from='client.? 172.21.15.98:0/3845079211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:01.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:01 smithi173 bash[27694]: cluster 2023-09-23T18:19:00.533051+0000 mon.a (mon.0) 749 : cluster [DBG] Standby manager daemon y started 2023-09-23T18:19:01.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:01 smithi173 bash[27694]: audit 2023-09-23T18:19:00.534254+0000 mon.c (mon.2) 128 : audit [DBG] from='mgr.? 172.21.15.98:0/1672589040' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T18:19:01.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:01 smithi173 bash[27694]: audit 2023-09-23T18:19:00.540454+0000 mon.c (mon.2) 129 : audit [DBG] from='mgr.? 172.21.15.98:0/1672589040' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:19:01.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:01 smithi173 bash[27694]: audit 2023-09-23T18:19:00.544792+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:01.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:01 smithi173 bash[27694]: audit 2023-09-23T18:19:00.545070+0000 mon.c (mon.2) 130 : audit [DBG] from='mgr.? 172.21.15.98:0/1672589040' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T18:19:01.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:01 smithi173 bash[27694]: audit 2023-09-23T18:19:00.545543+0000 mon.c (mon.2) 131 : audit [DBG] from='mgr.? 172.21.15.98:0/1672589040' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:19:02.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:02 smithi098 bash[27349]: cluster 2023-09-23T18:19:01.106684+0000 mgr.x (mgr.25562) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:02.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:02 smithi098 bash[27349]: cluster 2023-09-23T18:19:01.325538+0000 mon.a (mon.0) 751 : cluster [DBG] mgrmap e23: x(active, since 4s), standbys: y 2023-09-23T18:19:02.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:02 smithi098 bash[27349]: audit 2023-09-23T18:19:01.335367+0000 mon.b (mon.1) 57 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T18:19:02.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:02 smithi098 bash[33599]: cluster 2023-09-23T18:19:01.106684+0000 mgr.x (mgr.25562) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:02.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:02 smithi098 bash[33599]: cluster 2023-09-23T18:19:01.325538+0000 mon.a (mon.0) 751 : cluster [DBG] mgrmap e23: x(active, since 4s), standbys: y 2023-09-23T18:19:02.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:02 smithi098 bash[33599]: audit 2023-09-23T18:19:01.335367+0000 mon.b (mon.1) 57 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T18:19:02.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:02 smithi173 bash[27694]: cluster 2023-09-23T18:19:01.106684+0000 mgr.x (mgr.25562) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:02.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:02 smithi173 bash[27694]: cluster 2023-09-23T18:19:01.325538+0000 mon.a (mon.0) 751 : cluster [DBG] mgrmap e23: x(active, since 4s), standbys: y 2023-09-23T18:19:02.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:02 smithi173 bash[27694]: audit 2023-09-23T18:19:01.335367+0000 mon.b (mon.1) 57 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T18:19:03.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:03 smithi098 bash[33599]: audit 2023-09-23T18:19:02.832078+0000 mon.a (mon.0) 752 : audit [DBG] from='client.? 172.21.15.98:0/2687356752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:03.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:03 smithi098 bash[27349]: audit 2023-09-23T18:19:02.832078+0000 mon.a (mon.0) 752 : audit [DBG] from='client.? 172.21.15.98:0/2687356752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:03.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:03 smithi173 bash[27694]: audit 2023-09-23T18:19:02.832078+0000 mon.a (mon.0) 752 : audit [DBG] from='client.? 172.21.15.98:0/2687356752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:04.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:04 smithi098 bash[27349]: cluster 2023-09-23T18:19:03.107313+0000 mgr.x (mgr.25562) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:04.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:04 smithi098 bash[27349]: audit 2023-09-23T18:19:04.213748+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:04.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:04 smithi098 bash[33599]: cluster 2023-09-23T18:19:03.107313+0000 mgr.x (mgr.25562) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:04.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:04 smithi098 bash[33599]: audit 2023-09-23T18:19:04.213748+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:04.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:04 smithi173 bash[27694]: cluster 2023-09-23T18:19:03.107313+0000 mgr.x (mgr.25562) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:04.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:04 smithi173 bash[27694]: audit 2023-09-23T18:19:04.213748+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:05.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:05 smithi098 bash[33599]: audit 2023-09-23T18:19:05.286331+0000 mon.c (mon.2) 132 : audit [DBG] from='client.? 172.21.15.98:0/1284409405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:05.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:05 smithi098 bash[27349]: audit 2023-09-23T18:19:05.286331+0000 mon.c (mon.2) 132 : audit [DBG] from='client.? 172.21.15.98:0/1284409405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:05.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:05 smithi173 bash[27694]: audit 2023-09-23T18:19:05.286331+0000 mon.c (mon.2) 132 : audit [DBG] from='client.? 172.21.15.98:0/1284409405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:05.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:19:05 smithi173 bash[50615]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:19:05] "GET /metrics HTTP/1.1" 200 31029 "" "Prometheus/2.33.4" 2023-09-23T18:19:06.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:06 smithi098 bash[27349]: cluster 2023-09-23T18:19:05.108398+0000 mgr.x (mgr.25562) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 236 KiB/s wr, 31 op/s 2023-09-23T18:19:06.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:06 smithi098 bash[27349]: audit 2023-09-23T18:19:06.058637+0000 mon.a (mon.0) 754 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:06.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:06 smithi098 bash[27349]: audit 2023-09-23T18:19:06.067233+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:06.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:06 smithi098 bash[33599]: cluster 2023-09-23T18:19:05.108398+0000 mgr.x (mgr.25562) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 236 KiB/s wr, 31 op/s 2023-09-23T18:19:06.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:06 smithi098 bash[33599]: audit 2023-09-23T18:19:06.058637+0000 mon.a (mon.0) 754 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:06.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:06 smithi098 bash[33599]: audit 2023-09-23T18:19:06.067233+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:06.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:06 smithi173 bash[27694]: cluster 2023-09-23T18:19:05.108398+0000 mgr.x (mgr.25562) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 236 KiB/s wr, 31 op/s 2023-09-23T18:19:06.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:06 smithi173 bash[27694]: audit 2023-09-23T18:19:06.058637+0000 mon.a (mon.0) 754 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:06.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:06 smithi173 bash[27694]: audit 2023-09-23T18:19:06.067233+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:08.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:19:07 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:19:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:19:08.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:07 smithi098 bash[27349]: audit 2023-09-23T18:19:06.426452+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:08.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:07 smithi098 bash[27349]: audit 2023-09-23T18:19:06.440182+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:08.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:07 smithi098 bash[27349]: audit 2023-09-23T18:19:06.862515+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:08.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:07 smithi098 bash[27349]: audit 2023-09-23T18:19:06.871360+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:08.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:07 smithi098 bash[27349]: audit 2023-09-23T18:19:06.974828+0000 mon.a (mon.0) 760 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T18:19:08.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:07 smithi098 bash[33599]: audit 2023-09-23T18:19:06.426452+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:08.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:07 smithi098 bash[33599]: audit 2023-09-23T18:19:06.440182+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:08.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:07 smithi098 bash[33599]: audit 2023-09-23T18:19:06.862515+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:08.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:07 smithi098 bash[33599]: audit 2023-09-23T18:19:06.871360+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:08.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:07 smithi098 bash[33599]: audit 2023-09-23T18:19:06.974828+0000 mon.a (mon.0) 760 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T18:19:09.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:08 smithi173 bash[27694]: audit 2023-09-23T18:19:06.426452+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:09.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:08 smithi173 bash[27694]: audit 2023-09-23T18:19:06.440182+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:09.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:08 smithi173 bash[27694]: audit 2023-09-23T18:19:06.862515+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:09.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:08 smithi173 bash[27694]: audit 2023-09-23T18:19:06.871360+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:09.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:08 smithi173 bash[27694]: audit 2023-09-23T18:19:06.974828+0000 mon.a (mon.0) 760 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T18:19:10.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:09 smithi173 bash[27694]: audit 2023-09-23T18:19:06.979652+0000 mon.b (mon.1) 58 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:19:10.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:09 smithi173 bash[27694]: audit 2023-09-23T18:19:06.979989+0000 mon.b (mon.1) 59 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:19:10.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:09 smithi173 bash[27694]: audit 2023-09-23T18:19:06.980355+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:19:10.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:09 smithi173 bash[27694]: cluster 2023-09-23T18:19:07.109034+0000 mgr.x (mgr.25562) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 186 KiB/s wr, 24 op/s 2023-09-23T18:19:10.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:09 smithi173 bash[27694]: audit 2023-09-23T18:19:07.891406+0000 mon.a (mon.0) 761 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T18:19:10.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:09 smithi173 bash[27694]: audit 2023-09-23T18:19:07.900114+0000 mon.b (mon.1) 61 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T18:19:10.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:09 smithi173 bash[27694]: audit 2023-09-23T18:19:08.207862+0000 mon.c (mon.2) 133 : audit [DBG] from='client.? 172.21.15.98:0/1606025927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:10.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:09 smithi173 bash[27694]: audit 2023-09-23T18:19:08.812436+0000 mon.b (mon.1) 62 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T18:19:10.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:09 smithi173 bash[27694]: audit 2023-09-23T18:19:08.819932+0000 mon.b (mon.1) 63 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:19:10.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:09 smithi173 bash[27694]: audit 2023-09-23T18:19:08.820214+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:10.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:09 smithi173 bash[27694]: audit 2023-09-23T18:19:08.820239+0000 mon.b (mon.1) 64 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:19:10.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:09 smithi173 bash[27694]: audit 2023-09-23T18:19:08.820598+0000 mon.b (mon.1) 65 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:19:10.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:09 smithi173 bash[27694]: audit 2023-09-23T18:19:08.827902+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:19:10.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:09 smithi173 bash[27694]: audit 2023-09-23T18:19:08.828178+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:19:10.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:09 smithi173 bash[27694]: audit 2023-09-23T18:19:08.828471+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:19:10.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[33599]: audit 2023-09-23T18:19:06.979652+0000 mon.b (mon.1) 58 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:19:10.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[33599]: audit 2023-09-23T18:19:06.979989+0000 mon.b (mon.1) 59 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:19:10.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[33599]: audit 2023-09-23T18:19:06.980355+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:19:10.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[33599]: cluster 2023-09-23T18:19:07.109034+0000 mgr.x (mgr.25562) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 186 KiB/s wr, 24 op/s 2023-09-23T18:19:10.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[33599]: audit 2023-09-23T18:19:07.891406+0000 mon.a (mon.0) 761 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T18:19:10.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[33599]: audit 2023-09-23T18:19:07.900114+0000 mon.b (mon.1) 61 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T18:19:10.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[33599]: audit 2023-09-23T18:19:08.207862+0000 mon.c (mon.2) 133 : audit [DBG] from='client.? 172.21.15.98:0/1606025927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:10.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[33599]: audit 2023-09-23T18:19:08.812436+0000 mon.b (mon.1) 62 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T18:19:10.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[33599]: audit 2023-09-23T18:19:08.819932+0000 mon.b (mon.1) 63 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:19:10.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[33599]: audit 2023-09-23T18:19:08.820214+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:10.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[33599]: audit 2023-09-23T18:19:08.820239+0000 mon.b (mon.1) 64 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:19:10.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[33599]: audit 2023-09-23T18:19:08.820598+0000 mon.b (mon.1) 65 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:19:10.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[33599]: audit 2023-09-23T18:19:08.827902+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:19:10.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[33599]: audit 2023-09-23T18:19:08.828178+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:19:10.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[33599]: audit 2023-09-23T18:19:08.828471+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:19:10.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[27349]: audit 2023-09-23T18:19:06.979652+0000 mon.b (mon.1) 58 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:19:10.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[27349]: audit 2023-09-23T18:19:06.979989+0000 mon.b (mon.1) 59 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:19:10.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[27349]: audit 2023-09-23T18:19:06.980355+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:19:10.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[27349]: cluster 2023-09-23T18:19:07.109034+0000 mgr.x (mgr.25562) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 186 KiB/s wr, 24 op/s 2023-09-23T18:19:10.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[27349]: audit 2023-09-23T18:19:07.891406+0000 mon.a (mon.0) 761 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T18:19:10.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[27349]: audit 2023-09-23T18:19:07.900114+0000 mon.b (mon.1) 61 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T18:19:10.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[27349]: audit 2023-09-23T18:19:08.207862+0000 mon.c (mon.2) 133 : audit [DBG] from='client.? 172.21.15.98:0/1606025927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:10.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[27349]: audit 2023-09-23T18:19:08.812436+0000 mon.b (mon.1) 62 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T18:19:10.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[27349]: audit 2023-09-23T18:19:08.819932+0000 mon.b (mon.1) 63 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:19:10.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[27349]: audit 2023-09-23T18:19:08.820214+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:10.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[27349]: audit 2023-09-23T18:19:08.820239+0000 mon.b (mon.1) 64 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:19:10.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[27349]: audit 2023-09-23T18:19:08.820598+0000 mon.b (mon.1) 65 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:19:10.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[27349]: audit 2023-09-23T18:19:08.827902+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:19:10.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[27349]: audit 2023-09-23T18:19:08.828178+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:19:10.198 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:09 smithi098 bash[27349]: audit 2023-09-23T18:19:08.828471+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:19:11.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:10 smithi173 bash[27694]: audit 2023-09-23T18:19:08.823134+0000 mon.c (mon.2) 134 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T18:19:11.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:10 smithi173 bash[27694]: cluster 2023-09-23T18:19:09.110176+0000 mgr.x (mgr.25562) 13 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 215 KiB/s wr, 25 op/s 2023-09-23T18:19:11.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:10 smithi173 bash[27694]: audit 2023-09-23T18:19:09.721776+0000 mon.c (mon.2) 135 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T18:19:11.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:10 smithi173 bash[27694]: audit 2023-09-23T18:19:09.733415+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:11.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:10 smithi173 bash[27694]: audit 2023-09-23T18:19:10.652442+0000 mon.c (mon.2) 136 : audit [DBG] from='client.? 172.21.15.98:0/532693821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:11.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:10 smithi098 bash[27349]: audit 2023-09-23T18:19:08.823134+0000 mon.c (mon.2) 134 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T18:19:11.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:10 smithi098 bash[27349]: cluster 2023-09-23T18:19:09.110176+0000 mgr.x (mgr.25562) 13 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 215 KiB/s wr, 25 op/s 2023-09-23T18:19:11.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:10 smithi098 bash[27349]: audit 2023-09-23T18:19:09.721776+0000 mon.c (mon.2) 135 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T18:19:11.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:10 smithi098 bash[27349]: audit 2023-09-23T18:19:09.733415+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:11.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:10 smithi098 bash[27349]: audit 2023-09-23T18:19:10.652442+0000 mon.c (mon.2) 136 : audit [DBG] from='client.? 172.21.15.98:0/532693821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:11.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:10 smithi098 bash[33599]: audit 2023-09-23T18:19:08.823134+0000 mon.c (mon.2) 134 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T18:19:11.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:10 smithi098 bash[33599]: cluster 2023-09-23T18:19:09.110176+0000 mgr.x (mgr.25562) 13 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 215 KiB/s wr, 25 op/s 2023-09-23T18:19:11.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:10 smithi098 bash[33599]: audit 2023-09-23T18:19:09.721776+0000 mon.c (mon.2) 135 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T18:19:11.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:10 smithi098 bash[33599]: audit 2023-09-23T18:19:09.733415+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:11.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:10 smithi098 bash[33599]: audit 2023-09-23T18:19:10.652442+0000 mon.c (mon.2) 136 : audit [DBG] from='client.? 172.21.15.98:0/532693821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:12.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:11 smithi173 bash[27694]: cluster 2023-09-23T18:19:11.110899+0000 mgr.x (mgr.25562) 14 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 183 KiB/s wr, 21 op/s 2023-09-23T18:19:12.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:11 smithi098 bash[27349]: cluster 2023-09-23T18:19:11.110899+0000 mgr.x (mgr.25562) 14 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 183 KiB/s wr, 21 op/s 2023-09-23T18:19:12.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:11 smithi098 bash[33599]: cluster 2023-09-23T18:19:11.110899+0000 mgr.x (mgr.25562) 14 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 183 KiB/s wr, 21 op/s 2023-09-23T18:19:13.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:12 smithi173 bash[27694]: audit 2023-09-23T18:19:12.367141+0000 mon.b (mon.1) 69 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:19:13.161 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:12 smithi098 bash[27349]: audit 2023-09-23T18:19:12.367141+0000 mon.b (mon.1) 69 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:19:13.162 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:12 smithi098 bash[33599]: audit 2023-09-23T18:19:12.367141+0000 mon.b (mon.1) 69 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:19:14.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:13 smithi173 bash[27694]: audit 2023-09-23T18:19:13.106197+0000 mon.c (mon.2) 137 : audit [DBG] from='client.? 172.21.15.98:0/1350867670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:14.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:13 smithi173 bash[27694]: cluster 2023-09-23T18:19:13.111526+0000 mgr.x (mgr.25562) 15 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 179 KiB/s wr, 20 op/s 2023-09-23T18:19:14.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:13 smithi098 bash[33599]: audit 2023-09-23T18:19:13.106197+0000 mon.c (mon.2) 137 : audit [DBG] from='client.? 172.21.15.98:0/1350867670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:14.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:13 smithi098 bash[33599]: cluster 2023-09-23T18:19:13.111526+0000 mgr.x (mgr.25562) 15 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 179 KiB/s wr, 20 op/s 2023-09-23T18:19:14.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:13 smithi098 bash[27349]: audit 2023-09-23T18:19:13.106197+0000 mon.c (mon.2) 137 : audit [DBG] from='client.? 172.21.15.98:0/1350867670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:14.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:13 smithi098 bash[27349]: cluster 2023-09-23T18:19:13.111526+0000 mgr.x (mgr.25562) 15 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 179 KiB/s wr, 20 op/s 2023-09-23T18:19:15.736 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:19:15 smithi173 bash[50615]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:19:15] "GET /metrics HTTP/1.1" 200 33920 "" "Prometheus/2.33.4" 2023-09-23T18:19:16.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:16 smithi098 bash[27349]: cluster 2023-09-23T18:19:15.112793+0000 mgr.x (mgr.25562) 16 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-09-23T18:19:16.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:16 smithi098 bash[27349]: audit 2023-09-23T18:19:15.581271+0000 mon.c (mon.2) 138 : audit [DBG] from='client.? 172.21.15.98:0/2075610763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:16.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:16 smithi098 bash[33599]: cluster 2023-09-23T18:19:15.112793+0000 mgr.x (mgr.25562) 16 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-09-23T18:19:16.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:16 smithi098 bash[33599]: audit 2023-09-23T18:19:15.581271+0000 mon.c (mon.2) 138 : audit [DBG] from='client.? 172.21.15.98:0/2075610763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:16.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:16 smithi173 bash[27694]: cluster 2023-09-23T18:19:15.112793+0000 mgr.x (mgr.25562) 16 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-09-23T18:19:16.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:16 smithi173 bash[27694]: audit 2023-09-23T18:19:15.581271+0000 mon.c (mon.2) 138 : audit [DBG] from='client.? 172.21.15.98:0/2075610763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:17.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:17 smithi173 bash[27694]: audit 2023-09-23T18:19:16.254138+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:17.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:17 smithi173 bash[27694]: audit 2023-09-23T18:19:16.262668+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:17.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:17 smithi173 bash[27694]: audit 2023-09-23T18:19:16.265748+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:17.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:17 smithi173 bash[27694]: audit 2023-09-23T18:19:16.267829+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:17.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:17 smithi173 bash[27694]: audit 2023-09-23T18:19:16.269698+0000 mon.b (mon.1) 70 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:17.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:17 smithi173 bash[27694]: audit 2023-09-23T18:19:16.269745+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:17.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:17 smithi173 bash[27694]: audit 2023-09-23T18:19:16.271819+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:17.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:17 smithi173 bash[27694]: audit 2023-09-23T18:19:16.271920+0000 mon.b (mon.1) 71 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:17.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:17 smithi173 bash[27694]: audit 2023-09-23T18:19:16.273789+0000 mon.b (mon.1) 72 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:17.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:17 smithi173 bash[27694]: audit 2023-09-23T18:19:16.275754+0000 mon.b (mon.1) 73 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:17.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:17 smithi173 bash[27694]: cephadm 2023-09-23T18:19:16.277387+0000 mgr.x (mgr.25562) 17 : cephadm [INF] Adjusting osd_memory_target on smithi098 to 2524M 2023-09-23T18:19:17.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:17 smithi173 bash[27694]: audit 2023-09-23T18:19:16.279857+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:17.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:17 smithi098 bash[27349]: audit 2023-09-23T18:19:16.254138+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:17.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:17 smithi098 bash[27349]: audit 2023-09-23T18:19:16.262668+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:17.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:17 smithi098 bash[27349]: audit 2023-09-23T18:19:16.265748+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:17.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:17 smithi098 bash[27349]: audit 2023-09-23T18:19:16.267829+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:17.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:17 smithi098 bash[27349]: audit 2023-09-23T18:19:16.269698+0000 mon.b (mon.1) 70 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:17.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:17 smithi098 bash[27349]: audit 2023-09-23T18:19:16.269745+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:17.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:17 smithi098 bash[27349]: audit 2023-09-23T18:19:16.271819+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:17.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:17 smithi098 bash[27349]: audit 2023-09-23T18:19:16.271920+0000 mon.b (mon.1) 71 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:17.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:17 smithi098 bash[27349]: audit 2023-09-23T18:19:16.273789+0000 mon.b (mon.1) 72 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:17.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:17 smithi098 bash[27349]: audit 2023-09-23T18:19:16.275754+0000 mon.b (mon.1) 73 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:17.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:17 smithi098 bash[27349]: cephadm 2023-09-23T18:19:16.277387+0000 mgr.x (mgr.25562) 17 : cephadm [INF] Adjusting osd_memory_target on smithi098 to 2524M 2023-09-23T18:19:17.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:17 smithi098 bash[27349]: audit 2023-09-23T18:19:16.279857+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:17.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:17 smithi098 bash[33599]: audit 2023-09-23T18:19:16.254138+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:17.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:17 smithi098 bash[33599]: audit 2023-09-23T18:19:16.262668+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:17.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:17 smithi098 bash[33599]: audit 2023-09-23T18:19:16.265748+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:17.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:17 smithi098 bash[33599]: audit 2023-09-23T18:19:16.267829+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:17.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:17 smithi098 bash[33599]: audit 2023-09-23T18:19:16.269698+0000 mon.b (mon.1) 70 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:17.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:17 smithi098 bash[33599]: audit 2023-09-23T18:19:16.269745+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:17.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:17 smithi098 bash[33599]: audit 2023-09-23T18:19:16.271819+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:17.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:17 smithi098 bash[33599]: audit 2023-09-23T18:19:16.271920+0000 mon.b (mon.1) 71 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:17.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:17 smithi098 bash[33599]: audit 2023-09-23T18:19:16.273789+0000 mon.b (mon.1) 72 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:17.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:17 smithi098 bash[33599]: audit 2023-09-23T18:19:16.275754+0000 mon.b (mon.1) 73 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:17.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:17 smithi098 bash[33599]: cephadm 2023-09-23T18:19:16.277387+0000 mgr.x (mgr.25562) 17 : cephadm [INF] Adjusting osd_memory_target on smithi098 to 2524M 2023-09-23T18:19:17.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:17 smithi098 bash[33599]: audit 2023-09-23T18:19:16.279857+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:18.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:19:17 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:19:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:19:18.585 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:18 smithi173 bash[27694]: cluster 2023-09-23T18:19:17.113299+0000 mgr.x (mgr.25562) 18 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-23T18:19:18.585 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:18 smithi173 bash[27694]: audit 2023-09-23T18:19:18.018433+0000 mon.a (mon.0) 771 : audit [DBG] from='client.? 172.21.15.98:0/1044165782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:18.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:18 smithi098 bash[27349]: cluster 2023-09-23T18:19:17.113299+0000 mgr.x (mgr.25562) 18 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-23T18:19:18.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:18 smithi098 bash[27349]: audit 2023-09-23T18:19:18.018433+0000 mon.a (mon.0) 771 : audit [DBG] from='client.? 172.21.15.98:0/1044165782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:18.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:18 smithi098 bash[33599]: cluster 2023-09-23T18:19:17.113299+0000 mgr.x (mgr.25562) 18 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-23T18:19:18.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:18 smithi098 bash[33599]: audit 2023-09-23T18:19:18.018433+0000 mon.a (mon.0) 771 : audit [DBG] from='client.? 172.21.15.98:0/1044165782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:20.642 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:20 smithi098 systemd[1]: Stopping Ceph node-exporter.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T18:19:20.642 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[71568]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-node-exporter.a 2023-09-23T18:19:20.643 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: cluster 2023-09-23T18:19:19.114397+0000 mgr.x (mgr.25562) 19 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-23T18:19:20.643 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: audit 2023-09-23T18:19:19.337323+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.643 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: audit 2023-09-23T18:19:19.346771+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.643 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: audit 2023-09-23T18:19:19.351390+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:20.643 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: audit 2023-09-23T18:19:19.353452+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:20.643 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: audit 2023-09-23T18:19:19.355435+0000 mon.b (mon.1) 74 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:20.643 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: audit 2023-09-23T18:19:19.355539+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:20.643 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: audit 2023-09-23T18:19:19.357592+0000 mon.b (mon.1) 75 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:20.643 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: audit 2023-09-23T18:19:19.357765+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:20.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: audit 2023-09-23T18:19:19.359307+0000 mon.b (mon.1) 76 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:20.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: audit 2023-09-23T18:19:19.361610+0000 mon.b (mon.1) 77 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:20.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: cephadm 2023-09-23T18:19:19.363525+0000 mgr.x (mgr.25562) 20 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 2524M 2023-09-23T18:19:20.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: audit 2023-09-23T18:19:19.365504+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: audit 2023-09-23T18:19:19.373803+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:19:20.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: audit 2023-09-23T18:19:19.374903+0000 mon.b (mon.1) 79 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:19:20.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: cephadm 2023-09-23T18:19:19.376655+0000 mgr.x (mgr.25562) 21 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.conf 2023-09-23T18:19:20.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: cephadm 2023-09-23T18:19:19.377058+0000 mgr.x (mgr.25562) 22 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-09-23T18:19:20.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: audit 2023-09-23T18:19:20.041688+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: audit 2023-09-23T18:19:20.046736+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: audit 2023-09-23T18:19:20.051483+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: audit 2023-09-23T18:19:20.056251+0000 mon.a (mon.0) 782 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.645 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: audit 2023-09-23T18:19:20.063115+0000 mon.a (mon.0) 783 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.645 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: audit 2023-09-23T18:19:20.140682+0000 mon.a (mon.0) 784 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.645 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: audit 2023-09-23T18:19:20.148065+0000 mon.a (mon.0) 785 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.645 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: audit 2023-09-23T18:19:20.156513+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.645 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[33599]: audit 2023-09-23T18:19:20.164594+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.645 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: cluster 2023-09-23T18:19:19.114397+0000 mgr.x (mgr.25562) 19 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-23T18:19:20.645 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: audit 2023-09-23T18:19:19.337323+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.645 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: audit 2023-09-23T18:19:19.346771+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.646 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: audit 2023-09-23T18:19:19.351390+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:20.646 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: audit 2023-09-23T18:19:19.353452+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:20.646 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: audit 2023-09-23T18:19:19.355435+0000 mon.b (mon.1) 74 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:20.646 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: audit 2023-09-23T18:19:19.355539+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:20.646 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: audit 2023-09-23T18:19:19.357592+0000 mon.b (mon.1) 75 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:20.646 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: audit 2023-09-23T18:19:19.357765+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:20.646 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: audit 2023-09-23T18:19:19.359307+0000 mon.b (mon.1) 76 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:20.646 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: audit 2023-09-23T18:19:19.361610+0000 mon.b (mon.1) 77 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:20.646 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: cephadm 2023-09-23T18:19:19.363525+0000 mgr.x (mgr.25562) 20 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 2524M 2023-09-23T18:19:20.646 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: audit 2023-09-23T18:19:19.365504+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.646 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: audit 2023-09-23T18:19:19.373803+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:19:20.646 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: audit 2023-09-23T18:19:19.374903+0000 mon.b (mon.1) 79 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:19:20.647 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: cephadm 2023-09-23T18:19:19.376655+0000 mgr.x (mgr.25562) 21 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.conf 2023-09-23T18:19:20.647 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: cephadm 2023-09-23T18:19:19.377058+0000 mgr.x (mgr.25562) 22 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-09-23T18:19:20.647 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: audit 2023-09-23T18:19:20.041688+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.647 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: audit 2023-09-23T18:19:20.046736+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.647 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: audit 2023-09-23T18:19:20.051483+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.647 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: audit 2023-09-23T18:19:20.056251+0000 mon.a (mon.0) 782 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.647 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: audit 2023-09-23T18:19:20.063115+0000 mon.a (mon.0) 783 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.647 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: audit 2023-09-23T18:19:20.140682+0000 mon.a (mon.0) 784 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.647 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: audit 2023-09-23T18:19:20.148065+0000 mon.a (mon.0) 785 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.647 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: audit 2023-09-23T18:19:20.156513+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.647 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:20 smithi098 bash[27349]: audit 2023-09-23T18:19:20.164594+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: cluster 2023-09-23T18:19:19.114397+0000 mgr.x (mgr.25562) 19 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-23T18:19:20.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: audit 2023-09-23T18:19:19.337323+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: audit 2023-09-23T18:19:19.346771+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: audit 2023-09-23T18:19:19.351390+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:20.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: audit 2023-09-23T18:19:19.353452+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:20.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: audit 2023-09-23T18:19:19.355435+0000 mon.b (mon.1) 74 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:20.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: audit 2023-09-23T18:19:19.355539+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:20.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: audit 2023-09-23T18:19:19.357592+0000 mon.b (mon.1) 75 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:20.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: audit 2023-09-23T18:19:19.357765+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:20.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: audit 2023-09-23T18:19:19.359307+0000 mon.b (mon.1) 76 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:20.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: audit 2023-09-23T18:19:19.361610+0000 mon.b (mon.1) 77 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T18:19:20.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: cephadm 2023-09-23T18:19:19.363525+0000 mgr.x (mgr.25562) 20 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 2524M 2023-09-23T18:19:20.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: audit 2023-09-23T18:19:19.365504+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: audit 2023-09-23T18:19:19.373803+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:19:20.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: audit 2023-09-23T18:19:19.374903+0000 mon.b (mon.1) 79 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:19:20.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: cephadm 2023-09-23T18:19:19.376655+0000 mgr.x (mgr.25562) 21 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.conf 2023-09-23T18:19:20.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: cephadm 2023-09-23T18:19:19.377058+0000 mgr.x (mgr.25562) 22 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-09-23T18:19:20.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: audit 2023-09-23T18:19:20.041688+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: audit 2023-09-23T18:19:20.046736+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: audit 2023-09-23T18:19:20.051483+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: audit 2023-09-23T18:19:20.056251+0000 mon.a (mon.0) 782 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.780 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: audit 2023-09-23T18:19:20.063115+0000 mon.a (mon.0) 783 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.780 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: audit 2023-09-23T18:19:20.140682+0000 mon.a (mon.0) 784 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.780 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: audit 2023-09-23T18:19:20.148065+0000 mon.a (mon.0) 785 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.780 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: audit 2023-09-23T18:19:20.156513+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:20.780 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:20 smithi173 bash[27694]: audit 2023-09-23T18:19:20.164594+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:21.642 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:21 smithi098 bash[71607]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-node-exporter-a 2023-09-23T18:19:21.642 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:21 smithi098 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T18:19:21.642 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:21 smithi098 bash[71671]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-node-exporter-a 2023-09-23T18:19:21.642 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:21 smithi098 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-09-23T18:19:21.642 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:21 smithi098 systemd[1]: Stopped Ceph node-exporter.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:19:21.642 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:21 smithi098 systemd[1]: Started Ceph node-exporter.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:19:21.642 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:21 smithi098 bash[27349]: cephadm 2023-09-23T18:19:19.541502+0000 mgr.x (mgr.25562) 23 : cephadm [INF] Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:19:21.643 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:21 smithi098 bash[27349]: cephadm 2023-09-23T18:19:19.541805+0000 mgr.x (mgr.25562) 24 : cephadm [INF] Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:19:21.643 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:21 smithi098 bash[27349]: cephadm 2023-09-23T18:19:19.707202+0000 mgr.x (mgr.25562) 25 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:19:21.643 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:21 smithi098 bash[27349]: cephadm 2023-09-23T18:19:19.707761+0000 mgr.x (mgr.25562) 26 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:19:21.643 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:21 smithi098 bash[27349]: cephadm 2023-09-23T18:19:19.872285+0000 mgr.x (mgr.25562) 27 : cephadm [INF] Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:19:21.643 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:21 smithi098 bash[27349]: cephadm 2023-09-23T18:19:19.874045+0000 mgr.x (mgr.25562) 28 : cephadm [INF] Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:19:21.643 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:21 smithi098 bash[27349]: cephadm 2023-09-23T18:19:20.171454+0000 mgr.x (mgr.25562) 29 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T18:19:21.643 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:21 smithi098 bash[27349]: cephadm 2023-09-23T18:19:20.172271+0000 mgr.x (mgr.25562) 30 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi098 2023-09-23T18:19:21.643 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:21 smithi098 bash[27349]: audit 2023-09-23T18:19:20.474983+0000 mon.a (mon.0) 788 : audit [DBG] from='client.? 172.21.15.98:0/2448473679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:21.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:21 smithi098 bash[33599]: cephadm 2023-09-23T18:19:19.541502+0000 mgr.x (mgr.25562) 23 : cephadm [INF] Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:19:21.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:21 smithi098 bash[33599]: cephadm 2023-09-23T18:19:19.541805+0000 mgr.x (mgr.25562) 24 : cephadm [INF] Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:19:21.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:21 smithi098 bash[33599]: cephadm 2023-09-23T18:19:19.707202+0000 mgr.x (mgr.25562) 25 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:19:21.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:21 smithi098 bash[33599]: cephadm 2023-09-23T18:19:19.707761+0000 mgr.x (mgr.25562) 26 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:19:21.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:21 smithi098 bash[33599]: cephadm 2023-09-23T18:19:19.872285+0000 mgr.x (mgr.25562) 27 : cephadm [INF] Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:19:21.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:21 smithi098 bash[33599]: cephadm 2023-09-23T18:19:19.874045+0000 mgr.x (mgr.25562) 28 : cephadm [INF] Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:19:21.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:21 smithi098 bash[33599]: cephadm 2023-09-23T18:19:20.171454+0000 mgr.x (mgr.25562) 29 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T18:19:21.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:21 smithi098 bash[33599]: cephadm 2023-09-23T18:19:20.172271+0000 mgr.x (mgr.25562) 30 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi098 2023-09-23T18:19:21.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:21 smithi098 bash[33599]: audit 2023-09-23T18:19:20.474983+0000 mon.a (mon.0) 788 : audit [DBG] from='client.? 172.21.15.98:0/2448473679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:21.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:21 smithi173 bash[27694]: cephadm 2023-09-23T18:19:19.541502+0000 mgr.x (mgr.25562) 23 : cephadm [INF] Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:19:21.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:21 smithi173 bash[27694]: cephadm 2023-09-23T18:19:19.541805+0000 mgr.x (mgr.25562) 24 : cephadm [INF] Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:19:21.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:21 smithi173 bash[27694]: cephadm 2023-09-23T18:19:19.707202+0000 mgr.x (mgr.25562) 25 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:19:21.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:21 smithi173 bash[27694]: cephadm 2023-09-23T18:19:19.707761+0000 mgr.x (mgr.25562) 26 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:19:21.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:21 smithi173 bash[27694]: cephadm 2023-09-23T18:19:19.872285+0000 mgr.x (mgr.25562) 27 : cephadm [INF] Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:19:21.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:21 smithi173 bash[27694]: cephadm 2023-09-23T18:19:19.874045+0000 mgr.x (mgr.25562) 28 : cephadm [INF] Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:19:21.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:21 smithi173 bash[27694]: cephadm 2023-09-23T18:19:20.171454+0000 mgr.x (mgr.25562) 29 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T18:19:21.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:21 smithi173 bash[27694]: cephadm 2023-09-23T18:19:20.172271+0000 mgr.x (mgr.25562) 30 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi098 2023-09-23T18:19:21.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:21 smithi173 bash[27694]: audit 2023-09-23T18:19:20.474983+0000 mon.a (mon.0) 788 : audit [DBG] from='client.? 172.21.15.98:0/2448473679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:22.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:22 smithi098 bash[27349]: cluster 2023-09-23T18:19:21.115081+0000 mgr.x (mgr.25562) 31 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-09-23T18:19:22.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:22 smithi098 bash[27349]: audit 2023-09-23T18:19:21.617335+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:22.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:22 smithi098 bash[27349]: audit 2023-09-23T18:19:21.626784+0000 mon.a (mon.0) 790 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:22.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:22 smithi098 bash[33599]: cluster 2023-09-23T18:19:21.115081+0000 mgr.x (mgr.25562) 31 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-09-23T18:19:22.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:22 smithi098 bash[33599]: audit 2023-09-23T18:19:21.617335+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:22.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:22 smithi098 bash[33599]: audit 2023-09-23T18:19:21.626784+0000 mon.a (mon.0) 790 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:22.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:22 smithi173 bash[27694]: cluster 2023-09-23T18:19:21.115081+0000 mgr.x (mgr.25562) 31 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-09-23T18:19:22.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:22 smithi173 bash[27694]: audit 2023-09-23T18:19:21.617335+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:22.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:22 smithi173 bash[27694]: audit 2023-09-23T18:19:21.626784+0000 mon.a (mon.0) 790 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:22.777 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:22 smithi173 bash[48419]: ts=2023-09-23T18:19:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.98:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:19:23.195 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.021Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-09-23T18:19:23.195 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.021Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-09-23T18:19:23.195 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.021Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-09-23T18:19:23.195 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.021Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T18:19:23.195 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.022Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-09-23T18:19:23.195 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.022Z caller=node_exporter.go:115 level=info collector=arp 2023-09-23T18:19:23.196 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.022Z caller=node_exporter.go:115 level=info collector=bcache 2023-09-23T18:19:23.196 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.022Z caller=node_exporter.go:115 level=info collector=bonding 2023-09-23T18:19:23.196 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.022Z caller=node_exporter.go:115 level=info collector=btrfs 2023-09-23T18:19:23.196 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.022Z caller=node_exporter.go:115 level=info collector=conntrack 2023-09-23T18:19:23.196 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.022Z caller=node_exporter.go:115 level=info collector=cpu 2023-09-23T18:19:23.196 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.022Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-09-23T18:19:23.196 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.022Z caller=node_exporter.go:115 level=info collector=diskstats 2023-09-23T18:19:23.196 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=dmi 2023-09-23T18:19:23.196 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=edac 2023-09-23T18:19:23.196 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=entropy 2023-09-23T18:19:23.196 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-09-23T18:19:23.197 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=filefd 2023-09-23T18:19:23.197 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=filesystem 2023-09-23T18:19:23.197 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=hwmon 2023-09-23T18:19:23.197 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=infiniband 2023-09-23T18:19:23.197 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=ipvs 2023-09-23T18:19:23.197 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=loadavg 2023-09-23T18:19:23.197 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=mdadm 2023-09-23T18:19:23.197 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=meminfo 2023-09-23T18:19:23.197 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=netclass 2023-09-23T18:19:23.197 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=netdev 2023-09-23T18:19:23.197 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=netstat 2023-09-23T18:19:23.198 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=nfs 2023-09-23T18:19:23.198 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=nfsd 2023-09-23T18:19:23.198 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=nvme 2023-09-23T18:19:23.198 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=os 2023-09-23T18:19:23.198 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-09-23T18:19:23.198 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=pressure 2023-09-23T18:19:23.198 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=rapl 2023-09-23T18:19:23.198 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=schedstat 2023-09-23T18:19:23.198 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=sockstat 2023-09-23T18:19:23.198 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=softnet 2023-09-23T18:19:23.198 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=stat 2023-09-23T18:19:23.199 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=tapestats 2023-09-23T18:19:23.199 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=textfile 2023-09-23T18:19:23.199 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-09-23T18:19:23.199 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=time 2023-09-23T18:19:23.199 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-09-23T18:19:23.199 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=uname 2023-09-23T18:19:23.199 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=vmstat 2023-09-23T18:19:23.199 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=xfs 2023-09-23T18:19:23.199 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:115 level=info collector=zfs 2023-09-23T18:19:23.199 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-09-23T18:19:23.200 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[71699]: ts=2023-09-23T18:19:23.023Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-09-23T18:19:23.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[33599]: cephadm 2023-09-23T18:19:21.641874+0000 mgr.x (mgr.25562) 32 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T18:19:23.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[33599]: cephadm 2023-09-23T18:19:21.660846+0000 mgr.x (mgr.25562) 33 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi098 2023-09-23T18:19:23.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[33599]: audit 2023-09-23T18:19:22.904843+0000 mon.a (mon.0) 791 : audit [DBG] from='client.? 172.21.15.98:0/2811275482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:23.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[27349]: cephadm 2023-09-23T18:19:21.641874+0000 mgr.x (mgr.25562) 32 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T18:19:23.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[27349]: cephadm 2023-09-23T18:19:21.660846+0000 mgr.x (mgr.25562) 33 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi098 2023-09-23T18:19:23.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:23 smithi098 bash[27349]: audit 2023-09-23T18:19:22.904843+0000 mon.a (mon.0) 791 : audit [DBG] from='client.? 172.21.15.98:0/2811275482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:23.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:23 smithi173 bash[27694]: cephadm 2023-09-23T18:19:21.641874+0000 mgr.x (mgr.25562) 32 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T18:19:23.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:23 smithi173 bash[27694]: cephadm 2023-09-23T18:19:21.660846+0000 mgr.x (mgr.25562) 33 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi098 2023-09-23T18:19:23.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:23 smithi173 bash[27694]: audit 2023-09-23T18:19:22.904843+0000 mon.a (mon.0) 791 : audit [DBG] from='client.? 172.21.15.98:0/2811275482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:24.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:24 smithi098 bash[27349]: cluster 2023-09-23T18:19:23.115737+0000 mgr.x (mgr.25562) 34 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-09-23T18:19:24.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:24 smithi098 bash[33599]: cluster 2023-09-23T18:19:23.115737+0000 mgr.x (mgr.25562) 34 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-09-23T18:19:24.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:24 smithi173 bash[27694]: cluster 2023-09-23T18:19:23.115737+0000 mgr.x (mgr.25562) 34 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-09-23T18:19:25.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:25 smithi098 bash[27349]: audit 2023-09-23T18:19:25.340360+0000 mon.a (mon.0) 792 : audit [DBG] from='client.? 172.21.15.98:0/1484235030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:25.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:25 smithi098 bash[33599]: audit 2023-09-23T18:19:25.340360+0000 mon.a (mon.0) 792 : audit [DBG] from='client.? 172.21.15.98:0/1484235030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:25.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:25 smithi173 bash[27694]: audit 2023-09-23T18:19:25.340360+0000 mon.a (mon.0) 792 : audit [DBG] from='client.? 172.21.15.98:0/1484235030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:25.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:19:25 smithi173 bash[50615]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:19:25] "GET /metrics HTTP/1.1" 200 33920 "" "Prometheus/2.33.4" 2023-09-23T18:19:26.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:26 smithi098 bash[27349]: cluster 2023-09-23T18:19:25.116821+0000 mgr.x (mgr.25562) 35 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-09-23T18:19:26.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:26 smithi098 bash[33599]: cluster 2023-09-23T18:19:25.116821+0000 mgr.x (mgr.25562) 35 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-09-23T18:19:26.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:26 smithi173 bash[27694]: cluster 2023-09-23T18:19:25.116821+0000 mgr.x (mgr.25562) 35 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-09-23T18:19:27.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:27 smithi098 bash[27349]: audit 2023-09-23T18:19:27.367495+0000 mon.b (mon.1) 80 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:19:27.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:27 smithi098 bash[33599]: audit 2023-09-23T18:19:27.367495+0000 mon.b (mon.1) 80 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:19:27.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:27 smithi173 bash[27694]: audit 2023-09-23T18:19:27.367495+0000 mon.b (mon.1) 80 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:19:28.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:19:27 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:19:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:19:28.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:28 smithi098 bash[33599]: cluster 2023-09-23T18:19:27.117509+0000 mgr.x (mgr.25562) 36 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:28.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:28 smithi098 bash[33599]: audit 2023-09-23T18:19:27.786755+0000 mon.a (mon.0) 793 : audit [DBG] from='client.? 172.21.15.98:0/3834745659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:28.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:28 smithi098 bash[27349]: cluster 2023-09-23T18:19:27.117509+0000 mgr.x (mgr.25562) 36 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:28.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:28 smithi098 bash[27349]: audit 2023-09-23T18:19:27.786755+0000 mon.a (mon.0) 793 : audit [DBG] from='client.? 172.21.15.98:0/3834745659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:28.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:28 smithi173 bash[27694]: cluster 2023-09-23T18:19:27.117509+0000 mgr.x (mgr.25562) 36 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:28.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:28 smithi173 bash[27694]: audit 2023-09-23T18:19:27.786755+0000 mon.a (mon.0) 793 : audit [DBG] from='client.? 172.21.15.98:0/3834745659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:30.585 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:30 smithi098 bash[27349]: cluster 2023-09-23T18:19:29.118631+0000 mgr.x (mgr.25562) 37 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:30.585 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:30 smithi098 bash[27349]: audit 2023-09-23T18:19:30.224823+0000 mon.a (mon.0) 794 : audit [DBG] from='client.? 172.21.15.98:0/3567967975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:30.586 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:30 smithi098 bash[33599]: cluster 2023-09-23T18:19:29.118631+0000 mgr.x (mgr.25562) 37 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:30.586 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:30 smithi098 bash[33599]: audit 2023-09-23T18:19:30.224823+0000 mon.a (mon.0) 794 : audit [DBG] from='client.? 172.21.15.98:0/3567967975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:30.586 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:19:30 smithi098 systemd[1]: Stopping Ceph alertmanager.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T18:19:30.586 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:19:30 smithi098 bash[72062]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-alertmanager.a 2023-09-23T18:19:30.586 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:19:30 smithi098 bash[56623]: level=info ts=2023-09-23T18:19:30.491Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-09-23T18:19:30.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:30 smithi173 bash[27694]: cluster 2023-09-23T18:19:29.118631+0000 mgr.x (mgr.25562) 37 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:30.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:30 smithi173 bash[27694]: audit 2023-09-23T18:19:30.224823+0000 mon.a (mon.0) 794 : audit [DBG] from='client.? 172.21.15.98:0/3567967975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:31.482 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:19:31 smithi098 bash[72097]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-alertmanager-a 2023-09-23T18:19:31.482 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:19:31 smithi098 bash[72162]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-alertmanager-a 2023-09-23T18:19:31.482 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:19:31 smithi098 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T18:19:31.482 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:19:31 smithi098 systemd[1]: Stopped Ceph alertmanager.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:19:31.482 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:19:31 smithi098 systemd[1]: Started Ceph alertmanager.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:19:32.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:32 smithi098 bash[27349]: cluster 2023-09-23T18:19:31.119356+0000 mgr.x (mgr.25562) 38 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:32.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:32 smithi098 bash[27349]: audit 2023-09-23T18:19:31.460179+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:32.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:32 smithi098 bash[27349]: audit 2023-09-23T18:19:31.467854+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:32.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:32 smithi098 bash[27349]: cephadm 2023-09-23T18:19:31.504828+0000 mgr.x (mgr.25562) 39 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T18:19:32.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:32 smithi098 bash[33599]: cluster 2023-09-23T18:19:31.119356+0000 mgr.x (mgr.25562) 38 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:32.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:32 smithi098 bash[33599]: audit 2023-09-23T18:19:31.460179+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:32.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:32 smithi098 bash[33599]: audit 2023-09-23T18:19:31.467854+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:32.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:32 smithi098 bash[33599]: cephadm 2023-09-23T18:19:31.504828+0000 mgr.x (mgr.25562) 39 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T18:19:32.695 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:19:32 smithi098 bash[72190]: level=info ts=2023-09-23T18:19:32.429Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-09-23T18:19:32.695 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:19:32 smithi098 bash[72190]: level=info ts=2023-09-23T18:19:32.429Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-09-23T18:19:32.695 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:19:32 smithi098 bash[72190]: level=info ts=2023-09-23T18:19:32.432Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.98 port=9094 2023-09-23T18:19:32.695 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:19:32 smithi098 bash[72190]: level=info ts=2023-09-23T18:19:32.433Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T18:19:32.696 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:19:32 smithi098 bash[72190]: level=info ts=2023-09-23T18:19:32.480Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T18:19:32.696 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:19:32 smithi098 bash[72190]: level=info ts=2023-09-23T18:19:32.480Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T18:19:32.696 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:19:32 smithi098 bash[72190]: level=info ts=2023-09-23T18:19:32.484Z caller=main.go:518 msg=Listening address=:9093 2023-09-23T18:19:32.696 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:19:32 smithi098 bash[72190]: level=info ts=2023-09-23T18:19:32.484Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-09-23T18:19:32.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:32 smithi173 bash[27694]: cluster 2023-09-23T18:19:31.119356+0000 mgr.x (mgr.25562) 38 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:32.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:32 smithi173 bash[27694]: audit 2023-09-23T18:19:31.460179+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:32.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:32 smithi173 bash[27694]: audit 2023-09-23T18:19:31.467854+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:32.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:32 smithi173 bash[27694]: cephadm 2023-09-23T18:19:31.504828+0000 mgr.x (mgr.25562) 39 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T18:19:32.777 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:32 smithi173 bash[48419]: ts=2023-09-23T18:19:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.98:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:19:33.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:33 smithi173 bash[27694]: cephadm 2023-09-23T18:19:31.750066+0000 mgr.x (mgr.25562) 40 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi173 2023-09-23T18:19:33.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:33 smithi173 bash[27694]: audit 2023-09-23T18:19:32.683179+0000 mon.a (mon.0) 797 : audit [DBG] from='client.? 172.21.15.98:0/586088603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:33.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:33 smithi098 bash[33599]: cephadm 2023-09-23T18:19:31.750066+0000 mgr.x (mgr.25562) 40 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi173 2023-09-23T18:19:33.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:33 smithi098 bash[33599]: audit 2023-09-23T18:19:32.683179+0000 mon.a (mon.0) 797 : audit [DBG] from='client.? 172.21.15.98:0/586088603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:33.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:33 smithi098 bash[27349]: cephadm 2023-09-23T18:19:31.750066+0000 mgr.x (mgr.25562) 40 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi173 2023-09-23T18:19:33.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:33 smithi098 bash[27349]: audit 2023-09-23T18:19:32.683179+0000 mon.a (mon.0) 797 : audit [DBG] from='client.? 172.21.15.98:0/586088603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:34.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:34 smithi098 bash[27349]: cluster 2023-09-23T18:19:33.119886+0000 mgr.x (mgr.25562) 41 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:34.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:34 smithi098 bash[33599]: cluster 2023-09-23T18:19:33.119886+0000 mgr.x (mgr.25562) 41 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:34.695 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:19:34 smithi098 bash[72190]: level=info ts=2023-09-23T18:19:34.434Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000892774s 2023-09-23T18:19:34.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:34 smithi173 bash[27694]: cluster 2023-09-23T18:19:33.119886+0000 mgr.x (mgr.25562) 41 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:35.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:35 smithi173 bash[27694]: audit 2023-09-23T18:19:35.124885+0000 mon.c (mon.2) 139 : audit [DBG] from='client.? 172.21.15.98:0/3291981823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:35.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:19:35 smithi173 bash[50615]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:19:35] "GET /metrics HTTP/1.1" 200 33917 "" "Prometheus/2.33.4" 2023-09-23T18:19:35.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:35 smithi098 bash[27349]: audit 2023-09-23T18:19:35.124885+0000 mon.c (mon.2) 139 : audit [DBG] from='client.? 172.21.15.98:0/3291981823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:35.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:35 smithi098 bash[33599]: audit 2023-09-23T18:19:35.124885+0000 mon.c (mon.2) 139 : audit [DBG] from='client.? 172.21.15.98:0/3291981823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:36.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:36 smithi173 bash[27694]: cluster 2023-09-23T18:19:35.120806+0000 mgr.x (mgr.25562) 42 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:36.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:36 smithi098 bash[27349]: cluster 2023-09-23T18:19:35.120806+0000 mgr.x (mgr.25562) 42 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:36.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:36 smithi098 bash[33599]: cluster 2023-09-23T18:19:35.120806+0000 mgr.x (mgr.25562) 42 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:38.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:19:37 smithi098 bash[27618]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:19:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:19:38.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:38 smithi173 bash[27694]: cluster 2023-09-23T18:19:37.121473+0000 mgr.x (mgr.25562) 43 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:38.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:38 smithi173 bash[27694]: audit 2023-09-23T18:19:37.574804+0000 mon.c (mon.2) 140 : audit [DBG] from='client.? 172.21.15.98:0/2164050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:38.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:38 smithi098 bash[27349]: cluster 2023-09-23T18:19:37.121473+0000 mgr.x (mgr.25562) 43 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:38.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:38 smithi098 bash[27349]: audit 2023-09-23T18:19:37.574804+0000 mon.c (mon.2) 140 : audit [DBG] from='client.? 172.21.15.98:0/2164050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:38.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:38 smithi098 bash[33599]: cluster 2023-09-23T18:19:37.121473+0000 mgr.x (mgr.25562) 43 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:38.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:38 smithi098 bash[33599]: audit 2023-09-23T18:19:37.574804+0000 mon.c (mon.2) 140 : audit [DBG] from='client.? 172.21.15.98:0/2164050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:40.706 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:40 smithi173 bash[27694]: cluster 2023-09-23T18:19:39.122520+0000 mgr.x (mgr.25562) 44 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:40.706 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:40 smithi173 bash[27694]: audit 2023-09-23T18:19:40.003533+0000 mon.a (mon.0) 798 : audit [DBG] from='client.? 172.21.15.98:0/3469934879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:40.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:40 smithi098 bash[27349]: cluster 2023-09-23T18:19:39.122520+0000 mgr.x (mgr.25562) 44 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:40.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:40 smithi098 bash[27349]: audit 2023-09-23T18:19:40.003533+0000 mon.a (mon.0) 798 : audit [DBG] from='client.? 172.21.15.98:0/3469934879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:40.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:40 smithi098 bash[33599]: cluster 2023-09-23T18:19:39.122520+0000 mgr.x (mgr.25562) 44 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:40.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:40 smithi098 bash[33599]: audit 2023-09-23T18:19:40.003533+0000 mon.a (mon.0) 798 : audit [DBG] from='client.? 172.21.15.98:0/3469934879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:42.507 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:42 smithi173 bash[48419]: ts=2023-09-23T18:19:42.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.98:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:19:42.694 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 18:19:42 smithi098 bash[72190]: level=info ts=2023-09-23T18:19:42.437Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003418253s 2023-09-23T18:19:42.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:42 smithi098 bash[27349]: cluster 2023-09-23T18:19:41.123189+0000 mgr.x (mgr.25562) 45 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:42.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:42 smithi098 bash[27349]: audit 2023-09-23T18:19:42.367477+0000 mon.b (mon.1) 81 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:19:42.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:42 smithi098 bash[27349]: audit 2023-09-23T18:19:42.434598+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.98:0/4022246498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:42.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:42 smithi098 bash[33599]: cluster 2023-09-23T18:19:41.123189+0000 mgr.x (mgr.25562) 45 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:42.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:42 smithi098 bash[33599]: audit 2023-09-23T18:19:42.367477+0000 mon.b (mon.1) 81 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:19:42.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:42 smithi098 bash[33599]: audit 2023-09-23T18:19:42.434598+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.98:0/4022246498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:42.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:42 smithi173 bash[27694]: cluster 2023-09-23T18:19:41.123189+0000 mgr.x (mgr.25562) 45 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:42.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:42 smithi173 bash[27694]: audit 2023-09-23T18:19:42.367477+0000 mon.b (mon.1) 81 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:19:42.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:42 smithi173 bash[27694]: audit 2023-09-23T18:19:42.434598+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.98:0/4022246498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:43.527 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:43 smithi173 systemd[1]: Stopping Ceph prometheus.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T18:19:43.527 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:43 smithi173 bash[52892]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-prometheus.a 2023-09-23T18:19:43.527 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:43 smithi173 bash[48419]: ts=2023-09-23T18:19:43.305Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T18:19:43.528 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:43 smithi173 bash[48419]: ts=2023-09-23T18:19:43.306Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-09-23T18:19:43.528 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:43 smithi173 bash[48419]: ts=2023-09-23T18:19:43.306Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-09-23T18:19:43.528 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:43 smithi173 bash[48419]: ts=2023-09-23T18:19:43.306Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-09-23T18:19:43.528 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:43 smithi173 bash[48419]: ts=2023-09-23T18:19:43.306Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-09-23T18:19:43.528 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:43 smithi173 bash[48419]: ts=2023-09-23T18:19:43.306Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-09-23T18:19:43.528 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:43 smithi173 bash[48419]: ts=2023-09-23T18:19:43.306Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-09-23T18:19:43.528 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:43 smithi173 bash[48419]: ts=2023-09-23T18:19:43.306Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T18:19:43.528 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:43 smithi173 bash[48419]: ts=2023-09-23T18:19:43.306Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T18:19:43.528 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:43 smithi173 bash[48419]: ts=2023-09-23T18:19:43.307Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T18:19:43.528 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:43 smithi173 bash[48419]: ts=2023-09-23T18:19:43.307Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-09-23T18:19:43.528 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:43 smithi173 bash[48419]: ts=2023-09-23T18:19:43.307Z caller=main.go:1066 level=info msg="See you next time!" 2023-09-23T18:19:44.207 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:43 smithi173 bash[52926]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-prometheus-a 2023-09-23T18:19:44.207 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:44 smithi173 bash[52992]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-prometheus-a 2023-09-23T18:19:44.207 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:44 smithi173 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T18:19:44.208 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:44 smithi173 systemd[1]: Stopped Ceph prometheus.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:19:44.208 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:44 smithi173 systemd[1]: Started Ceph prometheus.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:19:44.777 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:44 smithi173 systemd[1]: Stopping Ceph node-exporter.b for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T18:19:44.777 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:44 smithi173 bash[53044]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-node-exporter.b 2023-09-23T18:19:44.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:44 smithi173 bash[27694]: cluster 2023-09-23T18:19:43.123602+0000 mgr.x (mgr.25562) 46 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:44.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:44 smithi173 bash[27694]: audit 2023-09-23T18:19:44.186323+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:44.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:44 smithi173 bash[27694]: audit 2023-09-23T18:19:44.196975+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:44.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:44 smithi098 bash[33599]: cluster 2023-09-23T18:19:43.123602+0000 mgr.x (mgr.25562) 46 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:44.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:44 smithi098 bash[33599]: audit 2023-09-23T18:19:44.186323+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:44.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:44 smithi098 bash[33599]: audit 2023-09-23T18:19:44.196975+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:44.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:44 smithi098 bash[27349]: cluster 2023-09-23T18:19:43.123602+0000 mgr.x (mgr.25562) 46 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:44.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:44 smithi098 bash[27349]: audit 2023-09-23T18:19:44.186323+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:44.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:44 smithi098 bash[27349]: audit 2023-09-23T18:19:44.196975+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:45.777 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:45 smithi173 bash[53081]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-node-exporter-b 2023-09-23T18:19:45.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:45 smithi173 bash[27694]: cephadm 2023-09-23T18:19:44.249913+0000 mgr.x (mgr.25562) 47 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T18:19:45.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:45 smithi173 bash[27694]: cephadm 2023-09-23T18:19:44.250670+0000 mgr.x (mgr.25562) 48 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi173 2023-09-23T18:19:45.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:45 smithi173 bash[27694]: audit 2023-09-23T18:19:44.874393+0000 mon.a (mon.0) 801 : audit [DBG] from='client.? 172.21.15.98:0/2610125211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:45.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:45 smithi098 bash[27349]: cephadm 2023-09-23T18:19:44.249913+0000 mgr.x (mgr.25562) 47 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T18:19:45.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:45 smithi098 bash[27349]: cephadm 2023-09-23T18:19:44.250670+0000 mgr.x (mgr.25562) 48 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi173 2023-09-23T18:19:45.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:45 smithi098 bash[27349]: audit 2023-09-23T18:19:44.874393+0000 mon.a (mon.0) 801 : audit [DBG] from='client.? 172.21.15.98:0/2610125211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:45.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:45 smithi098 bash[33599]: cephadm 2023-09-23T18:19:44.249913+0000 mgr.x (mgr.25562) 47 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T18:19:45.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:45 smithi098 bash[33599]: cephadm 2023-09-23T18:19:44.250670+0000 mgr.x (mgr.25562) 48 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi173 2023-09-23T18:19:45.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:45 smithi098 bash[33599]: audit 2023-09-23T18:19:44.874393+0000 mon.a (mon.0) 801 : audit [DBG] from='client.? 172.21.15.98:0/2610125211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:46.277 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[53021]: ts=2023-09-23T18:19:46.121Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-09-23T18:19:46.277 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[53021]: ts=2023-09-23T18:19:46.121Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-09-23T18:19:46.277 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[53021]: ts=2023-09-23T18:19:46.121Z caller=main.go:518 level=info host_details="(Linux 5.15.0-84-generic #93~20.04.1-Ubuntu SMP Wed Sep 6 16:15:40 UTC 2023 x86_64 smithi173 (none))" 2023-09-23T18:19:46.277 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[53021]: ts=2023-09-23T18:19:46.121Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T18:19:46.277 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[53021]: ts=2023-09-23T18:19:46.121Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T18:19:46.277 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[53021]: ts=2023-09-23T18:19:46.123Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T18:19:46.278 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[53021]: ts=2023-09-23T18:19:46.124Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-09-23T18:19:46.278 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[53021]: ts=2023-09-23T18:19:46.125Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-09-23T18:19:46.278 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[53021]: ts=2023-09-23T18:19:46.131Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T18:19:46.278 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[53021]: ts=2023-09-23T18:19:46.131Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.745µs 2023-09-23T18:19:46.278 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[53021]: ts=2023-09-23T18:19:46.131Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T18:19:46.278 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:46 smithi173 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-09-23T18:19:46.278 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[53180]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-node-exporter-b 2023-09-23T18:19:46.279 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:46 smithi173 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-09-23T18:19:46.279 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:46 smithi173 systemd[1]: Stopped Ceph node-exporter.b for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:19:46.279 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:46 smithi173 systemd[1]: Started Ceph node-exporter.b for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:19:46.526 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[53021]: ts=2023-09-23T18:19:46.307Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-09-23T18:19:46.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[27694]: cluster 2023-09-23T18:19:45.124270+0000 mgr.x (mgr.25562) 49 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:46.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[27694]: audit 2023-09-23T18:19:46.095971+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:46.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[27694]: audit 2023-09-23T18:19:46.107433+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:46.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[27694]: cephadm 2023-09-23T18:19:46.119762+0000 mgr.x (mgr.25562) 50 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-09-23T18:19:46.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[27694]: cephadm 2023-09-23T18:19:46.140131+0000 mgr.x (mgr.25562) 51 : cephadm [INF] Reconfiguring daemon grafana.a on smithi173 2023-09-23T18:19:46.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:46 smithi098 bash[27349]: cluster 2023-09-23T18:19:45.124270+0000 mgr.x (mgr.25562) 49 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:46.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:46 smithi098 bash[27349]: audit 2023-09-23T18:19:46.095971+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:46.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:46 smithi098 bash[27349]: audit 2023-09-23T18:19:46.107433+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:46.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:46 smithi098 bash[27349]: cephadm 2023-09-23T18:19:46.119762+0000 mgr.x (mgr.25562) 50 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-09-23T18:19:46.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:46 smithi098 bash[27349]: cephadm 2023-09-23T18:19:46.140131+0000 mgr.x (mgr.25562) 51 : cephadm [INF] Reconfiguring daemon grafana.a on smithi173 2023-09-23T18:19:46.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:46 smithi098 bash[33599]: cluster 2023-09-23T18:19:45.124270+0000 mgr.x (mgr.25562) 49 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:46.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:46 smithi098 bash[33599]: audit 2023-09-23T18:19:46.095971+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:46.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:46 smithi098 bash[33599]: audit 2023-09-23T18:19:46.107433+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:19:46.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:46 smithi098 bash[33599]: cephadm 2023-09-23T18:19:46.119762+0000 mgr.x (mgr.25562) 50 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-09-23T18:19:46.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:46 smithi098 bash[33599]: cephadm 2023-09-23T18:19:46.140131+0000 mgr.x (mgr.25562) 51 : cephadm [INF] Reconfiguring daemon grafana.a on smithi173 2023-09-23T18:19:47.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[53021]: ts=2023-09-23T18:19:46.577Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-09-23T18:19:47.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[53021]: ts=2023-09-23T18:19:46.578Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-09-23T18:19:47.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[53021]: ts=2023-09-23T18:19:46.578Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=50.032µs wal_replay_duration=446.608818ms total_replay_duration=446.682649ms 2023-09-23T18:19:47.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[53021]: ts=2023-09-23T18:19:46.582Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T18:19:47.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[53021]: ts=2023-09-23T18:19:46.582Z caller=main.go:947 level=info msg="TSDB started" 2023-09-23T18:19:47.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[53021]: ts=2023-09-23T18:19:46.582Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T18:19:47.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[53021]: ts=2023-09-23T18:19:46.647Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=64.45764ms db_storage=1.42µs remote_storage=2.384µs web_handler=528ns query_engine=1.182µs scrape=37.75152ms scrape_sd=149.553µs notify=27.418µs notify_sd=39.723µs rules=26.041616ms 2023-09-23T18:19:47.028 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:46 smithi173 bash[53021]: ts=2023-09-23T18:19:46.647Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-09-23T18:19:47.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:47 smithi098 bash[33599]: cluster 2023-09-23T18:19:47.124908+0000 mgr.x (mgr.25562) 52 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:47.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:47 smithi098 bash[33599]: audit 2023-09-23T18:19:47.322353+0000 mon.c (mon.2) 142 : audit [DBG] from='client.? 172.21.15.98:0/1047792310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:47.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:47 smithi098 bash[27349]: cluster 2023-09-23T18:19:47.124908+0000 mgr.x (mgr.25562) 52 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:47.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:47 smithi098 bash[27349]: audit 2023-09-23T18:19:47.322353+0000 mon.c (mon.2) 142 : audit [DBG] from='client.? 172.21.15.98:0/1047792310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:48.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[27694]: cluster 2023-09-23T18:19:47.124908+0000 mgr.x (mgr.25562) 52 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:48.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[27694]: audit 2023-09-23T18:19:47.322353+0000 mon.c (mon.2) 142 : audit [DBG] from='client.? 172.21.15.98:0/1047792310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:48.027 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.756Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-09-23T18:19:48.027 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.756Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-09-23T18:19:48.027 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.756Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-09-23T18:19:48.028 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.756Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T18:19:48.028 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-09-23T18:19:48.028 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=arp 2023-09-23T18:19:48.028 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=bcache 2023-09-23T18:19:48.028 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=bonding 2023-09-23T18:19:48.028 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=btrfs 2023-09-23T18:19:48.028 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=conntrack 2023-09-23T18:19:48.028 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=cpu 2023-09-23T18:19:48.028 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-09-23T18:19:48.028 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=diskstats 2023-09-23T18:19:48.028 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=dmi 2023-09-23T18:19:48.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=edac 2023-09-23T18:19:48.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=entropy 2023-09-23T18:19:48.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-09-23T18:19:48.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=filefd 2023-09-23T18:19:48.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=filesystem 2023-09-23T18:19:48.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=hwmon 2023-09-23T18:19:48.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=infiniband 2023-09-23T18:19:48.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=ipvs 2023-09-23T18:19:48.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=loadavg 2023-09-23T18:19:48.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=mdadm 2023-09-23T18:19:48.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=meminfo 2023-09-23T18:19:48.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=netclass 2023-09-23T18:19:48.029 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=netdev 2023-09-23T18:19:48.030 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=netstat 2023-09-23T18:19:48.030 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=nfs 2023-09-23T18:19:48.030 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=nfsd 2023-09-23T18:19:48.030 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=nvme 2023-09-23T18:19:48.030 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=os 2023-09-23T18:19:48.030 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-09-23T18:19:48.030 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=pressure 2023-09-23T18:19:48.031 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=rapl 2023-09-23T18:19:48.031 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=schedstat 2023-09-23T18:19:48.031 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=sockstat 2023-09-23T18:19:48.031 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=softnet 2023-09-23T18:19:48.031 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=stat 2023-09-23T18:19:48.031 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=tapestats 2023-09-23T18:19:48.031 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=textfile 2023-09-23T18:19:48.031 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-09-23T18:19:48.031 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=time 2023-09-23T18:19:48.031 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-09-23T18:19:48.031 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=uname 2023-09-23T18:19:48.031 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=vmstat 2023-09-23T18:19:48.031 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=xfs 2023-09-23T18:19:48.032 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:115 level=info collector=zfs 2023-09-23T18:19:48.032 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-09-23T18:19:48.032 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 18:19:47 smithi173 bash[53220]: ts=2023-09-23T18:19:47.758Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-09-23T18:19:50.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:50 smithi098 bash[33599]: cluster 2023-09-23T18:19:49.125726+0000 mgr.x (mgr.25562) 53 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:50.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:50 smithi098 bash[33599]: audit 2023-09-23T18:19:49.762758+0000 mon.a (mon.0) 804 : audit [DBG] from='client.? 172.21.15.98:0/2586565784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:50.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:50 smithi098 bash[27349]: cluster 2023-09-23T18:19:49.125726+0000 mgr.x (mgr.25562) 53 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:50.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:50 smithi098 bash[27349]: audit 2023-09-23T18:19:49.762758+0000 mon.a (mon.0) 804 : audit [DBG] from='client.? 172.21.15.98:0/2586565784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:50.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:50 smithi173 bash[27694]: cluster 2023-09-23T18:19:49.125726+0000 mgr.x (mgr.25562) 53 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:50.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:50 smithi173 bash[27694]: audit 2023-09-23T18:19:49.762758+0000 mon.a (mon.0) 804 : audit [DBG] from='client.? 172.21.15.98:0/2586565784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:52.686 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:52 smithi173 bash[27694]: cluster 2023-09-23T18:19:51.126322+0000 mgr.x (mgr.25562) 54 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:52.686 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:52 smithi173 bash[27694]: audit 2023-09-23T18:19:52.198254+0000 mon.c (mon.2) 143 : audit [DBG] from='client.? 172.21.15.98:0/1725926876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:52.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:52 smithi098 bash[27349]: cluster 2023-09-23T18:19:51.126322+0000 mgr.x (mgr.25562) 54 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:52.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:52 smithi098 bash[27349]: audit 2023-09-23T18:19:52.198254+0000 mon.c (mon.2) 143 : audit [DBG] from='client.? 172.21.15.98:0/1725926876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:52.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:52 smithi098 bash[33599]: cluster 2023-09-23T18:19:51.126322+0000 mgr.x (mgr.25562) 54 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:52.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:52 smithi098 bash[33599]: audit 2023-09-23T18:19:52.198254+0000 mon.c (mon.2) 143 : audit [DBG] from='client.? 172.21.15.98:0/1725926876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:53.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:19:52 smithi173 bash[50615]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:19:52] "GET /metrics HTTP/1.1" 200 33916 "" "Prometheus/2.33.4" 2023-09-23T18:19:54.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:54 smithi098 bash[27349]: cluster 2023-09-23T18:19:53.126797+0000 mgr.x (mgr.25562) 55 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:54.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:54 smithi098 bash[33599]: cluster 2023-09-23T18:19:53.126797+0000 mgr.x (mgr.25562) 55 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:54.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:54 smithi173 bash[27694]: cluster 2023-09-23T18:19:53.126797+0000 mgr.x (mgr.25562) 55 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:55.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:55 smithi098 bash[27349]: audit 2023-09-23T18:19:54.640312+0000 mon.a (mon.0) 805 : audit [DBG] from='client.? 172.21.15.98:0/1347533220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:55.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:55 smithi098 bash[33599]: audit 2023-09-23T18:19:54.640312+0000 mon.a (mon.0) 805 : audit [DBG] from='client.? 172.21.15.98:0/1347533220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:55.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:55 smithi173 bash[27694]: audit 2023-09-23T18:19:54.640312+0000 mon.a (mon.0) 805 : audit [DBG] from='client.? 172.21.15.98:0/1347533220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:56.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:56 smithi173 bash[27694]: cluster 2023-09-23T18:19:55.127571+0000 mgr.x (mgr.25562) 56 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:56.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:56 smithi098 bash[33599]: cluster 2023-09-23T18:19:55.127571+0000 mgr.x (mgr.25562) 56 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:56.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:56 smithi098 bash[27349]: cluster 2023-09-23T18:19:55.127571+0000 mgr.x (mgr.25562) 56 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:57.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:57 smithi173 bash[27694]: audit 2023-09-23T18:19:57.079153+0000 mon.c (mon.2) 144 : audit [DBG] from='client.? 172.21.15.98:0/1702359004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:57.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:57 smithi173 bash[27694]: audit 2023-09-23T18:19:57.372178+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:19:57.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:57 smithi098 bash[27349]: audit 2023-09-23T18:19:57.079153+0000 mon.c (mon.2) 144 : audit [DBG] from='client.? 172.21.15.98:0/1702359004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:57.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:57 smithi098 bash[27349]: audit 2023-09-23T18:19:57.372178+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:19:57.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:57 smithi098 bash[33599]: audit 2023-09-23T18:19:57.079153+0000 mon.c (mon.2) 144 : audit [DBG] from='client.? 172.21.15.98:0/1702359004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:19:57.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:57 smithi098 bash[33599]: audit 2023-09-23T18:19:57.372178+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:19:58.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:19:58 smithi173 bash[27694]: cluster 2023-09-23T18:19:57.128193+0000 mgr.x (mgr.25562) 57 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:58.777 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:19:58 smithi173 bash[53021]: ts=2023-09-23T18:19:58.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:19:58.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:19:58 smithi098 bash[33599]: cluster 2023-09-23T18:19:57.128193+0000 mgr.x (mgr.25562) 57 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:19:58.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:19:58 smithi098 bash[27349]: cluster 2023-09-23T18:19:57.128193+0000 mgr.x (mgr.25562) 57 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:00.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:00 smithi173 bash[27694]: cluster 2023-09-23T18:19:59.129357+0000 mgr.x (mgr.25562) 58 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:00.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:00 smithi173 bash[27694]: audit 2023-09-23T18:19:59.514375+0000 mon.a (mon.0) 806 : audit [DBG] from='client.? 172.21.15.98:0/1694469285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:00.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:00 smithi173 bash[27694]: cluster 2023-09-23T18:20:00.000161+0000 mon.a (mon.0) 807 : cluster [INF] overall HEALTH_OK 2023-09-23T18:20:00.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:00 smithi098 bash[33599]: cluster 2023-09-23T18:19:59.129357+0000 mgr.x (mgr.25562) 58 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:00.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:00 smithi098 bash[33599]: audit 2023-09-23T18:19:59.514375+0000 mon.a (mon.0) 806 : audit [DBG] from='client.? 172.21.15.98:0/1694469285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:00.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:00 smithi098 bash[33599]: cluster 2023-09-23T18:20:00.000161+0000 mon.a (mon.0) 807 : cluster [INF] overall HEALTH_OK 2023-09-23T18:20:00.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:00 smithi098 bash[27349]: cluster 2023-09-23T18:19:59.129357+0000 mgr.x (mgr.25562) 58 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:00.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:00 smithi098 bash[27349]: audit 2023-09-23T18:19:59.514375+0000 mon.a (mon.0) 806 : audit [DBG] from='client.? 172.21.15.98:0/1694469285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:00.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:00 smithi098 bash[27349]: cluster 2023-09-23T18:20:00.000161+0000 mon.a (mon.0) 807 : cluster [INF] overall HEALTH_OK 2023-09-23T18:20:02.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:02 smithi173 bash[27694]: cluster 2023-09-23T18:20:01.130387+0000 mgr.x (mgr.25562) 59 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:02.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:02 smithi173 bash[27694]: audit 2023-09-23T18:20:01.954886+0000 mon.a (mon.0) 808 : audit [DBG] from='client.? 172.21.15.98:0/2474754505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:02.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:02 smithi173 bash[50615]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:20:02] "GET /metrics HTTP/1.1" 200 33915 "" "Prometheus/2.33.4" 2023-09-23T18:20:02.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:02 smithi098 bash[27349]: cluster 2023-09-23T18:20:01.130387+0000 mgr.x (mgr.25562) 59 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:02.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:02 smithi098 bash[27349]: audit 2023-09-23T18:20:01.954886+0000 mon.a (mon.0) 808 : audit [DBG] from='client.? 172.21.15.98:0/2474754505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:02.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:02 smithi098 bash[33599]: cluster 2023-09-23T18:20:01.130387+0000 mgr.x (mgr.25562) 59 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:02.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:02 smithi098 bash[33599]: audit 2023-09-23T18:20:01.954886+0000 mon.a (mon.0) 808 : audit [DBG] from='client.? 172.21.15.98:0/2474754505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:04.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:04 smithi173 bash[27694]: cluster 2023-09-23T18:20:03.131055+0000 mgr.x (mgr.25562) 60 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:04.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:04 smithi173 bash[27694]: audit 2023-09-23T18:20:04.398913+0000 mon.c (mon.2) 145 : audit [DBG] from='client.? 172.21.15.98:0/3784780081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:04.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:04 smithi098 bash[27349]: cluster 2023-09-23T18:20:03.131055+0000 mgr.x (mgr.25562) 60 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:04.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:04 smithi098 bash[27349]: audit 2023-09-23T18:20:04.398913+0000 mon.c (mon.2) 145 : audit [DBG] from='client.? 172.21.15.98:0/3784780081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:04.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:04 smithi098 bash[33599]: cluster 2023-09-23T18:20:03.131055+0000 mgr.x (mgr.25562) 60 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:04.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:04 smithi098 bash[33599]: audit 2023-09-23T18:20:04.398913+0000 mon.c (mon.2) 145 : audit [DBG] from='client.? 172.21.15.98:0/3784780081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:06.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:06 smithi173 bash[27694]: cluster 2023-09-23T18:20:05.132160+0000 mgr.x (mgr.25562) 61 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:06.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:06 smithi098 bash[27349]: cluster 2023-09-23T18:20:05.132160+0000 mgr.x (mgr.25562) 61 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:06.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:06 smithi098 bash[33599]: cluster 2023-09-23T18:20:05.132160+0000 mgr.x (mgr.25562) 61 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:07.724 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:07 smithi173 bash[27694]: audit 2023-09-23T18:20:06.823746+0000 mon.a (mon.0) 809 : audit [DBG] from='client.? 172.21.15.98:0/28492093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:07.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:07 smithi098 bash[27349]: audit 2023-09-23T18:20:06.823746+0000 mon.a (mon.0) 809 : audit [DBG] from='client.? 172.21.15.98:0/28492093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:07.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:07 smithi098 bash[33599]: audit 2023-09-23T18:20:06.823746+0000 mon.a (mon.0) 809 : audit [DBG] from='client.? 172.21.15.98:0/28492093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:08.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:08 smithi173 bash[27694]: cluster 2023-09-23T18:20:07.132800+0000 mgr.x (mgr.25562) 62 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:08.777 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:08 smithi173 bash[53021]: ts=2023-09-23T18:20:08.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:20:08.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:08 smithi098 bash[27349]: cluster 2023-09-23T18:20:07.132800+0000 mgr.x (mgr.25562) 62 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:08.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:08 smithi098 bash[33599]: cluster 2023-09-23T18:20:07.132800+0000 mgr.x (mgr.25562) 62 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:09.667 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:09 smithi173 bash[27694]: audit 2023-09-23T18:20:09.273981+0000 mon.c (mon.2) 146 : audit [DBG] from='client.? 172.21.15.98:0/1295229754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:09.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:09 smithi098 bash[27349]: audit 2023-09-23T18:20:09.273981+0000 mon.c (mon.2) 146 : audit [DBG] from='client.? 172.21.15.98:0/1295229754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:09.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:09 smithi098 bash[33599]: audit 2023-09-23T18:20:09.273981+0000 mon.c (mon.2) 146 : audit [DBG] from='client.? 172.21.15.98:0/1295229754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:10.507 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:10 smithi173 systemd[1]: Stopping Ceph grafana.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T18:20:10.507 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:10 smithi173 bash[53475]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-grafana.a 2023-09-23T18:20:10.507 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:10 smithi173 bash[47708]: t=2023-09-23T18:20:10+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-09-23T18:20:10.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:10 smithi173 bash[27694]: cluster 2023-09-23T18:20:09.133911+0000 mgr.x (mgr.25562) 63 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:10.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:10 smithi098 bash[27349]: cluster 2023-09-23T18:20:09.133911+0000 mgr.x (mgr.25562) 63 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:10.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:10 smithi098 bash[33599]: cluster 2023-09-23T18:20:09.133911+0000 mgr.x (mgr.25562) 63 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:11.527 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:11 smithi173 bash[53511]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-grafana-a 2023-09-23T18:20:11.527 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:11 smithi173 bash[53576]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-grafana-a 2023-09-23T18:20:11.527 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:11 smithi173 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@grafana.a.service: Succeeded. 2023-09-23T18:20:11.527 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:11 smithi173 systemd[1]: Stopped Ceph grafana.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:20:11.527 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:11 smithi173 systemd[1]: Started Ceph grafana.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:20:12.686 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[27694]: cluster 2023-09-23T18:20:11.134448+0000 mgr.x (mgr.25562) 64 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:12.686 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[27694]: audit 2023-09-23T18:20:11.446812+0000 mon.a (mon.0) 810 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:12.686 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[27694]: audit 2023-09-23T18:20:11.454486+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:12.686 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[27694]: audit 2023-09-23T18:20:11.479206+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T18:20:12.686 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[27694]: audit 2023-09-23T18:20:11.479813+0000 mgr.x (mgr.25562) 65 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T18:20:12.686 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[27694]: audit 2023-09-23T18:20:11.484683+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T18:20:12.686 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[27694]: audit 2023-09-23T18:20:11.485535+0000 mgr.x (mgr.25562) 66 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T18:20:12.687 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[27694]: audit 2023-09-23T18:20:11.490603+0000 mon.b (mon.1) 85 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T18:20:12.687 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[27694]: audit 2023-09-23T18:20:11.491379+0000 mgr.x (mgr.25562) 67 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T18:20:12.687 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[27694]: audit 2023-09-23T18:20:11.499258+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:20:12.687 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[27694]: cephadm 2023-09-23T18:20:11.502535+0000 mgr.x (mgr.25562) 68 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T18:20:12.687 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[27694]: cephadm 2023-09-23T18:20:11.503172+0000 mgr.x (mgr.25562) 69 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T18:20:12.687 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[27694]: audit 2023-09-23T18:20:11.723692+0000 mon.c (mon.2) 147 : audit [DBG] from='client.? 172.21.15.98:0/414089511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:12.687 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[27694]: audit 2023-09-23T18:20:12.372972+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:20:12.687 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-09-23T18:20:12.687 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-09-23T18:20:12.687 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-09-23T18:20:12.687 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-09-23T18:20:12.688 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-09-23T18:20:12.688 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-09-23T18:20:12.688 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-09-23T18:20:12.688 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-09-23T18:20:12.688 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-09-23T18:20:12.688 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-09-23T18:20:12.688 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-09-23T18:20:12.688 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-09-23T18:20:12.688 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-09-23T18:20:12.688 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="App mode production" logger=settings 2023-09-23T18:20:12.688 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-09-23T18:20:12.688 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-09-23T18:20:12.688 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-09-23T18:20:12.689 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=658.461µs 2023-09-23T18:20:12.689 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-09-23T18:20:12.689 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-09-23T18:20:12.689 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-09-23T18:20:12.689 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-09-23T18:20:12.689 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-09-23T18:20:12.689 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-09-23T18:20:12.720 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[27349]: cluster 2023-09-23T18:20:11.134448+0000 mgr.x (mgr.25562) 64 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:12.720 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[27349]: audit 2023-09-23T18:20:11.446812+0000 mon.a (mon.0) 810 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:12.721 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[27349]: audit 2023-09-23T18:20:11.454486+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:12.721 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[27349]: audit 2023-09-23T18:20:11.479206+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T18:20:12.721 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[27349]: audit 2023-09-23T18:20:11.479813+0000 mgr.x (mgr.25562) 65 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T18:20:12.721 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[27349]: audit 2023-09-23T18:20:11.484683+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T18:20:12.721 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[27349]: audit 2023-09-23T18:20:11.485535+0000 mgr.x (mgr.25562) 66 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T18:20:12.721 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[27349]: audit 2023-09-23T18:20:11.490603+0000 mon.b (mon.1) 85 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T18:20:12.721 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[27349]: audit 2023-09-23T18:20:11.491379+0000 mgr.x (mgr.25562) 67 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T18:20:12.721 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[27349]: audit 2023-09-23T18:20:11.499258+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:20:12.721 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[27349]: cephadm 2023-09-23T18:20:11.502535+0000 mgr.x (mgr.25562) 68 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T18:20:12.721 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[27349]: cephadm 2023-09-23T18:20:11.503172+0000 mgr.x (mgr.25562) 69 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T18:20:12.721 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[27349]: audit 2023-09-23T18:20:11.723692+0000 mon.c (mon.2) 147 : audit [DBG] from='client.? 172.21.15.98:0/414089511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:12.721 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[27349]: audit 2023-09-23T18:20:12.372972+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:20:12.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[33599]: cluster 2023-09-23T18:20:11.134448+0000 mgr.x (mgr.25562) 64 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:12.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[33599]: audit 2023-09-23T18:20:11.446812+0000 mon.a (mon.0) 810 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:12.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[33599]: audit 2023-09-23T18:20:11.454486+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:12.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[33599]: audit 2023-09-23T18:20:11.479206+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T18:20:12.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[33599]: audit 2023-09-23T18:20:11.479813+0000 mgr.x (mgr.25562) 65 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T18:20:12.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[33599]: audit 2023-09-23T18:20:11.484683+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T18:20:12.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[33599]: audit 2023-09-23T18:20:11.485535+0000 mgr.x (mgr.25562) 66 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T18:20:12.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[33599]: audit 2023-09-23T18:20:11.490603+0000 mon.b (mon.1) 85 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T18:20:12.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[33599]: audit 2023-09-23T18:20:11.491379+0000 mgr.x (mgr.25562) 67 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T18:20:12.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[33599]: audit 2023-09-23T18:20:11.499258+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:20:12.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[33599]: cephadm 2023-09-23T18:20:11.502535+0000 mgr.x (mgr.25562) 68 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T18:20:12.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[33599]: cephadm 2023-09-23T18:20:11.503172+0000 mgr.x (mgr.25562) 69 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T18:20:12.723 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[33599]: audit 2023-09-23T18:20:11.723692+0000 mon.c (mon.2) 147 : audit [DBG] from='client.? 172.21.15.98:0/414089511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:12.723 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:12 smithi098 bash[33599]: audit 2023-09-23T18:20:12.372972+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:20:13.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[50615]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:20:12] "GET /metrics HTTP/1.1" 200 33914 "" "Prometheus/2.33.4" 2023-09-23T18:20:13.028 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2023-09-23T18:20:13.028 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-09-23T18:20:13.028 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2023-09-23T18:20:13.028 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-09-23T18:20:13.028 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-09-23T18:20:13.028 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 18:20:12 smithi173 bash[53604]: t=2023-09-23T18:20:12+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-09-23T18:20:14.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:14 smithi173 bash[27694]: cluster 2023-09-23T18:20:13.135213+0000 mgr.x (mgr.25562) 70 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:14.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:14 smithi173 bash[27694]: audit 2023-09-23T18:20:13.827891+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:14.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:14 smithi173 bash[27694]: audit 2023-09-23T18:20:13.832011+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:20:14.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:14 smithi173 bash[27694]: audit 2023-09-23T18:20:13.836078+0000 mon.b (mon.1) 88 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:20:14.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:14 smithi173 bash[27694]: audit 2023-09-23T18:20:13.838404+0000 mon.b (mon.1) 89 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:20:14.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:14 smithi173 bash[27694]: audit 2023-09-23T18:20:13.839808+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:20:14.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:14 smithi173 bash[27694]: audit 2023-09-23T18:20:14.177582+0000 mon.c (mon.2) 148 : audit [DBG] from='client.? 172.21.15.98:0/1341581330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:14.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:14 smithi098 bash[33599]: cluster 2023-09-23T18:20:13.135213+0000 mgr.x (mgr.25562) 70 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:14.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:14 smithi098 bash[33599]: audit 2023-09-23T18:20:13.827891+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:14.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:14 smithi098 bash[33599]: audit 2023-09-23T18:20:13.832011+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:20:14.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:14 smithi098 bash[33599]: audit 2023-09-23T18:20:13.836078+0000 mon.b (mon.1) 88 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:20:14.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:14 smithi098 bash[33599]: audit 2023-09-23T18:20:13.838404+0000 mon.b (mon.1) 89 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:20:14.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:14 smithi098 bash[33599]: audit 2023-09-23T18:20:13.839808+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:20:14.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:14 smithi098 bash[33599]: audit 2023-09-23T18:20:14.177582+0000 mon.c (mon.2) 148 : audit [DBG] from='client.? 172.21.15.98:0/1341581330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:14.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:14 smithi098 bash[27349]: cluster 2023-09-23T18:20:13.135213+0000 mgr.x (mgr.25562) 70 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:14.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:14 smithi098 bash[27349]: audit 2023-09-23T18:20:13.827891+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:14.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:14 smithi098 bash[27349]: audit 2023-09-23T18:20:13.832011+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:20:14.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:14 smithi098 bash[27349]: audit 2023-09-23T18:20:13.836078+0000 mon.b (mon.1) 88 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:20:14.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:14 smithi098 bash[27349]: audit 2023-09-23T18:20:13.838404+0000 mon.b (mon.1) 89 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:20:14.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:14 smithi098 bash[27349]: audit 2023-09-23T18:20:13.839808+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:20:14.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:14 smithi098 bash[27349]: audit 2023-09-23T18:20:14.177582+0000 mon.c (mon.2) 148 : audit [DBG] from='client.? 172.21.15.98:0/1341581330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:15.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:15 smithi098 bash[27349]: cephadm 2023-09-23T18:20:13.823190+0000 mgr.x (mgr.25562) 71 : cephadm [INF] Upgrade: Updating mgr.y 2023-09-23T18:20:15.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:15 smithi098 bash[27349]: cephadm 2023-09-23T18:20:13.841125+0000 mgr.x (mgr.25562) 72 : cephadm [INF] Deploying daemon mgr.y on smithi098 2023-09-23T18:20:15.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:15 smithi098 bash[33599]: cephadm 2023-09-23T18:20:13.823190+0000 mgr.x (mgr.25562) 71 : cephadm [INF] Upgrade: Updating mgr.y 2023-09-23T18:20:15.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:15 smithi098 bash[33599]: cephadm 2023-09-23T18:20:13.841125+0000 mgr.x (mgr.25562) 72 : cephadm [INF] Deploying daemon mgr.y on smithi098 2023-09-23T18:20:15.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:15 smithi173 bash[27694]: cephadm 2023-09-23T18:20:13.823190+0000 mgr.x (mgr.25562) 71 : cephadm [INF] Upgrade: Updating mgr.y 2023-09-23T18:20:15.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:15 smithi173 bash[27694]: cephadm 2023-09-23T18:20:13.841125+0000 mgr.x (mgr.25562) 72 : cephadm [INF] Deploying daemon mgr.y on smithi098 2023-09-23T18:20:16.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:16 smithi098 systemd[1]: Stopping Ceph mgr.y for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T18:20:16.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:16 smithi173 bash[27694]: cluster 2023-09-23T18:20:15.136350+0000 mgr.x (mgr.25562) 73 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:16.924 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:16 smithi098 bash[27349]: cluster 2023-09-23T18:20:15.136350+0000 mgr.x (mgr.25562) 73 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:16.925 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:16 smithi098 bash[33599]: cluster 2023-09-23T18:20:15.136350+0000 mgr.x (mgr.25562) 73 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:17.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:16 smithi098 bash[72942]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-mgr-y 2023-09-23T18:20:17.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:17 smithi098 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-09-23T18:20:17.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:17 smithi098 bash[73030]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-mgr-y 2023-09-23T18:20:17.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:17 smithi098 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-09-23T18:20:17.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:17 smithi098 systemd[1]: Stopped Ceph mgr.y for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:20:17.694 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:17 smithi098 systemd[1]: Started Ceph mgr.y for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:20:17.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:17 smithi098 bash[33599]: audit 2023-09-23T18:20:16.617408+0000 mon.c (mon.2) 149 : audit [DBG] from='client.? 172.21.15.98:0/1268275054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:17.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:17 smithi098 bash[33599]: audit 2023-09-23T18:20:17.432334+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:17.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:17 smithi098 bash[33599]: audit 2023-09-23T18:20:17.439704+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:17.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:17 smithi098 bash[33599]: audit 2023-09-23T18:20:17.465324+0000 mon.b (mon.1) 91 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:20:17.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:17 smithi098 bash[27349]: audit 2023-09-23T18:20:16.617408+0000 mon.c (mon.2) 149 : audit [DBG] from='client.? 172.21.15.98:0/1268275054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:17.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:17 smithi098 bash[27349]: audit 2023-09-23T18:20:17.432334+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:17.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:17 smithi098 bash[27349]: audit 2023-09-23T18:20:17.439704+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:17.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:17 smithi098 bash[27349]: audit 2023-09-23T18:20:17.465324+0000 mon.b (mon.1) 91 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:20:17.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:17 smithi173 bash[27694]: audit 2023-09-23T18:20:16.617408+0000 mon.c (mon.2) 149 : audit [DBG] from='client.? 172.21.15.98:0/1268275054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:17.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:17 smithi173 bash[27694]: audit 2023-09-23T18:20:17.432334+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:17.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:17 smithi173 bash[27694]: audit 2023-09-23T18:20:17.439704+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:17.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:17 smithi173 bash[27694]: audit 2023-09-23T18:20:17.465324+0000 mon.b (mon.1) 91 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:20:18.694 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:18 smithi098 bash[73085]: debug 2023-09-23T18:20:18.525+0000 7f36ac689700 1 -- 172.21.15.98:0/91887016 <== mon.2 v2:172.21.15.98:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559d554b31e0 con 0x559d56232800 2023-09-23T18:20:18.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:18 smithi098 bash[27349]: cluster 2023-09-23T18:20:17.136924+0000 mgr.x (mgr.25562) 74 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:18.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:18 smithi098 bash[33599]: cluster 2023-09-23T18:20:17.136924+0000 mgr.x (mgr.25562) 74 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:18.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:18 smithi173 bash[27694]: cluster 2023-09-23T18:20:17.136924+0000 mgr.x (mgr.25562) 74 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:18.778 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:18 smithi173 bash[53021]: ts=2023-09-23T18:20:18.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:20:19.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:18 smithi098 bash[73085]: debug 2023-09-23T18:20:18.689+0000 7f36b67d6200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T18:20:19.755 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:19 smithi098 bash[33599]: audit 2023-09-23T18:20:19.045537+0000 mon.a (mon.0) 816 : audit [DBG] from='client.? 172.21.15.98:0/3762614168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:19.755 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:19 smithi098 bash[27349]: audit 2023-09-23T18:20:19.045537+0000 mon.a (mon.0) 816 : audit [DBG] from='client.? 172.21.15.98:0/3762614168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:19.756 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:19 smithi098 bash[73085]: debug 2023-09-23T18:20:19.645+0000 7f36b67d6200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T18:20:19.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:19 smithi173 bash[27694]: audit 2023-09-23T18:20:19.045537+0000 mon.a (mon.0) 816 : audit [DBG] from='client.? 172.21.15.98:0/3762614168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:20.158 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:19 smithi098 bash[73085]: debug 2023-09-23T18:20:19.745+0000 7f36b67d6200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T18:20:20.158 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:19 smithi098 bash[73085]: debug 2023-09-23T18:20:19.861+0000 7f36b67d6200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T18:20:20.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:20 smithi098 bash[73085]: debug 2023-09-23T18:20:20.149+0000 7f36b67d6200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:20:20.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:20 smithi173 bash[27694]: cluster 2023-09-23T18:20:19.138085+0000 mgr.x (mgr.25562) 75 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:20.929 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:20 smithi098 bash[33599]: cluster 2023-09-23T18:20:19.138085+0000 mgr.x (mgr.25562) 75 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:20.930 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:20 smithi098 bash[27349]: cluster 2023-09-23T18:20:19.138085+0000 mgr.x (mgr.25562) 75 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:20.930 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:20 smithi098 bash[73085]: debug 2023-09-23T18:20:20.529+0000 7f36b67d6200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T18:20:20.930 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:20 smithi098 bash[73085]: debug 2023-09-23T18:20:20.689+0000 7f36b67d6200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T18:20:20.930 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:20 smithi098 bash[73085]: debug 2023-09-23T18:20:20.921+0000 7f36b67d6200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T18:20:21.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:21 smithi173 bash[27694]: audit 2023-09-23T18:20:21.477465+0000 mon.a (mon.0) 817 : audit [DBG] from='client.? 172.21.15.98:0/439649640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:21.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:21 smithi098 bash[27349]: audit 2023-09-23T18:20:21.477465+0000 mon.a (mon.0) 817 : audit [DBG] from='client.? 172.21.15.98:0/439649640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:21.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:21 smithi098 bash[33599]: audit 2023-09-23T18:20:21.477465+0000 mon.a (mon.0) 817 : audit [DBG] from='client.? 172.21.15.98:0/439649640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:22.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:22 smithi098 bash[73085]: debug 2023-09-23T18:20:21.997+0000 7f36b67d6200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T18:20:22.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:22 smithi098 bash[73085]: debug 2023-09-23T18:20:22.113+0000 7f36b67d6200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T18:20:22.445 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:22 smithi098 bash[73085]: debug 2023-09-23T18:20:22.237+0000 7f36b67d6200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T18:20:22.766 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:22 smithi098 bash[27349]: cluster 2023-09-23T18:20:21.138893+0000 mgr.x (mgr.25562) 76 : cluster [DBG] pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:22.766 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:22 smithi098 bash[33599]: cluster 2023-09-23T18:20:21.138893+0000 mgr.x (mgr.25562) 76 : cluster [DBG] pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:22.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:22 smithi173 bash[50615]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:20:22] "GET /metrics HTTP/1.1" 200 33914 "" "Prometheus/2.33.4" 2023-09-23T18:20:22.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:22 smithi173 bash[27694]: cluster 2023-09-23T18:20:21.138893+0000 mgr.x (mgr.25562) 76 : cluster [DBG] pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:23.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:22 smithi098 bash[73085]: debug 2023-09-23T18:20:22.757+0000 7f36b67d6200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T18:20:23.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:23 smithi098 bash[73085]: debug 2023-09-23T18:20:23.001+0000 7f36b67d6200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T18:20:23.944 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:23 smithi098 bash[73085]: debug 2023-09-23T18:20:23.525+0000 7f36b67d6200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T18:20:24.407 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:24 smithi098 bash[73085]: debug 2023-09-23T18:20:24.045+0000 7f36b67d6200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:20:24.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:24 smithi098 bash[27349]: cluster 2023-09-23T18:20:23.139537+0000 mgr.x (mgr.25562) 77 : cluster [DBG] pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:24.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:24 smithi098 bash[27349]: audit 2023-09-23T18:20:23.403527+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:24.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:24 smithi098 bash[27349]: audit 2023-09-23T18:20:23.414276+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:24.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:24 smithi098 bash[27349]: audit 2023-09-23T18:20:23.758757+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:24.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:24 smithi098 bash[27349]: audit 2023-09-23T18:20:23.771006+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:24.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:24 smithi098 bash[27349]: audit 2023-09-23T18:20:23.970162+0000 mon.c (mon.2) 150 : audit [DBG] from='client.? 172.21.15.98:0/1929460393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:24.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:24 smithi098 bash[27349]: audit 2023-09-23T18:20:24.269477+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:24.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:24 smithi098 bash[27349]: audit 2023-09-23T18:20:24.277511+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:24.695 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:24 smithi098 bash[73085]: debug 2023-09-23T18:20:24.645+0000 7f36b67d6200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T18:20:24.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:24 smithi098 bash[33599]: cluster 2023-09-23T18:20:23.139537+0000 mgr.x (mgr.25562) 77 : cluster [DBG] pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:24.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:24 smithi098 bash[33599]: audit 2023-09-23T18:20:23.403527+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:24.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:24 smithi098 bash[33599]: audit 2023-09-23T18:20:23.414276+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:24.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:24 smithi098 bash[33599]: audit 2023-09-23T18:20:23.758757+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:24.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:24 smithi098 bash[33599]: audit 2023-09-23T18:20:23.771006+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:24.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:24 smithi098 bash[33599]: audit 2023-09-23T18:20:23.970162+0000 mon.c (mon.2) 150 : audit [DBG] from='client.? 172.21.15.98:0/1929460393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:24.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:24 smithi098 bash[33599]: audit 2023-09-23T18:20:24.269477+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:24.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:24 smithi098 bash[33599]: audit 2023-09-23T18:20:24.277511+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:24.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:24 smithi173 bash[27694]: cluster 2023-09-23T18:20:23.139537+0000 mgr.x (mgr.25562) 77 : cluster [DBG] pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:24.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:24 smithi173 bash[27694]: audit 2023-09-23T18:20:23.403527+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:24.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:24 smithi173 bash[27694]: audit 2023-09-23T18:20:23.414276+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:24.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:24 smithi173 bash[27694]: audit 2023-09-23T18:20:23.758757+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:24.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:24 smithi173 bash[27694]: audit 2023-09-23T18:20:23.771006+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:24.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:24 smithi173 bash[27694]: audit 2023-09-23T18:20:23.970162+0000 mon.c (mon.2) 150 : audit [DBG] from='client.? 172.21.15.98:0/1929460393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:24.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:24 smithi173 bash[27694]: audit 2023-09-23T18:20:24.269477+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:24.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:24 smithi173 bash[27694]: audit 2023-09-23T18:20:24.277511+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:25.113 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:24 smithi098 bash[73085]: debug 2023-09-23T18:20:24.761+0000 7f36b67d6200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T18:20:25.373 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:25 smithi098 bash[73085]: debug 2023-09-23T18:20:25.105+0000 7f36b67d6200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T18:20:25.374 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:25 smithi098 bash[73085]: debug 2023-09-23T18:20:25.221+0000 7f36b67d6200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T18:20:25.624 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:25 smithi098 bash[73085]: debug 2023-09-23T18:20:25.365+0000 7f36b67d6200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T18:20:25.624 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:25 smithi098 bash[73085]: debug 2023-09-23T18:20:25.485+0000 7f36b67d6200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T18:20:25.944 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:25 smithi098 bash[73085]: debug 2023-09-23T18:20:25.617+0000 7f36b67d6200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T18:20:25.944 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:25 smithi098 bash[73085]: debug 2023-09-23T18:20:25.817+0000 7f36b67d6200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T18:20:26.412 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:25 smithi098 bash[73085]: debug 2023-09-23T18:20:25.949+0000 7f36b67d6200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T18:20:26.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:26 smithi098 bash[33599]: cluster 2023-09-23T18:20:25.140621+0000 mgr.x (mgr.25562) 78 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:26.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:26 smithi098 bash[27349]: cluster 2023-09-23T18:20:25.140621+0000 mgr.x (mgr.25562) 78 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:26.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:26 smithi173 bash[27694]: cluster 2023-09-23T18:20:25.140621+0000 mgr.x (mgr.25562) 78 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:27.695 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:27 smithi098 bash[73085]: debug 2023-09-23T18:20:27.217+0000 7f36b67d6200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T18:20:27.695 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:27 smithi098 bash[73085]: [23/Sep/2023:18:20:27] ENGINE Bus STARTING 2023-09-23T18:20:27.695 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:27 smithi098 bash[73085]: CherryPy Checker: 2023-09-23T18:20:27.695 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:27 smithi098 bash[73085]: The Application mounted at '' has an empty config. 2023-09-23T18:20:27.695 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:27 smithi098 bash[73085]: [23/Sep/2023:18:20:27] ENGINE Serving on http://:::9283 2023-09-23T18:20:27.695 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:27 smithi098 bash[73085]: [23/Sep/2023:18:20:27] ENGINE Bus STARTED 2023-09-23T18:20:27.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:27 smithi098 bash[27349]: audit 2023-09-23T18:20:26.410189+0000 mon.a (mon.0) 824 : audit [DBG] from='client.? 172.21.15.98:0/3563599599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:27.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:27 smithi098 bash[27349]: cluster 2023-09-23T18:20:27.242460+0000 mon.a (mon.0) 825 : cluster [DBG] Standby manager daemon y restarted 2023-09-23T18:20:27.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:27 smithi098 bash[27349]: cluster 2023-09-23T18:20:27.242727+0000 mon.a (mon.0) 826 : cluster [DBG] Standby manager daemon y started 2023-09-23T18:20:27.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:27 smithi098 bash[27349]: audit 2023-09-23T18:20:27.246879+0000 mon.c (mon.2) 151 : audit [DBG] from='mgr.? 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T18:20:27.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:27 smithi098 bash[27349]: audit 2023-09-23T18:20:27.247761+0000 mon.c (mon.2) 152 : audit [DBG] from='mgr.? 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:20:27.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:27 smithi098 bash[27349]: audit 2023-09-23T18:20:27.252062+0000 mon.c (mon.2) 153 : audit [DBG] from='mgr.? 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T18:20:27.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:27 smithi098 bash[27349]: audit 2023-09-23T18:20:27.252524+0000 mon.c (mon.2) 154 : audit [DBG] from='mgr.? 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:20:27.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:27 smithi098 bash[27349]: audit 2023-09-23T18:20:27.373190+0000 mon.b (mon.1) 92 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:20:27.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:27 smithi098 bash[33599]: audit 2023-09-23T18:20:26.410189+0000 mon.a (mon.0) 824 : audit [DBG] from='client.? 172.21.15.98:0/3563599599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:27.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:27 smithi098 bash[33599]: cluster 2023-09-23T18:20:27.242460+0000 mon.a (mon.0) 825 : cluster [DBG] Standby manager daemon y restarted 2023-09-23T18:20:27.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:27 smithi098 bash[33599]: cluster 2023-09-23T18:20:27.242727+0000 mon.a (mon.0) 826 : cluster [DBG] Standby manager daemon y started 2023-09-23T18:20:27.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:27 smithi098 bash[33599]: audit 2023-09-23T18:20:27.246879+0000 mon.c (mon.2) 151 : audit [DBG] from='mgr.? 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T18:20:27.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:27 smithi098 bash[33599]: audit 2023-09-23T18:20:27.247761+0000 mon.c (mon.2) 152 : audit [DBG] from='mgr.? 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:20:27.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:27 smithi098 bash[33599]: audit 2023-09-23T18:20:27.252062+0000 mon.c (mon.2) 153 : audit [DBG] from='mgr.? 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T18:20:27.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:27 smithi098 bash[33599]: audit 2023-09-23T18:20:27.252524+0000 mon.c (mon.2) 154 : audit [DBG] from='mgr.? 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:20:27.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:27 smithi098 bash[33599]: audit 2023-09-23T18:20:27.373190+0000 mon.b (mon.1) 92 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:20:27.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:27 smithi173 bash[27694]: audit 2023-09-23T18:20:26.410189+0000 mon.a (mon.0) 824 : audit [DBG] from='client.? 172.21.15.98:0/3563599599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:27.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:27 smithi173 bash[27694]: cluster 2023-09-23T18:20:27.242460+0000 mon.a (mon.0) 825 : cluster [DBG] Standby manager daemon y restarted 2023-09-23T18:20:27.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:27 smithi173 bash[27694]: cluster 2023-09-23T18:20:27.242727+0000 mon.a (mon.0) 826 : cluster [DBG] Standby manager daemon y started 2023-09-23T18:20:27.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:27 smithi173 bash[27694]: audit 2023-09-23T18:20:27.246879+0000 mon.c (mon.2) 151 : audit [DBG] from='mgr.? 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T18:20:27.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:27 smithi173 bash[27694]: audit 2023-09-23T18:20:27.247761+0000 mon.c (mon.2) 152 : audit [DBG] from='mgr.? 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:20:27.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:27 smithi173 bash[27694]: audit 2023-09-23T18:20:27.252062+0000 mon.c (mon.2) 153 : audit [DBG] from='mgr.? 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T18:20:27.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:27 smithi173 bash[27694]: audit 2023-09-23T18:20:27.252524+0000 mon.c (mon.2) 154 : audit [DBG] from='mgr.? 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:20:27.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:27 smithi173 bash[27694]: audit 2023-09-23T18:20:27.373190+0000 mon.b (mon.1) 92 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:20:28.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:28 smithi098 bash[27349]: cluster 2023-09-23T18:20:27.141291+0000 mgr.x (mgr.25562) 79 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:28.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:28 smithi098 bash[27349]: cluster 2023-09-23T18:20:27.423878+0000 mon.a (mon.0) 827 : cluster [DBG] mgrmap e24: x(active, since 90s), standbys: y 2023-09-23T18:20:28.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:28 smithi098 bash[33599]: cluster 2023-09-23T18:20:27.141291+0000 mgr.x (mgr.25562) 79 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:28.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:28 smithi098 bash[33599]: cluster 2023-09-23T18:20:27.423878+0000 mon.a (mon.0) 827 : cluster [DBG] mgrmap e24: x(active, since 90s), standbys: y 2023-09-23T18:20:28.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:28 smithi173 bash[27694]: cluster 2023-09-23T18:20:27.141291+0000 mgr.x (mgr.25562) 79 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:28.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:28 smithi173 bash[27694]: cluster 2023-09-23T18:20:27.423878+0000 mon.a (mon.0) 827 : cluster [DBG] mgrmap e24: x(active, since 90s), standbys: y 2023-09-23T18:20:28.777 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:28 smithi173 bash[53021]: ts=2023-09-23T18:20:28.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:20:29.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:29 smithi098 bash[33599]: cluster 2023-09-23T18:20:28.432542+0000 mon.a (mon.0) 828 : cluster [DBG] mgrmap e25: x(active, since 91s), standbys: y 2023-09-23T18:20:29.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:29 smithi098 bash[33599]: audit 2023-09-23T18:20:28.847528+0000 mon.a (mon.0) 829 : audit [DBG] from='client.? 172.21.15.98:0/3651487871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:29.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:29 smithi098 bash[27349]: cluster 2023-09-23T18:20:28.432542+0000 mon.a (mon.0) 828 : cluster [DBG] mgrmap e25: x(active, since 91s), standbys: y 2023-09-23T18:20:29.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:29 smithi098 bash[27349]: audit 2023-09-23T18:20:28.847528+0000 mon.a (mon.0) 829 : audit [DBG] from='client.? 172.21.15.98:0/3651487871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:29.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:29 smithi173 bash[27694]: cluster 2023-09-23T18:20:28.432542+0000 mon.a (mon.0) 828 : cluster [DBG] mgrmap e25: x(active, since 91s), standbys: y 2023-09-23T18:20:29.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:29 smithi173 bash[27694]: audit 2023-09-23T18:20:28.847528+0000 mon.a (mon.0) 829 : audit [DBG] from='client.? 172.21.15.98:0/3651487871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:30.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:30 smithi098 bash[27349]: cluster 2023-09-23T18:20:29.142151+0000 mgr.x (mgr.25562) 80 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T18:20:30.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:30 smithi098 bash[33599]: cluster 2023-09-23T18:20:29.142151+0000 mgr.x (mgr.25562) 80 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T18:20:30.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:30 smithi173 bash[27694]: cluster 2023-09-23T18:20:29.142151+0000 mgr.x (mgr.25562) 80 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T18:20:31.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:31 smithi098 bash[33599]: audit 2023-09-23T18:20:31.290168+0000 mon.a (mon.0) 830 : audit [DBG] from='client.? 172.21.15.98:0/608071267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:31.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:31 smithi098 bash[27349]: audit 2023-09-23T18:20:31.290168+0000 mon.a (mon.0) 830 : audit [DBG] from='client.? 172.21.15.98:0/608071267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:31.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:31 smithi173 bash[27694]: audit 2023-09-23T18:20:31.290168+0000 mon.a (mon.0) 830 : audit [DBG] from='client.? 172.21.15.98:0/608071267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:32.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:32 smithi098 bash[33599]: cluster 2023-09-23T18:20:31.142782+0000 mgr.x (mgr.25562) 81 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T18:20:32.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:32 smithi098 bash[27349]: cluster 2023-09-23T18:20:31.142782+0000 mgr.x (mgr.25562) 81 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T18:20:32.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:32 smithi173 bash[27694]: cluster 2023-09-23T18:20:31.142782+0000 mgr.x (mgr.25562) 81 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T18:20:32.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:32 smithi173 bash[50615]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:20:32] "GET /metrics HTTP/1.1" 200 33914 "" "Prometheus/2.33.4" 2023-09-23T18:20:34.583 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[73085]: [23/Sep/2023:18:20:34] ENGINE Bus STOPPING 2023-09-23T18:20:34.584 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[73085]: [23/Sep/2023:18:20:34] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T18:20:34.584 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[73085]: [23/Sep/2023:18:20:34] ENGINE Bus STOPPED 2023-09-23T18:20:34.584 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[73085]: [23/Sep/2023:18:20:34] ENGINE Bus STARTING 2023-09-23T18:20:34.584 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[73085]: debug 2023-09-23T18:20:34.520+0000 7f3610c6b700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:20:34.584 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[73085]: debug 2023-09-23T18:20:34.520+0000 7f3610c6b700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:20:34.584 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[73085]: debug 2023-09-23T18:20:34.520+0000 7f3610c6b700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:20:34.584 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[73085]: debug 2023-09-23T18:20:34.520+0000 7f3610c6b700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:20:34.584 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[73085]: debug 2023-09-23T18:20:34.520+0000 7f3610c6b700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:20:34.584 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[73085]: debug 2023-09-23T18:20:34.520+0000 7f3616c77700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:20:34.584 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[73085]: debug 2023-09-23T18:20:34.520+0000 7f3616c77700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:20:34.584 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[73085]: debug 2023-09-23T18:20:34.520+0000 7f3616c77700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:20:34.585 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[73085]: debug 2023-09-23T18:20:34.520+0000 7f3616c77700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:20:34.585 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[73085]: debug 2023-09-23T18:20:34.520+0000 7f3616c77700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:20:34.585 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[73085]: [23/Sep/2023:18:20:34] ENGINE Serving on http://:::9283 2023-09-23T18:20:34.585 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[73085]: [23/Sep/2023:18:20:34] ENGINE Bus STARTED 2023-09-23T18:20:34.585 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: cluster 2023-09-23T18:20:33.143469+0000 mgr.x (mgr.25562) 82 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T18:20:34.585 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:33.739836+0000 mon.a (mon.0) 831 : audit [DBG] from='client.? 172.21.15.98:0/2114820730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:34.585 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.180475+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:34.586 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.190560+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:34.586 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.197822+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:20:34.586 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.199260+0000 mon.b (mon.1) 94 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:20:34.586 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.201752+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:34.586 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.289533+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:20:34.586 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.294617+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T18:20:34.586 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.298637+0000 mon.b (mon.1) 96 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T18:20:34.586 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: cluster 2023-09-23T18:20:34.303879+0000 mon.a (mon.0) 836 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-09-23T18:20:34.586 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.304741+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T18:20:34.586 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: cluster 2023-09-23T18:20:34.304949+0000 mon.a (mon.0) 838 : cluster [DBG] mgrmap e26: y(active, starting, since 0.00950917s) 2023-09-23T18:20:34.586 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.308566+0000 mon.c (mon.2) 155 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:20:34.586 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.308770+0000 mon.c (mon.2) 156 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:20:34.587 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.308974+0000 mon.c (mon.2) 157 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:20:34.587 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.309387+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi173.qjefje"}]: dispatch 2023-09-23T18:20:34.587 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.309683+0000 mon.c (mon.2) 159 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi098.lowjqo"}]: dispatch 2023-09-23T18:20:34.587 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.310310+0000 mon.c (mon.2) 160 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T18:20:34.587 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.310645+0000 mon.c (mon.2) 161 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:20:34.587 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: cluster 2023-09-23T18:20:33.143469+0000 mgr.x (mgr.25562) 82 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T18:20:34.587 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:33.739836+0000 mon.a (mon.0) 831 : audit [DBG] from='client.? 172.21.15.98:0/2114820730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:34.587 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.180475+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:34.587 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.190560+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:34.588 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.197822+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:20:34.588 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.199260+0000 mon.b (mon.1) 94 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:20:34.588 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.201752+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:34.588 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.289533+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:20:34.588 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.294617+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T18:20:34.588 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.298637+0000 mon.b (mon.1) 96 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T18:20:34.588 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: cluster 2023-09-23T18:20:34.303879+0000 mon.a (mon.0) 836 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-09-23T18:20:34.588 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.304741+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T18:20:34.588 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: cluster 2023-09-23T18:20:34.304949+0000 mon.a (mon.0) 838 : cluster [DBG] mgrmap e26: y(active, starting, since 0.00950917s) 2023-09-23T18:20:34.588 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.308566+0000 mon.c (mon.2) 155 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:20:34.588 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.308770+0000 mon.c (mon.2) 156 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:20:34.588 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.308974+0000 mon.c (mon.2) 157 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:20:34.588 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.309387+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi173.qjefje"}]: dispatch 2023-09-23T18:20:34.589 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.309683+0000 mon.c (mon.2) 159 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi098.lowjqo"}]: dispatch 2023-09-23T18:20:34.589 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.310310+0000 mon.c (mon.2) 160 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T18:20:34.589 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.310645+0000 mon.c (mon.2) 161 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:20:34.589 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.311000+0000 mon.c (mon.2) 162 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:20:34.589 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.311365+0000 mon.c (mon.2) 163 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:20:34.589 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.311689+0000 mon.c (mon.2) 164 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:20:34.589 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.311989+0000 mon.c (mon.2) 165 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:20:34.589 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.312349+0000 mon.c (mon.2) 166 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:20:34.589 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.312645+0000 mon.c (mon.2) 167 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:20:34.590 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.312966+0000 mon.c (mon.2) 168 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:20:34.590 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.313306+0000 mon.c (mon.2) 169 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T18:20:34.590 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.313636+0000 mon.c (mon.2) 170 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T18:20:34.590 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.314223+0000 mon.c (mon.2) 171 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T18:20:34.590 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: cluster 2023-09-23T18:20:34.342068+0000 mon.a (mon.0) 839 : cluster [INF] Manager daemon y is now available 2023-09-23T18:20:34.590 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.377443+0000 mon.c (mon.2) 172 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:20:34.590 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.381438+0000 mon.c (mon.2) 173 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:20:34.590 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.416510+0000 mon.c (mon.2) 174 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:20:34.590 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.434051+0000 mon.c (mon.2) 175 : audit [INF] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:20:34.590 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[33599]: audit 2023-09-23T18:20:34.434596+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.25823 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:20:34.590 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.311000+0000 mon.c (mon.2) 162 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:20:34.591 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.311365+0000 mon.c (mon.2) 163 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:20:34.591 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.311689+0000 mon.c (mon.2) 164 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:20:34.591 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.311989+0000 mon.c (mon.2) 165 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:20:34.591 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.312349+0000 mon.c (mon.2) 166 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:20:34.591 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.312645+0000 mon.c (mon.2) 167 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:20:34.591 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.312966+0000 mon.c (mon.2) 168 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:20:34.591 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.313306+0000 mon.c (mon.2) 169 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T18:20:34.591 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.313636+0000 mon.c (mon.2) 170 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T18:20:34.591 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.314223+0000 mon.c (mon.2) 171 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T18:20:34.591 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: cluster 2023-09-23T18:20:34.342068+0000 mon.a (mon.0) 839 : cluster [INF] Manager daemon y is now available 2023-09-23T18:20:34.591 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.377443+0000 mon.c (mon.2) 172 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:20:34.591 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.381438+0000 mon.c (mon.2) 173 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:20:34.592 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.416510+0000 mon.c (mon.2) 174 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:20:34.592 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.434051+0000 mon.c (mon.2) 175 : audit [INF] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:20:34.592 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:34 smithi098 bash[27349]: audit 2023-09-23T18:20:34.434596+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.25823 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:20:34.593 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[50615]: debug 2023-09-23T18:20:34.305+0000 7fec72e5f700 -1 mgr handle_mgr_map I was active but no longer am 2023-09-23T18:20:34.593 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[50615]: ignoring --setuser ceph since I am not root 2023-09-23T18:20:34.593 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[50615]: ignoring --setgroup ceph since I am not root 2023-09-23T18:20:34.594 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[50615]: debug 2023-09-23T18:20:34.369+0000 7f88bda03700 1 -- 172.21.15.173:0/1579906161 <== mon.1 v2:172.21.15.173:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564d430251e0 con 0x564d43da4400 2023-09-23T18:20:34.594 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[50615]: debug 2023-09-23T18:20:34.545+0000 7f88c7732200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T18:20:34.594 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: cluster 2023-09-23T18:20:33.143469+0000 mgr.x (mgr.25562) 82 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T18:20:34.594 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:33.739836+0000 mon.a (mon.0) 831 : audit [DBG] from='client.? 172.21.15.98:0/2114820730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:34.594 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.180475+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:34.594 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.190560+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:34.594 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.197822+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:20:34.594 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.199260+0000 mon.b (mon.1) 94 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:20:34.595 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.201752+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.25562 ' entity='mgr.x' 2023-09-23T18:20:34.595 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.289533+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:20:34.595 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.294617+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T18:20:34.595 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.298637+0000 mon.b (mon.1) 96 : audit [INF] from='mgr.25562 172.21.15.173:0/645766685' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T18:20:34.595 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: cluster 2023-09-23T18:20:34.303879+0000 mon.a (mon.0) 836 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-09-23T18:20:34.595 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.304741+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.25562 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T18:20:34.595 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: cluster 2023-09-23T18:20:34.304949+0000 mon.a (mon.0) 838 : cluster [DBG] mgrmap e26: y(active, starting, since 0.00950917s) 2023-09-23T18:20:34.595 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.308566+0000 mon.c (mon.2) 155 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:20:34.595 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.308770+0000 mon.c (mon.2) 156 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:20:34.595 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.308974+0000 mon.c (mon.2) 157 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:20:34.595 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.309387+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi173.qjefje"}]: dispatch 2023-09-23T18:20:34.595 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.309683+0000 mon.c (mon.2) 159 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi098.lowjqo"}]: dispatch 2023-09-23T18:20:34.595 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.310310+0000 mon.c (mon.2) 160 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T18:20:34.596 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.310645+0000 mon.c (mon.2) 161 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:20:34.596 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.311000+0000 mon.c (mon.2) 162 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:20:34.596 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.311365+0000 mon.c (mon.2) 163 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:20:34.596 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.311689+0000 mon.c (mon.2) 164 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:20:34.596 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.311989+0000 mon.c (mon.2) 165 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:20:34.596 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.312349+0000 mon.c (mon.2) 166 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:20:34.596 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.312645+0000 mon.c (mon.2) 167 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:20:34.596 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.312966+0000 mon.c (mon.2) 168 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:20:34.596 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.313306+0000 mon.c (mon.2) 169 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T18:20:34.596 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.313636+0000 mon.c (mon.2) 170 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T18:20:34.596 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.314223+0000 mon.c (mon.2) 171 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T18:20:34.596 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: cluster 2023-09-23T18:20:34.342068+0000 mon.a (mon.0) 839 : cluster [INF] Manager daemon y is now available 2023-09-23T18:20:34.596 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.377443+0000 mon.c (mon.2) 172 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:20:34.597 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.381438+0000 mon.c (mon.2) 173 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:20:34.597 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.416510+0000 mon.c (mon.2) 174 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:20:34.597 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.434051+0000 mon.c (mon.2) 175 : audit [INF] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:20:34.597 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:34 smithi173 bash[27694]: audit 2023-09-23T18:20:34.434596+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.25823 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:20:35.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:35 smithi173 bash[27694]: audit 2023-09-23T18:20:34.475467+0000 mon.c (mon.2) 176 : audit [INF] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T18:20:35.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:35 smithi173 bash[27694]: audit 2023-09-23T18:20:34.476304+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.25823 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T18:20:35.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:35 smithi173 bash[27694]: cluster 2023-09-23T18:20:35.310038+0000 mon.a (mon.0) 842 : cluster [DBG] mgrmap e27: y(active, since 1.01459s) 2023-09-23T18:20:35.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:35 smithi173 bash[50615]: debug 2023-09-23T18:20:35.489+0000 7f88c7732200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T18:20:35.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:35 smithi173 bash[50615]: debug 2023-09-23T18:20:35.593+0000 7f88c7732200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T18:20:35.778 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:35 smithi173 bash[50615]: debug 2023-09-23T18:20:35.693+0000 7f88c7732200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T18:20:35.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:35 smithi098 bash[33599]: audit 2023-09-23T18:20:34.475467+0000 mon.c (mon.2) 176 : audit [INF] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T18:20:35.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:35 smithi098 bash[33599]: audit 2023-09-23T18:20:34.476304+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.25823 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T18:20:35.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:35 smithi098 bash[33599]: cluster 2023-09-23T18:20:35.310038+0000 mon.a (mon.0) 842 : cluster [DBG] mgrmap e27: y(active, since 1.01459s) 2023-09-23T18:20:35.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:35 smithi098 bash[27349]: audit 2023-09-23T18:20:34.475467+0000 mon.c (mon.2) 176 : audit [INF] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T18:20:35.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:35 smithi098 bash[27349]: audit 2023-09-23T18:20:34.476304+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.25823 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T18:20:35.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:35 smithi098 bash[27349]: cluster 2023-09-23T18:20:35.310038+0000 mon.a (mon.0) 842 : cluster [DBG] mgrmap e27: y(active, since 1.01459s) 2023-09-23T18:20:36.277 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:35 smithi173 bash[50615]: debug 2023-09-23T18:20:35.961+0000 7f88c7732200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:20:36.714 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:36 smithi173 bash[27694]: cluster 2023-09-23T18:20:35.352126+0000 mgr.y (mgr.25823) 1 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:36.715 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:36 smithi173 bash[27694]: audit 2023-09-23T18:20:36.162185+0000 mon.a (mon.0) 843 : audit [DBG] from='client.? 172.21.15.98:0/1429863013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:36.715 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:36 smithi173 bash[50615]: debug 2023-09-23T18:20:36.329+0000 7f88c7732200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T18:20:36.715 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:36 smithi173 bash[50615]: debug 2023-09-23T18:20:36.493+0000 7f88c7732200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T18:20:36.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:36 smithi098 bash[27349]: cluster 2023-09-23T18:20:35.352126+0000 mgr.y (mgr.25823) 1 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:36.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:36 smithi098 bash[27349]: audit 2023-09-23T18:20:36.162185+0000 mon.a (mon.0) 843 : audit [DBG] from='client.? 172.21.15.98:0/1429863013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:36.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:36 smithi098 bash[33599]: cluster 2023-09-23T18:20:35.352126+0000 mgr.y (mgr.25823) 1 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:36.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:36 smithi098 bash[33599]: audit 2023-09-23T18:20:36.162185+0000 mon.a (mon.0) 843 : audit [DBG] from='client.? 172.21.15.98:0/1429863013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:37.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:36 smithi173 bash[50615]: debug 2023-09-23T18:20:36.709+0000 7f88c7732200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T18:20:37.744 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:37 smithi173 bash[27694]: cluster 2023-09-23T18:20:36.311818+0000 mgr.y (mgr.25823) 2 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:37.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:37 smithi098 bash[27349]: cluster 2023-09-23T18:20:36.311818+0000 mgr.y (mgr.25823) 2 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:37.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:37 smithi098 bash[33599]: cluster 2023-09-23T18:20:36.311818+0000 mgr.y (mgr.25823) 2 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:38.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:37 smithi173 bash[50615]: debug 2023-09-23T18:20:37.737+0000 7f88c7732200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T18:20:38.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:37 smithi173 bash[50615]: debug 2023-09-23T18:20:37.853+0000 7f88c7732200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T18:20:38.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:37 smithi173 bash[50615]: debug 2023-09-23T18:20:37.969+0000 7f88c7732200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T18:20:38.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:38 smithi173 bash[50615]: debug 2023-09-23T18:20:38.461+0000 7f88c7732200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T18:20:38.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:38 smithi173 bash[50615]: debug 2023-09-23T18:20:38.689+0000 7f88c7732200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T18:20:38.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:38 smithi173 bash[27694]: cephadm 2023-09-23T18:20:36.747789+0000 mgr.y (mgr.25823) 3 : cephadm [INF] [23/Sep/2023:18:20:36] ENGINE Bus STARTING 2023-09-23T18:20:38.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:38 smithi173 bash[27694]: cephadm 2023-09-23T18:20:36.869010+0000 mgr.y (mgr.25823) 4 : cephadm [INF] [23/Sep/2023:18:20:36] ENGINE Serving on https://172.21.15.98:7150 2023-09-23T18:20:38.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:38 smithi173 bash[27694]: cephadm 2023-09-23T18:20:36.971240+0000 mgr.y (mgr.25823) 5 : cephadm [INF] [23/Sep/2023:18:20:36] ENGINE Serving on http://172.21.15.98:8765 2023-09-23T18:20:38.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:38 smithi173 bash[27694]: cephadm 2023-09-23T18:20:36.971641+0000 mgr.y (mgr.25823) 6 : cephadm [INF] [23/Sep/2023:18:20:36] ENGINE Bus STARTED 2023-09-23T18:20:38.778 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:38 smithi173 bash[53021]: ts=2023-09-23T18:20:38.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:20:38.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:38 smithi098 bash[33599]: cephadm 2023-09-23T18:20:36.747789+0000 mgr.y (mgr.25823) 3 : cephadm [INF] [23/Sep/2023:18:20:36] ENGINE Bus STARTING 2023-09-23T18:20:38.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:38 smithi098 bash[33599]: cephadm 2023-09-23T18:20:36.869010+0000 mgr.y (mgr.25823) 4 : cephadm [INF] [23/Sep/2023:18:20:36] ENGINE Serving on https://172.21.15.98:7150 2023-09-23T18:20:38.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:38 smithi098 bash[33599]: cephadm 2023-09-23T18:20:36.971240+0000 mgr.y (mgr.25823) 5 : cephadm [INF] [23/Sep/2023:18:20:36] ENGINE Serving on http://172.21.15.98:8765 2023-09-23T18:20:38.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:38 smithi098 bash[33599]: cephadm 2023-09-23T18:20:36.971641+0000 mgr.y (mgr.25823) 6 : cephadm [INF] [23/Sep/2023:18:20:36] ENGINE Bus STARTED 2023-09-23T18:20:38.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:38 smithi098 bash[27349]: cephadm 2023-09-23T18:20:36.747789+0000 mgr.y (mgr.25823) 3 : cephadm [INF] [23/Sep/2023:18:20:36] ENGINE Bus STARTING 2023-09-23T18:20:38.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:38 smithi098 bash[27349]: cephadm 2023-09-23T18:20:36.869010+0000 mgr.y (mgr.25823) 4 : cephadm [INF] [23/Sep/2023:18:20:36] ENGINE Serving on https://172.21.15.98:7150 2023-09-23T18:20:38.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:38 smithi098 bash[27349]: cephadm 2023-09-23T18:20:36.971240+0000 mgr.y (mgr.25823) 5 : cephadm [INF] [23/Sep/2023:18:20:36] ENGINE Serving on http://172.21.15.98:8765 2023-09-23T18:20:38.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:38 smithi098 bash[27349]: cephadm 2023-09-23T18:20:36.971641+0000 mgr.y (mgr.25823) 6 : cephadm [INF] [23/Sep/2023:18:20:36] ENGINE Bus STARTED 2023-09-23T18:20:39.486 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:39 smithi173 bash[50615]: debug 2023-09-23T18:20:39.157+0000 7f88c7732200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T18:20:39.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:39 smithi173 bash[50615]: debug 2023-09-23T18:20:39.665+0000 7f88c7732200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:20:39.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:39 smithi173 bash[27694]: cluster 2023-09-23T18:20:38.312295+0000 mgr.y (mgr.25823) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:39.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:39 smithi173 bash[27694]: audit 2023-09-23T18:20:38.606699+0000 mon.a (mon.0) 844 : audit [DBG] from='client.? 172.21.15.98:0/1320937788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:39.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:39 smithi098 bash[27349]: cluster 2023-09-23T18:20:38.312295+0000 mgr.y (mgr.25823) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:39.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:39 smithi098 bash[27349]: audit 2023-09-23T18:20:38.606699+0000 mon.a (mon.0) 844 : audit [DBG] from='client.? 172.21.15.98:0/1320937788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:39.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:39 smithi098 bash[33599]: cluster 2023-09-23T18:20:38.312295+0000 mgr.y (mgr.25823) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:39.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:39 smithi098 bash[33599]: audit 2023-09-23T18:20:38.606699+0000 mon.a (mon.0) 844 : audit [DBG] from='client.? 172.21.15.98:0/1320937788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:40.527 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:40 smithi173 bash[50615]: debug 2023-09-23T18:20:40.261+0000 7f88c7732200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T18:20:40.527 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:40 smithi173 bash[50615]: debug 2023-09-23T18:20:40.385+0000 7f88c7732200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T18:20:41.066 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:40 smithi173 bash[50615]: debug 2023-09-23T18:20:40.777+0000 7f88c7732200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T18:20:41.066 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:40 smithi173 bash[50615]: debug 2023-09-23T18:20:40.905+0000 7f88c7732200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T18:20:41.325 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:41 smithi173 bash[50615]: debug 2023-09-23T18:20:41.061+0000 7f88c7732200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T18:20:41.326 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:41 smithi173 bash[50615]: debug 2023-09-23T18:20:41.185+0000 7f88c7732200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T18:20:41.655 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:41 smithi173 bash[50615]: debug 2023-09-23T18:20:41.321+0000 7f88c7732200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T18:20:41.655 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:41 smithi173 bash[50615]: debug 2023-09-23T18:20:41.517+0000 7f88c7732200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T18:20:41.656 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:41 smithi173 bash[50615]: debug 2023-09-23T18:20:41.649+0000 7f88c7732200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T18:20:41.656 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:41 smithi173 bash[27694]: cluster 2023-09-23T18:20:40.312826+0000 mgr.y (mgr.25823) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:41.656 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:41 smithi173 bash[27694]: audit 2023-09-23T18:20:40.905153+0000 mon.a (mon.0) 845 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:41.656 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:41 smithi173 bash[27694]: audit 2023-09-23T18:20:40.914671+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:41.656 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:41 smithi173 bash[27694]: audit 2023-09-23T18:20:40.973894+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:41.656 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:41 smithi173 bash[27694]: audit 2023-09-23T18:20:40.982344+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:41.656 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:41 smithi173 bash[27694]: audit 2023-09-23T18:20:41.089059+0000 mon.c (mon.2) 177 : audit [DBG] from='client.? 172.21.15.98:0/2624507905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:41.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:41 smithi098 bash[27349]: cluster 2023-09-23T18:20:40.312826+0000 mgr.y (mgr.25823) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:41.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:41 smithi098 bash[27349]: audit 2023-09-23T18:20:40.905153+0000 mon.a (mon.0) 845 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:41.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:41 smithi098 bash[27349]: audit 2023-09-23T18:20:40.914671+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:41.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:41 smithi098 bash[27349]: audit 2023-09-23T18:20:40.973894+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:41.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:41 smithi098 bash[27349]: audit 2023-09-23T18:20:40.982344+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:41.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:41 smithi098 bash[27349]: audit 2023-09-23T18:20:41.089059+0000 mon.c (mon.2) 177 : audit [DBG] from='client.? 172.21.15.98:0/2624507905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:41.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:41 smithi098 bash[33599]: cluster 2023-09-23T18:20:40.312826+0000 mgr.y (mgr.25823) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:41.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:41 smithi098 bash[33599]: audit 2023-09-23T18:20:40.905153+0000 mon.a (mon.0) 845 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:41.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:41 smithi098 bash[33599]: audit 2023-09-23T18:20:40.914671+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:41.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:41 smithi098 bash[33599]: audit 2023-09-23T18:20:40.973894+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:41.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:41 smithi098 bash[33599]: audit 2023-09-23T18:20:40.982344+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:41.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:41 smithi098 bash[33599]: audit 2023-09-23T18:20:41.089059+0000 mon.c (mon.2) 177 : audit [DBG] from='client.? 172.21.15.98:0/2624507905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:43.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:42 smithi098 bash[33599]: audit 2023-09-23T18:20:41.785445+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:43.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:42 smithi098 bash[33599]: audit 2023-09-23T18:20:41.796979+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:43.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:42 smithi098 bash[33599]: audit 2023-09-23T18:20:41.816251+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:43.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:42 smithi098 bash[33599]: audit 2023-09-23T18:20:41.825322+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:43.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:42 smithi098 bash[33599]: cluster 2023-09-23T18:20:42.313823+0000 mgr.y (mgr.25823) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:43.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:42 smithi098 bash[27349]: audit 2023-09-23T18:20:41.785445+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:43.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:42 smithi098 bash[27349]: audit 2023-09-23T18:20:41.796979+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:43.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:42 smithi098 bash[27349]: audit 2023-09-23T18:20:41.816251+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:43.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:42 smithi098 bash[27349]: audit 2023-09-23T18:20:41.825322+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:43.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:42 smithi098 bash[27349]: cluster 2023-09-23T18:20:42.313823+0000 mgr.y (mgr.25823) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:43.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:42 smithi173 bash[27694]: audit 2023-09-23T18:20:41.785445+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:43.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:42 smithi173 bash[27694]: audit 2023-09-23T18:20:41.796979+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:43.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:42 smithi173 bash[27694]: audit 2023-09-23T18:20:41.816251+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:43.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:42 smithi173 bash[27694]: audit 2023-09-23T18:20:41.825322+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:43.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:42 smithi173 bash[27694]: cluster 2023-09-23T18:20:42.313823+0000 mgr.y (mgr.25823) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:43.278 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:42 smithi173 bash[50615]: debug 2023-09-23T18:20:42.909+0000 7f88c7732200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T18:20:43.278 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:42 smithi173 bash[50615]: [23/Sep/2023:18:20:42] ENGINE Bus STARTING 2023-09-23T18:20:43.278 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:43 smithi173 bash[50615]: CherryPy Checker: 2023-09-23T18:20:43.278 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:43 smithi173 bash[50615]: The Application mounted at '' has an empty config. 2023-09-23T18:20:43.278 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:43 smithi173 bash[50615]: [23/Sep/2023:18:20:43] ENGINE Serving on http://:::9283 2023-09-23T18:20:43.278 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:43 smithi173 bash[50615]: [23/Sep/2023:18:20:43] ENGINE Bus STARTED 2023-09-23T18:20:44.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:43 smithi098 bash[33599]: cluster 2023-09-23T18:20:42.911177+0000 mon.a (mon.0) 853 : cluster [DBG] Standby manager daemon x started 2023-09-23T18:20:44.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:43 smithi098 bash[33599]: audit 2023-09-23T18:20:42.914893+0000 mon.c (mon.2) 178 : audit [DBG] from='mgr.? 172.21.15.173:0/2238314545' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T18:20:44.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:43 smithi098 bash[33599]: audit 2023-09-23T18:20:42.916139+0000 mon.c (mon.2) 179 : audit [DBG] from='mgr.? 172.21.15.173:0/2238314545' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:20:44.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:43 smithi098 bash[33599]: audit 2023-09-23T18:20:42.920163+0000 mon.c (mon.2) 180 : audit [DBG] from='mgr.? 172.21.15.173:0/2238314545' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T18:20:44.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:43 smithi098 bash[33599]: audit 2023-09-23T18:20:42.921809+0000 mon.c (mon.2) 181 : audit [DBG] from='mgr.? 172.21.15.173:0/2238314545' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:20:44.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:43 smithi098 bash[33599]: audit 2023-09-23T18:20:43.562200+0000 mon.c (mon.2) 182 : audit [DBG] from='client.? 172.21.15.98:0/318574209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:44.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:43 smithi098 bash[27349]: cluster 2023-09-23T18:20:42.911177+0000 mon.a (mon.0) 853 : cluster [DBG] Standby manager daemon x started 2023-09-23T18:20:44.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:43 smithi098 bash[27349]: audit 2023-09-23T18:20:42.914893+0000 mon.c (mon.2) 178 : audit [DBG] from='mgr.? 172.21.15.173:0/2238314545' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T18:20:44.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:43 smithi098 bash[27349]: audit 2023-09-23T18:20:42.916139+0000 mon.c (mon.2) 179 : audit [DBG] from='mgr.? 172.21.15.173:0/2238314545' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:20:44.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:43 smithi098 bash[27349]: audit 2023-09-23T18:20:42.920163+0000 mon.c (mon.2) 180 : audit [DBG] from='mgr.? 172.21.15.173:0/2238314545' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T18:20:44.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:43 smithi098 bash[27349]: audit 2023-09-23T18:20:42.921809+0000 mon.c (mon.2) 181 : audit [DBG] from='mgr.? 172.21.15.173:0/2238314545' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:20:44.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:43 smithi098 bash[27349]: audit 2023-09-23T18:20:43.562200+0000 mon.c (mon.2) 182 : audit [DBG] from='client.? 172.21.15.98:0/318574209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:44.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:43 smithi173 bash[27694]: cluster 2023-09-23T18:20:42.911177+0000 mon.a (mon.0) 853 : cluster [DBG] Standby manager daemon x started 2023-09-23T18:20:44.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:43 smithi173 bash[27694]: audit 2023-09-23T18:20:42.914893+0000 mon.c (mon.2) 178 : audit [DBG] from='mgr.? 172.21.15.173:0/2238314545' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T18:20:44.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:43 smithi173 bash[27694]: audit 2023-09-23T18:20:42.916139+0000 mon.c (mon.2) 179 : audit [DBG] from='mgr.? 172.21.15.173:0/2238314545' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:20:44.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:43 smithi173 bash[27694]: audit 2023-09-23T18:20:42.920163+0000 mon.c (mon.2) 180 : audit [DBG] from='mgr.? 172.21.15.173:0/2238314545' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T18:20:44.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:43 smithi173 bash[27694]: audit 2023-09-23T18:20:42.921809+0000 mon.c (mon.2) 181 : audit [DBG] from='mgr.? 172.21.15.173:0/2238314545' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:20:44.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:43 smithi173 bash[27694]: audit 2023-09-23T18:20:43.562200+0000 mon.c (mon.2) 182 : audit [DBG] from='client.? 172.21.15.98:0/318574209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:45.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:44 smithi098 bash[27349]: cluster 2023-09-23T18:20:43.840795+0000 mon.a (mon.0) 854 : cluster [DBG] mgrmap e28: y(active, since 9s), standbys: x 2023-09-23T18:20:45.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:44 smithi098 bash[27349]: audit 2023-09-23T18:20:43.844090+0000 mon.c (mon.2) 183 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T18:20:45.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:44 smithi098 bash[27349]: cluster 2023-09-23T18:20:44.314493+0000 mgr.y (mgr.25823) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:45.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:44 smithi098 bash[33599]: cluster 2023-09-23T18:20:43.840795+0000 mon.a (mon.0) 854 : cluster [DBG] mgrmap e28: y(active, since 9s), standbys: x 2023-09-23T18:20:45.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:44 smithi098 bash[33599]: audit 2023-09-23T18:20:43.844090+0000 mon.c (mon.2) 183 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T18:20:45.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:44 smithi098 bash[33599]: cluster 2023-09-23T18:20:44.314493+0000 mgr.y (mgr.25823) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:45.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:44 smithi173 bash[27694]: cluster 2023-09-23T18:20:43.840795+0000 mon.a (mon.0) 854 : cluster [DBG] mgrmap e28: y(active, since 9s), standbys: x 2023-09-23T18:20:45.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:44 smithi173 bash[27694]: audit 2023-09-23T18:20:43.844090+0000 mon.c (mon.2) 183 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T18:20:45.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:44 smithi173 bash[27694]: cluster 2023-09-23T18:20:44.314493+0000 mgr.y (mgr.25823) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:46.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:46 smithi098 bash[27349]: audit 2023-09-23T18:20:46.013844+0000 mon.a (mon.0) 855 : audit [DBG] from='client.? 172.21.15.98:0/4178649806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:46.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:46 smithi098 bash[33599]: audit 2023-09-23T18:20:46.013844+0000 mon.a (mon.0) 855 : audit [DBG] from='client.? 172.21.15.98:0/4178649806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:46.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:46 smithi173 bash[27694]: audit 2023-09-23T18:20:46.013844+0000 mon.a (mon.0) 855 : audit [DBG] from='client.? 172.21.15.98:0/4178649806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:47.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:46 smithi173 bash[53021]: ts=2023-09-23T18:20:46.627Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.173:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.173:8765: connect: connection refused" 2023-09-23T18:20:47.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:46 smithi173 bash[53021]: ts=2023-09-23T18:20:46.629Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.173:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.173:8765: connect: connection refused" 2023-09-23T18:20:47.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:46 smithi173 bash[53021]: ts=2023-09-23T18:20:46.630Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.173:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.173:8765: connect: connection refused" 2023-09-23T18:20:47.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:46 smithi173 bash[53021]: ts=2023-09-23T18:20:46.630Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.173:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.173:8765: connect: connection refused" 2023-09-23T18:20:47.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:47 smithi098 bash[27349]: cluster 2023-09-23T18:20:46.315507+0000 mgr.y (mgr.25823) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:47.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:47 smithi098 bash[33599]: cluster 2023-09-23T18:20:46.315507+0000 mgr.y (mgr.25823) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:47.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:47 smithi173 bash[27694]: cluster 2023-09-23T18:20:46.315507+0000 mgr.y (mgr.25823) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:49.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:49 smithi098 bash[27349]: cluster 2023-09-23T18:20:48.315930+0000 mgr.y (mgr.25823) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:49.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:49 smithi098 bash[27349]: audit 2023-09-23T18:20:48.454074+0000 mon.a (mon.0) 856 : audit [DBG] from='client.? 172.21.15.98:0/2671068812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:49.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:49 smithi098 bash[27349]: audit 2023-09-23T18:20:49.416999+0000 mon.c (mon.2) 184 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:20:49.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:49 smithi098 bash[33599]: cluster 2023-09-23T18:20:48.315930+0000 mgr.y (mgr.25823) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:49.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:49 smithi098 bash[33599]: audit 2023-09-23T18:20:48.454074+0000 mon.a (mon.0) 856 : audit [DBG] from='client.? 172.21.15.98:0/2671068812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:49.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:49 smithi098 bash[33599]: audit 2023-09-23T18:20:49.416999+0000 mon.c (mon.2) 184 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:20:49.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:49 smithi173 bash[27694]: cluster 2023-09-23T18:20:48.315930+0000 mgr.y (mgr.25823) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:49.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:49 smithi173 bash[27694]: audit 2023-09-23T18:20:48.454074+0000 mon.a (mon.0) 856 : audit [DBG] from='client.? 172.21.15.98:0/2671068812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:49.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:49 smithi173 bash[27694]: audit 2023-09-23T18:20:49.416999+0000 mon.c (mon.2) 184 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:20:51.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:51 smithi173 bash[27694]: cluster 2023-09-23T18:20:50.316531+0000 mgr.y (mgr.25823) 13 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:51.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:51 smithi173 bash[27694]: audit 2023-09-23T18:20:50.901434+0000 mon.c (mon.2) 185 : audit [DBG] from='client.? 172.21.15.98:0/4086594638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:51.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:51 smithi098 bash[27349]: cluster 2023-09-23T18:20:50.316531+0000 mgr.y (mgr.25823) 13 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:51.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:51 smithi098 bash[27349]: audit 2023-09-23T18:20:50.901434+0000 mon.c (mon.2) 185 : audit [DBG] from='client.? 172.21.15.98:0/4086594638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:51.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:51 smithi098 bash[33599]: cluster 2023-09-23T18:20:50.316531+0000 mgr.y (mgr.25823) 13 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:51.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:51 smithi098 bash[33599]: audit 2023-09-23T18:20:50.901434+0000 mon.c (mon.2) 185 : audit [DBG] from='client.? 172.21.15.98:0/4086594638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:52.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[27349]: audit 2023-09-23T18:20:51.676586+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:52.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[27349]: audit 2023-09-23T18:20:51.685782+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:52.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[27349]: audit 2023-09-23T18:20:51.688719+0000 mon.c (mon.2) 186 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:20:52.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[27349]: audit 2023-09-23T18:20:51.690142+0000 mon.c (mon.2) 187 : audit [INF] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:20:52.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[27349]: cephadm 2023-09-23T18:20:51.692244+0000 mgr.y (mgr.25823) 14 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.conf 2023-09-23T18:20:52.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[27349]: cephadm 2023-09-23T18:20:51.692594+0000 mgr.y (mgr.25823) 15 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-09-23T18:20:52.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[27349]: cephadm 2023-09-23T18:20:51.857117+0000 mgr.y (mgr.25823) 16 : cephadm [INF] Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:20:52.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[27349]: cephadm 2023-09-23T18:20:51.857679+0000 mgr.y (mgr.25823) 17 : cephadm [INF] Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:20:52.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[27349]: cephadm 2023-09-23T18:20:52.022034+0000 mgr.y (mgr.25823) 18 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:20:52.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[27349]: cephadm 2023-09-23T18:20:52.022405+0000 mgr.y (mgr.25823) 19 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:20:52.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[27349]: cephadm 2023-09-23T18:20:52.237062+0000 mgr.y (mgr.25823) 20 : cephadm [INF] Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:20:52.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[27349]: cephadm 2023-09-23T18:20:52.238837+0000 mgr.y (mgr.25823) 21 : cephadm [INF] Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:20:52.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[27349]: cluster 2023-09-23T18:20:52.317109+0000 mgr.y (mgr.25823) 22 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:52.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[27349]: audit 2023-09-23T18:20:52.412371+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:52.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[27349]: audit 2023-09-23T18:20:52.423663+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:52.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[27349]: audit 2023-09-23T18:20:52.461750+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:52.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[27349]: audit 2023-09-23T18:20:52.470141+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:52.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[27349]: audit 2023-09-23T18:20:52.478559+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:52.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[33599]: audit 2023-09-23T18:20:51.676586+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:52.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[33599]: audit 2023-09-23T18:20:51.685782+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:52.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[33599]: audit 2023-09-23T18:20:51.688719+0000 mon.c (mon.2) 186 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:20:52.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[33599]: audit 2023-09-23T18:20:51.690142+0000 mon.c (mon.2) 187 : audit [INF] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:20:52.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[33599]: cephadm 2023-09-23T18:20:51.692244+0000 mgr.y (mgr.25823) 14 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.conf 2023-09-23T18:20:52.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[33599]: cephadm 2023-09-23T18:20:51.692594+0000 mgr.y (mgr.25823) 15 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-09-23T18:20:52.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[33599]: cephadm 2023-09-23T18:20:51.857117+0000 mgr.y (mgr.25823) 16 : cephadm [INF] Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:20:52.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[33599]: cephadm 2023-09-23T18:20:51.857679+0000 mgr.y (mgr.25823) 17 : cephadm [INF] Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:20:52.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[33599]: cephadm 2023-09-23T18:20:52.022034+0000 mgr.y (mgr.25823) 18 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:20:52.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[33599]: cephadm 2023-09-23T18:20:52.022405+0000 mgr.y (mgr.25823) 19 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:20:52.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[33599]: cephadm 2023-09-23T18:20:52.237062+0000 mgr.y (mgr.25823) 20 : cephadm [INF] Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:20:52.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[33599]: cephadm 2023-09-23T18:20:52.238837+0000 mgr.y (mgr.25823) 21 : cephadm [INF] Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:20:52.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[33599]: cluster 2023-09-23T18:20:52.317109+0000 mgr.y (mgr.25823) 22 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:52.948 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[33599]: audit 2023-09-23T18:20:52.412371+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:52.948 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[33599]: audit 2023-09-23T18:20:52.423663+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:52.948 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[33599]: audit 2023-09-23T18:20:52.461750+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:52.948 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[33599]: audit 2023-09-23T18:20:52.470141+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:52.948 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:52 smithi098 bash[33599]: audit 2023-09-23T18:20:52.478559+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:53.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:20:52 smithi173 bash[50615]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:20:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T18:20:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:52 smithi173 bash[27694]: audit 2023-09-23T18:20:51.676586+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:52 smithi173 bash[27694]: audit 2023-09-23T18:20:51.685782+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:52 smithi173 bash[27694]: audit 2023-09-23T18:20:51.688719+0000 mon.c (mon.2) 186 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:20:53.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:52 smithi173 bash[27694]: audit 2023-09-23T18:20:51.690142+0000 mon.c (mon.2) 187 : audit [INF] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:20:53.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:52 smithi173 bash[27694]: cephadm 2023-09-23T18:20:51.692244+0000 mgr.y (mgr.25823) 14 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.conf 2023-09-23T18:20:53.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:52 smithi173 bash[27694]: cephadm 2023-09-23T18:20:51.692594+0000 mgr.y (mgr.25823) 15 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-09-23T18:20:53.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:52 smithi173 bash[27694]: cephadm 2023-09-23T18:20:51.857117+0000 mgr.y (mgr.25823) 16 : cephadm [INF] Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:20:53.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:52 smithi173 bash[27694]: cephadm 2023-09-23T18:20:51.857679+0000 mgr.y (mgr.25823) 17 : cephadm [INF] Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:20:53.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:52 smithi173 bash[27694]: cephadm 2023-09-23T18:20:52.022034+0000 mgr.y (mgr.25823) 18 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:20:53.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:52 smithi173 bash[27694]: cephadm 2023-09-23T18:20:52.022405+0000 mgr.y (mgr.25823) 19 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:20:53.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:52 smithi173 bash[27694]: cephadm 2023-09-23T18:20:52.237062+0000 mgr.y (mgr.25823) 20 : cephadm [INF] Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:20:53.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:52 smithi173 bash[27694]: cephadm 2023-09-23T18:20:52.238837+0000 mgr.y (mgr.25823) 21 : cephadm [INF] Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:20:53.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:52 smithi173 bash[27694]: cluster 2023-09-23T18:20:52.317109+0000 mgr.y (mgr.25823) 22 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:53.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:52 smithi173 bash[27694]: audit 2023-09-23T18:20:52.412371+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:53.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:52 smithi173 bash[27694]: audit 2023-09-23T18:20:52.423663+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:53.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:52 smithi173 bash[27694]: audit 2023-09-23T18:20:52.461750+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:53.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:52 smithi173 bash[27694]: audit 2023-09-23T18:20:52.470141+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:53.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:52 smithi173 bash[27694]: audit 2023-09-23T18:20:52.478559+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:54.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:53 smithi173 bash[27694]: cephadm 2023-09-23T18:20:52.554522+0000 mgr.y (mgr.25823) 23 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T18:20:54.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:53 smithi173 bash[27694]: cephadm 2023-09-23T18:20:52.800973+0000 mgr.y (mgr.25823) 24 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi173 2023-09-23T18:20:54.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:53 smithi173 bash[27694]: audit 2023-09-23T18:20:53.337856+0000 mon.a (mon.0) 864 : audit [DBG] from='client.? 172.21.15.98:0/1673976608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:54.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:53 smithi098 bash[27349]: cephadm 2023-09-23T18:20:52.554522+0000 mgr.y (mgr.25823) 23 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T18:20:54.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:53 smithi098 bash[27349]: cephadm 2023-09-23T18:20:52.800973+0000 mgr.y (mgr.25823) 24 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi173 2023-09-23T18:20:54.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:53 smithi098 bash[27349]: audit 2023-09-23T18:20:53.337856+0000 mon.a (mon.0) 864 : audit [DBG] from='client.? 172.21.15.98:0/1673976608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:54.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:53 smithi098 bash[33599]: cephadm 2023-09-23T18:20:52.554522+0000 mgr.y (mgr.25823) 23 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T18:20:54.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:53 smithi098 bash[33599]: cephadm 2023-09-23T18:20:52.800973+0000 mgr.y (mgr.25823) 24 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi173 2023-09-23T18:20:54.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:53 smithi098 bash[33599]: audit 2023-09-23T18:20:53.337856+0000 mon.a (mon.0) 864 : audit [DBG] from='client.? 172.21.15.98:0/1673976608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:54.776 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:54 smithi173 bash[27694]: cluster 2023-09-23T18:20:54.317700+0000 mgr.y (mgr.25823) 25 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:55.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:54 smithi098 bash[33599]: cluster 2023-09-23T18:20:54.317700+0000 mgr.y (mgr.25823) 25 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:55.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:54 smithi098 bash[27349]: cluster 2023-09-23T18:20:54.317700+0000 mgr.y (mgr.25823) 25 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:56.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:55 smithi098 bash[33599]: audit 2023-09-23T18:20:55.788124+0000 mon.c (mon.2) 188 : audit [DBG] from='client.? 172.21.15.98:0/3904474758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:56.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:55 smithi098 bash[27349]: audit 2023-09-23T18:20:55.788124+0000 mon.c (mon.2) 188 : audit [DBG] from='client.? 172.21.15.98:0/3904474758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:56.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:55 smithi173 bash[27694]: audit 2023-09-23T18:20:55.788124+0000 mon.c (mon.2) 188 : audit [DBG] from='client.? 172.21.15.98:0/3904474758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:57.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:56 smithi173 bash[27694]: cluster 2023-09-23T18:20:56.318892+0000 mgr.y (mgr.25823) 26 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:57.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:56 smithi098 bash[33599]: cluster 2023-09-23T18:20:56.318892+0000 mgr.y (mgr.25823) 26 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:57.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:56 smithi098 bash[27349]: cluster 2023-09-23T18:20:56.318892+0000 mgr.y (mgr.25823) 26 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:57.777 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:57 smithi173 systemd[1]: Stopping Ceph prometheus.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T18:20:57.778 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:57 smithi173 bash[54851]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-prometheus.a 2023-09-23T18:20:57.778 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:57 smithi173 bash[53021]: ts=2023-09-23T18:20:57.595Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T18:20:57.778 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:57 smithi173 bash[53021]: ts=2023-09-23T18:20:57.595Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-09-23T18:20:57.778 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:57 smithi173 bash[53021]: ts=2023-09-23T18:20:57.595Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-09-23T18:20:57.778 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:57 smithi173 bash[53021]: ts=2023-09-23T18:20:57.595Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-09-23T18:20:57.778 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:57 smithi173 bash[53021]: ts=2023-09-23T18:20:57.595Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-09-23T18:20:57.778 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:57 smithi173 bash[53021]: ts=2023-09-23T18:20:57.595Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-09-23T18:20:57.778 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:57 smithi173 bash[53021]: ts=2023-09-23T18:20:57.596Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T18:20:57.778 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:57 smithi173 bash[53021]: ts=2023-09-23T18:20:57.596Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-09-23T18:20:57.778 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:57 smithi173 bash[53021]: ts=2023-09-23T18:20:57.596Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T18:20:57.778 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:57 smithi173 bash[53021]: ts=2023-09-23T18:20:57.596Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T18:20:57.779 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:57 smithi173 bash[53021]: ts=2023-09-23T18:20:57.596Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-09-23T18:20:57.779 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:57 smithi173 bash[53021]: ts=2023-09-23T18:20:57.596Z caller=main.go:1066 level=info msg="See you next time!" 2023-09-23T18:20:58.429 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:58 smithi173 bash[27694]: audit 2023-09-23T18:20:58.225598+0000 mon.c (mon.2) 189 : audit [DBG] from='client.? 172.21.15.98:0/2926055097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:58.430 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:58 smithi173 bash[54888]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-prometheus-a 2023-09-23T18:20:58.430 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:58 smithi173 bash[54953]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-prometheus-a 2023-09-23T18:20:58.430 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:58 smithi173 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T18:20:58.430 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:58 smithi173 systemd[1]: Stopped Ceph prometheus.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:20:58.430 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:58 smithi173 systemd[1]: Started Ceph prometheus.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:20:58.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:58 smithi098 bash[33599]: audit 2023-09-23T18:20:58.225598+0000 mon.c (mon.2) 189 : audit [DBG] from='client.? 172.21.15.98:0/2926055097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:58.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:58 smithi098 bash[27349]: audit 2023-09-23T18:20:58.225598+0000 mon.c (mon.2) 189 : audit [DBG] from='client.? 172.21.15.98:0/2926055097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:20:59.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:59 smithi098 bash[33599]: cluster 2023-09-23T18:20:58.319502+0000 mgr.y (mgr.25823) 27 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:59.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:59 smithi098 bash[33599]: audit 2023-09-23T18:20:58.400677+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:59.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:59 smithi098 bash[33599]: audit 2023-09-23T18:20:58.410187+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:59.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:59 smithi098 bash[33599]: audit 2023-09-23T18:20:58.445017+0000 mon.c (mon.2) 190 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T18:20:59.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:59 smithi098 bash[33599]: audit 2023-09-23T18:20:58.445931+0000 mgr.y (mgr.25823) 28 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T18:20:59.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:20:59 smithi098 bash[33599]: audit 2023-09-23T18:20:58.453207+0000 mon.c (mon.2) 191 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:20:59.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:59 smithi098 bash[27349]: cluster 2023-09-23T18:20:58.319502+0000 mgr.y (mgr.25823) 27 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:59.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:59 smithi098 bash[27349]: audit 2023-09-23T18:20:58.400677+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:59.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:59 smithi098 bash[27349]: audit 2023-09-23T18:20:58.410187+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:59.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:59 smithi098 bash[27349]: audit 2023-09-23T18:20:58.445017+0000 mon.c (mon.2) 190 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T18:20:59.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:59 smithi098 bash[27349]: audit 2023-09-23T18:20:58.445931+0000 mgr.y (mgr.25823) 28 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T18:20:59.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:20:59 smithi098 bash[27349]: audit 2023-09-23T18:20:58.453207+0000 mon.c (mon.2) 191 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:20:59.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:59 smithi173 bash[27694]: cluster 2023-09-23T18:20:58.319502+0000 mgr.y (mgr.25823) 27 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:20:59.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:59 smithi173 bash[27694]: audit 2023-09-23T18:20:58.400677+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:59.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:59 smithi173 bash[27694]: audit 2023-09-23T18:20:58.410187+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:20:59.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:59 smithi173 bash[27694]: audit 2023-09-23T18:20:58.445017+0000 mon.c (mon.2) 190 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T18:20:59.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:59 smithi173 bash[27694]: audit 2023-09-23T18:20:58.445931+0000 mgr.y (mgr.25823) 28 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T18:20:59.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:20:59 smithi173 bash[27694]: audit 2023-09-23T18:20:58.453207+0000 mon.c (mon.2) 191 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:21:00.102 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:59 smithi173 bash[54982]: ts=2023-09-23T18:20:59.952Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-09-23T18:21:00.102 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:59 smithi173 bash[54982]: ts=2023-09-23T18:20:59.952Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-09-23T18:21:00.102 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:59 smithi173 bash[54982]: ts=2023-09-23T18:20:59.952Z caller=main.go:518 level=info host_details="(Linux 5.15.0-84-generic #93~20.04.1-Ubuntu SMP Wed Sep 6 16:15:40 UTC 2023 x86_64 smithi173 (none))" 2023-09-23T18:21:00.102 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:59 smithi173 bash[54982]: ts=2023-09-23T18:20:59.952Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T18:21:00.102 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:59 smithi173 bash[54982]: ts=2023-09-23T18:20:59.952Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T18:21:00.102 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:59 smithi173 bash[54982]: ts=2023-09-23T18:20:59.954Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T18:21:00.102 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:59 smithi173 bash[54982]: ts=2023-09-23T18:20:59.955Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-09-23T18:21:00.102 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:59 smithi173 bash[54982]: ts=2023-09-23T18:20:59.958Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-09-23T18:21:00.103 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:59 smithi173 bash[54982]: ts=2023-09-23T18:20:59.962Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T18:21:00.103 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:59 smithi173 bash[54982]: ts=2023-09-23T18:20:59.962Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.844µs 2023-09-23T18:21:00.103 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:20:59 smithi173 bash[54982]: ts=2023-09-23T18:20:59.962Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T18:21:00.376 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:21:00 smithi173 bash[54982]: ts=2023-09-23T18:21:00.101Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-09-23T18:21:00.777 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:21:00 smithi173 bash[54982]: ts=2023-09-23T18:21:00.375Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-09-23T18:21:00.777 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:21:00 smithi173 bash[54982]: ts=2023-09-23T18:21:00.408Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-09-23T18:21:00.778 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:21:00 smithi173 bash[54982]: ts=2023-09-23T18:21:00.408Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-09-23T18:21:00.778 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:21:00 smithi173 bash[54982]: ts=2023-09-23T18:21:00.408Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=81.823µs wal_replay_duration=446.441872ms total_replay_duration=446.549625ms 2023-09-23T18:21:00.778 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:21:00 smithi173 bash[54982]: ts=2023-09-23T18:21:00.413Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T18:21:00.778 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:21:00 smithi173 bash[54982]: ts=2023-09-23T18:21:00.413Z caller=main.go:947 level=info msg="TSDB started" 2023-09-23T18:21:00.778 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:21:00 smithi173 bash[54982]: ts=2023-09-23T18:21:00.413Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T18:21:00.778 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:21:00 smithi173 bash[54982]: ts=2023-09-23T18:21:00.439Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=25.81506ms db_storage=1.18µs remote_storage=2.245µs web_handler=470ns query_engine=1.088µs scrape=402.365µs scrape_sd=177.536µs notify=28.64µs notify_sd=48.964µs rules=24.714704ms 2023-09-23T18:21:00.778 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:21:00 smithi173 bash[54982]: ts=2023-09-23T18:21:00.439Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-09-23T18:21:01.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:01 smithi098 bash[33599]: cluster 2023-09-23T18:21:00.320148+0000 mgr.y (mgr.25823) 29 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:01.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:01 smithi098 bash[33599]: audit 2023-09-23T18:21:00.672441+0000 mon.c (mon.2) 192 : audit [DBG] from='client.? 172.21.15.98:0/326554328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:01.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:01 smithi098 bash[33599]: audit 2023-09-23T18:21:00.979675+0000 mon.c (mon.2) 193 : audit [INF] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:21:01.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:01 smithi098 bash[33599]: audit 2023-09-23T18:21:00.980562+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.25823 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:21:01.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:01 smithi098 bash[33599]: audit 2023-09-23T18:21:00.983041+0000 mon.c (mon.2) 194 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:21:01.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:01 smithi098 bash[33599]: audit 2023-09-23T18:21:00.984947+0000 mon.c (mon.2) 195 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:21:01.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:01 smithi098 bash[27349]: cluster 2023-09-23T18:21:00.320148+0000 mgr.y (mgr.25823) 29 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:01.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:01 smithi098 bash[27349]: audit 2023-09-23T18:21:00.672441+0000 mon.c (mon.2) 192 : audit [DBG] from='client.? 172.21.15.98:0/326554328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:01.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:01 smithi098 bash[27349]: audit 2023-09-23T18:21:00.979675+0000 mon.c (mon.2) 193 : audit [INF] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:21:01.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:01 smithi098 bash[27349]: audit 2023-09-23T18:21:00.980562+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.25823 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:21:01.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:01 smithi098 bash[27349]: audit 2023-09-23T18:21:00.983041+0000 mon.c (mon.2) 194 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:21:01.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:01 smithi098 bash[27349]: audit 2023-09-23T18:21:00.984947+0000 mon.c (mon.2) 195 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:21:01.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:01 smithi173 bash[27694]: cluster 2023-09-23T18:21:00.320148+0000 mgr.y (mgr.25823) 29 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:01.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:01 smithi173 bash[27694]: audit 2023-09-23T18:21:00.672441+0000 mon.c (mon.2) 192 : audit [DBG] from='client.? 172.21.15.98:0/326554328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:01.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:01 smithi173 bash[27694]: audit 2023-09-23T18:21:00.979675+0000 mon.c (mon.2) 193 : audit [INF] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:21:01.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:01 smithi173 bash[27694]: audit 2023-09-23T18:21:00.980562+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.25823 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:21:01.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:01 smithi173 bash[27694]: audit 2023-09-23T18:21:00.983041+0000 mon.c (mon.2) 194 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:21:01.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:01 smithi173 bash[27694]: audit 2023-09-23T18:21:00.984947+0000 mon.c (mon.2) 195 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:21:02.651 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:02 smithi173 bash[27694]: cephadm 2023-09-23T18:21:00.978897+0000 mgr.y (mgr.25823) 30 : cephadm [INF] Upgrade: Updating mgr.x 2023-09-23T18:21:02.651 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:02 smithi173 bash[27694]: cephadm 2023-09-23T18:21:00.986695+0000 mgr.y (mgr.25823) 31 : cephadm [INF] Deploying daemon mgr.x on smithi173 2023-09-23T18:21:02.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:02 smithi098 bash[27349]: cephadm 2023-09-23T18:21:00.978897+0000 mgr.y (mgr.25823) 30 : cephadm [INF] Upgrade: Updating mgr.x 2023-09-23T18:21:02.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:02 smithi098 bash[27349]: cephadm 2023-09-23T18:21:00.986695+0000 mgr.y (mgr.25823) 31 : cephadm [INF] Deploying daemon mgr.x on smithi173 2023-09-23T18:21:02.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:02 smithi098 bash[33599]: cephadm 2023-09-23T18:21:00.978897+0000 mgr.y (mgr.25823) 30 : cephadm [INF] Upgrade: Updating mgr.x 2023-09-23T18:21:02.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:02 smithi098 bash[33599]: cephadm 2023-09-23T18:21:00.986695+0000 mgr.y (mgr.25823) 31 : cephadm [INF] Deploying daemon mgr.x on smithi173 2023-09-23T18:21:03.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:03 smithi173 bash[27694]: cluster 2023-09-23T18:21:02.321259+0000 mgr.y (mgr.25823) 32 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:03.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:03 smithi173 bash[27694]: audit 2023-09-23T18:21:03.118820+0000 mon.a (mon.0) 868 : audit [DBG] from='client.? 172.21.15.98:0/70114136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:03.527 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:03 smithi173 systemd[1]: Stopping Ceph mgr.x for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T18:21:03.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:03 smithi098 bash[27349]: cluster 2023-09-23T18:21:02.321259+0000 mgr.y (mgr.25823) 32 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:03.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:03 smithi098 bash[27349]: audit 2023-09-23T18:21:03.118820+0000 mon.a (mon.0) 868 : audit [DBG] from='client.? 172.21.15.98:0/70114136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:03.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:03 smithi098 bash[33599]: cluster 2023-09-23T18:21:02.321259+0000 mgr.y (mgr.25823) 32 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:03.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:03 smithi098 bash[33599]: audit 2023-09-23T18:21:03.118820+0000 mon.a (mon.0) 868 : audit [DBG] from='client.? 172.21.15.98:0/70114136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:04.527 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:04 smithi173 bash[55282]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-mgr-x 2023-09-23T18:21:04.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:04 smithi173 bash[27694]: audit 2023-09-23T18:21:04.417990+0000 mon.c (mon.2) 196 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:21:04.802 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:04 smithi173 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-09-23T18:21:04.802 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:04 smithi173 bash[55344]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-mgr-x 2023-09-23T18:21:04.802 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:04 smithi173 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-09-23T18:21:04.802 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:04 smithi173 systemd[1]: Stopped Ceph mgr.x for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:21:04.802 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:04 smithi173 systemd[1]: Started Ceph mgr.x for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:21:04.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:04 smithi098 bash[27349]: audit 2023-09-23T18:21:04.417990+0000 mon.c (mon.2) 196 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:21:04.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:04 smithi098 bash[33599]: audit 2023-09-23T18:21:04.417990+0000 mon.c (mon.2) 196 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:21:05.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:05 smithi173 bash[27694]: cluster 2023-09-23T18:21:04.321892+0000 mgr.y (mgr.25823) 33 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:05.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:05 smithi173 bash[27694]: audit 2023-09-23T18:21:04.836914+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:05.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:05 smithi173 bash[27694]: audit 2023-09-23T18:21:04.845386+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:05.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:05 smithi173 bash[27694]: audit 2023-09-23T18:21:04.862028+0000 mon.c (mon.2) 197 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:21:05.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:05 smithi098 bash[33599]: cluster 2023-09-23T18:21:04.321892+0000 mgr.y (mgr.25823) 33 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:05.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:05 smithi098 bash[33599]: audit 2023-09-23T18:21:04.836914+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:05.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:05 smithi098 bash[33599]: audit 2023-09-23T18:21:04.845386+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:05.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:05 smithi098 bash[33599]: audit 2023-09-23T18:21:04.862028+0000 mon.c (mon.2) 197 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:21:05.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:05 smithi098 bash[27349]: cluster 2023-09-23T18:21:04.321892+0000 mgr.y (mgr.25823) 33 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:05.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:05 smithi098 bash[27349]: audit 2023-09-23T18:21:04.836914+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:05.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:05 smithi098 bash[27349]: audit 2023-09-23T18:21:04.845386+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:05.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:05 smithi098 bash[27349]: audit 2023-09-23T18:21:04.862028+0000 mon.c (mon.2) 197 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:21:06.193 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:06 smithi173 bash[55400]: debug 2023-09-23T18:21:06.024+0000 7f3d10b06700 1 -- 172.21.15.173:0/2733705744 <== mon.1 v2:172.21.15.173:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d9a1e371e0 con 0x55d9a2bb6c00 2023-09-23T18:21:06.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:21:06] "GET /metrics HTTP/1.1" 200 33919 "" "Prometheus/2.33.4" 2023-09-23T18:21:06.464 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:06 smithi173 bash[55400]: debug 2023-09-23T18:21:06.188+0000 7f3d1ac53200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T18:21:06.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:06 smithi173 bash[27694]: audit 2023-09-23T18:21:05.563636+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.98:0/672630328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:06.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:06 smithi098 bash[33599]: audit 2023-09-23T18:21:05.563636+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.98:0/672630328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:06.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:06 smithi098 bash[27349]: audit 2023-09-23T18:21:05.563636+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.98:0/672630328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:07.468 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:07 smithi173 bash[55400]: debug 2023-09-23T18:21:07.104+0000 7f3d1ac53200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T18:21:07.468 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:07 smithi173 bash[55400]: debug 2023-09-23T18:21:07.212+0000 7f3d1ac53200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T18:21:07.468 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:07 smithi173 bash[55400]: debug 2023-09-23T18:21:07.316+0000 7f3d1ac53200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T18:21:07.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:07 smithi173 bash[27694]: cluster 2023-09-23T18:21:06.323156+0000 mgr.y (mgr.25823) 34 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:07.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:07 smithi173 bash[55400]: debug 2023-09-23T18:21:07.576+0000 7f3d1ac53200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:21:07.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:07 smithi098 bash[33599]: cluster 2023-09-23T18:21:06.323156+0000 mgr.y (mgr.25823) 34 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:07.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:07 smithi098 bash[27349]: cluster 2023-09-23T18:21:06.323156+0000 mgr.y (mgr.25823) 34 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:08.277 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:07 smithi173 bash[55400]: debug 2023-09-23T18:21:07.948+0000 7f3d1ac53200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T18:21:08.277 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:08 smithi173 bash[55400]: debug 2023-09-23T18:21:08.108+0000 7f3d1ac53200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T18:21:08.663 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:08 smithi173 bash[27694]: audit 2023-09-23T18:21:08.012836+0000 mon.a (mon.0) 872 : audit [DBG] from='client.? 172.21.15.98:0/787653274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:08.663 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:08 smithi173 bash[55400]: debug 2023-09-23T18:21:08.320+0000 7f3d1ac53200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T18:21:08.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:08 smithi098 bash[27349]: audit 2023-09-23T18:21:08.012836+0000 mon.a (mon.0) 872 : audit [DBG] from='client.? 172.21.15.98:0/787653274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:08.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:08 smithi098 bash[33599]: audit 2023-09-23T18:21:08.012836+0000 mon.a (mon.0) 872 : audit [DBG] from='client.? 172.21.15.98:0/787653274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:09.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:21:08 smithi173 bash[54982]: ts=2023-09-23T18:21:08.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi098:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:21:09.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:09 smithi173 bash[27694]: cluster 2023-09-23T18:21:08.323891+0000 mgr.y (mgr.25823) 35 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:09.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:09 smithi173 bash[55400]: debug 2023-09-23T18:21:09.344+0000 7f3d1ac53200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T18:21:09.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:09 smithi173 bash[55400]: debug 2023-09-23T18:21:09.456+0000 7f3d1ac53200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T18:21:09.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:09 smithi173 bash[55400]: debug 2023-09-23T18:21:09.576+0000 7f3d1ac53200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T18:21:09.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:09 smithi098 bash[27349]: cluster 2023-09-23T18:21:08.323891+0000 mgr.y (mgr.25823) 35 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:09.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:09 smithi098 bash[33599]: cluster 2023-09-23T18:21:08.323891+0000 mgr.y (mgr.25823) 35 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:10.486 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:10 smithi173 bash[55400]: debug 2023-09-23T18:21:10.072+0000 7f3d1ac53200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T18:21:10.486 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:10 smithi173 bash[55400]: debug 2023-09-23T18:21:10.308+0000 7f3d1ac53200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T18:21:10.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:10 smithi173 bash[27694]: audit 2023-09-23T18:21:10.466844+0000 mon.a (mon.0) 873 : audit [DBG] from='client.? 172.21.15.98:0/2874740327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:10.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:10 smithi098 bash[33599]: audit 2023-09-23T18:21:10.466844+0000 mon.a (mon.0) 873 : audit [DBG] from='client.? 172.21.15.98:0/2874740327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:10.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:10 smithi098 bash[27349]: audit 2023-09-23T18:21:10.466844+0000 mon.a (mon.0) 873 : audit [DBG] from='client.? 172.21.15.98:0/2874740327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:11.277 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:10 smithi173 bash[55400]: debug 2023-09-23T18:21:10.808+0000 7f3d1ac53200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T18:21:11.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:11 smithi173 bash[55400]: debug 2023-09-23T18:21:11.356+0000 7f3d1ac53200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:21:11.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:11 smithi173 bash[27694]: cluster 2023-09-23T18:21:10.324621+0000 mgr.y (mgr.25823) 36 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:11.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:11 smithi173 bash[27694]: audit 2023-09-23T18:21:11.011696+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:11.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:11 smithi173 bash[27694]: audit 2023-09-23T18:21:11.019783+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:11.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:11 smithi098 bash[33599]: cluster 2023-09-23T18:21:10.324621+0000 mgr.y (mgr.25823) 36 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:11.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:11 smithi098 bash[33599]: audit 2023-09-23T18:21:11.011696+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:11.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:11 smithi098 bash[33599]: audit 2023-09-23T18:21:11.019783+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:11.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:11 smithi098 bash[27349]: cluster 2023-09-23T18:21:10.324621+0000 mgr.y (mgr.25823) 36 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:11.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:11 smithi098 bash[27349]: audit 2023-09-23T18:21:11.011696+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:11.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:11 smithi098 bash[27349]: audit 2023-09-23T18:21:11.019783+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:12.277 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:11 smithi173 bash[55400]: debug 2023-09-23T18:21:11.968+0000 7f3d1ac53200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T18:21:12.277 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:12 smithi173 bash[55400]: debug 2023-09-23T18:21:12.096+0000 7f3d1ac53200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T18:21:12.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:12 smithi173 bash[55400]: debug 2023-09-23T18:21:12.488+0000 7f3d1ac53200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T18:21:12.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:12 smithi173 bash[55400]: debug 2023-09-23T18:21:12.616+0000 7f3d1ac53200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T18:21:13.048 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:12 smithi173 bash[27694]: audit 2023-09-23T18:21:11.821572+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:13.048 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:12 smithi173 bash[27694]: audit 2023-09-23T18:21:11.832124+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:13.048 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:12 smithi173 bash[27694]: cluster 2023-09-23T18:21:12.325775+0000 mgr.y (mgr.25823) 37 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:13.048 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:12 smithi173 bash[55400]: debug 2023-09-23T18:21:12.776+0000 7f3d1ac53200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T18:21:13.048 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:12 smithi173 bash[55400]: debug 2023-09-23T18:21:12.908+0000 7f3d1ac53200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T18:21:13.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:12 smithi098 bash[27349]: audit 2023-09-23T18:21:11.821572+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:13.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:12 smithi098 bash[27349]: audit 2023-09-23T18:21:11.832124+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:13.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:12 smithi098 bash[27349]: cluster 2023-09-23T18:21:12.325775+0000 mgr.y (mgr.25823) 37 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:13.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:12 smithi098 bash[33599]: audit 2023-09-23T18:21:11.821572+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:13.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:12 smithi098 bash[33599]: audit 2023-09-23T18:21:11.832124+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:13.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:12 smithi098 bash[33599]: cluster 2023-09-23T18:21:12.325775+0000 mgr.y (mgr.25823) 37 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:13.401 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:13 smithi173 bash[55400]: debug 2023-09-23T18:21:13.044+0000 7f3d1ac53200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T18:21:13.401 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:13 smithi173 bash[55400]: debug 2023-09-23T18:21:13.256+0000 7f3d1ac53200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T18:21:13.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:13 smithi173 bash[55400]: debug 2023-09-23T18:21:13.396+0000 7f3d1ac53200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T18:21:14.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:13 smithi098 bash[27349]: audit 2023-09-23T18:21:12.908870+0000 mon.c (mon.2) 198 : audit [DBG] from='client.? 172.21.15.98:0/2100450351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:14.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:13 smithi098 bash[33599]: audit 2023-09-23T18:21:12.908870+0000 mon.c (mon.2) 198 : audit [DBG] from='client.? 172.21.15.98:0/2100450351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:14.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:13 smithi173 bash[27694]: audit 2023-09-23T18:21:12.908870+0000 mon.c (mon.2) 198 : audit [DBG] from='client.? 172.21.15.98:0/2100450351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:15.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:14 smithi173 bash[55400]: debug 2023-09-23T18:21:14.708+0000 7f3d1ac53200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T18:21:15.028 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:14 smithi173 bash[55400]: [23/Sep/2023:18:21:14] ENGINE Bus STARTING 2023-09-23T18:21:15.028 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:14 smithi173 bash[55400]: CherryPy Checker: 2023-09-23T18:21:15.028 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:14 smithi173 bash[55400]: The Application mounted at '' has an empty config. 2023-09-23T18:21:15.028 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:14 smithi173 bash[55400]: [23/Sep/2023:18:21:14] ENGINE Serving on http://:::9283 2023-09-23T18:21:15.028 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:14 smithi173 bash[55400]: [23/Sep/2023:18:21:14] ENGINE Bus STARTED 2023-09-23T18:21:15.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:14 smithi173 bash[27694]: cluster 2023-09-23T18:21:14.326495+0000 mgr.y (mgr.25823) 38 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:15.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:14 smithi173 bash[27694]: cluster 2023-09-23T18:21:14.728879+0000 mon.a (mon.0) 878 : cluster [DBG] Standby manager daemon x restarted 2023-09-23T18:21:15.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:14 smithi173 bash[27694]: cluster 2023-09-23T18:21:14.729209+0000 mon.a (mon.0) 879 : cluster [DBG] Standby manager daemon x started 2023-09-23T18:21:15.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:14 smithi173 bash[27694]: audit 2023-09-23T18:21:14.737554+0000 mon.b (mon.1) 97 : audit [DBG] from='mgr.? 172.21.15.173:0/3867588676' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T18:21:15.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:14 smithi173 bash[27694]: audit 2023-09-23T18:21:14.739747+0000 mon.b (mon.1) 98 : audit [DBG] from='mgr.? 172.21.15.173:0/3867588676' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:21:15.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:14 smithi173 bash[27694]: audit 2023-09-23T18:21:14.745399+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.? 172.21.15.173:0/3867588676' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T18:21:15.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:14 smithi173 bash[27694]: audit 2023-09-23T18:21:14.745897+0000 mon.b (mon.1) 100 : audit [DBG] from='mgr.? 172.21.15.173:0/3867588676' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:21:15.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:14 smithi098 bash[27349]: cluster 2023-09-23T18:21:14.326495+0000 mgr.y (mgr.25823) 38 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:15.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:14 smithi098 bash[27349]: cluster 2023-09-23T18:21:14.728879+0000 mon.a (mon.0) 878 : cluster [DBG] Standby manager daemon x restarted 2023-09-23T18:21:15.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:14 smithi098 bash[27349]: cluster 2023-09-23T18:21:14.729209+0000 mon.a (mon.0) 879 : cluster [DBG] Standby manager daemon x started 2023-09-23T18:21:15.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:14 smithi098 bash[27349]: audit 2023-09-23T18:21:14.737554+0000 mon.b (mon.1) 97 : audit [DBG] from='mgr.? 172.21.15.173:0/3867588676' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T18:21:15.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:14 smithi098 bash[27349]: audit 2023-09-23T18:21:14.739747+0000 mon.b (mon.1) 98 : audit [DBG] from='mgr.? 172.21.15.173:0/3867588676' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:21:15.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:14 smithi098 bash[27349]: audit 2023-09-23T18:21:14.745399+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.? 172.21.15.173:0/3867588676' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T18:21:15.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:14 smithi098 bash[27349]: audit 2023-09-23T18:21:14.745897+0000 mon.b (mon.1) 100 : audit [DBG] from='mgr.? 172.21.15.173:0/3867588676' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:21:15.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:14 smithi098 bash[33599]: cluster 2023-09-23T18:21:14.326495+0000 mgr.y (mgr.25823) 38 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:15.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:14 smithi098 bash[33599]: cluster 2023-09-23T18:21:14.728879+0000 mon.a (mon.0) 878 : cluster [DBG] Standby manager daemon x restarted 2023-09-23T18:21:15.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:14 smithi098 bash[33599]: cluster 2023-09-23T18:21:14.729209+0000 mon.a (mon.0) 879 : cluster [DBG] Standby manager daemon x started 2023-09-23T18:21:15.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:14 smithi098 bash[33599]: audit 2023-09-23T18:21:14.737554+0000 mon.b (mon.1) 97 : audit [DBG] from='mgr.? 172.21.15.173:0/3867588676' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T18:21:15.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:14 smithi098 bash[33599]: audit 2023-09-23T18:21:14.739747+0000 mon.b (mon.1) 98 : audit [DBG] from='mgr.? 172.21.15.173:0/3867588676' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:21:15.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:14 smithi098 bash[33599]: audit 2023-09-23T18:21:14.745399+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.? 172.21.15.173:0/3867588676' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T18:21:15.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:14 smithi098 bash[33599]: audit 2023-09-23T18:21:14.745897+0000 mon.b (mon.1) 100 : audit [DBG] from='mgr.? 172.21.15.173:0/3867588676' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:21:16.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:21:16] "GET /metrics HTTP/1.1" 200 33919 "" "Prometheus/2.33.4" 2023-09-23T18:21:16.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:15 smithi098 bash[33599]: cluster 2023-09-23T18:21:14.852802+0000 mon.a (mon.0) 880 : cluster [DBG] mgrmap e29: y(active, since 40s), standbys: x 2023-09-23T18:21:16.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:15 smithi098 bash[33599]: audit 2023-09-23T18:21:15.348052+0000 mon.a (mon.0) 881 : audit [DBG] from='client.? 172.21.15.98:0/2265381476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:16.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:15 smithi098 bash[27349]: cluster 2023-09-23T18:21:14.852802+0000 mon.a (mon.0) 880 : cluster [DBG] mgrmap e29: y(active, since 40s), standbys: x 2023-09-23T18:21:16.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:15 smithi098 bash[27349]: audit 2023-09-23T18:21:15.348052+0000 mon.a (mon.0) 881 : audit [DBG] from='client.? 172.21.15.98:0/2265381476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:16.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:15 smithi173 bash[27694]: cluster 2023-09-23T18:21:14.852802+0000 mon.a (mon.0) 880 : cluster [DBG] mgrmap e29: y(active, since 40s), standbys: x 2023-09-23T18:21:16.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:15 smithi173 bash[27694]: audit 2023-09-23T18:21:15.348052+0000 mon.a (mon.0) 881 : audit [DBG] from='client.? 172.21.15.98:0/2265381476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:17.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:16 smithi173 bash[27694]: cluster 2023-09-23T18:21:16.327637+0000 mgr.y (mgr.25823) 39 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:17.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:16 smithi098 bash[27349]: cluster 2023-09-23T18:21:16.327637+0000 mgr.y (mgr.25823) 39 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:17.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:16 smithi098 bash[33599]: cluster 2023-09-23T18:21:16.327637+0000 mgr.y (mgr.25823) 39 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:18.183 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:17 smithi173 bash[27694]: audit 2023-09-23T18:21:17.783246+0000 mon.a (mon.0) 882 : audit [DBG] from='client.? 172.21.15.98:0/3411624267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:18.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:17 smithi098 bash[27349]: audit 2023-09-23T18:21:17.783246+0000 mon.a (mon.0) 882 : audit [DBG] from='client.? 172.21.15.98:0/3411624267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:18.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:17 smithi098 bash[33599]: audit 2023-09-23T18:21:17.783246+0000 mon.a (mon.0) 882 : audit [DBG] from='client.? 172.21.15.98:0/3411624267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:19.028 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:21:18 smithi173 bash[54982]: ts=2023-09-23T18:21:18.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi098:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:21:19.752 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:19 smithi173 bash[27694]: cluster 2023-09-23T18:21:18.328341+0000 mgr.y (mgr.25823) 40 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:19.752 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:19 smithi173 bash[27694]: audit 2023-09-23T18:21:19.418516+0000 mon.c (mon.2) 199 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:21:19.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:19 smithi098 bash[27349]: cluster 2023-09-23T18:21:18.328341+0000 mgr.y (mgr.25823) 40 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:19.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:19 smithi098 bash[27349]: audit 2023-09-23T18:21:19.418516+0000 mon.c (mon.2) 199 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:21:19.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:19 smithi098 bash[33599]: cluster 2023-09-23T18:21:18.328341+0000 mgr.y (mgr.25823) 40 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:19.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:19 smithi098 bash[33599]: audit 2023-09-23T18:21:19.418516+0000 mon.c (mon.2) 199 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:21:20.771 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:20 smithi173 bash[27694]: audit 2023-09-23T18:21:20.231493+0000 mon.a (mon.0) 883 : audit [DBG] from='client.? 172.21.15.98:0/4208612757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:20.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:20 smithi098 bash[27349]: audit 2023-09-23T18:21:20.231493+0000 mon.a (mon.0) 883 : audit [DBG] from='client.? 172.21.15.98:0/4208612757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:20.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:20 smithi098 bash[33599]: audit 2023-09-23T18:21:20.231493+0000 mon.a (mon.0) 883 : audit [DBG] from='client.? 172.21.15.98:0/4208612757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:21.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:21 smithi173 bash[27694]: cluster 2023-09-23T18:21:20.329032+0000 mgr.y (mgr.25823) 41 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:21.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:21 smithi173 bash[27694]: audit 2023-09-23T18:21:21.387965+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:21.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:21 smithi173 bash[27694]: audit 2023-09-23T18:21:21.396817+0000 mon.a (mon.0) 885 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:21.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:21 smithi173 bash[27694]: audit 2023-09-23T18:21:21.399428+0000 mon.c (mon.2) 200 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:21:21.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:21 smithi173 bash[27694]: audit 2023-09-23T18:21:21.401095+0000 mon.c (mon.2) 201 : audit [INF] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:21:21.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:21 smithi173 bash[27694]: audit 2023-09-23T18:21:21.409568+0000 mon.a (mon.0) 886 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:21.840 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:21 smithi098 bash[33599]: cluster 2023-09-23T18:21:20.329032+0000 mgr.y (mgr.25823) 41 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:21.841 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:21 smithi098 bash[33599]: audit 2023-09-23T18:21:21.387965+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:21.841 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:21 smithi098 bash[33599]: audit 2023-09-23T18:21:21.396817+0000 mon.a (mon.0) 885 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:21.841 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:21 smithi098 bash[33599]: audit 2023-09-23T18:21:21.399428+0000 mon.c (mon.2) 200 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:21:21.841 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:21 smithi098 bash[33599]: audit 2023-09-23T18:21:21.401095+0000 mon.c (mon.2) 201 : audit [INF] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:21:21.841 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:21 smithi098 bash[33599]: audit 2023-09-23T18:21:21.409568+0000 mon.a (mon.0) 886 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:21.841 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:21 smithi098 bash[27349]: cluster 2023-09-23T18:21:20.329032+0000 mgr.y (mgr.25823) 41 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:21.841 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:21 smithi098 bash[27349]: audit 2023-09-23T18:21:21.387965+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:21.842 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:21 smithi098 bash[27349]: audit 2023-09-23T18:21:21.396817+0000 mon.a (mon.0) 885 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:21.842 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:21 smithi098 bash[27349]: audit 2023-09-23T18:21:21.399428+0000 mon.c (mon.2) 200 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:21:21.842 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:21 smithi098 bash[27349]: audit 2023-09-23T18:21:21.401095+0000 mon.c (mon.2) 201 : audit [INF] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:21:21.842 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:21 smithi098 bash[27349]: audit 2023-09-23T18:21:21.409568+0000 mon.a (mon.0) 886 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:22.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:22 smithi173 bash[27694]: audit 2023-09-23T18:21:21.492985+0000 mon.c (mon.2) 202 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:21:22.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:22 smithi173 bash[27694]: audit 2023-09-23T18:21:21.497630+0000 mon.c (mon.2) 203 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:21:22.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:22 smithi173 bash[27694]: cephadm 2023-09-23T18:21:21.498663+0000 mgr.y (mgr.25823) 42 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-09-23T18:21:22.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:22 smithi173 bash[27694]: audit 2023-09-23T18:21:21.506792+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:22.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:22 smithi173 bash[27694]: audit 2023-09-23T18:21:21.508323+0000 mon.c (mon.2) 204 : audit [INF] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T18:21:22.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:22 smithi173 bash[27694]: audit 2023-09-23T18:21:21.508769+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.25823 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T18:21:22.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:22 smithi173 bash[27694]: audit 2023-09-23T18:21:21.515130+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.25823 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T18:21:22.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:22 smithi173 bash[27694]: audit 2023-09-23T18:21:21.520697+0000 mon.c (mon.2) 205 : audit [INF] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T18:21:22.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:22 smithi173 bash[27694]: audit 2023-09-23T18:21:21.521206+0000 mon.a (mon.0) 890 : audit [INF] from='mgr.25823 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T18:21:22.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:22 smithi173 bash[27694]: audit 2023-09-23T18:21:21.530455+0000 mon.a (mon.0) 891 : audit [INF] from='mgr.25823 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T18:21:22.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:22 smithi173 bash[27694]: audit 2023-09-23T18:21:21.535714+0000 mon.c (mon.2) 206 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T18:21:22.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:22 smithi173 bash[27694]: audit 2023-09-23T18:21:21.536784+0000 mon.c (mon.2) 207 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T18:21:22.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:22 smithi173 bash[27694]: cephadm 2023-09-23T18:21:21.537579+0000 mgr.y (mgr.25823) 43 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-09-23T18:21:22.930 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[27349]: audit 2023-09-23T18:21:21.492985+0000 mon.c (mon.2) 202 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:21:22.930 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[27349]: audit 2023-09-23T18:21:21.497630+0000 mon.c (mon.2) 203 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:21:22.930 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[27349]: cephadm 2023-09-23T18:21:21.498663+0000 mgr.y (mgr.25823) 42 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-09-23T18:21:22.930 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[27349]: audit 2023-09-23T18:21:21.506792+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:22.930 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[27349]: audit 2023-09-23T18:21:21.508323+0000 mon.c (mon.2) 204 : audit [INF] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T18:21:22.930 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[27349]: audit 2023-09-23T18:21:21.508769+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.25823 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T18:21:22.931 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[27349]: audit 2023-09-23T18:21:21.515130+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.25823 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T18:21:22.931 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[27349]: audit 2023-09-23T18:21:21.520697+0000 mon.c (mon.2) 205 : audit [INF] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T18:21:22.931 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[27349]: audit 2023-09-23T18:21:21.521206+0000 mon.a (mon.0) 890 : audit [INF] from='mgr.25823 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T18:21:22.931 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[27349]: audit 2023-09-23T18:21:21.530455+0000 mon.a (mon.0) 891 : audit [INF] from='mgr.25823 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T18:21:22.931 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[27349]: audit 2023-09-23T18:21:21.535714+0000 mon.c (mon.2) 206 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T18:21:22.931 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[27349]: audit 2023-09-23T18:21:21.536784+0000 mon.c (mon.2) 207 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T18:21:22.931 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[27349]: cephadm 2023-09-23T18:21:21.537579+0000 mgr.y (mgr.25823) 43 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-09-23T18:21:22.931 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[33599]: audit 2023-09-23T18:21:21.492985+0000 mon.c (mon.2) 202 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:21:22.931 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[33599]: audit 2023-09-23T18:21:21.497630+0000 mon.c (mon.2) 203 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:21:22.931 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[33599]: cephadm 2023-09-23T18:21:21.498663+0000 mgr.y (mgr.25823) 42 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-09-23T18:21:22.931 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[33599]: audit 2023-09-23T18:21:21.506792+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:22.932 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[33599]: audit 2023-09-23T18:21:21.508323+0000 mon.c (mon.2) 204 : audit [INF] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T18:21:22.932 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[33599]: audit 2023-09-23T18:21:21.508769+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.25823 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T18:21:22.932 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[33599]: audit 2023-09-23T18:21:21.515130+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.25823 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T18:21:22.932 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[33599]: audit 2023-09-23T18:21:21.520697+0000 mon.c (mon.2) 205 : audit [INF] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T18:21:22.932 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[33599]: audit 2023-09-23T18:21:21.521206+0000 mon.a (mon.0) 890 : audit [INF] from='mgr.25823 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T18:21:22.932 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[33599]: audit 2023-09-23T18:21:21.530455+0000 mon.a (mon.0) 891 : audit [INF] from='mgr.25823 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T18:21:22.932 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[33599]: audit 2023-09-23T18:21:21.535714+0000 mon.c (mon.2) 206 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T18:21:22.932 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[33599]: audit 2023-09-23T18:21:21.536784+0000 mon.c (mon.2) 207 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T18:21:22.932 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:22 smithi098 bash[33599]: cephadm 2023-09-23T18:21:21.537579+0000 mgr.y (mgr.25823) 43 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-09-23T18:21:23.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:23 smithi098 bash[33599]: cluster 2023-09-23T18:21:22.330084+0000 mgr.y (mgr.25823) 44 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:23.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:23 smithi098 bash[33599]: audit 2023-09-23T18:21:22.672518+0000 mon.a (mon.0) 892 : audit [DBG] from='client.? 172.21.15.98:0/1543582382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:23.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:23 smithi098 bash[27349]: cluster 2023-09-23T18:21:22.330084+0000 mgr.y (mgr.25823) 44 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:23.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:23 smithi098 bash[27349]: audit 2023-09-23T18:21:22.672518+0000 mon.a (mon.0) 892 : audit [DBG] from='client.? 172.21.15.98:0/1543582382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:24.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:23 smithi173 bash[27694]: cluster 2023-09-23T18:21:22.330084+0000 mgr.y (mgr.25823) 44 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:24.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:23 smithi173 bash[27694]: audit 2023-09-23T18:21:22.672518+0000 mon.a (mon.0) 892 : audit [DBG] from='client.? 172.21.15.98:0/1543582382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:25.356 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:25 smithi098 bash[27349]: cephadm 2023-09-23T18:21:24.056736+0000 mgr.y (mgr.25823) 45 : cephadm [INF] Upgrade: Updating mon.a 2023-09-23T18:21:25.356 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:25 smithi098 bash[27349]: audit 2023-09-23T18:21:24.066459+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:25.356 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:25 smithi098 bash[27349]: audit 2023-09-23T18:21:24.068447+0000 mon.c (mon.2) 208 : audit [INF] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:21:25.357 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:25 smithi098 bash[27349]: audit 2023-09-23T18:21:24.069460+0000 mon.c (mon.2) 209 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:21:25.357 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:25 smithi098 bash[27349]: audit 2023-09-23T18:21:24.070454+0000 mon.c (mon.2) 210 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:21:25.357 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:25 smithi098 bash[27349]: cephadm 2023-09-23T18:21:24.071691+0000 mgr.y (mgr.25823) 46 : cephadm [INF] Deploying daemon mon.a on smithi098 2023-09-23T18:21:25.357 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:25 smithi098 bash[27349]: cluster 2023-09-23T18:21:24.330709+0000 mgr.y (mgr.25823) 47 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:25.357 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:25 smithi098 bash[33599]: cephadm 2023-09-23T18:21:24.056736+0000 mgr.y (mgr.25823) 45 : cephadm [INF] Upgrade: Updating mon.a 2023-09-23T18:21:25.357 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:25 smithi098 bash[33599]: audit 2023-09-23T18:21:24.066459+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:25.357 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:25 smithi098 bash[33599]: audit 2023-09-23T18:21:24.068447+0000 mon.c (mon.2) 208 : audit [INF] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:21:25.357 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:25 smithi098 bash[33599]: audit 2023-09-23T18:21:24.069460+0000 mon.c (mon.2) 209 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:21:25.357 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:25 smithi098 bash[33599]: audit 2023-09-23T18:21:24.070454+0000 mon.c (mon.2) 210 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:21:25.358 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:25 smithi098 bash[33599]: cephadm 2023-09-23T18:21:24.071691+0000 mgr.y (mgr.25823) 46 : cephadm [INF] Deploying daemon mon.a on smithi098 2023-09-23T18:21:25.358 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:25 smithi098 bash[33599]: cluster 2023-09-23T18:21:24.330709+0000 mgr.y (mgr.25823) 47 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:25.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:25 smithi173 bash[27694]: cephadm 2023-09-23T18:21:24.056736+0000 mgr.y (mgr.25823) 45 : cephadm [INF] Upgrade: Updating mon.a 2023-09-23T18:21:25.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:25 smithi173 bash[27694]: audit 2023-09-23T18:21:24.066459+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.25823 ' entity='mgr.y' 2023-09-23T18:21:25.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:25 smithi173 bash[27694]: audit 2023-09-23T18:21:24.068447+0000 mon.c (mon.2) 208 : audit [INF] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:21:25.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:25 smithi173 bash[27694]: audit 2023-09-23T18:21:24.069460+0000 mon.c (mon.2) 209 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:21:25.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:25 smithi173 bash[27694]: audit 2023-09-23T18:21:24.070454+0000 mon.c (mon.2) 210 : audit [DBG] from='mgr.25823 172.21.15.98:0/1185826659' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:21:25.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:25 smithi173 bash[27694]: cephadm 2023-09-23T18:21:24.071691+0000 mgr.y (mgr.25823) 46 : cephadm [INF] Deploying daemon mon.a on smithi098 2023-09-23T18:21:25.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:25 smithi173 bash[27694]: cluster 2023-09-23T18:21:24.330709+0000 mgr.y (mgr.25823) 47 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:26.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:26 smithi098 bash[27349]: audit 2023-09-23T18:21:25.114659+0000 mon.a (mon.0) 894 : audit [DBG] from='client.? 172.21.15.98:0/120924969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:26.445 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:21:26] "GET /metrics HTTP/1.1" 200 33913 "" "Prometheus/2.33.4" 2023-09-23T18:21:26.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:26 smithi098 bash[33599]: audit 2023-09-23T18:21:25.114659+0000 mon.a (mon.0) 894 : audit [DBG] from='client.? 172.21.15.98:0/120924969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:26.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:26 smithi173 bash[27694]: audit 2023-09-23T18:21:25.114659+0000 mon.a (mon.0) 894 : audit [DBG] from='client.? 172.21.15.98:0/120924969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:26.849 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:26 smithi098 systemd[1]: Stopping Ceph mon.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T18:21:27.160 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:26 smithi098 bash[27349]: debug 2023-09-23T18:21:26.840+0000 7fe06f2f5700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-09-23T18:21:27.161 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:26 smithi098 bash[27349]: debug 2023-09-23T18:21:26.840+0000 7fe06f2f5700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-09-23T18:21:27.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:27 smithi098 bash[76673]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-mon-a 2023-09-23T18:21:27.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:27 smithi098 bash[76759]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-mon-a 2023-09-23T18:21:27.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:27 smithi098 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mon.a.service: Succeeded. 2023-09-23T18:21:27.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:27 smithi098 systemd[1]: Stopped Ceph mon.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:21:28.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:27 smithi098 systemd[1]: Started Ceph mon.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:21:29.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:21:28 smithi173 bash[54982]: ts=2023-09-23T18:21:28.664Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi098:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:21:29.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.119+0000 7fc8b11eac80 0 set uid:gid to 167:167 (ceph:ceph) 2023-09-23T18:21:29.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.119+0000 7fc8b11eac80 0 ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T18:21:29.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.119+0000 7fc8b11eac80 0 pidfile_write: ignore empty --pid-file 2023-09-23T18:21:29.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.123+0000 7fc8b11eac80 0 load: jerasure load: lrc 2023-09-23T18:21:29.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: RocksDB version: 7.9.2 2023-09-23T18:21:29.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Git sha 0 2023-09-23T18:21:29.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T18:21:29.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: DB SUMMARY 2023-09-23T18:21:29.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: DB Session ID: Y62N47BFGXDBGPXNDXTA 2023-09-23T18:21:29.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: CURRENT file: CURRENT 2023-09-23T18:21:29.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: IDENTITY file: IDENTITY 2023-09-23T18:21:29.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: MANIFEST file: MANIFEST-000008 size: 966 Bytes 2023-09-23T18:21:29.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000023.sst 2023-09-23T18:21:29.447 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000021.log size: 16542192 ; 2023-09-23T18:21:29.447 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.error_if_exists: 0 2023-09-23T18:21:29.447 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.create_if_missing: 0 2023-09-23T18:21:29.447 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.paranoid_checks: 1 2023-09-23T18:21:29.447 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T18:21:29.447 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T18:21:29.447 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T18:21:29.447 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.env: 0x55bf5d2cb720 2023-09-23T18:21:29.447 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.fs: PosixFileSystem 2023-09-23T18:21:29.447 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.info_log: 0x55bf5f2ff480 2023-09-23T18:21:29.447 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_file_opening_threads: 16 2023-09-23T18:21:29.447 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.statistics: (nil) 2023-09-23T18:21:29.447 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.use_fsync: 0 2023-09-23T18:21:29.448 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_log_file_size: 0 2023-09-23T18:21:29.449 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T18:21:29.449 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T18:21:29.449 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.keep_log_file_num: 1000 2023-09-23T18:21:29.449 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.recycle_log_file_num: 0 2023-09-23T18:21:29.449 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.allow_fallocate: 1 2023-09-23T18:21:29.449 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.allow_mmap_reads: 0 2023-09-23T18:21:29.449 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.allow_mmap_writes: 0 2023-09-23T18:21:29.449 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.use_direct_reads: 0 2023-09-23T18:21:29.449 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T18:21:29.449 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.create_missing_column_families: 0 2023-09-23T18:21:29.450 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.db_log_dir: 2023-09-23T18:21:29.450 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.wal_dir: 2023-09-23T18:21:29.450 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T18:21:29.450 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T18:21:29.450 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T18:21:29.450 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T18:21:29.450 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T18:21:29.450 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T18:21:29.450 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.advise_random_on_open: 1 2023-09-23T18:21:29.450 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.db_write_buffer_size: 0 2023-09-23T18:21:29.451 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.write_buffer_manager: 0x55bf5e58cc80 2023-09-23T18:21:29.451 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T18:21:29.451 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T18:21:29.451 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T18:21:29.451 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.rate_limiter: (nil) 2023-09-23T18:21:29.451 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T18:21:29.451 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.wal_recovery_mode: 2 2023-09-23T18:21:29.451 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.enable_thread_tracking: 0 2023-09-23T18:21:29.451 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.enable_pipelined_write: 0 2023-09-23T18:21:29.452 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.unordered_write: 0 2023-09-23T18:21:29.452 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T18:21:29.452 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T18:21:29.452 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T18:21:29.452 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T18:21:29.452 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.row_cache: None 2023-09-23T18:21:29.452 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.wal_filter: None 2023-09-23T18:21:29.452 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T18:21:29.452 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.allow_ingest_behind: 0 2023-09-23T18:21:29.452 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.two_write_queues: 0 2023-09-23T18:21:29.453 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.manual_wal_flush: 0 2023-09-23T18:21:29.453 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.wal_compression: 0 2023-09-23T18:21:29.453 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.atomic_flush: 0 2023-09-23T18:21:29.453 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T18:21:29.453 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T18:21:29.453 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T18:21:29.453 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.log_readahead_size: 0 2023-09-23T18:21:29.453 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T18:21:29.453 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.best_efforts_recovery: 0 2023-09-23T18:21:29.454 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T18:21:29.454 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T18:21:29.454 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.allow_data_in_errors: 0 2023-09-23T18:21:29.454 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.db_host_id: __hostname__ 2023-09-23T18:21:29.454 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T18:21:29.454 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_background_jobs: 2 2023-09-23T18:21:29.454 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_background_compactions: -1 2023-09-23T18:21:29.454 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_subcompactions: 1 2023-09-23T18:21:29.455 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T18:21:29.455 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T18:21:29.455 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T18:21:29.455 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_total_wal_size: 0 2023-09-23T18:21:29.455 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T18:21:29.455 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T18:21:29.455 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T18:21:29.455 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T18:21:29.455 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_open_files: -1 2023-09-23T18:21:29.456 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.bytes_per_sync: 0 2023-09-23T18:21:29.456 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T18:21:29.456 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T18:21:29.456 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.compaction_readahead_size: 0 2023-09-23T18:21:29.456 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_background_flushes: -1 2023-09-23T18:21:29.456 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Compression algorithms supported: 2023-09-23T18:21:29.456 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T18:21:29.456 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: kZSTD supported: 0 2023-09-23T18:21:29.456 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: kXpressCompression supported: 0 2023-09-23T18:21:29.456 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: kLZ4HCCompression supported: 1 2023-09-23T18:21:29.457 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: kZlibCompression supported: 1 2023-09-23T18:21:29.457 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: kSnappyCompression supported: 1 2023-09-23T18:21:29.457 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: kLZ4Compression supported: 1 2023-09-23T18:21:29.457 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: kBZip2Compression supported: 0 2023-09-23T18:21:29.457 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T18:21:29.457 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T18:21:29.457 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 2023-09-23T18:21:29.457 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T18:21:29.457 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T18:21:29.458 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.merge_operator: 2023-09-23T18:21:29.458 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.compaction_filter: None 2023-09-23T18:21:29.458 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.compaction_filter_factory: None 2023-09-23T18:21:29.458 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.sst_partitioner_factory: None 2023-09-23T18:21:29.458 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T18:21:29.458 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T18:21:29.458 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55bf5f2ff580) 2023-09-23T18:21:29.458 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: cache_index_and_filter_blocks: 1 2023-09-23T18:21:29.458 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T18:21:29.459 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T18:21:29.459 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: pin_top_level_index_and_filter: 1 2023-09-23T18:21:29.459 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: index_type: 0 2023-09-23T18:21:29.459 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: data_block_index_type: 0 2023-09-23T18:21:29.459 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: index_shortening: 1 2023-09-23T18:21:29.459 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: data_block_hash_table_util_ratio: 0.750000 2023-09-23T18:21:29.459 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: checksum: 4 2023-09-23T18:21:29.459 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: no_block_cache: 0 2023-09-23T18:21:29.459 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: block_cache: 0x55bf5e60f350 2023-09-23T18:21:29.459 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: block_cache_name: BinnedLRUCache 2023-09-23T18:21:29.460 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: block_cache_options: 2023-09-23T18:21:29.460 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: capacity : 536870912 2023-09-23T18:21:29.460 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: num_shard_bits : 4 2023-09-23T18:21:29.460 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: strict_capacity_limit : 0 2023-09-23T18:21:29.460 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: high_pri_pool_ratio: 0.000 2023-09-23T18:21:29.460 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: block_cache_compressed: (nil) 2023-09-23T18:21:29.460 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: persistent_cache: (nil) 2023-09-23T18:21:29.460 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: block_size: 4096 2023-09-23T18:21:29.460 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: block_size_deviation: 10 2023-09-23T18:21:29.460 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: block_restart_interval: 16 2023-09-23T18:21:29.461 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: index_block_restart_interval: 1 2023-09-23T18:21:29.461 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: metadata_block_size: 4096 2023-09-23T18:21:29.461 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: partition_filters: 0 2023-09-23T18:21:29.461 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: use_delta_encoding: 1 2023-09-23T18:21:29.461 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: filter_policy: bloomfilter 2023-09-23T18:21:29.461 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: whole_key_filtering: 1 2023-09-23T18:21:29.461 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: verify_compression: 0 2023-09-23T18:21:29.461 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: read_amp_bytes_per_bit: 0 2023-09-23T18:21:29.461 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: format_version: 5 2023-09-23T18:21:29.461 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: enable_index_compression: 1 2023-09-23T18:21:29.462 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: block_align: 0 2023-09-23T18:21:29.462 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: max_auto_readahead_size: 262144 2023-09-23T18:21:29.462 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: prepopulate_block_cache: 0 2023-09-23T18:21:29.462 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: initial_auto_readahead_size: 8192 2023-09-23T18:21:29.462 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: num_file_reads_for_auto_readahead: 2 2023-09-23T18:21:29.462 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.write_buffer_size: 33554432 2023-09-23T18:21:29.462 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_write_buffer_number: 2 2023-09-23T18:21:29.462 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.compression: NoCompression 2023-09-23T18:21:29.463 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.bottommost_compression: Disabled 2023-09-23T18:21:29.463 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.prefix_extractor: nullptr 2023-09-23T18:21:29.463 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T18:21:29.463 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.num_levels: 7 2023-09-23T18:21:29.463 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T18:21:29.463 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T18:21:29.463 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T18:21:29.463 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T18:21:29.463 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T18:21:29.463 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T18:21:29.464 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T18:21:29.464 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T18:21:29.464 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T18:21:29.464 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T18:21:29.464 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T18:21:29.464 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T18:21:29.464 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T18:21:29.464 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.compression_opts.level: 32767 2023-09-23T18:21:29.464 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.compression_opts.strategy: 0 2023-09-23T18:21:29.465 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T18:21:29.465 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T18:21:29.465 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T18:21:29.465 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T18:21:29.465 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.compression_opts.enabled: false 2023-09-23T18:21:29.465 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T18:21:29.465 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T18:21:29.465 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T18:21:29.465 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T18:21:29.465 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.target_file_size_base: 67108864 2023-09-23T18:21:29.465 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T18:21:29.465 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T18:21:29.465 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T18:21:29.466 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T18:21:29.466 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T18:21:29.466 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T18:21:29.466 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T18:21:29.466 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T18:21:29.466 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T18:21:29.466 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T18:21:29.466 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T18:21:29.466 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T18:21:29.466 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T18:21:29.466 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T18:21:29.466 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.arena_block_size: 1048576 2023-09-23T18:21:29.467 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T18:21:29.467 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T18:21:29.467 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.disable_auto_compactions: 0 2023-09-23T18:21:29.467 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T18:21:29.467 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T18:21:29.467 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T18:21:29.467 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T18:21:29.467 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T18:21:29.468 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T18:21:29.468 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T18:21:29.468 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T18:21:29.468 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T18:21:29.469 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T18:21:29.469 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.table_properties_collectors: 2023-09-23T18:21:29.469 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.inplace_update_support: 0 2023-09-23T18:21:29.469 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T18:21:29.469 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T18:21:29.469 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T18:21:29.469 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T18:21:29.469 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.bloom_locality: 0 2023-09-23T18:21:29.469 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.max_successive_merges: 0 2023-09-23T18:21:29.469 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T18:21:29.470 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.paranoid_file_checks: 0 2023-09-23T18:21:29.470 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.force_consistency_checks: 1 2023-09-23T18:21:29.470 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.report_bg_io_stats: 0 2023-09-23T18:21:29.470 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.ttl: 2592000 2023-09-23T18:21:29.470 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T18:21:29.470 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T18:21:29.470 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T18:21:29.470 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.enable_blob_files: false 2023-09-23T18:21:29.471 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.min_blob_size: 0 2023-09-23T18:21:29.471 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.blob_file_size: 268435456 2023-09-23T18:21:29.471 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T18:21:29.471 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T18:21:29.471 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T18:21:29.471 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T18:21:29.471 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T18:21:29.471 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.blob_file_starting_level: 0 2023-09-23T18:21:29.471 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T18:21:29.471 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8a6208700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 23.sst 2023-09-23T18:21:29.472 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T18:21:29.472 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 succeeded,manifest_file_number is 8, next_file_number is 25, last_sequence is 11333, log_number is 21,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T18:21:29.472 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 21 2023-09-23T18:21:29.472 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 3a84df5d-89f6-4446-87e9-0dc7d9733ad3 2023-09-23T18:21:29.472 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695493289132233, "job": 1, "event": "recovery_started", "wal_files": [21]} 2023-09-23T18:21:29.472 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.127+0000 7fc8b11eac80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #21 mode 2 2023-09-23T18:21:29.472 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.179+0000 7fc8b11eac80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695493289186188, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 26, "file_size": 16036941, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11334, "largest_seqno": 15351, "table_properties": {"data_size": 16013833, "index_size": 18438, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3653, "raw_key_size": 27865, "raw_average_key_size": 19, "raw_value_size": 15975308, "raw_average_value_size": 11101, "num_data_blocks": 800, "num_entries": 1439, "num_filter_entries": 1439, "num_deletions": 259, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695493289, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "3a84df5d-89f6-4446-87e9-0dc7d9733ad3", "db_session_id": "Y62N47BFGXDBGPXNDXTA", "orig_file_number": 26, "seqno_to_time_mapping": "N/A"}} 2023-09-23T18:21:29.472 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.183+0000 7fc8b11eac80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695493289186312, "job": 1, "event": "recovery_finished"} 2023-09-23T18:21:29.472 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.183+0000 7fc8b11eac80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 28 2023-09-23T18:21:29.473 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.183+0000 7fc8b11eac80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T18:21:29.473 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.183+0000 7fc8b11eac80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000021.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T18:21:29.473 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.183+0000 7fc8b11eac80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55bf5e6b0000 2023-09-23T18:21:29.473 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.183+0000 7fc8b11eac80 4 rocksdb: DB pointer 0x55bf5e69e000 2023-09-23T18:21:29.473 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.183+0000 7fc89f9fb700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T18:21:29.473 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.183+0000 7fc89f9fb700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T18:21:29.473 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: ** DB Stats ** 2023-09-23T18:21:29.473 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: Uptime(secs): 0.1 total, 0.1 interval 2023-09-23T18:21:29.473 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T18:21:29.474 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T18:21:29.474 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T18:21:29.474 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T18:21:29.474 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T18:21:29.474 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T18:21:29.474 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: ** Compaction Stats [default] ** 2023-09-23T18:21:29.474 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T18:21:29.474 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T18:21:29.474 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: L0 1/0 15.29 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 566.5 0.03 0.00 1 0.027 0 0 0.0 0.0 2023-09-23T18:21:29.474 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: L6 1/0 22.15 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T18:21:29.475 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: Sum 2/0 37.44 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 566.5 0.03 0.00 1 0.027 0 0 0.0 0.0 2023-09-23T18:21:29.475 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 566.5 0.03 0.00 1 0.027 0 0 0.0 0.0 2023-09-23T18:21:29.475 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: ** Compaction Stats [default] ** 2023-09-23T18:21:29.475 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T18:21:29.475 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T18:21:29.475 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 566.5 0.03 0.00 1 0.027 0 0 0.0 0.0 2023-09-23T18:21:29.475 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T18:21:29.475 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: Uptime(secs): 0.1 total, 0.1 interval 2023-09-23T18:21:29.475 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: Flush(GB): cumulative 0.015, interval 0.015 2023-09-23T18:21:29.475 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T18:21:29.475 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T18:21:29.475 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T18:21:29.475 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: AddFile(Keys): cumulative 0, interval 0 2023-09-23T18:21:29.476 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: Cumulative compaction: 0.01 GB write, 260.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T18:21:29.476 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: Interval compaction: 0.01 GB write, 260.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T18:21:29.476 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T18:21:29.476 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: Block cache BinnedLRUCache@0x55bf5e60f350#7 capacity: 512.00 MB usage: 24.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 2.5e-05 secs_since: 0 2023-09-23T18:21:29.476 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: Block cache entry stats(count,size,portion): FilterBlock(2,10.06 KB,0.00191927%) IndexBlock(1,20.08 KB,0.0038296%) Misc(1,0.00 KB,0%) 2023-09-23T18:21:29.476 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: ** File Read Latency Histogram By Level [default] ** 2023-09-23T18:21:29.476 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.183+0000 7fc8b11eac80 0 starting mon.a rank 0 at public addrs [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] at bind addrs [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:21:29.476 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.187+0000 7fc8b11eac80 1 mon.a@-1(???) e3 preinit fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:21:29.476 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.187+0000 7fc8b11eac80 0 mon.a@-1(???).mds e7 new map 2023-09-23T18:21:29.476 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.187+0000 7fc8b11eac80 0 mon.a@-1(???).mds e7 print_map 2023-09-23T18:21:29.476 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: e7 2023-09-23T18:21:29.476 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T18:21:29.477 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T18:21:29.477 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: legacy client fscid: 1 2023-09-23T18:21:29.477 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: 2023-09-23T18:21:29.477 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: Filesystem 'foo' (1) 2023-09-23T18:21:29.477 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: fs_name foo 2023-09-23T18:21:29.477 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: epoch 5 2023-09-23T18:21:29.477 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T18:21:29.477 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: created 2023-09-23T18:15:08.890010+0000 2023-09-23T18:21:29.477 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: modified 2023-09-23T18:15:16.817511+0000 2023-09-23T18:21:29.477 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: tableserver 0 2023-09-23T18:21:29.477 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: root 0 2023-09-23T18:21:29.477 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: session_timeout 60 2023-09-23T18:21:29.478 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: session_autoclose 300 2023-09-23T18:21:29.478 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: max_file_size 1099511627776 2023-09-23T18:21:29.478 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: required_client_features {} 2023-09-23T18:21:29.478 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: last_failure 0 2023-09-23T18:21:29.478 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: last_failure_osd_epoch 0 2023-09-23T18:21:29.478 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T18:21:29.478 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: max_mds 1 2023-09-23T18:21:29.478 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: in 0 2023-09-23T18:21:29.478 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: up {0=24545} 2023-09-23T18:21:29.478 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: failed 2023-09-23T18:21:29.478 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: damaged 2023-09-23T18:21:29.478 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: stopped 2023-09-23T18:21:29.479 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: data_pools [4] 2023-09-23T18:21:29.479 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: metadata_pool 3 2023-09-23T18:21:29.479 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: inline_data disabled 2023-09-23T18:21:29.479 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: balancer 2023-09-23T18:21:29.479 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: bal_rank_mask -1 2023-09-23T18:21:29.479 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: standby_count_wanted 1 2023-09-23T18:21:29.479 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: [mds.foo.smithi098.lowjqo{0:24545} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.98:6834/2708276357,v1:172.21.15.98:6835/2708276357] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T18:21:29.479 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: 2023-09-23T18:21:29.479 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: 2023-09-23T18:21:29.479 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: Standby daemons: 2023-09-23T18:21:29.479 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: 2023-09-23T18:21:29.479 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: [mds.foo.smithi173.qjefje{-1:24502} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.173:6832/621117609,v1:172.21.15.173:6833/621117609] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T18:21:29.479 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.187+0000 7fc8b11eac80 0 mon.a@-1(???).osd e83 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T18:21:29.480 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.187+0000 7fc8b11eac80 0 mon.a@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T18:21:29.480 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.187+0000 7fc8b11eac80 0 mon.a@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T18:21:29.480 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.187+0000 7fc8b11eac80 0 mon.a@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T18:21:29.480 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:21:29 smithi098 bash[76817]: debug 2023-09-23T18:21:29.187+0000 7fc8b11eac80 1 mon.a@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2023-09-23T18:21:31.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:30 smithi173 bash[55400]: ignoring --setuser ceph since I am not root 2023-09-23T18:21:31.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:30 smithi173 bash[55400]: ignoring --setgroup ceph since I am not root 2023-09-23T18:21:31.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:30 smithi173 bash[55400]: debug 2023-09-23T18:21:30.744+0000 7fcf1b34b700 1 -- 172.21.15.173:0/3647059411 <== mon.1 v2:172.21.15.173:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5649e0c7d1e0 con 0x5649e19fc800 2023-09-23T18:21:31.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:30 smithi173 bash[55400]: debug 2023-09-23T18:21:30.912+0000 7fcf2507a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T18:21:31.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:30 smithi173 bash[27694]: audit 2023-09-23T18:21:27.546551+0000 mon.c (mon.2) 211 : audit [DBG] from='client.? 172.21.15.98:0/2990783449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:31.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:30 smithi173 bash[27694]: cluster 2023-09-23T18:21:28.332113+0000 mgr.y (mgr.25823) 49 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:31.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:30 smithi173 bash[27694]: audit 2023-09-23T18:21:29.993769+0000 mon.c (mon.2) 212 : audit [DBG] from='client.? 172.21.15.98:0/4088546413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:31.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:30 smithi173 bash[27694]: cluster 2023-09-23T18:21:30.332733+0000 mgr.y (mgr.25823) 50 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:31.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:30 smithi173 bash[27694]: cluster 2023-09-23T18:21:30.612181+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-09-23T18:21:31.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:30 smithi173 bash[27694]: cluster 2023-09-23T18:21:30.634360+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T18:21:31.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:30 smithi173 bash[27694]: cluster 2023-09-23T18:21:30.646374+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0],b=[v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0],c=[v2:172.21.15.98:3301/0,v1:172.21.15.98:6790/0]} removed_ranks: {} 2023-09-23T18:21:31.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:30 smithi173 bash[27694]: cluster 2023-09-23T18:21:30.647098+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi098.lowjqo=up:active} 1 up:standby 2023-09-23T18:21:31.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:30 smithi173 bash[27694]: cluster 2023-09-23T18:21:30.647146+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-09-23T18:21:31.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:30 smithi173 bash[27694]: cluster 2023-09-23T18:21:30.648243+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e29: y(active, since 56s), standbys: x 2023-09-23T18:21:31.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:30 smithi173 bash[27694]: cluster 2023-09-23T18:21:30.681822+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e30: y(active, since 56s), standbys: x 2023-09-23T18:21:31.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:30 smithi173 bash[27694]: cluster 2023-09-23T18:21:30.681905+0000 mon.a (mon.0) 8 : cluster [INF] overall HEALTH_OK 2023-09-23T18:21:31.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:30 smithi173 bash[27694]: audit 2023-09-23T18:21:30.682093+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.25823 ' entity='' 2023-09-23T18:21:31.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:30 smithi098 bash[73085]: ignoring --setuser ceph since I am not root 2023-09-23T18:21:31.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:30 smithi098 bash[73085]: ignoring --setgroup ceph since I am not root 2023-09-23T18:21:31.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:30 smithi098 bash[73085]: debug 2023-09-23T18:21:30.747+0000 7f90e39c5700 1 -- 172.21.15.98:0/989984742 <== mon.1 v2:172.21.15.173:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56068e9891e0 con 0x56068f708400 2023-09-23T18:21:31.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:30 smithi098 bash[73085]: debug 2023-09-23T18:21:30.907+0000 7f90ed6f4200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T18:21:31.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:30 smithi098 bash[33599]: audit 2023-09-23T18:21:27.546551+0000 mon.c (mon.2) 211 : audit [DBG] from='client.? 172.21.15.98:0/2990783449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:31.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:30 smithi098 bash[33599]: cluster 2023-09-23T18:21:28.332113+0000 mgr.y (mgr.25823) 49 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:31.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:30 smithi098 bash[33599]: audit 2023-09-23T18:21:29.993769+0000 mon.c (mon.2) 212 : audit [DBG] from='client.? 172.21.15.98:0/4088546413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:31.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:30 smithi098 bash[33599]: cluster 2023-09-23T18:21:30.332733+0000 mgr.y (mgr.25823) 50 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:31.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:30 smithi098 bash[33599]: cluster 2023-09-23T18:21:30.612181+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-09-23T18:21:31.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:30 smithi098 bash[33599]: cluster 2023-09-23T18:21:30.634360+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T18:21:31.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:30 smithi098 bash[33599]: cluster 2023-09-23T18:21:30.646374+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0],b=[v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0],c=[v2:172.21.15.98:3301/0,v1:172.21.15.98:6790/0]} removed_ranks: {} 2023-09-23T18:21:31.197 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:30 smithi098 bash[33599]: cluster 2023-09-23T18:21:30.647098+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi098.lowjqo=up:active} 1 up:standby 2023-09-23T18:21:31.197 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:30 smithi098 bash[33599]: cluster 2023-09-23T18:21:30.647146+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-09-23T18:21:31.197 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:30 smithi098 bash[33599]: cluster 2023-09-23T18:21:30.648243+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e29: y(active, since 56s), standbys: x 2023-09-23T18:21:31.197 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:30 smithi098 bash[33599]: cluster 2023-09-23T18:21:30.681822+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e30: y(active, since 56s), standbys: x 2023-09-23T18:21:31.197 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:30 smithi098 bash[33599]: cluster 2023-09-23T18:21:30.681905+0000 mon.a (mon.0) 8 : cluster [INF] overall HEALTH_OK 2023-09-23T18:21:31.197 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:30 smithi098 bash[33599]: audit 2023-09-23T18:21:30.682093+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.25823 ' entity='' 2023-09-23T18:21:32.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:31 smithi098 bash[73085]: debug 2023-09-23T18:21:31.843+0000 7f90ed6f4200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T18:21:32.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:31 smithi098 bash[73085]: debug 2023-09-23T18:21:31.943+0000 7f90ed6f4200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T18:21:32.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:32 smithi098 bash[73085]: debug 2023-09-23T18:21:32.047+0000 7f90ed6f4200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T18:21:32.277 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:31 smithi173 bash[55400]: debug 2023-09-23T18:21:31.836+0000 7fcf2507a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T18:21:32.277 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:31 smithi173 bash[55400]: debug 2023-09-23T18:21:31.936+0000 7fcf2507a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T18:21:32.277 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:32 smithi173 bash[55400]: debug 2023-09-23T18:21:32.040+0000 7fcf2507a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T18:21:32.588 INFO:teuthology.orchestra.run.smithi098.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 1 2023-09-23T18:21:32.629 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:32 smithi173 bash[55400]: debug 2023-09-23T18:21:32.312+0000 7fcf2507a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:21:32.650 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:32 smithi098 bash[33599]: audit 2023-09-23T18:21:32.569030+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.98:0/2704000601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:32.650 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:32 smithi098 bash[73085]: debug 2023-09-23T18:21:32.383+0000 7f90ed6f4200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:21:32.901 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:32 smithi098 bash[73085]: debug 2023-09-23T18:21:32.751+0000 7f90ed6f4200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T18:21:33.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:32 smithi173 bash[55400]: debug 2023-09-23T18:21:32.680+0000 7fcf2507a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T18:21:33.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:32 smithi173 bash[55400]: debug 2023-09-23T18:21:32.836+0000 7fcf2507a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T18:21:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:32 smithi173 bash[27694]: audit 2023-09-23T18:21:32.569030+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.98:0/2704000601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:21:33.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:32 smithi098 bash[73085]: debug 2023-09-23T18:21:32.911+0000 7f90ed6f4200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T18:21:33.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:33 smithi098 bash[73085]: debug 2023-09-23T18:21:33.131+0000 7f90ed6f4200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T18:21:33.527 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:33 smithi173 bash[55400]: debug 2023-09-23T18:21:33.052+0000 7fcf2507a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T18:21:33.583 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T18:21:34.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:34 smithi098 bash[73085]: debug 2023-09-23T18:21:34.159+0000 7f90ed6f4200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T18:21:34.445 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:34 smithi098 bash[73085]: debug 2023-09-23T18:21:34.275+0000 7f90ed6f4200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T18:21:34.445 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:34 smithi098 bash[73085]: debug 2023-09-23T18:21:34.407+0000 7f90ed6f4200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T18:21:34.527 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:34 smithi173 bash[55400]: debug 2023-09-23T18:21:34.072+0000 7fcf2507a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T18:21:34.527 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:34 smithi173 bash[55400]: debug 2023-09-23T18:21:34.184+0000 7fcf2507a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T18:21:34.527 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:34 smithi173 bash[55400]: debug 2023-09-23T18:21:34.304+0000 7fcf2507a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T18:21:35.134 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:34 smithi098 bash[73085]: debug 2023-09-23T18:21:34.939+0000 7f90ed6f4200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T18:21:35.277 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:34 smithi173 bash[55400]: debug 2023-09-23T18:21:34.776+0000 7fcf2507a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T18:21:35.277 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:35 smithi173 bash[55400]: debug 2023-09-23T18:21:35.000+0000 7fcf2507a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T18:21:35.445 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:35 smithi098 bash[73085]: debug 2023-09-23T18:21:35.191+0000 7f90ed6f4200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T18:21:35.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:35 smithi173 bash[55400]: debug 2023-09-23T18:21:35.444+0000 7fcf2507a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T18:21:35.946 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:35 smithi098 bash[73085]: debug 2023-09-23T18:21:35.695+0000 7f90ed6f4200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T18:21:36.277 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:35 smithi173 bash[55400]: debug 2023-09-23T18:21:35.948+0000 7fcf2507a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:21:36.670 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:36 smithi098 bash[73085]: debug 2023-09-23T18:21:36.215+0000 7f90ed6f4200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T18:21:36.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:36 smithi173 bash[55400]: debug 2023-09-23T18:21:36.500+0000 7fcf2507a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T18:21:36.777 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:36 smithi173 bash[55400]: debug 2023-09-23T18:21:36.616+0000 7fcf2507a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T18:21:36.944 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:36 smithi098 bash[73085]: debug 2023-09-23T18:21:36.779+0000 7f90ed6f4200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T18:21:36.944 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:36 smithi098 bash[73085]: debug 2023-09-23T18:21:36.907+0000 7f90ed6f4200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T18:21:37.227 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:36 smithi173 bash[55400]: debug 2023-09-23T18:21:36.964+0000 7fcf2507a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T18:21:37.228 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:37 smithi173 bash[55400]: debug 2023-09-23T18:21:37.084+0000 7fcf2507a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T18:21:37.527 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:37 smithi173 bash[55400]: debug 2023-09-23T18:21:37.224+0000 7fcf2507a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T18:21:37.527 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:37 smithi173 bash[55400]: debug 2023-09-23T18:21:37.340+0000 7fcf2507a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T18:21:37.527 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:37 smithi173 bash[55400]: debug 2023-09-23T18:21:37.472+0000 7fcf2507a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T18:21:37.547 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:37 smithi098 bash[73085]: debug 2023-09-23T18:21:37.275+0000 7f90ed6f4200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T18:21:37.547 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:37 smithi098 bash[73085]: debug 2023-09-23T18:21:37.391+0000 7f90ed6f4200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T18:21:37.799 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:37 smithi098 bash[73085]: debug 2023-09-23T18:21:37.535+0000 7f90ed6f4200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T18:21:37.799 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:37 smithi098 bash[73085]: debug 2023-09-23T18:21:37.659+0000 7f90ed6f4200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T18:21:38.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:37 smithi173 bash[55400]: debug 2023-09-23T18:21:37.664+0000 7fcf2507a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T18:21:38.027 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:37 smithi173 bash[55400]: debug 2023-09-23T18:21:37.792+0000 7fcf2507a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T18:21:38.123 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:37 smithi098 bash[73085]: debug 2023-09-23T18:21:37.787+0000 7f90ed6f4200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T18:21:38.124 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:37 smithi098 bash[73085]: debug 2023-09-23T18:21:37.979+0000 7f90ed6f4200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T18:21:38.445 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:38 smithi098 bash[73085]: debug 2023-09-23T18:21:38.111+0000 7f90ed6f4200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T18:21:39.397 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:39 smithi098 bash[33599]: cluster 2023-09-23T18:21:39.052277+0000 mon.a (mon.0) 11 : cluster [DBG] Standby manager daemon x restarted 2023-09-23T18:21:39.397 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:39 smithi098 bash[33599]: cluster 2023-09-23T18:21:39.052459+0000 mon.a (mon.0) 12 : cluster [DBG] Standby manager daemon x started 2023-09-23T18:21:39.397 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:39 smithi098 bash[33599]: audit 2023-09-23T18:21:39.063941+0000 mon.b (mon.1) 101 : audit [DBG] from='mgr.? 172.21.15.173:0/624191507' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T18:21:39.397 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:39 smithi098 bash[33599]: audit 2023-09-23T18:21:39.066110+0000 mon.b (mon.1) 102 : audit [DBG] from='mgr.? 172.21.15.173:0/624191507' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:21:39.397 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:39 smithi098 bash[33599]: audit 2023-09-23T18:21:39.071394+0000 mon.b (mon.1) 103 : audit [DBG] from='mgr.? 172.21.15.173:0/624191507' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T18:21:39.397 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:39 smithi098 bash[33599]: audit 2023-09-23T18:21:39.071901+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.? 172.21.15.173:0/624191507' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:21:39.427 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:39 smithi173 bash[55400]: debug 2023-09-23T18:21:39.032+0000 7fcf2507a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T18:21:39.427 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:39 smithi173 bash[55400]: [23/Sep/2023:18:21:39] ENGINE Bus STARTING 2023-09-23T18:21:39.427 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:39 smithi173 bash[55400]: CherryPy Checker: 2023-09-23T18:21:39.427 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:39 smithi173 bash[55400]: The Application mounted at '' has an empty config. 2023-09-23T18:21:39.427 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:39 smithi173 bash[55400]: [23/Sep/2023:18:21:39] ENGINE Serving on http://:::9283 2023-09-23T18:21:39.427 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 18:21:39 smithi173 bash[55400]: [23/Sep/2023:18:21:39] ENGINE Bus STARTED 2023-09-23T18:21:39.428 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:39 smithi173 bash[27694]: cluster 2023-09-23T18:21:39.052277+0000 mon.a (mon.0) 11 : cluster [DBG] Standby manager daemon x restarted 2023-09-23T18:21:39.428 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:39 smithi173 bash[27694]: cluster 2023-09-23T18:21:39.052459+0000 mon.a (mon.0) 12 : cluster [DBG] Standby manager daemon x started 2023-09-23T18:21:39.428 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:39 smithi173 bash[27694]: audit 2023-09-23T18:21:39.063941+0000 mon.b (mon.1) 101 : audit [DBG] from='mgr.? 172.21.15.173:0/624191507' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T18:21:39.428 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:39 smithi173 bash[27694]: audit 2023-09-23T18:21:39.066110+0000 mon.b (mon.1) 102 : audit [DBG] from='mgr.? 172.21.15.173:0/624191507' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T18:21:39.428 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:39 smithi173 bash[27694]: audit 2023-09-23T18:21:39.071394+0000 mon.b (mon.1) 103 : audit [DBG] from='mgr.? 172.21.15.173:0/624191507' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T18:21:39.428 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:39 smithi173 bash[27694]: audit 2023-09-23T18:21:39.071901+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.? 172.21.15.173:0/624191507' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T18:21:39.428 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:39 smithi173 bash[27694]: debug 2023-09-23T18:21:39.424+0000 7f99183f5700 -1 mon.b@1(peon).osd e84 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T18:21:39.657 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:39 smithi098 bash[73085]: debug 2023-09-23T18:21:39.387+0000 7f90ed6f4200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T18:21:39.658 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:39 smithi098 bash[73085]: [23/Sep/2023:18:21:39] ENGINE Bus STARTING 2023-09-23T18:21:39.658 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:39 smithi098 bash[33599]: debug 2023-09-23T18:21:39.415+0000 7f0ef7813700 -1 mon.c@2(peon).osd e84 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T18:21:39.945 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:39 smithi098 bash[73085]: debug 2023-09-23T18:21:39.659+0000 7f9049a8d700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:21:39.945 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:39 smithi098 bash[73085]: debug 2023-09-23T18:21:39.659+0000 7f9049a8d700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:21:39.945 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:39 smithi098 bash[73085]: debug 2023-09-23T18:21:39.659+0000 7f9049a8d700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:21:39.945 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:39 smithi098 bash[73085]: debug 2023-09-23T18:21:39.659+0000 7f9049a8d700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:21:39.945 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:39 smithi098 bash[73085]: debug 2023-09-23T18:21:39.659+0000 7f9049a8d700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:21:39.945 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:39 smithi098 bash[73085]: debug 2023-09-23T18:21:39.659+0000 7f904828a700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:21:39.946 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:39 smithi098 bash[73085]: debug 2023-09-23T18:21:39.659+0000 7f904828a700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:21:39.946 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:39 smithi098 bash[73085]: debug 2023-09-23T18:21:39.659+0000 7f904828a700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:21:39.946 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:39 smithi098 bash[73085]: debug 2023-09-23T18:21:39.659+0000 7f904828a700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:21:39.946 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:39 smithi098 bash[73085]: debug 2023-09-23T18:21:39.659+0000 7f904828a700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T18:21:39.946 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:39 smithi098 bash[73085]: CherryPy Checker: 2023-09-23T18:21:39.946 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:39 smithi098 bash[73085]: The Application mounted at '' has an empty config. 2023-09-23T18:21:39.946 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:39 smithi098 bash[73085]: [23/Sep/2023:18:21:39] ENGINE Serving on http://:::9283 2023-09-23T18:21:39.947 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:39 smithi098 bash[73085]: [23/Sep/2023:18:21:39] ENGINE Bus STARTED 2023-09-23T18:21:40.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[73085]: debug 2023-09-23T18:21:40.435+0000 7f90a56ad700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-09-23T18:21:40.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: cluster 2023-09-23T18:21:39.115216+0000 mon.a (mon.0) 13 : cluster [DBG] mgrmap e31: y(active, since 64s), standbys: x 2023-09-23T18:21:40.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: cluster 2023-09-23T18:21:39.415228+0000 mon.a (mon.0) 14 : cluster [INF] Active manager daemon y restarted 2023-09-23T18:21:40.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: cluster 2023-09-23T18:21:39.415885+0000 mon.a (mon.0) 15 : cluster [INF] Activating manager daemon y 2023-09-23T18:21:40.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: cluster 2023-09-23T18:21:39.425055+0000 osd.0 (osd.0) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-09-23T18:21:40.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: cluster 2023-09-23T18:21:39.427723+0000 mon.a (mon.0) 16 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2023-09-23T18:21:40.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: cluster 2023-09-23T18:21:39.428315+0000 mon.a (mon.0) 17 : cluster [DBG] mgrmap e32: y(active, starting, since 0.0126028s), standbys: x 2023-09-23T18:21:40.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: audit 2023-09-23T18:21:39.432749+0000 mon.a (mon.0) 18 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:21:40.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: audit 2023-09-23T18:21:39.432968+0000 mon.a (mon.0) 19 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:21:40.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: audit 2023-09-23T18:21:39.433181+0000 mon.a (mon.0) 20 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:21:40.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: audit 2023-09-23T18:21:39.433406+0000 mon.a (mon.0) 21 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi173.qjefje"}]: dispatch 2023-09-23T18:21:40.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: audit 2023-09-23T18:21:39.433697+0000 mon.a (mon.0) 22 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi098.lowjqo"}]: dispatch 2023-09-23T18:21:40.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: audit 2023-09-23T18:21:39.434787+0000 mon.a (mon.0) 23 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T18:21:40.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: audit 2023-09-23T18:21:39.434980+0000 mon.a (mon.0) 24 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T18:21:40.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: audit 2023-09-23T18:21:39.435153+0000 mon.a (mon.0) 25 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:21:40.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: audit 2023-09-23T18:21:39.435428+0000 mon.a (mon.0) 26 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:21:40.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: audit 2023-09-23T18:21:39.435800+0000 mon.a (mon.0) 27 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:21:40.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: audit 2023-09-23T18:21:39.436102+0000 mon.a (mon.0) 28 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:21:40.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: audit 2023-09-23T18:21:39.436317+0000 mon.a (mon.0) 29 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:21:40.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: audit 2023-09-23T18:21:39.436509+0000 mon.a (mon.0) 30 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:21:40.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: audit 2023-09-23T18:21:39.436688+0000 mon.a (mon.0) 31 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:21:40.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: audit 2023-09-23T18:21:39.436879+0000 mon.a (mon.0) 32 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:21:40.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: audit 2023-09-23T18:21:39.437084+0000 mon.a (mon.0) 33 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T18:21:40.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: audit 2023-09-23T18:21:39.437428+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T18:21:40.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: audit 2023-09-23T18:21:39.437880+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T18:21:40.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: cluster 2023-09-23T18:21:39.439987+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-09-23T18:21:40.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: cluster 2023-09-23T18:21:39.471021+0000 mon.a (mon.0) 36 : cluster [INF] Manager daemon y is now available 2023-09-23T18:21:40.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: audit 2023-09-23T18:21:39.505516+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:21:40.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: audit 2023-09-23T18:21:39.511713+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:21:40.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: audit 2023-09-23T18:21:39.558048+0000 mon.a (mon.0) 39 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:21:40.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: audit 2023-09-23T18:21:39.565380+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:21:40.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: cluster 2023-09-23T18:21:39.597288+0000 osd.1 (osd.1) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-09-23T18:21:40.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:40 smithi098 bash[33599]: audit 2023-09-23T18:21:39.604361+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T18:21:40.457 INFO:teuthology.orchestra.run.smithi098.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T18:21:40.457 INFO:teuthology.orchestra.run.smithi098.stdout:alertmanager.a smithi098 *:9093,9094 running (2m) - 8m 13.6M - 0.23.0 ba2b418f427c 7c2bcaebb703 2023-09-23T18:21:40.457 INFO:teuthology.orchestra.run.smithi098.stdout:grafana.a smithi173 *:3000 running (88s) - 7m 41.4M - 8.3.5 dad864ee21e9 616bbe1127b7 2023-09-23T18:21:40.457 INFO:teuthology.orchestra.run.smithi098.stdout:mds.foo.smithi098.lowjqo smithi098 running (6m) - 6m 15.1M - 16.2.14-37-g2e1f1015 1e6a756174a8 e1555281d4fd 2023-09-23T18:21:40.457 INFO:teuthology.orchestra.run.smithi098.stdout:mds.foo.smithi173.qjefje smithi173 running (6m) - 6m 13.0M - 16.2.14-37-g2e1f1015 1e6a756174a8 fc5513981852 2023-09-23T18:21:40.457 INFO:teuthology.orchestra.run.smithi098.stdout:mgr.x smithi173 *:8443,9283,8765 running (35s) - 15m 70.6M - 18.2.0-460-gbb0daeff e32c2a64c126 070f62900469 2023-09-23T18:21:40.457 INFO:teuthology.orchestra.run.smithi098.stdout:mgr.y smithi098 *:8443,9283,8765 running (82s) - 17m 457M - 18.2.0-460-gbb0daeff e32c2a64c126 c11dea5b47dd 2023-09-23T18:21:40.457 INFO:teuthology.orchestra.run.smithi098.stdout:mon.a smithi098 running (17m) - 17m 63.1M 2048M 16.2.14-37-g2e1f1015 1e6a756174a8 35c9bdb63747 2023-09-23T18:21:40.458 INFO:teuthology.orchestra.run.smithi098.stdout:mon.b smithi173 running (15m) - 15m 61.1M 2048M 16.2.14-37-g2e1f1015 1e6a756174a8 0c6482ba189f 2023-09-23T18:21:40.458 INFO:teuthology.orchestra.run.smithi098.stdout:mon.c smithi098 running (15m) - 15m 59.7M 2048M 16.2.14-37-g2e1f1015 1e6a756174a8 5d5fc8fcca20 2023-09-23T18:21:40.458 INFO:teuthology.orchestra.run.smithi098.stdout:node-exporter.a smithi098 *:9100 running (2m) - 8m 8764k - 1.3.1 1dbe0e931976 bdffbddc2c81 2023-09-23T18:21:40.458 INFO:teuthology.orchestra.run.smithi098.stdout:node-exporter.b smithi173 *:9100 running (114s) - 8m 8624k - 1.3.1 1dbe0e931976 610050ff4ef2 2023-09-23T18:21:40.458 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0 smithi098 running (14m) - 14m 44.8M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 5900bdd3fa79 2023-09-23T18:21:40.458 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1 smithi098 running (13m) - 13m 47.4M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 7fd00c3be01c 2023-09-23T18:21:40.458 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2 smithi098 running (13m) - 13m 43.6M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 587769f25c85 2023-09-23T18:21:40.458 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3 smithi098 running (12m) - 12m 42.3M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 af525819942d 2023-09-23T18:21:40.458 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4 smithi173 running (11m) - 11m 42.5M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 0a479b881bee 2023-09-23T18:21:40.458 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5 smithi173 running (11m) - 11m 43.4M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 501903da3aa9 2023-09-23T18:21:40.458 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6 smithi173 running (10m) - 10m 44.9M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 b37149ed4271 2023-09-23T18:21:40.459 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7 smithi173 running (9m) - 9m 47.2M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 81723c51be2b 2023-09-23T18:21:40.459 INFO:teuthology.orchestra.run.smithi098.stdout:prometheus.a smithi173 *:9095 running (42s) - 8m 39.9M - 2.33.4 514e6a882f6e 7547c9472f93 2023-09-23T18:21:40.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: cluster 2023-09-23T18:21:39.115216+0000 mon.a (mon.0) 13 : cluster [DBG] mgrmap e31: y(active, since 64s), standbys: x 2023-09-23T18:21:40.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: cluster 2023-09-23T18:21:39.415228+0000 mon.a (mon.0) 14 : cluster [INF] Active manager daemon y restarted 2023-09-23T18:21:40.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: cluster 2023-09-23T18:21:39.415885+0000 mon.a (mon.0) 15 : cluster [INF] Activating manager daemon y 2023-09-23T18:21:40.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: cluster 2023-09-23T18:21:39.425055+0000 osd.0 (osd.0) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-09-23T18:21:40.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: cluster 2023-09-23T18:21:39.427723+0000 mon.a (mon.0) 16 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2023-09-23T18:21:40.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: cluster 2023-09-23T18:21:39.428315+0000 mon.a (mon.0) 17 : cluster [DBG] mgrmap e32: y(active, starting, since 0.0126028s), standbys: x 2023-09-23T18:21:40.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: audit 2023-09-23T18:21:39.432749+0000 mon.a (mon.0) 18 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:21:40.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: audit 2023-09-23T18:21:39.432968+0000 mon.a (mon.0) 19 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:21:40.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: audit 2023-09-23T18:21:39.433181+0000 mon.a (mon.0) 20 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:21:40.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: audit 2023-09-23T18:21:39.433406+0000 mon.a (mon.0) 21 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi173.qjefje"}]: dispatch 2023-09-23T18:21:40.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: audit 2023-09-23T18:21:39.433697+0000 mon.a (mon.0) 22 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi098.lowjqo"}]: dispatch 2023-09-23T18:21:40.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: audit 2023-09-23T18:21:39.434787+0000 mon.a (mon.0) 23 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T18:21:40.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: audit 2023-09-23T18:21:39.434980+0000 mon.a (mon.0) 24 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T18:21:40.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: audit 2023-09-23T18:21:39.435153+0000 mon.a (mon.0) 25 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:21:40.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: audit 2023-09-23T18:21:39.435428+0000 mon.a (mon.0) 26 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:21:40.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: audit 2023-09-23T18:21:39.435800+0000 mon.a (mon.0) 27 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:21:40.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: audit 2023-09-23T18:21:39.436102+0000 mon.a (mon.0) 28 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:21:40.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: audit 2023-09-23T18:21:39.436317+0000 mon.a (mon.0) 29 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:21:40.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: audit 2023-09-23T18:21:39.436509+0000 mon.a (mon.0) 30 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:21:40.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: audit 2023-09-23T18:21:39.436688+0000 mon.a (mon.0) 31 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T18:21:40.530 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: audit 2023-09-23T18:21:39.436879+0000 mon.a (mon.0) 32 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:21:40.530 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: audit 2023-09-23T18:21:39.437084+0000 mon.a (mon.0) 33 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T18:21:40.530 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: audit 2023-09-23T18:21:39.437428+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T18:21:40.530 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: audit 2023-09-23T18:21:39.437880+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T18:21:40.530 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: cluster 2023-09-23T18:21:39.439987+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-09-23T18:21:40.530 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: cluster 2023-09-23T18:21:39.471021+0000 mon.a (mon.0) 36 : cluster [INF] Manager daemon y is now available 2023-09-23T18:21:40.530 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: audit 2023-09-23T18:21:39.505516+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:21:40.530 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: audit 2023-09-23T18:21:39.511713+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:21:40.530 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: audit 2023-09-23T18:21:39.558048+0000 mon.a (mon.0) 39 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:21:40.530 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: audit 2023-09-23T18:21:39.565380+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T18:21:40.530 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: cluster 2023-09-23T18:21:39.597288+0000 osd.1 (osd.1) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-09-23T18:21:40.530 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:40 smithi173 bash[27694]: audit 2023-09-23T18:21:39.604361+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T18:21:41.365 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T18:21:41.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:41 smithi098 bash[33599]: cluster 2023-09-23T18:21:39.428547+0000 osd.2 (osd.2) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-09-23T18:21:41.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:41 smithi098 bash[33599]: cluster 2023-09-23T18:21:39.581112+0000 osd.5 (osd.5) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-09-23T18:21:41.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:41 smithi098 bash[33599]: cephadm 2023-09-23T18:21:40.327983+0000 mgr.y (mgr.26054) 1 : cephadm [INF] [23/Sep/2023:18:21:40] ENGINE Bus STARTING 2023-09-23T18:21:41.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:41 smithi098 bash[33599]: cluster 2023-09-23T18:21:40.433830+0000 mon.a (mon.0) 42 : cluster [DBG] mgrmap e33: y(active, since 1.01811s), standbys: x 2023-09-23T18:21:41.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:41 smithi098 bash[33599]: audit 2023-09-23T18:21:40.437161+0000 mgr.y (mgr.26054) 2 : audit [DBG] from='client.34119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:21:41.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:41 smithi098 bash[33599]: cluster 2023-09-23T18:21:40.443630+0000 mgr.y (mgr.26054) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:41.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:41 smithi098 bash[33599]: cephadm 2023-09-23T18:21:40.452194+0000 mgr.y (mgr.26054) 4 : cephadm [INF] [23/Sep/2023:18:21:40] ENGINE Serving on https://172.21.15.98:7150 2023-09-23T18:21:41.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:41 smithi098 bash[33599]: cephadm 2023-09-23T18:21:40.553952+0000 mgr.y (mgr.26054) 5 : cephadm [INF] [23/Sep/2023:18:21:40] ENGINE Serving on http://172.21.15.98:8765 2023-09-23T18:21:41.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:41 smithi098 bash[33599]: cephadm 2023-09-23T18:21:40.554456+0000 mgr.y (mgr.26054) 6 : cephadm [INF] [23/Sep/2023:18:21:40] ENGINE Bus STARTED 2023-09-23T18:21:41.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:41 smithi173 bash[27694]: cluster 2023-09-23T18:21:39.428547+0000 osd.2 (osd.2) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-09-23T18:21:41.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:41 smithi173 bash[27694]: cluster 2023-09-23T18:21:39.581112+0000 osd.5 (osd.5) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-09-23T18:21:41.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:41 smithi173 bash[27694]: cephadm 2023-09-23T18:21:40.327983+0000 mgr.y (mgr.26054) 1 : cephadm [INF] [23/Sep/2023:18:21:40] ENGINE Bus STARTING 2023-09-23T18:21:41.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:41 smithi173 bash[27694]: cluster 2023-09-23T18:21:40.433830+0000 mon.a (mon.0) 42 : cluster [DBG] mgrmap e33: y(active, since 1.01811s), standbys: x 2023-09-23T18:21:41.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:41 smithi173 bash[27694]: audit 2023-09-23T18:21:40.437161+0000 mgr.y (mgr.26054) 2 : audit [DBG] from='client.34119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:21:41.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:41 smithi173 bash[27694]: cluster 2023-09-23T18:21:40.443630+0000 mgr.y (mgr.26054) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:41.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:41 smithi173 bash[27694]: cephadm 2023-09-23T18:21:40.452194+0000 mgr.y (mgr.26054) 4 : cephadm [INF] [23/Sep/2023:18:21:40] ENGINE Serving on https://172.21.15.98:7150 2023-09-23T18:21:41.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:41 smithi173 bash[27694]: cephadm 2023-09-23T18:21:40.553952+0000 mgr.y (mgr.26054) 5 : cephadm [INF] [23/Sep/2023:18:21:40] ENGINE Serving on http://172.21.15.98:8765 2023-09-23T18:21:41.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:41 smithi173 bash[27694]: cephadm 2023-09-23T18:21:40.554456+0000 mgr.y (mgr.26054) 6 : cephadm [INF] [23/Sep/2023:18:21:40] ENGINE Bus STARTED 2023-09-23T18:21:42.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:42 smithi098 bash[33599]: cluster 2023-09-23T18:21:41.436845+0000 mgr.y (mgr.26054) 7 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:42.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:42 smithi098 bash[33599]: cluster 2023-09-23T18:21:42.132063+0000 mon.a (mon.0) 43 : cluster [DBG] mgrmap e34: y(active, since 2s), standbys: x 2023-09-23T18:21:42.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:42 smithi173 bash[27694]: cluster 2023-09-23T18:21:41.436845+0000 mgr.y (mgr.26054) 7 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:42.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:42 smithi173 bash[27694]: cluster 2023-09-23T18:21:42.132063+0000 mon.a (mon.0) 43 : cluster [DBG] mgrmap e34: y(active, since 2s), standbys: x 2023-09-23T18:21:44.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:44 smithi098 bash[33599]: cluster 2023-09-23T18:21:43.437420+0000 mgr.y (mgr.26054) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:44.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:44 smithi173 bash[27694]: cluster 2023-09-23T18:21:43.437420+0000 mgr.y (mgr.26054) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:44.829 INFO:teuthology.orchestra.run.smithi098.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:21:45.669 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T18:21:45.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:45 smithi098 bash[33599]: audit 2023-09-23T18:21:44.818772+0000 mgr.y (mgr.26054) 9 : audit [DBG] from='client.26108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:21:45.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:45 smithi098 bash[33599]: cephadm 2023-09-23T18:21:44.819526+0000 mgr.y (mgr.26054) 10 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:21:45.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:45 smithi098 bash[33599]: audit 2023-09-23T18:21:44.824174+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:45.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:45 smithi098 bash[33599]: cluster 2023-09-23T18:21:45.438011+0000 mgr.y (mgr.26054) 11 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:46.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:21:46] "GET /metrics HTTP/1.1" 200 31225 "" "Prometheus/2.33.4" 2023-09-23T18:21:46.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:45 smithi173 bash[27694]: audit 2023-09-23T18:21:44.818772+0000 mgr.y (mgr.26054) 9 : audit [DBG] from='client.26108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:21:46.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:45 smithi173 bash[27694]: cephadm 2023-09-23T18:21:44.819526+0000 mgr.y (mgr.26054) 10 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:21:46.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:45 smithi173 bash[27694]: audit 2023-09-23T18:21:44.824174+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:46.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:45 smithi173 bash[27694]: cluster 2023-09-23T18:21:45.438011+0000 mgr.y (mgr.26054) 11 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:47.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:47 smithi098 bash[33599]: audit 2023-09-23T18:21:46.020368+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:47.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:47 smithi098 bash[33599]: audit 2023-09-23T18:21:46.026473+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:47.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:47 smithi098 bash[33599]: audit 2023-09-23T18:21:46.169673+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:47.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:47 smithi098 bash[33599]: audit 2023-09-23T18:21:46.178995+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:47.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:47 smithi098 bash[33599]: audit 2023-09-23T18:21:46.818652+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:47.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:47 smithi098 bash[33599]: audit 2023-09-23T18:21:46.828233+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:47.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:47 smithi098 bash[33599]: audit 2023-09-23T18:21:46.971960+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:47.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:47 smithi098 bash[33599]: audit 2023-09-23T18:21:46.979202+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:47.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:47 smithi173 bash[27694]: audit 2023-09-23T18:21:46.020368+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:47.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:47 smithi173 bash[27694]: audit 2023-09-23T18:21:46.026473+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:47.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:47 smithi173 bash[27694]: audit 2023-09-23T18:21:46.169673+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:47.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:47 smithi173 bash[27694]: audit 2023-09-23T18:21:46.178995+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:47.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:47 smithi173 bash[27694]: audit 2023-09-23T18:21:46.818652+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:47 smithi173 bash[27694]: audit 2023-09-23T18:21:46.828233+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:47 smithi173 bash[27694]: audit 2023-09-23T18:21:46.971960+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:47 smithi173 bash[27694]: audit 2023-09-23T18:21:46.979202+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:48.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:48 smithi098 bash[33599]: cluster 2023-09-23T18:21:47.438988+0000 mgr.y (mgr.26054) 12 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:48.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:48 smithi173 bash[27694]: cluster 2023-09-23T18:21:47.438988+0000 mgr.y (mgr.26054) 12 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:49.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:21:48 smithi173 bash[54982]: ts=2023-09-23T18:21:48.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi098:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:21:50.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:50 smithi173 bash[27694]: cluster 2023-09-23T18:21:49.439623+0000 mgr.y (mgr.26054) 13 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:50.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:50 smithi098 bash[33599]: cluster 2023-09-23T18:21:49.439623+0000 mgr.y (mgr.26054) 13 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:52.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:52 smithi173 bash[27694]: cluster 2023-09-23T18:21:51.440783+0000 mgr.y (mgr.26054) 14 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:52.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:52 smithi098 bash[33599]: cluster 2023-09-23T18:21:51.440783+0000 mgr.y (mgr.26054) 14 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:54.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:54 smithi173 bash[27694]: cluster 2023-09-23T18:21:53.441517+0000 mgr.y (mgr.26054) 15 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:54.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:54 smithi098 bash[33599]: cluster 2023-09-23T18:21:53.441517+0000 mgr.y (mgr.26054) 15 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:55.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:55 smithi173 bash[27694]: audit 2023-09-23T18:21:54.558959+0000 mon.a (mon.0) 53 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:21:55.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:55 smithi098 bash[33599]: audit 2023-09-23T18:21:54.558959+0000 mon.a (mon.0) 53 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:21:56.445 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:21:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:21:56] "GET /metrics HTTP/1.1" 200 33916 "" "Prometheus/2.33.4" 2023-09-23T18:21:56.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:56 smithi173 bash[27694]: cluster 2023-09-23T18:21:55.442305+0000 mgr.y (mgr.26054) 16 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:56.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:56 smithi173 bash[27694]: audit 2023-09-23T18:21:56.280735+0000 mon.a (mon.0) 54 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:56.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:56 smithi173 bash[27694]: audit 2023-09-23T18:21:56.289022+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:56.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:56 smithi173 bash[27694]: audit 2023-09-23T18:21:56.291630+0000 mon.a (mon.0) 56 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:21:56.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:56 smithi173 bash[27694]: audit 2023-09-23T18:21:56.292837+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:21:56.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:56 smithi098 bash[33599]: cluster 2023-09-23T18:21:55.442305+0000 mgr.y (mgr.26054) 16 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:56.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:56 smithi098 bash[33599]: audit 2023-09-23T18:21:56.280735+0000 mon.a (mon.0) 54 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:56.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:56 smithi098 bash[33599]: audit 2023-09-23T18:21:56.289022+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:56.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:56 smithi098 bash[33599]: audit 2023-09-23T18:21:56.291630+0000 mon.a (mon.0) 56 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:21:56.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:56 smithi098 bash[33599]: audit 2023-09-23T18:21:56.292837+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:21:57.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:57 smithi173 bash[27694]: cephadm 2023-09-23T18:21:56.294780+0000 mgr.y (mgr.26054) 17 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.conf 2023-09-23T18:21:57.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:57 smithi173 bash[27694]: cephadm 2023-09-23T18:21:56.295124+0000 mgr.y (mgr.26054) 18 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-09-23T18:21:57.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:57 smithi173 bash[27694]: cephadm 2023-09-23T18:21:56.460368+0000 mgr.y (mgr.26054) 19 : cephadm [INF] Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:21:57.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:57 smithi173 bash[27694]: cephadm 2023-09-23T18:21:56.460725+0000 mgr.y (mgr.26054) 20 : cephadm [INF] Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:21:57.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:57 smithi173 bash[27694]: cephadm 2023-09-23T18:21:56.624879+0000 mgr.y (mgr.26054) 21 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:21:57.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:57 smithi173 bash[27694]: cephadm 2023-09-23T18:21:56.627173+0000 mgr.y (mgr.26054) 22 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:21:57.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:57 smithi173 bash[27694]: audit 2023-09-23T18:21:56.962832+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:57.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:57 smithi173 bash[27694]: audit 2023-09-23T18:21:56.969359+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:57.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:57 smithi173 bash[27694]: audit 2023-09-23T18:21:56.974168+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:57.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:57 smithi173 bash[27694]: audit 2023-09-23T18:21:56.978393+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:57.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:57 smithi173 bash[27694]: audit 2023-09-23T18:21:56.983652+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:57.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:57 smithi173 bash[27694]: audit 2023-09-23T18:21:57.063883+0000 mon.a (mon.0) 63 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:21:57.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:57 smithi173 bash[27694]: audit 2023-09-23T18:21:57.066631+0000 mon.a (mon.0) 64 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:21:57.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:57 smithi173 bash[27694]: audit 2023-09-23T18:21:57.067858+0000 mon.a (mon.0) 65 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:21:57.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:57 smithi173 bash[27694]: audit 2023-09-23T18:21:57.077077+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:57.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:57 smithi098 bash[33599]: cephadm 2023-09-23T18:21:56.294780+0000 mgr.y (mgr.26054) 17 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.conf 2023-09-23T18:21:57.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:57 smithi098 bash[33599]: cephadm 2023-09-23T18:21:56.295124+0000 mgr.y (mgr.26054) 18 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-09-23T18:21:57.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:57 smithi098 bash[33599]: cephadm 2023-09-23T18:21:56.460368+0000 mgr.y (mgr.26054) 19 : cephadm [INF] Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:21:57.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:57 smithi098 bash[33599]: cephadm 2023-09-23T18:21:56.460725+0000 mgr.y (mgr.26054) 20 : cephadm [INF] Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:21:57.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:57 smithi098 bash[33599]: cephadm 2023-09-23T18:21:56.624879+0000 mgr.y (mgr.26054) 21 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:21:57.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:57 smithi098 bash[33599]: cephadm 2023-09-23T18:21:56.627173+0000 mgr.y (mgr.26054) 22 : cephadm [INF] Updating smithi098:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:21:57.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:57 smithi098 bash[33599]: audit 2023-09-23T18:21:56.962832+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:57.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:57 smithi098 bash[33599]: audit 2023-09-23T18:21:56.969359+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:57.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:57 smithi098 bash[33599]: audit 2023-09-23T18:21:56.974168+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:57.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:57 smithi098 bash[33599]: audit 2023-09-23T18:21:56.978393+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:57.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:57 smithi098 bash[33599]: audit 2023-09-23T18:21:56.983652+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:57.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:57 smithi098 bash[33599]: audit 2023-09-23T18:21:57.063883+0000 mon.a (mon.0) 63 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:21:57.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:57 smithi098 bash[33599]: audit 2023-09-23T18:21:57.066631+0000 mon.a (mon.0) 64 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:21:57.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:57 smithi098 bash[33599]: audit 2023-09-23T18:21:57.067858+0000 mon.a (mon.0) 65 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:21:57.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:57 smithi098 bash[33599]: audit 2023-09-23T18:21:57.077077+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:21:58.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:58 smithi098 bash[33599]: cephadm 2023-09-23T18:21:56.789742+0000 mgr.y (mgr.26054) 23 : cephadm [INF] Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:21:58.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:58 smithi098 bash[33599]: cephadm 2023-09-23T18:21:56.791782+0000 mgr.y (mgr.26054) 24 : cephadm [INF] Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:21:58.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:21:58 smithi098 bash[33599]: cluster 2023-09-23T18:21:57.443412+0000 mgr.y (mgr.26054) 25 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:59.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:58 smithi173 bash[27694]: cephadm 2023-09-23T18:21:56.789742+0000 mgr.y (mgr.26054) 23 : cephadm [INF] Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:21:59.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:58 smithi173 bash[27694]: cephadm 2023-09-23T18:21:56.791782+0000 mgr.y (mgr.26054) 24 : cephadm [INF] Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:21:59.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:21:58 smithi173 bash[27694]: cluster 2023-09-23T18:21:57.443412+0000 mgr.y (mgr.26054) 25 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:21:59.028 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:21:58 smithi173 bash[54982]: ts=2023-09-23T18:21:58.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi098:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:22:00.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:00 smithi173 bash[27694]: cluster 2023-09-23T18:21:59.444235+0000 mgr.y (mgr.26054) 26 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:00.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:00 smithi098 bash[33599]: cluster 2023-09-23T18:21:59.444235+0000 mgr.y (mgr.26054) 26 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:02.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:02 smithi173 bash[27694]: cluster 2023-09-23T18:22:01.445424+0000 mgr.y (mgr.26054) 27 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:02.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:02 smithi098 bash[33599]: cluster 2023-09-23T18:22:01.445424+0000 mgr.y (mgr.26054) 27 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:04.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:04 smithi173 bash[27694]: cluster 2023-09-23T18:22:03.446032+0000 mgr.y (mgr.26054) 28 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:04.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:04 smithi098 bash[33599]: cluster 2023-09-23T18:22:03.446032+0000 mgr.y (mgr.26054) 28 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:06.445 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:22:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:22:06] "GET /metrics HTTP/1.1" 200 33916 "" "Prometheus/2.33.4" 2023-09-23T18:22:06.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:06 smithi173 bash[27694]: cluster 2023-09-23T18:22:05.447102+0000 mgr.y (mgr.26054) 29 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:06.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:06 smithi098 bash[33599]: cluster 2023-09-23T18:22:05.447102+0000 mgr.y (mgr.26054) 29 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:08.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:08 smithi173 bash[27694]: cluster 2023-09-23T18:22:07.447708+0000 mgr.y (mgr.26054) 30 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:08.777 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:22:08 smithi173 bash[54982]: ts=2023-09-23T18:22:08.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi098:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:22:08.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:08 smithi098 bash[33599]: cluster 2023-09-23T18:22:07.447708+0000 mgr.y (mgr.26054) 30 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:10.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:10 smithi173 bash[27694]: cluster 2023-09-23T18:22:09.448284+0000 mgr.y (mgr.26054) 31 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:10.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:10 smithi173 bash[27694]: audit 2023-09-23T18:22:09.559589+0000 mon.a (mon.0) 67 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:22:10.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:10 smithi098 bash[33599]: cluster 2023-09-23T18:22:09.448284+0000 mgr.y (mgr.26054) 31 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:10.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:10 smithi098 bash[33599]: audit 2023-09-23T18:22:09.559589+0000 mon.a (mon.0) 67 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:22:12.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:12 smithi173 bash[27694]: cluster 2023-09-23T18:22:11.449494+0000 mgr.y (mgr.26054) 32 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:12.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:12 smithi098 bash[33599]: cluster 2023-09-23T18:22:11.449494+0000 mgr.y (mgr.26054) 32 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:14.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:14 smithi173 bash[27694]: cluster 2023-09-23T18:22:13.450217+0000 mgr.y (mgr.26054) 33 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:14.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:14 smithi098 bash[33599]: cluster 2023-09-23T18:22:13.450217+0000 mgr.y (mgr.26054) 33 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:16.445 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:22:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:22:16] "GET /metrics HTTP/1.1" 200 33917 "" "Prometheus/2.33.4" 2023-09-23T18:22:16.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:16 smithi173 bash[27694]: cluster 2023-09-23T18:22:15.451280+0000 mgr.y (mgr.26054) 34 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:16.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:16 smithi098 bash[33599]: cluster 2023-09-23T18:22:15.451280+0000 mgr.y (mgr.26054) 34 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:18.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:18 smithi173 bash[27694]: cluster 2023-09-23T18:22:17.452053+0000 mgr.y (mgr.26054) 35 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:18.778 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:22:18 smithi173 bash[54982]: ts=2023-09-23T18:22:18.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi098:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:22:18.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:18 smithi098 bash[33599]: cluster 2023-09-23T18:22:17.452053+0000 mgr.y (mgr.26054) 35 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:20.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:20 smithi173 bash[27694]: cluster 2023-09-23T18:22:19.452863+0000 mgr.y (mgr.26054) 36 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:20.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:20 smithi098 bash[33599]: cluster 2023-09-23T18:22:19.452863+0000 mgr.y (mgr.26054) 36 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:22.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:22 smithi173 bash[27694]: cluster 2023-09-23T18:22:21.453942+0000 mgr.y (mgr.26054) 37 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:22.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:22 smithi098 bash[33599]: cluster 2023-09-23T18:22:21.453942+0000 mgr.y (mgr.26054) 37 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:24.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:24 smithi173 bash[27694]: cluster 2023-09-23T18:22:23.454638+0000 mgr.y (mgr.26054) 38 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:24.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:24 smithi098 bash[33599]: cluster 2023-09-23T18:22:23.454638+0000 mgr.y (mgr.26054) 38 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:25.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:25 smithi098 bash[33599]: audit 2023-09-23T18:22:24.559826+0000 mon.a (mon.0) 68 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:22:26.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:25 smithi173 bash[27694]: audit 2023-09-23T18:22:24.559826+0000 mon.a (mon.0) 68 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:22:26.445 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:22:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:22:26] "GET /metrics HTTP/1.1" 200 33915 "" "Prometheus/2.33.4" 2023-09-23T18:22:26.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:26 smithi098 bash[33599]: cluster 2023-09-23T18:22:25.455542+0000 mgr.y (mgr.26054) 39 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:27.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:26 smithi173 bash[27694]: cluster 2023-09-23T18:22:25.455542+0000 mgr.y (mgr.26054) 39 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:28.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:28 smithi098 bash[33599]: cluster 2023-09-23T18:22:27.456189+0000 mgr.y (mgr.26054) 40 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:29.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:28 smithi173 bash[27694]: cluster 2023-09-23T18:22:27.456189+0000 mgr.y (mgr.26054) 40 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:29.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:22:28 smithi173 bash[54982]: ts=2023-09-23T18:22:28.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi098:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:22:30.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:30 smithi173 bash[27694]: cluster 2023-09-23T18:22:29.456881+0000 mgr.y (mgr.26054) 41 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:30.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:30 smithi098 bash[33599]: cluster 2023-09-23T18:22:29.456881+0000 mgr.y (mgr.26054) 41 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:32.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:32 smithi173 bash[27694]: cluster 2023-09-23T18:22:31.457974+0000 mgr.y (mgr.26054) 42 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:32.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:32 smithi098 bash[33599]: cluster 2023-09-23T18:22:31.457974+0000 mgr.y (mgr.26054) 42 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:34.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:34 smithi173 bash[27694]: cluster 2023-09-23T18:22:33.458719+0000 mgr.y (mgr.26054) 43 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:34.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:34 smithi098 bash[33599]: cluster 2023-09-23T18:22:33.458719+0000 mgr.y (mgr.26054) 43 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:36.445 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:22:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:22:36] "GET /metrics HTTP/1.1" 200 33915 "" "Prometheus/2.33.4" 2023-09-23T18:22:36.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:36 smithi098 bash[33599]: cluster 2023-09-23T18:22:35.459609+0000 mgr.y (mgr.26054) 44 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:37.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:36 smithi173 bash[27694]: cluster 2023-09-23T18:22:35.459609+0000 mgr.y (mgr.26054) 44 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:38.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:38 smithi098 bash[33599]: cluster 2023-09-23T18:22:37.460322+0000 mgr.y (mgr.26054) 45 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:39.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:38 smithi173 bash[27694]: cluster 2023-09-23T18:22:37.460322+0000 mgr.y (mgr.26054) 45 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:39.028 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:22:38 smithi173 bash[54982]: ts=2023-09-23T18:22:38.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi098:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:22:40.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:40 smithi173 bash[27694]: cluster 2023-09-23T18:22:39.461091+0000 mgr.y (mgr.26054) 46 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:40.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:40 smithi173 bash[27694]: audit 2023-09-23T18:22:39.559776+0000 mon.a (mon.0) 69 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:22:40.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:40 smithi098 bash[33599]: cluster 2023-09-23T18:22:39.461091+0000 mgr.y (mgr.26054) 46 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:40.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:40 smithi098 bash[33599]: audit 2023-09-23T18:22:39.559776+0000 mon.a (mon.0) 69 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:22:42.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:42 smithi173 bash[27694]: cluster 2023-09-23T18:22:41.462331+0000 mgr.y (mgr.26054) 47 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:42.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:42 smithi098 bash[33599]: cluster 2023-09-23T18:22:41.462331+0000 mgr.y (mgr.26054) 47 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:44.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:44 smithi098 bash[33599]: cluster 2023-09-23T18:22:43.462956+0000 mgr.y (mgr.26054) 48 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:45.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:44 smithi173 bash[27694]: cluster 2023-09-23T18:22:43.462956+0000 mgr.y (mgr.26054) 48 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:46.445 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:22:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:22:46] "GET /metrics HTTP/1.1" 200 33917 "" "Prometheus/2.33.4" 2023-09-23T18:22:46.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:46 smithi098 bash[33599]: cluster 2023-09-23T18:22:45.463968+0000 mgr.y (mgr.26054) 49 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:47.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:46 smithi173 bash[27694]: cluster 2023-09-23T18:22:45.463968+0000 mgr.y (mgr.26054) 49 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:48.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:48 smithi098 bash[33599]: cluster 2023-09-23T18:22:47.464603+0000 mgr.y (mgr.26054) 50 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:49.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:48 smithi173 bash[27694]: cluster 2023-09-23T18:22:47.464603+0000 mgr.y (mgr.26054) 50 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:49.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:22:48 smithi173 bash[54982]: ts=2023-09-23T18:22:48.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi098:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:22:49.854 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T18:22:50.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:50 smithi098 bash[33599]: cluster 2023-09-23T18:22:49.464996+0000 mgr.y (mgr.26054) 51 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:50.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:50 smithi173 bash[27694]: cluster 2023-09-23T18:22:49.464996+0000 mgr.y (mgr.26054) 51 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:52.920 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:52 smithi098 bash[33599]: cluster 2023-09-23T18:22:51.465990+0000 mgr.y (mgr.26054) 52 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:52 smithi173 bash[27694]: cluster 2023-09-23T18:22:51.465990+0000 mgr.y (mgr.26054) 52 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:53.381 INFO:teuthology.orchestra.run.smithi098.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:22:54.181 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for majority of mons to upgrade' 2023-09-23T18:22:54.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:54 smithi098 bash[33599]: audit 2023-09-23T18:22:53.370857+0000 mgr.y (mgr.26054) 53 : audit [DBG] from='client.26114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:22:54.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:54 smithi098 bash[33599]: cephadm 2023-09-23T18:22:53.371233+0000 mgr.y (mgr.26054) 54 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:22:54.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:54 smithi098 bash[33599]: audit 2023-09-23T18:22:53.375263+0000 mon.a (mon.0) 70 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:22:54.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:54 smithi098 bash[33599]: audit 2023-09-23T18:22:53.425748+0000 mon.a (mon.0) 71 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:22:54.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:54 smithi098 bash[33599]: cluster 2023-09-23T18:22:53.466641+0000 mgr.y (mgr.26054) 55 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:54.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:54 smithi173 bash[27694]: audit 2023-09-23T18:22:53.370857+0000 mgr.y (mgr.26054) 53 : audit [DBG] from='client.26114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:22:54.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:54 smithi173 bash[27694]: cephadm 2023-09-23T18:22:53.371233+0000 mgr.y (mgr.26054) 54 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:22:54.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:54 smithi173 bash[27694]: audit 2023-09-23T18:22:53.375263+0000 mon.a (mon.0) 70 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:22:54.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:54 smithi173 bash[27694]: audit 2023-09-23T18:22:53.425748+0000 mon.a (mon.0) 71 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:22:54.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:54 smithi173 bash[27694]: cluster 2023-09-23T18:22:53.466641+0000 mgr.y (mgr.26054) 55 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:55.670 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:55 smithi098 bash[33599]: audit 2023-09-23T18:22:54.559958+0000 mon.a (mon.0) 72 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:22:55.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:55 smithi173 bash[27694]: audit 2023-09-23T18:22:54.559958+0000 mon.a (mon.0) 72 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:22:56.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:22:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:22:56] "GET /metrics HTTP/1.1" 200 33915 "" "Prometheus/2.33.4" 2023-09-23T18:22:56.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:56 smithi098 bash[33599]: cluster 2023-09-23T18:22:55.467695+0000 mgr.y (mgr.26054) 56 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:56.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:56 smithi173 bash[27694]: cluster 2023-09-23T18:22:55.467695+0000 mgr.y (mgr.26054) 56 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:57.467 INFO:teuthology.orchestra.run.smithi098.stdout:wait for majority of mons to upgrade 2023-09-23T18:22:58.344 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-09-23T18:22:58.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:22:58 smithi098 bash[33599]: cluster 2023-09-23T18:22:57.468213+0000 mgr.y (mgr.26054) 57 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:59.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:22:58 smithi173 bash[27694]: cluster 2023-09-23T18:22:57.468213+0000 mgr.y (mgr.26054) 57 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:22:59.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:22:58 smithi173 bash[54982]: ts=2023-09-23T18:22:58.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi098:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:23:01.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:00 smithi173 bash[27694]: cluster 2023-09-23T18:22:59.468829+0000 mgr.y (mgr.26054) 58 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:23:01.257 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:00 smithi173 bash[27694]: audit 2023-09-23T18:22:59.766395+0000 mon.a (mon.0) 73 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:23:01.257 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:00 smithi173 bash[27694]: audit 2023-09-23T18:22:59.772928+0000 mon.a (mon.0) 74 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:23:01.257 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:00 smithi173 bash[27694]: audit 2023-09-23T18:23:00.581622+0000 mon.a (mon.0) 75 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:23:01.257 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:00 smithi173 bash[27694]: audit 2023-09-23T18:23:00.588645+0000 mon.a (mon.0) 76 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:23:01.258 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:00 smithi098 bash[33599]: cluster 2023-09-23T18:22:59.468829+0000 mgr.y (mgr.26054) 58 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:23:01.258 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:00 smithi098 bash[33599]: audit 2023-09-23T18:22:59.766395+0000 mon.a (mon.0) 73 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:23:01.258 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:00 smithi098 bash[33599]: audit 2023-09-23T18:22:59.772928+0000 mon.a (mon.0) 74 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:23:01.258 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:00 smithi098 bash[33599]: audit 2023-09-23T18:23:00.581622+0000 mon.a (mon.0) 75 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:23:01.258 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:00 smithi098 bash[33599]: audit 2023-09-23T18:23:00.588645+0000 mon.a (mon.0) 76 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:23:02.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:01 smithi173 bash[27694]: audit 2023-09-23T18:23:01.055091+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:23:02.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:01 smithi173 bash[27694]: audit 2023-09-23T18:23:01.056707+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:23:02.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:01 smithi173 bash[27694]: audit 2023-09-23T18:23:01.066218+0000 mon.a (mon.0) 79 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:23:02.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:01 smithi173 bash[27694]: audit 2023-09-23T18:23:01.148154+0000 mon.a (mon.0) 80 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:23:02.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:01 smithi173 bash[27694]: audit 2023-09-23T18:23:01.151618+0000 mon.a (mon.0) 81 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:23:02.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:01 smithi173 bash[27694]: audit 2023-09-23T18:23:01.153268+0000 mon.a (mon.0) 82 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T18:23:02.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:01 smithi173 bash[27694]: audit 2023-09-23T18:23:01.154246+0000 mon.a (mon.0) 83 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T18:23:02.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:01 smithi098 bash[33599]: audit 2023-09-23T18:23:01.055091+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:23:02.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:01 smithi098 bash[33599]: audit 2023-09-23T18:23:01.056707+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:23:02.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:01 smithi098 bash[33599]: audit 2023-09-23T18:23:01.066218+0000 mon.a (mon.0) 79 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:23:02.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:01 smithi098 bash[33599]: audit 2023-09-23T18:23:01.148154+0000 mon.a (mon.0) 80 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:23:02.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:01 smithi098 bash[33599]: audit 2023-09-23T18:23:01.151618+0000 mon.a (mon.0) 81 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:23:02.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:01 smithi098 bash[33599]: audit 2023-09-23T18:23:01.153268+0000 mon.a (mon.0) 82 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T18:23:02.197 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:01 smithi098 bash[33599]: audit 2023-09-23T18:23:01.154246+0000 mon.a (mon.0) 83 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T18:23:02.846 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:02 smithi098 bash[33599]: cephadm 2023-09-23T18:23:01.154948+0000 mgr.y (mgr.26054) 59 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-09-23T18:23:02.846 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:02 smithi098 bash[33599]: cluster 2023-09-23T18:23:01.469506+0000 mgr.y (mgr.26054) 60 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:23:02.846 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:02 smithi098 bash[33599]: audit 2023-09-23T18:23:01.967110+0000 mon.a (mon.0) 84 : audit [DBG] from='client.? 172.21.15.98:0/2089910462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:23:03.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:02 smithi173 bash[27694]: cephadm 2023-09-23T18:23:01.154948+0000 mgr.y (mgr.26054) 59 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-09-23T18:23:03.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:02 smithi173 bash[27694]: cluster 2023-09-23T18:23:01.469506+0000 mgr.y (mgr.26054) 60 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:23:03.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:02 smithi173 bash[27694]: audit 2023-09-23T18:23:01.967110+0000 mon.a (mon.0) 84 : audit [DBG] from='client.? 172.21.15.98:0/2089910462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:23:04.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:04 smithi098 bash[33599]: cluster 2023-09-23T18:23:03.469941+0000 mgr.y (mgr.26054) 61 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:23:04.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:04 smithi098 bash[33599]: cephadm 2023-09-23T18:23:03.674657+0000 mgr.y (mgr.26054) 62 : cephadm [INF] Upgrade: Updating mon.c 2023-09-23T18:23:04.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:04 smithi098 bash[33599]: audit 2023-09-23T18:23:03.683308+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:23:04.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:04 smithi098 bash[33599]: audit 2023-09-23T18:23:03.684782+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:23:04.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:04 smithi098 bash[33599]: audit 2023-09-23T18:23:03.685776+0000 mon.a (mon.0) 87 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:23:04.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:04 smithi098 bash[33599]: audit 2023-09-23T18:23:03.686903+0000 mon.a (mon.0) 88 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:23:04.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:04 smithi098 bash[33599]: cephadm 2023-09-23T18:23:03.688259+0000 mgr.y (mgr.26054) 63 : cephadm [INF] Deploying daemon mon.c on smithi098 2023-09-23T18:23:04.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:04 smithi098 bash[33599]: audit 2023-09-23T18:23:04.399495+0000 mon.a (mon.0) 89 : audit [DBG] from='client.? 172.21.15.98:0/964655738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:23:05.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:04 smithi173 bash[27694]: cluster 2023-09-23T18:23:03.469941+0000 mgr.y (mgr.26054) 61 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:23:05.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:04 smithi173 bash[27694]: cephadm 2023-09-23T18:23:03.674657+0000 mgr.y (mgr.26054) 62 : cephadm [INF] Upgrade: Updating mon.c 2023-09-23T18:23:05.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:04 smithi173 bash[27694]: audit 2023-09-23T18:23:03.683308+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:23:05.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:04 smithi173 bash[27694]: audit 2023-09-23T18:23:03.684782+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:23:05.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:04 smithi173 bash[27694]: audit 2023-09-23T18:23:03.685776+0000 mon.a (mon.0) 87 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:23:05.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:04 smithi173 bash[27694]: audit 2023-09-23T18:23:03.686903+0000 mon.a (mon.0) 88 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:23:05.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:04 smithi173 bash[27694]: cephadm 2023-09-23T18:23:03.688259+0000 mgr.y (mgr.26054) 63 : cephadm [INF] Deploying daemon mon.c on smithi098 2023-09-23T18:23:05.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:04 smithi173 bash[27694]: audit 2023-09-23T18:23:04.399495+0000 mon.a (mon.0) 89 : audit [DBG] from='client.? 172.21.15.98:0/964655738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:23:06.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:23:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:23:06] "GET /metrics HTTP/1.1" 200 33915 "" "Prometheus/2.33.4" 2023-09-23T18:23:06.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:06 smithi098 systemd[1]: Stopping Ceph mon.c for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T18:23:06.598 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:06 smithi098 bash[33599]: debug 2023-09-23T18:23:06.258+0000 7f0efe020700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-09-23T18:23:06.598 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:06 smithi098 bash[33599]: debug 2023-09-23T18:23:06.258+0000 7f0efe020700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-09-23T18:23:07.398 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:06 smithi098 bash[79369]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-mon-c 2023-09-23T18:23:07.399 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:07 smithi098 bash[79460]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-mon-c 2023-09-23T18:23:07.399 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:07 smithi098 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mon.c.service: Succeeded. 2023-09-23T18:23:07.399 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:07 smithi098 systemd[1]: Stopped Ceph mon.c for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:23:07.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:07 smithi098 systemd[1]: Started Ceph mon.c for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:23:08.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.542+0000 7f5d29a9ac80 0 set uid:gid to 167:167 (ceph:ceph) 2023-09-23T18:23:08.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.542+0000 7f5d29a9ac80 0 ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T18:23:08.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.542+0000 7f5d29a9ac80 0 pidfile_write: ignore empty --pid-file 2023-09-23T18:23:08.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 0 load: jerasure load: lrc 2023-09-23T18:23:08.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: RocksDB version: 7.9.2 2023-09-23T18:23:08.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Git sha 0 2023-09-23T18:23:08.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T18:23:08.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: DB SUMMARY 2023-09-23T18:23:08.696 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: DB Session ID: 20IR59ID28AEXXAJN7Q2 2023-09-23T18:23:08.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: CURRENT file: CURRENT 2023-09-23T18:23:08.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: IDENTITY file: IDENTITY 2023-09-23T18:23:08.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 1277 Bytes 2023-09-23T18:23:08.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000026.sst 2023-09-23T18:23:08.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000024.log size: 1218455 ; 2023-09-23T18:23:08.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.error_if_exists: 0 2023-09-23T18:23:08.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.create_if_missing: 0 2023-09-23T18:23:08.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.paranoid_checks: 1 2023-09-23T18:23:08.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T18:23:08.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T18:23:08.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T18:23:08.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.env: 0x5651319a7720 2023-09-23T18:23:08.698 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.fs: PosixFileSystem 2023-09-23T18:23:08.698 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.info_log: 0x565134715480 2023-09-23T18:23:08.698 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_file_opening_threads: 16 2023-09-23T18:23:08.698 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.statistics: (nil) 2023-09-23T18:23:08.698 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.use_fsync: 0 2023-09-23T18:23:08.698 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_log_file_size: 0 2023-09-23T18:23:08.699 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T18:23:08.699 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T18:23:08.699 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.keep_log_file_num: 1000 2023-09-23T18:23:08.699 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.recycle_log_file_num: 0 2023-09-23T18:23:08.699 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.allow_fallocate: 1 2023-09-23T18:23:08.699 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.allow_mmap_reads: 0 2023-09-23T18:23:08.700 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.allow_mmap_writes: 0 2023-09-23T18:23:08.700 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.use_direct_reads: 0 2023-09-23T18:23:08.700 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T18:23:08.700 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.create_missing_column_families: 0 2023-09-23T18:23:08.700 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.db_log_dir: 2023-09-23T18:23:08.700 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.wal_dir: 2023-09-23T18:23:08.700 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T18:23:08.700 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T18:23:08.700 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T18:23:08.700 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T18:23:08.700 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T18:23:08.700 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T18:23:08.701 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.advise_random_on_open: 1 2023-09-23T18:23:08.701 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.db_write_buffer_size: 0 2023-09-23T18:23:08.701 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.write_buffer_manager: 0x5651339a2c80 2023-09-23T18:23:08.701 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T18:23:08.701 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T18:23:08.701 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T18:23:08.701 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.rate_limiter: (nil) 2023-09-23T18:23:08.701 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T18:23:08.701 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.wal_recovery_mode: 2 2023-09-23T18:23:08.701 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.enable_thread_tracking: 0 2023-09-23T18:23:08.702 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.enable_pipelined_write: 0 2023-09-23T18:23:08.702 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.unordered_write: 0 2023-09-23T18:23:08.702 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T18:23:08.702 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T18:23:08.702 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T18:23:08.702 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T18:23:08.702 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.row_cache: None 2023-09-23T18:23:08.702 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.wal_filter: None 2023-09-23T18:23:08.702 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T18:23:08.702 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.allow_ingest_behind: 0 2023-09-23T18:23:08.702 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.two_write_queues: 0 2023-09-23T18:23:08.702 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.manual_wal_flush: 0 2023-09-23T18:23:08.703 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.wal_compression: 0 2023-09-23T18:23:08.703 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.atomic_flush: 0 2023-09-23T18:23:08.703 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T18:23:08.703 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T18:23:08.703 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T18:23:08.703 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.log_readahead_size: 0 2023-09-23T18:23:08.703 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T18:23:08.703 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.best_efforts_recovery: 0 2023-09-23T18:23:08.703 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T18:23:08.703 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T18:23:08.703 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.allow_data_in_errors: 0 2023-09-23T18:23:08.703 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.db_host_id: __hostname__ 2023-09-23T18:23:08.703 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T18:23:08.704 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_background_jobs: 2 2023-09-23T18:23:08.704 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_background_compactions: -1 2023-09-23T18:23:08.704 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_subcompactions: 1 2023-09-23T18:23:08.704 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T18:23:08.704 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T18:23:08.704 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T18:23:08.704 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_total_wal_size: 0 2023-09-23T18:23:08.704 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T18:23:08.704 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T18:23:08.704 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T18:23:08.704 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T18:23:08.704 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_open_files: -1 2023-09-23T18:23:08.705 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.bytes_per_sync: 0 2023-09-23T18:23:08.705 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T18:23:08.705 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T18:23:08.705 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.compaction_readahead_size: 0 2023-09-23T18:23:08.705 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_background_flushes: -1 2023-09-23T18:23:08.705 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Compression algorithms supported: 2023-09-23T18:23:08.705 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T18:23:08.705 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: kZSTD supported: 0 2023-09-23T18:23:08.705 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: kXpressCompression supported: 0 2023-09-23T18:23:08.705 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: kLZ4HCCompression supported: 1 2023-09-23T18:23:08.705 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: kZlibCompression supported: 1 2023-09-23T18:23:08.705 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: kSnappyCompression supported: 1 2023-09-23T18:23:08.705 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: kLZ4Compression supported: 1 2023-09-23T18:23:08.706 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: kBZip2Compression supported: 0 2023-09-23T18:23:08.706 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T18:23:08.706 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T18:23:08.706 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2023-09-23T18:23:08.706 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T18:23:08.706 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T18:23:08.706 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.merge_operator: 2023-09-23T18:23:08.706 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.compaction_filter: None 2023-09-23T18:23:08.706 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.compaction_filter_factory: None 2023-09-23T18:23:08.706 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.sst_partitioner_factory: None 2023-09-23T18:23:08.706 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T18:23:08.706 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T18:23:08.707 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x565134715580) 2023-09-23T18:23:08.707 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: cache_index_and_filter_blocks: 1 2023-09-23T18:23:08.707 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T18:23:08.707 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T18:23:08.707 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: pin_top_level_index_and_filter: 1 2023-09-23T18:23:08.707 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: index_type: 0 2023-09-23T18:23:08.707 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: data_block_index_type: 0 2023-09-23T18:23:08.707 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: index_shortening: 1 2023-09-23T18:23:08.707 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: data_block_hash_table_util_ratio: 0.750000 2023-09-23T18:23:08.707 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: checksum: 4 2023-09-23T18:23:08.708 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: no_block_cache: 0 2023-09-23T18:23:08.708 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: block_cache: 0x565133a251f0 2023-09-23T18:23:08.708 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: block_cache_name: BinnedLRUCache 2023-09-23T18:23:08.708 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: block_cache_options: 2023-09-23T18:23:08.708 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: capacity : 536870912 2023-09-23T18:23:08.708 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: num_shard_bits : 4 2023-09-23T18:23:08.708 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: strict_capacity_limit : 0 2023-09-23T18:23:08.708 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: high_pri_pool_ratio: 0.000 2023-09-23T18:23:08.708 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: block_cache_compressed: (nil) 2023-09-23T18:23:08.708 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: persistent_cache: (nil) 2023-09-23T18:23:08.708 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: block_size: 4096 2023-09-23T18:23:08.708 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: block_size_deviation: 10 2023-09-23T18:23:08.709 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: block_restart_interval: 16 2023-09-23T18:23:08.709 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: index_block_restart_interval: 1 2023-09-23T18:23:08.709 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: metadata_block_size: 4096 2023-09-23T18:23:08.709 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: partition_filters: 0 2023-09-23T18:23:08.709 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: use_delta_encoding: 1 2023-09-23T18:23:08.709 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: filter_policy: bloomfilter 2023-09-23T18:23:08.709 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: whole_key_filtering: 1 2023-09-23T18:23:08.709 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: verify_compression: 0 2023-09-23T18:23:08.709 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: read_amp_bytes_per_bit: 0 2023-09-23T18:23:08.709 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: format_version: 5 2023-09-23T18:23:08.709 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: enable_index_compression: 1 2023-09-23T18:23:08.709 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: block_align: 0 2023-09-23T18:23:08.710 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: max_auto_readahead_size: 262144 2023-09-23T18:23:08.710 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: prepopulate_block_cache: 0 2023-09-23T18:23:08.710 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: initial_auto_readahead_size: 8192 2023-09-23T18:23:08.710 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: num_file_reads_for_auto_readahead: 2 2023-09-23T18:23:08.710 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.write_buffer_size: 33554432 2023-09-23T18:23:08.710 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_write_buffer_number: 2 2023-09-23T18:23:08.710 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.compression: NoCompression 2023-09-23T18:23:08.710 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.bottommost_compression: Disabled 2023-09-23T18:23:08.710 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.prefix_extractor: nullptr 2023-09-23T18:23:08.711 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T18:23:08.711 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.num_levels: 7 2023-09-23T18:23:08.711 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T18:23:08.711 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T18:23:08.711 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T18:23:08.711 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T18:23:08.711 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T18:23:08.711 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T18:23:08.711 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T18:23:08.711 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T18:23:08.711 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T18:23:08.712 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T18:23:08.712 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T18:23:08.712 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T18:23:08.712 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T18:23:08.712 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.compression_opts.level: 32767 2023-09-23T18:23:08.712 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.compression_opts.strategy: 0 2023-09-23T18:23:08.712 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T18:23:08.712 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T18:23:08.712 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T18:23:08.712 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T18:23:08.713 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.compression_opts.enabled: false 2023-09-23T18:23:08.713 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T18:23:08.713 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T18:23:08.713 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T18:23:08.713 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T18:23:08.713 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.target_file_size_base: 67108864 2023-09-23T18:23:08.713 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T18:23:08.713 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T18:23:08.713 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T18:23:08.713 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T18:23:08.713 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T18:23:08.713 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T18:23:08.714 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T18:23:08.714 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T18:23:08.714 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T18:23:08.714 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T18:23:08.714 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T18:23:08.714 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T18:23:08.714 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T18:23:08.714 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T18:23:08.714 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.arena_block_size: 1048576 2023-09-23T18:23:08.715 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T18:23:08.715 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T18:23:08.715 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.disable_auto_compactions: 0 2023-09-23T18:23:08.715 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T18:23:08.715 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T18:23:08.715 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T18:23:08.715 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T18:23:08.715 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T18:23:08.716 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T18:23:08.717 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T18:23:08.717 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T18:23:08.717 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T18:23:08.717 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T18:23:08.717 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.table_properties_collectors: 2023-09-23T18:23:08.717 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.inplace_update_support: 0 2023-09-23T18:23:08.717 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T18:23:08.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T18:23:08.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T18:23:08.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T18:23:08.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.bloom_locality: 0 2023-09-23T18:23:08.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.max_successive_merges: 0 2023-09-23T18:23:08.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T18:23:08.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.paranoid_file_checks: 0 2023-09-23T18:23:08.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.force_consistency_checks: 1 2023-09-23T18:23:08.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.report_bg_io_stats: 0 2023-09-23T18:23:08.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.ttl: 2592000 2023-09-23T18:23:08.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T18:23:08.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T18:23:08.719 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T18:23:08.719 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.enable_blob_files: false 2023-09-23T18:23:08.719 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.min_blob_size: 0 2023-09-23T18:23:08.719 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.blob_file_size: 268435456 2023-09-23T18:23:08.719 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T18:23:08.719 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T18:23:08.719 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T18:23:08.719 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T18:23:08.719 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T18:23:08.719 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.blob_file_starting_level: 0 2023-09-23T18:23:08.720 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d29a9ac80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T18:23:08.720 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.546+0000 7f5d1f2b9700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 26.sst 2023-09-23T18:23:08.720 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.550+0000 7f5d29a9ac80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 28, last_sequence is 17892, log_number is 24,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T18:23:08.720 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.550+0000 7f5d29a9ac80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 24 2023-09-23T18:23:08.720 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.550+0000 7f5d29a9ac80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: aa8a17c7-d124-4ac5-9c1c-01410b5460ad 2023-09-23T18:23:08.720 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.550+0000 7f5d29a9ac80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695493388552554, "job": 1, "event": "recovery_started", "wal_files": [24]} 2023-09-23T18:23:08.720 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.550+0000 7f5d29a9ac80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #24 mode 2 2023-09-23T18:23:08.720 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.554+0000 7f5d29a9ac80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695493388557659, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 29, "file_size": 810405, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 17887, "largest_seqno": 18145, "table_properties": {"data_size": 808017, "index_size": 1120, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 261, "raw_key_size": 2028, "raw_average_key_size": 22, "raw_value_size": 805374, "raw_average_value_size": 8948, "num_data_blocks": 48, "num_entries": 90, "num_filter_entries": 90, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695493388, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "aa8a17c7-d124-4ac5-9c1c-01410b5460ad", "db_session_id": "20IR59ID28AEXXAJN7Q2", "orig_file_number": 29, "seqno_to_time_mapping": "N/A"}} 2023-09-23T18:23:08.721 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.554+0000 7f5d29a9ac80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695493388557790, "job": 1, "event": "recovery_finished"} 2023-09-23T18:23:08.721 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.554+0000 7f5d29a9ac80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 31 2023-09-23T18:23:08.721 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.558+0000 7f5d29a9ac80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000024.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T18:23:08.721 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.558+0000 7f5d29a9ac80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x565133ac6000 2023-09-23T18:23:08.721 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.558+0000 7f5d29a9ac80 4 rocksdb: DB pointer 0x565133ab4000 2023-09-23T18:23:08.721 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.558+0000 7f5d182ab700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T18:23:08.721 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.558+0000 7f5d182ab700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T18:23:08.721 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: ** DB Stats ** 2023-09-23T18:23:08.721 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T18:23:08.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T18:23:08.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T18:23:08.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T18:23:08.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T18:23:08.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T18:23:08.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T18:23:08.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: ** Compaction Stats [default] ** 2023-09-23T18:23:08.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T18:23:08.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T18:23:08.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: L0 1/0 791.41 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 312.5 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-09-23T18:23:08.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: L6 1/0 32.81 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T18:23:08.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: Sum 2/0 33.59 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 312.5 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-09-23T18:23:08.723 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 312.5 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-09-23T18:23:08.723 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: ** Compaction Stats [default] ** 2023-09-23T18:23:08.723 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T18:23:08.723 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T18:23:08.723 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 312.5 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-09-23T18:23:08.723 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T18:23:08.723 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T18:23:08.723 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: Flush(GB): cumulative 0.001, interval 0.001 2023-09-23T18:23:08.723 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T18:23:08.723 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T18:23:08.723 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T18:23:08.723 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: AddFile(Keys): cumulative 0, interval 0 2023-09-23T18:23:08.724 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: Cumulative compaction: 0.00 GB write, 76.99 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T18:23:08.724 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: Interval compaction: 0.00 GB write, 76.99 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T18:23:08.724 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T18:23:08.724 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: Block cache BinnedLRUCache@0x565133a251f0#7 capacity: 512.00 MB usage: 1.52 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 2e-05 secs_since: 0 2023-09-23T18:23:08.724 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: Block cache entry stats(count,size,portion): FilterBlock(1,0.31 KB,5.96046e-05%) IndexBlock(1,1.20 KB,0.000229478%) Misc(1,0.00 KB,0%) 2023-09-23T18:23:08.724 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: ** File Read Latency Histogram By Level [default] ** 2023-09-23T18:23:08.724 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.558+0000 7f5d29a9ac80 0 starting mon.c rank 2 at public addrs [v2:172.21.15.98:3301/0,v1:172.21.15.98:6790/0] at bind addrs [v2:172.21.15.98:3301/0,v1:172.21.15.98:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:23:08.724 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.558+0000 7f5d29a9ac80 1 mon.c@-1(???) e3 preinit fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:23:08.724 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.558+0000 7f5d29a9ac80 0 mon.c@-1(???).mds e7 new map 2023-09-23T18:23:08.724 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.558+0000 7f5d29a9ac80 0 mon.c@-1(???).mds e7 print_map 2023-09-23T18:23:08.724 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: e7 2023-09-23T18:23:08.724 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T18:23:08.724 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T18:23:08.725 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: legacy client fscid: 1 2023-09-23T18:23:08.725 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: 2023-09-23T18:23:08.725 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: Filesystem 'foo' (1) 2023-09-23T18:23:08.725 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: fs_name foo 2023-09-23T18:23:08.725 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: epoch 5 2023-09-23T18:23:08.725 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T18:23:08.725 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: created 2023-09-23T18:15:08.890010+0000 2023-09-23T18:23:08.725 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: modified 2023-09-23T18:15:16.817511+0000 2023-09-23T18:23:08.725 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: tableserver 0 2023-09-23T18:23:08.725 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: root 0 2023-09-23T18:23:08.725 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: session_timeout 60 2023-09-23T18:23:08.726 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: session_autoclose 300 2023-09-23T18:23:08.726 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: max_file_size 1099511627776 2023-09-23T18:23:08.726 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: required_client_features {} 2023-09-23T18:23:08.726 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: last_failure 0 2023-09-23T18:23:08.726 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: last_failure_osd_epoch 0 2023-09-23T18:23:08.726 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T18:23:08.726 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: max_mds 1 2023-09-23T18:23:08.726 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: in 0 2023-09-23T18:23:08.726 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: up {0=24545} 2023-09-23T18:23:08.726 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: failed 2023-09-23T18:23:08.726 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: damaged 2023-09-23T18:23:08.726 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: stopped 2023-09-23T18:23:08.727 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: data_pools [4] 2023-09-23T18:23:08.727 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: metadata_pool 3 2023-09-23T18:23:08.727 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: inline_data disabled 2023-09-23T18:23:08.727 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: balancer 2023-09-23T18:23:08.727 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: bal_rank_mask -1 2023-09-23T18:23:08.727 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: standby_count_wanted 1 2023-09-23T18:23:08.727 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: [mds.foo.smithi098.lowjqo{0:24545} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.98:6834/2708276357,v1:172.21.15.98:6835/2708276357] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T18:23:08.727 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: 2023-09-23T18:23:08.727 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: 2023-09-23T18:23:08.727 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: Standby daemons: 2023-09-23T18:23:08.727 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: 2023-09-23T18:23:08.728 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: [mds.foo.smithi173.qjefje{-1:24502} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.173:6832/621117609,v1:172.21.15.173:6833/621117609] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T18:23:08.728 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.562+0000 7f5d29a9ac80 0 mon.c@-1(???).osd e84 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T18:23:08.728 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.562+0000 7f5d29a9ac80 0 mon.c@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T18:23:08.728 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.562+0000 7f5d29a9ac80 0 mon.c@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T18:23:08.728 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.562+0000 7f5d29a9ac80 0 mon.c@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T18:23:08.728 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:23:08 smithi098 bash[79516]: debug 2023-09-23T18:23:08.562+0000 7f5d29a9ac80 1 mon.c@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2023-09-23T18:23:09.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:23:08 smithi173 bash[54982]: ts=2023-09-23T18:23:08.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi098:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:23:14.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:13 smithi173 bash[27694]: cluster 2023-09-23T18:23:05.470951+0000 mgr.y (mgr.26054) 64 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:23:15.323 INFO:teuthology.orchestra.run.smithi098.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 2 2023-09-23T18:23:15.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:15 smithi173 bash[27694]: cluster 2023-09-23T18:23:08.772468+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-09-23T18:23:15.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:15 smithi173 bash[27694]: cluster 2023-09-23T18:23:14.794714+0000 mon.a (mon.0) 105 : cluster [INF] mon.a calling monitor election 2023-09-23T18:23:15.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:15 smithi173 bash[27694]: cluster 2023-09-23T18:23:14.799231+0000 mon.b (mon.1) 106 : cluster [INF] mon.b calling monitor election 2023-09-23T18:23:15.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:15 smithi173 bash[27694]: cluster 2023-09-23T18:23:14.836984+0000 mon.a (mon.0) 106 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T18:23:15.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:15 smithi173 bash[27694]: cluster 2023-09-23T18:23:14.847245+0000 mon.a (mon.0) 107 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0],b=[v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0],c=[v2:172.21.15.98:3301/0,v1:172.21.15.98:6790/0]} removed_ranks: {} 2023-09-23T18:23:15.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:15 smithi173 bash[27694]: cluster 2023-09-23T18:23:14.847878+0000 mon.a (mon.0) 108 : cluster [DBG] fsmap foo:1 {0=foo.smithi098.lowjqo=up:active} 1 up:standby 2023-09-23T18:23:15.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:15 smithi173 bash[27694]: cluster 2023-09-23T18:23:14.847928+0000 mon.a (mon.0) 109 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2023-09-23T18:23:15.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:15 smithi173 bash[27694]: cluster 2023-09-23T18:23:14.848622+0000 mon.a (mon.0) 110 : cluster [DBG] mgrmap e34: y(active, since 95s), standbys: x 2023-09-23T18:23:15.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:15 smithi173 bash[27694]: cluster 2023-09-23T18:23:14.849009+0000 mon.a (mon.0) 111 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T18:23:15.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:15 smithi173 bash[27694]: cluster 2023-09-23T18:23:14.849056+0000 mon.a (mon.0) 112 : cluster [INF] Cluster is now healthy 2023-09-23T18:23:15.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:15 smithi173 bash[27694]: cluster 2023-09-23T18:23:14.875925+0000 mon.a (mon.0) 113 : cluster [INF] overall HEALTH_OK 2023-09-23T18:23:16.307 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T18:23:16.445 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:23:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:23:16] "GET /metrics HTTP/1.1" 200 33908 "" "Prometheus/2.33.4" 2023-09-23T18:23:16.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:16 smithi173 bash[27694]: cluster 2023-09-23T18:23:15.250167+0000 mon.a (mon.0) 114 : cluster [DBG] mgrmap e35: y(active, since 95s), standbys: x 2023-09-23T18:23:16.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:16 smithi173 bash[27694]: audit 2023-09-23T18:23:15.299981+0000 mon.a (mon.0) 115 : audit [DBG] from='client.? 172.21.15.98:0/2700379740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:23:16.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:16 smithi173 bash[27694]: cluster 2023-09-23T18:23:15.474991+0000 mgr.y (mgr.26054) 69 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T18:23:18.945 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:23:18 smithi098 bash[73085]: debug 2023-09-23T18:23:18.566+0000 7f90a56ad700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-09-23T18:23:19.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:18 smithi173 bash[27694]: cluster 2023-09-23T18:23:17.475689+0000 mgr.y (mgr.26054) 70 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T18:23:19.028 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:23:18 smithi173 bash[54982]: ts=2023-09-23T18:23:18.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi098:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:23:19.939 INFO:teuthology.orchestra.run.smithi098.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T18:23:19.939 INFO:teuthology.orchestra.run.smithi098.stdout:alertmanager.a smithi098 *:9093,9094 running (3m) - 10m 14.2M - 0.23.0 ba2b418f427c 7c2bcaebb703 2023-09-23T18:23:19.939 INFO:teuthology.orchestra.run.smithi098.stdout:grafana.a smithi173 *:3000 running (3m) - 9m 40.9M - 8.3.5 dad864ee21e9 616bbe1127b7 2023-09-23T18:23:19.940 INFO:teuthology.orchestra.run.smithi098.stdout:mds.foo.smithi098.lowjqo smithi098 running (8m) - 8m 15.9M - 16.2.14-37-g2e1f1015 1e6a756174a8 e1555281d4fd 2023-09-23T18:23:19.940 INFO:teuthology.orchestra.run.smithi098.stdout:mds.foo.smithi173.qjefje smithi173 running (8m) - 8m 13.3M - 16.2.14-37-g2e1f1015 1e6a756174a8 fc5513981852 2023-09-23T18:23:19.940 INFO:teuthology.orchestra.run.smithi098.stdout:mgr.x smithi173 *:8443,9283,8765 running (2m) - 16m 426M - 18.2.0-460-gbb0daeff e32c2a64c126 070f62900469 2023-09-23T18:23:19.940 INFO:teuthology.orchestra.run.smithi098.stdout:mgr.y smithi098 *:8443,9283,8765 running (3m) - 19m 472M - 18.2.0-460-gbb0daeff e32c2a64c126 c11dea5b47dd 2023-09-23T18:23:19.940 INFO:teuthology.orchestra.run.smithi098.stdout:mon.a smithi098 running (111s) - 19m 47.2M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 d325a8e8ebe6 2023-09-23T18:23:19.940 INFO:teuthology.orchestra.run.smithi098.stdout:mon.b smithi173 running (17m) - 17m 53.2M 2048M 16.2.14-37-g2e1f1015 1e6a756174a8 0c6482ba189f 2023-09-23T18:23:19.940 INFO:teuthology.orchestra.run.smithi098.stdout:mon.c smithi098 starting - - - 2048M 2023-09-23T18:23:19.940 INFO:teuthology.orchestra.run.smithi098.stdout:node-exporter.a smithi098 *:9100 running (3m) - 10m 8643k - 1.3.1 1dbe0e931976 bdffbddc2c81 2023-09-23T18:23:19.940 INFO:teuthology.orchestra.run.smithi098.stdout:node-exporter.b smithi173 *:9100 running (3m) - 10m 9111k - 1.3.1 1dbe0e931976 610050ff4ef2 2023-09-23T18:23:19.940 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0 smithi098 running (16m) - 16m 45.5M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 5900bdd3fa79 2023-09-23T18:23:19.940 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1 smithi098 running (15m) - 15m 47.9M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 7fd00c3be01c 2023-09-23T18:23:19.941 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2 smithi098 running (14m) - 14m 44.0M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 587769f25c85 2023-09-23T18:23:19.941 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3 smithi098 running (14m) - 14m 42.8M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 af525819942d 2023-09-23T18:23:19.941 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4 smithi173 running (13m) - 13m 43.2M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 0a479b881bee 2023-09-23T18:23:19.941 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5 smithi173 running (12m) - 12m 44.0M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 501903da3aa9 2023-09-23T18:23:19.941 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6 smithi173 running (12m) - 12m 45.6M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 b37149ed4271 2023-09-23T18:23:19.941 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7 smithi173 running (11m) - 11m 47.9M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 81723c51be2b 2023-09-23T18:23:19.941 INFO:teuthology.orchestra.run.smithi098.stdout:prometheus.a smithi173 *:9095 running (2m) - 10m 53.9M - 2.33.4 514e6a882f6e 7547c9472f93 2023-09-23T18:23:20.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:20 smithi173 bash[27694]: cluster 2023-09-23T18:23:19.476240+0000 mgr.y (mgr.26054) 71 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T18:23:20.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:20 smithi173 bash[27694]: audit 2023-09-23T18:23:20.173017+0000 mon.a (mon.0) 116 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:23:20.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:20 smithi173 bash[27694]: audit 2023-09-23T18:23:20.179314+0000 mon.a (mon.0) 117 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:23:20.764 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T18:23:21.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:21 smithi173 bash[27694]: audit 2023-09-23T18:23:19.915529+0000 mgr.y (mgr.26054) 72 : audit [DBG] from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:23:21.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:21 smithi173 bash[27694]: audit 2023-09-23T18:23:20.969345+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:23:21.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:21 smithi173 bash[27694]: audit 2023-09-23T18:23:20.976358+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:23:22.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:22 smithi173 bash[27694]: cluster 2023-09-23T18:23:21.477399+0000 mgr.y (mgr.26054) 73 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T18:23:24.278 INFO:teuthology.orchestra.run.smithi098.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:23:25.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:24 smithi173 bash[27694]: cluster 2023-09-23T18:23:23.478056+0000 mgr.y (mgr.26054) 74 : cluster [DBG] pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:23:25.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:24 smithi173 bash[27694]: audit 2023-09-23T18:23:24.273012+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:23:25.104 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T18:23:26.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:25 smithi173 bash[27694]: audit 2023-09-23T18:23:24.267992+0000 mgr.y (mgr.26054) 75 : audit [DBG] from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:23:26.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:25 smithi173 bash[27694]: cephadm 2023-09-23T18:23:24.268376+0000 mgr.y (mgr.26054) 76 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:23:26.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:25 smithi173 bash[27694]: audit 2023-09-23T18:23:24.560575+0000 mon.a (mon.0) 121 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:23:26.391 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:23:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:23:26] "GET /metrics HTTP/1.1" 200 33906 "" "Prometheus/2.33.4" 2023-09-23T18:23:27.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:26 smithi173 bash[27694]: cluster 2023-09-23T18:23:25.479105+0000 mgr.y (mgr.26054) 77 : cluster [DBG] pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:23:29.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:28 smithi173 bash[27694]: cluster 2023-09-23T18:23:27.479516+0000 mgr.y (mgr.26054) 78 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:23:29.028 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:23:28 smithi173 bash[54982]: ts=2023-09-23T18:23:28.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi098:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:23:30.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:30 smithi173 bash[27694]: cluster 2023-09-23T18:23:29.479949+0000 mgr.y (mgr.26054) 79 : cluster [DBG] pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:23:33.033 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:32 smithi173 bash[27694]: cluster 2023-09-23T18:23:31.480933+0000 mgr.y (mgr.26054) 80 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:23:33.033 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:32 smithi173 bash[27694]: audit 2023-09-23T18:23:32.227002+0000 mon.a (mon.0) 122 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:23:33.033 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:32 smithi173 bash[27694]: audit 2023-09-23T18:23:32.236684+0000 mon.a (mon.0) 123 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:23:33.033 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:32 smithi173 bash[27694]: audit 2023-09-23T18:23:32.238605+0000 mon.a (mon.0) 124 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:23:33.033 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:32 smithi173 bash[27694]: audit 2023-09-23T18:23:32.240135+0000 mon.a (mon.0) 125 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:23:33.033 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:32 smithi173 bash[27694]: audit 2023-09-23T18:23:32.249311+0000 mon.a (mon.0) 126 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:23:33.033 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:32 smithi173 bash[27694]: audit 2023-09-23T18:23:32.331608+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:23:33.033 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:32 smithi173 bash[27694]: audit 2023-09-23T18:23:32.334630+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:23:33.034 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:32 smithi173 bash[27694]: audit 2023-09-23T18:23:32.336045+0000 mon.a (mon.0) 129 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:23:33.034 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:32 smithi173 bash[27694]: audit 2023-09-23T18:23:32.345504+0000 mon.a (mon.0) 130 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:23:35.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:34 smithi173 bash[27694]: cluster 2023-09-23T18:23:33.481526+0000 mgr.y (mgr.26054) 81 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:23:36.445 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:23:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:23:36] "GET /metrics HTTP/1.1" 200 33906 "" "Prometheus/2.33.4" 2023-09-23T18:23:37.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:36 smithi173 bash[27694]: cluster 2023-09-23T18:23:35.482520+0000 mgr.y (mgr.26054) 82 : cluster [DBG] pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:23:39.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:38 smithi173 bash[27694]: cluster 2023-09-23T18:23:37.483127+0000 mgr.y (mgr.26054) 83 : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:23:39.028 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:23:38 smithi173 bash[54982]: ts=2023-09-23T18:23:38.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi098:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:23:40.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:40 smithi173 bash[27694]: cluster 2023-09-23T18:23:39.483760+0000 mgr.y (mgr.26054) 84 : cluster [DBG] pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:23:40.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:40 smithi173 bash[27694]: audit 2023-09-23T18:23:39.566806+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:23:43.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:42 smithi173 bash[27694]: cluster 2023-09-23T18:23:41.484838+0000 mgr.y (mgr.26054) 85 : cluster [DBG] pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:23:45.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:44 smithi173 bash[27694]: cluster 2023-09-23T18:23:43.485392+0000 mgr.y (mgr.26054) 86 : cluster [DBG] pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:23:46.445 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:23:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:23:46] "GET /metrics HTTP/1.1" 200 33905 "" "Prometheus/2.33.4" 2023-09-23T18:23:47.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:46 smithi173 bash[27694]: cluster 2023-09-23T18:23:45.486368+0000 mgr.y (mgr.26054) 87 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:23:49.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:48 smithi173 bash[27694]: cluster 2023-09-23T18:23:47.487094+0000 mgr.y (mgr.26054) 88 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:23:49.028 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:23:48 smithi173 bash[54982]: ts=2023-09-23T18:23:48.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi098:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:23:50.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:50 smithi173 bash[27694]: cluster 2023-09-23T18:23:49.487773+0000 mgr.y (mgr.26054) 89 : cluster [DBG] pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:23:53.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:52 smithi173 bash[27694]: cluster 2023-09-23T18:23:51.488901+0000 mgr.y (mgr.26054) 90 : cluster [DBG] pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:23:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:54 smithi173 bash[27694]: cluster 2023-09-23T18:23:53.489667+0000 mgr.y (mgr.26054) 91 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:23:56.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:55 smithi173 bash[27694]: audit 2023-09-23T18:23:54.567165+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:23:56.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:23:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:23:56] "GET /metrics HTTP/1.1" 200 33916 "" "Prometheus/2.33.4" 2023-09-23T18:23:57.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:56 smithi173 bash[27694]: cluster 2023-09-23T18:23:55.490753+0000 mgr.y (mgr.26054) 92 : cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:23:59.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:23:58 smithi173 bash[27694]: cluster 2023-09-23T18:23:57.491392+0000 mgr.y (mgr.26054) 93 : cluster [DBG] pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:23:59.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:23:58 smithi173 bash[54982]: ts=2023-09-23T18:23:58.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi098:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:24:00.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:00 smithi173 bash[27694]: cluster 2023-09-23T18:23:59.491955+0000 mgr.y (mgr.26054) 94 : cluster [DBG] pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:03.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:02 smithi173 bash[27694]: cluster 2023-09-23T18:24:01.492983+0000 mgr.y (mgr.26054) 95 : cluster [DBG] pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:05.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:04 smithi173 bash[27694]: cluster 2023-09-23T18:24:03.493658+0000 mgr.y (mgr.26054) 96 : cluster [DBG] pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:06.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:24:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:24:06] "GET /metrics HTTP/1.1" 200 33916 "" "Prometheus/2.33.4" 2023-09-23T18:24:07.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:06 smithi173 bash[27694]: cluster 2023-09-23T18:24:05.494700+0000 mgr.y (mgr.26054) 97 : cluster [DBG] pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:09.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:08 smithi173 bash[27694]: cluster 2023-09-23T18:24:07.495328+0000 mgr.y (mgr.26054) 98 : cluster [DBG] pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:09.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:24:08 smithi173 bash[54982]: ts=2023-09-23T18:24:08.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi098:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:24:10.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:10 smithi173 bash[27694]: cluster 2023-09-23T18:24:09.495998+0000 mgr.y (mgr.26054) 99 : cluster [DBG] pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:10.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:10 smithi173 bash[27694]: audit 2023-09-23T18:24:09.567498+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:13.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:12 smithi173 bash[27694]: cluster 2023-09-23T18:24:11.497024+0000 mgr.y (mgr.26054) 100 : cluster [DBG] pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:15.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:14 smithi173 bash[27694]: cluster 2023-09-23T18:24:13.497764+0000 mgr.y (mgr.26054) 101 : cluster [DBG] pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:16.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:24:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:24:16] "GET /metrics HTTP/1.1" 200 33916 "" "Prometheus/2.33.4" 2023-09-23T18:24:17.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:16 smithi173 bash[27694]: cluster 2023-09-23T18:24:15.498920+0000 mgr.y (mgr.26054) 102 : cluster [DBG] pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:19.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:18 smithi173 bash[27694]: cluster 2023-09-23T18:24:17.499626+0000 mgr.y (mgr.26054) 103 : cluster [DBG] pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:19.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:24:18 smithi173 bash[54982]: ts=2023-09-23T18:24:18.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi098:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:24:20.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:20 smithi173 bash[27694]: cluster 2023-09-23T18:24:19.500320+0000 mgr.y (mgr.26054) 104 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:23.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:22 smithi173 bash[27694]: cluster 2023-09-23T18:24:21.501388+0000 mgr.y (mgr.26054) 105 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:25.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:24 smithi173 bash[27694]: cluster 2023-09-23T18:24:23.501967+0000 mgr.y (mgr.26054) 106 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:26.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:25 smithi173 bash[27694]: audit 2023-09-23T18:24:24.568031+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:26.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:24:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:24:26] "GET /metrics HTTP/1.1" 200 33916 "" "Prometheus/2.33.4" 2023-09-23T18:24:27.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:26 smithi173 bash[27694]: cluster 2023-09-23T18:24:25.503004+0000 mgr.y (mgr.26054) 107 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:29.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:28 smithi173 bash[27694]: cluster 2023-09-23T18:24:27.503606+0000 mgr.y (mgr.26054) 108 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:29.027 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 18:24:28 smithi173 bash[54982]: ts=2023-09-23T18:24:28.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.98\", device_class=\"ssd\", hostname=\"smithi098\", instance=\"smithi098:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.98\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T18:24:30.280 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T18:24:30.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:30 smithi173 bash[27694]: cluster 2023-09-23T18:24:29.504045+0000 mgr.y (mgr.26054) 109 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:32 smithi173 bash[27694]: cluster 2023-09-23T18:24:31.505077+0000 mgr.y (mgr.26054) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:32 smithi173 bash[27694]: audit 2023-09-23T18:24:32.427368+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:24:33.759 INFO:teuthology.orchestra.run.smithi098.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:24:34.530 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for all mons to upgrade' 2023-09-23T18:24:35.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:34 smithi173 bash[27694]: cluster 2023-09-23T18:24:33.505540+0000 mgr.y (mgr.26054) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:35.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:34 smithi173 bash[27694]: audit 2023-09-23T18:24:33.746712+0000 mgr.y (mgr.26054) 112 : audit [DBG] from='client.34209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:24:35.030 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:34 smithi173 bash[27694]: cephadm 2023-09-23T18:24:33.747342+0000 mgr.y (mgr.26054) 113 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:24:35.030 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:34 smithi173 bash[27694]: audit 2023-09-23T18:24:33.753708+0000 mon.a (mon.0) 136 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:36.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:24:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:24:36] "GET /metrics HTTP/1.1" 200 33916 "" "Prometheus/2.33.4" 2023-09-23T18:24:37.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:36 smithi173 bash[27694]: cluster 2023-09-23T18:24:35.506563+0000 mgr.y (mgr.26054) 114 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:37.709 INFO:teuthology.orchestra.run.smithi098.stdout:wait for all mons to upgrade 2023-09-23T18:24:38.509 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-09-23T18:24:39.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:38 smithi173 bash[27694]: cluster 2023-09-23T18:24:37.506932+0000 mgr.y (mgr.26054) 115 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:39.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:38 smithi173 bash[27694]: audit 2023-09-23T18:24:38.268428+0000 mon.a (mon.0) 137 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:39.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:38 smithi173 bash[27694]: audit 2023-09-23T18:24:38.276457+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:40.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:39 smithi173 bash[27694]: audit 2023-09-23T18:24:38.773548+0000 mon.a (mon.0) 139 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:40.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:39 smithi173 bash[27694]: audit 2023-09-23T18:24:38.796961+0000 mon.a (mon.0) 140 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:40.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:39 smithi173 bash[27694]: audit 2023-09-23T18:24:39.270808+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:24:40.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:39 smithi173 bash[27694]: audit 2023-09-23T18:24:39.273180+0000 mon.a (mon.0) 142 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:24:40.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:39 smithi173 bash[27694]: audit 2023-09-23T18:24:39.283626+0000 mon.a (mon.0) 143 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:40.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:39 smithi173 bash[27694]: audit 2023-09-23T18:24:39.366510+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:24:40.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:39 smithi173 bash[27694]: audit 2023-09-23T18:24:39.370803+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:24:40.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:39 smithi173 bash[27694]: audit 2023-09-23T18:24:39.372804+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T18:24:40.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:39 smithi173 bash[27694]: audit 2023-09-23T18:24:39.374149+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T18:24:40.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:39 smithi173 bash[27694]: audit 2023-09-23T18:24:39.568290+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:40.982 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:40 smithi173 bash[27694]: cephadm 2023-09-23T18:24:39.374993+0000 mgr.y (mgr.26054) 116 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2023-09-23T18:24:40.982 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:40 smithi173 bash[27694]: cluster 2023-09-23T18:24:39.507390+0000 mgr.y (mgr.26054) 117 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:42.984 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:42 smithi173 bash[27694]: cluster 2023-09-23T18:24:41.508443+0000 mgr.y (mgr.26054) 118 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:42.984 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:42 smithi173 bash[27694]: cephadm 2023-09-23T18:24:41.696111+0000 mgr.y (mgr.26054) 119 : cephadm [INF] Upgrade: Updating mon.b 2023-09-23T18:24:42.985 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:42 smithi173 bash[27694]: audit 2023-09-23T18:24:41.703858+0000 mon.a (mon.0) 149 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:42.985 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:42 smithi173 bash[27694]: audit 2023-09-23T18:24:41.705897+0000 mon.a (mon.0) 150 : audit [INF] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:24:42.985 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:42 smithi173 bash[27694]: audit 2023-09-23T18:24:41.707041+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:24:42.985 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:42 smithi173 bash[27694]: audit 2023-09-23T18:24:41.714652+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:24:42.985 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:42 smithi173 bash[27694]: cephadm 2023-09-23T18:24:41.716007+0000 mgr.y (mgr.26054) 120 : cephadm [INF] Deploying daemon mon.b on smithi173 2023-09-23T18:24:42.985 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:42 smithi173 bash[27694]: audit 2023-09-23T18:24:41.975073+0000 mon.a (mon.0) 153 : audit [DBG] from='client.? 172.21.15.98:0/2074759034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:24:44.007 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:43 smithi173 systemd[1]: Stopping Ceph mon.b for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T18:24:44.269 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:44 smithi173 bash[27694]: debug 2023-09-23T18:24:44.004+0000 7f991ec02700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-09-23T18:24:44.269 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:44 smithi173 bash[27694]: debug 2023-09-23T18:24:44.004+0000 7f991ec02700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-09-23T18:24:45.242 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:44 smithi173 bash[59301]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-mon-b 2023-09-23T18:24:45.242 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:44 smithi173 bash[59365]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-mon-b 2023-09-23T18:24:45.243 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:45 smithi173 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mon.b.service: Succeeded. 2023-09-23T18:24:45.243 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:45 smithi173 systemd[1]: Stopped Ceph mon.b for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:24:45.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:45 smithi173 systemd[1]: Started Ceph mon.b for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:24:46.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:24:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:24:46] "GET /metrics HTTP/1.1" 200 33914 "" "Prometheus/2.33.4" 2023-09-23T18:24:46.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.324+0000 7f077b5e7c80 0 set uid:gid to 167:167 (ceph:ceph) 2023-09-23T18:24:46.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.324+0000 7f077b5e7c80 0 ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T18:24:46.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.324+0000 7f077b5e7c80 0 pidfile_write: ignore empty --pid-file 2023-09-23T18:24:46.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.328+0000 7f077b5e7c80 0 load: jerasure load: lrc 2023-09-23T18:24:46.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: RocksDB version: 7.9.2 2023-09-23T18:24:46.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Git sha 0 2023-09-23T18:24:46.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T18:24:46.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: DB SUMMARY 2023-09-23T18:24:46.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: DB Session ID: T37H7DF7F5IMKAI3KN8P 2023-09-23T18:24:46.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: CURRENT file: CURRENT 2023-09-23T18:24:46.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: IDENTITY file: IDENTITY 2023-09-23T18:24:46.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 1853 Bytes 2023-09-23T18:24:46.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000035.sst 2023-09-23T18:24:46.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000033.log size: 129663 ; 2023-09-23T18:24:46.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.error_if_exists: 0 2023-09-23T18:24:46.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.create_if_missing: 0 2023-09-23T18:24:46.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.paranoid_checks: 1 2023-09-23T18:24:46.530 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T18:24:46.530 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T18:24:46.530 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T18:24:46.530 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.env: 0x55efeec7c720 2023-09-23T18:24:46.530 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.fs: PosixFileSystem 2023-09-23T18:24:46.530 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.info_log: 0x55eff211b480 2023-09-23T18:24:46.530 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_file_opening_threads: 16 2023-09-23T18:24:46.530 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.statistics: (nil) 2023-09-23T18:24:46.530 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.use_fsync: 0 2023-09-23T18:24:46.531 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_log_file_size: 0 2023-09-23T18:24:46.532 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T18:24:46.532 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T18:24:46.532 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.keep_log_file_num: 1000 2023-09-23T18:24:46.532 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.recycle_log_file_num: 0 2023-09-23T18:24:46.532 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.allow_fallocate: 1 2023-09-23T18:24:46.532 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.allow_mmap_reads: 0 2023-09-23T18:24:46.532 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.allow_mmap_writes: 0 2023-09-23T18:24:46.532 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.use_direct_reads: 0 2023-09-23T18:24:46.532 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T18:24:46.533 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.create_missing_column_families: 0 2023-09-23T18:24:46.533 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.db_log_dir: 2023-09-23T18:24:46.533 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.wal_dir: 2023-09-23T18:24:46.533 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T18:24:46.533 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T18:24:46.533 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T18:24:46.533 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T18:24:46.533 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T18:24:46.533 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T18:24:46.533 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.advise_random_on_open: 1 2023-09-23T18:24:46.533 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.db_write_buffer_size: 0 2023-09-23T18:24:46.533 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.write_buffer_manager: 0x55eff13a8c80 2023-09-23T18:24:46.534 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T18:24:46.534 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T18:24:46.534 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T18:24:46.534 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.rate_limiter: (nil) 2023-09-23T18:24:46.534 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T18:24:46.534 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.wal_recovery_mode: 2 2023-09-23T18:24:46.534 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.enable_thread_tracking: 0 2023-09-23T18:24:46.534 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.enable_pipelined_write: 0 2023-09-23T18:24:46.534 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.unordered_write: 0 2023-09-23T18:24:46.534 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T18:24:46.534 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T18:24:46.535 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T18:24:46.535 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T18:24:46.535 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.row_cache: None 2023-09-23T18:24:46.535 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.wal_filter: None 2023-09-23T18:24:46.535 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T18:24:46.535 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.allow_ingest_behind: 0 2023-09-23T18:24:46.535 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.two_write_queues: 0 2023-09-23T18:24:46.535 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.manual_wal_flush: 0 2023-09-23T18:24:46.535 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.wal_compression: 0 2023-09-23T18:24:46.535 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.atomic_flush: 0 2023-09-23T18:24:46.535 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T18:24:46.536 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T18:24:46.536 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T18:24:46.536 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.log_readahead_size: 0 2023-09-23T18:24:46.536 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T18:24:46.536 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.best_efforts_recovery: 0 2023-09-23T18:24:46.536 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T18:24:46.536 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T18:24:46.536 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.allow_data_in_errors: 0 2023-09-23T18:24:46.536 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.db_host_id: __hostname__ 2023-09-23T18:24:46.536 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T18:24:46.536 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_background_jobs: 2 2023-09-23T18:24:46.536 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_background_compactions: -1 2023-09-23T18:24:46.537 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_subcompactions: 1 2023-09-23T18:24:46.537 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T18:24:46.537 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T18:24:46.537 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T18:24:46.537 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_total_wal_size: 0 2023-09-23T18:24:46.537 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T18:24:46.537 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T18:24:46.537 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T18:24:46.537 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T18:24:46.537 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_open_files: -1 2023-09-23T18:24:46.537 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.bytes_per_sync: 0 2023-09-23T18:24:46.537 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T18:24:46.538 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T18:24:46.538 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.compaction_readahead_size: 0 2023-09-23T18:24:46.538 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_background_flushes: -1 2023-09-23T18:24:46.538 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Compression algorithms supported: 2023-09-23T18:24:46.538 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T18:24:46.538 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: kZSTD supported: 0 2023-09-23T18:24:46.538 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: kXpressCompression supported: 0 2023-09-23T18:24:46.538 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: kLZ4HCCompression supported: 1 2023-09-23T18:24:46.538 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: kZlibCompression supported: 1 2023-09-23T18:24:46.538 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: kSnappyCompression supported: 1 2023-09-23T18:24:46.538 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: kLZ4Compression supported: 1 2023-09-23T18:24:46.538 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: kBZip2Compression supported: 0 2023-09-23T18:24:46.539 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T18:24:46.539 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T18:24:46.539 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2023-09-23T18:24:46.539 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T18:24:46.539 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T18:24:46.539 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.merge_operator: 2023-09-23T18:24:46.539 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.compaction_filter: None 2023-09-23T18:24:46.539 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.compaction_filter_factory: None 2023-09-23T18:24:46.539 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.sst_partitioner_factory: None 2023-09-23T18:24:46.539 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T18:24:46.539 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T18:24:46.539 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55eff211b580) 2023-09-23T18:24:46.539 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: cache_index_and_filter_blocks: 1 2023-09-23T18:24:46.540 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T18:24:46.540 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T18:24:46.540 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: pin_top_level_index_and_filter: 1 2023-09-23T18:24:46.540 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: index_type: 0 2023-09-23T18:24:46.540 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: data_block_index_type: 0 2023-09-23T18:24:46.540 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: index_shortening: 1 2023-09-23T18:24:46.540 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: data_block_hash_table_util_ratio: 0.750000 2023-09-23T18:24:46.540 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: checksum: 4 2023-09-23T18:24:46.540 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: no_block_cache: 0 2023-09-23T18:24:46.540 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: block_cache: 0x55eff142b1f0 2023-09-23T18:24:46.540 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: block_cache_name: BinnedLRUCache 2023-09-23T18:24:46.540 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: block_cache_options: 2023-09-23T18:24:46.541 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: capacity : 536870912 2023-09-23T18:24:46.541 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: num_shard_bits : 4 2023-09-23T18:24:46.541 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: strict_capacity_limit : 0 2023-09-23T18:24:46.541 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: high_pri_pool_ratio: 0.000 2023-09-23T18:24:46.541 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: block_cache_compressed: (nil) 2023-09-23T18:24:46.541 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: persistent_cache: (nil) 2023-09-23T18:24:46.541 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: block_size: 4096 2023-09-23T18:24:46.541 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: block_size_deviation: 10 2023-09-23T18:24:46.541 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: block_restart_interval: 16 2023-09-23T18:24:46.541 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: index_block_restart_interval: 1 2023-09-23T18:24:46.541 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: metadata_block_size: 4096 2023-09-23T18:24:46.541 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: partition_filters: 0 2023-09-23T18:24:46.542 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: use_delta_encoding: 1 2023-09-23T18:24:46.542 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: filter_policy: bloomfilter 2023-09-23T18:24:46.542 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: whole_key_filtering: 1 2023-09-23T18:24:46.542 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: verify_compression: 0 2023-09-23T18:24:46.542 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: read_amp_bytes_per_bit: 0 2023-09-23T18:24:46.542 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: format_version: 5 2023-09-23T18:24:46.542 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: enable_index_compression: 1 2023-09-23T18:24:46.542 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: block_align: 0 2023-09-23T18:24:46.542 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: max_auto_readahead_size: 262144 2023-09-23T18:24:46.542 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: prepopulate_block_cache: 0 2023-09-23T18:24:46.542 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: initial_auto_readahead_size: 8192 2023-09-23T18:24:46.542 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: num_file_reads_for_auto_readahead: 2 2023-09-23T18:24:46.543 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.write_buffer_size: 33554432 2023-09-23T18:24:46.543 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_write_buffer_number: 2 2023-09-23T18:24:46.543 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.compression: NoCompression 2023-09-23T18:24:46.543 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.bottommost_compression: Disabled 2023-09-23T18:24:46.543 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.prefix_extractor: nullptr 2023-09-23T18:24:46.543 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T18:24:46.543 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.num_levels: 7 2023-09-23T18:24:46.543 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T18:24:46.543 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T18:24:46.544 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T18:24:46.544 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T18:24:46.544 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T18:24:46.544 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T18:24:46.544 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T18:24:46.544 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T18:24:46.544 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T18:24:46.544 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T18:24:46.544 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T18:24:46.544 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T18:24:46.544 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T18:24:46.544 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.compression_opts.level: 32767 2023-09-23T18:24:46.545 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.compression_opts.strategy: 0 2023-09-23T18:24:46.545 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T18:24:46.545 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T18:24:46.545 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T18:24:46.545 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T18:24:46.545 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.compression_opts.enabled: false 2023-09-23T18:24:46.545 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T18:24:46.545 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T18:24:46.545 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T18:24:46.545 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T18:24:46.545 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.target_file_size_base: 67108864 2023-09-23T18:24:46.546 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T18:24:46.546 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T18:24:46.546 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T18:24:46.546 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T18:24:46.546 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T18:24:46.546 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T18:24:46.546 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T18:24:46.546 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T18:24:46.546 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T18:24:46.546 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T18:24:46.547 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T18:24:46.547 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T18:24:46.547 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T18:24:46.547 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T18:24:46.547 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.arena_block_size: 1048576 2023-09-23T18:24:46.547 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T18:24:46.547 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T18:24:46.547 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.disable_auto_compactions: 0 2023-09-23T18:24:46.547 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T18:24:46.547 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T18:24:46.547 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T18:24:46.548 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T18:24:46.548 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T18:24:46.549 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T18:24:46.549 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T18:24:46.549 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T18:24:46.549 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T18:24:46.549 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T18:24:46.549 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.table_properties_collectors: 2023-09-23T18:24:46.550 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.inplace_update_support: 0 2023-09-23T18:24:46.550 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T18:24:46.550 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T18:24:46.550 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T18:24:46.550 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T18:24:46.550 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.bloom_locality: 0 2023-09-23T18:24:46.550 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.max_successive_merges: 0 2023-09-23T18:24:46.551 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T18:24:46.551 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.paranoid_file_checks: 0 2023-09-23T18:24:46.551 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.force_consistency_checks: 1 2023-09-23T18:24:46.551 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.report_bg_io_stats: 0 2023-09-23T18:24:46.551 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.ttl: 2592000 2023-09-23T18:24:46.551 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T18:24:46.551 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T18:24:46.551 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T18:24:46.551 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.enable_blob_files: false 2023-09-23T18:24:46.552 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.min_blob_size: 0 2023-09-23T18:24:46.552 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.blob_file_size: 268435456 2023-09-23T18:24:46.552 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T18:24:46.552 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T18:24:46.552 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T18:24:46.552 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T18:24:46.552 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T18:24:46.552 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.blob_file_starting_level: 0 2023-09-23T18:24:46.552 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T18:24:46.553 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f076fe04700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 35.sst 2023-09-23T18:24:46.553 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 37, last_sequence is 20460, log_number is 33,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T18:24:46.553 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 33 2023-09-23T18:24:46.553 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 9823c3c2-aba1-428d-90a8-f069be168166 2023-09-23T18:24:46.553 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695493486336238, "job": 1, "event": "recovery_started", "wal_files": [33]} 2023-09-23T18:24:46.553 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.332+0000 7f077b5e7c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #33 mode 2 2023-09-23T18:24:46.553 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.336+0000 7f077b5e7c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695493486338064, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 38, "file_size": 87510, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 20465, "largest_seqno": 20485, "table_properties": {"data_size": 86279, "index_size": 157, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 307, "raw_average_key_size": 21, "raw_value_size": 85860, "raw_average_value_size": 6132, "num_data_blocks": 7, "num_entries": 14, "num_filter_entries": 14, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695493486, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "9823c3c2-aba1-428d-90a8-f069be168166", "db_session_id": "T37H7DF7F5IMKAI3KN8P", "orig_file_number": 38, "seqno_to_time_mapping": "N/A"}} 2023-09-23T18:24:46.553 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.336+0000 7f077b5e7c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695493486338224, "job": 1, "event": "recovery_finished"} 2023-09-23T18:24:46.554 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.336+0000 7f077b5e7c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 40 2023-09-23T18:24:46.554 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.336+0000 7f077b5e7c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000033.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T18:24:46.554 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.336+0000 7f077b5e7c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55eff14cc000 2023-09-23T18:24:46.554 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.336+0000 7f077b5e7c80 4 rocksdb: DB pointer 0x55eff14ba000 2023-09-23T18:24:46.554 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.336+0000 7f0769df8700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T18:24:46.554 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.336+0000 7f0769df8700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T18:24:46.554 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: ** DB Stats ** 2023-09-23T18:24:46.555 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T18:24:46.555 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T18:24:46.555 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T18:24:46.555 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T18:24:46.555 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T18:24:46.555 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T18:24:46.555 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T18:24:46.555 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: ** Compaction Stats [default] ** 2023-09-23T18:24:46.555 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T18:24:46.556 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T18:24:46.556 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: L0 1/0 85.46 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 84.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-09-23T18:24:46.556 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: L6 1/0 26.55 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T18:24:46.556 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: Sum 2/0 26.63 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 84.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-09-23T18:24:46.556 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 84.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-09-23T18:24:46.556 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: ** Compaction Stats [default] ** 2023-09-23T18:24:46.556 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T18:24:46.556 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T18:24:46.557 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 84.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-09-23T18:24:46.557 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T18:24:46.557 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T18:24:46.557 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T18:24:46.557 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T18:24:46.557 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T18:24:46.557 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T18:24:46.557 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: AddFile(Keys): cumulative 0, interval 0 2023-09-23T18:24:46.557 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: Cumulative compaction: 0.00 GB write, 13.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T18:24:46.558 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: Interval compaction: 0.00 GB write, 13.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T18:24:46.558 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T18:24:46.558 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: Block cache BinnedLRUCache@0x55eff142b1f0#7 capacity: 512.00 MB usage: 0.34 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.6e-05 secs_since: 0 2023-09-23T18:24:46.558 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.23 KB,4.47035e-05%) Misc(1,0.00 KB,0%) 2023-09-23T18:24:46.558 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: ** File Read Latency Histogram By Level [default] ** 2023-09-23T18:24:46.558 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.340+0000 7f077b5e7c80 0 starting mon.b rank 1 at public addrs [v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0] at bind addrs [v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:24:46.558 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.340+0000 7f077b5e7c80 1 mon.b@-1(???) e3 preinit fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:24:46.558 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.340+0000 7f077b5e7c80 0 mon.b@-1(???).mds e7 new map 2023-09-23T18:24:46.559 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.340+0000 7f077b5e7c80 0 mon.b@-1(???).mds e7 print_map 2023-09-23T18:24:46.559 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: e7 2023-09-23T18:24:46.559 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T18:24:46.559 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T18:24:46.559 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: legacy client fscid: 1 2023-09-23T18:24:46.559 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: 2023-09-23T18:24:46.559 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: Filesystem 'foo' (1) 2023-09-23T18:24:46.559 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: fs_name foo 2023-09-23T18:24:46.559 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: epoch 5 2023-09-23T18:24:46.560 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T18:24:46.560 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: created 2023-09-23T18:15:08.890010+0000 2023-09-23T18:24:46.560 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: modified 2023-09-23T18:15:16.817511+0000 2023-09-23T18:24:46.560 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: tableserver 0 2023-09-23T18:24:46.560 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: root 0 2023-09-23T18:24:46.560 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: session_timeout 60 2023-09-23T18:24:46.560 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: session_autoclose 300 2023-09-23T18:24:46.560 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: max_file_size 1099511627776 2023-09-23T18:24:46.561 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: required_client_features {} 2023-09-23T18:24:46.561 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: last_failure 0 2023-09-23T18:24:46.561 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: last_failure_osd_epoch 0 2023-09-23T18:24:46.561 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T18:24:46.561 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: max_mds 1 2023-09-23T18:24:46.561 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: in 0 2023-09-23T18:24:46.561 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: up {0=24545} 2023-09-23T18:24:46.561 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: failed 2023-09-23T18:24:46.561 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: damaged 2023-09-23T18:24:46.561 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: stopped 2023-09-23T18:24:46.561 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: data_pools [4] 2023-09-23T18:24:46.562 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: metadata_pool 3 2023-09-23T18:24:46.562 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: inline_data disabled 2023-09-23T18:24:46.562 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: balancer 2023-09-23T18:24:46.562 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: bal_rank_mask -1 2023-09-23T18:24:46.562 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: standby_count_wanted 1 2023-09-23T18:24:46.562 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: [mds.foo.smithi098.lowjqo{0:24545} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.98:6834/2708276357,v1:172.21.15.98:6835/2708276357] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T18:24:46.562 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: 2023-09-23T18:24:46.562 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: 2023-09-23T18:24:46.562 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: Standby daemons: 2023-09-23T18:24:46.562 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: 2023-09-23T18:24:46.562 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: [mds.foo.smithi173.qjefje{-1:24502} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.173:6832/621117609,v1:172.21.15.173:6833/621117609] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T18:24:46.563 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.340+0000 7f077b5e7c80 0 mon.b@-1(???).osd e84 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T18:24:46.563 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.340+0000 7f077b5e7c80 0 mon.b@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T18:24:46.563 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.340+0000 7f077b5e7c80 0 mon.b@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T18:24:46.563 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.340+0000 7f077b5e7c80 0 mon.b@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T18:24:46.563 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:46 smithi173 bash[59422]: debug 2023-09-23T18:24:46.340+0000 7f077b5e7c80 1 mon.b@-1(???).paxosservice(auth 1..23) refresh upgraded, format 0 -> 3 2023-09-23T18:24:48.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:04:11.580933+0000 mon.a (mon.0) 0 : 1 mkfs 756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:24:48.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:19:44.249913+0000 mgr.x (mgr.25562) 47 : cephadm 1 Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T18:24:48.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:19:44.250670+0000 mgr.x (mgr.25562) 48 : cephadm 1 Reconfiguring daemon node-exporter.b on smithi173 2023-09-23T18:24:48.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:19:46.119762+0000 mgr.x (mgr.25562) 50 : cephadm 1 Reconfiguring grafana.a (dependencies changed)... 2023-09-23T18:24:48.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:19:46.140131+0000 mgr.x (mgr.25562) 51 : cephadm 1 Reconfiguring daemon grafana.a on smithi173 2023-09-23T18:24:48.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:20:11.502535+0000 mgr.x (mgr.25562) 68 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T18:24:48.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:20:11.503172+0000 mgr.x (mgr.25562) 69 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T18:24:48.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:20:13.823190+0000 mgr.x (mgr.25562) 71 : cephadm 1 Upgrade: Updating mgr.y 2023-09-23T18:24:48.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:20:13.841125+0000 mgr.x (mgr.25562) 72 : cephadm 1 Deploying daemon mgr.y on smithi098 2023-09-23T18:24:48.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:20:36.747789+0000 mgr.y (mgr.25823) 3 : cephadm 1 [23/Sep/2023:18:20:36] ENGINE Bus STARTING 2023-09-23T18:24:48.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:20:36.869010+0000 mgr.y (mgr.25823) 4 : cephadm 1 [23/Sep/2023:18:20:36] ENGINE Serving on https://172.21.15.98:7150 2023-09-23T18:24:48.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:20:36.971240+0000 mgr.y (mgr.25823) 5 : cephadm 1 [23/Sep/2023:18:20:36] ENGINE Serving on http://172.21.15.98:8765 2023-09-23T18:24:48.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:20:36.971641+0000 mgr.y (mgr.25823) 6 : cephadm 1 [23/Sep/2023:18:20:36] ENGINE Bus STARTED 2023-09-23T18:24:48.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:20:51.692244+0000 mgr.y (mgr.25823) 14 : cephadm 1 Updating smithi098:/etc/ceph/ceph.conf 2023-09-23T18:24:48.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:20:51.692594+0000 mgr.y (mgr.25823) 15 : cephadm 1 Updating smithi173:/etc/ceph/ceph.conf 2023-09-23T18:24:48.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:20:51.857117+0000 mgr.y (mgr.25823) 16 : cephadm 1 Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:24:48.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:20:51.857679+0000 mgr.y (mgr.25823) 17 : cephadm 1 Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:24:48.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:20:52.022034+0000 mgr.y (mgr.25823) 18 : cephadm 1 Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:24:48.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:20:52.022405+0000 mgr.y (mgr.25823) 19 : cephadm 1 Updating smithi098:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:24:48.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:20:52.237062+0000 mgr.y (mgr.25823) 20 : cephadm 1 Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:24:48.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:20:52.238837+0000 mgr.y (mgr.25823) 21 : cephadm 1 Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:24:48.696 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:20:52.554522+0000 mgr.y (mgr.25823) 23 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T18:24:48.697 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:20:52.800973+0000 mgr.y (mgr.25823) 24 : cephadm 1 Reconfiguring daemon prometheus.a on smithi173 2023-09-23T18:24:48.697 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:21:00.978897+0000 mgr.y (mgr.25823) 30 : cephadm 1 Upgrade: Updating mgr.x 2023-09-23T18:24:48.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:04:11.580933+0000 mon.a (mon.0) 0 : 1 mkfs 756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:24:48.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:19:44.249913+0000 mgr.x (mgr.25562) 47 : cephadm 1 Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T18:24:48.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:19:44.250670+0000 mgr.x (mgr.25562) 48 : cephadm 1 Reconfiguring daemon node-exporter.b on smithi173 2023-09-23T18:24:48.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:19:46.119762+0000 mgr.x (mgr.25562) 50 : cephadm 1 Reconfiguring grafana.a (dependencies changed)... 2023-09-23T18:24:48.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:19:46.140131+0000 mgr.x (mgr.25562) 51 : cephadm 1 Reconfiguring daemon grafana.a on smithi173 2023-09-23T18:24:48.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:20:11.502535+0000 mgr.x (mgr.25562) 68 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T18:24:48.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:20:11.503172+0000 mgr.x (mgr.25562) 69 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T18:24:48.698 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:20:13.823190+0000 mgr.x (mgr.25562) 71 : cephadm 1 Upgrade: Updating mgr.y 2023-09-23T18:24:48.698 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:20:13.841125+0000 mgr.x (mgr.25562) 72 : cephadm 1 Deploying daemon mgr.y on smithi098 2023-09-23T18:24:48.698 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:20:36.747789+0000 mgr.y (mgr.25823) 3 : cephadm 1 [23/Sep/2023:18:20:36] ENGINE Bus STARTING 2023-09-23T18:24:48.698 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:20:36.869010+0000 mgr.y (mgr.25823) 4 : cephadm 1 [23/Sep/2023:18:20:36] ENGINE Serving on https://172.21.15.98:7150 2023-09-23T18:24:48.698 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:20:36.971240+0000 mgr.y (mgr.25823) 5 : cephadm 1 [23/Sep/2023:18:20:36] ENGINE Serving on http://172.21.15.98:8765 2023-09-23T18:24:48.698 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:20:36.971641+0000 mgr.y (mgr.25823) 6 : cephadm 1 [23/Sep/2023:18:20:36] ENGINE Bus STARTED 2023-09-23T18:24:48.698 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:20:51.692244+0000 mgr.y (mgr.25823) 14 : cephadm 1 Updating smithi098:/etc/ceph/ceph.conf 2023-09-23T18:24:48.698 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:20:51.692594+0000 mgr.y (mgr.25823) 15 : cephadm 1 Updating smithi173:/etc/ceph/ceph.conf 2023-09-23T18:24:48.698 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:20:51.857117+0000 mgr.y (mgr.25823) 16 : cephadm 1 Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:24:48.698 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:20:51.857679+0000 mgr.y (mgr.25823) 17 : cephadm 1 Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:24:48.698 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:20:52.022034+0000 mgr.y (mgr.25823) 18 : cephadm 1 Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:24:48.698 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:20:52.022405+0000 mgr.y (mgr.25823) 19 : cephadm 1 Updating smithi098:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:24:48.698 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:20:52.237062+0000 mgr.y (mgr.25823) 20 : cephadm 1 Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:24:48.699 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:20:52.238837+0000 mgr.y (mgr.25823) 21 : cephadm 1 Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:24:48.699 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:20:52.554522+0000 mgr.y (mgr.25823) 23 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T18:24:48.699 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:20:52.800973+0000 mgr.y (mgr.25823) 24 : cephadm 1 Reconfiguring daemon prometheus.a on smithi173 2023-09-23T18:24:48.699 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:21:00.978897+0000 mgr.y (mgr.25823) 30 : cephadm 1 Upgrade: Updating mgr.x 2023-09-23T18:24:48.699 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:21:00.986695+0000 mgr.y (mgr.25823) 31 : cephadm 1 Deploying daemon mgr.x on smithi173 2023-09-23T18:24:48.699 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:21:21.498663+0000 mgr.y (mgr.25823) 42 : cephadm 1 Upgrade: Setting container_image for all mgr 2023-09-23T18:24:48.699 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:21:21.537579+0000 mgr.y (mgr.25823) 43 : cephadm 1 Upgrade: It appears safe to stop mon.a 2023-09-23T18:24:48.699 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:21:24.056736+0000 mgr.y (mgr.25823) 45 : cephadm 1 Upgrade: Updating mon.a 2023-09-23T18:24:48.699 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:21:24.071691+0000 mgr.y (mgr.25823) 46 : cephadm 1 Deploying daemon mon.a on smithi098 2023-09-23T18:24:48.700 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:21:40.327983+0000 mgr.y (mgr.26054) 1 : cephadm 1 [23/Sep/2023:18:21:40] ENGINE Bus STARTING 2023-09-23T18:24:48.700 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:21:40.452194+0000 mgr.y (mgr.26054) 4 : cephadm 1 [23/Sep/2023:18:21:40] ENGINE Serving on https://172.21.15.98:7150 2023-09-23T18:24:48.700 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:21:40.553952+0000 mgr.y (mgr.26054) 5 : cephadm 1 [23/Sep/2023:18:21:40] ENGINE Serving on http://172.21.15.98:8765 2023-09-23T18:24:48.700 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:21:40.554456+0000 mgr.y (mgr.26054) 6 : cephadm 1 [23/Sep/2023:18:21:40] ENGINE Bus STARTED 2023-09-23T18:24:48.700 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:21:44.819526+0000 mgr.y (mgr.26054) 10 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:24:48.700 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:21:56.294780+0000 mgr.y (mgr.26054) 17 : cephadm 1 Updating smithi098:/etc/ceph/ceph.conf 2023-09-23T18:24:48.700 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:21:56.295124+0000 mgr.y (mgr.26054) 18 : cephadm 1 Updating smithi173:/etc/ceph/ceph.conf 2023-09-23T18:24:48.700 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:21:56.460368+0000 mgr.y (mgr.26054) 19 : cephadm 1 Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:24:48.700 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:21:56.460725+0000 mgr.y (mgr.26054) 20 : cephadm 1 Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:24:48.700 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:21:56.624879+0000 mgr.y (mgr.26054) 21 : cephadm 1 Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:24:48.700 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:21:56.627173+0000 mgr.y (mgr.26054) 22 : cephadm 1 Updating smithi098:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:24:48.700 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:21:56.789742+0000 mgr.y (mgr.26054) 23 : cephadm 1 Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:24:48.700 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:21:56.791782+0000 mgr.y (mgr.26054) 24 : cephadm 1 Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:24:48.701 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:22:53.371233+0000 mgr.y (mgr.26054) 54 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:24:48.701 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:01.153268+0000 mon.a (mon.0) 82 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T18:24:48.701 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:01.154246+0000 mon.a (mon.0) 83 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T18:24:48.701 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:23:01.154948+0000 mgr.y (mgr.26054) 59 : cephadm 1 Upgrade: It appears safe to stop mon.c 2023-09-23T18:24:48.701 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:01.967110+0000 mon.a (mon.0) 84 : audit 0 from='client.? 172.21.15.98:0/2089910462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:24:48.701 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:21:00.986695+0000 mgr.y (mgr.25823) 31 : cephadm 1 Deploying daemon mgr.x on smithi173 2023-09-23T18:24:48.701 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:21:21.498663+0000 mgr.y (mgr.25823) 42 : cephadm 1 Upgrade: Setting container_image for all mgr 2023-09-23T18:24:48.701 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:21:21.537579+0000 mgr.y (mgr.25823) 43 : cephadm 1 Upgrade: It appears safe to stop mon.a 2023-09-23T18:24:48.701 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:21:24.056736+0000 mgr.y (mgr.25823) 45 : cephadm 1 Upgrade: Updating mon.a 2023-09-23T18:24:48.702 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:21:24.071691+0000 mgr.y (mgr.25823) 46 : cephadm 1 Deploying daemon mon.a on smithi098 2023-09-23T18:24:48.702 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:21:40.327983+0000 mgr.y (mgr.26054) 1 : cephadm 1 [23/Sep/2023:18:21:40] ENGINE Bus STARTING 2023-09-23T18:24:48.702 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:21:40.452194+0000 mgr.y (mgr.26054) 4 : cephadm 1 [23/Sep/2023:18:21:40] ENGINE Serving on https://172.21.15.98:7150 2023-09-23T18:24:48.702 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:21:40.553952+0000 mgr.y (mgr.26054) 5 : cephadm 1 [23/Sep/2023:18:21:40] ENGINE Serving on http://172.21.15.98:8765 2023-09-23T18:24:48.702 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:21:40.554456+0000 mgr.y (mgr.26054) 6 : cephadm 1 [23/Sep/2023:18:21:40] ENGINE Bus STARTED 2023-09-23T18:24:48.702 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:21:44.819526+0000 mgr.y (mgr.26054) 10 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:24:48.702 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:21:56.294780+0000 mgr.y (mgr.26054) 17 : cephadm 1 Updating smithi098:/etc/ceph/ceph.conf 2023-09-23T18:24:48.702 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:21:56.295124+0000 mgr.y (mgr.26054) 18 : cephadm 1 Updating smithi173:/etc/ceph/ceph.conf 2023-09-23T18:24:48.702 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:21:56.460368+0000 mgr.y (mgr.26054) 19 : cephadm 1 Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:24:48.702 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:21:56.460725+0000 mgr.y (mgr.26054) 20 : cephadm 1 Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:24:48.702 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:21:56.624879+0000 mgr.y (mgr.26054) 21 : cephadm 1 Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:24:48.702 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:21:56.627173+0000 mgr.y (mgr.26054) 22 : cephadm 1 Updating smithi098:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:24:48.703 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:21:56.789742+0000 mgr.y (mgr.26054) 23 : cephadm 1 Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:24:48.703 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:21:56.791782+0000 mgr.y (mgr.26054) 24 : cephadm 1 Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:24:48.703 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:22:53.371233+0000 mgr.y (mgr.26054) 54 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:24:48.703 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:01.153268+0000 mon.a (mon.0) 82 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T18:24:48.703 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:01.154246+0000 mon.a (mon.0) 83 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T18:24:48.703 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:23:01.154948+0000 mgr.y (mgr.26054) 59 : cephadm 1 Upgrade: It appears safe to stop mon.c 2023-09-23T18:24:48.703 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:01.967110+0000 mon.a (mon.0) 84 : audit 0 from='client.? 172.21.15.98:0/2089910462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:24:48.703 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:23:03.674657+0000 mgr.y (mgr.26054) 62 : cephadm 1 Upgrade: Updating mon.c 2023-09-23T18:24:48.703 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:03.683308+0000 mon.a (mon.0) 85 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.703 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:03.684782+0000 mon.a (mon.0) 86 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:24:48.704 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:03.685776+0000 mon.a (mon.0) 87 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:24:48.704 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:03.686903+0000 mon.a (mon.0) 88 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:24:48.704 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:23:03.688259+0000 mgr.y (mgr.26054) 63 : cephadm 1 Deploying daemon mon.c on smithi098 2023-09-23T18:24:48.704 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:04.399495+0000 mon.a (mon.0) 89 : audit 0 from='client.? 172.21.15.98:0/964655738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:24:48.704 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:14.847928+0000 mon.a (mon.0) 109 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2023-09-23T18:24:48.704 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:14.848622+0000 mon.a (mon.0) 110 : cluster 0 mgrmap e34: y(active, since 95s), standbys: x 2023-09-23T18:24:48.704 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:14.849009+0000 mon.a (mon.0) 111 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T18:24:48.704 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:14.849056+0000 mon.a (mon.0) 112 : cluster 1 Cluster is now healthy 2023-09-23T18:24:48.704 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:14.875925+0000 mon.a (mon.0) 113 : cluster 1 overall HEALTH_OK 2023-09-23T18:24:48.704 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:15.250167+0000 mon.a (mon.0) 114 : cluster 0 mgrmap e35: y(active, since 95s), standbys: x 2023-09-23T18:24:48.704 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:15.299981+0000 mon.a (mon.0) 115 : audit 0 from='client.? 172.21.15.98:0/2700379740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:24:48.704 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:15.474991+0000 mgr.y (mgr.26054) 69 : cluster 0 pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T18:24:48.704 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:17.475689+0000 mgr.y (mgr.26054) 70 : cluster 0 pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T18:24:48.705 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:19.476240+0000 mgr.y (mgr.26054) 71 : cluster 0 pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T18:24:48.705 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:19.915529+0000 mgr.y (mgr.26054) 72 : audit 0 from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:24:48.705 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:20.173017+0000 mon.a (mon.0) 116 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.705 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:20.179314+0000 mon.a (mon.0) 117 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.705 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:20.969345+0000 mon.a (mon.0) 118 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.705 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:20.976358+0000 mon.a (mon.0) 119 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.705 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:21.477399+0000 mgr.y (mgr.26054) 73 : cluster 0 pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T18:24:48.705 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:23:03.674657+0000 mgr.y (mgr.26054) 62 : cephadm 1 Upgrade: Updating mon.c 2023-09-23T18:24:48.705 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:03.683308+0000 mon.a (mon.0) 85 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.706 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:03.684782+0000 mon.a (mon.0) 86 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:24:48.706 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:03.685776+0000 mon.a (mon.0) 87 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:24:48.706 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:03.686903+0000 mon.a (mon.0) 88 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:24:48.706 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:23:03.688259+0000 mgr.y (mgr.26054) 63 : cephadm 1 Deploying daemon mon.c on smithi098 2023-09-23T18:24:48.706 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:04.399495+0000 mon.a (mon.0) 89 : audit 0 from='client.? 172.21.15.98:0/964655738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:24:48.706 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:14.847928+0000 mon.a (mon.0) 109 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2023-09-23T18:24:48.706 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:14.848622+0000 mon.a (mon.0) 110 : cluster 0 mgrmap e34: y(active, since 95s), standbys: x 2023-09-23T18:24:48.706 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:14.849009+0000 mon.a (mon.0) 111 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T18:24:48.706 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:14.849056+0000 mon.a (mon.0) 112 : cluster 1 Cluster is now healthy 2023-09-23T18:24:48.706 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:14.875925+0000 mon.a (mon.0) 113 : cluster 1 overall HEALTH_OK 2023-09-23T18:24:48.706 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:15.250167+0000 mon.a (mon.0) 114 : cluster 0 mgrmap e35: y(active, since 95s), standbys: x 2023-09-23T18:24:48.706 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:15.299981+0000 mon.a (mon.0) 115 : audit 0 from='client.? 172.21.15.98:0/2700379740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:24:48.707 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:15.474991+0000 mgr.y (mgr.26054) 69 : cluster 0 pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T18:24:48.707 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:17.475689+0000 mgr.y (mgr.26054) 70 : cluster 0 pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T18:24:48.707 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:19.476240+0000 mgr.y (mgr.26054) 71 : cluster 0 pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T18:24:48.707 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:19.915529+0000 mgr.y (mgr.26054) 72 : audit 0 from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:24:48.707 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:20.173017+0000 mon.a (mon.0) 116 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.707 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:20.179314+0000 mon.a (mon.0) 117 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.707 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:20.969345+0000 mon.a (mon.0) 118 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.707 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:20.976358+0000 mon.a (mon.0) 119 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.707 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:21.477399+0000 mgr.y (mgr.26054) 73 : cluster 0 pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T18:24:48.707 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:23.478056+0000 mgr.y (mgr.26054) 74 : cluster 0 pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.708 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:24.267992+0000 mgr.y (mgr.26054) 75 : audit 0 from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:24:48.708 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:23:24.268376+0000 mgr.y (mgr.26054) 76 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:24:48.708 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:24.273012+0000 mon.a (mon.0) 120 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.708 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:24.560575+0000 mon.a (mon.0) 121 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:48.708 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:25.479105+0000 mgr.y (mgr.26054) 77 : cluster 0 pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.708 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:27.479516+0000 mgr.y (mgr.26054) 78 : cluster 0 pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.708 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:29.479949+0000 mgr.y (mgr.26054) 79 : cluster 0 pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.708 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:31.480933+0000 mgr.y (mgr.26054) 80 : cluster 0 pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.708 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:32.227002+0000 mon.a (mon.0) 122 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.708 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:32.236684+0000 mon.a (mon.0) 123 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.708 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:32.238605+0000 mon.a (mon.0) 124 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:24:48.708 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:32.240135+0000 mon.a (mon.0) 125 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:24:48.709 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:32.249311+0000 mon.a (mon.0) 126 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.709 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:32.331608+0000 mon.a (mon.0) 127 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:24:48.709 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:32.334630+0000 mon.a (mon.0) 128 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:24:48.709 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:32.336045+0000 mon.a (mon.0) 129 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:24:48.709 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:32.345504+0000 mon.a (mon.0) 130 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.709 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:33.481526+0000 mgr.y (mgr.26054) 81 : cluster 0 pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.709 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:35.482520+0000 mgr.y (mgr.26054) 82 : cluster 0 pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.709 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:23.478056+0000 mgr.y (mgr.26054) 74 : cluster 0 pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.709 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:24.267992+0000 mgr.y (mgr.26054) 75 : audit 0 from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:24:48.710 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:23:24.268376+0000 mgr.y (mgr.26054) 76 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:24:48.710 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:24.273012+0000 mon.a (mon.0) 120 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.710 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:24.560575+0000 mon.a (mon.0) 121 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:48.710 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:25.479105+0000 mgr.y (mgr.26054) 77 : cluster 0 pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.710 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:27.479516+0000 mgr.y (mgr.26054) 78 : cluster 0 pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.710 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:29.479949+0000 mgr.y (mgr.26054) 79 : cluster 0 pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.710 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:31.480933+0000 mgr.y (mgr.26054) 80 : cluster 0 pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.710 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:32.227002+0000 mon.a (mon.0) 122 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.710 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:32.236684+0000 mon.a (mon.0) 123 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.710 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:32.238605+0000 mon.a (mon.0) 124 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:24:48.710 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:32.240135+0000 mon.a (mon.0) 125 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:24:48.710 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:32.249311+0000 mon.a (mon.0) 126 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.711 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:32.331608+0000 mon.a (mon.0) 127 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:24:48.711 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:32.334630+0000 mon.a (mon.0) 128 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:24:48.711 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:32.336045+0000 mon.a (mon.0) 129 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:24:48.711 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:32.345504+0000 mon.a (mon.0) 130 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.711 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:33.481526+0000 mgr.y (mgr.26054) 81 : cluster 0 pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.711 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:35.482520+0000 mgr.y (mgr.26054) 82 : cluster 0 pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.711 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:37.483127+0000 mgr.y (mgr.26054) 83 : cluster 0 pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.711 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:39.483760+0000 mgr.y (mgr.26054) 84 : cluster 0 pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.711 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:39.566806+0000 mon.a (mon.0) 131 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:48.711 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:41.484838+0000 mgr.y (mgr.26054) 85 : cluster 0 pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.712 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:43.485392+0000 mgr.y (mgr.26054) 86 : cluster 0 pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.712 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:45.486368+0000 mgr.y (mgr.26054) 87 : cluster 0 pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.712 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:47.487094+0000 mgr.y (mgr.26054) 88 : cluster 0 pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.712 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:49.487773+0000 mgr.y (mgr.26054) 89 : cluster 0 pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.712 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:51.488901+0000 mgr.y (mgr.26054) 90 : cluster 0 pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.712 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:53.489667+0000 mgr.y (mgr.26054) 91 : cluster 0 pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.712 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:23:54.567165+0000 mon.a (mon.0) 132 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:48.712 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:55.490753+0000 mgr.y (mgr.26054) 92 : cluster 0 pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.712 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:57.491392+0000 mgr.y (mgr.26054) 93 : cluster 0 pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.712 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:23:59.491955+0000 mgr.y (mgr.26054) 94 : cluster 0 pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.712 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:01.492983+0000 mgr.y (mgr.26054) 95 : cluster 0 pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.712 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:03.493658+0000 mgr.y (mgr.26054) 96 : cluster 0 pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.713 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:05.494700+0000 mgr.y (mgr.26054) 97 : cluster 0 pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.713 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:07.495328+0000 mgr.y (mgr.26054) 98 : cluster 0 pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.713 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:09.495998+0000 mgr.y (mgr.26054) 99 : cluster 0 pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.713 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:09.567498+0000 mon.a (mon.0) 133 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:48.713 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:37.483127+0000 mgr.y (mgr.26054) 83 : cluster 0 pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.713 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:39.483760+0000 mgr.y (mgr.26054) 84 : cluster 0 pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.713 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:39.566806+0000 mon.a (mon.0) 131 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:48.713 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:41.484838+0000 mgr.y (mgr.26054) 85 : cluster 0 pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.713 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:43.485392+0000 mgr.y (mgr.26054) 86 : cluster 0 pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.713 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:45.486368+0000 mgr.y (mgr.26054) 87 : cluster 0 pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.714 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:47.487094+0000 mgr.y (mgr.26054) 88 : cluster 0 pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.714 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:49.487773+0000 mgr.y (mgr.26054) 89 : cluster 0 pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.714 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:51.488901+0000 mgr.y (mgr.26054) 90 : cluster 0 pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.714 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:53.489667+0000 mgr.y (mgr.26054) 91 : cluster 0 pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.714 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:23:54.567165+0000 mon.a (mon.0) 132 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:48.714 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:55.490753+0000 mgr.y (mgr.26054) 92 : cluster 0 pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.714 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:57.491392+0000 mgr.y (mgr.26054) 93 : cluster 0 pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.714 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:23:59.491955+0000 mgr.y (mgr.26054) 94 : cluster 0 pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.714 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:01.492983+0000 mgr.y (mgr.26054) 95 : cluster 0 pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.714 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:03.493658+0000 mgr.y (mgr.26054) 96 : cluster 0 pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.714 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:05.494700+0000 mgr.y (mgr.26054) 97 : cluster 0 pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.714 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:07.495328+0000 mgr.y (mgr.26054) 98 : cluster 0 pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.715 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:09.495998+0000 mgr.y (mgr.26054) 99 : cluster 0 pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.715 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:09.567498+0000 mon.a (mon.0) 133 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:48.715 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:11.497024+0000 mgr.y (mgr.26054) 100 : cluster 0 pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.715 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:13.497764+0000 mgr.y (mgr.26054) 101 : cluster 0 pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.715 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:15.498920+0000 mgr.y (mgr.26054) 102 : cluster 0 pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.715 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:17.499626+0000 mgr.y (mgr.26054) 103 : cluster 0 pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.715 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:19.500320+0000 mgr.y (mgr.26054) 104 : cluster 0 pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.715 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:21.501388+0000 mgr.y (mgr.26054) 105 : cluster 0 pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.715 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:23.501967+0000 mgr.y (mgr.26054) 106 : cluster 0 pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.716 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:24.568031+0000 mon.a (mon.0) 134 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:48.716 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:25.503004+0000 mgr.y (mgr.26054) 107 : cluster 0 pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.716 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:27.503606+0000 mgr.y (mgr.26054) 108 : cluster 0 pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.716 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:29.504045+0000 mgr.y (mgr.26054) 109 : cluster 0 pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.716 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:31.505077+0000 mgr.y (mgr.26054) 110 : cluster 0 pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.716 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:32.427368+0000 mon.a (mon.0) 135 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:24:48.716 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:33.505540+0000 mgr.y (mgr.26054) 111 : cluster 0 pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.716 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:33.746712+0000 mgr.y (mgr.26054) 112 : audit 0 from='client.34209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:24:48.716 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:24:33.747342+0000 mgr.y (mgr.26054) 113 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:24:48.716 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:33.753708+0000 mon.a (mon.0) 136 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.716 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:35.506563+0000 mgr.y (mgr.26054) 114 : cluster 0 pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.716 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:37.506932+0000 mgr.y (mgr.26054) 115 : cluster 0 pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.717 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:38.268428+0000 mon.a (mon.0) 137 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.717 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:11.497024+0000 mgr.y (mgr.26054) 100 : cluster 0 pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.717 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:13.497764+0000 mgr.y (mgr.26054) 101 : cluster 0 pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.717 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:15.498920+0000 mgr.y (mgr.26054) 102 : cluster 0 pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.717 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:17.499626+0000 mgr.y (mgr.26054) 103 : cluster 0 pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.717 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:19.500320+0000 mgr.y (mgr.26054) 104 : cluster 0 pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.717 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:21.501388+0000 mgr.y (mgr.26054) 105 : cluster 0 pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.717 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:23.501967+0000 mgr.y (mgr.26054) 106 : cluster 0 pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.717 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:24.568031+0000 mon.a (mon.0) 134 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:48.717 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:25.503004+0000 mgr.y (mgr.26054) 107 : cluster 0 pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:27.503606+0000 mgr.y (mgr.26054) 108 : cluster 0 pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:29.504045+0000 mgr.y (mgr.26054) 109 : cluster 0 pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:31.505077+0000 mgr.y (mgr.26054) 110 : cluster 0 pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:32.427368+0000 mon.a (mon.0) 135 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:24:48.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:33.505540+0000 mgr.y (mgr.26054) 111 : cluster 0 pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:33.746712+0000 mgr.y (mgr.26054) 112 : audit 0 from='client.34209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:24:48.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:24:33.747342+0000 mgr.y (mgr.26054) 113 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:24:48.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:33.753708+0000 mon.a (mon.0) 136 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:35.506563+0000 mgr.y (mgr.26054) 114 : cluster 0 pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:37.506932+0000 mgr.y (mgr.26054) 115 : cluster 0 pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:38.268428+0000 mon.a (mon.0) 137 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.719 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:38.276457+0000 mon.a (mon.0) 138 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.719 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:38.773548+0000 mon.a (mon.0) 139 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.719 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:38.796961+0000 mon.a (mon.0) 140 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.719 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:39.270808+0000 mon.a (mon.0) 141 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:24:48.719 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:39.273180+0000 mon.a (mon.0) 142 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:24:48.719 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:39.283626+0000 mon.a (mon.0) 143 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.719 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:39.366510+0000 mon.a (mon.0) 144 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:24:48.719 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:39.370803+0000 mon.a (mon.0) 145 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:24:48.719 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:39.372804+0000 mon.a (mon.0) 146 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T18:24:48.719 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:39.374149+0000 mon.a (mon.0) 147 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T18:24:48.719 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:24:39.374993+0000 mgr.y (mgr.26054) 116 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-09-23T18:24:48.720 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:39.507390+0000 mgr.y (mgr.26054) 117 : cluster 0 pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.720 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:39.568290+0000 mon.a (mon.0) 148 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:48.720 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:41.508443+0000 mgr.y (mgr.26054) 118 : cluster 0 pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.720 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:24:41.696111+0000 mgr.y (mgr.26054) 119 : cephadm 1 Upgrade: Updating mon.b 2023-09-23T18:24:48.720 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:41.703858+0000 mon.a (mon.0) 149 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.720 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:41.705897+0000 mon.a (mon.0) 150 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:24:48.720 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:41.707041+0000 mon.a (mon.0) 151 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:24:48.720 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:41.714652+0000 mon.a (mon.0) 152 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:24:48.720 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cephadm 2023-09-23T18:24:41.716007+0000 mgr.y (mgr.26054) 120 : cephadm 1 Deploying daemon mon.b on smithi173 2023-09-23T18:24:48.720 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:41.975073+0000 mon.a (mon.0) 153 : audit 0 from='client.? 172.21.15.98:0/2074759034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:24:48.720 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:47.358200+0000 mon.a (mon.0) 162 : cluster 1 overall HEALTH_OK 2023-09-23T18:24:48.721 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:38.276457+0000 mon.a (mon.0) 138 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.721 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:38.773548+0000 mon.a (mon.0) 139 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.721 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:38.796961+0000 mon.a (mon.0) 140 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.721 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:39.270808+0000 mon.a (mon.0) 141 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:24:48.721 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:39.273180+0000 mon.a (mon.0) 142 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:24:48.721 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:39.283626+0000 mon.a (mon.0) 143 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.721 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:39.366510+0000 mon.a (mon.0) 144 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:24:48.721 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:39.370803+0000 mon.a (mon.0) 145 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:24:48.721 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:39.372804+0000 mon.a (mon.0) 146 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T18:24:48.721 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:39.374149+0000 mon.a (mon.0) 147 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T18:24:48.721 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:24:39.374993+0000 mgr.y (mgr.26054) 116 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-09-23T18:24:48.721 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:39.507390+0000 mgr.y (mgr.26054) 117 : cluster 0 pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:39.568290+0000 mon.a (mon.0) 148 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:48.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:41.508443+0000 mgr.y (mgr.26054) 118 : cluster 0 pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:24:41.696111+0000 mgr.y (mgr.26054) 119 : cephadm 1 Upgrade: Updating mon.b 2023-09-23T18:24:48.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:41.703858+0000 mon.a (mon.0) 149 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:41.705897+0000 mon.a (mon.0) 150 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:24:48.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:41.707041+0000 mon.a (mon.0) 151 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:24:48.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:41.714652+0000 mon.a (mon.0) 152 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:24:48.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cephadm 2023-09-23T18:24:41.716007+0000 mgr.y (mgr.26054) 120 : cephadm 1 Deploying daemon mon.b on smithi173 2023-09-23T18:24:48.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:41.975073+0000 mon.a (mon.0) 153 : audit 0 from='client.? 172.21.15.98:0/2074759034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:24:48.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:47.358200+0000 mon.a (mon.0) 162 : cluster 1 overall HEALTH_OK 2023-09-23T18:24:48.723 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:47.358725+0000 mon.a (mon.0) 163 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:24:48.723 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:47.359026+0000 mon.a (mon.0) 164 : cluster 1 mon.a calling monitor election 2023-09-23T18:24:48.723 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:47.359169+0000 mon.c (mon.2) 2 : cluster 1 mon.c calling monitor election 2023-09-23T18:24:48.723 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:47.360127+0000 mon.a (mon.0) 165 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:24:48.723 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:47.360257+0000 mon.a (mon.0) 166 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:24:48.723 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:47.361366+0000 mon.a (mon.0) 167 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T18:24:48.723 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:47.363168+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-09-23T18:24:48.723 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:47.371108+0000 mon.a (mon.0) 168 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0],b=[v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0],c=[v2:172.21.15.98:3301/0,v1:172.21.15.98:6790/0]} removed_ranks: {} 2023-09-23T18:24:48.723 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:47.371869+0000 mon.a (mon.0) 169 : cluster 0 fsmap foo:1 {0=foo.smithi098.lowjqo=up:active} 1 up:standby 2023-09-23T18:24:48.723 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:47.371913+0000 mon.a (mon.0) 170 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2023-09-23T18:24:48.723 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:47.372346+0000 mon.a (mon.0) 171 : cluster 0 mgrmap e35: y(active, since 3m), standbys: x 2023-09-23T18:24:48.724 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:47.377633+0000 mon.a (mon.0) 172 : cluster 1 overall HEALTH_OK 2023-09-23T18:24:48.724 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:47.381718+0000 mon.a (mon.0) 173 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.724 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:47.388933+0000 mon.a (mon.0) 174 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.724 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: audit 2023-09-23T18:24:47.394883+0000 mon.a (mon.0) 175 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:24:48.724 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[76817]: cluster 2023-09-23T18:24:47.510671+0000 mgr.y (mgr.26054) 123 : cluster 0 pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.724 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:47.358725+0000 mon.a (mon.0) 163 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:24:48.724 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:47.359026+0000 mon.a (mon.0) 164 : cluster 1 mon.a calling monitor election 2023-09-23T18:24:48.724 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:47.359169+0000 mon.c (mon.2) 2 : cluster 1 mon.c calling monitor election 2023-09-23T18:24:48.724 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:47.360127+0000 mon.a (mon.0) 165 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:24:48.724 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:47.360257+0000 mon.a (mon.0) 166 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:24:48.724 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:47.361366+0000 mon.a (mon.0) 167 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T18:24:48.725 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:47.363168+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-09-23T18:24:48.725 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:47.371108+0000 mon.a (mon.0) 168 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0],b=[v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0],c=[v2:172.21.15.98:3301/0,v1:172.21.15.98:6790/0]} removed_ranks: {} 2023-09-23T18:24:48.725 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:47.371869+0000 mon.a (mon.0) 169 : cluster 0 fsmap foo:1 {0=foo.smithi098.lowjqo=up:active} 1 up:standby 2023-09-23T18:24:48.725 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:47.371913+0000 mon.a (mon.0) 170 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2023-09-23T18:24:48.725 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:47.372346+0000 mon.a (mon.0) 171 : cluster 0 mgrmap e35: y(active, since 3m), standbys: x 2023-09-23T18:24:48.725 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:47.377633+0000 mon.a (mon.0) 172 : cluster 1 overall HEALTH_OK 2023-09-23T18:24:48.725 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:47.381718+0000 mon.a (mon.0) 173 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.725 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:47.388933+0000 mon.a (mon.0) 174 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.725 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: audit 2023-09-23T18:24:47.394883+0000 mon.a (mon.0) 175 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:24:48.725 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:48 smithi098 bash[79516]: cluster 2023-09-23T18:24:47.510671+0000 mgr.y (mgr.26054) 123 : cluster 0 pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:04:11.580933+0000 mon.a (mon.0) 0 : 1 mkfs 756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:24:48.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:19:44.249913+0000 mgr.x (mgr.25562) 47 : cephadm 1 Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T18:24:48.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:19:44.250670+0000 mgr.x (mgr.25562) 48 : cephadm 1 Reconfiguring daemon node-exporter.b on smithi173 2023-09-23T18:24:48.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:19:46.119762+0000 mgr.x (mgr.25562) 50 : cephadm 1 Reconfiguring grafana.a (dependencies changed)... 2023-09-23T18:24:48.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:19:46.140131+0000 mgr.x (mgr.25562) 51 : cephadm 1 Reconfiguring daemon grafana.a on smithi173 2023-09-23T18:24:48.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:20:11.502535+0000 mgr.x (mgr.25562) 68 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T18:24:48.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:20:11.503172+0000 mgr.x (mgr.25562) 69 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T18:24:48.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:20:13.823190+0000 mgr.x (mgr.25562) 71 : cephadm 1 Upgrade: Updating mgr.y 2023-09-23T18:24:48.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:20:13.841125+0000 mgr.x (mgr.25562) 72 : cephadm 1 Deploying daemon mgr.y on smithi098 2023-09-23T18:24:48.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:20:36.747789+0000 mgr.y (mgr.25823) 3 : cephadm 1 [23/Sep/2023:18:20:36] ENGINE Bus STARTING 2023-09-23T18:24:48.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:20:36.869010+0000 mgr.y (mgr.25823) 4 : cephadm 1 [23/Sep/2023:18:20:36] ENGINE Serving on https://172.21.15.98:7150 2023-09-23T18:24:48.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:20:36.971240+0000 mgr.y (mgr.25823) 5 : cephadm 1 [23/Sep/2023:18:20:36] ENGINE Serving on http://172.21.15.98:8765 2023-09-23T18:24:48.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:20:36.971641+0000 mgr.y (mgr.25823) 6 : cephadm 1 [23/Sep/2023:18:20:36] ENGINE Bus STARTED 2023-09-23T18:24:48.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:20:51.692244+0000 mgr.y (mgr.25823) 14 : cephadm 1 Updating smithi098:/etc/ceph/ceph.conf 2023-09-23T18:24:48.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:20:51.692594+0000 mgr.y (mgr.25823) 15 : cephadm 1 Updating smithi173:/etc/ceph/ceph.conf 2023-09-23T18:24:48.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:20:51.857117+0000 mgr.y (mgr.25823) 16 : cephadm 1 Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:24:48.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:20:51.857679+0000 mgr.y (mgr.25823) 17 : cephadm 1 Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:24:48.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:20:52.022034+0000 mgr.y (mgr.25823) 18 : cephadm 1 Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:24:48.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:20:52.022405+0000 mgr.y (mgr.25823) 19 : cephadm 1 Updating smithi098:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:24:48.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:20:52.237062+0000 mgr.y (mgr.25823) 20 : cephadm 1 Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:24:48.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:20:52.238837+0000 mgr.y (mgr.25823) 21 : cephadm 1 Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:24:48.780 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:20:52.554522+0000 mgr.y (mgr.25823) 23 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T18:24:48.780 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:20:52.800973+0000 mgr.y (mgr.25823) 24 : cephadm 1 Reconfiguring daemon prometheus.a on smithi173 2023-09-23T18:24:48.780 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:21:00.978897+0000 mgr.y (mgr.25823) 30 : cephadm 1 Upgrade: Updating mgr.x 2023-09-23T18:24:48.781 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:21:00.986695+0000 mgr.y (mgr.25823) 31 : cephadm 1 Deploying daemon mgr.x on smithi173 2023-09-23T18:24:48.781 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:21:21.498663+0000 mgr.y (mgr.25823) 42 : cephadm 1 Upgrade: Setting container_image for all mgr 2023-09-23T18:24:48.781 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:21:21.537579+0000 mgr.y (mgr.25823) 43 : cephadm 1 Upgrade: It appears safe to stop mon.a 2023-09-23T18:24:48.781 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:21:24.056736+0000 mgr.y (mgr.25823) 45 : cephadm 1 Upgrade: Updating mon.a 2023-09-23T18:24:48.781 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:21:24.071691+0000 mgr.y (mgr.25823) 46 : cephadm 1 Deploying daemon mon.a on smithi098 2023-09-23T18:24:48.781 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:21:40.327983+0000 mgr.y (mgr.26054) 1 : cephadm 1 [23/Sep/2023:18:21:40] ENGINE Bus STARTING 2023-09-23T18:24:48.781 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:21:40.452194+0000 mgr.y (mgr.26054) 4 : cephadm 1 [23/Sep/2023:18:21:40] ENGINE Serving on https://172.21.15.98:7150 2023-09-23T18:24:48.781 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:21:40.553952+0000 mgr.y (mgr.26054) 5 : cephadm 1 [23/Sep/2023:18:21:40] ENGINE Serving on http://172.21.15.98:8765 2023-09-23T18:24:48.782 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:21:40.554456+0000 mgr.y (mgr.26054) 6 : cephadm 1 [23/Sep/2023:18:21:40] ENGINE Bus STARTED 2023-09-23T18:24:48.782 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:21:44.819526+0000 mgr.y (mgr.26054) 10 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:24:48.782 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:21:56.294780+0000 mgr.y (mgr.26054) 17 : cephadm 1 Updating smithi098:/etc/ceph/ceph.conf 2023-09-23T18:24:48.782 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:21:56.295124+0000 mgr.y (mgr.26054) 18 : cephadm 1 Updating smithi173:/etc/ceph/ceph.conf 2023-09-23T18:24:48.782 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:21:56.460368+0000 mgr.y (mgr.26054) 19 : cephadm 1 Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:24:48.782 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:21:56.460725+0000 mgr.y (mgr.26054) 20 : cephadm 1 Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T18:24:48.782 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:21:56.624879+0000 mgr.y (mgr.26054) 21 : cephadm 1 Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:24:48.782 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:21:56.627173+0000 mgr.y (mgr.26054) 22 : cephadm 1 Updating smithi098:/etc/ceph/ceph.client.admin.keyring 2023-09-23T18:24:48.782 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:21:56.789742+0000 mgr.y (mgr.26054) 23 : cephadm 1 Updating smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:24:48.782 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:21:56.791782+0000 mgr.y (mgr.26054) 24 : cephadm 1 Updating smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T18:24:48.782 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:22:53.371233+0000 mgr.y (mgr.26054) 54 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:24:48.782 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:01.153268+0000 mon.a (mon.0) 82 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T18:24:48.783 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:01.154246+0000 mon.a (mon.0) 83 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T18:24:48.783 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:23:01.154948+0000 mgr.y (mgr.26054) 59 : cephadm 1 Upgrade: It appears safe to stop mon.c 2023-09-23T18:24:48.783 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:01.967110+0000 mon.a (mon.0) 84 : audit 0 from='client.? 172.21.15.98:0/2089910462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:24:48.783 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:23:03.674657+0000 mgr.y (mgr.26054) 62 : cephadm 1 Upgrade: Updating mon.c 2023-09-23T18:24:48.783 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:03.683308+0000 mon.a (mon.0) 85 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.783 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:03.684782+0000 mon.a (mon.0) 86 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:24:48.783 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:03.685776+0000 mon.a (mon.0) 87 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:24:48.783 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:03.686903+0000 mon.a (mon.0) 88 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:24:48.783 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:23:03.688259+0000 mgr.y (mgr.26054) 63 : cephadm 1 Deploying daemon mon.c on smithi098 2023-09-23T18:24:48.783 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:04.399495+0000 mon.a (mon.0) 89 : audit 0 from='client.? 172.21.15.98:0/964655738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:24:48.783 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:14.847928+0000 mon.a (mon.0) 109 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2023-09-23T18:24:48.783 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:14.848622+0000 mon.a (mon.0) 110 : cluster 0 mgrmap e34: y(active, since 95s), standbys: x 2023-09-23T18:24:48.783 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:14.849009+0000 mon.a (mon.0) 111 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T18:24:48.784 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:14.849056+0000 mon.a (mon.0) 112 : cluster 1 Cluster is now healthy 2023-09-23T18:24:48.784 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:14.875925+0000 mon.a (mon.0) 113 : cluster 1 overall HEALTH_OK 2023-09-23T18:24:48.784 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:15.250167+0000 mon.a (mon.0) 114 : cluster 0 mgrmap e35: y(active, since 95s), standbys: x 2023-09-23T18:24:48.784 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:15.299981+0000 mon.a (mon.0) 115 : audit 0 from='client.? 172.21.15.98:0/2700379740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:24:48.784 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:15.474991+0000 mgr.y (mgr.26054) 69 : cluster 0 pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T18:24:48.784 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:17.475689+0000 mgr.y (mgr.26054) 70 : cluster 0 pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T18:24:48.784 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:19.476240+0000 mgr.y (mgr.26054) 71 : cluster 0 pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T18:24:48.784 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:19.915529+0000 mgr.y (mgr.26054) 72 : audit 0 from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:24:48.784 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:20.173017+0000 mon.a (mon.0) 116 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.784 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:20.179314+0000 mon.a (mon.0) 117 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.784 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:20.969345+0000 mon.a (mon.0) 118 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.784 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:20.976358+0000 mon.a (mon.0) 119 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.785 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:21.477399+0000 mgr.y (mgr.26054) 73 : cluster 0 pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T18:24:48.785 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:23.478056+0000 mgr.y (mgr.26054) 74 : cluster 0 pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.785 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:24.267992+0000 mgr.y (mgr.26054) 75 : audit 0 from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:24:48.785 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:23:24.268376+0000 mgr.y (mgr.26054) 76 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:24:48.785 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:24.273012+0000 mon.a (mon.0) 120 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.785 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:24.560575+0000 mon.a (mon.0) 121 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:48.785 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:25.479105+0000 mgr.y (mgr.26054) 77 : cluster 0 pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.785 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:27.479516+0000 mgr.y (mgr.26054) 78 : cluster 0 pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.785 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:29.479949+0000 mgr.y (mgr.26054) 79 : cluster 0 pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.785 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:31.480933+0000 mgr.y (mgr.26054) 80 : cluster 0 pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.785 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:32.227002+0000 mon.a (mon.0) 122 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.785 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:32.236684+0000 mon.a (mon.0) 123 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.785 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:32.238605+0000 mon.a (mon.0) 124 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:24:48.786 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:32.240135+0000 mon.a (mon.0) 125 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:24:48.786 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:32.249311+0000 mon.a (mon.0) 126 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.786 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:32.331608+0000 mon.a (mon.0) 127 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:24:48.786 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:32.334630+0000 mon.a (mon.0) 128 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:24:48.786 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:32.336045+0000 mon.a (mon.0) 129 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:24:48.786 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:32.345504+0000 mon.a (mon.0) 130 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.786 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:33.481526+0000 mgr.y (mgr.26054) 81 : cluster 0 pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.786 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:35.482520+0000 mgr.y (mgr.26054) 82 : cluster 0 pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.786 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:37.483127+0000 mgr.y (mgr.26054) 83 : cluster 0 pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.786 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:39.483760+0000 mgr.y (mgr.26054) 84 : cluster 0 pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.786 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:39.566806+0000 mon.a (mon.0) 131 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:48.786 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:41.484838+0000 mgr.y (mgr.26054) 85 : cluster 0 pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.787 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:43.485392+0000 mgr.y (mgr.26054) 86 : cluster 0 pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.787 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:45.486368+0000 mgr.y (mgr.26054) 87 : cluster 0 pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.787 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:47.487094+0000 mgr.y (mgr.26054) 88 : cluster 0 pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.787 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:49.487773+0000 mgr.y (mgr.26054) 89 : cluster 0 pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.787 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:51.488901+0000 mgr.y (mgr.26054) 90 : cluster 0 pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.787 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:53.489667+0000 mgr.y (mgr.26054) 91 : cluster 0 pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.787 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:23:54.567165+0000 mon.a (mon.0) 132 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:48.787 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:55.490753+0000 mgr.y (mgr.26054) 92 : cluster 0 pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.787 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:57.491392+0000 mgr.y (mgr.26054) 93 : cluster 0 pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.787 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:23:59.491955+0000 mgr.y (mgr.26054) 94 : cluster 0 pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.787 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:01.492983+0000 mgr.y (mgr.26054) 95 : cluster 0 pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.787 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:03.493658+0000 mgr.y (mgr.26054) 96 : cluster 0 pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.787 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:05.494700+0000 mgr.y (mgr.26054) 97 : cluster 0 pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.788 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:07.495328+0000 mgr.y (mgr.26054) 98 : cluster 0 pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.788 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:09.495998+0000 mgr.y (mgr.26054) 99 : cluster 0 pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.788 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:09.567498+0000 mon.a (mon.0) 133 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:48.788 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:11.497024+0000 mgr.y (mgr.26054) 100 : cluster 0 pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.788 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:13.497764+0000 mgr.y (mgr.26054) 101 : cluster 0 pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.788 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:15.498920+0000 mgr.y (mgr.26054) 102 : cluster 0 pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.788 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:17.499626+0000 mgr.y (mgr.26054) 103 : cluster 0 pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.788 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:19.500320+0000 mgr.y (mgr.26054) 104 : cluster 0 pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.788 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:21.501388+0000 mgr.y (mgr.26054) 105 : cluster 0 pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.788 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:23.501967+0000 mgr.y (mgr.26054) 106 : cluster 0 pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.788 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:24.568031+0000 mon.a (mon.0) 134 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:48.788 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:25.503004+0000 mgr.y (mgr.26054) 107 : cluster 0 pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.788 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:27.503606+0000 mgr.y (mgr.26054) 108 : cluster 0 pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.789 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:29.504045+0000 mgr.y (mgr.26054) 109 : cluster 0 pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.789 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:31.505077+0000 mgr.y (mgr.26054) 110 : cluster 0 pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.789 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:32.427368+0000 mon.a (mon.0) 135 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:24:48.789 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:33.505540+0000 mgr.y (mgr.26054) 111 : cluster 0 pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.789 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:33.746712+0000 mgr.y (mgr.26054) 112 : audit 0 from='client.34209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:24:48.789 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:24:33.747342+0000 mgr.y (mgr.26054) 113 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:24:48.789 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:33.753708+0000 mon.a (mon.0) 136 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.789 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:35.506563+0000 mgr.y (mgr.26054) 114 : cluster 0 pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.789 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:37.506932+0000 mgr.y (mgr.26054) 115 : cluster 0 pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.789 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:38.268428+0000 mon.a (mon.0) 137 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.789 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:38.276457+0000 mon.a (mon.0) 138 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.789 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:38.773548+0000 mon.a (mon.0) 139 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.790 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:38.796961+0000 mon.a (mon.0) 140 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.790 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:39.270808+0000 mon.a (mon.0) 141 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:24:48.790 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:39.273180+0000 mon.a (mon.0) 142 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:24:48.790 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:39.283626+0000 mon.a (mon.0) 143 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.790 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:39.366510+0000 mon.a (mon.0) 144 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:24:48.790 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:39.370803+0000 mon.a (mon.0) 145 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:24:48.790 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:39.372804+0000 mon.a (mon.0) 146 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T18:24:48.790 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:39.374149+0000 mon.a (mon.0) 147 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T18:24:48.790 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:24:39.374993+0000 mgr.y (mgr.26054) 116 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-09-23T18:24:48.790 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:39.507390+0000 mgr.y (mgr.26054) 117 : cluster 0 pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.790 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:39.568290+0000 mon.a (mon.0) 148 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:48.790 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:41.508443+0000 mgr.y (mgr.26054) 118 : cluster 0 pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:48.790 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:24:41.696111+0000 mgr.y (mgr.26054) 119 : cephadm 1 Upgrade: Updating mon.b 2023-09-23T18:24:48.791 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:41.703858+0000 mon.a (mon.0) 149 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.791 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:41.705897+0000 mon.a (mon.0) 150 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:24:48.791 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:41.707041+0000 mon.a (mon.0) 151 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:24:48.791 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:41.714652+0000 mon.a (mon.0) 152 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:24:48.791 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cephadm 2023-09-23T18:24:41.716007+0000 mgr.y (mgr.26054) 120 : cephadm 1 Deploying daemon mon.b on smithi173 2023-09-23T18:24:48.791 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:41.975073+0000 mon.a (mon.0) 153 : audit 0 from='client.? 172.21.15.98:0/2074759034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:24:48.791 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:47.358200+0000 mon.a (mon.0) 162 : cluster 1 overall HEALTH_OK 2023-09-23T18:24:48.791 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:47.358725+0000 mon.a (mon.0) 163 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T18:24:48.791 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:47.359026+0000 mon.a (mon.0) 164 : cluster 1 mon.a calling monitor election 2023-09-23T18:24:48.791 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:47.359169+0000 mon.c (mon.2) 2 : cluster 1 mon.c calling monitor election 2023-09-23T18:24:48.791 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:47.360127+0000 mon.a (mon.0) 165 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T18:24:48.791 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:47.360257+0000 mon.a (mon.0) 166 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T18:24:48.791 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:47.361366+0000 mon.a (mon.0) 167 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T18:24:48.792 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:47.363168+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-09-23T18:24:48.792 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:47.371108+0000 mon.a (mon.0) 168 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0],b=[v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0],c=[v2:172.21.15.98:3301/0,v1:172.21.15.98:6790/0]} removed_ranks: {} 2023-09-23T18:24:48.792 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:47.371869+0000 mon.a (mon.0) 169 : cluster 0 fsmap foo:1 {0=foo.smithi098.lowjqo=up:active} 1 up:standby 2023-09-23T18:24:48.792 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:47.371913+0000 mon.a (mon.0) 170 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2023-09-23T18:24:48.792 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:47.372346+0000 mon.a (mon.0) 171 : cluster 0 mgrmap e35: y(active, since 3m), standbys: x 2023-09-23T18:24:48.792 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:47.377633+0000 mon.a (mon.0) 172 : cluster 1 overall HEALTH_OK 2023-09-23T18:24:48.792 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:47.381718+0000 mon.a (mon.0) 173 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.792 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:47.388933+0000 mon.a (mon.0) 174 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:48.792 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: audit 2023-09-23T18:24:47.394883+0000 mon.a (mon.0) 175 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:24:48.792 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:48 smithi173 bash[59422]: cluster 2023-09-23T18:24:47.510671+0000 mgr.y (mgr.26054) 123 : cluster 0 pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:49.333 INFO:teuthology.orchestra.run.smithi098.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 3 2023-09-23T18:24:50.226 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T18:24:50.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:50 smithi098 bash[79516]: audit 2023-09-23T18:24:49.313685+0000 mon.c (mon.2) 3 : audit 0 from='client.? 172.21.15.98:0/766963932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:24:50.697 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:50 smithi098 bash[79516]: cluster 2023-09-23T18:24:49.511142+0000 mgr.y (mgr.26054) 124 : cluster 0 pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:50.697 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:50 smithi098 bash[76817]: audit 2023-09-23T18:24:49.313685+0000 mon.c (mon.2) 3 : audit 0 from='client.? 172.21.15.98:0/766963932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:24:50.697 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:50 smithi098 bash[76817]: cluster 2023-09-23T18:24:49.511142+0000 mgr.y (mgr.26054) 124 : cluster 0 pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:50.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:50 smithi173 bash[59422]: audit 2023-09-23T18:24:49.313685+0000 mon.c (mon.2) 3 : audit 0 from='client.? 172.21.15.98:0/766963932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T18:24:50.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:50 smithi173 bash[59422]: cluster 2023-09-23T18:24:49.511142+0000 mgr.y (mgr.26054) 124 : cluster 0 pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:51.694 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:24:51 smithi098 bash[73085]: debug 2023-09-23T18:24:51.346+0000 7f90a56ad700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-09-23T18:24:52.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:52 smithi098 bash[76817]: cluster 2023-09-23T18:24:51.512195+0000 mgr.y (mgr.26054) 125 : cluster 0 pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:52.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:52 smithi098 bash[79516]: cluster 2023-09-23T18:24:51.512195+0000 mgr.y (mgr.26054) 125 : cluster 0 pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:52.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:52 smithi173 bash[59422]: cluster 2023-09-23T18:24:51.512195+0000 mgr.y (mgr.26054) 125 : cluster 0 pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:53.567 INFO:teuthology.orchestra.run.smithi098.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T18:24:53.568 INFO:teuthology.orchestra.run.smithi098.stdout:alertmanager.a smithi098 *:9093,9094 running (5m) - 11m 13.4M - 0.23.0 ba2b418f427c 7c2bcaebb703 2023-09-23T18:24:53.568 INFO:teuthology.orchestra.run.smithi098.stdout:grafana.a smithi173 *:3000 running (4m) - 11m 40.8M - 8.3.5 dad864ee21e9 616bbe1127b7 2023-09-23T18:24:53.568 INFO:teuthology.orchestra.run.smithi098.stdout:mds.foo.smithi098.lowjqo smithi098 running (9m) - 9m 16.2M - 16.2.14-37-g2e1f1015 1e6a756174a8 e1555281d4fd 2023-09-23T18:24:53.568 INFO:teuthology.orchestra.run.smithi098.stdout:mds.foo.smithi173.qjefje smithi173 running (9m) - 9m 13.6M - 16.2.14-37-g2e1f1015 1e6a756174a8 fc5513981852 2023-09-23T18:24:53.568 INFO:teuthology.orchestra.run.smithi098.stdout:mgr.x smithi173 *:8443,9283,8765 running (3m) - 18m 427M - 18.2.0-460-gbb0daeff e32c2a64c126 070f62900469 2023-09-23T18:24:53.568 INFO:teuthology.orchestra.run.smithi098.stdout:mgr.y smithi098 *:8443,9283,8765 running (4m) - 20m 475M - 18.2.0-460-gbb0daeff e32c2a64c126 c11dea5b47dd 2023-09-23T18:24:53.568 INFO:teuthology.orchestra.run.smithi098.stdout:mon.a smithi098 running (3m) - 20m 59.2M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 d325a8e8ebe6 2023-09-23T18:24:53.568 INFO:teuthology.orchestra.run.smithi098.stdout:mon.b smithi173 starting - - - 2048M 2023-09-23T18:24:53.568 INFO:teuthology.orchestra.run.smithi098.stdout:mon.c smithi098 running (106s) - 18m 34.5M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 3bbca4740a2d 2023-09-23T18:24:53.568 INFO:teuthology.orchestra.run.smithi098.stdout:node-exporter.a smithi098 *:9100 running (5m) - 11m 9511k - 1.3.1 1dbe0e931976 bdffbddc2c81 2023-09-23T18:24:53.569 INFO:teuthology.orchestra.run.smithi098.stdout:node-exporter.b smithi173 *:9100 running (5m) - 11m 9404k - 1.3.1 1dbe0e931976 610050ff4ef2 2023-09-23T18:24:53.569 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0 smithi098 running (17m) - 17m 46.1M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 5900bdd3fa79 2023-09-23T18:24:53.569 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1 smithi098 running (17m) - 17m 48.1M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 7fd00c3be01c 2023-09-23T18:24:53.569 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2 smithi098 running (16m) - 16m 44.2M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 587769f25c85 2023-09-23T18:24:53.569 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3 smithi098 running (15m) - 15m 43.1M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 af525819942d 2023-09-23T18:24:53.569 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4 smithi173 running (14m) - 15m 43.5M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 0a479b881bee 2023-09-23T18:24:53.569 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5 smithi173 running (14m) - 14m 44.2M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 501903da3aa9 2023-09-23T18:24:53.569 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6 smithi173 running (13m) - 13m 45.9M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 b37149ed4271 2023-09-23T18:24:53.569 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7 smithi173 running (12m) - 12m 48.3M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 81723c51be2b 2023-09-23T18:24:53.569 INFO:teuthology.orchestra.run.smithi098.stdout:prometheus.a smithi173 *:9095 running (3m) - 11m 54.8M - 2.33.4 514e6a882f6e 7547c9472f93 2023-09-23T18:24:54.372 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T18:24:54.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:54 smithi098 bash[79516]: cluster 2023-09-23T18:24:53.512852+0000 mgr.y (mgr.26054) 126 : cluster 0 pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:54.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:54 smithi098 bash[79516]: audit 2023-09-23T18:24:53.543595+0000 mgr.y (mgr.26054) 127 : audit 0 from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:24:54.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:54 smithi098 bash[79516]: audit 2023-09-23T18:24:53.637923+0000 mon.a (mon.0) 176 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:54.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:54 smithi098 bash[79516]: audit 2023-09-23T18:24:53.644564+0000 mon.a (mon.0) 177 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:54.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:54 smithi098 bash[79516]: audit 2023-09-23T18:24:54.441296+0000 mon.a (mon.0) 178 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:54.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:54 smithi098 bash[79516]: audit 2023-09-23T18:24:54.451241+0000 mon.a (mon.0) 179 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:54.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:54 smithi098 bash[79516]: audit 2023-09-23T18:24:54.568079+0000 mon.a (mon.0) 180 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:54.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:54 smithi098 bash[76817]: cluster 2023-09-23T18:24:53.512852+0000 mgr.y (mgr.26054) 126 : cluster 0 pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:54.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:54 smithi098 bash[76817]: audit 2023-09-23T18:24:53.543595+0000 mgr.y (mgr.26054) 127 : audit 0 from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:24:54.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:54 smithi098 bash[76817]: audit 2023-09-23T18:24:53.637923+0000 mon.a (mon.0) 176 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:54.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:54 smithi098 bash[76817]: audit 2023-09-23T18:24:53.644564+0000 mon.a (mon.0) 177 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:54.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:54 smithi098 bash[76817]: audit 2023-09-23T18:24:54.441296+0000 mon.a (mon.0) 178 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:54.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:54 smithi098 bash[76817]: audit 2023-09-23T18:24:54.451241+0000 mon.a (mon.0) 179 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:54.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:54 smithi098 bash[76817]: audit 2023-09-23T18:24:54.568079+0000 mon.a (mon.0) 180 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:54 smithi173 bash[59422]: cluster 2023-09-23T18:24:53.512852+0000 mgr.y (mgr.26054) 126 : cluster 0 pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:54 smithi173 bash[59422]: audit 2023-09-23T18:24:53.543595+0000 mgr.y (mgr.26054) 127 : audit 0 from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:24:55.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:54 smithi173 bash[59422]: audit 2023-09-23T18:24:53.637923+0000 mon.a (mon.0) 176 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:55.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:54 smithi173 bash[59422]: audit 2023-09-23T18:24:53.644564+0000 mon.a (mon.0) 177 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:55.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:54 smithi173 bash[59422]: audit 2023-09-23T18:24:54.441296+0000 mon.a (mon.0) 178 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:55.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:54 smithi173 bash[59422]: audit 2023-09-23T18:24:54.451241+0000 mon.a (mon.0) 179 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:55.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:54 smithi173 bash[59422]: audit 2023-09-23T18:24:54.568079+0000 mon.a (mon.0) 180 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:56.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:24:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:24:56] "GET /metrics HTTP/1.1" 200 33909 "" "Prometheus/2.33.4" 2023-09-23T18:24:56.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:56 smithi098 bash[79516]: cluster 2023-09-23T18:24:55.513963+0000 mgr.y (mgr.26054) 128 : cluster 0 pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:56.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:56 smithi098 bash[76817]: cluster 2023-09-23T18:24:55.513963+0000 mgr.y (mgr.26054) 128 : cluster 0 pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:57.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:56 smithi173 bash[59422]: cluster 2023-09-23T18:24:55.513963+0000 mgr.y (mgr.26054) 128 : cluster 0 pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:58.046 INFO:teuthology.orchestra.run.smithi098.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:24:58.800 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T18:24:58.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:58 smithi098 bash[76817]: cluster 2023-09-23T18:24:57.514622+0000 mgr.y (mgr.26054) 129 : cluster 0 pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:58.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:58 smithi098 bash[76817]: audit 2023-09-23T18:24:58.041327+0000 mon.a (mon.0) 181 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:58.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:58 smithi098 bash[79516]: cluster 2023-09-23T18:24:57.514622+0000 mgr.y (mgr.26054) 129 : cluster 0 pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:58.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:58 smithi098 bash[79516]: audit 2023-09-23T18:24:58.041327+0000 mon.a (mon.0) 181 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:59.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:58 smithi173 bash[59422]: cluster 2023-09-23T18:24:57.514622+0000 mgr.y (mgr.26054) 129 : cluster 0 pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:24:59.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:58 smithi173 bash[59422]: audit 2023-09-23T18:24:58.041327+0000 mon.a (mon.0) 181 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:24:59.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:59 smithi098 bash[79516]: audit 2023-09-23T18:24:58.031838+0000 mgr.y (mgr.26054) 130 : audit 0 from='client.34245 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:24:59.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:24:59 smithi098 bash[79516]: cephadm 2023-09-23T18:24:58.032503+0000 mgr.y (mgr.26054) 131 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:24:59.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:59 smithi098 bash[76817]: audit 2023-09-23T18:24:58.031838+0000 mgr.y (mgr.26054) 130 : audit 0 from='client.34245 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:24:59.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:24:59 smithi098 bash[76817]: cephadm 2023-09-23T18:24:58.032503+0000 mgr.y (mgr.26054) 131 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:25:00.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:59 smithi173 bash[59422]: audit 2023-09-23T18:24:58.031838+0000 mgr.y (mgr.26054) 130 : audit 0 from='client.34245 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:25:00.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:24:59 smithi173 bash[59422]: cephadm 2023-09-23T18:24:58.032503+0000 mgr.y (mgr.26054) 131 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:25:00.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:00 smithi098 bash[76817]: cluster 2023-09-23T18:24:59.515189+0000 mgr.y (mgr.26054) 132 : cluster 0 pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:00.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:00 smithi098 bash[79516]: cluster 2023-09-23T18:24:59.515189+0000 mgr.y (mgr.26054) 132 : cluster 0 pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:00.728 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:00 smithi173 bash[59422]: cluster 2023-09-23T18:24:59.515189+0000 mgr.y (mgr.26054) 132 : cluster 0 pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:02.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:02 smithi173 bash[59422]: cluster 2023-09-23T18:25:01.516211+0000 mgr.y (mgr.26054) 133 : cluster 0 pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:02.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:02 smithi098 bash[76817]: cluster 2023-09-23T18:25:01.516211+0000 mgr.y (mgr.26054) 133 : cluster 0 pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:02.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:02 smithi098 bash[79516]: cluster 2023-09-23T18:25:01.516211+0000 mgr.y (mgr.26054) 133 : cluster 0 pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:04.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:04 smithi098 bash[79516]: cluster 2023-09-23T18:25:03.516861+0000 mgr.y (mgr.26054) 134 : cluster 0 pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:04.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:04 smithi098 bash[79516]: audit 2023-09-23T18:25:04.052601+0000 mon.a (mon.0) 182 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:04.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:04 smithi098 bash[79516]: audit 2023-09-23T18:25:04.063602+0000 mon.a (mon.0) 183 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:04.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:04 smithi098 bash[79516]: audit 2023-09-23T18:25:04.065534+0000 mon.a (mon.0) 184 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:04.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:04 smithi098 bash[79516]: audit 2023-09-23T18:25:04.067011+0000 mon.a (mon.0) 185 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:25:04.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:04 smithi098 bash[79516]: audit 2023-09-23T18:25:04.076504+0000 mon.a (mon.0) 186 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:04.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:04 smithi098 bash[79516]: audit 2023-09-23T18:25:04.149303+0000 mon.a (mon.0) 187 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:25:04.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:04 smithi098 bash[79516]: audit 2023-09-23T18:25:04.151081+0000 mon.a (mon.0) 188 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:25:04.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:04 smithi098 bash[79516]: audit 2023-09-23T18:25:04.152702+0000 mon.a (mon.0) 189 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:04.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:04 smithi098 bash[76817]: cluster 2023-09-23T18:25:03.516861+0000 mgr.y (mgr.26054) 134 : cluster 0 pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:04.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:04 smithi098 bash[76817]: audit 2023-09-23T18:25:04.052601+0000 mon.a (mon.0) 182 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:04.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:04 smithi098 bash[76817]: audit 2023-09-23T18:25:04.063602+0000 mon.a (mon.0) 183 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:04.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:04 smithi098 bash[76817]: audit 2023-09-23T18:25:04.065534+0000 mon.a (mon.0) 184 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:04.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:04 smithi098 bash[76817]: audit 2023-09-23T18:25:04.067011+0000 mon.a (mon.0) 185 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:25:04.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:04 smithi098 bash[76817]: audit 2023-09-23T18:25:04.076504+0000 mon.a (mon.0) 186 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:04.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:04 smithi098 bash[76817]: audit 2023-09-23T18:25:04.149303+0000 mon.a (mon.0) 187 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:25:04.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:04 smithi098 bash[76817]: audit 2023-09-23T18:25:04.151081+0000 mon.a (mon.0) 188 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:25:04.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:04 smithi098 bash[76817]: audit 2023-09-23T18:25:04.152702+0000 mon.a (mon.0) 189 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:05.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:04 smithi173 bash[59422]: cluster 2023-09-23T18:25:03.516861+0000 mgr.y (mgr.26054) 134 : cluster 0 pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:05.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:04 smithi173 bash[59422]: audit 2023-09-23T18:25:04.052601+0000 mon.a (mon.0) 182 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:05.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:04 smithi173 bash[59422]: audit 2023-09-23T18:25:04.063602+0000 mon.a (mon.0) 183 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:05.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:04 smithi173 bash[59422]: audit 2023-09-23T18:25:04.065534+0000 mon.a (mon.0) 184 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:05.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:04 smithi173 bash[59422]: audit 2023-09-23T18:25:04.067011+0000 mon.a (mon.0) 185 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:25:05.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:04 smithi173 bash[59422]: audit 2023-09-23T18:25:04.076504+0000 mon.a (mon.0) 186 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:05.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:04 smithi173 bash[59422]: audit 2023-09-23T18:25:04.149303+0000 mon.a (mon.0) 187 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:25:05.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:04 smithi173 bash[59422]: audit 2023-09-23T18:25:04.151081+0000 mon.a (mon.0) 188 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:25:05.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:04 smithi173 bash[59422]: audit 2023-09-23T18:25:04.152702+0000 mon.a (mon.0) 189 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:05.939 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:05 smithi098 bash[76817]: cephadm 2023-09-23T18:25:04.148565+0000 mgr.y (mgr.26054) 135 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-09-23T18:25:05.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:05 smithi098 bash[76817]: cephadm 2023-09-23T18:25:04.154389+0000 mgr.y (mgr.26054) 136 : cephadm 1 Reconfiguring daemon mon.a on smithi098 2023-09-23T18:25:05.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:05 smithi098 bash[79516]: cephadm 2023-09-23T18:25:04.148565+0000 mgr.y (mgr.26054) 135 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-09-23T18:25:05.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:05 smithi098 bash[79516]: cephadm 2023-09-23T18:25:04.154389+0000 mgr.y (mgr.26054) 136 : cephadm 1 Reconfiguring daemon mon.a on smithi098 2023-09-23T18:25:06.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:05 smithi173 bash[59422]: cephadm 2023-09-23T18:25:04.148565+0000 mgr.y (mgr.26054) 135 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-09-23T18:25:06.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:05 smithi173 bash[59422]: cephadm 2023-09-23T18:25:04.154389+0000 mgr.y (mgr.26054) 136 : cephadm 1 Reconfiguring daemon mon.a on smithi098 2023-09-23T18:25:06.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:25:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:25:06] "GET /metrics HTTP/1.1" 200 33909 "" "Prometheus/2.33.4" 2023-09-23T18:25:06.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:06 smithi098 bash[76817]: cluster 2023-09-23T18:25:05.517884+0000 mgr.y (mgr.26054) 137 : cluster 0 pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:06.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:06 smithi098 bash[76817]: audit 2023-09-23T18:25:06.564868+0000 mon.a (mon.0) 190 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:06.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:06 smithi098 bash[76817]: audit 2023-09-23T18:25:06.572074+0000 mon.a (mon.0) 191 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:06.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:06 smithi098 bash[76817]: audit 2023-09-23T18:25:06.574906+0000 mon.a (mon.0) 192 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:25:06.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:06 smithi098 bash[76817]: audit 2023-09-23T18:25:06.576006+0000 mon.a (mon.0) 193 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:25:06.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:06 smithi098 bash[76817]: audit 2023-09-23T18:25:06.577050+0000 mon.a (mon.0) 194 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:06.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:06 smithi098 bash[79516]: cluster 2023-09-23T18:25:05.517884+0000 mgr.y (mgr.26054) 137 : cluster 0 pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:06.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:06 smithi098 bash[79516]: audit 2023-09-23T18:25:06.564868+0000 mon.a (mon.0) 190 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:06.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:06 smithi098 bash[79516]: audit 2023-09-23T18:25:06.572074+0000 mon.a (mon.0) 191 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:06.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:06 smithi098 bash[79516]: audit 2023-09-23T18:25:06.574906+0000 mon.a (mon.0) 192 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:25:06.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:06 smithi098 bash[79516]: audit 2023-09-23T18:25:06.576006+0000 mon.a (mon.0) 193 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:25:06.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:06 smithi098 bash[79516]: audit 2023-09-23T18:25:06.577050+0000 mon.a (mon.0) 194 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:07.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:06 smithi173 bash[59422]: cluster 2023-09-23T18:25:05.517884+0000 mgr.y (mgr.26054) 137 : cluster 0 pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:07.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:06 smithi173 bash[59422]: audit 2023-09-23T18:25:06.564868+0000 mon.a (mon.0) 190 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:07.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:06 smithi173 bash[59422]: audit 2023-09-23T18:25:06.572074+0000 mon.a (mon.0) 191 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:07.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:06 smithi173 bash[59422]: audit 2023-09-23T18:25:06.574906+0000 mon.a (mon.0) 192 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:25:07.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:06 smithi173 bash[59422]: audit 2023-09-23T18:25:06.576006+0000 mon.a (mon.0) 193 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:25:07.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:06 smithi173 bash[59422]: audit 2023-09-23T18:25:06.577050+0000 mon.a (mon.0) 194 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:07.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:07 smithi098 bash[79516]: cephadm 2023-09-23T18:25:06.574511+0000 mgr.y (mgr.26054) 138 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-09-23T18:25:07.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:07 smithi098 bash[79516]: cephadm 2023-09-23T18:25:06.578215+0000 mgr.y (mgr.26054) 139 : cephadm 1 Reconfiguring daemon mgr.y on smithi098 2023-09-23T18:25:07.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:07 smithi098 bash[76817]: cephadm 2023-09-23T18:25:06.574511+0000 mgr.y (mgr.26054) 138 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-09-23T18:25:07.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:07 smithi098 bash[76817]: cephadm 2023-09-23T18:25:06.578215+0000 mgr.y (mgr.26054) 139 : cephadm 1 Reconfiguring daemon mgr.y on smithi098 2023-09-23T18:25:08.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:07 smithi173 bash[59422]: cephadm 2023-09-23T18:25:06.574511+0000 mgr.y (mgr.26054) 138 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-09-23T18:25:08.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:07 smithi173 bash[59422]: cephadm 2023-09-23T18:25:06.578215+0000 mgr.y (mgr.26054) 139 : cephadm 1 Reconfiguring daemon mgr.y on smithi098 2023-09-23T18:25:08.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:08 smithi098 bash[79516]: cluster 2023-09-23T18:25:07.518469+0000 mgr.y (mgr.26054) 140 : cluster 0 pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:08.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:08 smithi098 bash[76817]: cluster 2023-09-23T18:25:07.518469+0000 mgr.y (mgr.26054) 140 : cluster 0 pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:09.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:08 smithi173 bash[59422]: cluster 2023-09-23T18:25:07.518469+0000 mgr.y (mgr.26054) 140 : cluster 0 pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:10.075 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:09 smithi098 bash[79516]: audit 2023-09-23T18:25:08.795861+0000 mon.a (mon.0) 195 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:10.075 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:09 smithi098 bash[79516]: audit 2023-09-23T18:25:08.801497+0000 mon.a (mon.0) 196 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:10.075 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:09 smithi098 bash[79516]: audit 2023-09-23T18:25:08.848035+0000 mon.a (mon.0) 197 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:25:10.075 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:09 smithi098 bash[79516]: audit 2023-09-23T18:25:08.849695+0000 mon.a (mon.0) 198 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:25:10.075 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:09 smithi098 bash[79516]: audit 2023-09-23T18:25:08.851264+0000 mon.a (mon.0) 199 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:10.075 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:09 smithi098 bash[79516]: audit 2023-09-23T18:25:09.569003+0000 mon.a (mon.0) 200 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:25:10.075 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:09 smithi098 bash[76817]: audit 2023-09-23T18:25:08.795861+0000 mon.a (mon.0) 195 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:10.076 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:09 smithi098 bash[76817]: audit 2023-09-23T18:25:08.801497+0000 mon.a (mon.0) 196 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:10.076 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:09 smithi098 bash[76817]: audit 2023-09-23T18:25:08.848035+0000 mon.a (mon.0) 197 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:25:10.076 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:09 smithi098 bash[76817]: audit 2023-09-23T18:25:08.849695+0000 mon.a (mon.0) 198 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:25:10.076 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:09 smithi098 bash[76817]: audit 2023-09-23T18:25:08.851264+0000 mon.a (mon.0) 199 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:10.076 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:09 smithi098 bash[76817]: audit 2023-09-23T18:25:09.569003+0000 mon.a (mon.0) 200 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:25:10.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:09 smithi173 bash[59422]: audit 2023-09-23T18:25:08.795861+0000 mon.a (mon.0) 195 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:10.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:09 smithi173 bash[59422]: audit 2023-09-23T18:25:08.801497+0000 mon.a (mon.0) 196 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:10.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:09 smithi173 bash[59422]: audit 2023-09-23T18:25:08.848035+0000 mon.a (mon.0) 197 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:25:10.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:09 smithi173 bash[59422]: audit 2023-09-23T18:25:08.849695+0000 mon.a (mon.0) 198 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:25:10.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:09 smithi173 bash[59422]: audit 2023-09-23T18:25:08.851264+0000 mon.a (mon.0) 199 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:10.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:09 smithi173 bash[59422]: audit 2023-09-23T18:25:09.569003+0000 mon.a (mon.0) 200 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:25:11.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:10 smithi098 bash[76817]: cephadm 2023-09-23T18:25:08.847397+0000 mgr.y (mgr.26054) 141 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-09-23T18:25:11.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:10 smithi098 bash[76817]: cephadm 2023-09-23T18:25:08.853119+0000 mgr.y (mgr.26054) 142 : cephadm 1 Reconfiguring daemon mon.c on smithi098 2023-09-23T18:25:11.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:10 smithi098 bash[76817]: cluster 2023-09-23T18:25:09.519042+0000 mgr.y (mgr.26054) 143 : cluster 0 pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:11.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:10 smithi098 bash[79516]: cephadm 2023-09-23T18:25:08.847397+0000 mgr.y (mgr.26054) 141 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-09-23T18:25:11.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:10 smithi098 bash[79516]: cephadm 2023-09-23T18:25:08.853119+0000 mgr.y (mgr.26054) 142 : cephadm 1 Reconfiguring daemon mon.c on smithi098 2023-09-23T18:25:11.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:10 smithi098 bash[79516]: cluster 2023-09-23T18:25:09.519042+0000 mgr.y (mgr.26054) 143 : cluster 0 pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:11.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:10 smithi173 bash[59422]: cephadm 2023-09-23T18:25:08.847397+0000 mgr.y (mgr.26054) 141 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-09-23T18:25:11.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:10 smithi173 bash[59422]: cephadm 2023-09-23T18:25:08.853119+0000 mgr.y (mgr.26054) 142 : cephadm 1 Reconfiguring daemon mon.c on smithi098 2023-09-23T18:25:11.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:10 smithi173 bash[59422]: cluster 2023-09-23T18:25:09.519042+0000 mgr.y (mgr.26054) 143 : cluster 0 pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:12.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:12 smithi098 bash[76817]: audit 2023-09-23T18:25:11.048877+0000 mon.a (mon.0) 201 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:12.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:12 smithi098 bash[76817]: audit 2023-09-23T18:25:11.057500+0000 mon.a (mon.0) 202 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:12.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:12 smithi098 bash[76817]: cephadm 2023-09-23T18:25:11.072496+0000 mgr.y (mgr.26054) 144 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-09-23T18:25:12.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:12 smithi098 bash[76817]: audit 2023-09-23T18:25:11.073035+0000 mon.a (mon.0) 203 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T18:25:12.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:12 smithi098 bash[76817]: audit 2023-09-23T18:25:11.074643+0000 mon.a (mon.0) 204 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:12.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:12 smithi098 bash[76817]: cephadm 2023-09-23T18:25:11.078997+0000 mgr.y (mgr.26054) 145 : cephadm 1 Reconfiguring daemon osd.0 on smithi098 2023-09-23T18:25:12.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:12 smithi098 bash[76817]: cluster 2023-09-23T18:25:11.520121+0000 mgr.y (mgr.26054) 146 : cluster 0 pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:12.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:12 smithi098 bash[79516]: audit 2023-09-23T18:25:11.048877+0000 mon.a (mon.0) 201 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:12.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:12 smithi098 bash[79516]: audit 2023-09-23T18:25:11.057500+0000 mon.a (mon.0) 202 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:12.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:12 smithi098 bash[79516]: cephadm 2023-09-23T18:25:11.072496+0000 mgr.y (mgr.26054) 144 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-09-23T18:25:12.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:12 smithi098 bash[79516]: audit 2023-09-23T18:25:11.073035+0000 mon.a (mon.0) 203 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T18:25:12.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:12 smithi098 bash[79516]: audit 2023-09-23T18:25:11.074643+0000 mon.a (mon.0) 204 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:12.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:12 smithi098 bash[79516]: cephadm 2023-09-23T18:25:11.078997+0000 mgr.y (mgr.26054) 145 : cephadm 1 Reconfiguring daemon osd.0 on smithi098 2023-09-23T18:25:12.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:12 smithi098 bash[79516]: cluster 2023-09-23T18:25:11.520121+0000 mgr.y (mgr.26054) 146 : cluster 0 pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:12.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:12 smithi173 bash[59422]: audit 2023-09-23T18:25:11.048877+0000 mon.a (mon.0) 201 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:12.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:12 smithi173 bash[59422]: audit 2023-09-23T18:25:11.057500+0000 mon.a (mon.0) 202 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:12.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:12 smithi173 bash[59422]: cephadm 2023-09-23T18:25:11.072496+0000 mgr.y (mgr.26054) 144 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-09-23T18:25:12.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:12 smithi173 bash[59422]: audit 2023-09-23T18:25:11.073035+0000 mon.a (mon.0) 203 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T18:25:12.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:12 smithi173 bash[59422]: audit 2023-09-23T18:25:11.074643+0000 mon.a (mon.0) 204 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:12.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:12 smithi173 bash[59422]: cephadm 2023-09-23T18:25:11.078997+0000 mgr.y (mgr.26054) 145 : cephadm 1 Reconfiguring daemon osd.0 on smithi098 2023-09-23T18:25:12.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:12 smithi173 bash[59422]: cluster 2023-09-23T18:25:11.520121+0000 mgr.y (mgr.26054) 146 : cluster 0 pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:14.625 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:14 smithi098 bash[79516]: audit 2023-09-23T18:25:13.336692+0000 mon.a (mon.0) 205 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:14.626 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:14 smithi098 bash[79516]: audit 2023-09-23T18:25:13.346133+0000 mon.a (mon.0) 206 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:14.626 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:14 smithi098 bash[79516]: cephadm 2023-09-23T18:25:13.397588+0000 mgr.y (mgr.26054) 147 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-09-23T18:25:14.626 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:14 smithi098 bash[79516]: audit 2023-09-23T18:25:13.398308+0000 mon.a (mon.0) 207 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T18:25:14.626 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:14 smithi098 bash[79516]: audit 2023-09-23T18:25:13.399923+0000 mon.a (mon.0) 208 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:14.626 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:14 smithi098 bash[79516]: cephadm 2023-09-23T18:25:13.404227+0000 mgr.y (mgr.26054) 148 : cephadm 1 Reconfiguring daemon osd.1 on smithi098 2023-09-23T18:25:14.626 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:14 smithi098 bash[79516]: cluster 2023-09-23T18:25:13.520713+0000 mgr.y (mgr.26054) 149 : cluster 0 pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:14.626 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:14 smithi098 bash[76817]: audit 2023-09-23T18:25:13.336692+0000 mon.a (mon.0) 205 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:14.627 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:14 smithi098 bash[76817]: audit 2023-09-23T18:25:13.346133+0000 mon.a (mon.0) 206 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:14.627 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:14 smithi098 bash[76817]: cephadm 2023-09-23T18:25:13.397588+0000 mgr.y (mgr.26054) 147 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-09-23T18:25:14.627 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:14 smithi098 bash[76817]: audit 2023-09-23T18:25:13.398308+0000 mon.a (mon.0) 207 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T18:25:14.627 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:14 smithi098 bash[76817]: audit 2023-09-23T18:25:13.399923+0000 mon.a (mon.0) 208 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:14.627 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:14 smithi098 bash[76817]: cephadm 2023-09-23T18:25:13.404227+0000 mgr.y (mgr.26054) 148 : cephadm 1 Reconfiguring daemon osd.1 on smithi098 2023-09-23T18:25:14.627 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:14 smithi098 bash[76817]: cluster 2023-09-23T18:25:13.520713+0000 mgr.y (mgr.26054) 149 : cluster 0 pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:14.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:14 smithi173 bash[59422]: audit 2023-09-23T18:25:13.336692+0000 mon.a (mon.0) 205 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:14.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:14 smithi173 bash[59422]: audit 2023-09-23T18:25:13.346133+0000 mon.a (mon.0) 206 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:14.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:14 smithi173 bash[59422]: cephadm 2023-09-23T18:25:13.397588+0000 mgr.y (mgr.26054) 147 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-09-23T18:25:14.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:14 smithi173 bash[59422]: audit 2023-09-23T18:25:13.398308+0000 mon.a (mon.0) 207 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T18:25:14.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:14 smithi173 bash[59422]: audit 2023-09-23T18:25:13.399923+0000 mon.a (mon.0) 208 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:14.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:14 smithi173 bash[59422]: cephadm 2023-09-23T18:25:13.404227+0000 mgr.y (mgr.26054) 148 : cephadm 1 Reconfiguring daemon osd.1 on smithi098 2023-09-23T18:25:14.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:14 smithi173 bash[59422]: cluster 2023-09-23T18:25:13.520713+0000 mgr.y (mgr.26054) 149 : cluster 0 pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:16.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:25:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:25:16] "GET /metrics HTTP/1.1" 200 33913 "" "Prometheus/2.33.4" 2023-09-23T18:25:16.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:16 smithi098 bash[79516]: cluster 2023-09-23T18:25:15.521708+0000 mgr.y (mgr.26054) 150 : cluster 0 pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:16.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:16 smithi098 bash[79516]: audit 2023-09-23T18:25:15.662396+0000 mon.a (mon.0) 209 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:16.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:16 smithi098 bash[79516]: audit 2023-09-23T18:25:15.671393+0000 mon.a (mon.0) 210 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:16.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:16 smithi098 bash[79516]: cephadm 2023-09-23T18:25:15.723084+0000 mgr.y (mgr.26054) 151 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-09-23T18:25:16.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:16 smithi098 bash[79516]: audit 2023-09-23T18:25:15.723725+0000 mon.a (mon.0) 211 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T18:25:16.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:16 smithi098 bash[79516]: audit 2023-09-23T18:25:15.725434+0000 mon.a (mon.0) 212 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:16.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:16 smithi098 bash[79516]: cephadm 2023-09-23T18:25:15.729935+0000 mgr.y (mgr.26054) 152 : cephadm 1 Reconfiguring daemon osd.2 on smithi098 2023-09-23T18:25:16.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:16 smithi098 bash[76817]: cluster 2023-09-23T18:25:15.521708+0000 mgr.y (mgr.26054) 150 : cluster 0 pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:16.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:16 smithi098 bash[76817]: audit 2023-09-23T18:25:15.662396+0000 mon.a (mon.0) 209 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:16.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:16 smithi098 bash[76817]: audit 2023-09-23T18:25:15.671393+0000 mon.a (mon.0) 210 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:16.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:16 smithi098 bash[76817]: cephadm 2023-09-23T18:25:15.723084+0000 mgr.y (mgr.26054) 151 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-09-23T18:25:16.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:16 smithi098 bash[76817]: audit 2023-09-23T18:25:15.723725+0000 mon.a (mon.0) 211 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T18:25:16.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:16 smithi098 bash[76817]: audit 2023-09-23T18:25:15.725434+0000 mon.a (mon.0) 212 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:16.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:16 smithi098 bash[76817]: cephadm 2023-09-23T18:25:15.729935+0000 mgr.y (mgr.26054) 152 : cephadm 1 Reconfiguring daemon osd.2 on smithi098 2023-09-23T18:25:17.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:16 smithi173 bash[59422]: cluster 2023-09-23T18:25:15.521708+0000 mgr.y (mgr.26054) 150 : cluster 0 pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:17.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:16 smithi173 bash[59422]: audit 2023-09-23T18:25:15.662396+0000 mon.a (mon.0) 209 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:17.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:16 smithi173 bash[59422]: audit 2023-09-23T18:25:15.671393+0000 mon.a (mon.0) 210 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:17.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:16 smithi173 bash[59422]: cephadm 2023-09-23T18:25:15.723084+0000 mgr.y (mgr.26054) 151 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-09-23T18:25:17.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:16 smithi173 bash[59422]: audit 2023-09-23T18:25:15.723725+0000 mon.a (mon.0) 211 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T18:25:17.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:16 smithi173 bash[59422]: audit 2023-09-23T18:25:15.725434+0000 mon.a (mon.0) 212 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:17.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:16 smithi173 bash[59422]: cephadm 2023-09-23T18:25:15.729935+0000 mgr.y (mgr.26054) 152 : cephadm 1 Reconfiguring daemon osd.2 on smithi098 2023-09-23T18:25:18.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:18 smithi098 bash[76817]: cluster 2023-09-23T18:25:17.522369+0000 mgr.y (mgr.26054) 153 : cluster 0 pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:18.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:18 smithi098 bash[76817]: audit 2023-09-23T18:25:17.917368+0000 mon.a (mon.0) 213 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:18.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:18 smithi098 bash[76817]: audit 2023-09-23T18:25:17.925770+0000 mon.a (mon.0) 214 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:18.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:18 smithi098 bash[76817]: audit 2023-09-23T18:25:17.949267+0000 mon.a (mon.0) 215 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T18:25:18.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:18 smithi098 bash[76817]: audit 2023-09-23T18:25:17.950988+0000 mon.a (mon.0) 216 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:18.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:18 smithi098 bash[79516]: cluster 2023-09-23T18:25:17.522369+0000 mgr.y (mgr.26054) 153 : cluster 0 pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:18.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:18 smithi098 bash[79516]: audit 2023-09-23T18:25:17.917368+0000 mon.a (mon.0) 213 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:18.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:18 smithi098 bash[79516]: audit 2023-09-23T18:25:17.925770+0000 mon.a (mon.0) 214 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:18.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:18 smithi098 bash[79516]: audit 2023-09-23T18:25:17.949267+0000 mon.a (mon.0) 215 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T18:25:18.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:18 smithi098 bash[79516]: audit 2023-09-23T18:25:17.950988+0000 mon.a (mon.0) 216 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:19.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:18 smithi173 bash[59422]: cluster 2023-09-23T18:25:17.522369+0000 mgr.y (mgr.26054) 153 : cluster 0 pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:19.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:18 smithi173 bash[59422]: audit 2023-09-23T18:25:17.917368+0000 mon.a (mon.0) 213 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:19.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:18 smithi173 bash[59422]: audit 2023-09-23T18:25:17.925770+0000 mon.a (mon.0) 214 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:19.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:18 smithi173 bash[59422]: audit 2023-09-23T18:25:17.949267+0000 mon.a (mon.0) 215 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T18:25:19.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:18 smithi173 bash[59422]: audit 2023-09-23T18:25:17.950988+0000 mon.a (mon.0) 216 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:19.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:19 smithi098 bash[76817]: cephadm 2023-09-23T18:25:17.948720+0000 mgr.y (mgr.26054) 154 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-09-23T18:25:19.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:19 smithi098 bash[76817]: cephadm 2023-09-23T18:25:17.955119+0000 mgr.y (mgr.26054) 155 : cephadm 1 Reconfiguring daemon osd.3 on smithi098 2023-09-23T18:25:19.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:19 smithi098 bash[79516]: cephadm 2023-09-23T18:25:17.948720+0000 mgr.y (mgr.26054) 154 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-09-23T18:25:19.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:19 smithi098 bash[79516]: cephadm 2023-09-23T18:25:17.955119+0000 mgr.y (mgr.26054) 155 : cephadm 1 Reconfiguring daemon osd.3 on smithi098 2023-09-23T18:25:20.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:19 smithi173 bash[59422]: cephadm 2023-09-23T18:25:17.948720+0000 mgr.y (mgr.26054) 154 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-09-23T18:25:20.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:19 smithi173 bash[59422]: cephadm 2023-09-23T18:25:17.955119+0000 mgr.y (mgr.26054) 155 : cephadm 1 Reconfiguring daemon osd.3 on smithi098 2023-09-23T18:25:20.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:20 smithi098 bash[76817]: cluster 2023-09-23T18:25:19.522959+0000 mgr.y (mgr.26054) 156 : cluster 0 pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:20.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:20 smithi098 bash[76817]: audit 2023-09-23T18:25:20.394693+0000 mon.a (mon.0) 217 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:20.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:20 smithi098 bash[76817]: audit 2023-09-23T18:25:20.401996+0000 mon.a (mon.0) 218 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:20.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:20 smithi098 bash[76817]: audit 2023-09-23T18:25:20.425711+0000 mon.a (mon.0) 219 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi098.lowjqo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T18:25:20.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:20 smithi098 bash[76817]: audit 2023-09-23T18:25:20.427724+0000 mon.a (mon.0) 220 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:20.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:20 smithi098 bash[79516]: cluster 2023-09-23T18:25:19.522959+0000 mgr.y (mgr.26054) 156 : cluster 0 pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:20.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:20 smithi098 bash[79516]: audit 2023-09-23T18:25:20.394693+0000 mon.a (mon.0) 217 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:20.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:20 smithi098 bash[79516]: audit 2023-09-23T18:25:20.401996+0000 mon.a (mon.0) 218 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:20.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:20 smithi098 bash[79516]: audit 2023-09-23T18:25:20.425711+0000 mon.a (mon.0) 219 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi098.lowjqo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T18:25:20.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:20 smithi098 bash[79516]: audit 2023-09-23T18:25:20.427724+0000 mon.a (mon.0) 220 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:21.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:20 smithi173 bash[59422]: cluster 2023-09-23T18:25:19.522959+0000 mgr.y (mgr.26054) 156 : cluster 0 pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:21.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:20 smithi173 bash[59422]: audit 2023-09-23T18:25:20.394693+0000 mon.a (mon.0) 217 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:21.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:20 smithi173 bash[59422]: audit 2023-09-23T18:25:20.401996+0000 mon.a (mon.0) 218 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:21.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:20 smithi173 bash[59422]: audit 2023-09-23T18:25:20.425711+0000 mon.a (mon.0) 219 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi098.lowjqo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T18:25:21.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:20 smithi173 bash[59422]: audit 2023-09-23T18:25:20.427724+0000 mon.a (mon.0) 220 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:21.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:21 smithi098 bash[76817]: cephadm 2023-09-23T18:25:20.424954+0000 mgr.y (mgr.26054) 157 : cephadm 1 Reconfiguring mds.foo.smithi098.lowjqo (monmap changed)... 2023-09-23T18:25:21.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:21 smithi098 bash[76817]: cephadm 2023-09-23T18:25:20.429881+0000 mgr.y (mgr.26054) 158 : cephadm 1 Reconfiguring daemon mds.foo.smithi098.lowjqo on smithi098 2023-09-23T18:25:21.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:21 smithi098 bash[79516]: cephadm 2023-09-23T18:25:20.424954+0000 mgr.y (mgr.26054) 157 : cephadm 1 Reconfiguring mds.foo.smithi098.lowjqo (monmap changed)... 2023-09-23T18:25:21.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:21 smithi098 bash[79516]: cephadm 2023-09-23T18:25:20.429881+0000 mgr.y (mgr.26054) 158 : cephadm 1 Reconfiguring daemon mds.foo.smithi098.lowjqo on smithi098 2023-09-23T18:25:22.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:21 smithi173 bash[59422]: cephadm 2023-09-23T18:25:20.424954+0000 mgr.y (mgr.26054) 157 : cephadm 1 Reconfiguring mds.foo.smithi098.lowjqo (monmap changed)... 2023-09-23T18:25:22.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:21 smithi173 bash[59422]: cephadm 2023-09-23T18:25:20.429881+0000 mgr.y (mgr.26054) 158 : cephadm 1 Reconfiguring daemon mds.foo.smithi098.lowjqo on smithi098 2023-09-23T18:25:22.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:22 smithi098 bash[79516]: cluster 2023-09-23T18:25:21.524055+0000 mgr.y (mgr.26054) 159 : cluster 0 pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:22.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:22 smithi098 bash[79516]: audit 2023-09-23T18:25:22.619197+0000 mon.a (mon.0) 221 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:22.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:22 smithi098 bash[79516]: audit 2023-09-23T18:25:22.628553+0000 mon.a (mon.0) 222 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:22.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:22 smithi098 bash[79516]: audit 2023-09-23T18:25:22.649290+0000 mon.a (mon.0) 223 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:25:22.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:22 smithi098 bash[79516]: audit 2023-09-23T18:25:22.650831+0000 mon.a (mon.0) 224 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:25:22.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:22 smithi098 bash[79516]: audit 2023-09-23T18:25:22.652227+0000 mon.a (mon.0) 225 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:22.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:22 smithi098 bash[76817]: cluster 2023-09-23T18:25:21.524055+0000 mgr.y (mgr.26054) 159 : cluster 0 pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:22.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:22 smithi098 bash[76817]: audit 2023-09-23T18:25:22.619197+0000 mon.a (mon.0) 221 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:22.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:22 smithi098 bash[76817]: audit 2023-09-23T18:25:22.628553+0000 mon.a (mon.0) 222 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:22.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:22 smithi098 bash[76817]: audit 2023-09-23T18:25:22.649290+0000 mon.a (mon.0) 223 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:25:22.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:22 smithi098 bash[76817]: audit 2023-09-23T18:25:22.650831+0000 mon.a (mon.0) 224 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:25:22.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:22 smithi098 bash[76817]: audit 2023-09-23T18:25:22.652227+0000 mon.a (mon.0) 225 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:22.959 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:22 smithi173 bash[59422]: cluster 2023-09-23T18:25:21.524055+0000 mgr.y (mgr.26054) 159 : cluster 0 pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:22.959 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:22 smithi173 bash[59422]: audit 2023-09-23T18:25:22.619197+0000 mon.a (mon.0) 221 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:22.959 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:22 smithi173 bash[59422]: audit 2023-09-23T18:25:22.628553+0000 mon.a (mon.0) 222 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:22.959 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:22 smithi173 bash[59422]: audit 2023-09-23T18:25:22.649290+0000 mon.a (mon.0) 223 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T18:25:22.959 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:22 smithi173 bash[59422]: audit 2023-09-23T18:25:22.650831+0000 mon.a (mon.0) 224 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T18:25:22.959 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:22 smithi173 bash[59422]: audit 2023-09-23T18:25:22.652227+0000 mon.a (mon.0) 225 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:23.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:23 smithi098 bash[76817]: cephadm 2023-09-23T18:25:22.648647+0000 mgr.y (mgr.26054) 160 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-09-23T18:25:23.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:23 smithi098 bash[76817]: cephadm 2023-09-23T18:25:22.654020+0000 mgr.y (mgr.26054) 161 : cephadm 1 Reconfiguring daemon mon.b on smithi173 2023-09-23T18:25:23.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:23 smithi098 bash[79516]: cephadm 2023-09-23T18:25:22.648647+0000 mgr.y (mgr.26054) 160 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-09-23T18:25:23.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:23 smithi098 bash[79516]: cephadm 2023-09-23T18:25:22.654020+0000 mgr.y (mgr.26054) 161 : cephadm 1 Reconfiguring daemon mon.b on smithi173 2023-09-23T18:25:24.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:23 smithi173 bash[59422]: cephadm 2023-09-23T18:25:22.648647+0000 mgr.y (mgr.26054) 160 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-09-23T18:25:24.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:23 smithi173 bash[59422]: cephadm 2023-09-23T18:25:22.654020+0000 mgr.y (mgr.26054) 161 : cephadm 1 Reconfiguring daemon mon.b on smithi173 2023-09-23T18:25:25.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:24 smithi173 bash[59422]: cluster 2023-09-23T18:25:23.524712+0000 mgr.y (mgr.26054) 162 : cluster 0 pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:25.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:24 smithi173 bash[59422]: audit 2023-09-23T18:25:24.569680+0000 mon.a (mon.0) 226 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:25:25.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:24 smithi098 bash[76817]: cluster 2023-09-23T18:25:23.524712+0000 mgr.y (mgr.26054) 162 : cluster 0 pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:25.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:24 smithi098 bash[76817]: audit 2023-09-23T18:25:24.569680+0000 mon.a (mon.0) 226 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:25:25.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:24 smithi098 bash[79516]: cluster 2023-09-23T18:25:23.524712+0000 mgr.y (mgr.26054) 162 : cluster 0 pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:25.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:24 smithi098 bash[79516]: audit 2023-09-23T18:25:24.569680+0000 mon.a (mon.0) 226 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:25:26.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:26 smithi173 bash[59422]: audit 2023-09-23T18:25:25.011628+0000 mon.a (mon.0) 227 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:26.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:26 smithi173 bash[59422]: audit 2023-09-23T18:25:25.019252+0000 mon.a (mon.0) 228 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:26.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:26 smithi173 bash[59422]: cephadm 2023-09-23T18:25:25.025782+0000 mgr.y (mgr.26054) 163 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-09-23T18:25:26.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:26 smithi173 bash[59422]: audit 2023-09-23T18:25:25.026329+0000 mon.a (mon.0) 229 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:25:26.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:26 smithi173 bash[59422]: audit 2023-09-23T18:25:25.027786+0000 mon.a (mon.0) 230 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:25:26.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:26 smithi173 bash[59422]: audit 2023-09-23T18:25:25.028997+0000 mon.a (mon.0) 231 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:26.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:26 smithi173 bash[59422]: cephadm 2023-09-23T18:25:25.030494+0000 mgr.y (mgr.26054) 164 : cephadm 1 Reconfiguring daemon mgr.x on smithi173 2023-09-23T18:25:26.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:26 smithi173 bash[59422]: cluster 2023-09-23T18:25:25.525685+0000 mgr.y (mgr.26054) 165 : cluster 0 pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:26.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:25:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:25:26] "GET /metrics HTTP/1.1" 200 33909 "" "Prometheus/2.33.4" 2023-09-23T18:25:26.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:26 smithi098 bash[76817]: audit 2023-09-23T18:25:25.011628+0000 mon.a (mon.0) 227 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:26.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:26 smithi098 bash[76817]: audit 2023-09-23T18:25:25.019252+0000 mon.a (mon.0) 228 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:26.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:26 smithi098 bash[76817]: cephadm 2023-09-23T18:25:25.025782+0000 mgr.y (mgr.26054) 163 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-09-23T18:25:26.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:26 smithi098 bash[76817]: audit 2023-09-23T18:25:25.026329+0000 mon.a (mon.0) 229 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:25:26.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:26 smithi098 bash[76817]: audit 2023-09-23T18:25:25.027786+0000 mon.a (mon.0) 230 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:25:26.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:26 smithi098 bash[76817]: audit 2023-09-23T18:25:25.028997+0000 mon.a (mon.0) 231 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:26.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:26 smithi098 bash[76817]: cephadm 2023-09-23T18:25:25.030494+0000 mgr.y (mgr.26054) 164 : cephadm 1 Reconfiguring daemon mgr.x on smithi173 2023-09-23T18:25:26.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:26 smithi098 bash[76817]: cluster 2023-09-23T18:25:25.525685+0000 mgr.y (mgr.26054) 165 : cluster 0 pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:26.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:26 smithi098 bash[79516]: audit 2023-09-23T18:25:25.011628+0000 mon.a (mon.0) 227 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:26.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:26 smithi098 bash[79516]: audit 2023-09-23T18:25:25.019252+0000 mon.a (mon.0) 228 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:26.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:26 smithi098 bash[79516]: cephadm 2023-09-23T18:25:25.025782+0000 mgr.y (mgr.26054) 163 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-09-23T18:25:26.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:26 smithi098 bash[79516]: audit 2023-09-23T18:25:25.026329+0000 mon.a (mon.0) 229 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T18:25:26.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:26 smithi098 bash[79516]: audit 2023-09-23T18:25:25.027786+0000 mon.a (mon.0) 230 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T18:25:26.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:26 smithi098 bash[79516]: audit 2023-09-23T18:25:25.028997+0000 mon.a (mon.0) 231 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:26.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:26 smithi098 bash[79516]: cephadm 2023-09-23T18:25:25.030494+0000 mgr.y (mgr.26054) 164 : cephadm 1 Reconfiguring daemon mgr.x on smithi173 2023-09-23T18:25:26.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:26 smithi098 bash[79516]: cluster 2023-09-23T18:25:25.525685+0000 mgr.y (mgr.26054) 165 : cluster 0 pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:28.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:28 smithi173 bash[59422]: audit 2023-09-23T18:25:27.255260+0000 mon.a (mon.0) 232 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:28.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:28 smithi173 bash[59422]: audit 2023-09-23T18:25:27.263346+0000 mon.a (mon.0) 233 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:28.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:28 smithi173 bash[59422]: cephadm 2023-09-23T18:25:27.300999+0000 mgr.y (mgr.26054) 166 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-09-23T18:25:28.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:28 smithi173 bash[59422]: audit 2023-09-23T18:25:27.301639+0000 mon.a (mon.0) 234 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T18:25:28.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:28 smithi173 bash[59422]: audit 2023-09-23T18:25:27.303389+0000 mon.a (mon.0) 235 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:28.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:28 smithi173 bash[59422]: cephadm 2023-09-23T18:25:27.307707+0000 mgr.y (mgr.26054) 167 : cephadm 1 Reconfiguring daemon osd.4 on smithi173 2023-09-23T18:25:28.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:28 smithi173 bash[59422]: cluster 2023-09-23T18:25:27.526317+0000 mgr.y (mgr.26054) 168 : cluster 0 pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:28.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:28 smithi098 bash[79516]: audit 2023-09-23T18:25:27.255260+0000 mon.a (mon.0) 232 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:28.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:28 smithi098 bash[79516]: audit 2023-09-23T18:25:27.263346+0000 mon.a (mon.0) 233 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:28.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:28 smithi098 bash[79516]: cephadm 2023-09-23T18:25:27.300999+0000 mgr.y (mgr.26054) 166 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-09-23T18:25:28.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:28 smithi098 bash[79516]: audit 2023-09-23T18:25:27.301639+0000 mon.a (mon.0) 234 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T18:25:28.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:28 smithi098 bash[79516]: audit 2023-09-23T18:25:27.303389+0000 mon.a (mon.0) 235 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:28.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:28 smithi098 bash[79516]: cephadm 2023-09-23T18:25:27.307707+0000 mgr.y (mgr.26054) 167 : cephadm 1 Reconfiguring daemon osd.4 on smithi173 2023-09-23T18:25:28.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:28 smithi098 bash[79516]: cluster 2023-09-23T18:25:27.526317+0000 mgr.y (mgr.26054) 168 : cluster 0 pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:28.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:28 smithi098 bash[76817]: audit 2023-09-23T18:25:27.255260+0000 mon.a (mon.0) 232 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:28.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:28 smithi098 bash[76817]: audit 2023-09-23T18:25:27.263346+0000 mon.a (mon.0) 233 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:28.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:28 smithi098 bash[76817]: cephadm 2023-09-23T18:25:27.300999+0000 mgr.y (mgr.26054) 166 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-09-23T18:25:28.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:28 smithi098 bash[76817]: audit 2023-09-23T18:25:27.301639+0000 mon.a (mon.0) 234 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T18:25:28.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:28 smithi098 bash[76817]: audit 2023-09-23T18:25:27.303389+0000 mon.a (mon.0) 235 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:28.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:28 smithi098 bash[76817]: cephadm 2023-09-23T18:25:27.307707+0000 mgr.y (mgr.26054) 167 : cephadm 1 Reconfiguring daemon osd.4 on smithi173 2023-09-23T18:25:28.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:28 smithi098 bash[76817]: cluster 2023-09-23T18:25:27.526317+0000 mgr.y (mgr.26054) 168 : cluster 0 pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:30.931 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:30 smithi173 bash[59422]: cluster 2023-09-23T18:25:29.526947+0000 mgr.y (mgr.26054) 169 : cluster 0 pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:30.932 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:30 smithi173 bash[59422]: audit 2023-09-23T18:25:29.667886+0000 mon.a (mon.0) 236 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:30.932 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:30 smithi173 bash[59422]: audit 2023-09-23T18:25:29.675679+0000 mon.a (mon.0) 237 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:30.932 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:30 smithi173 bash[59422]: cephadm 2023-09-23T18:25:29.677289+0000 mgr.y (mgr.26054) 170 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-09-23T18:25:30.932 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:30 smithi173 bash[59422]: audit 2023-09-23T18:25:29.677725+0000 mon.a (mon.0) 238 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T18:25:30.932 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:30 smithi173 bash[59422]: audit 2023-09-23T18:25:29.679140+0000 mon.a (mon.0) 239 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:30.932 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:30 smithi173 bash[59422]: cephadm 2023-09-23T18:25:29.681947+0000 mgr.y (mgr.26054) 171 : cephadm 1 Reconfiguring daemon osd.5 on smithi173 2023-09-23T18:25:30.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:30 smithi098 bash[76817]: cluster 2023-09-23T18:25:29.526947+0000 mgr.y (mgr.26054) 169 : cluster 0 pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:30.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:30 smithi098 bash[76817]: audit 2023-09-23T18:25:29.667886+0000 mon.a (mon.0) 236 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:30.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:30 smithi098 bash[76817]: audit 2023-09-23T18:25:29.675679+0000 mon.a (mon.0) 237 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:30.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:30 smithi098 bash[76817]: cephadm 2023-09-23T18:25:29.677289+0000 mgr.y (mgr.26054) 170 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-09-23T18:25:30.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:30 smithi098 bash[76817]: audit 2023-09-23T18:25:29.677725+0000 mon.a (mon.0) 238 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T18:25:30.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:30 smithi098 bash[76817]: audit 2023-09-23T18:25:29.679140+0000 mon.a (mon.0) 239 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:30.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:30 smithi098 bash[76817]: cephadm 2023-09-23T18:25:29.681947+0000 mgr.y (mgr.26054) 171 : cephadm 1 Reconfiguring daemon osd.5 on smithi173 2023-09-23T18:25:30.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:30 smithi098 bash[79516]: cluster 2023-09-23T18:25:29.526947+0000 mgr.y (mgr.26054) 169 : cluster 0 pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:30.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:30 smithi098 bash[79516]: audit 2023-09-23T18:25:29.667886+0000 mon.a (mon.0) 236 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:30.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:30 smithi098 bash[79516]: audit 2023-09-23T18:25:29.675679+0000 mon.a (mon.0) 237 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:30.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:30 smithi098 bash[79516]: cephadm 2023-09-23T18:25:29.677289+0000 mgr.y (mgr.26054) 170 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-09-23T18:25:30.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:30 smithi098 bash[79516]: audit 2023-09-23T18:25:29.677725+0000 mon.a (mon.0) 238 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T18:25:30.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:30 smithi098 bash[79516]: audit 2023-09-23T18:25:29.679140+0000 mon.a (mon.0) 239 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:30.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:30 smithi098 bash[79516]: cephadm 2023-09-23T18:25:29.681947+0000 mgr.y (mgr.26054) 171 : cephadm 1 Reconfiguring daemon osd.5 on smithi173 2023-09-23T18:25:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:32 smithi173 bash[59422]: cluster 2023-09-23T18:25:31.527959+0000 mgr.y (mgr.26054) 172 : cluster 0 pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:32 smithi173 bash[59422]: audit 2023-09-23T18:25:31.707431+0000 mon.a (mon.0) 240 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:33.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:32 smithi173 bash[59422]: audit 2023-09-23T18:25:31.717164+0000 mon.a (mon.0) 241 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:33.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:32 smithi173 bash[59422]: cephadm 2023-09-23T18:25:31.750345+0000 mgr.y (mgr.26054) 173 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-09-23T18:25:33.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:32 smithi173 bash[59422]: audit 2023-09-23T18:25:31.750990+0000 mon.a (mon.0) 242 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T18:25:33.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:32 smithi173 bash[59422]: audit 2023-09-23T18:25:31.752794+0000 mon.a (mon.0) 243 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:33.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:32 smithi173 bash[59422]: cephadm 2023-09-23T18:25:31.757033+0000 mgr.y (mgr.26054) 174 : cephadm 1 Reconfiguring daemon osd.6 on smithi173 2023-09-23T18:25:33.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:32 smithi098 bash[79516]: cluster 2023-09-23T18:25:31.527959+0000 mgr.y (mgr.26054) 172 : cluster 0 pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:33.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:32 smithi098 bash[79516]: audit 2023-09-23T18:25:31.707431+0000 mon.a (mon.0) 240 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:33.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:32 smithi098 bash[79516]: audit 2023-09-23T18:25:31.717164+0000 mon.a (mon.0) 241 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:33.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:32 smithi098 bash[79516]: cephadm 2023-09-23T18:25:31.750345+0000 mgr.y (mgr.26054) 173 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-09-23T18:25:33.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:32 smithi098 bash[79516]: audit 2023-09-23T18:25:31.750990+0000 mon.a (mon.0) 242 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T18:25:33.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:32 smithi098 bash[79516]: audit 2023-09-23T18:25:31.752794+0000 mon.a (mon.0) 243 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:33.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:32 smithi098 bash[79516]: cephadm 2023-09-23T18:25:31.757033+0000 mgr.y (mgr.26054) 174 : cephadm 1 Reconfiguring daemon osd.6 on smithi173 2023-09-23T18:25:33.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:32 smithi098 bash[76817]: cluster 2023-09-23T18:25:31.527959+0000 mgr.y (mgr.26054) 172 : cluster 0 pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:33.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:32 smithi098 bash[76817]: audit 2023-09-23T18:25:31.707431+0000 mon.a (mon.0) 240 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:33.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:32 smithi098 bash[76817]: audit 2023-09-23T18:25:31.717164+0000 mon.a (mon.0) 241 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:33.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:32 smithi098 bash[76817]: cephadm 2023-09-23T18:25:31.750345+0000 mgr.y (mgr.26054) 173 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-09-23T18:25:33.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:32 smithi098 bash[76817]: audit 2023-09-23T18:25:31.750990+0000 mon.a (mon.0) 242 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T18:25:33.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:32 smithi098 bash[76817]: audit 2023-09-23T18:25:31.752794+0000 mon.a (mon.0) 243 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:33.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:32 smithi098 bash[76817]: cephadm 2023-09-23T18:25:31.757033+0000 mgr.y (mgr.26054) 174 : cephadm 1 Reconfiguring daemon osd.6 on smithi173 2023-09-23T18:25:35.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:34 smithi173 bash[59422]: cluster 2023-09-23T18:25:33.528591+0000 mgr.y (mgr.26054) 175 : cluster 0 pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:35.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:34 smithi173 bash[59422]: audit 2023-09-23T18:25:33.743652+0000 mon.a (mon.0) 244 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:35.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:34 smithi173 bash[59422]: audit 2023-09-23T18:25:33.754905+0000 mon.a (mon.0) 245 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:35.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:34 smithi173 bash[59422]: cephadm 2023-09-23T18:25:33.775935+0000 mgr.y (mgr.26054) 176 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-09-23T18:25:35.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:34 smithi173 bash[59422]: audit 2023-09-23T18:25:33.776571+0000 mon.a (mon.0) 246 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T18:25:35.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:34 smithi173 bash[59422]: audit 2023-09-23T18:25:33.778247+0000 mon.a (mon.0) 247 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:35.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:34 smithi173 bash[59422]: cephadm 2023-09-23T18:25:33.782686+0000 mgr.y (mgr.26054) 177 : cephadm 1 Reconfiguring daemon osd.7 on smithi173 2023-09-23T18:25:35.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:34 smithi098 bash[76817]: cluster 2023-09-23T18:25:33.528591+0000 mgr.y (mgr.26054) 175 : cluster 0 pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:35.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:34 smithi098 bash[76817]: audit 2023-09-23T18:25:33.743652+0000 mon.a (mon.0) 244 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:35.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:34 smithi098 bash[76817]: audit 2023-09-23T18:25:33.754905+0000 mon.a (mon.0) 245 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:35.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:34 smithi098 bash[76817]: cephadm 2023-09-23T18:25:33.775935+0000 mgr.y (mgr.26054) 176 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-09-23T18:25:35.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:34 smithi098 bash[76817]: audit 2023-09-23T18:25:33.776571+0000 mon.a (mon.0) 246 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T18:25:35.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:34 smithi098 bash[76817]: audit 2023-09-23T18:25:33.778247+0000 mon.a (mon.0) 247 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:35.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:34 smithi098 bash[76817]: cephadm 2023-09-23T18:25:33.782686+0000 mgr.y (mgr.26054) 177 : cephadm 1 Reconfiguring daemon osd.7 on smithi173 2023-09-23T18:25:35.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:34 smithi098 bash[79516]: cluster 2023-09-23T18:25:33.528591+0000 mgr.y (mgr.26054) 175 : cluster 0 pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:35.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:34 smithi098 bash[79516]: audit 2023-09-23T18:25:33.743652+0000 mon.a (mon.0) 244 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:35.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:34 smithi098 bash[79516]: audit 2023-09-23T18:25:33.754905+0000 mon.a (mon.0) 245 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:35.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:34 smithi098 bash[79516]: cephadm 2023-09-23T18:25:33.775935+0000 mgr.y (mgr.26054) 176 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-09-23T18:25:35.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:34 smithi098 bash[79516]: audit 2023-09-23T18:25:33.776571+0000 mon.a (mon.0) 246 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T18:25:35.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:34 smithi098 bash[79516]: audit 2023-09-23T18:25:33.778247+0000 mon.a (mon.0) 247 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:35.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:34 smithi098 bash[79516]: cephadm 2023-09-23T18:25:33.782686+0000 mgr.y (mgr.26054) 177 : cephadm 1 Reconfiguring daemon osd.7 on smithi173 2023-09-23T18:25:36.443 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:25:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:25:36] "GET /metrics HTTP/1.1" 200 33909 "" "Prometheus/2.33.4" 2023-09-23T18:25:37.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:36 smithi173 bash[59422]: cluster 2023-09-23T18:25:35.529597+0000 mgr.y (mgr.26054) 178 : cluster 0 pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:37.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:36 smithi173 bash[59422]: audit 2023-09-23T18:25:35.888296+0000 mon.a (mon.0) 248 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:37.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:36 smithi173 bash[59422]: audit 2023-09-23T18:25:35.895913+0000 mon.a (mon.0) 249 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:37.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:36 smithi173 bash[59422]: audit 2023-09-23T18:25:35.903890+0000 mon.a (mon.0) 250 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi173.qjefje", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T18:25:37.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:36 smithi173 bash[59422]: audit 2023-09-23T18:25:35.905310+0000 mon.a (mon.0) 251 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:37.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:36 smithi098 bash[76817]: cluster 2023-09-23T18:25:35.529597+0000 mgr.y (mgr.26054) 178 : cluster 0 pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:37.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:36 smithi098 bash[76817]: audit 2023-09-23T18:25:35.888296+0000 mon.a (mon.0) 248 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:37.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:36 smithi098 bash[76817]: audit 2023-09-23T18:25:35.895913+0000 mon.a (mon.0) 249 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:37.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:36 smithi098 bash[76817]: audit 2023-09-23T18:25:35.903890+0000 mon.a (mon.0) 250 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi173.qjefje", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T18:25:37.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:36 smithi098 bash[76817]: audit 2023-09-23T18:25:35.905310+0000 mon.a (mon.0) 251 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:37.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:36 smithi098 bash[79516]: cluster 2023-09-23T18:25:35.529597+0000 mgr.y (mgr.26054) 178 : cluster 0 pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:37.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:36 smithi098 bash[79516]: audit 2023-09-23T18:25:35.888296+0000 mon.a (mon.0) 248 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:37.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:36 smithi098 bash[79516]: audit 2023-09-23T18:25:35.895913+0000 mon.a (mon.0) 249 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:37.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:36 smithi098 bash[79516]: audit 2023-09-23T18:25:35.903890+0000 mon.a (mon.0) 250 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi173.qjefje", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T18:25:37.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:36 smithi098 bash[79516]: audit 2023-09-23T18:25:35.905310+0000 mon.a (mon.0) 251 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:38.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:37 smithi173 bash[59422]: cephadm 2023-09-23T18:25:35.903433+0000 mgr.y (mgr.26054) 179 : cephadm 1 Reconfiguring mds.foo.smithi173.qjefje (monmap changed)... 2023-09-23T18:25:38.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:37 smithi173 bash[59422]: cephadm 2023-09-23T18:25:35.906624+0000 mgr.y (mgr.26054) 180 : cephadm 1 Reconfiguring daemon mds.foo.smithi173.qjefje on smithi173 2023-09-23T18:25:38.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:37 smithi098 bash[79516]: cephadm 2023-09-23T18:25:35.903433+0000 mgr.y (mgr.26054) 179 : cephadm 1 Reconfiguring mds.foo.smithi173.qjefje (monmap changed)... 2023-09-23T18:25:38.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:37 smithi098 bash[79516]: cephadm 2023-09-23T18:25:35.906624+0000 mgr.y (mgr.26054) 180 : cephadm 1 Reconfiguring daemon mds.foo.smithi173.qjefje on smithi173 2023-09-23T18:25:38.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:37 smithi098 bash[76817]: cephadm 2023-09-23T18:25:35.903433+0000 mgr.y (mgr.26054) 179 : cephadm 1 Reconfiguring mds.foo.smithi173.qjefje (monmap changed)... 2023-09-23T18:25:38.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:37 smithi098 bash[76817]: cephadm 2023-09-23T18:25:35.906624+0000 mgr.y (mgr.26054) 180 : cephadm 1 Reconfiguring daemon mds.foo.smithi173.qjefje on smithi173 2023-09-23T18:25:39.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:38 smithi173 bash[59422]: cluster 2023-09-23T18:25:37.530191+0000 mgr.y (mgr.26054) 181 : cluster 0 pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:39.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:38 smithi173 bash[59422]: audit 2023-09-23T18:25:37.874517+0000 mon.a (mon.0) 252 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:39.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:38 smithi173 bash[59422]: audit 2023-09-23T18:25:37.884963+0000 mon.a (mon.0) 253 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:39.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:38 smithi173 bash[59422]: audit 2023-09-23T18:25:37.936095+0000 mon.a (mon.0) 254 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:25:39.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:38 smithi173 bash[59422]: audit 2023-09-23T18:25:38.312837+0000 mon.a (mon.0) 255 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:39.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:38 smithi173 bash[59422]: audit 2023-09-23T18:25:38.323331+0000 mon.a (mon.0) 256 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:39.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:38 smithi098 bash[76817]: cluster 2023-09-23T18:25:37.530191+0000 mgr.y (mgr.26054) 181 : cluster 0 pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:39.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:38 smithi098 bash[76817]: audit 2023-09-23T18:25:37.874517+0000 mon.a (mon.0) 252 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:39.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:38 smithi098 bash[76817]: audit 2023-09-23T18:25:37.884963+0000 mon.a (mon.0) 253 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:39.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:38 smithi098 bash[76817]: audit 2023-09-23T18:25:37.936095+0000 mon.a (mon.0) 254 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:25:39.235 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:38 smithi098 bash[76817]: audit 2023-09-23T18:25:38.312837+0000 mon.a (mon.0) 255 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:39.235 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:38 smithi098 bash[76817]: audit 2023-09-23T18:25:38.323331+0000 mon.a (mon.0) 256 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:39.235 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:38 smithi098 bash[79516]: cluster 2023-09-23T18:25:37.530191+0000 mgr.y (mgr.26054) 181 : cluster 0 pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:39.235 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:38 smithi098 bash[79516]: audit 2023-09-23T18:25:37.874517+0000 mon.a (mon.0) 252 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:39.235 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:38 smithi098 bash[79516]: audit 2023-09-23T18:25:37.884963+0000 mon.a (mon.0) 253 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:39.235 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:38 smithi098 bash[79516]: audit 2023-09-23T18:25:37.936095+0000 mon.a (mon.0) 254 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:25:39.236 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:38 smithi098 bash[79516]: audit 2023-09-23T18:25:38.312837+0000 mon.a (mon.0) 255 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:39.236 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:38 smithi098 bash[79516]: audit 2023-09-23T18:25:38.323331+0000 mon.a (mon.0) 256 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:40.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:40 smithi098 bash[76817]: cluster 2023-09-23T18:25:39.530842+0000 mgr.y (mgr.26054) 182 : cluster 0 pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:40.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:40 smithi098 bash[76817]: audit 2023-09-23T18:25:39.570098+0000 mon.a (mon.0) 257 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:25:40.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:40 smithi098 bash[79516]: cluster 2023-09-23T18:25:39.530842+0000 mgr.y (mgr.26054) 182 : cluster 0 pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:40.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:40 smithi098 bash[79516]: audit 2023-09-23T18:25:39.570098+0000 mon.a (mon.0) 257 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:25:40.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:40 smithi173 bash[59422]: cluster 2023-09-23T18:25:39.530842+0000 mgr.y (mgr.26054) 182 : cluster 0 pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:40.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:40 smithi173 bash[59422]: audit 2023-09-23T18:25:39.570098+0000 mon.a (mon.0) 257 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:25:42.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:42 smithi098 bash[76817]: cluster 2023-09-23T18:25:41.531862+0000 mgr.y (mgr.26054) 183 : cluster 0 pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:42.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:42 smithi098 bash[79516]: cluster 2023-09-23T18:25:41.531862+0000 mgr.y (mgr.26054) 183 : cluster 0 pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:43.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:42 smithi173 bash[59422]: cluster 2023-09-23T18:25:41.531862+0000 mgr.y (mgr.26054) 183 : cluster 0 pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:44.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:44 smithi098 bash[76817]: cluster 2023-09-23T18:25:43.532439+0000 mgr.y (mgr.26054) 184 : cluster 0 pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:44.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:44 smithi098 bash[76817]: audit 2023-09-23T18:25:44.377189+0000 mon.a (mon.0) 258 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:44.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:44 smithi098 bash[76817]: audit 2023-09-23T18:25:44.385446+0000 mon.a (mon.0) 259 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:44.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:44 smithi098 bash[79516]: cluster 2023-09-23T18:25:43.532439+0000 mgr.y (mgr.26054) 184 : cluster 0 pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:44.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:44 smithi098 bash[79516]: audit 2023-09-23T18:25:44.377189+0000 mon.a (mon.0) 258 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:44.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:44 smithi098 bash[79516]: audit 2023-09-23T18:25:44.385446+0000 mon.a (mon.0) 259 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:45.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:44 smithi173 bash[59422]: cluster 2023-09-23T18:25:43.532439+0000 mgr.y (mgr.26054) 184 : cluster 0 pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:45.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:44 smithi173 bash[59422]: audit 2023-09-23T18:25:44.377189+0000 mon.a (mon.0) 258 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:45.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:44 smithi173 bash[59422]: audit 2023-09-23T18:25:44.385446+0000 mon.a (mon.0) 259 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:45.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:45 smithi098 bash[76817]: audit 2023-09-23T18:25:44.863328+0000 mon.a (mon.0) 260 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:45.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:45 smithi098 bash[76817]: audit 2023-09-23T18:25:44.865146+0000 mon.a (mon.0) 261 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:25:45.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:45 smithi098 bash[76817]: audit 2023-09-23T18:25:44.873652+0000 mon.a (mon.0) 262 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:45.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:45 smithi098 bash[79516]: audit 2023-09-23T18:25:44.863328+0000 mon.a (mon.0) 260 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:45.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:45 smithi098 bash[79516]: audit 2023-09-23T18:25:44.865146+0000 mon.a (mon.0) 261 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:25:45.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:45 smithi098 bash[79516]: audit 2023-09-23T18:25:44.873652+0000 mon.a (mon.0) 262 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:46.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:45 smithi173 bash[59422]: audit 2023-09-23T18:25:44.863328+0000 mon.a (mon.0) 260 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:25:46.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:45 smithi173 bash[59422]: audit 2023-09-23T18:25:44.865146+0000 mon.a (mon.0) 261 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:25:46.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:45 smithi173 bash[59422]: audit 2023-09-23T18:25:44.873652+0000 mon.a (mon.0) 262 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:25:46.443 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:25:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:25:46] "GET /metrics HTTP/1.1" 200 33910 "" "Prometheus/2.33.4" 2023-09-23T18:25:46.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:46 smithi098 bash[76817]: cluster 2023-09-23T18:25:45.533464+0000 mgr.y (mgr.26054) 185 : cluster 0 pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:46.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:46 smithi098 bash[79516]: cluster 2023-09-23T18:25:45.533464+0000 mgr.y (mgr.26054) 185 : cluster 0 pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:47.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:46 smithi173 bash[59422]: cluster 2023-09-23T18:25:45.533464+0000 mgr.y (mgr.26054) 185 : cluster 0 pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:48.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:48 smithi098 bash[76817]: cluster 2023-09-23T18:25:47.534096+0000 mgr.y (mgr.26054) 186 : cluster 0 pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:48.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:48 smithi098 bash[79516]: cluster 2023-09-23T18:25:47.534096+0000 mgr.y (mgr.26054) 186 : cluster 0 pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:49.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:48 smithi173 bash[59422]: cluster 2023-09-23T18:25:47.534096+0000 mgr.y (mgr.26054) 186 : cluster 0 pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:50.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:50 smithi098 bash[76817]: cluster 2023-09-23T18:25:49.534729+0000 mgr.y (mgr.26054) 187 : cluster 0 pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:50.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:50 smithi098 bash[79516]: cluster 2023-09-23T18:25:49.534729+0000 mgr.y (mgr.26054) 187 : cluster 0 pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:50.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:50 smithi173 bash[59422]: cluster 2023-09-23T18:25:49.534729+0000 mgr.y (mgr.26054) 187 : cluster 0 pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:52.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:52 smithi098 bash[79516]: cluster 2023-09-23T18:25:51.535775+0000 mgr.y (mgr.26054) 188 : cluster 0 pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:52.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:52 smithi098 bash[76817]: cluster 2023-09-23T18:25:51.535775+0000 mgr.y (mgr.26054) 188 : cluster 0 pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:52 smithi173 bash[59422]: cluster 2023-09-23T18:25:51.535775+0000 mgr.y (mgr.26054) 188 : cluster 0 pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:54.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:54 smithi098 bash[76817]: cluster 2023-09-23T18:25:53.536289+0000 mgr.y (mgr.26054) 189 : cluster 0 pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:54.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:54 smithi098 bash[76817]: audit 2023-09-23T18:25:54.570305+0000 mon.a (mon.0) 263 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:25:54.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:54 smithi098 bash[79516]: cluster 2023-09-23T18:25:53.536289+0000 mgr.y (mgr.26054) 189 : cluster 0 pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:54.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:54 smithi098 bash[79516]: audit 2023-09-23T18:25:54.570305+0000 mon.a (mon.0) 263 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:25:55.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:54 smithi173 bash[59422]: cluster 2023-09-23T18:25:53.536289+0000 mgr.y (mgr.26054) 189 : cluster 0 pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:55.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:54 smithi173 bash[59422]: audit 2023-09-23T18:25:54.570305+0000 mon.a (mon.0) 263 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:25:56.443 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:25:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:25:56] "GET /metrics HTTP/1.1" 200 33908 "" "Prometheus/2.33.4" 2023-09-23T18:25:56.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:56 smithi098 bash[79516]: cluster 2023-09-23T18:25:55.537253+0000 mgr.y (mgr.26054) 190 : cluster 0 pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:56.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:56 smithi098 bash[76817]: cluster 2023-09-23T18:25:55.537253+0000 mgr.y (mgr.26054) 190 : cluster 0 pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:57.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:56 smithi173 bash[59422]: cluster 2023-09-23T18:25:55.537253+0000 mgr.y (mgr.26054) 190 : cluster 0 pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:58.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:25:58 smithi098 bash[76817]: cluster 2023-09-23T18:25:57.537880+0000 mgr.y (mgr.26054) 191 : cluster 0 pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:58.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:25:58 smithi098 bash[79516]: cluster 2023-09-23T18:25:57.537880+0000 mgr.y (mgr.26054) 191 : cluster 0 pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:25:59.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:25:58 smithi173 bash[59422]: cluster 2023-09-23T18:25:57.537880+0000 mgr.y (mgr.26054) 191 : cluster 0 pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:00.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:00 smithi098 bash[76817]: cluster 2023-09-23T18:25:59.538495+0000 mgr.y (mgr.26054) 192 : cluster 0 pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:00.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:00 smithi098 bash[79516]: cluster 2023-09-23T18:25:59.538495+0000 mgr.y (mgr.26054) 192 : cluster 0 pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:00.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:00 smithi173 bash[59422]: cluster 2023-09-23T18:25:59.538495+0000 mgr.y (mgr.26054) 192 : cluster 0 pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:02.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:02 smithi098 bash[79516]: cluster 2023-09-23T18:26:01.539495+0000 mgr.y (mgr.26054) 193 : cluster 0 pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:02.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:02 smithi098 bash[76817]: cluster 2023-09-23T18:26:01.539495+0000 mgr.y (mgr.26054) 193 : cluster 0 pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:03.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:02 smithi173 bash[59422]: cluster 2023-09-23T18:26:01.539495+0000 mgr.y (mgr.26054) 193 : cluster 0 pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:03.035 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T18:26:04.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:04 smithi098 bash[79516]: cluster 2023-09-23T18:26:03.540018+0000 mgr.y (mgr.26054) 194 : cluster 0 pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:04.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:04 smithi098 bash[76817]: cluster 2023-09-23T18:26:03.540018+0000 mgr.y (mgr.26054) 194 : cluster 0 pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:05.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:04 smithi173 bash[59422]: cluster 2023-09-23T18:26:03.540018+0000 mgr.y (mgr.26054) 194 : cluster 0 pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:06.443 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:26:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:26:06] "GET /metrics HTTP/1.1" 200 33908 "" "Prometheus/2.33.4" 2023-09-23T18:26:06.576 INFO:teuthology.orchestra.run.smithi098.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:26:06.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:06 smithi098 bash[76817]: cluster 2023-09-23T18:26:05.541028+0000 mgr.y (mgr.26054) 195 : cluster 0 pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:06.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:06 smithi098 bash[76817]: audit 2023-09-23T18:26:06.571859+0000 mon.a (mon.0) 264 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:06.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:06 smithi098 bash[76817]: audit 2023-09-23T18:26:06.576076+0000 mon.a (mon.0) 265 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:26:06.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:06 smithi098 bash[79516]: cluster 2023-09-23T18:26:05.541028+0000 mgr.y (mgr.26054) 195 : cluster 0 pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:06.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:06 smithi098 bash[79516]: audit 2023-09-23T18:26:06.571859+0000 mon.a (mon.0) 264 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:06.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:06 smithi098 bash[79516]: audit 2023-09-23T18:26:06.576076+0000 mon.a (mon.0) 265 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:26:07.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:06 smithi173 bash[59422]: cluster 2023-09-23T18:26:05.541028+0000 mgr.y (mgr.26054) 195 : cluster 0 pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:07.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:06 smithi173 bash[59422]: audit 2023-09-23T18:26:06.571859+0000 mon.a (mon.0) 264 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:07.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:06 smithi173 bash[59422]: audit 2023-09-23T18:26:06.576076+0000 mon.a (mon.0) 265 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:26:07.449 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for half of osds to upgrade' 2023-09-23T18:26:07.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:07 smithi098 bash[76817]: audit 2023-09-23T18:26:06.564259+0000 mgr.y (mgr.26054) 196 : audit 0 from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:26:07.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:07 smithi098 bash[76817]: cephadm 2023-09-23T18:26:06.564958+0000 mgr.y (mgr.26054) 197 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:26:07.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:07 smithi098 bash[79516]: audit 2023-09-23T18:26:06.564259+0000 mgr.y (mgr.26054) 196 : audit 0 from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:26:07.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:07 smithi098 bash[79516]: cephadm 2023-09-23T18:26:06.564958+0000 mgr.y (mgr.26054) 197 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:26:08.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:07 smithi173 bash[59422]: audit 2023-09-23T18:26:06.564259+0000 mgr.y (mgr.26054) 196 : audit 0 from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:26:08.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:07 smithi173 bash[59422]: cephadm 2023-09-23T18:26:06.564958+0000 mgr.y (mgr.26054) 197 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:26:08.907 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:08 smithi098 bash[79516]: cluster 2023-09-23T18:26:07.541635+0000 mgr.y (mgr.26054) 198 : cluster 0 pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:08.907 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:08 smithi098 bash[76817]: cluster 2023-09-23T18:26:07.541635+0000 mgr.y (mgr.26054) 198 : cluster 0 pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:09.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:08 smithi173 bash[59422]: cluster 2023-09-23T18:26:07.541635+0000 mgr.y (mgr.26054) 198 : cluster 0 pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:10.643 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:10 smithi098 bash[76817]: cluster 2023-09-23T18:26:09.542256+0000 mgr.y (mgr.26054) 199 : cluster 0 pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:10.643 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:10 smithi098 bash[76817]: audit 2023-09-23T18:26:09.570624+0000 mon.a (mon.0) 266 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:26:10.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:10 smithi098 bash[79516]: cluster 2023-09-23T18:26:09.542256+0000 mgr.y (mgr.26054) 199 : cluster 0 pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:10.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:10 smithi098 bash[79516]: audit 2023-09-23T18:26:09.570624+0000 mon.a (mon.0) 266 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:26:10.757 INFO:teuthology.orchestra.run.smithi098.stdout:wait for half of osds to upgrade 2023-09-23T18:26:10.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:10 smithi173 bash[59422]: cluster 2023-09-23T18:26:09.542256+0000 mgr.y (mgr.26054) 199 : cluster 0 pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:10.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:10 smithi173 bash[59422]: audit 2023-09-23T18:26:09.570624+0000 mon.a (mon.0) 266 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:26:11.657 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-09-23T18:26:12.861 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:12 smithi098 bash[79516]: cluster 2023-09-23T18:26:11.543237+0000 mgr.y (mgr.26054) 200 : cluster 0 pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:12.861 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:12 smithi098 bash[76817]: cluster 2023-09-23T18:26:11.543237+0000 mgr.y (mgr.26054) 200 : cluster 0 pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:13.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:12 smithi173 bash[59422]: cluster 2023-09-23T18:26:11.543237+0000 mgr.y (mgr.26054) 200 : cluster 0 pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:13.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[76817]: audit 2023-09-23T18:26:12.765799+0000 mon.a (mon.0) 267 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:13.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[76817]: audit 2023-09-23T18:26:12.773648+0000 mon.a (mon.0) 268 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:13.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[76817]: audit 2023-09-23T18:26:13.244441+0000 mon.a (mon.0) 269 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:26:13.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[76817]: audit 2023-09-23T18:26:13.245799+0000 mon.a (mon.0) 270 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:26:13.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[76817]: audit 2023-09-23T18:26:13.253951+0000 mon.a (mon.0) 271 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:13.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[76817]: audit 2023-09-23T18:26:13.335896+0000 mon.a (mon.0) 272 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:26:13.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[76817]: audit 2023-09-23T18:26:13.339827+0000 mon.a (mon.0) 273 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:26:13.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[76817]: audit 2023-09-23T18:26:13.341923+0000 mon.a (mon.0) 274 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:26:13.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[76817]: audit 2023-09-23T18:26:13.348732+0000 mon.a (mon.0) 275 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:13.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[76817]: audit 2023-09-23T18:26:13.350051+0000 mon.a (mon.0) 276 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T18:26:13.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[76817]: audit 2023-09-23T18:26:13.355004+0000 mon.a (mon.0) 277 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T18:26:13.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[76817]: audit 2023-09-23T18:26:13.356367+0000 mon.a (mon.0) 278 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T18:26:13.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[76817]: audit 2023-09-23T18:26:13.360761+0000 mon.a (mon.0) 279 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T18:26:13.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[76817]: audit 2023-09-23T18:26:13.361978+0000 mon.a (mon.0) 280 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T18:26:13.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[76817]: audit 2023-09-23T18:26:13.373133+0000 mon.a (mon.0) 281 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T18:26:13.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[76817]: audit 2023-09-23T18:26:13.374596+0000 mon.a (mon.0) 282 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:26:13.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[76817]: audit 2023-09-23T18:26:13.379942+0000 mon.a (mon.0) 283 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:13.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[76817]: audit 2023-09-23T18:26:13.381751+0000 mon.a (mon.0) 284 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T18:26:13.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[79516]: audit 2023-09-23T18:26:12.765799+0000 mon.a (mon.0) 267 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:13.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[79516]: audit 2023-09-23T18:26:12.773648+0000 mon.a (mon.0) 268 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:13.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[79516]: audit 2023-09-23T18:26:13.244441+0000 mon.a (mon.0) 269 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:26:13.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[79516]: audit 2023-09-23T18:26:13.245799+0000 mon.a (mon.0) 270 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:26:13.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[79516]: audit 2023-09-23T18:26:13.253951+0000 mon.a (mon.0) 271 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:13.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[79516]: audit 2023-09-23T18:26:13.335896+0000 mon.a (mon.0) 272 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:26:13.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[79516]: audit 2023-09-23T18:26:13.339827+0000 mon.a (mon.0) 273 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:26:13.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[79516]: audit 2023-09-23T18:26:13.341923+0000 mon.a (mon.0) 274 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:26:13.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[79516]: audit 2023-09-23T18:26:13.348732+0000 mon.a (mon.0) 275 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:13.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[79516]: audit 2023-09-23T18:26:13.350051+0000 mon.a (mon.0) 276 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T18:26:13.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[79516]: audit 2023-09-23T18:26:13.355004+0000 mon.a (mon.0) 277 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T18:26:13.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[79516]: audit 2023-09-23T18:26:13.356367+0000 mon.a (mon.0) 278 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T18:26:13.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[79516]: audit 2023-09-23T18:26:13.360761+0000 mon.a (mon.0) 279 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T18:26:13.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[79516]: audit 2023-09-23T18:26:13.361978+0000 mon.a (mon.0) 280 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T18:26:13.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[79516]: audit 2023-09-23T18:26:13.373133+0000 mon.a (mon.0) 281 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T18:26:13.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[79516]: audit 2023-09-23T18:26:13.374596+0000 mon.a (mon.0) 282 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:26:13.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[79516]: audit 2023-09-23T18:26:13.379942+0000 mon.a (mon.0) 283 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:13.947 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:13 smithi098 bash[79516]: audit 2023-09-23T18:26:13.381751+0000 mon.a (mon.0) 284 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T18:26:14.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:13 smithi173 bash[59422]: audit 2023-09-23T18:26:12.765799+0000 mon.a (mon.0) 267 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:14.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:13 smithi173 bash[59422]: audit 2023-09-23T18:26:12.773648+0000 mon.a (mon.0) 268 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:14.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:13 smithi173 bash[59422]: audit 2023-09-23T18:26:13.244441+0000 mon.a (mon.0) 269 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:26:14.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:13 smithi173 bash[59422]: audit 2023-09-23T18:26:13.245799+0000 mon.a (mon.0) 270 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:26:14.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:13 smithi173 bash[59422]: audit 2023-09-23T18:26:13.253951+0000 mon.a (mon.0) 271 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:14.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:13 smithi173 bash[59422]: audit 2023-09-23T18:26:13.335896+0000 mon.a (mon.0) 272 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:26:14.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:13 smithi173 bash[59422]: audit 2023-09-23T18:26:13.339827+0000 mon.a (mon.0) 273 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:26:14.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:13 smithi173 bash[59422]: audit 2023-09-23T18:26:13.341923+0000 mon.a (mon.0) 274 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:26:14.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:13 smithi173 bash[59422]: audit 2023-09-23T18:26:13.348732+0000 mon.a (mon.0) 275 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:14.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:13 smithi173 bash[59422]: audit 2023-09-23T18:26:13.350051+0000 mon.a (mon.0) 276 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T18:26:14.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:13 smithi173 bash[59422]: audit 2023-09-23T18:26:13.355004+0000 mon.a (mon.0) 277 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T18:26:14.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:13 smithi173 bash[59422]: audit 2023-09-23T18:26:13.356367+0000 mon.a (mon.0) 278 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T18:26:14.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:13 smithi173 bash[59422]: audit 2023-09-23T18:26:13.360761+0000 mon.a (mon.0) 279 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T18:26:14.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:13 smithi173 bash[59422]: audit 2023-09-23T18:26:13.361978+0000 mon.a (mon.0) 280 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T18:26:14.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:13 smithi173 bash[59422]: audit 2023-09-23T18:26:13.373133+0000 mon.a (mon.0) 281 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T18:26:14.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:13 smithi173 bash[59422]: audit 2023-09-23T18:26:13.374596+0000 mon.a (mon.0) 282 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:26:14.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:13 smithi173 bash[59422]: audit 2023-09-23T18:26:13.379942+0000 mon.a (mon.0) 283 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:14.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:13 smithi173 bash[59422]: audit 2023-09-23T18:26:13.381751+0000 mon.a (mon.0) 284 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T18:26:15.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:14 smithi173 bash[59422]: cephadm 2023-09-23T18:26:13.342805+0000 mgr.y (mgr.26054) 201 : cephadm 1 Upgrade: Setting container_image for all mon 2023-09-23T18:26:15.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:14 smithi173 bash[59422]: cephadm 2023-09-23T18:26:13.375350+0000 mgr.y (mgr.26054) 202 : cephadm 1 Upgrade: Setting container_image for all crash 2023-09-23T18:26:15.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:14 smithi173 bash[59422]: audit 2023-09-23T18:26:13.381943+0000 mgr.y (mgr.26054) 203 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T18:26:15.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:14 smithi173 bash[59422]: cephadm 2023-09-23T18:26:13.383351+0000 mgr.y (mgr.26054) 204 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-09-23T18:26:15.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:14 smithi173 bash[59422]: cluster 2023-09-23T18:26:13.543865+0000 mgr.y (mgr.26054) 205 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:15.093 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:14 smithi098 bash[79516]: cephadm 2023-09-23T18:26:13.342805+0000 mgr.y (mgr.26054) 201 : cephadm 1 Upgrade: Setting container_image for all mon 2023-09-23T18:26:15.093 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:14 smithi098 bash[79516]: cephadm 2023-09-23T18:26:13.375350+0000 mgr.y (mgr.26054) 202 : cephadm 1 Upgrade: Setting container_image for all crash 2023-09-23T18:26:15.093 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:14 smithi098 bash[79516]: audit 2023-09-23T18:26:13.381943+0000 mgr.y (mgr.26054) 203 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T18:26:15.093 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:14 smithi098 bash[79516]: cephadm 2023-09-23T18:26:13.383351+0000 mgr.y (mgr.26054) 204 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-09-23T18:26:15.094 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:14 smithi098 bash[79516]: cluster 2023-09-23T18:26:13.543865+0000 mgr.y (mgr.26054) 205 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:15.094 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:14 smithi098 bash[76817]: cephadm 2023-09-23T18:26:13.342805+0000 mgr.y (mgr.26054) 201 : cephadm 1 Upgrade: Setting container_image for all mon 2023-09-23T18:26:15.094 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:14 smithi098 bash[76817]: cephadm 2023-09-23T18:26:13.375350+0000 mgr.y (mgr.26054) 202 : cephadm 1 Upgrade: Setting container_image for all crash 2023-09-23T18:26:15.094 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:14 smithi098 bash[76817]: audit 2023-09-23T18:26:13.381943+0000 mgr.y (mgr.26054) 203 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T18:26:15.094 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:14 smithi098 bash[76817]: cephadm 2023-09-23T18:26:13.383351+0000 mgr.y (mgr.26054) 204 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-09-23T18:26:15.094 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:14 smithi098 bash[76817]: cluster 2023-09-23T18:26:13.543865+0000 mgr.y (mgr.26054) 205 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:15.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:15 smithi098 bash[79516]: audit 2023-09-23T18:26:15.542906+0000 mon.a (mon.0) 285 : audit 0 from='client.? 172.21.15.98:0/3809611986' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:15.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:15 smithi098 bash[76817]: audit 2023-09-23T18:26:15.542906+0000 mon.a (mon.0) 285 : audit 0 from='client.? 172.21.15.98:0/3809611986' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:16.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:15 smithi173 bash[59422]: audit 2023-09-23T18:26:15.542906+0000 mon.a (mon.0) 285 : audit 0 from='client.? 172.21.15.98:0/3809611986' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:16.443 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:26:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:26:16] "GET /metrics HTTP/1.1" 200 33909 "" "Prometheus/2.33.4" 2023-09-23T18:26:17.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:16 smithi098 bash[76817]: cluster 2023-09-23T18:26:15.544799+0000 mgr.y (mgr.26054) 206 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:17.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:16 smithi098 bash[76817]: audit 2023-09-23T18:26:16.264569+0000 mon.a (mon.0) 286 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:17.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:16 smithi098 bash[76817]: audit 2023-09-23T18:26:16.265996+0000 mon.a (mon.0) 287 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T18:26:17.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:16 smithi098 bash[76817]: audit 2023-09-23T18:26:16.267081+0000 mon.a (mon.0) 288 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:26:17.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:16 smithi098 bash[79516]: cluster 2023-09-23T18:26:15.544799+0000 mgr.y (mgr.26054) 206 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:17.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:16 smithi098 bash[79516]: audit 2023-09-23T18:26:16.264569+0000 mon.a (mon.0) 286 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:17.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:16 smithi098 bash[79516]: audit 2023-09-23T18:26:16.265996+0000 mon.a (mon.0) 287 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T18:26:17.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:16 smithi098 bash[79516]: audit 2023-09-23T18:26:16.267081+0000 mon.a (mon.0) 288 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:26:17.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:16 smithi173 bash[59422]: cluster 2023-09-23T18:26:15.544799+0000 mgr.y (mgr.26054) 206 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:17.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:16 smithi173 bash[59422]: audit 2023-09-23T18:26:16.264569+0000 mon.a (mon.0) 286 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:17.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:16 smithi173 bash[59422]: audit 2023-09-23T18:26:16.265996+0000 mon.a (mon.0) 287 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T18:26:17.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:16 smithi173 bash[59422]: audit 2023-09-23T18:26:16.267081+0000 mon.a (mon.0) 288 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:26:18.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:17 smithi098 bash[79516]: cephadm 2023-09-23T18:26:16.254621+0000 mgr.y (mgr.26054) 207 : cephadm 1 Upgrade: Updating osd.0 2023-09-23T18:26:18.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:17 smithi098 bash[79516]: cephadm 2023-09-23T18:26:16.269817+0000 mgr.y (mgr.26054) 208 : cephadm 1 Deploying daemon osd.0 on smithi098 2023-09-23T18:26:18.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:17 smithi098 bash[76817]: cephadm 2023-09-23T18:26:16.254621+0000 mgr.y (mgr.26054) 207 : cephadm 1 Upgrade: Updating osd.0 2023-09-23T18:26:18.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:17 smithi098 bash[76817]: cephadm 2023-09-23T18:26:16.269817+0000 mgr.y (mgr.26054) 208 : cephadm 1 Deploying daemon osd.0 on smithi098 2023-09-23T18:26:18.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:17 smithi173 bash[59422]: cephadm 2023-09-23T18:26:16.254621+0000 mgr.y (mgr.26054) 207 : cephadm 1 Upgrade: Updating osd.0 2023-09-23T18:26:18.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:17 smithi173 bash[59422]: cephadm 2023-09-23T18:26:16.269817+0000 mgr.y (mgr.26054) 208 : cephadm 1 Deploying daemon osd.0 on smithi098 2023-09-23T18:26:19.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:18 smithi098 bash[76817]: cluster 2023-09-23T18:26:17.545418+0000 mgr.y (mgr.26054) 209 : cluster 0 pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:19.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:18 smithi098 bash[76817]: audit 2023-09-23T18:26:17.903858+0000 mon.c (mon.2) 4 : audit 0 from='client.? 172.21.15.98:0/982711542' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:19.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:18 smithi098 bash[79516]: cluster 2023-09-23T18:26:17.545418+0000 mgr.y (mgr.26054) 209 : cluster 0 pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:19.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:18 smithi098 bash[79516]: audit 2023-09-23T18:26:17.903858+0000 mon.c (mon.2) 4 : audit 0 from='client.? 172.21.15.98:0/982711542' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:19.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:18 smithi173 bash[59422]: cluster 2023-09-23T18:26:17.545418+0000 mgr.y (mgr.26054) 209 : cluster 0 pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:19.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:18 smithi173 bash[59422]: audit 2023-09-23T18:26:17.903858+0000 mon.c (mon.2) 4 : audit 0 from='client.? 172.21.15.98:0/982711542' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:20.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:20 smithi098 bash[76817]: cluster 2023-09-23T18:26:19.546042+0000 mgr.y (mgr.26054) 210 : cluster 0 pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:20.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:20 smithi098 bash[76817]: audit 2023-09-23T18:26:20.256658+0000 mon.c (mon.2) 5 : audit 0 from='client.? 172.21.15.98:0/805030381' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:20.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:20 smithi098 bash[79516]: cluster 2023-09-23T18:26:19.546042+0000 mgr.y (mgr.26054) 210 : cluster 0 pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:20.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:20 smithi098 bash[79516]: audit 2023-09-23T18:26:20.256658+0000 mon.c (mon.2) 5 : audit 0 from='client.? 172.21.15.98:0/805030381' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:20.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:20 smithi173 bash[59422]: cluster 2023-09-23T18:26:19.546042+0000 mgr.y (mgr.26054) 210 : cluster 0 pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:20.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:20 smithi173 bash[59422]: audit 2023-09-23T18:26:20.256658+0000 mon.c (mon.2) 5 : audit 0 from='client.? 172.21.15.98:0/805030381' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:21.107 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 18:26:20 smithi098 systemd[1]: Stopping Ceph osd.0 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T18:26:21.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:21 smithi098 bash[76817]: cluster 2023-09-23T18:26:21.104147+0000 mon.a (mon.0) 289 : cluster 1 osd.0 marked itself down and dead 2023-09-23T18:26:21.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:21 smithi098 bash[79516]: cluster 2023-09-23T18:26:21.104147+0000 mon.a (mon.0) 289 : cluster 1 osd.0 marked itself down and dead 2023-09-23T18:26:21.441 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 18:26:21 smithi098 bash[36780]: debug 2023-09-23T18:26:21.101+0000 7fb6bc6e2700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T18:26:21.441 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 18:26:21 smithi098 bash[36780]: debug 2023-09-23T18:26:21.101+0000 7fb6bc6e2700 -1 osd.0 84 *** Got signal Terminated *** 2023-09-23T18:26:21.441 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 18:26:21 smithi098 bash[36780]: debug 2023-09-23T18:26:21.101+0000 7fb6bc6e2700 -1 osd.0 84 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T18:26:21.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:21 smithi173 bash[59422]: cluster 2023-09-23T18:26:21.104147+0000 mon.a (mon.0) 289 : cluster 1 osd.0 marked itself down and dead 2023-09-23T18:26:22.360 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 18:26:22 smithi098 bash[84771]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-0 2023-09-23T18:26:22.360 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 18:26:22 smithi098 bash[84836]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-0 2023-09-23T18:26:22.360 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:22 smithi098 bash[76817]: cluster 2023-09-23T18:26:21.315468+0000 mon.a (mon.0) 290 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:26:22.361 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:22 smithi098 bash[76817]: cluster 2023-09-23T18:26:21.326198+0000 mon.a (mon.0) 291 : cluster 0 osdmap e85: 8 total, 7 up, 8 in 2023-09-23T18:26:22.361 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:22 smithi098 bash[76817]: cluster 2023-09-23T18:26:21.327652+0000 osd.7 (osd.7) 3 : cluster 3 failed to encode map e85 with expected crc 2023-09-23T18:26:22.361 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:22 smithi098 bash[76817]: cluster 2023-09-23T18:26:21.328404+0000 osd.4 (osd.4) 4 : cluster 3 failed to encode map e85 with expected crc 2023-09-23T18:26:22.361 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:22 smithi098 bash[76817]: cluster 2023-09-23T18:26:21.546809+0000 mgr.y (mgr.26054) 211 : cluster 0 pgmap v145: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:22.361 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:22 smithi098 bash[79516]: cluster 2023-09-23T18:26:21.315468+0000 mon.a (mon.0) 290 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:26:22.361 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:22 smithi098 bash[79516]: cluster 2023-09-23T18:26:21.326198+0000 mon.a (mon.0) 291 : cluster 0 osdmap e85: 8 total, 7 up, 8 in 2023-09-23T18:26:22.361 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:22 smithi098 bash[79516]: cluster 2023-09-23T18:26:21.327652+0000 osd.7 (osd.7) 3 : cluster 3 failed to encode map e85 with expected crc 2023-09-23T18:26:22.361 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:22 smithi098 bash[79516]: cluster 2023-09-23T18:26:21.328404+0000 osd.4 (osd.4) 4 : cluster 3 failed to encode map e85 with expected crc 2023-09-23T18:26:22.361 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:22 smithi098 bash[79516]: cluster 2023-09-23T18:26:21.546809+0000 mgr.y (mgr.26054) 211 : cluster 0 pgmap v145: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:22.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:22 smithi173 bash[59422]: cluster 2023-09-23T18:26:21.315468+0000 mon.a (mon.0) 290 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:26:22.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:22 smithi173 bash[59422]: cluster 2023-09-23T18:26:21.326198+0000 mon.a (mon.0) 291 : cluster 0 osdmap e85: 8 total, 7 up, 8 in 2023-09-23T18:26:22.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:22 smithi173 bash[59422]: cluster 2023-09-23T18:26:21.327652+0000 osd.7 (osd.7) 3 : cluster 3 failed to encode map e85 with expected crc 2023-09-23T18:26:22.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:22 smithi173 bash[59422]: cluster 2023-09-23T18:26:21.328404+0000 osd.4 (osd.4) 4 : cluster 3 failed to encode map e85 with expected crc 2023-09-23T18:26:22.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:22 smithi173 bash[59422]: cluster 2023-09-23T18:26:21.546809+0000 mgr.y (mgr.26054) 211 : cluster 0 pgmap v145: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:23.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:23 smithi098 bash[76817]: cluster 2023-09-23T18:26:22.335461+0000 mon.a (mon.0) 292 : cluster 0 osdmap e86: 8 total, 7 up, 8 in 2023-09-23T18:26:23.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:23 smithi098 bash[76817]: cluster 2023-09-23T18:26:22.335909+0000 osd.3 (osd.3) 3 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:23 smithi098 bash[76817]: cluster 2023-09-23T18:26:22.335934+0000 osd.1 (osd.1) 4 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:23 smithi098 bash[76817]: cluster 2023-09-23T18:26:22.336243+0000 osd.7 (osd.7) 4 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:23 smithi098 bash[76817]: cluster 2023-09-23T18:26:22.337904+0000 osd.4 (osd.4) 5 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:23 smithi098 bash[76817]: cluster 2023-09-23T18:26:22.338608+0000 osd.2 (osd.2) 4 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:23 smithi098 bash[76817]: cluster 2023-09-23T18:26:22.338873+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:23 smithi098 bash[76817]: cluster 2023-09-23T18:26:22.338882+0000 osd.5 (osd.5) 4 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:23 smithi098 bash[76817]: cluster 2023-09-23T18:26:22.339634+0000 osd.6 (osd.6) 3 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:23 smithi098 bash[76817]: cluster 2023-09-23T18:26:22.340627+0000 osd.4 (osd.4) 6 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:23 smithi098 bash[76817]: audit 2023-09-23T18:26:22.627925+0000 mon.c (mon.2) 6 : audit 0 from='client.? 172.21.15.98:0/2293330692' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:23.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:23 smithi098 bash[79516]: cluster 2023-09-23T18:26:22.335461+0000 mon.a (mon.0) 292 : cluster 0 osdmap e86: 8 total, 7 up, 8 in 2023-09-23T18:26:23.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:23 smithi098 bash[79516]: cluster 2023-09-23T18:26:22.335909+0000 osd.3 (osd.3) 3 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:23 smithi098 bash[79516]: cluster 2023-09-23T18:26:22.335934+0000 osd.1 (osd.1) 4 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:23 smithi098 bash[79516]: cluster 2023-09-23T18:26:22.336243+0000 osd.7 (osd.7) 4 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:23 smithi098 bash[79516]: cluster 2023-09-23T18:26:22.337904+0000 osd.4 (osd.4) 5 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:23 smithi098 bash[79516]: cluster 2023-09-23T18:26:22.338608+0000 osd.2 (osd.2) 4 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:23 smithi098 bash[79516]: cluster 2023-09-23T18:26:22.338873+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:23 smithi098 bash[79516]: cluster 2023-09-23T18:26:22.338882+0000 osd.5 (osd.5) 4 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:23 smithi098 bash[79516]: cluster 2023-09-23T18:26:22.339634+0000 osd.6 (osd.6) 3 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:23 smithi098 bash[79516]: cluster 2023-09-23T18:26:22.340627+0000 osd.4 (osd.4) 6 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:23 smithi098 bash[79516]: audit 2023-09-23T18:26:22.627925+0000 mon.c (mon.2) 6 : audit 0 from='client.? 172.21.15.98:0/2293330692' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:23.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:23 smithi173 bash[59422]: cluster 2023-09-23T18:26:22.335461+0000 mon.a (mon.0) 292 : cluster 0 osdmap e86: 8 total, 7 up, 8 in 2023-09-23T18:26:23.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:23 smithi173 bash[59422]: cluster 2023-09-23T18:26:22.335909+0000 osd.3 (osd.3) 3 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:23 smithi173 bash[59422]: cluster 2023-09-23T18:26:22.335934+0000 osd.1 (osd.1) 4 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:23 smithi173 bash[59422]: cluster 2023-09-23T18:26:22.336243+0000 osd.7 (osd.7) 4 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:23 smithi173 bash[59422]: cluster 2023-09-23T18:26:22.337904+0000 osd.4 (osd.4) 5 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:23 smithi173 bash[59422]: cluster 2023-09-23T18:26:22.338608+0000 osd.2 (osd.2) 4 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:23 smithi173 bash[59422]: cluster 2023-09-23T18:26:22.338873+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:23 smithi173 bash[59422]: cluster 2023-09-23T18:26:22.338882+0000 osd.5 (osd.5) 4 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:23 smithi173 bash[59422]: cluster 2023-09-23T18:26:22.339634+0000 osd.6 (osd.6) 3 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:23 smithi173 bash[59422]: cluster 2023-09-23T18:26:22.340627+0000 osd.4 (osd.4) 6 : cluster 3 failed to encode map e86 with expected crc 2023-09-23T18:26:23.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:23 smithi173 bash[59422]: audit 2023-09-23T18:26:22.627925+0000 mon.c (mon.2) 6 : audit 0 from='client.? 172.21.15.98:0/2293330692' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:24.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:24 smithi098 bash[76817]: cluster 2023-09-23T18:26:23.547378+0000 mgr.y (mgr.26054) 212 : cluster 0 pgmap v147: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:24.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:24 smithi098 bash[79516]: cluster 2023-09-23T18:26:23.547378+0000 mgr.y (mgr.26054) 212 : cluster 0 pgmap v147: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:24.693 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 18:26:24 smithi098 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.0.service: Succeeded. 2023-09-23T18:26:24.693 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 18:26:24 smithi098 systemd[1]: Stopped Ceph osd.0 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:26:24.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:24 smithi173 bash[59422]: cluster 2023-09-23T18:26:23.547378+0000 mgr.y (mgr.26054) 212 : cluster 0 pgmap v147: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:25.193 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 18:26:24 smithi098 systemd[1]: Started Ceph osd.0 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:26:25.895 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:25 smithi098 bash[79516]: audit 2023-09-23T18:26:24.576357+0000 mon.a (mon.0) 293 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:25.895 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:25 smithi098 bash[79516]: audit 2023-09-23T18:26:24.578517+0000 mon.a (mon.0) 294 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:26:25.895 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:25 smithi098 bash[79516]: audit 2023-09-23T18:26:24.835684+0000 mon.a (mon.0) 295 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:25.895 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:25 smithi098 bash[79516]: audit 2023-09-23T18:26:24.844327+0000 mon.a (mon.0) 296 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:25.895 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:25 smithi098 bash[79516]: audit 2023-09-23T18:26:24.855588+0000 mon.a (mon.0) 297 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:26:25.896 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:25 smithi098 bash[79516]: audit 2023-09-23T18:26:24.995483+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.98:0/3958705286' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:25.896 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:25 smithi098 bash[79516]: audit 2023-09-23T18:26:25.231730+0000 mon.a (mon.0) 298 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:25.896 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:25 smithi098 bash[79516]: audit 2023-09-23T18:26:25.239037+0000 mon.a (mon.0) 299 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:25.896 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:25 smithi098 bash[76817]: audit 2023-09-23T18:26:24.576357+0000 mon.a (mon.0) 293 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:25.896 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:25 smithi098 bash[76817]: audit 2023-09-23T18:26:24.578517+0000 mon.a (mon.0) 294 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:26:25.896 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:25 smithi098 bash[76817]: audit 2023-09-23T18:26:24.835684+0000 mon.a (mon.0) 295 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:25.896 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:25 smithi098 bash[76817]: audit 2023-09-23T18:26:24.844327+0000 mon.a (mon.0) 296 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:25.896 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:25 smithi098 bash[76817]: audit 2023-09-23T18:26:24.855588+0000 mon.a (mon.0) 297 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:26:25.896 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:25 smithi098 bash[76817]: audit 2023-09-23T18:26:24.995483+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.98:0/3958705286' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:25.896 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:25 smithi098 bash[76817]: audit 2023-09-23T18:26:25.231730+0000 mon.a (mon.0) 298 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:25.896 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:25 smithi098 bash[76817]: audit 2023-09-23T18:26:25.239037+0000 mon.a (mon.0) 299 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:26.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:25 smithi173 bash[59422]: audit 2023-09-23T18:26:24.576357+0000 mon.a (mon.0) 293 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:26.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:25 smithi173 bash[59422]: audit 2023-09-23T18:26:24.578517+0000 mon.a (mon.0) 294 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:26:26.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:25 smithi173 bash[59422]: audit 2023-09-23T18:26:24.835684+0000 mon.a (mon.0) 295 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:26.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:25 smithi173 bash[59422]: audit 2023-09-23T18:26:24.844327+0000 mon.a (mon.0) 296 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:26.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:25 smithi173 bash[59422]: audit 2023-09-23T18:26:24.855588+0000 mon.a (mon.0) 297 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:26:26.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:25 smithi173 bash[59422]: audit 2023-09-23T18:26:24.995483+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.98:0/3958705286' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:26.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:25 smithi173 bash[59422]: audit 2023-09-23T18:26:25.231730+0000 mon.a (mon.0) 298 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:26.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:25 smithi173 bash[59422]: audit 2023-09-23T18:26:25.239037+0000 mon.a (mon.0) 299 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:26.424 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:26:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:26:26] "GET /metrics HTTP/1.1" 200 33973 "" "Prometheus/2.33.4" 2023-09-23T18:26:26.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:26 smithi098 bash[79516]: cluster 2023-09-23T18:26:25.548638+0000 mgr.y (mgr.26054) 213 : cluster 0 pgmap v148: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T18:26:26.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:26 smithi098 bash[79516]: cluster 2023-09-23T18:26:26.236636+0000 mon.a (mon.0) 300 : cluster 3 Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:26:26.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:26 smithi098 bash[76817]: cluster 2023-09-23T18:26:25.548638+0000 mgr.y (mgr.26054) 213 : cluster 0 pgmap v148: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T18:26:26.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:26 smithi098 bash[76817]: cluster 2023-09-23T18:26:26.236636+0000 mon.a (mon.0) 300 : cluster 3 Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:26:26.694 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 18:26:26 smithi098 bash[85040]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T18:26:26.694 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 18:26:26 smithi098 bash[85040]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T18:26:26.694 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 18:26:26 smithi098 bash[85040]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-09-23T18:26:26.694 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 18:26:26 smithi098 bash[85040]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-09-23T18:26:26.694 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 18:26:26 smithi098 bash[85040]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-09-23T18:26:26.694 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 18:26:26 smithi098 bash[85040]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-09-23T18:26:26.694 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 18:26:26 smithi098 bash[85040]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T18:26:26.694 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 18:26:26 smithi098 bash[85040]: --> ceph-volume lvm activate successful for osd ID: 0 2023-09-23T18:26:27.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:26 smithi173 bash[59422]: cluster 2023-09-23T18:26:25.548638+0000 mgr.y (mgr.26054) 213 : cluster 0 pgmap v148: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T18:26:27.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:26 smithi173 bash[59422]: cluster 2023-09-23T18:26:26.236636+0000 mon.a (mon.0) 300 : cluster 3 Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:26:27.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:27 smithi098 bash[79516]: audit 2023-09-23T18:26:27.374417+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.98:0/3381971798' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:27.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:27 smithi098 bash[76817]: audit 2023-09-23T18:26:27.374417+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.98:0/3381971798' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:28.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:27 smithi173 bash[59422]: audit 2023-09-23T18:26:27.374417+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.98:0/3381971798' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:28.443 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 18:26:28 smithi098 bash[85332]: debug 2023-09-23T18:26:28.237+0000 7f76b6667700 1 -- 172.21.15.98:0/3581483407 <== mon.0 v2:172.21.15.98:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5624c560a4e0 con 0x5624c5664c00 2023-09-23T18:26:28.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:28 smithi098 bash[79516]: cluster 2023-09-23T18:26:27.549446+0000 mgr.y (mgr.26054) 214 : cluster 0 pgmap v149: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T18:26:28.943 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 18:26:28 smithi098 bash[85332]: debug 2023-09-23T18:26:28.801+0000 7f76bf7ec640 -1 Falling back to public interface 2023-09-23T18:26:28.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:28 smithi098 bash[76817]: cluster 2023-09-23T18:26:27.549446+0000 mgr.y (mgr.26054) 214 : cluster 0 pgmap v149: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T18:26:29.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:28 smithi173 bash[59422]: cluster 2023-09-23T18:26:27.549446+0000 mgr.y (mgr.26054) 214 : cluster 0 pgmap v149: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T18:26:30.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:30 smithi098 bash[79516]: cluster 2023-09-23T18:26:29.550149+0000 mgr.y (mgr.26054) 215 : cluster 0 pgmap v150: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T18:26:30.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:30 smithi098 bash[79516]: audit 2023-09-23T18:26:29.746479+0000 mon.c (mon.2) 9 : audit 0 from='client.? 172.21.15.98:0/1708610527' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:30.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:30 smithi098 bash[76817]: cluster 2023-09-23T18:26:29.550149+0000 mgr.y (mgr.26054) 215 : cluster 0 pgmap v150: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T18:26:30.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:30 smithi098 bash[76817]: audit 2023-09-23T18:26:29.746479+0000 mon.c (mon.2) 9 : audit 0 from='client.? 172.21.15.98:0/1708610527' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:30.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:30 smithi173 bash[59422]: cluster 2023-09-23T18:26:29.550149+0000 mgr.y (mgr.26054) 215 : cluster 0 pgmap v150: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T18:26:30.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:30 smithi173 bash[59422]: audit 2023-09-23T18:26:29.746479+0000 mon.c (mon.2) 9 : audit 0 from='client.? 172.21.15.98:0/1708610527' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:32.610 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 18:26:32 smithi098 bash[85332]: debug 2023-09-23T18:26:32.325+0000 7f76bf7ec640 -1 osd.0 84 log_to_monitors true 2023-09-23T18:26:32.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:32 smithi098 bash[79516]: cluster 2023-09-23T18:26:31.551106+0000 mgr.y (mgr.26054) 216 : cluster 0 pgmap v151: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T18:26:32.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:32 smithi098 bash[79516]: audit 2023-09-23T18:26:31.948835+0000 mon.a (mon.0) 301 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:32.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:32 smithi098 bash[79516]: audit 2023-09-23T18:26:31.957294+0000 mon.a (mon.0) 302 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:32.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:32 smithi098 bash[79516]: audit 2023-09-23T18:26:32.153371+0000 mon.a (mon.0) 303 : audit 0 from='client.? 172.21.15.98:0/3951751517' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:32.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:32 smithi098 bash[79516]: audit 2023-09-23T18:26:32.333108+0000 mon.c (mon.2) 10 : audit 1 from='osd.0 [v2:172.21.15.98:6802/1320995478,v1:172.21.15.98:6803/1320995478]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T18:26:32.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:32 smithi098 bash[79516]: audit 2023-09-23T18:26:32.333676+0000 mon.a (mon.0) 304 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T18:26:32.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:32 smithi098 bash[76817]: cluster 2023-09-23T18:26:31.551106+0000 mgr.y (mgr.26054) 216 : cluster 0 pgmap v151: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T18:26:32.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:32 smithi098 bash[76817]: audit 2023-09-23T18:26:31.948835+0000 mon.a (mon.0) 301 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:32.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:32 smithi098 bash[76817]: audit 2023-09-23T18:26:31.957294+0000 mon.a (mon.0) 302 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:32.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:32 smithi098 bash[76817]: audit 2023-09-23T18:26:32.153371+0000 mon.a (mon.0) 303 : audit 0 from='client.? 172.21.15.98:0/3951751517' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:32.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:32 smithi098 bash[76817]: audit 2023-09-23T18:26:32.333108+0000 mon.c (mon.2) 10 : audit 1 from='osd.0 [v2:172.21.15.98:6802/1320995478,v1:172.21.15.98:6803/1320995478]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T18:26:32.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:32 smithi098 bash[76817]: audit 2023-09-23T18:26:32.333676+0000 mon.a (mon.0) 304 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T18:26:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:32 smithi173 bash[59422]: cluster 2023-09-23T18:26:31.551106+0000 mgr.y (mgr.26054) 216 : cluster 0 pgmap v151: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T18:26:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:32 smithi173 bash[59422]: audit 2023-09-23T18:26:31.948835+0000 mon.a (mon.0) 301 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:32 smithi173 bash[59422]: audit 2023-09-23T18:26:31.957294+0000 mon.a (mon.0) 302 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:32 smithi173 bash[59422]: audit 2023-09-23T18:26:32.153371+0000 mon.a (mon.0) 303 : audit 0 from='client.? 172.21.15.98:0/3951751517' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:32 smithi173 bash[59422]: audit 2023-09-23T18:26:32.333108+0000 mon.c (mon.2) 10 : audit 1 from='osd.0 [v2:172.21.15.98:6802/1320995478,v1:172.21.15.98:6803/1320995478]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T18:26:33.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:32 smithi173 bash[59422]: audit 2023-09-23T18:26:32.333676+0000 mon.a (mon.0) 304 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T18:26:33.443 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 18:26:33 smithi098 bash[85332]: debug 2023-09-23T18:26:33.005+0000 7f76acc51700 -1 osd.0 84 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T18:26:34.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:33 smithi173 bash[59422]: audit 2023-09-23T18:26:32.753551+0000 mon.a (mon.0) 305 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:34.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:33 smithi173 bash[59422]: audit 2023-09-23T18:26:32.760695+0000 mon.a (mon.0) 306 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:34.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:33 smithi173 bash[59422]: audit 2023-09-23T18:26:32.962194+0000 mon.a (mon.0) 307 : audit 1 from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T18:26:34.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:33 smithi173 bash[59422]: cluster 2023-09-23T18:26:32.962355+0000 mon.a (mon.0) 308 : cluster 0 osdmap e87: 8 total, 7 up, 8 in 2023-09-23T18:26:34.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:33 smithi173 bash[59422]: audit 2023-09-23T18:26:32.963381+0000 mon.c (mon.2) 11 : audit 1 from='osd.0 [v2:172.21.15.98:6802/1320995478,v1:172.21.15.98:6803/1320995478]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:26:34.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:33 smithi173 bash[59422]: audit 2023-09-23T18:26:32.963787+0000 mon.a (mon.0) 309 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:26:34.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:33 smithi098 bash[76817]: audit 2023-09-23T18:26:32.753551+0000 mon.a (mon.0) 305 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:34.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:33 smithi098 bash[76817]: audit 2023-09-23T18:26:32.760695+0000 mon.a (mon.0) 306 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:34.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:33 smithi098 bash[76817]: audit 2023-09-23T18:26:32.962194+0000 mon.a (mon.0) 307 : audit 1 from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T18:26:34.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:33 smithi098 bash[76817]: cluster 2023-09-23T18:26:32.962355+0000 mon.a (mon.0) 308 : cluster 0 osdmap e87: 8 total, 7 up, 8 in 2023-09-23T18:26:34.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:33 smithi098 bash[76817]: audit 2023-09-23T18:26:32.963381+0000 mon.c (mon.2) 11 : audit 1 from='osd.0 [v2:172.21.15.98:6802/1320995478,v1:172.21.15.98:6803/1320995478]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:26:34.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:33 smithi098 bash[76817]: audit 2023-09-23T18:26:32.963787+0000 mon.a (mon.0) 309 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:26:34.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:33 smithi098 bash[79516]: audit 2023-09-23T18:26:32.753551+0000 mon.a (mon.0) 305 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:34.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:33 smithi098 bash[79516]: audit 2023-09-23T18:26:32.760695+0000 mon.a (mon.0) 306 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:34.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:33 smithi098 bash[79516]: audit 2023-09-23T18:26:32.962194+0000 mon.a (mon.0) 307 : audit 1 from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T18:26:34.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:33 smithi098 bash[79516]: cluster 2023-09-23T18:26:32.962355+0000 mon.a (mon.0) 308 : cluster 0 osdmap e87: 8 total, 7 up, 8 in 2023-09-23T18:26:34.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:33 smithi098 bash[79516]: audit 2023-09-23T18:26:32.963381+0000 mon.c (mon.2) 11 : audit 1 from='osd.0 [v2:172.21.15.98:6802/1320995478,v1:172.21.15.98:6803/1320995478]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:26:34.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:33 smithi098 bash[79516]: audit 2023-09-23T18:26:32.963787+0000 mon.a (mon.0) 309 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:26:35.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:34 smithi173 bash[59422]: cluster 2023-09-23T18:26:32.959084+0000 osd.1 (osd.1) 5 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T18:26:35.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:34 smithi173 bash[59422]: cluster 2023-09-23T18:26:32.962077+0000 osd.4 (osd.4) 7 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T18:26:35.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:34 smithi173 bash[59422]: cluster 2023-09-23T18:26:33.551802+0000 mgr.y (mgr.26054) 217 : cluster 0 pgmap v153: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T18:26:35.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:34 smithi173 bash[59422]: cluster 2023-09-23T18:26:33.869879+0000 osd.5 (osd.5) 5 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T18:26:35.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:34 smithi173 bash[59422]: cluster 2023-09-23T18:26:33.958420+0000 mon.a (mon.0) 310 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:26:35.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:34 smithi173 bash[59422]: cluster 2023-09-23T18:26:33.969769+0000 osd.3 (osd.3) 4 : cluster 3 failed to encode map e88 with expected crc 2023-09-23T18:26:35.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:34 smithi173 bash[59422]: cluster 2023-09-23T18:26:33.971836+0000 mon.a (mon.0) 311 : cluster 1 osd.0 [v2:172.21.15.98:6802/1320995478,v1:172.21.15.98:6803/1320995478] boot 2023-09-23T18:26:35.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:34 smithi173 bash[59422]: cluster 2023-09-23T18:26:33.971889+0000 mon.a (mon.0) 312 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2023-09-23T18:26:35.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:34 smithi173 bash[59422]: audit 2023-09-23T18:26:33.972181+0000 mon.a (mon.0) 313 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:26:35.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:34 smithi173 bash[59422]: cluster 2023-09-23T18:26:33.975426+0000 osd.6 (osd.6) 4 : cluster 3 failed to encode map e88 with expected crc 2023-09-23T18:26:35.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:34 smithi173 bash[59422]: audit 2023-09-23T18:26:34.517664+0000 mon.c (mon.2) 12 : audit 0 from='client.? 172.21.15.98:0/298338257' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:35.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:34 smithi098 bash[76817]: cluster 2023-09-23T18:26:32.959084+0000 osd.1 (osd.1) 5 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T18:26:35.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:34 smithi098 bash[76817]: cluster 2023-09-23T18:26:32.962077+0000 osd.4 (osd.4) 7 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T18:26:35.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:34 smithi098 bash[76817]: cluster 2023-09-23T18:26:33.551802+0000 mgr.y (mgr.26054) 217 : cluster 0 pgmap v153: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T18:26:35.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:34 smithi098 bash[76817]: cluster 2023-09-23T18:26:33.869879+0000 osd.5 (osd.5) 5 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T18:26:35.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:34 smithi098 bash[76817]: cluster 2023-09-23T18:26:33.958420+0000 mon.a (mon.0) 310 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:26:35.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:34 smithi098 bash[76817]: cluster 2023-09-23T18:26:33.969769+0000 osd.3 (osd.3) 4 : cluster 3 failed to encode map e88 with expected crc 2023-09-23T18:26:35.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:34 smithi098 bash[76817]: cluster 2023-09-23T18:26:33.971836+0000 mon.a (mon.0) 311 : cluster 1 osd.0 [v2:172.21.15.98:6802/1320995478,v1:172.21.15.98:6803/1320995478] boot 2023-09-23T18:26:35.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:34 smithi098 bash[76817]: cluster 2023-09-23T18:26:33.971889+0000 mon.a (mon.0) 312 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2023-09-23T18:26:35.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:34 smithi098 bash[76817]: audit 2023-09-23T18:26:33.972181+0000 mon.a (mon.0) 313 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:26:35.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:34 smithi098 bash[76817]: cluster 2023-09-23T18:26:33.975426+0000 osd.6 (osd.6) 4 : cluster 3 failed to encode map e88 with expected crc 2023-09-23T18:26:35.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:34 smithi098 bash[76817]: audit 2023-09-23T18:26:34.517664+0000 mon.c (mon.2) 12 : audit 0 from='client.? 172.21.15.98:0/298338257' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:35.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:34 smithi098 bash[79516]: cluster 2023-09-23T18:26:32.959084+0000 osd.1 (osd.1) 5 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T18:26:35.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:34 smithi098 bash[79516]: cluster 2023-09-23T18:26:32.962077+0000 osd.4 (osd.4) 7 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T18:26:35.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:34 smithi098 bash[79516]: cluster 2023-09-23T18:26:33.551802+0000 mgr.y (mgr.26054) 217 : cluster 0 pgmap v153: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T18:26:35.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:34 smithi098 bash[79516]: cluster 2023-09-23T18:26:33.869879+0000 osd.5 (osd.5) 5 : cluster 3 failed to encode map e87 with expected crc 2023-09-23T18:26:35.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:34 smithi098 bash[79516]: cluster 2023-09-23T18:26:33.958420+0000 mon.a (mon.0) 310 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:26:35.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:34 smithi098 bash[79516]: cluster 2023-09-23T18:26:33.969769+0000 osd.3 (osd.3) 4 : cluster 3 failed to encode map e88 with expected crc 2023-09-23T18:26:35.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:34 smithi098 bash[79516]: cluster 2023-09-23T18:26:33.971836+0000 mon.a (mon.0) 311 : cluster 1 osd.0 [v2:172.21.15.98:6802/1320995478,v1:172.21.15.98:6803/1320995478] boot 2023-09-23T18:26:35.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:34 smithi098 bash[79516]: cluster 2023-09-23T18:26:33.971889+0000 mon.a (mon.0) 312 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2023-09-23T18:26:35.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:34 smithi098 bash[79516]: audit 2023-09-23T18:26:33.972181+0000 mon.a (mon.0) 313 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T18:26:35.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:34 smithi098 bash[79516]: cluster 2023-09-23T18:26:33.975426+0000 osd.6 (osd.6) 4 : cluster 3 failed to encode map e88 with expected crc 2023-09-23T18:26:35.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:34 smithi098 bash[79516]: audit 2023-09-23T18:26:34.517664+0000 mon.c (mon.2) 12 : audit 0 from='client.? 172.21.15.98:0/298338257' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:36.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:35 smithi173 bash[59422]: cluster 2023-09-23T18:26:34.972527+0000 osd.1 (osd.1) 6 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T18:26:36.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:35 smithi173 bash[59422]: cluster 2023-09-23T18:26:34.976570+0000 mon.a (mon.0) 314 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2023-09-23T18:26:36.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:35 smithi173 bash[59422]: cluster 2023-09-23T18:26:34.976962+0000 osd.1 (osd.1) 7 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T18:26:36.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:35 smithi173 bash[59422]: cluster 2023-09-23T18:26:34.976974+0000 osd.3 (osd.3) 5 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T18:26:36.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:35 smithi173 bash[59422]: cluster 2023-09-23T18:26:34.977052+0000 osd.4 (osd.4) 8 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T18:26:36.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:35 smithi173 bash[59422]: cluster 2023-09-23T18:26:34.977210+0000 osd.2 (osd.2) 5 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T18:26:36.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:35 smithi173 bash[59422]: cluster 2023-09-23T18:26:34.980020+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T18:26:36.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:35 smithi173 bash[59422]: cluster 2023-09-23T18:26:34.981278+0000 osd.4 (osd.4) 9 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T18:26:36.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:35 smithi173 bash[59422]: cluster 2023-09-23T18:26:34.981347+0000 osd.6 (osd.6) 5 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T18:26:36.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:35 smithi173 bash[59422]: cluster 2023-09-23T18:26:35.552874+0000 mgr.y (mgr.26054) 218 : cluster 0 pgmap v156: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T18:26:36.443 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:26:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:26:36] "GET /metrics HTTP/1.1" 200 33973 "" "Prometheus/2.33.4" 2023-09-23T18:26:36.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:35 smithi098 bash[76817]: cluster 2023-09-23T18:26:34.972527+0000 osd.1 (osd.1) 6 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T18:26:36.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:35 smithi098 bash[76817]: cluster 2023-09-23T18:26:34.976570+0000 mon.a (mon.0) 314 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2023-09-23T18:26:36.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:35 smithi098 bash[76817]: cluster 2023-09-23T18:26:34.976962+0000 osd.1 (osd.1) 7 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T18:26:36.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:35 smithi098 bash[76817]: cluster 2023-09-23T18:26:34.976974+0000 osd.3 (osd.3) 5 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T18:26:36.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:35 smithi098 bash[76817]: cluster 2023-09-23T18:26:34.977052+0000 osd.4 (osd.4) 8 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T18:26:36.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:35 smithi098 bash[76817]: cluster 2023-09-23T18:26:34.977210+0000 osd.2 (osd.2) 5 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T18:26:36.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:35 smithi098 bash[76817]: cluster 2023-09-23T18:26:34.980020+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T18:26:36.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:35 smithi098 bash[76817]: cluster 2023-09-23T18:26:34.981278+0000 osd.4 (osd.4) 9 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T18:26:36.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:35 smithi098 bash[76817]: cluster 2023-09-23T18:26:34.981347+0000 osd.6 (osd.6) 5 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T18:26:36.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:35 smithi098 bash[76817]: cluster 2023-09-23T18:26:35.552874+0000 mgr.y (mgr.26054) 218 : cluster 0 pgmap v156: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T18:26:36.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:35 smithi098 bash[79516]: cluster 2023-09-23T18:26:34.972527+0000 osd.1 (osd.1) 6 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T18:26:36.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:35 smithi098 bash[79516]: cluster 2023-09-23T18:26:34.976570+0000 mon.a (mon.0) 314 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2023-09-23T18:26:36.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:35 smithi098 bash[79516]: cluster 2023-09-23T18:26:34.976962+0000 osd.1 (osd.1) 7 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T18:26:36.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:35 smithi098 bash[79516]: cluster 2023-09-23T18:26:34.976974+0000 osd.3 (osd.3) 5 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T18:26:36.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:35 smithi098 bash[79516]: cluster 2023-09-23T18:26:34.977052+0000 osd.4 (osd.4) 8 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T18:26:36.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:35 smithi098 bash[79516]: cluster 2023-09-23T18:26:34.977210+0000 osd.2 (osd.2) 5 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T18:26:36.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:35 smithi098 bash[79516]: cluster 2023-09-23T18:26:34.980020+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T18:26:36.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:35 smithi098 bash[79516]: cluster 2023-09-23T18:26:34.981278+0000 osd.4 (osd.4) 9 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T18:26:36.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:35 smithi098 bash[79516]: cluster 2023-09-23T18:26:34.981347+0000 osd.6 (osd.6) 5 : cluster 3 failed to encode map e89 with expected crc 2023-09-23T18:26:36.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:35 smithi098 bash[79516]: cluster 2023-09-23T18:26:35.552874+0000 mgr.y (mgr.26054) 218 : cluster 0 pgmap v156: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T18:26:37.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:36 smithi173 bash[59422]: cluster 2023-09-23T18:26:35.971337+0000 mon.a (mon.0) 315 : cluster 3 Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:26:37.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:36 smithi173 bash[59422]: audit 2023-09-23T18:26:36.880874+0000 mon.a (mon.0) 316 : audit 0 from='client.? 172.21.15.98:0/3253035249' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:37.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:36 smithi098 bash[76817]: cluster 2023-09-23T18:26:35.971337+0000 mon.a (mon.0) 315 : cluster 3 Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:26:37.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:36 smithi098 bash[76817]: audit 2023-09-23T18:26:36.880874+0000 mon.a (mon.0) 316 : audit 0 from='client.? 172.21.15.98:0/3253035249' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:37.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:36 smithi098 bash[79516]: cluster 2023-09-23T18:26:35.971337+0000 mon.a (mon.0) 315 : cluster 3 Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:26:37.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:36 smithi098 bash[79516]: audit 2023-09-23T18:26:36.880874+0000 mon.a (mon.0) 316 : audit 0 from='client.? 172.21.15.98:0/3253035249' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:38.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:38 smithi173 bash[59422]: cluster 2023-09-23T18:26:37.553585+0000 mgr.y (mgr.26054) 219 : cluster 0 pgmap v157: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T18:26:38.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:37 smithi098 bash[76817]: cluster 2023-09-23T18:26:37.553585+0000 mgr.y (mgr.26054) 219 : cluster 0 pgmap v157: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T18:26:38.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:37 smithi098 bash[79516]: cluster 2023-09-23T18:26:37.553585+0000 mgr.y (mgr.26054) 219 : cluster 0 pgmap v157: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T18:26:39.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:39 smithi098 bash[76817]: audit 2023-09-23T18:26:39.242795+0000 mon.a (mon.0) 317 : audit 0 from='client.? 172.21.15.98:0/3970115456' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:39.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:39 smithi098 bash[79516]: audit 2023-09-23T18:26:39.242795+0000 mon.a (mon.0) 317 : audit 0 from='client.? 172.21.15.98:0/3970115456' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:39.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:39 smithi173 bash[59422]: audit 2023-09-23T18:26:39.242795+0000 mon.a (mon.0) 317 : audit 0 from='client.? 172.21.15.98:0/3970115456' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:40.870 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:40 smithi098 bash[76817]: cluster 2023-09-23T18:26:39.554214+0000 mgr.y (mgr.26054) 220 : cluster 0 pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:40.871 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:40 smithi098 bash[76817]: audit 2023-09-23T18:26:39.577758+0000 mon.a (mon.0) 318 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:40.871 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:40 smithi098 bash[76817]: audit 2023-09-23T18:26:39.579404+0000 mon.a (mon.0) 319 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:26:40.871 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:40 smithi098 bash[76817]: cluster 2023-09-23T18:26:40.311887+0000 mon.a (mon.0) 320 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-23T18:26:40.871 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:40 smithi098 bash[76817]: cluster 2023-09-23T18:26:40.311942+0000 mon.a (mon.0) 321 : cluster 1 Cluster is now healthy 2023-09-23T18:26:40.871 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:40 smithi098 bash[79516]: cluster 2023-09-23T18:26:39.554214+0000 mgr.y (mgr.26054) 220 : cluster 0 pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:40.871 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:40 smithi098 bash[79516]: audit 2023-09-23T18:26:39.577758+0000 mon.a (mon.0) 318 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:40.871 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:40 smithi098 bash[79516]: audit 2023-09-23T18:26:39.579404+0000 mon.a (mon.0) 319 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:26:40.871 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:40 smithi098 bash[79516]: cluster 2023-09-23T18:26:40.311887+0000 mon.a (mon.0) 320 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-23T18:26:40.871 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:40 smithi098 bash[79516]: cluster 2023-09-23T18:26:40.311942+0000 mon.a (mon.0) 321 : cluster 1 Cluster is now healthy 2023-09-23T18:26:41.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:40 smithi173 bash[59422]: cluster 2023-09-23T18:26:39.554214+0000 mgr.y (mgr.26054) 220 : cluster 0 pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:41.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:40 smithi173 bash[59422]: audit 2023-09-23T18:26:39.577758+0000 mon.a (mon.0) 318 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:41.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:40 smithi173 bash[59422]: audit 2023-09-23T18:26:39.579404+0000 mon.a (mon.0) 319 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:26:41.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:40 smithi173 bash[59422]: cluster 2023-09-23T18:26:40.311887+0000 mon.a (mon.0) 320 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-23T18:26:41.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:40 smithi173 bash[59422]: cluster 2023-09-23T18:26:40.311942+0000 mon.a (mon.0) 321 : cluster 1 Cluster is now healthy 2023-09-23T18:26:42.934 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:42 smithi098 bash[76817]: cluster 2023-09-23T18:26:41.554904+0000 mgr.y (mgr.26054) 221 : cluster 0 pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:42.934 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:42 smithi098 bash[76817]: audit 2023-09-23T18:26:41.641462+0000 mon.a (mon.0) 322 : audit 0 from='client.? 172.21.15.98:0/2008222807' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:42.934 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:42 smithi098 bash[76817]: audit 2023-09-23T18:26:42.516011+0000 mon.a (mon.0) 323 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:42.934 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:42 smithi098 bash[76817]: audit 2023-09-23T18:26:42.524343+0000 mon.a (mon.0) 324 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:42.934 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:42 smithi098 bash[76817]: audit 2023-09-23T18:26:42.525953+0000 mon.a (mon.0) 325 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:26:42.934 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:42 smithi098 bash[76817]: audit 2023-09-23T18:26:42.527295+0000 mon.a (mon.0) 326 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:26:42.935 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:42 smithi098 bash[76817]: audit 2023-09-23T18:26:42.535453+0000 mon.a (mon.0) 327 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:42.935 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:42 smithi098 bash[79516]: cluster 2023-09-23T18:26:41.554904+0000 mgr.y (mgr.26054) 221 : cluster 0 pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:42.935 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:42 smithi098 bash[79516]: audit 2023-09-23T18:26:41.641462+0000 mon.a (mon.0) 322 : audit 0 from='client.? 172.21.15.98:0/2008222807' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:42.935 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:42 smithi098 bash[79516]: audit 2023-09-23T18:26:42.516011+0000 mon.a (mon.0) 323 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:42.935 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:42 smithi098 bash[79516]: audit 2023-09-23T18:26:42.524343+0000 mon.a (mon.0) 324 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:42.935 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:42 smithi098 bash[79516]: audit 2023-09-23T18:26:42.525953+0000 mon.a (mon.0) 325 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:26:42.935 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:42 smithi098 bash[79516]: audit 2023-09-23T18:26:42.527295+0000 mon.a (mon.0) 326 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:26:42.935 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:42 smithi098 bash[79516]: audit 2023-09-23T18:26:42.535453+0000 mon.a (mon.0) 327 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:43.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:42 smithi173 bash[59422]: cluster 2023-09-23T18:26:41.554904+0000 mgr.y (mgr.26054) 221 : cluster 0 pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:43.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:42 smithi173 bash[59422]: audit 2023-09-23T18:26:41.641462+0000 mon.a (mon.0) 322 : audit 0 from='client.? 172.21.15.98:0/2008222807' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:43.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:42 smithi173 bash[59422]: audit 2023-09-23T18:26:42.516011+0000 mon.a (mon.0) 323 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:43.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:42 smithi173 bash[59422]: audit 2023-09-23T18:26:42.524343+0000 mon.a (mon.0) 324 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:43.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:42 smithi173 bash[59422]: audit 2023-09-23T18:26:42.525953+0000 mon.a (mon.0) 325 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:26:43.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:42 smithi173 bash[59422]: audit 2023-09-23T18:26:42.527295+0000 mon.a (mon.0) 326 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:26:43.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:42 smithi173 bash[59422]: audit 2023-09-23T18:26:42.535453+0000 mon.a (mon.0) 327 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:43.938 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:43 smithi098 bash[76817]: audit 2023-09-23T18:26:42.619653+0000 mon.a (mon.0) 328 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:26:43.939 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:43 smithi098 bash[76817]: audit 2023-09-23T18:26:42.624386+0000 mon.a (mon.0) 329 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:26:43.939 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:43 smithi098 bash[76817]: audit 2023-09-23T18:26:42.626953+0000 mon.a (mon.0) 330 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:26:43.939 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:43 smithi098 bash[76817]: audit 2023-09-23T18:26:42.628960+0000 mon.a (mon.0) 331 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:26:43.939 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:43 smithi098 bash[76817]: audit 2023-09-23T18:26:42.631269+0000 mon.a (mon.0) 332 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T18:26:43.939 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:43 smithi098 bash[76817]: audit 2023-09-23T18:26:42.631556+0000 mgr.y (mgr.26054) 222 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T18:26:43.939 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:43 smithi098 bash[76817]: cephadm 2023-09-23T18:26:42.632948+0000 mgr.y (mgr.26054) 223 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-09-23T18:26:43.939 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:43 smithi098 bash[79516]: audit 2023-09-23T18:26:42.619653+0000 mon.a (mon.0) 328 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:26:43.939 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:43 smithi098 bash[79516]: audit 2023-09-23T18:26:42.624386+0000 mon.a (mon.0) 329 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:26:43.939 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:43 smithi098 bash[79516]: audit 2023-09-23T18:26:42.626953+0000 mon.a (mon.0) 330 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:26:43.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:43 smithi098 bash[79516]: audit 2023-09-23T18:26:42.628960+0000 mon.a (mon.0) 331 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:26:43.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:43 smithi098 bash[79516]: audit 2023-09-23T18:26:42.631269+0000 mon.a (mon.0) 332 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T18:26:43.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:43 smithi098 bash[79516]: audit 2023-09-23T18:26:42.631556+0000 mgr.y (mgr.26054) 222 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T18:26:43.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:43 smithi098 bash[79516]: cephadm 2023-09-23T18:26:42.632948+0000 mgr.y (mgr.26054) 223 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-09-23T18:26:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:43 smithi173 bash[59422]: audit 2023-09-23T18:26:42.619653+0000 mon.a (mon.0) 328 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:26:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:43 smithi173 bash[59422]: audit 2023-09-23T18:26:42.624386+0000 mon.a (mon.0) 329 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:26:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:43 smithi173 bash[59422]: audit 2023-09-23T18:26:42.626953+0000 mon.a (mon.0) 330 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:26:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:43 smithi173 bash[59422]: audit 2023-09-23T18:26:42.628960+0000 mon.a (mon.0) 331 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:26:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:43 smithi173 bash[59422]: audit 2023-09-23T18:26:42.631269+0000 mon.a (mon.0) 332 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T18:26:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:43 smithi173 bash[59422]: audit 2023-09-23T18:26:42.631556+0000 mgr.y (mgr.26054) 222 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T18:26:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:43 smithi173 bash[59422]: cephadm 2023-09-23T18:26:42.632948+0000 mgr.y (mgr.26054) 223 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-09-23T18:26:44.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:44 smithi098 bash[79516]: cluster 2023-09-23T18:26:43.555587+0000 mgr.y (mgr.26054) 224 : cluster 0 pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:44.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:44 smithi098 bash[79516]: audit 2023-09-23T18:26:44.007711+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.98:0/814175361' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:44.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:44 smithi098 bash[76817]: cluster 2023-09-23T18:26:43.555587+0000 mgr.y (mgr.26054) 224 : cluster 0 pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:44.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:44 smithi098 bash[76817]: audit 2023-09-23T18:26:44.007711+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.98:0/814175361' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:45.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:44 smithi173 bash[59422]: cluster 2023-09-23T18:26:43.555587+0000 mgr.y (mgr.26054) 224 : cluster 0 pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:45.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:44 smithi173 bash[59422]: audit 2023-09-23T18:26:44.007711+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.98:0/814175361' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:45.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:45 smithi098 bash[76817]: audit 2023-09-23T18:26:44.627351+0000 mon.a (mon.0) 333 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:45.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:45 smithi098 bash[76817]: audit 2023-09-23T18:26:44.961263+0000 mon.a (mon.0) 334 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:45.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:45 smithi098 bash[76817]: audit 2023-09-23T18:26:44.962900+0000 mon.a (mon.0) 335 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T18:26:45.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:45 smithi098 bash[76817]: audit 2023-09-23T18:26:44.964170+0000 mon.a (mon.0) 336 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:26:45.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:45 smithi098 bash[79516]: audit 2023-09-23T18:26:44.627351+0000 mon.a (mon.0) 333 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:45.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:45 smithi098 bash[79516]: audit 2023-09-23T18:26:44.961263+0000 mon.a (mon.0) 334 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:45.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:45 smithi098 bash[79516]: audit 2023-09-23T18:26:44.962900+0000 mon.a (mon.0) 335 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T18:26:45.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:45 smithi098 bash[79516]: audit 2023-09-23T18:26:44.964170+0000 mon.a (mon.0) 336 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:26:46.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:45 smithi173 bash[59422]: audit 2023-09-23T18:26:44.627351+0000 mon.a (mon.0) 333 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:46.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:45 smithi173 bash[59422]: audit 2023-09-23T18:26:44.961263+0000 mon.a (mon.0) 334 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:46.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:45 smithi173 bash[59422]: audit 2023-09-23T18:26:44.962900+0000 mon.a (mon.0) 335 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T18:26:46.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:45 smithi173 bash[59422]: audit 2023-09-23T18:26:44.964170+0000 mon.a (mon.0) 336 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:26:46.341 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:26:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:26:46] "GET /metrics HTTP/1.1" 200 34035 "" "Prometheus/2.33.4" 2023-09-23T18:26:46.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:46 smithi098 bash[76817]: cephadm 2023-09-23T18:26:44.952061+0000 mgr.y (mgr.26054) 225 : cephadm 1 Upgrade: Updating osd.1 2023-09-23T18:26:46.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:46 smithi098 bash[76817]: cephadm 2023-09-23T18:26:44.967419+0000 mgr.y (mgr.26054) 226 : cephadm 1 Deploying daemon osd.1 on smithi098 2023-09-23T18:26:46.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:46 smithi098 bash[76817]: cluster 2023-09-23T18:26:45.556562+0000 mgr.y (mgr.26054) 227 : cluster 0 pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:46.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:46 smithi098 bash[76817]: audit 2023-09-23T18:26:46.364252+0000 mon.a (mon.0) 337 : audit 0 from='client.? 172.21.15.98:0/2435074837' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:46.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:46 smithi098 bash[79516]: cephadm 2023-09-23T18:26:44.952061+0000 mgr.y (mgr.26054) 225 : cephadm 1 Upgrade: Updating osd.1 2023-09-23T18:26:46.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:46 smithi098 bash[79516]: cephadm 2023-09-23T18:26:44.967419+0000 mgr.y (mgr.26054) 226 : cephadm 1 Deploying daemon osd.1 on smithi098 2023-09-23T18:26:46.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:46 smithi098 bash[79516]: cluster 2023-09-23T18:26:45.556562+0000 mgr.y (mgr.26054) 227 : cluster 0 pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:46.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:46 smithi098 bash[79516]: audit 2023-09-23T18:26:46.364252+0000 mon.a (mon.0) 337 : audit 0 from='client.? 172.21.15.98:0/2435074837' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:47.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:46 smithi173 bash[59422]: cephadm 2023-09-23T18:26:44.952061+0000 mgr.y (mgr.26054) 225 : cephadm 1 Upgrade: Updating osd.1 2023-09-23T18:26:47.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:46 smithi173 bash[59422]: cephadm 2023-09-23T18:26:44.967419+0000 mgr.y (mgr.26054) 226 : cephadm 1 Deploying daemon osd.1 on smithi098 2023-09-23T18:26:47.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:46 smithi173 bash[59422]: cluster 2023-09-23T18:26:45.556562+0000 mgr.y (mgr.26054) 227 : cluster 0 pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:47.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:46 smithi173 bash[59422]: audit 2023-09-23T18:26:46.364252+0000 mon.a (mon.0) 337 : audit 0 from='client.? 172.21.15.98:0/2435074837' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:48.910 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:48 smithi098 bash[79516]: cluster 2023-09-23T18:26:47.557217+0000 mgr.y (mgr.26054) 228 : cluster 0 pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:48.910 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:48 smithi098 bash[76817]: cluster 2023-09-23T18:26:47.557217+0000 mgr.y (mgr.26054) 228 : cluster 0 pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:49.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:48 smithi173 bash[59422]: cluster 2023-09-23T18:26:47.557217+0000 mgr.y (mgr.26054) 228 : cluster 0 pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:49.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:49 smithi098 bash[76817]: audit 2023-09-23T18:26:48.734200+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.98:0/4236342539' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:49.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:49 smithi098 bash[79516]: audit 2023-09-23T18:26:48.734200+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.98:0/4236342539' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:49.943 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:26:49 smithi098 systemd[1]: Stopping Ceph osd.1 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T18:26:50.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:49 smithi173 bash[59422]: audit 2023-09-23T18:26:48.734200+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.98:0/4236342539' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:50.443 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:26:50 smithi098 bash[40828]: debug 2023-09-23T18:26:50.037+0000 7f2759dde700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T18:26:50.443 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:26:50 smithi098 bash[40828]: debug 2023-09-23T18:26:50.037+0000 7f2759dde700 -1 osd.1 89 *** Got signal Terminated *** 2023-09-23T18:26:50.443 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:26:50 smithi098 bash[40828]: debug 2023-09-23T18:26:50.037+0000 7f2759dde700 -1 osd.1 89 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T18:26:50.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:50 smithi098 bash[76817]: cluster 2023-09-23T18:26:49.557895+0000 mgr.y (mgr.26054) 229 : cluster 0 pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:50.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:50 smithi098 bash[76817]: cluster 2023-09-23T18:26:50.039950+0000 mon.a (mon.0) 338 : cluster 1 osd.1 marked itself down and dead 2023-09-23T18:26:50.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:50 smithi098 bash[79516]: cluster 2023-09-23T18:26:49.557895+0000 mgr.y (mgr.26054) 229 : cluster 0 pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:50.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:50 smithi098 bash[79516]: cluster 2023-09-23T18:26:50.039950+0000 mon.a (mon.0) 338 : cluster 1 osd.1 marked itself down and dead 2023-09-23T18:26:51.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:50 smithi173 bash[59422]: cluster 2023-09-23T18:26:49.557895+0000 mgr.y (mgr.26054) 229 : cluster 0 pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:51.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:50 smithi173 bash[59422]: cluster 2023-09-23T18:26:50.039950+0000 mon.a (mon.0) 338 : cluster 1 osd.1 marked itself down and dead 2023-09-23T18:26:51.611 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:26:51 smithi098 bash[87499]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-1 2023-09-23T18:26:51.611 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:26:51 smithi098 bash[87587]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-1 2023-09-23T18:26:51.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:51 smithi098 bash[79516]: cluster 2023-09-23T18:26:50.639420+0000 mon.a (mon.0) 339 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:26:51.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:51 smithi098 bash[79516]: cluster 2023-09-23T18:26:50.645890+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e90 with expected crc 2023-09-23T18:26:51.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:51 smithi098 bash[79516]: cluster 2023-09-23T18:26:50.646665+0000 osd.4 (osd.4) 10 : cluster 3 failed to encode map e90 with expected crc 2023-09-23T18:26:51.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:51 smithi098 bash[79516]: cluster 2023-09-23T18:26:50.646855+0000 mon.a (mon.0) 340 : cluster 0 osdmap e90: 8 total, 7 up, 8 in 2023-09-23T18:26:51.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:51 smithi098 bash[79516]: audit 2023-09-23T18:26:51.083703+0000 mon.a (mon.0) 341 : audit 0 from='client.? 172.21.15.98:0/4154011666' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:51.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:51 smithi098 bash[76817]: cluster 2023-09-23T18:26:50.639420+0000 mon.a (mon.0) 339 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:26:51.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:51 smithi098 bash[76817]: cluster 2023-09-23T18:26:50.645890+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e90 with expected crc 2023-09-23T18:26:51.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:51 smithi098 bash[76817]: cluster 2023-09-23T18:26:50.646665+0000 osd.4 (osd.4) 10 : cluster 3 failed to encode map e90 with expected crc 2023-09-23T18:26:51.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:51 smithi098 bash[76817]: cluster 2023-09-23T18:26:50.646855+0000 mon.a (mon.0) 340 : cluster 0 osdmap e90: 8 total, 7 up, 8 in 2023-09-23T18:26:51.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:51 smithi098 bash[76817]: audit 2023-09-23T18:26:51.083703+0000 mon.a (mon.0) 341 : audit 0 from='client.? 172.21.15.98:0/4154011666' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:52.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:51 smithi173 bash[59422]: cluster 2023-09-23T18:26:50.639420+0000 mon.a (mon.0) 339 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:26:52.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:51 smithi173 bash[59422]: cluster 2023-09-23T18:26:50.645890+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e90 with expected crc 2023-09-23T18:26:52.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:51 smithi173 bash[59422]: cluster 2023-09-23T18:26:50.646665+0000 osd.4 (osd.4) 10 : cluster 3 failed to encode map e90 with expected crc 2023-09-23T18:26:52.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:51 smithi173 bash[59422]: cluster 2023-09-23T18:26:50.646855+0000 mon.a (mon.0) 340 : cluster 0 osdmap e90: 8 total, 7 up, 8 in 2023-09-23T18:26:52.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:51 smithi173 bash[59422]: audit 2023-09-23T18:26:51.083703+0000 mon.a (mon.0) 341 : audit 0 from='client.? 172.21.15.98:0/4154011666' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:52.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:52 smithi098 bash[79516]: cluster 2023-09-23T18:26:51.558570+0000 mgr.y (mgr.26054) 230 : cluster 0 pgmap v165: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:52.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:52 smithi098 bash[79516]: cluster 2023-09-23T18:26:51.657540+0000 osd.2 (osd.2) 6 : cluster 3 failed to encode map e91 with expected crc 2023-09-23T18:26:52.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:52 smithi098 bash[79516]: cluster 2023-09-23T18:26:51.661758+0000 osd.4 (osd.4) 11 : cluster 3 failed to encode map e91 with expected crc 2023-09-23T18:26:52.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:52 smithi098 bash[79516]: cluster 2023-09-23T18:26:51.664456+0000 mon.a (mon.0) 342 : cluster 0 osdmap e91: 8 total, 7 up, 8 in 2023-09-23T18:26:52.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:52 smithi098 bash[79516]: cluster 2023-09-23T18:26:51.664831+0000 osd.3 (osd.3) 6 : cluster 3 failed to encode map e91 with expected crc 2023-09-23T18:26:52.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:52 smithi098 bash[79516]: cluster 2023-09-23T18:26:51.664868+0000 osd.2 (osd.2) 7 : cluster 3 failed to encode map e91 with expected crc 2023-09-23T18:26:52.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:52 smithi098 bash[79516]: cluster 2023-09-23T18:26:51.667568+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e91 with expected crc 2023-09-23T18:26:52.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:52 smithi098 bash[79516]: cluster 2023-09-23T18:26:51.667727+0000 osd.6 (osd.6) 6 : cluster 3 failed to encode map e91 with expected crc 2023-09-23T18:26:52.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:52 smithi098 bash[79516]: cluster 2023-09-23T18:26:51.667751+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e91 with expected crc 2023-09-23T18:26:52.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:52 smithi098 bash[76817]: cluster 2023-09-23T18:26:51.558570+0000 mgr.y (mgr.26054) 230 : cluster 0 pgmap v165: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:52.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:52 smithi098 bash[76817]: cluster 2023-09-23T18:26:51.657540+0000 osd.2 (osd.2) 6 : cluster 3 failed to encode map e91 with expected crc 2023-09-23T18:26:52.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:52 smithi098 bash[76817]: cluster 2023-09-23T18:26:51.661758+0000 osd.4 (osd.4) 11 : cluster 3 failed to encode map e91 with expected crc 2023-09-23T18:26:52.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:52 smithi098 bash[76817]: cluster 2023-09-23T18:26:51.664456+0000 mon.a (mon.0) 342 : cluster 0 osdmap e91: 8 total, 7 up, 8 in 2023-09-23T18:26:52.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:52 smithi098 bash[76817]: cluster 2023-09-23T18:26:51.664831+0000 osd.3 (osd.3) 6 : cluster 3 failed to encode map e91 with expected crc 2023-09-23T18:26:52.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:52 smithi098 bash[76817]: cluster 2023-09-23T18:26:51.664868+0000 osd.2 (osd.2) 7 : cluster 3 failed to encode map e91 with expected crc 2023-09-23T18:26:52.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:52 smithi098 bash[76817]: cluster 2023-09-23T18:26:51.667568+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e91 with expected crc 2023-09-23T18:26:52.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:52 smithi098 bash[76817]: cluster 2023-09-23T18:26:51.667727+0000 osd.6 (osd.6) 6 : cluster 3 failed to encode map e91 with expected crc 2023-09-23T18:26:52.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:52 smithi098 bash[76817]: cluster 2023-09-23T18:26:51.667751+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e91 with expected crc 2023-09-23T18:26:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:52 smithi173 bash[59422]: cluster 2023-09-23T18:26:51.558570+0000 mgr.y (mgr.26054) 230 : cluster 0 pgmap v165: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:26:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:52 smithi173 bash[59422]: cluster 2023-09-23T18:26:51.657540+0000 osd.2 (osd.2) 6 : cluster 3 failed to encode map e91 with expected crc 2023-09-23T18:26:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:52 smithi173 bash[59422]: cluster 2023-09-23T18:26:51.661758+0000 osd.4 (osd.4) 11 : cluster 3 failed to encode map e91 with expected crc 2023-09-23T18:26:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:52 smithi173 bash[59422]: cluster 2023-09-23T18:26:51.664456+0000 mon.a (mon.0) 342 : cluster 0 osdmap e91: 8 total, 7 up, 8 in 2023-09-23T18:26:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:52 smithi173 bash[59422]: cluster 2023-09-23T18:26:51.664831+0000 osd.3 (osd.3) 6 : cluster 3 failed to encode map e91 with expected crc 2023-09-23T18:26:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:52 smithi173 bash[59422]: cluster 2023-09-23T18:26:51.664868+0000 osd.2 (osd.2) 7 : cluster 3 failed to encode map e91 with expected crc 2023-09-23T18:26:53.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:52 smithi173 bash[59422]: cluster 2023-09-23T18:26:51.667568+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e91 with expected crc 2023-09-23T18:26:53.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:52 smithi173 bash[59422]: cluster 2023-09-23T18:26:51.667727+0000 osd.6 (osd.6) 6 : cluster 3 failed to encode map e91 with expected crc 2023-09-23T18:26:53.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:52 smithi173 bash[59422]: cluster 2023-09-23T18:26:51.667751+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e91 with expected crc 2023-09-23T18:26:53.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:53 smithi098 bash[79516]: audit 2023-09-23T18:26:53.448852+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.98:0/2696949452' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:53.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:53 smithi098 bash[76817]: audit 2023-09-23T18:26:53.448852+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.98:0/2696949452' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:53.943 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:26:53 smithi098 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.1.service: Succeeded. 2023-09-23T18:26:53.943 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:26:53 smithi098 systemd[1]: Stopped Ceph osd.1 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:26:54.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:53 smithi173 bash[59422]: audit 2023-09-23T18:26:53.448852+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.98:0/2696949452' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:54.443 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:26:54 smithi098 systemd[1]: Started Ceph osd.1 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:26:54.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:54 smithi098 bash[76817]: cluster 2023-09-23T18:26:53.559298+0000 mgr.y (mgr.26054) 231 : cluster 0 pgmap v167: 73 pgs: 2 active+undersized+degraded, 9 stale+active+clean, 1 active+undersized, 61 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T18:26:54.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:54 smithi098 bash[76817]: cluster 2023-09-23T18:26:53.659010+0000 mon.a (mon.0) 343 : cluster 3 Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:26:54.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:54 smithi098 bash[76817]: audit 2023-09-23T18:26:54.162134+0000 mon.a (mon.0) 344 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:54.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:54 smithi098 bash[76817]: audit 2023-09-23T18:26:54.171259+0000 mon.a (mon.0) 345 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:54.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:54 smithi098 bash[76817]: audit 2023-09-23T18:26:54.203230+0000 mon.a (mon.0) 346 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:26:54.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:54 smithi098 bash[76817]: audit 2023-09-23T18:26:54.579433+0000 mon.a (mon.0) 347 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:54.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:54 smithi098 bash[76817]: audit 2023-09-23T18:26:54.580775+0000 mon.a (mon.0) 348 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:26:54.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:54 smithi098 bash[79516]: cluster 2023-09-23T18:26:53.559298+0000 mgr.y (mgr.26054) 231 : cluster 0 pgmap v167: 73 pgs: 2 active+undersized+degraded, 9 stale+active+clean, 1 active+undersized, 61 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T18:26:54.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:54 smithi098 bash[79516]: cluster 2023-09-23T18:26:53.659010+0000 mon.a (mon.0) 343 : cluster 3 Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:26:54.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:54 smithi098 bash[79516]: audit 2023-09-23T18:26:54.162134+0000 mon.a (mon.0) 344 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:54.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:54 smithi098 bash[79516]: audit 2023-09-23T18:26:54.171259+0000 mon.a (mon.0) 345 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:54.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:54 smithi098 bash[79516]: audit 2023-09-23T18:26:54.203230+0000 mon.a (mon.0) 346 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:26:54.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:54 smithi098 bash[79516]: audit 2023-09-23T18:26:54.579433+0000 mon.a (mon.0) 347 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:54.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:54 smithi098 bash[79516]: audit 2023-09-23T18:26:54.580775+0000 mon.a (mon.0) 348 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:26:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:54 smithi173 bash[59422]: cluster 2023-09-23T18:26:53.559298+0000 mgr.y (mgr.26054) 231 : cluster 0 pgmap v167: 73 pgs: 2 active+undersized+degraded, 9 stale+active+clean, 1 active+undersized, 61 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T18:26:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:54 smithi173 bash[59422]: cluster 2023-09-23T18:26:53.659010+0000 mon.a (mon.0) 343 : cluster 3 Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:26:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:54 smithi173 bash[59422]: audit 2023-09-23T18:26:54.162134+0000 mon.a (mon.0) 344 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:54 smithi173 bash[59422]: audit 2023-09-23T18:26:54.171259+0000 mon.a (mon.0) 345 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:54 smithi173 bash[59422]: audit 2023-09-23T18:26:54.203230+0000 mon.a (mon.0) 346 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:26:55.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:54 smithi173 bash[59422]: audit 2023-09-23T18:26:54.579433+0000 mon.a (mon.0) 347 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:26:55.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:54 smithi173 bash[59422]: audit 2023-09-23T18:26:54.580775+0000 mon.a (mon.0) 348 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:26:56.087 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:26:55 smithi098 bash[87771]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T18:26:56.088 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:26:55 smithi098 bash[87771]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T18:26:56.088 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:26:55 smithi098 bash[87771]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-09-23T18:26:56.088 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:26:55 smithi098 bash[87771]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-09-23T18:26:56.088 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:26:55 smithi098 bash[87771]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-09-23T18:26:56.088 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:26:55 smithi098 bash[87771]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-09-23T18:26:56.088 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:26:55 smithi098 bash[87771]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T18:26:56.088 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:26:55 smithi098 bash[87771]: --> ceph-volume lvm activate successful for osd ID: 1 2023-09-23T18:26:56.443 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:26:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:26:56] "GET /metrics HTTP/1.1" 200 34035 "" "Prometheus/2.33.4" 2023-09-23T18:26:56.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:56 smithi098 bash[76817]: cluster 2023-09-23T18:26:55.560306+0000 mgr.y (mgr.26054) 232 : cluster 0 pgmap v168: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:26:56.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:56 smithi098 bash[76817]: audit 2023-09-23T18:26:55.808145+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.98:0/3942413639' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:56.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:56 smithi098 bash[79516]: cluster 2023-09-23T18:26:55.560306+0000 mgr.y (mgr.26054) 232 : cluster 0 pgmap v168: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:26:56.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:56 smithi098 bash[79516]: audit 2023-09-23T18:26:55.808145+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.98:0/3942413639' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:57.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:56 smithi173 bash[59422]: cluster 2023-09-23T18:26:55.560306+0000 mgr.y (mgr.26054) 232 : cluster 0 pgmap v168: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:26:57.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:56 smithi173 bash[59422]: audit 2023-09-23T18:26:55.808145+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.98:0/3942413639' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:57.942 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:26:57 smithi098 bash[88087]: debug 2023-09-23T18:26:57.781+0000 7fdd0e187700 1 -- 172.21.15.98:0/1321461202 <== mon.2 v2:172.21.15.98:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x557b9448a4e0 con 0x557b944e4c00 2023-09-23T18:26:58.683 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:26:58 smithi098 bash[88087]: debug 2023-09-23T18:26:58.337+0000 7fdd1730c640 -1 Falling back to public interface 2023-09-23T18:26:58.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:58 smithi098 bash[79516]: cluster 2023-09-23T18:26:57.561113+0000 mgr.y (mgr.26054) 233 : cluster 0 pgmap v169: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:26:58.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:58 smithi098 bash[79516]: audit 2023-09-23T18:26:58.166555+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.98:0/1376722373' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:58.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:58 smithi098 bash[76817]: cluster 2023-09-23T18:26:57.561113+0000 mgr.y (mgr.26054) 233 : cluster 0 pgmap v169: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:26:58.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:58 smithi098 bash[76817]: audit 2023-09-23T18:26:58.166555+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.98:0/1376722373' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:59.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:58 smithi173 bash[59422]: cluster 2023-09-23T18:26:57.561113+0000 mgr.y (mgr.26054) 233 : cluster 0 pgmap v169: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:26:59.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:58 smithi173 bash[59422]: audit 2023-09-23T18:26:58.166555+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.98:0/1376722373' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:26:59.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:26:59 smithi098 bash[76817]: cluster 2023-09-23T18:26:59.314419+0000 mon.a (mon.0) 349 : cluster 3 Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T18:26:59.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:26:59 smithi098 bash[79516]: cluster 2023-09-23T18:26:59.314419+0000 mon.a (mon.0) 349 : cluster 3 Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T18:27:00.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:26:59 smithi173 bash[59422]: cluster 2023-09-23T18:26:59.314419+0000 mon.a (mon.0) 349 : cluster 3 Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T18:27:00.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:00 smithi098 bash[79516]: cluster 2023-09-23T18:26:59.562013+0000 mgr.y (mgr.26054) 234 : cluster 0 pgmap v170: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:27:00.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:00 smithi098 bash[79516]: audit 2023-09-23T18:27:00.551793+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.98:0/1944955074' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:00.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:00 smithi098 bash[76817]: cluster 2023-09-23T18:26:59.562013+0000 mgr.y (mgr.26054) 234 : cluster 0 pgmap v170: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:27:00.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:00 smithi098 bash[76817]: audit 2023-09-23T18:27:00.551793+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.98:0/1944955074' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:01.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:00 smithi173 bash[59422]: cluster 2023-09-23T18:26:59.562013+0000 mgr.y (mgr.26054) 234 : cluster 0 pgmap v170: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:27:01.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:00 smithi173 bash[59422]: audit 2023-09-23T18:27:00.551793+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.98:0/1944955074' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:02.192 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:27:01 smithi098 bash[88087]: debug 2023-09-23T18:27:01.889+0000 7fdd1730c640 -1 osd.1 89 log_to_monitors true 2023-09-23T18:27:02.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:02 smithi173 bash[59422]: audit 2023-09-23T18:27:01.197899+0000 mon.a (mon.0) 350 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:02.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:02 smithi173 bash[59422]: audit 2023-09-23T18:27:01.204843+0000 mon.a (mon.0) 351 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:02.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:02 smithi173 bash[59422]: cluster 2023-09-23T18:27:01.562893+0000 mgr.y (mgr.26054) 235 : cluster 0 pgmap v171: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:27:02.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:02 smithi173 bash[59422]: audit 2023-09-23T18:27:01.897574+0000 mon.a (mon.0) 352 : audit 1 from='osd.1 [v2:172.21.15.98:6810/1561159155,v1:172.21.15.98:6811/1561159155]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T18:27:02.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:02 smithi173 bash[59422]: audit 2023-09-23T18:27:01.996654+0000 mon.a (mon.0) 353 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:02.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:02 smithi173 bash[59422]: audit 2023-09-23T18:27:02.004465+0000 mon.a (mon.0) 354 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:02.692 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:27:02 smithi098 bash[88087]: debug 2023-09-23T18:27:02.249+0000 7fdd06f76700 -1 osd.1 89 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T18:27:02.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:02 smithi098 bash[79516]: audit 2023-09-23T18:27:01.197899+0000 mon.a (mon.0) 350 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:02.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:02 smithi098 bash[79516]: audit 2023-09-23T18:27:01.204843+0000 mon.a (mon.0) 351 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:02.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:02 smithi098 bash[79516]: cluster 2023-09-23T18:27:01.562893+0000 mgr.y (mgr.26054) 235 : cluster 0 pgmap v171: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:27:02.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:02 smithi098 bash[79516]: audit 2023-09-23T18:27:01.897574+0000 mon.a (mon.0) 352 : audit 1 from='osd.1 [v2:172.21.15.98:6810/1561159155,v1:172.21.15.98:6811/1561159155]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T18:27:02.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:02 smithi098 bash[79516]: audit 2023-09-23T18:27:01.996654+0000 mon.a (mon.0) 353 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:02.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:02 smithi098 bash[79516]: audit 2023-09-23T18:27:02.004465+0000 mon.a (mon.0) 354 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:02.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:02 smithi098 bash[76817]: audit 2023-09-23T18:27:01.197899+0000 mon.a (mon.0) 350 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:02.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:02 smithi098 bash[76817]: audit 2023-09-23T18:27:01.204843+0000 mon.a (mon.0) 351 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:02.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:02 smithi098 bash[76817]: cluster 2023-09-23T18:27:01.562893+0000 mgr.y (mgr.26054) 235 : cluster 0 pgmap v171: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:27:02.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:02 smithi098 bash[76817]: audit 2023-09-23T18:27:01.897574+0000 mon.a (mon.0) 352 : audit 1 from='osd.1 [v2:172.21.15.98:6810/1561159155,v1:172.21.15.98:6811/1561159155]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T18:27:02.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:02 smithi098 bash[76817]: audit 2023-09-23T18:27:01.996654+0000 mon.a (mon.0) 353 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:02.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:02 smithi098 bash[76817]: audit 2023-09-23T18:27:02.004465+0000 mon.a (mon.0) 354 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:03.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:03 smithi173 bash[59422]: audit 2023-09-23T18:27:02.210732+0000 mon.a (mon.0) 355 : audit 1 from='osd.1 [v2:172.21.15.98:6810/1561159155,v1:172.21.15.98:6811/1561159155]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T18:27:03.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:03 smithi173 bash[59422]: cluster 2023-09-23T18:27:02.210798+0000 mon.a (mon.0) 356 : cluster 0 osdmap e92: 8 total, 7 up, 8 in 2023-09-23T18:27:03.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:03 smithi173 bash[59422]: audit 2023-09-23T18:27:02.211398+0000 mon.a (mon.0) 357 : audit 1 from='osd.1 [v2:172.21.15.98:6810/1561159155,v1:172.21.15.98:6811/1561159155]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:27:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:03 smithi173 bash[59422]: cluster 2023-09-23T18:27:02.211469+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T18:27:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:03 smithi173 bash[59422]: cluster 2023-09-23T18:27:02.211589+0000 osd.4 (osd.4) 12 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T18:27:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:03 smithi173 bash[59422]: audit 2023-09-23T18:27:02.920774+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.98:0/316698641' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:03.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:03 smithi098 bash[76817]: audit 2023-09-23T18:27:02.210732+0000 mon.a (mon.0) 355 : audit 1 from='osd.1 [v2:172.21.15.98:6810/1561159155,v1:172.21.15.98:6811/1561159155]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T18:27:03.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:03 smithi098 bash[76817]: cluster 2023-09-23T18:27:02.210798+0000 mon.a (mon.0) 356 : cluster 0 osdmap e92: 8 total, 7 up, 8 in 2023-09-23T18:27:03.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:03 smithi098 bash[76817]: audit 2023-09-23T18:27:02.211398+0000 mon.a (mon.0) 357 : audit 1 from='osd.1 [v2:172.21.15.98:6810/1561159155,v1:172.21.15.98:6811/1561159155]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:27:03.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:03 smithi098 bash[76817]: cluster 2023-09-23T18:27:02.211469+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T18:27:03.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:03 smithi098 bash[76817]: cluster 2023-09-23T18:27:02.211589+0000 osd.4 (osd.4) 12 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T18:27:03.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:03 smithi098 bash[76817]: audit 2023-09-23T18:27:02.920774+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.98:0/316698641' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:03.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:03 smithi098 bash[79516]: audit 2023-09-23T18:27:02.210732+0000 mon.a (mon.0) 355 : audit 1 from='osd.1 [v2:172.21.15.98:6810/1561159155,v1:172.21.15.98:6811/1561159155]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T18:27:03.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:03 smithi098 bash[79516]: cluster 2023-09-23T18:27:02.210798+0000 mon.a (mon.0) 356 : cluster 0 osdmap e92: 8 total, 7 up, 8 in 2023-09-23T18:27:03.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:03 smithi098 bash[79516]: audit 2023-09-23T18:27:02.211398+0000 mon.a (mon.0) 357 : audit 1 from='osd.1 [v2:172.21.15.98:6810/1561159155,v1:172.21.15.98:6811/1561159155]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:27:03.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:03 smithi098 bash[79516]: cluster 2023-09-23T18:27:02.211469+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T18:27:03.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:03 smithi098 bash[79516]: cluster 2023-09-23T18:27:02.211589+0000 osd.4 (osd.4) 12 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T18:27:03.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:03 smithi098 bash[79516]: audit 2023-09-23T18:27:02.920774+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.98:0/316698641' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:04.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:04 smithi173 bash[59422]: cluster 2023-09-23T18:27:02.791228+0000 osd.6 (osd.6) 7 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T18:27:04.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:04 smithi173 bash[59422]: cluster 2023-09-23T18:27:03.155504+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T18:27:04.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:04 smithi173 bash[59422]: cluster 2023-09-23T18:27:03.209467+0000 mon.a (mon.0) 358 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:27:04.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:04 smithi173 bash[59422]: cluster 2023-09-23T18:27:03.221130+0000 mon.a (mon.0) 359 : cluster 1 osd.1 [v2:172.21.15.98:6810/1561159155,v1:172.21.15.98:6811/1561159155] boot 2023-09-23T18:27:04.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:04 smithi173 bash[59422]: cluster 2023-09-23T18:27:03.221238+0000 mon.a (mon.0) 360 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2023-09-23T18:27:04.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:04 smithi173 bash[59422]: cluster 2023-09-23T18:27:03.221909+0000 osd.4 (osd.4) 13 : cluster 3 failed to encode map e93 with expected crc 2023-09-23T18:27:04.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:04 smithi173 bash[59422]: audit 2023-09-23T18:27:03.225302+0000 mon.a (mon.0) 361 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:27:04.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:04 smithi173 bash[59422]: cluster 2023-09-23T18:27:03.227276+0000 osd.3 (osd.3) 7 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T18:27:04.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:04 smithi173 bash[59422]: cluster 2023-09-23T18:27:03.228938+0000 osd.2 (osd.2) 8 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T18:27:04.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:04 smithi173 bash[59422]: cluster 2023-09-23T18:27:03.563560+0000 mgr.y (mgr.26054) 236 : cluster 0 pgmap v174: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:27:04.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:04 smithi098 bash[76817]: cluster 2023-09-23T18:27:02.791228+0000 osd.6 (osd.6) 7 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T18:27:04.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:04 smithi098 bash[76817]: cluster 2023-09-23T18:27:03.155504+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T18:27:04.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:04 smithi098 bash[76817]: cluster 2023-09-23T18:27:03.209467+0000 mon.a (mon.0) 358 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:27:04.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:04 smithi098 bash[76817]: cluster 2023-09-23T18:27:03.221130+0000 mon.a (mon.0) 359 : cluster 1 osd.1 [v2:172.21.15.98:6810/1561159155,v1:172.21.15.98:6811/1561159155] boot 2023-09-23T18:27:04.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:04 smithi098 bash[76817]: cluster 2023-09-23T18:27:03.221238+0000 mon.a (mon.0) 360 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2023-09-23T18:27:04.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:04 smithi098 bash[76817]: cluster 2023-09-23T18:27:03.221909+0000 osd.4 (osd.4) 13 : cluster 3 failed to encode map e93 with expected crc 2023-09-23T18:27:04.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:04 smithi098 bash[76817]: audit 2023-09-23T18:27:03.225302+0000 mon.a (mon.0) 361 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:27:04.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:04 smithi098 bash[76817]: cluster 2023-09-23T18:27:03.227276+0000 osd.3 (osd.3) 7 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T18:27:04.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:04 smithi098 bash[76817]: cluster 2023-09-23T18:27:03.228938+0000 osd.2 (osd.2) 8 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T18:27:04.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:04 smithi098 bash[76817]: cluster 2023-09-23T18:27:03.563560+0000 mgr.y (mgr.26054) 236 : cluster 0 pgmap v174: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:27:04.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:04 smithi098 bash[79516]: cluster 2023-09-23T18:27:02.791228+0000 osd.6 (osd.6) 7 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T18:27:04.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:04 smithi098 bash[79516]: cluster 2023-09-23T18:27:03.155504+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T18:27:04.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:04 smithi098 bash[79516]: cluster 2023-09-23T18:27:03.209467+0000 mon.a (mon.0) 358 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:27:04.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:04 smithi098 bash[79516]: cluster 2023-09-23T18:27:03.221130+0000 mon.a (mon.0) 359 : cluster 1 osd.1 [v2:172.21.15.98:6810/1561159155,v1:172.21.15.98:6811/1561159155] boot 2023-09-23T18:27:04.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:04 smithi098 bash[79516]: cluster 2023-09-23T18:27:03.221238+0000 mon.a (mon.0) 360 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2023-09-23T18:27:04.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:04 smithi098 bash[79516]: cluster 2023-09-23T18:27:03.221909+0000 osd.4 (osd.4) 13 : cluster 3 failed to encode map e93 with expected crc 2023-09-23T18:27:04.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:04 smithi098 bash[79516]: audit 2023-09-23T18:27:03.225302+0000 mon.a (mon.0) 361 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:27:04.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:04 smithi098 bash[79516]: cluster 2023-09-23T18:27:03.227276+0000 osd.3 (osd.3) 7 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T18:27:04.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:04 smithi098 bash[79516]: cluster 2023-09-23T18:27:03.228938+0000 osd.2 (osd.2) 8 : cluster 3 failed to encode map e92 with expected crc 2023-09-23T18:27:04.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:04 smithi098 bash[79516]: cluster 2023-09-23T18:27:03.563560+0000 mgr.y (mgr.26054) 236 : cluster 0 pgmap v174: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:27:05.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:05 smithi173 bash[59422]: cluster 2023-09-23T18:27:04.231246+0000 mon.a (mon.0) 362 : cluster 0 osdmap e94: 8 total, 8 up, 8 in 2023-09-23T18:27:05.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:05 smithi173 bash[59422]: cluster 2023-09-23T18:27:04.231671+0000 osd.3 (osd.3) 8 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:05 smithi173 bash[59422]: cluster 2023-09-23T18:27:04.231922+0000 osd.2 (osd.2) 9 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:05 smithi173 bash[59422]: cluster 2023-09-23T18:27:04.232412+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:05 smithi173 bash[59422]: cluster 2023-09-23T18:27:04.234491+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:05 smithi173 bash[59422]: cluster 2023-09-23T18:27:04.234869+0000 osd.4 (osd.4) 14 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:05 smithi173 bash[59422]: cluster 2023-09-23T18:27:04.235758+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:05 smithi173 bash[59422]: cluster 2023-09-23T18:27:04.237602+0000 osd.4 (osd.4) 15 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:05 smithi173 bash[59422]: cluster 2023-09-23T18:27:04.237849+0000 osd.6 (osd.6) 8 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:05 smithi173 bash[59422]: cluster 2023-09-23T18:27:04.238654+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:05 smithi173 bash[59422]: cluster 2023-09-23T18:27:04.239551+0000 osd.3 (osd.3) 9 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:05 smithi173 bash[59422]: cluster 2023-09-23T18:27:04.241956+0000 osd.4 (osd.4) 16 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:05 smithi173 bash[59422]: audit 2023-09-23T18:27:04.649650+0000 mon.a (mon.0) 363 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:05.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[79516]: cluster 2023-09-23T18:27:04.231246+0000 mon.a (mon.0) 362 : cluster 0 osdmap e94: 8 total, 8 up, 8 in 2023-09-23T18:27:05.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[79516]: cluster 2023-09-23T18:27:04.231671+0000 osd.3 (osd.3) 8 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[79516]: cluster 2023-09-23T18:27:04.231922+0000 osd.2 (osd.2) 9 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[79516]: cluster 2023-09-23T18:27:04.232412+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[79516]: cluster 2023-09-23T18:27:04.234491+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[79516]: cluster 2023-09-23T18:27:04.234869+0000 osd.4 (osd.4) 14 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[79516]: cluster 2023-09-23T18:27:04.235758+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[79516]: cluster 2023-09-23T18:27:04.237602+0000 osd.4 (osd.4) 15 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[79516]: cluster 2023-09-23T18:27:04.237849+0000 osd.6 (osd.6) 8 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[79516]: cluster 2023-09-23T18:27:04.238654+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[79516]: cluster 2023-09-23T18:27:04.239551+0000 osd.3 (osd.3) 9 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[79516]: cluster 2023-09-23T18:27:04.241956+0000 osd.4 (osd.4) 16 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[79516]: audit 2023-09-23T18:27:04.649650+0000 mon.a (mon.0) 363 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:05.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[76817]: cluster 2023-09-23T18:27:04.231246+0000 mon.a (mon.0) 362 : cluster 0 osdmap e94: 8 total, 8 up, 8 in 2023-09-23T18:27:05.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[76817]: cluster 2023-09-23T18:27:04.231671+0000 osd.3 (osd.3) 8 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[76817]: cluster 2023-09-23T18:27:04.231922+0000 osd.2 (osd.2) 9 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[76817]: cluster 2023-09-23T18:27:04.232412+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[76817]: cluster 2023-09-23T18:27:04.234491+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[76817]: cluster 2023-09-23T18:27:04.234869+0000 osd.4 (osd.4) 14 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[76817]: cluster 2023-09-23T18:27:04.235758+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[76817]: cluster 2023-09-23T18:27:04.237602+0000 osd.4 (osd.4) 15 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[76817]: cluster 2023-09-23T18:27:04.237849+0000 osd.6 (osd.6) 8 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[76817]: cluster 2023-09-23T18:27:04.238654+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[76817]: cluster 2023-09-23T18:27:04.239551+0000 osd.3 (osd.3) 9 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[76817]: cluster 2023-09-23T18:27:04.241956+0000 osd.4 (osd.4) 16 : cluster 3 failed to encode map e94 with expected crc 2023-09-23T18:27:05.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:05 smithi098 bash[76817]: audit 2023-09-23T18:27:04.649650+0000 mon.a (mon.0) 363 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:06.442 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:27:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:27:06] "GET /metrics HTTP/1.1" 200 34035 "" "Prometheus/2.33.4" 2023-09-23T18:27:06.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:06 smithi098 bash[76817]: audit 2023-09-23T18:27:05.284723+0000 mon.c (mon.2) 20 : audit 0 from='client.? 172.21.15.98:0/259583082' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:06.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:06 smithi098 bash[76817]: cluster 2023-09-23T18:27:05.564578+0000 mgr.y (mgr.26054) 237 : cluster 0 pgmap v176: 73 pgs: 2 active+undersized+degraded, 10 active+undersized, 61 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T18:27:06.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:06 smithi098 bash[79516]: audit 2023-09-23T18:27:05.284723+0000 mon.c (mon.2) 20 : audit 0 from='client.? 172.21.15.98:0/259583082' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:06.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:06 smithi098 bash[79516]: cluster 2023-09-23T18:27:05.564578+0000 mgr.y (mgr.26054) 237 : cluster 0 pgmap v176: 73 pgs: 2 active+undersized+degraded, 10 active+undersized, 61 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T18:27:06.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:06 smithi173 bash[59422]: audit 2023-09-23T18:27:05.284723+0000 mon.c (mon.2) 20 : audit 0 from='client.? 172.21.15.98:0/259583082' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:06.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:06 smithi173 bash[59422]: cluster 2023-09-23T18:27:05.564578+0000 mgr.y (mgr.26054) 237 : cluster 0 pgmap v176: 73 pgs: 2 active+undersized+degraded, 10 active+undersized, 61 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T18:27:07.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:07 smithi173 bash[59422]: cluster 2023-09-23T18:27:06.228726+0000 mon.a (mon.0) 364 : cluster 3 Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:27:07.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:07 smithi098 bash[76817]: cluster 2023-09-23T18:27:06.228726+0000 mon.a (mon.0) 364 : cluster 3 Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:27:07.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:07 smithi098 bash[79516]: cluster 2023-09-23T18:27:06.228726+0000 mon.a (mon.0) 364 : cluster 3 Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:27:08.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:08 smithi098 bash[76817]: cluster 2023-09-23T18:27:07.565327+0000 mgr.y (mgr.26054) 238 : cluster 0 pgmap v177: 73 pgs: 1 active+undersized+degraded, 1 active+undersized, 71 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T18:27:08.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:08 smithi098 bash[76817]: audit 2023-09-23T18:27:07.649103+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.98:0/4214817922' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:08.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:08 smithi098 bash[79516]: cluster 2023-09-23T18:27:07.565327+0000 mgr.y (mgr.26054) 238 : cluster 0 pgmap v177: 73 pgs: 1 active+undersized+degraded, 1 active+undersized, 71 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T18:27:08.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:08 smithi098 bash[79516]: audit 2023-09-23T18:27:07.649103+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.98:0/4214817922' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:08.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:08 smithi173 bash[59422]: cluster 2023-09-23T18:27:07.565327+0000 mgr.y (mgr.26054) 238 : cluster 0 pgmap v177: 73 pgs: 1 active+undersized+degraded, 1 active+undersized, 71 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T18:27:08.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:08 smithi173 bash[59422]: audit 2023-09-23T18:27:07.649103+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.98:0/4214817922' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:10.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:10 smithi098 bash[79516]: cluster 2023-09-23T18:27:09.566103+0000 mgr.y (mgr.26054) 239 : cluster 0 pgmap v178: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:27:10.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:10 smithi098 bash[79516]: audit 2023-09-23T18:27:09.575465+0000 mon.a (mon.0) 365 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:10.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:10 smithi098 bash[79516]: audit 2023-09-23T18:27:09.576840+0000 mon.a (mon.0) 366 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:27:10.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:10 smithi098 bash[79516]: audit 2023-09-23T18:27:10.022419+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.98:0/3855781237' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:10.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:10 smithi098 bash[79516]: cluster 2023-09-23T18:27:10.321271+0000 mon.a (mon.0) 367 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-09-23T18:27:10.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:10 smithi098 bash[79516]: cluster 2023-09-23T18:27:10.321332+0000 mon.a (mon.0) 368 : cluster 1 Cluster is now healthy 2023-09-23T18:27:10.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:10 smithi098 bash[76817]: cluster 2023-09-23T18:27:09.566103+0000 mgr.y (mgr.26054) 239 : cluster 0 pgmap v178: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:27:10.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:10 smithi098 bash[76817]: audit 2023-09-23T18:27:09.575465+0000 mon.a (mon.0) 365 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:10.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:10 smithi098 bash[76817]: audit 2023-09-23T18:27:09.576840+0000 mon.a (mon.0) 366 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:27:10.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:10 smithi098 bash[76817]: audit 2023-09-23T18:27:10.022419+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.98:0/3855781237' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:10.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:10 smithi098 bash[76817]: cluster 2023-09-23T18:27:10.321271+0000 mon.a (mon.0) 367 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-09-23T18:27:10.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:10 smithi098 bash[76817]: cluster 2023-09-23T18:27:10.321332+0000 mon.a (mon.0) 368 : cluster 1 Cluster is now healthy 2023-09-23T18:27:11.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:10 smithi173 bash[59422]: cluster 2023-09-23T18:27:09.566103+0000 mgr.y (mgr.26054) 239 : cluster 0 pgmap v178: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:27:11.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:10 smithi173 bash[59422]: audit 2023-09-23T18:27:09.575465+0000 mon.a (mon.0) 365 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:11.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:10 smithi173 bash[59422]: audit 2023-09-23T18:27:09.576840+0000 mon.a (mon.0) 366 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:27:11.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:10 smithi173 bash[59422]: audit 2023-09-23T18:27:10.022419+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.98:0/3855781237' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:11.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:10 smithi173 bash[59422]: cluster 2023-09-23T18:27:10.321271+0000 mon.a (mon.0) 367 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-09-23T18:27:11.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:10 smithi173 bash[59422]: cluster 2023-09-23T18:27:10.321332+0000 mon.a (mon.0) 368 : cluster 1 Cluster is now healthy 2023-09-23T18:27:12.869 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[76817]: audit 2023-09-23T18:27:11.554949+0000 mon.a (mon.0) 369 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:12.869 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[76817]: audit 2023-09-23T18:27:11.563601+0000 mon.a (mon.0) 370 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:12.869 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[76817]: audit 2023-09-23T18:27:11.565156+0000 mon.a (mon.0) 371 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:27:12.869 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[76817]: audit 2023-09-23T18:27:11.566543+0000 mon.a (mon.0) 372 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:27:12.869 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[76817]: cluster 2023-09-23T18:27:11.566810+0000 mgr.y (mgr.26054) 240 : cluster 0 pgmap v179: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:27:12.869 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[76817]: audit 2023-09-23T18:27:11.576989+0000 mon.a (mon.0) 373 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:12.869 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[76817]: audit 2023-09-23T18:27:11.661510+0000 mon.a (mon.0) 374 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:27:12.870 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[76817]: audit 2023-09-23T18:27:11.665786+0000 mon.a (mon.0) 375 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:27:12.870 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[76817]: audit 2023-09-23T18:27:11.668108+0000 mon.a (mon.0) 376 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:27:12.870 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[76817]: audit 2023-09-23T18:27:11.669802+0000 mon.a (mon.0) 377 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:27:12.870 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[76817]: audit 2023-09-23T18:27:11.672298+0000 mon.a (mon.0) 378 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T18:27:12.870 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[76817]: audit 2023-09-23T18:27:11.672635+0000 mgr.y (mgr.26054) 241 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T18:27:12.870 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[76817]: cephadm 2023-09-23T18:27:11.674108+0000 mgr.y (mgr.26054) 242 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-09-23T18:27:12.870 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[76817]: audit 2023-09-23T18:27:12.377585+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.98:0/3165778540' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:12.870 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[79516]: audit 2023-09-23T18:27:11.554949+0000 mon.a (mon.0) 369 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:12.871 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[79516]: audit 2023-09-23T18:27:11.563601+0000 mon.a (mon.0) 370 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:12.871 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[79516]: audit 2023-09-23T18:27:11.565156+0000 mon.a (mon.0) 371 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:27:12.871 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[79516]: audit 2023-09-23T18:27:11.566543+0000 mon.a (mon.0) 372 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:27:12.871 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[79516]: cluster 2023-09-23T18:27:11.566810+0000 mgr.y (mgr.26054) 240 : cluster 0 pgmap v179: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:27:12.871 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[79516]: audit 2023-09-23T18:27:11.576989+0000 mon.a (mon.0) 373 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:12.871 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[79516]: audit 2023-09-23T18:27:11.661510+0000 mon.a (mon.0) 374 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:27:12.871 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[79516]: audit 2023-09-23T18:27:11.665786+0000 mon.a (mon.0) 375 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:27:12.871 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[79516]: audit 2023-09-23T18:27:11.668108+0000 mon.a (mon.0) 376 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:27:12.871 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[79516]: audit 2023-09-23T18:27:11.669802+0000 mon.a (mon.0) 377 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:27:12.871 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[79516]: audit 2023-09-23T18:27:11.672298+0000 mon.a (mon.0) 378 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T18:27:12.872 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[79516]: audit 2023-09-23T18:27:11.672635+0000 mgr.y (mgr.26054) 241 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T18:27:12.872 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[79516]: cephadm 2023-09-23T18:27:11.674108+0000 mgr.y (mgr.26054) 242 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-09-23T18:27:12.872 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:12 smithi098 bash[79516]: audit 2023-09-23T18:27:12.377585+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.98:0/3165778540' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:13.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:12 smithi173 bash[59422]: audit 2023-09-23T18:27:11.554949+0000 mon.a (mon.0) 369 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:13.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:12 smithi173 bash[59422]: audit 2023-09-23T18:27:11.563601+0000 mon.a (mon.0) 370 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:13.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:12 smithi173 bash[59422]: audit 2023-09-23T18:27:11.565156+0000 mon.a (mon.0) 371 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:27:13.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:12 smithi173 bash[59422]: audit 2023-09-23T18:27:11.566543+0000 mon.a (mon.0) 372 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:27:13.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:12 smithi173 bash[59422]: cluster 2023-09-23T18:27:11.566810+0000 mgr.y (mgr.26054) 240 : cluster 0 pgmap v179: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:27:13.030 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:12 smithi173 bash[59422]: audit 2023-09-23T18:27:11.576989+0000 mon.a (mon.0) 373 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:13.030 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:12 smithi173 bash[59422]: audit 2023-09-23T18:27:11.661510+0000 mon.a (mon.0) 374 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:27:13.030 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:12 smithi173 bash[59422]: audit 2023-09-23T18:27:11.665786+0000 mon.a (mon.0) 375 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:27:13.030 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:12 smithi173 bash[59422]: audit 2023-09-23T18:27:11.668108+0000 mon.a (mon.0) 376 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:27:13.030 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:12 smithi173 bash[59422]: audit 2023-09-23T18:27:11.669802+0000 mon.a (mon.0) 377 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:27:13.030 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:12 smithi173 bash[59422]: audit 2023-09-23T18:27:11.672298+0000 mon.a (mon.0) 378 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T18:27:13.030 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:12 smithi173 bash[59422]: audit 2023-09-23T18:27:11.672635+0000 mgr.y (mgr.26054) 241 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T18:27:13.030 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:12 smithi173 bash[59422]: cephadm 2023-09-23T18:27:11.674108+0000 mgr.y (mgr.26054) 242 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-09-23T18:27:13.030 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:12 smithi173 bash[59422]: audit 2023-09-23T18:27:12.377585+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.98:0/3165778540' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:14.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:14 smithi098 bash[76817]: cluster 2023-09-23T18:27:13.567618+0000 mgr.y (mgr.26054) 243 : cluster 0 pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:27:14.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:14 smithi098 bash[76817]: audit 2023-09-23T18:27:13.953026+0000 mon.a (mon.0) 379 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:14.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:14 smithi098 bash[76817]: audit 2023-09-23T18:27:13.954622+0000 mon.a (mon.0) 380 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T18:27:14.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:14 smithi098 bash[76817]: audit 2023-09-23T18:27:13.955924+0000 mon.a (mon.0) 381 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:27:14.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:14 smithi098 bash[79516]: cluster 2023-09-23T18:27:13.567618+0000 mgr.y (mgr.26054) 243 : cluster 0 pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:27:14.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:14 smithi098 bash[79516]: audit 2023-09-23T18:27:13.953026+0000 mon.a (mon.0) 379 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:14.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:14 smithi098 bash[79516]: audit 2023-09-23T18:27:13.954622+0000 mon.a (mon.0) 380 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T18:27:14.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:14 smithi098 bash[79516]: audit 2023-09-23T18:27:13.955924+0000 mon.a (mon.0) 381 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:27:15.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:14 smithi173 bash[59422]: cluster 2023-09-23T18:27:13.567618+0000 mgr.y (mgr.26054) 243 : cluster 0 pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:27:15.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:14 smithi173 bash[59422]: audit 2023-09-23T18:27:13.953026+0000 mon.a (mon.0) 379 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:15.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:14 smithi173 bash[59422]: audit 2023-09-23T18:27:13.954622+0000 mon.a (mon.0) 380 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T18:27:15.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:14 smithi173 bash[59422]: audit 2023-09-23T18:27:13.955924+0000 mon.a (mon.0) 381 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:27:15.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:15 smithi098 bash[76817]: cephadm 2023-09-23T18:27:13.943405+0000 mgr.y (mgr.26054) 244 : cephadm 1 Upgrade: Updating osd.2 2023-09-23T18:27:15.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:15 smithi098 bash[76817]: cephadm 2023-09-23T18:27:13.959745+0000 mgr.y (mgr.26054) 245 : cephadm 1 Deploying daemon osd.2 on smithi098 2023-09-23T18:27:15.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:15 smithi098 bash[76817]: audit 2023-09-23T18:27:14.663901+0000 mon.a (mon.0) 382 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:15.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:15 smithi098 bash[76817]: audit 2023-09-23T18:27:14.743028+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.98:0/1689925970' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:15.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:15 smithi098 bash[79516]: cephadm 2023-09-23T18:27:13.943405+0000 mgr.y (mgr.26054) 244 : cephadm 1 Upgrade: Updating osd.2 2023-09-23T18:27:15.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:15 smithi098 bash[79516]: cephadm 2023-09-23T18:27:13.959745+0000 mgr.y (mgr.26054) 245 : cephadm 1 Deploying daemon osd.2 on smithi098 2023-09-23T18:27:15.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:15 smithi098 bash[79516]: audit 2023-09-23T18:27:14.663901+0000 mon.a (mon.0) 382 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:15.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:15 smithi098 bash[79516]: audit 2023-09-23T18:27:14.743028+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.98:0/1689925970' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:16.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:15 smithi173 bash[59422]: cephadm 2023-09-23T18:27:13.943405+0000 mgr.y (mgr.26054) 244 : cephadm 1 Upgrade: Updating osd.2 2023-09-23T18:27:16.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:15 smithi173 bash[59422]: cephadm 2023-09-23T18:27:13.959745+0000 mgr.y (mgr.26054) 245 : cephadm 1 Deploying daemon osd.2 on smithi098 2023-09-23T18:27:16.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:15 smithi173 bash[59422]: audit 2023-09-23T18:27:14.663901+0000 mon.a (mon.0) 382 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:16.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:15 smithi173 bash[59422]: audit 2023-09-23T18:27:14.743028+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.98:0/1689925970' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:16.442 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:27:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:27:16] "GET /metrics HTTP/1.1" 200 34030 "" "Prometheus/2.33.4" 2023-09-23T18:27:16.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:16 smithi098 bash[79516]: cluster 2023-09-23T18:27:15.568547+0000 mgr.y (mgr.26054) 246 : cluster 0 pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:27:16.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:16 smithi098 bash[76817]: cluster 2023-09-23T18:27:15.568547+0000 mgr.y (mgr.26054) 246 : cluster 0 pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:27:17.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:16 smithi173 bash[59422]: cluster 2023-09-23T18:27:15.568547+0000 mgr.y (mgr.26054) 246 : cluster 0 pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:27:17.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:17 smithi098 bash[76817]: audit 2023-09-23T18:27:17.107889+0000 mon.a (mon.0) 383 : audit 0 from='client.? 172.21.15.98:0/2343495003' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:17.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:17 smithi098 bash[79516]: audit 2023-09-23T18:27:17.107889+0000 mon.a (mon.0) 383 : audit 0 from='client.? 172.21.15.98:0/2343495003' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:18.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:17 smithi173 bash[59422]: audit 2023-09-23T18:27:17.107889+0000 mon.a (mon.0) 383 : audit 0 from='client.? 172.21.15.98:0/2343495003' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:18.898 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:18 smithi098 bash[76817]: cluster 2023-09-23T18:27:17.569272+0000 mgr.y (mgr.26054) 247 : cluster 0 pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:27:18.899 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:18 smithi098 bash[79516]: cluster 2023-09-23T18:27:17.569272+0000 mgr.y (mgr.26054) 247 : cluster 0 pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:27:18.899 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 18:27:18 smithi098 systemd[1]: Stopping Ceph osd.2 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T18:27:19.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:18 smithi173 bash[59422]: cluster 2023-09-23T18:27:17.569272+0000 mgr.y (mgr.26054) 247 : cluster 0 pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:27:19.193 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 18:27:18 smithi098 bash[44997]: debug 2023-09-23T18:27:18.893+0000 7f2b41e2a700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T18:27:19.193 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 18:27:18 smithi098 bash[44997]: debug 2023-09-23T18:27:18.893+0000 7f2b41e2a700 -1 osd.2 94 *** Got signal Terminated *** 2023-09-23T18:27:19.193 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 18:27:18 smithi098 bash[44997]: debug 2023-09-23T18:27:18.893+0000 7f2b41e2a700 -1 osd.2 94 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T18:27:19.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:19 smithi098 bash[76817]: cluster 2023-09-23T18:27:18.896271+0000 mon.a (mon.0) 384 : cluster 1 osd.2 marked itself down and dead 2023-09-23T18:27:19.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:19 smithi098 bash[76817]: audit 2023-09-23T18:27:19.465031+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.98:0/987700525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:19.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:19 smithi098 bash[79516]: cluster 2023-09-23T18:27:18.896271+0000 mon.a (mon.0) 384 : cluster 1 osd.2 marked itself down and dead 2023-09-23T18:27:19.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:19 smithi098 bash[79516]: audit 2023-09-23T18:27:19.465031+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.98:0/987700525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:20.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:19 smithi173 bash[59422]: cluster 2023-09-23T18:27:18.896271+0000 mon.a (mon.0) 384 : cluster 1 osd.2 marked itself down and dead 2023-09-23T18:27:20.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:19 smithi173 bash[59422]: audit 2023-09-23T18:27:19.465031+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.98:0/987700525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:20.686 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 18:27:20 smithi098 bash[90346]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-2 2023-09-23T18:27:20.686 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 18:27:20 smithi098 bash[90435]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-2 2023-09-23T18:27:20.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:20 smithi098 bash[76817]: cluster 2023-09-23T18:27:19.570055+0000 mgr.y (mgr.26054) 248 : cluster 0 pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:27:20.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:20 smithi098 bash[76817]: cluster 2023-09-23T18:27:19.680550+0000 mon.a (mon.0) 385 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:27:20.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:20 smithi098 bash[76817]: cluster 2023-09-23T18:27:19.692252+0000 mon.a (mon.0) 386 : cluster 0 osdmap e95: 8 total, 7 up, 8 in 2023-09-23T18:27:20.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:20 smithi098 bash[76817]: cluster 2023-09-23T18:27:19.692596+0000 osd.4 (osd.4) 17 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T18:27:20.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:20 smithi098 bash[76817]: cluster 2023-09-23T18:27:19.693155+0000 osd.3 (osd.3) 10 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T18:27:20.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:20 smithi098 bash[76817]: cluster 2023-09-23T18:27:19.695253+0000 osd.6 (osd.6) 9 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T18:27:20.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:20 smithi098 bash[76817]: cluster 2023-09-23T18:27:19.695644+0000 osd.4 (osd.4) 18 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T18:27:20.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:20 smithi098 bash[76817]: cluster 2023-09-23T18:27:19.697080+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T18:27:20.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:20 smithi098 bash[76817]: cluster 2023-09-23T18:27:19.700057+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T18:27:20.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:20 smithi098 bash[79516]: cluster 2023-09-23T18:27:19.570055+0000 mgr.y (mgr.26054) 248 : cluster 0 pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:27:20.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:20 smithi098 bash[79516]: cluster 2023-09-23T18:27:19.680550+0000 mon.a (mon.0) 385 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:27:20.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:20 smithi098 bash[79516]: cluster 2023-09-23T18:27:19.692252+0000 mon.a (mon.0) 386 : cluster 0 osdmap e95: 8 total, 7 up, 8 in 2023-09-23T18:27:20.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:20 smithi098 bash[79516]: cluster 2023-09-23T18:27:19.692596+0000 osd.4 (osd.4) 17 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T18:27:20.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:20 smithi098 bash[79516]: cluster 2023-09-23T18:27:19.693155+0000 osd.3 (osd.3) 10 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T18:27:20.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:20 smithi098 bash[79516]: cluster 2023-09-23T18:27:19.695253+0000 osd.6 (osd.6) 9 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T18:27:20.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:20 smithi098 bash[79516]: cluster 2023-09-23T18:27:19.695644+0000 osd.4 (osd.4) 18 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T18:27:20.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:20 smithi098 bash[79516]: cluster 2023-09-23T18:27:19.697080+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T18:27:20.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:20 smithi098 bash[79516]: cluster 2023-09-23T18:27:19.700057+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T18:27:21.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:20 smithi173 bash[59422]: cluster 2023-09-23T18:27:19.570055+0000 mgr.y (mgr.26054) 248 : cluster 0 pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:27:21.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:20 smithi173 bash[59422]: cluster 2023-09-23T18:27:19.680550+0000 mon.a (mon.0) 385 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:27:21.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:20 smithi173 bash[59422]: cluster 2023-09-23T18:27:19.692252+0000 mon.a (mon.0) 386 : cluster 0 osdmap e95: 8 total, 7 up, 8 in 2023-09-23T18:27:21.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:20 smithi173 bash[59422]: cluster 2023-09-23T18:27:19.692596+0000 osd.4 (osd.4) 17 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T18:27:21.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:20 smithi173 bash[59422]: cluster 2023-09-23T18:27:19.693155+0000 osd.3 (osd.3) 10 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T18:27:21.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:20 smithi173 bash[59422]: cluster 2023-09-23T18:27:19.695253+0000 osd.6 (osd.6) 9 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T18:27:21.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:20 smithi173 bash[59422]: cluster 2023-09-23T18:27:19.695644+0000 osd.4 (osd.4) 18 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T18:27:21.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:20 smithi173 bash[59422]: cluster 2023-09-23T18:27:19.697080+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T18:27:21.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:20 smithi173 bash[59422]: cluster 2023-09-23T18:27:19.700057+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e95 with expected crc 2023-09-23T18:27:21.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:21 smithi098 bash[76817]: cluster 2023-09-23T18:27:20.694801+0000 osd.3 (osd.3) 11 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T18:27:21.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:21 smithi098 bash[76817]: cluster 2023-09-23T18:27:20.698862+0000 mon.a (mon.0) 387 : cluster 0 osdmap e96: 8 total, 7 up, 8 in 2023-09-23T18:27:21.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:21 smithi098 bash[76817]: cluster 2023-09-23T18:27:20.699151+0000 osd.3 (osd.3) 12 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T18:27:21.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:21 smithi098 bash[76817]: cluster 2023-09-23T18:27:20.700716+0000 osd.6 (osd.6) 10 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T18:27:21.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:21 smithi098 bash[76817]: cluster 2023-09-23T18:27:20.701991+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T18:27:21.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:21 smithi098 bash[76817]: cluster 2023-09-23T18:27:20.702003+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T18:27:21.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:21 smithi098 bash[76817]: cluster 2023-09-23T18:27:20.703473+0000 osd.4 (osd.4) 19 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T18:27:21.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:21 smithi098 bash[79516]: cluster 2023-09-23T18:27:20.694801+0000 osd.3 (osd.3) 11 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T18:27:21.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:21 smithi098 bash[79516]: cluster 2023-09-23T18:27:20.698862+0000 mon.a (mon.0) 387 : cluster 0 osdmap e96: 8 total, 7 up, 8 in 2023-09-23T18:27:21.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:21 smithi098 bash[79516]: cluster 2023-09-23T18:27:20.699151+0000 osd.3 (osd.3) 12 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T18:27:21.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:21 smithi098 bash[79516]: cluster 2023-09-23T18:27:20.700716+0000 osd.6 (osd.6) 10 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T18:27:21.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:21 smithi098 bash[79516]: cluster 2023-09-23T18:27:20.701991+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T18:27:21.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:21 smithi098 bash[79516]: cluster 2023-09-23T18:27:20.702003+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T18:27:21.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:21 smithi098 bash[79516]: cluster 2023-09-23T18:27:20.703473+0000 osd.4 (osd.4) 19 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T18:27:22.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:21 smithi173 bash[59422]: cluster 2023-09-23T18:27:20.694801+0000 osd.3 (osd.3) 11 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T18:27:22.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:21 smithi173 bash[59422]: cluster 2023-09-23T18:27:20.698862+0000 mon.a (mon.0) 387 : cluster 0 osdmap e96: 8 total, 7 up, 8 in 2023-09-23T18:27:22.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:21 smithi173 bash[59422]: cluster 2023-09-23T18:27:20.699151+0000 osd.3 (osd.3) 12 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T18:27:22.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:21 smithi173 bash[59422]: cluster 2023-09-23T18:27:20.700716+0000 osd.6 (osd.6) 10 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T18:27:22.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:21 smithi173 bash[59422]: cluster 2023-09-23T18:27:20.701991+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T18:27:22.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:21 smithi173 bash[59422]: cluster 2023-09-23T18:27:20.702003+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T18:27:22.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:21 smithi173 bash[59422]: cluster 2023-09-23T18:27:20.703473+0000 osd.4 (osd.4) 19 : cluster 3 failed to encode map e96 with expected crc 2023-09-23T18:27:22.967 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 18:27:22 smithi098 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.2.service: Succeeded. 2023-09-23T18:27:22.967 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 18:27:22 smithi098 systemd[1]: Stopped Ceph osd.2 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:27:22.968 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:22 smithi098 bash[79516]: cluster 2023-09-23T18:27:21.570769+0000 mgr.y (mgr.26054) 249 : cluster 0 pgmap v186: 73 pgs: 11 peering, 1 stale+active+clean, 61 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:27:22.968 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:22 smithi098 bash[79516]: cluster 2023-09-23T18:27:21.692247+0000 mon.a (mon.0) 388 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:27:22.968 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:22 smithi098 bash[79516]: audit 2023-09-23T18:27:21.829242+0000 mon.a (mon.0) 389 : audit 0 from='client.? 172.21.15.98:0/1733246426' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:22.968 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:22 smithi098 bash[76817]: cluster 2023-09-23T18:27:21.570769+0000 mgr.y (mgr.26054) 249 : cluster 0 pgmap v186: 73 pgs: 11 peering, 1 stale+active+clean, 61 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:27:22.968 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:22 smithi098 bash[76817]: cluster 2023-09-23T18:27:21.692247+0000 mon.a (mon.0) 388 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:27:22.968 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:22 smithi098 bash[76817]: audit 2023-09-23T18:27:21.829242+0000 mon.a (mon.0) 389 : audit 0 from='client.? 172.21.15.98:0/1733246426' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:23.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:22 smithi173 bash[59422]: cluster 2023-09-23T18:27:21.570769+0000 mgr.y (mgr.26054) 249 : cluster 0 pgmap v186: 73 pgs: 11 peering, 1 stale+active+clean, 61 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T18:27:23.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:22 smithi173 bash[59422]: cluster 2023-09-23T18:27:21.692247+0000 mon.a (mon.0) 388 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:27:23.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:22 smithi173 bash[59422]: audit 2023-09-23T18:27:21.829242+0000 mon.a (mon.0) 389 : audit 0 from='client.? 172.21.15.98:0/1733246426' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:23.266 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 18:27:23 smithi098 systemd[1]: Started Ceph osd.2 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:27:24.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:24 smithi098 bash[79516]: audit 2023-09-23T18:27:23.239446+0000 mon.a (mon.0) 390 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:24.552 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:24 smithi098 bash[79516]: audit 2023-09-23T18:27:23.249214+0000 mon.a (mon.0) 391 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:24.553 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:24 smithi098 bash[79516]: audit 2023-09-23T18:27:23.299975+0000 mon.a (mon.0) 392 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:27:24.553 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:24 smithi098 bash[79516]: cluster 2023-09-23T18:27:23.571890+0000 mgr.y (mgr.26054) 250 : cluster 0 pgmap v187: 73 pgs: 1 active+undersized+degraded, 11 peering, 7 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T18:27:24.553 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:24 smithi098 bash[79516]: audit 2023-09-23T18:27:24.185441+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.98:0/2082295834' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:24.553 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:24 smithi098 bash[76817]: audit 2023-09-23T18:27:23.239446+0000 mon.a (mon.0) 390 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:24.553 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:24 smithi098 bash[76817]: audit 2023-09-23T18:27:23.249214+0000 mon.a (mon.0) 391 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:24.553 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:24 smithi098 bash[76817]: audit 2023-09-23T18:27:23.299975+0000 mon.a (mon.0) 392 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:27:24.553 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:24 smithi098 bash[76817]: cluster 2023-09-23T18:27:23.571890+0000 mgr.y (mgr.26054) 250 : cluster 0 pgmap v187: 73 pgs: 1 active+undersized+degraded, 11 peering, 7 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T18:27:24.554 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:24 smithi098 bash[76817]: audit 2023-09-23T18:27:24.185441+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.98:0/2082295834' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:24.554 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:24 smithi173 bash[59422]: audit 2023-09-23T18:27:23.239446+0000 mon.a (mon.0) 390 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:24.554 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:24 smithi173 bash[59422]: audit 2023-09-23T18:27:23.249214+0000 mon.a (mon.0) 391 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:24.554 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:24 smithi173 bash[59422]: audit 2023-09-23T18:27:23.299975+0000 mon.a (mon.0) 392 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:27:24.554 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:24 smithi173 bash[59422]: cluster 2023-09-23T18:27:23.571890+0000 mgr.y (mgr.26054) 250 : cluster 0 pgmap v187: 73 pgs: 1 active+undersized+degraded, 11 peering, 7 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T18:27:24.554 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:24 smithi173 bash[59422]: audit 2023-09-23T18:27:24.185441+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.98:0/2082295834' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:25.080 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 18:27:24 smithi098 bash[90617]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T18:27:25.081 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 18:27:24 smithi098 bash[90617]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T18:27:25.081 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 18:27:24 smithi098 bash[90617]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-09-23T18:27:25.081 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 18:27:24 smithi098 bash[90617]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-09-23T18:27:25.081 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 18:27:24 smithi098 bash[90617]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-09-23T18:27:25.081 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 18:27:24 smithi098 bash[90617]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-09-23T18:27:25.081 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 18:27:24 smithi098 bash[90617]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T18:27:25.081 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 18:27:24 smithi098 bash[90617]: --> ceph-volume lvm activate successful for osd ID: 2 2023-09-23T18:27:25.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:25 smithi098 bash[79516]: cluster 2023-09-23T18:27:24.245529+0000 mon.a (mon.0) 393 : cluster 3 Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:27:25.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:25 smithi098 bash[79516]: audit 2023-09-23T18:27:24.577477+0000 mon.a (mon.0) 394 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:25.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:25 smithi098 bash[79516]: audit 2023-09-23T18:27:24.578909+0000 mon.a (mon.0) 395 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:27:25.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:25 smithi098 bash[76817]: cluster 2023-09-23T18:27:24.245529+0000 mon.a (mon.0) 393 : cluster 3 Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:27:25.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:25 smithi098 bash[76817]: audit 2023-09-23T18:27:24.577477+0000 mon.a (mon.0) 394 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:25.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:25 smithi098 bash[76817]: audit 2023-09-23T18:27:24.578909+0000 mon.a (mon.0) 395 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:27:25.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:25 smithi173 bash[59422]: cluster 2023-09-23T18:27:24.245529+0000 mon.a (mon.0) 393 : cluster 3 Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:27:25.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:25 smithi173 bash[59422]: audit 2023-09-23T18:27:24.577477+0000 mon.a (mon.0) 394 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:25.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:25 smithi173 bash[59422]: audit 2023-09-23T18:27:24.578909+0000 mon.a (mon.0) 395 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:27:26.442 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:27:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:27:26] "GET /metrics HTTP/1.1" 200 34100 "" "Prometheus/2.33.4" 2023-09-23T18:27:26.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:26 smithi098 bash[79516]: cluster 2023-09-23T18:27:25.572546+0000 mgr.y (mgr.26054) 251 : cluster 0 pgmap v188: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T18:27:26.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:26 smithi098 bash[76817]: cluster 2023-09-23T18:27:25.572546+0000 mgr.y (mgr.26054) 251 : cluster 0 pgmap v188: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T18:27:26.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:26 smithi173 bash[59422]: cluster 2023-09-23T18:27:25.572546+0000 mgr.y (mgr.26054) 251 : cluster 0 pgmap v188: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T18:27:26.942 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 18:27:26 smithi098 bash[90938]: debug 2023-09-23T18:27:26.729+0000 7fac53d9d700 1 -- 172.21.15.98:0/1335209959 <== mon.2 v2:172.21.15.98:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55cf1c2044e0 con 0x55cf1c25e800 2023-09-23T18:27:27.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:27 smithi173 bash[59422]: cluster 2023-09-23T18:27:26.239106+0000 mon.a (mon.0) 396 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T18:27:27.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:27 smithi173 bash[59422]: audit 2023-09-23T18:27:26.547262+0000 mon.a (mon.0) 397 : audit 0 from='client.? 172.21.15.98:0/410357521' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:27.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:27 smithi098 bash[76817]: cluster 2023-09-23T18:27:26.239106+0000 mon.a (mon.0) 396 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T18:27:27.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:27 smithi098 bash[76817]: audit 2023-09-23T18:27:26.547262+0000 mon.a (mon.0) 397 : audit 0 from='client.? 172.21.15.98:0/410357521' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:27.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:27 smithi098 bash[79516]: cluster 2023-09-23T18:27:26.239106+0000 mon.a (mon.0) 396 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T18:27:27.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:27 smithi098 bash[79516]: audit 2023-09-23T18:27:26.547262+0000 mon.a (mon.0) 397 : audit 0 from='client.? 172.21.15.98:0/410357521' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:27.693 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 18:27:27 smithi098 bash[90938]: debug 2023-09-23T18:27:27.277+0000 7fac5cf22640 -1 Falling back to public interface 2023-09-23T18:27:28.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:28 smithi173 bash[59422]: cluster 2023-09-23T18:27:27.573331+0000 mgr.y (mgr.26054) 252 : cluster 0 pgmap v189: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T18:27:28.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:28 smithi098 bash[76817]: cluster 2023-09-23T18:27:27.573331+0000 mgr.y (mgr.26054) 252 : cluster 0 pgmap v189: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T18:27:28.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:28 smithi098 bash[79516]: cluster 2023-09-23T18:27:27.573331+0000 mgr.y (mgr.26054) 252 : cluster 0 pgmap v189: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T18:27:29.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:29 smithi173 bash[59422]: audit 2023-09-23T18:27:28.923617+0000 mon.a (mon.0) 398 : audit 0 from='client.? 172.21.15.98:0/2820798581' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:29.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:29 smithi098 bash[76817]: audit 2023-09-23T18:27:28.923617+0000 mon.a (mon.0) 398 : audit 0 from='client.? 172.21.15.98:0/2820798581' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:29.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:29 smithi098 bash[79516]: audit 2023-09-23T18:27:28.923617+0000 mon.a (mon.0) 398 : audit 0 from='client.? 172.21.15.98:0/2820798581' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:30.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:30 smithi173 bash[59422]: cluster 2023-09-23T18:27:29.324654+0000 mon.a (mon.0) 399 : cluster 3 Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T18:27:30.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:30 smithi173 bash[59422]: audit 2023-09-23T18:27:29.544147+0000 mon.a (mon.0) 400 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:30.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:30 smithi173 bash[59422]: audit 2023-09-23T18:27:29.551052+0000 mon.a (mon.0) 401 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:30.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:30 smithi173 bash[59422]: cluster 2023-09-23T18:27:29.573961+0000 mgr.y (mgr.26054) 253 : cluster 0 pgmap v190: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T18:27:30.563 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:30 smithi098 bash[79516]: cluster 2023-09-23T18:27:29.324654+0000 mon.a (mon.0) 399 : cluster 3 Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T18:27:30.564 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:30 smithi098 bash[79516]: audit 2023-09-23T18:27:29.544147+0000 mon.a (mon.0) 400 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:30.564 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:30 smithi098 bash[79516]: audit 2023-09-23T18:27:29.551052+0000 mon.a (mon.0) 401 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:30.564 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:30 smithi098 bash[79516]: cluster 2023-09-23T18:27:29.573961+0000 mgr.y (mgr.26054) 253 : cluster 0 pgmap v190: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T18:27:30.564 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:30 smithi098 bash[76817]: cluster 2023-09-23T18:27:29.324654+0000 mon.a (mon.0) 399 : cluster 3 Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T18:27:30.564 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:30 smithi098 bash[76817]: audit 2023-09-23T18:27:29.544147+0000 mon.a (mon.0) 400 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:30.565 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:30 smithi098 bash[76817]: audit 2023-09-23T18:27:29.551052+0000 mon.a (mon.0) 401 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:30.565 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:30 smithi098 bash[76817]: cluster 2023-09-23T18:27:29.573961+0000 mgr.y (mgr.26054) 253 : cluster 0 pgmap v190: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T18:27:30.565 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 18:27:30 smithi098 bash[90938]: debug 2023-09-23T18:27:30.557+0000 7fac5cf22640 -1 osd.2 94 log_to_monitors true 2023-09-23T18:27:31.692 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 18:27:31 smithi098 bash[90938]: debug 2023-09-23T18:27:31.601+0000 7fac4c38b700 -1 osd.2 94 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T18:27:31.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:31 smithi098 bash[76817]: audit 2023-09-23T18:27:30.361858+0000 mon.a (mon.0) 402 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:31.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:31 smithi098 bash[76817]: audit 2023-09-23T18:27:30.368690+0000 mon.a (mon.0) 403 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:31.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:31 smithi098 bash[76817]: audit 2023-09-23T18:27:30.564815+0000 mon.c (mon.2) 27 : audit 1 from='osd.2 [v2:172.21.15.98:6818/4249816143,v1:172.21.15.98:6819/4249816143]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T18:27:31.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:31 smithi098 bash[76817]: audit 2023-09-23T18:27:30.565436+0000 mon.a (mon.0) 404 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T18:27:31.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:31 smithi098 bash[76817]: audit 2023-09-23T18:27:31.157490+0000 mon.a (mon.0) 405 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:31.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:31 smithi098 bash[76817]: audit 2023-09-23T18:27:31.164899+0000 mon.a (mon.0) 406 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:31.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:31 smithi098 bash[76817]: audit 2023-09-23T18:27:31.293885+0000 mon.a (mon.0) 407 : audit 0 from='client.? 172.21.15.98:0/1798028832' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:31.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:31 smithi098 bash[79516]: audit 2023-09-23T18:27:30.361858+0000 mon.a (mon.0) 402 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:31.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:31 smithi098 bash[79516]: audit 2023-09-23T18:27:30.368690+0000 mon.a (mon.0) 403 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:31.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:31 smithi098 bash[79516]: audit 2023-09-23T18:27:30.564815+0000 mon.c (mon.2) 27 : audit 1 from='osd.2 [v2:172.21.15.98:6818/4249816143,v1:172.21.15.98:6819/4249816143]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T18:27:31.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:31 smithi098 bash[79516]: audit 2023-09-23T18:27:30.565436+0000 mon.a (mon.0) 404 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T18:27:31.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:31 smithi098 bash[79516]: audit 2023-09-23T18:27:31.157490+0000 mon.a (mon.0) 405 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:31.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:31 smithi098 bash[79516]: audit 2023-09-23T18:27:31.164899+0000 mon.a (mon.0) 406 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:31.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:31 smithi098 bash[79516]: audit 2023-09-23T18:27:31.293885+0000 mon.a (mon.0) 407 : audit 0 from='client.? 172.21.15.98:0/1798028832' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:31.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:31 smithi173 bash[59422]: audit 2023-09-23T18:27:30.361858+0000 mon.a (mon.0) 402 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:31.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:31 smithi173 bash[59422]: audit 2023-09-23T18:27:30.368690+0000 mon.a (mon.0) 403 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:31.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:31 smithi173 bash[59422]: audit 2023-09-23T18:27:30.564815+0000 mon.c (mon.2) 27 : audit 1 from='osd.2 [v2:172.21.15.98:6818/4249816143,v1:172.21.15.98:6819/4249816143]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T18:27:31.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:31 smithi173 bash[59422]: audit 2023-09-23T18:27:30.565436+0000 mon.a (mon.0) 404 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T18:27:31.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:31 smithi173 bash[59422]: audit 2023-09-23T18:27:31.157490+0000 mon.a (mon.0) 405 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:31.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:31 smithi173 bash[59422]: audit 2023-09-23T18:27:31.164899+0000 mon.a (mon.0) 406 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:31.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:31 smithi173 bash[59422]: audit 2023-09-23T18:27:31.293885+0000 mon.a (mon.0) 407 : audit 0 from='client.? 172.21.15.98:0/1798028832' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:32.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:32 smithi098 bash[76817]: cluster 2023-09-23T18:27:31.554387+0000 osd.3 (osd.3) 13 : cluster 3 failed to encode map e97 with expected crc 2023-09-23T18:27:32.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:32 smithi098 bash[76817]: audit 2023-09-23T18:27:31.557316+0000 mon.a (mon.0) 408 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T18:27:32.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:32 smithi098 bash[76817]: cluster 2023-09-23T18:27:31.557458+0000 mon.a (mon.0) 409 : cluster 0 osdmap e97: 8 total, 7 up, 8 in 2023-09-23T18:27:32.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:32 smithi098 bash[76817]: cluster 2023-09-23T18:27:31.557810+0000 osd.6 (osd.6) 11 : cluster 3 failed to encode map e97 with expected crc 2023-09-23T18:27:32.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:32 smithi098 bash[76817]: audit 2023-09-23T18:27:31.560630+0000 mon.c (mon.2) 28 : audit 1 from='osd.2 [v2:172.21.15.98:6818/4249816143,v1:172.21.15.98:6819/4249816143]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:27:32.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:32 smithi098 bash[76817]: audit 2023-09-23T18:27:31.561061+0000 mon.a (mon.0) 410 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:27:32.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:32 smithi098 bash[76817]: cluster 2023-09-23T18:27:31.574556+0000 mgr.y (mgr.26054) 254 : cluster 0 pgmap v192: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T18:27:32.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:32 smithi098 bash[79516]: cluster 2023-09-23T18:27:31.554387+0000 osd.3 (osd.3) 13 : cluster 3 failed to encode map e97 with expected crc 2023-09-23T18:27:32.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:32 smithi098 bash[79516]: audit 2023-09-23T18:27:31.557316+0000 mon.a (mon.0) 408 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T18:27:32.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:32 smithi098 bash[79516]: cluster 2023-09-23T18:27:31.557458+0000 mon.a (mon.0) 409 : cluster 0 osdmap e97: 8 total, 7 up, 8 in 2023-09-23T18:27:32.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:32 smithi098 bash[79516]: cluster 2023-09-23T18:27:31.557810+0000 osd.6 (osd.6) 11 : cluster 3 failed to encode map e97 with expected crc 2023-09-23T18:27:32.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:32 smithi098 bash[79516]: audit 2023-09-23T18:27:31.560630+0000 mon.c (mon.2) 28 : audit 1 from='osd.2 [v2:172.21.15.98:6818/4249816143,v1:172.21.15.98:6819/4249816143]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:27:32.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:32 smithi098 bash[79516]: audit 2023-09-23T18:27:31.561061+0000 mon.a (mon.0) 410 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:27:32.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:32 smithi098 bash[79516]: cluster 2023-09-23T18:27:31.574556+0000 mgr.y (mgr.26054) 254 : cluster 0 pgmap v192: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T18:27:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:32 smithi173 bash[59422]: cluster 2023-09-23T18:27:31.554387+0000 osd.3 (osd.3) 13 : cluster 3 failed to encode map e97 with expected crc 2023-09-23T18:27:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:32 smithi173 bash[59422]: audit 2023-09-23T18:27:31.557316+0000 mon.a (mon.0) 408 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T18:27:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:32 smithi173 bash[59422]: cluster 2023-09-23T18:27:31.557458+0000 mon.a (mon.0) 409 : cluster 0 osdmap e97: 8 total, 7 up, 8 in 2023-09-23T18:27:33.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:32 smithi173 bash[59422]: cluster 2023-09-23T18:27:31.557810+0000 osd.6 (osd.6) 11 : cluster 3 failed to encode map e97 with expected crc 2023-09-23T18:27:33.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:32 smithi173 bash[59422]: audit 2023-09-23T18:27:31.560630+0000 mon.c (mon.2) 28 : audit 1 from='osd.2 [v2:172.21.15.98:6818/4249816143,v1:172.21.15.98:6819/4249816143]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:27:33.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:32 smithi173 bash[59422]: audit 2023-09-23T18:27:31.561061+0000 mon.a (mon.0) 410 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:27:33.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:32 smithi173 bash[59422]: cluster 2023-09-23T18:27:31.574556+0000 mgr.y (mgr.26054) 254 : cluster 0 pgmap v192: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T18:27:33.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:33 smithi098 bash[76817]: cluster 2023-09-23T18:27:32.554505+0000 mon.a (mon.0) 411 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:27:33.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:33 smithi098 bash[76817]: cluster 2023-09-23T18:27:32.561901+0000 osd.3 (osd.3) 14 : cluster 3 failed to encode map e98 with expected crc 2023-09-23T18:27:33.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:33 smithi098 bash[76817]: cluster 2023-09-23T18:27:32.565782+0000 mon.a (mon.0) 412 : cluster 1 osd.2 [v2:172.21.15.98:6818/4249816143,v1:172.21.15.98:6819/4249816143] boot 2023-09-23T18:27:33.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:33 smithi098 bash[76817]: cluster 2023-09-23T18:27:32.565866+0000 mon.a (mon.0) 413 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2023-09-23T18:27:33.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:33 smithi098 bash[76817]: audit 2023-09-23T18:27:32.566426+0000 mon.a (mon.0) 414 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:27:33.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:33 smithi098 bash[76817]: cluster 2023-09-23T18:27:32.567668+0000 osd.4 (osd.4) 20 : cluster 3 failed to encode map e98 with expected crc 2023-09-23T18:27:33.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:33 smithi098 bash[79516]: cluster 2023-09-23T18:27:32.554505+0000 mon.a (mon.0) 411 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:27:33.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:33 smithi098 bash[79516]: cluster 2023-09-23T18:27:32.561901+0000 osd.3 (osd.3) 14 : cluster 3 failed to encode map e98 with expected crc 2023-09-23T18:27:33.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:33 smithi098 bash[79516]: cluster 2023-09-23T18:27:32.565782+0000 mon.a (mon.0) 412 : cluster 1 osd.2 [v2:172.21.15.98:6818/4249816143,v1:172.21.15.98:6819/4249816143] boot 2023-09-23T18:27:33.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:33 smithi098 bash[79516]: cluster 2023-09-23T18:27:32.565866+0000 mon.a (mon.0) 413 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2023-09-23T18:27:33.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:33 smithi098 bash[79516]: audit 2023-09-23T18:27:32.566426+0000 mon.a (mon.0) 414 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:27:33.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:33 smithi098 bash[79516]: cluster 2023-09-23T18:27:32.567668+0000 osd.4 (osd.4) 20 : cluster 3 failed to encode map e98 with expected crc 2023-09-23T18:27:34.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:33 smithi173 bash[59422]: cluster 2023-09-23T18:27:32.554505+0000 mon.a (mon.0) 411 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:27:34.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:33 smithi173 bash[59422]: cluster 2023-09-23T18:27:32.561901+0000 osd.3 (osd.3) 14 : cluster 3 failed to encode map e98 with expected crc 2023-09-23T18:27:34.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:33 smithi173 bash[59422]: cluster 2023-09-23T18:27:32.565782+0000 mon.a (mon.0) 412 : cluster 1 osd.2 [v2:172.21.15.98:6818/4249816143,v1:172.21.15.98:6819/4249816143] boot 2023-09-23T18:27:34.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:33 smithi173 bash[59422]: cluster 2023-09-23T18:27:32.565866+0000 mon.a (mon.0) 413 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2023-09-23T18:27:34.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:33 smithi173 bash[59422]: audit 2023-09-23T18:27:32.566426+0000 mon.a (mon.0) 414 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:27:34.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:33 smithi173 bash[59422]: cluster 2023-09-23T18:27:32.567668+0000 osd.4 (osd.4) 20 : cluster 3 failed to encode map e98 with expected crc 2023-09-23T18:27:34.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[76817]: cluster 2023-09-23T18:27:33.563749+0000 osd.3 (osd.3) 15 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:34.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[76817]: cluster 2023-09-23T18:27:33.566363+0000 mon.a (mon.0) 415 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2023-09-23T18:27:34.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[76817]: cluster 2023-09-23T18:27:33.566776+0000 osd.3 (osd.3) 16 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:34.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[76817]: cluster 2023-09-23T18:27:33.567220+0000 osd.6 (osd.6) 12 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:34.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[76817]: cluster 2023-09-23T18:27:33.569341+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:34.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[76817]: cluster 2023-09-23T18:27:33.569406+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:34.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[76817]: cluster 2023-09-23T18:27:33.571687+0000 osd.4 (osd.4) 21 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:34.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[76817]: cluster 2023-09-23T18:27:33.571914+0000 osd.6 (osd.6) 13 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:34.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[76817]: cluster 2023-09-23T18:27:33.573325+0000 osd.3 (osd.3) 17 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:34.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[76817]: cluster 2023-09-23T18:27:33.574096+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:34.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[76817]: cluster 2023-09-23T18:27:33.574565+0000 osd.4 (osd.4) 22 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:34.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[76817]: cluster 2023-09-23T18:27:33.574599+0000 osd.6 (osd.6) 14 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:34.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[76817]: cluster 2023-09-23T18:27:33.575327+0000 osd.3 (osd.3) 18 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:34.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[76817]: cluster 2023-09-23T18:27:33.575747+0000 mgr.y (mgr.26054) 255 : cluster 0 pgmap v195: 73 pgs: 3 peering, 2 active+undersized+degraded, 14 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T18:27:34.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[76817]: audit 2023-09-23T18:27:33.689094+0000 mon.a (mon.0) 416 : audit 0 from='client.? 172.21.15.98:0/766728298' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:34.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[79516]: cluster 2023-09-23T18:27:33.563749+0000 osd.3 (osd.3) 15 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:34.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[79516]: cluster 2023-09-23T18:27:33.566363+0000 mon.a (mon.0) 415 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2023-09-23T18:27:34.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[79516]: cluster 2023-09-23T18:27:33.566776+0000 osd.3 (osd.3) 16 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:34.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[79516]: cluster 2023-09-23T18:27:33.567220+0000 osd.6 (osd.6) 12 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:34.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[79516]: cluster 2023-09-23T18:27:33.569341+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:34.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[79516]: cluster 2023-09-23T18:27:33.569406+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:34.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[79516]: cluster 2023-09-23T18:27:33.571687+0000 osd.4 (osd.4) 21 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:34.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[79516]: cluster 2023-09-23T18:27:33.571914+0000 osd.6 (osd.6) 13 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:34.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[79516]: cluster 2023-09-23T18:27:33.573325+0000 osd.3 (osd.3) 17 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:34.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[79516]: cluster 2023-09-23T18:27:33.574096+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:34.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[79516]: cluster 2023-09-23T18:27:33.574565+0000 osd.4 (osd.4) 22 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:34.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[79516]: cluster 2023-09-23T18:27:33.574599+0000 osd.6 (osd.6) 14 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:34.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[79516]: cluster 2023-09-23T18:27:33.575327+0000 osd.3 (osd.3) 18 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:34.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[79516]: cluster 2023-09-23T18:27:33.575747+0000 mgr.y (mgr.26054) 255 : cluster 0 pgmap v195: 73 pgs: 3 peering, 2 active+undersized+degraded, 14 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T18:27:34.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:34 smithi098 bash[79516]: audit 2023-09-23T18:27:33.689094+0000 mon.a (mon.0) 416 : audit 0 from='client.? 172.21.15.98:0/766728298' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:35.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:34 smithi173 bash[59422]: cluster 2023-09-23T18:27:33.563749+0000 osd.3 (osd.3) 15 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:35.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:34 smithi173 bash[59422]: cluster 2023-09-23T18:27:33.566363+0000 mon.a (mon.0) 415 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2023-09-23T18:27:35.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:34 smithi173 bash[59422]: cluster 2023-09-23T18:27:33.566776+0000 osd.3 (osd.3) 16 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:35.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:34 smithi173 bash[59422]: cluster 2023-09-23T18:27:33.567220+0000 osd.6 (osd.6) 12 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:35.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:34 smithi173 bash[59422]: cluster 2023-09-23T18:27:33.569341+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:35.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:34 smithi173 bash[59422]: cluster 2023-09-23T18:27:33.569406+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:35.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:34 smithi173 bash[59422]: cluster 2023-09-23T18:27:33.571687+0000 osd.4 (osd.4) 21 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:35.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:34 smithi173 bash[59422]: cluster 2023-09-23T18:27:33.571914+0000 osd.6 (osd.6) 13 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:35.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:34 smithi173 bash[59422]: cluster 2023-09-23T18:27:33.573325+0000 osd.3 (osd.3) 17 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:35.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:34 smithi173 bash[59422]: cluster 2023-09-23T18:27:33.574096+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:35.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:34 smithi173 bash[59422]: cluster 2023-09-23T18:27:33.574565+0000 osd.4 (osd.4) 22 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:35.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:34 smithi173 bash[59422]: cluster 2023-09-23T18:27:33.574599+0000 osd.6 (osd.6) 14 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:35.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:34 smithi173 bash[59422]: cluster 2023-09-23T18:27:33.575327+0000 osd.3 (osd.3) 18 : cluster 3 failed to encode map e99 with expected crc 2023-09-23T18:27:35.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:34 smithi173 bash[59422]: cluster 2023-09-23T18:27:33.575747+0000 mgr.y (mgr.26054) 255 : cluster 0 pgmap v195: 73 pgs: 3 peering, 2 active+undersized+degraded, 14 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T18:27:35.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:34 smithi173 bash[59422]: audit 2023-09-23T18:27:33.689094+0000 mon.a (mon.0) 416 : audit 0 from='client.? 172.21.15.98:0/766728298' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:35.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:35 smithi098 bash[79516]: cluster 2023-09-23T18:27:34.574686+0000 mon.a (mon.0) 417 : cluster 3 Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:27:35.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:35 smithi098 bash[76817]: cluster 2023-09-23T18:27:34.574686+0000 mon.a (mon.0) 417 : cluster 3 Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:27:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:35 smithi173 bash[59422]: cluster 2023-09-23T18:27:34.574686+0000 mon.a (mon.0) 417 : cluster 3 Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:27:36.442 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:27:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:27:36] "GET /metrics HTTP/1.1" 200 34100 "" "Prometheus/2.33.4" 2023-09-23T18:27:36.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:36 smithi098 bash[76817]: cluster 2023-09-23T18:27:35.576699+0000 mgr.y (mgr.26054) 256 : cluster 0 pgmap v196: 73 pgs: 3 peering, 6 active+undersized, 64 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:36.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:36 smithi098 bash[76817]: cluster 2023-09-23T18:27:35.583673+0000 mon.a (mon.0) 418 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 2 pgs degraded) 2023-09-23T18:27:36.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:36 smithi098 bash[76817]: cluster 2023-09-23T18:27:35.583745+0000 mon.a (mon.0) 419 : cluster 1 Cluster is now healthy 2023-09-23T18:27:36.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:36 smithi098 bash[76817]: audit 2023-09-23T18:27:36.058905+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.98:0/2985902759' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:36.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:36 smithi098 bash[79516]: cluster 2023-09-23T18:27:35.576699+0000 mgr.y (mgr.26054) 256 : cluster 0 pgmap v196: 73 pgs: 3 peering, 6 active+undersized, 64 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:36.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:36 smithi098 bash[79516]: cluster 2023-09-23T18:27:35.583673+0000 mon.a (mon.0) 418 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 2 pgs degraded) 2023-09-23T18:27:36.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:36 smithi098 bash[79516]: cluster 2023-09-23T18:27:35.583745+0000 mon.a (mon.0) 419 : cluster 1 Cluster is now healthy 2023-09-23T18:27:36.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:36 smithi098 bash[79516]: audit 2023-09-23T18:27:36.058905+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.98:0/2985902759' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:37.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:36 smithi173 bash[59422]: cluster 2023-09-23T18:27:35.576699+0000 mgr.y (mgr.26054) 256 : cluster 0 pgmap v196: 73 pgs: 3 peering, 6 active+undersized, 64 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:37.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:36 smithi173 bash[59422]: cluster 2023-09-23T18:27:35.583673+0000 mon.a (mon.0) 418 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 2 pgs degraded) 2023-09-23T18:27:37.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:36 smithi173 bash[59422]: cluster 2023-09-23T18:27:35.583745+0000 mon.a (mon.0) 419 : cluster 1 Cluster is now healthy 2023-09-23T18:27:37.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:36 smithi173 bash[59422]: audit 2023-09-23T18:27:36.058905+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.98:0/2985902759' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:38.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:38 smithi098 bash[76817]: cluster 2023-09-23T18:27:37.577526+0000 mgr.y (mgr.26054) 257 : cluster 0 pgmap v197: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:38.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:38 smithi098 bash[76817]: audit 2023-09-23T18:27:38.436411+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.98:0/1884949373' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:38.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:38 smithi098 bash[79516]: cluster 2023-09-23T18:27:37.577526+0000 mgr.y (mgr.26054) 257 : cluster 0 pgmap v197: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:38.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:38 smithi098 bash[79516]: audit 2023-09-23T18:27:38.436411+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.98:0/1884949373' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:39.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:38 smithi173 bash[59422]: cluster 2023-09-23T18:27:37.577526+0000 mgr.y (mgr.26054) 257 : cluster 0 pgmap v197: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:39.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:38 smithi173 bash[59422]: audit 2023-09-23T18:27:38.436411+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.98:0/1884949373' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:40.958 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:40 smithi098 bash[76817]: cluster 2023-09-23T18:27:39.578280+0000 mgr.y (mgr.26054) 258 : cluster 0 pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:40.959 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:40 smithi098 bash[76817]: audit 2023-09-23T18:27:39.578504+0000 mon.a (mon.0) 420 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:40.959 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:40 smithi098 bash[76817]: audit 2023-09-23T18:27:39.580273+0000 mon.a (mon.0) 421 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:27:40.959 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:40 smithi098 bash[79516]: cluster 2023-09-23T18:27:39.578280+0000 mgr.y (mgr.26054) 258 : cluster 0 pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:40.959 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:40 smithi098 bash[79516]: audit 2023-09-23T18:27:39.578504+0000 mon.a (mon.0) 420 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:40.959 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:40 smithi098 bash[79516]: audit 2023-09-23T18:27:39.580273+0000 mon.a (mon.0) 421 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:27:41.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:40 smithi173 bash[59422]: cluster 2023-09-23T18:27:39.578280+0000 mgr.y (mgr.26054) 258 : cluster 0 pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:41.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:40 smithi173 bash[59422]: audit 2023-09-23T18:27:39.578504+0000 mon.a (mon.0) 420 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:41.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:40 smithi173 bash[59422]: audit 2023-09-23T18:27:39.580273+0000 mon.a (mon.0) 421 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:27:41.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:41 smithi098 bash[76817]: audit 2023-09-23T18:27:40.801424+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.98:0/2714355650' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:41.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:41 smithi098 bash[76817]: audit 2023-09-23T18:27:41.065576+0000 mon.a (mon.0) 422 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:41.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:41 smithi098 bash[76817]: audit 2023-09-23T18:27:41.073936+0000 mon.a (mon.0) 423 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:41.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:41 smithi098 bash[76817]: audit 2023-09-23T18:27:41.075483+0000 mon.a (mon.0) 424 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:27:41.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:41 smithi098 bash[76817]: audit 2023-09-23T18:27:41.076948+0000 mon.a (mon.0) 425 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:27:41.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:41 smithi098 bash[76817]: audit 2023-09-23T18:27:41.084749+0000 mon.a (mon.0) 426 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:41.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:41 smithi098 bash[76817]: audit 2023-09-23T18:27:41.170487+0000 mon.a (mon.0) 427 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:27:41.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:41 smithi098 bash[76817]: audit 2023-09-23T18:27:41.175130+0000 mon.a (mon.0) 428 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:27:41.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:41 smithi098 bash[76817]: audit 2023-09-23T18:27:41.177554+0000 mon.a (mon.0) 429 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:27:41.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:41 smithi098 bash[76817]: audit 2023-09-23T18:27:41.179601+0000 mon.a (mon.0) 430 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:27:41.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:41 smithi098 bash[76817]: audit 2023-09-23T18:27:41.181724+0000 mon.a (mon.0) 431 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T18:27:41.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:41 smithi098 bash[79516]: audit 2023-09-23T18:27:40.801424+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.98:0/2714355650' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:41.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:41 smithi098 bash[79516]: audit 2023-09-23T18:27:41.065576+0000 mon.a (mon.0) 422 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:41.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:41 smithi098 bash[79516]: audit 2023-09-23T18:27:41.073936+0000 mon.a (mon.0) 423 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:41.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:41 smithi098 bash[79516]: audit 2023-09-23T18:27:41.075483+0000 mon.a (mon.0) 424 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:27:41.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:41 smithi098 bash[79516]: audit 2023-09-23T18:27:41.076948+0000 mon.a (mon.0) 425 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:27:41.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:41 smithi098 bash[79516]: audit 2023-09-23T18:27:41.084749+0000 mon.a (mon.0) 426 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:41.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:41 smithi098 bash[79516]: audit 2023-09-23T18:27:41.170487+0000 mon.a (mon.0) 427 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:27:41.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:41 smithi098 bash[79516]: audit 2023-09-23T18:27:41.175130+0000 mon.a (mon.0) 428 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:27:41.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:41 smithi098 bash[79516]: audit 2023-09-23T18:27:41.177554+0000 mon.a (mon.0) 429 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:27:41.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:41 smithi098 bash[79516]: audit 2023-09-23T18:27:41.179601+0000 mon.a (mon.0) 430 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:27:41.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:41 smithi098 bash[79516]: audit 2023-09-23T18:27:41.181724+0000 mon.a (mon.0) 431 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T18:27:42.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:41 smithi173 bash[59422]: audit 2023-09-23T18:27:40.801424+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.98:0/2714355650' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:42.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:41 smithi173 bash[59422]: audit 2023-09-23T18:27:41.065576+0000 mon.a (mon.0) 422 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:42.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:41 smithi173 bash[59422]: audit 2023-09-23T18:27:41.073936+0000 mon.a (mon.0) 423 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:42.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:41 smithi173 bash[59422]: audit 2023-09-23T18:27:41.075483+0000 mon.a (mon.0) 424 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:27:42.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:41 smithi173 bash[59422]: audit 2023-09-23T18:27:41.076948+0000 mon.a (mon.0) 425 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:27:42.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:41 smithi173 bash[59422]: audit 2023-09-23T18:27:41.084749+0000 mon.a (mon.0) 426 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:42.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:41 smithi173 bash[59422]: audit 2023-09-23T18:27:41.170487+0000 mon.a (mon.0) 427 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:27:42.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:41 smithi173 bash[59422]: audit 2023-09-23T18:27:41.175130+0000 mon.a (mon.0) 428 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:27:42.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:41 smithi173 bash[59422]: audit 2023-09-23T18:27:41.177554+0000 mon.a (mon.0) 429 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:27:42.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:41 smithi173 bash[59422]: audit 2023-09-23T18:27:41.179601+0000 mon.a (mon.0) 430 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:27:42.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:41 smithi173 bash[59422]: audit 2023-09-23T18:27:41.181724+0000 mon.a (mon.0) 431 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T18:27:42.827 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:42 smithi098 bash[79516]: audit 2023-09-23T18:27:41.182035+0000 mgr.y (mgr.26054) 259 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T18:27:42.827 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:42 smithi098 bash[79516]: cephadm 2023-09-23T18:27:41.183352+0000 mgr.y (mgr.26054) 260 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-09-23T18:27:42.828 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:42 smithi098 bash[79516]: cluster 2023-09-23T18:27:41.578920+0000 mgr.y (mgr.26054) 261 : cluster 0 pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:42.828 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:42 smithi098 bash[76817]: audit 2023-09-23T18:27:41.182035+0000 mgr.y (mgr.26054) 259 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T18:27:42.828 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:42 smithi098 bash[76817]: cephadm 2023-09-23T18:27:41.183352+0000 mgr.y (mgr.26054) 260 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-09-23T18:27:42.828 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:42 smithi098 bash[76817]: cluster 2023-09-23T18:27:41.578920+0000 mgr.y (mgr.26054) 261 : cluster 0 pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:43.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:42 smithi173 bash[59422]: audit 2023-09-23T18:27:41.182035+0000 mgr.y (mgr.26054) 259 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T18:27:43.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:42 smithi173 bash[59422]: cephadm 2023-09-23T18:27:41.183352+0000 mgr.y (mgr.26054) 260 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-09-23T18:27:43.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:42 smithi173 bash[59422]: cluster 2023-09-23T18:27:41.578920+0000 mgr.y (mgr.26054) 261 : cluster 0 pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:43.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:43 smithi098 bash[79516]: audit 2023-09-23T18:27:43.162747+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.98:0/3169749424' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:43.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:43 smithi098 bash[76817]: audit 2023-09-23T18:27:43.162747+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.98:0/3169749424' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:43 smithi173 bash[59422]: audit 2023-09-23T18:27:43.162747+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.98:0/3169749424' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:44.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:44 smithi098 bash[76817]: cluster 2023-09-23T18:27:43.579707+0000 mgr.y (mgr.26054) 262 : cluster 0 pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:44.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:44 smithi098 bash[76817]: cephadm 2023-09-23T18:27:43.652831+0000 mgr.y (mgr.26054) 263 : cephadm 1 Upgrade: Updating osd.3 2023-09-23T18:27:44.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:44 smithi098 bash[76817]: audit 2023-09-23T18:27:43.662677+0000 mon.a (mon.0) 432 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:44.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:44 smithi098 bash[76817]: audit 2023-09-23T18:27:43.664317+0000 mon.a (mon.0) 433 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T18:27:44.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:44 smithi098 bash[76817]: audit 2023-09-23T18:27:43.665637+0000 mon.a (mon.0) 434 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:27:44.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:44 smithi098 bash[76817]: cephadm 2023-09-23T18:27:43.669463+0000 mgr.y (mgr.26054) 264 : cephadm 1 Deploying daemon osd.3 on smithi098 2023-09-23T18:27:44.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:44 smithi098 bash[79516]: cluster 2023-09-23T18:27:43.579707+0000 mgr.y (mgr.26054) 262 : cluster 0 pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:44.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:44 smithi098 bash[79516]: cephadm 2023-09-23T18:27:43.652831+0000 mgr.y (mgr.26054) 263 : cephadm 1 Upgrade: Updating osd.3 2023-09-23T18:27:44.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:44 smithi098 bash[79516]: audit 2023-09-23T18:27:43.662677+0000 mon.a (mon.0) 432 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:44.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:44 smithi098 bash[79516]: audit 2023-09-23T18:27:43.664317+0000 mon.a (mon.0) 433 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T18:27:44.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:44 smithi098 bash[79516]: audit 2023-09-23T18:27:43.665637+0000 mon.a (mon.0) 434 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:27:44.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:44 smithi098 bash[79516]: cephadm 2023-09-23T18:27:43.669463+0000 mgr.y (mgr.26054) 264 : cephadm 1 Deploying daemon osd.3 on smithi098 2023-09-23T18:27:45.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:44 smithi173 bash[59422]: cluster 2023-09-23T18:27:43.579707+0000 mgr.y (mgr.26054) 262 : cluster 0 pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:45.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:44 smithi173 bash[59422]: cephadm 2023-09-23T18:27:43.652831+0000 mgr.y (mgr.26054) 263 : cephadm 1 Upgrade: Updating osd.3 2023-09-23T18:27:45.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:44 smithi173 bash[59422]: audit 2023-09-23T18:27:43.662677+0000 mon.a (mon.0) 432 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:45.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:44 smithi173 bash[59422]: audit 2023-09-23T18:27:43.664317+0000 mon.a (mon.0) 433 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T18:27:45.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:44 smithi173 bash[59422]: audit 2023-09-23T18:27:43.665637+0000 mon.a (mon.0) 434 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:27:45.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:44 smithi173 bash[59422]: cephadm 2023-09-23T18:27:43.669463+0000 mgr.y (mgr.26054) 264 : cephadm 1 Deploying daemon osd.3 on smithi098 2023-09-23T18:27:46.006 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:45 smithi098 bash[79516]: audit 2023-09-23T18:27:44.689548+0000 mon.a (mon.0) 435 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:46.006 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:45 smithi098 bash[79516]: audit 2023-09-23T18:27:45.518471+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.98:0/3381491582' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:46.007 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:45 smithi098 bash[76817]: audit 2023-09-23T18:27:44.689548+0000 mon.a (mon.0) 435 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:46.007 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:45 smithi098 bash[76817]: audit 2023-09-23T18:27:45.518471+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.98:0/3381491582' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:46.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:45 smithi173 bash[59422]: audit 2023-09-23T18:27:44.689548+0000 mon.a (mon.0) 435 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:46.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:45 smithi173 bash[59422]: audit 2023-09-23T18:27:45.518471+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.98:0/3381491582' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:46.442 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:27:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:27:46] "GET /metrics HTTP/1.1" 200 34100 "" "Prometheus/2.33.4" 2023-09-23T18:27:47.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:46 smithi173 bash[59422]: cluster 2023-09-23T18:27:45.580439+0000 mgr.y (mgr.26054) 265 : cluster 0 pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:47.055 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:46 smithi098 bash[76817]: cluster 2023-09-23T18:27:45.580439+0000 mgr.y (mgr.26054) 265 : cluster 0 pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:47.056 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:46 smithi098 bash[79516]: cluster 2023-09-23T18:27:45.580439+0000 mgr.y (mgr.26054) 265 : cluster 0 pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:48.692 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 18:27:48 smithi098 systemd[1]: Stopping Ceph osd.3 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T18:27:49.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:48 smithi173 bash[59422]: cluster 2023-09-23T18:27:47.581081+0000 mgr.y (mgr.26054) 266 : cluster 0 pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:49.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:48 smithi173 bash[59422]: audit 2023-09-23T18:27:47.869945+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.98:0/934774176' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:49.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:48 smithi098 bash[79516]: cluster 2023-09-23T18:27:47.581081+0000 mgr.y (mgr.26054) 266 : cluster 0 pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:49.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:48 smithi098 bash[79516]: audit 2023-09-23T18:27:47.869945+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.98:0/934774176' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:49.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:48 smithi098 bash[76817]: cluster 2023-09-23T18:27:47.581081+0000 mgr.y (mgr.26054) 266 : cluster 0 pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:49.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:48 smithi098 bash[76817]: audit 2023-09-23T18:27:47.869945+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.98:0/934774176' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:49.193 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 18:27:48 smithi098 bash[49221]: debug 2023-09-23T18:27:48.801+0000 7f00dad0e700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T18:27:49.193 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 18:27:48 smithi098 bash[49221]: debug 2023-09-23T18:27:48.801+0000 7f00dad0e700 -1 osd.3 99 *** Got signal Terminated *** 2023-09-23T18:27:49.193 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 18:27:48 smithi098 bash[49221]: debug 2023-09-23T18:27:48.801+0000 7f00dad0e700 -1 osd.3 99 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T18:27:50.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:49 smithi173 bash[59422]: cluster 2023-09-23T18:27:48.803894+0000 mon.a (mon.0) 436 : cluster 1 osd.3 marked itself down and dead 2023-09-23T18:27:50.067 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:49 smithi098 bash[76817]: cluster 2023-09-23T18:27:48.803894+0000 mon.a (mon.0) 436 : cluster 1 osd.3 marked itself down and dead 2023-09-23T18:27:50.067 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:49 smithi098 bash[79516]: cluster 2023-09-23T18:27:48.803894+0000 mon.a (mon.0) 436 : cluster 1 osd.3 marked itself down and dead 2023-09-23T18:27:50.942 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 18:27:50 smithi098 bash[93162]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-3 2023-09-23T18:27:50.942 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 18:27:50 smithi098 bash[93255]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-3 2023-09-23T18:27:50.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:50 smithi098 bash[79516]: cluster 2023-09-23T18:27:49.581900+0000 mgr.y (mgr.26054) 267 : cluster 0 pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:50.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:50 smithi098 bash[79516]: cluster 2023-09-23T18:27:49.699812+0000 mon.a (mon.0) 437 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:27:50.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:50 smithi098 bash[79516]: cluster 2023-09-23T18:27:49.718125+0000 mon.a (mon.0) 438 : cluster 0 osdmap e100: 8 total, 7 up, 8 in 2023-09-23T18:27:50.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:50 smithi098 bash[79516]: cluster 2023-09-23T18:27:49.718195+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T18:27:50.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:50 smithi098 bash[79516]: cluster 2023-09-23T18:27:49.726154+0000 osd.4 (osd.4) 23 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T18:27:50.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:50 smithi098 bash[79516]: cluster 2023-09-23T18:27:49.726637+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T18:27:50.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:50 smithi098 bash[79516]: cluster 2023-09-23T18:27:49.729525+0000 osd.4 (osd.4) 24 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T18:27:50.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:50 smithi098 bash[79516]: audit 2023-09-23T18:27:50.235637+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.98:0/1778378091' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:50.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:50 smithi098 bash[76817]: cluster 2023-09-23T18:27:49.581900+0000 mgr.y (mgr.26054) 267 : cluster 0 pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:50.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:50 smithi098 bash[76817]: cluster 2023-09-23T18:27:49.699812+0000 mon.a (mon.0) 437 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:27:50.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:50 smithi098 bash[76817]: cluster 2023-09-23T18:27:49.718125+0000 mon.a (mon.0) 438 : cluster 0 osdmap e100: 8 total, 7 up, 8 in 2023-09-23T18:27:50.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:50 smithi098 bash[76817]: cluster 2023-09-23T18:27:49.718195+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T18:27:50.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:50 smithi098 bash[76817]: cluster 2023-09-23T18:27:49.726154+0000 osd.4 (osd.4) 23 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T18:27:50.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:50 smithi098 bash[76817]: cluster 2023-09-23T18:27:49.726637+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T18:27:50.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:50 smithi098 bash[76817]: cluster 2023-09-23T18:27:49.729525+0000 osd.4 (osd.4) 24 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T18:27:50.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:50 smithi098 bash[76817]: audit 2023-09-23T18:27:50.235637+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.98:0/1778378091' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:51.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:50 smithi173 bash[59422]: cluster 2023-09-23T18:27:49.581900+0000 mgr.y (mgr.26054) 267 : cluster 0 pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:51.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:50 smithi173 bash[59422]: cluster 2023-09-23T18:27:49.699812+0000 mon.a (mon.0) 437 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:27:51.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:50 smithi173 bash[59422]: cluster 2023-09-23T18:27:49.718125+0000 mon.a (mon.0) 438 : cluster 0 osdmap e100: 8 total, 7 up, 8 in 2023-09-23T18:27:51.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:50 smithi173 bash[59422]: cluster 2023-09-23T18:27:49.718195+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T18:27:51.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:50 smithi173 bash[59422]: cluster 2023-09-23T18:27:49.726154+0000 osd.4 (osd.4) 23 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T18:27:51.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:50 smithi173 bash[59422]: cluster 2023-09-23T18:27:49.726637+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T18:27:51.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:50 smithi173 bash[59422]: cluster 2023-09-23T18:27:49.729525+0000 osd.4 (osd.4) 24 : cluster 3 failed to encode map e100 with expected crc 2023-09-23T18:27:51.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:50 smithi173 bash[59422]: audit 2023-09-23T18:27:50.235637+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.98:0/1778378091' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:51.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:51 smithi098 bash[76817]: cluster 2023-09-23T18:27:50.717253+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T18:27:51.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:51 smithi098 bash[76817]: cluster 2023-09-23T18:27:50.717607+0000 osd.4 (osd.4) 25 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T18:27:51.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:51 smithi098 bash[76817]: cluster 2023-09-23T18:27:50.717622+0000 mon.a (mon.0) 439 : cluster 0 osdmap e101: 8 total, 7 up, 8 in 2023-09-23T18:27:51.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:51 smithi098 bash[76817]: cluster 2023-09-23T18:27:50.720216+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T18:27:51.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:51 smithi098 bash[76817]: cluster 2023-09-23T18:27:50.720227+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T18:27:51.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:51 smithi098 bash[76817]: cluster 2023-09-23T18:27:50.720844+0000 osd.4 (osd.4) 26 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T18:27:51.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:51 smithi098 bash[76817]: cluster 2023-09-23T18:27:50.720891+0000 osd.6 (osd.6) 15 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T18:27:51.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:51 smithi098 bash[76817]: cluster 2023-09-23T18:27:50.723050+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T18:27:51.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:51 smithi098 bash[76817]: cluster 2023-09-23T18:27:50.725040+0000 osd.6 (osd.6) 16 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T18:27:51.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:51 smithi098 bash[79516]: cluster 2023-09-23T18:27:50.717253+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T18:27:51.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:51 smithi098 bash[79516]: cluster 2023-09-23T18:27:50.717607+0000 osd.4 (osd.4) 25 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T18:27:51.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:51 smithi098 bash[79516]: cluster 2023-09-23T18:27:50.717622+0000 mon.a (mon.0) 439 : cluster 0 osdmap e101: 8 total, 7 up, 8 in 2023-09-23T18:27:51.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:51 smithi098 bash[79516]: cluster 2023-09-23T18:27:50.720216+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T18:27:51.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:51 smithi098 bash[79516]: cluster 2023-09-23T18:27:50.720227+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T18:27:51.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:51 smithi098 bash[79516]: cluster 2023-09-23T18:27:50.720844+0000 osd.4 (osd.4) 26 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T18:27:51.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:51 smithi098 bash[79516]: cluster 2023-09-23T18:27:50.720891+0000 osd.6 (osd.6) 15 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T18:27:51.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:51 smithi098 bash[79516]: cluster 2023-09-23T18:27:50.723050+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T18:27:51.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:51 smithi098 bash[79516]: cluster 2023-09-23T18:27:50.725040+0000 osd.6 (osd.6) 16 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T18:27:52.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:51 smithi173 bash[59422]: cluster 2023-09-23T18:27:50.717253+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T18:27:52.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:51 smithi173 bash[59422]: cluster 2023-09-23T18:27:50.717607+0000 osd.4 (osd.4) 25 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T18:27:52.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:51 smithi173 bash[59422]: cluster 2023-09-23T18:27:50.717622+0000 mon.a (mon.0) 439 : cluster 0 osdmap e101: 8 total, 7 up, 8 in 2023-09-23T18:27:52.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:51 smithi173 bash[59422]: cluster 2023-09-23T18:27:50.720216+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T18:27:52.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:51 smithi173 bash[59422]: cluster 2023-09-23T18:27:50.720227+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T18:27:52.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:51 smithi173 bash[59422]: cluster 2023-09-23T18:27:50.720844+0000 osd.4 (osd.4) 26 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T18:27:52.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:51 smithi173 bash[59422]: cluster 2023-09-23T18:27:50.720891+0000 osd.6 (osd.6) 15 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T18:27:52.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:51 smithi173 bash[59422]: cluster 2023-09-23T18:27:50.723050+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T18:27:52.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:51 smithi173 bash[59422]: cluster 2023-09-23T18:27:50.725040+0000 osd.6 (osd.6) 16 : cluster 3 failed to encode map e101 with expected crc 2023-09-23T18:27:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:52 smithi173 bash[59422]: cluster 2023-09-23T18:27:51.582738+0000 mgr.y (mgr.26054) 268 : cluster 0 pgmap v206: 73 pgs: 14 peering, 6 stale+active+clean, 3 active+undersized, 50 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:52 smithi173 bash[59422]: cluster 2023-09-23T18:27:51.714280+0000 mon.a (mon.0) 440 : cluster 3 Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T18:27:53.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:52 smithi173 bash[59422]: audit 2023-09-23T18:27:52.586908+0000 mon.a (mon.0) 441 : audit 0 from='client.? 172.21.15.98:0/363372036' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:53.136 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:52 smithi098 bash[79516]: cluster 2023-09-23T18:27:51.582738+0000 mgr.y (mgr.26054) 268 : cluster 0 pgmap v206: 73 pgs: 14 peering, 6 stale+active+clean, 3 active+undersized, 50 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:53.136 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:52 smithi098 bash[79516]: cluster 2023-09-23T18:27:51.714280+0000 mon.a (mon.0) 440 : cluster 3 Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T18:27:53.136 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:52 smithi098 bash[79516]: audit 2023-09-23T18:27:52.586908+0000 mon.a (mon.0) 441 : audit 0 from='client.? 172.21.15.98:0/363372036' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:53.137 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 18:27:52 smithi098 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.3.service: Succeeded. 2023-09-23T18:27:53.137 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 18:27:52 smithi098 systemd[1]: Stopped Ceph osd.3 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:27:53.137 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:52 smithi098 bash[76817]: cluster 2023-09-23T18:27:51.582738+0000 mgr.y (mgr.26054) 268 : cluster 0 pgmap v206: 73 pgs: 14 peering, 6 stale+active+clean, 3 active+undersized, 50 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:27:53.137 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:52 smithi098 bash[76817]: cluster 2023-09-23T18:27:51.714280+0000 mon.a (mon.0) 440 : cluster 3 Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T18:27:53.137 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:52 smithi098 bash[76817]: audit 2023-09-23T18:27:52.586908+0000 mon.a (mon.0) 441 : audit 0 from='client.? 172.21.15.98:0/363372036' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:53.442 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 18:27:53 smithi098 systemd[1]: Started Ceph osd.3 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:27:54.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:54 smithi098 bash[79516]: audit 2023-09-23T18:27:53.169806+0000 mon.a (mon.0) 442 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:54.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:54 smithi098 bash[79516]: audit 2023-09-23T18:27:53.177844+0000 mon.a (mon.0) 443 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:54.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:54 smithi098 bash[79516]: audit 2023-09-23T18:27:53.205735+0000 mon.a (mon.0) 444 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:27:54.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:54 smithi098 bash[79516]: cluster 2023-09-23T18:27:53.583608+0000 mgr.y (mgr.26054) 269 : cluster 0 pgmap v207: 73 pgs: 1 active+undersized+degraded, 14 peering, 16 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T18:27:54.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:54 smithi098 bash[76817]: audit 2023-09-23T18:27:53.169806+0000 mon.a (mon.0) 442 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:54.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:54 smithi098 bash[76817]: audit 2023-09-23T18:27:53.177844+0000 mon.a (mon.0) 443 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:54.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:54 smithi098 bash[76817]: audit 2023-09-23T18:27:53.205735+0000 mon.a (mon.0) 444 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:27:54.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:54 smithi098 bash[76817]: cluster 2023-09-23T18:27:53.583608+0000 mgr.y (mgr.26054) 269 : cluster 0 pgmap v207: 73 pgs: 1 active+undersized+degraded, 14 peering, 16 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T18:27:54.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:54 smithi173 bash[59422]: audit 2023-09-23T18:27:53.169806+0000 mon.a (mon.0) 442 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:54.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:54 smithi173 bash[59422]: audit 2023-09-23T18:27:53.177844+0000 mon.a (mon.0) 443 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:54.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:54 smithi173 bash[59422]: audit 2023-09-23T18:27:53.205735+0000 mon.a (mon.0) 444 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:27:54.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:54 smithi173 bash[59422]: cluster 2023-09-23T18:27:53.583608+0000 mgr.y (mgr.26054) 269 : cluster 0 pgmap v207: 73 pgs: 1 active+undersized+degraded, 14 peering, 16 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T18:27:54.901 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 18:27:54 smithi098 bash[93440]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T18:27:54.901 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 18:27:54 smithi098 bash[93440]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T18:27:54.901 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 18:27:54 smithi098 bash[93440]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-09-23T18:27:54.901 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 18:27:54 smithi098 bash[93440]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-09-23T18:27:54.901 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 18:27:54 smithi098 bash[93440]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-09-23T18:27:54.901 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 18:27:54 smithi098 bash[93440]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-09-23T18:27:54.902 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 18:27:54 smithi098 bash[93440]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T18:27:54.902 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 18:27:54 smithi098 bash[93440]: --> ceph-volume lvm activate successful for osd ID: 3 2023-09-23T18:27:55.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:55 smithi098 bash[79516]: cluster 2023-09-23T18:27:54.175354+0000 mon.a (mon.0) 445 : cluster 3 Health check failed: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:27:55.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:55 smithi098 bash[79516]: audit 2023-09-23T18:27:54.581513+0000 mon.a (mon.0) 446 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:55.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:55 smithi098 bash[79516]: audit 2023-09-23T18:27:54.583098+0000 mon.a (mon.0) 447 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:27:55.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:55 smithi098 bash[79516]: audit 2023-09-23T18:27:54.945475+0000 mon.a (mon.0) 448 : audit 0 from='client.? 172.21.15.98:0/2167670964' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:55.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:55 smithi098 bash[76817]: cluster 2023-09-23T18:27:54.175354+0000 mon.a (mon.0) 445 : cluster 3 Health check failed: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:27:55.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:55 smithi098 bash[76817]: audit 2023-09-23T18:27:54.581513+0000 mon.a (mon.0) 446 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:55.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:55 smithi098 bash[76817]: audit 2023-09-23T18:27:54.583098+0000 mon.a (mon.0) 447 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:27:55.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:55 smithi098 bash[76817]: audit 2023-09-23T18:27:54.945475+0000 mon.a (mon.0) 448 : audit 0 from='client.? 172.21.15.98:0/2167670964' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:55.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:55 smithi173 bash[59422]: cluster 2023-09-23T18:27:54.175354+0000 mon.a (mon.0) 445 : cluster 3 Health check failed: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:27:55.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:55 smithi173 bash[59422]: audit 2023-09-23T18:27:54.581513+0000 mon.a (mon.0) 446 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:27:55.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:55 smithi173 bash[59422]: audit 2023-09-23T18:27:54.583098+0000 mon.a (mon.0) 447 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:27:55.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:55 smithi173 bash[59422]: audit 2023-09-23T18:27:54.945475+0000 mon.a (mon.0) 448 : audit 0 from='client.? 172.21.15.98:0/2167670964' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:56.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:56 smithi098 bash[76817]: cluster 2023-09-23T18:27:55.584657+0000 mgr.y (mgr.26054) 270 : cluster 0 pgmap v208: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T18:27:56.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:56 smithi098 bash[79516]: cluster 2023-09-23T18:27:55.584657+0000 mgr.y (mgr.26054) 270 : cluster 0 pgmap v208: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T18:27:56.440 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:27:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:27:56] "GET /metrics HTTP/1.1" 200 34098 "" "Prometheus/2.33.4" 2023-09-23T18:27:56.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:56 smithi173 bash[59422]: cluster 2023-09-23T18:27:55.584657+0000 mgr.y (mgr.26054) 270 : cluster 0 pgmap v208: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T18:27:56.692 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 18:27:56 smithi098 bash[93735]: debug 2023-09-23T18:27:56.573+0000 7fa999c0c700 1 -- 172.21.15.98:0/4103407393 <== mon.0 v2:172.21.15.98:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c252fa84e0 con 0x55c253002800 2023-09-23T18:27:57.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:57 smithi098 bash[76817]: cluster 2023-09-23T18:27:56.170886+0000 mon.a (mon.0) 449 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2023-09-23T18:27:57.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:57 smithi098 bash[79516]: cluster 2023-09-23T18:27:56.170886+0000 mon.a (mon.0) 449 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2023-09-23T18:27:57.442 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 18:27:57 smithi098 bash[93735]: debug 2023-09-23T18:27:57.117+0000 7fa9a2d91640 -1 Falling back to public interface 2023-09-23T18:27:57.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:57 smithi173 bash[59422]: cluster 2023-09-23T18:27:56.170886+0000 mon.a (mon.0) 449 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2023-09-23T18:27:58.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:58 smithi173 bash[59422]: audit 2023-09-23T18:27:57.318535+0000 mon.a (mon.0) 450 : audit 0 from='client.? 172.21.15.98:0/602156293' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:58.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:58 smithi173 bash[59422]: cluster 2023-09-23T18:27:57.585590+0000 mgr.y (mgr.26054) 271 : cluster 0 pgmap v209: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T18:27:58.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:58 smithi098 bash[76817]: audit 2023-09-23T18:27:57.318535+0000 mon.a (mon.0) 450 : audit 0 from='client.? 172.21.15.98:0/602156293' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:58.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:58 smithi098 bash[76817]: cluster 2023-09-23T18:27:57.585590+0000 mgr.y (mgr.26054) 271 : cluster 0 pgmap v209: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T18:27:58.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:58 smithi098 bash[79516]: audit 2023-09-23T18:27:57.318535+0000 mon.a (mon.0) 450 : audit 0 from='client.? 172.21.15.98:0/602156293' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:27:58.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:58 smithi098 bash[79516]: cluster 2023-09-23T18:27:57.585590+0000 mgr.y (mgr.26054) 271 : cluster 0 pgmap v209: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T18:27:59.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:27:59 smithi098 bash[79516]: cluster 2023-09-23T18:27:59.334452+0000 mon.a (mon.0) 451 : cluster 3 Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T18:27:59.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:27:59 smithi098 bash[76817]: cluster 2023-09-23T18:27:59.334452+0000 mon.a (mon.0) 451 : cluster 3 Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T18:27:59.781 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:27:59 smithi173 bash[59422]: cluster 2023-09-23T18:27:59.334452+0000 mon.a (mon.0) 451 : cluster 3 Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T18:28:00.692 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 18:28:00 smithi098 bash[93735]: debug 2023-09-23T18:28:00.289+0000 7fa9a2d91640 -1 osd.3 99 log_to_monitors true 2023-09-23T18:28:00.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:00 smithi098 bash[76817]: cluster 2023-09-23T18:27:59.586178+0000 mgr.y (mgr.26054) 272 : cluster 0 pgmap v210: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T18:28:00.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:00 smithi098 bash[76817]: audit 2023-09-23T18:27:59.711093+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.98:0/2173297222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:28:00.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:00 smithi098 bash[76817]: audit 2023-09-23T18:28:00.250393+0000 mon.a (mon.0) 452 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:00.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:00 smithi098 bash[76817]: audit 2023-09-23T18:28:00.259364+0000 mon.a (mon.0) 453 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:00.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:00 smithi098 bash[76817]: audit 2023-09-23T18:28:00.294600+0000 mon.a (mon.0) 454 : audit 1 from='osd.3 [v2:172.21.15.98:6826/2042614518,v1:172.21.15.98:6827/2042614518]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T18:28:00.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:00 smithi098 bash[79516]: cluster 2023-09-23T18:27:59.586178+0000 mgr.y (mgr.26054) 272 : cluster 0 pgmap v210: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T18:28:00.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:00 smithi098 bash[79516]: audit 2023-09-23T18:27:59.711093+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.98:0/2173297222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:28:00.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:00 smithi098 bash[79516]: audit 2023-09-23T18:28:00.250393+0000 mon.a (mon.0) 452 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:00.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:00 smithi098 bash[79516]: audit 2023-09-23T18:28:00.259364+0000 mon.a (mon.0) 453 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:00.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:00 smithi098 bash[79516]: audit 2023-09-23T18:28:00.294600+0000 mon.a (mon.0) 454 : audit 1 from='osd.3 [v2:172.21.15.98:6826/2042614518,v1:172.21.15.98:6827/2042614518]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T18:28:00.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:00 smithi173 bash[59422]: cluster 2023-09-23T18:27:59.586178+0000 mgr.y (mgr.26054) 272 : cluster 0 pgmap v210: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T18:28:00.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:00 smithi173 bash[59422]: audit 2023-09-23T18:27:59.711093+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.98:0/2173297222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:28:00.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:00 smithi173 bash[59422]: audit 2023-09-23T18:28:00.250393+0000 mon.a (mon.0) 452 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:00.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:00 smithi173 bash[59422]: audit 2023-09-23T18:28:00.259364+0000 mon.a (mon.0) 453 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:00.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:00 smithi173 bash[59422]: audit 2023-09-23T18:28:00.294600+0000 mon.a (mon.0) 454 : audit 1 from='osd.3 [v2:172.21.15.98:6826/2042614518,v1:172.21.15.98:6827/2042614518]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T18:28:01.692 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 18:28:01 smithi098 bash[93735]: debug 2023-09-23T18:28:01.317+0000 7fa9929fb700 -1 osd.3 99 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T18:28:02.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:02 smithi098 bash[79516]: audit 2023-09-23T18:28:01.057069+0000 mon.a (mon.0) 455 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:02.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:02 smithi098 bash[79516]: audit 2023-09-23T18:28:01.066162+0000 mon.a (mon.0) 456 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:02.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:02 smithi098 bash[79516]: audit 2023-09-23T18:28:01.265920+0000 mon.a (mon.0) 457 : audit 1 from='osd.3 [v2:172.21.15.98:6826/2042614518,v1:172.21.15.98:6827/2042614518]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T18:28:02.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:02 smithi098 bash[79516]: cluster 2023-09-23T18:28:01.266086+0000 mon.a (mon.0) 458 : cluster 0 osdmap e102: 8 total, 7 up, 8 in 2023-09-23T18:28:02.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:02 smithi098 bash[79516]: audit 2023-09-23T18:28:01.266979+0000 mon.a (mon.0) 459 : audit 1 from='osd.3 [v2:172.21.15.98:6826/2042614518,v1:172.21.15.98:6827/2042614518]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:28:02.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:02 smithi098 bash[79516]: cluster 2023-09-23T18:28:01.586919+0000 mgr.y (mgr.26054) 273 : cluster 0 pgmap v212: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T18:28:02.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:02 smithi098 bash[76817]: audit 2023-09-23T18:28:01.057069+0000 mon.a (mon.0) 455 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:02.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:02 smithi098 bash[76817]: audit 2023-09-23T18:28:01.066162+0000 mon.a (mon.0) 456 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:02.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:02 smithi098 bash[76817]: audit 2023-09-23T18:28:01.265920+0000 mon.a (mon.0) 457 : audit 1 from='osd.3 [v2:172.21.15.98:6826/2042614518,v1:172.21.15.98:6827/2042614518]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T18:28:02.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:02 smithi098 bash[76817]: cluster 2023-09-23T18:28:01.266086+0000 mon.a (mon.0) 458 : cluster 0 osdmap e102: 8 total, 7 up, 8 in 2023-09-23T18:28:02.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:02 smithi098 bash[76817]: audit 2023-09-23T18:28:01.266979+0000 mon.a (mon.0) 459 : audit 1 from='osd.3 [v2:172.21.15.98:6826/2042614518,v1:172.21.15.98:6827/2042614518]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:28:02.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:02 smithi098 bash[76817]: cluster 2023-09-23T18:28:01.586919+0000 mgr.y (mgr.26054) 273 : cluster 0 pgmap v212: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T18:28:02.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:02 smithi173 bash[59422]: audit 2023-09-23T18:28:01.057069+0000 mon.a (mon.0) 455 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:02.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:02 smithi173 bash[59422]: audit 2023-09-23T18:28:01.066162+0000 mon.a (mon.0) 456 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:02.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:02 smithi173 bash[59422]: audit 2023-09-23T18:28:01.265920+0000 mon.a (mon.0) 457 : audit 1 from='osd.3 [v2:172.21.15.98:6826/2042614518,v1:172.21.15.98:6827/2042614518]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T18:28:02.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:02 smithi173 bash[59422]: cluster 2023-09-23T18:28:01.266086+0000 mon.a (mon.0) 458 : cluster 0 osdmap e102: 8 total, 7 up, 8 in 2023-09-23T18:28:02.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:02 smithi173 bash[59422]: audit 2023-09-23T18:28:01.266979+0000 mon.a (mon.0) 459 : audit 1 from='osd.3 [v2:172.21.15.98:6826/2042614518,v1:172.21.15.98:6827/2042614518]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:28:02.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:02 smithi173 bash[59422]: cluster 2023-09-23T18:28:01.586919+0000 mgr.y (mgr.26054) 273 : cluster 0 pgmap v212: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T18:28:03.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:03 smithi098 bash[76817]: audit 2023-09-23T18:28:02.076333+0000 mon.a (mon.0) 460 : audit 0 from='client.? 172.21.15.98:0/2733584499' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:28:03.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:03 smithi098 bash[76817]: cluster 2023-09-23T18:28:02.262725+0000 mon.a (mon.0) 461 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:28:03.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:03 smithi098 bash[76817]: cluster 2023-09-23T18:28:02.272547+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e102 with expected crc 2023-09-23T18:28:03.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:03 smithi098 bash[76817]: cluster 2023-09-23T18:28:02.272555+0000 mon.a (mon.0) 462 : cluster 1 osd.3 [v2:172.21.15.98:6826/2042614518,v1:172.21.15.98:6827/2042614518] boot 2023-09-23T18:28:03.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:03 smithi098 bash[76817]: cluster 2023-09-23T18:28:02.272639+0000 mon.a (mon.0) 463 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2023-09-23T18:28:03.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:03 smithi098 bash[76817]: audit 2023-09-23T18:28:02.273092+0000 mon.a (mon.0) 464 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:28:03.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:03 smithi098 bash[76817]: cluster 2023-09-23T18:28:02.286068+0000 osd.4 (osd.4) 27 : cluster 3 failed to encode map e102 with expected crc 2023-09-23T18:28:03.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:03 smithi098 bash[76817]: cluster 2023-09-23T18:28:02.287628+0000 osd.4 (osd.4) 28 : cluster 3 failed to encode map e102 with expected crc 2023-09-23T18:28:03.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:03 smithi098 bash[79516]: audit 2023-09-23T18:28:02.076333+0000 mon.a (mon.0) 460 : audit 0 from='client.? 172.21.15.98:0/2733584499' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:28:03.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:03 smithi098 bash[79516]: cluster 2023-09-23T18:28:02.262725+0000 mon.a (mon.0) 461 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:28:03.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:03 smithi098 bash[79516]: cluster 2023-09-23T18:28:02.272547+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e102 with expected crc 2023-09-23T18:28:03.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:03 smithi098 bash[79516]: cluster 2023-09-23T18:28:02.272555+0000 mon.a (mon.0) 462 : cluster 1 osd.3 [v2:172.21.15.98:6826/2042614518,v1:172.21.15.98:6827/2042614518] boot 2023-09-23T18:28:03.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:03 smithi098 bash[79516]: cluster 2023-09-23T18:28:02.272639+0000 mon.a (mon.0) 463 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2023-09-23T18:28:03.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:03 smithi098 bash[79516]: audit 2023-09-23T18:28:02.273092+0000 mon.a (mon.0) 464 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:28:03.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:03 smithi098 bash[79516]: cluster 2023-09-23T18:28:02.286068+0000 osd.4 (osd.4) 27 : cluster 3 failed to encode map e102 with expected crc 2023-09-23T18:28:03.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:03 smithi098 bash[79516]: cluster 2023-09-23T18:28:02.287628+0000 osd.4 (osd.4) 28 : cluster 3 failed to encode map e102 with expected crc 2023-09-23T18:28:03.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:03 smithi173 bash[59422]: audit 2023-09-23T18:28:02.076333+0000 mon.a (mon.0) 460 : audit 0 from='client.? 172.21.15.98:0/2733584499' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:28:03.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:03 smithi173 bash[59422]: cluster 2023-09-23T18:28:02.262725+0000 mon.a (mon.0) 461 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:28:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:03 smithi173 bash[59422]: cluster 2023-09-23T18:28:02.272547+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e102 with expected crc 2023-09-23T18:28:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:03 smithi173 bash[59422]: cluster 2023-09-23T18:28:02.272555+0000 mon.a (mon.0) 462 : cluster 1 osd.3 [v2:172.21.15.98:6826/2042614518,v1:172.21.15.98:6827/2042614518] boot 2023-09-23T18:28:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:03 smithi173 bash[59422]: cluster 2023-09-23T18:28:02.272639+0000 mon.a (mon.0) 463 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2023-09-23T18:28:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:03 smithi173 bash[59422]: audit 2023-09-23T18:28:02.273092+0000 mon.a (mon.0) 464 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:28:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:03 smithi173 bash[59422]: cluster 2023-09-23T18:28:02.286068+0000 osd.4 (osd.4) 27 : cluster 3 failed to encode map e102 with expected crc 2023-09-23T18:28:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:03 smithi173 bash[59422]: cluster 2023-09-23T18:28:02.287628+0000 osd.4 (osd.4) 28 : cluster 3 failed to encode map e102 with expected crc 2023-09-23T18:28:04.468 INFO:teuthology.orchestra.run.smithi098.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 4 2023-09-23T18:28:04.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:04 smithi098 bash[76817]: cluster 2023-09-23T18:28:03.278015+0000 osd.4 (osd.4) 29 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T18:28:04.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:04 smithi098 bash[76817]: cluster 2023-09-23T18:28:03.279123+0000 mon.a (mon.0) 465 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2023-09-23T18:28:04.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:04 smithi098 bash[76817]: cluster 2023-09-23T18:28:03.281402+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T18:28:04.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:04 smithi098 bash[76817]: cluster 2023-09-23T18:28:03.281666+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T18:28:04.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:04 smithi098 bash[76817]: cluster 2023-09-23T18:28:03.283259+0000 osd.4 (osd.4) 30 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T18:28:04.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:04 smithi098 bash[76817]: cluster 2023-09-23T18:28:03.284175+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T18:28:04.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:04 smithi098 bash[76817]: cluster 2023-09-23T18:28:03.287738+0000 osd.4 (osd.4) 31 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T18:28:04.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:04 smithi098 bash[76817]: cluster 2023-09-23T18:28:03.288403+0000 osd.6 (osd.6) 17 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T18:28:04.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:04 smithi098 bash[76817]: cluster 2023-09-23T18:28:03.587779+0000 mgr.y (mgr.26054) 274 : cluster 0 pgmap v215: 73 pgs: 5 peering, 3 active+undersized+degraded, 23 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T18:28:04.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:04 smithi098 bash[79516]: cluster 2023-09-23T18:28:03.278015+0000 osd.4 (osd.4) 29 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T18:28:04.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:04 smithi098 bash[79516]: cluster 2023-09-23T18:28:03.279123+0000 mon.a (mon.0) 465 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2023-09-23T18:28:04.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:04 smithi098 bash[79516]: cluster 2023-09-23T18:28:03.281402+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T18:28:04.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:04 smithi098 bash[79516]: cluster 2023-09-23T18:28:03.281666+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T18:28:04.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:04 smithi098 bash[79516]: cluster 2023-09-23T18:28:03.283259+0000 osd.4 (osd.4) 30 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T18:28:04.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:04 smithi098 bash[79516]: cluster 2023-09-23T18:28:03.284175+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T18:28:04.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:04 smithi098 bash[79516]: cluster 2023-09-23T18:28:03.287738+0000 osd.4 (osd.4) 31 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T18:28:04.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:04 smithi098 bash[79516]: cluster 2023-09-23T18:28:03.288403+0000 osd.6 (osd.6) 17 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T18:28:04.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:04 smithi098 bash[79516]: cluster 2023-09-23T18:28:03.587779+0000 mgr.y (mgr.26054) 274 : cluster 0 pgmap v215: 73 pgs: 5 peering, 3 active+undersized+degraded, 23 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T18:28:04.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:04 smithi173 bash[59422]: cluster 2023-09-23T18:28:03.278015+0000 osd.4 (osd.4) 29 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T18:28:04.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:04 smithi173 bash[59422]: cluster 2023-09-23T18:28:03.279123+0000 mon.a (mon.0) 465 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2023-09-23T18:28:04.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:04 smithi173 bash[59422]: cluster 2023-09-23T18:28:03.281402+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T18:28:04.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:04 smithi173 bash[59422]: cluster 2023-09-23T18:28:03.281666+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T18:28:04.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:04 smithi173 bash[59422]: cluster 2023-09-23T18:28:03.283259+0000 osd.4 (osd.4) 30 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T18:28:04.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:04 smithi173 bash[59422]: cluster 2023-09-23T18:28:03.284175+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T18:28:04.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:04 smithi173 bash[59422]: cluster 2023-09-23T18:28:03.287738+0000 osd.4 (osd.4) 31 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T18:28:04.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:04 smithi173 bash[59422]: cluster 2023-09-23T18:28:03.288403+0000 osd.6 (osd.6) 17 : cluster 3 failed to encode map e104 with expected crc 2023-09-23T18:28:04.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:04 smithi173 bash[59422]: cluster 2023-09-23T18:28:03.587779+0000 mgr.y (mgr.26054) 274 : cluster 0 pgmap v215: 73 pgs: 5 peering, 3 active+undersized+degraded, 23 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T18:28:05.444 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T18:28:05.639 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:05 smithi098 bash[79516]: audit 2023-09-23T18:28:04.451128+0000 mon.a (mon.0) 466 : audit 0 from='client.? 172.21.15.98:0/496707555' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:28:05.639 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:05 smithi098 bash[76817]: audit 2023-09-23T18:28:04.451128+0000 mon.a (mon.0) 466 : audit 0 from='client.? 172.21.15.98:0/496707555' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:28:05.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:05 smithi173 bash[59422]: audit 2023-09-23T18:28:04.451128+0000 mon.a (mon.0) 466 : audit 0 from='client.? 172.21.15.98:0/496707555' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T18:28:06.352 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:28:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:28:06] "GET /metrics HTTP/1.1" 200 34098 "" "Prometheus/2.33.4" 2023-09-23T18:28:06.698 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:06 smithi098 bash[76817]: cluster 2023-09-23T18:28:05.588636+0000 mgr.y (mgr.26054) 275 : cluster 0 pgmap v216: 73 pgs: 5 peering, 2 active+undersized+degraded, 16 active+undersized, 50 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T18:28:06.698 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:06 smithi098 bash[79516]: cluster 2023-09-23T18:28:05.588636+0000 mgr.y (mgr.26054) 275 : cluster 0 pgmap v216: 73 pgs: 5 peering, 2 active+undersized+degraded, 16 active+undersized, 50 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T18:28:06.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:06 smithi173 bash[59422]: cluster 2023-09-23T18:28:05.588636+0000 mgr.y (mgr.26054) 275 : cluster 0 pgmap v216: 73 pgs: 5 peering, 2 active+undersized+degraded, 16 active+undersized, 50 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T18:28:07.600 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:07 smithi098 bash[79516]: cluster 2023-09-23T18:28:06.342860+0000 mon.a (mon.0) 467 : cluster 3 Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:28:07.600 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:07 smithi098 bash[76817]: cluster 2023-09-23T18:28:06.342860+0000 mon.a (mon.0) 467 : cluster 3 Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:28:07.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:07 smithi173 bash[59422]: cluster 2023-09-23T18:28:06.342860+0000 mon.a (mon.0) 467 : cluster 3 Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:28:08.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:08 smithi098 bash[76817]: cluster 2023-09-23T18:28:07.589403+0000 mgr.y (mgr.26054) 276 : cluster 0 pgmap v217: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:08.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:08 smithi098 bash[79516]: cluster 2023-09-23T18:28:07.589403+0000 mgr.y (mgr.26054) 276 : cluster 0 pgmap v217: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:08.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:08 smithi173 bash[59422]: cluster 2023-09-23T18:28:07.589403+0000 mgr.y (mgr.26054) 276 : cluster 0 pgmap v217: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:09.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:09 smithi098 bash[76817]: cluster 2023-09-23T18:28:08.352476+0000 mon.a (mon.0) 468 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-09-23T18:28:09.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:09 smithi098 bash[76817]: cluster 2023-09-23T18:28:08.352555+0000 mon.a (mon.0) 469 : cluster 1 Cluster is now healthy 2023-09-23T18:28:09.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:09 smithi098 bash[79516]: cluster 2023-09-23T18:28:08.352476+0000 mon.a (mon.0) 468 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-09-23T18:28:09.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:09 smithi098 bash[79516]: cluster 2023-09-23T18:28:08.352555+0000 mon.a (mon.0) 469 : cluster 1 Cluster is now healthy 2023-09-23T18:28:09.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:09 smithi173 bash[59422]: cluster 2023-09-23T18:28:08.352476+0000 mon.a (mon.0) 468 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-09-23T18:28:09.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:09 smithi173 bash[59422]: cluster 2023-09-23T18:28:08.352555+0000 mon.a (mon.0) 469 : cluster 1 Cluster is now healthy 2023-09-23T18:28:10.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:10 smithi098 bash[76817]: audit 2023-09-23T18:28:09.582034+0000 mon.a (mon.0) 470 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:10.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:10 smithi098 bash[76817]: audit 2023-09-23T18:28:09.583371+0000 mon.a (mon.0) 471 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:28:10.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:10 smithi098 bash[76817]: cluster 2023-09-23T18:28:09.589879+0000 mgr.y (mgr.26054) 277 : cluster 0 pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:10.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:10 smithi098 bash[79516]: audit 2023-09-23T18:28:09.582034+0000 mon.a (mon.0) 470 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:10.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:10 smithi098 bash[79516]: audit 2023-09-23T18:28:09.583371+0000 mon.a (mon.0) 471 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:28:10.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:10 smithi098 bash[79516]: cluster 2023-09-23T18:28:09.589879+0000 mgr.y (mgr.26054) 277 : cluster 0 pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:11.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:10 smithi173 bash[59422]: audit 2023-09-23T18:28:09.582034+0000 mon.a (mon.0) 470 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:11.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:10 smithi173 bash[59422]: audit 2023-09-23T18:28:09.583371+0000 mon.a (mon.0) 471 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:28:11.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:10 smithi173 bash[59422]: cluster 2023-09-23T18:28:09.589879+0000 mgr.y (mgr.26054) 277 : cluster 0 pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:11.118 INFO:teuthology.orchestra.run.smithi098.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:28:12.333 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T18:28:12.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:12 smithi098 bash[76817]: audit 2023-09-23T18:28:11.110603+0000 mgr.y (mgr.26054) 278 : audit 0 from='client.34563 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:28:12.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:12 smithi098 bash[76817]: cephadm 2023-09-23T18:28:11.111240+0000 mgr.y (mgr.26054) 279 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:28:12.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:12 smithi098 bash[76817]: audit 2023-09-23T18:28:11.115771+0000 mon.a (mon.0) 472 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:12.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:12 smithi098 bash[76817]: cluster 2023-09-23T18:28:11.590804+0000 mgr.y (mgr.26054) 280 : cluster 0 pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:12.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:12 smithi098 bash[79516]: audit 2023-09-23T18:28:11.110603+0000 mgr.y (mgr.26054) 278 : audit 0 from='client.34563 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:28:12.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:12 smithi098 bash[79516]: cephadm 2023-09-23T18:28:11.111240+0000 mgr.y (mgr.26054) 279 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:28:12.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:12 smithi098 bash[79516]: audit 2023-09-23T18:28:11.115771+0000 mon.a (mon.0) 472 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:12.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:12 smithi098 bash[79516]: cluster 2023-09-23T18:28:11.590804+0000 mgr.y (mgr.26054) 280 : cluster 0 pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:12.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:12 smithi173 bash[59422]: audit 2023-09-23T18:28:11.110603+0000 mgr.y (mgr.26054) 278 : audit 0 from='client.34563 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:28:12.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:12 smithi173 bash[59422]: cephadm 2023-09-23T18:28:11.111240+0000 mgr.y (mgr.26054) 279 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T18:28:12.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:12 smithi173 bash[59422]: audit 2023-09-23T18:28:11.115771+0000 mon.a (mon.0) 472 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:12.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:12 smithi173 bash[59422]: cluster 2023-09-23T18:28:11.590804+0000 mgr.y (mgr.26054) 280 : cluster 0 pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:14.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:14 smithi098 bash[76817]: audit 2023-09-23T18:28:13.271119+0000 mon.a (mon.0) 473 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:14.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:14 smithi098 bash[76817]: audit 2023-09-23T18:28:13.277066+0000 mon.a (mon.0) 474 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:14.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:14 smithi098 bash[76817]: audit 2023-09-23T18:28:13.278650+0000 mon.a (mon.0) 475 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:28:14.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:14 smithi098 bash[76817]: audit 2023-09-23T18:28:13.280069+0000 mon.a (mon.0) 476 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:28:14.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:14 smithi098 bash[76817]: audit 2023-09-23T18:28:13.286711+0000 mon.a (mon.0) 477 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:14.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:14 smithi098 bash[76817]: audit 2023-09-23T18:28:13.364796+0000 mon.a (mon.0) 478 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:28:14.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:14 smithi098 bash[76817]: audit 2023-09-23T18:28:13.370804+0000 mon.a (mon.0) 479 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:28:14.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:14 smithi098 bash[76817]: audit 2023-09-23T18:28:13.372016+0000 mon.a (mon.0) 480 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:28:14.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:14 smithi098 bash[76817]: audit 2023-09-23T18:28:13.379176+0000 mon.a (mon.0) 481 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:14.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:14 smithi098 bash[76817]: cluster 2023-09-23T18:28:13.591405+0000 mgr.y (mgr.26054) 281 : cluster 0 pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:14.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:14 smithi098 bash[79516]: audit 2023-09-23T18:28:13.271119+0000 mon.a (mon.0) 473 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:14.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:14 smithi098 bash[79516]: audit 2023-09-23T18:28:13.277066+0000 mon.a (mon.0) 474 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:14.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:14 smithi098 bash[79516]: audit 2023-09-23T18:28:13.278650+0000 mon.a (mon.0) 475 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:28:14.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:14 smithi098 bash[79516]: audit 2023-09-23T18:28:13.280069+0000 mon.a (mon.0) 476 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:28:14.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:14 smithi098 bash[79516]: audit 2023-09-23T18:28:13.286711+0000 mon.a (mon.0) 477 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:14.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:14 smithi098 bash[79516]: audit 2023-09-23T18:28:13.364796+0000 mon.a (mon.0) 478 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:28:14.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:14 smithi098 bash[79516]: audit 2023-09-23T18:28:13.370804+0000 mon.a (mon.0) 479 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:28:14.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:14 smithi098 bash[79516]: audit 2023-09-23T18:28:13.372016+0000 mon.a (mon.0) 480 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:28:14.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:14 smithi098 bash[79516]: audit 2023-09-23T18:28:13.379176+0000 mon.a (mon.0) 481 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:14.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:14 smithi098 bash[79516]: cluster 2023-09-23T18:28:13.591405+0000 mgr.y (mgr.26054) 281 : cluster 0 pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:14.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:14 smithi173 bash[59422]: audit 2023-09-23T18:28:13.271119+0000 mon.a (mon.0) 473 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:14.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:14 smithi173 bash[59422]: audit 2023-09-23T18:28:13.277066+0000 mon.a (mon.0) 474 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:14.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:14 smithi173 bash[59422]: audit 2023-09-23T18:28:13.278650+0000 mon.a (mon.0) 475 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:28:14.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:14 smithi173 bash[59422]: audit 2023-09-23T18:28:13.280069+0000 mon.a (mon.0) 476 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:28:14.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:14 smithi173 bash[59422]: audit 2023-09-23T18:28:13.286711+0000 mon.a (mon.0) 477 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:14.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:14 smithi173 bash[59422]: audit 2023-09-23T18:28:13.364796+0000 mon.a (mon.0) 478 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:28:14.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:14 smithi173 bash[59422]: audit 2023-09-23T18:28:13.370804+0000 mon.a (mon.0) 479 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:28:14.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:14 smithi173 bash[59422]: audit 2023-09-23T18:28:13.372016+0000 mon.a (mon.0) 480 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:28:14.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:14 smithi173 bash[59422]: audit 2023-09-23T18:28:13.379176+0000 mon.a (mon.0) 481 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:14.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:14 smithi173 bash[59422]: cluster 2023-09-23T18:28:13.591405+0000 mgr.y (mgr.26054) 281 : cluster 0 pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:15.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:15 smithi098 bash[76817]: audit 2023-09-23T18:28:14.710206+0000 mon.a (mon.0) 482 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:15.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:15 smithi098 bash[79516]: audit 2023-09-23T18:28:14.710206+0000 mon.a (mon.0) 482 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:15.978 INFO:teuthology.orchestra.run.smithi098.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T18:28:15.978 INFO:teuthology.orchestra.run.smithi098.stdout:alertmanager.a smithi098 *:9093,9094 running (8m) - 14m 13.5M - 0.23.0 ba2b418f427c 7c2bcaebb703 2023-09-23T18:28:15.978 INFO:teuthology.orchestra.run.smithi098.stdout:grafana.a smithi173 *:3000 running (8m) - 14m 41.5M - 8.3.5 dad864ee21e9 616bbe1127b7 2023-09-23T18:28:15.978 INFO:teuthology.orchestra.run.smithi098.stdout:mds.foo.smithi098.lowjqo smithi098 running (13m) - 13m 17.3M - 16.2.14-37-g2e1f1015 1e6a756174a8 e1555281d4fd 2023-09-23T18:28:15.978 INFO:teuthology.orchestra.run.smithi098.stdout:mds.foo.smithi173.qjefje smithi173 running (12m) - 12m 14.2M - 16.2.14-37-g2e1f1015 1e6a756174a8 fc5513981852 2023-09-23T18:28:15.979 INFO:teuthology.orchestra.run.smithi098.stdout:mgr.x smithi173 *:8443,9283,8765 running (7m) - 21m 429M - 18.2.0-460-gbb0daeff e32c2a64c126 070f62900469 2023-09-23T18:28:15.979 INFO:teuthology.orchestra.run.smithi098.stdout:mgr.y smithi098 *:8443,9283,8765 running (7m) - 23m 483M - 18.2.0-460-gbb0daeff e32c2a64c126 c11dea5b47dd 2023-09-23T18:28:15.979 INFO:teuthology.orchestra.run.smithi098.stdout:mon.a smithi098 running (6m) - 24m 73.9M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 d325a8e8ebe6 2023-09-23T18:28:15.979 INFO:teuthology.orchestra.run.smithi098.stdout:mon.b smithi173 running (3m) - 22m 35.9M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 8067956be572 2023-09-23T18:28:15.979 INFO:teuthology.orchestra.run.smithi098.stdout:mon.c smithi098 running (5m) - 22m 47.6M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 3bbca4740a2d 2023-09-23T18:28:15.979 INFO:teuthology.orchestra.run.smithi098.stdout:node-exporter.a smithi098 *:9100 running (8m) - 15m 9.80M - 1.3.1 1dbe0e931976 bdffbddc2c81 2023-09-23T18:28:15.979 INFO:teuthology.orchestra.run.smithi098.stdout:node-exporter.b smithi173 *:9100 running (8m) - 15m 9356k - 1.3.1 1dbe0e931976 610050ff4ef2 2023-09-23T18:28:15.979 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0 smithi098 running (108s) - 21m 43.7M 2524M 18.2.0-460-gbb0daeff e32c2a64c126 9a2ceadecf07 2023-09-23T18:28:15.979 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1 smithi098 running (79s) - 20m 42.9M 2524M 18.2.0-460-gbb0daeff e32c2a64c126 75b3f1d0770f 2023-09-23T18:28:15.979 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2 smithi098 running (50s) - 19m 40.3M 2524M 18.2.0-460-gbb0daeff e32c2a64c126 aa52f48e56f8 2023-09-23T18:28:15.979 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3 smithi098 running (20s) - 19m - 2524M 18.2.0-460-gbb0daeff e32c2a64c126 4092763f2af2 2023-09-23T18:28:15.979 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4 smithi173 running (18m) - 18m 47.3M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 0a479b881bee 2023-09-23T18:28:15.979 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5 smithi173 running (17m) - 17m 45.7M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 501903da3aa9 2023-09-23T18:28:15.980 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6 smithi173 running (16m) - 16m 47.3M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 b37149ed4271 2023-09-23T18:28:15.980 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7 smithi173 running (16m) - 16m 50.4M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 81723c51be2b 2023-09-23T18:28:15.980 INFO:teuthology.orchestra.run.smithi098.stdout:prometheus.a smithi173 *:9095 running (7m) - 15m 57.3M - 2.33.4 514e6a882f6e 7547c9472f93 2023-09-23T18:28:16.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:15 smithi173 bash[59422]: audit 2023-09-23T18:28:14.710206+0000 mon.a (mon.0) 482 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:28:16.442 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:28:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:28:16] "GET /metrics HTTP/1.1" 200 34095 "" "Prometheus/2.33.4" 2023-09-23T18:28:16.901 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T18:28:17.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:16 smithi173 bash[59422]: cluster 2023-09-23T18:28:15.592287+0000 mgr.y (mgr.26054) 282 : cluster 0 pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:17.097 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:16 smithi098 bash[76817]: cluster 2023-09-23T18:28:15.592287+0000 mgr.y (mgr.26054) 282 : cluster 0 pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:17.097 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:16 smithi098 bash[79516]: cluster 2023-09-23T18:28:15.592287+0000 mgr.y (mgr.26054) 282 : cluster 0 pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:18.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:17 smithi173 bash[59422]: audit 2023-09-23T18:28:15.957149+0000 mgr.y (mgr.26054) 283 : audit 0 from='client.34569 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:28:18.129 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:17 smithi098 bash[76817]: audit 2023-09-23T18:28:15.957149+0000 mgr.y (mgr.26054) 283 : audit 0 from='client.34569 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:28:18.129 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:17 smithi098 bash[79516]: audit 2023-09-23T18:28:15.957149+0000 mgr.y (mgr.26054) 283 : audit 0 from='client.34569 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:28:18.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:18 smithi098 bash[76817]: cluster 2023-09-23T18:28:17.593147+0000 mgr.y (mgr.26054) 284 : cluster 0 pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:18.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:18 smithi098 bash[79516]: cluster 2023-09-23T18:28:17.593147+0000 mgr.y (mgr.26054) 284 : cluster 0 pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:19.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:18 smithi173 bash[59422]: cluster 2023-09-23T18:28:17.593147+0000 mgr.y (mgr.26054) 284 : cluster 0 pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:20.503 INFO:teuthology.orchestra.run.smithi098.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T18:28:20.503 INFO:teuthology.orchestra.run.smithi098.stdout:alertmanager.a smithi098 *:9093,9094 running (8m) - 15m 13.5M - 0.23.0 ba2b418f427c 7c2bcaebb703 2023-09-23T18:28:20.504 INFO:teuthology.orchestra.run.smithi098.stdout:grafana.a smithi173 *:3000 running (8m) - 14m 41.5M - 8.3.5 dad864ee21e9 616bbe1127b7 2023-09-23T18:28:20.504 INFO:teuthology.orchestra.run.smithi098.stdout:mds.foo.smithi098.lowjqo smithi098 running (13m) - 13m 17.3M - 16.2.14-37-g2e1f1015 1e6a756174a8 e1555281d4fd 2023-09-23T18:28:20.504 INFO:teuthology.orchestra.run.smithi098.stdout:mds.foo.smithi173.qjefje smithi173 running (13m) - 13m 14.2M - 16.2.14-37-g2e1f1015 1e6a756174a8 fc5513981852 2023-09-23T18:28:20.504 INFO:teuthology.orchestra.run.smithi098.stdout:mgr.x smithi173 *:8443,9283,8765 running (7m) - 21m 429M - 18.2.0-460-gbb0daeff e32c2a64c126 070f62900469 2023-09-23T18:28:20.504 INFO:teuthology.orchestra.run.smithi098.stdout:mgr.y smithi098 *:8443,9283,8765 running (8m) - 24m 483M - 18.2.0-460-gbb0daeff e32c2a64c126 c11dea5b47dd 2023-09-23T18:28:20.504 INFO:teuthology.orchestra.run.smithi098.stdout:mon.a smithi098 running (6m) - 24m 73.9M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 d325a8e8ebe6 2023-09-23T18:28:20.504 INFO:teuthology.orchestra.run.smithi098.stdout:mon.b smithi173 running (3m) - 22m 35.9M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 8067956be572 2023-09-23T18:28:20.504 INFO:teuthology.orchestra.run.smithi098.stdout:mon.c smithi098 running (5m) - 22m 47.6M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 3bbca4740a2d 2023-09-23T18:28:20.504 INFO:teuthology.orchestra.run.smithi098.stdout:node-exporter.a smithi098 *:9100 running (8m) - 15m 9.80M - 1.3.1 1dbe0e931976 bdffbddc2c81 2023-09-23T18:28:20.504 INFO:teuthology.orchestra.run.smithi098.stdout:node-exporter.b smithi173 *:9100 running (8m) - 15m 9356k - 1.3.1 1dbe0e931976 610050ff4ef2 2023-09-23T18:28:20.504 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0 smithi098 running (113s) - 21m 43.7M 2524M 18.2.0-460-gbb0daeff e32c2a64c126 9a2ceadecf07 2023-09-23T18:28:20.504 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1 smithi098 running (83s) - 20m 42.9M 2524M 18.2.0-460-gbb0daeff e32c2a64c126 75b3f1d0770f 2023-09-23T18:28:20.505 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2 smithi098 running (54s) - 19m 40.3M 2524M 18.2.0-460-gbb0daeff e32c2a64c126 aa52f48e56f8 2023-09-23T18:28:20.505 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3 smithi098 running (24s) - 19m - 2524M 18.2.0-460-gbb0daeff e32c2a64c126 4092763f2af2 2023-09-23T18:28:20.505 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4 smithi173 running (18m) - 18m 47.3M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 0a479b881bee 2023-09-23T18:28:20.505 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5 smithi173 running (17m) - 17m 45.7M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 501903da3aa9 2023-09-23T18:28:20.505 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6 smithi173 running (17m) - 17m 47.3M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 b37149ed4271 2023-09-23T18:28:20.505 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7 smithi173 running (16m) - 16m 50.4M 2524M 16.2.14-37-g2e1f1015 1e6a756174a8 81723c51be2b 2023-09-23T18:28:20.505 INFO:teuthology.orchestra.run.smithi098.stdout:prometheus.a smithi173 *:9095 running (7m) - 15m 57.3M - 2.33.4 514e6a882f6e 7547c9472f93 2023-09-23T18:28:20.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:20 smithi098 bash[76817]: cluster 2023-09-23T18:28:19.593886+0000 mgr.y (mgr.26054) 285 : cluster 0 pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:20.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:20 smithi098 bash[79516]: cluster 2023-09-23T18:28:19.593886+0000 mgr.y (mgr.26054) 285 : cluster 0 pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:20.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:20 smithi173 bash[59422]: cluster 2023-09-23T18:28:19.593886+0000 mgr.y (mgr.26054) 285 : cluster 0 pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:21.289 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph versions' 2023-09-23T18:28:21.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:21 smithi098 bash[76817]: audit 2023-09-23T18:28:20.487929+0000 mgr.y (mgr.26054) 286 : audit 0 from='client.44480 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:28:21.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:21 smithi098 bash[79516]: audit 2023-09-23T18:28:20.487929+0000 mgr.y (mgr.26054) 286 : audit 0 from='client.44480 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:28:21.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:21 smithi173 bash[59422]: audit 2023-09-23T18:28:20.487929+0000 mgr.y (mgr.26054) 286 : audit 0 from='client.44480 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:28:22.620 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:22 smithi098 bash[76817]: cluster 2023-09-23T18:28:21.594655+0000 mgr.y (mgr.26054) 287 : cluster 0 pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:22.620 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:22 smithi098 bash[79516]: cluster 2023-09-23T18:28:21.594655+0000 mgr.y (mgr.26054) 287 : cluster 0 pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:22.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:22 smithi173 bash[59422]: cluster 2023-09-23T18:28:21.594655+0000 mgr.y (mgr.26054) 287 : cluster 0 pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:24.905 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:28:24.906 INFO:teuthology.orchestra.run.smithi098.stdout: "mon": { 2023-09-23T18:28:24.906 INFO:teuthology.orchestra.run.smithi098.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 3 2023-09-23T18:28:24.906 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:28:24.907 INFO:teuthology.orchestra.run.smithi098.stdout: "mgr": { 2023-09-23T18:28:24.907 INFO:teuthology.orchestra.run.smithi098.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 2 2023-09-23T18:28:24.907 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:28:24.907 INFO:teuthology.orchestra.run.smithi098.stdout: "osd": { 2023-09-23T18:28:24.907 INFO:teuthology.orchestra.run.smithi098.stdout: "ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)": 4, 2023-09-23T18:28:24.907 INFO:teuthology.orchestra.run.smithi098.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 4 2023-09-23T18:28:24.907 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:28:24.907 INFO:teuthology.orchestra.run.smithi098.stdout: "mds": { 2023-09-23T18:28:24.907 INFO:teuthology.orchestra.run.smithi098.stdout: "ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)": 2 2023-09-23T18:28:24.907 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:28:24.907 INFO:teuthology.orchestra.run.smithi098.stdout: "overall": { 2023-09-23T18:28:24.907 INFO:teuthology.orchestra.run.smithi098.stdout: "ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)": 6, 2023-09-23T18:28:24.907 INFO:teuthology.orchestra.run.smithi098.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 9 2023-09-23T18:28:24.908 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:28:24.908 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:28:24.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:24 smithi098 bash[76817]: cluster 2023-09-23T18:28:23.595489+0000 mgr.y (mgr.26054) 288 : cluster 0 pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:24.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:24 smithi098 bash[76817]: audit 2023-09-23T18:28:24.574519+0000 mon.a (mon.0) 483 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:28:24.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:24 smithi098 bash[79516]: cluster 2023-09-23T18:28:23.595489+0000 mgr.y (mgr.26054) 288 : cluster 0 pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:24.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:24 smithi098 bash[79516]: audit 2023-09-23T18:28:24.574519+0000 mon.a (mon.0) 483 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:28:25.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:24 smithi173 bash[59422]: cluster 2023-09-23T18:28:23.595489+0000 mgr.y (mgr.26054) 288 : cluster 0 pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:25.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:24 smithi173 bash[59422]: audit 2023-09-23T18:28:24.574519+0000 mon.a (mon.0) 483 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:28:25.746 DEBUG:teuthology.parallel:result is None 2023-09-23T18:28:25.747 INFO:teuthology.run_tasks:Running task print... 2023-09-23T18:28:25.758 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-09-23T18:28:25.758 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-09-23T18:28:25.804 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T18:28:25.805 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T18:28:25.805 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T18:28:25.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:25 smithi098 bash[76817]: audit 2023-09-23T18:28:24.903019+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.98:0/3069052177' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:28:25.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:25 smithi098 bash[79516]: audit 2023-09-23T18:28:24.903019+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.98:0/3069052177' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:28:26.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:25 smithi173 bash[59422]: audit 2023-09-23T18:28:24.903019+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.98:0/3069052177' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T18:28:26.442 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:28:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:28:26] "GET /metrics HTTP/1.1" 200 34089 "" "Prometheus/2.33.4" 2023-09-23T18:28:26.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:26 smithi098 bash[76817]: cluster 2023-09-23T18:28:25.596385+0000 mgr.y (mgr.26054) 289 : cluster 0 pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:26.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:26 smithi098 bash[79516]: cluster 2023-09-23T18:28:25.596385+0000 mgr.y (mgr.26054) 289 : cluster 0 pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:27.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:26 smithi173 bash[59422]: cluster 2023-09-23T18:28:25.596385+0000 mgr.y (mgr.26054) 289 : cluster 0 pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:28.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:28 smithi098 bash[76817]: cluster 2023-09-23T18:28:27.597214+0000 mgr.y (mgr.26054) 290 : cluster 0 pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:28.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:28 smithi098 bash[79516]: cluster 2023-09-23T18:28:27.597214+0000 mgr.y (mgr.26054) 290 : cluster 0 pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:29.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:28 smithi173 bash[59422]: cluster 2023-09-23T18:28:27.597214+0000 mgr.y (mgr.26054) 290 : cluster 0 pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:30.578 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:28:30.608 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:30 smithi098 bash[76817]: cluster 2023-09-23T18:28:29.597989+0000 mgr.y (mgr.26054) 291 : cluster 0 pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:30.609 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:30 smithi098 bash[79516]: cluster 2023-09-23T18:28:29.597989+0000 mgr.y (mgr.26054) 291 : cluster 0 pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:30.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:30 smithi173 bash[59422]: cluster 2023-09-23T18:28:29.597989+0000 mgr.y (mgr.26054) 291 : cluster 0 pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:32.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:32 smithi098 bash[76817]: cluster 2023-09-23T18:28:31.598917+0000 mgr.y (mgr.26054) 292 : cluster 0 pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:32.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:32 smithi098 bash[79516]: cluster 2023-09-23T18:28:31.598917+0000 mgr.y (mgr.26054) 292 : cluster 0 pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:32 smithi173 bash[59422]: cluster 2023-09-23T18:28:31.598917+0000 mgr.y (mgr.26054) 292 : cluster 0 pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:33.964 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:28:34.767 INFO:teuthology.orchestra.run.smithi098.stdout:{"epoch":35,"active_gid":26054,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6800","nonce":1095325224},{"type":"v1","addr":"172.21.15.98:6801","nonce":1095325224}]},"active_addr":"172.21.15.98:6801/1095325224","active_change":"2023-09-23T18:21:39.415698+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25684,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.98:8443/","prometheus":"http://172.21.15.98:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":84,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.98:0","nonce":3562361602}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.98:0","nonce":2479904968}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.98:0","nonce":1130329224}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.98:0","nonce":1101711125}]}]} 2023-09-23T18:28:34.772 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T18:28:34.772 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T18:28:34.772 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T18:28:34.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:34 smithi098 bash[79516]: cluster 2023-09-23T18:28:33.599580+0000 mgr.y (mgr.26054) 293 : cluster 0 pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:34.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:34 smithi098 bash[79516]: audit 2023-09-23T18:28:33.950650+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.98:0/2409921957' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T18:28:34.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:34 smithi098 bash[76817]: cluster 2023-09-23T18:28:33.599580+0000 mgr.y (mgr.26054) 293 : cluster 0 pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:34.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:34 smithi098 bash[76817]: audit 2023-09-23T18:28:33.950650+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.98:0/2409921957' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T18:28:35.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:34 smithi173 bash[59422]: cluster 2023-09-23T18:28:33.599580+0000 mgr.y (mgr.26054) 293 : cluster 0 pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:35.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:34 smithi173 bash[59422]: audit 2023-09-23T18:28:33.950650+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.98:0/2409921957' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T18:28:36.441 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:28:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:28:36] "GET /metrics HTTP/1.1" 200 34089 "" "Prometheus/2.33.4" 2023-09-23T18:28:36.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:36 smithi098 bash[79516]: cluster 2023-09-23T18:28:35.600451+0000 mgr.y (mgr.26054) 294 : cluster 0 pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:36.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:36 smithi098 bash[76817]: cluster 2023-09-23T18:28:35.600451+0000 mgr.y (mgr.26054) 294 : cluster 0 pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:37.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:36 smithi173 bash[59422]: cluster 2023-09-23T18:28:35.600451+0000 mgr.y (mgr.26054) 294 : cluster 0 pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:38.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:38 smithi098 bash[76817]: cluster 2023-09-23T18:28:37.601291+0000 mgr.y (mgr.26054) 295 : cluster 0 pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:38.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:38 smithi098 bash[79516]: cluster 2023-09-23T18:28:37.601291+0000 mgr.y (mgr.26054) 295 : cluster 0 pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:39.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:38 smithi173 bash[59422]: cluster 2023-09-23T18:28:37.601291+0000 mgr.y (mgr.26054) 295 : cluster 0 pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:39.549 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:28:40.632 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:40 smithi098 bash[79516]: audit 2023-09-23T18:28:39.574447+0000 mon.a (mon.0) 484 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:28:40.633 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:40 smithi098 bash[79516]: cluster 2023-09-23T18:28:39.601689+0000 mgr.y (mgr.26054) 296 : cluster 0 pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:40.633 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:40 smithi098 bash[76817]: audit 2023-09-23T18:28:39.574447+0000 mon.a (mon.0) 484 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:28:40.633 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:40 smithi098 bash[76817]: cluster 2023-09-23T18:28:39.601689+0000 mgr.y (mgr.26054) 296 : cluster 0 pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:40.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:40 smithi173 bash[59422]: audit 2023-09-23T18:28:39.574447+0000 mon.a (mon.0) 484 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:28:40.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:40 smithi173 bash[59422]: cluster 2023-09-23T18:28:39.601689+0000 mgr.y (mgr.26054) 296 : cluster 0 pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:42.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:42 smithi098 bash[76817]: cluster 2023-09-23T18:28:41.602602+0000 mgr.y (mgr.26054) 297 : cluster 0 pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:42.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:42 smithi098 bash[79516]: cluster 2023-09-23T18:28:41.602602+0000 mgr.y (mgr.26054) 297 : cluster 0 pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:42.962 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:28:42.962 INFO:teuthology.orchestra.run.smithi098.stdout:{"epoch":104,"fsid":"756c96d4-5a3b-11ee-8db4-212e2dc638e7","created":"2023-09-23T18:04:11.580781+0000","modified":"2023-09-23T18:28:03.269518+0000","last_up_change":"2023-09-23T18:28:02.261645+0000","last_in_change":"2023-09-23T18:11:43.419979+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T18:08:35.146924+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"82","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T18:13:12.421802+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"77","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":77,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T18:15:07.820408+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T18:15:08.333052+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4f35f176-4d07-4157-aeb9-dd8f71da3a63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":84,"up_from":88,"up_thru":103,"down_at":85,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6802","nonce":1320995478},{"type":"v1","addr":"172.21.15.98:6803","nonce":1320995478}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6804","nonce":1320995478},{"type":"v1","addr":"172.21.15.98:6805","nonce":1320995478}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6808","nonce":1320995478},{"type":"v1","addr":"172.21.15.98:6809","nonce":1320995478}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6806","nonce":1320995478},{"type":"v1","addr":"172.21.15.98:6807","nonce":1320995478}]},"public_addr":"172.21.15.98:6803/1320995478","cluster_addr":"172.21.15.98:6805/1320995478","heartbeat_back_addr":"172.21.15.98:6809/1320995478","heartbeat_front_addr":"172.21.15.98:6807/1320995478","state":["exists","up"]},{"osd":1,"uuid":"5d777d1f-3bd3-49ba-b0de-0ac532ee3934","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":89,"up_from":93,"up_thru":103,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6810","nonce":1561159155},{"type":"v1","addr":"172.21.15.98:6811","nonce":1561159155}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6812","nonce":1561159155},{"type":"v1","addr":"172.21.15.98:6813","nonce":1561159155}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6816","nonce":1561159155},{"type":"v1","addr":"172.21.15.98:6817","nonce":1561159155}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6814","nonce":1561159155},{"type":"v1","addr":"172.21.15.98:6815","nonce":1561159155}]},"public_addr":"172.21.15.98:6811/1561159155","cluster_addr":"172.21.15.98:6813/1561159155","heartbeat_back_addr":"172.21.15.98:6817/1561159155","heartbeat_front_addr":"172.21.15.98:6815/1561159155","state":["exists","up"]},{"osd":2,"uuid":"63978288-ac36-4cf2-bbb7-46dc828e6ce5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":94,"up_from":98,"up_thru":103,"down_at":95,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6818","nonce":4249816143},{"type":"v1","addr":"172.21.15.98:6819","nonce":4249816143}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6820","nonce":4249816143},{"type":"v1","addr":"172.21.15.98:6821","nonce":4249816143}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6824","nonce":4249816143},{"type":"v1","addr":"172.21.15.98:6825","nonce":4249816143}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6822","nonce":4249816143},{"type":"v1","addr":"172.21.15.98:6823","nonce":4249816143}]},"public_addr":"172.21.15.98:6819/4249816143","cluster_addr":"172.21.15.98:6821/4249816143","heartbeat_back_addr":"172.21.15.98:6825/4249816143","heartbeat_front_addr":"172.21.15.98:6823/4249816143","state":["exists","up"]},{"osd":3,"uuid":"48596d91-46da-4815-b810-d493bbf01dcc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":99,"up_from":103,"up_thru":103,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6826","nonce":2042614518},{"type":"v1","addr":"172.21.15.98:6827","nonce":2042614518}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6828","nonce":2042614518},{"type":"v1","addr":"172.21.15.98:6829","nonce":2042614518}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6832","nonce":2042614518},{"type":"v1","addr":"172.21.15.98:6833","nonce":2042614518}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6830","nonce":2042614518},{"type":"v1","addr":"172.21.15.98:6831","nonce":2042614518}]},"public_addr":"172.21.15.98:6827/2042614518","cluster_addr":"172.21.15.98:6829/2042614518","heartbeat_back_addr":"172.21.15.98:6833/2042614518","heartbeat_front_addr":"172.21.15.98:6831/2042614518","state":["exists","up"]},{"osd":4,"uuid":"a562afc6-ffe6-44bb-91a9-8a04018c3c6f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6800","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6801","nonce":165906554}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6802","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6803","nonce":165906554}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6806","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6807","nonce":165906554}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6804","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6805","nonce":165906554}]},"public_addr":"172.21.15.173:6801/165906554","cluster_addr":"172.21.15.173:6803/165906554","heartbeat_back_addr":"172.21.15.173:6807/165906554","heartbeat_front_addr":"172.21.15.173:6805/165906554","state":["exists","up"]},{"osd":5,"uuid":"add28146-8301-4b4c-9616-967e9df3a8d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6808","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6809","nonce":2894909344}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6810","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6811","nonce":2894909344}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6814","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6815","nonce":2894909344}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6812","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6813","nonce":2894909344}]},"public_addr":"172.21.15.173:6809/2894909344","cluster_addr":"172.21.15.173:6811/2894909344","heartbeat_back_addr":"172.21.15.173:6815/2894909344","heartbeat_front_addr":"172.21.15.173:6813/2894909344","state":["exists","up"]},{"osd":6,"uuid":"f30854de-bc26-465d-8b0e-a9211ae2d9fa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":100,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6816","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6817","nonce":3550013548}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6818","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6819","nonce":3550013548}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6822","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6823","nonce":3550013548}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6820","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6821","nonce":3550013548}]},"public_addr":"172.21.15.173:6817/3550013548","cluster_addr":"172.21.15.173:6819/3550013548","heartbeat_back_addr":"172.21.15.173:6823/3550013548","heartbeat_front_addr":"172.21.15.173:6821/3550013548","state":["exists","up"]},{"osd":7,"uuid":"5f540de3-48f9-4bc8-b14a-9aadf838f16b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6824","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6825","nonce":135608222}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6826","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6827","nonce":135608222}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6830","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6831","nonce":135608222}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6828","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6829","nonce":135608222}]},"public_addr":"172.21.15.173:6825/135608222","cluster_addr":"172.21.15.173:6827/135608222","heartbeat_back_addr":"172.21.15.173:6831/135608222","heartbeat_front_addr":"172.21.15.173:6829/135608222","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:07:04.739243+0000","dead_epoch":84},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:07:48.583198+0000","dead_epoch":89},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:08:33.173348+0000","dead_epoch":94},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:09:18.218502+0000","dead_epoch":99},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:09:59.765019+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:10:42.490689+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:11:24.357920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:12:08.168900+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.98:0/325576677":"2023-09-24T18:21:39.415268+0000","172.21.15.98:0/3213365423":"2023-09-24T18:04:40.596823+0000","172.21.15.98:0/2288964511":"2023-09-24T18:21:39.415268+0000","172.21.15.98:0/2098141849":"2023-09-24T18:18:53.712554+0000","172.21.15.98:0/910147733":"2023-09-24T18:21:39.415268+0000","172.21.15.98:6800/3510470804":"2023-09-24T18:21:39.415268+0000","172.21.15.173:0/2105184133":"2023-09-24T18:20:34.294726+0000","172.21.15.98:6800/3706591260":"2023-09-24T18:18:53.712554+0000","172.21.15.98:0/1142988558":"2023-09-24T18:12:42.525837+0000","172.21.15.98:6801/3510470804":"2023-09-24T18:21:39.415268+0000","172.21.15.98:0/2194193535":"2023-09-24T18:12:42.525837+0000","172.21.15.98:0/4085572202":"2023-09-24T18:18:53.712554+0000","172.21.15.98:6800/3368447653":"2023-09-24T18:04:40.596823+0000","172.21.15.173:6834/1425795384":"2023-09-24T18:20:34.294726+0000","172.21.15.98:6800/483469857":"2023-09-24T18:12:42.525837+0000","172.21.15.98:0/3410138197":"2023-09-24T18:21:39.415268+0000","172.21.15.98:0/3976553993":"2023-09-24T18:04:40.596823+0000","172.21.15.98:0/3472030975":"2023-09-24T18:05:11.060631+0000","172.21.15.173:0/4281585910":"2023-09-24T18:20:34.294726+0000","172.21.15.98:6801/3706591260":"2023-09-24T18:18:53.712554+0000","172.21.15.98:0/1603044865":"2023-09-24T18:05:11.060631+0000","172.21.15.173:6835/1425795384":"2023-09-24T18:20:34.294726+0000","172.21.15.173:0/2782586450":"2023-09-24T18:20:34.294726+0000","172.21.15.98:0/3501988932":"2023-09-24T18:21:39.415268+0000","172.21.15.98:6801/483469857":"2023-09-24T18:12:42.525837+0000","172.21.15.98:6801/3368447653":"2023-09-24T18:04:40.596823+0000","172.21.15.98:0/335394369":"2023-09-24T18:21:39.415268+0000","172.21.15.173:0/2534791896":"2023-09-24T18:20:34.294726+0000","172.21.15.98:6800/1392949322":"2023-09-24T18:05:11.060631+0000","172.21.15.98:6801/1392949322":"2023-09-24T18:05:11.060631+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T18:28:43.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:42 smithi173 bash[59422]: cluster 2023-09-23T18:28:41.602602+0000 mgr.y (mgr.26054) 297 : cluster 0 pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:43.803 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T18:28:43.803 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T18:28:43.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:43 smithi098 bash[79516]: audit 2023-09-23T18:28:42.956671+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.98:0/2946291514' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:28:43.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:43 smithi098 bash[76817]: audit 2023-09-23T18:28:42.956671+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.98:0/2946291514' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:28:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:43 smithi173 bash[59422]: audit 2023-09-23T18:28:42.956671+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.98:0/2946291514' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:28:44.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:44 smithi098 bash[76817]: cluster 2023-09-23T18:28:43.603333+0000 mgr.y (mgr.26054) 298 : cluster 0 pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:44.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:44 smithi098 bash[79516]: cluster 2023-09-23T18:28:43.603333+0000 mgr.y (mgr.26054) 298 : cluster 0 pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:45.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:44 smithi173 bash[59422]: cluster 2023-09-23T18:28:43.603333+0000 mgr.y (mgr.26054) 298 : cluster 0 pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:46.442 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:28:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:28:46] "GET /metrics HTTP/1.1" 200 34097 "" "Prometheus/2.33.4" 2023-09-23T18:28:46.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:46 smithi098 bash[76817]: cluster 2023-09-23T18:28:45.604201+0000 mgr.y (mgr.26054) 299 : cluster 0 pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:46.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:46 smithi098 bash[79516]: cluster 2023-09-23T18:28:45.604201+0000 mgr.y (mgr.26054) 299 : cluster 0 pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:47.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:46 smithi173 bash[59422]: cluster 2023-09-23T18:28:45.604201+0000 mgr.y (mgr.26054) 299 : cluster 0 pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:48.583 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:28:48.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:48 smithi098 bash[79516]: cluster 2023-09-23T18:28:47.605014+0000 mgr.y (mgr.26054) 300 : cluster 0 pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:48.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:48 smithi098 bash[76817]: cluster 2023-09-23T18:28:47.605014+0000 mgr.y (mgr.26054) 300 : cluster 0 pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:49.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:48 smithi173 bash[59422]: cluster 2023-09-23T18:28:47.605014+0000 mgr.y (mgr.26054) 300 : cluster 0 pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:50.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:50 smithi098 bash[76817]: cluster 2023-09-23T18:28:49.605753+0000 mgr.y (mgr.26054) 301 : cluster 0 pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:50.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:50 smithi098 bash[79516]: cluster 2023-09-23T18:28:49.605753+0000 mgr.y (mgr.26054) 301 : cluster 0 pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:50.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:50 smithi173 bash[59422]: cluster 2023-09-23T18:28:49.605753+0000 mgr.y (mgr.26054) 301 : cluster 0 pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:52.148 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:28:52.148 INFO:teuthology.orchestra.run.smithi098.stdout:{"epoch":104,"fsid":"756c96d4-5a3b-11ee-8db4-212e2dc638e7","created":"2023-09-23T18:04:11.580781+0000","modified":"2023-09-23T18:28:03.269518+0000","last_up_change":"2023-09-23T18:28:02.261645+0000","last_in_change":"2023-09-23T18:11:43.419979+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T18:08:35.146924+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"82","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T18:13:12.421802+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"77","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":77,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T18:15:07.820408+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T18:15:08.333052+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4f35f176-4d07-4157-aeb9-dd8f71da3a63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":84,"up_from":88,"up_thru":103,"down_at":85,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6802","nonce":1320995478},{"type":"v1","addr":"172.21.15.98:6803","nonce":1320995478}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6804","nonce":1320995478},{"type":"v1","addr":"172.21.15.98:6805","nonce":1320995478}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6808","nonce":1320995478},{"type":"v1","addr":"172.21.15.98:6809","nonce":1320995478}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6806","nonce":1320995478},{"type":"v1","addr":"172.21.15.98:6807","nonce":1320995478}]},"public_addr":"172.21.15.98:6803/1320995478","cluster_addr":"172.21.15.98:6805/1320995478","heartbeat_back_addr":"172.21.15.98:6809/1320995478","heartbeat_front_addr":"172.21.15.98:6807/1320995478","state":["exists","up"]},{"osd":1,"uuid":"5d777d1f-3bd3-49ba-b0de-0ac532ee3934","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":89,"up_from":93,"up_thru":103,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6810","nonce":1561159155},{"type":"v1","addr":"172.21.15.98:6811","nonce":1561159155}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6812","nonce":1561159155},{"type":"v1","addr":"172.21.15.98:6813","nonce":1561159155}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6816","nonce":1561159155},{"type":"v1","addr":"172.21.15.98:6817","nonce":1561159155}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6814","nonce":1561159155},{"type":"v1","addr":"172.21.15.98:6815","nonce":1561159155}]},"public_addr":"172.21.15.98:6811/1561159155","cluster_addr":"172.21.15.98:6813/1561159155","heartbeat_back_addr":"172.21.15.98:6817/1561159155","heartbeat_front_addr":"172.21.15.98:6815/1561159155","state":["exists","up"]},{"osd":2,"uuid":"63978288-ac36-4cf2-bbb7-46dc828e6ce5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":94,"up_from":98,"up_thru":103,"down_at":95,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6818","nonce":4249816143},{"type":"v1","addr":"172.21.15.98:6819","nonce":4249816143}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6820","nonce":4249816143},{"type":"v1","addr":"172.21.15.98:6821","nonce":4249816143}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6824","nonce":4249816143},{"type":"v1","addr":"172.21.15.98:6825","nonce":4249816143}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6822","nonce":4249816143},{"type":"v1","addr":"172.21.15.98:6823","nonce":4249816143}]},"public_addr":"172.21.15.98:6819/4249816143","cluster_addr":"172.21.15.98:6821/4249816143","heartbeat_back_addr":"172.21.15.98:6825/4249816143","heartbeat_front_addr":"172.21.15.98:6823/4249816143","state":["exists","up"]},{"osd":3,"uuid":"48596d91-46da-4815-b810-d493bbf01dcc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":99,"up_from":103,"up_thru":103,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6826","nonce":2042614518},{"type":"v1","addr":"172.21.15.98:6827","nonce":2042614518}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6828","nonce":2042614518},{"type":"v1","addr":"172.21.15.98:6829","nonce":2042614518}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6832","nonce":2042614518},{"type":"v1","addr":"172.21.15.98:6833","nonce":2042614518}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6830","nonce":2042614518},{"type":"v1","addr":"172.21.15.98:6831","nonce":2042614518}]},"public_addr":"172.21.15.98:6827/2042614518","cluster_addr":"172.21.15.98:6829/2042614518","heartbeat_back_addr":"172.21.15.98:6833/2042614518","heartbeat_front_addr":"172.21.15.98:6831/2042614518","state":["exists","up"]},{"osd":4,"uuid":"a562afc6-ffe6-44bb-91a9-8a04018c3c6f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6800","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6801","nonce":165906554}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6802","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6803","nonce":165906554}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6806","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6807","nonce":165906554}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6804","nonce":165906554},{"type":"v1","addr":"172.21.15.173:6805","nonce":165906554}]},"public_addr":"172.21.15.173:6801/165906554","cluster_addr":"172.21.15.173:6803/165906554","heartbeat_back_addr":"172.21.15.173:6807/165906554","heartbeat_front_addr":"172.21.15.173:6805/165906554","state":["exists","up"]},{"osd":5,"uuid":"add28146-8301-4b4c-9616-967e9df3a8d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6808","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6809","nonce":2894909344}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6810","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6811","nonce":2894909344}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6814","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6815","nonce":2894909344}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6812","nonce":2894909344},{"type":"v1","addr":"172.21.15.173:6813","nonce":2894909344}]},"public_addr":"172.21.15.173:6809/2894909344","cluster_addr":"172.21.15.173:6811/2894909344","heartbeat_back_addr":"172.21.15.173:6815/2894909344","heartbeat_front_addr":"172.21.15.173:6813/2894909344","state":["exists","up"]},{"osd":6,"uuid":"f30854de-bc26-465d-8b0e-a9211ae2d9fa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":100,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6816","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6817","nonce":3550013548}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6818","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6819","nonce":3550013548}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6822","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6823","nonce":3550013548}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6820","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6821","nonce":3550013548}]},"public_addr":"172.21.15.173:6817/3550013548","cluster_addr":"172.21.15.173:6819/3550013548","heartbeat_back_addr":"172.21.15.173:6823/3550013548","heartbeat_front_addr":"172.21.15.173:6821/3550013548","state":["exists","up"]},{"osd":7,"uuid":"5f540de3-48f9-4bc8-b14a-9aadf838f16b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6824","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6825","nonce":135608222}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6826","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6827","nonce":135608222}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6830","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6831","nonce":135608222}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6828","nonce":135608222},{"type":"v1","addr":"172.21.15.173:6829","nonce":135608222}]},"public_addr":"172.21.15.173:6825/135608222","cluster_addr":"172.21.15.173:6827/135608222","heartbeat_back_addr":"172.21.15.173:6831/135608222","heartbeat_front_addr":"172.21.15.173:6829/135608222","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:07:04.739243+0000","dead_epoch":84},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:07:48.583198+0000","dead_epoch":89},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:08:33.173348+0000","dead_epoch":94},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:09:18.218502+0000","dead_epoch":99},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:09:59.765019+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:10:42.490689+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:11:24.357920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:12:08.168900+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.98:0/325576677":"2023-09-24T18:21:39.415268+0000","172.21.15.98:0/3213365423":"2023-09-24T18:04:40.596823+0000","172.21.15.98:0/2288964511":"2023-09-24T18:21:39.415268+0000","172.21.15.98:0/2098141849":"2023-09-24T18:18:53.712554+0000","172.21.15.98:0/910147733":"2023-09-24T18:21:39.415268+0000","172.21.15.98:6800/3510470804":"2023-09-24T18:21:39.415268+0000","172.21.15.173:0/2105184133":"2023-09-24T18:20:34.294726+0000","172.21.15.98:6800/3706591260":"2023-09-24T18:18:53.712554+0000","172.21.15.98:0/1142988558":"2023-09-24T18:12:42.525837+0000","172.21.15.98:6801/3510470804":"2023-09-24T18:21:39.415268+0000","172.21.15.98:0/2194193535":"2023-09-24T18:12:42.525837+0000","172.21.15.98:0/4085572202":"2023-09-24T18:18:53.712554+0000","172.21.15.98:6800/3368447653":"2023-09-24T18:04:40.596823+0000","172.21.15.173:6834/1425795384":"2023-09-24T18:20:34.294726+0000","172.21.15.98:6800/483469857":"2023-09-24T18:12:42.525837+0000","172.21.15.98:0/3410138197":"2023-09-24T18:21:39.415268+0000","172.21.15.98:0/3976553993":"2023-09-24T18:04:40.596823+0000","172.21.15.98:0/3472030975":"2023-09-24T18:05:11.060631+0000","172.21.15.173:0/4281585910":"2023-09-24T18:20:34.294726+0000","172.21.15.98:6801/3706591260":"2023-09-24T18:18:53.712554+0000","172.21.15.98:0/1603044865":"2023-09-24T18:05:11.060631+0000","172.21.15.173:6835/1425795384":"2023-09-24T18:20:34.294726+0000","172.21.15.173:0/2782586450":"2023-09-24T18:20:34.294726+0000","172.21.15.98:0/3501988932":"2023-09-24T18:21:39.415268+0000","172.21.15.98:6801/483469857":"2023-09-24T18:12:42.525837+0000","172.21.15.98:6801/3368447653":"2023-09-24T18:04:40.596823+0000","172.21.15.98:0/335394369":"2023-09-24T18:21:39.415268+0000","172.21.15.173:0/2534791896":"2023-09-24T18:20:34.294726+0000","172.21.15.98:6800/1392949322":"2023-09-24T18:05:11.060631+0000","172.21.15.98:6801/1392949322":"2023-09-24T18:05:11.060631+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T18:28:52.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:52 smithi098 bash[76817]: cluster 2023-09-23T18:28:51.606698+0000 mgr.y (mgr.26054) 302 : cluster 0 pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:52.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:52 smithi098 bash[76817]: audit 2023-09-23T18:28:52.142333+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.98:0/1782858169' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:28:52.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:52 smithi098 bash[79516]: cluster 2023-09-23T18:28:51.606698+0000 mgr.y (mgr.26054) 302 : cluster 0 pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:52.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:52 smithi098 bash[79516]: audit 2023-09-23T18:28:52.142333+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.98:0/1782858169' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:28:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:52 smithi173 bash[59422]: cluster 2023-09-23T18:28:51.606698+0000 mgr.y (mgr.26054) 302 : cluster 0 pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:52 smithi173 bash[59422]: audit 2023-09-23T18:28:52.142333+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.98:0/1782858169' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:28:53.124 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T18:28:53.124 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T18:28:53.125 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T18:28:53.125 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T18:28:53.126 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T18:28:53.126 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T18:28:53.127 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T18:28:53.127 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T18:28:54.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:54 smithi098 bash[76817]: cluster 2023-09-23T18:28:53.607482+0000 mgr.y (mgr.26054) 303 : cluster 0 pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:54.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:54 smithi098 bash[76817]: audit 2023-09-23T18:28:54.574782+0000 mon.a (mon.0) 485 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:28:54.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:54 smithi098 bash[79516]: cluster 2023-09-23T18:28:53.607482+0000 mgr.y (mgr.26054) 303 : cluster 0 pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:54.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:54 smithi098 bash[79516]: audit 2023-09-23T18:28:54.574782+0000 mon.a (mon.0) 485 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:28:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:54 smithi173 bash[59422]: cluster 2023-09-23T18:28:53.607482+0000 mgr.y (mgr.26054) 303 : cluster 0 pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:54 smithi173 bash[59422]: audit 2023-09-23T18:28:54.574782+0000 mon.a (mon.0) 485 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:28:56.441 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:28:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:28:56] "GET /metrics HTTP/1.1" 200 34091 "" "Prometheus/2.33.4" 2023-09-23T18:28:56.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:56 smithi098 bash[79516]: cluster 2023-09-23T18:28:55.608378+0000 mgr.y (mgr.26054) 304 : cluster 0 pgmap v241: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:56.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:56 smithi098 bash[76817]: cluster 2023-09-23T18:28:55.608378+0000 mgr.y (mgr.26054) 304 : cluster 0 pgmap v241: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:57.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:56 smithi173 bash[59422]: cluster 2023-09-23T18:28:55.608378+0000 mgr.y (mgr.26054) 304 : cluster 0 pgmap v241: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:58.008 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:28:58.008 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:28:58.009 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:28:58.009 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:28:58.011 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:28:58.011 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:28:58.015 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:28:58.015 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:28:58.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:28:58 smithi098 bash[76817]: cluster 2023-09-23T18:28:57.609251+0000 mgr.y (mgr.26054) 305 : cluster 0 pgmap v242: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:58.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:28:58 smithi098 bash[79516]: cluster 2023-09-23T18:28:57.609251+0000 mgr.y (mgr.26054) 305 : cluster 0 pgmap v242: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:28:59.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:28:58 smithi173 bash[59422]: cluster 2023-09-23T18:28:57.609251+0000 mgr.y (mgr.26054) 305 : cluster 0 pgmap v242: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:00.481 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:00 smithi098 bash[79516]: cluster 2023-09-23T18:28:59.609846+0000 mgr.y (mgr.26054) 306 : cluster 0 pgmap v243: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:00.481 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:00 smithi098 bash[76817]: cluster 2023-09-23T18:28:59.609846+0000 mgr.y (mgr.26054) 306 : cluster 0 pgmap v243: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:00.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:00 smithi173 bash[59422]: cluster 2023-09-23T18:28:59.609846+0000 mgr.y (mgr.26054) 306 : cluster 0 pgmap v243: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:02.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:02 smithi098 bash[79516]: cluster 2023-09-23T18:29:01.610648+0000 mgr.y (mgr.26054) 307 : cluster 0 pgmap v244: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:02.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:02 smithi098 bash[76817]: cluster 2023-09-23T18:29:01.610648+0000 mgr.y (mgr.26054) 307 : cluster 0 pgmap v244: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:03.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:02 smithi173 bash[59422]: cluster 2023-09-23T18:29:01.610648+0000 mgr.y (mgr.26054) 307 : cluster 0 pgmap v244: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:04.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:04 smithi098 bash[76817]: cluster 2023-09-23T18:29:03.611234+0000 mgr.y (mgr.26054) 308 : cluster 0 pgmap v245: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:04.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:04 smithi098 bash[79516]: cluster 2023-09-23T18:29:03.611234+0000 mgr.y (mgr.26054) 308 : cluster 0 pgmap v245: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:05.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:04 smithi173 bash[59422]: cluster 2023-09-23T18:29:03.611234+0000 mgr.y (mgr.26054) 308 : cluster 0 pgmap v245: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:06.441 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:29:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:29:06] "GET /metrics HTTP/1.1" 200 34091 "" "Prometheus/2.33.4" 2023-09-23T18:29:06.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:06 smithi098 bash[76817]: cluster 2023-09-23T18:29:05.611957+0000 mgr.y (mgr.26054) 309 : cluster 0 pgmap v246: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:06.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:06 smithi098 bash[79516]: cluster 2023-09-23T18:29:05.611957+0000 mgr.y (mgr.26054) 309 : cluster 0 pgmap v246: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:07.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:06 smithi173 bash[59422]: cluster 2023-09-23T18:29:05.611957+0000 mgr.y (mgr.26054) 309 : cluster 0 pgmap v246: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:08.959 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:08 smithi098 bash[79516]: cluster 2023-09-23T18:29:07.612619+0000 mgr.y (mgr.26054) 310 : cluster 0 pgmap v247: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:08.960 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:08 smithi098 bash[76817]: cluster 2023-09-23T18:29:07.612619+0000 mgr.y (mgr.26054) 310 : cluster 0 pgmap v247: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:09.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:08 smithi173 bash[59422]: cluster 2023-09-23T18:29:07.612619+0000 mgr.y (mgr.26054) 310 : cluster 0 pgmap v247: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:10.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:10 smithi098 bash[79516]: audit 2023-09-23T18:29:09.574532+0000 mon.a (mon.0) 486 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:29:10.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:10 smithi098 bash[79516]: cluster 2023-09-23T18:29:09.613102+0000 mgr.y (mgr.26054) 311 : cluster 0 pgmap v248: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:10.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:10 smithi098 bash[76817]: audit 2023-09-23T18:29:09.574532+0000 mon.a (mon.0) 486 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:29:10.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:10 smithi098 bash[76817]: cluster 2023-09-23T18:29:09.613102+0000 mgr.y (mgr.26054) 311 : cluster 0 pgmap v248: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:10.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:10 smithi173 bash[59422]: audit 2023-09-23T18:29:09.574532+0000 mon.a (mon.0) 486 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:29:10.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:10 smithi173 bash[59422]: cluster 2023-09-23T18:29:09.613102+0000 mgr.y (mgr.26054) 311 : cluster 0 pgmap v248: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:12.930 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:12 smithi098 bash[76817]: cluster 2023-09-23T18:29:11.613999+0000 mgr.y (mgr.26054) 312 : cluster 0 pgmap v249: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:12.931 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:12 smithi098 bash[79516]: cluster 2023-09-23T18:29:11.613999+0000 mgr.y (mgr.26054) 312 : cluster 0 pgmap v249: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:13.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:12 smithi173 bash[59422]: cluster 2023-09-23T18:29:11.613999+0000 mgr.y (mgr.26054) 312 : cluster 0 pgmap v249: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:13.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:13 smithi098 bash[76817]: audit 2023-09-23T18:29:13.454580+0000 mon.a (mon.0) 487 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:29:13.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:13 smithi098 bash[79516]: audit 2023-09-23T18:29:13.454580+0000 mon.a (mon.0) 487 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:29:14.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:13 smithi173 bash[59422]: audit 2023-09-23T18:29:13.454580+0000 mon.a (mon.0) 487 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:29:14.149 INFO:teuthology.orchestra.run.smithi098.stdout:154618822881 2023-09-23T18:29:14.150 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T18:29:14.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:14 smithi098 bash[76817]: cluster 2023-09-23T18:29:13.614835+0000 mgr.y (mgr.26054) 313 : cluster 0 pgmap v250: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:14.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:14 smithi098 bash[79516]: cluster 2023-09-23T18:29:13.614835+0000 mgr.y (mgr.26054) 313 : cluster 0 pgmap v250: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:15.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:14 smithi173 bash[59422]: cluster 2023-09-23T18:29:13.614835+0000 mgr.y (mgr.26054) 313 : cluster 0 pgmap v250: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:15.956 INFO:teuthology.orchestra.run.smithi098.stdout:399431958556 2023-09-23T18:29:15.957 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T18:29:16.238 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:29:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:29:16] "GET /metrics HTTP/1.1" 200 34094 "" "Prometheus/2.33.4" 2023-09-23T18:29:16.538 INFO:teuthology.orchestra.run.smithi098.stdout:128849019114 2023-09-23T18:29:16.538 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T18:29:16.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:16 smithi098 bash[76817]: cluster 2023-09-23T18:29:15.615865+0000 mgr.y (mgr.26054) 314 : cluster 0 pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:16.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:16 smithi098 bash[79516]: cluster 2023-09-23T18:29:15.615865+0000 mgr.y (mgr.26054) 314 : cluster 0 pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:17.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:16 smithi173 bash[59422]: cluster 2023-09-23T18:29:15.615865+0000 mgr.y (mgr.26054) 314 : cluster 0 pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:18.704 INFO:teuthology.orchestra.run.smithi098.stdout:180388626648 2023-09-23T18:29:18.704 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T18:29:18.709 INFO:teuthology.orchestra.run.smithi098.stdout:206158430417 2023-09-23T18:29:18.710 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T18:29:18.710 INFO:teuthology.orchestra.run.smithi098.stdout:442381631504 2023-09-23T18:29:18.711 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T18:29:18.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:18 smithi098 bash[76817]: cluster 2023-09-23T18:29:17.616732+0000 mgr.y (mgr.26054) 315 : cluster 0 pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:18.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:18 smithi098 bash[79516]: cluster 2023-09-23T18:29:17.616732+0000 mgr.y (mgr.26054) 315 : cluster 0 pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:19.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:18 smithi173 bash[59422]: cluster 2023-09-23T18:29:17.616732+0000 mgr.y (mgr.26054) 315 : cluster 0 pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:19.382 INFO:teuthology.orchestra.run.smithi098.stdout:420906795031 2023-09-23T18:29:19.383 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T18:29:19.584 INFO:teuthology.orchestra.run.smithi098.stdout:377957122082 2023-09-23T18:29:19.584 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T18:29:21.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:20 smithi173 bash[59422]: cluster 2023-09-23T18:29:19.617453+0000 mgr.y (mgr.26054) 316 : cluster 0 pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:21.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:20 smithi173 bash[59422]: audit 2023-09-23T18:29:19.744504+0000 mon.a (mon.0) 488 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:29:21.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:20 smithi173 bash[59422]: audit 2023-09-23T18:29:19.751922+0000 mon.a (mon.0) 489 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:29:21.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:20 smithi098 bash[76817]: cluster 2023-09-23T18:29:19.617453+0000 mgr.y (mgr.26054) 316 : cluster 0 pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:21.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:20 smithi098 bash[76817]: audit 2023-09-23T18:29:19.744504+0000 mon.a (mon.0) 488 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:29:21.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:20 smithi098 bash[76817]: audit 2023-09-23T18:29:19.751922+0000 mon.a (mon.0) 489 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:29:21.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:20 smithi098 bash[79516]: cluster 2023-09-23T18:29:19.617453+0000 mgr.y (mgr.26054) 316 : cluster 0 pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:21.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:20 smithi098 bash[79516]: audit 2023-09-23T18:29:19.744504+0000 mon.a (mon.0) 488 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:29:21.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:20 smithi098 bash[79516]: audit 2023-09-23T18:29:19.751922+0000 mon.a (mon.0) 489 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:29:22.365 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:29:22.367 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:29:22.367 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:29:23.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:22 smithi173 bash[59422]: cluster 2023-09-23T18:29:21.618452+0000 mgr.y (mgr.26054) 317 : cluster 0 pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:23.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:22 smithi098 bash[76817]: cluster 2023-09-23T18:29:21.618452+0000 mgr.y (mgr.26054) 317 : cluster 0 pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:23.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:22 smithi098 bash[79516]: cluster 2023-09-23T18:29:21.618452+0000 mgr.y (mgr.26054) 317 : cluster 0 pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:23.381 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:29:23.381 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:29:23.382 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:29:23.383 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:29:23.387 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:29:24.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:24 smithi098 bash[76817]: audit 2023-09-23T18:29:23.405529+0000 mon.a (mon.0) 490 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:29:24.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:24 smithi098 bash[76817]: audit 2023-09-23T18:29:23.418724+0000 mon.a (mon.0) 491 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:29:24.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:24 smithi098 bash[76817]: cluster 2023-09-23T18:29:23.618892+0000 mgr.y (mgr.26054) 318 : cluster 0 pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:24.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:24 smithi098 bash[76817]: audit 2023-09-23T18:29:23.840334+0000 mon.a (mon.0) 492 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:29:24.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:24 smithi098 bash[76817]: audit 2023-09-23T18:29:23.841442+0000 mon.a (mon.0) 493 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:29:24.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:24 smithi098 bash[76817]: audit 2023-09-23T18:29:23.847757+0000 mon.a (mon.0) 494 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:29:24.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:24 smithi098 bash[79516]: audit 2023-09-23T18:29:23.405529+0000 mon.a (mon.0) 490 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:29:24.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:24 smithi098 bash[79516]: audit 2023-09-23T18:29:23.418724+0000 mon.a (mon.0) 491 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:29:24.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:24 smithi098 bash[79516]: cluster 2023-09-23T18:29:23.618892+0000 mgr.y (mgr.26054) 318 : cluster 0 pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:24.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:24 smithi098 bash[79516]: audit 2023-09-23T18:29:23.840334+0000 mon.a (mon.0) 492 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:29:24.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:24 smithi098 bash[79516]: audit 2023-09-23T18:29:23.841442+0000 mon.a (mon.0) 493 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:29:24.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:24 smithi098 bash[79516]: audit 2023-09-23T18:29:23.847757+0000 mon.a (mon.0) 494 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:29:24.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:24 smithi173 bash[59422]: audit 2023-09-23T18:29:23.405529+0000 mon.a (mon.0) 490 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:29:24.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:24 smithi173 bash[59422]: audit 2023-09-23T18:29:23.418724+0000 mon.a (mon.0) 491 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:29:24.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:24 smithi173 bash[59422]: cluster 2023-09-23T18:29:23.618892+0000 mgr.y (mgr.26054) 318 : cluster 0 pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:24.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:24 smithi173 bash[59422]: audit 2023-09-23T18:29:23.840334+0000 mon.a (mon.0) 492 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:29:24.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:24 smithi173 bash[59422]: audit 2023-09-23T18:29:23.841442+0000 mon.a (mon.0) 493 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:29:24.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:24 smithi173 bash[59422]: audit 2023-09-23T18:29:23.847757+0000 mon.a (mon.0) 494 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:29:25.639 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:25 smithi098 bash[76817]: audit 2023-09-23T18:29:24.575254+0000 mon.a (mon.0) 495 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:29:25.639 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:25 smithi098 bash[79516]: audit 2023-09-23T18:29:24.575254+0000 mon.a (mon.0) 495 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:29:25.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:25 smithi173 bash[59422]: audit 2023-09-23T18:29:24.575254+0000 mon.a (mon.0) 495 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:29:26.330 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:29:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:29:26] "GET /metrics HTTP/1.1" 200 34094 "" "Prometheus/2.33.4" 2023-09-23T18:29:26.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:26 smithi098 bash[76817]: cluster 2023-09-23T18:29:25.619555+0000 mgr.y (mgr.26054) 319 : cluster 0 pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:26.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:26 smithi098 bash[79516]: cluster 2023-09-23T18:29:25.619555+0000 mgr.y (mgr.26054) 319 : cluster 0 pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:26.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:26 smithi173 bash[59422]: cluster 2023-09-23T18:29:25.619555+0000 mgr.y (mgr.26054) 319 : cluster 0 pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:28.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:28 smithi098 bash[79516]: cluster 2023-09-23T18:29:27.620063+0000 mgr.y (mgr.26054) 320 : cluster 0 pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:28.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:28 smithi098 bash[76817]: cluster 2023-09-23T18:29:27.620063+0000 mgr.y (mgr.26054) 320 : cluster 0 pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:29.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:28 smithi173 bash[59422]: cluster 2023-09-23T18:29:27.620063+0000 mgr.y (mgr.26054) 320 : cluster 0 pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:30.375 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:30 smithi098 bash[76817]: cluster 2023-09-23T18:29:29.620751+0000 mgr.y (mgr.26054) 321 : cluster 0 pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:30.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:30 smithi173 bash[59422]: cluster 2023-09-23T18:29:29.620751+0000 mgr.y (mgr.26054) 321 : cluster 0 pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:30.783 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:30 smithi098 bash[79516]: cluster 2023-09-23T18:29:29.620751+0000 mgr.y (mgr.26054) 321 : cluster 0 pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:32.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:32 smithi098 bash[79516]: cluster 2023-09-23T18:29:31.621756+0000 mgr.y (mgr.26054) 322 : cluster 0 pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:32.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:32 smithi098 bash[76817]: cluster 2023-09-23T18:29:31.621756+0000 mgr.y (mgr.26054) 322 : cluster 0 pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:32 smithi173 bash[59422]: cluster 2023-09-23T18:29:31.621756+0000 mgr.y (mgr.26054) 322 : cluster 0 pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:34.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:34 smithi098 bash[76817]: cluster 2023-09-23T18:29:33.622556+0000 mgr.y (mgr.26054) 323 : cluster 0 pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:34.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:34 smithi098 bash[79516]: cluster 2023-09-23T18:29:33.622556+0000 mgr.y (mgr.26054) 323 : cluster 0 pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:35.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:34 smithi173 bash[59422]: cluster 2023-09-23T18:29:33.622556+0000 mgr.y (mgr.26054) 323 : cluster 0 pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:35.188 INFO:teuthology.orchestra.run.smithi098.stdout:154618822886 2023-09-23T18:29:35.269 INFO:teuthology.orchestra.run.smithi098.stdout:128849019118 2023-09-23T18:29:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:35 smithi173 bash[59422]: audit 2023-09-23T18:29:35.186901+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.98:0/3082827336' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T18:29:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:35 smithi173 bash[59422]: audit 2023-09-23T18:29:35.267878+0000 mon.a (mon.0) 496 : audit 0 from='client.? 172.21.15.98:0/3206375799' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T18:29:36.086 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:35 smithi098 bash[76817]: audit 2023-09-23T18:29:35.186901+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.98:0/3082827336' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T18:29:36.086 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:35 smithi098 bash[76817]: audit 2023-09-23T18:29:35.267878+0000 mon.a (mon.0) 496 : audit 0 from='client.? 172.21.15.98:0/3206375799' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T18:29:36.086 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:35 smithi098 bash[79516]: audit 2023-09-23T18:29:35.186901+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.98:0/3082827336' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T18:29:36.086 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:35 smithi098 bash[79516]: audit 2023-09-23T18:29:35.267878+0000 mon.a (mon.0) 496 : audit 0 from='client.? 172.21.15.98:0/3206375799' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T18:29:36.485 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:29:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:29:36] "GET /metrics HTTP/1.1" 200 34094 "" "Prometheus/2.33.4" 2023-09-23T18:29:37.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:36 smithi173 bash[59422]: cluster 2023-09-23T18:29:35.623353+0000 mgr.y (mgr.26054) 324 : cluster 0 pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:37.180 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:36 smithi098 bash[76817]: cluster 2023-09-23T18:29:35.623353+0000 mgr.y (mgr.26054) 324 : cluster 0 pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:37.180 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:36 smithi098 bash[79516]: cluster 2023-09-23T18:29:35.623353+0000 mgr.y (mgr.26054) 324 : cluster 0 pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:37.736 INFO:teuthology.orchestra.run.smithi098.stdout:399431958561 2023-09-23T18:29:37.917 INFO:teuthology.orchestra.run.smithi098.stdout:206158430421 2023-09-23T18:29:38.615 INFO:teuthology.orchestra.run.smithi098.stdout:180388626653 2023-09-23T18:29:39.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:38 smithi173 bash[59422]: cluster 2023-09-23T18:29:37.623782+0000 mgr.y (mgr.26054) 325 : cluster 0 pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:39.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:38 smithi173 bash[59422]: audit 2023-09-23T18:29:37.734753+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.98:0/1384701181' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T18:29:39.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:38 smithi173 bash[59422]: audit 2023-09-23T18:29:37.915728+0000 mon.a (mon.0) 497 : audit 0 from='client.? 172.21.15.98:0/2942168037' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T18:29:39.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:38 smithi173 bash[59422]: audit 2023-09-23T18:29:38.613666+0000 mon.c (mon.2) 43 : audit 0 from='client.? 172.21.15.98:0/3654745378' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T18:29:39.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:38 smithi098 bash[76817]: cluster 2023-09-23T18:29:37.623782+0000 mgr.y (mgr.26054) 325 : cluster 0 pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:39.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:38 smithi098 bash[76817]: audit 2023-09-23T18:29:37.734753+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.98:0/1384701181' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T18:29:39.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:38 smithi098 bash[76817]: audit 2023-09-23T18:29:37.915728+0000 mon.a (mon.0) 497 : audit 0 from='client.? 172.21.15.98:0/2942168037' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T18:29:39.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:38 smithi098 bash[76817]: audit 2023-09-23T18:29:38.613666+0000 mon.c (mon.2) 43 : audit 0 from='client.? 172.21.15.98:0/3654745378' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T18:29:39.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:38 smithi098 bash[79516]: cluster 2023-09-23T18:29:37.623782+0000 mgr.y (mgr.26054) 325 : cluster 0 pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:39.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:38 smithi098 bash[79516]: audit 2023-09-23T18:29:37.734753+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.98:0/1384701181' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T18:29:39.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:38 smithi098 bash[79516]: audit 2023-09-23T18:29:37.915728+0000 mon.a (mon.0) 497 : audit 0 from='client.? 172.21.15.98:0/2942168037' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T18:29:39.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:38 smithi098 bash[79516]: audit 2023-09-23T18:29:38.613666+0000 mon.c (mon.2) 43 : audit 0 from='client.? 172.21.15.98:0/3654745378' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T18:29:39.660 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822881 got 154618822886 for osd.5 2023-09-23T18:29:39.660 DEBUG:teuthology.parallel:result is None 2023-09-23T18:29:40.339 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019114 got 128849019118 for osd.4 2023-09-23T18:29:40.339 DEBUG:teuthology.parallel:result is None 2023-09-23T18:29:40.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:40 smithi098 bash[76817]: audit 2023-09-23T18:29:39.575560+0000 mon.a (mon.0) 498 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:29:40.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:40 smithi098 bash[76817]: cluster 2023-09-23T18:29:39.624492+0000 mgr.y (mgr.26054) 326 : cluster 0 pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:40.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:40 smithi098 bash[79516]: audit 2023-09-23T18:29:39.575560+0000 mon.a (mon.0) 498 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:29:40.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:40 smithi098 bash[79516]: cluster 2023-09-23T18:29:39.624492+0000 mgr.y (mgr.26054) 326 : cluster 0 pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:40.760 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430417 got 206158430421 for osd.7 2023-09-23T18:29:40.760 DEBUG:teuthology.parallel:result is None 2023-09-23T18:29:40.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:40 smithi173 bash[59422]: audit 2023-09-23T18:29:39.575560+0000 mon.a (mon.0) 498 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:29:40.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:40 smithi173 bash[59422]: cluster 2023-09-23T18:29:39.624492+0000 mgr.y (mgr.26054) 326 : cluster 0 pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:41.164 INFO:teuthology.orchestra.run.smithi098.stdout:442381631509 2023-09-23T18:29:41.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:41 smithi098 bash[76817]: audit 2023-09-23T18:29:41.163554+0000 mon.a (mon.0) 499 : audit 0 from='client.? 172.21.15.98:0/360526984' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T18:29:41.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:41 smithi098 bash[79516]: audit 2023-09-23T18:29:41.163554+0000 mon.a (mon.0) 499 : audit 0 from='client.? 172.21.15.98:0/360526984' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T18:29:41.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:41 smithi173 bash[59422]: audit 2023-09-23T18:29:41.163554+0000 mon.a (mon.0) 499 : audit 0 from='client.? 172.21.15.98:0/360526984' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T18:29:42.303 INFO:teuthology.orchestra.run.smithi098.stdout:377957122087 2023-09-23T18:29:42.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:42 smithi098 bash[79516]: cluster 2023-09-23T18:29:41.625170+0000 mgr.y (mgr.26054) 327 : cluster 0 pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:42.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:42 smithi098 bash[79516]: audit 2023-09-23T18:29:42.301811+0000 mon.c (mon.2) 44 : audit 0 from='client.? 172.21.15.98:0/4055346643' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T18:29:42.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:42 smithi098 bash[76817]: cluster 2023-09-23T18:29:41.625170+0000 mgr.y (mgr.26054) 327 : cluster 0 pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:42.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:42 smithi098 bash[76817]: audit 2023-09-23T18:29:42.301811+0000 mon.c (mon.2) 44 : audit 0 from='client.? 172.21.15.98:0/4055346643' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T18:29:42.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:42 smithi173 bash[59422]: cluster 2023-09-23T18:29:41.625170+0000 mgr.y (mgr.26054) 327 : cluster 0 pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:42.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:42 smithi173 bash[59422]: audit 2023-09-23T18:29:42.301811+0000 mon.c (mon.2) 44 : audit 0 from='client.? 172.21.15.98:0/4055346643' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T18:29:42.836 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626648 got 180388626653 for osd.6 2023-09-23T18:29:42.836 DEBUG:teuthology.parallel:result is None 2023-09-23T18:29:43.356 INFO:tasks.cephadm.ceph_manager.ceph:need seq 399431958556 got 399431958561 for osd.1 2023-09-23T18:29:43.356 DEBUG:teuthology.parallel:result is None 2023-09-23T18:29:44.198 INFO:tasks.cephadm.ceph_manager.ceph:need seq 442381631504 got 442381631509 for osd.3 2023-09-23T18:29:44.199 DEBUG:teuthology.parallel:result is None 2023-09-23T18:29:44.250 INFO:teuthology.orchestra.run.smithi098.stdout:420906795036 2023-09-23T18:29:44.494 INFO:tasks.cephadm.ceph_manager.ceph:need seq 377957122082 got 377957122087 for osd.0 2023-09-23T18:29:44.494 DEBUG:teuthology.parallel:result is None 2023-09-23T18:29:44.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:44 smithi098 bash[79516]: cluster 2023-09-23T18:29:43.625919+0000 mgr.y (mgr.26054) 328 : cluster 0 pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:44.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:44 smithi098 bash[79516]: audit 2023-09-23T18:29:44.249002+0000 mon.c (mon.2) 45 : audit 0 from='client.? 172.21.15.98:0/1721353919' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T18:29:44.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:44 smithi098 bash[76817]: cluster 2023-09-23T18:29:43.625919+0000 mgr.y (mgr.26054) 328 : cluster 0 pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:44.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:44 smithi098 bash[76817]: audit 2023-09-23T18:29:44.249002+0000 mon.c (mon.2) 45 : audit 0 from='client.? 172.21.15.98:0/1721353919' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T18:29:45.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:44 smithi173 bash[59422]: cluster 2023-09-23T18:29:43.625919+0000 mgr.y (mgr.26054) 328 : cluster 0 pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:45.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:44 smithi173 bash[59422]: audit 2023-09-23T18:29:44.249002+0000 mon.c (mon.2) 45 : audit 0 from='client.? 172.21.15.98:0/1721353919' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T18:29:45.115 INFO:tasks.cephadm.ceph_manager.ceph:need seq 420906795031 got 420906795036 for osd.2 2023-09-23T18:29:45.115 DEBUG:teuthology.parallel:result is None 2023-09-23T18:29:45.115 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T18:29:45.115 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:29:46.441 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:29:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:29:46] "GET /metrics HTTP/1.1" 200 34095 "" "Prometheus/2.33.4" 2023-09-23T18:29:46.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:46 smithi098 bash[76817]: cluster 2023-09-23T18:29:45.626881+0000 mgr.y (mgr.26054) 329 : cluster 0 pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:46.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:46 smithi098 bash[79516]: cluster 2023-09-23T18:29:45.626881+0000 mgr.y (mgr.26054) 329 : cluster 0 pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:47.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:46 smithi173 bash[59422]: cluster 2023-09-23T18:29:45.626881+0000 mgr.y (mgr.26054) 329 : cluster 0 pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:48.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:48 smithi098 bash[76817]: cluster 2023-09-23T18:29:47.627678+0000 mgr.y (mgr.26054) 330 : cluster 0 pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:48.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:48 smithi098 bash[79516]: cluster 2023-09-23T18:29:47.627678+0000 mgr.y (mgr.26054) 330 : cluster 0 pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:49.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:48 smithi173 bash[59422]: cluster 2023-09-23T18:29:47.627678+0000 mgr.y (mgr.26054) 330 : cluster 0 pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:49.893 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:29:50.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:50 smithi098 bash[79516]: cluster 2023-09-23T18:29:49.628374+0000 mgr.y (mgr.26054) 331 : cluster 0 pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:50.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:50 smithi098 bash[76817]: cluster 2023-09-23T18:29:49.628374+0000 mgr.y (mgr.26054) 331 : cluster 0 pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:50.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:50 smithi173 bash[59422]: cluster 2023-09-23T18:29:49.628374+0000 mgr.y (mgr.26054) 331 : cluster 0 pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:52.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:52 smithi098 bash[79516]: cluster 2023-09-23T18:29:51.629311+0000 mgr.y (mgr.26054) 332 : cluster 0 pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:52.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:52 smithi098 bash[76817]: cluster 2023-09-23T18:29:51.629311+0000 mgr.y (mgr.26054) 332 : cluster 0 pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:52 smithi173 bash[59422]: cluster 2023-09-23T18:29:51.629311+0000 mgr.y (mgr.26054) 332 : cluster 0 pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:53.233 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:29:53.235 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-23T18:29:54.103 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":269,"stamp":"2023-09-23T18:29:51.628646+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6303,"num_read_kb":13760,"num_write":1106,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1341076,"kb_used_data":22548,"kb_used_omap":31,"kb_used_meta":1318432,"kb_avail":748620140,"statfs":{"total":767960285184,"available":766587023360,"internally_reserved":0,"allocated":23089152,"data_stored":19014596,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32507,"internal_metadata":1350074629},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004622"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":63,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.289993+0000","last_change":"2023-09-23T18:15:43.682873+0000","last_active":"2023-09-23T18:28:02.289993+0000","last_peered":"2023-09-23T18:28:02.289993+0000","last_clean":"2023-09-23T18:28:02.289993+0000","last_became_active":"2023-09-23T18:15:43.682465+0000","last_became_peered":"2023-09-23T18:15:43.682465+0000","last_unstale":"2023-09-23T18:28:02.289993+0000","last_undegraded":"2023-09-23T18:28:02.289993+0000","last_fullsized":"2023-09-23T18:28:02.289993+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.280010+0000","last_change":"2023-09-23T18:27:33.583935+0000","last_active":"2023-09-23T18:28:03.280010+0000","last_peered":"2023-09-23T18:28:03.280010+0000","last_clean":"2023-09-23T18:28:03.280010+0000","last_became_active":"2023-09-23T18:27:33.582561+0000","last_became_peered":"2023-09-23T18:27:33.582561+0000","last_unstale":"2023-09-23T18:28:03.280010+0000","last_undegraded":"2023-09-23T18:28:03.280010+0000","last_fullsized":"2023-09-23T18:28:03.280010+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:43.589363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":90,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.299520+0000","last_change":"2023-09-23T18:28:03.299520+0000","last_active":"2023-09-23T18:28:03.299520+0000","last_peered":"2023-09-23T18:28:03.299520+0000","last_clean":"2023-09-23T18:28:03.299520+0000","last_became_active":"2023-09-23T18:28:03.299158+0000","last_became_peered":"2023-09-23T18:28:03.299158+0000","last_unstale":"2023-09-23T18:28:03.299520+0000","last_undegraded":"2023-09-23T18:28:03.299520+0000","last_fullsized":"2023-09-23T18:28:03.299520+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:26.247064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":77,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.296850+0000","last_change":"2023-09-23T18:28:03.296850+0000","last_active":"2023-09-23T18:28:03.296850+0000","last_peered":"2023-09-23T18:28:03.296850+0000","last_clean":"2023-09-23T18:28:03.296850+0000","last_became_active":"2023-09-23T18:28:03.295842+0000","last_became_peered":"2023-09-23T18:28:03.295842+0000","last_unstale":"2023-09-23T18:28:03.296850+0000","last_undegraded":"2023-09-23T18:28:03.296850+0000","last_fullsized":"2023-09-23T18:28:03.296850+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:53.048579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.302261+0000","last_change":"2023-09-23T18:28:03.302261+0000","last_active":"2023-09-23T18:28:03.302261+0000","last_peered":"2023-09-23T18:28:03.302261+0000","last_clean":"2023-09-23T18:28:03.302261+0000","last_became_active":"2023-09-23T18:28:03.301911+0000","last_became_peered":"2023-09-23T18:28:03.301911+0000","last_unstale":"2023-09-23T18:28:03.302261+0000","last_undegraded":"2023-09-23T18:28:03.302261+0000","last_fullsized":"2023-09-23T18:28:03.302261+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":57,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.279119+0000","last_change":"2023-09-23T18:15:08.902664+0000","last_active":"2023-09-23T18:28:02.279119+0000","last_peered":"2023-09-23T18:28:02.279119+0000","last_clean":"2023-09-23T18:28:02.279119+0000","last_became_active":"2023-09-23T18:15:08.901252+0000","last_became_peered":"2023-09-23T18:15:08.901252+0000","last_unstale":"2023-09-23T18:28:02.279119+0000","last_undegraded":"2023-09-23T18:28:02.279119+0000","last_fullsized":"2023-09-23T18:28:02.279119+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.300718+0000","last_change":"2023-09-23T18:28:03.300718+0000","last_active":"2023-09-23T18:28:03.300718+0000","last_peered":"2023-09-23T18:28:03.300718+0000","last_clean":"2023-09-23T18:28:03.300718+0000","last_became_active":"2023-09-23T18:28:03.300298+0000","last_became_peered":"2023-09-23T18:28:03.300298+0000","last_unstale":"2023-09-23T18:28:03.300718+0000","last_undegraded":"2023-09-23T18:28:03.300718+0000","last_fullsized":"2023-09-23T18:28:03.300718+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.278761+0000","last_change":"2023-09-23T18:27:04.250817+0000","last_active":"2023-09-23T18:28:02.278761+0000","last_peered":"2023-09-23T18:28:02.278761+0000","last_clean":"2023-09-23T18:28:02.278761+0000","last_became_active":"2023-09-23T18:27:04.250397+0000","last_became_peered":"2023-09-23T18:27:04.250397+0000","last_unstale":"2023-09-23T18:28:02.278761+0000","last_undegraded":"2023-09-23T18:28:02.278761+0000","last_fullsized":"2023-09-23T18:28:02.278761+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":91,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.288092+0000","last_change":"2023-09-23T18:28:03.288092+0000","last_active":"2023-09-23T18:28:03.288092+0000","last_peered":"2023-09-23T18:28:03.288092+0000","last_clean":"2023-09-23T18:28:03.288092+0000","last_became_active":"2023-09-23T18:28:03.287643+0000","last_became_peered":"2023-09-23T18:28:03.287643+0000","last_unstale":"2023-09-23T18:28:03.288092+0000","last_undegraded":"2023-09-23T18:28:03.288092+0000","last_fullsized":"2023-09-23T18:28:03.288092+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:44.163816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":41,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.280816+0000","last_change":"2023-09-23T18:26:34.994176+0000","last_active":"2023-09-23T18:28:03.280816+0000","last_peered":"2023-09-23T18:28:03.280816+0000","last_clean":"2023-09-23T18:28:03.280816+0000","last_became_active":"2023-09-23T18:26:34.993829+0000","last_became_peered":"2023-09-23T18:26:34.993829+0000","last_unstale":"2023-09-23T18:28:03.280816+0000","last_undegraded":"2023-09-23T18:28:03.280816+0000","last_fullsized":"2023-09-23T18:28:03.280816+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:27.111857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.293000+0000","last_change":"2023-09-23T18:15:09.125623+0000","last_active":"2023-09-23T18:28:02.293000+0000","last_peered":"2023-09-23T18:28:02.293000+0000","last_clean":"2023-09-23T18:28:02.293000+0000","last_became_active":"2023-09-23T18:15:09.125018+0000","last_became_peered":"2023-09-23T18:15:09.125018+0000","last_unstale":"2023-09-23T18:28:02.293000+0000","last_undegraded":"2023-09-23T18:28:02.293000+0000","last_fullsized":"2023-09-23T18:28:02.293000+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.281601+0000","last_change":"2023-09-23T18:27:04.247514+0000","last_active":"2023-09-23T18:28:03.281601+0000","last_peered":"2023-09-23T18:28:03.281601+0000","last_clean":"2023-09-23T18:28:03.281601+0000","last_became_active":"2023-09-23T18:27:04.247025+0000","last_became_peered":"2023-09-23T18:27:04.247025+0000","last_unstale":"2023-09-23T18:28:03.281601+0000","last_undegraded":"2023-09-23T18:28:03.281601+0000","last_fullsized":"2023-09-23T18:28:03.281601+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:32.498164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"77'112","reported_seq":594,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.284556+0000","last_change":"2023-09-23T18:27:33.587478+0000","last_active":"2023-09-23T18:28:02.284556+0000","last_peered":"2023-09-23T18:28:02.284556+0000","last_clean":"2023-09-23T18:28:02.284556+0000","last_became_active":"2023-09-23T18:27:33.586958+0000","last_became_peered":"2023-09-23T18:27:33.586958+0000","last_unstale":"2023-09-23T18:28:02.284556+0000","last_undegraded":"2023-09-23T18:28:02.284556+0000","last_fullsized":"2023-09-23T18:28:02.284556+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":893,"num_read_kb":3172,"num_write":69,"num_write_kb":2237,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.292295+0000","last_change":"2023-09-23T18:28:03.292295+0000","last_active":"2023-09-23T18:28:03.292295+0000","last_peered":"2023-09-23T18:28:03.292295+0000","last_clean":"2023-09-23T18:28:03.292295+0000","last_became_active":"2023-09-23T18:28:03.291902+0000","last_became_peered":"2023-09-23T18:28:03.291902+0000","last_unstale":"2023-09-23T18:28:03.292295+0000","last_undegraded":"2023-09-23T18:28:03.292295+0000","last_fullsized":"2023-09-23T18:28:03.292295+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:38.415256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'181","reported_seq":591,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.282681+0000","last_change":"2023-09-23T18:27:04.256906+0000","last_active":"2023-09-23T18:28:03.282681+0000","last_peered":"2023-09-23T18:28:03.282681+0000","last_clean":"2023-09-23T18:28:03.282681+0000","last_became_active":"2023-09-23T18:27:04.255734+0000","last_became_peered":"2023-09-23T18:27:04.255734+0000","last_unstale":"2023-09-23T18:28:03.282681+0000","last_undegraded":"2023-09-23T18:28:03.282681+0000","last_fullsized":"2023-09-23T18:28:03.282681+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":181,"log_dups_size":0,"ondisk_log_size":181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:53.270831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":909,"num_read_kb":734,"num_write":148,"num_write_kb":75,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.298805+0000","last_change":"2023-09-23T18:28:03.298805+0000","last_active":"2023-09-23T18:28:03.298805+0000","last_peered":"2023-09-23T18:28:03.298805+0000","last_clean":"2023-09-23T18:28:03.298805+0000","last_became_active":"2023-09-23T18:28:03.298437+0000","last_became_peered":"2023-09-23T18:28:03.298437+0000","last_unstale":"2023-09-23T18:28:03.298805+0000","last_undegraded":"2023-09-23T18:28:03.298805+0000","last_fullsized":"2023-09-23T18:28:03.298805+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:27.831669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.279808+0000","last_change":"2023-09-23T18:26:34.990770+0000","last_active":"2023-09-23T18:28:03.279808+0000","last_peered":"2023-09-23T18:28:03.279808+0000","last_clean":"2023-09-23T18:28:03.279808+0000","last_became_active":"2023-09-23T18:26:34.990262+0000","last_became_peered":"2023-09-23T18:26:34.990262+0000","last_unstale":"2023-09-23T18:28:03.279808+0000","last_undegraded":"2023-09-23T18:28:03.279808+0000","last_fullsized":"2023-09-23T18:28:03.279808+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:07.982834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.281672+0000","last_change":"2023-09-23T18:27:33.579367+0000","last_active":"2023-09-23T18:28:03.281672+0000","last_peered":"2023-09-23T18:28:03.281672+0000","last_clean":"2023-09-23T18:28:03.281672+0000","last_became_active":"2023-09-23T18:27:33.578607+0000","last_became_peered":"2023-09-23T18:27:33.578607+0000","last_unstale":"2023-09-23T18:28:03.281672+0000","last_undegraded":"2023-09-23T18:28:03.281672+0000","last_fullsized":"2023-09-23T18:28:03.281672+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:58:35.628296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'78","reported_seq":340,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.286740+0000","last_change":"2023-09-23T18:26:34.992345+0000","last_active":"2023-09-23T18:28:02.286740+0000","last_peered":"2023-09-23T18:28:02.286740+0000","last_clean":"2023-09-23T18:28:02.286740+0000","last_became_active":"2023-09-23T18:26:34.991774+0000","last_became_peered":"2023-09-23T18:26:34.991774+0000","last_unstale":"2023-09-23T18:28:02.286740+0000","last_undegraded":"2023-09-23T18:28:02.286740+0000","last_fullsized":"2023-09-23T18:28:02.286740+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":78,"log_dups_size":0,"ondisk_log_size":78,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":428,"num_read_kb":1125,"num_write":73,"num_write_kb":431,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.281734+0000","last_change":"2023-09-23T18:27:33.577544+0000","last_active":"2023-09-23T18:28:03.281734+0000","last_peered":"2023-09-23T18:28:03.281734+0000","last_clean":"2023-09-23T18:28:03.281734+0000","last_became_active":"2023-09-23T18:27:33.577194+0000","last_became_peered":"2023-09-23T18:27:33.577194+0000","last_unstale":"2023-09-23T18:28:03.281734+0000","last_undegraded":"2023-09-23T18:28:03.281734+0000","last_fullsized":"2023-09-23T18:28:03.281734+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:49.633444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.299894+0000","last_change":"2023-09-23T18:28:03.299508+0000","last_active":"2023-09-23T18:28:03.299894+0000","last_peered":"2023-09-23T18:28:03.299894+0000","last_clean":"2023-09-23T18:28:03.299894+0000","last_became_active":"2023-09-23T18:28:03.298682+0000","last_became_peered":"2023-09-23T18:28:03.298682+0000","last_unstale":"2023-09-23T18:28:03.299894+0000","last_undegraded":"2023-09-23T18:28:03.299894+0000","last_fullsized":"2023-09-23T18:28:03.299894+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:08.398302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'120","reported_seq":550,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.282230+0000","last_change":"2023-09-23T18:27:04.257709+0000","last_active":"2023-09-23T18:28:02.282230+0000","last_peered":"2023-09-23T18:28:02.282230+0000","last_clean":"2023-09-23T18:28:02.282230+0000","last_became_active":"2023-09-23T18:27:04.257331+0000","last_became_peered":"2023-09-23T18:27:04.257331+0000","last_unstale":"2023-09-23T18:28:02.282230+0000","last_undegraded":"2023-09-23T18:28:02.282230+0000","last_fullsized":"2023-09-23T18:28:02.282230+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":120,"log_dups_size":0,"ondisk_log_size":120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":809,"num_read_kb":2749,"num_write":98,"num_write_kb":2087,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.283659+0000","last_change":"2023-09-23T18:15:08.921629+0000","last_active":"2023-09-23T18:28:02.283659+0000","last_peered":"2023-09-23T18:28:02.283659+0000","last_clean":"2023-09-23T18:28:02.283659+0000","last_became_active":"2023-09-23T18:15:08.921283+0000","last_became_peered":"2023-09-23T18:15:08.921283+0000","last_unstale":"2023-09-23T18:28:02.283659+0000","last_undegraded":"2023-09-23T18:28:02.283659+0000","last_fullsized":"2023-09-23T18:28:02.283659+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":62,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.295577+0000","last_change":"2023-09-23T18:28:03.295577+0000","last_active":"2023-09-23T18:28:03.295577+0000","last_peered":"2023-09-23T18:28:03.295577+0000","last_clean":"2023-09-23T18:28:03.295577+0000","last_became_active":"2023-09-23T18:28:03.293777+0000","last_became_peered":"2023-09-23T18:28:03.293777+0000","last_unstale":"2023-09-23T18:28:03.295577+0000","last_undegraded":"2023-09-23T18:28:03.295577+0000","last_fullsized":"2023-09-23T18:28:03.295577+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:49.714764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":306,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.282044+0000","last_change":"2023-09-23T18:26:34.999402+0000","last_active":"2023-09-23T18:28:02.282044+0000","last_peered":"2023-09-23T18:28:02.282044+0000","last_clean":"2023-09-23T18:28:02.282044+0000","last_became_active":"2023-09-23T18:26:34.998828+0000","last_became_peered":"2023-09-23T18:26:34.998828+0000","last_unstale":"2023-09-23T18:28:02.282044+0000","last_undegraded":"2023-09-23T18:28:02.282044+0000","last_fullsized":"2023-09-23T18:28:02.282044+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_clean_scrub_stamp":"2023-09-23T18:08:35.701374+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"77'147","reported_seq":503,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.277795+0000","last_change":"2023-09-23T18:27:33.583558+0000","last_active":"2023-09-23T18:28:02.277795+0000","last_peered":"2023-09-23T18:28:02.277795+0000","last_clean":"2023-09-23T18:28:02.277795+0000","last_became_active":"2023-09-23T18:27:33.583003+0000","last_became_peered":"2023-09-23T18:27:33.583003+0000","last_unstale":"2023-09-23T18:28:02.277795+0000","last_undegraded":"2023-09-23T18:28:02.277795+0000","last_fullsized":"2023-09-23T18:28:02.277795+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":625,"num_read_kb":1869,"num_write":143,"num_write_kb":5368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":78,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.282670+0000","last_change":"2023-09-23T18:27:04.242813+0000","last_active":"2023-09-23T18:28:03.282670+0000","last_peered":"2023-09-23T18:28:03.282670+0000","last_clean":"2023-09-23T18:28:03.282670+0000","last_became_active":"2023-09-23T18:27:04.242409+0000","last_became_peered":"2023-09-23T18:27:04.242409+0000","last_unstale":"2023-09-23T18:28:03.282670+0000","last_undegraded":"2023-09-23T18:28:03.282670+0000","last_fullsized":"2023-09-23T18:28:03.282670+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:01.836849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.275061+0000","last_change":"2023-09-23T18:27:33.585279+0000","last_active":"2023-09-23T18:28:03.275061+0000","last_peered":"2023-09-23T18:28:03.275061+0000","last_clean":"2023-09-23T18:28:03.275061+0000","last_became_active":"2023-09-23T18:27:33.584928+0000","last_became_peered":"2023-09-23T18:27:33.584928+0000","last_unstale":"2023-09-23T18:28:03.275061+0000","last_undegraded":"2023-09-23T18:28:03.275061+0000","last_fullsized":"2023-09-23T18:28:03.275061+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:07.000233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'112","reported_seq":539,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.297866+0000","last_change":"2023-09-23T18:28:03.297540+0000","last_active":"2023-09-23T18:28:03.297866+0000","last_peered":"2023-09-23T18:28:03.297866+0000","last_clean":"2023-09-23T18:28:03.297866+0000","last_became_active":"2023-09-23T18:28:03.296649+0000","last_became_peered":"2023-09-23T18:28:03.296649+0000","last_unstale":"2023-09-23T18:28:03.297866+0000","last_undegraded":"2023-09-23T18:28:03.297866+0000","last_fullsized":"2023-09-23T18:28:03.297866+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:32.531775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":774,"num_read_kb":596,"num_write":102,"num_write_kb":52,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.289185+0000","last_change":"2023-09-23T18:28:03.288707+0000","last_active":"2023-09-23T18:28:03.289185+0000","last_peered":"2023-09-23T18:28:03.289185+0000","last_clean":"2023-09-23T18:28:03.289185+0000","last_became_active":"2023-09-23T18:28:03.288335+0000","last_became_peered":"2023-09-23T18:28:03.288335+0000","last_unstale":"2023-09-23T18:28:03.289185+0000","last_undegraded":"2023-09-23T18:28:03.289185+0000","last_fullsized":"2023-09-23T18:28:03.289185+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:57.652706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.292438+0000","last_change":"2023-09-23T18:26:34.992535+0000","last_active":"2023-09-23T18:28:02.292438+0000","last_peered":"2023-09-23T18:28:02.292438+0000","last_clean":"2023-09-23T18:28:02.292438+0000","last_became_active":"2023-09-23T18:26:34.992177+0000","last_became_peered":"2023-09-23T18:26:34.992177+0000","last_unstale":"2023-09-23T18:28:02.292438+0000","last_undegraded":"2023-09-23T18:28:02.292438+0000","last_fullsized":"2023-09-23T18:28:02.292438+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"77'96","reported_seq":619,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.277710+0000","last_change":"2023-09-23T18:27:04.250427+0000","last_active":"2023-09-23T18:28:02.277710+0000","last_peered":"2023-09-23T18:28:02.277710+0000","last_clean":"2023-09-23T18:28:02.277710+0000","last_became_active":"2023-09-23T18:27:04.249907+0000","last_became_peered":"2023-09-23T18:27:04.249907+0000","last_unstale":"2023-09-23T18:28:02.277710+0000","last_undegraded":"2023-09-23T18:28:02.277710+0000","last_fullsized":"2023-09-23T18:28:02.277710+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":96,"log_dups_size":0,"ondisk_log_size":96,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":785,"num_read_kb":1291,"num_write":87,"num_write_kb":338,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.275431+0000","last_change":"2023-09-23T18:27:04.251552+0000","last_active":"2023-09-23T18:28:03.275431+0000","last_peered":"2023-09-23T18:28:03.275431+0000","last_clean":"2023-09-23T18:28:03.275431+0000","last_became_active":"2023-09-23T18:27:04.250901+0000","last_became_peered":"2023-09-23T18:27:04.250901+0000","last_unstale":"2023-09-23T18:28:03.275431+0000","last_undegraded":"2023-09-23T18:28:03.275431+0000","last_fullsized":"2023-09-23T18:28:03.275431+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:22.322137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.297594+0000","last_change":"2023-09-23T18:28:03.297594+0000","last_active":"2023-09-23T18:28:03.297594+0000","last_peered":"2023-09-23T18:28:03.297594+0000","last_clean":"2023-09-23T18:28:03.297594+0000","last_became_active":"2023-09-23T18:28:03.296725+0000","last_became_peered":"2023-09-23T18:28:03.296725+0000","last_unstale":"2023-09-23T18:28:03.297594+0000","last_undegraded":"2023-09-23T18:28:03.297594+0000","last_fullsized":"2023-09-23T18:28:03.297594+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'130","reported_seq":583,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.281567+0000","last_change":"2023-09-23T18:27:04.250265+0000","last_active":"2023-09-23T18:28:03.281567+0000","last_peered":"2023-09-23T18:28:03.281567+0000","last_clean":"2023-09-23T18:28:03.281567+0000","last_became_active":"2023-09-23T18:27:04.249097+0000","last_became_peered":"2023-09-23T18:27:04.249097+0000","last_unstale":"2023-09-23T18:28:03.281567+0000","last_undegraded":"2023-09-23T18:28:03.281567+0000","last_fullsized":"2023-09-23T18:28:03.281567+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:27.143225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":772,"num_read_kb":1712,"num_write":126,"num_write_kb":2126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.295387+0000","last_change":"2023-09-23T18:28:03.294782+0000","last_active":"2023-09-23T18:28:03.295387+0000","last_peered":"2023-09-23T18:28:03.295387+0000","last_clean":"2023-09-23T18:28:03.295387+0000","last_became_active":"2023-09-23T18:28:03.294430+0000","last_became_peered":"2023-09-23T18:28:03.294430+0000","last_unstale":"2023-09-23T18:28:03.295387+0000","last_undegraded":"2023-09-23T18:28:03.295387+0000","last_fullsized":"2023-09-23T18:28:03.295387+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:20.226197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.287471+0000","last_change":"2023-09-23T18:28:03.287471+0000","last_active":"2023-09-23T18:28:03.287471+0000","last_peered":"2023-09-23T18:28:03.287471+0000","last_clean":"2023-09-23T18:28:03.287471+0000","last_became_active":"2023-09-23T18:28:03.286923+0000","last_became_peered":"2023-09-23T18:28:03.286923+0000","last_unstale":"2023-09-23T18:28:03.287471+0000","last_undegraded":"2023-09-23T18:28:03.287471+0000","last_fullsized":"2023-09-23T18:28:03.287471+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:10.505004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.291109+0000","last_change":"2023-09-23T18:27:04.254684+0000","last_active":"2023-09-23T18:28:02.291109+0000","last_peered":"2023-09-23T18:28:02.291109+0000","last_clean":"2023-09-23T18:28:02.291109+0000","last_became_active":"2023-09-23T18:27:04.254014+0000","last_became_peered":"2023-09-23T18:27:04.254014+0000","last_unstale":"2023-09-23T18:28:02.291109+0000","last_undegraded":"2023-09-23T18:28:02.291109+0000","last_fullsized":"2023-09-23T18:28:02.291109+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.293127+0000","last_change":"2023-09-23T18:27:33.585714+0000","last_active":"2023-09-23T18:28:02.293127+0000","last_peered":"2023-09-23T18:28:02.293127+0000","last_clean":"2023-09-23T18:28:02.293127+0000","last_became_active":"2023-09-23T18:27:33.584641+0000","last_became_peered":"2023-09-23T18:27:33.584641+0000","last_unstale":"2023-09-23T18:28:02.293127+0000","last_undegraded":"2023-09-23T18:28:02.293127+0000","last_fullsized":"2023-09-23T18:28:02.293127+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":84,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.291696+0000","last_change":"2023-09-23T18:27:33.588308+0000","last_active":"2023-09-23T18:28:02.291696+0000","last_peered":"2023-09-23T18:28:02.291696+0000","last_clean":"2023-09-23T18:28:02.291696+0000","last_became_active":"2023-09-23T18:27:33.587974+0000","last_became_peered":"2023-09-23T18:27:33.587974+0000","last_unstale":"2023-09-23T18:28:02.291696+0000","last_undegraded":"2023-09-23T18:28:02.291696+0000","last_fullsized":"2023-09-23T18:28:02.291696+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.284595+0000","last_change":"2023-09-23T18:27:04.252119+0000","last_active":"2023-09-23T18:28:02.284595+0000","last_peered":"2023-09-23T18:28:02.284595+0000","last_clean":"2023-09-23T18:28:02.284595+0000","last_became_active":"2023-09-23T18:27:04.251714+0000","last_became_peered":"2023-09-23T18:27:04.251714+0000","last_unstale":"2023-09-23T18:28:02.284595+0000","last_undegraded":"2023-09-23T18:28:02.284595+0000","last_fullsized":"2023-09-23T18:28:02.284595+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.302326+0000","last_change":"2023-09-23T18:28:03.302326+0000","last_active":"2023-09-23T18:28:03.302326+0000","last_peered":"2023-09-23T18:28:03.302326+0000","last_clean":"2023-09-23T18:28:03.302326+0000","last_became_active":"2023-09-23T18:28:03.301970+0000","last_became_peered":"2023-09-23T18:28:03.301970+0000","last_unstale":"2023-09-23T18:28:03.302326+0000","last_undegraded":"2023-09-23T18:28:03.302326+0000","last_fullsized":"2023-09-23T18:28:03.302326+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.297628+0000","last_change":"2023-09-23T18:28:03.297628+0000","last_active":"2023-09-23T18:28:03.297628+0000","last_peered":"2023-09-23T18:28:03.297628+0000","last_clean":"2023-09-23T18:28:03.297628+0000","last_became_active":"2023-09-23T18:28:03.296694+0000","last_became_peered":"2023-09-23T18:28:03.296694+0000","last_unstale":"2023-09-23T18:28:03.297628+0000","last_undegraded":"2023-09-23T18:28:03.297628+0000","last_fullsized":"2023-09-23T18:28:03.297628+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:43.375652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":65,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.280061+0000","last_change":"2023-09-23T18:27:33.582033+0000","last_active":"2023-09-23T18:28:03.280061+0000","last_peered":"2023-09-23T18:28:03.280061+0000","last_clean":"2023-09-23T18:28:03.280061+0000","last_became_active":"2023-09-23T18:27:33.581667+0000","last_became_peered":"2023-09-23T18:27:33.581667+0000","last_unstale":"2023-09-23T18:28:03.280061+0000","last_undegraded":"2023-09-23T18:28:03.280061+0000","last_fullsized":"2023-09-23T18:28:03.280061+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:36.039922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.298611+0000","last_change":"2023-09-23T18:28:03.298611+0000","last_active":"2023-09-23T18:28:03.298611+0000","last_peered":"2023-09-23T18:28:03.298611+0000","last_clean":"2023-09-23T18:28:03.298611+0000","last_became_active":"2023-09-23T18:28:03.298233+0000","last_became_peered":"2023-09-23T18:28:03.298233+0000","last_unstale":"2023-09-23T18:28:03.298611+0000","last_undegraded":"2023-09-23T18:28:03.298611+0000","last_fullsized":"2023-09-23T18:28:03.298611+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":78,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.284278+0000","last_change":"2023-09-23T18:27:33.589116+0000","last_active":"2023-09-23T18:28:02.284278+0000","last_peered":"2023-09-23T18:28:02.284278+0000","last_clean":"2023-09-23T18:28:02.284278+0000","last_became_active":"2023-09-23T18:27:33.588809+0000","last_became_peered":"2023-09-23T18:27:33.588809+0000","last_unstale":"2023-09-23T18:28:02.284278+0000","last_undegraded":"2023-09-23T18:28:02.284278+0000","last_fullsized":"2023-09-23T18:28:02.284278+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":56,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.286875+0000","last_change":"2023-09-23T18:15:08.908376+0000","last_active":"2023-09-23T18:28:02.286875+0000","last_peered":"2023-09-23T18:28:02.286875+0000","last_clean":"2023-09-23T18:28:02.286875+0000","last_became_active":"2023-09-23T18:15:08.907752+0000","last_became_peered":"2023-09-23T18:15:08.907752+0000","last_unstale":"2023-09-23T18:28:02.286875+0000","last_undegraded":"2023-09-23T18:28:02.286875+0000","last_fullsized":"2023-09-23T18:28:02.286875+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.302292+0000","last_change":"2023-09-23T18:28:03.302292+0000","last_active":"2023-09-23T18:28:03.302292+0000","last_peered":"2023-09-23T18:28:03.302292+0000","last_clean":"2023-09-23T18:28:03.302292+0000","last_became_active":"2023-09-23T18:28:03.301927+0000","last_became_peered":"2023-09-23T18:28:03.301927+0000","last_unstale":"2023-09-23T18:28:03.302292+0000","last_undegraded":"2023-09-23T18:28:03.302292+0000","last_fullsized":"2023-09-23T18:28:03.302292+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.286163+0000","last_change":"2023-09-23T18:27:04.253991+0000","last_active":"2023-09-23T18:28:02.286163+0000","last_peered":"2023-09-23T18:28:02.286163+0000","last_clean":"2023-09-23T18:28:02.286163+0000","last_became_active":"2023-09-23T18:27:04.253593+0000","last_became_peered":"2023-09-23T18:27:04.253593+0000","last_unstale":"2023-09-23T18:28:02.286163+0000","last_undegraded":"2023-09-23T18:28:02.286163+0000","last_fullsized":"2023-09-23T18:28:02.286163+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.277381+0000","last_change":"2023-09-23T18:15:10.327919+0000","last_active":"2023-09-23T18:28:02.277381+0000","last_peered":"2023-09-23T18:28:02.277381+0000","last_clean":"2023-09-23T18:28:02.277381+0000","last_became_active":"2023-09-23T18:15:10.327052+0000","last_became_peered":"2023-09-23T18:15:10.327052+0000","last_unstale":"2023-09-23T18:28:02.277381+0000","last_undegraded":"2023-09-23T18:28:02.277381+0000","last_fullsized":"2023-09-23T18:28:02.277381+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.286482+0000","last_change":"2023-09-23T18:26:34.999684+0000","last_active":"2023-09-23T18:28:02.286482+0000","last_peered":"2023-09-23T18:28:02.286482+0000","last_clean":"2023-09-23T18:28:02.286482+0000","last_became_active":"2023-09-23T18:26:34.999076+0000","last_became_peered":"2023-09-23T18:26:34.999076+0000","last_unstale":"2023-09-23T18:28:02.286482+0000","last_undegraded":"2023-09-23T18:28:02.286482+0000","last_fullsized":"2023-09-23T18:28:02.286482+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.295838+0000","last_change":"2023-09-23T18:28:03.295838+0000","last_active":"2023-09-23T18:28:03.295838+0000","last_peered":"2023-09-23T18:28:03.295838+0000","last_clean":"2023-09-23T18:28:03.295838+0000","last_became_active":"2023-09-23T18:28:03.291774+0000","last_became_peered":"2023-09-23T18:28:03.291774+0000","last_unstale":"2023-09-23T18:28:03.295838+0000","last_undegraded":"2023-09-23T18:28:03.295838+0000","last_fullsized":"2023-09-23T18:28:03.295838+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:23.166353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":78,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.284152+0000","last_change":"2023-09-23T18:26:34.995333+0000","last_active":"2023-09-23T18:28:02.284152+0000","last_peered":"2023-09-23T18:28:02.284152+0000","last_clean":"2023-09-23T18:28:02.284152+0000","last_became_active":"2023-09-23T18:26:34.994983+0000","last_became_peered":"2023-09-23T18:26:34.994983+0000","last_unstale":"2023-09-23T18:28:02.284152+0000","last_undegraded":"2023-09-23T18:28:02.284152+0000","last_fullsized":"2023-09-23T18:28:02.284152+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.290583+0000","last_change":"2023-09-23T18:28:03.290217+0000","last_active":"2023-09-23T18:28:03.290583+0000","last_peered":"2023-09-23T18:28:03.290583+0000","last_clean":"2023-09-23T18:28:03.290583+0000","last_became_active":"2023-09-23T18:28:03.289888+0000","last_became_peered":"2023-09-23T18:28:03.289888+0000","last_unstale":"2023-09-23T18:28:03.290583+0000","last_undegraded":"2023-09-23T18:28:03.290583+0000","last_fullsized":"2023-09-23T18:28:03.290583+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:51:45.400847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.281244+0000","last_change":"2023-09-23T18:15:08.916658+0000","last_active":"2023-09-23T18:28:02.281244+0000","last_peered":"2023-09-23T18:28:02.281244+0000","last_clean":"2023-09-23T18:28:02.281244+0000","last_became_active":"2023-09-23T18:15:08.916301+0000","last_became_peered":"2023-09-23T18:15:08.916301+0000","last_unstale":"2023-09-23T18:28:02.281244+0000","last_undegraded":"2023-09-23T18:28:02.281244+0000","last_fullsized":"2023-09-23T18:28:02.281244+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.297841+0000","last_change":"2023-09-23T18:28:03.297841+0000","last_active":"2023-09-23T18:28:03.297841+0000","last_peered":"2023-09-23T18:28:03.297841+0000","last_clean":"2023-09-23T18:28:03.297841+0000","last_became_active":"2023-09-23T18:28:03.297216+0000","last_became_peered":"2023-09-23T18:28:03.297216+0000","last_unstale":"2023-09-23T18:28:03.297841+0000","last_undegraded":"2023-09-23T18:28:03.297841+0000","last_fullsized":"2023-09-23T18:28:03.297841+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.290591+0000","last_change":"2023-09-23T18:28:03.290316+0000","last_active":"2023-09-23T18:28:03.290591+0000","last_peered":"2023-09-23T18:28:03.290591+0000","last_clean":"2023-09-23T18:28:03.290591+0000","last_became_active":"2023-09-23T18:28:03.289698+0000","last_became_peered":"2023-09-23T18:28:03.289698+0000","last_unstale":"2023-09-23T18:28:03.290591+0000","last_undegraded":"2023-09-23T18:28:03.290591+0000","last_fullsized":"2023-09-23T18:28:03.290591+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:00.779542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.292363+0000","last_change":"2023-09-23T18:28:03.292363+0000","last_active":"2023-09-23T18:28:03.292363+0000","last_peered":"2023-09-23T18:28:03.292363+0000","last_clean":"2023-09-23T18:28:03.292363+0000","last_became_active":"2023-09-23T18:28:03.291796+0000","last_became_peered":"2023-09-23T18:28:03.291796+0000","last_unstale":"2023-09-23T18:28:03.292363+0000","last_undegraded":"2023-09-23T18:28:03.292363+0000","last_fullsized":"2023-09-23T18:28:03.292363+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:13:35.340945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.283954+0000","last_change":"2023-09-23T18:27:33.585121+0000","last_active":"2023-09-23T18:28:02.283954+0000","last_peered":"2023-09-23T18:28:02.283954+0000","last_clean":"2023-09-23T18:28:02.283954+0000","last_became_active":"2023-09-23T18:27:33.584620+0000","last_became_peered":"2023-09-23T18:27:33.584620+0000","last_unstale":"2023-09-23T18:28:02.283954+0000","last_undegraded":"2023-09-23T18:28:02.283954+0000","last_fullsized":"2023-09-23T18:28:02.283954+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.290588+0000","last_change":"2023-09-23T18:27:04.256030+0000","last_active":"2023-09-23T18:28:02.290588+0000","last_peered":"2023-09-23T18:28:02.290588+0000","last_clean":"2023-09-23T18:28:02.290588+0000","last_became_active":"2023-09-23T18:27:04.255549+0000","last_became_peered":"2023-09-23T18:27:04.255549+0000","last_unstale":"2023-09-23T18:28:02.290588+0000","last_undegraded":"2023-09-23T18:28:02.290588+0000","last_fullsized":"2023-09-23T18:28:02.290588+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":77,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.292732+0000","last_change":"2023-09-23T18:15:08.909339+0000","last_active":"2023-09-23T18:28:02.292732+0000","last_peered":"2023-09-23T18:28:02.292732+0000","last_clean":"2023-09-23T18:28:02.292732+0000","last_became_active":"2023-09-23T18:15:08.908912+0000","last_became_peered":"2023-09-23T18:15:08.908912+0000","last_unstale":"2023-09-23T18:28:02.292732+0000","last_undegraded":"2023-09-23T18:28:02.292732+0000","last_fullsized":"2023-09-23T18:28:02.292732+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.275687+0000","last_change":"2023-09-23T18:27:33.585072+0000","last_active":"2023-09-23T18:28:03.275687+0000","last_peered":"2023-09-23T18:28:03.275687+0000","last_clean":"2023-09-23T18:28:03.275687+0000","last_became_active":"2023-09-23T18:27:33.584118+0000","last_became_peered":"2023-09-23T18:27:33.584118+0000","last_unstale":"2023-09-23T18:28:03.275687+0000","last_undegraded":"2023-09-23T18:28:03.275687+0000","last_fullsized":"2023-09-23T18:28:03.275687+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:30:12.156524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.296913+0000","last_change":"2023-09-23T18:28:03.296913+0000","last_active":"2023-09-23T18:28:03.296913+0000","last_peered":"2023-09-23T18:28:03.296913+0000","last_clean":"2023-09-23T18:28:03.296913+0000","last_became_active":"2023-09-23T18:28:03.296499+0000","last_became_peered":"2023-09-23T18:28:03.296499+0000","last_unstale":"2023-09-23T18:28:03.296913+0000","last_undegraded":"2023-09-23T18:28:03.296913+0000","last_fullsized":"2023-09-23T18:28:03.296913+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":61,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.298214+0000","last_change":"2023-09-23T18:28:03.298214+0000","last_active":"2023-09-23T18:28:03.298214+0000","last_peered":"2023-09-23T18:28:03.298214+0000","last_clean":"2023-09-23T18:28:03.298214+0000","last_became_active":"2023-09-23T18:28:03.297852+0000","last_became_peered":"2023-09-23T18:28:03.297852+0000","last_unstale":"2023-09-23T18:28:03.298214+0000","last_undegraded":"2023-09-23T18:28:03.298214+0000","last_fullsized":"2023-09-23T18:28:03.298214+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.146541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.278178+0000","last_change":"2023-09-23T18:27:04.250764+0000","last_active":"2023-09-23T18:28:02.278178+0000","last_peered":"2023-09-23T18:28:02.278178+0000","last_clean":"2023-09-23T18:28:02.278178+0000","last_became_active":"2023-09-23T18:27:04.250360+0000","last_became_peered":"2023-09-23T18:27:04.250360+0000","last_unstale":"2023-09-23T18:28:02.278178+0000","last_undegraded":"2023-09-23T18:28:02.278178+0000","last_fullsized":"2023-09-23T18:28:02.278178+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.295734+0000","last_change":"2023-09-23T18:28:03.295734+0000","last_active":"2023-09-23T18:28:03.295734+0000","last_peered":"2023-09-23T18:28:03.295734+0000","last_clean":"2023-09-23T18:28:03.295734+0000","last_became_active":"2023-09-23T18:28:03.293761+0000","last_became_peered":"2023-09-23T18:28:03.293761+0000","last_unstale":"2023-09-23T18:28:03.295734+0000","last_undegraded":"2023-09-23T18:28:03.295734+0000","last_fullsized":"2023-09-23T18:28:03.295734+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:55.285705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.297725+0000","last_change":"2023-09-23T18:28:03.297439+0000","last_active":"2023-09-23T18:28:03.297725+0000","last_peered":"2023-09-23T18:28:03.297725+0000","last_clean":"2023-09-23T18:28:03.297725+0000","last_became_active":"2023-09-23T18:28:03.297112+0000","last_became_peered":"2023-09-23T18:28:03.297112+0000","last_unstale":"2023-09-23T18:28:03.297725+0000","last_undegraded":"2023-09-23T18:28:03.297725+0000","last_fullsized":"2023-09-23T18:28:03.297725+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:50.442238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.284400+0000","last_change":"2023-09-23T18:27:04.247225+0000","last_active":"2023-09-23T18:28:02.284400+0000","last_peered":"2023-09-23T18:28:02.284400+0000","last_clean":"2023-09-23T18:28:02.284400+0000","last_became_active":"2023-09-23T18:27:04.246843+0000","last_became_peered":"2023-09-23T18:27:04.246843+0000","last_unstale":"2023-09-23T18:28:02.284400+0000","last_undegraded":"2023-09-23T18:28:02.284400+0000","last_fullsized":"2023-09-23T18:28:02.284400+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":87,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.287830+0000","last_change":"2023-09-23T18:28:03.287830+0000","last_active":"2023-09-23T18:28:03.287830+0000","last_peered":"2023-09-23T18:28:03.287830+0000","last_clean":"2023-09-23T18:28:03.287830+0000","last_became_active":"2023-09-23T18:28:03.287461+0000","last_became_peered":"2023-09-23T18:28:03.287461+0000","last_unstale":"2023-09-23T18:28:03.287830+0000","last_undegraded":"2023-09-23T18:28:03.287830+0000","last_fullsized":"2023-09-23T18:28:03.287830+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:21:51.267964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.290067+0000","last_change":"2023-09-23T18:28:03.289500+0000","last_active":"2023-09-23T18:28:03.290067+0000","last_peered":"2023-09-23T18:28:03.290067+0000","last_clean":"2023-09-23T18:28:03.290067+0000","last_became_active":"2023-09-23T18:28:03.289122+0000","last_became_peered":"2023-09-23T18:28:03.289122+0000","last_unstale":"2023-09-23T18:28:03.290067+0000","last_undegraded":"2023-09-23T18:28:03.290067+0000","last_fullsized":"2023-09-23T18:28:03.290067+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:22:26.904660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.295399+0000","last_change":"2023-09-23T18:28:03.295009+0000","last_active":"2023-09-23T18:28:03.295399+0000","last_peered":"2023-09-23T18:28:03.295399+0000","last_clean":"2023-09-23T18:28:03.295399+0000","last_became_active":"2023-09-23T18:28:03.294501+0000","last_became_peered":"2023-09-23T18:28:03.294501+0000","last_unstale":"2023-09-23T18:28:03.295399+0000","last_undegraded":"2023-09-23T18:28:03.295399+0000","last_fullsized":"2023-09-23T18:28:03.295399+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:00.671680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.299495+0000","last_change":"2023-09-23T18:28:03.299495+0000","last_active":"2023-09-23T18:28:03.299495+0000","last_peered":"2023-09-23T18:28:03.299495+0000","last_clean":"2023-09-23T18:28:03.299495+0000","last_became_active":"2023-09-23T18:28:03.297293+0000","last_became_peered":"2023-09-23T18:28:03.297293+0000","last_unstale":"2023-09-23T18:28:03.299495+0000","last_undegraded":"2023-09-23T18:28:03.299495+0000","last_fullsized":"2023-09-23T18:28:03.299495+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:01.235861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":86,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.291372+0000","last_change":"2023-09-23T18:27:33.588552+0000","last_active":"2023-09-23T18:28:02.291372+0000","last_peered":"2023-09-23T18:28:02.291372+0000","last_clean":"2023-09-23T18:28:02.291372+0000","last_became_active":"2023-09-23T18:27:33.588118+0000","last_became_peered":"2023-09-23T18:27:33.588118+0000","last_unstale":"2023-09-23T18:28:02.291372+0000","last_undegraded":"2023-09-23T18:28:02.291372+0000","last_fullsized":"2023-09-23T18:28:02.291372+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26886,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5621,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430424,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306928,"kb_used_data":5736,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93438224,"statfs":{"total":95995035648,"available":95680741376,"internally_reserved":0,"allocated":5873664,"data_stored":5365985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:29:42 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.91900000000000004,"15min":0.92700000000000005},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.2909999999999999,"5min":1.4670000000000001,"15min":1.4670000000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.93700000000000006,"15min":0.95899999999999996},"min":{"1min":0.69899999999999995,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.175,"5min":1.302,"15min":1.302},"last":0.85299999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:29:05 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.91300000000000003,"15min":0.91600000000000004},"min":{"1min":0.61199999999999999,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.1080000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":1.014},{"interface":"front","average":{"1min":1.028,"5min":0.97599999999999998,"15min":0.96799999999999997},"min":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":0.76000000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:29:36 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"min":{"1min":0.71099999999999997,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":1.2529999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":0.89900000000000002},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.94899999999999995,"15min":0.95499999999999996},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.2989999999999999,"5min":1.3380000000000001,"15min":1.3380000000000001},"last":0.92100000000000004}]},{"osd":3,"last update":"Sat Sep 23 18:29:02 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":0.999,"15min":0.999},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.383,"5min":1.383,"15min":1.383},"last":0.83999999999999997},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"min":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"max":{"1min":1.115,"5min":1.115,"15min":1.115},"last":1.048}]},{"osd":4,"last update":"Sat Sep 23 18:29:50 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.92400000000000004,"15min":0.88900000000000001},"min":{"1min":0.625,"5min":0.60099999999999998,"15min":0.371},"max":{"1min":1.214,"5min":1.2589999999999999,"15min":1.3839999999999999},"last":1.0740000000000001},{"interface":"front","average":{"1min":1.028,"5min":1.0469999999999999,"15min":1.028},"min":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.50700000000000001},"max":{"1min":1.341,"5min":1.4430000000000001,"15min":1.4430000000000001},"last":0.99399999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:29:50 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.94099999999999995,"15min":0.95699999999999996},"min":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.505},"max":{"1min":1.298,"5min":1.298,"15min":1.4470000000000001},"last":0.95099999999999996},{"interface":"front","average":{"1min":1.042,"5min":1.0580000000000001,"15min":1.0129999999999999},"min":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.40899999999999997},"max":{"1min":1.2889999999999999,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":0.93700000000000006}]},{"osd":6,"last update":"Sat Sep 23 18:29:50 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":1.0980000000000001,"15min":1.05},"min":{"1min":0.95999999999999996,"5min":0.72699999999999998,"15min":0.442},"max":{"1min":1.3560000000000001,"5min":1.4399999999999999,"15min":1.4690000000000001},"last":1.3200000000000001},{"interface":"front","average":{"1min":1.095,"5min":1.145,"15min":1.131},"min":{"1min":0.67400000000000004,"5min":0.63600000000000001,"15min":0.53200000000000003},"max":{"1min":1.3380000000000001,"5min":1.548,"15min":1.548},"last":0.88100000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626656,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302820,"kb_used_data":1628,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93442332,"statfs":{"total":95995035648,"available":95684947968,"internally_reserved":0,"allocated":1667072,"data_stored":1171239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:29:40 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.91600000000000004,"15min":0.91000000000000003},"min":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.244,"5min":1.2470000000000001,"15min":1.2470000000000001},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.99399999999999999,"5min":1.0409999999999999,"15min":1.0449999999999999},"min":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"max":{"1min":1.1850000000000001,"5min":1.3540000000000001,"15min":1.3540000000000001},"last":0.94999999999999996}]},{"osd":1,"last update":"Sat Sep 23 18:29:07 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.90700000000000003,"15min":0.90300000000000002},"min":{"1min":0.61699999999999999,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.2210000000000001,"5min":1.2210000000000001,"15min":1.2210000000000001},"last":0.85799999999999998},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.92900000000000005,"15min":0.92500000000000004},"min":{"1min":0.71799999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.2470000000000001,"5min":1.2470000000000001,"15min":1.2470000000000001},"last":1.1439999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.999,"15min":1.0049999999999999},"min":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"max":{"1min":1.2789999999999999,"5min":1.2789999999999999,"15min":1.2789999999999999},"last":1.202},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.95699999999999996,"15min":0.95899999999999996},"min":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.196,"5min":1.196,"15min":1.196},"last":1.177}]},{"osd":3,"last update":"Sat Sep 23 18:29:04 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"min":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.3069999999999999,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":1.3260000000000001},{"interface":"front","average":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"min":{"1min":0.745,"5min":0.745,"15min":0.745},"max":{"1min":1.288,"5min":1.288,"15min":1.288},"last":1.089}]},{"osd":4,"last update":"Sat Sep 23 18:28:54 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.90400000000000003,"15min":0.91700000000000004},"min":{"1min":0.64500000000000002,"5min":0.53400000000000003,"15min":0.44800000000000001},"max":{"1min":1.339,"5min":1.339,"15min":1.4790000000000001},"last":1.024},{"interface":"front","average":{"1min":0.98599999999999999,"5min":0.98399999999999999,"15min":0.97199999999999998},"min":{"1min":0.68300000000000005,"5min":0.56499999999999995,"15min":0.41199999999999998},"max":{"1min":1.4299999999999999,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":1.248}]},{"osd":5,"last update":"Sat Sep 23 18:28:54 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.008,"15min":1.018},"min":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.36699999999999999},"max":{"1min":1.3819999999999999,"5min":1.431,"15min":1.6160000000000001},"last":1.121},{"interface":"front","average":{"1min":1.056,"5min":1.0429999999999999,"15min":1.048},"min":{"1min":0.66700000000000004,"5min":0.59199999999999997,"15min":0.50900000000000001},"max":{"1min":1.504,"5min":1.504,"15min":1.5700000000000001},"last":1.403}]},{"osd":7,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.056,"15min":1.0760000000000001},"min":{"1min":0.76000000000000001,"5min":0.59499999999999997,"15min":0.48999999999999999},"max":{"1min":1.274,"5min":1.764,"15min":1.764},"last":1.2829999999999999},{"interface":"front","average":{"1min":1.173,"5min":1.133,"15min":1.1339999999999999},"min":{"1min":0.83399999999999996,"5min":0.64400000000000002,"15min":0.499},"max":{"1min":1.3759999999999999,"5min":1.6659999999999999,"15min":1.6659999999999999},"last":1.2629999999999999}]}]},{"osd":1,"up_from":93,"seq":399431958563,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32648,"kb_used_data":1088,"kb_used_omap":7,"kb_used_meta":31544,"kb_avail":93712504,"statfs":{"total":95995035648,"available":95961604096,"internally_reserved":0,"allocated":1114112,"data_stored":586888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":32301838},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:29:08 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.66800000000000004,"15min":0.67600000000000005},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.83999999999999997,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.72299999999999998,"15min":0.73499999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.88900000000000001,"5min":1.0960000000000001,"15min":1.0960000000000001},"last":0.65300000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:29:40 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.60099999999999998,"15min":0.59899999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":1.1000000000000001,"5min":1.1000000000000001,"15min":1.1000000000000001},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.71399999999999997,"15min":0.71999999999999997},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.0449999999999999,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.88200000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:29:02 2023","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":0.753,"15min":0.753},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.157,"5min":1.157,"15min":1.157},"last":0.80600000000000005},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.294,"5min":1.294,"15min":1.294},"last":0.68000000000000005}]},{"osd":4,"last update":"Sat Sep 23 18:29:08 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.92600000000000005,"15min":0.92100000000000004},"min":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.204,"5min":1.204,"15min":1.204},"last":1.0940000000000001},{"interface":"front","average":{"1min":0.874,"5min":1.0129999999999999,"15min":1.036},"min":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.2969999999999999,"5min":1.4950000000000001,"15min":1.4950000000000001},"last":0.86299999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:29:08 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":0.98199999999999998,"15min":0.999},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.143,"5min":1.5149999999999999,"15min":1.5149999999999999},"last":0.84499999999999997},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.97899999999999998,"15min":0.98499999999999999},"min":{"1min":0.65100000000000002,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.294,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":1.244}]},{"osd":6,"last update":"Sat Sep 23 18:29:08 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":1.018,"15min":1.03},"min":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"max":{"1min":1.2270000000000001,"5min":1.6100000000000001,"15min":1.6100000000000001},"last":1.028},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.0649999999999999,"15min":1.0740000000000001},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.3600000000000001,"5min":1.4419999999999999,"15min":1.4419999999999999},"last":1.319}]},{"osd":7,"last update":"Sat Sep 23 18:29:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.095,"15min":1.109},"min":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.4019999999999999,"5min":1.55,"15min":1.55},"last":1.173},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.1120000000000001,"15min":1.1339999999999999},"min":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.51,"5min":1.575,"15min":1.575},"last":0.98099999999999998}]}]},{"osd":0,"up_from":88,"seq":377957122089,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33240,"kb_used_data":1680,"kb_used_omap":8,"kb_used_meta":31543,"kb_avail":93711912,"statfs":{"total":95995035648,"available":95960997888,"internally_reserved":0,"allocated":1720320,"data_stored":1178335,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":32300926},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:29:08 2023","interfaces":[{"interface":"back","average":{"1min":0.71799999999999997,"5min":0.65800000000000003,"15min":0.64800000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.2170000000000001,"5min":1.2170000000000001,"15min":1.2170000000000001},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.83399999999999996,"5min":0.80800000000000005,"15min":0.80400000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.3480000000000001,"5min":1.3480000000000001,"15min":1.3480000000000001},"last":0.70999999999999996}]},{"osd":2,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.80500000000000005,"15min":0.82399999999999995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.202,"5min":1.254,"15min":1.254},"last":0.88600000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.67000000000000004,"15min":0.68300000000000005},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.90100000000000002,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.86099999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:29:02 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.2849999999999999,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":0.91400000000000003},{"interface":"front","average":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.3500000000000001,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.64300000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:29:41 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.97299999999999998,"15min":0.97399999999999998},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.4970000000000001,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":0.80600000000000005},{"interface":"front","average":{"1min":0.90700000000000003,"5min":0.98399999999999999,"15min":1.0089999999999999},"min":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.29,"5min":1.29,"15min":1.29},"last":1.014}]},{"osd":5,"last update":"Sat Sep 23 18:29:41 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.99099999999999999,"15min":1.0149999999999999},"min":{"1min":0.68000000000000005,"5min":0.67200000000000004,"15min":0.67200000000000004},"max":{"1min":1.23,"5min":1.524,"15min":1.524},"last":0.83099999999999996},{"interface":"front","average":{"1min":0.96399999999999997,"5min":1.0129999999999999,"15min":1.0349999999999999},"min":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"max":{"1min":1.544,"5min":1.544,"15min":1.544},"last":1.22}]},{"osd":6,"last update":"Sat Sep 23 18:29:41 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":1.036,"15min":1.0640000000000001},"min":{"1min":0.69999999999999996,"5min":0.64100000000000001,"15min":0.64100000000000001},"max":{"1min":1.343,"5min":1.446,"15min":1.446},"last":0.96899999999999997},{"interface":"front","average":{"1min":0.96199999999999997,"5min":1.0329999999999999,"15min":1.0469999999999999},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.458,"5min":1.4610000000000001,"15min":1.4610000000000001},"last":1.115}]},{"osd":7,"last update":"Sat Sep 23 18:29:41 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":1.034,"15min":1.054},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.619,"5min":1.619,"15min":1.619},"last":1.1679999999999999},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.0660000000000001,"15min":1.0840000000000001},"min":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"max":{"1min":1.421,"5min":1.5720000000000001,"15min":1.5720000000000001},"last":1.0329999999999999}]}]},{"osd":2,"up_from":98,"seq":420906795038,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36672,"kb_used_data":5176,"kb_used_omap":11,"kb_used_meta":31476,"kb_avail":93708480,"statfs":{"total":95995035648,"available":95957483520,"internally_reserved":0,"allocated":5300224,"data_stored":4779645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11949,"internal_metadata":32231763},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.66700000000000004,"15min":0.67700000000000005},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":1.012,"5min":1.4430000000000001,"15min":1.4430000000000001},"last":1.1830000000000001},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.1799999999999999,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.66400000000000003}]},{"osd":1,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64900000000000002,"15min":0.65100000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.96499999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.69299999999999995,"5min":0.74099999999999999,"15min":0.749},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":1.006,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.77900000000000003}]},{"osd":3,"last update":"Sat Sep 23 18:29:04 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.117,"5min":1.117,"15min":1.117},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.5189999999999999,"5min":1.5189999999999999,"15min":1.5189999999999999},"last":0.86299999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.92300000000000004,"15min":0.92800000000000005},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.83399999999999996},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.97499999999999998,"15min":0.98099999999999998},"min":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.393,"5min":1.579,"15min":1.579},"last":1.0600000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":1.002,"15min":1.0189999999999999},"min":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.333,"5min":1.6990000000000001,"15min":1.6990000000000001},"last":0.97599999999999998},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1,"15min":0.995},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.488,"5min":1.488,"15min":1.488},"last":1.0009999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":1.0309999999999999,"15min":1.038},"min":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.373,"5min":1.4199999999999999,"15min":1.4199999999999999},"last":0.90900000000000003},{"interface":"front","average":{"1min":1.026,"5min":1.0660000000000001,"15min":1.073},"min":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"max":{"1min":1.4430000000000001,"5min":1.631,"15min":1.631},"last":1.1359999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.0589999999999999,"15min":1.0680000000000001},"min":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":1.538,"5min":1.538,"15min":1.538},"last":1.032},{"interface":"front","average":{"1min":0.999,"5min":1.1020000000000001,"15min":1.119},"min":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.2889999999999999,"5min":1.7609999999999999,"15min":1.7609999999999999},"last":1.232}]}]},{"osd":3,"up_from":103,"seq":442381631512,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28388,"kb_used_data":1052,"kb_used_omap":4,"kb_used_meta":27323,"kb_avail":93716764,"statfs":{"total":95995035648,"available":95965966336,"internally_reserved":0,"allocated":1077248,"data_stored":580581,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4826,"internal_metadata":27979046},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:29:03 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.90000000000000002},{"interface":"front","average":{"1min":0.749,"5min":0.749,"15min":0.749},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.84299999999999997}]},{"osd":1,"last update":"Sat Sep 23 18:29:03 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":1.1439999999999999,"5min":1.1439999999999999,"15min":1.1439999999999999},"last":0.98899999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:29:03 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":0.75,"15min":0.75},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":1.0760000000000001},{"interface":"front","average":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"min":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.0600000000000001,"5min":1.0600000000000001,"15min":1.0600000000000001},"last":0.87}]},{"osd":4,"last update":"Sat Sep 23 18:29:03 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"min":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.204,"5min":1.204,"15min":1.204},"last":1.3260000000000001},{"interface":"front","average":{"1min":1.04,"5min":1.04,"15min":1.04},"min":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"max":{"1min":1.431,"5min":1.431,"15min":1.431},"last":1.1899999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:29:03 2023","interfaces":[{"interface":"back","average":{"1min":1.089,"5min":1.089,"15min":1.089},"min":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"max":{"1min":1.444,"5min":1.444,"15min":1.444},"last":1.2949999999999999},{"interface":"front","average":{"1min":1.149,"5min":1.149,"15min":1.149},"min":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"max":{"1min":1.5409999999999999,"5min":1.5409999999999999,"15min":1.5409999999999999},"last":1.2589999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:29:03 2023","interfaces":[{"interface":"back","average":{"1min":1.141,"5min":1.141,"15min":1.141},"min":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"max":{"1min":1.492,"5min":1.492,"15min":1.492},"last":1.167},{"interface":"front","average":{"1min":1.099,"5min":1.099,"15min":1.099},"min":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"max":{"1min":1.425,"5min":1.425,"15min":1.425},"last":1.22}]},{"osd":7,"last update":"Sat Sep 23 18:29:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.1379999999999999,"15min":1.1379999999999999},"min":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"max":{"1min":1.5529999999999999,"5min":1.5529999999999999,"15min":1.5529999999999999},"last":1.121},{"interface":"front","average":{"1min":1.169,"5min":1.169,"15min":1.169},"min":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"max":{"1min":1.4530000000000001,"5min":1.4530000000000001,"15min":1.4530000000000001},"last":1.381}]}]},{"osd":4,"up_from":30,"seq":128849019122,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298152,"kb_used_data":1056,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447000,"statfs":{"total":95995035648,"available":95689728000,"internally_reserved":0,"allocated":1081344,"data_stored":581243,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:29:40 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.92700000000000005,"15min":0.92900000000000005},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.171,"5min":1.2809999999999999,"15min":1.2809999999999999},"last":1.1799999999999999},{"interface":"front","average":{"1min":0.93200000000000005,"5min":1.014,"15min":1.028},"min":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.2310000000000001,"5min":1.258,"15min":1.258},"last":1.2030000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:29:06 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.85099999999999998,"15min":0.84299999999999997},"min":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.1359999999999999,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":1.0109999999999999},{"interface":"front","average":{"1min":0.879,"5min":0.88500000000000001,"15min":0.88600000000000001},"min":{"1min":0.63400000000000001,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.198,"5min":1.224,"15min":1.224},"last":0.96099999999999997}]},{"osd":2,"last update":"Sat Sep 23 18:29:40 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":1.048,"15min":1.0489999999999999},"min":{"1min":0.72899999999999998,"5min":0.63,"15min":0.63},"max":{"1min":1.302,"5min":1.302,"15min":1.302},"last":1.1299999999999999},{"interface":"front","average":{"1min":0.95999999999999996,"5min":1.002,"15min":1.0089999999999999},"min":{"1min":0.68300000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"max":{"1min":1.2190000000000001,"5min":1.242,"15min":1.242},"last":1.2470000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:29:06 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"min":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"max":{"1min":1.1899999999999999,"5min":1.1899999999999999,"15min":1.1899999999999999},"last":0.91900000000000004},{"interface":"front","average":{"1min":1.048,"5min":1.048,"15min":1.048},"min":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"max":{"1min":1.2589999999999999,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":1.0669999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:29:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":0.97699999999999998,"15min":0.97299999999999998},"min":{"1min":0.72199999999999998,"5min":0.60199999999999998,"15min":0.51600000000000001},"max":{"1min":1.264,"5min":1.3919999999999999,"15min":2.266},"last":1.1539999999999999},{"interface":"front","average":{"1min":0.996,"5min":0.97599999999999998,"15min":0.97199999999999998},"min":{"1min":0.71499999999999997,"5min":0.58999999999999997,"15min":0.48199999999999998},"max":{"1min":1.298,"5min":1.411,"15min":2.4929999999999999},"last":1.0449999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:28:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.0209999999999999,"15min":1.022},"min":{"1min":0.66300000000000003,"5min":0.57699999999999996,"15min":0.55300000000000005},"max":{"1min":1.302,"5min":1.458,"15min":2.351},"last":1.2270000000000001},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.0409999999999999,"15min":1.008},"min":{"1min":0.74199999999999999,"5min":0.63300000000000001,"15min":0.42899999999999999},"max":{"1min":1.357,"5min":1.3660000000000001,"15min":2.621},"last":1.2769999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:29:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.105,"15min":1.083},"min":{"1min":0.92000000000000004,"5min":0.66200000000000003,"15min":0.50800000000000001},"max":{"1min":1.3600000000000001,"5min":1.4339999999999999,"15min":2.6819999999999999},"last":0.86699999999999999},{"interface":"front","average":{"1min":1.151,"5min":1.1579999999999999,"15min":1.169},"min":{"1min":0.86499999999999999,"5min":0.80400000000000005,"15min":0.66400000000000003},"max":{"1min":1.3520000000000001,"5min":1.4490000000000001,"15min":2.4140000000000001},"last":1.3089999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822890,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302228,"kb_used_data":5132,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93442924,"statfs":{"total":95995035648,"available":95685554176,"internally_reserved":0,"allocated":5255168,"data_stored":4770680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:29:42 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":0.85099999999999998,"15min":0.85099999999999998},"min":{"1min":0.57899999999999996,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.228,"5min":1.244,"15min":1.244},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.83199999999999996,"5min":0.86399999999999999,"15min":0.872},"min":{"1min":0.56200000000000006,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.123,"5min":1.1930000000000001,"15min":1.1930000000000001},"last":1.093}]},{"osd":1,"last update":"Sat Sep 23 18:29:09 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":0.86099999999999999,"15min":0.85299999999999998},"min":{"1min":0.61699999999999999,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.2210000000000001,"5min":1.2210000000000001,"15min":1.2210000000000001},"last":0.878},{"interface":"front","average":{"1min":0.83399999999999996,"5min":0.91900000000000004,"15min":0.93300000000000005},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.073,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":0.94599999999999995}]},{"osd":2,"last update":"Sat Sep 23 18:29:38 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":0.90900000000000003,"15min":0.91200000000000003},"min":{"1min":0.628,"5min":0.628,"15min":0.628},"max":{"1min":1.274,"5min":1.274,"15min":1.274},"last":0.81399999999999995},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.96199999999999997,"15min":0.95899999999999996},"min":{"1min":0.59999999999999998,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.2350000000000001,"5min":1.2350000000000001,"15min":1.2350000000000001},"last":1.044}]},{"osd":3,"last update":"Sat Sep 23 18:29:02 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"min":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.1319999999999999,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":0.97399999999999998},{"interface":"front","average":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"min":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.1140000000000001,"5min":1.1140000000000001,"15min":1.1140000000000001},"last":1.0720000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:29:09 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.92100000000000004,"15min":0.89500000000000002},"min":{"1min":0.55100000000000005,"5min":0.55000000000000004,"15min":0.39600000000000002},"max":{"1min":1.2270000000000001,"5min":1.3879999999999999,"15min":2.0259999999999998},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.90400000000000003,"5min":0.93400000000000005,"15min":0.93200000000000005},"min":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.36699999999999999},"max":{"1min":1.194,"5min":1.319,"15min":1.73},"last":0.996}]},{"osd":6,"last update":"Sat Sep 23 18:29:02 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":1.014,"15min":0.99299999999999999},"min":{"1min":0.68999999999999995,"5min":0.64700000000000002,"15min":0.47399999999999998},"max":{"1min":1.2350000000000001,"5min":1.464,"15min":2.137},"last":1.0309999999999999},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.097,"15min":1.0669999999999999},"min":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.51100000000000001},"max":{"1min":1.321,"5min":1.4199999999999999,"15min":1.8779999999999999},"last":1.129}]},{"osd":7,"last update":"Sat Sep 23 18:29:42 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.046,"15min":1.0509999999999999},"min":{"1min":0.66600000000000004,"5min":0.65400000000000003,"15min":0.53500000000000003},"max":{"1min":1.3180000000000001,"5min":1.5589999999999999,"15min":2.1549999999999998},"last":1.1539999999999999},{"interface":"front","average":{"1min":1.087,"5min":1.0820000000000001,"15min":1.083},"min":{"1min":0.83999999999999997,"5min":0.65200000000000002,"15min":0.54300000000000004},"max":{"1min":1.3069999999999999,"5min":1.6990000000000001,"15min":2.2189999999999999},"last":1.1759999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4224,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7725,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3429,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:29:54.108 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:29:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:54 smithi173 bash[59422]: audit 2023-09-23T18:29:53.212582+0000 mgr.y (mgr.26054) 333 : audit 0 from='client.34698 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:29:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:54 smithi173 bash[59422]: cluster 2023-09-23T18:29:53.629822+0000 mgr.y (mgr.26054) 334 : cluster 0 pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:54 smithi173 bash[59422]: audit 2023-09-23T18:29:54.576450+0000 mon.a (mon.0) 500 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:29:55.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:54 smithi098 bash[76817]: audit 2023-09-23T18:29:53.212582+0000 mgr.y (mgr.26054) 333 : audit 0 from='client.34698 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:29:55.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:54 smithi098 bash[76817]: cluster 2023-09-23T18:29:53.629822+0000 mgr.y (mgr.26054) 334 : cluster 0 pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:55.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:54 smithi098 bash[76817]: audit 2023-09-23T18:29:54.576450+0000 mon.a (mon.0) 500 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:29:55.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:54 smithi098 bash[79516]: audit 2023-09-23T18:29:53.212582+0000 mgr.y (mgr.26054) 333 : audit 0 from='client.34698 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:29:55.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:54 smithi098 bash[79516]: cluster 2023-09-23T18:29:53.629822+0000 mgr.y (mgr.26054) 334 : cluster 0 pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:55.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:54 smithi098 bash[79516]: audit 2023-09-23T18:29:54.576450+0000 mon.a (mon.0) 500 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:29:56.441 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:29:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:29:56] "GET /metrics HTTP/1.1" 200 34093 "" "Prometheus/2.33.4" 2023-09-23T18:29:57.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:56 smithi173 bash[59422]: cluster 2023-09-23T18:29:55.630680+0000 mgr.y (mgr.26054) 335 : cluster 0 pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:57.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:56 smithi098 bash[76817]: cluster 2023-09-23T18:29:55.630680+0000 mgr.y (mgr.26054) 335 : cluster 0 pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:57.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:56 smithi098 bash[79516]: cluster 2023-09-23T18:29:55.630680+0000 mgr.y (mgr.26054) 335 : cluster 0 pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:58.887 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:29:59.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:29:58 smithi173 bash[59422]: cluster 2023-09-23T18:29:57.631433+0000 mgr.y (mgr.26054) 336 : cluster 0 pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:59.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:29:58 smithi098 bash[76817]: cluster 2023-09-23T18:29:57.631433+0000 mgr.y (mgr.26054) 336 : cluster 0 pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:29:59.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:29:58 smithi098 bash[79516]: cluster 2023-09-23T18:29:57.631433+0000 mgr.y (mgr.26054) 336 : cluster 0 pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:00.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:00 smithi098 bash[76817]: cluster 2023-09-23T18:29:59.632178+0000 mgr.y (mgr.26054) 337 : cluster 0 pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:00.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:00 smithi098 bash[76817]: cluster 2023-09-23T18:30:00.000176+0000 mon.a (mon.0) 501 : cluster 1 overall HEALTH_OK 2023-09-23T18:30:00.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:00 smithi098 bash[79516]: cluster 2023-09-23T18:29:59.632178+0000 mgr.y (mgr.26054) 337 : cluster 0 pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:00.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:00 smithi098 bash[79516]: cluster 2023-09-23T18:30:00.000176+0000 mon.a (mon.0) 501 : cluster 1 overall HEALTH_OK 2023-09-23T18:30:00.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:00 smithi173 bash[59422]: cluster 2023-09-23T18:29:59.632178+0000 mgr.y (mgr.26054) 337 : cluster 0 pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:00.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:00 smithi173 bash[59422]: cluster 2023-09-23T18:30:00.000176+0000 mon.a (mon.0) 501 : cluster 1 overall HEALTH_OK 2023-09-23T18:30:02.133 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:30:02.138 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-23T18:30:03.002 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:02 smithi098 bash[76817]: cluster 2023-09-23T18:30:01.633183+0000 mgr.y (mgr.26054) 338 : cluster 0 pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:03.003 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:02 smithi098 bash[79516]: cluster 2023-09-23T18:30:01.633183+0000 mgr.y (mgr.26054) 338 : cluster 0 pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:03.003 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":274,"stamp":"2023-09-23T18:30:01.632468+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6303,"num_read_kb":13760,"num_write":1106,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1341076,"kb_used_data":22548,"kb_used_omap":31,"kb_used_meta":1318432,"kb_avail":748620140,"statfs":{"total":767960285184,"available":766587023360,"internally_reserved":0,"allocated":23089152,"data_stored":19014596,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32507,"internal_metadata":1350074629},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004468"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":63,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.289993+0000","last_change":"2023-09-23T18:15:43.682873+0000","last_active":"2023-09-23T18:28:02.289993+0000","last_peered":"2023-09-23T18:28:02.289993+0000","last_clean":"2023-09-23T18:28:02.289993+0000","last_became_active":"2023-09-23T18:15:43.682465+0000","last_became_peered":"2023-09-23T18:15:43.682465+0000","last_unstale":"2023-09-23T18:28:02.289993+0000","last_undegraded":"2023-09-23T18:28:02.289993+0000","last_fullsized":"2023-09-23T18:28:02.289993+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.280010+0000","last_change":"2023-09-23T18:27:33.583935+0000","last_active":"2023-09-23T18:28:03.280010+0000","last_peered":"2023-09-23T18:28:03.280010+0000","last_clean":"2023-09-23T18:28:03.280010+0000","last_became_active":"2023-09-23T18:27:33.582561+0000","last_became_peered":"2023-09-23T18:27:33.582561+0000","last_unstale":"2023-09-23T18:28:03.280010+0000","last_undegraded":"2023-09-23T18:28:03.280010+0000","last_fullsized":"2023-09-23T18:28:03.280010+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:43.589363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":90,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.299520+0000","last_change":"2023-09-23T18:28:03.299520+0000","last_active":"2023-09-23T18:28:03.299520+0000","last_peered":"2023-09-23T18:28:03.299520+0000","last_clean":"2023-09-23T18:28:03.299520+0000","last_became_active":"2023-09-23T18:28:03.299158+0000","last_became_peered":"2023-09-23T18:28:03.299158+0000","last_unstale":"2023-09-23T18:28:03.299520+0000","last_undegraded":"2023-09-23T18:28:03.299520+0000","last_fullsized":"2023-09-23T18:28:03.299520+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:26.247064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":78,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.297944+0000","last_change":"2023-09-23T18:28:03.296850+0000","last_active":"2023-09-23T18:28:03.297944+0000","last_peered":"2023-09-23T18:28:03.297944+0000","last_clean":"2023-09-23T18:28:03.297944+0000","last_became_active":"2023-09-23T18:28:03.295842+0000","last_became_peered":"2023-09-23T18:28:03.295842+0000","last_unstale":"2023-09-23T18:28:03.297944+0000","last_undegraded":"2023-09-23T18:28:03.297944+0000","last_fullsized":"2023-09-23T18:28:03.297944+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:53.048579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.302261+0000","last_change":"2023-09-23T18:28:03.302261+0000","last_active":"2023-09-23T18:28:03.302261+0000","last_peered":"2023-09-23T18:28:03.302261+0000","last_clean":"2023-09-23T18:28:03.302261+0000","last_became_active":"2023-09-23T18:28:03.301911+0000","last_became_peered":"2023-09-23T18:28:03.301911+0000","last_unstale":"2023-09-23T18:28:03.302261+0000","last_undegraded":"2023-09-23T18:28:03.302261+0000","last_fullsized":"2023-09-23T18:28:03.302261+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":57,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.279119+0000","last_change":"2023-09-23T18:15:08.902664+0000","last_active":"2023-09-23T18:28:02.279119+0000","last_peered":"2023-09-23T18:28:02.279119+0000","last_clean":"2023-09-23T18:28:02.279119+0000","last_became_active":"2023-09-23T18:15:08.901252+0000","last_became_peered":"2023-09-23T18:15:08.901252+0000","last_unstale":"2023-09-23T18:28:02.279119+0000","last_undegraded":"2023-09-23T18:28:02.279119+0000","last_fullsized":"2023-09-23T18:28:02.279119+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.300718+0000","last_change":"2023-09-23T18:28:03.300718+0000","last_active":"2023-09-23T18:28:03.300718+0000","last_peered":"2023-09-23T18:28:03.300718+0000","last_clean":"2023-09-23T18:28:03.300718+0000","last_became_active":"2023-09-23T18:28:03.300298+0000","last_became_peered":"2023-09-23T18:28:03.300298+0000","last_unstale":"2023-09-23T18:28:03.300718+0000","last_undegraded":"2023-09-23T18:28:03.300718+0000","last_fullsized":"2023-09-23T18:28:03.300718+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.278761+0000","last_change":"2023-09-23T18:27:04.250817+0000","last_active":"2023-09-23T18:28:02.278761+0000","last_peered":"2023-09-23T18:28:02.278761+0000","last_clean":"2023-09-23T18:28:02.278761+0000","last_became_active":"2023-09-23T18:27:04.250397+0000","last_became_peered":"2023-09-23T18:27:04.250397+0000","last_unstale":"2023-09-23T18:28:02.278761+0000","last_undegraded":"2023-09-23T18:28:02.278761+0000","last_fullsized":"2023-09-23T18:28:02.278761+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":92,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.288758+0000","last_change":"2023-09-23T18:28:03.288092+0000","last_active":"2023-09-23T18:28:03.288758+0000","last_peered":"2023-09-23T18:28:03.288758+0000","last_clean":"2023-09-23T18:28:03.288758+0000","last_became_active":"2023-09-23T18:28:03.287643+0000","last_became_peered":"2023-09-23T18:28:03.287643+0000","last_unstale":"2023-09-23T18:28:03.288758+0000","last_undegraded":"2023-09-23T18:28:03.288758+0000","last_fullsized":"2023-09-23T18:28:03.288758+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:44.163816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":41,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.280816+0000","last_change":"2023-09-23T18:26:34.994176+0000","last_active":"2023-09-23T18:28:03.280816+0000","last_peered":"2023-09-23T18:28:03.280816+0000","last_clean":"2023-09-23T18:28:03.280816+0000","last_became_active":"2023-09-23T18:26:34.993829+0000","last_became_peered":"2023-09-23T18:26:34.993829+0000","last_unstale":"2023-09-23T18:28:03.280816+0000","last_undegraded":"2023-09-23T18:28:03.280816+0000","last_fullsized":"2023-09-23T18:28:03.280816+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:27.111857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.293000+0000","last_change":"2023-09-23T18:15:09.125623+0000","last_active":"2023-09-23T18:28:02.293000+0000","last_peered":"2023-09-23T18:28:02.293000+0000","last_clean":"2023-09-23T18:28:02.293000+0000","last_became_active":"2023-09-23T18:15:09.125018+0000","last_became_peered":"2023-09-23T18:15:09.125018+0000","last_unstale":"2023-09-23T18:28:02.293000+0000","last_undegraded":"2023-09-23T18:28:02.293000+0000","last_fullsized":"2023-09-23T18:28:02.293000+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.281601+0000","last_change":"2023-09-23T18:27:04.247514+0000","last_active":"2023-09-23T18:28:03.281601+0000","last_peered":"2023-09-23T18:28:03.281601+0000","last_clean":"2023-09-23T18:28:03.281601+0000","last_became_active":"2023-09-23T18:27:04.247025+0000","last_became_peered":"2023-09-23T18:27:04.247025+0000","last_unstale":"2023-09-23T18:28:03.281601+0000","last_undegraded":"2023-09-23T18:28:03.281601+0000","last_fullsized":"2023-09-23T18:28:03.281601+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:32.498164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"77'112","reported_seq":594,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.284556+0000","last_change":"2023-09-23T18:27:33.587478+0000","last_active":"2023-09-23T18:28:02.284556+0000","last_peered":"2023-09-23T18:28:02.284556+0000","last_clean":"2023-09-23T18:28:02.284556+0000","last_became_active":"2023-09-23T18:27:33.586958+0000","last_became_peered":"2023-09-23T18:27:33.586958+0000","last_unstale":"2023-09-23T18:28:02.284556+0000","last_undegraded":"2023-09-23T18:28:02.284556+0000","last_fullsized":"2023-09-23T18:28:02.284556+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":893,"num_read_kb":3172,"num_write":69,"num_write_kb":2237,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.293466+0000","last_change":"2023-09-23T18:28:03.292295+0000","last_active":"2023-09-23T18:28:03.293466+0000","last_peered":"2023-09-23T18:28:03.293466+0000","last_clean":"2023-09-23T18:28:03.293466+0000","last_became_active":"2023-09-23T18:28:03.291902+0000","last_became_peered":"2023-09-23T18:28:03.291902+0000","last_unstale":"2023-09-23T18:28:03.293466+0000","last_undegraded":"2023-09-23T18:28:03.293466+0000","last_fullsized":"2023-09-23T18:28:03.293466+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:38.415256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'181","reported_seq":591,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.282681+0000","last_change":"2023-09-23T18:27:04.256906+0000","last_active":"2023-09-23T18:28:03.282681+0000","last_peered":"2023-09-23T18:28:03.282681+0000","last_clean":"2023-09-23T18:28:03.282681+0000","last_became_active":"2023-09-23T18:27:04.255734+0000","last_became_peered":"2023-09-23T18:27:04.255734+0000","last_unstale":"2023-09-23T18:28:03.282681+0000","last_undegraded":"2023-09-23T18:28:03.282681+0000","last_fullsized":"2023-09-23T18:28:03.282681+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":181,"log_dups_size":0,"ondisk_log_size":181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:53.270831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":909,"num_read_kb":734,"num_write":148,"num_write_kb":75,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.298805+0000","last_change":"2023-09-23T18:28:03.298805+0000","last_active":"2023-09-23T18:28:03.298805+0000","last_peered":"2023-09-23T18:28:03.298805+0000","last_clean":"2023-09-23T18:28:03.298805+0000","last_became_active":"2023-09-23T18:28:03.298437+0000","last_became_peered":"2023-09-23T18:28:03.298437+0000","last_unstale":"2023-09-23T18:28:03.298805+0000","last_undegraded":"2023-09-23T18:28:03.298805+0000","last_fullsized":"2023-09-23T18:28:03.298805+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:27.831669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.279808+0000","last_change":"2023-09-23T18:26:34.990770+0000","last_active":"2023-09-23T18:28:03.279808+0000","last_peered":"2023-09-23T18:28:03.279808+0000","last_clean":"2023-09-23T18:28:03.279808+0000","last_became_active":"2023-09-23T18:26:34.990262+0000","last_became_peered":"2023-09-23T18:26:34.990262+0000","last_unstale":"2023-09-23T18:28:03.279808+0000","last_undegraded":"2023-09-23T18:28:03.279808+0000","last_fullsized":"2023-09-23T18:28:03.279808+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:07.982834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.281672+0000","last_change":"2023-09-23T18:27:33.579367+0000","last_active":"2023-09-23T18:28:03.281672+0000","last_peered":"2023-09-23T18:28:03.281672+0000","last_clean":"2023-09-23T18:28:03.281672+0000","last_became_active":"2023-09-23T18:27:33.578607+0000","last_became_peered":"2023-09-23T18:27:33.578607+0000","last_unstale":"2023-09-23T18:28:03.281672+0000","last_undegraded":"2023-09-23T18:28:03.281672+0000","last_fullsized":"2023-09-23T18:28:03.281672+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:58:35.628296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'78","reported_seq":340,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.286740+0000","last_change":"2023-09-23T18:26:34.992345+0000","last_active":"2023-09-23T18:28:02.286740+0000","last_peered":"2023-09-23T18:28:02.286740+0000","last_clean":"2023-09-23T18:28:02.286740+0000","last_became_active":"2023-09-23T18:26:34.991774+0000","last_became_peered":"2023-09-23T18:26:34.991774+0000","last_unstale":"2023-09-23T18:28:02.286740+0000","last_undegraded":"2023-09-23T18:28:02.286740+0000","last_fullsized":"2023-09-23T18:28:02.286740+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":78,"log_dups_size":0,"ondisk_log_size":78,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":428,"num_read_kb":1125,"num_write":73,"num_write_kb":431,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.281734+0000","last_change":"2023-09-23T18:27:33.577544+0000","last_active":"2023-09-23T18:28:03.281734+0000","last_peered":"2023-09-23T18:28:03.281734+0000","last_clean":"2023-09-23T18:28:03.281734+0000","last_became_active":"2023-09-23T18:27:33.577194+0000","last_became_peered":"2023-09-23T18:27:33.577194+0000","last_unstale":"2023-09-23T18:28:03.281734+0000","last_undegraded":"2023-09-23T18:28:03.281734+0000","last_fullsized":"2023-09-23T18:28:03.281734+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:49.633444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.299894+0000","last_change":"2023-09-23T18:28:03.299508+0000","last_active":"2023-09-23T18:28:03.299894+0000","last_peered":"2023-09-23T18:28:03.299894+0000","last_clean":"2023-09-23T18:28:03.299894+0000","last_became_active":"2023-09-23T18:28:03.298682+0000","last_became_peered":"2023-09-23T18:28:03.298682+0000","last_unstale":"2023-09-23T18:28:03.299894+0000","last_undegraded":"2023-09-23T18:28:03.299894+0000","last_fullsized":"2023-09-23T18:28:03.299894+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:08.398302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'120","reported_seq":550,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.282230+0000","last_change":"2023-09-23T18:27:04.257709+0000","last_active":"2023-09-23T18:28:02.282230+0000","last_peered":"2023-09-23T18:28:02.282230+0000","last_clean":"2023-09-23T18:28:02.282230+0000","last_became_active":"2023-09-23T18:27:04.257331+0000","last_became_peered":"2023-09-23T18:27:04.257331+0000","last_unstale":"2023-09-23T18:28:02.282230+0000","last_undegraded":"2023-09-23T18:28:02.282230+0000","last_fullsized":"2023-09-23T18:28:02.282230+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":120,"log_dups_size":0,"ondisk_log_size":120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":809,"num_read_kb":2749,"num_write":98,"num_write_kb":2087,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.283659+0000","last_change":"2023-09-23T18:15:08.921629+0000","last_active":"2023-09-23T18:28:02.283659+0000","last_peered":"2023-09-23T18:28:02.283659+0000","last_clean":"2023-09-23T18:28:02.283659+0000","last_became_active":"2023-09-23T18:15:08.921283+0000","last_became_peered":"2023-09-23T18:15:08.921283+0000","last_unstale":"2023-09-23T18:28:02.283659+0000","last_undegraded":"2023-09-23T18:28:02.283659+0000","last_fullsized":"2023-09-23T18:28:02.283659+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":62,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.295577+0000","last_change":"2023-09-23T18:28:03.295577+0000","last_active":"2023-09-23T18:28:03.295577+0000","last_peered":"2023-09-23T18:28:03.295577+0000","last_clean":"2023-09-23T18:28:03.295577+0000","last_became_active":"2023-09-23T18:28:03.293777+0000","last_became_peered":"2023-09-23T18:28:03.293777+0000","last_unstale":"2023-09-23T18:28:03.295577+0000","last_undegraded":"2023-09-23T18:28:03.295577+0000","last_fullsized":"2023-09-23T18:28:03.295577+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:49.714764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":306,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.282044+0000","last_change":"2023-09-23T18:26:34.999402+0000","last_active":"2023-09-23T18:28:02.282044+0000","last_peered":"2023-09-23T18:28:02.282044+0000","last_clean":"2023-09-23T18:28:02.282044+0000","last_became_active":"2023-09-23T18:26:34.998828+0000","last_became_peered":"2023-09-23T18:26:34.998828+0000","last_unstale":"2023-09-23T18:28:02.282044+0000","last_undegraded":"2023-09-23T18:28:02.282044+0000","last_fullsized":"2023-09-23T18:28:02.282044+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_clean_scrub_stamp":"2023-09-23T18:08:35.701374+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"77'147","reported_seq":503,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.277795+0000","last_change":"2023-09-23T18:27:33.583558+0000","last_active":"2023-09-23T18:28:02.277795+0000","last_peered":"2023-09-23T18:28:02.277795+0000","last_clean":"2023-09-23T18:28:02.277795+0000","last_became_active":"2023-09-23T18:27:33.583003+0000","last_became_peered":"2023-09-23T18:27:33.583003+0000","last_unstale":"2023-09-23T18:28:02.277795+0000","last_undegraded":"2023-09-23T18:28:02.277795+0000","last_fullsized":"2023-09-23T18:28:02.277795+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":625,"num_read_kb":1869,"num_write":143,"num_write_kb":5368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":78,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.282670+0000","last_change":"2023-09-23T18:27:04.242813+0000","last_active":"2023-09-23T18:28:03.282670+0000","last_peered":"2023-09-23T18:28:03.282670+0000","last_clean":"2023-09-23T18:28:03.282670+0000","last_became_active":"2023-09-23T18:27:04.242409+0000","last_became_peered":"2023-09-23T18:27:04.242409+0000","last_unstale":"2023-09-23T18:28:03.282670+0000","last_undegraded":"2023-09-23T18:28:03.282670+0000","last_fullsized":"2023-09-23T18:28:03.282670+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:01.836849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.275061+0000","last_change":"2023-09-23T18:27:33.585279+0000","last_active":"2023-09-23T18:28:03.275061+0000","last_peered":"2023-09-23T18:28:03.275061+0000","last_clean":"2023-09-23T18:28:03.275061+0000","last_became_active":"2023-09-23T18:27:33.584928+0000","last_became_peered":"2023-09-23T18:27:33.584928+0000","last_unstale":"2023-09-23T18:28:03.275061+0000","last_undegraded":"2023-09-23T18:28:03.275061+0000","last_fullsized":"2023-09-23T18:28:03.275061+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:07.000233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'112","reported_seq":539,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.297866+0000","last_change":"2023-09-23T18:28:03.297540+0000","last_active":"2023-09-23T18:28:03.297866+0000","last_peered":"2023-09-23T18:28:03.297866+0000","last_clean":"2023-09-23T18:28:03.297866+0000","last_became_active":"2023-09-23T18:28:03.296649+0000","last_became_peered":"2023-09-23T18:28:03.296649+0000","last_unstale":"2023-09-23T18:28:03.297866+0000","last_undegraded":"2023-09-23T18:28:03.297866+0000","last_fullsized":"2023-09-23T18:28:03.297866+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:32.531775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":774,"num_read_kb":596,"num_write":102,"num_write_kb":52,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.289185+0000","last_change":"2023-09-23T18:28:03.288707+0000","last_active":"2023-09-23T18:28:03.289185+0000","last_peered":"2023-09-23T18:28:03.289185+0000","last_clean":"2023-09-23T18:28:03.289185+0000","last_became_active":"2023-09-23T18:28:03.288335+0000","last_became_peered":"2023-09-23T18:28:03.288335+0000","last_unstale":"2023-09-23T18:28:03.289185+0000","last_undegraded":"2023-09-23T18:28:03.289185+0000","last_fullsized":"2023-09-23T18:28:03.289185+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:57.652706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.292438+0000","last_change":"2023-09-23T18:26:34.992535+0000","last_active":"2023-09-23T18:28:02.292438+0000","last_peered":"2023-09-23T18:28:02.292438+0000","last_clean":"2023-09-23T18:28:02.292438+0000","last_became_active":"2023-09-23T18:26:34.992177+0000","last_became_peered":"2023-09-23T18:26:34.992177+0000","last_unstale":"2023-09-23T18:28:02.292438+0000","last_undegraded":"2023-09-23T18:28:02.292438+0000","last_fullsized":"2023-09-23T18:28:02.292438+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"77'96","reported_seq":619,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.277710+0000","last_change":"2023-09-23T18:27:04.250427+0000","last_active":"2023-09-23T18:28:02.277710+0000","last_peered":"2023-09-23T18:28:02.277710+0000","last_clean":"2023-09-23T18:28:02.277710+0000","last_became_active":"2023-09-23T18:27:04.249907+0000","last_became_peered":"2023-09-23T18:27:04.249907+0000","last_unstale":"2023-09-23T18:28:02.277710+0000","last_undegraded":"2023-09-23T18:28:02.277710+0000","last_fullsized":"2023-09-23T18:28:02.277710+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":96,"log_dups_size":0,"ondisk_log_size":96,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":785,"num_read_kb":1291,"num_write":87,"num_write_kb":338,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.275431+0000","last_change":"2023-09-23T18:27:04.251552+0000","last_active":"2023-09-23T18:28:03.275431+0000","last_peered":"2023-09-23T18:28:03.275431+0000","last_clean":"2023-09-23T18:28:03.275431+0000","last_became_active":"2023-09-23T18:27:04.250901+0000","last_became_peered":"2023-09-23T18:27:04.250901+0000","last_unstale":"2023-09-23T18:28:03.275431+0000","last_undegraded":"2023-09-23T18:28:03.275431+0000","last_fullsized":"2023-09-23T18:28:03.275431+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:22.322137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.297594+0000","last_change":"2023-09-23T18:28:03.297594+0000","last_active":"2023-09-23T18:28:03.297594+0000","last_peered":"2023-09-23T18:28:03.297594+0000","last_clean":"2023-09-23T18:28:03.297594+0000","last_became_active":"2023-09-23T18:28:03.296725+0000","last_became_peered":"2023-09-23T18:28:03.296725+0000","last_unstale":"2023-09-23T18:28:03.297594+0000","last_undegraded":"2023-09-23T18:28:03.297594+0000","last_fullsized":"2023-09-23T18:28:03.297594+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'130","reported_seq":583,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.281567+0000","last_change":"2023-09-23T18:27:04.250265+0000","last_active":"2023-09-23T18:28:03.281567+0000","last_peered":"2023-09-23T18:28:03.281567+0000","last_clean":"2023-09-23T18:28:03.281567+0000","last_became_active":"2023-09-23T18:27:04.249097+0000","last_became_peered":"2023-09-23T18:27:04.249097+0000","last_unstale":"2023-09-23T18:28:03.281567+0000","last_undegraded":"2023-09-23T18:28:03.281567+0000","last_fullsized":"2023-09-23T18:28:03.281567+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:27.143225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":772,"num_read_kb":1712,"num_write":126,"num_write_kb":2126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.295387+0000","last_change":"2023-09-23T18:28:03.294782+0000","last_active":"2023-09-23T18:28:03.295387+0000","last_peered":"2023-09-23T18:28:03.295387+0000","last_clean":"2023-09-23T18:28:03.295387+0000","last_became_active":"2023-09-23T18:28:03.294430+0000","last_became_peered":"2023-09-23T18:28:03.294430+0000","last_unstale":"2023-09-23T18:28:03.295387+0000","last_undegraded":"2023-09-23T18:28:03.295387+0000","last_fullsized":"2023-09-23T18:28:03.295387+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:20.226197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.287471+0000","last_change":"2023-09-23T18:28:03.287471+0000","last_active":"2023-09-23T18:28:03.287471+0000","last_peered":"2023-09-23T18:28:03.287471+0000","last_clean":"2023-09-23T18:28:03.287471+0000","last_became_active":"2023-09-23T18:28:03.286923+0000","last_became_peered":"2023-09-23T18:28:03.286923+0000","last_unstale":"2023-09-23T18:28:03.287471+0000","last_undegraded":"2023-09-23T18:28:03.287471+0000","last_fullsized":"2023-09-23T18:28:03.287471+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:10.505004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.291109+0000","last_change":"2023-09-23T18:27:04.254684+0000","last_active":"2023-09-23T18:28:02.291109+0000","last_peered":"2023-09-23T18:28:02.291109+0000","last_clean":"2023-09-23T18:28:02.291109+0000","last_became_active":"2023-09-23T18:27:04.254014+0000","last_became_peered":"2023-09-23T18:27:04.254014+0000","last_unstale":"2023-09-23T18:28:02.291109+0000","last_undegraded":"2023-09-23T18:28:02.291109+0000","last_fullsized":"2023-09-23T18:28:02.291109+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.293127+0000","last_change":"2023-09-23T18:27:33.585714+0000","last_active":"2023-09-23T18:28:02.293127+0000","last_peered":"2023-09-23T18:28:02.293127+0000","last_clean":"2023-09-23T18:28:02.293127+0000","last_became_active":"2023-09-23T18:27:33.584641+0000","last_became_peered":"2023-09-23T18:27:33.584641+0000","last_unstale":"2023-09-23T18:28:02.293127+0000","last_undegraded":"2023-09-23T18:28:02.293127+0000","last_fullsized":"2023-09-23T18:28:02.293127+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":84,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.291696+0000","last_change":"2023-09-23T18:27:33.588308+0000","last_active":"2023-09-23T18:28:02.291696+0000","last_peered":"2023-09-23T18:28:02.291696+0000","last_clean":"2023-09-23T18:28:02.291696+0000","last_became_active":"2023-09-23T18:27:33.587974+0000","last_became_peered":"2023-09-23T18:27:33.587974+0000","last_unstale":"2023-09-23T18:28:02.291696+0000","last_undegraded":"2023-09-23T18:28:02.291696+0000","last_fullsized":"2023-09-23T18:28:02.291696+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.284595+0000","last_change":"2023-09-23T18:27:04.252119+0000","last_active":"2023-09-23T18:28:02.284595+0000","last_peered":"2023-09-23T18:28:02.284595+0000","last_clean":"2023-09-23T18:28:02.284595+0000","last_became_active":"2023-09-23T18:27:04.251714+0000","last_became_peered":"2023-09-23T18:27:04.251714+0000","last_unstale":"2023-09-23T18:28:02.284595+0000","last_undegraded":"2023-09-23T18:28:02.284595+0000","last_fullsized":"2023-09-23T18:28:02.284595+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.302326+0000","last_change":"2023-09-23T18:28:03.302326+0000","last_active":"2023-09-23T18:28:03.302326+0000","last_peered":"2023-09-23T18:28:03.302326+0000","last_clean":"2023-09-23T18:28:03.302326+0000","last_became_active":"2023-09-23T18:28:03.301970+0000","last_became_peered":"2023-09-23T18:28:03.301970+0000","last_unstale":"2023-09-23T18:28:03.302326+0000","last_undegraded":"2023-09-23T18:28:03.302326+0000","last_fullsized":"2023-09-23T18:28:03.302326+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.297628+0000","last_change":"2023-09-23T18:28:03.297628+0000","last_active":"2023-09-23T18:28:03.297628+0000","last_peered":"2023-09-23T18:28:03.297628+0000","last_clean":"2023-09-23T18:28:03.297628+0000","last_became_active":"2023-09-23T18:28:03.296694+0000","last_became_peered":"2023-09-23T18:28:03.296694+0000","last_unstale":"2023-09-23T18:28:03.297628+0000","last_undegraded":"2023-09-23T18:28:03.297628+0000","last_fullsized":"2023-09-23T18:28:03.297628+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:43.375652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":65,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.280061+0000","last_change":"2023-09-23T18:27:33.582033+0000","last_active":"2023-09-23T18:28:03.280061+0000","last_peered":"2023-09-23T18:28:03.280061+0000","last_clean":"2023-09-23T18:28:03.280061+0000","last_became_active":"2023-09-23T18:27:33.581667+0000","last_became_peered":"2023-09-23T18:27:33.581667+0000","last_unstale":"2023-09-23T18:28:03.280061+0000","last_undegraded":"2023-09-23T18:28:03.280061+0000","last_fullsized":"2023-09-23T18:28:03.280061+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:36.039922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.298611+0000","last_change":"2023-09-23T18:28:03.298611+0000","last_active":"2023-09-23T18:28:03.298611+0000","last_peered":"2023-09-23T18:28:03.298611+0000","last_clean":"2023-09-23T18:28:03.298611+0000","last_became_active":"2023-09-23T18:28:03.298233+0000","last_became_peered":"2023-09-23T18:28:03.298233+0000","last_unstale":"2023-09-23T18:28:03.298611+0000","last_undegraded":"2023-09-23T18:28:03.298611+0000","last_fullsized":"2023-09-23T18:28:03.298611+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":78,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.284278+0000","last_change":"2023-09-23T18:27:33.589116+0000","last_active":"2023-09-23T18:28:02.284278+0000","last_peered":"2023-09-23T18:28:02.284278+0000","last_clean":"2023-09-23T18:28:02.284278+0000","last_became_active":"2023-09-23T18:27:33.588809+0000","last_became_peered":"2023-09-23T18:27:33.588809+0000","last_unstale":"2023-09-23T18:28:02.284278+0000","last_undegraded":"2023-09-23T18:28:02.284278+0000","last_fullsized":"2023-09-23T18:28:02.284278+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":56,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.286875+0000","last_change":"2023-09-23T18:15:08.908376+0000","last_active":"2023-09-23T18:28:02.286875+0000","last_peered":"2023-09-23T18:28:02.286875+0000","last_clean":"2023-09-23T18:28:02.286875+0000","last_became_active":"2023-09-23T18:15:08.907752+0000","last_became_peered":"2023-09-23T18:15:08.907752+0000","last_unstale":"2023-09-23T18:28:02.286875+0000","last_undegraded":"2023-09-23T18:28:02.286875+0000","last_fullsized":"2023-09-23T18:28:02.286875+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.302292+0000","last_change":"2023-09-23T18:28:03.302292+0000","last_active":"2023-09-23T18:28:03.302292+0000","last_peered":"2023-09-23T18:28:03.302292+0000","last_clean":"2023-09-23T18:28:03.302292+0000","last_became_active":"2023-09-23T18:28:03.301927+0000","last_became_peered":"2023-09-23T18:28:03.301927+0000","last_unstale":"2023-09-23T18:28:03.302292+0000","last_undegraded":"2023-09-23T18:28:03.302292+0000","last_fullsized":"2023-09-23T18:28:03.302292+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.286163+0000","last_change":"2023-09-23T18:27:04.253991+0000","last_active":"2023-09-23T18:28:02.286163+0000","last_peered":"2023-09-23T18:28:02.286163+0000","last_clean":"2023-09-23T18:28:02.286163+0000","last_became_active":"2023-09-23T18:27:04.253593+0000","last_became_peered":"2023-09-23T18:27:04.253593+0000","last_unstale":"2023-09-23T18:28:02.286163+0000","last_undegraded":"2023-09-23T18:28:02.286163+0000","last_fullsized":"2023-09-23T18:28:02.286163+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.277381+0000","last_change":"2023-09-23T18:15:10.327919+0000","last_active":"2023-09-23T18:28:02.277381+0000","last_peered":"2023-09-23T18:28:02.277381+0000","last_clean":"2023-09-23T18:28:02.277381+0000","last_became_active":"2023-09-23T18:15:10.327052+0000","last_became_peered":"2023-09-23T18:15:10.327052+0000","last_unstale":"2023-09-23T18:28:02.277381+0000","last_undegraded":"2023-09-23T18:28:02.277381+0000","last_fullsized":"2023-09-23T18:28:02.277381+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.286482+0000","last_change":"2023-09-23T18:26:34.999684+0000","last_active":"2023-09-23T18:28:02.286482+0000","last_peered":"2023-09-23T18:28:02.286482+0000","last_clean":"2023-09-23T18:28:02.286482+0000","last_became_active":"2023-09-23T18:26:34.999076+0000","last_became_peered":"2023-09-23T18:26:34.999076+0000","last_unstale":"2023-09-23T18:28:02.286482+0000","last_undegraded":"2023-09-23T18:28:02.286482+0000","last_fullsized":"2023-09-23T18:28:02.286482+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.295838+0000","last_change":"2023-09-23T18:28:03.295838+0000","last_active":"2023-09-23T18:28:03.295838+0000","last_peered":"2023-09-23T18:28:03.295838+0000","last_clean":"2023-09-23T18:28:03.295838+0000","last_became_active":"2023-09-23T18:28:03.291774+0000","last_became_peered":"2023-09-23T18:28:03.291774+0000","last_unstale":"2023-09-23T18:28:03.295838+0000","last_undegraded":"2023-09-23T18:28:03.295838+0000","last_fullsized":"2023-09-23T18:28:03.295838+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:23.166353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":78,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.284152+0000","last_change":"2023-09-23T18:26:34.995333+0000","last_active":"2023-09-23T18:28:02.284152+0000","last_peered":"2023-09-23T18:28:02.284152+0000","last_clean":"2023-09-23T18:28:02.284152+0000","last_became_active":"2023-09-23T18:26:34.994983+0000","last_became_peered":"2023-09-23T18:26:34.994983+0000","last_unstale":"2023-09-23T18:28:02.284152+0000","last_undegraded":"2023-09-23T18:28:02.284152+0000","last_fullsized":"2023-09-23T18:28:02.284152+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.290583+0000","last_change":"2023-09-23T18:28:03.290217+0000","last_active":"2023-09-23T18:28:03.290583+0000","last_peered":"2023-09-23T18:28:03.290583+0000","last_clean":"2023-09-23T18:28:03.290583+0000","last_became_active":"2023-09-23T18:28:03.289888+0000","last_became_peered":"2023-09-23T18:28:03.289888+0000","last_unstale":"2023-09-23T18:28:03.290583+0000","last_undegraded":"2023-09-23T18:28:03.290583+0000","last_fullsized":"2023-09-23T18:28:03.290583+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:51:45.400847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.281244+0000","last_change":"2023-09-23T18:15:08.916658+0000","last_active":"2023-09-23T18:28:02.281244+0000","last_peered":"2023-09-23T18:28:02.281244+0000","last_clean":"2023-09-23T18:28:02.281244+0000","last_became_active":"2023-09-23T18:15:08.916301+0000","last_became_peered":"2023-09-23T18:15:08.916301+0000","last_unstale":"2023-09-23T18:28:02.281244+0000","last_undegraded":"2023-09-23T18:28:02.281244+0000","last_fullsized":"2023-09-23T18:28:02.281244+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.297841+0000","last_change":"2023-09-23T18:28:03.297841+0000","last_active":"2023-09-23T18:28:03.297841+0000","last_peered":"2023-09-23T18:28:03.297841+0000","last_clean":"2023-09-23T18:28:03.297841+0000","last_became_active":"2023-09-23T18:28:03.297216+0000","last_became_peered":"2023-09-23T18:28:03.297216+0000","last_unstale":"2023-09-23T18:28:03.297841+0000","last_undegraded":"2023-09-23T18:28:03.297841+0000","last_fullsized":"2023-09-23T18:28:03.297841+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.290591+0000","last_change":"2023-09-23T18:28:03.290316+0000","last_active":"2023-09-23T18:28:03.290591+0000","last_peered":"2023-09-23T18:28:03.290591+0000","last_clean":"2023-09-23T18:28:03.290591+0000","last_became_active":"2023-09-23T18:28:03.289698+0000","last_became_peered":"2023-09-23T18:28:03.289698+0000","last_unstale":"2023-09-23T18:28:03.290591+0000","last_undegraded":"2023-09-23T18:28:03.290591+0000","last_fullsized":"2023-09-23T18:28:03.290591+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:00.779542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.292363+0000","last_change":"2023-09-23T18:28:03.292363+0000","last_active":"2023-09-23T18:28:03.292363+0000","last_peered":"2023-09-23T18:28:03.292363+0000","last_clean":"2023-09-23T18:28:03.292363+0000","last_became_active":"2023-09-23T18:28:03.291796+0000","last_became_peered":"2023-09-23T18:28:03.291796+0000","last_unstale":"2023-09-23T18:28:03.292363+0000","last_undegraded":"2023-09-23T18:28:03.292363+0000","last_fullsized":"2023-09-23T18:28:03.292363+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:13:35.340945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.283954+0000","last_change":"2023-09-23T18:27:33.585121+0000","last_active":"2023-09-23T18:28:02.283954+0000","last_peered":"2023-09-23T18:28:02.283954+0000","last_clean":"2023-09-23T18:28:02.283954+0000","last_became_active":"2023-09-23T18:27:33.584620+0000","last_became_peered":"2023-09-23T18:27:33.584620+0000","last_unstale":"2023-09-23T18:28:02.283954+0000","last_undegraded":"2023-09-23T18:28:02.283954+0000","last_fullsized":"2023-09-23T18:28:02.283954+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.290588+0000","last_change":"2023-09-23T18:27:04.256030+0000","last_active":"2023-09-23T18:28:02.290588+0000","last_peered":"2023-09-23T18:28:02.290588+0000","last_clean":"2023-09-23T18:28:02.290588+0000","last_became_active":"2023-09-23T18:27:04.255549+0000","last_became_peered":"2023-09-23T18:27:04.255549+0000","last_unstale":"2023-09-23T18:28:02.290588+0000","last_undegraded":"2023-09-23T18:28:02.290588+0000","last_fullsized":"2023-09-23T18:28:02.290588+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":77,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.292732+0000","last_change":"2023-09-23T18:15:08.909339+0000","last_active":"2023-09-23T18:28:02.292732+0000","last_peered":"2023-09-23T18:28:02.292732+0000","last_clean":"2023-09-23T18:28:02.292732+0000","last_became_active":"2023-09-23T18:15:08.908912+0000","last_became_peered":"2023-09-23T18:15:08.908912+0000","last_unstale":"2023-09-23T18:28:02.292732+0000","last_undegraded":"2023-09-23T18:28:02.292732+0000","last_fullsized":"2023-09-23T18:28:02.292732+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.275687+0000","last_change":"2023-09-23T18:27:33.585072+0000","last_active":"2023-09-23T18:28:03.275687+0000","last_peered":"2023-09-23T18:28:03.275687+0000","last_clean":"2023-09-23T18:28:03.275687+0000","last_became_active":"2023-09-23T18:27:33.584118+0000","last_became_peered":"2023-09-23T18:27:33.584118+0000","last_unstale":"2023-09-23T18:28:03.275687+0000","last_undegraded":"2023-09-23T18:28:03.275687+0000","last_fullsized":"2023-09-23T18:28:03.275687+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:30:12.156524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.296913+0000","last_change":"2023-09-23T18:28:03.296913+0000","last_active":"2023-09-23T18:28:03.296913+0000","last_peered":"2023-09-23T18:28:03.296913+0000","last_clean":"2023-09-23T18:28:03.296913+0000","last_became_active":"2023-09-23T18:28:03.296499+0000","last_became_peered":"2023-09-23T18:28:03.296499+0000","last_unstale":"2023-09-23T18:28:03.296913+0000","last_undegraded":"2023-09-23T18:28:03.296913+0000","last_fullsized":"2023-09-23T18:28:03.296913+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":61,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.298214+0000","last_change":"2023-09-23T18:28:03.298214+0000","last_active":"2023-09-23T18:28:03.298214+0000","last_peered":"2023-09-23T18:28:03.298214+0000","last_clean":"2023-09-23T18:28:03.298214+0000","last_became_active":"2023-09-23T18:28:03.297852+0000","last_became_peered":"2023-09-23T18:28:03.297852+0000","last_unstale":"2023-09-23T18:28:03.298214+0000","last_undegraded":"2023-09-23T18:28:03.298214+0000","last_fullsized":"2023-09-23T18:28:03.298214+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.146541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.278178+0000","last_change":"2023-09-23T18:27:04.250764+0000","last_active":"2023-09-23T18:28:02.278178+0000","last_peered":"2023-09-23T18:28:02.278178+0000","last_clean":"2023-09-23T18:28:02.278178+0000","last_became_active":"2023-09-23T18:27:04.250360+0000","last_became_peered":"2023-09-23T18:27:04.250360+0000","last_unstale":"2023-09-23T18:28:02.278178+0000","last_undegraded":"2023-09-23T18:28:02.278178+0000","last_fullsized":"2023-09-23T18:28:02.278178+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.295734+0000","last_change":"2023-09-23T18:28:03.295734+0000","last_active":"2023-09-23T18:28:03.295734+0000","last_peered":"2023-09-23T18:28:03.295734+0000","last_clean":"2023-09-23T18:28:03.295734+0000","last_became_active":"2023-09-23T18:28:03.293761+0000","last_became_peered":"2023-09-23T18:28:03.293761+0000","last_unstale":"2023-09-23T18:28:03.295734+0000","last_undegraded":"2023-09-23T18:28:03.295734+0000","last_fullsized":"2023-09-23T18:28:03.295734+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:55.285705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.297725+0000","last_change":"2023-09-23T18:28:03.297439+0000","last_active":"2023-09-23T18:28:03.297725+0000","last_peered":"2023-09-23T18:28:03.297725+0000","last_clean":"2023-09-23T18:28:03.297725+0000","last_became_active":"2023-09-23T18:28:03.297112+0000","last_became_peered":"2023-09-23T18:28:03.297112+0000","last_unstale":"2023-09-23T18:28:03.297725+0000","last_undegraded":"2023-09-23T18:28:03.297725+0000","last_fullsized":"2023-09-23T18:28:03.297725+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:50.442238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.284400+0000","last_change":"2023-09-23T18:27:04.247225+0000","last_active":"2023-09-23T18:28:02.284400+0000","last_peered":"2023-09-23T18:28:02.284400+0000","last_clean":"2023-09-23T18:28:02.284400+0000","last_became_active":"2023-09-23T18:27:04.246843+0000","last_became_peered":"2023-09-23T18:27:04.246843+0000","last_unstale":"2023-09-23T18:28:02.284400+0000","last_undegraded":"2023-09-23T18:28:02.284400+0000","last_fullsized":"2023-09-23T18:28:02.284400+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":87,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.287830+0000","last_change":"2023-09-23T18:28:03.287830+0000","last_active":"2023-09-23T18:28:03.287830+0000","last_peered":"2023-09-23T18:28:03.287830+0000","last_clean":"2023-09-23T18:28:03.287830+0000","last_became_active":"2023-09-23T18:28:03.287461+0000","last_became_peered":"2023-09-23T18:28:03.287461+0000","last_unstale":"2023-09-23T18:28:03.287830+0000","last_undegraded":"2023-09-23T18:28:03.287830+0000","last_fullsized":"2023-09-23T18:28:03.287830+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:21:51.267964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.290067+0000","last_change":"2023-09-23T18:28:03.289500+0000","last_active":"2023-09-23T18:28:03.290067+0000","last_peered":"2023-09-23T18:28:03.290067+0000","last_clean":"2023-09-23T18:28:03.290067+0000","last_became_active":"2023-09-23T18:28:03.289122+0000","last_became_peered":"2023-09-23T18:28:03.289122+0000","last_unstale":"2023-09-23T18:28:03.290067+0000","last_undegraded":"2023-09-23T18:28:03.290067+0000","last_fullsized":"2023-09-23T18:28:03.290067+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:22:26.904660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.295399+0000","last_change":"2023-09-23T18:28:03.295009+0000","last_active":"2023-09-23T18:28:03.295399+0000","last_peered":"2023-09-23T18:28:03.295399+0000","last_clean":"2023-09-23T18:28:03.295399+0000","last_became_active":"2023-09-23T18:28:03.294501+0000","last_became_peered":"2023-09-23T18:28:03.294501+0000","last_unstale":"2023-09-23T18:28:03.295399+0000","last_undegraded":"2023-09-23T18:28:03.295399+0000","last_fullsized":"2023-09-23T18:28:03.295399+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:00.671680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.299495+0000","last_change":"2023-09-23T18:28:03.299495+0000","last_active":"2023-09-23T18:28:03.299495+0000","last_peered":"2023-09-23T18:28:03.299495+0000","last_clean":"2023-09-23T18:28:03.299495+0000","last_became_active":"2023-09-23T18:28:03.297293+0000","last_became_peered":"2023-09-23T18:28:03.297293+0000","last_unstale":"2023-09-23T18:28:03.299495+0000","last_undegraded":"2023-09-23T18:28:03.299495+0000","last_fullsized":"2023-09-23T18:28:03.299495+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:01.235861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":86,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.291372+0000","last_change":"2023-09-23T18:27:33.588552+0000","last_active":"2023-09-23T18:28:02.291372+0000","last_peered":"2023-09-23T18:28:02.291372+0000","last_clean":"2023-09-23T18:28:02.291372+0000","last_became_active":"2023-09-23T18:27:33.588118+0000","last_became_peered":"2023-09-23T18:27:33.588118+0000","last_unstale":"2023-09-23T18:28:02.291372+0000","last_undegraded":"2023-09-23T18:28:02.291372+0000","last_fullsized":"2023-09-23T18:28:02.291372+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26886,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5621,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430426,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306928,"kb_used_data":5736,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93438224,"statfs":{"total":95995035648,"available":95680741376,"internally_reserved":0,"allocated":5873664,"data_stored":5365985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:29:42 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.91900000000000004,"15min":0.92700000000000005},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.2909999999999999,"5min":1.4670000000000001,"15min":1.4670000000000001},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.93700000000000006,"15min":0.95899999999999996},"min":{"1min":0.69899999999999995,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.175,"5min":1.302,"15min":1.302},"last":1.1679999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:29:05 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.91300000000000003,"15min":0.91600000000000004},"min":{"1min":0.61199999999999999,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.1080000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":1.1040000000000001},{"interface":"front","average":{"1min":1.028,"5min":0.97599999999999998,"15min":0.96799999999999997},"min":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":0.88600000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:29:36 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"min":{"1min":0.71099999999999997,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":1.2529999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":1.2589999999999999},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.94899999999999995,"15min":0.95499999999999996},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.2989999999999999,"5min":1.3380000000000001,"15min":1.3380000000000001},"last":1.0649999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:29:02 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":0.999,"15min":0.999},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.383,"5min":1.383,"15min":1.383},"last":0.95499999999999996},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"min":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"max":{"1min":1.115,"5min":1.115,"15min":1.115},"last":1.0209999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:29:50 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.92400000000000004,"15min":0.88900000000000001},"min":{"1min":0.625,"5min":0.60099999999999998,"15min":0.371},"max":{"1min":1.214,"5min":1.2589999999999999,"15min":1.3839999999999999},"last":0.97699999999999998},{"interface":"front","average":{"1min":1.028,"5min":1.0469999999999999,"15min":1.028},"min":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.50700000000000001},"max":{"1min":1.341,"5min":1.4430000000000001,"15min":1.4430000000000001},"last":1.1359999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:29:50 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.94099999999999995,"15min":0.95699999999999996},"min":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.505},"max":{"1min":1.298,"5min":1.298,"15min":1.4470000000000001},"last":0.90800000000000003},{"interface":"front","average":{"1min":1.042,"5min":1.0580000000000001,"15min":1.0129999999999999},"min":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.40899999999999997},"max":{"1min":1.2889999999999999,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":1.298}]},{"osd":6,"last update":"Sat Sep 23 18:29:50 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":1.0980000000000001,"15min":1.05},"min":{"1min":0.95999999999999996,"5min":0.72699999999999998,"15min":0.442},"max":{"1min":1.3560000000000001,"5min":1.4399999999999999,"15min":1.4690000000000001},"last":1.218},{"interface":"front","average":{"1min":1.095,"5min":1.145,"15min":1.131},"min":{"1min":0.67400000000000004,"5min":0.63600000000000001,"15min":0.53200000000000003},"max":{"1min":1.3380000000000001,"5min":1.548,"15min":1.548},"last":1.1890000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626658,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302820,"kb_used_data":1628,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93442332,"statfs":{"total":95995035648,"available":95684947968,"internally_reserved":0,"allocated":1667072,"data_stored":1171239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:29:40 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.91600000000000004,"15min":0.91000000000000003},"min":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.244,"5min":1.2470000000000001,"15min":1.2470000000000001},"last":0.97999999999999998},{"interface":"front","average":{"1min":0.99399999999999999,"5min":1.0409999999999999,"15min":1.0449999999999999},"min":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"max":{"1min":1.1850000000000001,"5min":1.3540000000000001,"15min":1.3540000000000001},"last":1.038}]},{"osd":1,"last update":"Sat Sep 23 18:29:07 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.90700000000000003,"15min":0.90300000000000002},"min":{"1min":0.61699999999999999,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.2210000000000001,"5min":1.2210000000000001,"15min":1.2210000000000001},"last":0.89800000000000002},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.92900000000000005,"15min":0.92500000000000004},"min":{"1min":0.71799999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.2470000000000001,"5min":1.2470000000000001,"15min":1.2470000000000001},"last":0.87}]},{"osd":2,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.999,"15min":1.0049999999999999},"min":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"max":{"1min":1.2789999999999999,"5min":1.2789999999999999,"15min":1.2789999999999999},"last":1.012},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.95699999999999996,"15min":0.95899999999999996},"min":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.196,"5min":1.196,"15min":1.196},"last":1.129}]},{"osd":3,"last update":"Sat Sep 23 18:29:04 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"min":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.3069999999999999,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":1.0980000000000001},{"interface":"front","average":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"min":{"1min":0.745,"5min":0.745,"15min":0.745},"max":{"1min":1.288,"5min":1.288,"15min":1.288},"last":1.0549999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:29:55 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":0.91200000000000003,"15min":0.92400000000000004},"min":{"1min":0.61699999999999999,"5min":0.53400000000000003,"15min":0.44800000000000001},"max":{"1min":1.3069999999999999,"5min":1.339,"15min":1.4399999999999999},"last":0.67500000000000004},{"interface":"front","average":{"1min":1.085,"5min":1.006,"15min":0.98099999999999998},"min":{"1min":0.78900000000000003,"5min":0.56499999999999995,"15min":0.41199999999999998},"max":{"1min":1.369,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.748}]},{"osd":5,"last update":"Sat Sep 23 18:29:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":1.0209999999999999,"15min":1.0249999999999999},"min":{"1min":0.92600000000000005,"5min":0.58999999999999997,"15min":0.36699999999999999},"max":{"1min":1.4219999999999999,"5min":1.431,"15min":1.5469999999999999},"last":0.96999999999999997},{"interface":"front","average":{"1min":1.1399999999999999,"5min":1.0580000000000001,"15min":1.0580000000000001},"min":{"1min":0.83199999999999996,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.403,"5min":1.504,"15min":1.5700000000000001},"last":1.161}]},{"osd":7,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.056,"15min":1.0760000000000001},"min":{"1min":0.76000000000000001,"5min":0.59499999999999997,"15min":0.48999999999999999},"max":{"1min":1.274,"5min":1.764,"15min":1.764},"last":0.93999999999999995},{"interface":"front","average":{"1min":1.173,"5min":1.133,"15min":1.1339999999999999},"min":{"1min":0.83399999999999996,"5min":0.64400000000000002,"15min":0.499},"max":{"1min":1.3759999999999999,"5min":1.6659999999999999,"15min":1.6659999999999999},"last":1.0740000000000001}]}]},{"osd":1,"up_from":93,"seq":399431958565,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32648,"kb_used_data":1088,"kb_used_omap":7,"kb_used_meta":31544,"kb_avail":93712504,"statfs":{"total":95995035648,"available":95961604096,"internally_reserved":0,"allocated":1114112,"data_stored":586888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":32301838},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:29:08 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.66800000000000004,"15min":0.67600000000000005},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.83999999999999997,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.746},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.72299999999999998,"15min":0.73499999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.88900000000000001,"5min":1.0960000000000001,"15min":1.0960000000000001},"last":0.78800000000000003}]},{"osd":2,"last update":"Sat Sep 23 18:29:40 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.60099999999999998,"15min":0.59899999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":1.1000000000000001,"5min":1.1000000000000001,"15min":1.1000000000000001},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.71399999999999997,"15min":0.71999999999999997},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.0449999999999999,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.71799999999999997}]},{"osd":3,"last update":"Sat Sep 23 18:29:02 2023","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":0.753,"15min":0.753},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.157,"5min":1.157,"15min":1.157},"last":0.86499999999999999},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.294,"5min":1.294,"15min":1.294},"last":0.83599999999999997}]},{"osd":4,"last update":"Sat Sep 23 18:29:08 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.92600000000000005,"15min":0.92100000000000004},"min":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.204,"5min":1.204,"15min":1.204},"last":1.0289999999999999},{"interface":"front","average":{"1min":0.874,"5min":1.0129999999999999,"15min":1.036},"min":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.2969999999999999,"5min":1.4950000000000001,"15min":1.4950000000000001},"last":0.88800000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:29:08 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":0.98199999999999998,"15min":0.999},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.143,"5min":1.5149999999999999,"15min":1.5149999999999999},"last":0.90600000000000003},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.97899999999999998,"15min":0.98499999999999999},"min":{"1min":0.65100000000000002,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.294,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":0.96899999999999997}]},{"osd":6,"last update":"Sat Sep 23 18:29:08 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":1.018,"15min":1.03},"min":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"max":{"1min":1.2270000000000001,"5min":1.6100000000000001,"15min":1.6100000000000001},"last":0.95399999999999996},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.0649999999999999,"15min":1.0740000000000001},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.3600000000000001,"5min":1.4419999999999999,"15min":1.4419999999999999},"last":1.073}]},{"osd":7,"last update":"Sat Sep 23 18:29:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.095,"15min":1.109},"min":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.4019999999999999,"5min":1.55,"15min":1.55},"last":1.1579999999999999},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.1120000000000001,"15min":1.1339999999999999},"min":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.51,"5min":1.575,"15min":1.575},"last":0.93899999999999995}]}]},{"osd":0,"up_from":88,"seq":377957122091,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33240,"kb_used_data":1680,"kb_used_omap":8,"kb_used_meta":31543,"kb_avail":93711912,"statfs":{"total":95995035648,"available":95960997888,"internally_reserved":0,"allocated":1720320,"data_stored":1178335,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":32300926},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:29:08 2023","interfaces":[{"interface":"back","average":{"1min":0.71799999999999997,"5min":0.65800000000000003,"15min":0.64800000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.2170000000000001,"5min":1.2170000000000001,"15min":1.2170000000000001},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.83399999999999996,"5min":0.80800000000000005,"15min":0.80400000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.3480000000000001,"5min":1.3480000000000001,"15min":1.3480000000000001},"last":0.73899999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.80500000000000005,"15min":0.82399999999999995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.202,"5min":1.254,"15min":1.254},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.67000000000000004,"15min":0.68300000000000005},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.90100000000000002,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.78800000000000003}]},{"osd":3,"last update":"Sat Sep 23 18:29:02 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.2849999999999999,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":1.0189999999999999},{"interface":"front","average":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.3500000000000001,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.61199999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:29:41 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.97299999999999998,"15min":0.97399999999999998},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.4970000000000001,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":0.97699999999999998},{"interface":"front","average":{"1min":0.90700000000000003,"5min":0.98399999999999999,"15min":1.0089999999999999},"min":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.29,"5min":1.29,"15min":1.29},"last":1.196}]},{"osd":5,"last update":"Sat Sep 23 18:29:41 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.99099999999999999,"15min":1.0149999999999999},"min":{"1min":0.68000000000000005,"5min":0.67200000000000004,"15min":0.67200000000000004},"max":{"1min":1.23,"5min":1.524,"15min":1.524},"last":1.3},{"interface":"front","average":{"1min":0.96399999999999997,"5min":1.0129999999999999,"15min":1.0349999999999999},"min":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"max":{"1min":1.544,"5min":1.544,"15min":1.544},"last":1.244}]},{"osd":6,"last update":"Sat Sep 23 18:29:41 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":1.036,"15min":1.0640000000000001},"min":{"1min":0.69999999999999996,"5min":0.64100000000000001,"15min":0.64100000000000001},"max":{"1min":1.343,"5min":1.446,"15min":1.446},"last":1.3919999999999999},{"interface":"front","average":{"1min":0.96199999999999997,"5min":1.0329999999999999,"15min":1.0469999999999999},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.458,"5min":1.4610000000000001,"15min":1.4610000000000001},"last":1.111}]},{"osd":7,"last update":"Sat Sep 23 18:29:41 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":1.034,"15min":1.054},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.619,"5min":1.619,"15min":1.619},"last":1.155},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.0660000000000001,"15min":1.0840000000000001},"min":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"max":{"1min":1.421,"5min":1.5720000000000001,"15min":1.5720000000000001},"last":0.91100000000000003}]}]},{"osd":2,"up_from":98,"seq":420906795040,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36672,"kb_used_data":5176,"kb_used_omap":11,"kb_used_meta":31476,"kb_avail":93708480,"statfs":{"total":95995035648,"available":95957483520,"internally_reserved":0,"allocated":5300224,"data_stored":4779645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11949,"internal_metadata":32231763},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.66700000000000004,"15min":0.67700000000000005},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":1.012,"5min":1.4430000000000001,"15min":1.4430000000000001},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.1799999999999999,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.79200000000000004}]},{"osd":1,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64900000000000002,"15min":0.65100000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.96499999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.94199999999999995},{"interface":"front","average":{"1min":0.69299999999999995,"5min":0.74099999999999999,"15min":0.749},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":1.006,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.90800000000000003}]},{"osd":3,"last update":"Sat Sep 23 18:29:04 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.117,"5min":1.117,"15min":1.117},"last":0.76600000000000001},{"interface":"front","average":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.5189999999999999,"5min":1.5189999999999999,"15min":1.5189999999999999},"last":0.85399999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.92300000000000004,"15min":0.92800000000000005},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":1.129},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.97499999999999998,"15min":0.98099999999999998},"min":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.393,"5min":1.579,"15min":1.579},"last":1.073}]},{"osd":5,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":1.002,"15min":1.0189999999999999},"min":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.333,"5min":1.6990000000000001,"15min":1.6990000000000001},"last":0.98899999999999999},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1,"15min":0.995},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.488,"5min":1.488,"15min":1.488},"last":1.3089999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":1.0309999999999999,"15min":1.038},"min":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.373,"5min":1.4199999999999999,"15min":1.4199999999999999},"last":1.1839999999999999},{"interface":"front","average":{"1min":1.026,"5min":1.0660000000000001,"15min":1.073},"min":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"max":{"1min":1.4430000000000001,"5min":1.631,"15min":1.631},"last":1.198}]},{"osd":7,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.0589999999999999,"15min":1.0680000000000001},"min":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":1.538,"5min":1.538,"15min":1.538},"last":1.264},{"interface":"front","average":{"1min":0.999,"5min":1.1020000000000001,"15min":1.119},"min":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.2889999999999999,"5min":1.7609999999999999,"15min":1.7609999999999999},"last":1.2809999999999999}]}]},{"osd":3,"up_from":103,"seq":442381631514,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28388,"kb_used_data":1052,"kb_used_omap":4,"kb_used_meta":27323,"kb_avail":93716764,"statfs":{"total":95995035648,"available":95965966336,"internally_reserved":0,"allocated":1077248,"data_stored":580581,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4826,"internal_metadata":27979046},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:29:03 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.749,"5min":0.749,"15min":0.749},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":1.1240000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:29:03 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":1.1439999999999999,"5min":1.1439999999999999,"15min":1.1439999999999999},"last":0.93500000000000005}]},{"osd":2,"last update":"Sat Sep 23 18:29:03 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":0.75,"15min":0.75},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.83099999999999996},{"interface":"front","average":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"min":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.0600000000000001,"5min":1.0600000000000001,"15min":1.0600000000000001},"last":1.1719999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:29:03 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"min":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.204,"5min":1.204,"15min":1.204},"last":0.97699999999999998},{"interface":"front","average":{"1min":1.04,"5min":1.04,"15min":1.04},"min":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"max":{"1min":1.431,"5min":1.431,"15min":1.431},"last":1.0529999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:29:03 2023","interfaces":[{"interface":"back","average":{"1min":1.089,"5min":1.089,"15min":1.089},"min":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"max":{"1min":1.444,"5min":1.444,"15min":1.444},"last":1.151},{"interface":"front","average":{"1min":1.149,"5min":1.149,"15min":1.149},"min":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"max":{"1min":1.5409999999999999,"5min":1.5409999999999999,"15min":1.5409999999999999},"last":1.21}]},{"osd":6,"last update":"Sat Sep 23 18:29:03 2023","interfaces":[{"interface":"back","average":{"1min":1.141,"5min":1.141,"15min":1.141},"min":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"max":{"1min":1.492,"5min":1.492,"15min":1.492},"last":1.0169999999999999},{"interface":"front","average":{"1min":1.099,"5min":1.099,"15min":1.099},"min":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"max":{"1min":1.425,"5min":1.425,"15min":1.425},"last":1.25}]},{"osd":7,"last update":"Sat Sep 23 18:29:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.1379999999999999,"15min":1.1379999999999999},"min":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"max":{"1min":1.5529999999999999,"5min":1.5529999999999999,"15min":1.5529999999999999},"last":1.302},{"interface":"front","average":{"1min":1.169,"5min":1.169,"15min":1.169},"min":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"max":{"1min":1.4530000000000001,"5min":1.4530000000000001,"15min":1.4530000000000001},"last":1.111}]}]},{"osd":4,"up_from":30,"seq":128849019124,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298152,"kb_used_data":1056,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447000,"statfs":{"total":95995035648,"available":95689728000,"internally_reserved":0,"allocated":1081344,"data_stored":581243,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:29:40 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.92700000000000005,"15min":0.92900000000000005},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.171,"5min":1.2809999999999999,"15min":1.2809999999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.93200000000000005,"5min":1.014,"15min":1.028},"min":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.2310000000000001,"5min":1.258,"15min":1.258},"last":0.99099999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:29:06 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.85099999999999998,"15min":0.84299999999999997},"min":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.1359999999999999,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":0.90500000000000003},{"interface":"front","average":{"1min":0.879,"5min":0.88500000000000001,"15min":0.88600000000000001},"min":{"1min":0.63400000000000001,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.198,"5min":1.224,"15min":1.224},"last":0.70699999999999996}]},{"osd":2,"last update":"Sat Sep 23 18:29:40 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":1.048,"15min":1.0489999999999999},"min":{"1min":0.72899999999999998,"5min":0.63,"15min":0.63},"max":{"1min":1.302,"5min":1.302,"15min":1.302},"last":1.1639999999999999},{"interface":"front","average":{"1min":0.95999999999999996,"5min":1.002,"15min":1.0089999999999999},"min":{"1min":0.68300000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"max":{"1min":1.2190000000000001,"5min":1.242,"15min":1.242},"last":0.79400000000000004}]},{"osd":3,"last update":"Sat Sep 23 18:29:06 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"min":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"max":{"1min":1.1899999999999999,"5min":1.1899999999999999,"15min":1.1899999999999999},"last":0.59399999999999997},{"interface":"front","average":{"1min":1.048,"5min":1.048,"15min":1.048},"min":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"max":{"1min":1.2589999999999999,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":1.0589999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:29:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":0.97699999999999998,"15min":0.97299999999999998},"min":{"1min":0.72199999999999998,"5min":0.60199999999999998,"15min":0.51600000000000001},"max":{"1min":1.264,"5min":1.3919999999999999,"15min":2.266},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.996,"5min":0.97599999999999998,"15min":0.97199999999999998},"min":{"1min":0.71499999999999997,"5min":0.58999999999999997,"15min":0.48199999999999998},"max":{"1min":1.298,"5min":1.411,"15min":2.4929999999999999},"last":0.86199999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:29:58 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.0109999999999999,"15min":1.0269999999999999},"min":{"1min":0.83599999999999997,"5min":0.65000000000000002,"15min":0.57699999999999996},"max":{"1min":1.323,"5min":1.323,"15min":2.351},"last":0.84599999999999997},{"interface":"front","average":{"1min":1.044,"5min":1.0389999999999999,"15min":1.0169999999999999},"min":{"1min":0.66800000000000004,"5min":0.63300000000000001,"15min":0.42899999999999999},"max":{"1min":1.2769999999999999,"5min":1.3660000000000001,"15min":2.621},"last":1.04}]},{"osd":7,"last update":"Sat Sep 23 18:29:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.105,"15min":1.083},"min":{"1min":0.92000000000000004,"5min":0.66200000000000003,"15min":0.50800000000000001},"max":{"1min":1.3600000000000001,"5min":1.4339999999999999,"15min":2.6819999999999999},"last":1.1080000000000001},{"interface":"front","average":{"1min":1.151,"5min":1.1579999999999999,"15min":1.169},"min":{"1min":0.86499999999999999,"5min":0.80400000000000005,"15min":0.66400000000000003},"max":{"1min":1.3520000000000001,"5min":1.4490000000000001,"15min":2.4140000000000001},"last":1.1830000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822892,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302228,"kb_used_data":5132,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93442924,"statfs":{"total":95995035648,"available":95685554176,"internally_reserved":0,"allocated":5255168,"data_stored":4770680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:29:42 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":0.85099999999999998,"15min":0.85099999999999998},"min":{"1min":0.57899999999999996,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.228,"5min":1.244,"15min":1.244},"last":0.88700000000000001},{"interface":"front","average":{"1min":0.83199999999999996,"5min":0.86399999999999999,"15min":0.872},"min":{"1min":0.56200000000000006,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.123,"5min":1.1930000000000001,"15min":1.1930000000000001},"last":0.80000000000000004}]},{"osd":1,"last update":"Sat Sep 23 18:29:09 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":0.86099999999999999,"15min":0.85299999999999998},"min":{"1min":0.61699999999999999,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.2210000000000001,"5min":1.2210000000000001,"15min":1.2210000000000001},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.83399999999999996,"5min":0.91900000000000004,"15min":0.93300000000000005},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.073,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":0.67900000000000005}]},{"osd":2,"last update":"Sat Sep 23 18:29:38 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":0.90900000000000003,"15min":0.91200000000000003},"min":{"1min":0.628,"5min":0.628,"15min":0.628},"max":{"1min":1.274,"5min":1.274,"15min":1.274},"last":1.1699999999999999},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.96199999999999997,"15min":0.95899999999999996},"min":{"1min":0.59999999999999998,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.2350000000000001,"5min":1.2350000000000001,"15min":1.2350000000000001},"last":0.99399999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:29:02 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"min":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.1319999999999999,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":1.1020000000000001},{"interface":"front","average":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"min":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.1140000000000001,"5min":1.1140000000000001,"15min":1.1140000000000001},"last":1.071}]},{"osd":4,"last update":"Sat Sep 23 18:29:09 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.92100000000000004,"15min":0.89500000000000002},"min":{"1min":0.55100000000000005,"5min":0.55000000000000004,"15min":0.39600000000000002},"max":{"1min":1.2270000000000001,"5min":1.3879999999999999,"15min":2.0259999999999998},"last":0.85699999999999998},{"interface":"front","average":{"1min":0.90400000000000003,"5min":0.93400000000000005,"15min":0.93200000000000005},"min":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.36699999999999999},"max":{"1min":1.194,"5min":1.319,"15min":1.73},"last":1.252}]},{"osd":6,"last update":"Sat Sep 23 18:29:02 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":1.014,"15min":0.99299999999999999},"min":{"1min":0.68999999999999995,"5min":0.64700000000000002,"15min":0.47399999999999998},"max":{"1min":1.2350000000000001,"5min":1.464,"15min":2.137},"last":1.2729999999999999},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.097,"15min":1.0669999999999999},"min":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.51100000000000001},"max":{"1min":1.321,"5min":1.4199999999999999,"15min":1.8779999999999999},"last":1.3169999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:29:42 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.046,"15min":1.0509999999999999},"min":{"1min":0.66600000000000004,"5min":0.65400000000000003,"15min":0.53500000000000003},"max":{"1min":1.3180000000000001,"5min":1.5589999999999999,"15min":2.1549999999999998},"last":1.2270000000000001},{"interface":"front","average":{"1min":1.087,"5min":1.0820000000000001,"15min":1.083},"min":{"1min":0.83999999999999997,"5min":0.65200000000000002,"15min":0.54300000000000004},"max":{"1min":1.3069999999999999,"5min":1.6990000000000001,"15min":2.2189999999999999},"last":1.2070000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4224,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7725,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3429,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:30:03.009 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T18:30:03.009 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T18:30:03.009 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T18:30:03.009 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T18:30:03.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:02 smithi173 bash[59422]: cluster 2023-09-23T18:30:01.633183+0000 mgr.y (mgr.26054) 338 : cluster 0 pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:04.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:03 smithi173 bash[59422]: audit 2023-09-23T18:30:02.115612+0000 mgr.y (mgr.26054) 339 : audit 0 from='client.44612 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:30:04.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:03 smithi098 bash[76817]: audit 2023-09-23T18:30:02.115612+0000 mgr.y (mgr.26054) 339 : audit 0 from='client.44612 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:30:04.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:03 smithi098 bash[79516]: audit 2023-09-23T18:30:02.115612+0000 mgr.y (mgr.26054) 339 : audit 0 from='client.44612 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:30:05.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:04 smithi173 bash[59422]: cluster 2023-09-23T18:30:03.633928+0000 mgr.y (mgr.26054) 340 : cluster 0 pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:05.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:04 smithi098 bash[76817]: cluster 2023-09-23T18:30:03.633928+0000 mgr.y (mgr.26054) 340 : cluster 0 pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:05.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:04 smithi098 bash[79516]: cluster 2023-09-23T18:30:03.633928+0000 mgr.y (mgr.26054) 340 : cluster 0 pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:06.441 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:30:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:30:06] "GET /metrics HTTP/1.1" 200 34093 "" "Prometheus/2.33.4" 2023-09-23T18:30:07.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:06 smithi173 bash[59422]: cluster 2023-09-23T18:30:05.634725+0000 mgr.y (mgr.26054) 341 : cluster 0 pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:07.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:06 smithi098 bash[79516]: cluster 2023-09-23T18:30:05.634725+0000 mgr.y (mgr.26054) 341 : cluster 0 pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:07.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:06 smithi098 bash[76817]: cluster 2023-09-23T18:30:05.634725+0000 mgr.y (mgr.26054) 341 : cluster 0 pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:07.823 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:30:08.999 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:08 smithi098 bash[76817]: cluster 2023-09-23T18:30:07.635460+0000 mgr.y (mgr.26054) 342 : cluster 0 pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:09.000 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:08 smithi098 bash[79516]: cluster 2023-09-23T18:30:07.635460+0000 mgr.y (mgr.26054) 342 : cluster 0 pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:09.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:08 smithi173 bash[59422]: cluster 2023-09-23T18:30:07.635460+0000 mgr.y (mgr.26054) 342 : cluster 0 pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:10.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:10 smithi098 bash[79516]: audit 2023-09-23T18:30:09.576653+0000 mon.a (mon.0) 502 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:30:10.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:10 smithi098 bash[79516]: cluster 2023-09-23T18:30:09.636202+0000 mgr.y (mgr.26054) 343 : cluster 0 pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:10.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:10 smithi098 bash[76817]: audit 2023-09-23T18:30:09.576653+0000 mon.a (mon.0) 502 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:30:10.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:10 smithi098 bash[76817]: cluster 2023-09-23T18:30:09.636202+0000 mgr.y (mgr.26054) 343 : cluster 0 pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:10.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:10 smithi173 bash[59422]: audit 2023-09-23T18:30:09.576653+0000 mon.a (mon.0) 502 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:30:10.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:10 smithi173 bash[59422]: cluster 2023-09-23T18:30:09.636202+0000 mgr.y (mgr.26054) 343 : cluster 0 pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:10.996 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:30:10.997 INFO:teuthology.orchestra.run.smithi098.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T18:30:11.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:11 smithi098 bash[76817]: audit 2023-09-23T18:30:10.995448+0000 mon.a (mon.0) 503 : audit 0 from='client.? 172.21.15.98:0/4114580241' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T18:30:11.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:11 smithi098 bash[79516]: audit 2023-09-23T18:30:10.995448+0000 mon.a (mon.0) 503 : audit 0 from='client.? 172.21.15.98:0/4114580241' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T18:30:11.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:11 smithi173 bash[59422]: audit 2023-09-23T18:30:10.995448+0000 mon.a (mon.0) 503 : audit 0 from='client.? 172.21.15.98:0/4114580241' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T18:30:11.862 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T18:30:11.862 INFO:teuthology.run_tasks:Running task print... 2023-09-23T18:30:11.871 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-09-23T18:30:11.871 INFO:teuthology.run_tasks:Running task parallel... 2023-09-23T18:30:11.915 INFO:teuthology.task.parallel:starting parallel... 2023-09-23T18:30:11.915 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T18:30:11.916 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-09-23T18:30:11.917 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-09-23T18:30:11.917 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}}, 'kernel': {'hwe': True}, 'workunit': {'branch': 'reef', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}} 2023-09-23T18:30:11.917 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-09-23T18:30:11.917 INFO:tasks.thrashosds:Beginning thrashosds... 2023-09-23T18:30:11.917 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T18:30:11.918 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:30:12.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:12 smithi098 bash[79516]: cluster 2023-09-23T18:30:11.637154+0000 mgr.y (mgr.26054) 344 : cluster 0 pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:12.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:12 smithi098 bash[76817]: cluster 2023-09-23T18:30:11.637154+0000 mgr.y (mgr.26054) 344 : cluster 0 pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:12.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:12 smithi173 bash[59422]: cluster 2023-09-23T18:30:11.637154+0000 mgr.y (mgr.26054) 344 : cluster 0 pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:15.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:14 smithi173 bash[59422]: cluster 2023-09-23T18:30:13.637879+0000 mgr.y (mgr.26054) 345 : cluster 0 pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:15.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:14 smithi098 bash[76817]: cluster 2023-09-23T18:30:13.637879+0000 mgr.y (mgr.26054) 345 : cluster 0 pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:15.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:14 smithi098 bash[79516]: cluster 2023-09-23T18:30:13.637879+0000 mgr.y (mgr.26054) 345 : cluster 0 pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:16.441 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:30:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:30:16] "GET /metrics HTTP/1.1" 200 34089 "" "Prometheus/2.33.4" 2023-09-23T18:30:16.696 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:30:17.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:16 smithi173 bash[59422]: cluster 2023-09-23T18:30:15.638775+0000 mgr.y (mgr.26054) 346 : cluster 0 pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:17.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:16 smithi098 bash[79516]: cluster 2023-09-23T18:30:15.638775+0000 mgr.y (mgr.26054) 346 : cluster 0 pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:17.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:16 smithi098 bash[76817]: cluster 2023-09-23T18:30:15.638775+0000 mgr.y (mgr.26054) 346 : cluster 0 pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:18.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:18 smithi098 bash[76817]: cluster 2023-09-23T18:30:17.639550+0000 mgr.y (mgr.26054) 347 : cluster 0 pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:18.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:18 smithi098 bash[79516]: cluster 2023-09-23T18:30:17.639550+0000 mgr.y (mgr.26054) 347 : cluster 0 pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:19.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:18 smithi173 bash[59422]: cluster 2023-09-23T18:30:17.639550+0000 mgr.y (mgr.26054) 347 : cluster 0 pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:19.747 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:30:19.752 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-23T18:30:20.537 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":283,"stamp":"2023-09-23T18:30:19.639827+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6303,"num_read_kb":13760,"num_write":1106,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1341076,"kb_used_data":22548,"kb_used_omap":31,"kb_used_meta":1318432,"kb_avail":748620140,"statfs":{"total":767960285184,"available":766587023360,"internally_reserved":0,"allocated":23089152,"data_stored":19014596,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32507,"internal_metadata":1350074629},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004867"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":63,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.289993+0000","last_change":"2023-09-23T18:15:43.682873+0000","last_active":"2023-09-23T18:28:02.289993+0000","last_peered":"2023-09-23T18:28:02.289993+0000","last_clean":"2023-09-23T18:28:02.289993+0000","last_became_active":"2023-09-23T18:15:43.682465+0000","last_became_peered":"2023-09-23T18:15:43.682465+0000","last_unstale":"2023-09-23T18:28:02.289993+0000","last_undegraded":"2023-09-23T18:28:02.289993+0000","last_fullsized":"2023-09-23T18:28:02.289993+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.280010+0000","last_change":"2023-09-23T18:27:33.583935+0000","last_active":"2023-09-23T18:28:03.280010+0000","last_peered":"2023-09-23T18:28:03.280010+0000","last_clean":"2023-09-23T18:28:03.280010+0000","last_became_active":"2023-09-23T18:27:33.582561+0000","last_became_peered":"2023-09-23T18:27:33.582561+0000","last_unstale":"2023-09-23T18:28:03.280010+0000","last_undegraded":"2023-09-23T18:28:03.280010+0000","last_fullsized":"2023-09-23T18:28:03.280010+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:43.589363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":90,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.299520+0000","last_change":"2023-09-23T18:28:03.299520+0000","last_active":"2023-09-23T18:28:03.299520+0000","last_peered":"2023-09-23T18:28:03.299520+0000","last_clean":"2023-09-23T18:28:03.299520+0000","last_became_active":"2023-09-23T18:28:03.299158+0000","last_became_peered":"2023-09-23T18:28:03.299158+0000","last_unstale":"2023-09-23T18:28:03.299520+0000","last_undegraded":"2023-09-23T18:28:03.299520+0000","last_fullsized":"2023-09-23T18:28:03.299520+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:26.247064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":78,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.297944+0000","last_change":"2023-09-23T18:28:03.296850+0000","last_active":"2023-09-23T18:28:03.297944+0000","last_peered":"2023-09-23T18:28:03.297944+0000","last_clean":"2023-09-23T18:28:03.297944+0000","last_became_active":"2023-09-23T18:28:03.295842+0000","last_became_peered":"2023-09-23T18:28:03.295842+0000","last_unstale":"2023-09-23T18:28:03.297944+0000","last_undegraded":"2023-09-23T18:28:03.297944+0000","last_fullsized":"2023-09-23T18:28:03.297944+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:53.048579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.302261+0000","last_change":"2023-09-23T18:28:03.302261+0000","last_active":"2023-09-23T18:28:03.302261+0000","last_peered":"2023-09-23T18:28:03.302261+0000","last_clean":"2023-09-23T18:28:03.302261+0000","last_became_active":"2023-09-23T18:28:03.301911+0000","last_became_peered":"2023-09-23T18:28:03.301911+0000","last_unstale":"2023-09-23T18:28:03.302261+0000","last_undegraded":"2023-09-23T18:28:03.302261+0000","last_fullsized":"2023-09-23T18:28:03.302261+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":57,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.279119+0000","last_change":"2023-09-23T18:15:08.902664+0000","last_active":"2023-09-23T18:28:02.279119+0000","last_peered":"2023-09-23T18:28:02.279119+0000","last_clean":"2023-09-23T18:28:02.279119+0000","last_became_active":"2023-09-23T18:15:08.901252+0000","last_became_peered":"2023-09-23T18:15:08.901252+0000","last_unstale":"2023-09-23T18:28:02.279119+0000","last_undegraded":"2023-09-23T18:28:02.279119+0000","last_fullsized":"2023-09-23T18:28:02.279119+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.300718+0000","last_change":"2023-09-23T18:28:03.300718+0000","last_active":"2023-09-23T18:28:03.300718+0000","last_peered":"2023-09-23T18:28:03.300718+0000","last_clean":"2023-09-23T18:28:03.300718+0000","last_became_active":"2023-09-23T18:28:03.300298+0000","last_became_peered":"2023-09-23T18:28:03.300298+0000","last_unstale":"2023-09-23T18:28:03.300718+0000","last_undegraded":"2023-09-23T18:28:03.300718+0000","last_fullsized":"2023-09-23T18:28:03.300718+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.278761+0000","last_change":"2023-09-23T18:27:04.250817+0000","last_active":"2023-09-23T18:28:02.278761+0000","last_peered":"2023-09-23T18:28:02.278761+0000","last_clean":"2023-09-23T18:28:02.278761+0000","last_became_active":"2023-09-23T18:27:04.250397+0000","last_became_peered":"2023-09-23T18:27:04.250397+0000","last_unstale":"2023-09-23T18:28:02.278761+0000","last_undegraded":"2023-09-23T18:28:02.278761+0000","last_fullsized":"2023-09-23T18:28:02.278761+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":92,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.288758+0000","last_change":"2023-09-23T18:28:03.288092+0000","last_active":"2023-09-23T18:28:03.288758+0000","last_peered":"2023-09-23T18:28:03.288758+0000","last_clean":"2023-09-23T18:28:03.288758+0000","last_became_active":"2023-09-23T18:28:03.287643+0000","last_became_peered":"2023-09-23T18:28:03.287643+0000","last_unstale":"2023-09-23T18:28:03.288758+0000","last_undegraded":"2023-09-23T18:28:03.288758+0000","last_fullsized":"2023-09-23T18:28:03.288758+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:44.163816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":41,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.280816+0000","last_change":"2023-09-23T18:26:34.994176+0000","last_active":"2023-09-23T18:28:03.280816+0000","last_peered":"2023-09-23T18:28:03.280816+0000","last_clean":"2023-09-23T18:28:03.280816+0000","last_became_active":"2023-09-23T18:26:34.993829+0000","last_became_peered":"2023-09-23T18:26:34.993829+0000","last_unstale":"2023-09-23T18:28:03.280816+0000","last_undegraded":"2023-09-23T18:28:03.280816+0000","last_fullsized":"2023-09-23T18:28:03.280816+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:27.111857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.293000+0000","last_change":"2023-09-23T18:15:09.125623+0000","last_active":"2023-09-23T18:28:02.293000+0000","last_peered":"2023-09-23T18:28:02.293000+0000","last_clean":"2023-09-23T18:28:02.293000+0000","last_became_active":"2023-09-23T18:15:09.125018+0000","last_became_peered":"2023-09-23T18:15:09.125018+0000","last_unstale":"2023-09-23T18:28:02.293000+0000","last_undegraded":"2023-09-23T18:28:02.293000+0000","last_fullsized":"2023-09-23T18:28:02.293000+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.281601+0000","last_change":"2023-09-23T18:27:04.247514+0000","last_active":"2023-09-23T18:28:03.281601+0000","last_peered":"2023-09-23T18:28:03.281601+0000","last_clean":"2023-09-23T18:28:03.281601+0000","last_became_active":"2023-09-23T18:27:04.247025+0000","last_became_peered":"2023-09-23T18:27:04.247025+0000","last_unstale":"2023-09-23T18:28:03.281601+0000","last_undegraded":"2023-09-23T18:28:03.281601+0000","last_fullsized":"2023-09-23T18:28:03.281601+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:32.498164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"77'112","reported_seq":594,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.284556+0000","last_change":"2023-09-23T18:27:33.587478+0000","last_active":"2023-09-23T18:28:02.284556+0000","last_peered":"2023-09-23T18:28:02.284556+0000","last_clean":"2023-09-23T18:28:02.284556+0000","last_became_active":"2023-09-23T18:27:33.586958+0000","last_became_peered":"2023-09-23T18:27:33.586958+0000","last_unstale":"2023-09-23T18:28:02.284556+0000","last_undegraded":"2023-09-23T18:28:02.284556+0000","last_fullsized":"2023-09-23T18:28:02.284556+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":893,"num_read_kb":3172,"num_write":69,"num_write_kb":2237,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.293466+0000","last_change":"2023-09-23T18:28:03.292295+0000","last_active":"2023-09-23T18:28:03.293466+0000","last_peered":"2023-09-23T18:28:03.293466+0000","last_clean":"2023-09-23T18:28:03.293466+0000","last_became_active":"2023-09-23T18:28:03.291902+0000","last_became_peered":"2023-09-23T18:28:03.291902+0000","last_unstale":"2023-09-23T18:28:03.293466+0000","last_undegraded":"2023-09-23T18:28:03.293466+0000","last_fullsized":"2023-09-23T18:28:03.293466+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:38.415256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'181","reported_seq":591,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.282681+0000","last_change":"2023-09-23T18:27:04.256906+0000","last_active":"2023-09-23T18:28:03.282681+0000","last_peered":"2023-09-23T18:28:03.282681+0000","last_clean":"2023-09-23T18:28:03.282681+0000","last_became_active":"2023-09-23T18:27:04.255734+0000","last_became_peered":"2023-09-23T18:27:04.255734+0000","last_unstale":"2023-09-23T18:28:03.282681+0000","last_undegraded":"2023-09-23T18:28:03.282681+0000","last_fullsized":"2023-09-23T18:28:03.282681+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":181,"log_dups_size":0,"ondisk_log_size":181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:53.270831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":909,"num_read_kb":734,"num_write":148,"num_write_kb":75,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.299389+0000","last_change":"2023-09-23T18:28:03.298805+0000","last_active":"2023-09-23T18:28:03.299389+0000","last_peered":"2023-09-23T18:28:03.299389+0000","last_clean":"2023-09-23T18:28:03.299389+0000","last_became_active":"2023-09-23T18:28:03.298437+0000","last_became_peered":"2023-09-23T18:28:03.298437+0000","last_unstale":"2023-09-23T18:28:03.299389+0000","last_undegraded":"2023-09-23T18:28:03.299389+0000","last_fullsized":"2023-09-23T18:28:03.299389+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:27.831669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.279808+0000","last_change":"2023-09-23T18:26:34.990770+0000","last_active":"2023-09-23T18:28:03.279808+0000","last_peered":"2023-09-23T18:28:03.279808+0000","last_clean":"2023-09-23T18:28:03.279808+0000","last_became_active":"2023-09-23T18:26:34.990262+0000","last_became_peered":"2023-09-23T18:26:34.990262+0000","last_unstale":"2023-09-23T18:28:03.279808+0000","last_undegraded":"2023-09-23T18:28:03.279808+0000","last_fullsized":"2023-09-23T18:28:03.279808+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:07.982834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.281672+0000","last_change":"2023-09-23T18:27:33.579367+0000","last_active":"2023-09-23T18:28:03.281672+0000","last_peered":"2023-09-23T18:28:03.281672+0000","last_clean":"2023-09-23T18:28:03.281672+0000","last_became_active":"2023-09-23T18:27:33.578607+0000","last_became_peered":"2023-09-23T18:27:33.578607+0000","last_unstale":"2023-09-23T18:28:03.281672+0000","last_undegraded":"2023-09-23T18:28:03.281672+0000","last_fullsized":"2023-09-23T18:28:03.281672+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:58:35.628296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'78","reported_seq":340,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.286740+0000","last_change":"2023-09-23T18:26:34.992345+0000","last_active":"2023-09-23T18:28:02.286740+0000","last_peered":"2023-09-23T18:28:02.286740+0000","last_clean":"2023-09-23T18:28:02.286740+0000","last_became_active":"2023-09-23T18:26:34.991774+0000","last_became_peered":"2023-09-23T18:26:34.991774+0000","last_unstale":"2023-09-23T18:28:02.286740+0000","last_undegraded":"2023-09-23T18:28:02.286740+0000","last_fullsized":"2023-09-23T18:28:02.286740+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":78,"log_dups_size":0,"ondisk_log_size":78,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":428,"num_read_kb":1125,"num_write":73,"num_write_kb":431,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.281734+0000","last_change":"2023-09-23T18:27:33.577544+0000","last_active":"2023-09-23T18:28:03.281734+0000","last_peered":"2023-09-23T18:28:03.281734+0000","last_clean":"2023-09-23T18:28:03.281734+0000","last_became_active":"2023-09-23T18:27:33.577194+0000","last_became_peered":"2023-09-23T18:27:33.577194+0000","last_unstale":"2023-09-23T18:28:03.281734+0000","last_undegraded":"2023-09-23T18:28:03.281734+0000","last_fullsized":"2023-09-23T18:28:03.281734+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:49.633444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.299894+0000","last_change":"2023-09-23T18:28:03.299508+0000","last_active":"2023-09-23T18:28:03.299894+0000","last_peered":"2023-09-23T18:28:03.299894+0000","last_clean":"2023-09-23T18:28:03.299894+0000","last_became_active":"2023-09-23T18:28:03.298682+0000","last_became_peered":"2023-09-23T18:28:03.298682+0000","last_unstale":"2023-09-23T18:28:03.299894+0000","last_undegraded":"2023-09-23T18:28:03.299894+0000","last_fullsized":"2023-09-23T18:28:03.299894+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:08.398302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'120","reported_seq":550,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.282230+0000","last_change":"2023-09-23T18:27:04.257709+0000","last_active":"2023-09-23T18:28:02.282230+0000","last_peered":"2023-09-23T18:28:02.282230+0000","last_clean":"2023-09-23T18:28:02.282230+0000","last_became_active":"2023-09-23T18:27:04.257331+0000","last_became_peered":"2023-09-23T18:27:04.257331+0000","last_unstale":"2023-09-23T18:28:02.282230+0000","last_undegraded":"2023-09-23T18:28:02.282230+0000","last_fullsized":"2023-09-23T18:28:02.282230+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":120,"log_dups_size":0,"ondisk_log_size":120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":809,"num_read_kb":2749,"num_write":98,"num_write_kb":2087,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.283659+0000","last_change":"2023-09-23T18:15:08.921629+0000","last_active":"2023-09-23T18:28:02.283659+0000","last_peered":"2023-09-23T18:28:02.283659+0000","last_clean":"2023-09-23T18:28:02.283659+0000","last_became_active":"2023-09-23T18:15:08.921283+0000","last_became_peered":"2023-09-23T18:15:08.921283+0000","last_unstale":"2023-09-23T18:28:02.283659+0000","last_undegraded":"2023-09-23T18:28:02.283659+0000","last_fullsized":"2023-09-23T18:28:02.283659+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":63,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.298886+0000","last_change":"2023-09-23T18:28:03.295577+0000","last_active":"2023-09-23T18:28:03.298886+0000","last_peered":"2023-09-23T18:28:03.298886+0000","last_clean":"2023-09-23T18:28:03.298886+0000","last_became_active":"2023-09-23T18:28:03.293777+0000","last_became_peered":"2023-09-23T18:28:03.293777+0000","last_unstale":"2023-09-23T18:28:03.298886+0000","last_undegraded":"2023-09-23T18:28:03.298886+0000","last_fullsized":"2023-09-23T18:28:03.298886+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:49.714764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":306,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.282044+0000","last_change":"2023-09-23T18:26:34.999402+0000","last_active":"2023-09-23T18:28:02.282044+0000","last_peered":"2023-09-23T18:28:02.282044+0000","last_clean":"2023-09-23T18:28:02.282044+0000","last_became_active":"2023-09-23T18:26:34.998828+0000","last_became_peered":"2023-09-23T18:26:34.998828+0000","last_unstale":"2023-09-23T18:28:02.282044+0000","last_undegraded":"2023-09-23T18:28:02.282044+0000","last_fullsized":"2023-09-23T18:28:02.282044+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_clean_scrub_stamp":"2023-09-23T18:08:35.701374+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"77'147","reported_seq":503,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.277795+0000","last_change":"2023-09-23T18:27:33.583558+0000","last_active":"2023-09-23T18:28:02.277795+0000","last_peered":"2023-09-23T18:28:02.277795+0000","last_clean":"2023-09-23T18:28:02.277795+0000","last_became_active":"2023-09-23T18:27:33.583003+0000","last_became_peered":"2023-09-23T18:27:33.583003+0000","last_unstale":"2023-09-23T18:28:02.277795+0000","last_undegraded":"2023-09-23T18:28:02.277795+0000","last_fullsized":"2023-09-23T18:28:02.277795+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":625,"num_read_kb":1869,"num_write":143,"num_write_kb":5368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":78,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.282670+0000","last_change":"2023-09-23T18:27:04.242813+0000","last_active":"2023-09-23T18:28:03.282670+0000","last_peered":"2023-09-23T18:28:03.282670+0000","last_clean":"2023-09-23T18:28:03.282670+0000","last_became_active":"2023-09-23T18:27:04.242409+0000","last_became_peered":"2023-09-23T18:27:04.242409+0000","last_unstale":"2023-09-23T18:28:03.282670+0000","last_undegraded":"2023-09-23T18:28:03.282670+0000","last_fullsized":"2023-09-23T18:28:03.282670+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:01.836849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.275061+0000","last_change":"2023-09-23T18:27:33.585279+0000","last_active":"2023-09-23T18:28:03.275061+0000","last_peered":"2023-09-23T18:28:03.275061+0000","last_clean":"2023-09-23T18:28:03.275061+0000","last_became_active":"2023-09-23T18:27:33.584928+0000","last_became_peered":"2023-09-23T18:27:33.584928+0000","last_unstale":"2023-09-23T18:28:03.275061+0000","last_undegraded":"2023-09-23T18:28:03.275061+0000","last_fullsized":"2023-09-23T18:28:03.275061+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:07.000233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'112","reported_seq":539,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.297866+0000","last_change":"2023-09-23T18:28:03.297540+0000","last_active":"2023-09-23T18:28:03.297866+0000","last_peered":"2023-09-23T18:28:03.297866+0000","last_clean":"2023-09-23T18:28:03.297866+0000","last_became_active":"2023-09-23T18:28:03.296649+0000","last_became_peered":"2023-09-23T18:28:03.296649+0000","last_unstale":"2023-09-23T18:28:03.297866+0000","last_undegraded":"2023-09-23T18:28:03.297866+0000","last_fullsized":"2023-09-23T18:28:03.297866+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:32.531775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":774,"num_read_kb":596,"num_write":102,"num_write_kb":52,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.289185+0000","last_change":"2023-09-23T18:28:03.288707+0000","last_active":"2023-09-23T18:28:03.289185+0000","last_peered":"2023-09-23T18:28:03.289185+0000","last_clean":"2023-09-23T18:28:03.289185+0000","last_became_active":"2023-09-23T18:28:03.288335+0000","last_became_peered":"2023-09-23T18:28:03.288335+0000","last_unstale":"2023-09-23T18:28:03.289185+0000","last_undegraded":"2023-09-23T18:28:03.289185+0000","last_fullsized":"2023-09-23T18:28:03.289185+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:57.652706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.292438+0000","last_change":"2023-09-23T18:26:34.992535+0000","last_active":"2023-09-23T18:28:02.292438+0000","last_peered":"2023-09-23T18:28:02.292438+0000","last_clean":"2023-09-23T18:28:02.292438+0000","last_became_active":"2023-09-23T18:26:34.992177+0000","last_became_peered":"2023-09-23T18:26:34.992177+0000","last_unstale":"2023-09-23T18:28:02.292438+0000","last_undegraded":"2023-09-23T18:28:02.292438+0000","last_fullsized":"2023-09-23T18:28:02.292438+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"77'96","reported_seq":619,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.277710+0000","last_change":"2023-09-23T18:27:04.250427+0000","last_active":"2023-09-23T18:28:02.277710+0000","last_peered":"2023-09-23T18:28:02.277710+0000","last_clean":"2023-09-23T18:28:02.277710+0000","last_became_active":"2023-09-23T18:27:04.249907+0000","last_became_peered":"2023-09-23T18:27:04.249907+0000","last_unstale":"2023-09-23T18:28:02.277710+0000","last_undegraded":"2023-09-23T18:28:02.277710+0000","last_fullsized":"2023-09-23T18:28:02.277710+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":96,"log_dups_size":0,"ondisk_log_size":96,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":785,"num_read_kb":1291,"num_write":87,"num_write_kb":338,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.275431+0000","last_change":"2023-09-23T18:27:04.251552+0000","last_active":"2023-09-23T18:28:03.275431+0000","last_peered":"2023-09-23T18:28:03.275431+0000","last_clean":"2023-09-23T18:28:03.275431+0000","last_became_active":"2023-09-23T18:27:04.250901+0000","last_became_peered":"2023-09-23T18:27:04.250901+0000","last_unstale":"2023-09-23T18:28:03.275431+0000","last_undegraded":"2023-09-23T18:28:03.275431+0000","last_fullsized":"2023-09-23T18:28:03.275431+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:22.322137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.297594+0000","last_change":"2023-09-23T18:28:03.297594+0000","last_active":"2023-09-23T18:28:03.297594+0000","last_peered":"2023-09-23T18:28:03.297594+0000","last_clean":"2023-09-23T18:28:03.297594+0000","last_became_active":"2023-09-23T18:28:03.296725+0000","last_became_peered":"2023-09-23T18:28:03.296725+0000","last_unstale":"2023-09-23T18:28:03.297594+0000","last_undegraded":"2023-09-23T18:28:03.297594+0000","last_fullsized":"2023-09-23T18:28:03.297594+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'130","reported_seq":583,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.281567+0000","last_change":"2023-09-23T18:27:04.250265+0000","last_active":"2023-09-23T18:28:03.281567+0000","last_peered":"2023-09-23T18:28:03.281567+0000","last_clean":"2023-09-23T18:28:03.281567+0000","last_became_active":"2023-09-23T18:27:04.249097+0000","last_became_peered":"2023-09-23T18:27:04.249097+0000","last_unstale":"2023-09-23T18:28:03.281567+0000","last_undegraded":"2023-09-23T18:28:03.281567+0000","last_fullsized":"2023-09-23T18:28:03.281567+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:27.143225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":772,"num_read_kb":1712,"num_write":126,"num_write_kb":2126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.295387+0000","last_change":"2023-09-23T18:28:03.294782+0000","last_active":"2023-09-23T18:28:03.295387+0000","last_peered":"2023-09-23T18:28:03.295387+0000","last_clean":"2023-09-23T18:28:03.295387+0000","last_became_active":"2023-09-23T18:28:03.294430+0000","last_became_peered":"2023-09-23T18:28:03.294430+0000","last_unstale":"2023-09-23T18:28:03.295387+0000","last_undegraded":"2023-09-23T18:28:03.295387+0000","last_fullsized":"2023-09-23T18:28:03.295387+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:20.226197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.291041+0000","last_change":"2023-09-23T18:28:03.287471+0000","last_active":"2023-09-23T18:28:03.291041+0000","last_peered":"2023-09-23T18:28:03.291041+0000","last_clean":"2023-09-23T18:28:03.291041+0000","last_became_active":"2023-09-23T18:28:03.286923+0000","last_became_peered":"2023-09-23T18:28:03.286923+0000","last_unstale":"2023-09-23T18:28:03.291041+0000","last_undegraded":"2023-09-23T18:28:03.291041+0000","last_fullsized":"2023-09-23T18:28:03.291041+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:10.505004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.291109+0000","last_change":"2023-09-23T18:27:04.254684+0000","last_active":"2023-09-23T18:28:02.291109+0000","last_peered":"2023-09-23T18:28:02.291109+0000","last_clean":"2023-09-23T18:28:02.291109+0000","last_became_active":"2023-09-23T18:27:04.254014+0000","last_became_peered":"2023-09-23T18:27:04.254014+0000","last_unstale":"2023-09-23T18:28:02.291109+0000","last_undegraded":"2023-09-23T18:28:02.291109+0000","last_fullsized":"2023-09-23T18:28:02.291109+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.293127+0000","last_change":"2023-09-23T18:27:33.585714+0000","last_active":"2023-09-23T18:28:02.293127+0000","last_peered":"2023-09-23T18:28:02.293127+0000","last_clean":"2023-09-23T18:28:02.293127+0000","last_became_active":"2023-09-23T18:27:33.584641+0000","last_became_peered":"2023-09-23T18:27:33.584641+0000","last_unstale":"2023-09-23T18:28:02.293127+0000","last_undegraded":"2023-09-23T18:28:02.293127+0000","last_fullsized":"2023-09-23T18:28:02.293127+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":84,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.291696+0000","last_change":"2023-09-23T18:27:33.588308+0000","last_active":"2023-09-23T18:28:02.291696+0000","last_peered":"2023-09-23T18:28:02.291696+0000","last_clean":"2023-09-23T18:28:02.291696+0000","last_became_active":"2023-09-23T18:27:33.587974+0000","last_became_peered":"2023-09-23T18:27:33.587974+0000","last_unstale":"2023-09-23T18:28:02.291696+0000","last_undegraded":"2023-09-23T18:28:02.291696+0000","last_fullsized":"2023-09-23T18:28:02.291696+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.284595+0000","last_change":"2023-09-23T18:27:04.252119+0000","last_active":"2023-09-23T18:28:02.284595+0000","last_peered":"2023-09-23T18:28:02.284595+0000","last_clean":"2023-09-23T18:28:02.284595+0000","last_became_active":"2023-09-23T18:27:04.251714+0000","last_became_peered":"2023-09-23T18:27:04.251714+0000","last_unstale":"2023-09-23T18:28:02.284595+0000","last_undegraded":"2023-09-23T18:28:02.284595+0000","last_fullsized":"2023-09-23T18:28:02.284595+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.302326+0000","last_change":"2023-09-23T18:28:03.302326+0000","last_active":"2023-09-23T18:28:03.302326+0000","last_peered":"2023-09-23T18:28:03.302326+0000","last_clean":"2023-09-23T18:28:03.302326+0000","last_became_active":"2023-09-23T18:28:03.301970+0000","last_became_peered":"2023-09-23T18:28:03.301970+0000","last_unstale":"2023-09-23T18:28:03.302326+0000","last_undegraded":"2023-09-23T18:28:03.302326+0000","last_fullsized":"2023-09-23T18:28:03.302326+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.298459+0000","last_change":"2023-09-23T18:28:03.297628+0000","last_active":"2023-09-23T18:28:03.298459+0000","last_peered":"2023-09-23T18:28:03.298459+0000","last_clean":"2023-09-23T18:28:03.298459+0000","last_became_active":"2023-09-23T18:28:03.296694+0000","last_became_peered":"2023-09-23T18:28:03.296694+0000","last_unstale":"2023-09-23T18:28:03.298459+0000","last_undegraded":"2023-09-23T18:28:03.298459+0000","last_fullsized":"2023-09-23T18:28:03.298459+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:43.375652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":65,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.280061+0000","last_change":"2023-09-23T18:27:33.582033+0000","last_active":"2023-09-23T18:28:03.280061+0000","last_peered":"2023-09-23T18:28:03.280061+0000","last_clean":"2023-09-23T18:28:03.280061+0000","last_became_active":"2023-09-23T18:27:33.581667+0000","last_became_peered":"2023-09-23T18:27:33.581667+0000","last_unstale":"2023-09-23T18:28:03.280061+0000","last_undegraded":"2023-09-23T18:28:03.280061+0000","last_fullsized":"2023-09-23T18:28:03.280061+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:36.039922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.298611+0000","last_change":"2023-09-23T18:28:03.298611+0000","last_active":"2023-09-23T18:28:03.298611+0000","last_peered":"2023-09-23T18:28:03.298611+0000","last_clean":"2023-09-23T18:28:03.298611+0000","last_became_active":"2023-09-23T18:28:03.298233+0000","last_became_peered":"2023-09-23T18:28:03.298233+0000","last_unstale":"2023-09-23T18:28:03.298611+0000","last_undegraded":"2023-09-23T18:28:03.298611+0000","last_fullsized":"2023-09-23T18:28:03.298611+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":78,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.284278+0000","last_change":"2023-09-23T18:27:33.589116+0000","last_active":"2023-09-23T18:28:02.284278+0000","last_peered":"2023-09-23T18:28:02.284278+0000","last_clean":"2023-09-23T18:28:02.284278+0000","last_became_active":"2023-09-23T18:27:33.588809+0000","last_became_peered":"2023-09-23T18:27:33.588809+0000","last_unstale":"2023-09-23T18:28:02.284278+0000","last_undegraded":"2023-09-23T18:28:02.284278+0000","last_fullsized":"2023-09-23T18:28:02.284278+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":56,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.286875+0000","last_change":"2023-09-23T18:15:08.908376+0000","last_active":"2023-09-23T18:28:02.286875+0000","last_peered":"2023-09-23T18:28:02.286875+0000","last_clean":"2023-09-23T18:28:02.286875+0000","last_became_active":"2023-09-23T18:15:08.907752+0000","last_became_peered":"2023-09-23T18:15:08.907752+0000","last_unstale":"2023-09-23T18:28:02.286875+0000","last_undegraded":"2023-09-23T18:28:02.286875+0000","last_fullsized":"2023-09-23T18:28:02.286875+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.302292+0000","last_change":"2023-09-23T18:28:03.302292+0000","last_active":"2023-09-23T18:28:03.302292+0000","last_peered":"2023-09-23T18:28:03.302292+0000","last_clean":"2023-09-23T18:28:03.302292+0000","last_became_active":"2023-09-23T18:28:03.301927+0000","last_became_peered":"2023-09-23T18:28:03.301927+0000","last_unstale":"2023-09-23T18:28:03.302292+0000","last_undegraded":"2023-09-23T18:28:03.302292+0000","last_fullsized":"2023-09-23T18:28:03.302292+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.286163+0000","last_change":"2023-09-23T18:27:04.253991+0000","last_active":"2023-09-23T18:28:02.286163+0000","last_peered":"2023-09-23T18:28:02.286163+0000","last_clean":"2023-09-23T18:28:02.286163+0000","last_became_active":"2023-09-23T18:27:04.253593+0000","last_became_peered":"2023-09-23T18:27:04.253593+0000","last_unstale":"2023-09-23T18:28:02.286163+0000","last_undegraded":"2023-09-23T18:28:02.286163+0000","last_fullsized":"2023-09-23T18:28:02.286163+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.277381+0000","last_change":"2023-09-23T18:15:10.327919+0000","last_active":"2023-09-23T18:28:02.277381+0000","last_peered":"2023-09-23T18:28:02.277381+0000","last_clean":"2023-09-23T18:28:02.277381+0000","last_became_active":"2023-09-23T18:15:10.327052+0000","last_became_peered":"2023-09-23T18:15:10.327052+0000","last_unstale":"2023-09-23T18:28:02.277381+0000","last_undegraded":"2023-09-23T18:28:02.277381+0000","last_fullsized":"2023-09-23T18:28:02.277381+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.286482+0000","last_change":"2023-09-23T18:26:34.999684+0000","last_active":"2023-09-23T18:28:02.286482+0000","last_peered":"2023-09-23T18:28:02.286482+0000","last_clean":"2023-09-23T18:28:02.286482+0000","last_became_active":"2023-09-23T18:26:34.999076+0000","last_became_peered":"2023-09-23T18:26:34.999076+0000","last_unstale":"2023-09-23T18:28:02.286482+0000","last_undegraded":"2023-09-23T18:28:02.286482+0000","last_fullsized":"2023-09-23T18:28:02.286482+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.295838+0000","last_change":"2023-09-23T18:28:03.295838+0000","last_active":"2023-09-23T18:28:03.295838+0000","last_peered":"2023-09-23T18:28:03.295838+0000","last_clean":"2023-09-23T18:28:03.295838+0000","last_became_active":"2023-09-23T18:28:03.291774+0000","last_became_peered":"2023-09-23T18:28:03.291774+0000","last_unstale":"2023-09-23T18:28:03.295838+0000","last_undegraded":"2023-09-23T18:28:03.295838+0000","last_fullsized":"2023-09-23T18:28:03.295838+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:23.166353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":78,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.284152+0000","last_change":"2023-09-23T18:26:34.995333+0000","last_active":"2023-09-23T18:28:02.284152+0000","last_peered":"2023-09-23T18:28:02.284152+0000","last_clean":"2023-09-23T18:28:02.284152+0000","last_became_active":"2023-09-23T18:26:34.994983+0000","last_became_peered":"2023-09-23T18:26:34.994983+0000","last_unstale":"2023-09-23T18:28:02.284152+0000","last_undegraded":"2023-09-23T18:28:02.284152+0000","last_fullsized":"2023-09-23T18:28:02.284152+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.290583+0000","last_change":"2023-09-23T18:28:03.290217+0000","last_active":"2023-09-23T18:28:03.290583+0000","last_peered":"2023-09-23T18:28:03.290583+0000","last_clean":"2023-09-23T18:28:03.290583+0000","last_became_active":"2023-09-23T18:28:03.289888+0000","last_became_peered":"2023-09-23T18:28:03.289888+0000","last_unstale":"2023-09-23T18:28:03.290583+0000","last_undegraded":"2023-09-23T18:28:03.290583+0000","last_fullsized":"2023-09-23T18:28:03.290583+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:51:45.400847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.281244+0000","last_change":"2023-09-23T18:15:08.916658+0000","last_active":"2023-09-23T18:28:02.281244+0000","last_peered":"2023-09-23T18:28:02.281244+0000","last_clean":"2023-09-23T18:28:02.281244+0000","last_became_active":"2023-09-23T18:15:08.916301+0000","last_became_peered":"2023-09-23T18:15:08.916301+0000","last_unstale":"2023-09-23T18:28:02.281244+0000","last_undegraded":"2023-09-23T18:28:02.281244+0000","last_fullsized":"2023-09-23T18:28:02.281244+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.297841+0000","last_change":"2023-09-23T18:28:03.297841+0000","last_active":"2023-09-23T18:28:03.297841+0000","last_peered":"2023-09-23T18:28:03.297841+0000","last_clean":"2023-09-23T18:28:03.297841+0000","last_became_active":"2023-09-23T18:28:03.297216+0000","last_became_peered":"2023-09-23T18:28:03.297216+0000","last_unstale":"2023-09-23T18:28:03.297841+0000","last_undegraded":"2023-09-23T18:28:03.297841+0000","last_fullsized":"2023-09-23T18:28:03.297841+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.290591+0000","last_change":"2023-09-23T18:28:03.290316+0000","last_active":"2023-09-23T18:28:03.290591+0000","last_peered":"2023-09-23T18:28:03.290591+0000","last_clean":"2023-09-23T18:28:03.290591+0000","last_became_active":"2023-09-23T18:28:03.289698+0000","last_became_peered":"2023-09-23T18:28:03.289698+0000","last_unstale":"2023-09-23T18:28:03.290591+0000","last_undegraded":"2023-09-23T18:28:03.290591+0000","last_fullsized":"2023-09-23T18:28:03.290591+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:00.779542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.292363+0000","last_change":"2023-09-23T18:28:03.292363+0000","last_active":"2023-09-23T18:28:03.292363+0000","last_peered":"2023-09-23T18:28:03.292363+0000","last_clean":"2023-09-23T18:28:03.292363+0000","last_became_active":"2023-09-23T18:28:03.291796+0000","last_became_peered":"2023-09-23T18:28:03.291796+0000","last_unstale":"2023-09-23T18:28:03.292363+0000","last_undegraded":"2023-09-23T18:28:03.292363+0000","last_fullsized":"2023-09-23T18:28:03.292363+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:13:35.340945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.283954+0000","last_change":"2023-09-23T18:27:33.585121+0000","last_active":"2023-09-23T18:28:02.283954+0000","last_peered":"2023-09-23T18:28:02.283954+0000","last_clean":"2023-09-23T18:28:02.283954+0000","last_became_active":"2023-09-23T18:27:33.584620+0000","last_became_peered":"2023-09-23T18:27:33.584620+0000","last_unstale":"2023-09-23T18:28:02.283954+0000","last_undegraded":"2023-09-23T18:28:02.283954+0000","last_fullsized":"2023-09-23T18:28:02.283954+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.290588+0000","last_change":"2023-09-23T18:27:04.256030+0000","last_active":"2023-09-23T18:28:02.290588+0000","last_peered":"2023-09-23T18:28:02.290588+0000","last_clean":"2023-09-23T18:28:02.290588+0000","last_became_active":"2023-09-23T18:27:04.255549+0000","last_became_peered":"2023-09-23T18:27:04.255549+0000","last_unstale":"2023-09-23T18:28:02.290588+0000","last_undegraded":"2023-09-23T18:28:02.290588+0000","last_fullsized":"2023-09-23T18:28:02.290588+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":77,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.292732+0000","last_change":"2023-09-23T18:15:08.909339+0000","last_active":"2023-09-23T18:28:02.292732+0000","last_peered":"2023-09-23T18:28:02.292732+0000","last_clean":"2023-09-23T18:28:02.292732+0000","last_became_active":"2023-09-23T18:15:08.908912+0000","last_became_peered":"2023-09-23T18:15:08.908912+0000","last_unstale":"2023-09-23T18:28:02.292732+0000","last_undegraded":"2023-09-23T18:28:02.292732+0000","last_fullsized":"2023-09-23T18:28:02.292732+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.275687+0000","last_change":"2023-09-23T18:27:33.585072+0000","last_active":"2023-09-23T18:28:03.275687+0000","last_peered":"2023-09-23T18:28:03.275687+0000","last_clean":"2023-09-23T18:28:03.275687+0000","last_became_active":"2023-09-23T18:27:33.584118+0000","last_became_peered":"2023-09-23T18:27:33.584118+0000","last_unstale":"2023-09-23T18:28:03.275687+0000","last_undegraded":"2023-09-23T18:28:03.275687+0000","last_fullsized":"2023-09-23T18:28:03.275687+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:30:12.156524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.296913+0000","last_change":"2023-09-23T18:28:03.296913+0000","last_active":"2023-09-23T18:28:03.296913+0000","last_peered":"2023-09-23T18:28:03.296913+0000","last_clean":"2023-09-23T18:28:03.296913+0000","last_became_active":"2023-09-23T18:28:03.296499+0000","last_became_peered":"2023-09-23T18:28:03.296499+0000","last_unstale":"2023-09-23T18:28:03.296913+0000","last_undegraded":"2023-09-23T18:28:03.296913+0000","last_fullsized":"2023-09-23T18:28:03.296913+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":61,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.298214+0000","last_change":"2023-09-23T18:28:03.298214+0000","last_active":"2023-09-23T18:28:03.298214+0000","last_peered":"2023-09-23T18:28:03.298214+0000","last_clean":"2023-09-23T18:28:03.298214+0000","last_became_active":"2023-09-23T18:28:03.297852+0000","last_became_peered":"2023-09-23T18:28:03.297852+0000","last_unstale":"2023-09-23T18:28:03.298214+0000","last_undegraded":"2023-09-23T18:28:03.298214+0000","last_fullsized":"2023-09-23T18:28:03.298214+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.146541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.278178+0000","last_change":"2023-09-23T18:27:04.250764+0000","last_active":"2023-09-23T18:28:02.278178+0000","last_peered":"2023-09-23T18:28:02.278178+0000","last_clean":"2023-09-23T18:28:02.278178+0000","last_became_active":"2023-09-23T18:27:04.250360+0000","last_became_peered":"2023-09-23T18:27:04.250360+0000","last_unstale":"2023-09-23T18:28:02.278178+0000","last_undegraded":"2023-09-23T18:28:02.278178+0000","last_fullsized":"2023-09-23T18:28:02.278178+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.295734+0000","last_change":"2023-09-23T18:28:03.295734+0000","last_active":"2023-09-23T18:28:03.295734+0000","last_peered":"2023-09-23T18:28:03.295734+0000","last_clean":"2023-09-23T18:28:03.295734+0000","last_became_active":"2023-09-23T18:28:03.293761+0000","last_became_peered":"2023-09-23T18:28:03.293761+0000","last_unstale":"2023-09-23T18:28:03.295734+0000","last_undegraded":"2023-09-23T18:28:03.295734+0000","last_fullsized":"2023-09-23T18:28:03.295734+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:55.285705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.297725+0000","last_change":"2023-09-23T18:28:03.297439+0000","last_active":"2023-09-23T18:28:03.297725+0000","last_peered":"2023-09-23T18:28:03.297725+0000","last_clean":"2023-09-23T18:28:03.297725+0000","last_became_active":"2023-09-23T18:28:03.297112+0000","last_became_peered":"2023-09-23T18:28:03.297112+0000","last_unstale":"2023-09-23T18:28:03.297725+0000","last_undegraded":"2023-09-23T18:28:03.297725+0000","last_fullsized":"2023-09-23T18:28:03.297725+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:50.442238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.284400+0000","last_change":"2023-09-23T18:27:04.247225+0000","last_active":"2023-09-23T18:28:02.284400+0000","last_peered":"2023-09-23T18:28:02.284400+0000","last_clean":"2023-09-23T18:28:02.284400+0000","last_became_active":"2023-09-23T18:27:04.246843+0000","last_became_peered":"2023-09-23T18:27:04.246843+0000","last_unstale":"2023-09-23T18:28:02.284400+0000","last_undegraded":"2023-09-23T18:28:02.284400+0000","last_fullsized":"2023-09-23T18:28:02.284400+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":87,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.287830+0000","last_change":"2023-09-23T18:28:03.287830+0000","last_active":"2023-09-23T18:28:03.287830+0000","last_peered":"2023-09-23T18:28:03.287830+0000","last_clean":"2023-09-23T18:28:03.287830+0000","last_became_active":"2023-09-23T18:28:03.287461+0000","last_became_peered":"2023-09-23T18:28:03.287461+0000","last_unstale":"2023-09-23T18:28:03.287830+0000","last_undegraded":"2023-09-23T18:28:03.287830+0000","last_fullsized":"2023-09-23T18:28:03.287830+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:21:51.267964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.290067+0000","last_change":"2023-09-23T18:28:03.289500+0000","last_active":"2023-09-23T18:28:03.290067+0000","last_peered":"2023-09-23T18:28:03.290067+0000","last_clean":"2023-09-23T18:28:03.290067+0000","last_became_active":"2023-09-23T18:28:03.289122+0000","last_became_peered":"2023-09-23T18:28:03.289122+0000","last_unstale":"2023-09-23T18:28:03.290067+0000","last_undegraded":"2023-09-23T18:28:03.290067+0000","last_fullsized":"2023-09-23T18:28:03.290067+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:22:26.904660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.295399+0000","last_change":"2023-09-23T18:28:03.295009+0000","last_active":"2023-09-23T18:28:03.295399+0000","last_peered":"2023-09-23T18:28:03.295399+0000","last_clean":"2023-09-23T18:28:03.295399+0000","last_became_active":"2023-09-23T18:28:03.294501+0000","last_became_peered":"2023-09-23T18:28:03.294501+0000","last_unstale":"2023-09-23T18:28:03.295399+0000","last_undegraded":"2023-09-23T18:28:03.295399+0000","last_fullsized":"2023-09-23T18:28:03.295399+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:00.671680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.299495+0000","last_change":"2023-09-23T18:28:03.299495+0000","last_active":"2023-09-23T18:28:03.299495+0000","last_peered":"2023-09-23T18:28:03.299495+0000","last_clean":"2023-09-23T18:28:03.299495+0000","last_became_active":"2023-09-23T18:28:03.297293+0000","last_became_peered":"2023-09-23T18:28:03.297293+0000","last_unstale":"2023-09-23T18:28:03.299495+0000","last_undegraded":"2023-09-23T18:28:03.299495+0000","last_fullsized":"2023-09-23T18:28:03.299495+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:01.235861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":86,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.291372+0000","last_change":"2023-09-23T18:27:33.588552+0000","last_active":"2023-09-23T18:28:02.291372+0000","last_peered":"2023-09-23T18:28:02.291372+0000","last_clean":"2023-09-23T18:28:02.291372+0000","last_became_active":"2023-09-23T18:27:33.588118+0000","last_became_peered":"2023-09-23T18:27:33.588118+0000","last_unstale":"2023-09-23T18:28:02.291372+0000","last_undegraded":"2023-09-23T18:28:02.291372+0000","last_fullsized":"2023-09-23T18:28:02.291372+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26886,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5621,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430429,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306928,"kb_used_data":5736,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93438224,"statfs":{"total":95995035648,"available":95680741376,"internally_reserved":0,"allocated":5873664,"data_stored":5365985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:29:42 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.91900000000000004,"15min":0.92700000000000005},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.2909999999999999,"5min":1.4670000000000001,"15min":1.4670000000000001},"last":0.97599999999999998},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.93700000000000006,"15min":0.95899999999999996},"min":{"1min":0.69899999999999995,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.175,"5min":1.302,"15min":1.302},"last":0.85999999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:30:07 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.91100000000000003,"15min":0.91500000000000004},"min":{"1min":0.58599999999999997,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.4039999999999999,"5min":1.4039999999999999,"15min":1.4039999999999999},"last":0.94999999999999996},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.97499999999999998,"15min":0.96699999999999997},"min":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.254,"5min":1.306,"15min":1.306},"last":1.105}]},{"osd":2,"last update":"Sat Sep 23 18:29:36 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"min":{"1min":0.71099999999999997,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":1.2529999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":1.1539999999999999},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.94899999999999995,"15min":0.95499999999999996},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.2989999999999999,"5min":1.3380000000000001,"15min":1.3380000000000001},"last":1.2689999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:30:04 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":0.99199999999999999,"15min":0.996},"min":{"1min":0.626,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.254,"5min":1.383,"15min":1.383},"last":0.89100000000000001},{"interface":"front","average":{"1min":0.91500000000000004,"5min":0.94499999999999995,"15min":0.94999999999999996},"min":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"max":{"1min":1.3360000000000001,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":1.018}]},{"osd":4,"last update":"Sat Sep 23 18:29:50 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.92400000000000004,"15min":0.88900000000000001},"min":{"1min":0.625,"5min":0.60099999999999998,"15min":0.371},"max":{"1min":1.214,"5min":1.2589999999999999,"15min":1.3839999999999999},"last":1.0580000000000001},{"interface":"front","average":{"1min":1.028,"5min":1.0469999999999999,"15min":1.028},"min":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.50700000000000001},"max":{"1min":1.341,"5min":1.4430000000000001,"15min":1.4430000000000001},"last":1.2070000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:29:50 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.94099999999999995,"15min":0.95699999999999996},"min":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.505},"max":{"1min":1.298,"5min":1.298,"15min":1.4470000000000001},"last":0.93100000000000005},{"interface":"front","average":{"1min":1.042,"5min":1.0580000000000001,"15min":1.0129999999999999},"min":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.40899999999999997},"max":{"1min":1.2889999999999999,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":1.282}]},{"osd":6,"last update":"Sat Sep 23 18:29:50 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":1.0980000000000001,"15min":1.05},"min":{"1min":0.95999999999999996,"5min":0.72699999999999998,"15min":0.442},"max":{"1min":1.3560000000000001,"5min":1.4399999999999999,"15min":1.4690000000000001},"last":1.2310000000000001},{"interface":"front","average":{"1min":1.095,"5min":1.145,"15min":1.131},"min":{"1min":0.67400000000000004,"5min":0.63600000000000001,"15min":0.53200000000000003},"max":{"1min":1.3380000000000001,"5min":1.548,"15min":1.548},"last":1.3169999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626661,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302820,"kb_used_data":1628,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93442332,"statfs":{"total":95995035648,"available":95684947968,"internally_reserved":0,"allocated":1667072,"data_stored":1171239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:29:40 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.91600000000000004,"15min":0.91000000000000003},"min":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.244,"5min":1.2470000000000001,"15min":1.2470000000000001},"last":1.171},{"interface":"front","average":{"1min":0.99399999999999999,"5min":1.0409999999999999,"15min":1.0449999999999999},"min":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"max":{"1min":1.1850000000000001,"5min":1.3540000000000001,"15min":1.3540000000000001},"last":0.80500000000000005}]},{"osd":1,"last update":"Sat Sep 23 18:30:10 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90400000000000003},"min":{"1min":0.70699999999999996,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.1140000000000001,"5min":1.2210000000000001,"15min":1.2210000000000001},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.93100000000000005,"15min":0.92500000000000004},"min":{"1min":0.61599999999999999,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.403,"5min":1.403,"15min":1.403},"last":1.232}]},{"osd":2,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.999,"15min":1.0049999999999999},"min":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"max":{"1min":1.2789999999999999,"5min":1.2789999999999999,"15min":1.2789999999999999},"last":0.90100000000000002},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.95699999999999996,"15min":0.95899999999999996},"min":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.196,"5min":1.196,"15min":1.196},"last":0.97199999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":0.99399999999999999,"15min":0.98599999999999999},"min":{"1min":0.749,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.3260000000000001,"5min":1.3260000000000001,"15min":1.3260000000000001},"last":1.0389999999999999},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.97699999999999998,"15min":0.98399999999999999},"min":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"max":{"1min":1.1850000000000001,"5min":1.288,"15min":1.288},"last":0.878}]},{"osd":4,"last update":"Sat Sep 23 18:29:55 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":0.91200000000000003,"15min":0.92400000000000004},"min":{"1min":0.61699999999999999,"5min":0.53400000000000003,"15min":0.44800000000000001},"max":{"1min":1.3069999999999999,"5min":1.339,"15min":1.4399999999999999},"last":1.0089999999999999},{"interface":"front","average":{"1min":1.085,"5min":1.006,"15min":0.98099999999999998},"min":{"1min":0.78900000000000003,"5min":0.56499999999999995,"15min":0.41199999999999998},"max":{"1min":1.369,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.92100000000000004}]},{"osd":5,"last update":"Sat Sep 23 18:29:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":1.0209999999999999,"15min":1.0249999999999999},"min":{"1min":0.92600000000000005,"5min":0.58999999999999997,"15min":0.36699999999999999},"max":{"1min":1.4219999999999999,"5min":1.431,"15min":1.5469999999999999},"last":0.84499999999999997},{"interface":"front","average":{"1min":1.1399999999999999,"5min":1.0580000000000001,"15min":1.0580000000000001},"min":{"1min":0.83199999999999996,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.403,"5min":1.504,"15min":1.5700000000000001},"last":1.1080000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.056,"15min":1.0760000000000001},"min":{"1min":0.76000000000000001,"5min":0.59499999999999997,"15min":0.48999999999999999},"max":{"1min":1.274,"5min":1.764,"15min":1.764},"last":1.0620000000000001},{"interface":"front","average":{"1min":1.173,"5min":1.133,"15min":1.1339999999999999},"min":{"1min":0.83399999999999996,"5min":0.64400000000000002,"15min":0.499},"max":{"1min":1.3759999999999999,"5min":1.6659999999999999,"15min":1.6659999999999999},"last":1.2849999999999999}]}]},{"osd":1,"up_from":93,"seq":399431958569,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32648,"kb_used_data":1088,"kb_used_omap":7,"kb_used_meta":31544,"kb_avail":93712504,"statfs":{"total":95995035648,"available":95961604096,"internally_reserved":0,"allocated":1114112,"data_stored":586888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":32301838},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:30:13 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.66100000000000003,"15min":0.67400000000000004},"min":{"1min":0.44500000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.216,"5min":1.216,"15min":1.216},"last":0.79800000000000004},{"interface":"front","average":{"1min":0.748,"5min":0.72399999999999998,"15min":0.73499999999999999},"min":{"1min":0.46800000000000003,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.252,"5min":1.252,"15min":1.252},"last":0.83699999999999997}]},{"osd":2,"last update":"Sat Sep 23 18:29:40 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.60099999999999998,"15min":0.59899999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":1.1000000000000001,"5min":1.1000000000000001,"15min":1.1000000000000001},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.71399999999999997,"15min":0.71999999999999997},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.0449999999999999,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.91000000000000003}]},{"osd":3,"last update":"Sat Sep 23 18:30:03 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.745,"15min":0.75},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.179,"5min":1.179,"15min":1.179},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.70099999999999996,"15min":0.71499999999999997},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.88,"5min":1.294,"15min":1.294},"last":0.68500000000000005}]},{"osd":4,"last update":"Sat Sep 23 18:30:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":0.94599999999999995,"15min":0.92800000000000005},"min":{"1min":0.70899999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.77200000000000002},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.97999999999999998,"15min":1.0249999999999999},"min":{"1min":0.67800000000000005,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.0269999999999999,"5min":1.4950000000000001,"15min":1.4950000000000001},"last":0.88400000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:30:13 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.96099999999999997,"15min":0.99199999999999999},"min":{"1min":0.624,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.2170000000000001,"5min":1.5149999999999999,"15min":1.5149999999999999},"last":0.98799999999999999},{"interface":"front","average":{"1min":1.012,"5min":0.98299999999999998,"15min":0.98599999999999999},"min":{"1min":0.73099999999999998,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.296,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":0.96499999999999997}]},{"osd":6,"last update":"Sat Sep 23 18:30:13 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.99199999999999999,"15min":1.022},"min":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.2230000000000001,"5min":1.6100000000000001,"15min":1.6100000000000001},"last":1.0509999999999999},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.0589999999999999,"15min":1.0720000000000001},"min":{"1min":0.68899999999999995,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.3839999999999999,"5min":1.4419999999999999,"15min":1.4419999999999999},"last":0.81799999999999995}]},{"osd":7,"last update":"Sat Sep 23 18:30:13 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.0860000000000001,"15min":1.1060000000000001},"min":{"1min":0.70499999999999996,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.3580000000000001,"5min":1.55,"15min":1.55},"last":1.0149999999999999},{"interface":"front","average":{"1min":1.006,"5min":1.0840000000000001,"15min":1.1240000000000001},"min":{"1min":0.79600000000000004,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.268,"5min":1.575,"15min":1.575},"last":1.319}]}]},{"osd":0,"up_from":88,"seq":377957122095,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33240,"kb_used_data":1680,"kb_used_omap":8,"kb_used_meta":31543,"kb_avail":93711912,"statfs":{"total":95995035648,"available":95960997888,"internally_reserved":0,"allocated":1720320,"data_stored":1178335,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":32300926},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:30:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.63600000000000001,"15min":0.64000000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.77500000000000002,"5min":1.2170000000000001,"15min":1.2170000000000001},"last":0.627},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.77900000000000003,"15min":0.79400000000000004},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.97899999999999998,"5min":1.3480000000000001,"15min":1.3480000000000001},"last":0.79400000000000004}]},{"osd":2,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.80500000000000005,"15min":0.82399999999999995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.202,"5min":1.254,"15min":1.254},"last":1.135},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.67000000000000004,"15min":0.68300000000000005},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.90100000000000002,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.67200000000000004}]},{"osd":3,"last update":"Sat Sep 23 18:30:02 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.77300000000000002,"15min":0.78500000000000003},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.103,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":1.1910000000000001},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.84599999999999997,"15min":0.87},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.0429999999999999,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":1.1639999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:29:41 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.97299999999999998,"15min":0.97399999999999998},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.4970000000000001,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":0.96799999999999997},{"interface":"front","average":{"1min":0.90700000000000003,"5min":0.98399999999999999,"15min":1.0089999999999999},"min":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.29,"5min":1.29,"15min":1.29},"last":0.89600000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:29:41 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.99099999999999999,"15min":1.0149999999999999},"min":{"1min":0.68000000000000005,"5min":0.67200000000000004,"15min":0.67200000000000004},"max":{"1min":1.23,"5min":1.524,"15min":1.524},"last":1.0309999999999999},{"interface":"front","average":{"1min":0.96399999999999997,"5min":1.0129999999999999,"15min":1.0349999999999999},"min":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"max":{"1min":1.544,"5min":1.544,"15min":1.544},"last":1.244}]},{"osd":6,"last update":"Sat Sep 23 18:29:41 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":1.036,"15min":1.0640000000000001},"min":{"1min":0.69999999999999996,"5min":0.64100000000000001,"15min":0.64100000000000001},"max":{"1min":1.343,"5min":1.446,"15min":1.446},"last":1.1000000000000001},{"interface":"front","average":{"1min":0.96199999999999997,"5min":1.0329999999999999,"15min":1.0469999999999999},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.458,"5min":1.4610000000000001,"15min":1.4610000000000001},"last":1.0760000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:29:41 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":1.034,"15min":1.054},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.619,"5min":1.619,"15min":1.619},"last":1.008},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.0660000000000001,"15min":1.0840000000000001},"min":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"max":{"1min":1.421,"5min":1.5720000000000001,"15min":1.5720000000000001},"last":1.222}]}]},{"osd":2,"up_from":98,"seq":420906795043,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36672,"kb_used_data":5176,"kb_used_omap":11,"kb_used_meta":31476,"kb_avail":93708480,"statfs":{"total":95995035648,"available":95957483520,"internally_reserved":0,"allocated":5300224,"data_stored":4779645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11949,"internal_metadata":32231763},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.66700000000000004,"15min":0.67700000000000005},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":1.012,"5min":1.4430000000000001,"15min":1.4430000000000001},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.1799999999999999,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.74299999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64900000000000002,"15min":0.65100000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.96499999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":1.0209999999999999},{"interface":"front","average":{"1min":0.69299999999999995,"5min":0.74099999999999999,"15min":0.749},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":1.006,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.88600000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.71499999999999997,"15min":0.72399999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.3080000000000001,"5min":1.3080000000000001,"15min":1.3080000000000001},"last":0.84399999999999997},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.70899999999999996,"15min":0.71799999999999997},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.0680000000000001,"5min":1.5189999999999999,"15min":1.5189999999999999},"last":0.72799999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.92300000000000004,"15min":0.92800000000000005},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":1.091},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.97499999999999998,"15min":0.98099999999999998},"min":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.393,"5min":1.579,"15min":1.579},"last":0.80100000000000005}]},{"osd":5,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":1.002,"15min":1.0189999999999999},"min":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.333,"5min":1.6990000000000001,"15min":1.6990000000000001},"last":0.91300000000000003},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1,"15min":0.995},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.488,"5min":1.488,"15min":1.488},"last":0.96399999999999997}]},{"osd":6,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":1.0309999999999999,"15min":1.038},"min":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.373,"5min":1.4199999999999999,"15min":1.4199999999999999},"last":1.2410000000000001},{"interface":"front","average":{"1min":1.026,"5min":1.0660000000000001,"15min":1.073},"min":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"max":{"1min":1.4430000000000001,"5min":1.631,"15min":1.631},"last":1.107}]},{"osd":7,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.0589999999999999,"15min":1.0680000000000001},"min":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":1.538,"5min":1.538,"15min":1.538},"last":1.1970000000000001},{"interface":"front","average":{"1min":0.999,"5min":1.1020000000000001,"15min":1.119},"min":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.2889999999999999,"5min":1.7609999999999999,"15min":1.7609999999999999},"last":1.054}]}]},{"osd":3,"up_from":103,"seq":442381631517,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28388,"kb_used_data":1052,"kb_used_omap":4,"kb_used_meta":27323,"kb_avail":93716764,"statfs":{"total":95995035648,"available":95965966336,"internally_reserved":0,"allocated":1077248,"data_stored":580581,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4826,"internal_metadata":27979046},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:30:04 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.67500000000000004,"15min":0.68899999999999995},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.93600000000000005,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.85399999999999998},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.72499999999999998,"15min":0.74099999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.1240000000000001,"5min":1.1240000000000001,"15min":1.1240000000000001},"last":0.78600000000000003}]},{"osd":1,"last update":"Sat Sep 23 18:30:04 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.70299999999999996,"15min":0.71499999999999997},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":1.022,"5min":1.022,"15min":1.022},"last":1.034},{"interface":"front","average":{"1min":0.69599999999999995,"5min":0.76400000000000001,"15min":0.77600000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.002,"5min":1.1439999999999999,"15min":1.1439999999999999},"last":0.96199999999999997}]},{"osd":2,"last update":"Sat Sep 23 18:30:04 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.72499999999999998,"15min":0.74099999999999999},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.0760000000000001,"5min":1.0760000000000001,"15min":1.0760000000000001},"last":0.997},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.79400000000000004,"15min":0.80600000000000005},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.1719999999999999,"5min":1.1719999999999999,"15min":1.1719999999999999},"last":0.63}]},{"osd":4,"last update":"Sat Sep 23 18:30:04 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":0.95599999999999996,"15min":0.96499999999999997},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.3260000000000001,"5min":1.3260000000000001,"15min":1.3260000000000001},"last":1.1799999999999999},{"interface":"front","average":{"1min":0.94899999999999995,"5min":1.0209999999999999,"15min":1.0329999999999999},"min":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.2,"5min":1.431,"15min":1.431},"last":1.153}]},{"osd":5,"last update":"Sat Sep 23 18:30:04 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":1.0580000000000001,"15min":1.0780000000000001},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.2949999999999999,"5min":1.444,"15min":1.444},"last":1.0960000000000001},{"interface":"front","average":{"1min":0.93500000000000005,"5min":1.1060000000000001,"15min":1.1339999999999999},"min":{"1min":0.623,"5min":0.623,"15min":0.623},"max":{"1min":1.2649999999999999,"5min":1.5409999999999999,"15min":1.5409999999999999},"last":0.90800000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:30:04 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":1.1000000000000001,"15min":1.127},"min":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.357,"5min":1.492,"15min":1.492},"last":1.0700000000000001},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.075,"15min":1.091},"min":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"max":{"1min":1.4059999999999999,"5min":1.425,"15min":1.425},"last":0.93500000000000005}]},{"osd":7,"last update":"Sat Sep 23 18:30:04 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":1.107,"15min":1.127},"min":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"max":{"1min":1.4850000000000001,"5min":1.5529999999999999,"15min":1.5529999999999999},"last":1.1259999999999999},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.1419999999999999,"15min":1.1599999999999999},"min":{"1min":0.77800000000000002,"5min":0.71699999999999997,"15min":0.71699999999999997},"max":{"1min":1.381,"5min":1.4530000000000001,"15min":1.4530000000000001},"last":1.2609999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019127,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298152,"kb_used_data":1056,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447000,"statfs":{"total":95995035648,"available":95689728000,"internally_reserved":0,"allocated":1081344,"data_stored":581243,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:29:40 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.92700000000000005,"15min":0.92900000000000005},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.171,"5min":1.2809999999999999,"15min":1.2809999999999999},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.93200000000000005,"5min":1.014,"15min":1.028},"min":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.2310000000000001,"5min":1.258,"15min":1.258},"last":0.89500000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:30:08 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.86499999999999999,"15min":0.84699999999999998},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.2110000000000001,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.88700000000000001,"5min":0.88500000000000001,"15min":0.88600000000000001},"min":{"1min":0.63300000000000001,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.103,"5min":1.224,"15min":1.224},"last":0.93300000000000005}]},{"osd":2,"last update":"Sat Sep 23 18:29:40 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":1.048,"15min":1.0489999999999999},"min":{"1min":0.72899999999999998,"5min":0.63,"15min":0.63},"max":{"1min":1.302,"5min":1.302,"15min":1.302},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.95999999999999996,"5min":1.002,"15min":1.0089999999999999},"min":{"1min":0.68300000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"max":{"1min":1.2190000000000001,"5min":1.242,"15min":1.242},"last":0.72599999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:30:08 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":0.89700000000000002,"15min":0.90500000000000003},"min":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.008,"5min":1.1899999999999999,"15min":1.1899999999999999},"last":0.98499999999999999},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.0309999999999999,"15min":1.042},"min":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"max":{"1min":1.1930000000000001,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":1.1060000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:29:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":0.97699999999999998,"15min":0.97299999999999998},"min":{"1min":0.72199999999999998,"5min":0.60199999999999998,"15min":0.51600000000000001},"max":{"1min":1.264,"5min":1.3919999999999999,"15min":2.266},"last":1.198},{"interface":"front","average":{"1min":0.996,"5min":0.97599999999999998,"15min":0.97199999999999998},"min":{"1min":0.71499999999999997,"5min":0.58999999999999997,"15min":0.48199999999999998},"max":{"1min":1.298,"5min":1.411,"15min":2.4929999999999999},"last":1.095}]},{"osd":6,"last update":"Sat Sep 23 18:29:58 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.0109999999999999,"15min":1.0269999999999999},"min":{"1min":0.83599999999999997,"5min":0.65000000000000002,"15min":0.57699999999999996},"max":{"1min":1.323,"5min":1.323,"15min":2.351},"last":1.278},{"interface":"front","average":{"1min":1.044,"5min":1.0389999999999999,"15min":1.0169999999999999},"min":{"1min":0.66800000000000004,"5min":0.63300000000000001,"15min":0.42899999999999999},"max":{"1min":1.2769999999999999,"5min":1.3660000000000001,"15min":2.621},"last":1.034}]},{"osd":7,"last update":"Sat Sep 23 18:29:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.105,"15min":1.083},"min":{"1min":0.92000000000000004,"5min":0.66200000000000003,"15min":0.50800000000000001},"max":{"1min":1.3600000000000001,"5min":1.4339999999999999,"15min":2.6819999999999999},"last":1.1359999999999999},{"interface":"front","average":{"1min":1.151,"5min":1.1579999999999999,"15min":1.169},"min":{"1min":0.86499999999999999,"5min":0.80400000000000005,"15min":0.66400000000000003},"max":{"1min":1.3520000000000001,"5min":1.4490000000000001,"15min":2.4140000000000001},"last":1.3440000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822895,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302228,"kb_used_data":5132,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93442924,"statfs":{"total":95995035648,"available":95685554176,"internally_reserved":0,"allocated":5255168,"data_stored":4770680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:29:42 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":0.85099999999999998,"15min":0.85099999999999998},"min":{"1min":0.57899999999999996,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.228,"5min":1.244,"15min":1.244},"last":1.002},{"interface":"front","average":{"1min":0.83199999999999996,"5min":0.86399999999999999,"15min":0.872},"min":{"1min":0.56200000000000006,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.123,"5min":1.1930000000000001,"15min":1.1930000000000001},"last":1.0429999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:30:10 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.87,"15min":0.85599999999999998},"min":{"1min":0.64200000000000002,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.256,"5min":1.256,"15min":1.256},"last":0.82199999999999995},{"interface":"front","average":{"1min":0.82499999999999996,"5min":0.89600000000000002,"15min":0.92600000000000005},"min":{"1min":0.54600000000000004,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.081,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":1.093}]},{"osd":2,"last update":"Sat Sep 23 18:29:38 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":0.90900000000000003,"15min":0.91200000000000003},"min":{"1min":0.628,"5min":0.628,"15min":0.628},"max":{"1min":1.274,"5min":1.274,"15min":1.274},"last":0.89300000000000002},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.96199999999999997,"15min":0.95899999999999996},"min":{"1min":0.59999999999999998,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.2350000000000001,"5min":1.2350000000000001,"15min":1.2350000000000001},"last":1.1040000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:30:05 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":0.94999999999999996,"15min":0.94499999999999995},"min":{"1min":0.73699999999999999,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.181,"5min":1.181,"15min":1.181},"last":1.1579999999999999},{"interface":"front","average":{"1min":0.89100000000000001,"5min":0.90100000000000002,"15min":0.90300000000000002},"min":{"1min":0.745,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.0720000000000001,"5min":1.1140000000000001,"15min":1.1140000000000001},"last":1.0700000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:30:10 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.90000000000000002,"15min":0.89500000000000002},"min":{"1min":0.64600000000000002,"5min":0.55000000000000004,"15min":0.39600000000000002},"max":{"1min":1.2250000000000001,"5min":1.3879999999999999,"15min":2.0259999999999998},"last":0.84599999999999997},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.91800000000000004,"15min":0.93000000000000005},"min":{"1min":0.64700000000000002,"5min":0.60399999999999998,"15min":0.36699999999999999},"max":{"1min":1.252,"5min":1.319,"15min":1.73},"last":1.1799999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":0.99099999999999999,"15min":0.99099999999999999},"min":{"1min":0.77400000000000002,"5min":0.64700000000000002,"15min":0.47399999999999998},"max":{"1min":1.2729999999999999,"5min":1.464,"15min":2.137},"last":0.98699999999999999},{"interface":"front","average":{"1min":1.1020000000000001,"5min":1.085,"15min":1.069},"min":{"1min":0.86499999999999999,"5min":0.79200000000000004,"15min":0.51100000000000001},"max":{"1min":1.3320000000000001,"5min":1.4199999999999999,"15min":1.8779999999999999},"last":1.288}]},{"osd":7,"last update":"Sat Sep 23 18:29:42 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.046,"15min":1.0509999999999999},"min":{"1min":0.66600000000000004,"5min":0.65400000000000003,"15min":0.53500000000000003},"max":{"1min":1.3180000000000001,"5min":1.5589999999999999,"15min":2.1549999999999998},"last":1.2310000000000001},{"interface":"front","average":{"1min":1.087,"5min":1.0820000000000001,"15min":1.083},"min":{"1min":0.83999999999999997,"5min":0.65200000000000002,"15min":0.54300000000000004},"max":{"1min":1.3069999999999999,"5min":1.6990000000000001,"15min":2.2189999999999999},"last":1.2130000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4224,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7725,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3429,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:30:20.543 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:30:20.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:20 smithi098 bash[76817]: cluster 2023-09-23T18:30:19.640223+0000 mgr.y (mgr.26054) 348 : cluster 0 pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:20.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:20 smithi098 bash[76817]: audit 2023-09-23T18:30:19.726513+0000 mgr.y (mgr.26054) 349 : audit 0 from='client.34716 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:30:20.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:20 smithi098 bash[79516]: cluster 2023-09-23T18:30:19.640223+0000 mgr.y (mgr.26054) 348 : cluster 0 pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:20.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:20 smithi098 bash[79516]: audit 2023-09-23T18:30:19.726513+0000 mgr.y (mgr.26054) 349 : audit 0 from='client.34716 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:30:20.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:20 smithi173 bash[59422]: cluster 2023-09-23T18:30:19.640223+0000 mgr.y (mgr.26054) 348 : cluster 0 pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:20.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:20 smithi173 bash[59422]: audit 2023-09-23T18:30:19.726513+0000 mgr.y (mgr.26054) 349 : audit 0 from='client.34716 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:30:23.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:22 smithi173 bash[59422]: cluster 2023-09-23T18:30:21.641197+0000 mgr.y (mgr.26054) 350 : cluster 0 pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:23.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:22 smithi098 bash[76817]: cluster 2023-09-23T18:30:21.641197+0000 mgr.y (mgr.26054) 350 : cluster 0 pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:23.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:22 smithi098 bash[79516]: cluster 2023-09-23T18:30:21.641197+0000 mgr.y (mgr.26054) 350 : cluster 0 pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:25.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:24 smithi173 bash[59422]: cluster 2023-09-23T18:30:23.641997+0000 mgr.y (mgr.26054) 351 : cluster 0 pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:25.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:24 smithi173 bash[59422]: audit 2023-09-23T18:30:23.922417+0000 mon.a (mon.0) 504 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:30:25.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:24 smithi173 bash[59422]: audit 2023-09-23T18:30:24.577105+0000 mon.a (mon.0) 505 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:30:25.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:24 smithi098 bash[79516]: cluster 2023-09-23T18:30:23.641997+0000 mgr.y (mgr.26054) 351 : cluster 0 pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:25.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:24 smithi098 bash[79516]: audit 2023-09-23T18:30:23.922417+0000 mon.a (mon.0) 504 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:30:25.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:24 smithi098 bash[79516]: audit 2023-09-23T18:30:24.577105+0000 mon.a (mon.0) 505 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:30:25.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:24 smithi098 bash[76817]: cluster 2023-09-23T18:30:23.641997+0000 mgr.y (mgr.26054) 351 : cluster 0 pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:25.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:24 smithi098 bash[76817]: audit 2023-09-23T18:30:23.922417+0000 mon.a (mon.0) 504 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:30:25.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:24 smithi098 bash[76817]: audit 2023-09-23T18:30:24.577105+0000 mon.a (mon.0) 505 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:30:25.364 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:30:26.375 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:30:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:30:26] "GET /metrics HTTP/1.1" 200 34094 "" "Prometheus/2.33.4" 2023-09-23T18:30:27.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:26 smithi173 bash[59422]: cluster 2023-09-23T18:30:25.642780+0000 mgr.y (mgr.26054) 352 : cluster 0 pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:27.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:26 smithi098 bash[76817]: cluster 2023-09-23T18:30:25.642780+0000 mgr.y (mgr.26054) 352 : cluster 0 pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:27.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:26 smithi098 bash[79516]: cluster 2023-09-23T18:30:25.642780+0000 mgr.y (mgr.26054) 352 : cluster 0 pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:28.651 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:30:28.652 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-23T18:30:29.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:28 smithi173 bash[59422]: cluster 2023-09-23T18:30:27.643557+0000 mgr.y (mgr.26054) 353 : cluster 0 pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:29.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:28 smithi098 bash[76817]: cluster 2023-09-23T18:30:27.643557+0000 mgr.y (mgr.26054) 353 : cluster 0 pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:29.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:28 smithi098 bash[79516]: cluster 2023-09-23T18:30:27.643557+0000 mgr.y (mgr.26054) 353 : cluster 0 pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:29.476 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":287,"stamp":"2023-09-23T18:30:27.643031+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6303,"num_read_kb":13760,"num_write":1106,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1341076,"kb_used_data":22548,"kb_used_omap":31,"kb_used_meta":1318432,"kb_avail":748620140,"statfs":{"total":767960285184,"available":766587023360,"internally_reserved":0,"allocated":23089152,"data_stored":19014596,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32507,"internal_metadata":1350074629},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004845"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":63,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.289993+0000","last_change":"2023-09-23T18:15:43.682873+0000","last_active":"2023-09-23T18:28:02.289993+0000","last_peered":"2023-09-23T18:28:02.289993+0000","last_clean":"2023-09-23T18:28:02.289993+0000","last_became_active":"2023-09-23T18:15:43.682465+0000","last_became_peered":"2023-09-23T18:15:43.682465+0000","last_unstale":"2023-09-23T18:28:02.289993+0000","last_undegraded":"2023-09-23T18:28:02.289993+0000","last_fullsized":"2023-09-23T18:28:02.289993+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.280010+0000","last_change":"2023-09-23T18:27:33.583935+0000","last_active":"2023-09-23T18:28:03.280010+0000","last_peered":"2023-09-23T18:28:03.280010+0000","last_clean":"2023-09-23T18:28:03.280010+0000","last_became_active":"2023-09-23T18:27:33.582561+0000","last_became_peered":"2023-09-23T18:27:33.582561+0000","last_unstale":"2023-09-23T18:28:03.280010+0000","last_undegraded":"2023-09-23T18:28:03.280010+0000","last_fullsized":"2023-09-23T18:28:03.280010+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:43.589363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":91,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.300455+0000","last_change":"2023-09-23T18:28:03.299520+0000","last_active":"2023-09-23T18:28:03.300455+0000","last_peered":"2023-09-23T18:28:03.300455+0000","last_clean":"2023-09-23T18:28:03.300455+0000","last_became_active":"2023-09-23T18:28:03.299158+0000","last_became_peered":"2023-09-23T18:28:03.299158+0000","last_unstale":"2023-09-23T18:28:03.300455+0000","last_undegraded":"2023-09-23T18:28:03.300455+0000","last_fullsized":"2023-09-23T18:28:03.300455+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:26.247064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":78,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.297944+0000","last_change":"2023-09-23T18:28:03.296850+0000","last_active":"2023-09-23T18:28:03.297944+0000","last_peered":"2023-09-23T18:28:03.297944+0000","last_clean":"2023-09-23T18:28:03.297944+0000","last_became_active":"2023-09-23T18:28:03.295842+0000","last_became_peered":"2023-09-23T18:28:03.295842+0000","last_unstale":"2023-09-23T18:28:03.297944+0000","last_undegraded":"2023-09-23T18:28:03.297944+0000","last_fullsized":"2023-09-23T18:28:03.297944+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:53.048579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.302261+0000","last_change":"2023-09-23T18:28:03.302261+0000","last_active":"2023-09-23T18:28:03.302261+0000","last_peered":"2023-09-23T18:28:03.302261+0000","last_clean":"2023-09-23T18:28:03.302261+0000","last_became_active":"2023-09-23T18:28:03.301911+0000","last_became_peered":"2023-09-23T18:28:03.301911+0000","last_unstale":"2023-09-23T18:28:03.302261+0000","last_undegraded":"2023-09-23T18:28:03.302261+0000","last_fullsized":"2023-09-23T18:28:03.302261+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":57,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.279119+0000","last_change":"2023-09-23T18:15:08.902664+0000","last_active":"2023-09-23T18:28:02.279119+0000","last_peered":"2023-09-23T18:28:02.279119+0000","last_clean":"2023-09-23T18:28:02.279119+0000","last_became_active":"2023-09-23T18:15:08.901252+0000","last_became_peered":"2023-09-23T18:15:08.901252+0000","last_unstale":"2023-09-23T18:28:02.279119+0000","last_undegraded":"2023-09-23T18:28:02.279119+0000","last_fullsized":"2023-09-23T18:28:02.279119+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.300718+0000","last_change":"2023-09-23T18:28:03.300718+0000","last_active":"2023-09-23T18:28:03.300718+0000","last_peered":"2023-09-23T18:28:03.300718+0000","last_clean":"2023-09-23T18:28:03.300718+0000","last_became_active":"2023-09-23T18:28:03.300298+0000","last_became_peered":"2023-09-23T18:28:03.300298+0000","last_unstale":"2023-09-23T18:28:03.300718+0000","last_undegraded":"2023-09-23T18:28:03.300718+0000","last_fullsized":"2023-09-23T18:28:03.300718+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.278761+0000","last_change":"2023-09-23T18:27:04.250817+0000","last_active":"2023-09-23T18:28:02.278761+0000","last_peered":"2023-09-23T18:28:02.278761+0000","last_clean":"2023-09-23T18:28:02.278761+0000","last_became_active":"2023-09-23T18:27:04.250397+0000","last_became_peered":"2023-09-23T18:27:04.250397+0000","last_unstale":"2023-09-23T18:28:02.278761+0000","last_undegraded":"2023-09-23T18:28:02.278761+0000","last_fullsized":"2023-09-23T18:28:02.278761+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":92,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.288758+0000","last_change":"2023-09-23T18:28:03.288092+0000","last_active":"2023-09-23T18:28:03.288758+0000","last_peered":"2023-09-23T18:28:03.288758+0000","last_clean":"2023-09-23T18:28:03.288758+0000","last_became_active":"2023-09-23T18:28:03.287643+0000","last_became_peered":"2023-09-23T18:28:03.287643+0000","last_unstale":"2023-09-23T18:28:03.288758+0000","last_undegraded":"2023-09-23T18:28:03.288758+0000","last_fullsized":"2023-09-23T18:28:03.288758+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:44.163816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":41,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.280816+0000","last_change":"2023-09-23T18:26:34.994176+0000","last_active":"2023-09-23T18:28:03.280816+0000","last_peered":"2023-09-23T18:28:03.280816+0000","last_clean":"2023-09-23T18:28:03.280816+0000","last_became_active":"2023-09-23T18:26:34.993829+0000","last_became_peered":"2023-09-23T18:26:34.993829+0000","last_unstale":"2023-09-23T18:28:03.280816+0000","last_undegraded":"2023-09-23T18:28:03.280816+0000","last_fullsized":"2023-09-23T18:28:03.280816+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:27.111857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.293000+0000","last_change":"2023-09-23T18:15:09.125623+0000","last_active":"2023-09-23T18:28:02.293000+0000","last_peered":"2023-09-23T18:28:02.293000+0000","last_clean":"2023-09-23T18:28:02.293000+0000","last_became_active":"2023-09-23T18:15:09.125018+0000","last_became_peered":"2023-09-23T18:15:09.125018+0000","last_unstale":"2023-09-23T18:28:02.293000+0000","last_undegraded":"2023-09-23T18:28:02.293000+0000","last_fullsized":"2023-09-23T18:28:02.293000+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.281601+0000","last_change":"2023-09-23T18:27:04.247514+0000","last_active":"2023-09-23T18:28:03.281601+0000","last_peered":"2023-09-23T18:28:03.281601+0000","last_clean":"2023-09-23T18:28:03.281601+0000","last_became_active":"2023-09-23T18:27:04.247025+0000","last_became_peered":"2023-09-23T18:27:04.247025+0000","last_unstale":"2023-09-23T18:28:03.281601+0000","last_undegraded":"2023-09-23T18:28:03.281601+0000","last_fullsized":"2023-09-23T18:28:03.281601+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:32.498164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"77'112","reported_seq":594,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.284556+0000","last_change":"2023-09-23T18:27:33.587478+0000","last_active":"2023-09-23T18:28:02.284556+0000","last_peered":"2023-09-23T18:28:02.284556+0000","last_clean":"2023-09-23T18:28:02.284556+0000","last_became_active":"2023-09-23T18:27:33.586958+0000","last_became_peered":"2023-09-23T18:27:33.586958+0000","last_unstale":"2023-09-23T18:28:02.284556+0000","last_undegraded":"2023-09-23T18:28:02.284556+0000","last_fullsized":"2023-09-23T18:28:02.284556+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":893,"num_read_kb":3172,"num_write":69,"num_write_kb":2237,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.293466+0000","last_change":"2023-09-23T18:28:03.292295+0000","last_active":"2023-09-23T18:28:03.293466+0000","last_peered":"2023-09-23T18:28:03.293466+0000","last_clean":"2023-09-23T18:28:03.293466+0000","last_became_active":"2023-09-23T18:28:03.291902+0000","last_became_peered":"2023-09-23T18:28:03.291902+0000","last_unstale":"2023-09-23T18:28:03.293466+0000","last_undegraded":"2023-09-23T18:28:03.293466+0000","last_fullsized":"2023-09-23T18:28:03.293466+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:38.415256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'181","reported_seq":591,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.282681+0000","last_change":"2023-09-23T18:27:04.256906+0000","last_active":"2023-09-23T18:28:03.282681+0000","last_peered":"2023-09-23T18:28:03.282681+0000","last_clean":"2023-09-23T18:28:03.282681+0000","last_became_active":"2023-09-23T18:27:04.255734+0000","last_became_peered":"2023-09-23T18:27:04.255734+0000","last_unstale":"2023-09-23T18:28:03.282681+0000","last_undegraded":"2023-09-23T18:28:03.282681+0000","last_fullsized":"2023-09-23T18:28:03.282681+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":181,"log_dups_size":0,"ondisk_log_size":181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:53.270831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":909,"num_read_kb":734,"num_write":148,"num_write_kb":75,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.299389+0000","last_change":"2023-09-23T18:28:03.298805+0000","last_active":"2023-09-23T18:28:03.299389+0000","last_peered":"2023-09-23T18:28:03.299389+0000","last_clean":"2023-09-23T18:28:03.299389+0000","last_became_active":"2023-09-23T18:28:03.298437+0000","last_became_peered":"2023-09-23T18:28:03.298437+0000","last_unstale":"2023-09-23T18:28:03.299389+0000","last_undegraded":"2023-09-23T18:28:03.299389+0000","last_fullsized":"2023-09-23T18:28:03.299389+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:27.831669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.279808+0000","last_change":"2023-09-23T18:26:34.990770+0000","last_active":"2023-09-23T18:28:03.279808+0000","last_peered":"2023-09-23T18:28:03.279808+0000","last_clean":"2023-09-23T18:28:03.279808+0000","last_became_active":"2023-09-23T18:26:34.990262+0000","last_became_peered":"2023-09-23T18:26:34.990262+0000","last_unstale":"2023-09-23T18:28:03.279808+0000","last_undegraded":"2023-09-23T18:28:03.279808+0000","last_fullsized":"2023-09-23T18:28:03.279808+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:07.982834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.281672+0000","last_change":"2023-09-23T18:27:33.579367+0000","last_active":"2023-09-23T18:28:03.281672+0000","last_peered":"2023-09-23T18:28:03.281672+0000","last_clean":"2023-09-23T18:28:03.281672+0000","last_became_active":"2023-09-23T18:27:33.578607+0000","last_became_peered":"2023-09-23T18:27:33.578607+0000","last_unstale":"2023-09-23T18:28:03.281672+0000","last_undegraded":"2023-09-23T18:28:03.281672+0000","last_fullsized":"2023-09-23T18:28:03.281672+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:58:35.628296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'78","reported_seq":340,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.286740+0000","last_change":"2023-09-23T18:26:34.992345+0000","last_active":"2023-09-23T18:28:02.286740+0000","last_peered":"2023-09-23T18:28:02.286740+0000","last_clean":"2023-09-23T18:28:02.286740+0000","last_became_active":"2023-09-23T18:26:34.991774+0000","last_became_peered":"2023-09-23T18:26:34.991774+0000","last_unstale":"2023-09-23T18:28:02.286740+0000","last_undegraded":"2023-09-23T18:28:02.286740+0000","last_fullsized":"2023-09-23T18:28:02.286740+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":78,"log_dups_size":0,"ondisk_log_size":78,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":428,"num_read_kb":1125,"num_write":73,"num_write_kb":431,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.281734+0000","last_change":"2023-09-23T18:27:33.577544+0000","last_active":"2023-09-23T18:28:03.281734+0000","last_peered":"2023-09-23T18:28:03.281734+0000","last_clean":"2023-09-23T18:28:03.281734+0000","last_became_active":"2023-09-23T18:27:33.577194+0000","last_became_peered":"2023-09-23T18:27:33.577194+0000","last_unstale":"2023-09-23T18:28:03.281734+0000","last_undegraded":"2023-09-23T18:28:03.281734+0000","last_fullsized":"2023-09-23T18:28:03.281734+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:49.633444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.299894+0000","last_change":"2023-09-23T18:28:03.299508+0000","last_active":"2023-09-23T18:28:03.299894+0000","last_peered":"2023-09-23T18:28:03.299894+0000","last_clean":"2023-09-23T18:28:03.299894+0000","last_became_active":"2023-09-23T18:28:03.298682+0000","last_became_peered":"2023-09-23T18:28:03.298682+0000","last_unstale":"2023-09-23T18:28:03.299894+0000","last_undegraded":"2023-09-23T18:28:03.299894+0000","last_fullsized":"2023-09-23T18:28:03.299894+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:08.398302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'120","reported_seq":550,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.282230+0000","last_change":"2023-09-23T18:27:04.257709+0000","last_active":"2023-09-23T18:28:02.282230+0000","last_peered":"2023-09-23T18:28:02.282230+0000","last_clean":"2023-09-23T18:28:02.282230+0000","last_became_active":"2023-09-23T18:27:04.257331+0000","last_became_peered":"2023-09-23T18:27:04.257331+0000","last_unstale":"2023-09-23T18:28:02.282230+0000","last_undegraded":"2023-09-23T18:28:02.282230+0000","last_fullsized":"2023-09-23T18:28:02.282230+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":120,"log_dups_size":0,"ondisk_log_size":120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":809,"num_read_kb":2749,"num_write":98,"num_write_kb":2087,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.283659+0000","last_change":"2023-09-23T18:15:08.921629+0000","last_active":"2023-09-23T18:28:02.283659+0000","last_peered":"2023-09-23T18:28:02.283659+0000","last_clean":"2023-09-23T18:28:02.283659+0000","last_became_active":"2023-09-23T18:15:08.921283+0000","last_became_peered":"2023-09-23T18:15:08.921283+0000","last_unstale":"2023-09-23T18:28:02.283659+0000","last_undegraded":"2023-09-23T18:28:02.283659+0000","last_fullsized":"2023-09-23T18:28:02.283659+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":63,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.298886+0000","last_change":"2023-09-23T18:28:03.295577+0000","last_active":"2023-09-23T18:28:03.298886+0000","last_peered":"2023-09-23T18:28:03.298886+0000","last_clean":"2023-09-23T18:28:03.298886+0000","last_became_active":"2023-09-23T18:28:03.293777+0000","last_became_peered":"2023-09-23T18:28:03.293777+0000","last_unstale":"2023-09-23T18:28:03.298886+0000","last_undegraded":"2023-09-23T18:28:03.298886+0000","last_fullsized":"2023-09-23T18:28:03.298886+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:49.714764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":306,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.282044+0000","last_change":"2023-09-23T18:26:34.999402+0000","last_active":"2023-09-23T18:28:02.282044+0000","last_peered":"2023-09-23T18:28:02.282044+0000","last_clean":"2023-09-23T18:28:02.282044+0000","last_became_active":"2023-09-23T18:26:34.998828+0000","last_became_peered":"2023-09-23T18:26:34.998828+0000","last_unstale":"2023-09-23T18:28:02.282044+0000","last_undegraded":"2023-09-23T18:28:02.282044+0000","last_fullsized":"2023-09-23T18:28:02.282044+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_clean_scrub_stamp":"2023-09-23T18:08:35.701374+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"77'147","reported_seq":503,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.277795+0000","last_change":"2023-09-23T18:27:33.583558+0000","last_active":"2023-09-23T18:28:02.277795+0000","last_peered":"2023-09-23T18:28:02.277795+0000","last_clean":"2023-09-23T18:28:02.277795+0000","last_became_active":"2023-09-23T18:27:33.583003+0000","last_became_peered":"2023-09-23T18:27:33.583003+0000","last_unstale":"2023-09-23T18:28:02.277795+0000","last_undegraded":"2023-09-23T18:28:02.277795+0000","last_fullsized":"2023-09-23T18:28:02.277795+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":625,"num_read_kb":1869,"num_write":143,"num_write_kb":5368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":78,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.282670+0000","last_change":"2023-09-23T18:27:04.242813+0000","last_active":"2023-09-23T18:28:03.282670+0000","last_peered":"2023-09-23T18:28:03.282670+0000","last_clean":"2023-09-23T18:28:03.282670+0000","last_became_active":"2023-09-23T18:27:04.242409+0000","last_became_peered":"2023-09-23T18:27:04.242409+0000","last_unstale":"2023-09-23T18:28:03.282670+0000","last_undegraded":"2023-09-23T18:28:03.282670+0000","last_fullsized":"2023-09-23T18:28:03.282670+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:01.836849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.275061+0000","last_change":"2023-09-23T18:27:33.585279+0000","last_active":"2023-09-23T18:28:03.275061+0000","last_peered":"2023-09-23T18:28:03.275061+0000","last_clean":"2023-09-23T18:28:03.275061+0000","last_became_active":"2023-09-23T18:27:33.584928+0000","last_became_peered":"2023-09-23T18:27:33.584928+0000","last_unstale":"2023-09-23T18:28:03.275061+0000","last_undegraded":"2023-09-23T18:28:03.275061+0000","last_fullsized":"2023-09-23T18:28:03.275061+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:07.000233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'112","reported_seq":539,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.297866+0000","last_change":"2023-09-23T18:28:03.297540+0000","last_active":"2023-09-23T18:28:03.297866+0000","last_peered":"2023-09-23T18:28:03.297866+0000","last_clean":"2023-09-23T18:28:03.297866+0000","last_became_active":"2023-09-23T18:28:03.296649+0000","last_became_peered":"2023-09-23T18:28:03.296649+0000","last_unstale":"2023-09-23T18:28:03.297866+0000","last_undegraded":"2023-09-23T18:28:03.297866+0000","last_fullsized":"2023-09-23T18:28:03.297866+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:32.531775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":774,"num_read_kb":596,"num_write":102,"num_write_kb":52,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.289185+0000","last_change":"2023-09-23T18:28:03.288707+0000","last_active":"2023-09-23T18:28:03.289185+0000","last_peered":"2023-09-23T18:28:03.289185+0000","last_clean":"2023-09-23T18:28:03.289185+0000","last_became_active":"2023-09-23T18:28:03.288335+0000","last_became_peered":"2023-09-23T18:28:03.288335+0000","last_unstale":"2023-09-23T18:28:03.289185+0000","last_undegraded":"2023-09-23T18:28:03.289185+0000","last_fullsized":"2023-09-23T18:28:03.289185+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:57.652706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.292438+0000","last_change":"2023-09-23T18:26:34.992535+0000","last_active":"2023-09-23T18:28:02.292438+0000","last_peered":"2023-09-23T18:28:02.292438+0000","last_clean":"2023-09-23T18:28:02.292438+0000","last_became_active":"2023-09-23T18:26:34.992177+0000","last_became_peered":"2023-09-23T18:26:34.992177+0000","last_unstale":"2023-09-23T18:28:02.292438+0000","last_undegraded":"2023-09-23T18:28:02.292438+0000","last_fullsized":"2023-09-23T18:28:02.292438+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"77'96","reported_seq":619,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.277710+0000","last_change":"2023-09-23T18:27:04.250427+0000","last_active":"2023-09-23T18:28:02.277710+0000","last_peered":"2023-09-23T18:28:02.277710+0000","last_clean":"2023-09-23T18:28:02.277710+0000","last_became_active":"2023-09-23T18:27:04.249907+0000","last_became_peered":"2023-09-23T18:27:04.249907+0000","last_unstale":"2023-09-23T18:28:02.277710+0000","last_undegraded":"2023-09-23T18:28:02.277710+0000","last_fullsized":"2023-09-23T18:28:02.277710+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":96,"log_dups_size":0,"ondisk_log_size":96,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":785,"num_read_kb":1291,"num_write":87,"num_write_kb":338,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.275431+0000","last_change":"2023-09-23T18:27:04.251552+0000","last_active":"2023-09-23T18:28:03.275431+0000","last_peered":"2023-09-23T18:28:03.275431+0000","last_clean":"2023-09-23T18:28:03.275431+0000","last_became_active":"2023-09-23T18:27:04.250901+0000","last_became_peered":"2023-09-23T18:27:04.250901+0000","last_unstale":"2023-09-23T18:28:03.275431+0000","last_undegraded":"2023-09-23T18:28:03.275431+0000","last_fullsized":"2023-09-23T18:28:03.275431+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:22.322137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.297594+0000","last_change":"2023-09-23T18:28:03.297594+0000","last_active":"2023-09-23T18:28:03.297594+0000","last_peered":"2023-09-23T18:28:03.297594+0000","last_clean":"2023-09-23T18:28:03.297594+0000","last_became_active":"2023-09-23T18:28:03.296725+0000","last_became_peered":"2023-09-23T18:28:03.296725+0000","last_unstale":"2023-09-23T18:28:03.297594+0000","last_undegraded":"2023-09-23T18:28:03.297594+0000","last_fullsized":"2023-09-23T18:28:03.297594+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'130","reported_seq":583,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.281567+0000","last_change":"2023-09-23T18:27:04.250265+0000","last_active":"2023-09-23T18:28:03.281567+0000","last_peered":"2023-09-23T18:28:03.281567+0000","last_clean":"2023-09-23T18:28:03.281567+0000","last_became_active":"2023-09-23T18:27:04.249097+0000","last_became_peered":"2023-09-23T18:27:04.249097+0000","last_unstale":"2023-09-23T18:28:03.281567+0000","last_undegraded":"2023-09-23T18:28:03.281567+0000","last_fullsized":"2023-09-23T18:28:03.281567+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:27.143225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":772,"num_read_kb":1712,"num_write":126,"num_write_kb":2126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.295387+0000","last_change":"2023-09-23T18:28:03.294782+0000","last_active":"2023-09-23T18:28:03.295387+0000","last_peered":"2023-09-23T18:28:03.295387+0000","last_clean":"2023-09-23T18:28:03.295387+0000","last_became_active":"2023-09-23T18:28:03.294430+0000","last_became_peered":"2023-09-23T18:28:03.294430+0000","last_unstale":"2023-09-23T18:28:03.295387+0000","last_undegraded":"2023-09-23T18:28:03.295387+0000","last_fullsized":"2023-09-23T18:28:03.295387+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:20.226197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.291041+0000","last_change":"2023-09-23T18:28:03.287471+0000","last_active":"2023-09-23T18:28:03.291041+0000","last_peered":"2023-09-23T18:28:03.291041+0000","last_clean":"2023-09-23T18:28:03.291041+0000","last_became_active":"2023-09-23T18:28:03.286923+0000","last_became_peered":"2023-09-23T18:28:03.286923+0000","last_unstale":"2023-09-23T18:28:03.291041+0000","last_undegraded":"2023-09-23T18:28:03.291041+0000","last_fullsized":"2023-09-23T18:28:03.291041+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:10.505004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.291109+0000","last_change":"2023-09-23T18:27:04.254684+0000","last_active":"2023-09-23T18:28:02.291109+0000","last_peered":"2023-09-23T18:28:02.291109+0000","last_clean":"2023-09-23T18:28:02.291109+0000","last_became_active":"2023-09-23T18:27:04.254014+0000","last_became_peered":"2023-09-23T18:27:04.254014+0000","last_unstale":"2023-09-23T18:28:02.291109+0000","last_undegraded":"2023-09-23T18:28:02.291109+0000","last_fullsized":"2023-09-23T18:28:02.291109+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.293127+0000","last_change":"2023-09-23T18:27:33.585714+0000","last_active":"2023-09-23T18:28:02.293127+0000","last_peered":"2023-09-23T18:28:02.293127+0000","last_clean":"2023-09-23T18:28:02.293127+0000","last_became_active":"2023-09-23T18:27:33.584641+0000","last_became_peered":"2023-09-23T18:27:33.584641+0000","last_unstale":"2023-09-23T18:28:02.293127+0000","last_undegraded":"2023-09-23T18:28:02.293127+0000","last_fullsized":"2023-09-23T18:28:02.293127+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":84,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.291696+0000","last_change":"2023-09-23T18:27:33.588308+0000","last_active":"2023-09-23T18:28:02.291696+0000","last_peered":"2023-09-23T18:28:02.291696+0000","last_clean":"2023-09-23T18:28:02.291696+0000","last_became_active":"2023-09-23T18:27:33.587974+0000","last_became_peered":"2023-09-23T18:27:33.587974+0000","last_unstale":"2023-09-23T18:28:02.291696+0000","last_undegraded":"2023-09-23T18:28:02.291696+0000","last_fullsized":"2023-09-23T18:28:02.291696+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.284595+0000","last_change":"2023-09-23T18:27:04.252119+0000","last_active":"2023-09-23T18:28:02.284595+0000","last_peered":"2023-09-23T18:28:02.284595+0000","last_clean":"2023-09-23T18:28:02.284595+0000","last_became_active":"2023-09-23T18:27:04.251714+0000","last_became_peered":"2023-09-23T18:27:04.251714+0000","last_unstale":"2023-09-23T18:28:02.284595+0000","last_undegraded":"2023-09-23T18:28:02.284595+0000","last_fullsized":"2023-09-23T18:28:02.284595+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.302326+0000","last_change":"2023-09-23T18:28:03.302326+0000","last_active":"2023-09-23T18:28:03.302326+0000","last_peered":"2023-09-23T18:28:03.302326+0000","last_clean":"2023-09-23T18:28:03.302326+0000","last_became_active":"2023-09-23T18:28:03.301970+0000","last_became_peered":"2023-09-23T18:28:03.301970+0000","last_unstale":"2023-09-23T18:28:03.302326+0000","last_undegraded":"2023-09-23T18:28:03.302326+0000","last_fullsized":"2023-09-23T18:28:03.302326+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.298459+0000","last_change":"2023-09-23T18:28:03.297628+0000","last_active":"2023-09-23T18:28:03.298459+0000","last_peered":"2023-09-23T18:28:03.298459+0000","last_clean":"2023-09-23T18:28:03.298459+0000","last_became_active":"2023-09-23T18:28:03.296694+0000","last_became_peered":"2023-09-23T18:28:03.296694+0000","last_unstale":"2023-09-23T18:28:03.298459+0000","last_undegraded":"2023-09-23T18:28:03.298459+0000","last_fullsized":"2023-09-23T18:28:03.298459+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:43.375652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":65,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.280061+0000","last_change":"2023-09-23T18:27:33.582033+0000","last_active":"2023-09-23T18:28:03.280061+0000","last_peered":"2023-09-23T18:28:03.280061+0000","last_clean":"2023-09-23T18:28:03.280061+0000","last_became_active":"2023-09-23T18:27:33.581667+0000","last_became_peered":"2023-09-23T18:27:33.581667+0000","last_unstale":"2023-09-23T18:28:03.280061+0000","last_undegraded":"2023-09-23T18:28:03.280061+0000","last_fullsized":"2023-09-23T18:28:03.280061+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:36.039922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.298611+0000","last_change":"2023-09-23T18:28:03.298611+0000","last_active":"2023-09-23T18:28:03.298611+0000","last_peered":"2023-09-23T18:28:03.298611+0000","last_clean":"2023-09-23T18:28:03.298611+0000","last_became_active":"2023-09-23T18:28:03.298233+0000","last_became_peered":"2023-09-23T18:28:03.298233+0000","last_unstale":"2023-09-23T18:28:03.298611+0000","last_undegraded":"2023-09-23T18:28:03.298611+0000","last_fullsized":"2023-09-23T18:28:03.298611+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":78,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.284278+0000","last_change":"2023-09-23T18:27:33.589116+0000","last_active":"2023-09-23T18:28:02.284278+0000","last_peered":"2023-09-23T18:28:02.284278+0000","last_clean":"2023-09-23T18:28:02.284278+0000","last_became_active":"2023-09-23T18:27:33.588809+0000","last_became_peered":"2023-09-23T18:27:33.588809+0000","last_unstale":"2023-09-23T18:28:02.284278+0000","last_undegraded":"2023-09-23T18:28:02.284278+0000","last_fullsized":"2023-09-23T18:28:02.284278+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":56,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.286875+0000","last_change":"2023-09-23T18:15:08.908376+0000","last_active":"2023-09-23T18:28:02.286875+0000","last_peered":"2023-09-23T18:28:02.286875+0000","last_clean":"2023-09-23T18:28:02.286875+0000","last_became_active":"2023-09-23T18:15:08.907752+0000","last_became_peered":"2023-09-23T18:15:08.907752+0000","last_unstale":"2023-09-23T18:28:02.286875+0000","last_undegraded":"2023-09-23T18:28:02.286875+0000","last_fullsized":"2023-09-23T18:28:02.286875+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.302292+0000","last_change":"2023-09-23T18:28:03.302292+0000","last_active":"2023-09-23T18:28:03.302292+0000","last_peered":"2023-09-23T18:28:03.302292+0000","last_clean":"2023-09-23T18:28:03.302292+0000","last_became_active":"2023-09-23T18:28:03.301927+0000","last_became_peered":"2023-09-23T18:28:03.301927+0000","last_unstale":"2023-09-23T18:28:03.302292+0000","last_undegraded":"2023-09-23T18:28:03.302292+0000","last_fullsized":"2023-09-23T18:28:03.302292+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.286163+0000","last_change":"2023-09-23T18:27:04.253991+0000","last_active":"2023-09-23T18:28:02.286163+0000","last_peered":"2023-09-23T18:28:02.286163+0000","last_clean":"2023-09-23T18:28:02.286163+0000","last_became_active":"2023-09-23T18:27:04.253593+0000","last_became_peered":"2023-09-23T18:27:04.253593+0000","last_unstale":"2023-09-23T18:28:02.286163+0000","last_undegraded":"2023-09-23T18:28:02.286163+0000","last_fullsized":"2023-09-23T18:28:02.286163+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.277381+0000","last_change":"2023-09-23T18:15:10.327919+0000","last_active":"2023-09-23T18:28:02.277381+0000","last_peered":"2023-09-23T18:28:02.277381+0000","last_clean":"2023-09-23T18:28:02.277381+0000","last_became_active":"2023-09-23T18:15:10.327052+0000","last_became_peered":"2023-09-23T18:15:10.327052+0000","last_unstale":"2023-09-23T18:28:02.277381+0000","last_undegraded":"2023-09-23T18:28:02.277381+0000","last_fullsized":"2023-09-23T18:28:02.277381+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.286482+0000","last_change":"2023-09-23T18:26:34.999684+0000","last_active":"2023-09-23T18:28:02.286482+0000","last_peered":"2023-09-23T18:28:02.286482+0000","last_clean":"2023-09-23T18:28:02.286482+0000","last_became_active":"2023-09-23T18:26:34.999076+0000","last_became_peered":"2023-09-23T18:26:34.999076+0000","last_unstale":"2023-09-23T18:28:02.286482+0000","last_undegraded":"2023-09-23T18:28:02.286482+0000","last_fullsized":"2023-09-23T18:28:02.286482+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.298115+0000","last_change":"2023-09-23T18:28:03.295838+0000","last_active":"2023-09-23T18:28:03.298115+0000","last_peered":"2023-09-23T18:28:03.298115+0000","last_clean":"2023-09-23T18:28:03.298115+0000","last_became_active":"2023-09-23T18:28:03.291774+0000","last_became_peered":"2023-09-23T18:28:03.291774+0000","last_unstale":"2023-09-23T18:28:03.298115+0000","last_undegraded":"2023-09-23T18:28:03.298115+0000","last_fullsized":"2023-09-23T18:28:03.298115+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:23.166353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":78,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.284152+0000","last_change":"2023-09-23T18:26:34.995333+0000","last_active":"2023-09-23T18:28:02.284152+0000","last_peered":"2023-09-23T18:28:02.284152+0000","last_clean":"2023-09-23T18:28:02.284152+0000","last_became_active":"2023-09-23T18:26:34.994983+0000","last_became_peered":"2023-09-23T18:26:34.994983+0000","last_unstale":"2023-09-23T18:28:02.284152+0000","last_undegraded":"2023-09-23T18:28:02.284152+0000","last_fullsized":"2023-09-23T18:28:02.284152+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.290583+0000","last_change":"2023-09-23T18:28:03.290217+0000","last_active":"2023-09-23T18:28:03.290583+0000","last_peered":"2023-09-23T18:28:03.290583+0000","last_clean":"2023-09-23T18:28:03.290583+0000","last_became_active":"2023-09-23T18:28:03.289888+0000","last_became_peered":"2023-09-23T18:28:03.289888+0000","last_unstale":"2023-09-23T18:28:03.290583+0000","last_undegraded":"2023-09-23T18:28:03.290583+0000","last_fullsized":"2023-09-23T18:28:03.290583+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:51:45.400847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.281244+0000","last_change":"2023-09-23T18:15:08.916658+0000","last_active":"2023-09-23T18:28:02.281244+0000","last_peered":"2023-09-23T18:28:02.281244+0000","last_clean":"2023-09-23T18:28:02.281244+0000","last_became_active":"2023-09-23T18:15:08.916301+0000","last_became_peered":"2023-09-23T18:15:08.916301+0000","last_unstale":"2023-09-23T18:28:02.281244+0000","last_undegraded":"2023-09-23T18:28:02.281244+0000","last_fullsized":"2023-09-23T18:28:02.281244+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.297841+0000","last_change":"2023-09-23T18:28:03.297841+0000","last_active":"2023-09-23T18:28:03.297841+0000","last_peered":"2023-09-23T18:28:03.297841+0000","last_clean":"2023-09-23T18:28:03.297841+0000","last_became_active":"2023-09-23T18:28:03.297216+0000","last_became_peered":"2023-09-23T18:28:03.297216+0000","last_unstale":"2023-09-23T18:28:03.297841+0000","last_undegraded":"2023-09-23T18:28:03.297841+0000","last_fullsized":"2023-09-23T18:28:03.297841+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.290591+0000","last_change":"2023-09-23T18:28:03.290316+0000","last_active":"2023-09-23T18:28:03.290591+0000","last_peered":"2023-09-23T18:28:03.290591+0000","last_clean":"2023-09-23T18:28:03.290591+0000","last_became_active":"2023-09-23T18:28:03.289698+0000","last_became_peered":"2023-09-23T18:28:03.289698+0000","last_unstale":"2023-09-23T18:28:03.290591+0000","last_undegraded":"2023-09-23T18:28:03.290591+0000","last_fullsized":"2023-09-23T18:28:03.290591+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:00.779542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.293709+0000","last_change":"2023-09-23T18:28:03.292363+0000","last_active":"2023-09-23T18:28:03.293709+0000","last_peered":"2023-09-23T18:28:03.293709+0000","last_clean":"2023-09-23T18:28:03.293709+0000","last_became_active":"2023-09-23T18:28:03.291796+0000","last_became_peered":"2023-09-23T18:28:03.291796+0000","last_unstale":"2023-09-23T18:28:03.293709+0000","last_undegraded":"2023-09-23T18:28:03.293709+0000","last_fullsized":"2023-09-23T18:28:03.293709+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:13:35.340945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.283954+0000","last_change":"2023-09-23T18:27:33.585121+0000","last_active":"2023-09-23T18:28:02.283954+0000","last_peered":"2023-09-23T18:28:02.283954+0000","last_clean":"2023-09-23T18:28:02.283954+0000","last_became_active":"2023-09-23T18:27:33.584620+0000","last_became_peered":"2023-09-23T18:27:33.584620+0000","last_unstale":"2023-09-23T18:28:02.283954+0000","last_undegraded":"2023-09-23T18:28:02.283954+0000","last_fullsized":"2023-09-23T18:28:02.283954+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.290588+0000","last_change":"2023-09-23T18:27:04.256030+0000","last_active":"2023-09-23T18:28:02.290588+0000","last_peered":"2023-09-23T18:28:02.290588+0000","last_clean":"2023-09-23T18:28:02.290588+0000","last_became_active":"2023-09-23T18:27:04.255549+0000","last_became_peered":"2023-09-23T18:27:04.255549+0000","last_unstale":"2023-09-23T18:28:02.290588+0000","last_undegraded":"2023-09-23T18:28:02.290588+0000","last_fullsized":"2023-09-23T18:28:02.290588+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":77,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.292732+0000","last_change":"2023-09-23T18:15:08.909339+0000","last_active":"2023-09-23T18:28:02.292732+0000","last_peered":"2023-09-23T18:28:02.292732+0000","last_clean":"2023-09-23T18:28:02.292732+0000","last_became_active":"2023-09-23T18:15:08.908912+0000","last_became_peered":"2023-09-23T18:15:08.908912+0000","last_unstale":"2023-09-23T18:28:02.292732+0000","last_undegraded":"2023-09-23T18:28:02.292732+0000","last_fullsized":"2023-09-23T18:28:02.292732+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.275687+0000","last_change":"2023-09-23T18:27:33.585072+0000","last_active":"2023-09-23T18:28:03.275687+0000","last_peered":"2023-09-23T18:28:03.275687+0000","last_clean":"2023-09-23T18:28:03.275687+0000","last_became_active":"2023-09-23T18:27:33.584118+0000","last_became_peered":"2023-09-23T18:27:33.584118+0000","last_unstale":"2023-09-23T18:28:03.275687+0000","last_undegraded":"2023-09-23T18:28:03.275687+0000","last_fullsized":"2023-09-23T18:28:03.275687+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:30:12.156524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.296913+0000","last_change":"2023-09-23T18:28:03.296913+0000","last_active":"2023-09-23T18:28:03.296913+0000","last_peered":"2023-09-23T18:28:03.296913+0000","last_clean":"2023-09-23T18:28:03.296913+0000","last_became_active":"2023-09-23T18:28:03.296499+0000","last_became_peered":"2023-09-23T18:28:03.296499+0000","last_unstale":"2023-09-23T18:28:03.296913+0000","last_undegraded":"2023-09-23T18:28:03.296913+0000","last_fullsized":"2023-09-23T18:28:03.296913+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":62,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.298989+0000","last_change":"2023-09-23T18:28:03.298214+0000","last_active":"2023-09-23T18:28:03.298989+0000","last_peered":"2023-09-23T18:28:03.298989+0000","last_clean":"2023-09-23T18:28:03.298989+0000","last_became_active":"2023-09-23T18:28:03.297852+0000","last_became_peered":"2023-09-23T18:28:03.297852+0000","last_unstale":"2023-09-23T18:28:03.298989+0000","last_undegraded":"2023-09-23T18:28:03.298989+0000","last_fullsized":"2023-09-23T18:28:03.298989+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.146541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.278178+0000","last_change":"2023-09-23T18:27:04.250764+0000","last_active":"2023-09-23T18:28:02.278178+0000","last_peered":"2023-09-23T18:28:02.278178+0000","last_clean":"2023-09-23T18:28:02.278178+0000","last_became_active":"2023-09-23T18:27:04.250360+0000","last_became_peered":"2023-09-23T18:27:04.250360+0000","last_unstale":"2023-09-23T18:28:02.278178+0000","last_undegraded":"2023-09-23T18:28:02.278178+0000","last_fullsized":"2023-09-23T18:28:02.278178+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.296849+0000","last_change":"2023-09-23T18:28:03.295734+0000","last_active":"2023-09-23T18:28:03.296849+0000","last_peered":"2023-09-23T18:28:03.296849+0000","last_clean":"2023-09-23T18:28:03.296849+0000","last_became_active":"2023-09-23T18:28:03.293761+0000","last_became_peered":"2023-09-23T18:28:03.293761+0000","last_unstale":"2023-09-23T18:28:03.296849+0000","last_undegraded":"2023-09-23T18:28:03.296849+0000","last_fullsized":"2023-09-23T18:28:03.296849+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:55.285705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.297725+0000","last_change":"2023-09-23T18:28:03.297439+0000","last_active":"2023-09-23T18:28:03.297725+0000","last_peered":"2023-09-23T18:28:03.297725+0000","last_clean":"2023-09-23T18:28:03.297725+0000","last_became_active":"2023-09-23T18:28:03.297112+0000","last_became_peered":"2023-09-23T18:28:03.297112+0000","last_unstale":"2023-09-23T18:28:03.297725+0000","last_undegraded":"2023-09-23T18:28:03.297725+0000","last_fullsized":"2023-09-23T18:28:03.297725+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:50.442238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.284400+0000","last_change":"2023-09-23T18:27:04.247225+0000","last_active":"2023-09-23T18:28:02.284400+0000","last_peered":"2023-09-23T18:28:02.284400+0000","last_clean":"2023-09-23T18:28:02.284400+0000","last_became_active":"2023-09-23T18:27:04.246843+0000","last_became_peered":"2023-09-23T18:27:04.246843+0000","last_unstale":"2023-09-23T18:28:02.284400+0000","last_undegraded":"2023-09-23T18:28:02.284400+0000","last_fullsized":"2023-09-23T18:28:02.284400+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":88,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.291087+0000","last_change":"2023-09-23T18:28:03.287830+0000","last_active":"2023-09-23T18:28:03.291087+0000","last_peered":"2023-09-23T18:28:03.291087+0000","last_clean":"2023-09-23T18:28:03.291087+0000","last_became_active":"2023-09-23T18:28:03.287461+0000","last_became_peered":"2023-09-23T18:28:03.287461+0000","last_unstale":"2023-09-23T18:28:03.291087+0000","last_undegraded":"2023-09-23T18:28:03.291087+0000","last_fullsized":"2023-09-23T18:28:03.291087+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:21:51.267964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.290067+0000","last_change":"2023-09-23T18:28:03.289500+0000","last_active":"2023-09-23T18:28:03.290067+0000","last_peered":"2023-09-23T18:28:03.290067+0000","last_clean":"2023-09-23T18:28:03.290067+0000","last_became_active":"2023-09-23T18:28:03.289122+0000","last_became_peered":"2023-09-23T18:28:03.289122+0000","last_unstale":"2023-09-23T18:28:03.290067+0000","last_undegraded":"2023-09-23T18:28:03.290067+0000","last_fullsized":"2023-09-23T18:28:03.290067+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:22:26.904660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.295399+0000","last_change":"2023-09-23T18:28:03.295009+0000","last_active":"2023-09-23T18:28:03.295399+0000","last_peered":"2023-09-23T18:28:03.295399+0000","last_clean":"2023-09-23T18:28:03.295399+0000","last_became_active":"2023-09-23T18:28:03.294501+0000","last_became_peered":"2023-09-23T18:28:03.294501+0000","last_unstale":"2023-09-23T18:28:03.295399+0000","last_undegraded":"2023-09-23T18:28:03.295399+0000","last_fullsized":"2023-09-23T18:28:03.295399+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:00.671680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-09-23T18:28:03.299495+0000","last_change":"2023-09-23T18:28:03.299495+0000","last_active":"2023-09-23T18:28:03.299495+0000","last_peered":"2023-09-23T18:28:03.299495+0000","last_clean":"2023-09-23T18:28:03.299495+0000","last_became_active":"2023-09-23T18:28:03.297293+0000","last_became_peered":"2023-09-23T18:28:03.297293+0000","last_unstale":"2023-09-23T18:28:03.299495+0000","last_undegraded":"2023-09-23T18:28:03.299495+0000","last_fullsized":"2023-09-23T18:28:03.299495+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:01.235861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":86,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-09-23T18:28:02.291372+0000","last_change":"2023-09-23T18:27:33.588552+0000","last_active":"2023-09-23T18:28:02.291372+0000","last_peered":"2023-09-23T18:28:02.291372+0000","last_clean":"2023-09-23T18:28:02.291372+0000","last_became_active":"2023-09-23T18:27:33.588118+0000","last_became_peered":"2023-09-23T18:27:33.588118+0000","last_unstale":"2023-09-23T18:28:02.291372+0000","last_undegraded":"2023-09-23T18:28:02.291372+0000","last_fullsized":"2023-09-23T18:28:02.291372+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26886,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5621,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430431,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306928,"kb_used_data":5736,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93438224,"statfs":{"total":95995035648,"available":95680741376,"internally_reserved":0,"allocated":5873664,"data_stored":5365985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:29:42 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.91900000000000004,"15min":0.92700000000000005},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.2909999999999999,"5min":1.4670000000000001,"15min":1.4670000000000001},"last":0.86799999999999999},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.93700000000000006,"15min":0.95899999999999996},"min":{"1min":0.69899999999999995,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.175,"5min":1.302,"15min":1.302},"last":1.1879999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:30:07 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.91100000000000003,"15min":0.91500000000000004},"min":{"1min":0.58599999999999997,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.4039999999999999,"5min":1.4039999999999999,"15min":1.4039999999999999},"last":1.022},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.97499999999999998,"15min":0.96699999999999997},"min":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.254,"5min":1.306,"15min":1.306},"last":0.98899999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:29:36 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"min":{"1min":0.71099999999999997,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":1.2529999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":1.105},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.94899999999999995,"15min":0.95499999999999996},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.2989999999999999,"5min":1.3380000000000001,"15min":1.3380000000000001},"last":1.05}]},{"osd":3,"last update":"Sat Sep 23 18:30:04 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":0.99199999999999999,"15min":0.996},"min":{"1min":0.626,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.254,"5min":1.383,"15min":1.383},"last":0.77300000000000002},{"interface":"front","average":{"1min":0.91500000000000004,"5min":0.94499999999999995,"15min":0.94999999999999996},"min":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"max":{"1min":1.3360000000000001,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":1.071}]},{"osd":4,"last update":"Sat Sep 23 18:29:50 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.92400000000000004,"15min":0.88900000000000001},"min":{"1min":0.625,"5min":0.60099999999999998,"15min":0.371},"max":{"1min":1.214,"5min":1.2589999999999999,"15min":1.3839999999999999},"last":0.82299999999999995},{"interface":"front","average":{"1min":1.028,"5min":1.0469999999999999,"15min":1.028},"min":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.50700000000000001},"max":{"1min":1.341,"5min":1.4430000000000001,"15min":1.4430000000000001},"last":0.67300000000000004}]},{"osd":5,"last update":"Sat Sep 23 18:29:50 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.94099999999999995,"15min":0.95699999999999996},"min":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.505},"max":{"1min":1.298,"5min":1.298,"15min":1.4470000000000001},"last":0.90000000000000002},{"interface":"front","average":{"1min":1.042,"5min":1.0580000000000001,"15min":1.0129999999999999},"min":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.40899999999999997},"max":{"1min":1.2889999999999999,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":0.73099999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:29:50 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":1.0980000000000001,"15min":1.05},"min":{"1min":0.95999999999999996,"5min":0.72699999999999998,"15min":0.442},"max":{"1min":1.3560000000000001,"5min":1.4399999999999999,"15min":1.4690000000000001},"last":0.94099999999999995},{"interface":"front","average":{"1min":1.095,"5min":1.145,"15min":1.131},"min":{"1min":0.67400000000000004,"5min":0.63600000000000001,"15min":0.53200000000000003},"max":{"1min":1.3380000000000001,"5min":1.548,"15min":1.548},"last":1.1399999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626663,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302820,"kb_used_data":1628,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93442332,"statfs":{"total":95995035648,"available":95684947968,"internally_reserved":0,"allocated":1667072,"data_stored":1171239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:29:40 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.91600000000000004,"15min":0.91000000000000003},"min":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.244,"5min":1.2470000000000001,"15min":1.2470000000000001},"last":1.22},{"interface":"front","average":{"1min":0.99399999999999999,"5min":1.0409999999999999,"15min":1.0449999999999999},"min":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"max":{"1min":1.1850000000000001,"5min":1.3540000000000001,"15min":1.3540000000000001},"last":1.284}]},{"osd":1,"last update":"Sat Sep 23 18:30:10 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90400000000000003},"min":{"1min":0.70699999999999996,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.1140000000000001,"5min":1.2210000000000001,"15min":1.2210000000000001},"last":1.02},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.93100000000000005,"15min":0.92500000000000004},"min":{"1min":0.61599999999999999,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.403,"5min":1.403,"15min":1.403},"last":1.1379999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.999,"15min":1.0049999999999999},"min":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"max":{"1min":1.2789999999999999,"5min":1.2789999999999999,"15min":1.2789999999999999},"last":1.169},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.95699999999999996,"15min":0.95899999999999996},"min":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.196,"5min":1.196,"15min":1.196},"last":0.99099999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":0.99399999999999999,"15min":0.98599999999999999},"min":{"1min":0.749,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.3260000000000001,"5min":1.3260000000000001,"15min":1.3260000000000001},"last":0.85599999999999998},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.97699999999999998,"15min":0.98399999999999999},"min":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"max":{"1min":1.1850000000000001,"5min":1.288,"15min":1.288},"last":1.073}]},{"osd":4,"last update":"Sat Sep 23 18:29:55 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":0.91200000000000003,"15min":0.92400000000000004},"min":{"1min":0.61699999999999999,"5min":0.53400000000000003,"15min":0.44800000000000001},"max":{"1min":1.3069999999999999,"5min":1.339,"15min":1.4399999999999999},"last":0.82199999999999995},{"interface":"front","average":{"1min":1.085,"5min":1.006,"15min":0.98099999999999998},"min":{"1min":0.78900000000000003,"5min":0.56499999999999995,"15min":0.41199999999999998},"max":{"1min":1.369,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.94899999999999995}]},{"osd":5,"last update":"Sat Sep 23 18:29:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":1.0209999999999999,"15min":1.0249999999999999},"min":{"1min":0.92600000000000005,"5min":0.58999999999999997,"15min":0.36699999999999999},"max":{"1min":1.4219999999999999,"5min":1.431,"15min":1.5469999999999999},"last":0.92700000000000005},{"interface":"front","average":{"1min":1.1399999999999999,"5min":1.0580000000000001,"15min":1.0580000000000001},"min":{"1min":0.83199999999999996,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.403,"5min":1.504,"15min":1.5700000000000001},"last":1.0469999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.056,"15min":1.0760000000000001},"min":{"1min":0.76000000000000001,"5min":0.59499999999999997,"15min":0.48999999999999999},"max":{"1min":1.274,"5min":1.764,"15min":1.764},"last":0.89300000000000002},{"interface":"front","average":{"1min":1.173,"5min":1.133,"15min":1.1339999999999999},"min":{"1min":0.83399999999999996,"5min":0.64400000000000002,"15min":0.499},"max":{"1min":1.3759999999999999,"5min":1.6659999999999999,"15min":1.6659999999999999},"last":1.3200000000000001}]}]},{"osd":1,"up_from":93,"seq":399431958571,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32648,"kb_used_data":1088,"kb_used_omap":7,"kb_used_meta":31544,"kb_avail":93712504,"statfs":{"total":95995035648,"available":95961604096,"internally_reserved":0,"allocated":1114112,"data_stored":586888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":32301838},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:30:13 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.66100000000000003,"15min":0.67400000000000004},"min":{"1min":0.44500000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.216,"5min":1.216,"15min":1.216},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.748,"5min":0.72399999999999998,"15min":0.73499999999999999},"min":{"1min":0.46800000000000003,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.252,"5min":1.252,"15min":1.252},"last":0.55200000000000005}]},{"osd":2,"last update":"Sat Sep 23 18:29:40 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.60099999999999998,"15min":0.59899999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":1.1000000000000001,"5min":1.1000000000000001,"15min":1.1000000000000001},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.71399999999999997,"15min":0.71999999999999997},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.0449999999999999,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.751}]},{"osd":3,"last update":"Sat Sep 23 18:30:03 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.745,"15min":0.75},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.179,"5min":1.179,"15min":1.179},"last":0.84599999999999997},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.70099999999999996,"15min":0.71499999999999997},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.88,"5min":1.294,"15min":1.294},"last":0.70599999999999996}]},{"osd":4,"last update":"Sat Sep 23 18:30:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":0.94599999999999995,"15min":0.92800000000000005},"min":{"1min":0.70899999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":1.081},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.97999999999999998,"15min":1.0249999999999999},"min":{"1min":0.67800000000000005,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.0269999999999999,"5min":1.4950000000000001,"15min":1.4950000000000001},"last":0.873}]},{"osd":5,"last update":"Sat Sep 23 18:30:13 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.96099999999999997,"15min":0.99199999999999999},"min":{"1min":0.624,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.2170000000000001,"5min":1.5149999999999999,"15min":1.5149999999999999},"last":0.89400000000000002},{"interface":"front","average":{"1min":1.012,"5min":0.98299999999999998,"15min":0.98599999999999999},"min":{"1min":0.73099999999999998,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.296,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":1.163}]},{"osd":6,"last update":"Sat Sep 23 18:30:13 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.99199999999999999,"15min":1.022},"min":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.2230000000000001,"5min":1.6100000000000001,"15min":1.6100000000000001},"last":1.0620000000000001},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.0589999999999999,"15min":1.0720000000000001},"min":{"1min":0.68899999999999995,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.3839999999999999,"5min":1.4419999999999999,"15min":1.4419999999999999},"last":0.97999999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:30:13 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.0860000000000001,"15min":1.1060000000000001},"min":{"1min":0.70499999999999996,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.3580000000000001,"5min":1.55,"15min":1.55},"last":1.21},{"interface":"front","average":{"1min":1.006,"5min":1.0840000000000001,"15min":1.1240000000000001},"min":{"1min":0.79600000000000004,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.268,"5min":1.575,"15min":1.575},"last":1.03}]}]},{"osd":0,"up_from":88,"seq":377957122096,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33240,"kb_used_data":1680,"kb_used_omap":8,"kb_used_meta":31543,"kb_avail":93711912,"statfs":{"total":95995035648,"available":95960997888,"internally_reserved":0,"allocated":1720320,"data_stored":1178335,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":32300926},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:30:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.63600000000000001,"15min":0.64000000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.77500000000000002,"5min":1.2170000000000001,"15min":1.2170000000000001},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.77900000000000003,"15min":0.79400000000000004},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.97899999999999998,"5min":1.3480000000000001,"15min":1.3480000000000001},"last":0.81999999999999995}]},{"osd":2,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.80500000000000005,"15min":0.82399999999999995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.202,"5min":1.254,"15min":1.254},"last":0.76100000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.67000000000000004,"15min":0.68300000000000005},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.90100000000000002,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.54700000000000004}]},{"osd":3,"last update":"Sat Sep 23 18:30:02 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.77300000000000002,"15min":0.78500000000000003},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.103,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":1.0720000000000001},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.84599999999999997,"15min":0.87},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.0429999999999999,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.96999999999999997}]},{"osd":4,"last update":"Sat Sep 23 18:29:41 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.97299999999999998,"15min":0.97399999999999998},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.4970000000000001,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":0.98699999999999999},{"interface":"front","average":{"1min":0.90700000000000003,"5min":0.98399999999999999,"15min":1.0089999999999999},"min":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.29,"5min":1.29,"15min":1.29},"last":1.2010000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:29:41 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.99099999999999999,"15min":1.0149999999999999},"min":{"1min":0.68000000000000005,"5min":0.67200000000000004,"15min":0.67200000000000004},"max":{"1min":1.23,"5min":1.524,"15min":1.524},"last":1.2669999999999999},{"interface":"front","average":{"1min":0.96399999999999997,"5min":1.0129999999999999,"15min":1.0349999999999999},"min":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"max":{"1min":1.544,"5min":1.544,"15min":1.544},"last":0.79300000000000004}]},{"osd":6,"last update":"Sat Sep 23 18:29:41 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":1.036,"15min":1.0640000000000001},"min":{"1min":0.69999999999999996,"5min":0.64100000000000001,"15min":0.64100000000000001},"max":{"1min":1.343,"5min":1.446,"15min":1.446},"last":1.3120000000000001},{"interface":"front","average":{"1min":0.96199999999999997,"5min":1.0329999999999999,"15min":1.0469999999999999},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.458,"5min":1.4610000000000001,"15min":1.4610000000000001},"last":0.92800000000000005}]},{"osd":7,"last update":"Sat Sep 23 18:29:41 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":1.034,"15min":1.054},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.619,"5min":1.619,"15min":1.619},"last":0.879},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.0660000000000001,"15min":1.0840000000000001},"min":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"max":{"1min":1.421,"5min":1.5720000000000001,"15min":1.5720000000000001},"last":0.90800000000000003}]}]},{"osd":2,"up_from":98,"seq":420906795045,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36672,"kb_used_data":5176,"kb_used_omap":11,"kb_used_meta":31476,"kb_avail":93708480,"statfs":{"total":95995035648,"available":95957483520,"internally_reserved":0,"allocated":5300224,"data_stored":4779645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11949,"internal_metadata":32231763},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.66700000000000004,"15min":0.67700000000000005},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":1.012,"5min":1.4430000000000001,"15min":1.4430000000000001},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.1799999999999999,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":1.0389999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64900000000000002,"15min":0.65100000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.96499999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.69299999999999995,"5min":0.74099999999999999,"15min":0.749},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":1.006,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.82999999999999996}]},{"osd":3,"last update":"Sat Sep 23 18:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.71499999999999997,"15min":0.72399999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.3080000000000001,"5min":1.3080000000000001,"15min":1.3080000000000001},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.70899999999999996,"15min":0.71799999999999997},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.0680000000000001,"5min":1.5189999999999999,"15min":1.5189999999999999},"last":0.995}]},{"osd":4,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.92300000000000004,"15min":0.92800000000000005},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":1.3939999999999999},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.97499999999999998,"15min":0.98099999999999998},"min":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.393,"5min":1.579,"15min":1.579},"last":1.3129999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":1.002,"15min":1.0189999999999999},"min":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.333,"5min":1.6990000000000001,"15min":1.6990000000000001},"last":1.466},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1,"15min":0.995},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.488,"5min":1.488,"15min":1.488},"last":1.2769999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":1.0309999999999999,"15min":1.038},"min":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.373,"5min":1.4199999999999999,"15min":1.4199999999999999},"last":1.1719999999999999},{"interface":"front","average":{"1min":1.026,"5min":1.0660000000000001,"15min":1.073},"min":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"max":{"1min":1.4430000000000001,"5min":1.631,"15min":1.631},"last":1.6339999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:29:35 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.0589999999999999,"15min":1.0680000000000001},"min":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":1.538,"5min":1.538,"15min":1.538},"last":1.3440000000000001},{"interface":"front","average":{"1min":0.999,"5min":1.1020000000000001,"15min":1.119},"min":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.2889999999999999,"5min":1.7609999999999999,"15min":1.7609999999999999},"last":1.5780000000000001}]}]},{"osd":3,"up_from":103,"seq":442381631519,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28388,"kb_used_data":1052,"kb_used_omap":4,"kb_used_meta":27323,"kb_avail":93716764,"statfs":{"total":95995035648,"available":95965966336,"internally_reserved":0,"allocated":1077248,"data_stored":580581,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4826,"internal_metadata":27979046},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:30:04 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.67500000000000004,"15min":0.68899999999999995},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.93600000000000005,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.72499999999999998,"15min":0.74099999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.1240000000000001,"5min":1.1240000000000001,"15min":1.1240000000000001},"last":0.497}]},{"osd":1,"last update":"Sat Sep 23 18:30:04 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.70299999999999996,"15min":0.71499999999999997},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":1.022,"5min":1.022,"15min":1.022},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.69599999999999995,"5min":0.76400000000000001,"15min":0.77600000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.002,"5min":1.1439999999999999,"15min":1.1439999999999999},"last":0.55700000000000005}]},{"osd":2,"last update":"Sat Sep 23 18:30:04 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.72499999999999998,"15min":0.74099999999999999},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.0760000000000001,"5min":1.0760000000000001,"15min":1.0760000000000001},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.79400000000000004,"15min":0.80600000000000005},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.1719999999999999,"5min":1.1719999999999999,"15min":1.1719999999999999},"last":0.66200000000000003}]},{"osd":4,"last update":"Sat Sep 23 18:30:04 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":0.95599999999999996,"15min":0.96499999999999997},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.3260000000000001,"5min":1.3260000000000001,"15min":1.3260000000000001},"last":0.94399999999999995},{"interface":"front","average":{"1min":0.94899999999999995,"5min":1.0209999999999999,"15min":1.0329999999999999},"min":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.2,"5min":1.431,"15min":1.431},"last":0.78200000000000003}]},{"osd":5,"last update":"Sat Sep 23 18:30:04 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":1.0580000000000001,"15min":1.0780000000000001},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.2949999999999999,"5min":1.444,"15min":1.444},"last":0.873},{"interface":"front","average":{"1min":0.93500000000000005,"5min":1.1060000000000001,"15min":1.1339999999999999},"min":{"1min":0.623,"5min":0.623,"15min":0.623},"max":{"1min":1.2649999999999999,"5min":1.5409999999999999,"15min":1.5409999999999999},"last":0.84999999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:30:04 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":1.1000000000000001,"15min":1.127},"min":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.357,"5min":1.492,"15min":1.492},"last":0.91700000000000004},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.075,"15min":1.091},"min":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"max":{"1min":1.4059999999999999,"5min":1.425,"15min":1.425},"last":1.0569999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:30:04 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":1.107,"15min":1.127},"min":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"max":{"1min":1.4850000000000001,"5min":1.5529999999999999,"15min":1.5529999999999999},"last":1.163},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.1419999999999999,"15min":1.1599999999999999},"min":{"1min":0.77800000000000002,"5min":0.71699999999999997,"15min":0.71699999999999997},"max":{"1min":1.381,"5min":1.4530000000000001,"15min":1.4530000000000001},"last":0.96699999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019129,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298152,"kb_used_data":1056,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447000,"statfs":{"total":95995035648,"available":95689728000,"internally_reserved":0,"allocated":1081344,"data_stored":581243,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:29:40 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.92700000000000005,"15min":0.92900000000000005},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.171,"5min":1.2809999999999999,"15min":1.2809999999999999},"last":0.95699999999999996},{"interface":"front","average":{"1min":0.93200000000000005,"5min":1.014,"15min":1.028},"min":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.2310000000000001,"5min":1.258,"15min":1.258},"last":1.171}]},{"osd":1,"last update":"Sat Sep 23 18:30:08 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.86499999999999999,"15min":0.84699999999999998},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.2110000000000001,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":1.05},{"interface":"front","average":{"1min":0.88700000000000001,"5min":0.88500000000000001,"15min":0.88600000000000001},"min":{"1min":0.63300000000000001,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.103,"5min":1.224,"15min":1.224},"last":1.135}]},{"osd":2,"last update":"Sat Sep 23 18:29:40 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":1.048,"15min":1.0489999999999999},"min":{"1min":0.72899999999999998,"5min":0.63,"15min":0.63},"max":{"1min":1.302,"5min":1.302,"15min":1.302},"last":1.274},{"interface":"front","average":{"1min":0.95999999999999996,"5min":1.002,"15min":1.0089999999999999},"min":{"1min":0.68300000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"max":{"1min":1.2190000000000001,"5min":1.242,"15min":1.242},"last":1.2430000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:30:08 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":0.89700000000000002,"15min":0.90500000000000003},"min":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.008,"5min":1.1899999999999999,"15min":1.1899999999999999},"last":1.571},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.0309999999999999,"15min":1.042},"min":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"max":{"1min":1.1930000000000001,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":1.3140000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:29:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":0.97699999999999998,"15min":0.97299999999999998},"min":{"1min":0.72199999999999998,"5min":0.60199999999999998,"15min":0.51600000000000001},"max":{"1min":1.264,"5min":1.3919999999999999,"15min":2.266},"last":1.468},{"interface":"front","average":{"1min":0.996,"5min":0.97599999999999998,"15min":0.97199999999999998},"min":{"1min":0.71499999999999997,"5min":0.58999999999999997,"15min":0.48199999999999998},"max":{"1min":1.298,"5min":1.411,"15min":2.4929999999999999},"last":1.3919999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:29:58 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.0109999999999999,"15min":1.0269999999999999},"min":{"1min":0.83599999999999997,"5min":0.65000000000000002,"15min":0.57699999999999996},"max":{"1min":1.323,"5min":1.323,"15min":2.351},"last":1.647},{"interface":"front","average":{"1min":1.044,"5min":1.0389999999999999,"15min":1.0169999999999999},"min":{"1min":0.66800000000000004,"5min":0.63300000000000001,"15min":0.42899999999999999},"max":{"1min":1.2769999999999999,"5min":1.3660000000000001,"15min":2.621},"last":1.4590000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:29:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.105,"15min":1.083},"min":{"1min":0.92000000000000004,"5min":0.66200000000000003,"15min":0.50800000000000001},"max":{"1min":1.3600000000000001,"5min":1.4339999999999999,"15min":2.6819999999999999},"last":1.357},{"interface":"front","average":{"1min":1.151,"5min":1.1579999999999999,"15min":1.169},"min":{"1min":0.86499999999999999,"5min":0.80400000000000005,"15min":0.66400000000000003},"max":{"1min":1.3520000000000001,"5min":1.4490000000000001,"15min":2.4140000000000001},"last":1.7130000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822897,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302228,"kb_used_data":5132,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93442924,"statfs":{"total":95995035648,"available":95685554176,"internally_reserved":0,"allocated":5255168,"data_stored":4770680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:29:42 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":0.85099999999999998,"15min":0.85099999999999998},"min":{"1min":0.57899999999999996,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.228,"5min":1.244,"15min":1.244},"last":0.81200000000000006},{"interface":"front","average":{"1min":0.83199999999999996,"5min":0.86399999999999999,"15min":0.872},"min":{"1min":0.56200000000000006,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.123,"5min":1.1930000000000001,"15min":1.1930000000000001},"last":1.05}]},{"osd":1,"last update":"Sat Sep 23 18:30:10 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.87,"15min":0.85599999999999998},"min":{"1min":0.64200000000000002,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.256,"5min":1.256,"15min":1.256},"last":1.01},{"interface":"front","average":{"1min":0.82499999999999996,"5min":0.89600000000000002,"15min":0.92600000000000005},"min":{"1min":0.54600000000000004,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.081,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":0.85399999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:29:38 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":0.90900000000000003,"15min":0.91200000000000003},"min":{"1min":0.628,"5min":0.628,"15min":0.628},"max":{"1min":1.274,"5min":1.274,"15min":1.274},"last":1.367},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.96199999999999997,"15min":0.95899999999999996},"min":{"1min":0.59999999999999998,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.2350000000000001,"5min":1.2350000000000001,"15min":1.2350000000000001},"last":1.26}]},{"osd":3,"last update":"Sat Sep 23 18:30:05 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":0.94999999999999996,"15min":0.94499999999999995},"min":{"1min":0.73699999999999999,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.181,"5min":1.181,"15min":1.181},"last":1.1180000000000001},{"interface":"front","average":{"1min":0.89100000000000001,"5min":0.90100000000000002,"15min":0.90300000000000002},"min":{"1min":0.745,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.0720000000000001,"5min":1.1140000000000001,"15min":1.1140000000000001},"last":1.1919999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:30:10 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.90000000000000002,"15min":0.89500000000000002},"min":{"1min":0.64600000000000002,"5min":0.55000000000000004,"15min":0.39600000000000002},"max":{"1min":1.2250000000000001,"5min":1.3879999999999999,"15min":2.0259999999999998},"last":1.236},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.91800000000000004,"15min":0.93000000000000005},"min":{"1min":0.64700000000000002,"5min":0.60399999999999998,"15min":0.36699999999999999},"max":{"1min":1.252,"5min":1.319,"15min":1.73},"last":1.3049999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":0.99099999999999999,"15min":0.99099999999999999},"min":{"1min":0.77400000000000002,"5min":0.64700000000000002,"15min":0.47399999999999998},"max":{"1min":1.2729999999999999,"5min":1.464,"15min":2.137},"last":1.331},{"interface":"front","average":{"1min":1.1020000000000001,"5min":1.085,"15min":1.069},"min":{"1min":0.86499999999999999,"5min":0.79200000000000004,"15min":0.51100000000000001},"max":{"1min":1.3320000000000001,"5min":1.4199999999999999,"15min":1.8779999999999999},"last":1.4139999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:29:42 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.046,"15min":1.0509999999999999},"min":{"1min":0.66600000000000004,"5min":0.65400000000000003,"15min":0.53500000000000003},"max":{"1min":1.3180000000000001,"5min":1.5589999999999999,"15min":2.1549999999999998},"last":1.347},{"interface":"front","average":{"1min":1.087,"5min":1.0820000000000001,"15min":1.083},"min":{"1min":0.83999999999999997,"5min":0.65200000000000002,"15min":0.54300000000000004},"max":{"1min":1.3069999999999999,"5min":1.6990000000000001,"15min":2.2189999999999999},"last":1.3919999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4224,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7725,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3429,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:30:29.481 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T18:30:29.482 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd dump 2023-09-23T18:30:29.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:29 smithi098 bash[79516]: audit 2023-09-23T18:30:28.630873+0000 mgr.y (mgr.26054) 354 : audit 0 from='client.44630 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:30:29.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:29 smithi098 bash[79516]: audit 2023-09-23T18:30:29.517312+0000 mon.a (mon.0) 506 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:30:29.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:29 smithi098 bash[79516]: audit 2023-09-23T18:30:29.525493+0000 mon.a (mon.0) 507 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:30:29.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:29 smithi098 bash[76817]: audit 2023-09-23T18:30:28.630873+0000 mgr.y (mgr.26054) 354 : audit 0 from='client.44630 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:30:29.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:29 smithi098 bash[76817]: audit 2023-09-23T18:30:29.517312+0000 mon.a (mon.0) 506 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:30:29.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:29 smithi098 bash[76817]: audit 2023-09-23T18:30:29.525493+0000 mon.a (mon.0) 507 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:30:30.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:29 smithi173 bash[59422]: audit 2023-09-23T18:30:28.630873+0000 mgr.y (mgr.26054) 354 : audit 0 from='client.44630 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:30:30.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:29 smithi173 bash[59422]: audit 2023-09-23T18:30:29.517312+0000 mon.a (mon.0) 506 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:30:30.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:29 smithi173 bash[59422]: audit 2023-09-23T18:30:29.525493+0000 mon.a (mon.0) 507 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:30:31.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:30 smithi173 bash[59422]: cluster 2023-09-23T18:30:29.644223+0000 mgr.y (mgr.26054) 355 : cluster 0 pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:31.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:30 smithi173 bash[59422]: audit 2023-09-23T18:30:30.129151+0000 mon.a (mon.0) 508 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:30:31.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:30 smithi173 bash[59422]: audit 2023-09-23T18:30:30.137004+0000 mon.a (mon.0) 509 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:30:31.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:30 smithi173 bash[59422]: audit 2023-09-23T18:30:30.612212+0000 mon.a (mon.0) 510 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:30:31.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:30 smithi173 bash[59422]: audit 2023-09-23T18:30:30.613966+0000 mon.a (mon.0) 511 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:30:31.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:30 smithi173 bash[59422]: audit 2023-09-23T18:30:30.622818+0000 mon.a (mon.0) 512 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:30:31.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:30 smithi098 bash[76817]: cluster 2023-09-23T18:30:29.644223+0000 mgr.y (mgr.26054) 355 : cluster 0 pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:31.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:30 smithi098 bash[76817]: audit 2023-09-23T18:30:30.129151+0000 mon.a (mon.0) 508 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:30:31.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:30 smithi098 bash[76817]: audit 2023-09-23T18:30:30.137004+0000 mon.a (mon.0) 509 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:30:31.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:30 smithi098 bash[76817]: audit 2023-09-23T18:30:30.612212+0000 mon.a (mon.0) 510 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:30:31.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:30 smithi098 bash[76817]: audit 2023-09-23T18:30:30.613966+0000 mon.a (mon.0) 511 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:30:31.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:30 smithi098 bash[76817]: audit 2023-09-23T18:30:30.622818+0000 mon.a (mon.0) 512 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:30:31.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:30 smithi098 bash[79516]: cluster 2023-09-23T18:30:29.644223+0000 mgr.y (mgr.26054) 355 : cluster 0 pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:31.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:30 smithi098 bash[79516]: audit 2023-09-23T18:30:30.129151+0000 mon.a (mon.0) 508 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:30:31.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:30 smithi098 bash[79516]: audit 2023-09-23T18:30:30.137004+0000 mon.a (mon.0) 509 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:30:31.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:30 smithi098 bash[79516]: audit 2023-09-23T18:30:30.612212+0000 mon.a (mon.0) 510 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:30:31.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:30 smithi098 bash[79516]: audit 2023-09-23T18:30:30.613966+0000 mon.a (mon.0) 511 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:30:31.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:30 smithi098 bash[79516]: audit 2023-09-23T18:30:30.622818+0000 mon.a (mon.0) 512 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:30:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:32 smithi173 bash[59422]: cluster 2023-09-23T18:30:31.645181+0000 mgr.y (mgr.26054) 356 : cluster 0 pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:33.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:32 smithi098 bash[76817]: cluster 2023-09-23T18:30:31.645181+0000 mgr.y (mgr.26054) 356 : cluster 0 pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:33.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:32 smithi098 bash[79516]: cluster 2023-09-23T18:30:31.645181+0000 mgr.y (mgr.26054) 356 : cluster 0 pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:34.260 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:30:35.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:34 smithi173 bash[59422]: cluster 2023-09-23T18:30:33.645906+0000 mgr.y (mgr.26054) 357 : cluster 0 pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:35.167 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:34 smithi098 bash[79516]: cluster 2023-09-23T18:30:33.645906+0000 mgr.y (mgr.26054) 357 : cluster 0 pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:35.167 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:34 smithi098 bash[76817]: cluster 2023-09-23T18:30:33.645906+0000 mgr.y (mgr.26054) 357 : cluster 0 pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:36.440 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:30:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:30:36] "GET /metrics HTTP/1.1" 200 34094 "" "Prometheus/2.33.4" 2023-09-23T18:30:37.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:36 smithi173 bash[59422]: cluster 2023-09-23T18:30:35.646770+0000 mgr.y (mgr.26054) 358 : cluster 0 pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:37.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:36 smithi098 bash[76817]: cluster 2023-09-23T18:30:35.646770+0000 mgr.y (mgr.26054) 358 : cluster 0 pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:37.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:36 smithi098 bash[79516]: cluster 2023-09-23T18:30:35.646770+0000 mgr.y (mgr.26054) 358 : cluster 0 pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:37.646 INFO:teuthology.orchestra.run.smithi098.stdout:epoch 104 2023-09-23T18:30:37.647 INFO:teuthology.orchestra.run.smithi098.stdout:fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 2023-09-23T18:30:37.647 INFO:teuthology.orchestra.run.smithi098.stdout:created 2023-09-23T18:04:11.580781+0000 2023-09-23T18:30:37.647 INFO:teuthology.orchestra.run.smithi098.stdout:modified 2023-09-23T18:28:03.269518+0000 2023-09-23T18:30:37.647 INFO:teuthology.orchestra.run.smithi098.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-09-23T18:30:37.647 INFO:teuthology.orchestra.run.smithi098.stdout:crush_version 22 2023-09-23T18:30:37.647 INFO:teuthology.orchestra.run.smithi098.stdout:full_ratio 0.95 2023-09-23T18:30:37.647 INFO:teuthology.orchestra.run.smithi098.stdout:backfillfull_ratio 0.9 2023-09-23T18:30:37.647 INFO:teuthology.orchestra.run.smithi098.stdout:nearfull_ratio 0.85 2023-09-23T18:30:37.648 INFO:teuthology.orchestra.run.smithi098.stdout:require_min_compat_client luminous 2023-09-23T18:30:37.648 INFO:teuthology.orchestra.run.smithi098.stdout:min_compat_client luminous 2023-09-23T18:30:37.648 INFO:teuthology.orchestra.run.smithi098.stdout:require_osd_release pacific 2023-09-23T18:30:37.648 INFO:teuthology.orchestra.run.smithi098.stdout:stretch_mode_enabled false 2023-09-23T18:30:37.648 INFO:teuthology.orchestra.run.smithi098.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 82 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr,mgr_devicehealth read_balance_score 7.89 2023-09-23T18:30:37.648 INFO:teuthology.orchestra.run.smithi098.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 77 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2023-09-23T18:30:37.648 INFO:teuthology.orchestra.run.smithi098.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 59 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-09-23T18:30:37.648 INFO:teuthology.orchestra.run.smithi098.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 60 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2023-09-23T18:30:37.648 INFO:teuthology.orchestra.run.smithi098.stdout:max_osd 8 2023-09-23T18:30:37.648 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0 up in weight 1 up_from 88 up_thru 103 down_at 85 last_clean_interval [8,84) [v2:172.21.15.98:6802/1320995478,v1:172.21.15.98:6803/1320995478] [v2:172.21.15.98:6804/1320995478,v1:172.21.15.98:6805/1320995478] exists,up 4f35f176-4d07-4157-aeb9-dd8f71da3a63 2023-09-23T18:30:37.648 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1 up in weight 1 up_from 93 up_thru 103 down_at 90 last_clean_interval [13,89) [v2:172.21.15.98:6810/1561159155,v1:172.21.15.98:6811/1561159155] [v2:172.21.15.98:6812/1561159155,v1:172.21.15.98:6813/1561159155] exists,up 5d777d1f-3bd3-49ba-b0de-0ac532ee3934 2023-09-23T18:30:37.648 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2 up in weight 1 up_from 98 up_thru 103 down_at 95 last_clean_interval [18,94) [v2:172.21.15.98:6818/4249816143,v1:172.21.15.98:6819/4249816143] [v2:172.21.15.98:6820/4249816143,v1:172.21.15.98:6821/4249816143] exists,up 63978288-ac36-4cf2-bbb7-46dc828e6ce5 2023-09-23T18:30:37.649 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3 up in weight 1 up_from 103 up_thru 103 down_at 100 last_clean_interval [25,99) [v2:172.21.15.98:6826/2042614518,v1:172.21.15.98:6827/2042614518] [v2:172.21.15.98:6828/2042614518,v1:172.21.15.98:6829/2042614518] exists,up 48596d91-46da-4815-b810-d493bbf01dcc 2023-09-23T18:30:37.649 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4 up in weight 1 up_from 30 up_thru 103 down_at 0 last_clean_interval [0,0) [v2:172.21.15.173:6800/165906554,v1:172.21.15.173:6801/165906554] [v2:172.21.15.173:6802/165906554,v1:172.21.15.173:6803/165906554] exists,up a562afc6-ffe6-44bb-91a9-8a04018c3c6f 2023-09-23T18:30:37.649 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5 up in weight 1 up_from 36 up_thru 103 down_at 0 last_clean_interval [0,0) [v2:172.21.15.173:6808/2894909344,v1:172.21.15.173:6809/2894909344] [v2:172.21.15.173:6810/2894909344,v1:172.21.15.173:6811/2894909344] exists,up add28146-8301-4b4c-9616-967e9df3a8d8 2023-09-23T18:30:37.649 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6 up in weight 1 up_from 42 up_thru 100 down_at 0 last_clean_interval [0,0) [v2:172.21.15.173:6816/3550013548,v1:172.21.15.173:6817/3550013548] [v2:172.21.15.173:6818/3550013548,v1:172.21.15.173:6819/3550013548] exists,up f30854de-bc26-465d-8b0e-a9211ae2d9fa 2023-09-23T18:30:37.649 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7 up in weight 1 up_from 48 up_thru 103 down_at 0 last_clean_interval [0,0) [v2:172.21.15.173:6824/135608222,v1:172.21.15.173:6825/135608222] [v2:172.21.15.173:6826/135608222,v1:172.21.15.173:6827/135608222] exists,up 5f540de3-48f9-4bc8-b14a-9aadf838f16b 2023-09-23T18:30:37.649 INFO:teuthology.orchestra.run.smithi098.stdout:pg_upmap_items 4.a [1,2] 2023-09-23T18:30:37.649 INFO:teuthology.orchestra.run.smithi098.stdout:pg_upmap_items 4.b [1,2] 2023-09-23T18:30:37.649 INFO:teuthology.orchestra.run.smithi098.stdout:pg_upmap_items 4.18 [1,5] 2023-09-23T18:30:37.649 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.98:0/325576677 expires 2023-09-24T18:21:39.415268+0000 2023-09-23T18:30:37.649 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.98:0/3213365423 expires 2023-09-24T18:04:40.596823+0000 2023-09-23T18:30:37.649 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.98:0/2288964511 expires 2023-09-24T18:21:39.415268+0000 2023-09-23T18:30:37.649 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.98:0/2098141849 expires 2023-09-24T18:18:53.712554+0000 2023-09-23T18:30:37.650 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.98:0/910147733 expires 2023-09-24T18:21:39.415268+0000 2023-09-23T18:30:37.650 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.98:6800/3510470804 expires 2023-09-24T18:21:39.415268+0000 2023-09-23T18:30:37.650 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.173:0/2105184133 expires 2023-09-24T18:20:34.294726+0000 2023-09-23T18:30:37.650 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.98:6800/3706591260 expires 2023-09-24T18:18:53.712554+0000 2023-09-23T18:30:37.650 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.98:0/1142988558 expires 2023-09-24T18:12:42.525837+0000 2023-09-23T18:30:37.650 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.98:6801/3510470804 expires 2023-09-24T18:21:39.415268+0000 2023-09-23T18:30:37.650 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.98:0/2194193535 expires 2023-09-24T18:12:42.525837+0000 2023-09-23T18:30:37.650 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.98:0/4085572202 expires 2023-09-24T18:18:53.712554+0000 2023-09-23T18:30:37.650 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.98:6800/3368447653 expires 2023-09-24T18:04:40.596823+0000 2023-09-23T18:30:37.650 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.173:6834/1425795384 expires 2023-09-24T18:20:34.294726+0000 2023-09-23T18:30:37.650 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.98:6800/483469857 expires 2023-09-24T18:12:42.525837+0000 2023-09-23T18:30:37.651 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.98:0/3410138197 expires 2023-09-24T18:21:39.415268+0000 2023-09-23T18:30:37.651 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.98:0/3976553993 expires 2023-09-24T18:04:40.596823+0000 2023-09-23T18:30:37.651 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.98:0/3472030975 expires 2023-09-24T18:05:11.060631+0000 2023-09-23T18:30:37.651 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.173:0/4281585910 expires 2023-09-24T18:20:34.294726+0000 2023-09-23T18:30:37.651 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.98:6801/3706591260 expires 2023-09-24T18:18:53.712554+0000 2023-09-23T18:30:37.651 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.98:0/1603044865 expires 2023-09-24T18:05:11.060631+0000 2023-09-23T18:30:37.651 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.173:6835/1425795384 expires 2023-09-24T18:20:34.294726+0000 2023-09-23T18:30:37.651 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.173:0/2782586450 expires 2023-09-24T18:20:34.294726+0000 2023-09-23T18:30:37.651 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.98:0/3501988932 expires 2023-09-24T18:21:39.415268+0000 2023-09-23T18:30:37.651 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.98:6801/483469857 expires 2023-09-24T18:12:42.525837+0000 2023-09-23T18:30:37.651 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.98:6801/3368447653 expires 2023-09-24T18:04:40.596823+0000 2023-09-23T18:30:37.651 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.98:0/335394369 expires 2023-09-24T18:21:39.415268+0000 2023-09-23T18:30:37.652 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.173:0/2534791896 expires 2023-09-24T18:20:34.294726+0000 2023-09-23T18:30:37.652 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.98:6800/1392949322 expires 2023-09-24T18:05:11.060631+0000 2023-09-23T18:30:37.652 INFO:teuthology.orchestra.run.smithi098.stdout:blocklist 172.21.15.98:6801/1392949322 expires 2023-09-24T18:05:11.060631+0000 2023-09-23T18:30:37.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:37 smithi098 bash[79516]: audit 2023-09-23T18:30:37.644037+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.98:0/1743972102' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T18:30:37.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:37 smithi098 bash[76817]: audit 2023-09-23T18:30:37.644037+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.98:0/1743972102' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T18:30:38.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:37 smithi173 bash[59422]: audit 2023-09-23T18:30:37.644037+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.98:0/1743972102' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T18:30:38.454 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 88 up_thru 103 down_at 85 last_clean_interval [8,84) [v2:172.21.15.98:6802/1320995478,v1:172.21.15.98:6803/1320995478] [v2:172.21.15.98:6804/1320995478,v1:172.21.15.98:6805/1320995478] exists,up 4f35f176-4d07-4157-aeb9-dd8f71da3a63', 'osd.1 up in weight 1 up_from 93 up_thru 103 down_at 90 last_clean_interval [13,89) [v2:172.21.15.98:6810/1561159155,v1:172.21.15.98:6811/1561159155] [v2:172.21.15.98:6812/1561159155,v1:172.21.15.98:6813/1561159155] exists,up 5d777d1f-3bd3-49ba-b0de-0ac532ee3934', 'osd.2 up in weight 1 up_from 98 up_thru 103 down_at 95 last_clean_interval [18,94) [v2:172.21.15.98:6818/4249816143,v1:172.21.15.98:6819/4249816143] [v2:172.21.15.98:6820/4249816143,v1:172.21.15.98:6821/4249816143] exists,up 63978288-ac36-4cf2-bbb7-46dc828e6ce5', 'osd.3 up in weight 1 up_from 103 up_thru 103 down_at 100 last_clean_interval [25,99) [v2:172.21.15.98:6826/2042614518,v1:172.21.15.98:6827/2042614518] [v2:172.21.15.98:6828/2042614518,v1:172.21.15.98:6829/2042614518] exists,up 48596d91-46da-4815-b810-d493bbf01dcc', 'osd.4 up in weight 1 up_from 30 up_thru 103 down_at 0 last_clean_interval [0,0) [v2:172.21.15.173:6800/165906554,v1:172.21.15.173:6801/165906554] [v2:172.21.15.173:6802/165906554,v1:172.21.15.173:6803/165906554] exists,up a562afc6-ffe6-44bb-91a9-8a04018c3c6f', 'osd.5 up in weight 1 up_from 36 up_thru 103 down_at 0 last_clean_interval [0,0) [v2:172.21.15.173:6808/2894909344,v1:172.21.15.173:6809/2894909344] [v2:172.21.15.173:6810/2894909344,v1:172.21.15.173:6811/2894909344] exists,up add28146-8301-4b4c-9616-967e9df3a8d8', 'osd.6 up in weight 1 up_from 42 up_thru 100 down_at 0 last_clean_interval [0,0) [v2:172.21.15.173:6816/3550013548,v1:172.21.15.173:6817/3550013548] [v2:172.21.15.173:6818/3550013548,v1:172.21.15.173:6819/3550013548] exists,up f30854de-bc26-465d-8b0e-a9211ae2d9fa', 'osd.7 up in weight 1 up_from 48 up_thru 103 down_at 0 last_clean_interval [0,0) [v2:172.21.15.173:6824/135608222,v1:172.21.15.173:6825/135608222] [v2:172.21.15.173:6826/135608222,v1:172.21.15.173:6827/135608222] exists,up 5f540de3-48f9-4bc8-b14a-9aadf838f16b'] 2023-09-23T18:30:38.455 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon mon.a config show 2023-09-23T18:30:39.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:38 smithi173 bash[59422]: cluster 2023-09-23T18:30:37.647523+0000 mgr.y (mgr.26054) 359 : cluster 0 pgmap v292: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:39.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:38 smithi098 bash[76817]: cluster 2023-09-23T18:30:37.647523+0000 mgr.y (mgr.26054) 359 : cluster 0 pgmap v292: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:39.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:38 smithi098 bash[79516]: cluster 2023-09-23T18:30:37.647523+0000 mgr.y (mgr.26054) 359 : cluster 0 pgmap v292: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:40.690 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:40 smithi098 bash[79516]: audit 2023-09-23T18:30:39.577437+0000 mon.a (mon.0) 513 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:30:40.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:40 smithi098 bash[79516]: cluster 2023-09-23T18:30:39.648171+0000 mgr.y (mgr.26054) 360 : cluster 0 pgmap v293: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:40.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:40 smithi098 bash[76817]: audit 2023-09-23T18:30:39.577437+0000 mon.a (mon.0) 513 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:30:40.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:40 smithi098 bash[76817]: cluster 2023-09-23T18:30:39.648171+0000 mgr.y (mgr.26054) 360 : cluster 0 pgmap v293: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:40.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:40 smithi173 bash[59422]: audit 2023-09-23T18:30:39.577437+0000 mon.a (mon.0) 513 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:30:40.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:40 smithi173 bash[59422]: cluster 2023-09-23T18:30:39.648171+0000 mgr.y (mgr.26054) 360 : cluster 0 pgmap v293: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:43.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:42 smithi173 bash[59422]: cluster 2023-09-23T18:30:41.649128+0000 mgr.y (mgr.26054) 361 : cluster 0 pgmap v294: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:43.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:42 smithi098 bash[76817]: cluster 2023-09-23T18:30:41.649128+0000 mgr.y (mgr.26054) 361 : cluster 0 pgmap v294: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:43.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:42 smithi098 bash[79516]: cluster 2023-09-23T18:30:41.649128+0000 mgr.y (mgr.26054) 361 : cluster 0 pgmap v294: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:43.227 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:30:44.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:44 smithi098 bash[76817]: cluster 2023-09-23T18:30:43.649944+0000 mgr.y (mgr.26054) 362 : cluster 0 pgmap v295: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:44.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:44 smithi098 bash[79516]: cluster 2023-09-23T18:30:43.649944+0000 mgr.y (mgr.26054) 362 : cluster 0 pgmap v295: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:45.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:44 smithi173 bash[59422]: cluster 2023-09-23T18:30:43.649944+0000 mgr.y (mgr.26054) 362 : cluster 0 pgmap v295: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:46.240 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:30:46.240 INFO:teuthology.orchestra.run.smithi098.stdout: "name": "mon.a", 2023-09-23T18:30:46.240 INFO:teuthology.orchestra.run.smithi098.stdout: "cluster": "ceph", 2023-09-23T18:30:46.240 INFO:teuthology.orchestra.run.smithi098.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-09-23T18:30:46.240 INFO:teuthology.orchestra.run.smithi098.stdout: "admin_socket_mode": "", 2023-09-23T18:30:46.240 INFO:teuthology.orchestra.run.smithi098.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-09-23T18:30:46.240 INFO:teuthology.orchestra.run.smithi098.stdout: "auth_client_required": "cephx, none", 2023-09-23T18:30:46.241 INFO:teuthology.orchestra.run.smithi098.stdout: "auth_cluster_required": "cephx", 2023-09-23T18:30:46.241 INFO:teuthology.orchestra.run.smithi098.stdout: "auth_debug": "true", 2023-09-23T18:30:46.241 INFO:teuthology.orchestra.run.smithi098.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-09-23T18:30:46.241 INFO:teuthology.orchestra.run.smithi098.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-09-23T18:30:46.241 INFO:teuthology.orchestra.run.smithi098.stdout: "auth_service_required": "cephx", 2023-09-23T18:30:46.241 INFO:teuthology.orchestra.run.smithi098.stdout: "auth_service_ticket_ttl": "240.000000", 2023-09-23T18:30:46.241 INFO:teuthology.orchestra.run.smithi098.stdout: "auth_supported": "", 2023-09-23T18:30:46.241 INFO:teuthology.orchestra.run.smithi098.stdout: "bdev_aio": "true", 2023-09-23T18:30:46.241 INFO:teuthology.orchestra.run.smithi098.stdout: "bdev_aio_max_queue_depth": "1024", 2023-09-23T18:30:46.241 INFO:teuthology.orchestra.run.smithi098.stdout: "bdev_aio_poll_ms": "250", 2023-09-23T18:30:46.241 INFO:teuthology.orchestra.run.smithi098.stdout: "bdev_aio_reap_max": "16", 2023-09-23T18:30:46.241 INFO:teuthology.orchestra.run.smithi098.stdout: "bdev_async_discard": "false", 2023-09-23T18:30:46.242 INFO:teuthology.orchestra.run.smithi098.stdout: "bdev_block_size": "4096", 2023-09-23T18:30:46.242 INFO:teuthology.orchestra.run.smithi098.stdout: "bdev_debug_aio": "false", 2023-09-23T18:30:46.242 INFO:teuthology.orchestra.run.smithi098.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-09-23T18:30:46.242 INFO:teuthology.orchestra.run.smithi098.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-09-23T18:30:46.242 INFO:teuthology.orchestra.run.smithi098.stdout: "bdev_debug_inflight_ios": "false", 2023-09-23T18:30:46.242 INFO:teuthology.orchestra.run.smithi098.stdout: "bdev_enable_discard": "false", 2023-09-23T18:30:46.242 INFO:teuthology.orchestra.run.smithi098.stdout: "bdev_flock_retry": "3", 2023-09-23T18:30:46.242 INFO:teuthology.orchestra.run.smithi098.stdout: "bdev_flock_retry_interval": "0.100000", 2023-09-23T18:30:46.242 INFO:teuthology.orchestra.run.smithi098.stdout: "bdev_inject_crash": "0", 2023-09-23T18:30:46.242 INFO:teuthology.orchestra.run.smithi098.stdout: "bdev_inject_crash_flush_delay": "2", 2023-09-23T18:30:46.242 INFO:teuthology.orchestra.run.smithi098.stdout: "bdev_ioring": "false", 2023-09-23T18:30:46.242 INFO:teuthology.orchestra.run.smithi098.stdout: "bdev_ioring_hipri": "false", 2023-09-23T18:30:46.243 INFO:teuthology.orchestra.run.smithi098.stdout: "bdev_ioring_sqthread_poll": "false", 2023-09-23T18:30:46.243 INFO:teuthology.orchestra.run.smithi098.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-09-23T18:30:46.243 INFO:teuthology.orchestra.run.smithi098.stdout: "bdev_read_buffer_alignment": "4096", 2023-09-23T18:30:46.243 INFO:teuthology.orchestra.run.smithi098.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-09-23T18:30:46.243 INFO:teuthology.orchestra.run.smithi098.stdout: "bdev_type": "", 2023-09-23T18:30:46.243 INFO:teuthology.orchestra.run.smithi098.stdout: "bluefs_alloc_size": "1048576", 2023-09-23T18:30:46.243 INFO:teuthology.orchestra.run.smithi098.stdout: "bluefs_allocator": "hybrid", 2023-09-23T18:30:46.243 INFO:teuthology.orchestra.run.smithi098.stdout: "bluefs_buffered_io": "true", 2023-09-23T18:30:46.243 INFO:teuthology.orchestra.run.smithi098.stdout: "bluefs_check_for_zeros": "false", 2023-09-23T18:30:46.243 INFO:teuthology.orchestra.run.smithi098.stdout: "bluefs_check_volume_selector_often": "false", 2023-09-23T18:30:46.243 INFO:teuthology.orchestra.run.smithi098.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-09-23T18:30:46.243 INFO:teuthology.orchestra.run.smithi098.stdout: "bluefs_compact_log_sync": "false", 2023-09-23T18:30:46.243 INFO:teuthology.orchestra.run.smithi098.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-09-23T18:30:46.244 INFO:teuthology.orchestra.run.smithi098.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-09-23T18:30:46.244 INFO:teuthology.orchestra.run.smithi098.stdout: "bluefs_log_compact_min_size": "16777216", 2023-09-23T18:30:46.244 INFO:teuthology.orchestra.run.smithi098.stdout: "bluefs_log_replay_check_allocations": "true", 2023-09-23T18:30:46.244 INFO:teuthology.orchestra.run.smithi098.stdout: "bluefs_max_log_runway": "4194304", 2023-09-23T18:30:46.244 INFO:teuthology.orchestra.run.smithi098.stdout: "bluefs_max_prefetch": "1048576", 2023-09-23T18:30:46.244 INFO:teuthology.orchestra.run.smithi098.stdout: "bluefs_min_flush_size": "524288", 2023-09-23T18:30:46.244 INFO:teuthology.orchestra.run.smithi098.stdout: "bluefs_min_log_runway": "1048576", 2023-09-23T18:30:46.244 INFO:teuthology.orchestra.run.smithi098.stdout: "bluefs_replay_recovery": "false", 2023-09-23T18:30:46.244 INFO:teuthology.orchestra.run.smithi098.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-09-23T18:30:46.244 INFO:teuthology.orchestra.run.smithi098.stdout: "bluefs_shared_alloc_size": "65536", 2023-09-23T18:30:46.244 INFO:teuthology.orchestra.run.smithi098.stdout: "bluefs_sync_write": "false", 2023-09-23T18:30:46.244 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-09-23T18:30:46.244 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-09-23T18:30:46.245 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-09-23T18:30:46.245 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_allocation_from_file": "true", 2023-09-23T18:30:46.245 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_allocator": "hybrid", 2023-09-23T18:30:46.245 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-09-23T18:30:46.245 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-09-23T18:30:46.245 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-09-23T18:30:46.245 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-09-23T18:30:46.245 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-09-23T18:30:46.245 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-09-23T18:30:46.245 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_blobid_prealloc": "10240", 2023-09-23T18:30:46.245 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_block_create": "true", 2023-09-23T18:30:46.245 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_block_db_create": "false", 2023-09-23T18:30:46.246 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_block_db_path": "", 2023-09-23T18:30:46.246 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_block_db_size": "0", 2023-09-23T18:30:46.246 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_block_path": "", 2023-09-23T18:30:46.246 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_block_preallocate_file": "false", 2023-09-23T18:30:46.246 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_block_size": "107374182400", 2023-09-23T18:30:46.246 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_block_wal_create": "false", 2023-09-23T18:30:46.246 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_block_wal_path": "", 2023-09-23T18:30:46.246 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_block_wal_size": "100663296", 2023-09-23T18:30:46.246 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_bluefs": "true", 2023-09-23T18:30:46.246 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-09-23T18:30:46.246 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_bluefs_env_mirror": "false", 2023-09-23T18:30:46.246 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-09-23T18:30:46.246 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-09-23T18:30:46.247 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T18:30:46.247 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T18:30:46.247 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-09-23T18:30:46.247 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T18:30:46.247 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_cache_autotune": "true", 2023-09-23T18:30:46.247 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-09-23T18:30:46.247 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-09-23T18:30:46.247 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-09-23T18:30:46.247 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-09-23T18:30:46.247 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_cache_size": "0", 2023-09-23T18:30:46.247 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-09-23T18:30:46.247 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-09-23T18:30:46.247 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-09-23T18:30:46.247 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-09-23T18:30:46.248 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_cache_type": "2q", 2023-09-23T18:30:46.249 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-09-23T18:30:46.249 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_clone_cow": "true", 2023-09-23T18:30:46.249 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_compression_algorithm": "snappy", 2023-09-23T18:30:46.249 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_compression_max_blob_size": "0", 2023-09-23T18:30:46.249 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-09-23T18:30:46.249 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-09-23T18:30:46.249 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_compression_min_blob_size": "0", 2023-09-23T18:30:46.250 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-09-23T18:30:46.250 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-09-23T18:30:46.250 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_compression_mode": "none", 2023-09-23T18:30:46.250 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-09-23T18:30:46.250 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_csum_type": "crc32c", 2023-09-23T18:30:46.250 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_debug_enforce_settings": "default", 2023-09-23T18:30:46.250 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_debug_freelist": "false", 2023-09-23T18:30:46.250 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_debug_fsck_abort": "false", 2023-09-23T18:30:46.250 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-09-23T18:30:46.250 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_debug_inject_bug21040": "false", 2023-09-23T18:30:46.250 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-09-23T18:30:46.251 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_debug_inject_read_err": "false", 2023-09-23T18:30:46.251 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_debug_legacy_omap": "false", 2023-09-23T18:30:46.251 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-09-23T18:30:46.251 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-09-23T18:30:46.251 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-09-23T18:30:46.251 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-09-23T18:30:46.251 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_debug_prefill": "0.000000", 2023-09-23T18:30:46.251 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-09-23T18:30:46.251 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-09-23T18:30:46.251 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-09-23T18:30:46.251 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_debug_small_allocations": "0", 2023-09-23T18:30:46.251 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-09-23T18:30:46.251 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_default_buffered_read": "true", 2023-09-23T18:30:46.252 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_default_buffered_write": "false", 2023-09-23T18:30:46.252 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_deferred_batch_ops": "0", 2023-09-23T18:30:46.252 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-09-23T18:30:46.252 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-09-23T18:30:46.252 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-09-23T18:30:46.252 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-09-23T18:30:46.252 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-09-23T18:30:46.252 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-09-23T18:30:46.252 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-09-23T18:30:46.252 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_fail_eio": "false", 2023-09-23T18:30:46.252 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-09-23T18:30:46.252 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-09-23T18:30:46.252 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-09-23T18:30:46.253 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-09-23T18:30:46.253 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_fsck_on_mkfs": "true", 2023-09-23T18:30:46.253 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-09-23T18:30:46.253 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_fsck_on_mount": "false", 2023-09-23T18:30:46.253 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-09-23T18:30:46.253 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_fsck_on_umount": "false", 2023-09-23T18:30:46.253 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-09-23T18:30:46.253 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-09-23T18:30:46.253 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-09-23T18:30:46.253 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-09-23T18:30:46.253 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-09-23T18:30:46.253 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-09-23T18:30:46.254 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-09-23T18:30:46.254 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-09-23T18:30:46.254 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_ignore_data_csum": "false", 2023-09-23T18:30:46.254 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-09-23T18:30:46.254 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_kvbackend": "rocksdb", 2023-09-23T18:30:46.254 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-09-23T18:30:46.254 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-09-23T18:30:46.254 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_log_op_age": "5.000000", 2023-09-23T18:30:46.254 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_max_alloc_size": "0", 2023-09-23T18:30:46.254 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_max_blob_size": "0", 2023-09-23T18:30:46.254 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-09-23T18:30:46.254 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-09-23T18:30:46.254 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_max_defer_interval": "3.000000", 2023-09-23T18:30:46.255 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_max_deferred_txc": "32", 2023-09-23T18:30:46.255 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_min_alloc_size": "0", 2023-09-23T18:30:46.255 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-09-23T18:30:46.255 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-09-23T18:30:46.255 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_nid_prealloc": "1024", 2023-09-23T18:30:46.255 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_prefer_deferred_size": "0", 2023-09-23T18:30:46.255 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-09-23T18:30:46.255 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-09-23T18:30:46.255 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_qfsck_on_mount": "true", 2023-09-23T18:30:46.255 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_retry_disk_reads": "3", 2023-09-23T18:30:46.255 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_rocksdb_cf": "true", 2023-09-23T18:30:46.255 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-09-23T18:30:46.256 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-09-23T18:30:46.256 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_rocksdb_options_annex": "", 2023-09-23T18:30:46.256 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_spdk_coremask": "0x1", 2023-09-23T18:30:46.256 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_spdk_io_sleep": "5", 2023-09-23T18:30:46.256 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_spdk_max_io_completion": "0", 2023-09-23T18:30:46.256 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_spdk_mem": "512", 2023-09-23T18:30:46.256 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_sync_submit_transaction": "false", 2023-09-23T18:30:46.256 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_throttle_bytes": "67108864", 2023-09-23T18:30:46.256 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_throttle_cost_per_io": "0", 2023-09-23T18:30:46.256 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-09-23T18:30:46.256 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-09-23T18:30:46.256 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-09-23T18:30:46.257 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-09-23T18:30:46.257 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_tracing": "false", 2023-09-23T18:30:46.257 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-09-23T18:30:46.257 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-09-23T18:30:46.257 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_volume_selection_reserved": "0", 2023-09-23T18:30:46.257 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-09-23T18:30:46.257 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-09-23T18:30:46.257 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-09-23T18:30:46.257 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-09-23T18:30:46.257 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-09-23T18:30:46.257 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-09-23T18:30:46.257 INFO:teuthology.orchestra.run.smithi098.stdout: "bluestore_zero_block_detection": "false", 2023-09-23T18:30:46.258 INFO:teuthology.orchestra.run.smithi098.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-09-23T18:30:46.258 INFO:teuthology.orchestra.run.smithi098.stdout: "cephfs_mirror_action_update_interval": "2", 2023-09-23T18:30:46.258 INFO:teuthology.orchestra.run.smithi098.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-09-23T18:30:46.258 INFO:teuthology.orchestra.run.smithi098.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-09-23T18:30:46.258 INFO:teuthology.orchestra.run.smithi098.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-09-23T18:30:46.258 INFO:teuthology.orchestra.run.smithi098.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-09-23T18:30:46.258 INFO:teuthology.orchestra.run.smithi098.stdout: "cephfs_mirror_mount_timeout": "10", 2023-09-23T18:30:46.258 INFO:teuthology.orchestra.run.smithi098.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-09-23T18:30:46.258 INFO:teuthology.orchestra.run.smithi098.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-09-23T18:30:46.258 INFO:teuthology.orchestra.run.smithi098.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-09-23T18:30:46.258 INFO:teuthology.orchestra.run.smithi098.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-09-23T18:30:46.258 INFO:teuthology.orchestra.run.smithi098.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-09-23T18:30:46.259 INFO:teuthology.orchestra.run.smithi098.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-09-23T18:30:46.259 INFO:teuthology.orchestra.run.smithi098.stdout: "cephx_cluster_require_signatures": "false", 2023-09-23T18:30:46.259 INFO:teuthology.orchestra.run.smithi098.stdout: "cephx_cluster_require_version": "2", 2023-09-23T18:30:46.259 INFO:teuthology.orchestra.run.smithi098.stdout: "cephx_require_signatures": "false", 2023-09-23T18:30:46.259 INFO:teuthology.orchestra.run.smithi098.stdout: "cephx_require_version": "2", 2023-09-23T18:30:46.259 INFO:teuthology.orchestra.run.smithi098.stdout: "cephx_service_require_signatures": "false", 2023-09-23T18:30:46.259 INFO:teuthology.orchestra.run.smithi098.stdout: "cephx_service_require_version": "2", 2023-09-23T18:30:46.259 INFO:teuthology.orchestra.run.smithi098.stdout: "cephx_sign_messages": "true", 2023-09-23T18:30:46.259 INFO:teuthology.orchestra.run.smithi098.stdout: "chdir": "", 2023-09-23T18:30:46.259 INFO:teuthology.orchestra.run.smithi098.stdout: "client_acl_type": "", 2023-09-23T18:30:46.259 INFO:teuthology.orchestra.run.smithi098.stdout: "client_asio_thread_count": "2", 2023-09-23T18:30:46.259 INFO:teuthology.orchestra.run.smithi098.stdout: "client_cache_mid": "0.750000", 2023-09-23T18:30:46.260 INFO:teuthology.orchestra.run.smithi098.stdout: "client_cache_size": "16384", 2023-09-23T18:30:46.260 INFO:teuthology.orchestra.run.smithi098.stdout: "client_caps_release_delay": "5", 2023-09-23T18:30:46.260 INFO:teuthology.orchestra.run.smithi098.stdout: "client_check_pool_perm": "true", 2023-09-23T18:30:46.260 INFO:teuthology.orchestra.run.smithi098.stdout: "client_collect_and_send_global_metrics": "false", 2023-09-23T18:30:46.260 INFO:teuthology.orchestra.run.smithi098.stdout: "client_debug_force_sync_read": "false", 2023-09-23T18:30:46.260 INFO:teuthology.orchestra.run.smithi098.stdout: "client_debug_getattr_caps": "false", 2023-09-23T18:30:46.260 INFO:teuthology.orchestra.run.smithi098.stdout: "client_debug_inject_tick_delay": "0", 2023-09-23T18:30:46.260 INFO:teuthology.orchestra.run.smithi098.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-09-23T18:30:46.260 INFO:teuthology.orchestra.run.smithi098.stdout: "client_die_on_failed_remount": "false", 2023-09-23T18:30:46.260 INFO:teuthology.orchestra.run.smithi098.stdout: "client_dirsize_rbytes": "true", 2023-09-23T18:30:46.260 INFO:teuthology.orchestra.run.smithi098.stdout: "client_force_lazyio": "false", 2023-09-23T18:30:46.260 INFO:teuthology.orchestra.run.smithi098.stdout: "client_fs": "", 2023-09-23T18:30:46.260 INFO:teuthology.orchestra.run.smithi098.stdout: "client_inject_fixed_oldest_tid": "false", 2023-09-23T18:30:46.261 INFO:teuthology.orchestra.run.smithi098.stdout: "client_inject_release_failure": "false", 2023-09-23T18:30:46.261 INFO:teuthology.orchestra.run.smithi098.stdout: "client_max_inline_size": "4096", 2023-09-23T18:30:46.261 INFO:teuthology.orchestra.run.smithi098.stdout: "client_max_retries_on_remount_failure": "5", 2023-09-23T18:30:46.261 INFO:teuthology.orchestra.run.smithi098.stdout: "client_mds_namespace": "", 2023-09-23T18:30:46.261 INFO:teuthology.orchestra.run.smithi098.stdout: "client_metadata": "", 2023-09-23T18:30:46.261 INFO:teuthology.orchestra.run.smithi098.stdout: "client_mount_gid": "-1", 2023-09-23T18:30:46.261 INFO:teuthology.orchestra.run.smithi098.stdout: "client_mount_timeout": "300", 2023-09-23T18:30:46.261 INFO:teuthology.orchestra.run.smithi098.stdout: "client_mount_uid": "-1", 2023-09-23T18:30:46.261 INFO:teuthology.orchestra.run.smithi098.stdout: "client_mountpoint": "/", 2023-09-23T18:30:46.261 INFO:teuthology.orchestra.run.smithi098.stdout: "client_notify_timeout": "10", 2023-09-23T18:30:46.261 INFO:teuthology.orchestra.run.smithi098.stdout: "client_oc": "true", 2023-09-23T18:30:46.261 INFO:teuthology.orchestra.run.smithi098.stdout: "client_oc_max_dirty": "104857600", 2023-09-23T18:30:46.262 INFO:teuthology.orchestra.run.smithi098.stdout: "client_oc_max_dirty_age": "5.000000", 2023-09-23T18:30:46.262 INFO:teuthology.orchestra.run.smithi098.stdout: "client_oc_max_objects": "1000", 2023-09-23T18:30:46.262 INFO:teuthology.orchestra.run.smithi098.stdout: "client_oc_size": "209715200", 2023-09-23T18:30:46.262 INFO:teuthology.orchestra.run.smithi098.stdout: "client_oc_target_dirty": "8388608", 2023-09-23T18:30:46.262 INFO:teuthology.orchestra.run.smithi098.stdout: "client_permissions": "true", 2023-09-23T18:30:46.262 INFO:teuthology.orchestra.run.smithi098.stdout: "client_quota": "true", 2023-09-23T18:30:46.262 INFO:teuthology.orchestra.run.smithi098.stdout: "client_quota_df": "true", 2023-09-23T18:30:46.262 INFO:teuthology.orchestra.run.smithi098.stdout: "client_readahead_max_bytes": "0", 2023-09-23T18:30:46.262 INFO:teuthology.orchestra.run.smithi098.stdout: "client_readahead_max_periods": "4", 2023-09-23T18:30:46.262 INFO:teuthology.orchestra.run.smithi098.stdout: "client_readahead_min": "131072", 2023-09-23T18:30:46.262 INFO:teuthology.orchestra.run.smithi098.stdout: "client_reconnect_stale": "false", 2023-09-23T18:30:46.262 INFO:teuthology.orchestra.run.smithi098.stdout: "client_shutdown_timeout": "30", 2023-09-23T18:30:46.263 INFO:teuthology.orchestra.run.smithi098.stdout: "client_snapdir": ".snap", 2023-09-23T18:30:46.263 INFO:teuthology.orchestra.run.smithi098.stdout: "client_tick_interval": "1", 2023-09-23T18:30:46.263 INFO:teuthology.orchestra.run.smithi098.stdout: "client_trace": "", 2023-09-23T18:30:46.263 INFO:teuthology.orchestra.run.smithi098.stdout: "client_try_dentry_invalidate": "false", 2023-09-23T18:30:46.263 INFO:teuthology.orchestra.run.smithi098.stdout: "client_use_faked_inos": "false", 2023-09-23T18:30:46.263 INFO:teuthology.orchestra.run.smithi098.stdout: "client_use_random_mds": "false", 2023-09-23T18:30:46.263 INFO:teuthology.orchestra.run.smithi098.stdout: "clog_to_graylog": "false", 2023-09-23T18:30:46.263 INFO:teuthology.orchestra.run.smithi098.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-09-23T18:30:46.263 INFO:teuthology.orchestra.run.smithi098.stdout: "clog_to_graylog_port": "12201", 2023-09-23T18:30:46.263 INFO:teuthology.orchestra.run.smithi098.stdout: "clog_to_monitors": "default=true", 2023-09-23T18:30:46.263 INFO:teuthology.orchestra.run.smithi098.stdout: "clog_to_syslog": "false", 2023-09-23T18:30:46.263 INFO:teuthology.orchestra.run.smithi098.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-09-23T18:30:46.263 INFO:teuthology.orchestra.run.smithi098.stdout: "clog_to_syslog_level": "info", 2023-09-23T18:30:46.264 INFO:teuthology.orchestra.run.smithi098.stdout: "cluster_addr": "-", 2023-09-23T18:30:46.264 INFO:teuthology.orchestra.run.smithi098.stdout: "cluster_network": "", 2023-09-23T18:30:46.264 INFO:teuthology.orchestra.run.smithi098.stdout: "cluster_network_interface": "", 2023-09-23T18:30:46.264 INFO:teuthology.orchestra.run.smithi098.stdout: "compressor_zlib_isal": "false", 2023-09-23T18:30:46.264 INFO:teuthology.orchestra.run.smithi098.stdout: "compressor_zlib_level": "5", 2023-09-23T18:30:46.264 INFO:teuthology.orchestra.run.smithi098.stdout: "compressor_zlib_winsize": "-15", 2023-09-23T18:30:46.264 INFO:teuthology.orchestra.run.smithi098.stdout: "compressor_zstd_level": "1", 2023-09-23T18:30:46.264 INFO:teuthology.orchestra.run.smithi098.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d", 2023-09-23T18:30:46.264 INFO:teuthology.orchestra.run.smithi098.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-09-23T18:30:46.264 INFO:teuthology.orchestra.run.smithi098.stdout: "crimson_alien_op_num_threads": "6", 2023-09-23T18:30:46.264 INFO:teuthology.orchestra.run.smithi098.stdout: "crimson_alien_thread_cpu_cores": "", 2023-09-23T18:30:46.264 INFO:teuthology.orchestra.run.smithi098.stdout: "crimson_osd_obc_lru_size": "10", 2023-09-23T18:30:46.264 INFO:teuthology.orchestra.run.smithi098.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-09-23T18:30:46.265 INFO:teuthology.orchestra.run.smithi098.stdout: "crimson_seastar_smp": "1", 2023-09-23T18:30:46.265 INFO:teuthology.orchestra.run.smithi098.stdout: "crush_location": "", 2023-09-23T18:30:46.265 INFO:teuthology.orchestra.run.smithi098.stdout: "crush_location_hook": "", 2023-09-23T18:30:46.265 INFO:teuthology.orchestra.run.smithi098.stdout: "crush_location_hook_timeout": "10", 2023-09-23T18:30:46.265 INFO:teuthology.orchestra.run.smithi098.stdout: "daemonize": "false", 2023-09-23T18:30:46.265 INFO:teuthology.orchestra.run.smithi098.stdout: "daos_pool": "tank", 2023-09-23T18:30:46.265 INFO:teuthology.orchestra.run.smithi098.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-09-23T18:30:46.265 INFO:teuthology.orchestra.run.smithi098.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-09-23T18:30:46.265 INFO:teuthology.orchestra.run.smithi098.stdout: "dbstore_db_name_prefix": "dbstore", 2023-09-23T18:30:46.265 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_alienstore": "0/5", 2023-09-23T18:30:46.265 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_allow_any_pool_priority": "false", 2023-09-23T18:30:46.265 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_asok": "1/5", 2023-09-23T18:30:46.266 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_asok_assert_abort": "false", 2023-09-23T18:30:46.266 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_asserts_on_shutdown": "true", 2023-09-23T18:30:46.266 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_auth": "1/5", 2023-09-23T18:30:46.266 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_bdev": "1/3", 2023-09-23T18:30:46.266 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_bluefs": "1/5", 2023-09-23T18:30:46.266 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_bluestore": "1/5", 2023-09-23T18:30:46.266 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_buffer": "0/1", 2023-09-23T18:30:46.266 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_ceph_exporter": "1/5", 2023-09-23T18:30:46.266 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_cephfs_mirror": "0/5", 2023-09-23T18:30:46.266 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_cephsqlite": "0/5", 2023-09-23T18:30:46.266 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_client": "0/5", 2023-09-23T18:30:46.267 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_compressor": "1/5", 2023-09-23T18:30:46.267 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_context": "0/1", 2023-09-23T18:30:46.267 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_crush": "1/1", 2023-09-23T18:30:46.267 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_crypto": "1/5", 2023-09-23T18:30:46.267 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_cyanstore": "0/5", 2023-09-23T18:30:46.267 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_deliberately_leak_memory": "false", 2023-09-23T18:30:46.267 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_disable_randomized_ping": "false", 2023-09-23T18:30:46.267 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_dpdk": "1/5", 2023-09-23T18:30:46.267 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_eventtrace": "1/5", 2023-09-23T18:30:46.267 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_filer": "0/1", 2023-09-23T18:30:46.267 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_filestore": "1/3", 2023-09-23T18:30:46.267 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_finisher": "1/1", 2023-09-23T18:30:46.267 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_fuse": "1/5", 2023-09-23T18:30:46.268 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_heartbeat_testing_span": "0", 2023-09-23T18:30:46.268 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_heartbeatmap": "1/5", 2023-09-23T18:30:46.268 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_immutable_obj_cache": "0/5", 2023-09-23T18:30:46.268 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_javaclient": "1/5", 2023-09-23T18:30:46.268 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_journal": "1/3", 2023-09-23T18:30:46.268 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_journaler": "0/5", 2023-09-23T18:30:46.268 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_kstore": "1/5", 2023-09-23T18:30:46.268 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_leveldb": "4/5", 2023-09-23T18:30:46.268 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_lockdep": "0/1", 2023-09-23T18:30:46.268 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_mclock": "1/5", 2023-09-23T18:30:46.268 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_mds": "1/5", 2023-09-23T18:30:46.268 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_mds_balancer": "1/5", 2023-09-23T18:30:46.269 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_mds_locker": "1/5", 2023-09-23T18:30:46.269 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_mds_log": "1/5", 2023-09-23T18:30:46.269 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_mds_log_expire": "1/5", 2023-09-23T18:30:46.269 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_mds_migrator": "1/5", 2023-09-23T18:30:46.269 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_memstore": "1/5", 2023-09-23T18:30:46.269 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_mgr": "2/5", 2023-09-23T18:30:46.269 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_mgrc": "1/5", 2023-09-23T18:30:46.269 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_mon": "20/20", 2023-09-23T18:30:46.269 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_monc": "0/10", 2023-09-23T18:30:46.269 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_ms": "1/1", 2023-09-23T18:30:46.269 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_none": "0/5", 2023-09-23T18:30:46.270 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_objclass": "0/5", 2023-09-23T18:30:46.270 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_objectcacher": "0/5", 2023-09-23T18:30:46.270 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_objecter": "0/1", 2023-09-23T18:30:46.270 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_optracker": "0/5", 2023-09-23T18:30:46.270 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_osd": "1/5", 2023-09-23T18:30:46.270 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_paxos": "20/20", 2023-09-23T18:30:46.270 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_perfcounter": "1/5", 2023-09-23T18:30:46.270 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_prioritycache": "1/5", 2023-09-23T18:30:46.270 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_rados": "0/5", 2023-09-23T18:30:46.270 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_rbd": "0/5", 2023-09-23T18:30:46.270 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_rbd_mirror": "0/5", 2023-09-23T18:30:46.270 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_rbd_pwl": "0/5", 2023-09-23T18:30:46.271 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_rbd_replay": "0/5", 2023-09-23T18:30:46.271 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_refs": "0/0", 2023-09-23T18:30:46.271 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_reserver": "1/1", 2023-09-23T18:30:46.271 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_rgw": "1/5", 2023-09-23T18:30:46.271 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_rgw_access": "1/5", 2023-09-23T18:30:46.271 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_rgw_datacache": "1/5", 2023-09-23T18:30:46.271 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_rgw_dbstore": "1/5", 2023-09-23T18:30:46.271 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_rgw_flight": "1/5", 2023-09-23T18:30:46.271 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_rgw_sync": "1/5", 2023-09-23T18:30:46.271 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_rocksdb": "4/5", 2023-09-23T18:30:46.271 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_seastore": "0/5", 2023-09-23T18:30:46.271 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_seastore_backref": "0/5", 2023-09-23T18:30:46.272 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_seastore_cache": "0/5", 2023-09-23T18:30:46.272 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_seastore_cleaner": "0/5", 2023-09-23T18:30:46.272 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_seastore_device": "0/5", 2023-09-23T18:30:46.272 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_seastore_epm": "0/5", 2023-09-23T18:30:46.272 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-09-23T18:30:46.272 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_seastore_journal": "0/5", 2023-09-23T18:30:46.272 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_seastore_lba": "0/5", 2023-09-23T18:30:46.272 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_seastore_odata": "0/5", 2023-09-23T18:30:46.272 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_seastore_omap": "0/5", 2023-09-23T18:30:46.272 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_seastore_onode": "0/5", 2023-09-23T18:30:46.272 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_seastore_t": "0/5", 2023-09-23T18:30:46.272 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_seastore_tm": "0/5", 2023-09-23T18:30:46.273 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_striper": "0/1", 2023-09-23T18:30:46.273 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_test": "0/5", 2023-09-23T18:30:46.273 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_throttle": "1/1", 2023-09-23T18:30:46.273 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_timer": "0/1", 2023-09-23T18:30:46.273 INFO:teuthology.orchestra.run.smithi098.stdout: "debug_tp": "0/5", 2023-09-23T18:30:46.273 INFO:teuthology.orchestra.run.smithi098.stdout: "device_failure_prediction_mode": "none", 2023-09-23T18:30:46.273 INFO:teuthology.orchestra.run.smithi098.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-09-23T18:30:46.273 INFO:teuthology.orchestra.run.smithi098.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-09-23T18:30:46.273 INFO:teuthology.orchestra.run.smithi098.stdout: "err_to_graylog": "false", 2023-09-23T18:30:46.273 INFO:teuthology.orchestra.run.smithi098.stdout: "err_to_journald": "false", 2023-09-23T18:30:46.273 INFO:teuthology.orchestra.run.smithi098.stdout: "err_to_stderr": "true", 2023-09-23T18:30:46.274 INFO:teuthology.orchestra.run.smithi098.stdout: "err_to_syslog": "false", 2023-09-23T18:30:46.274 INFO:teuthology.orchestra.run.smithi098.stdout: "event_tracing": "false", 2023-09-23T18:30:46.274 INFO:teuthology.orchestra.run.smithi098.stdout: "exporter_addr": "0.0.0.0", 2023-09-23T18:30:46.274 INFO:teuthology.orchestra.run.smithi098.stdout: "exporter_http_port": "9926", 2023-09-23T18:30:46.274 INFO:teuthology.orchestra.run.smithi098.stdout: "exporter_prio_limit": "5", 2023-09-23T18:30:46.274 INFO:teuthology.orchestra.run.smithi098.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-09-23T18:30:46.274 INFO:teuthology.orchestra.run.smithi098.stdout: "exporter_sort_metrics": "true", 2023-09-23T18:30:46.274 INFO:teuthology.orchestra.run.smithi098.stdout: "exporter_stats_period": "5", 2023-09-23T18:30:46.274 INFO:teuthology.orchestra.run.smithi098.stdout: "fake_statfs_for_testing": "0", 2023-09-23T18:30:46.274 INFO:teuthology.orchestra.run.smithi098.stdout: "fatal_signal_handlers": "true", 2023-09-23T18:30:46.274 INFO:teuthology.orchestra.run.smithi098.stdout: "filer_max_purge_ops": "10", 2023-09-23T18:30:46.274 INFO:teuthology.orchestra.run.smithi098.stdout: "filer_max_truncate_ops": "128", 2023-09-23T18:30:46.275 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_apply_finisher_threads": "1", 2023-09-23T18:30:46.275 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_blackhole": "false", 2023-09-23T18:30:46.275 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_btrfs_clone_range": "true", 2023-09-23T18:30:46.275 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_btrfs_snap": "true", 2023-09-23T18:30:46.275 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_caller_concurrency": "10", 2023-09-23T18:30:46.275 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_collect_device_partition_information": "true", 2023-09-23T18:30:46.275 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_commit_timeout": "600.000000", 2023-09-23T18:30:46.275 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_debug_inject_read_err": "false", 2023-09-23T18:30:46.275 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_debug_omap_check": "false", 2023-09-23T18:30:46.275 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_debug_random_read_err": "0.000000", 2023-09-23T18:30:46.275 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_debug_verify_split": "false", 2023-09-23T18:30:46.275 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_dump_file": "", 2023-09-23T18:30:46.276 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-09-23T18:30:46.276 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-09-23T18:30:46.276 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_fadvise": "true", 2023-09-23T18:30:46.276 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_fail_eio": "true", 2023-09-23T18:30:46.276 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_fd_cache_shards": "16", 2023-09-23T18:30:46.276 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_fd_cache_size": "128", 2023-09-23T18:30:46.276 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_fiemap": "false", 2023-09-23T18:30:46.276 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_fiemap_threshold": "4096", 2023-09-23T18:30:46.276 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-09-23T18:30:46.276 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_index_retry_probability": "0.000000", 2023-09-23T18:30:46.276 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_inject_stall": "0", 2023-09-23T18:30:46.276 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_journal_parallel": "false", 2023-09-23T18:30:46.277 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_journal_trailing": "false", 2023-09-23T18:30:46.277 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_journal_writeahead": "false", 2023-09-23T18:30:46.277 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_kill_at": "0", 2023-09-23T18:30:46.277 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-09-23T18:30:46.277 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_max_inline_xattr_size": "0", 2023-09-23T18:30:46.277 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-09-23T18:30:46.277 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-09-23T18:30:46.277 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-09-23T18:30:46.277 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_max_inline_xattrs": "0", 2023-09-23T18:30:46.277 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-09-23T18:30:46.277 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_max_inline_xattrs_other": "2", 2023-09-23T18:30:46.277 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-09-23T18:30:46.278 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_max_sync_interval": "5.000000", 2023-09-23T18:30:46.278 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_max_xattr_value_size": "0", 2023-09-23T18:30:46.278 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-09-23T18:30:46.278 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-09-23T18:30:46.278 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-09-23T18:30:46.278 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_merge_threshold": "-10", 2023-09-23T18:30:46.278 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_min_sync_interval": "0.010000", 2023-09-23T18:30:46.278 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_odsync_write": "false", 2023-09-23T18:30:46.278 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_omap_backend": "rocksdb", 2023-09-23T18:30:46.278 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_omap_backend_path": "", 2023-09-23T18:30:46.278 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_omap_header_cache_size": "1024", 2023-09-23T18:30:46.279 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_ondisk_finisher_threads": "1", 2023-09-23T18:30:46.279 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-09-23T18:30:46.279 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_op_thread_timeout": "60", 2023-09-23T18:30:46.279 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_op_threads": "2", 2023-09-23T18:30:46.279 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_punch_hole": "false", 2023-09-23T18:30:46.279 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-09-23T18:30:46.279 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-09-23T18:30:46.279 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-09-23T18:30:46.279 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-09-23T18:30:46.279 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-09-23T18:30:46.279 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_queue_max_bytes": "104857600", 2023-09-23T18:30:46.279 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-09-23T18:30:46.280 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-09-23T18:30:46.280 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-09-23T18:30:46.280 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_queue_max_ops": "50", 2023-09-23T18:30:46.280 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-09-23T18:30:46.280 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_seek_data_hole": "false", 2023-09-23T18:30:46.280 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_sloppy_crc": "false", 2023-09-23T18:30:46.280 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-09-23T18:30:46.280 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_splice": "false", 2023-09-23T18:30:46.280 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_split_multiple": "2", 2023-09-23T18:30:46.280 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_split_rand_factor": "20", 2023-09-23T18:30:46.280 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_update_to": "1000", 2023-09-23T18:30:46.280 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-09-23T18:30:46.281 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-09-23T18:30:46.281 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-09-23T18:30:46.281 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-09-23T18:30:46.281 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-09-23T18:30:46.281 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-09-23T18:30:46.281 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_wbthrottle_enable": "true", 2023-09-23T18:30:46.281 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-09-23T18:30:46.281 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-09-23T18:30:46.281 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-09-23T18:30:46.282 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-09-23T18:30:46.282 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-09-23T18:30:46.282 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-09-23T18:30:46.282 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_xfs_extsize": "false", 2023-09-23T18:30:46.282 INFO:teuthology.orchestra.run.smithi098.stdout: "filestore_zfs_snap": "false", 2023-09-23T18:30:46.283 INFO:teuthology.orchestra.run.smithi098.stdout: "fio_dir": "/tmp/fio", 2023-09-23T18:30:46.283 INFO:teuthology.orchestra.run.smithi098.stdout: "fsid": "756c96d4-5a3b-11ee-8db4-212e2dc638e7", 2023-09-23T18:30:46.283 INFO:teuthology.orchestra.run.smithi098.stdout: "fuse_allow_other": "true", 2023-09-23T18:30:46.283 INFO:teuthology.orchestra.run.smithi098.stdout: "fuse_atomic_o_trunc": "true", 2023-09-23T18:30:46.283 INFO:teuthology.orchestra.run.smithi098.stdout: "fuse_big_writes": "true", 2023-09-23T18:30:46.283 INFO:teuthology.orchestra.run.smithi098.stdout: "fuse_debug": "false", 2023-09-23T18:30:46.283 INFO:teuthology.orchestra.run.smithi098.stdout: "fuse_default_permissions": "false", 2023-09-23T18:30:46.283 INFO:teuthology.orchestra.run.smithi098.stdout: "fuse_disable_pagecache": "false", 2023-09-23T18:30:46.283 INFO:teuthology.orchestra.run.smithi098.stdout: "fuse_max_write": "0", 2023-09-23T18:30:46.283 INFO:teuthology.orchestra.run.smithi098.stdout: "fuse_multithreaded": "true", 2023-09-23T18:30:46.283 INFO:teuthology.orchestra.run.smithi098.stdout: "fuse_require_active_mds": "true", 2023-09-23T18:30:46.283 INFO:teuthology.orchestra.run.smithi098.stdout: "fuse_set_user_groups": "true", 2023-09-23T18:30:46.283 INFO:teuthology.orchestra.run.smithi098.stdout: "fuse_splice_move": "true", 2023-09-23T18:30:46.284 INFO:teuthology.orchestra.run.smithi098.stdout: "fuse_splice_read": "true", 2023-09-23T18:30:46.284 INFO:teuthology.orchestra.run.smithi098.stdout: "fuse_splice_write": "true", 2023-09-23T18:30:46.284 INFO:teuthology.orchestra.run.smithi098.stdout: "fuse_syncfs_on_mksnap": "true", 2023-09-23T18:30:46.284 INFO:teuthology.orchestra.run.smithi098.stdout: "fuse_use_invalidate_cb": "true", 2023-09-23T18:30:46.284 INFO:teuthology.orchestra.run.smithi098.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-09-23T18:30:46.284 INFO:teuthology.orchestra.run.smithi098.stdout: "gss_target_name": "ceph", 2023-09-23T18:30:46.284 INFO:teuthology.orchestra.run.smithi098.stdout: "heartbeat_file": "", 2023-09-23T18:30:46.284 INFO:teuthology.orchestra.run.smithi098.stdout: "heartbeat_inject_failure": "0", 2023-09-23T18:30:46.284 INFO:teuthology.orchestra.run.smithi098.stdout: "heartbeat_interval": "5", 2023-09-23T18:30:46.284 INFO:teuthology.orchestra.run.smithi098.stdout: "host": "smithi098", 2023-09-23T18:30:46.284 INFO:teuthology.orchestra.run.smithi098.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-09-23T18:30:46.284 INFO:teuthology.orchestra.run.smithi098.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-09-23T18:30:46.285 INFO:teuthology.orchestra.run.smithi098.stdout: "immutable_object_cache_max_size": "1073741824", 2023-09-23T18:30:46.285 INFO:teuthology.orchestra.run.smithi098.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-09-23T18:30:46.285 INFO:teuthology.orchestra.run.smithi098.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-09-23T18:30:46.285 INFO:teuthology.orchestra.run.smithi098.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-09-23T18:30:46.285 INFO:teuthology.orchestra.run.smithi098.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-09-23T18:30:46.285 INFO:teuthology.orchestra.run.smithi098.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-09-23T18:30:46.285 INFO:teuthology.orchestra.run.smithi098.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-09-23T18:30:46.285 INFO:teuthology.orchestra.run.smithi098.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-09-23T18:30:46.285 INFO:teuthology.orchestra.run.smithi098.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-09-23T18:30:46.285 INFO:teuthology.orchestra.run.smithi098.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-09-23T18:30:46.285 INFO:teuthology.orchestra.run.smithi098.stdout: "immutable_object_cache_watermark": "0.900000", 2023-09-23T18:30:46.285 INFO:teuthology.orchestra.run.smithi098.stdout: "inject_early_sigterm": "false", 2023-09-23T18:30:46.285 INFO:teuthology.orchestra.run.smithi098.stdout: "jaeger_agent_port": "6799", 2023-09-23T18:30:46.286 INFO:teuthology.orchestra.run.smithi098.stdout: "jaeger_tracing_enable": "false", 2023-09-23T18:30:46.286 INFO:teuthology.orchestra.run.smithi098.stdout: "journal_aio": "true", 2023-09-23T18:30:46.286 INFO:teuthology.orchestra.run.smithi098.stdout: "journal_align_min_size": "65536", 2023-09-23T18:30:46.286 INFO:teuthology.orchestra.run.smithi098.stdout: "journal_block_align": "true", 2023-09-23T18:30:46.286 INFO:teuthology.orchestra.run.smithi098.stdout: "journal_block_size": "4096", 2023-09-23T18:30:46.286 INFO:teuthology.orchestra.run.smithi098.stdout: "journal_dio": "true", 2023-09-23T18:30:46.286 INFO:teuthology.orchestra.run.smithi098.stdout: "journal_discard": "false", 2023-09-23T18:30:46.286 INFO:teuthology.orchestra.run.smithi098.stdout: "journal_force_aio": "false", 2023-09-23T18:30:46.286 INFO:teuthology.orchestra.run.smithi098.stdout: "journal_ignore_corruption": "false", 2023-09-23T18:30:46.286 INFO:teuthology.orchestra.run.smithi098.stdout: "journal_max_write_bytes": "10485760", 2023-09-23T18:30:46.286 INFO:teuthology.orchestra.run.smithi098.stdout: "journal_max_write_entries": "100", 2023-09-23T18:30:46.286 INFO:teuthology.orchestra.run.smithi098.stdout: "journal_replay_from": "0", 2023-09-23T18:30:46.287 INFO:teuthology.orchestra.run.smithi098.stdout: "journal_throttle_high_multiple": "0.000000", 2023-09-23T18:30:46.287 INFO:teuthology.orchestra.run.smithi098.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-09-23T18:30:46.287 INFO:teuthology.orchestra.run.smithi098.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-09-23T18:30:46.287 INFO:teuthology.orchestra.run.smithi098.stdout: "journal_throttle_max_multiple": "0.000000", 2023-09-23T18:30:46.287 INFO:teuthology.orchestra.run.smithi098.stdout: "journal_write_header_frequency": "0", 2023-09-23T18:30:46.287 INFO:teuthology.orchestra.run.smithi098.stdout: "journal_zero_on_create": "false", 2023-09-23T18:30:46.287 INFO:teuthology.orchestra.run.smithi098.stdout: "journaler_prefetch_periods": "10", 2023-09-23T18:30:46.287 INFO:teuthology.orchestra.run.smithi098.stdout: "journaler_prezero_periods": "5", 2023-09-23T18:30:46.287 INFO:teuthology.orchestra.run.smithi098.stdout: "journaler_write_head_interval": "15", 2023-09-23T18:30:46.287 INFO:teuthology.orchestra.run.smithi098.stdout: "key": "", 2023-09-23T18:30:46.287 INFO:teuthology.orchestra.run.smithi098.stdout: "keyfile": "", 2023-09-23T18:30:46.287 INFO:teuthology.orchestra.run.smithi098.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-09-23T18:30:46.287 INFO:teuthology.orchestra.run.smithi098.stdout: "kstore_backend": "rocksdb", 2023-09-23T18:30:46.288 INFO:teuthology.orchestra.run.smithi098.stdout: "kstore_default_stripe_size": "65536", 2023-09-23T18:30:46.288 INFO:teuthology.orchestra.run.smithi098.stdout: "kstore_fsck_on_mount": "false", 2023-09-23T18:30:46.288 INFO:teuthology.orchestra.run.smithi098.stdout: "kstore_fsck_on_mount_deep": "true", 2023-09-23T18:30:46.288 INFO:teuthology.orchestra.run.smithi098.stdout: "kstore_max_bytes": "67108864", 2023-09-23T18:30:46.288 INFO:teuthology.orchestra.run.smithi098.stdout: "kstore_max_ops": "512", 2023-09-23T18:30:46.288 INFO:teuthology.orchestra.run.smithi098.stdout: "kstore_nid_prealloc": "1024", 2023-09-23T18:30:46.288 INFO:teuthology.orchestra.run.smithi098.stdout: "kstore_onode_map_size": "1024", 2023-09-23T18:30:46.288 INFO:teuthology.orchestra.run.smithi098.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-09-23T18:30:46.288 INFO:teuthology.orchestra.run.smithi098.stdout: "kstore_sync_submit_transaction": "false", 2023-09-23T18:30:46.288 INFO:teuthology.orchestra.run.smithi098.stdout: "kstore_sync_transaction": "false", 2023-09-23T18:30:46.288 INFO:teuthology.orchestra.run.smithi098.stdout: "leveldb_block_size": "65536", 2023-09-23T18:30:46.288 INFO:teuthology.orchestra.run.smithi098.stdout: "leveldb_bloom_size": "0", 2023-09-23T18:30:46.289 INFO:teuthology.orchestra.run.smithi098.stdout: "leveldb_cache_size": "536870912", 2023-09-23T18:30:46.289 INFO:teuthology.orchestra.run.smithi098.stdout: "leveldb_compact_on_mount": "false", 2023-09-23T18:30:46.289 INFO:teuthology.orchestra.run.smithi098.stdout: "leveldb_compression": "false", 2023-09-23T18:30:46.289 INFO:teuthology.orchestra.run.smithi098.stdout: "leveldb_log": "", 2023-09-23T18:30:46.289 INFO:teuthology.orchestra.run.smithi098.stdout: "leveldb_log_to_ceph_log": "true", 2023-09-23T18:30:46.289 INFO:teuthology.orchestra.run.smithi098.stdout: "leveldb_max_open_files": "0", 2023-09-23T18:30:46.289 INFO:teuthology.orchestra.run.smithi098.stdout: "leveldb_paranoid": "false", 2023-09-23T18:30:46.289 INFO:teuthology.orchestra.run.smithi098.stdout: "leveldb_write_buffer_size": "33554432", 2023-09-23T18:30:46.289 INFO:teuthology.orchestra.run.smithi098.stdout: "librados_thread_count": "2", 2023-09-23T18:30:46.289 INFO:teuthology.orchestra.run.smithi098.stdout: "lockdep": "false", 2023-09-23T18:30:46.289 INFO:teuthology.orchestra.run.smithi098.stdout: "lockdep_force_backtrace": "false", 2023-09-23T18:30:46.289 INFO:teuthology.orchestra.run.smithi098.stdout: "log_coarse_timestamps": "true", 2023-09-23T18:30:46.289 INFO:teuthology.orchestra.run.smithi098.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-09-23T18:30:46.290 INFO:teuthology.orchestra.run.smithi098.stdout: "log_flush_on_exit": "false", 2023-09-23T18:30:46.290 INFO:teuthology.orchestra.run.smithi098.stdout: "log_graylog_host": "127.0.0.1", 2023-09-23T18:30:46.290 INFO:teuthology.orchestra.run.smithi098.stdout: "log_graylog_port": "12201", 2023-09-23T18:30:46.290 INFO:teuthology.orchestra.run.smithi098.stdout: "log_max_new": "1000", 2023-09-23T18:30:46.290 INFO:teuthology.orchestra.run.smithi098.stdout: "log_max_recent": "10000", 2023-09-23T18:30:46.290 INFO:teuthology.orchestra.run.smithi098.stdout: "log_stderr_prefix": "debug ", 2023-09-23T18:30:46.290 INFO:teuthology.orchestra.run.smithi098.stdout: "log_stop_at_utilization": "0.970000", 2023-09-23T18:30:46.290 INFO:teuthology.orchestra.run.smithi098.stdout: "log_to_file": "true", 2023-09-23T18:30:46.290 INFO:teuthology.orchestra.run.smithi098.stdout: "log_to_graylog": "false", 2023-09-23T18:30:46.290 INFO:teuthology.orchestra.run.smithi098.stdout: "log_to_journald": "false", 2023-09-23T18:30:46.290 INFO:teuthology.orchestra.run.smithi098.stdout: "log_to_stderr": "false", 2023-09-23T18:30:46.291 INFO:teuthology.orchestra.run.smithi098.stdout: "log_to_syslog": "false", 2023-09-23T18:30:46.291 INFO:teuthology.orchestra.run.smithi098.stdout: "max_rotating_auth_attempts": "10", 2023-09-23T18:30:46.291 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-09-23T18:30:46.291 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_action_on_write_error": "1", 2023-09-23T18:30:46.291 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_alternate_name_max": "8192", 2023-09-23T18:30:46.291 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_asio_thread_count": "2", 2023-09-23T18:30:46.291 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_bal_export_pin": "true", 2023-09-23T18:30:46.291 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_bal_fragment_dirs": "true", 2023-09-23T18:30:46.291 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-09-23T18:30:46.291 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_bal_fragment_interval": "5", 2023-09-23T18:30:46.291 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_bal_fragment_size_max": "100000", 2023-09-23T18:30:46.292 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_bal_idle_threshold": "0.000000", 2023-09-23T18:30:46.292 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_bal_interval": "10", 2023-09-23T18:30:46.292 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_bal_max": "-1", 2023-09-23T18:30:46.292 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_bal_max_until": "-1", 2023-09-23T18:30:46.292 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_bal_merge_size": "50", 2023-09-23T18:30:46.292 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_bal_midchunk": "0.300000", 2023-09-23T18:30:46.292 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_bal_min_rebalance": "0.100000", 2023-09-23T18:30:46.292 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_bal_min_start": "0.200000", 2023-09-23T18:30:46.292 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_bal_minchunk": "0.001000", 2023-09-23T18:30:46.292 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_bal_mode": "0", 2023-09-23T18:30:46.292 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_bal_need_max": "1.200000", 2023-09-23T18:30:46.292 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_bal_need_min": "0.800000", 2023-09-23T18:30:46.293 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-09-23T18:30:46.293 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_bal_sample_interval": "3.000000", 2023-09-23T18:30:46.293 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_bal_split_bits": "3", 2023-09-23T18:30:46.293 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_bal_split_rd": "25000.000000", 2023-09-23T18:30:46.293 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_bal_split_size": "10000", 2023-09-23T18:30:46.293 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_bal_split_wr": "10000.000000", 2023-09-23T18:30:46.293 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_bal_target_decay": "10.000000", 2023-09-23T18:30:46.293 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-09-23T18:30:46.293 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_beacon_grace": "15.000000", 2023-09-23T18:30:46.293 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_beacon_interval": "4.000000", 2023-09-23T18:30:46.293 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_beacon_mon_down_grace": "60", 2023-09-23T18:30:46.293 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_cache_memory_limit": "4294967296", 2023-09-23T18:30:46.293 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_cache_mid": "0.700000", 2023-09-23T18:30:46.294 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_cache_release_free_interval": "10", 2023-09-23T18:30:46.294 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_cache_reservation": "0.050000", 2023-09-23T18:30:46.294 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-09-23T18:30:46.294 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_cache_trim_interval": "1", 2023-09-23T18:30:46.294 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_cache_trim_threshold": "262144", 2023-09-23T18:30:46.294 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-09-23T18:30:46.294 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-09-23T18:30:46.294 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_client_delegate_inos_pct": "50", 2023-09-23T18:30:46.294 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_client_prealloc_inos": "1000", 2023-09-23T18:30:46.294 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-09-23T18:30:46.294 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_connect_bootstrapping": "false", 2023-09-23T18:30:46.294 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_damage_table_max_entries": "10000", 2023-09-23T18:30:46.295 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-09-23T18:30:46.295 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_debug_auth_pins": "false", 2023-09-23T18:30:46.295 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_debug_frag": "false", 2023-09-23T18:30:46.295 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_debug_scatterstat": "false", 2023-09-23T18:30:46.295 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_debug_subtrees": "false", 2023-09-23T18:30:46.295 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_decay_halflife": "5.000000", 2023-09-23T18:30:46.295 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_default_dir_hash": "2", 2023-09-23T18:30:46.295 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_defer_session_stale": "true", 2023-09-23T18:30:46.295 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_deny_all_reconnect": "false", 2023-09-23T18:30:46.295 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_dir_keys_per_op": "16384", 2023-09-23T18:30:46.295 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_dir_max_commit_size": "10", 2023-09-23T18:30:46.295 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_dir_max_entries": "0", 2023-09-23T18:30:46.296 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_dir_prefetch": "true", 2023-09-23T18:30:46.296 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_dirstat_min_interval": "1.000000", 2023-09-23T18:30:46.296 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_dump_cache_after_rejoin": "false", 2023-09-23T18:30:46.296 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_dump_cache_on_map": "false", 2023-09-23T18:30:46.296 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_dump_cache_threshold_file": "0", 2023-09-23T18:30:46.296 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-09-23T18:30:46.296 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_early_reply": "true", 2023-09-23T18:30:46.296 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_enable_op_tracker": "true", 2023-09-23T18:30:46.296 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_enforce_unique_name": "true", 2023-09-23T18:30:46.296 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_export_ephemeral_distributed": "true", 2023-09-23T18:30:46.296 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-09-23T18:30:46.296 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_export_ephemeral_random": "true", 2023-09-23T18:30:46.296 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-09-23T18:30:46.297 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-09-23T18:30:46.297 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_forward_all_requests_to_auth": "false", 2023-09-23T18:30:46.297 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-09-23T18:30:46.297 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-09-23T18:30:46.297 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-09-23T18:30:46.297 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-09-23T18:30:46.297 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_health_cache_threshold": "1.500000", 2023-09-23T18:30:46.297 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_health_summarize_threshold": "10", 2023-09-23T18:30:46.297 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_heartbeat_grace": "15.000000", 2023-09-23T18:30:46.297 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_heartbeat_reset_grace": "1000", 2023-09-23T18:30:46.297 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_inject_health_dummy": "false", 2023-09-23T18:30:46.297 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-09-23T18:30:46.298 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_inject_migrator_session_race": "false", 2023-09-23T18:30:46.298 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-09-23T18:30:46.298 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-09-23T18:30:46.298 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-09-23T18:30:46.298 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_join_fs": "", 2023-09-23T18:30:46.298 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_journal_format": "1", 2023-09-23T18:30:46.298 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_kill_create_at": "0", 2023-09-23T18:30:46.298 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_kill_export_at": "0", 2023-09-23T18:30:46.298 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_kill_import_at": "0", 2023-09-23T18:30:46.298 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_kill_journal_at": "0", 2023-09-23T18:30:46.298 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_kill_journal_expire_at": "0", 2023-09-23T18:30:46.298 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_kill_journal_replay_at": "0", 2023-09-23T18:30:46.298 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_kill_link_at": "0", 2023-09-23T18:30:46.299 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_kill_mdstable_at": "0", 2023-09-23T18:30:46.299 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_kill_openc_at": "0", 2023-09-23T18:30:46.299 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_kill_rename_at": "0", 2023-09-23T18:30:46.299 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-09-23T18:30:46.299 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_log_events_per_segment": "1024", 2023-09-23T18:30:46.299 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_log_max_events": "-1", 2023-09-23T18:30:46.299 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_log_max_segments": "128", 2023-09-23T18:30:46.299 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_log_pause": "false", 2023-09-23T18:30:46.299 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_log_segment_size": "0", 2023-09-23T18:30:46.299 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_log_skip_corrupt_events": "false", 2023-09-23T18:30:46.299 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_log_warn_factor": "2.000000", 2023-09-23T18:30:46.299 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_max_caps_per_client": "1048576", 2023-09-23T18:30:46.299 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_max_completed_flushes": "100000", 2023-09-23T18:30:46.300 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_max_completed_requests": "100000", 2023-09-23T18:30:46.300 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_max_export_size": "20971520", 2023-09-23T18:30:46.300 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_max_file_recover": "32", 2023-09-23T18:30:46.300 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_max_purge_files": "64", 2023-09-23T18:30:46.300 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_max_purge_ops": "8192", 2023-09-23T18:30:46.300 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-09-23T18:30:46.300 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-09-23T18:30:46.300 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_max_snaps_per_dir": "100", 2023-09-23T18:30:46.300 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_max_xattr_pairs_size": "65536", 2023-09-23T18:30:46.300 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_metrics_update_interval": "2", 2023-09-23T18:30:46.300 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_min_caps_per_client": "100", 2023-09-23T18:30:46.300 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_min_caps_working_set": "10000", 2023-09-23T18:30:46.300 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-09-23T18:30:46.301 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_numa_node": "-1", 2023-09-23T18:30:46.301 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-09-23T18:30:46.301 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_op_complaint_time": "30.000000", 2023-09-23T18:30:46.301 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_op_history_duration": "600", 2023-09-23T18:30:46.301 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_op_history_size": "20", 2023-09-23T18:30:46.301 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_op_log_threshold": "5", 2023-09-23T18:30:46.301 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_ping_grace": "15", 2023-09-23T18:30:46.301 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_ping_interval": "5", 2023-09-23T18:30:46.301 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-09-23T18:30:46.301 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-09-23T18:30:46.301 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_recall_max_caps": "30000", 2023-09-23T18:30:46.301 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-09-23T18:30:46.302 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_recall_max_decay_threshold": "131072", 2023-09-23T18:30:46.302 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-09-23T18:30:46.302 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_recall_warning_threshold": "262144", 2023-09-23T18:30:46.302 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_reconnect_timeout": "45.000000", 2023-09-23T18:30:46.302 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_replay_interval": "1.000000", 2023-09-23T18:30:46.302 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-09-23T18:30:46.302 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-09-23T18:30:46.302 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_root_ino_gid": "0", 2023-09-23T18:30:46.302 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_root_ino_uid": "0", 2023-09-23T18:30:46.302 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-09-23T18:30:46.302 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_session_blocklist_on_evict": "true", 2023-09-23T18:30:46.302 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_session_blocklist_on_timeout": "true", 2023-09-23T18:30:46.303 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-09-23T18:30:46.303 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-09-23T18:30:46.303 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-09-23T18:30:46.303 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-09-23T18:30:46.303 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-09-23T18:30:46.303 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-09-23T18:30:46.303 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_shutdown_check": "0", 2023-09-23T18:30:46.303 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_skip_ino": "0", 2023-09-23T18:30:46.303 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_sleep_rank_change": "0.000000", 2023-09-23T18:30:46.303 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_snap_max_uid": "4294967294", 2023-09-23T18:30:46.303 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_snap_min_uid": "0", 2023-09-23T18:30:46.303 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_snap_rstat": "false", 2023-09-23T18:30:46.303 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_standby_replay_damaged": "false", 2023-09-23T18:30:46.304 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_symlink_recovery": "true", 2023-09-23T18:30:46.304 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_task_status_update_interval": "2.000000", 2023-09-23T18:30:46.304 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_thrash_exports": "0", 2023-09-23T18:30:46.304 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_thrash_fragments": "0", 2023-09-23T18:30:46.304 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_tick_interval": "5.000000", 2023-09-23T18:30:46.304 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_valgrind_exit": "false", 2023-09-23T18:30:46.304 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_verify_backtrace": "1", 2023-09-23T18:30:46.304 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_verify_scatter": "false", 2023-09-23T18:30:46.304 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_wipe_ino_prealloc": "false", 2023-09-23T18:30:46.304 INFO:teuthology.orchestra.run.smithi098.stdout: "mds_wipe_sessions": "false", 2023-09-23T18:30:46.304 INFO:teuthology.orchestra.run.smithi098.stdout: "mempool_debug": "false", 2023-09-23T18:30:46.304 INFO:teuthology.orchestra.run.smithi098.stdout: "memstore_debug_omit_block_device_write": "false", 2023-09-23T18:30:46.305 INFO:teuthology.orchestra.run.smithi098.stdout: "memstore_device_bytes": "1073741824", 2023-09-23T18:30:46.305 INFO:teuthology.orchestra.run.smithi098.stdout: "memstore_page_set": "false", 2023-09-23T18:30:46.305 INFO:teuthology.orchestra.run.smithi098.stdout: "memstore_page_size": "65536", 2023-09-23T18:30:46.305 INFO:teuthology.orchestra.run.smithi098.stdout: "mgr_client_bytes": "134217728", 2023-09-23T18:30:46.305 INFO:teuthology.orchestra.run.smithi098.stdout: "mgr_client_messages": "512", 2023-09-23T18:30:46.305 INFO:teuthology.orchestra.run.smithi098.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-09-23T18:30:46.305 INFO:teuthology.orchestra.run.smithi098.stdout: "mgr_connect_retry_interval": "1.000000", 2023-09-23T18:30:46.305 INFO:teuthology.orchestra.run.smithi098.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-09-23T18:30:46.305 INFO:teuthology.orchestra.run.smithi098.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-09-23T18:30:46.305 INFO:teuthology.orchestra.run.smithi098.stdout: "mgr_disabled_modules": "", 2023-09-23T18:30:46.305 INFO:teuthology.orchestra.run.smithi098.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-09-23T18:30:46.305 INFO:teuthology.orchestra.run.smithi098.stdout: "mgr_max_pg_num_change": "128", 2023-09-23T18:30:46.305 INFO:teuthology.orchestra.run.smithi098.stdout: "mgr_mds_bytes": "134217728", 2023-09-23T18:30:46.306 INFO:teuthology.orchestra.run.smithi098.stdout: "mgr_mds_messages": "128", 2023-09-23T18:30:46.306 INFO:teuthology.orchestra.run.smithi098.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-09-23T18:30:46.306 INFO:teuthology.orchestra.run.smithi098.stdout: "mgr_mon_bytes": "134217728", 2023-09-23T18:30:46.306 INFO:teuthology.orchestra.run.smithi098.stdout: "mgr_mon_messages": "128", 2023-09-23T18:30:46.306 INFO:teuthology.orchestra.run.smithi098.stdout: "mgr_osd_bytes": "536870912", 2023-09-23T18:30:46.306 INFO:teuthology.orchestra.run.smithi098.stdout: "mgr_osd_messages": "8192", 2023-09-23T18:30:46.306 INFO:teuthology.orchestra.run.smithi098.stdout: "mgr_pool": "true", 2023-09-23T18:30:46.306 INFO:teuthology.orchestra.run.smithi098.stdout: "mgr_service_beacon_grace": "60.000000", 2023-09-23T18:30:46.306 INFO:teuthology.orchestra.run.smithi098.stdout: "mgr_standby_modules": "true", 2023-09-23T18:30:46.306 INFO:teuthology.orchestra.run.smithi098.stdout: "mgr_stats_period": "5", 2023-09-23T18:30:46.306 INFO:teuthology.orchestra.run.smithi098.stdout: "mgr_stats_threshold": "5", 2023-09-23T18:30:46.306 INFO:teuthology.orchestra.run.smithi098.stdout: "mgr_tick_period": "2", 2023-09-23T18:30:46.307 INFO:teuthology.orchestra.run.smithi098.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-09-23T18:30:46.307 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_accept_timeout_factor": "2.000000", 2023-09-23T18:30:46.307 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_allow_pool_delete": "true", 2023-09-23T18:30:46.307 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_allow_pool_size_one": "false", 2023-09-23T18:30:46.307 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_auth_validate_all_caps": "true", 2023-09-23T18:30:46.307 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-09-23T18:30:46.307 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-09-23T18:30:46.307 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_client_bytes": "104857600", 2023-09-23T18:30:46.307 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_client_directed_command_retry": "2", 2023-09-23T18:30:46.307 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_client_hunt_interval": "3.000000", 2023-09-23T18:30:46.307 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-09-23T18:30:46.307 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-09-23T18:30:46.308 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-09-23T18:30:46.308 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_client_hunt_parallel": "3", 2023-09-23T18:30:46.308 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_client_log_interval": "1.000000", 2023-09-23T18:30:46.308 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-09-23T18:30:46.308 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_client_ping_interval": "10.000000", 2023-09-23T18:30:46.308 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_client_ping_timeout": "30.000000", 2023-09-23T18:30:46.308 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_clock_drift_allowed": "1.000000", 2023-09-23T18:30:46.308 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-09-23T18:30:46.308 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-09-23T18:30:46.308 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_cluster_log_file_level": "debug", 2023-09-23T18:30:46.308 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_cluster_log_to_file": "false", 2023-09-23T18:30:46.308 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_cluster_log_to_graylog": "false", 2023-09-23T18:30:46.308 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-09-23T18:30:46.309 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-09-23T18:30:46.309 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_cluster_log_to_journald": "false", 2023-09-23T18:30:46.309 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_cluster_log_to_stderr": "true", 2023-09-23T18:30:46.309 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-09-23T18:30:46.309 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-09-23T18:30:46.309 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-09-23T18:30:46.309 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_compact_on_bootstrap": "false", 2023-09-23T18:30:46.309 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_compact_on_start": "false", 2023-09-23T18:30:46.309 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_compact_on_trim": "true", 2023-09-23T18:30:46.309 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_con_tracker_persist_interval": "10", 2023-09-23T18:30:46.309 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_con_tracker_score_halflife": "43200", 2023-09-23T18:30:46.309 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_config_key_max_entry_size": "65536", 2023-09-23T18:30:46.310 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_cpu_threads": "4", 2023-09-23T18:30:46.310 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_crush_min_required_version": "hammer", 2023-09-23T18:30:46.310 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_daemon_bytes": "419430400", 2023-09-23T18:30:46.310 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-09-23T18:30:46.310 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_data_avail_crit": "5", 2023-09-23T18:30:46.310 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_data_avail_warn": "5", 2023-09-23T18:30:46.310 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_data_size_warn": "16106127360", 2023-09-23T18:30:46.310 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_debug_block_osdmap_trim": "false", 2023-09-23T18:30:46.310 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-09-23T18:30:46.310 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_debug_dump_json": "false", 2023-09-23T18:30:46.310 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-09-23T18:30:46.310 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_debug_dump_transactions": "false", 2023-09-23T18:30:46.311 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_debug_extra_checks": "false", 2023-09-23T18:30:46.311 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-09-23T18:30:46.311 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-09-23T18:30:46.311 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_debug_no_require_quincy": "false", 2023-09-23T18:30:46.311 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_debug_no_require_reef": "false", 2023-09-23T18:30:46.311 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-09-23T18:30:46.311 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_delta_reset_interval": "10.000000", 2023-09-23T18:30:46.311 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_dns_srv_name": "ceph-mon", 2023-09-23T18:30:46.311 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_down_mkfs_grace": "60", 2023-09-23T18:30:46.311 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_election_default_strategy": "3", 2023-09-23T18:30:46.311 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_election_timeout": "5.000000", 2023-09-23T18:30:46.311 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-09-23T18:30:46.312 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_elector_ping_divisor": "2", 2023-09-23T18:30:46.312 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_elector_ping_timeout": "2.000000", 2023-09-23T18:30:46.312 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_enable_op_tracker": "true", 2023-09-23T18:30:46.312 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_fake_pool_delete": "false", 2023-09-23T18:30:46.312 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_force_quorum_join": "false", 2023-09-23T18:30:46.312 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_globalid_prealloc": "10000", 2023-09-23T18:30:46.312 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_health_detail_to_clog": "true", 2023-09-23T18:30:46.312 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_health_log_update_period": "5", 2023-09-23T18:30:46.312 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_health_max_detail": "50", 2023-09-23T18:30:46.312 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_health_to_clog": "true", 2023-09-23T18:30:46.312 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_health_to_clog_interval": "600", 2023-09-23T18:30:46.312 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-09-23T18:30:46.312 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_host": "[v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] [v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0] [v2:172.21.15.98:3301/0,v1:172.21.15.98:6790/0]", 2023-09-23T18:30:46.313 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_host_override": "", 2023-09-23T18:30:46.313 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_initial_members": "", 2023-09-23T18:30:46.313 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-09-23T18:30:46.313 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-09-23T18:30:46.313 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-09-23T18:30:46.313 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-09-23T18:30:46.313 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_keyvaluedb": "rocksdb", 2023-09-23T18:30:46.313 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_lease": "5.000000", 2023-09-23T18:30:46.313 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-09-23T18:30:46.313 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-09-23T18:30:46.313 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_log_full_interval": "50", 2023-09-23T18:30:46.313 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_log_max": "10000", 2023-09-23T18:30:46.313 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_log_max_summary": "50", 2023-09-23T18:30:46.314 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_max_log_entries_per_event": "4096", 2023-09-23T18:30:46.314 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_max_log_epochs": "500", 2023-09-23T18:30:46.314 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_max_mdsmap_epochs": "500", 2023-09-23T18:30:46.314 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_max_mgrmap_epochs": "500", 2023-09-23T18:30:46.314 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_max_osd": "10000", 2023-09-23T18:30:46.314 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_max_pg_per_osd": "10000", 2023-09-23T18:30:46.314 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_max_pool_pg_num": "65536", 2023-09-23T18:30:46.314 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-09-23T18:30:46.314 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-09-23T18:30:46.314 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_mds_force_trim_to": "0", 2023-09-23T18:30:46.314 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_mds_skip_sanity": "true", 2023-09-23T18:30:46.314 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_memory_autotune": "true", 2023-09-23T18:30:46.315 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_memory_target": "2147483648", 2023-09-23T18:30:46.315 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_mgr_beacon_grace": "30", 2023-09-23T18:30:46.315 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-09-23T18:30:46.315 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_mgr_digest_period": "5", 2023-09-23T18:30:46.315 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_mgr_inactive_grace": "60", 2023-09-23T18:30:46.315 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_mgr_mkfs_grace": "240", 2023-09-23T18:30:46.315 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-09-23T18:30:46.315 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_min_osdmap_epochs": "500", 2023-09-23T18:30:46.315 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_op_complaint_time": "30", 2023-09-23T18:30:46.315 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_op_history_duration": "600", 2023-09-23T18:30:46.315 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_op_history_size": "20", 2023-09-23T18:30:46.315 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_op_history_slow_op_size": "20", 2023-09-23T18:30:46.315 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_op_history_slow_op_threshold": "10", 2023-09-23T18:30:46.316 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_op_log_threshold": "5", 2023-09-23T18:30:46.316 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-09-23T18:30:46.316 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-09-23T18:30:46.316 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-09-23T18:30:46.316 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_auto_mark_in": "false", 2023-09-23T18:30:46.316 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_auto_mark_new_in": "true", 2023-09-23T18:30:46.316 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-09-23T18:30:46.316 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-09-23T18:30:46.316 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_cache_size": "500", 2023-09-23T18:30:46.316 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_cache_size_min": "134217728", 2023-09-23T18:30:46.316 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_crush_smoke_test": "true", 2023-09-23T18:30:46.316 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_destroyed_out_interval": "600", 2023-09-23T18:30:46.317 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_down_out_interval": "600", 2023-09-23T18:30:46.317 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-09-23T18:30:46.317 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-09-23T18:30:46.317 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_force_trim_to": "0", 2023-09-23T18:30:46.317 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_full_ratio": "0.950000", 2023-09-23T18:30:46.317 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-09-23T18:30:46.317 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_laggy_halflife": "3600", 2023-09-23T18:30:46.317 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_laggy_max_interval": "300", 2023-09-23T18:30:46.317 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_laggy_weight": "0.300000", 2023-09-23T18:30:46.317 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-09-23T18:30:46.317 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_max_creating_pgs": "1024", 2023-09-23T18:30:46.317 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_max_initial_pgs": "1024", 2023-09-23T18:30:46.317 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_min_down_reporters": "2", 2023-09-23T18:30:46.318 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-09-23T18:30:46.318 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-09-23T18:30:46.318 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-09-23T18:30:46.318 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_prime_pg_temp": "true", 2023-09-23T18:30:46.318 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-09-23T18:30:46.318 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-09-23T18:30:46.318 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_report_timeout": "900", 2023-09-23T18:30:46.318 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-09-23T18:30:46.318 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-09-23T18:30:46.318 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_warn_num_repaired": "10", 2023-09-23T18:30:46.318 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osd_warn_op_age": "32.000000", 2023-09-23T18:30:46.318 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-09-23T18:30:46.319 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osdmap_full_prune_interval": "10", 2023-09-23T18:30:46.319 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osdmap_full_prune_min": "10000", 2023-09-23T18:30:46.319 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-09-23T18:30:46.319 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-09-23T18:30:46.319 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_pg_stuck_threshold": "60", 2023-09-23T18:30:46.319 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-09-23T18:30:46.319 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_pg_warn_min_objects": "10000", 2023-09-23T18:30:46.319 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_pg_warn_min_per_osd": "0", 2023-09-23T18:30:46.319 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-09-23T18:30:46.319 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_pool_quota_crit_threshold": "0", 2023-09-23T18:30:46.319 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_pool_quota_warn_threshold": "0", 2023-09-23T18:30:46.319 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_probe_timeout": "2.000000", 2023-09-23T18:30:46.319 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_reweight_max_change": "0.050000", 2023-09-23T18:30:46.320 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_reweight_max_osds": "4", 2023-09-23T18:30:46.320 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-09-23T18:30:46.320 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-09-23T18:30:46.320 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-09-23T18:30:46.320 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-09-23T18:30:46.320 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-09-23T18:30:46.320 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_scrub_interval": "86400", 2023-09-23T18:30:46.320 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_scrub_max_keys": "100", 2023-09-23T18:30:46.320 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_scrub_timeout": "300", 2023-09-23T18:30:46.320 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_session_timeout": "300", 2023-09-23T18:30:46.320 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_smart_report_timeout": "5", 2023-09-23T18:30:46.320 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_stat_smooth_intervals": "6", 2023-09-23T18:30:46.320 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-09-23T18:30:46.321 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_stretch_pool_min_size": "2", 2023-09-23T18:30:46.321 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_stretch_pool_size": "4", 2023-09-23T18:30:46.321 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-09-23T18:30:46.321 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_subscribe_interval": "86400.000000", 2023-09-23T18:30:46.321 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_sync_debug": "false", 2023-09-23T18:30:46.321 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_sync_max_payload_keys": "2000", 2023-09-23T18:30:46.321 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_sync_max_payload_size": "1048576", 2023-09-23T18:30:46.321 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_sync_provider_kill_at": "0", 2023-09-23T18:30:46.321 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_sync_requester_kill_at": "0", 2023-09-23T18:30:46.321 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_sync_timeout": "60.000000", 2023-09-23T18:30:46.321 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_target_pg_per_osd": "100", 2023-09-23T18:30:46.321 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_tick_interval": "5", 2023-09-23T18:30:46.321 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_timecheck_interval": "300.000000", 2023-09-23T18:30:46.322 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-09-23T18:30:46.322 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_use_min_delay_socket": "false", 2023-09-23T18:30:46.322 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_warn_older_version_delay": "604800", 2023-09-23T18:30:46.322 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-09-23T18:30:46.322 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-09-23T18:30:46.322 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-09-23T18:30:46.322 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_warn_on_filestore_osds": "true", 2023-09-23T18:30:46.322 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-09-23T18:30:46.322 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-09-23T18:30:46.322 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-09-23T18:30:46.322 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_warn_on_misplaced": "false", 2023-09-23T18:30:46.323 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-09-23T18:30:46.323 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_warn_on_older_version": "true", 2023-09-23T18:30:46.323 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-09-23T18:30:46.323 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_warn_on_pool_no_app": "true", 2023-09-23T18:30:46.323 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-09-23T18:30:46.323 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-09-23T18:30:46.323 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-09-23T18:30:46.323 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-09-23T18:30:46.323 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_warn_on_too_few_osds": "false", 2023-09-23T18:30:46.323 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-09-23T18:30:46.323 INFO:teuthology.orchestra.run.smithi098.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-09-23T18:30:46.323 INFO:teuthology.orchestra.run.smithi098.stdout: "monmap": "", 2023-09-23T18:30:46.323 INFO:teuthology.orchestra.run.smithi098.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-09-23T18:30:46.324 INFO:teuthology.orchestra.run.smithi098.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-09-23T18:30:46.324 INFO:teuthology.orchestra.run.smithi098.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-09-23T18:30:46.324 INFO:teuthology.orchestra.run.smithi098.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-09-23T18:30:46.324 INFO:teuthology.orchestra.run.smithi098.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-09-23T18:30:46.324 INFO:teuthology.orchestra.run.smithi098.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-09-23T18:30:46.324 INFO:teuthology.orchestra.run.smithi098.stdout: "motr_tracing_enabled": "false", 2023-09-23T18:30:46.324 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_async_op_threads": "3", 2023-09-23T18:30:46.324 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_async_rdma_buffer_size": "131072", 2023-09-23T18:30:46.324 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_async_rdma_cm": "false", 2023-09-23T18:30:46.324 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_async_rdma_device_name": "", 2023-09-23T18:30:46.324 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_async_rdma_dscp": "96", 2023-09-23T18:30:46.324 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-09-23T18:30:46.325 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_async_rdma_gid_idx": "0", 2023-09-23T18:30:46.325 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_async_rdma_local_gid": "", 2023-09-23T18:30:46.325 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_async_rdma_polling_us": "1000", 2023-09-23T18:30:46.325 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_async_rdma_port_num": "1", 2023-09-23T18:30:46.325 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-09-23T18:30:46.325 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-09-23T18:30:46.325 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_async_rdma_roce_ver": "1", 2023-09-23T18:30:46.325 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_async_rdma_send_buffers": "1024", 2023-09-23T18:30:46.325 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_async_rdma_sl": "3", 2023-09-23T18:30:46.325 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_async_rdma_support_srq": "true", 2023-09-23T18:30:46.325 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_async_rdma_type": "ib", 2023-09-23T18:30:46.325 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_async_reap_threshold": "5", 2023-09-23T18:30:46.326 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_bind_before_connect": "false", 2023-09-23T18:30:46.326 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_bind_ipv4": "true", 2023-09-23T18:30:46.326 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_bind_ipv6": "false", 2023-09-23T18:30:46.326 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_bind_msgr1": "true", 2023-09-23T18:30:46.326 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_bind_msgr2": "true", 2023-09-23T18:30:46.326 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_bind_port_max": "7568", 2023-09-23T18:30:46.326 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_bind_port_min": "6800", 2023-09-23T18:30:46.326 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_bind_prefer_ipv4": "false", 2023-09-23T18:30:46.326 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_bind_retry_count": "3", 2023-09-23T18:30:46.326 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_bind_retry_delay": "5", 2023-09-23T18:30:46.326 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_blackhole_client": "false", 2023-09-23T18:30:46.326 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_blackhole_mds": "false", 2023-09-23T18:30:46.327 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_blackhole_mgr": "false", 2023-09-23T18:30:46.327 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_blackhole_mon": "false", 2023-09-23T18:30:46.327 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_blackhole_osd": "false", 2023-09-23T18:30:46.327 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_client_mode": "crc secure", 2023-09-23T18:30:46.327 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_cluster_mode": "crc secure", 2023-09-23T18:30:46.327 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_cluster_type": "", 2023-09-23T18:30:46.327 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_compress_secure": "false", 2023-09-23T18:30:46.327 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_connection_idle_timeout": "900", 2023-09-23T18:30:46.327 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_connection_ready_timeout": "10", 2023-09-23T18:30:46.327 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_crc_data": "true", 2023-09-23T18:30:46.327 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_crc_header": "true", 2023-09-23T18:30:46.327 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_die_on_bad_msg": "false", 2023-09-23T18:30:46.328 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_die_on_bug": "true", 2023-09-23T18:30:46.328 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_die_on_old_message": "true", 2023-09-23T18:30:46.328 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_die_on_skipped_message": "false", 2023-09-23T18:30:46.328 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_die_on_unhandled_msg": "false", 2023-09-23T18:30:46.328 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-09-23T18:30:46.328 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_dpdk_coremask": "0xF", 2023-09-23T18:30:46.328 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-09-23T18:30:46.328 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_dpdk_devs_allowlist": "", 2023-09-23T18:30:46.328 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_dpdk_enable_tso": "true", 2023-09-23T18:30:46.328 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-09-23T18:30:46.328 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-09-23T18:30:46.328 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_dpdk_hugepages": "", 2023-09-23T18:30:46.329 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_dpdk_hw_flow_control": "true", 2023-09-23T18:30:46.329 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-09-23T18:30:46.329 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_dpdk_lro": "true", 2023-09-23T18:30:46.329 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_dpdk_memory_channel": "4", 2023-09-23T18:30:46.329 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-09-23T18:30:46.329 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_dpdk_pmd": "", 2023-09-23T18:30:46.329 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_dpdk_port_id": "0", 2023-09-23T18:30:46.329 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-09-23T18:30:46.329 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_dump_corrupt_message_level": "1", 2023-09-23T18:30:46.329 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_dump_on_send": "false", 2023-09-23T18:30:46.329 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_initial_backoff": "0.200000", 2023-09-23T18:30:46.329 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_inject_delay_max": "1.000000", 2023-09-23T18:30:46.329 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_inject_delay_probability": "0.000000", 2023-09-23T18:30:46.330 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_inject_delay_type": "", 2023-09-23T18:30:46.330 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_inject_internal_delays": "0.000000", 2023-09-23T18:30:46.330 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_inject_network_congestion": "0", 2023-09-23T18:30:46.330 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_inject_socket_failures": "0", 2023-09-23T18:30:46.330 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_learn_addr_from_peer": "true", 2023-09-23T18:30:46.330 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_max_accept_failures": "4", 2023-09-23T18:30:46.330 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_max_backoff": "15.000000", 2023-09-23T18:30:46.330 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_mon_client_mode": "secure crc", 2023-09-23T18:30:46.330 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_mon_cluster_mode": "secure crc", 2023-09-23T18:30:46.330 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_mon_service_mode": "secure crc", 2023-09-23T18:30:46.330 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_osd_compress_min_size": "1024", 2023-09-23T18:30:46.330 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_osd_compress_mode": "none", 2023-09-23T18:30:46.331 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_osd_compression_algorithm": "snappy", 2023-09-23T18:30:46.331 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-09-23T18:30:46.331 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_pq_min_cost": "65536", 2023-09-23T18:30:46.331 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_public_type": "", 2023-09-23T18:30:46.331 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_service_mode": "crc secure", 2023-09-23T18:30:46.331 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_tcp_listen_backlog": "512", 2023-09-23T18:30:46.331 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_tcp_nodelay": "true", 2023-09-23T18:30:46.331 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-09-23T18:30:46.331 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_tcp_rcvbuf": "0", 2023-09-23T18:30:46.331 INFO:teuthology.orchestra.run.smithi098.stdout: "ms_type": "async+posix", 2023-09-23T18:30:46.331 INFO:teuthology.orchestra.run.smithi098.stdout: "no_config_file": "false", 2023-09-23T18:30:46.331 INFO:teuthology.orchestra.run.smithi098.stdout: "objecter_completion_locks_per_session": "32", 2023-09-23T18:30:46.332 INFO:teuthology.orchestra.run.smithi098.stdout: "objecter_debug_inject_relock_delay": "false", 2023-09-23T18:30:46.332 INFO:teuthology.orchestra.run.smithi098.stdout: "objecter_inflight_op_bytes": "104857600", 2023-09-23T18:30:46.332 INFO:teuthology.orchestra.run.smithi098.stdout: "objecter_inflight_ops": "1024", 2023-09-23T18:30:46.332 INFO:teuthology.orchestra.run.smithi098.stdout: "objecter_inject_no_watch_ping": "false", 2023-09-23T18:30:46.332 INFO:teuthology.orchestra.run.smithi098.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-09-23T18:30:46.332 INFO:teuthology.orchestra.run.smithi098.stdout: "objecter_tick_interval": "5.000000", 2023-09-23T18:30:46.332 INFO:teuthology.orchestra.run.smithi098.stdout: "objecter_timeout": "10.000000", 2023-09-23T18:30:46.332 INFO:teuthology.orchestra.run.smithi098.stdout: "objectstore_blackhole": "false", 2023-09-23T18:30:46.332 INFO:teuthology.orchestra.run.smithi098.stdout: "openssl_engine_opts": "", 2023-09-23T18:30:46.332 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_agent_delay_time": "5.000000", 2023-09-23T18:30:46.332 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_agent_hist_halflife": "1000", 2023-09-23T18:30:46.332 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_agent_max_low_ops": "2", 2023-09-23T18:30:46.333 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_agent_max_ops": "4", 2023-09-23T18:30:46.333 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-09-23T18:30:46.333 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_agent_quantize_effort": "0.100000", 2023-09-23T18:30:46.333 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_agent_slop": "0.020000", 2023-09-23T18:30:46.333 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-09-23T18:30:46.333 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_allow_recovery_below_min_size": "true", 2023-09-23T18:30:46.333 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_asio_thread_count": "2", 2023-09-23T18:30:46.333 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_async_recovery_min_cost": "100", 2023-09-23T18:30:46.333 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_auto_mark_unfound_lost": "false", 2023-09-23T18:30:46.333 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_backfill_retry_interval": "30.000000", 2023-09-23T18:30:46.333 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_backfill_scan_max": "512", 2023-09-23T18:30:46.333 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_backfill_scan_min": "64", 2023-09-23T18:30:46.334 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_backoff_on_degraded": "false", 2023-09-23T18:30:46.334 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_backoff_on_peering": "false", 2023-09-23T18:30:46.334 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_backoff_on_unfound": "true", 2023-09-23T18:30:46.334 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_beacon_report_interval": "300", 2023-09-23T18:30:46.334 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_bench_duration": "30", 2023-09-23T18:30:46.334 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-09-23T18:30:46.334 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_bench_max_block_size": "67108864", 2023-09-23T18:30:46.334 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_bench_small_size_max_iops": "100", 2023-09-23T18:30:46.334 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_blkin_trace_all": "false", 2023-09-23T18:30:46.334 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_blocked_scrub_grace_period": "120", 2023-09-23T18:30:46.334 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-09-23T18:30:46.334 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-09-23T18:30:46.335 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-09-23T18:30:46.335 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_check_for_log_corruption": "false", 2023-09-23T18:30:46.335 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-09-23T18:30:46.335 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-09-23T18:30:46.335 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-09-23T18:30:46.335 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-09-23T18:30:46.335 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_class_update_on_start": "true", 2023-09-23T18:30:46.335 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_client_message_cap": "256", 2023-09-23T18:30:46.335 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_client_message_size_cap": "524288000", 2023-09-23T18:30:46.335 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_client_op_priority": "63", 2023-09-23T18:30:46.335 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_client_watch_timeout": "30", 2023-09-23T18:30:46.335 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_command_max_records": "256", 2023-09-23T18:30:46.336 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_compact_on_start": "false", 2023-09-23T18:30:46.336 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-09-23T18:30:46.336 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_crush_chooseleaf_type": "1", 2023-09-23T18:30:46.336 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_crush_initial_weight": "-1.000000", 2023-09-23T18:30:46.336 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_crush_update_on_start": "true", 2023-09-23T18:30:46.336 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_crush_update_weight_set": "true", 2023-09-23T18:30:46.336 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-09-23T18:30:46.336 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-09-23T18:30:46.336 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-09-23T18:30:46.336 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_debug_drop_ping_duration": "0", 2023-09-23T18:30:46.336 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-09-23T18:30:46.336 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_debug_feed_pullee": "-1", 2023-09-23T18:30:46.337 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-09-23T18:30:46.337 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-09-23T18:30:46.337 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-09-23T18:30:46.337 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_debug_misdirected_ops": "false", 2023-09-23T18:30:46.337 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_debug_no_acting_change": "false", 2023-09-23T18:30:46.337 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_debug_no_purge_strays": "false", 2023-09-23T18:30:46.337 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_debug_op_order": "false", 2023-09-23T18:30:46.337 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_debug_pg_log_writeout": "false", 2023-09-23T18:30:46.337 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_debug_pretend_recovery_active": "false", 2023-09-23T18:30:46.337 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-09-23T18:30:46.337 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-09-23T18:30:46.337 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_debug_shutdown": "false", 2023-09-23T18:30:46.338 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-09-23T18:30:46.338 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-09-23T18:30:46.338 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_debug_verify_cached_snaps": "false", 2023-09-23T18:30:46.338 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_debug_verify_missing_on_start": "false", 2023-09-23T18:30:46.338 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_debug_verify_snaps": "false", 2023-09-23T18:30:46.338 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-09-23T18:30:46.338 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-09-23T18:30:46.338 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_deep_scrub_keys": "1024", 2023-09-23T18:30:46.338 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-09-23T18:30:46.338 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-09-23T18:30:46.338 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-09-23T18:30:46.338 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_deep_scrub_stride": "524288", 2023-09-23T18:30:46.339 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-09-23T18:30:46.339 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_default_data_pool_replay_window": "45", 2023-09-23T18:30:46.339 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_default_notify_timeout": "30", 2023-09-23T18:30:46.339 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_delete_sleep": "0.000000", 2023-09-23T18:30:46.339 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-09-23T18:30:46.339 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-09-23T18:30:46.339 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-09-23T18:30:46.339 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_discard_disconnected_ops": "true", 2023-09-23T18:30:46.339 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_enable_op_tracker": "true", 2023-09-23T18:30:46.339 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-09-23T18:30:46.339 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_extblkdev_plugins": "vdo", 2023-09-23T18:30:46.339 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-09-23T18:30:46.340 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-09-23T18:30:46.340 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_fast_info": "true", 2023-09-23T18:30:46.340 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_fast_shutdown": "true", 2023-09-23T18:30:46.340 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-09-23T18:30:46.340 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_fast_shutdown_timeout": "15", 2023-09-23T18:30:46.340 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-09-23T18:30:46.340 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-09-23T18:30:46.340 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-09-23T18:30:46.340 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_function_tracing": "false", 2023-09-23T18:30:46.340 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_heartbeat_grace": "20", 2023-09-23T18:30:46.340 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_heartbeat_interval": "6", 2023-09-23T18:30:46.340 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-09-23T18:30:46.340 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_heartbeat_min_peers": "10", 2023-09-23T18:30:46.341 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_heartbeat_min_size": "2000", 2023-09-23T18:30:46.341 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_heartbeat_stale": "600", 2023-09-23T18:30:46.341 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-09-23T18:30:46.341 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_hit_set_max_size": "100000", 2023-09-23T18:30:46.341 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_hit_set_min_size": "1000", 2023-09-23T18:30:46.341 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-09-23T18:30:46.341 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-09-23T18:30:46.341 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-09-23T18:30:46.341 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-09-23T18:30:46.341 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-09-23T18:30:46.341 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_journal_flush_on_shutdown": "true", 2023-09-23T18:30:46.341 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_journal_size": "5120", 2023-09-23T18:30:46.342 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_kill_backfill_at": "0", 2023-09-23T18:30:46.342 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_loop_before_reset_tphandle": "64", 2023-09-23T18:30:46.342 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_map_cache_size": "50", 2023-09-23T18:30:46.342 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_map_dedup": "true", 2023-09-23T18:30:46.342 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_map_message_max": "40", 2023-09-23T18:30:46.342 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_map_message_max_bytes": "10485760", 2023-09-23T18:30:46.342 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_map_share_max_epochs": "40", 2023-09-23T18:30:46.342 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_max_attr_name_len": "100", 2023-09-23T18:30:46.342 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_max_attr_size": "0", 2023-09-23T18:30:46.342 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_max_backfills": "1", 2023-09-23T18:30:46.342 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_max_markdown_count": "5", 2023-09-23T18:30:46.342 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_max_markdown_period": "600", 2023-09-23T18:30:46.343 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_max_object_name_len": "2048", 2023-09-23T18:30:46.343 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_max_object_namespace_len": "256", 2023-09-23T18:30:46.343 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_max_object_size": "134217728", 2023-09-23T18:30:46.343 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-09-23T18:30:46.343 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_max_omap_entries_per_request": "1024", 2023-09-23T18:30:46.343 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_max_pg_blocked_by": "16", 2023-09-23T18:30:46.343 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_max_pg_log_entries": "10000", 2023-09-23T18:30:46.343 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-09-23T18:30:46.343 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_max_pgls": "1024", 2023-09-23T18:30:46.343 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_max_push_cost": "8388608", 2023-09-23T18:30:46.343 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_max_push_objects": "10", 2023-09-23T18:30:46.343 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_max_scrubs": "1", 2023-09-23T18:30:46.343 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-09-23T18:30:46.344 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_max_trimming_pgs": "2", 2023-09-23T18:30:46.344 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_max_write_op_reply_len": "64", 2023-09-23T18:30:46.344 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_max_write_size": "90", 2023-09-23T18:30:46.344 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-09-23T18:30:46.344 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-09-23T18:30:46.344 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-09-23T18:30:46.344 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-09-23T18:30:46.344 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-09-23T18:30:46.344 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-09-23T18:30:46.344 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-09-23T18:30:46.344 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_mclock_override_recovery_settings": "false", 2023-09-23T18:30:46.344 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_mclock_profile": "balanced", 2023-09-23T18:30:46.345 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-09-23T18:30:46.345 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-09-23T18:30:46.345 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-09-23T18:30:46.345 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-09-23T18:30:46.345 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-09-23T18:30:46.345 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-09-23T18:30:46.345 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-09-23T18:30:46.345 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-09-23T18:30:46.345 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-09-23T18:30:46.345 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-09-23T18:30:46.345 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_mclock_skip_benchmark": "false", 2023-09-23T18:30:46.345 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_memory_base": "805306368", 2023-09-23T18:30:46.346 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_memory_cache_min": "134217728", 2023-09-23T18:30:46.346 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-09-23T18:30:46.346 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-09-23T18:30:46.346 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_memory_target": "4294967296", 2023-09-23T18:30:46.346 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_memory_target_autotune": "false", 2023-09-23T18:30:46.346 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-09-23T18:30:46.346 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_min_pg_log_entries": "250", 2023-09-23T18:30:46.346 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_min_recovery_priority": "0", 2023-09-23T18:30:46.346 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_mon_heartbeat_interval": "30", 2023-09-23T18:30:46.346 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-09-23T18:30:46.346 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_mon_report_interval": "5", 2023-09-23T18:30:46.346 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_mon_report_max_in_flight": "2", 2023-09-23T18:30:46.347 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-09-23T18:30:46.347 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_num_cache_shards": "32", 2023-09-23T18:30:46.347 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_num_op_tracker_shard": "32", 2023-09-23T18:30:46.347 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_numa_auto_affinity": "true", 2023-09-23T18:30:46.347 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_numa_node": "-1", 2023-09-23T18:30:46.347 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_numa_prefer_iface": "true", 2023-09-23T18:30:46.347 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-09-23T18:30:46.347 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_objecter_finishers": "1", 2023-09-23T18:30:46.347 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_objectstore": "bluestore", 2023-09-23T18:30:46.347 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_objectstore_fuse": "false", 2023-09-23T18:30:46.347 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_objectstore_tracing": "false", 2023-09-23T18:30:46.347 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_op_complaint_time": "30.000000", 2023-09-23T18:30:46.348 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_op_history_duration": "600", 2023-09-23T18:30:46.348 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_op_history_size": "20", 2023-09-23T18:30:46.348 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_op_history_slow_op_size": "20", 2023-09-23T18:30:46.348 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-09-23T18:30:46.348 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_op_log_threshold": "5", 2023-09-23T18:30:46.348 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_op_num_shards": "0", 2023-09-23T18:30:46.348 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_op_num_shards_hdd": "5", 2023-09-23T18:30:46.348 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_op_num_shards_ssd": "8", 2023-09-23T18:30:46.348 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_op_num_threads_per_shard": "0", 2023-09-23T18:30:46.348 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-09-23T18:30:46.348 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-09-23T18:30:46.348 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-09-23T18:30:46.348 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_op_pq_min_cost": "65536", 2023-09-23T18:30:46.349 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_op_queue": "mclock_scheduler", 2023-09-23T18:30:46.349 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_op_queue_cut_off": "high", 2023-09-23T18:30:46.349 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_op_thread_suicide_timeout": "150", 2023-09-23T18:30:46.349 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_op_thread_timeout": "15", 2023-09-23T18:30:46.349 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_open_classes_on_start": "true", 2023-09-23T18:30:46.349 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_os_flags": "0", 2023-09-23T18:30:46.349 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_peering_op_priority": "255", 2023-09-23T18:30:46.349 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pg_delete_cost": "1048576", 2023-09-23T18:30:46.349 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pg_delete_priority": "5", 2023-09-23T18:30:46.349 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-09-23T18:30:46.349 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-09-23T18:30:46.349 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pg_log_dups_tracked": "3000", 2023-09-23T18:30:46.350 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pg_log_trim_max": "10000", 2023-09-23T18:30:46.350 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pg_log_trim_min": "100", 2023-09-23T18:30:46.350 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-09-23T18:30:46.350 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pg_object_context_cache_count": "64", 2023-09-23T18:30:46.350 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pg_stat_report_interval_max": "500", 2023-09-23T18:30:46.350 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-09-23T18:30:46.350 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-09-23T18:30:46.350 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-09-23T18:30:46.350 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-09-23T18:30:46.350 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-09-23T18:30:46.350 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-09-23T18:30:46.350 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_default_crimson": "false", 2023-09-23T18:30:46.350 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_default_crush_rule": "-1", 2023-09-23T18:30:46.351 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_default_ec_fast_read": "false", 2023-09-23T18:30:46.351 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-09-23T18:30:46.351 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_default_flag_bulk": "false", 2023-09-23T18:30:46.351 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-09-23T18:30:46.351 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_default_flag_nodelete": "false", 2023-09-23T18:30:46.351 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-09-23T18:30:46.351 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-09-23T18:30:46.351 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_default_flags": "0", 2023-09-23T18:30:46.351 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-09-23T18:30:46.351 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_default_min_size": "0", 2023-09-23T18:30:46.351 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-09-23T18:30:46.351 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_default_pg_num": "32", 2023-09-23T18:30:46.352 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_default_pgp_num": "0", 2023-09-23T18:30:46.352 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-09-23T18:30:46.352 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_default_size": "3", 2023-09-23T18:30:46.352 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_default_type": "replicated", 2023-09-23T18:30:46.352 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-09-23T18:30:46.352 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_pool_use_gmt_hitset": "true", 2023-09-23T18:30:46.352 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_push_per_object_cost": "1000", 2023-09-23T18:30:46.352 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_read_ec_check_for_errors": "false", 2023-09-23T18:30:46.352 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_recover_clone_overlap": "true", 2023-09-23T18:30:46.352 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_recover_clone_overlap_limit": "10", 2023-09-23T18:30:46.352 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_recovery_cost": "20971520", 2023-09-23T18:30:46.352 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_recovery_delay_start": "0.000000", 2023-09-23T18:30:46.353 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_recovery_max_active": "0", 2023-09-23T18:30:46.353 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_recovery_max_active_hdd": "3", 2023-09-23T18:30:46.353 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_recovery_max_active_ssd": "10", 2023-09-23T18:30:46.353 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_recovery_max_chunk": "8388608", 2023-09-23T18:30:46.353 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-09-23T18:30:46.353 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_recovery_max_single_start": "1", 2023-09-23T18:30:46.353 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_recovery_op_priority": "3", 2023-09-23T18:30:46.353 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_recovery_op_warn_multiple": "16", 2023-09-23T18:30:46.353 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_recovery_priority": "5", 2023-09-23T18:30:46.353 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_recovery_retry_interval": "30.000000", 2023-09-23T18:30:46.353 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_recovery_sleep": "0.000000", 2023-09-23T18:30:46.353 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-09-23T18:30:46.354 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-09-23T18:30:46.354 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-09-23T18:30:46.354 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_repair_during_recovery": "false", 2023-09-23T18:30:46.354 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_requested_scrub_priority": "120", 2023-09-23T18:30:46.354 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-09-23T18:30:46.354 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_rollback_to_cluster_snap": "", 2023-09-23T18:30:46.354 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_scrub_auto_repair": "false", 2023-09-23T18:30:46.354 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-09-23T18:30:46.354 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-09-23T18:30:46.354 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_scrub_begin_hour": "0", 2023-09-23T18:30:46.354 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_scrub_begin_week_day": "0", 2023-09-23T18:30:46.354 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_scrub_chunk_max": "25", 2023-09-23T18:30:46.355 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_scrub_chunk_min": "5", 2023-09-23T18:30:46.355 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_scrub_cost": "52428800", 2023-09-23T18:30:46.355 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_scrub_during_recovery": "false", 2023-09-23T18:30:46.355 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_scrub_end_hour": "0", 2023-09-23T18:30:46.355 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_scrub_end_week_day": "0", 2023-09-23T18:30:46.355 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_scrub_event_cost": "4096", 2023-09-23T18:30:46.355 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-09-23T18:30:46.355 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-09-23T18:30:46.355 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_scrub_invalid_stats": "true", 2023-09-23T18:30:46.355 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_scrub_load_threshold": "0.500000", 2023-09-23T18:30:46.355 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_scrub_max_interval": "604800.000000", 2023-09-23T18:30:46.355 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_scrub_max_preemptions": "5", 2023-09-23T18:30:46.355 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_scrub_min_interval": "86400.000000", 2023-09-23T18:30:46.356 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_scrub_priority": "5", 2023-09-23T18:30:46.356 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_scrub_reservation_timeout": "5000", 2023-09-23T18:30:46.356 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_scrub_sleep": "0.000000", 2023-09-23T18:30:46.356 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-09-23T18:30:46.356 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-09-23T18:30:46.356 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-09-23T18:30:46.356 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_shutdown_pgref_assert": "false", 2023-09-23T18:30:46.356 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_skip_data_digest": "false", 2023-09-23T18:30:46.356 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_smart_report_timeout": "5", 2023-09-23T18:30:46.356 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_snap_trim_cost": "1048576", 2023-09-23T18:30:46.356 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_snap_trim_priority": "5", 2023-09-23T18:30:46.356 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_snap_trim_sleep": "0.000000", 2023-09-23T18:30:46.357 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-09-23T18:30:46.357 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-09-23T18:30:46.357 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-09-23T18:30:46.357 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-09-23T18:30:46.357 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_stats_update_period_scrubbing": "15", 2023-09-23T18:30:46.357 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-09-23T18:30:46.357 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_target_transaction_size": "30", 2023-09-23T18:30:46.357 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-09-23T18:30:46.357 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-09-23T18:30:46.357 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-09-23T18:30:46.357 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-09-23T18:30:46.357 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-09-23T18:30:46.357 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-09-23T18:30:46.358 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-09-23T18:30:46.358 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_tier_default_cache_mode": "writeback", 2023-09-23T18:30:46.358 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-09-23T18:30:46.358 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-09-23T18:30:46.358 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_tracing": "false", 2023-09-23T18:30:46.358 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_use_stale_snap": "false", 2023-09-23T18:30:46.358 INFO:teuthology.orchestra.run.smithi098.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-09-23T18:30:46.358 INFO:teuthology.orchestra.run.smithi098.stdout: "osdc_blkin_trace_all": "false", 2023-09-23T18:30:46.358 INFO:teuthology.orchestra.run.smithi098.stdout: "paxos_kill_at": "0", 2023-09-23T18:30:46.358 INFO:teuthology.orchestra.run.smithi098.stdout: "paxos_max_join_drift": "10", 2023-09-23T18:30:46.358 INFO:teuthology.orchestra.run.smithi098.stdout: "paxos_min": "500", 2023-09-23T18:30:46.359 INFO:teuthology.orchestra.run.smithi098.stdout: "paxos_min_wait": "0.050000", 2023-09-23T18:30:46.359 INFO:teuthology.orchestra.run.smithi098.stdout: "paxos_propose_interval": "1.000000", 2023-09-23T18:30:46.359 INFO:teuthology.orchestra.run.smithi098.stdout: "paxos_service_trim_max": "500", 2023-09-23T18:30:46.359 INFO:teuthology.orchestra.run.smithi098.stdout: "paxos_service_trim_max_multiplier": "20", 2023-09-23T18:30:46.359 INFO:teuthology.orchestra.run.smithi098.stdout: "paxos_service_trim_min": "250", 2023-09-23T18:30:46.359 INFO:teuthology.orchestra.run.smithi098.stdout: "paxos_stash_full_interval": "25", 2023-09-23T18:30:46.359 INFO:teuthology.orchestra.run.smithi098.stdout: "paxos_trim_max": "500", 2023-09-23T18:30:46.359 INFO:teuthology.orchestra.run.smithi098.stdout: "paxos_trim_min": "250", 2023-09-23T18:30:46.359 INFO:teuthology.orchestra.run.smithi098.stdout: "perf": "true", 2023-09-23T18:30:46.359 INFO:teuthology.orchestra.run.smithi098.stdout: "pid_file": "", 2023-09-23T18:30:46.359 INFO:teuthology.orchestra.run.smithi098.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-09-23T18:30:46.359 INFO:teuthology.orchestra.run.smithi098.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-09-23T18:30:46.360 INFO:teuthology.orchestra.run.smithi098.stdout: "public_addr": "-", 2023-09-23T18:30:46.360 INFO:teuthology.orchestra.run.smithi098.stdout: "public_addrv": "", 2023-09-23T18:30:46.360 INFO:teuthology.orchestra.run.smithi098.stdout: "public_bind_addr": "-", 2023-09-23T18:30:46.360 INFO:teuthology.orchestra.run.smithi098.stdout: "public_network": "172.21.15.254/32,172.21.0.0/20", 2023-09-23T18:30:46.360 INFO:teuthology.orchestra.run.smithi098.stdout: "public_network_interface": "", 2023-09-23T18:30:46.360 INFO:teuthology.orchestra.run.smithi098.stdout: "qat_compressor_enabled": "false", 2023-09-23T18:30:46.360 INFO:teuthology.orchestra.run.smithi098.stdout: "qat_compressor_session_max_number": "256", 2023-09-23T18:30:46.360 INFO:teuthology.orchestra.run.smithi098.stdout: "rados_mon_op_timeout": "0", 2023-09-23T18:30:46.360 INFO:teuthology.orchestra.run.smithi098.stdout: "rados_osd_op_timeout": "0", 2023-09-23T18:30:46.360 INFO:teuthology.orchestra.run.smithi098.stdout: "rados_tracing": "false", 2023-09-23T18:30:46.360 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_atime_update_interval": "60", 2023-09-23T18:30:46.360 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-09-23T18:30:46.360 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_balance_parent_reads": "false", 2023-09-23T18:30:46.361 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_balance_snap_reads": "false", 2023-09-23T18:30:46.361 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_blkin_trace_all": "false", 2023-09-23T18:30:46.361 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_blocklist_expire_seconds": "0", 2023-09-23T18:30:46.361 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_blocklist_on_break_lock": "true", 2023-09-23T18:30:46.361 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_cache": "true", 2023-09-23T18:30:46.361 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_cache_block_writes_upfront": "false", 2023-09-23T18:30:46.361 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_cache_max_dirty": "25165824", 2023-09-23T18:30:46.361 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-09-23T18:30:46.361 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_cache_max_dirty_object": "0", 2023-09-23T18:30:46.361 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_cache_policy": "writearound", 2023-09-23T18:30:46.361 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_cache_size": "33554432", 2023-09-23T18:30:46.361 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_cache_target_dirty": "16777216", 2023-09-23T18:30:46.362 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-09-23T18:30:46.362 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_clone_copy_on_read": "false", 2023-09-23T18:30:46.362 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_compression_hint": "none", 2023-09-23T18:30:46.362 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_concurrent_management_ops": "10", 2023-09-23T18:30:46.362 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-09-23T18:30:46.362 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_default_clone_format": "auto", 2023-09-23T18:30:46.362 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_default_data_pool": "", 2023-09-23T18:30:46.362 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_default_features": "61", 2023-09-23T18:30:46.362 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_default_format": "2", 2023-09-23T18:30:46.362 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_default_map_options": "", 2023-09-23T18:30:46.362 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_default_order": "22", 2023-09-23T18:30:46.362 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_default_pool": "rbd", 2023-09-23T18:30:46.363 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-09-23T18:30:46.363 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_default_stripe_count": "0", 2023-09-23T18:30:46.363 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_default_stripe_unit": "0", 2023-09-23T18:30:46.363 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_disable_zero_copy_writes": "true", 2023-09-23T18:30:46.363 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_discard_granularity_bytes": "65536", 2023-09-23T18:30:46.363 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-09-23T18:30:46.363 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_enable_alloc_hint": "true", 2023-09-23T18:30:46.363 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-09-23T18:30:46.363 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_io_scheduler": "simple", 2023-09-23T18:30:46.363 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-09-23T18:30:46.363 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_journal_commit_age": "5.000000", 2023-09-23T18:30:46.363 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-09-23T18:30:46.363 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-09-23T18:30:46.364 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-09-23T18:30:46.364 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-09-23T18:30:46.364 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_journal_object_flush_interval": "0", 2023-09-23T18:30:46.364 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-09-23T18:30:46.364 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-09-23T18:30:46.364 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_journal_order": "24", 2023-09-23T18:30:46.364 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_journal_pool": "", 2023-09-23T18:30:46.364 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_journal_splay_width": "4", 2023-09-23T18:30:46.364 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_localize_parent_reads": "false", 2023-09-23T18:30:46.364 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_localize_snap_reads": "false", 2023-09-23T18:30:46.364 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-09-23T18:30:46.364 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-09-23T18:30:46.365 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-09-23T18:30:46.365 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-09-23T18:30:46.365 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-09-23T18:30:46.365 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-09-23T18:30:46.365 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirror_image_policy_type": "simple", 2023-09-23T18:30:46.365 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-09-23T18:30:46.365 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-09-23T18:30:46.365 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-09-23T18:30:46.365 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-09-23T18:30:46.365 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-09-23T18:30:46.365 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-09-23T18:30:46.365 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-09-23T18:30:46.365 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirror_memory_autotune": "true", 2023-09-23T18:30:46.366 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirror_memory_base": "805306368", 2023-09-23T18:30:46.366 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-09-23T18:30:46.366 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-09-23T18:30:46.366 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-09-23T18:30:46.366 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-09-23T18:30:46.366 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirror_memory_target": "4294967296", 2023-09-23T18:30:46.366 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-09-23T18:30:46.366 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-09-23T18:30:46.366 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-09-23T18:30:46.366 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirroring_delete_delay": "0", 2023-09-23T18:30:46.366 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-09-23T18:30:46.366 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirroring_replay_delay": "0", 2023-09-23T18:30:46.367 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-09-23T18:30:46.367 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-09-23T18:30:46.367 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_move_to_trash_on_remove": "false", 2023-09-23T18:30:46.367 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-09-23T18:30:46.367 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_mtime_update_interval": "60", 2023-09-23T18:30:46.367 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_non_blocking_aio": "true", 2023-09-23T18:30:46.367 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_op_thread_timeout": "60", 2023-09-23T18:30:46.367 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_op_threads": "1", 2023-09-23T18:30:46.367 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_parent_cache_enabled": "false", 2023-09-23T18:30:46.367 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_persistent_cache_mode": "disabled", 2023-09-23T18:30:46.367 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_persistent_cache_path": "/tmp", 2023-09-23T18:30:46.367 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_persistent_cache_size": "1073741824", 2023-09-23T18:30:46.368 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_plugins": "", 2023-09-23T18:30:46.368 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_qos_bps_burst": "0", 2023-09-23T18:30:46.368 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-09-23T18:30:46.368 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_qos_bps_limit": "0", 2023-09-23T18:30:46.368 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_qos_exclude_ops": "0", 2023-09-23T18:30:46.368 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_qos_iops_burst": "0", 2023-09-23T18:30:46.368 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-09-23T18:30:46.368 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_qos_iops_limit": "0", 2023-09-23T18:30:46.368 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_qos_read_bps_burst": "0", 2023-09-23T18:30:46.368 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-09-23T18:30:46.368 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_qos_read_bps_limit": "0", 2023-09-23T18:30:46.368 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_qos_read_iops_burst": "0", 2023-09-23T18:30:46.368 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-09-23T18:30:46.369 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_qos_read_iops_limit": "0", 2023-09-23T18:30:46.369 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_qos_schedule_tick_min": "50", 2023-09-23T18:30:46.369 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_qos_write_bps_burst": "0", 2023-09-23T18:30:46.369 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-09-23T18:30:46.369 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_qos_write_bps_limit": "0", 2023-09-23T18:30:46.369 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_qos_write_iops_burst": "0", 2023-09-23T18:30:46.369 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-09-23T18:30:46.369 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_qos_write_iops_limit": "0", 2023-09-23T18:30:46.369 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_quiesce_notification_attempts": "10", 2023-09-23T18:30:46.369 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_read_from_replica_policy": "default", 2023-09-23T18:30:46.369 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-09-23T18:30:46.369 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_readahead_max_bytes": "524288", 2023-09-23T18:30:46.370 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_readahead_trigger_requests": "10", 2023-09-23T18:30:46.370 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_request_timed_out_seconds": "30", 2023-09-23T18:30:46.370 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_skip_partial_discard": "true", 2023-09-23T18:30:46.370 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-09-23T18:30:46.370 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_tracing": "false", 2023-09-23T18:30:46.370 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_validate_names": "true", 2023-09-23T18:30:46.370 INFO:teuthology.orchestra.run.smithi098.stdout: "rbd_validate_pool": "true", 2023-09-23T18:30:46.370 INFO:teuthology.orchestra.run.smithi098.stdout: "restapi_base_url": "", 2023-09-23T18:30:46.370 INFO:teuthology.orchestra.run.smithi098.stdout: "restapi_log_level": "", 2023-09-23T18:30:46.370 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_acl_grants_max_num": "100", 2023-09-23T18:30:46.370 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_admin_entry": "admin", 2023-09-23T18:30:46.370 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-09-23T18:30:46.371 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_backend_store": "rados", 2023-09-23T18:30:46.371 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_barbican_url": "", 2023-09-23T18:30:46.371 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_beast_enable_async": "true", 2023-09-23T18:30:46.371 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-09-23T18:30:46.371 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-09-23T18:30:46.371 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_bucket_index_max_aio": "128", 2023-09-23T18:30:46.371 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-09-23T18:30:46.371 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-09-23T18:30:46.371 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_bucket_quota_ttl": "600", 2023-09-23T18:30:46.371 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-09-23T18:30:46.371 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_cache_enabled": "true", 2023-09-23T18:30:46.371 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_cache_expiry_interval": "900", 2023-09-23T18:30:46.371 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_cache_lru_size": "10000", 2023-09-23T18:30:46.372 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_config_store": "rados", 2023-09-23T18:30:46.372 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_content_length_compat": "false", 2023-09-23T18:30:46.372 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_copy_obj_progress": "true", 2023-09-23T18:30:46.372 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-09-23T18:30:46.372 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_cors_rules_max_num": "100", 2023-09-23T18:30:46.372 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_cross_domain_policy": "", 2023-09-23T18:30:46.372 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_default_encryption_key": "", 2023-09-23T18:30:46.372 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_kmip_addr": "", 2023-09-23T18:30:46.372 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_kmip_ca_path": "", 2023-09-23T18:30:46.372 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_kmip_client_cert": "", 2023-09-23T18:30:46.372 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_kmip_client_key": "", 2023-09-23T18:30:46.372 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-09-23T18:30:46.373 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_kmip_password": "", 2023-09-23T18:30:46.373 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-09-23T18:30:46.373 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_kmip_username": "", 2023-09-23T18:30:46.373 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_require_ssl": "true", 2023-09-23T18:30:46.373 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-09-23T18:30:46.373 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-09-23T18:30:46.373 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-09-23T18:30:46.373 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-09-23T18:30:46.373 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-09-23T18:30:46.373 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-09-23T18:30:46.373 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-09-23T18:30:46.373 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-09-23T18:30:46.373 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-09-23T18:30:46.374 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-09-23T18:30:46.374 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-09-23T18:30:46.374 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-09-23T18:30:46.374 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-09-23T18:30:46.374 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-09-23T18:30:46.374 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_suppress_logs": "true", 2023-09-23T18:30:46.374 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_vault_addr": "", 2023-09-23T18:30:46.374 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_vault_auth": "token", 2023-09-23T18:30:46.374 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_vault_namespace": "", 2023-09-23T18:30:46.374 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_vault_prefix": "", 2023-09-23T18:30:46.374 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-09-23T18:30:46.374 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-09-23T18:30:46.375 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-09-23T18:30:46.375 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-09-23T18:30:46.375 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_vault_token_file": "", 2023-09-23T18:30:46.375 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-09-23T18:30:46.375 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_curl_buffersize": "524288", 2023-09-23T18:30:46.375 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_curl_low_speed_limit": "1024", 2023-09-23T18:30:46.375 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_curl_low_speed_time": "300", 2023-09-23T18:30:46.375 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_curl_tcp_keepalive": "0", 2023-09-23T18:30:46.375 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-09-23T18:30:46.375 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-09-23T18:30:46.375 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-09-23T18:30:46.375 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-09-23T18:30:46.376 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-09-23T18:30:46.376 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_d3n_l1_fadvise": "4", 2023-09-23T18:30:46.376 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-09-23T18:30:46.376 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-09-23T18:30:46.376 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-09-23T18:30:46.376 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-09-23T18:30:46.376 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_data_log_changes_size": "1000", 2023-09-23T18:30:46.376 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_data_log_num_shards": "128", 2023-09-23T18:30:46.376 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-09-23T18:30:46.376 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_data_log_window": "30", 2023-09-23T18:30:46.376 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_data_notify_interval_msec": "0", 2023-09-23T18:30:46.376 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_data_sync_poll_interval": "20", 2023-09-23T18:30:46.376 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_data_sync_spawn_window": "20", 2023-09-23T18:30:46.377 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-09-23T18:30:46.377 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-09-23T18:30:46.377 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_default_data_log_backing": "fifo", 2023-09-23T18:30:46.377 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-09-23T18:30:46.377 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_default_region_info_oid": "default.region", 2023-09-23T18:30:46.377 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-09-23T18:30:46.377 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-09-23T18:30:46.377 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_defer_to_bucket_acls": "", 2023-09-23T18:30:46.377 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-09-23T18:30:46.377 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-09-23T18:30:46.377 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-09-23T18:30:46.377 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-09-23T18:30:46.378 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-09-23T18:30:46.378 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-09-23T18:30:46.378 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-09-23T18:30:46.378 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-09-23T18:30:46.378 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_dmclock_data_res": "500.000000", 2023-09-23T18:30:46.378 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-09-23T18:30:46.378 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-09-23T18:30:46.378 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-09-23T18:30:46.378 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-09-23T18:30:46.378 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_dns_name": "", 2023-09-23T18:30:46.378 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_dns_s3website_name": "", 2023-09-23T18:30:46.378 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_dynamic_resharding": "true", 2023-09-23T18:30:46.378 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-09-23T18:30:46.379 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_enable_gc_threads": "true", 2023-09-23T18:30:46.379 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_enable_lc_threads": "true", 2023-09-23T18:30:46.379 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_enable_ops_log": "false", 2023-09-23T18:30:46.379 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_enable_quota_threads": "true", 2023-09-23T18:30:46.379 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_enable_static_website": "false", 2023-09-23T18:30:46.379 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_enable_usage_log": "false", 2023-09-23T18:30:46.379 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_enforce_swift_acls": "true", 2023-09-23T18:30:46.379 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_exit_timeout_secs": "120", 2023-09-23T18:30:46.379 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_expose_bucket": "false", 2023-09-23T18:30:46.379 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_extended_http_attrs": "", 2023-09-23T18:30:46.379 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_filter": "none", 2023-09-23T18:30:46.379 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-09-23T18:30:46.379 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_frontends": "beast port=7480", 2023-09-23T18:30:46.380 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_gc_max_concurrent_io": "10", 2023-09-23T18:30:46.380 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_gc_max_deferred": "50", 2023-09-23T18:30:46.380 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-09-23T18:30:46.380 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_gc_max_objs": "32", 2023-09-23T18:30:46.380 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_gc_max_queue_size": "134213632", 2023-09-23T18:30:46.380 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_gc_max_trim_chunk": "16", 2023-09-23T18:30:46.380 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_gc_obj_min_wait": "7200", 2023-09-23T18:30:46.380 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_gc_processor_max_time": "3600", 2023-09-23T18:30:46.380 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_gc_processor_period": "3600", 2023-09-23T18:30:46.380 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-09-23T18:30:46.380 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_get_obj_window_size": "16777216", 2023-09-23T18:30:46.380 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_healthcheck_disabling_path": "", 2023-09-23T18:30:46.381 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_ignore_get_invalid_range": "false", 2023-09-23T18:30:46.381 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_init_timeout": "300", 2023-09-23T18:30:46.381 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-09-23T18:30:46.381 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-09-23T18:30:46.381 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-09-23T18:30:46.381 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-09-23T18:30:46.381 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_keystone_admin_domain": "", 2023-09-23T18:30:46.381 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_keystone_admin_password": "", 2023-09-23T18:30:46.381 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_keystone_admin_password_path": "", 2023-09-23T18:30:46.381 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_keystone_admin_project": "", 2023-09-23T18:30:46.381 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_keystone_admin_tenant": "", 2023-09-23T18:30:46.381 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_keystone_admin_token": "", 2023-09-23T18:30:46.381 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_keystone_admin_token_path": "", 2023-09-23T18:30:46.382 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_keystone_admin_user": "", 2023-09-23T18:30:46.382 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_keystone_api_version": "2", 2023-09-23T18:30:46.382 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_keystone_barbican_domain": "", 2023-09-23T18:30:46.382 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_keystone_barbican_password": "", 2023-09-23T18:30:46.382 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_keystone_barbican_project": "", 2023-09-23T18:30:46.382 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_keystone_barbican_tenant": "", 2023-09-23T18:30:46.382 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_keystone_barbican_user": "", 2023-09-23T18:30:46.382 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-09-23T18:30:46.382 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_keystone_implicit_tenants": "false", 2023-09-23T18:30:46.382 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-09-23T18:30:46.382 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_keystone_service_token_enabled": "false", 2023-09-23T18:30:46.382 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_keystone_token_cache_size": "10000", 2023-09-23T18:30:46.383 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_keystone_url": "", 2023-09-23T18:30:46.383 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_keystone_verify_ssl": "true", 2023-09-23T18:30:46.383 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_lc_debug_interval": "-1", 2023-09-23T18:30:46.383 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_lc_lock_max_time": "90", 2023-09-23T18:30:46.383 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_lc_max_objs": "32", 2023-09-23T18:30:46.383 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_lc_max_rules": "1000", 2023-09-23T18:30:46.383 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_lc_max_worker": "3", 2023-09-23T18:30:46.383 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_lc_max_wp_worker": "3", 2023-09-23T18:30:46.383 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_lc_thread_delay": "0", 2023-09-23T18:30:46.383 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-09-23T18:30:46.383 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_ldap_dnattr": "uid", 2023-09-23T18:30:46.383 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-09-23T18:30:46.383 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_ldap_searchfilter": "", 2023-09-23T18:30:46.384 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-09-23T18:30:46.384 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_ldap_uri": "ldaps://", 2023-09-23T18:30:46.384 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-09-23T18:30:46.384 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-09-23T18:30:46.384 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-09-23T18:30:46.384 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_log_http_headers": "", 2023-09-23T18:30:46.384 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_log_nonexistent_bucket": "false", 2023-09-23T18:30:46.384 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-09-23T18:30:46.384 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_log_object_name_utc": "false", 2023-09-23T18:30:46.384 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-09-23T18:30:46.384 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_max_attr_name_len": "0", 2023-09-23T18:30:46.385 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_max_attr_size": "0", 2023-09-23T18:30:46.385 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_max_attrs_num_in_req": "0", 2023-09-23T18:30:46.385 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_max_chunk_size": "4194304", 2023-09-23T18:30:46.385 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_max_concurrent_requests": "1024", 2023-09-23T18:30:46.385 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-09-23T18:30:46.385 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_max_dynamic_shards": "1999", 2023-09-23T18:30:46.385 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_max_listing_results": "1000", 2023-09-23T18:30:46.385 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_max_notify_retries": "10", 2023-09-23T18:30:46.385 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_max_objs_per_shard": "100000", 2023-09-23T18:30:46.385 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_max_put_param_size": "1048576", 2023-09-23T18:30:46.385 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_max_put_size": "5368709120", 2023-09-23T18:30:46.385 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_max_slo_entries": "1000", 2023-09-23T18:30:46.386 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_md_log_max_shards": "64", 2023-09-23T18:30:46.386 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_md_notify_interval_msec": "200", 2023-09-23T18:30:46.386 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_meta_sync_poll_interval": "20", 2023-09-23T18:30:46.386 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_meta_sync_spawn_window": "20", 2023-09-23T18:30:46.386 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-09-23T18:30:46.386 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_mp_lock_max_time": "600", 2023-09-23T18:30:46.386 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-09-23T18:30:46.386 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_multipart_min_part_size": "5242880", 2023-09-23T18:30:46.386 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-09-23T18:30:46.386 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-09-23T18:30:46.386 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_nfs_fhcache_size": "2017", 2023-09-23T18:30:46.386 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-09-23T18:30:46.387 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-09-23T18:30:46.387 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_nfs_lru_lanes": "5", 2023-09-23T18:30:46.387 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_nfs_max_gc": "300", 2023-09-23T18:30:46.387 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-09-23T18:30:46.387 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_nfs_run_gc_threads": "false", 2023-09-23T18:30:46.387 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_nfs_run_lc_threads": "false", 2023-09-23T18:30:46.387 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_nfs_run_quota_threads": "false", 2023-09-23T18:30:46.387 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_nfs_run_sync_thread": "false", 2023-09-23T18:30:46.387 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-09-23T18:30:46.387 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-09-23T18:30:46.387 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_num_async_rados_threads": "32", 2023-09-23T18:30:46.387 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_num_control_oids": "8", 2023-09-23T18:30:46.388 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_numa_node": "-1", 2023-09-23T18:30:46.388 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_obj_stripe_size": "4194304", 2023-09-23T18:30:46.388 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-09-23T18:30:46.388 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_objexp_chunk_size": "100", 2023-09-23T18:30:46.388 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_objexp_gc_interval": "600", 2023-09-23T18:30:46.388 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_objexp_hints_num_shards": "127", 2023-09-23T18:30:46.388 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-09-23T18:30:46.388 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-09-23T18:30:46.388 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_op_thread_timeout": "600", 2023-09-23T18:30:46.388 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_op_tracing": "false", 2023-09-23T18:30:46.388 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_opa_token": "", 2023-09-23T18:30:46.388 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_opa_url": "", 2023-09-23T18:30:46.388 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_opa_verify_ssl": "true", 2023-09-23T18:30:46.389 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-09-23T18:30:46.389 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-09-23T18:30:46.389 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_ops_log_rados": "false", 2023-09-23T18:30:46.389 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_ops_log_socket_path": "", 2023-09-23T18:30:46.389 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-09-23T18:30:46.389 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-09-23T18:30:46.389 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-09-23T18:30:46.389 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_period_push_interval": "2.000000", 2023-09-23T18:30:46.389 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_period_push_interval_max": "30.000000", 2023-09-23T18:30:46.389 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_period_root_pool": ".rgw.root", 2023-09-23T18:30:46.389 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-09-23T18:30:46.389 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_print_continue": "true", 2023-09-23T18:30:46.390 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_print_prohibited_content_length": "false", 2023-09-23T18:30:46.390 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-09-23T18:30:46.390 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-09-23T18:30:46.390 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-09-23T18:30:46.390 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-09-23T18:30:46.390 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_rados_tracing": "false", 2023-09-23T18:30:46.390 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_realm": "", 2023-09-23T18:30:46.390 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_realm_id": "", 2023-09-23T18:30:46.390 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-09-23T18:30:46.390 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_region": "", 2023-09-23T18:30:46.390 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_region_root_pool": ".rgw.root", 2023-09-23T18:30:46.390 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_relaxed_region_enforcement": "false", 2023-09-23T18:30:46.391 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-09-23T18:30:46.391 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-09-23T18:30:46.391 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_request_uri": "", 2023-09-23T18:30:46.391 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_reshard_batch_size": "64", 2023-09-23T18:30:46.391 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-09-23T18:30:46.391 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_reshard_max_aio": "128", 2023-09-23T18:30:46.391 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_reshard_num_logs": "16", 2023-09-23T18:30:46.391 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_reshard_thread_interval": "600", 2023-09-23T18:30:46.391 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_resolve_cname": "false", 2023-09-23T18:30:46.391 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_rest_getusage_op_compat": "false", 2023-09-23T18:30:46.391 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_run_sync_thread": "true", 2023-09-23T18:30:46.391 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-09-23T18:30:46.392 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_s3_auth_use_keystone": "false", 2023-09-23T18:30:46.392 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_s3_auth_use_ldap": "false", 2023-09-23T18:30:46.392 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_s3_auth_use_rados": "true", 2023-09-23T18:30:46.392 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_s3_auth_use_sts": "false", 2023-09-23T18:30:46.392 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-09-23T18:30:46.392 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_s3_success_create_obj_status": "0", 2023-09-23T18:30:46.392 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-09-23T18:30:46.392 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_scheduler_type": "throttler", 2023-09-23T18:30:46.392 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_script_uri": "", 2023-09-23T18:30:46.392 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_service_provider_name": "", 2023-09-23T18:30:46.392 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-09-23T18:30:46.392 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_sts_client_id": "", 2023-09-23T18:30:46.392 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_sts_client_secret": "", 2023-09-23T18:30:46.393 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_sts_entry": "sts", 2023-09-23T18:30:46.393 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_sts_key": "sts", 2023-09-23T18:30:46.393 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_sts_max_session_duration": "43200", 2023-09-23T18:30:46.393 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_sts_min_session_duration": "900", 2023-09-23T18:30:46.393 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_sts_token_introspection_url": "", 2023-09-23T18:30:46.393 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_swift_account_in_url": "false", 2023-09-23T18:30:46.393 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_swift_auth_entry": "auth", 2023-09-23T18:30:46.393 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_swift_auth_url": "", 2023-09-23T18:30:46.393 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_swift_custom_header": "", 2023-09-23T18:30:46.393 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_swift_enforce_content_length": "false", 2023-09-23T18:30:46.393 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_swift_need_stats": "true", 2023-09-23T18:30:46.393 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_swift_tenant_name": "", 2023-09-23T18:30:46.394 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_swift_token_expiration": "86400", 2023-09-23T18:30:46.394 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_swift_url": "", 2023-09-23T18:30:46.394 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_swift_url_prefix": "swift", 2023-09-23T18:30:46.394 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_swift_versioning_enabled": "false", 2023-09-23T18:30:46.394 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-09-23T18:30:46.394 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-09-23T18:30:46.394 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_sync_lease_period": "120", 2023-09-23T18:30:46.394 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-09-23T18:30:46.394 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_sync_log_trim_interval": "1200", 2023-09-23T18:30:46.394 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-09-23T18:30:46.394 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-09-23T18:30:46.394 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-09-23T18:30:46.394 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_sync_obj_etag_verify": "false", 2023-09-23T18:30:46.395 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_sync_trace_history_size": "4096", 2023-09-23T18:30:46.395 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-09-23T18:30:46.395 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-09-23T18:30:46.395 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_thread_pool_size": "512", 2023-09-23T18:30:46.395 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_torrent_comment": "", 2023-09-23T18:30:46.395 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_torrent_createby": "", 2023-09-23T18:30:46.395 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_torrent_encoding": "", 2023-09-23T18:30:46.395 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_torrent_flag": "false", 2023-09-23T18:30:46.395 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_torrent_origin": "", 2023-09-23T18:30:46.395 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_torrent_sha_unit": "524288", 2023-09-23T18:30:46.395 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_torrent_tracker": "", 2023-09-23T18:30:46.395 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_trust_forwarded_https": "false", 2023-09-23T18:30:46.395 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-09-23T18:30:46.396 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_usage_log_tick_interval": "30", 2023-09-23T18:30:46.396 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_usage_max_shards": "32", 2023-09-23T18:30:46.396 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_usage_max_user_shards": "1", 2023-09-23T18:30:46.396 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_use_opa_authz": "false", 2023-09-23T18:30:46.396 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-09-23T18:30:46.396 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_user_default_quota_max_size": "-1", 2023-09-23T18:30:46.396 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_user_max_buckets": "1000", 2023-09-23T18:30:46.396 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_user_policies_max_num": "100", 2023-09-23T18:30:46.396 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-09-23T18:30:46.396 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-09-23T18:30:46.396 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_user_quota_sync_interval": "86400", 2023-09-23T18:30:46.396 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-09-23T18:30:46.397 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_user_unique_email": "true", 2023-09-23T18:30:46.397 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_verify_ssl": "true", 2023-09-23T18:30:46.397 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_website_routing_rules_max_num": "50", 2023-09-23T18:30:46.397 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_zone": "", 2023-09-23T18:30:46.397 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_zone_id": "", 2023-09-23T18:30:46.397 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-09-23T18:30:46.397 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_zonegroup": "", 2023-09-23T18:30:46.397 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_zonegroup_id": "", 2023-09-23T18:30:46.397 INFO:teuthology.orchestra.run.smithi098.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-09-23T18:30:46.397 INFO:teuthology.orchestra.run.smithi098.stdout: "rgwlc_auto_session_clear": "true", 2023-09-23T18:30:46.397 INFO:teuthology.orchestra.run.smithi098.stdout: "rgwlc_skip_bucket_step": "false", 2023-09-23T18:30:46.397 INFO:teuthology.orchestra.run.smithi098.stdout: "rocksdb_block_size": "4096", 2023-09-23T18:30:46.397 INFO:teuthology.orchestra.run.smithi098.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-09-23T18:30:46.398 INFO:teuthology.orchestra.run.smithi098.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-09-23T18:30:46.398 INFO:teuthology.orchestra.run.smithi098.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-09-23T18:30:46.398 INFO:teuthology.orchestra.run.smithi098.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-09-23T18:30:46.398 INFO:teuthology.orchestra.run.smithi098.stdout: "rocksdb_cache_shard_bits": "4", 2023-09-23T18:30:46.398 INFO:teuthology.orchestra.run.smithi098.stdout: "rocksdb_cache_size": "536870912", 2023-09-23T18:30:46.398 INFO:teuthology.orchestra.run.smithi098.stdout: "rocksdb_cache_type": "binned_lru", 2023-09-23T18:30:46.398 INFO:teuthology.orchestra.run.smithi098.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-09-23T18:30:46.398 INFO:teuthology.orchestra.run.smithi098.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-09-23T18:30:46.398 INFO:teuthology.orchestra.run.smithi098.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-09-23T18:30:46.398 INFO:teuthology.orchestra.run.smithi098.stdout: "rocksdb_collect_compaction_stats": "false", 2023-09-23T18:30:46.398 INFO:teuthology.orchestra.run.smithi098.stdout: "rocksdb_collect_extended_stats": "false", 2023-09-23T18:30:46.398 INFO:teuthology.orchestra.run.smithi098.stdout: "rocksdb_collect_memory_stats": "false", 2023-09-23T18:30:46.399 INFO:teuthology.orchestra.run.smithi098.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-09-23T18:30:46.399 INFO:teuthology.orchestra.run.smithi098.stdout: "rocksdb_index_type": "binary_search", 2023-09-23T18:30:46.399 INFO:teuthology.orchestra.run.smithi098.stdout: "rocksdb_log_to_ceph_log": "true", 2023-09-23T18:30:46.399 INFO:teuthology.orchestra.run.smithi098.stdout: "rocksdb_metadata_block_size": "4096", 2023-09-23T18:30:46.399 INFO:teuthology.orchestra.run.smithi098.stdout: "rocksdb_partition_filters": "false", 2023-09-23T18:30:46.399 INFO:teuthology.orchestra.run.smithi098.stdout: "rocksdb_perf": "false", 2023-09-23T18:30:46.399 INFO:teuthology.orchestra.run.smithi098.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-09-23T18:30:46.399 INFO:teuthology.orchestra.run.smithi098.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-09-23T18:30:46.399 INFO:teuthology.orchestra.run.smithi098.stdout: "rotating_keys_renewal_timeout": "10", 2023-09-23T18:30:46.399 INFO:teuthology.orchestra.run.smithi098.stdout: "run_dir": "/var/run/ceph", 2023-09-23T18:30:46.399 INFO:teuthology.orchestra.run.smithi098.stdout: "seastore_block_create": "true", 2023-09-23T18:30:46.399 INFO:teuthology.orchestra.run.smithi098.stdout: "seastore_cache_lru_size": "67108864", 2023-09-23T18:30:46.400 INFO:teuthology.orchestra.run.smithi098.stdout: "seastore_cbjournal_size": "5368709120", 2023-09-23T18:30:46.400 INFO:teuthology.orchestra.run.smithi098.stdout: "seastore_default_max_object_size": "16777216", 2023-09-23T18:30:46.400 INFO:teuthology.orchestra.run.smithi098.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-09-23T18:30:46.400 INFO:teuthology.orchestra.run.smithi098.stdout: "seastore_device_size": "53687091200", 2023-09-23T18:30:46.400 INFO:teuthology.orchestra.run.smithi098.stdout: "seastore_journal_batch_capacity": "16", 2023-09-23T18:30:46.400 INFO:teuthology.orchestra.run.smithi098.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-09-23T18:30:46.400 INFO:teuthology.orchestra.run.smithi098.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-09-23T18:30:46.400 INFO:teuthology.orchestra.run.smithi098.stdout: "seastore_journal_iodepth_limit": "5", 2023-09-23T18:30:46.400 INFO:teuthology.orchestra.run.smithi098.stdout: "seastore_main_device_type": "SSD", 2023-09-23T18:30:46.400 INFO:teuthology.orchestra.run.smithi098.stdout: "seastore_max_concurrent_transactions": "8", 2023-09-23T18:30:46.400 INFO:teuthology.orchestra.run.smithi098.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-09-23T18:30:46.400 INFO:teuthology.orchestra.run.smithi098.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-09-23T18:30:46.401 INFO:teuthology.orchestra.run.smithi098.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-09-23T18:30:46.401 INFO:teuthology.orchestra.run.smithi098.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-09-23T18:30:46.401 INFO:teuthology.orchestra.run.smithi098.stdout: "seastore_segment_size": "67108864", 2023-09-23T18:30:46.401 INFO:teuthology.orchestra.run.smithi098.stdout: "set_keepcaps": "false", 2023-09-23T18:30:46.401 INFO:teuthology.orchestra.run.smithi098.stdout: "setgroup": "ceph", 2023-09-23T18:30:46.401 INFO:teuthology.orchestra.run.smithi098.stdout: "setuser": "ceph", 2023-09-23T18:30:46.401 INFO:teuthology.orchestra.run.smithi098.stdout: "setuser_match_path": "", 2023-09-23T18:30:46.401 INFO:teuthology.orchestra.run.smithi098.stdout: "target_max_misplaced_ratio": "0.050000", 2023-09-23T18:30:46.401 INFO:teuthology.orchestra.run.smithi098.stdout: "thp": "false", 2023-09-23T18:30:46.401 INFO:teuthology.orchestra.run.smithi098.stdout: "threadpool_default_timeout": "60", 2023-09-23T18:30:46.401 INFO:teuthology.orchestra.run.smithi098.stdout: "threadpool_empty_queue_max_wait": "2", 2023-09-23T18:30:46.401 INFO:teuthology.orchestra.run.smithi098.stdout: "throttler_perf_counter": "true" 2023-09-23T18:30:46.401 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:30:46.402 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:30:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:30:46] "GET /metrics HTTP/1.1" 200 34094 "" "Prometheus/2.33.4" 2023-09-23T18:30:46.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:46 smithi098 bash[76817]: cluster 2023-09-23T18:30:45.650798+0000 mgr.y (mgr.26054) 363 : cluster 0 pgmap v296: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:46.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:46 smithi098 bash[79516]: cluster 2023-09-23T18:30:45.650798+0000 mgr.y (mgr.26054) 363 : cluster 0 pgmap v296: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:47.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:46 smithi173 bash[59422]: cluster 2023-09-23T18:30:45.650798+0000 mgr.y (mgr.26054) 363 : cluster 0 pgmap v296: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:47.042 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-09-23T18:30:49.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:48 smithi173 bash[59422]: cluster 2023-09-23T18:30:47.651576+0000 mgr.y (mgr.26054) 364 : cluster 0 pgmap v297: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:49.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:48 smithi098 bash[76817]: cluster 2023-09-23T18:30:47.651576+0000 mgr.y (mgr.26054) 364 : cluster 0 pgmap v297: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:49.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:48 smithi098 bash[79516]: cluster 2023-09-23T18:30:47.651576+0000 mgr.y (mgr.26054) 364 : cluster 0 pgmap v297: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:50.690 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:50 smithi098 bash[76817]: cluster 2023-09-23T18:30:49.652404+0000 mgr.y (mgr.26054) 365 : cluster 0 pgmap v298: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:50.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:50 smithi098 bash[79516]: cluster 2023-09-23T18:30:49.652404+0000 mgr.y (mgr.26054) 365 : cluster 0 pgmap v298: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:50.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:50 smithi173 bash[59422]: cluster 2023-09-23T18:30:49.652404+0000 mgr.y (mgr.26054) 365 : cluster 0 pgmap v298: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:51.818 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:30:52.980 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:52 smithi098 bash[76817]: cluster 2023-09-23T18:30:51.653433+0000 mgr.y (mgr.26054) 366 : cluster 0 pgmap v299: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:52.980 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:52 smithi098 bash[79516]: cluster 2023-09-23T18:30:51.653433+0000 mgr.y (mgr.26054) 366 : cluster 0 pgmap v299: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:52 smithi173 bash[59422]: cluster 2023-09-23T18:30:51.653433+0000 mgr.y (mgr.26054) 366 : cluster 0 pgmap v299: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:54.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:54 smithi098 bash[79516]: cluster 2023-09-23T18:30:53.654214+0000 mgr.y (mgr.26054) 367 : cluster 0 pgmap v300: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:54.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:54 smithi098 bash[79516]: audit 2023-09-23T18:30:54.577902+0000 mon.a (mon.0) 514 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:30:54.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:54 smithi098 bash[76817]: cluster 2023-09-23T18:30:53.654214+0000 mgr.y (mgr.26054) 367 : cluster 0 pgmap v300: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:54.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:54 smithi098 bash[76817]: audit 2023-09-23T18:30:54.577902+0000 mon.a (mon.0) 514 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:30:54.958 INFO:teuthology.orchestra.run.smithi098.stdout:mon.a: {} 2023-09-23T18:30:54.977 INFO:teuthology.orchestra.run.smithi098.stdout:mon.b: {} 2023-09-23T18:30:54.995 INFO:teuthology.orchestra.run.smithi098.stdout:mon.c: {} 2023-09-23T18:30:54.997 INFO:teuthology.orchestra.run.smithi098.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T18:30:54.997 INFO:teuthology.orchestra.run.smithi098.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T18:30:54.997 INFO:teuthology.orchestra.run.smithi098.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T18:30:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:54 smithi173 bash[59422]: cluster 2023-09-23T18:30:53.654214+0000 mgr.y (mgr.26054) 367 : cluster 0 pgmap v300: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:54 smithi173 bash[59422]: audit 2023-09-23T18:30:54.577902+0000 mon.a (mon.0) 514 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:30:55.813 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-09-23T18:30:55.813 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-09-23T18:30:55.814 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T18:30:55.816 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T18:30:55.816 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-09-23T18:30:55.816 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-09-23T18:30:55.817 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-09-23T18:30:55.817 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T18:30:55.817 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:30:55.817 INFO:tasks.thrashosds.thrasher:Killing osd 1, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T18:30:55.817 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-09-23T18:30:55.817 DEBUG:teuthology.orchestra.run.smithi098:> sudo systemctl stop ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T18:30:55.818 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-09-23T18:30:55.818 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:30:55.818 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-09-23T18:30:55.819 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:30:55.819 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-09-23T18:30:55.819 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T18:30:55.820 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-09-23T18:30:55.820 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:30:55.854 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:30:55.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:55.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:30:55.957 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:30:55.987 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:30:55.988 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:30:55 smithi098 systemd[1]: Stopping Ceph osd.1 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T18:30:55.989 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:55 smithi098 bash[79516]: audit 2023-09-23T18:30:54.944056+0000 mon.c (mon.2) 47 : audit 0 from='client.? 172.21.15.98:0/1707905045' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T18:30:55.989 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:55 smithi098 bash[76817]: audit 2023-09-23T18:30:54.944056+0000 mon.c (mon.2) 47 : audit 0 from='client.? 172.21.15.98:0/1707905045' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T18:30:55.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:55.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:30:56.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:55 smithi173 bash[59422]: audit 2023-09-23T18:30:54.944056+0000 mon.c (mon.2) 47 : audit 0 from='client.? 172.21.15.98:0/1707905045' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T18:30:56.091 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:30:56.120 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:30:56.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:56.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:30:56.223 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:30:56.253 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:30:56.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:56.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:30:56.357 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:30:56.369 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:30:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:30:56] "GET /metrics HTTP/1.1" 200 34094 "" "Prometheus/2.33.4" 2023-09-23T18:30:56.369 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:30:56 smithi098 bash[88087]: debug 2023-09-23T18:30:55.996+0000 7fdd0d986700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T18:30:56.369 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:30:56 smithi098 bash[88087]: debug 2023-09-23T18:30:55.996+0000 7fdd0d986700 -1 osd.1 104 *** Got signal Terminated *** 2023-09-23T18:30:56.369 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:30:56 smithi098 bash[88087]: debug 2023-09-23T18:30:55.996+0000 7fdd0d986700 -1 osd.1 104 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T18:30:56.386 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:30:56.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:56.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:30:56.490 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:30:56.526 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:30:56.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:56.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:30:56.630 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:30:56.662 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:30:56.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:56.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:30:56.950 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:30:56.980 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:30:56.982 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:56 smithi098 bash[79516]: cluster 2023-09-23T18:30:55.655043+0000 mgr.y (mgr.26054) 368 : cluster 0 pgmap v301: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:56.982 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:56 smithi098 bash[79516]: cluster 2023-09-23T18:30:56.002329+0000 mon.a (mon.0) 515 : cluster 1 osd.1 marked itself down and dead 2023-09-23T18:30:56.982 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:56 smithi098 bash[76817]: cluster 2023-09-23T18:30:55.655043+0000 mgr.y (mgr.26054) 368 : cluster 0 pgmap v301: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:56.982 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:56 smithi098 bash[76817]: cluster 2023-09-23T18:30:56.002329+0000 mon.a (mon.0) 515 : cluster 1 osd.1 marked itself down and dead 2023-09-23T18:30:56.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:56.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:30:57.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:56 smithi173 bash[59422]: cluster 2023-09-23T18:30:55.655043+0000 mgr.y (mgr.26054) 368 : cluster 0 pgmap v301: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:57.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:56 smithi173 bash[59422]: cluster 2023-09-23T18:30:56.002329+0000 mon.a (mon.0) 515 : cluster 1 osd.1 marked itself down and dead 2023-09-23T18:30:57.085 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:30:57.115 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:30:57.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:57.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:30:57.219 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:30:57.248 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:30:57.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:57.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:30:57.352 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:30:57.385 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:30:57.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:57.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:30:57.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:30:57.520 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:30:57.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:57.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:30:57.623 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:30:57.657 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:30:57.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:57.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:30:57.760 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:30:57.782 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:30:57.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:57.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:30:57.885 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:30:57.896 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:57 smithi173 bash[59422]: cluster 2023-09-23T18:30:56.723828+0000 mon.a (mon.0) 516 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:30:57.896 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:57 smithi173 bash[59422]: cluster 2023-09-23T18:30:56.738991+0000 osd.4 (osd.4) 32 : cluster 3 failed to encode map e105 with expected crc 2023-09-23T18:30:57.896 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:57 smithi173 bash[59422]: cluster 2023-09-23T18:30:56.740915+0000 mon.a (mon.0) 517 : cluster 0 osdmap e105: 8 total, 7 up, 8 in 2023-09-23T18:30:57.897 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:57 smithi173 bash[59422]: cluster 2023-09-23T18:30:56.743119+0000 osd.6 (osd.6) 18 : cluster 3 failed to encode map e105 with expected crc 2023-09-23T18:30:57.897 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:57 smithi173 bash[59422]: cluster 2023-09-23T18:30:56.743242+0000 osd.4 (osd.4) 33 : cluster 3 failed to encode map e105 with expected crc 2023-09-23T18:30:57.897 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:57 smithi173 bash[59422]: cluster 2023-09-23T18:30:56.743507+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e105 with expected crc 2023-09-23T18:30:57.897 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:57 smithi173 bash[59422]: cluster 2023-09-23T18:30:56.751140+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e105 with expected crc 2023-09-23T18:30:57.914 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:30:57.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:57.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:30:58.018 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:30:58.035 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:30:57 smithi098 bash[102848]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-1 2023-09-23T18:30:58.035 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:30:57 smithi098 bash[103049]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-1 2023-09-23T18:30:58.036 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:57 smithi098 bash[76817]: cluster 2023-09-23T18:30:56.723828+0000 mon.a (mon.0) 516 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:30:58.036 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:57 smithi098 bash[76817]: cluster 2023-09-23T18:30:56.738991+0000 osd.4 (osd.4) 32 : cluster 3 failed to encode map e105 with expected crc 2023-09-23T18:30:58.036 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:57 smithi098 bash[76817]: cluster 2023-09-23T18:30:56.740915+0000 mon.a (mon.0) 517 : cluster 0 osdmap e105: 8 total, 7 up, 8 in 2023-09-23T18:30:58.036 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:57 smithi098 bash[76817]: cluster 2023-09-23T18:30:56.743119+0000 osd.6 (osd.6) 18 : cluster 3 failed to encode map e105 with expected crc 2023-09-23T18:30:58.036 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:57 smithi098 bash[76817]: cluster 2023-09-23T18:30:56.743242+0000 osd.4 (osd.4) 33 : cluster 3 failed to encode map e105 with expected crc 2023-09-23T18:30:58.036 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:57 smithi098 bash[76817]: cluster 2023-09-23T18:30:56.743507+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e105 with expected crc 2023-09-23T18:30:58.036 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:57 smithi098 bash[76817]: cluster 2023-09-23T18:30:56.751140+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e105 with expected crc 2023-09-23T18:30:58.037 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:57 smithi098 bash[79516]: cluster 2023-09-23T18:30:56.723828+0000 mon.a (mon.0) 516 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:30:58.037 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:57 smithi098 bash[79516]: cluster 2023-09-23T18:30:56.738991+0000 osd.4 (osd.4) 32 : cluster 3 failed to encode map e105 with expected crc 2023-09-23T18:30:58.037 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:57 smithi098 bash[79516]: cluster 2023-09-23T18:30:56.740915+0000 mon.a (mon.0) 517 : cluster 0 osdmap e105: 8 total, 7 up, 8 in 2023-09-23T18:30:58.037 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:57 smithi098 bash[79516]: cluster 2023-09-23T18:30:56.743119+0000 osd.6 (osd.6) 18 : cluster 3 failed to encode map e105 with expected crc 2023-09-23T18:30:58.037 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:57 smithi098 bash[79516]: cluster 2023-09-23T18:30:56.743242+0000 osd.4 (osd.4) 33 : cluster 3 failed to encode map e105 with expected crc 2023-09-23T18:30:58.037 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:57 smithi098 bash[79516]: cluster 2023-09-23T18:30:56.743507+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e105 with expected crc 2023-09-23T18:30:58.037 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:57 smithi098 bash[79516]: cluster 2023-09-23T18:30:56.751140+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e105 with expected crc 2023-09-23T18:30:58.050 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:30:58.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:58.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:30:58.154 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:30:58.190 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:30:58.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:58.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:30:58.294 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:30:58.325 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:30:58.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:58.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:30:58.430 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:30:58.462 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:30:58.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:58.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:30:58.567 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:30:58.597 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:30:58.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:58.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:30:58.700 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:30:58.732 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:30:58.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:58.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:30:58.836 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:30:58.867 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:30:58.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:58.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:30:58.971 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:30:58.997 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:58 smithi098 bash[76817]: cluster 2023-09-23T18:30:57.655638+0000 mgr.y (mgr.26054) 369 : cluster 0 pgmap v303: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:58.997 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:58 smithi098 bash[76817]: cluster 2023-09-23T18:30:57.742890+0000 mon.a (mon.0) 518 : cluster 0 osdmap e106: 8 total, 7 up, 8 in 2023-09-23T18:30:58.997 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:58 smithi098 bash[76817]: cluster 2023-09-23T18:30:57.744630+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:58.997 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:58 smithi098 bash[76817]: cluster 2023-09-23T18:30:57.744630+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:58.997 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:58 smithi098 bash[76817]: cluster 2023-09-23T18:30:57.744994+0000 osd.4 (osd.4) 34 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:58.997 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:58 smithi098 bash[76817]: cluster 2023-09-23T18:30:57.747438+0000 osd.4 (osd.4) 35 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:58.997 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:58 smithi098 bash[76817]: cluster 2023-09-23T18:30:57.748206+0000 osd.6 (osd.6) 19 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:58.998 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:58 smithi098 bash[76817]: cluster 2023-09-23T18:30:57.748789+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:58.998 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:58 smithi098 bash[76817]: cluster 2023-09-23T18:30:57.748942+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:58.998 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:58 smithi098 bash[76817]: cluster 2023-09-23T18:30:57.750110+0000 osd.4 (osd.4) 36 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:58.998 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:58 smithi098 bash[76817]: cluster 2023-09-23T18:30:57.750836+0000 osd.6 (osd.6) 20 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:58.998 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:30:58 smithi098 bash[76817]: cluster 2023-09-23T18:30:57.752313+0000 osd.4 (osd.4) 37 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:58.998 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:58 smithi098 bash[79516]: cluster 2023-09-23T18:30:57.655638+0000 mgr.y (mgr.26054) 369 : cluster 0 pgmap v303: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:58.998 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:58 smithi098 bash[79516]: cluster 2023-09-23T18:30:57.742890+0000 mon.a (mon.0) 518 : cluster 0 osdmap e106: 8 total, 7 up, 8 in 2023-09-23T18:30:58.998 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:58 smithi098 bash[79516]: cluster 2023-09-23T18:30:57.744630+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:58.998 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:58 smithi098 bash[79516]: cluster 2023-09-23T18:30:57.744630+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:58.998 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:58 smithi098 bash[79516]: cluster 2023-09-23T18:30:57.744994+0000 osd.4 (osd.4) 34 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:58.999 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:58 smithi098 bash[79516]: cluster 2023-09-23T18:30:57.747438+0000 osd.4 (osd.4) 35 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:58.999 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:58 smithi098 bash[79516]: cluster 2023-09-23T18:30:57.748206+0000 osd.6 (osd.6) 19 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:58.999 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:58 smithi098 bash[79516]: cluster 2023-09-23T18:30:57.748789+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:58.999 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:58 smithi098 bash[79516]: cluster 2023-09-23T18:30:57.748942+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:58.999 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:58 smithi098 bash[79516]: cluster 2023-09-23T18:30:57.750110+0000 osd.4 (osd.4) 36 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:58.999 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:58 smithi098 bash[79516]: cluster 2023-09-23T18:30:57.750836+0000 osd.6 (osd.6) 20 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:58.999 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:30:58 smithi098 bash[79516]: cluster 2023-09-23T18:30:57.752313+0000 osd.4 (osd.4) 37 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:59.006 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:30:59.008 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:58 smithi173 bash[59422]: cluster 2023-09-23T18:30:57.655638+0000 mgr.y (mgr.26054) 369 : cluster 0 pgmap v303: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T18:30:59.008 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:58 smithi173 bash[59422]: cluster 2023-09-23T18:30:57.742890+0000 mon.a (mon.0) 518 : cluster 0 osdmap e106: 8 total, 7 up, 8 in 2023-09-23T18:30:59.008 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:58 smithi173 bash[59422]: cluster 2023-09-23T18:30:57.744630+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:59.009 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:58 smithi173 bash[59422]: cluster 2023-09-23T18:30:57.744630+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:59.009 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:58 smithi173 bash[59422]: cluster 2023-09-23T18:30:57.744994+0000 osd.4 (osd.4) 34 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:59.009 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:58 smithi173 bash[59422]: cluster 2023-09-23T18:30:57.747438+0000 osd.4 (osd.4) 35 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:59.009 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:58 smithi173 bash[59422]: cluster 2023-09-23T18:30:57.748206+0000 osd.6 (osd.6) 19 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:59.009 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:58 smithi173 bash[59422]: cluster 2023-09-23T18:30:57.748789+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:59.009 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:58 smithi173 bash[59422]: cluster 2023-09-23T18:30:57.748942+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:59.009 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:58 smithi173 bash[59422]: cluster 2023-09-23T18:30:57.750110+0000 osd.4 (osd.4) 36 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:59.009 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:58 smithi173 bash[59422]: cluster 2023-09-23T18:30:57.750836+0000 osd.6 (osd.6) 20 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:59.009 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:30:58 smithi173 bash[59422]: cluster 2023-09-23T18:30:57.752313+0000 osd.4 (osd.4) 37 : cluster 3 failed to encode map e106 with expected crc 2023-09-23T18:30:59.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:59.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:30:59.111 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:30:59.142 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:30:59.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:59.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:30:59.246 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:30:59.273 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:30:59.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:59.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:30:59.377 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:30:59.406 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:30:59.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:59.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:30:59.510 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:30:59.550 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:30:59.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:59.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:30:59.654 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:30:59.689 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:30:59.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:59.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:30:59.794 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:30:59.831 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:30:59.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:59.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:30:59.936 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:30:59.965 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:30:59.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:30:59.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:00.070 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:00.109 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:00.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:00.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:00.213 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:00.248 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:00.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:00.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:00.288 DEBUG:teuthology.orchestra.run.smithi098:> sudo pkill -f 'journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.1.service' 2023-09-23T18:31:00.332 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T18:31:00.332 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-09-23T18:31:00.351 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:00.386 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:00.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:00.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:00.490 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:00.523 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:00.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:00.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:00.626 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:00.642 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:00 smithi173 bash[59422]: cluster 2023-09-23T18:30:59.656438+0000 mgr.y (mgr.26054) 370 : cluster 0 pgmap v305: 73 pgs: 2 active+undersized+degraded, 9 stale+active+clean, 1 active+undersized, 61 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T18:31:00.663 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:00.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:00.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:00.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:00 smithi098 bash[79516]: cluster 2023-09-23T18:30:59.656438+0000 mgr.y (mgr.26054) 370 : cluster 0 pgmap v305: 73 pgs: 2 active+undersized+degraded, 9 stale+active+clean, 1 active+undersized, 61 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T18:31:00.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:00 smithi098 bash[76817]: cluster 2023-09-23T18:30:59.656438+0000 mgr.y (mgr.26054) 370 : cluster 0 pgmap v305: 73 pgs: 2 active+undersized+degraded, 9 stale+active+clean, 1 active+undersized, 61 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T18:31:00.767 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:00.791 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:31:00.791 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:31:00.792 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:31:00.792 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:31:00.797 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:00.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:00.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:00.901 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:00.928 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:00.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:00.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:01.031 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:01.066 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:01.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:01.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:01.172 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:01.208 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:01.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:01.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:01.310 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:01.344 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:01.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:01.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:01.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:01 smithi098 bash[79516]: cluster 2023-09-23T18:31:00.405786+0000 mon.a (mon.0) 519 : cluster 3 Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:31:01.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:01 smithi098 bash[76817]: cluster 2023-09-23T18:31:00.405786+0000 mon.a (mon.0) 519 : cluster 3 Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:31:01.447 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:01.477 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:01.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:01.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:01.581 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:01.611 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:01.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:01.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:01.715 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:01.742 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:01.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:01.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:01.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:01 smithi173 bash[59422]: cluster 2023-09-23T18:31:00.405786+0000 mon.a (mon.0) 519 : cluster 3 Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:31:01.845 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:01.871 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:01.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:01.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:01.974 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:02.004 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:02.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:02.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:02.107 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:02.138 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:02.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:02.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:02.243 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:02.273 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:02.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:02.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:02.377 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:02.406 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:02.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:02.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:02.511 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:02.545 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:02.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:02.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:02.648 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:02.661 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:02 smithi173 bash[59422]: cluster 2023-09-23T18:31:01.657695+0000 mgr.y (mgr.26054) 371 : cluster 0 pgmap v306: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:02.683 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:02.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:02.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:02.688 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:02 smithi098 bash[79516]: cluster 2023-09-23T18:31:01.657695+0000 mgr.y (mgr.26054) 371 : cluster 0 pgmap v306: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:02.688 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:02 smithi098 bash[76817]: cluster 2023-09-23T18:31:01.657695+0000 mgr.y (mgr.26054) 371 : cluster 0 pgmap v306: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:02.788 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:02.815 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:02.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:02.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:02.918 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:02.952 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:02.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:02.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:03.055 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:03.083 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:03.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:03.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:03.187 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:03.215 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:03.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:03.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:03.319 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:03.343 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:03.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:03.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:03.447 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:03.476 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:03.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:03.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:03.579 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:03.611 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:03.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:03.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:03.714 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:03.745 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:03.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:03.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:03.849 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:03.879 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:03.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:03.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:03.983 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:04.012 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:04.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:04.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:04.116 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:04.149 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:04.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:04.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:04.253 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:04.287 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:04.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:04.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:04.392 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:04.419 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:04.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:04.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:04.522 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:04.554 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:04.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:04.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:04.657 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:04.685 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:04.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:04.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:04.789 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:04.812 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:04.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:04.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:04.914 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:04.925 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:04 smithi173 bash[59422]: cluster 2023-09-23T18:31:03.658631+0000 mgr.y (mgr.26054) 372 : cluster 0 pgmap v307: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:04.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:04 smithi098 bash[79516]: cluster 2023-09-23T18:31:03.658631+0000 mgr.y (mgr.26054) 372 : cluster 0 pgmap v307: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:04.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:04 smithi098 bash[76817]: cluster 2023-09-23T18:31:03.658631+0000 mgr.y (mgr.26054) 372 : cluster 0 pgmap v307: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:04.942 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:04.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:04.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:05.046 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:05.078 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:05.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:05.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:05.182 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:05.217 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:05.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:05.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:05.321 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:05.333 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T18:31:05.333 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:31:05.352 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:05.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:05.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:05.455 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:05.479 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:05.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:05.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:05.583 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:05.616 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:05.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:05.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:05.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:05.753 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:05.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:05.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:05.858 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:05.885 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:05.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:05.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:05.989 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:06.024 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:06.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:06.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:06.128 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:06.139 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:31:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:31:06] "GET /metrics HTTP/1.1" 200 34094 "" "Prometheus/2.33.4" 2023-09-23T18:31:06.158 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:06.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:06.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:06.262 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:06.295 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:06.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:06.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:06.398 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:06.430 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:06.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:06.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:06.534 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:06.562 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:06.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:06.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:06.665 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:06.698 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:06.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:06.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:06.802 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:06.835 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:06.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:06.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:06.939 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:06.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:06 smithi098 bash[76817]: cluster 2023-09-23T18:31:05.659544+0000 mgr.y (mgr.26054) 373 : cluster 0 pgmap v308: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:06.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:06 smithi098 bash[79516]: cluster 2023-09-23T18:31:05.659544+0000 mgr.y (mgr.26054) 373 : cluster 0 pgmap v308: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:06.952 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:06 smithi173 bash[59422]: cluster 2023-09-23T18:31:05.659544+0000 mgr.y (mgr.26054) 373 : cluster 0 pgmap v308: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:06.972 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:06.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:06.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:07.076 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:07.109 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:07.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:07.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:07.212 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:07.246 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:07.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:07.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:07.349 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:07.381 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:07.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:07.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:07.486 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:07.516 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:07.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:07.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:07.619 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:07.653 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:07.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:07.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:07.757 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:07.788 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:07.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:07.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:07.892 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:07.926 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:07.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:07.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:08.029 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:08.062 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:08.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:08.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:08.165 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:08.198 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:08.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:08.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:08.302 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:08.331 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:08.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:08.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:08.435 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:08.465 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:08.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:08.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:08.568 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:08.599 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:08.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:08.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:08.704 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:08.737 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:08.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:08.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:08.841 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:08.868 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:08.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:08.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:08.971 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:08.979 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:08 smithi098 bash[79516]: cluster 2023-09-23T18:31:07.660322+0000 mgr.y (mgr.26054) 374 : cluster 0 pgmap v309: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:08.979 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:08 smithi098 bash[76817]: cluster 2023-09-23T18:31:07.660322+0000 mgr.y (mgr.26054) 374 : cluster 0 pgmap v309: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:09.000 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:09.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:09.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:09.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:08 smithi173 bash[59422]: cluster 2023-09-23T18:31:07.660322+0000 mgr.y (mgr.26054) 374 : cluster 0 pgmap v309: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:09.103 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:09.135 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:09.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:09.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:09.239 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:09.265 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:09.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:09.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:09.369 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:09.405 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:09.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:09.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:09.492 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:31:09.508 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:09.533 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:31:09.541 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:09.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:09.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:09.572 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:31:09.611 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:31:09.628 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:31:09.629 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:31:09.637 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:31:09.644 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:09.674 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:09.676 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:31:09.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:09.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:09.698 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:31:09.699 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:31:09.699 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T18:31:09.699 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:31:09.699 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:31:09.699 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:31:09.699 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:31:09.699 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:31:09.699 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:31:09.728 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:09 smithi173 bash[59422]: cluster 2023-09-23T18:31:09.405758+0000 mon.a (mon.0) 520 : cluster 3 Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T18:31:09.728 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:09 smithi173 bash[59422]: audit 2023-09-23T18:31:09.467776+0000 mon.a (mon.0) 521 : audit 0 from='client.? 172.21.15.98:0/3676643268' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:31:09.728 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:09 smithi173 bash[59422]: audit 2023-09-23T18:31:09.583948+0000 mon.a (mon.0) 522 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:31:09.779 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:09.791 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:09 smithi098 bash[79516]: cluster 2023-09-23T18:31:09.405758+0000 mon.a (mon.0) 520 : cluster 3 Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T18:31:09.791 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:09 smithi098 bash[79516]: audit 2023-09-23T18:31:09.467776+0000 mon.a (mon.0) 521 : audit 0 from='client.? 172.21.15.98:0/3676643268' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:31:09.792 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:09 smithi098 bash[79516]: audit 2023-09-23T18:31:09.583948+0000 mon.a (mon.0) 522 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:31:09.792 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:09 smithi098 bash[79516]: audit 2023-09-23T18:31:09.585246+0000 mon.a (mon.0) 523 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:31:09.792 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:09 smithi098 bash[76817]: cluster 2023-09-23T18:31:09.405758+0000 mon.a (mon.0) 520 : cluster 3 Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T18:31:09.792 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:09 smithi098 bash[76817]: audit 2023-09-23T18:31:09.467776+0000 mon.a (mon.0) 521 : audit 0 from='client.? 172.21.15.98:0/3676643268' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:31:09.792 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:09 smithi098 bash[76817]: audit 2023-09-23T18:31:09.583948+0000 mon.a (mon.0) 522 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:31:09.792 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:09 smithi098 bash[76817]: audit 2023-09-23T18:31:09.585246+0000 mon.a (mon.0) 523 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:31:09.807 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:09.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:09.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:09.911 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:09.921 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:09 smithi173 bash[59422]: audit 2023-09-23T18:31:09.585246+0000 mon.a (mon.0) 523 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:31:09.941 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:09.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:09.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:10.044 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:10.063 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:10.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:10.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:10.167 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:10.198 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:10.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:10.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:10.301 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:10.334 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:10.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:10.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:10.437 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:10.466 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:10.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:10.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:10.569 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:10.596 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:10.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:10.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:10.699 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:10.733 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:10.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:10.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:10.779 INFO:teuthology.orchestra.run.smithi098.stderr:pool 'unique_pool_0' created 2023-09-23T18:31:10.779 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is set 2023-09-23T18:31:10.832 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:10 smithi098 bash[76817]: cluster 2023-09-23T18:31:09.660924+0000 mgr.y (mgr.26054) 375 : cluster 0 pgmap v310: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:10.833 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:10 smithi098 bash[76817]: audit 2023-09-23T18:31:09.762875+0000 mon.a (mon.0) 524 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:31:10.833 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:10 smithi098 bash[76817]: audit 2023-09-23T18:31:09.882621+0000 mon.c (mon.2) 48 : audit 1 from='client.? 172.21.15.98:0/1221643784' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:31:10.833 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:10 smithi098 bash[76817]: audit 2023-09-23T18:31:09.883070+0000 mon.a (mon.0) 525 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:31:10.833 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:10 smithi098 bash[76817]: audit 2023-09-23T18:31:09.926350+0000 mon.a (mon.0) 526 : audit 1 from='client.? 172.21.15.98:0/1613190346' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T18:31:10.833 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:10 smithi098 bash[79516]: cluster 2023-09-23T18:31:09.660924+0000 mgr.y (mgr.26054) 375 : cluster 0 pgmap v310: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:10.833 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:10 smithi098 bash[79516]: audit 2023-09-23T18:31:09.762875+0000 mon.a (mon.0) 524 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:31:10.833 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:10 smithi098 bash[79516]: audit 2023-09-23T18:31:09.882621+0000 mon.c (mon.2) 48 : audit 1 from='client.? 172.21.15.98:0/1221643784' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:31:10.834 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:10 smithi098 bash[79516]: audit 2023-09-23T18:31:09.883070+0000 mon.a (mon.0) 525 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:31:10.834 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:10 smithi098 bash[79516]: audit 2023-09-23T18:31:09.926350+0000 mon.a (mon.0) 526 : audit 1 from='client.? 172.21.15.98:0/1613190346' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T18:31:10.837 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:10.869 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:10.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:10.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:10.972 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:10.993 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:10.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:10.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:11.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:10 smithi173 bash[59422]: cluster 2023-09-23T18:31:09.660924+0000 mgr.y (mgr.26054) 375 : cluster 0 pgmap v310: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:11.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:10 smithi173 bash[59422]: audit 2023-09-23T18:31:09.762875+0000 mon.a (mon.0) 524 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:31:11.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:10 smithi173 bash[59422]: audit 2023-09-23T18:31:09.882621+0000 mon.c (mon.2) 48 : audit 1 from='client.? 172.21.15.98:0/1221643784' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:31:11.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:10 smithi173 bash[59422]: audit 2023-09-23T18:31:09.883070+0000 mon.a (mon.0) 525 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:31:11.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:10 smithi173 bash[59422]: audit 2023-09-23T18:31:09.926350+0000 mon.a (mon.0) 526 : audit 1 from='client.? 172.21.15.98:0/1613190346' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T18:31:11.035 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T18:31:11.079 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:31:11.080 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:31:11.097 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:11.131 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:11.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:11.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:11.235 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:11.268 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:11.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:11.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:11.372 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:11.400 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:11.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:11.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:11.503 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:11.530 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:11.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:11.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:11.633 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:11.659 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:11.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:11.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:11.763 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:11.792 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:11.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:11.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:11.896 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:11.920 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:11.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:11.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:11.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:11 smithi098 bash[79516]: cluster 2023-09-23T18:31:10.760259+0000 mon.a (mon.0) 527 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:31:11.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:11 smithi098 bash[79516]: cluster 2023-09-23T18:31:10.770013+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e107 with expected crc 2023-09-23T18:31:11.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:11 smithi098 bash[79516]: cluster 2023-09-23T18:31:10.770267+0000 osd.6 (osd.6) 21 : cluster 3 failed to encode map e107 with expected crc 2023-09-23T18:31:11.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:11 smithi098 bash[79516]: audit 2023-09-23T18:31:10.771949+0000 mon.a (mon.0) 528 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:31:11.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:11 smithi098 bash[79516]: audit 2023-09-23T18:31:10.772074+0000 mon.a (mon.0) 529 : audit 1 from='client.? 172.21.15.98:0/1613190346' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T18:31:11.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:11 smithi098 bash[79516]: cluster 2023-09-23T18:31:10.772168+0000 mon.a (mon.0) 530 : cluster 0 osdmap e107: 8 total, 7 up, 8 in 2023-09-23T18:31:11.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:11 smithi098 bash[76817]: cluster 2023-09-23T18:31:10.760259+0000 mon.a (mon.0) 527 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:31:11.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:11 smithi098 bash[76817]: cluster 2023-09-23T18:31:10.770013+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e107 with expected crc 2023-09-23T18:31:11.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:11 smithi098 bash[76817]: cluster 2023-09-23T18:31:10.770267+0000 osd.6 (osd.6) 21 : cluster 3 failed to encode map e107 with expected crc 2023-09-23T18:31:11.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:11 smithi098 bash[76817]: audit 2023-09-23T18:31:10.771949+0000 mon.a (mon.0) 528 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:31:11.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:11 smithi098 bash[76817]: audit 2023-09-23T18:31:10.772074+0000 mon.a (mon.0) 529 : audit 1 from='client.? 172.21.15.98:0/1613190346' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T18:31:11.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:11 smithi098 bash[76817]: cluster 2023-09-23T18:31:10.772168+0000 mon.a (mon.0) 530 : cluster 0 osdmap e107: 8 total, 7 up, 8 in 2023-09-23T18:31:12.023 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:12.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:11 smithi173 bash[59422]: cluster 2023-09-23T18:31:10.760259+0000 mon.a (mon.0) 527 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:31:12.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:11 smithi173 bash[59422]: cluster 2023-09-23T18:31:10.770013+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e107 with expected crc 2023-09-23T18:31:12.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:11 smithi173 bash[59422]: cluster 2023-09-23T18:31:10.770267+0000 osd.6 (osd.6) 21 : cluster 3 failed to encode map e107 with expected crc 2023-09-23T18:31:12.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:11 smithi173 bash[59422]: audit 2023-09-23T18:31:10.771949+0000 mon.a (mon.0) 528 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:31:12.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:11 smithi173 bash[59422]: audit 2023-09-23T18:31:10.772074+0000 mon.a (mon.0) 529 : audit 1 from='client.? 172.21.15.98:0/1613190346' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T18:31:12.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:11 smithi173 bash[59422]: cluster 2023-09-23T18:31:10.772168+0000 mon.a (mon.0) 530 : cluster 0 osdmap e107: 8 total, 7 up, 8 in 2023-09-23T18:31:12.050 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:12.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:12.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:12.154 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:12.185 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:12.186 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-09-23T18:31:12.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:12.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:12.290 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:12.322 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:12.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:12.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:12.426 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:12.455 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:12.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:12.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:12.558 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:12.591 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:12.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:12.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:12.694 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:12.728 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:12.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:12.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:12.833 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:12.846 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:12 smithi173 bash[59422]: cluster 2023-09-23T18:31:11.661484+0000 mgr.y (mgr.26054) 376 : cluster 0 pgmap v312: 89 pgs: 6 creating+peering, 10 unknown, 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:12.846 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:12 smithi173 bash[59422]: cluster 2023-09-23T18:31:11.778974+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T18:31:12.846 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:12 smithi173 bash[59422]: cluster 2023-09-23T18:31:11.780071+0000 mon.a (mon.0) 531 : cluster 0 osdmap e108: 8 total, 7 up, 8 in 2023-09-23T18:31:12.846 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:12 smithi173 bash[59422]: cluster 2023-09-23T18:31:11.781204+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T18:31:12.847 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:12 smithi173 bash[59422]: cluster 2023-09-23T18:31:11.781377+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T18:31:12.847 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:12 smithi173 bash[59422]: cluster 2023-09-23T18:31:11.787502+0000 osd.6 (osd.6) 22 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T18:31:12.847 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:12 smithi173 bash[59422]: cluster 2023-09-23T18:31:11.787601+0000 osd.4 (osd.4) 38 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T18:31:12.864 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:12.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:12.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:12.968 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:13.001 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:13.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:13.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:13.081 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:31:13.096 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:12 smithi098 bash[76817]: cluster 2023-09-23T18:31:11.661484+0000 mgr.y (mgr.26054) 376 : cluster 0 pgmap v312: 89 pgs: 6 creating+peering, 10 unknown, 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:13.096 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:12 smithi098 bash[76817]: cluster 2023-09-23T18:31:11.778974+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T18:31:13.096 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:12 smithi098 bash[76817]: cluster 2023-09-23T18:31:11.780071+0000 mon.a (mon.0) 531 : cluster 0 osdmap e108: 8 total, 7 up, 8 in 2023-09-23T18:31:13.096 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:12 smithi098 bash[76817]: cluster 2023-09-23T18:31:11.781204+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T18:31:13.096 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:12 smithi098 bash[76817]: cluster 2023-09-23T18:31:11.781377+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T18:31:13.096 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:12 smithi098 bash[76817]: cluster 2023-09-23T18:31:11.787502+0000 osd.6 (osd.6) 22 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T18:31:13.096 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:12 smithi098 bash[76817]: cluster 2023-09-23T18:31:11.787601+0000 osd.4 (osd.4) 38 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T18:31:13.097 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:12 smithi098 bash[79516]: cluster 2023-09-23T18:31:11.661484+0000 mgr.y (mgr.26054) 376 : cluster 0 pgmap v312: 89 pgs: 6 creating+peering, 10 unknown, 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:13.097 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:12 smithi098 bash[79516]: cluster 2023-09-23T18:31:11.778974+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T18:31:13.097 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:12 smithi098 bash[79516]: cluster 2023-09-23T18:31:11.780071+0000 mon.a (mon.0) 531 : cluster 0 osdmap e108: 8 total, 7 up, 8 in 2023-09-23T18:31:13.097 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:12 smithi098 bash[79516]: cluster 2023-09-23T18:31:11.781204+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T18:31:13.097 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:12 smithi098 bash[79516]: cluster 2023-09-23T18:31:11.781377+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T18:31:13.097 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:12 smithi098 bash[79516]: cluster 2023-09-23T18:31:11.787502+0000 osd.6 (osd.6) 22 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T18:31:13.097 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:12 smithi098 bash[79516]: cluster 2023-09-23T18:31:11.787601+0000 osd.4 (osd.4) 38 : cluster 3 failed to encode map e108 with expected crc 2023-09-23T18:31:13.104 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:13.136 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:13.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:13.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:13.241 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:13.275 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:13.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:13.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:13.378 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:13.410 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:13.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:13.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:13.514 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:13.548 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:13.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:13.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:13.652 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:13.687 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:13.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:13.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:13.790 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:13.820 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:13.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:13.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:13.924 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:13.956 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:13.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:13.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:14.061 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:14.072 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:13 smithi173 bash[59422]: cluster 2023-09-23T18:31:12.793213+0000 osd.7 (osd.7) 23 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T18:31:14.072 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:13 smithi173 bash[59422]: cluster 2023-09-23T18:31:12.795163+0000 mon.a (mon.0) 532 : cluster 0 osdmap e109: 8 total, 7 up, 8 in 2023-09-23T18:31:14.072 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:13 smithi173 bash[59422]: cluster 2023-09-23T18:31:13.088413+0000 osd.6 (osd.6) 23 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T18:31:14.073 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:13 smithi173 bash[59422]: cluster 2023-09-23T18:31:13.091086+0000 osd.6 (osd.6) 24 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T18:31:14.090 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:14.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:14.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:14.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:13 smithi098 bash[79516]: cluster 2023-09-23T18:31:12.793213+0000 osd.7 (osd.7) 23 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T18:31:14.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:13 smithi098 bash[79516]: cluster 2023-09-23T18:31:12.795163+0000 mon.a (mon.0) 532 : cluster 0 osdmap e109: 8 total, 7 up, 8 in 2023-09-23T18:31:14.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:13 smithi098 bash[79516]: cluster 2023-09-23T18:31:13.088413+0000 osd.6 (osd.6) 23 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T18:31:14.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:13 smithi098 bash[79516]: cluster 2023-09-23T18:31:13.091086+0000 osd.6 (osd.6) 24 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T18:31:14.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:13 smithi098 bash[76817]: cluster 2023-09-23T18:31:12.793213+0000 osd.7 (osd.7) 23 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T18:31:14.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:13 smithi098 bash[76817]: cluster 2023-09-23T18:31:12.795163+0000 mon.a (mon.0) 532 : cluster 0 osdmap e109: 8 total, 7 up, 8 in 2023-09-23T18:31:14.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:13 smithi098 bash[76817]: cluster 2023-09-23T18:31:13.088413+0000 osd.6 (osd.6) 23 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T18:31:14.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:13 smithi098 bash[76817]: cluster 2023-09-23T18:31:13.091086+0000 osd.6 (osd.6) 24 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T18:31:14.194 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:14.204 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:31:14.227 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:14.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:14.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:14.331 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:14.363 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:14.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:14.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:14.466 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:14.499 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:14.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:14.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:14.602 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:14.637 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:14.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:14.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:14.740 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:14.773 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:14.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:14.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:14.877 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:14.894 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:14 smithi173 bash[59422]: cluster 2023-09-23T18:31:13.662334+0000 mgr.y (mgr.26054) 377 : cluster 0 pgmap v315: 89 pgs: 6 creating+peering, 8 unknown, 5 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:14.913 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:14.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:14.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:15.017 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:15.046 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:15.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:15.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:15.150 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:15.183 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:15.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:15.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:15.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:14 smithi098 bash[76817]: cluster 2023-09-23T18:31:13.662334+0000 mgr.y (mgr.26054) 377 : cluster 0 pgmap v315: 89 pgs: 6 creating+peering, 8 unknown, 5 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:15.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:14 smithi098 bash[79516]: cluster 2023-09-23T18:31:13.662334+0000 mgr.y (mgr.26054) 377 : cluster 0 pgmap v315: 89 pgs: 6 creating+peering, 8 unknown, 5 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:15.289 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:15.318 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:15.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:15.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:15.422 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:15.452 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:15.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:15.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:15.556 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:15.587 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:15.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:15.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:15.691 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:15.723 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:15.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:15.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:15.827 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:15.860 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:15.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:15.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:15.888 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:31:15.963 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:15.977 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:15 smithi173 bash[59422]: cluster 2023-09-23T18:31:13.990926+0000 osd.4 (osd.4) 39 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T18:31:15.977 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:15 smithi173 bash[59422]: cluster 2023-09-23T18:31:13.993565+0000 osd.4 (osd.4) 40 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T18:31:15.997 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:15.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:16.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:16.086 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:15 smithi098 bash[76817]: cluster 2023-09-23T18:31:13.990926+0000 osd.4 (osd.4) 39 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T18:31:16.086 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:15 smithi098 bash[76817]: cluster 2023-09-23T18:31:13.993565+0000 osd.4 (osd.4) 40 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T18:31:16.087 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:15 smithi098 bash[79516]: cluster 2023-09-23T18:31:13.990926+0000 osd.4 (osd.4) 39 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T18:31:16.087 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:15 smithi098 bash[79516]: cluster 2023-09-23T18:31:13.993565+0000 osd.4 (osd.4) 40 : cluster 3 failed to encode map e109 with expected crc 2023-09-23T18:31:16.101 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:16.126 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:16.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:16.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:16.190 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:31:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:31:16] "GET /metrics HTTP/1.1" 200 34092 "" "Prometheus/2.33.4" 2023-09-23T18:31:16.230 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:16.258 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:16.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:16.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:16.361 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:16.394 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:16.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:16.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:16.498 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:16.527 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:16.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:16.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:16.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:16.665 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:16.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:16.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:16.768 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:16.799 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:16.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:16.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:16.902 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:16.903 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:31:16.905 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:31:16.931 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:16.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:16.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:17.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:16 smithi173 bash[59422]: cluster 2023-09-23T18:31:15.663344+0000 mgr.y (mgr.26054) 378 : cluster 0 pgmap v316: 89 pgs: 6 creating+peering, 5 active+undersized+degraded, 30 active+undersized, 48 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:17.034 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:17.059 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:17.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:17.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:17.162 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:17.170 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:16 smithi098 bash[76817]: cluster 2023-09-23T18:31:15.663344+0000 mgr.y (mgr.26054) 378 : cluster 0 pgmap v316: 89 pgs: 6 creating+peering, 5 active+undersized+degraded, 30 active+undersized, 48 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:17.170 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:16 smithi098 bash[79516]: cluster 2023-09-23T18:31:15.663344+0000 mgr.y (mgr.26054) 378 : cluster 0 pgmap v316: 89 pgs: 6 creating+peering, 5 active+undersized+degraded, 30 active+undersized, 48 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:17.191 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:17.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:17.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:17.294 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:17.324 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:17.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:17.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:17.428 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:17.460 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:17.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:17.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:17.564 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:17.601 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:17.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:17.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:17.705 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:17.741 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:17.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:17.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:17.843 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:17.872 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:17.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:17.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:17.976 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:18.001 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:18.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:18.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:18.104 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:18.136 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:18.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:18.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:18.240 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:18.265 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:18.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:18.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:18.369 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:18.400 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:18.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:18.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:18.504 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:18.529 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:18.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:18.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:18.632 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:18.664 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:18.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:18.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:18.769 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:18.791 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:18.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:18.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:18.894 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:18.929 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:18.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:18.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:19.032 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:19.064 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:19.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:19.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:19.167 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:19.180 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:18 smithi098 bash[76817]: cluster 2023-09-23T18:31:17.664279+0000 mgr.y (mgr.26054) 379 : cluster 0 pgmap v317: 89 pgs: 5 active+undersized+degraded, 32 active+undersized, 52 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:19.180 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:18 smithi098 bash[79516]: cluster 2023-09-23T18:31:17.664279+0000 mgr.y (mgr.26054) 379 : cluster 0 pgmap v317: 89 pgs: 5 active+undersized+degraded, 32 active+undersized, 52 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:19.198 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:19.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:19.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:19.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:18 smithi173 bash[59422]: cluster 2023-09-23T18:31:17.664279+0000 mgr.y (mgr.26054) 379 : cluster 0 pgmap v317: 89 pgs: 5 active+undersized+degraded, 32 active+undersized, 52 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:19.301 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:19.335 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:19.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:19.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:19.439 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:19.467 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:19.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:19.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:19.571 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:19.649 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:19.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:19.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:19.753 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:19.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:19.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:19.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:19.890 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:19.917 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:19.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:19.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:20.003 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:31:20.021 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:20.045 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:20.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:20.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:20.148 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:20.180 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:20.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:20.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:20.284 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:20.319 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:20.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:20.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:20.423 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:20.456 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:20.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:20.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:20.560 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:20.591 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:20.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:20.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:20.690 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:20 smithi098 bash[79516]: cluster 2023-09-23T18:31:19.664849+0000 mgr.y (mgr.26054) 380 : cluster 0 pgmap v318: 89 pgs: 5 active+undersized+degraded, 32 active+undersized, 52 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:20.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:20 smithi098 bash[76817]: cluster 2023-09-23T18:31:19.664849+0000 mgr.y (mgr.26054) 380 : cluster 0 pgmap v318: 89 pgs: 5 active+undersized+degraded, 32 active+undersized, 52 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:20.695 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:20.709 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:20 smithi173 bash[59422]: cluster 2023-09-23T18:31:19.664849+0000 mgr.y (mgr.26054) 380 : cluster 0 pgmap v318: 89 pgs: 5 active+undersized+degraded, 32 active+undersized, 52 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:20.727 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:20.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:20.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:20.831 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:20.857 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:20.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:20.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:20.961 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:20.995 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:20.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:20.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:21.099 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:21.120 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:21.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:21.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:21.223 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:21.245 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:21.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:21.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:21.348 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:21.380 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:21.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:21.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:21.483 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:21.511 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:21.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:21.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:21.614 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:21.646 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:21.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:21.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:21.749 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:21.776 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:21.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:21.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:21.879 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:21.905 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:21.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:21.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:22.008 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:22.033 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:22.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:22.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:22.136 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:22.165 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:22.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:22.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:22.268 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:22.299 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:22.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:22.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:22.403 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:22.433 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:22.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:22.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:22.537 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:22.567 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:22.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:22.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:22.670 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:22.701 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:22.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:22.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:22.805 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:22.826 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:22 smithi173 bash[59422]: cluster 2023-09-23T18:31:21.665886+0000 mgr.y (mgr.26054) 381 : cluster 0 pgmap v319: 89 pgs: 5 active+undersized+degraded, 32 active+undersized, 52 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:22.843 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:22.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:22.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:22.947 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:22.953 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:22 smithi098 bash[79516]: cluster 2023-09-23T18:31:21.665886+0000 mgr.y (mgr.26054) 381 : cluster 0 pgmap v319: 89 pgs: 5 active+undersized+degraded, 32 active+undersized, 52 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:22.953 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:22 smithi098 bash[76817]: cluster 2023-09-23T18:31:21.665886+0000 mgr.y (mgr.26054) 381 : cluster 0 pgmap v319: 89 pgs: 5 active+undersized+degraded, 32 active+undersized, 52 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:22.984 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:22.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:22.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:23.088 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:23.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:23.132 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:23.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:23.233 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:23.273 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:23.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:23.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:23.379 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:23.411 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:23.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:23.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:23.515 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:23.538 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:23.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:23.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:23.642 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:23.668 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:23.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:23.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:23.771 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:23.799 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:23.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:23.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:23.902 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:23.936 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:23.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:23.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:24.040 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:24.079 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:24.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:24.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:24.183 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:24.208 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:24.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:24.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:24.265 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:31:24.265 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:31:24.312 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:24.341 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:24.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:24.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:24.444 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:24.478 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:24.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:24.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:24.582 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:24.612 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:24.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:24.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:24.715 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:24.747 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:24.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:24.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:24.851 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:24.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:24.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:24.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:24.990 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:25.002 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:24 smithi173 bash[59422]: cluster 2023-09-23T18:31:23.666453+0000 mgr.y (mgr.26054) 382 : cluster 0 pgmap v320: 89 pgs: 5 active+undersized+degraded, 32 active+undersized, 52 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:25.002 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:24 smithi173 bash[59422]: audit 2023-09-23T18:31:24.586479+0000 mon.a (mon.0) 533 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:31:25.002 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:24 smithi173 bash[59422]: audit 2023-09-23T18:31:24.588140+0000 mon.a (mon.0) 534 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:31:25.020 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:25.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:25.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:25.074 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:24 smithi098 bash[79516]: cluster 2023-09-23T18:31:23.666453+0000 mgr.y (mgr.26054) 382 : cluster 0 pgmap v320: 89 pgs: 5 active+undersized+degraded, 32 active+undersized, 52 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:25.074 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:24 smithi098 bash[79516]: audit 2023-09-23T18:31:24.586479+0000 mon.a (mon.0) 533 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:31:25.074 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:24 smithi098 bash[79516]: audit 2023-09-23T18:31:24.588140+0000 mon.a (mon.0) 534 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:31:25.074 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:24 smithi098 bash[76817]: cluster 2023-09-23T18:31:23.666453+0000 mgr.y (mgr.26054) 382 : cluster 0 pgmap v320: 89 pgs: 5 active+undersized+degraded, 32 active+undersized, 52 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:25.075 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:24 smithi098 bash[76817]: audit 2023-09-23T18:31:24.586479+0000 mon.a (mon.0) 533 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:31:25.075 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:24 smithi098 bash[76817]: audit 2023-09-23T18:31:24.588140+0000 mon.a (mon.0) 534 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:31:25.124 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:25.159 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:25.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:25.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:25.262 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:25.294 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:25.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:25.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:25.397 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:25.421 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:25.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:25.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:25.524 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:25.546 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:25.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:25.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:25.649 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:25.680 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:25.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:25.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:25.784 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:25.821 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:25.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:25.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:25.924 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:25.961 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:25.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:25.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:26.065 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:26.097 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:26.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:26.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:26.202 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:26.227 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:26.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:26.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:26.332 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:26.362 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:26.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:26.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:26.440 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:31:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:31:26] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.33.4" 2023-09-23T18:31:26.449 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T18:31:26.465 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:26.499 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:26.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:26.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:26.603 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:26.638 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:26.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:26.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:26.741 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:26.750 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:26 smithi098 bash[79516]: cluster 2023-09-23T18:31:25.667471+0000 mgr.y (mgr.26054) 383 : cluster 0 pgmap v321: 89 pgs: 5 active+undersized+degraded, 32 active+undersized, 52 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:26.750 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:26 smithi098 bash[76817]: cluster 2023-09-23T18:31:25.667471+0000 mgr.y (mgr.26054) 383 : cluster 0 pgmap v321: 89 pgs: 5 active+undersized+degraded, 32 active+undersized, 52 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:26.764 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:26.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:26.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:26.868 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:26.889 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:26.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:26.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:26.993 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:27.006 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:26 smithi173 bash[59422]: cluster 2023-09-23T18:31:25.667471+0000 mgr.y (mgr.26054) 383 : cluster 0 pgmap v321: 89 pgs: 5 active+undersized+degraded, 32 active+undersized, 52 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:27.025 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:27.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:27.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:27.129 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:27.156 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:27.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:27.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:27.259 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:27.287 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:27.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:27.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:27.391 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:27.419 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:27.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:27.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:27.524 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:27.557 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:27.558 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:31:27.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:27.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:27.581 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:31:27.604 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:31:27.629 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:31:27.651 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:31:27.662 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:27.675 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:31:27.688 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:27.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:27.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:27.713 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:31:27.713 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:31:27.713 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T18:31:27.713 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:31:27.713 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:31:27.714 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:31:27.714 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:31:27.714 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:31:27.714 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:31:27.790 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:27.810 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:27.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:27.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:27.913 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:27.925 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:27 smithi173 bash[59422]: audit 2023-09-23T18:31:27.537374+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.98:0/2258194256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:31:27.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:27 smithi098 bash[76817]: audit 2023-09-23T18:31:27.537374+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.98:0/2258194256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:31:27.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:27 smithi098 bash[79516]: audit 2023-09-23T18:31:27.537374+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.98:0/2258194256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:31:27.945 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:27.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:27.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:28.049 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:28.085 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:28.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:28.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:28.188 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:28.214 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:28.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:28.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:28.318 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:28.349 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:28.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:28.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:28.452 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:28.473 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:28.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:28.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:28.577 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:28.608 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:28.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:28.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:28.712 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:28.743 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:28.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:28.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:28.764 INFO:teuthology.orchestra.run.smithi098.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-09-23T18:31:28.847 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:28.861 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:28 smithi173 bash[59422]: cluster 2023-09-23T18:31:27.668129+0000 mgr.y (mgr.26054) 384 : cluster 0 pgmap v322: 89 pgs: 5 active+undersized+degraded, 32 active+undersized, 52 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:28.861 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:28 smithi173 bash[59422]: audit 2023-09-23T18:31:27.798995+0000 mon.c (mon.2) 50 : audit 1 from='client.? 172.21.15.98:0/2124088708' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:31:28.861 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:28 smithi173 bash[59422]: audit 2023-09-23T18:31:27.799458+0000 mon.a (mon.0) 535 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:31:28.878 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:28.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:28.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:28.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:28 smithi098 bash[76817]: cluster 2023-09-23T18:31:27.668129+0000 mgr.y (mgr.26054) 384 : cluster 0 pgmap v322: 89 pgs: 5 active+undersized+degraded, 32 active+undersized, 52 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:28.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:28 smithi098 bash[76817]: audit 2023-09-23T18:31:27.798995+0000 mon.c (mon.2) 50 : audit 1 from='client.? 172.21.15.98:0/2124088708' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:31:28.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:28 smithi098 bash[76817]: audit 2023-09-23T18:31:27.799458+0000 mon.a (mon.0) 535 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:31:28.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:28 smithi098 bash[79516]: cluster 2023-09-23T18:31:27.668129+0000 mgr.y (mgr.26054) 384 : cluster 0 pgmap v322: 89 pgs: 5 active+undersized+degraded, 32 active+undersized, 52 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:28.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:28 smithi098 bash[79516]: audit 2023-09-23T18:31:27.798995+0000 mon.c (mon.2) 50 : audit 1 from='client.? 172.21.15.98:0/2124088708' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:31:28.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:28 smithi098 bash[79516]: audit 2023-09-23T18:31:27.799458+0000 mon.a (mon.0) 535 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:31:28.982 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:29.007 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:29.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:29.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:29.110 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:29.140 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:29.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:29.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:29.243 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:29.274 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:29.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:29.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:29.379 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:29.410 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:29.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:29.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:29.514 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:29.544 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:29.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:29.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:29.650 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:29.658 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:31:29.659 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:31:29.685 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:29.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:29.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:29.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:29 smithi173 bash[59422]: cluster 2023-09-23T18:31:28.756945+0000 osd.6 (osd.6) 25 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T18:31:29.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:29 smithi173 bash[59422]: audit 2023-09-23T18:31:28.758057+0000 mon.a (mon.0) 536 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:31:29.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:29 smithi173 bash[59422]: cluster 2023-09-23T18:31:28.758150+0000 mon.a (mon.0) 537 : cluster 0 osdmap e110: 8 total, 7 up, 8 in 2023-09-23T18:31:29.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:29 smithi173 bash[59422]: cluster 2023-09-23T18:31:29.145901+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T18:31:29.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:29 smithi173 bash[59422]: cluster 2023-09-23T18:31:29.146325+0000 osd.7 (osd.7) 24 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T18:31:29.788 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:29.819 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:29.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:29.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:29.923 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:29.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:29 smithi098 bash[76817]: cluster 2023-09-23T18:31:28.756945+0000 osd.6 (osd.6) 25 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T18:31:29.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:29 smithi098 bash[76817]: audit 2023-09-23T18:31:28.758057+0000 mon.a (mon.0) 536 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:31:29.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:29 smithi098 bash[76817]: cluster 2023-09-23T18:31:28.758150+0000 mon.a (mon.0) 537 : cluster 0 osdmap e110: 8 total, 7 up, 8 in 2023-09-23T18:31:29.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:29 smithi098 bash[76817]: cluster 2023-09-23T18:31:29.145901+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T18:31:29.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:29 smithi098 bash[76817]: cluster 2023-09-23T18:31:29.146325+0000 osd.7 (osd.7) 24 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T18:31:29.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:29 smithi098 bash[79516]: cluster 2023-09-23T18:31:28.756945+0000 osd.6 (osd.6) 25 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T18:31:29.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:29 smithi098 bash[79516]: audit 2023-09-23T18:31:28.758057+0000 mon.a (mon.0) 536 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:31:29.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:29 smithi098 bash[79516]: cluster 2023-09-23T18:31:28.758150+0000 mon.a (mon.0) 537 : cluster 0 osdmap e110: 8 total, 7 up, 8 in 2023-09-23T18:31:29.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:29 smithi098 bash[79516]: cluster 2023-09-23T18:31:29.145901+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T18:31:29.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:29 smithi098 bash[79516]: cluster 2023-09-23T18:31:29.146325+0000 osd.7 (osd.7) 24 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T18:31:29.954 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:29.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:29.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:30.057 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:30.088 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:30.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:30.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:30.191 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:30.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:30.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:30.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:30.329 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:30.364 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:30.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:30.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:30.468 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:30.502 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:30.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:30.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:30.606 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:30.638 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:30.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:30.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:30.743 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:30.762 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:30.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:30.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:30.866 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:30.879 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:30 smithi173 bash[59422]: cluster 2023-09-23T18:31:29.145970+0000 osd.4 (osd.4) 41 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T18:31:30.880 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:30 smithi173 bash[59422]: cluster 2023-09-23T18:31:29.668843+0000 mgr.y (mgr.26054) 385 : cluster 0 pgmap v324: 89 pgs: 5 active+undersized+degraded, 32 active+undersized, 52 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:30.880 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:30 smithi173 bash[59422]: audit 2023-09-23T18:31:30.705385+0000 mon.a (mon.0) 538 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:31:30.897 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:30.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:30.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:31.001 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:31.011 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:30 smithi098 bash[79516]: cluster 2023-09-23T18:31:29.145970+0000 osd.4 (osd.4) 41 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T18:31:31.011 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:30 smithi098 bash[79516]: cluster 2023-09-23T18:31:29.668843+0000 mgr.y (mgr.26054) 385 : cluster 0 pgmap v324: 89 pgs: 5 active+undersized+degraded, 32 active+undersized, 52 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:31.011 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:30 smithi098 bash[79516]: audit 2023-09-23T18:31:30.705385+0000 mon.a (mon.0) 538 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:31:31.012 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:30 smithi098 bash[76817]: cluster 2023-09-23T18:31:29.145970+0000 osd.4 (osd.4) 41 : cluster 3 failed to encode map e110 with expected crc 2023-09-23T18:31:31.012 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:30 smithi098 bash[76817]: cluster 2023-09-23T18:31:29.668843+0000 mgr.y (mgr.26054) 385 : cluster 0 pgmap v324: 89 pgs: 5 active+undersized+degraded, 32 active+undersized, 52 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T18:31:31.012 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:30 smithi098 bash[76817]: audit 2023-09-23T18:31:30.705385+0000 mon.a (mon.0) 538 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:31:31.027 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:31.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:31.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:31.131 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:31.161 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:31.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:31.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:31.265 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:31.299 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:31.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:31.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:31.403 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:31.436 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:31.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:31.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:31.537 DEBUG:teuthology.orchestra.run.smithi098:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-09-23T18:31:31.538 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-09-23T18:31:31.538 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:31:31.540 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:31.540 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T18:31:31.541 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T18:31:31.570 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:31.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:31.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:31.586 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:hints = 1 2023-09-23T18:31:31.586 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T18:31:31.586 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Object prefix: benchmark_data_smithi098_105788 2023-09-23T18:31:31.587 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:31:31.587 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 0 0 0 0 0 0 - 0 2023-09-23T18:31:31.660 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:31:31.673 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:31.699 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:31.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:31.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:31.782 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is set 2023-09-23T18:31:31.802 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:31.811 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:31 smithi098 bash[76817]: audit 2023-09-23T18:31:30.916368+0000 mon.c (mon.2) 51 : audit 1 from='client.? 172.21.15.98:0/2367134142' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:31:31.811 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:31 smithi098 bash[76817]: audit 2023-09-23T18:31:30.917105+0000 mon.a (mon.0) 539 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:31:31.811 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:31 smithi098 bash[79516]: audit 2023-09-23T18:31:30.916368+0000 mon.c (mon.2) 51 : audit 1 from='client.? 172.21.15.98:0/2367134142' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:31:31.811 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:31 smithi098 bash[79516]: audit 2023-09-23T18:31:30.917105+0000 mon.a (mon.0) 539 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:31:31.840 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:31.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:31.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:31.945 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:31.954 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:31 smithi173 bash[59422]: audit 2023-09-23T18:31:30.916368+0000 mon.c (mon.2) 51 : audit 1 from='client.? 172.21.15.98:0/2367134142' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:31:31.955 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:31 smithi173 bash[59422]: audit 2023-09-23T18:31:30.917105+0000 mon.a (mon.0) 539 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:31:31.972 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:31.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:31.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:32.075 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:32.102 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:32.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:32.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:32.208 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:32.228 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:32.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:32.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:32.330 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:32.359 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:32.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:32.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:32.463 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:32.494 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:32.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:32.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:32.587 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 1 16 3189 3173 198.302 198.312 0.00354821 0.00502097 2023-09-23T18:31:32.597 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:32.626 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:32.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:32.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:32.729 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:32.761 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:32.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:32.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:32.864 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:32.888 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:32.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:32.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:32.992 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:33.017 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:33.019 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:32 smithi098 bash[79516]: cluster 2023-09-23T18:31:31.669683+0000 mgr.y (mgr.26054) 386 : cluster 0 pgmap v325: 89 pgs: 6 active+undersized+degraded, 31 active+undersized, 52 active+clean; 5.3 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s wr, 1 op/s; 8/117 objects degraded (6.838%) 2023-09-23T18:31:33.019 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:32 smithi098 bash[79516]: cluster 2023-09-23T18:31:31.758328+0000 mon.a (mon.0) 540 : cluster 3 Health check update: Degraded data redundancy: 8/117 objects degraded (6.838%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:31:33.019 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:32 smithi098 bash[79516]: cluster 2023-09-23T18:31:31.758669+0000 mon.a (mon.0) 541 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:31:33.019 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:32 smithi098 bash[79516]: cluster 2023-09-23T18:31:31.772026+0000 osd.7 (osd.7) 25 : cluster 3 failed to encode map e111 with expected crc 2023-09-23T18:31:33.019 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:32 smithi098 bash[79516]: audit 2023-09-23T18:31:31.773445+0000 mon.a (mon.0) 542 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:31:33.020 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:32 smithi098 bash[79516]: cluster 2023-09-23T18:31:31.773613+0000 mon.a (mon.0) 543 : cluster 0 osdmap e111: 8 total, 7 up, 8 in 2023-09-23T18:31:33.020 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:32 smithi098 bash[79516]: cluster 2023-09-23T18:31:31.776028+0000 osd.6 (osd.6) 26 : cluster 3 failed to encode map e111 with expected crc 2023-09-23T18:31:33.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:33.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:33.021 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:32 smithi098 bash[76817]: cluster 2023-09-23T18:31:31.669683+0000 mgr.y (mgr.26054) 386 : cluster 0 pgmap v325: 89 pgs: 6 active+undersized+degraded, 31 active+undersized, 52 active+clean; 5.3 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s wr, 1 op/s; 8/117 objects degraded (6.838%) 2023-09-23T18:31:33.021 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:32 smithi098 bash[76817]: cluster 2023-09-23T18:31:31.758328+0000 mon.a (mon.0) 540 : cluster 3 Health check update: Degraded data redundancy: 8/117 objects degraded (6.838%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:31:33.021 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:32 smithi098 bash[76817]: cluster 2023-09-23T18:31:31.758669+0000 mon.a (mon.0) 541 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:31:33.021 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:32 smithi098 bash[76817]: cluster 2023-09-23T18:31:31.772026+0000 osd.7 (osd.7) 25 : cluster 3 failed to encode map e111 with expected crc 2023-09-23T18:31:33.022 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:32 smithi098 bash[76817]: audit 2023-09-23T18:31:31.773445+0000 mon.a (mon.0) 542 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:31:33.022 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:32 smithi098 bash[76817]: cluster 2023-09-23T18:31:31.773613+0000 mon.a (mon.0) 543 : cluster 0 osdmap e111: 8 total, 7 up, 8 in 2023-09-23T18:31:33.022 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:32 smithi098 bash[76817]: cluster 2023-09-23T18:31:31.776028+0000 osd.6 (osd.6) 26 : cluster 3 failed to encode map e111 with expected crc 2023-09-23T18:31:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:32 smithi173 bash[59422]: cluster 2023-09-23T18:31:31.669683+0000 mgr.y (mgr.26054) 386 : cluster 0 pgmap v325: 89 pgs: 6 active+undersized+degraded, 31 active+undersized, 52 active+clean; 5.3 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s wr, 1 op/s; 8/117 objects degraded (6.838%) 2023-09-23T18:31:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:32 smithi173 bash[59422]: cluster 2023-09-23T18:31:31.758328+0000 mon.a (mon.0) 540 : cluster 3 Health check update: Degraded data redundancy: 8/117 objects degraded (6.838%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:31:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:32 smithi173 bash[59422]: cluster 2023-09-23T18:31:31.758669+0000 mon.a (mon.0) 541 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:31:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:32 smithi173 bash[59422]: cluster 2023-09-23T18:31:31.772026+0000 osd.7 (osd.7) 25 : cluster 3 failed to encode map e111 with expected crc 2023-09-23T18:31:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:32 smithi173 bash[59422]: audit 2023-09-23T18:31:31.773445+0000 mon.a (mon.0) 542 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:31:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:32 smithi173 bash[59422]: cluster 2023-09-23T18:31:31.773613+0000 mon.a (mon.0) 543 : cluster 0 osdmap e111: 8 total, 7 up, 8 in 2023-09-23T18:31:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:32 smithi173 bash[59422]: cluster 2023-09-23T18:31:31.776028+0000 osd.6 (osd.6) 26 : cluster 3 failed to encode map e111 with expected crc 2023-09-23T18:31:33.122 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:33.149 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:33.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:33.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:33.254 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:33.281 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:33.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:33.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:33.387 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:33.415 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:33.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:33.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:33.518 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:33.544 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:33.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:33.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:33.587 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 2 16 6391 6375 199.205 200.125 0.00633713 0.00500801 2023-09-23T18:31:33.643 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:31:33.647 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:33.686 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:33.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:33.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:33.790 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:33.818 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:33.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:33.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:33.922 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:33.951 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:33.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:33.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:34.055 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:34.083 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:34.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:34.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:34.188 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:34.212 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:34.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:34.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:34.315 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:34.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:34.345 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:34.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:34.446 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:34.472 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:34.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:34.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:34.578 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:34.587 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 3 16 9367 9351 194.797 186 0.00920838 0.00512222 2023-09-23T18:31:34.603 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:34.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:34.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:34.707 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:34.739 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:34.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:34.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:34.843 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:34.853 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:34 smithi173 bash[59422]: cluster 2023-09-23T18:31:33.670187+0000 mgr.y (mgr.26054) 387 : cluster 0 pgmap v327: 89 pgs: 7 active+undersized+degraded, 30 active+undersized, 52 active+clean; 40 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s wr, 71 op/s; 292/1800 objects degraded (16.222%) 2023-09-23T18:31:34.871 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:34.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:34.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:34.975 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:35.001 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:35.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:35.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:35.104 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:35.138 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:35.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:35.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:35.162 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:31:35.171 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:34 smithi098 bash[76817]: cluster 2023-09-23T18:31:33.670187+0000 mgr.y (mgr.26054) 387 : cluster 0 pgmap v327: 89 pgs: 7 active+undersized+degraded, 30 active+undersized, 52 active+clean; 40 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s wr, 71 op/s; 292/1800 objects degraded (16.222%) 2023-09-23T18:31:35.171 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:34 smithi098 bash[79516]: cluster 2023-09-23T18:31:33.670187+0000 mgr.y (mgr.26054) 387 : cluster 0 pgmap v327: 89 pgs: 7 active+undersized+degraded, 30 active+undersized, 52 active+clean; 40 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s wr, 71 op/s; 292/1800 objects degraded (16.222%) 2023-09-23T18:31:35.243 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:35.269 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:35.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:35.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:35.373 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:35.410 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:35.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:35.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:35.514 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:35.538 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:35.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:35.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:35.587 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 4 16 12028 12012 187.671 166.312 0.00366114 0.00531882 2023-09-23T18:31:35.642 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:35.669 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:35.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:35.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:35.774 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:35.798 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:35.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:35.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:35.908 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:35.942 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:35.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:35.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:36.046 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:36.074 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:36.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:36.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:36.182 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:36.211 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:36.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:36.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:36.317 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:36.349 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:36.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:36.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:36.436 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:31:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:31:36] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.33.4" 2023-09-23T18:31:36.454 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:36.489 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:36.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:36.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:36.587 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 5 16 14121 14105 176.297 130.812 0.00493422 0.00566183 2023-09-23T18:31:36.592 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:36.644 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:36.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:36.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:36.748 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:36.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:36.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:36.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:36.890 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:36.949 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:36.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:36.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:37.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:36 smithi173 bash[59422]: cluster 2023-09-23T18:31:35.670867+0000 mgr.y (mgr.26054) 388 : cluster 0 pgmap v328: 89 pgs: 9 active+undersized+degraded, 28 active+undersized, 52 active+clean; 412 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 51 MiB/s wr, 813 op/s; 1807/19623 objects degraded (9.209%) 2023-09-23T18:31:37.052 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:37.088 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:37.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:37.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:37.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:36 smithi098 bash[79516]: cluster 2023-09-23T18:31:35.670867+0000 mgr.y (mgr.26054) 388 : cluster 0 pgmap v328: 89 pgs: 9 active+undersized+degraded, 28 active+undersized, 52 active+clean; 412 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 51 MiB/s wr, 813 op/s; 1807/19623 objects degraded (9.209%) 2023-09-23T18:31:37.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:36 smithi098 bash[76817]: cluster 2023-09-23T18:31:35.670867+0000 mgr.y (mgr.26054) 388 : cluster 0 pgmap v328: 89 pgs: 9 active+undersized+degraded, 28 active+undersized, 52 active+clean; 412 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 51 MiB/s wr, 813 op/s; 1807/19623 objects degraded (9.209%) 2023-09-23T18:31:37.195 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:37.231 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:37.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:37.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:37.334 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:37.368 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:37.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:37.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:37.471 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:37.521 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:37.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:37.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:37.587 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 6 16 16172 16156 168.277 128.188 0.0049885 0.00593299 2023-09-23T18:31:37.627 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:37.659 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:37.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:37.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:37.763 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:37.788 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:37.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:37.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:37.870 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:31:37.874 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:31:37.891 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:37.919 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:37.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:37.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:38.022 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:38.059 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:38 smithi098 bash[76817]: audit 2023-09-23T18:31:37.059318+0000 mon.a (mon.0) 544 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:31:38.059 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:38 smithi098 bash[76817]: audit 2023-09-23T18:31:37.070375+0000 mon.a (mon.0) 545 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:31:38.064 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:38.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:38.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:38.167 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:38.177 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:38 smithi173 bash[59422]: audit 2023-09-23T18:31:37.059318+0000 mon.a (mon.0) 544 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:31:38.177 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:38 smithi173 bash[59422]: audit 2023-09-23T18:31:37.070375+0000 mon.a (mon.0) 545 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:31:38.177 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:38 smithi173 bash[59422]: audit 2023-09-23T18:31:37.151641+0000 mon.a (mon.0) 546 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:31:38.178 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:38 smithi173 bash[59422]: audit 2023-09-23T18:31:37.163333+0000 mon.a (mon.0) 547 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:31:38.178 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:38 smithi173 bash[59422]: cluster 2023-09-23T18:31:37.671680+0000 mgr.y (mgr.26054) 389 : cluster 0 pgmap v329: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 738 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 3515/35277 objects degraded (9.964%) 2023-09-23T18:31:38.178 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:38 smithi173 bash[59422]: audit 2023-09-23T18:31:37.806219+0000 mon.a (mon.0) 548 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:31:38.178 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:38 smithi173 bash[59422]: audit 2023-09-23T18:31:37.807581+0000 mon.a (mon.0) 549 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:31:38.178 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:38 smithi173 bash[59422]: audit 2023-09-23T18:31:37.814969+0000 mon.a (mon.0) 550 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:31:38.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:38 smithi098 bash[79516]: audit 2023-09-23T18:31:37.059318+0000 mon.a (mon.0) 544 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:31:38.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:38 smithi098 bash[79516]: audit 2023-09-23T18:31:37.070375+0000 mon.a (mon.0) 545 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:31:38.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:38 smithi098 bash[79516]: audit 2023-09-23T18:31:37.151641+0000 mon.a (mon.0) 546 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:31:38.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:38 smithi098 bash[79516]: audit 2023-09-23T18:31:37.163333+0000 mon.a (mon.0) 547 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:31:38.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:38 smithi098 bash[79516]: cluster 2023-09-23T18:31:37.671680+0000 mgr.y (mgr.26054) 389 : cluster 0 pgmap v329: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 738 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 3515/35277 objects degraded (9.964%) 2023-09-23T18:31:38.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:38 smithi098 bash[79516]: audit 2023-09-23T18:31:37.806219+0000 mon.a (mon.0) 548 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:31:38.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:38 smithi098 bash[79516]: audit 2023-09-23T18:31:37.807581+0000 mon.a (mon.0) 549 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:31:38.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:38 smithi098 bash[79516]: audit 2023-09-23T18:31:37.814969+0000 mon.a (mon.0) 550 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:31:38.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:38 smithi098 bash[76817]: audit 2023-09-23T18:31:37.151641+0000 mon.a (mon.0) 546 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:31:38.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:38 smithi098 bash[76817]: audit 2023-09-23T18:31:37.163333+0000 mon.a (mon.0) 547 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:31:38.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:38 smithi098 bash[76817]: cluster 2023-09-23T18:31:37.671680+0000 mgr.y (mgr.26054) 389 : cluster 0 pgmap v329: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 738 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 3515/35277 objects degraded (9.964%) 2023-09-23T18:31:38.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:38 smithi098 bash[76817]: audit 2023-09-23T18:31:37.806219+0000 mon.a (mon.0) 548 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:31:38.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:38 smithi098 bash[76817]: audit 2023-09-23T18:31:37.807581+0000 mon.a (mon.0) 549 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:31:38.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:38 smithi098 bash[76817]: audit 2023-09-23T18:31:37.814969+0000 mon.a (mon.0) 550 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:31:38.196 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:38.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:38.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:38.299 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:38.338 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:38.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:38.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:38.441 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:38.472 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:38.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:38.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:38.575 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:38.587 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 7 16 18451 18435 164.583 142.438 0.00206051 0.00603865 2023-09-23T18:31:38.599 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:38.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:38.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:38.702 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:38.728 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:38.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:38.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:38.832 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:38.860 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:38.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:38.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:38.964 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:38.994 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:38.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:38.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:39.098 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:39.128 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:39.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:39.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:39.231 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:39.255 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:39.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:39.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:39.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:39 smithi173 bash[59422]: cluster 2023-09-23T18:31:38.160614+0000 mon.a (mon.0) 551 : cluster 3 Health check update: Degraded data redundancy: 3515/35277 objects degraded (9.964%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T18:31:39.358 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:39.386 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:39.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:39.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:39.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:39 smithi098 bash[76817]: cluster 2023-09-23T18:31:38.160614+0000 mon.a (mon.0) 551 : cluster 3 Health check update: Degraded data redundancy: 3515/35277 objects degraded (9.964%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T18:31:39.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:39 smithi098 bash[79516]: cluster 2023-09-23T18:31:38.160614+0000 mon.a (mon.0) 551 : cluster 3 Health check update: Degraded data redundancy: 3515/35277 objects degraded (9.964%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T18:31:39.492 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:39.518 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:39.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:39.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:39.587 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 8 16 20764 20748 162.079 144.562 0.00428135 0.00616088 2023-09-23T18:31:39.622 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:39.656 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:39.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:39.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:39.759 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:39.760 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:31:39.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:39.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:39.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:39.888 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:39.925 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:39.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:39.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:40.028 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:40.052 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:40.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:40.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:40.156 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:40.181 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:40.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:40.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:40.285 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:40.314 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:40.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:40.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:40.417 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:40.447 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:40.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:40.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:40.552 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:40.582 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:40.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:40.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:40.587 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 9 16 23058 23042 159.999 143.375 0.00426331 0.00624304 2023-09-23T18:31:40.685 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:40.690 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:40 smithi098 bash[79516]: audit 2023-09-23T18:31:39.578392+0000 mon.a (mon.0) 552 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:31:40.690 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:40 smithi098 bash[79516]: cluster 2023-09-23T18:31:39.672192+0000 mgr.y (mgr.26054) 390 : cluster 0 pgmap v330: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 836 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 4291/39981 objects degraded (10.733%) 2023-09-23T18:31:40.690 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:40 smithi098 bash[76817]: audit 2023-09-23T18:31:39.578392+0000 mon.a (mon.0) 552 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:31:40.690 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:40 smithi098 bash[76817]: cluster 2023-09-23T18:31:39.672192+0000 mgr.y (mgr.26054) 390 : cluster 0 pgmap v330: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 836 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 4291/39981 objects degraded (10.733%) 2023-09-23T18:31:40.694 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:40 smithi173 bash[59422]: audit 2023-09-23T18:31:39.578392+0000 mon.a (mon.0) 552 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:31:40.694 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:40 smithi173 bash[59422]: cluster 2023-09-23T18:31:39.672192+0000 mgr.y (mgr.26054) 390 : cluster 0 pgmap v330: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 836 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 4291/39981 objects degraded (10.733%) 2023-09-23T18:31:40.710 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:40.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:40.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:40.816 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:40.846 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:40.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:40.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:40.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:40.977 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:40.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:40.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:41.080 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:41.108 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:41.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:41.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:41.211 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:41.238 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:41.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:41.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:41.344 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:41.380 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:41.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:41.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:41.484 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:41.513 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:41.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:41.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:41.588 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 10 16 25180 25164 157.261 132.625 0.00692343 0.00635383 2023-09-23T18:31:41.615 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:41.639 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:41.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:41.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:41.743 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:41.768 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:41.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:41.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:41.871 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:41.895 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:41.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:41.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:41.999 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:42.022 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:42.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:42.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:42.127 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:42.153 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:42.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:42.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:42.256 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:42.287 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:42.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:42.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:42.389 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:42.420 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:42.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:42.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:42.524 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:42.589 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 11 16 27185 27169 154.345 125.312 0.00698773 0.00647174 2023-09-23T18:31:42.593 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:42.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:42.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:42.697 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:42.736 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:42.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:42.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:42.845 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:42.847 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:31:42.847 INFO:teuthology.orchestra.run.smithi098.stdout: "size": 20, 2023-09-23T18:31:42.847 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 600, 2023-09-23T18:31:42.847 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [ 2023-09-23T18:31:42.847 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.848 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.44693.0:1957 5.e e111/107 5:784b46bc:::benchmark_data_smithi098_105788_object1956:head v 111'133, mlcod=111'130)", 2023-09-23T18:31:42.848 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:31:32.219023+0000", 2023-09-23T18:31:42.848 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 10.622056117, 2023-09-23T18:31:42.848 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.022748847999999999, 2023-09-23T18:31:42.848 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:31:42.848 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:31:42.848 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:31:42.848 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.848 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:31:42.848 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.219023+0000", 2023-09-23T18:31:42.848 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.848 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.849 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.849 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:31:42.849 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.219023+0000", 2023-09-23T18:31:42.849 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.849 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.849 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.849 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:31:42.849 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.219023+0000", 2023-09-23T18:31:42.849 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:31:42.849 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.849 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.849 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:31:42.849 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.219067+0000", 2023-09-23T18:31:42.850 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.3874e-05 2023-09-23T18:31:42.850 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.850 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.850 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:31:42.850 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.219075+0000", 2023-09-23T18:31:42.850 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.4640000000000006e-06 2023-09-23T18:31:42.850 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.850 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.850 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:31:42.850 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.219080+0000", 2023-09-23T18:31:42.850 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.8339999999999998e-06 2023-09-23T18:31:42.851 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.851 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.851 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:31:42.851 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.219115+0000", 2023-09-23T18:31:42.851 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.4626999999999999e-05 2023-09-23T18:31:42.851 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.851 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.851 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:31:42.851 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.219131+0000", 2023-09-23T18:31:42.851 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.6269999999999998e-05 2023-09-23T18:31:42.851 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.851 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.852 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:31:42.852 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.241732+0000", 2023-09-23T18:31:42.852 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.022601514999999999 2023-09-23T18:31:42.852 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.852 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.852 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:31:42.852 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.241772+0000", 2023-09-23T18:31:42.852 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.9870999999999998e-05 2023-09-23T18:31:42.852 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.852 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:31:42.852 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.852 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.853 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.853 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.44693.0:1959 5.e e111/107 5:72d977ed:::benchmark_data_smithi098_105788_object1958:head v 111'134, mlcod=111'130)", 2023-09-23T18:31:42.853 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:31:32.219088+0000", 2023-09-23T18:31:42.853 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 10.621991663999999, 2023-09-23T18:31:42.853 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.023006645999999999, 2023-09-23T18:31:42.853 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:31:42.853 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:31:42.853 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:31:42.853 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.853 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:31:42.853 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.219088+0000", 2023-09-23T18:31:42.853 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.854 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.854 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.854 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:31:42.854 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.219088+0000", 2023-09-23T18:31:42.854 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.854 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.854 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.854 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:31:42.854 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.219087+0000", 2023-09-23T18:31:42.854 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:31:42.854 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.854 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.855 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:31:42.855 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.219180+0000", 2023-09-23T18:31:42.855 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.2503000000000005e-05 2023-09-23T18:31:42.855 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.855 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.855 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:31:42.855 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.240582+0000", 2023-09-23T18:31:42.855 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.021402233 2023-09-23T18:31:42.855 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.855 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.855 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:31:42.855 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.240597+0000", 2023-09-23T18:31:42.856 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.4313e-05 2023-09-23T18:31:42.856 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.856 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.856 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:31:42.856 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.240939+0000", 2023-09-23T18:31:42.856 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00034248599999999997 2023-09-23T18:31:42.856 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.856 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.856 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:31:42.856 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.240964+0000", 2023-09-23T18:31:42.856 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.4871999999999998e-05 2023-09-23T18:31:42.857 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.857 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.857 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:31:42.857 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.242065+0000", 2023-09-23T18:31:42.857 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.001100717 2023-09-23T18:31:42.857 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.857 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.857 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:31:42.857 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.242094+0000", 2023-09-23T18:31:42.857 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.9824000000000001e-05 2023-09-23T18:31:42.857 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.857 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:31:42.857 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.858 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.858 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.858 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.44693.0:1963 5.3 5:cfac3deb:::benchmark_data_smithi098_105788_object1962:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-09-23T18:31:42.858 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:31:32.219517+0000", 2023-09-23T18:31:42.858 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 10.621562215999999, 2023-09-23T18:31:42.858 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.026330727000000002, 2023-09-23T18:31:42.858 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:31:42.858 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:31:42.858 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:31:42.858 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.44693", 2023-09-23T18:31:42.858 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:31:42.858 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 1963 2023-09-23T18:31:42.859 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.859 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:31:42.859 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.859 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:31:42.859 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.219517+0000", 2023-09-23T18:31:42.859 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.859 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.859 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.859 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:31:42.859 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.219517+0000", 2023-09-23T18:31:42.859 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.859 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.860 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.860 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:31:42.860 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.219516+0000", 2023-09-23T18:31:42.860 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:31:42.860 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.860 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.860 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:31:42.860 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.219550+0000", 2023-09-23T18:31:42.860 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.4267000000000002e-05 2023-09-23T18:31:42.860 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.860 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.860 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:31:42.860 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.240569+0000", 2023-09-23T18:31:42.861 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.021018800000000001 2023-09-23T18:31:42.861 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.861 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.861 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:31:42.861 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.240590+0000", 2023-09-23T18:31:42.861 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.0435999999999999e-05 2023-09-23T18:31:42.861 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.861 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.861 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:31:42.861 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.240682+0000", 2023-09-23T18:31:42.861 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.2137000000000002e-05 2023-09-23T18:31:42.861 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.862 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.862 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:31:42.862 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.240860+0000", 2023-09-23T18:31:42.862 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00017820100000000001 2023-09-23T18:31:42.862 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.862 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.862 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 5,6", 2023-09-23T18:31:42.862 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.241042+0000", 2023-09-23T18:31:42.862 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000181547 2023-09-23T18:31:42.862 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.862 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.863 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:31:42.863 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.241938+0000", 2023-09-23T18:31:42.863 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00089673599999999997 2023-09-23T18:31:42.863 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.863 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.863 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:31:42.863 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.243133+0000", 2023-09-23T18:31:42.863 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.001195024 2023-09-23T18:31:42.863 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.863 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.863 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:31:42.863 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.245734+0000", 2023-09-23T18:31:42.864 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.002601108 2023-09-23T18:31:42.864 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.864 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.864 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:31:42.864 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.245810+0000", 2023-09-23T18:31:42.864 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.5104000000000005e-05 2023-09-23T18:31:42.864 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.864 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.864 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:31:42.864 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:32.245848+0000", 2023-09-23T18:31:42.864 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.8457000000000001e-05 2023-09-23T18:31:42.865 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.865 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:31:42.865 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.865 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.865 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.865 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.44693.0:13756 5.5 5:ad72819b:::benchmark_data_smithi098_105788_object13755:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-09-23T18:31:42.865 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:31:36.408072+0000", 2023-09-23T18:31:42.865 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 6.4330077719999998, 2023-09-23T18:31:42.865 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.019866591999999999, 2023-09-23T18:31:42.865 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:31:42.865 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:31:42.865 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:31:42.866 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.44693", 2023-09-23T18:31:42.866 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:31:42.866 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 13756 2023-09-23T18:31:42.866 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.866 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:31:42.866 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.866 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:31:42.866 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.408072+0000", 2023-09-23T18:31:42.866 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.866 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.866 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.866 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:31:42.867 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.408072+0000", 2023-09-23T18:31:42.867 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.867 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.867 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.867 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:31:42.867 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.408071+0000", 2023-09-23T18:31:42.867 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:31:42.867 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.867 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.867 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:31:42.867 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.408092+0000", 2023-09-23T18:31:42.867 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.0852e-05 2023-09-23T18:31:42.868 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.868 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.868 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:31:42.868 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.408101+0000", 2023-09-23T18:31:42.868 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.755e-06 2023-09-23T18:31:42.868 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.868 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.868 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:31:42.868 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.408107+0000", 2023-09-23T18:31:42.868 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.0129999999999999e-06 2023-09-23T18:31:42.868 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.868 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.869 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:31:42.869 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.409083+0000", 2023-09-23T18:31:42.869 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00097656000000000001 2023-09-23T18:31:42.869 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.869 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.869 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:31:42.869 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.409214+0000", 2023-09-23T18:31:42.869 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00013096700000000001 2023-09-23T18:31:42.869 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.869 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.869 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 4", 2023-09-23T18:31:42.869 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.409363+0000", 2023-09-23T18:31:42.870 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00014857800000000001 2023-09-23T18:31:42.870 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.870 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.870 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:31:42.870 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.410490+0000", 2023-09-23T18:31:42.870 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0011271 2023-09-23T18:31:42.870 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.870 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.870 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:31:42.870 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.427745+0000", 2023-09-23T18:31:42.870 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.017255360000000001 2023-09-23T18:31:42.870 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.871 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.871 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:31:42.871 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.427816+0000", 2023-09-23T18:31:42.871 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.0564999999999999e-05 2023-09-23T18:31:42.871 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.871 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.871 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:31:42.871 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.427938+0000", 2023-09-23T18:31:42.871 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000122533 2023-09-23T18:31:42.871 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.871 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:31:42.871 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.872 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.872 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.872 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.44693.0:14264 5.5 5:a8b48a2f:::benchmark_data_smithi098_105788_object14263:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-09-23T18:31:42.872 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:31:36.660610+0000", 2023-09-23T18:31:42.872 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 6.1804693290000001, 2023-09-23T18:31:42.872 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.020951212, 2023-09-23T18:31:42.872 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:31:42.872 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:31:42.872 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:31:42.872 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.44693", 2023-09-23T18:31:42.872 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:31:42.872 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 14264 2023-09-23T18:31:42.873 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.873 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:31:42.873 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.873 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:31:42.873 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.660610+0000", 2023-09-23T18:31:42.873 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.873 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.873 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.873 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:31:42.873 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.660610+0000", 2023-09-23T18:31:42.873 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.873 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.874 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.874 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:31:42.874 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.660609+0000", 2023-09-23T18:31:42.874 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:31:42.874 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.874 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.874 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:31:42.874 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.660637+0000", 2023-09-23T18:31:42.874 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.7667000000000001e-05 2023-09-23T18:31:42.874 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.874 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.874 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:31:42.874 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.660645+0000", 2023-09-23T18:31:42.875 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.6449999999999994e-06 2023-09-23T18:31:42.875 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.875 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.875 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:31:42.875 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.660651+0000", 2023-09-23T18:31:42.875 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.3940000000000004e-06 2023-09-23T18:31:42.875 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.875 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.875 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:31:42.875 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.660700+0000", 2023-09-23T18:31:42.875 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.9308000000000003e-05 2023-09-23T18:31:42.875 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.876 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.876 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:31:42.876 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.660857+0000", 2023-09-23T18:31:42.876 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000156606 2023-09-23T18:31:42.876 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.876 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.876 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 4", 2023-09-23T18:31:42.876 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.661051+0000", 2023-09-23T18:31:42.876 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000194453 2023-09-23T18:31:42.876 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.876 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.876 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:31:42.877 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.662444+0000", 2023-09-23T18:31:42.877 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.001392922 2023-09-23T18:31:42.877 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.877 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.877 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:31:42.877 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.681441+0000", 2023-09-23T18:31:42.877 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.018996585 2023-09-23T18:31:42.877 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.877 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.877 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:31:42.877 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.681487+0000", 2023-09-23T18:31:42.877 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.6789999999999998e-05 2023-09-23T18:31:42.878 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.878 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.878 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:31:42.878 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:36.681561+0000", 2023-09-23T18:31:42.878 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.3962999999999996e-05 2023-09-23T18:31:42.878 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.878 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:31:42.878 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.878 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.878 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.878 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.44693.0:14934 5.3 5:ca12d128:::benchmark_data_smithi098_105788_object14933:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-09-23T18:31:42.879 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:31:37.005871+0000", 2023-09-23T18:31:42.879 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 5.8352082789999997, 2023-09-23T18:31:42.879 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.021338564000000001, 2023-09-23T18:31:42.879 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:31:42.879 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:31:42.879 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:31:42.879 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.44693", 2023-09-23T18:31:42.879 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:31:42.879 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 14934 2023-09-23T18:31:42.879 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.879 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:31:42.879 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.880 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:31:42.880 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.005871+0000", 2023-09-23T18:31:42.880 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.880 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.880 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.880 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:31:42.880 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.005871+0000", 2023-09-23T18:31:42.880 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.880 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.880 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.880 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:31:42.880 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.005870+0000", 2023-09-23T18:31:42.880 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:31:42.881 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.881 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.881 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:31:42.881 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.005894+0000", 2023-09-23T18:31:42.881 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.4326999999999999e-05 2023-09-23T18:31:42.881 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.881 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.881 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:31:42.881 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.005905+0000", 2023-09-23T18:31:42.881 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1249e-05 2023-09-23T18:31:42.881 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.881 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.881 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:31:42.882 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.005913+0000", 2023-09-23T18:31:42.882 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.4240000000000004e-06 2023-09-23T18:31:42.882 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.882 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.882 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:31:42.882 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.008669+0000", 2023-09-23T18:31:42.882 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0027563850000000001 2023-09-23T18:31:42.882 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.882 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.882 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:31:42.882 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.008832+0000", 2023-09-23T18:31:42.883 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00016232099999999999 2023-09-23T18:31:42.883 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.883 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.883 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 5,6", 2023-09-23T18:31:42.883 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.009008+0000", 2023-09-23T18:31:42.883 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000176064 2023-09-23T18:31:42.883 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.883 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.883 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:31:42.883 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.010014+0000", 2023-09-23T18:31:42.883 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.001006481 2023-09-23T18:31:42.883 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.883 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.884 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:31:42.884 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.010634+0000", 2023-09-23T18:31:42.884 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000620069 2023-09-23T18:31:42.884 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.884 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.884 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:31:42.884 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.027142+0000", 2023-09-23T18:31:42.884 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.016507457999999999 2023-09-23T18:31:42.884 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.884 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.884 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:31:42.884 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.027187+0000", 2023-09-23T18:31:42.885 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.5779999999999999e-05 2023-09-23T18:31:42.885 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.885 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.885 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:31:42.885 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.027210+0000", 2023-09-23T18:31:42.885 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.2376000000000001e-05 2023-09-23T18:31:42.885 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.885 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:31:42.885 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.885 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.885 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.885 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.44693.0:16811 5.2 e111/107 5:4e21425b:::benchmark_data_smithi098_105788_object16810:head v 111'1060, mlcod=111'1058)", 2023-09-23T18:31:42.885 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:31:37.864440+0000", 2023-09-23T18:31:42.886 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 4.9766398689999995, 2023-09-23T18:31:42.886 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.022202778999999999, 2023-09-23T18:31:42.886 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:31:42.886 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:31:42.886 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:31:42.886 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.886 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:31:42.886 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.864440+0000", 2023-09-23T18:31:42.886 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.886 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.886 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.887 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:31:42.887 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.864440+0000", 2023-09-23T18:31:42.887 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.887 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.887 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.887 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:31:42.887 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.864439+0000", 2023-09-23T18:31:42.887 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:31:42.887 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.887 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.887 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:31:42.887 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.864470+0000", 2023-09-23T18:31:42.887 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.1361e-05 2023-09-23T18:31:42.888 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.888 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.888 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:31:42.888 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.864495+0000", 2023-09-23T18:31:42.888 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.5264000000000001e-05 2023-09-23T18:31:42.888 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.888 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.888 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:31:42.888 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.864505+0000", 2023-09-23T18:31:42.888 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.5810000000000003e-06 2023-09-23T18:31:42.888 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.888 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.888 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:31:42.889 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.864578+0000", 2023-09-23T18:31:42.889 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.2799000000000006e-05 2023-09-23T18:31:42.889 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.889 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.889 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:31:42.889 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.864596+0000", 2023-09-23T18:31:42.889 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.7827000000000001e-05 2023-09-23T18:31:42.889 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.889 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.889 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:31:42.889 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.886587+0000", 2023-09-23T18:31:42.889 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.021991012000000001 2023-09-23T18:31:42.889 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.890 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.890 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:31:42.890 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.886642+0000", 2023-09-23T18:31:42.890 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.5683999999999997e-05 2023-09-23T18:31:42.890 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.890 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:31:42.890 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.890 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.890 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.890 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.44693.0:16818 5.2 e111/107 5:49a0ff67:::benchmark_data_smithi098_105788_object16817:head v 111'1061, mlcod=111'1058)", 2023-09-23T18:31:42.890 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:31:37.865885+0000", 2023-09-23T18:31:42.890 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 4.9751944379999999, 2023-09-23T18:31:42.891 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.020773375, 2023-09-23T18:31:42.891 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:31:42.891 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:31:42.891 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:31:42.891 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.891 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:31:42.891 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.865885+0000", 2023-09-23T18:31:42.891 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.891 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.891 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.891 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:31:42.891 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.865885+0000", 2023-09-23T18:31:42.891 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.892 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.892 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.892 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:31:42.892 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.865884+0000", 2023-09-23T18:31:42.892 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:31:42.892 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.892 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.892 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:31:42.892 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.866073+0000", 2023-09-23T18:31:42.892 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00018862100000000001 2023-09-23T18:31:42.892 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.892 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.892 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:31:42.893 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.866088+0000", 2023-09-23T18:31:42.893 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.5141e-05 2023-09-23T18:31:42.893 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.893 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.893 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:31:42.893 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.866097+0000", 2023-09-23T18:31:42.893 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.7320000000000004e-06 2023-09-23T18:31:42.893 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.893 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.893 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:31:42.893 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.883491+0000", 2023-09-23T18:31:42.893 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.017394353000000001 2023-09-23T18:31:42.893 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.894 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.894 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:31:42.894 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.883521+0000", 2023-09-23T18:31:42.894 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.0363000000000001e-05 2023-09-23T18:31:42.894 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.894 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.894 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:31:42.894 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.886645+0000", 2023-09-23T18:31:42.894 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0031236990000000002 2023-09-23T18:31:42.894 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.894 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.894 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:31:42.895 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.886658+0000", 2023-09-23T18:31:42.895 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3227999999999999e-05 2023-09-23T18:31:42.895 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.895 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:31:42.895 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.895 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.895 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.895 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.44693.0:16820 5.2 e111/107 5:408ef824:::benchmark_data_smithi098_105788_object16819:head v 111'1062, mlcod=111'1058)", 2023-09-23T18:31:42.895 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:31:37.866821+0000", 2023-09-23T18:31:42.895 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 4.9742583390000004, 2023-09-23T18:31:42.895 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.020062703000000001, 2023-09-23T18:31:42.895 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:31:42.896 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:31:42.896 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:31:42.896 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.896 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:31:42.896 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.866821+0000", 2023-09-23T18:31:42.896 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.896 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.896 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.896 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:31:42.896 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.866821+0000", 2023-09-23T18:31:42.896 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.896 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.897 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.897 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:31:42.897 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.866820+0000", 2023-09-23T18:31:42.897 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:31:42.897 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.897 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.897 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:31:42.897 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.866893+0000", 2023-09-23T18:31:42.897 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.2476000000000002e-05 2023-09-23T18:31:42.897 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.897 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.898 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:31:42.898 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.866908+0000", 2023-09-23T18:31:42.898 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.5335000000000001e-05 2023-09-23T18:31:42.898 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.898 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.898 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:31:42.898 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.866925+0000", 2023-09-23T18:31:42.898 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.6484999999999999e-05 2023-09-23T18:31:42.898 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.898 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.898 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:31:42.899 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.884619+0000", 2023-09-23T18:31:42.899 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.017694239 2023-09-23T18:31:42.899 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.899 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.899 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:31:42.899 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.884641+0000", 2023-09-23T18:31:42.899 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.1696999999999999e-05 2023-09-23T18:31:42.899 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.899 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.899 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:31:42.899 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.886852+0000", 2023-09-23T18:31:42.899 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0022112690000000001 2023-09-23T18:31:42.899 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.900 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.900 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:31:42.900 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:37.886884+0000", 2023-09-23T18:31:42.900 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.1927000000000001e-05 2023-09-23T18:31:42.900 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.900 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:31:42.900 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.900 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.900 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.900 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.44693.0:23973 5.3 5:c15839e2:::benchmark_data_smithi098_105788_object23972:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-09-23T18:31:42.900 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:31:40.987850+0000", 2023-09-23T18:31:42.900 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 1.8532297899999999, 2023-09-23T18:31:42.901 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.064116271000000002, 2023-09-23T18:31:42.901 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:31:42.901 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:31:42.901 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:31:42.901 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.44693", 2023-09-23T18:31:42.901 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:31:42.901 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 23973 2023-09-23T18:31:42.901 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.901 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:31:42.901 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.901 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:31:42.901 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.987850+0000", 2023-09-23T18:31:42.901 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.902 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.902 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.902 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:31:42.902 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.987850+0000", 2023-09-23T18:31:42.902 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.902 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.902 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.902 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:31:42.902 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.987849+0000", 2023-09-23T18:31:42.902 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:31:42.902 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.902 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.903 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:31:42.903 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.987873+0000", 2023-09-23T18:31:42.903 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.4142e-05 2023-09-23T18:31:42.903 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.903 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.903 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:31:42.903 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.987882+0000", 2023-09-23T18:31:42.903 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.4169999999999998e-06 2023-09-23T18:31:42.903 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.903 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.903 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:31:42.904 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.987888+0000", 2023-09-23T18:31:42.904 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.6269999999999997e-06 2023-09-23T18:31:42.904 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.904 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.904 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:31:42.904 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.987943+0000", 2023-09-23T18:31:42.904 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.5068000000000002e-05 2023-09-23T18:31:42.904 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.904 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.904 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:31:42.905 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.988138+0000", 2023-09-23T18:31:42.905 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00019465800000000001 2023-09-23T18:31:42.905 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.905 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.905 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 5,6", 2023-09-23T18:31:42.905 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.988299+0000", 2023-09-23T18:31:42.906 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000161569 2023-09-23T18:31:42.906 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.906 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.906 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:31:42.906 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.989468+0000", 2023-09-23T18:31:42.906 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0011692079999999999 2023-09-23T18:31:42.906 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.906 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.906 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:31:42.906 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.051825+0000", 2023-09-23T18:31:42.907 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.062356574999999997 2023-09-23T18:31:42.907 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.907 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.907 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:31:42.907 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.051862+0000", 2023-09-23T18:31:42.907 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.7373999999999998e-05 2023-09-23T18:31:42.907 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.907 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.907 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:31:42.907 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.051924+0000", 2023-09-23T18:31:42.907 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.1445999999999994e-05 2023-09-23T18:31:42.907 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.907 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.908 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:31:42.908 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.051966+0000", 2023-09-23T18:31:42.908 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.2055999999999998e-05 2023-09-23T18:31:42.908 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.908 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:31:42.908 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.908 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.908 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.908 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.44693.0:23974 5.5 5:ab85e4ec:::benchmark_data_smithi098_105788_object23973:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-09-23T18:31:42.908 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:31:40.988041+0000", 2023-09-23T18:31:42.908 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 1.8530382059999999, 2023-09-23T18:31:42.908 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.066822688000000005, 2023-09-23T18:31:42.908 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:31:42.909 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:31:42.909 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:31:42.909 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.44693", 2023-09-23T18:31:42.909 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:31:42.909 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 23974 2023-09-23T18:31:42.909 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.909 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:31:42.909 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.909 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:31:42.909 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.988041+0000", 2023-09-23T18:31:42.909 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.909 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.909 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.910 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:31:42.910 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.988041+0000", 2023-09-23T18:31:42.910 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.910 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.910 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.910 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:31:42.910 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.988040+0000", 2023-09-23T18:31:42.910 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:31:42.910 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.910 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.910 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:31:42.910 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.988071+0000", 2023-09-23T18:31:42.911 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.1072000000000003e-05 2023-09-23T18:31:42.911 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.911 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.911 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:31:42.911 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.988080+0000", 2023-09-23T18:31:42.911 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.4570000000000005e-06 2023-09-23T18:31:42.911 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.911 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.911 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:31:42.911 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.988085+0000", 2023-09-23T18:31:42.911 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.8210000000000001e-06 2023-09-23T18:31:42.911 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.912 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.912 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:31:42.912 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.988291+0000", 2023-09-23T18:31:42.912 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000205966 2023-09-23T18:31:42.912 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.912 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.912 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:31:42.912 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.988375+0000", 2023-09-23T18:31:42.912 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.4110999999999995e-05 2023-09-23T18:31:42.912 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.912 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.912 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 4", 2023-09-23T18:31:42.912 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.988495+0000", 2023-09-23T18:31:42.913 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000120479 2023-09-23T18:31:42.913 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.913 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.913 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:31:42.913 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.989710+0000", 2023-09-23T18:31:42.913 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.001214358 2023-09-23T18:31:42.913 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.913 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.913 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:31:42.913 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.053387+0000", 2023-09-23T18:31:42.913 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.063677833000000003 2023-09-23T18:31:42.913 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.913 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.914 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:31:42.914 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.053456+0000", 2023-09-23T18:31:42.914 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.8091999999999998e-05 2023-09-23T18:31:42.914 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.914 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.914 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:31:42.914 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.054864+0000", 2023-09-23T18:31:42.914 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0014083979999999999 2023-09-23T18:31:42.914 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.914 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:31:42.914 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.914 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.915 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.915 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.44693.0:23977 5.5 5:af7cf0d4:::benchmark_data_smithi098_105788_object23976:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-09-23T18:31:42.915 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:31:40.988336+0000", 2023-09-23T18:31:42.915 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 1.8527435479999999, 2023-09-23T18:31:42.915 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.066995752000000006, 2023-09-23T18:31:42.915 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:31:42.915 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:31:42.915 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:31:42.915 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.44693", 2023-09-23T18:31:42.915 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:31:42.915 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 23977 2023-09-23T18:31:42.915 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.916 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:31:42.916 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.916 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:31:42.916 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.988336+0000", 2023-09-23T18:31:42.916 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.916 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.916 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.916 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:31:42.916 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.988336+0000", 2023-09-23T18:31:42.916 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.916 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.916 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.916 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:31:42.916 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.988335+0000", 2023-09-23T18:31:42.917 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:31:42.917 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.917 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.917 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:31:42.917 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.988373+0000", 2023-09-23T18:31:42.917 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.8345e-05 2023-09-23T18:31:42.917 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.917 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.917 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:31:42.917 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.988387+0000", 2023-09-23T18:31:42.917 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3981999999999999e-05 2023-09-23T18:31:42.917 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.918 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.918 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:31:42.918 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.988394+0000", 2023-09-23T18:31:42.918 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.5349999999999999e-06 2023-09-23T18:31:42.918 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.918 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.918 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:31:42.918 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.989096+0000", 2023-09-23T18:31:42.918 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00070168499999999996 2023-09-23T18:31:42.918 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.918 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.918 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:31:42.918 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.989188+0000", 2023-09-23T18:31:42.919 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.1667000000000002e-05 2023-09-23T18:31:42.919 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.919 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.919 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 4", 2023-09-23T18:31:42.919 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.989278+0000", 2023-09-23T18:31:42.919 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.0231999999999999e-05 2023-09-23T18:31:42.919 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.919 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.919 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:31:42.919 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.989862+0000", 2023-09-23T18:31:42.919 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00058384099999999998 2023-09-23T18:31:42.919 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.919 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.920 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:31:42.920 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.055205+0000", 2023-09-23T18:31:42.920 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.065343340999999999 2023-09-23T18:31:42.920 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.920 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.920 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:31:42.920 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.055256+0000", 2023-09-23T18:31:42.920 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.1184000000000003e-05 2023-09-23T18:31:42.920 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.920 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.920 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:31:42.920 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.055332+0000", 2023-09-23T18:31:42.921 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.5291999999999997e-05 2023-09-23T18:31:42.921 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.921 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:31:42.921 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.921 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.921 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.921 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.44693.0:23972 5.7 e111/107 5:e1e21634:::benchmark_data_smithi098_105788_object23971:head v 111'1493, mlcod=111'1490)", 2023-09-23T18:31:42.921 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:31:40.990538+0000", 2023-09-23T18:31:42.921 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 1.8505415219999999, 2023-09-23T18:31:42.921 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.064154151000000006, 2023-09-23T18:31:42.921 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:31:42.922 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:31:42.922 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:31:42.922 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.922 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:31:42.922 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.990538+0000", 2023-09-23T18:31:42.922 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.922 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.922 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.922 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:31:42.922 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.990538+0000", 2023-09-23T18:31:42.922 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.922 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.923 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.923 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:31:42.923 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.990538+0000", 2023-09-23T18:31:42.923 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:31:42.923 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.923 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.923 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:31:42.923 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.990611+0000", 2023-09-23T18:31:42.923 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.3100000000000001e-05 2023-09-23T18:31:42.923 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.923 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.923 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:31:42.924 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.990619+0000", 2023-09-23T18:31:42.924 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.3000000000000002e-06 2023-09-23T18:31:42.924 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.924 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.924 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:31:42.924 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.990624+0000", 2023-09-23T18:31:42.924 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.8160000000000001e-06 2023-09-23T18:31:42.924 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.924 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.924 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:31:42.924 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.990648+0000", 2023-09-23T18:31:42.924 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.4607e-05 2023-09-23T18:31:42.925 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.925 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.925 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:31:42.925 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.990661+0000", 2023-09-23T18:31:42.925 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3079e-05 2023-09-23T18:31:42.925 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.925 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.925 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:31:42.925 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.054627+0000", 2023-09-23T18:31:42.925 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.063966016000000001 2023-09-23T18:31:42.925 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.925 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.925 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:31:42.926 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.054692+0000", 2023-09-23T18:31:42.926 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.4655999999999999e-05 2023-09-23T18:31:42.926 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.926 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:31:42.926 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.926 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.926 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.926 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop_reply(client.44693.0:23973 5.3 e111/107 ondisk, result = 0)", 2023-09-23T18:31:42.926 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:31:40.991583+0000", 2023-09-23T18:31:42.926 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 1.849496392, 2023-09-23T18:31:42.926 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.060389527999999998, 2023-09-23T18:31:42.927 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:31:42.927 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T18:31:42.927 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:31:42.927 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.927 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:31:42.927 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.991583+0000", 2023-09-23T18:31:42.927 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.927 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.927 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.927 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:31:42.927 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.991583+0000", 2023-09-23T18:31:42.928 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.928 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.928 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.928 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:31:42.928 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.991582+0000", 2023-09-23T18:31:42.928 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:31:42.928 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.928 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.928 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:31:42.928 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.991587+0000", 2023-09-23T18:31:42.928 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.4229999999999998e-06 2023-09-23T18:31:42.928 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.929 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.929 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:31:42.929 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.051215+0000", 2023-09-23T18:31:42.929 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.059628079 2023-09-23T18:31:42.929 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.929 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.929 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:31:42.929 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.051242+0000", 2023-09-23T18:31:42.929 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.6704e-05 2023-09-23T18:31:42.929 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.929 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.930 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:31:42.930 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.051850+0000", 2023-09-23T18:31:42.930 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00060809899999999997 2023-09-23T18:31:42.930 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.930 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.930 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:31:42.930 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.051857+0000", 2023-09-23T18:31:42.930 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.1760000000000004e-06 2023-09-23T18:31:42.930 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.930 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.930 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:31:42.931 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.051973+0000", 2023-09-23T18:31:42.931 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00011568100000000001 2023-09-23T18:31:42.931 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.931 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:31:42.931 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.931 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.931 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.931 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop_reply(client.44693.0:23973 5.3 e111/107 ondisk, result = 0)", 2023-09-23T18:31:42.931 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:31:40.992322+0000", 2023-09-23T18:31:42.931 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 1.8487577229999999, 2023-09-23T18:31:42.931 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.059507559000000002, 2023-09-23T18:31:42.931 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:31:42.931 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T18:31:42.932 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:31:42.932 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.932 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:31:42.932 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.992322+0000", 2023-09-23T18:31:42.932 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.932 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.932 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.932 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:31:42.932 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.992322+0000", 2023-09-23T18:31:42.932 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.932 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.932 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.933 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:31:42.933 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.992321+0000", 2023-09-23T18:31:42.933 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:31:42.933 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.933 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.933 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:31:42.933 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.992326+0000", 2023-09-23T18:31:42.933 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.5449999999999997e-06 2023-09-23T18:31:42.933 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.933 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.933 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:31:42.934 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.051201+0000", 2023-09-23T18:31:42.934 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.058875584000000002 2023-09-23T18:31:42.934 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.934 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.934 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:31:42.934 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.051214+0000", 2023-09-23T18:31:42.934 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.2989e-05 2023-09-23T18:31:42.934 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.934 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.934 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:31:42.934 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.051808+0000", 2023-09-23T18:31:42.934 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00059396999999999998 2023-09-23T18:31:42.935 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.935 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.935 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:31:42.935 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.051818+0000", 2023-09-23T18:31:42.935 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.8449999999999998e-06 2023-09-23T18:31:42.935 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.935 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.935 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:31:42.935 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.051829+0000", 2023-09-23T18:31:42.935 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1282e-05 2023-09-23T18:31:42.935 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.935 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:31:42.936 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.936 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.936 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.936 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.44693.0:23992 5.5 5:ae418cef:::benchmark_data_smithi098_105788_object23991:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-09-23T18:31:42.936 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:31:40.999321+0000", 2023-09-23T18:31:42.936 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 1.84175887, 2023-09-23T18:31:42.936 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.063899532999999994, 2023-09-23T18:31:42.936 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:31:42.936 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:31:42.936 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:31:42.936 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.44693", 2023-09-23T18:31:42.936 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:31:42.937 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 23992 2023-09-23T18:31:42.937 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.937 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:31:42.937 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.937 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:31:42.937 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.999321+0000", 2023-09-23T18:31:42.937 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.937 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.937 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.937 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:31:42.937 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.999321+0000", 2023-09-23T18:31:42.937 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.937 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.938 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.938 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:31:42.938 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.999319+0000", 2023-09-23T18:31:42.938 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T18:31:42.938 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.938 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.938 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:31:42.938 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:40.999348+0000", 2023-09-23T18:31:42.938 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.8875999999999999e-05 2023-09-23T18:31:42.938 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.938 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.939 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:31:42.939 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.051247+0000", 2023-09-23T18:31:42.939 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.051898950999999999 2023-09-23T18:31:42.939 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.939 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.939 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:31:42.939 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.051277+0000", 2023-09-23T18:31:42.939 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.0300000000000001e-05 2023-09-23T18:31:42.939 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.939 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.939 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:31:42.939 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.051496+0000", 2023-09-23T18:31:42.940 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00021914300000000001 2023-09-23T18:31:42.940 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.940 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.940 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:31:42.940 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.051686+0000", 2023-09-23T18:31:42.940 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00018957300000000001 2023-09-23T18:31:42.940 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.940 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.940 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 4", 2023-09-23T18:31:42.940 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.051878+0000", 2023-09-23T18:31:42.940 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000192759 2023-09-23T18:31:42.941 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.941 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.941 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:31:42.941 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.054886+0000", 2023-09-23T18:31:42.941 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.003007366 2023-09-23T18:31:42.941 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.941 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.941 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:31:42.941 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.063050+0000", 2023-09-23T18:31:42.941 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0081645469999999994 2023-09-23T18:31:42.941 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.941 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.942 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:31:42.942 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.063102+0000", 2023-09-23T18:31:42.942 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.1232e-05 2023-09-23T18:31:42.942 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.942 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.942 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:31:42.942 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.063220+0000", 2023-09-23T18:31:42.942 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000118484 2023-09-23T18:31:42.942 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.942 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:31:42.942 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.942 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.943 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.943 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.44693.0:25573 5.7 e111/107 5:e2d87f26:::benchmark_data_smithi098_105788_object25572:head v 111'1587, mlcod=111'1585)", 2023-09-23T18:31:42.943 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:31:41.774948+0000", 2023-09-23T18:31:42.943 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 1.0661310509999999, 2023-09-23T18:31:42.943 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.020602141000000001, 2023-09-23T18:31:42.943 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:31:42.943 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:31:42.943 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:31:42.943 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.943 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:31:42.943 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.774948+0000", 2023-09-23T18:31:42.943 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.944 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.944 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.944 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:31:42.944 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.774948+0000", 2023-09-23T18:31:42.944 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.944 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.944 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.944 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:31:42.944 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.774948+0000", 2023-09-23T18:31:42.944 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:31:42.944 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.944 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.945 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:31:42.945 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.775018+0000", 2023-09-23T18:31:42.945 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.0020999999999999e-05 2023-09-23T18:31:42.945 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.945 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.945 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:31:42.945 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.775027+0000", 2023-09-23T18:31:42.945 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.6020000000000006e-06 2023-09-23T18:31:42.945 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.945 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.945 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:31:42.945 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.775033+0000", 2023-09-23T18:31:42.946 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.6110000000000001e-06 2023-09-23T18:31:42.946 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.946 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.946 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:31:42.946 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.775077+0000", 2023-09-23T18:31:42.946 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.3952000000000002e-05 2023-09-23T18:31:42.946 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.946 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.946 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:31:42.946 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.775091+0000", 2023-09-23T18:31:42.946 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3923999999999999e-05 2023-09-23T18:31:42.946 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.947 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.947 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:31:42.947 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.775628+0000", 2023-09-23T18:31:42.947 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00053703799999999999 2023-09-23T18:31:42.947 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.947 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.947 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:31:42.947 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:41.795551+0000", 2023-09-23T18:31:42.947 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.019922583000000001 2023-09-23T18:31:42.947 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.947 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:31:42.947 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.948 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.948 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.948 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.44693.0:27100 5.3 5:cbb58f49:::benchmark_data_smithi098_105788_object27099:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-09-23T18:31:42.948 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:31:42.538031+0000", 2023-09-23T18:31:42.948 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.30304848600000001, 2023-09-23T18:31:42.948 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.023496535999999998, 2023-09-23T18:31:42.948 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:31:42.948 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:31:42.948 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:31:42.948 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.44693", 2023-09-23T18:31:42.948 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:31:42.948 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 27100 2023-09-23T18:31:42.948 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.949 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:31:42.949 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.949 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:31:42.949 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.538031+0000", 2023-09-23T18:31:42.949 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.949 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.949 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.949 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:31:42.949 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.538031+0000", 2023-09-23T18:31:42.949 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.949 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.949 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.950 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:31:42.950 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.538030+0000", 2023-09-23T18:31:42.950 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:31:42.950 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.950 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.950 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:31:42.950 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.538048+0000", 2023-09-23T18:31:42.950 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.7388e-05 2023-09-23T18:31:42.950 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.950 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.950 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:31:42.950 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.538066+0000", 2023-09-23T18:31:42.951 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.8433000000000001e-05 2023-09-23T18:31:42.951 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.951 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.951 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:31:42.951 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.538072+0000", 2023-09-23T18:31:42.951 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.6370000000000004e-06 2023-09-23T18:31:42.951 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.951 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.951 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:31:42.951 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.541707+0000", 2023-09-23T18:31:42.951 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00363532 2023-09-23T18:31:42.951 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.951 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.952 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:31:42.952 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.541839+0000", 2023-09-23T18:31:42.952 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00013172300000000001 2023-09-23T18:31:42.952 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.952 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.952 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 5,6", 2023-09-23T18:31:42.952 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.541958+0000", 2023-09-23T18:31:42.952 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000119227 2023-09-23T18:31:42.952 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.952 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.952 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:31:42.952 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.544431+0000", 2023-09-23T18:31:42.952 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0024731459999999999 2023-09-23T18:31:42.953 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.953 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.953 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:31:42.953 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.544714+0000", 2023-09-23T18:31:42.953 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00028299899999999998 2023-09-23T18:31:42.953 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.953 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.953 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:31:42.953 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.561443+0000", 2023-09-23T18:31:42.953 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.016729223000000001 2023-09-23T18:31:42.953 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.953 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.954 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:31:42.954 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.561492+0000", 2023-09-23T18:31:42.954 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.8309000000000001e-05 2023-09-23T18:31:42.954 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.954 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.954 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:31:42.954 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.561527+0000", 2023-09-23T18:31:42.954 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.5836e-05 2023-09-23T18:31:42.954 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.954 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:31:42.954 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.955 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.955 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.955 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.44693.0:27445 5.3 5:c01414e2:::benchmark_data_smithi098_105788_object27444:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-09-23T18:31:42.955 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:31:42.720527+0000", 2023-09-23T18:31:42.955 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.12055228, 2023-09-23T18:31:42.955 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.027367328, 2023-09-23T18:31:42.955 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:31:42.955 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:31:42.955 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:31:42.955 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.44693", 2023-09-23T18:31:42.955 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:31:42.955 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 27445 2023-09-23T18:31:42.955 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.956 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:31:42.956 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.956 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:31:42.956 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.720527+0000", 2023-09-23T18:31:42.956 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.956 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.956 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.956 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:31:42.956 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.720527+0000", 2023-09-23T18:31:42.956 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.956 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.956 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.957 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:31:42.957 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.720525+0000", 2023-09-23T18:31:42.957 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T18:31:42.957 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.957 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.957 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:31:42.957 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.720556+0000", 2023-09-23T18:31:42.957 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.0889000000000002e-05 2023-09-23T18:31:42.957 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.957 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.957 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:31:42.957 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.720572+0000", 2023-09-23T18:31:42.958 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.5894000000000001e-05 2023-09-23T18:31:42.958 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.958 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.958 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:31:42.958 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.720581+0000", 2023-09-23T18:31:42.958 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.0569999999999999e-06 2023-09-23T18:31:42.958 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.958 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.958 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:31:42.958 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.720638+0000", 2023-09-23T18:31:42.958 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.6653000000000001e-05 2023-09-23T18:31:42.958 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.959 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.959 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:31:42.959 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.720785+0000", 2023-09-23T18:31:42.959 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000147675 2023-09-23T18:31:42.959 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.959 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.959 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 5,6", 2023-09-23T18:31:42.959 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.720927+0000", 2023-09-23T18:31:42.959 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00014196400000000001 2023-09-23T18:31:42.959 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.959 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.960 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:31:42.960 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.722842+0000", 2023-09-23T18:31:42.960 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0019144139999999999 2023-09-23T18:31:42.960 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.960 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.960 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:31:42.960 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.728258+0000", 2023-09-23T18:31:42.960 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0054166020000000004 2023-09-23T18:31:42.960 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.960 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.960 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:31:42.960 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.747794+0000", 2023-09-23T18:31:42.961 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.019536010999999999 2023-09-23T18:31:42.961 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.961 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.961 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:31:42.961 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.747852+0000", 2023-09-23T18:31:42.961 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.7361999999999997e-05 2023-09-23T18:31:42.961 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.961 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.961 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:31:42.961 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.747894+0000", 2023-09-23T18:31:42.961 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.2941000000000001e-05 2023-09-23T18:31:42.961 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.962 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:31:42.962 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.962 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.962 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.962 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.44693.0:27450 5.3 5:c2a7f55b:::benchmark_data_smithi098_105788_object27449:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-09-23T18:31:42.962 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:31:42.724456+0000", 2023-09-23T18:31:42.962 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.11662307800000001, 2023-09-23T18:31:42.962 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.023620477000000001, 2023-09-23T18:31:42.962 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:31:42.962 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:31:42.962 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:31:42.963 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.44693", 2023-09-23T18:31:42.963 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:31:42.963 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 27450 2023-09-23T18:31:42.963 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.963 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:31:42.963 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.963 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:31:42.963 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.724456+0000", 2023-09-23T18:31:42.963 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.963 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.963 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.963 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:31:42.963 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.724456+0000", 2023-09-23T18:31:42.964 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:31:42.964 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.964 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.964 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:31:42.964 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.724454+0000", 2023-09-23T18:31:42.964 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999971 2023-09-23T18:31:42.964 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.964 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.964 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:31:42.964 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.724487+0000", 2023-09-23T18:31:42.964 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.3506000000000001e-05 2023-09-23T18:31:42.964 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.965 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.965 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:31:42.965 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.724508+0000", 2023-09-23T18:31:42.965 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.0562000000000001e-05 2023-09-23T18:31:42.965 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.965 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.965 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:31:42.965 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.724515+0000", 2023-09-23T18:31:42.965 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.5360000000000004e-06 2023-09-23T18:31:42.965 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.965 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.965 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:31:42.966 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.724578+0000", 2023-09-23T18:31:42.966 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.2516999999999998e-05 2023-09-23T18:31:42.966 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.966 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.966 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:31:42.966 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.724696+0000", 2023-09-23T18:31:42.966 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000118208 2023-09-23T18:31:42.966 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.966 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.966 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 5,6", 2023-09-23T18:31:42.966 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.724804+0000", 2023-09-23T18:31:42.966 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00010770200000000001 2023-09-23T18:31:42.966 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.967 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.967 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:31:42.967 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.726175+0000", 2023-09-23T18:31:42.967 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0013715279999999999 2023-09-23T18:31:42.967 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.967 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.967 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:31:42.967 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.728266+0000", 2023-09-23T18:31:42.967 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0020911990000000002 2023-09-23T18:31:42.967 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.967 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.967 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:31:42.968 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.748006+0000", 2023-09-23T18:31:42.968 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.019739300000000001 2023-09-23T18:31:42.968 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.968 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.968 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:31:42.968 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.748045+0000", 2023-09-23T18:31:42.968 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.9217000000000001e-05 2023-09-23T18:31:42.968 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:31:42.968 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:31:42.968 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:31:42.968 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:31:42.748077+0000", 2023-09-23T18:31:42.968 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.1967000000000001e-05 2023-09-23T18:31:42.969 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.969 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:31:42.969 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.969 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:31:42.969 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:31:42.969 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:31:42.970 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:42 smithi098 bash[76817]: cluster 2023-09-23T18:31:41.672884+0000 mgr.y (mgr.26054) 391 : cluster 0 pgmap v331: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 1.4 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 144 MiB/s wr, 2.30k op/s; 7042/69189 objects degraded (10.178%) 2023-09-23T18:31:42.970 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:42 smithi098 bash[79516]: cluster 2023-09-23T18:31:41.672884+0000 mgr.y (mgr.26054) 391 : cluster 0 pgmap v331: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 1.4 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 144 MiB/s wr, 2.30k op/s; 7042/69189 objects degraded (10.178%) 2023-09-23T18:31:42.998 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:43.000 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:42 smithi173 bash[59422]: cluster 2023-09-23T18:31:41.672884+0000 mgr.y (mgr.26054) 391 : cluster 0 pgmap v331: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 1.4 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 144 MiB/s wr, 2.30k op/s; 7042/69189 objects degraded (10.178%) 2023-09-23T18:31:43.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:43.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:43.101 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:43.130 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:43.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:43.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:43.236 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:43.261 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:43.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:43.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:43.365 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:43.392 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:43.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:43.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:43.495 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:43.520 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:43.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:43.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:43.589 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 12 16 29159 29143 151.763 123.375 0.0052192 0.00658306 2023-09-23T18:31:43.624 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:43.652 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:43.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:43.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:43.756 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:43.784 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:43.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:43.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:43.891 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:43.915 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:43.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:43.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:44.019 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:44.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:43 smithi173 bash[59422]: cluster 2023-09-23T18:31:43.731433+0000 mon.a (mon.0) 553 : cluster 3 Health check update: Degraded data redundancy: 7707/73182 objects degraded (10.531%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T18:31:44.029 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:43 smithi098 bash[76817]: cluster 2023-09-23T18:31:43.731433+0000 mon.a (mon.0) 553 : cluster 3 Health check update: Degraded data redundancy: 7707/73182 objects degraded (10.531%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T18:31:44.029 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:43 smithi098 bash[79516]: cluster 2023-09-23T18:31:43.731433+0000 mon.a (mon.0) 553 : cluster 3 Health check update: Degraded data redundancy: 7707/73182 objects degraded (10.531%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T18:31:44.052 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:44.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:44.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:44.156 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:44.183 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:44.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:44.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:44.288 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:44.317 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:44.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:44.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:44.422 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:44.449 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:44.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:44.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:44.553 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:44.582 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:44.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:44.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:44.588 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 13 16 31253 31237 150.156 130.875 0.00949702 0.00665403 2023-09-23T18:31:44.686 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:44.716 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:44.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:44.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:44.819 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:44.827 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:44 smithi173 bash[59422]: cluster 2023-09-23T18:31:43.673604+0000 mgr.y (mgr.26054) 392 : cluster 0 pgmap v332: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 1.5 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s; 7707/73182 objects degraded (10.531%) 2023-09-23T18:31:44.845 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:44.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:44.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:44.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:44 smithi098 bash[79516]: cluster 2023-09-23T18:31:43.673604+0000 mgr.y (mgr.26054) 392 : cluster 0 pgmap v332: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 1.5 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s; 7707/73182 objects degraded (10.531%) 2023-09-23T18:31:44.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:44 smithi098 bash[76817]: cluster 2023-09-23T18:31:43.673604+0000 mgr.y (mgr.26054) 392 : cluster 0 pgmap v332: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 1.5 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s; 7707/73182 objects degraded (10.531%) 2023-09-23T18:31:44.949 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:44.980 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:44.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:44.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:45.086 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:45.109 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:45.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:45.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:45.212 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:45.238 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:45.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:45.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:45.328 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T18:31:45.342 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:45.368 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:45.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:45.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:45.471 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:45.496 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:45.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:45.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:45.591 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 14 16 33166 33150 147.942 119.562 0.0132154 0.00675329 2023-09-23T18:31:45.600 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:45.627 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:45.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:45.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:45.730 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:45.754 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:45.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:45.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:45.860 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:45.889 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:45.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:45.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:45.993 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:46.020 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:46.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:46.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:46.087 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:31:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:31:46] "GET /metrics HTTP/1.1" 200 36507 "" "Prometheus/2.33.4" 2023-09-23T18:31:46.123 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:46.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:46.155 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:46.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:46.256 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:46.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:46.297 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:46.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:46.399 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:46.422 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:46.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:46.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:46.525 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:46.553 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:46.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:46.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:46.594 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 15 16 35037 35021 145.875 116.938 0.00482502 0.00684918 2023-09-23T18:31:46.656 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:46.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:46.690 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:46.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:46.792 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:46.824 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:46.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:46.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:46.857 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:46 smithi098 bash[79516]: cluster 2023-09-23T18:31:45.674964+0000 mgr.y (mgr.26054) 393 : cluster 0 pgmap v333: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 1.8 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s; 8964/88326 objects degraded (10.149%) 2023-09-23T18:31:46.930 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:46.960 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:46.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:46.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:47.065 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:47.094 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:47.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:47.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:47.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:46 smithi098 bash[76817]: cluster 2023-09-23T18:31:45.674964+0000 mgr.y (mgr.26054) 393 : cluster 0 pgmap v333: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 1.8 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s; 8964/88326 objects degraded (10.149%) 2023-09-23T18:31:47.199 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:47.232 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:47.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:47.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:47.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:46 smithi173 bash[59422]: cluster 2023-09-23T18:31:45.674964+0000 mgr.y (mgr.26054) 393 : cluster 0 pgmap v333: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 1.8 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s; 8964/88326 objects degraded (10.149%) 2023-09-23T18:31:47.335 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:47.360 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:47.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:47.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:47.463 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:47.498 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:47.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:47.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:47.592 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 16 16 36877 36861 143.945 115 0.0061729 0.0069416 2023-09-23T18:31:47.603 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:47.633 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:47.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:47.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:47.736 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:47.764 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:47.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:47.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:47.867 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:47.893 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:47.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:47.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:47.997 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:48.024 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:48.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:48.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:48.127 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:48.155 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:48.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:48.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:48.258 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:48.282 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:48.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:48.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:48.382 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:31:48.390 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:48.414 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:48.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:48.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:48.459 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:31:48.499 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:31:48.517 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:48.546 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:48.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:48.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:48.561 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:31:48.592 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 17 16 38524 38508 141.533 102.938 0.0129208 0.00705866 2023-09-23T18:31:48.594 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:31:48.630 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:31:48.652 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:48.671 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:31:48.672 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:31:48.672 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T18:31:48.672 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:31:48.672 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:31:48.672 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:31:48.672 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:31:48.672 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:31:48.672 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:31:48.684 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:48.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:48.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:48.788 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:48.817 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:48.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:48.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:48.824 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is unset 2023-09-23T18:31:48.921 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:48.922 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:31:48.933 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-23T18:31:48.951 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:48.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:48.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:49.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:48 smithi173 bash[59422]: cluster 2023-09-23T18:31:47.675582+0000 mgr.y (mgr.26054) 394 : cluster 0 pgmap v334: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 2.0 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s; 10248/99354 objects degraded (10.315%) 2023-09-23T18:31:49.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:48 smithi173 bash[59422]: audit 2023-09-23T18:31:48.330895+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.98:0/1540912243' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:31:49.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:48 smithi173 bash[59422]: audit 2023-09-23T18:31:48.796903+0000 mon.c (mon.2) 53 : audit 1 from='client.? 172.21.15.98:0/3299569312' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:31:49.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:48 smithi173 bash[59422]: audit 2023-09-23T18:31:48.797768+0000 mon.a (mon.0) 554 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:31:49.055 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:49.063 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:48 smithi098 bash[76817]: cluster 2023-09-23T18:31:47.675582+0000 mgr.y (mgr.26054) 394 : cluster 0 pgmap v334: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 2.0 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s; 10248/99354 objects degraded (10.315%) 2023-09-23T18:31:49.064 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:48 smithi098 bash[76817]: audit 2023-09-23T18:31:48.330895+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.98:0/1540912243' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:31:49.064 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:48 smithi098 bash[76817]: audit 2023-09-23T18:31:48.796903+0000 mon.c (mon.2) 53 : audit 1 from='client.? 172.21.15.98:0/3299569312' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:31:49.064 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:48 smithi098 bash[76817]: audit 2023-09-23T18:31:48.797768+0000 mon.a (mon.0) 554 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:31:49.064 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:48 smithi098 bash[79516]: cluster 2023-09-23T18:31:47.675582+0000 mgr.y (mgr.26054) 394 : cluster 0 pgmap v334: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 2.0 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s; 10248/99354 objects degraded (10.315%) 2023-09-23T18:31:49.064 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:48 smithi098 bash[79516]: audit 2023-09-23T18:31:48.330895+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.98:0/1540912243' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:31:49.064 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:48 smithi098 bash[79516]: audit 2023-09-23T18:31:48.796903+0000 mon.c (mon.2) 53 : audit 1 from='client.? 172.21.15.98:0/3299569312' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:31:49.064 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:48 smithi098 bash[79516]: audit 2023-09-23T18:31:48.797768+0000 mon.a (mon.0) 554 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:31:49.080 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:49.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:49.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:49.183 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:49.208 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:49.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:49.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:49.312 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:49.339 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:49.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:49.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:49.442 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:49.477 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:49.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:49.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:49.583 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:49.592 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 18 16 40227 40211 139.583 106.438 0.00394422 0.00715669 2023-09-23T18:31:49.607 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:49.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:49.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:49.710 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:49.739 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:49.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:49.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:49.844 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:49.854 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:49 smithi098 bash[79516]: cluster 2023-09-23T18:31:48.806650+0000 mon.a (mon.0) 555 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:31:49.854 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:49 smithi098 bash[79516]: audit 2023-09-23T18:31:48.813132+0000 mon.a (mon.0) 556 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:31:49.854 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:49 smithi098 bash[79516]: cluster 2023-09-23T18:31:48.813186+0000 mon.a (mon.0) 557 : cluster 0 osdmap e112: 8 total, 7 up, 8 in 2023-09-23T18:31:49.854 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:49 smithi098 bash[79516]: cluster 2023-09-23T18:31:48.813756+0000 osd.6 (osd.6) 27 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T18:31:49.854 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:49 smithi098 bash[79516]: cluster 2023-09-23T18:31:48.819629+0000 osd.4 (osd.4) 42 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T18:31:49.854 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:49 smithi098 bash[79516]: cluster 2023-09-23T18:31:48.829362+0000 osd.4 (osd.4) 43 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T18:31:49.855 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:49 smithi098 bash[79516]: cluster 2023-09-23T18:31:48.829412+0000 osd.7 (osd.7) 26 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T18:31:49.855 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:49 smithi098 bash[79516]: cluster 2023-09-23T18:31:48.829464+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T18:31:49.855 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:49 smithi098 bash[79516]: cluster 2023-09-23T18:31:49.417439+0000 mon.a (mon.0) 558 : cluster 3 Health check update: Degraded data redundancy: 10248/99354 objects degraded (10.315%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T18:31:49.855 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:49 smithi098 bash[76817]: cluster 2023-09-23T18:31:48.806650+0000 mon.a (mon.0) 555 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:31:49.855 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:49 smithi098 bash[76817]: audit 2023-09-23T18:31:48.813132+0000 mon.a (mon.0) 556 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:31:49.856 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:49 smithi098 bash[76817]: cluster 2023-09-23T18:31:48.813186+0000 mon.a (mon.0) 557 : cluster 0 osdmap e112: 8 total, 7 up, 8 in 2023-09-23T18:31:49.856 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:49 smithi098 bash[76817]: cluster 2023-09-23T18:31:48.813756+0000 osd.6 (osd.6) 27 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T18:31:49.856 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:49 smithi098 bash[76817]: cluster 2023-09-23T18:31:48.819629+0000 osd.4 (osd.4) 42 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T18:31:49.856 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:49 smithi098 bash[76817]: cluster 2023-09-23T18:31:48.829362+0000 osd.4 (osd.4) 43 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T18:31:49.856 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:49 smithi098 bash[76817]: cluster 2023-09-23T18:31:48.829412+0000 osd.7 (osd.7) 26 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T18:31:49.856 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:49 smithi098 bash[76817]: cluster 2023-09-23T18:31:48.829464+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T18:31:49.856 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:49 smithi098 bash[76817]: cluster 2023-09-23T18:31:49.417439+0000 mon.a (mon.0) 558 : cluster 3 Health check update: Degraded data redundancy: 10248/99354 objects degraded (10.315%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T18:31:49.871 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:49.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:49.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:49.974 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:49.998 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:50.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:50.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:50.102 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:50.130 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:50.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:50.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:50.233 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:50.241 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:49 smithi173 bash[59422]: cluster 2023-09-23T18:31:48.806650+0000 mon.a (mon.0) 555 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:31:50.241 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:49 smithi173 bash[59422]: audit 2023-09-23T18:31:48.813132+0000 mon.a (mon.0) 556 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:31:50.242 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:49 smithi173 bash[59422]: cluster 2023-09-23T18:31:48.813186+0000 mon.a (mon.0) 557 : cluster 0 osdmap e112: 8 total, 7 up, 8 in 2023-09-23T18:31:50.242 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:49 smithi173 bash[59422]: cluster 2023-09-23T18:31:48.813756+0000 osd.6 (osd.6) 27 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T18:31:50.242 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:49 smithi173 bash[59422]: cluster 2023-09-23T18:31:48.819629+0000 osd.4 (osd.4) 42 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T18:31:50.242 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:49 smithi173 bash[59422]: cluster 2023-09-23T18:31:48.829362+0000 osd.4 (osd.4) 43 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T18:31:50.242 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:49 smithi173 bash[59422]: cluster 2023-09-23T18:31:48.829412+0000 osd.7 (osd.7) 26 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T18:31:50.242 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:49 smithi173 bash[59422]: cluster 2023-09-23T18:31:48.829464+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e112 with expected crc 2023-09-23T18:31:50.242 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:49 smithi173 bash[59422]: cluster 2023-09-23T18:31:49.417439+0000 mon.a (mon.0) 558 : cluster 3 Health check update: Degraded data redundancy: 10248/99354 objects degraded (10.315%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T18:31:50.269 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:50.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:50.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:50.375 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:50.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:50.404 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:50.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:50.505 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:50.530 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:50.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:50.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:50.592 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 19 16 41886 41870 137.693 103.688 0.0063801 0.00725708 2023-09-23T18:31:50.634 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:50.657 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:50.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:50.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:50.761 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:50.794 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:50.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:50.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:50.899 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:50.924 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:50.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:50.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:51.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:50 smithi173 bash[59422]: audit 2023-09-23T18:31:48.905688+0000 mgr.y (mgr.26054) 395 : audit 0 from='client.34800 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:31:51.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:50 smithi173 bash[59422]: cluster 2023-09-23T18:31:49.676054+0000 mgr.y (mgr.26054) 396 : cluster 0 pgmap v336: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 2.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 132 MiB/s wr, 2.10k op/s; 10881/103161 objects degraded (10.548%) 2023-09-23T18:31:51.030 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:51.064 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:51.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:51.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:51.167 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:51.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:50 smithi098 bash[79516]: audit 2023-09-23T18:31:48.905688+0000 mgr.y (mgr.26054) 395 : audit 0 from='client.34800 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:31:51.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:50 smithi098 bash[79516]: cluster 2023-09-23T18:31:49.676054+0000 mgr.y (mgr.26054) 396 : cluster 0 pgmap v336: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 2.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 132 MiB/s wr, 2.10k op/s; 10881/103161 objects degraded (10.548%) 2023-09-23T18:31:51.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:50 smithi098 bash[76817]: audit 2023-09-23T18:31:48.905688+0000 mgr.y (mgr.26054) 395 : audit 0 from='client.34800 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:31:51.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:50 smithi098 bash[76817]: cluster 2023-09-23T18:31:49.676054+0000 mgr.y (mgr.26054) 396 : cluster 0 pgmap v336: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 2.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 132 MiB/s wr, 2.10k op/s; 10881/103161 objects degraded (10.548%) 2023-09-23T18:31:51.192 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:51.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:51.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:51.296 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:51.320 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:51.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:51.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:51.423 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:51.449 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:51.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:51.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:51.552 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:51.576 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:51.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:51.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:51.592 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:2023-09-23T18:31:51.591917+0000 min lat: 0.00116463 max lat: 0.0924006 avg lat: 0.00733818 2023-09-23T18:31:51.592 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:31:51.592 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 20 16 43595 43579 136.149 106.812 0.00508317 0.00733818 2023-09-23T18:31:51.680 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:51.714 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:51.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:51.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:51.771 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:31:51.771 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:31:51.817 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:51.841 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:51.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:51.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:51.895 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":335,"stamp":"2023-09-23T18:31:48.810581+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2256541619,"num_objects":34387,"num_object_clones":0,"num_object_copies":103161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10881,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":34387,"num_whiteouts":0,"num_read":6303,"num_read_kb":13760,"num_write":35465,"num_write_kb":2214135,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":35529,"ondisk_log_size":35529,"up":230,"acting":230,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":262,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7175448,"kb_used_data":5809304,"kb_used_omap":31,"kb_used_meta":1366048,"kb_avail":742785768,"statfs":{"total":767960285184,"available":760612626432,"internally_reserved":0,"allocated":5948727296,"data_stored":5944573891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32507,"internal_metadata":1398833413},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[8,4,6,5,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1482948608,"num_objects":22628,"num_object_clones":0,"num_object_copies":67884,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7366,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22628,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22628,"num_write_kb":1448192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.139390"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":99,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:29.150331+0000","last_change":"2023-09-23T18:30:57.765424+0000","last_active":"2023-09-23T18:31:29.150331+0000","last_peered":"2023-09-23T18:31:29.150331+0000","last_clean":"2023-09-23T18:28:03.292943+0000","last_became_active":"2023-09-23T18:30:57.765424+0000","last_became_peered":"2023-09-23T18:30:57.765424+0000","last_unstale":"2023-09-23T18:31:29.150331+0000","last_undegraded":"2023-09-23T18:31:29.150331+0000","last_fullsized":"2023-09-23T18:30:57.757189+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:28.757614+0000","last_change":"2023-09-23T18:30:57.761478+0000","last_active":"2023-09-23T18:31:28.757614+0000","last_peered":"2023-09-23T18:31:28.757614+0000","last_clean":"2023-09-23T18:30:31.307538+0000","last_became_active":"2023-09-23T18:30:57.761478+0000","last_became_peered":"2023-09-23T18:30:57.761478+0000","last_unstale":"2023-09-23T18:31:28.757614+0000","last_undegraded":"2023-09-23T18:31:28.757614+0000","last_fullsized":"2023-09-23T18:30:57.742716+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:23.036992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:28.757387+0000","last_change":"2023-09-23T18:30:57.756090+0000","last_active":"2023-09-23T18:31:28.757387+0000","last_peered":"2023-09-23T18:31:28.757387+0000","last_clean":"2023-09-23T18:27:33.568258+0000","last_became_active":"2023-09-23T18:30:57.756090+0000","last_became_peered":"2023-09-23T18:30:57.756090+0000","last_unstale":"2023-09-23T18:31:28.757387+0000","last_undegraded":"2023-09-23T18:31:28.757387+0000","last_fullsized":"2023-09-23T18:30:57.742018+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:14.861981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":61,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.146622+0000","last_change":"2023-09-23T18:28:03.289500+0000","last_active":"2023-09-23T18:31:29.146622+0000","last_peered":"2023-09-23T18:31:29.146622+0000","last_clean":"2023-09-23T18:31:29.146622+0000","last_became_active":"2023-09-23T18:28:03.289122+0000","last_became_peered":"2023-09-23T18:28:03.289122+0000","last_unstale":"2023-09-23T18:31:29.146622+0000","last_undegraded":"2023-09-23T18:31:29.146622+0000","last_fullsized":"2023-09-23T18:31:29.146622+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:22:26.904660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":101,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:28.757938+0000","last_change":"2023-09-23T18:30:57.745533+0000","last_active":"2023-09-23T18:31:28.757938+0000","last_peered":"2023-09-23T18:31:28.757938+0000","last_clean":"2023-09-23T18:30:26.306231+0000","last_became_active":"2023-09-23T18:30:57.745533+0000","last_became_peered":"2023-09-23T18:30:57.745533+0000","last_unstale":"2023-09-23T18:31:28.757938+0000","last_undegraded":"2023-09-23T18:31:28.757938+0000","last_fullsized":"2023-09-23T18:30:57.741719+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:13.887774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":82,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:28.763568+0000","last_change":"2023-09-23T18:30:57.763593+0000","last_active":"2023-09-23T18:31:28.763568+0000","last_peered":"2023-09-23T18:31:28.763568+0000","last_clean":"2023-09-23T18:28:03.295680+0000","last_became_active":"2023-09-23T18:30:57.763593+0000","last_became_peered":"2023-09-23T18:30:57.763593+0000","last_unstale":"2023-09-23T18:31:28.763568+0000","last_undegraded":"2023-09-23T18:31:28.763568+0000","last_fullsized":"2023-09-23T18:30:57.758604+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":60,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.146932+0000","last_change":"2023-09-23T18:28:03.297439+0000","last_active":"2023-09-23T18:31:29.146932+0000","last_peered":"2023-09-23T18:31:29.146932+0000","last_clean":"2023-09-23T18:31:29.146932+0000","last_became_active":"2023-09-23T18:28:03.297112+0000","last_became_peered":"2023-09-23T18:28:03.297112+0000","last_unstale":"2023-09-23T18:31:29.146932+0000","last_undegraded":"2023-09-23T18:31:29.146932+0000","last_fullsized":"2023-09-23T18:31:29.146932+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:50.442238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:28.756614+0000","last_change":"2023-09-23T18:30:57.755753+0000","last_active":"2023-09-23T18:31:28.756614+0000","last_peered":"2023-09-23T18:31:28.756614+0000","last_clean":"2023-09-23T18:30:21.304949+0000","last_became_active":"2023-09-23T18:30:57.755753+0000","last_became_peered":"2023-09-23T18:30:57.755753+0000","last_unstale":"2023-09-23T18:31:28.756614+0000","last_undegraded":"2023-09-23T18:31:28.756614+0000","last_fullsized":"2023-09-23T18:30:57.741711+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:47.235882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":83,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:29.154225+0000","last_change":"2023-09-23T18:30:57.763985+0000","last_active":"2023-09-23T18:31:29.154225+0000","last_peered":"2023-09-23T18:31:29.154225+0000","last_clean":"2023-09-23T18:28:03.290643+0000","last_became_active":"2023-09-23T18:30:57.763985+0000","last_became_peered":"2023-09-23T18:30:57.763985+0000","last_unstale":"2023-09-23T18:31:29.154225+0000","last_undegraded":"2023-09-23T18:31:29.154225+0000","last_fullsized":"2023-09-23T18:30:57.753877+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:28.756937+0000","last_change":"2023-09-23T18:28:03.298214+0000","last_active":"2023-09-23T18:31:28.756937+0000","last_peered":"2023-09-23T18:31:28.756937+0000","last_clean":"2023-09-23T18:31:28.756937+0000","last_became_active":"2023-09-23T18:28:03.297852+0000","last_became_peered":"2023-09-23T18:28:03.297852+0000","last_unstale":"2023-09-23T18:31:28.756937+0000","last_undegraded":"2023-09-23T18:31:28.756937+0000","last_fullsized":"2023-09-23T18:31:28.756937+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.146541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":80,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.153013+0000","last_change":"2023-09-23T18:28:03.296913+0000","last_active":"2023-09-23T18:31:29.153013+0000","last_peered":"2023-09-23T18:31:29.153013+0000","last_clean":"2023-09-23T18:31:29.153013+0000","last_became_active":"2023-09-23T18:28:03.296499+0000","last_became_peered":"2023-09-23T18:28:03.296499+0000","last_unstale":"2023-09-23T18:31:29.153013+0000","last_undegraded":"2023-09-23T18:31:29.153013+0000","last_fullsized":"2023-09-23T18:31:29.153013+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:28.762646+0000","last_change":"2023-09-23T18:30:57.765407+0000","last_active":"2023-09-23T18:31:28.762646+0000","last_peered":"2023-09-23T18:31:28.762646+0000","last_clean":"2023-09-23T18:27:04.257256+0000","last_became_active":"2023-09-23T18:30:57.765407+0000","last_became_peered":"2023-09-23T18:30:57.765407+0000","last_unstale":"2023-09-23T18:31:28.762646+0000","last_undegraded":"2023-09-23T18:31:28.762646+0000","last_fullsized":"2023-09-23T18:30:57.754597+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":84,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.150508+0000","last_change":"2023-09-23T18:15:08.909339+0000","last_active":"2023-09-23T18:31:29.150508+0000","last_peered":"2023-09-23T18:31:29.150508+0000","last_clean":"2023-09-23T18:31:29.150508+0000","last_became_active":"2023-09-23T18:15:08.908912+0000","last_became_peered":"2023-09-23T18:15:08.908912+0000","last_unstale":"2023-09-23T18:31:29.150508+0000","last_undegraded":"2023-09-23T18:31:29.150508+0000","last_fullsized":"2023-09-23T18:31:29.150508+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":82,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:29.150477+0000","last_change":"2023-09-23T18:30:57.762665+0000","last_active":"2023-09-23T18:31:29.150477+0000","last_peered":"2023-09-23T18:31:29.150477+0000","last_clean":"2023-09-23T18:28:03.293385+0000","last_became_active":"2023-09-23T18:30:57.762665+0000","last_became_peered":"2023-09-23T18:30:57.762665+0000","last_unstale":"2023-09-23T18:31:29.150477+0000","last_undegraded":"2023-09-23T18:31:29.150477+0000","last_fullsized":"2023-09-23T18:30:57.757694+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":79,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.152521+0000","last_change":"2023-09-23T18:27:33.585121+0000","last_active":"2023-09-23T18:31:29.152521+0000","last_peered":"2023-09-23T18:31:29.152521+0000","last_clean":"2023-09-23T18:31:29.152521+0000","last_became_active":"2023-09-23T18:27:33.584620+0000","last_became_peered":"2023-09-23T18:27:33.584620+0000","last_unstale":"2023-09-23T18:31:29.152521+0000","last_undegraded":"2023-09-23T18:31:29.152521+0000","last_fullsized":"2023-09-23T18:31:29.152521+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:28.757397+0000","last_change":"2023-09-23T18:30:57.755267+0000","last_active":"2023-09-23T18:31:28.757397+0000","last_peered":"2023-09-23T18:31:28.757397+0000","last_clean":"2023-09-23T18:30:21.305279+0000","last_became_active":"2023-09-23T18:30:57.755267+0000","last_became_peered":"2023-09-23T18:30:57.755267+0000","last_unstale":"2023-09-23T18:31:28.757397+0000","last_undegraded":"2023-09-23T18:31:28.757397+0000","last_fullsized":"2023-09-23T18:30:57.743249+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:25.575440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":60,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.146096+0000","last_change":"2023-09-23T18:28:03.290316+0000","last_active":"2023-09-23T18:31:29.146096+0000","last_peered":"2023-09-23T18:31:29.146096+0000","last_clean":"2023-09-23T18:31:29.146096+0000","last_became_active":"2023-09-23T18:28:03.289698+0000","last_became_peered":"2023-09-23T18:28:03.289698+0000","last_unstale":"2023-09-23T18:31:29.146096+0000","last_undegraded":"2023-09-23T18:31:29.146096+0000","last_fullsized":"2023-09-23T18:31:29.146096+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:00.779542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.153984+0000","last_change":"2023-09-23T18:28:03.297841+0000","last_active":"2023-09-23T18:31:29.153984+0000","last_peered":"2023-09-23T18:31:29.153984+0000","last_clean":"2023-09-23T18:31:29.153984+0000","last_became_active":"2023-09-23T18:28:03.297216+0000","last_became_peered":"2023-09-23T18:28:03.297216+0000","last_unstale":"2023-09-23T18:31:29.153984+0000","last_undegraded":"2023-09-23T18:31:29.153984+0000","last_fullsized":"2023-09-23T18:31:29.153984+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":62,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.152620+0000","last_change":"2023-09-23T18:15:08.916658+0000","last_active":"2023-09-23T18:31:29.152620+0000","last_peered":"2023-09-23T18:31:29.152620+0000","last_clean":"2023-09-23T18:31:29.152620+0000","last_became_active":"2023-09-23T18:15:08.916301+0000","last_became_peered":"2023-09-23T18:15:08.916301+0000","last_unstale":"2023-09-23T18:31:29.152620+0000","last_undegraded":"2023-09-23T18:31:29.152620+0000","last_fullsized":"2023-09-23T18:31:29.152620+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":61,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.146558+0000","last_change":"2023-09-23T18:28:03.290217+0000","last_active":"2023-09-23T18:31:29.146558+0000","last_peered":"2023-09-23T18:31:29.146558+0000","last_clean":"2023-09-23T18:31:29.146558+0000","last_became_active":"2023-09-23T18:28:03.289888+0000","last_became_peered":"2023-09-23T18:28:03.289888+0000","last_unstale":"2023-09-23T18:31:29.146558+0000","last_undegraded":"2023-09-23T18:31:29.146558+0000","last_fullsized":"2023-09-23T18:31:29.146558+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:51:45.400847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:28.762785+0000","last_change":"2023-09-23T18:26:34.995333+0000","last_active":"2023-09-23T18:31:28.762785+0000","last_peered":"2023-09-23T18:31:28.762785+0000","last_clean":"2023-09-23T18:31:28.762785+0000","last_became_active":"2023-09-23T18:26:34.994983+0000","last_became_peered":"2023-09-23T18:26:34.994983+0000","last_unstale":"2023-09-23T18:31:28.762785+0000","last_undegraded":"2023-09-23T18:31:28.762785+0000","last_fullsized":"2023-09-23T18:31:28.762785+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:28.757034+0000","last_change":"2023-09-23T18:30:57.754720+0000","last_active":"2023-09-23T18:31:28.757034+0000","last_peered":"2023-09-23T18:31:28.757034+0000","last_clean":"2023-09-23T18:30:26.306588+0000","last_became_active":"2023-09-23T18:30:57.754720+0000","last_became_peered":"2023-09-23T18:30:57.754720+0000","last_unstale":"2023-09-23T18:31:28.757034+0000","last_undegraded":"2023-09-23T18:31:28.757034+0000","last_fullsized":"2023-09-23T18:30:57.742671+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:46.816007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"111'2130","reported_seq":2142,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T18:31:45.505404+0000","last_change":"2023-09-23T18:31:11.799201+0000","last_active":"2023-09-23T18:31:45.505404+0000","last_peered":"2023-09-23T18:31:45.505404+0000","last_clean":"2023-09-23T18:31:45.505404+0000","last_became_active":"2023-09-23T18:31:11.798833+0000","last_became_peered":"2023-09-23T18:31:11.798833+0000","last_unstale":"2023-09-23T18:31:45.505404+0000","last_undegraded":"2023-09-23T18:31:45.505404+0000","last_fullsized":"2023-09-23T18:31:45.505404+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":2130,"log_dups_size":0,"ondisk_log_size":2130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139591680,"num_objects":2130,"num_object_clones":0,"num_object_copies":6390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2130,"num_write_kb":136320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.153609+0000","last_change":"2023-09-23T18:26:34.999684+0000","last_active":"2023-09-23T18:31:29.153609+0000","last_peered":"2023-09-23T18:31:29.153609+0000","last_clean":"2023-09-23T18:31:29.153609+0000","last_became_active":"2023-09-23T18:26:34.999076+0000","last_became_peered":"2023-09-23T18:26:34.999076+0000","last_unstale":"2023-09-23T18:31:29.153609+0000","last_undegraded":"2023-09-23T18:31:29.153609+0000","last_fullsized":"2023-09-23T18:31:29.153609+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":61,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.152383+0000","last_change":"2023-09-23T18:15:10.327919+0000","last_active":"2023-09-23T18:31:29.152383+0000","last_peered":"2023-09-23T18:31:29.152383+0000","last_clean":"2023-09-23T18:31:29.152383+0000","last_became_active":"2023-09-23T18:15:10.327052+0000","last_became_peered":"2023-09-23T18:15:10.327052+0000","last_unstale":"2023-09-23T18:31:29.152383+0000","last_undegraded":"2023-09-23T18:31:29.152383+0000","last_fullsized":"2023-09-23T18:31:29.152383+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"111'2224","reported_seq":2234,"reported_epoch":111,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:31:46.592412+0000","last_change":"2023-09-23T18:31:31.595872+0000","last_active":"2023-09-23T18:31:46.592412+0000","last_peered":"2023-09-23T18:31:46.592412+0000","last_clean":"2023-09-23T18:31:10.759555+0000","last_became_active":"2023-09-23T18:31:11.789789+0000","last_became_peered":"2023-09-23T18:31:11.789789+0000","last_unstale":"2023-09-23T18:31:46.592412+0000","last_undegraded":"2023-09-23T18:31:31.591874+0000","last_fullsized":"2023-09-23T18:31:11.784114+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":2224,"log_dups_size":0,"ondisk_log_size":2224,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:02.167863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145752064,"num_objects":2224,"num_object_clones":0,"num_object_copies":6672,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2224,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2224,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2224,"num_write_kb":142336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":2224}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":85,"reported_epoch":110,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:31:29.153483+0000","last_change":"2023-09-23T18:30:57.760718+0000","last_active":"2023-09-23T18:31:29.153483+0000","last_peered":"2023-09-23T18:31:29.153483+0000","last_clean":"2023-09-23T18:28:03.287970+0000","last_became_active":"2023-09-23T18:30:57.760718+0000","last_became_peered":"2023-09-23T18:30:57.760718+0000","last_unstale":"2023-09-23T18:31:29.153483+0000","last_undegraded":"2023-09-23T18:30:57.754075+0000","last_fullsized":"2023-09-23T18:30:57.753412+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":97,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:29.150080+0000","last_change":"2023-09-23T18:30:57.759991+0000","last_active":"2023-09-23T18:31:29.150080+0000","last_peered":"2023-09-23T18:31:29.150080+0000","last_clean":"2023-09-23T18:28:03.302737+0000","last_became_active":"2023-09-23T18:30:57.759991+0000","last_became_peered":"2023-09-23T18:30:57.759991+0000","last_unstale":"2023-09-23T18:31:29.150080+0000","last_undegraded":"2023-09-23T18:31:29.150080+0000","last_fullsized":"2023-09-23T18:30:57.755928+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"111'2131","reported_seq":2144,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T18:31:46.572410+0000","last_change":"2023-09-23T18:31:11.792393+0000","last_active":"2023-09-23T18:31:46.572410+0000","last_peered":"2023-09-23T18:31:46.572410+0000","last_clean":"2023-09-23T18:31:46.572410+0000","last_became_active":"2023-09-23T18:31:11.792104+0000","last_became_peered":"2023-09-23T18:31:11.792104+0000","last_unstale":"2023-09-23T18:31:46.572410+0000","last_undegraded":"2023-09-23T18:31:46.572410+0000","last_fullsized":"2023-09-23T18:31:46.572410+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":2131,"log_dups_size":0,"ondisk_log_size":2131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:14.788041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139657216,"num_objects":2131,"num_object_clones":0,"num_object_copies":6393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2131,"num_write_kb":136384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":63,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.153149+0000","last_change":"2023-09-23T18:15:08.908376+0000","last_active":"2023-09-23T18:31:29.153149+0000","last_peered":"2023-09-23T18:31:29.153149+0000","last_clean":"2023-09-23T18:31:29.153149+0000","last_became_active":"2023-09-23T18:15:08.907752+0000","last_became_peered":"2023-09-23T18:15:08.907752+0000","last_unstale":"2023-09-23T18:31:29.153149+0000","last_undegraded":"2023-09-23T18:31:29.153149+0000","last_fullsized":"2023-09-23T18:31:29.153149+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:28.762902+0000","last_change":"2023-09-23T18:27:33.589116+0000","last_active":"2023-09-23T18:31:28.762902+0000","last_peered":"2023-09-23T18:31:28.762902+0000","last_clean":"2023-09-23T18:31:28.762902+0000","last_became_active":"2023-09-23T18:27:33.588809+0000","last_became_peered":"2023-09-23T18:27:33.588809+0000","last_unstale":"2023-09-23T18:31:28.762902+0000","last_undegraded":"2023-09-23T18:31:28.762902+0000","last_fullsized":"2023-09-23T18:31:28.762902+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"111'2121","reported_seq":2134,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T18:31:46.592637+0000","last_change":"2023-09-23T18:31:11.796358+0000","last_active":"2023-09-23T18:31:46.592637+0000","last_peered":"2023-09-23T18:31:46.592637+0000","last_clean":"2023-09-23T18:31:46.592637+0000","last_became_active":"2023-09-23T18:31:11.795962+0000","last_became_peered":"2023-09-23T18:31:11.795962+0000","last_unstale":"2023-09-23T18:31:46.592637+0000","last_undegraded":"2023-09-23T18:31:46.592637+0000","last_fullsized":"2023-09-23T18:31:46.592637+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":2121,"log_dups_size":0,"ondisk_log_size":2121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:29.481931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139001856,"num_objects":2121,"num_object_clones":0,"num_object_copies":6363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2121,"num_write_kb":135744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":77,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.153082+0000","last_change":"2023-09-23T18:28:03.298611+0000","last_active":"2023-09-23T18:31:29.153082+0000","last_peered":"2023-09-23T18:31:29.153082+0000","last_clean":"2023-09-23T18:31:29.153082+0000","last_became_active":"2023-09-23T18:28:03.298233+0000","last_became_peered":"2023-09-23T18:28:03.298233+0000","last_unstale":"2023-09-23T18:31:29.153082+0000","last_undegraded":"2023-09-23T18:31:29.153082+0000","last_fullsized":"2023-09-23T18:31:29.153082+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.146142+0000","last_change":"2023-09-23T18:27:33.582033+0000","last_active":"2023-09-23T18:31:29.146142+0000","last_peered":"2023-09-23T18:31:29.146142+0000","last_clean":"2023-09-23T18:31:29.146142+0000","last_became_active":"2023-09-23T18:27:33.581667+0000","last_became_peered":"2023-09-23T18:27:33.581667+0000","last_unstale":"2023-09-23T18:31:29.146142+0000","last_undegraded":"2023-09-23T18:31:29.146142+0000","last_fullsized":"2023-09-23T18:31:29.146142+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:36.039922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"111'2259","reported_seq":2272,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T18:31:46.595887+0000","last_change":"2023-09-23T18:31:11.794977+0000","last_active":"2023-09-23T18:31:46.595887+0000","last_peered":"2023-09-23T18:31:46.595887+0000","last_clean":"2023-09-23T18:31:46.595887+0000","last_became_active":"2023-09-23T18:31:11.794567+0000","last_became_peered":"2023-09-23T18:31:11.794567+0000","last_unstale":"2023-09-23T18:31:46.595887+0000","last_undegraded":"2023-09-23T18:31:46.595887+0000","last_fullsized":"2023-09-23T18:31:46.595887+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":2259,"log_dups_size":0,"ondisk_log_size":2259,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:57.658734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148045824,"num_objects":2259,"num_object_clones":0,"num_object_copies":6777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2259,"num_write_kb":144576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:28.757900+0000","last_change":"2023-09-23T18:28:03.297628+0000","last_active":"2023-09-23T18:31:28.757900+0000","last_peered":"2023-09-23T18:31:28.757900+0000","last_clean":"2023-09-23T18:31:28.757900+0000","last_became_active":"2023-09-23T18:28:03.296694+0000","last_became_peered":"2023-09-23T18:28:03.296694+0000","last_unstale":"2023-09-23T18:31:28.757900+0000","last_undegraded":"2023-09-23T18:31:28.757900+0000","last_fullsized":"2023-09-23T18:31:28.757900+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:43.375652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.150082+0000","last_change":"2023-09-23T18:28:03.302326+0000","last_active":"2023-09-23T18:31:29.150082+0000","last_peered":"2023-09-23T18:31:29.150082+0000","last_clean":"2023-09-23T18:31:29.150082+0000","last_became_active":"2023-09-23T18:28:03.301970+0000","last_became_peered":"2023-09-23T18:28:03.301970+0000","last_unstale":"2023-09-23T18:31:29.150082+0000","last_undegraded":"2023-09-23T18:31:29.150082+0000","last_fullsized":"2023-09-23T18:31:29.150082+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"111'2028","reported_seq":2037,"reported_epoch":111,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:31:45.496215+0000","last_change":"2023-09-23T18:31:31.595358+0000","last_active":"2023-09-23T18:31:45.496215+0000","last_peered":"2023-09-23T18:31:45.496215+0000","last_clean":"2023-09-23T18:31:10.759555+0000","last_became_active":"2023-09-23T18:31:11.793471+0000","last_became_peered":"2023-09-23T18:31:11.793471+0000","last_unstale":"2023-09-23T18:31:45.496215+0000","last_undegraded":"2023-09-23T18:31:31.595206+0000","last_fullsized":"2023-09-23T18:31:11.790449+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":2028,"log_dups_size":0,"ondisk_log_size":2028,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132907008,"num_objects":2028,"num_object_clones":0,"num_object_copies":6084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2028,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2028,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2028,"num_write_kb":129792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":2028}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":83,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:28.763110+0000","last_change":"2023-09-23T18:30:57.763833+0000","last_active":"2023-09-23T18:31:28.763110+0000","last_peered":"2023-09-23T18:31:28.763110+0000","last_clean":"2023-09-23T18:28:03.294831+0000","last_became_active":"2023-09-23T18:30:57.763833+0000","last_became_peered":"2023-09-23T18:30:57.763833+0000","last_unstale":"2023-09-23T18:31:28.763110+0000","last_undegraded":"2023-09-23T18:31:28.763110+0000","last_fullsized":"2023-09-23T18:30:57.755260+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":97,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:29.150449+0000","last_change":"2023-09-23T18:30:57.765518+0000","last_active":"2023-09-23T18:31:29.150449+0000","last_peered":"2023-09-23T18:31:29.150449+0000","last_clean":"2023-09-23T18:28:03.294665+0000","last_became_active":"2023-09-23T18:30:57.765518+0000","last_became_peered":"2023-09-23T18:30:57.765518+0000","last_unstale":"2023-09-23T18:31:29.150449+0000","last_undegraded":"2023-09-23T18:31:29.150449+0000","last_fullsized":"2023-09-23T18:30:57.757909+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"111'2087","reported_seq":2099,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T18:31:45.506365+0000","last_change":"2023-09-23T18:31:11.800084+0000","last_active":"2023-09-23T18:31:45.506365+0000","last_peered":"2023-09-23T18:31:45.506365+0000","last_clean":"2023-09-23T18:31:45.506365+0000","last_became_active":"2023-09-23T18:31:11.799400+0000","last_became_peered":"2023-09-23T18:31:11.799400+0000","last_unstale":"2023-09-23T18:31:45.506365+0000","last_undegraded":"2023-09-23T18:31:45.506365+0000","last_fullsized":"2023-09-23T18:31:45.506365+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":2087,"log_dups_size":0,"ondisk_log_size":2087,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136773632,"num_objects":2087,"num_object_clones":0,"num_object_copies":6261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2087,"num_write_kb":133568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":77,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.149698+0000","last_change":"2023-09-23T18:27:33.585714+0000","last_active":"2023-09-23T18:31:29.149698+0000","last_peered":"2023-09-23T18:31:29.149698+0000","last_clean":"2023-09-23T18:31:29.149698+0000","last_became_active":"2023-09-23T18:27:33.584641+0000","last_became_peered":"2023-09-23T18:27:33.584641+0000","last_unstale":"2023-09-23T18:31:29.149698+0000","last_undegraded":"2023-09-23T18:31:29.149698+0000","last_fullsized":"2023-09-23T18:31:29.149698+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":82,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:29.150590+0000","last_change":"2023-09-23T18:30:57.762661+0000","last_active":"2023-09-23T18:31:29.150590+0000","last_peered":"2023-09-23T18:31:29.150590+0000","last_clean":"2023-09-23T18:28:03.292763+0000","last_became_active":"2023-09-23T18:30:57.762661+0000","last_became_peered":"2023-09-23T18:30:57.762661+0000","last_unstale":"2023-09-23T18:31:29.150590+0000","last_undegraded":"2023-09-23T18:31:29.150590+0000","last_fullsized":"2023-09-23T18:30:57.760181+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"111'1988","reported_seq":2000,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T18:31:45.506722+0000","last_change":"2023-09-23T18:31:11.798036+0000","last_active":"2023-09-23T18:31:45.506722+0000","last_peered":"2023-09-23T18:31:45.506722+0000","last_clean":"2023-09-23T18:31:45.506722+0000","last_became_active":"2023-09-23T18:31:11.797684+0000","last_became_peered":"2023-09-23T18:31:11.797684+0000","last_unstale":"2023-09-23T18:31:45.506722+0000","last_undegraded":"2023-09-23T18:31:45.506722+0000","last_fullsized":"2023-09-23T18:31:45.506722+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":1988,"log_dups_size":0,"ondisk_log_size":1988,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130285568,"num_objects":1988,"num_object_clones":0,"num_object_copies":5964,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1988,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1988,"num_write_kb":127232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:28.757071+0000","last_change":"2023-09-23T18:30:57.758546+0000","last_active":"2023-09-23T18:31:28.757071+0000","last_peered":"2023-09-23T18:31:28.757071+0000","last_clean":"2023-09-23T18:30:11.303354+0000","last_became_active":"2023-09-23T18:30:57.758546+0000","last_became_peered":"2023-09-23T18:30:57.758546+0000","last_unstale":"2023-09-23T18:31:28.757071+0000","last_undegraded":"2023-09-23T18:31:28.757071+0000","last_fullsized":"2023-09-23T18:30:57.742833+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:09:29.647331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:28.758162+0000","last_change":"2023-09-23T18:30:57.759447+0000","last_active":"2023-09-23T18:31:28.758162+0000","last_peered":"2023-09-23T18:31:28.758162+0000","last_clean":"2023-09-23T18:27:33.569231+0000","last_became_active":"2023-09-23T18:30:57.759447+0000","last_became_peered":"2023-09-23T18:30:57.759447+0000","last_unstale":"2023-09-23T18:31:28.758162+0000","last_undegraded":"2023-09-23T18:31:28.758162+0000","last_fullsized":"2023-09-23T18:30:57.743500+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:37.916625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"77'130","reported_seq":553,"reported_epoch":110,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:31:29.146581+0000","last_change":"2023-09-23T18:30:57.754736+0000","last_active":"2023-09-23T18:31:29.146581+0000","last_peered":"2023-09-23T18:31:29.146581+0000","last_clean":"2023-09-23T18:17:47.121124+0000","last_became_active":"2023-09-23T18:30:57.754736+0000","last_became_peered":"2023-09-23T18:30:57.754736+0000","last_unstale":"2023-09-23T18:31:29.146581+0000","last_undegraded":"2023-09-23T18:30:57.744730+0000","last_fullsized":"2023-09-23T18:30:57.744572+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:31.511177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":772,"num_read_kb":1712,"num_write":126,"num_write_kb":2126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":92,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.154122+0000","last_change":"2023-09-23T18:28:03.297594+0000","last_active":"2023-09-23T18:31:29.154122+0000","last_peered":"2023-09-23T18:31:29.154122+0000","last_clean":"2023-09-23T18:31:29.154122+0000","last_became_active":"2023-09-23T18:28:03.296725+0000","last_became_peered":"2023-09-23T18:28:03.296725+0000","last_unstale":"2023-09-23T18:31:29.154122+0000","last_undegraded":"2023-09-23T18:31:29.154122+0000","last_fullsized":"2023-09-23T18:31:29.154122+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":27,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:29.152800+0000","last_change":"2023-09-23T18:30:57.758933+0000","last_active":"2023-09-23T18:31:29.152800+0000","last_peered":"2023-09-23T18:31:29.152800+0000","last_clean":"2023-09-23T18:15:08.873352+0000","last_became_active":"2023-09-23T18:30:57.758933+0000","last_became_peered":"2023-09-23T18:30:57.758933+0000","last_unstale":"2023-09-23T18:31:29.152800+0000","last_undegraded":"2023-09-23T18:31:29.152800+0000","last_fullsized":"2023-09-23T18:30:57.754757+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.3","version":"111'2371","reported_seq":2384,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T18:31:48.018645+0000","last_change":"2023-09-23T18:31:11.798309+0000","last_active":"2023-09-23T18:31:48.018645+0000","last_peered":"2023-09-23T18:31:48.018645+0000","last_clean":"2023-09-23T18:31:48.018645+0000","last_became_active":"2023-09-23T18:31:11.797917+0000","last_became_peered":"2023-09-23T18:31:11.797917+0000","last_unstale":"2023-09-23T18:31:48.018645+0000","last_undegraded":"2023-09-23T18:31:48.018645+0000","last_fullsized":"2023-09-23T18:31:48.018645+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":2371,"log_dups_size":0,"ondisk_log_size":2371,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:25.706907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155385856,"num_objects":2371,"num_object_clones":0,"num_object_copies":7113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2371,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2371,"num_write_kb":151744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"111'2358","reported_seq":2368,"reported_epoch":111,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:31:47.994722+0000","last_change":"2023-09-23T18:31:31.595287+0000","last_active":"2023-09-23T18:31:47.994722+0000","last_peered":"2023-09-23T18:31:47.994722+0000","last_clean":"2023-09-23T18:31:10.759555+0000","last_became_active":"2023-09-23T18:31:11.792758+0000","last_became_peered":"2023-09-23T18:31:11.792758+0000","last_unstale":"2023-09-23T18:31:47.994722+0000","last_undegraded":"2023-09-23T18:31:31.594926+0000","last_fullsized":"2023-09-23T18:31:11.788267+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":2358,"log_dups_size":0,"ondisk_log_size":2358,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:34:46.104736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154533888,"num_objects":2358,"num_object_clones":0,"num_object_copies":7074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2358,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2358,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":2358}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'96","reported_seq":632,"reported_epoch":110,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:31:29.153055+0000","last_change":"2023-09-23T18:30:57.762323+0000","last_active":"2023-09-23T18:31:29.153055+0000","last_peered":"2023-09-23T18:31:29.153055+0000","last_clean":"2023-09-23T18:28:03.290301+0000","last_became_active":"2023-09-23T18:30:57.762323+0000","last_became_peered":"2023-09-23T18:30:57.762323+0000","last_unstale":"2023-09-23T18:31:29.153055+0000","last_undegraded":"2023-09-23T18:30:57.755511+0000","last_fullsized":"2023-09-23T18:30:57.755373+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":96,"log_dups_size":0,"ondisk_log_size":96,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":785,"num_read_kb":1291,"num_write":87,"num_write_kb":338,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":77,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.150250+0000","last_change":"2023-09-23T18:26:34.992535+0000","last_active":"2023-09-23T18:31:29.150250+0000","last_peered":"2023-09-23T18:31:29.150250+0000","last_clean":"2023-09-23T18:31:29.150250+0000","last_became_active":"2023-09-23T18:26:34.992177+0000","last_became_peered":"2023-09-23T18:26:34.992177+0000","last_unstale":"2023-09-23T18:31:29.150250+0000","last_undegraded":"2023-09-23T18:31:29.150250+0000","last_fullsized":"2023-09-23T18:31:29.150250+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:29.149342+0000","last_change":"2023-09-23T18:30:57.750780+0000","last_active":"2023-09-23T18:31:29.149342+0000","last_peered":"2023-09-23T18:31:29.149342+0000","last_clean":"2023-09-23T18:27:33.585799+0000","last_became_active":"2023-09-23T18:30:57.750780+0000","last_became_peered":"2023-09-23T18:30:57.750780+0000","last_unstale":"2023-09-23T18:31:29.149342+0000","last_undegraded":"2023-09-23T18:31:29.149342+0000","last_fullsized":"2023-09-23T18:30:57.747747+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:02.927953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.6","version":"111'2200","reported_seq":2213,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T18:31:46.599450+0000","last_change":"2023-09-23T18:31:11.795570+0000","last_active":"2023-09-23T18:31:46.599450+0000","last_peered":"2023-09-23T18:31:46.599450+0000","last_clean":"2023-09-23T18:31:46.599450+0000","last_became_active":"2023-09-23T18:31:11.795067+0000","last_became_peered":"2023-09-23T18:31:11.795067+0000","last_unstale":"2023-09-23T18:31:46.599450+0000","last_undegraded":"2023-09-23T18:31:46.599450+0000","last_fullsized":"2023-09-23T18:31:46.599450+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":2200,"log_dups_size":0,"ondisk_log_size":2200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:08:17.723025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144179200,"num_objects":2200,"num_object_clones":0,"num_object_copies":6600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2200,"num_write_kb":140800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"77'112","reported_seq":547,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T18:31:31.782483+0000","last_change":"2023-09-23T18:28:03.297540+0000","last_active":"2023-09-23T18:31:31.782483+0000","last_peered":"2023-09-23T18:31:31.782483+0000","last_clean":"2023-09-23T18:31:31.782483+0000","last_became_active":"2023-09-23T18:28:03.296649+0000","last_became_peered":"2023-09-23T18:28:03.296649+0000","last_unstale":"2023-09-23T18:31:31.782483+0000","last_undegraded":"2023-09-23T18:31:31.782483+0000","last_fullsized":"2023-09-23T18:31:31.782483+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:32.531775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":774,"num_read_kb":596,"num_write":102,"num_write_kb":52,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:29.146960+0000","last_change":"2023-09-23T18:30:57.760762+0000","last_active":"2023-09-23T18:31:29.146960+0000","last_peered":"2023-09-23T18:31:29.146960+0000","last_clean":"2023-09-23T18:27:04.256214+0000","last_became_active":"2023-09-23T18:30:57.760762+0000","last_became_peered":"2023-09-23T18:30:57.760762+0000","last_unstale":"2023-09-23T18:31:29.146960+0000","last_undegraded":"2023-09-23T18:31:29.146960+0000","last_fullsized":"2023-09-23T18:30:57.747320+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:22.564483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:29.154305+0000","last_change":"2023-09-23T18:30:57.756578+0000","last_active":"2023-09-23T18:31:29.154305+0000","last_peered":"2023-09-23T18:31:29.154305+0000","last_clean":"2023-09-23T18:21:39.602195+0000","last_became_active":"2023-09-23T18:30:57.756578+0000","last_became_peered":"2023-09-23T18:30:57.756578+0000","last_unstale":"2023-09-23T18:31:29.154305+0000","last_undegraded":"2023-09-23T18:31:29.154305+0000","last_fullsized":"2023-09-23T18:30:57.753868+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"77'147","reported_seq":510,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.153646+0000","last_change":"2023-09-23T18:27:33.583558+0000","last_active":"2023-09-23T18:31:29.153646+0000","last_peered":"2023-09-23T18:31:29.153646+0000","last_clean":"2023-09-23T18:31:29.153646+0000","last_became_active":"2023-09-23T18:27:33.583003+0000","last_became_peered":"2023-09-23T18:27:33.583003+0000","last_unstale":"2023-09-23T18:31:29.153646+0000","last_undegraded":"2023-09-23T18:31:29.153646+0000","last_fullsized":"2023-09-23T18:31:29.153646+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":625,"num_read_kb":1869,"num_write":143,"num_write_kb":5368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":313,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.152159+0000","last_change":"2023-09-23T18:26:34.999402+0000","last_active":"2023-09-23T18:31:29.152159+0000","last_peered":"2023-09-23T18:31:29.152159+0000","last_clean":"2023-09-23T18:31:29.152159+0000","last_became_active":"2023-09-23T18:26:34.998828+0000","last_became_peered":"2023-09-23T18:26:34.998828+0000","last_unstale":"2023-09-23T18:31:29.152159+0000","last_undegraded":"2023-09-23T18:31:29.152159+0000","last_fullsized":"2023-09-23T18:31:29.152159+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_clean_scrub_stamp":"2023-09-23T18:08:35.701374+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:28.756896+0000","last_change":"2023-09-23T18:28:03.295577+0000","last_active":"2023-09-23T18:31:28.756896+0000","last_peered":"2023-09-23T18:31:28.756896+0000","last_clean":"2023-09-23T18:31:28.756896+0000","last_became_active":"2023-09-23T18:28:03.293777+0000","last_became_peered":"2023-09-23T18:28:03.293777+0000","last_unstale":"2023-09-23T18:31:28.756896+0000","last_undegraded":"2023-09-23T18:31:28.756896+0000","last_fullsized":"2023-09-23T18:31:28.756896+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:49.714764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":61,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:28.763262+0000","last_change":"2023-09-23T18:15:08.921629+0000","last_active":"2023-09-23T18:31:28.763262+0000","last_peered":"2023-09-23T18:31:28.763262+0000","last_clean":"2023-09-23T18:31:28.763262+0000","last_became_active":"2023-09-23T18:15:08.921283+0000","last_became_peered":"2023-09-23T18:15:08.921283+0000","last_unstale":"2023-09-23T18:31:28.763262+0000","last_undegraded":"2023-09-23T18:31:28.763262+0000","last_fullsized":"2023-09-23T18:31:28.763262+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"111'2041","reported_seq":2053,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T18:31:45.512855+0000","last_change":"2023-09-23T18:31:11.795769+0000","last_active":"2023-09-23T18:31:45.512855+0000","last_peered":"2023-09-23T18:31:45.512855+0000","last_clean":"2023-09-23T18:31:45.512855+0000","last_became_active":"2023-09-23T18:31:11.795438+0000","last_became_peered":"2023-09-23T18:31:11.795438+0000","last_unstale":"2023-09-23T18:31:45.512855+0000","last_undegraded":"2023-09-23T18:31:45.512855+0000","last_fullsized":"2023-09-23T18:31:45.512855+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":2041,"log_dups_size":0,"ondisk_log_size":2041,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133758976,"num_objects":2041,"num_object_clones":0,"num_object_copies":6123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2041,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2041,"num_write_kb":130624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"111'2066","reported_seq":2075,"reported_epoch":111,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:31:45.506780+0000","last_change":"2023-09-23T18:31:31.606526+0000","last_active":"2023-09-23T18:31:45.506780+0000","last_peered":"2023-09-23T18:31:45.506780+0000","last_clean":"2023-09-23T18:31:10.759555+0000","last_became_active":"2023-09-23T18:31:11.796206+0000","last_became_peered":"2023-09-23T18:31:11.796206+0000","last_unstale":"2023-09-23T18:31:45.506780+0000","last_undegraded":"2023-09-23T18:31:31.592910+0000","last_fullsized":"2023-09-23T18:31:11.785631+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":2066,"log_dups_size":0,"ondisk_log_size":2066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135397376,"num_objects":2066,"num_object_clones":0,"num_object_copies":6198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2066,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2066,"num_write_kb":132224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":2066}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"77'120","reported_seq":563,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:29.152289+0000","last_change":"2023-09-23T18:30:57.756612+0000","last_active":"2023-09-23T18:31:29.152289+0000","last_peered":"2023-09-23T18:31:29.152289+0000","last_clean":"2023-09-23T18:28:03.286680+0000","last_became_active":"2023-09-23T18:30:57.756612+0000","last_became_peered":"2023-09-23T18:30:57.756612+0000","last_unstale":"2023-09-23T18:31:29.152289+0000","last_undegraded":"2023-09-23T18:31:29.152289+0000","last_fullsized":"2023-09-23T18:30:57.751463+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":120,"log_dups_size":0,"ondisk_log_size":120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":809,"num_read_kb":2749,"num_write":98,"num_write_kb":2087,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":60,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.145901+0000","last_change":"2023-09-23T18:28:03.299508+0000","last_active":"2023-09-23T18:31:29.145901+0000","last_peered":"2023-09-23T18:31:29.145901+0000","last_clean":"2023-09-23T18:31:29.145901+0000","last_became_active":"2023-09-23T18:28:03.298682+0000","last_became_peered":"2023-09-23T18:28:03.298682+0000","last_unstale":"2023-09-23T18:31:29.145901+0000","last_undegraded":"2023-09-23T18:31:29.145901+0000","last_fullsized":"2023-09-23T18:31:29.145901+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:08.398302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":82,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:29.146884+0000","last_change":"2023-09-23T18:30:57.748793+0000","last_active":"2023-09-23T18:31:29.146884+0000","last_peered":"2023-09-23T18:31:29.146884+0000","last_clean":"2023-09-23T18:30:53.021930+0000","last_became_active":"2023-09-23T18:30:57.748793+0000","last_became_peered":"2023-09-23T18:30:57.748793+0000","last_unstale":"2023-09-23T18:31:29.146884+0000","last_undegraded":"2023-09-23T18:31:29.146884+0000","last_fullsized":"2023-09-23T18:30:57.745371+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:43.511022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'78","reported_seq":347,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.153117+0000","last_change":"2023-09-23T18:26:34.992345+0000","last_active":"2023-09-23T18:31:29.153117+0000","last_peered":"2023-09-23T18:31:29.153117+0000","last_clean":"2023-09-23T18:31:29.153117+0000","last_became_active":"2023-09-23T18:26:34.991774+0000","last_became_peered":"2023-09-23T18:26:34.991774+0000","last_unstale":"2023-09-23T18:31:29.153117+0000","last_undegraded":"2023-09-23T18:31:29.153117+0000","last_fullsized":"2023-09-23T18:31:29.153117+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":78,"log_dups_size":0,"ondisk_log_size":78,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":428,"num_read_kb":1125,"num_write":73,"num_write_kb":431,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:29.149528+0000","last_change":"2023-09-23T18:30:57.756695+0000","last_active":"2023-09-23T18:31:29.149528+0000","last_peered":"2023-09-23T18:31:29.149528+0000","last_clean":"2023-09-23T18:27:04.256055+0000","last_became_active":"2023-09-23T18:30:57.756695+0000","last_became_peered":"2023-09-23T18:30:57.756695+0000","last_unstale":"2023-09-23T18:31:29.149528+0000","last_undegraded":"2023-09-23T18:31:29.149528+0000","last_fullsized":"2023-09-23T18:30:57.748551+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:51.257454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":46,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.146698+0000","last_change":"2023-09-23T18:26:34.990770+0000","last_active":"2023-09-23T18:31:29.146698+0000","last_peered":"2023-09-23T18:31:29.146698+0000","last_clean":"2023-09-23T18:31:29.146698+0000","last_became_active":"2023-09-23T18:26:34.990262+0000","last_became_peered":"2023-09-23T18:26:34.990262+0000","last_unstale":"2023-09-23T18:31:29.146698+0000","last_undegraded":"2023-09-23T18:31:29.146698+0000","last_fullsized":"2023-09-23T18:31:29.146698+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:07.982834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"111'2082","reported_seq":2094,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T18:31:45.497320+0000","last_change":"2023-09-23T18:31:11.800031+0000","last_active":"2023-09-23T18:31:45.497320+0000","last_peered":"2023-09-23T18:31:45.497320+0000","last_clean":"2023-09-23T18:31:45.497320+0000","last_became_active":"2023-09-23T18:31:11.799641+0000","last_became_peered":"2023-09-23T18:31:11.799641+0000","last_unstale":"2023-09-23T18:31:45.497320+0000","last_undegraded":"2023-09-23T18:31:45.497320+0000","last_fullsized":"2023-09-23T18:31:45.497320+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":2082,"log_dups_size":0,"ondisk_log_size":2082,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136445952,"num_objects":2082,"num_object_clones":0,"num_object_copies":6246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2082,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2082,"num_write_kb":133248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:28.758090+0000","last_change":"2023-09-23T18:28:03.298805+0000","last_active":"2023-09-23T18:31:28.758090+0000","last_peered":"2023-09-23T18:31:28.758090+0000","last_clean":"2023-09-23T18:31:28.758090+0000","last_became_active":"2023-09-23T18:28:03.298437+0000","last_became_peered":"2023-09-23T18:28:03.298437+0000","last_unstale":"2023-09-23T18:31:28.758090+0000","last_undegraded":"2023-09-23T18:31:28.758090+0000","last_fullsized":"2023-09-23T18:31:28.758090+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:27.831669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'181","reported_seq":582,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:28.763412+0000","last_change":"2023-09-23T18:30:57.764331+0000","last_active":"2023-09-23T18:31:28.763412+0000","last_peered":"2023-09-23T18:31:28.763412+0000","last_clean":"2023-09-23T18:17:46.151116+0000","last_became_active":"2023-09-23T18:30:57.764331+0000","last_became_peered":"2023-09-23T18:30:57.764331+0000","last_unstale":"2023-09-23T18:31:28.763412+0000","last_undegraded":"2023-09-23T18:31:28.763412+0000","last_fullsized":"2023-09-23T18:30:57.758120+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":181,"log_dups_size":0,"ondisk_log_size":181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":909,"num_read_kb":734,"num_write":148,"num_write_kb":75,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:28.756895+0000","last_change":"2023-09-23T18:28:03.292295+0000","last_active":"2023-09-23T18:31:28.756895+0000","last_peered":"2023-09-23T18:31:28.756895+0000","last_clean":"2023-09-23T18:31:28.756895+0000","last_became_active":"2023-09-23T18:28:03.291902+0000","last_became_peered":"2023-09-23T18:28:03.291902+0000","last_unstale":"2023-09-23T18:31:28.756895+0000","last_undegraded":"2023-09-23T18:31:28.756895+0000","last_fullsized":"2023-09-23T18:31:28.756895+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:38.415256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"111'2073","reported_seq":2085,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T18:31:45.489221+0000","last_change":"2023-09-23T18:31:11.796383+0000","last_active":"2023-09-23T18:31:45.489221+0000","last_peered":"2023-09-23T18:31:45.489221+0000","last_clean":"2023-09-23T18:31:45.489221+0000","last_became_active":"2023-09-23T18:31:11.796024+0000","last_became_peered":"2023-09-23T18:31:11.796024+0000","last_unstale":"2023-09-23T18:31:45.489221+0000","last_undegraded":"2023-09-23T18:31:45.489221+0000","last_fullsized":"2023-09-23T18:31:45.489221+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":2073,"log_dups_size":0,"ondisk_log_size":2073,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135856128,"num_objects":2073,"num_object_clones":0,"num_object_copies":6219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2073,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2073,"num_write_kb":132672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'112","reported_seq":601,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:28.763441+0000","last_change":"2023-09-23T18:27:33.587478+0000","last_active":"2023-09-23T18:31:28.763441+0000","last_peered":"2023-09-23T18:31:28.763441+0000","last_clean":"2023-09-23T18:31:28.763441+0000","last_became_active":"2023-09-23T18:27:33.586958+0000","last_became_peered":"2023-09-23T18:27:33.586958+0000","last_unstale":"2023-09-23T18:31:28.763441+0000","last_undegraded":"2023-09-23T18:31:28.763441+0000","last_fullsized":"2023-09-23T18:31:28.763441+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":893,"num_read_kb":3172,"num_write":69,"num_write_kb":2237,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":70,"reported_epoch":110,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:31:29.146715+0000","last_change":"2023-09-23T18:30:57.759899+0000","last_active":"2023-09-23T18:31:29.146715+0000","last_peered":"2023-09-23T18:31:29.146715+0000","last_clean":"2023-09-23T18:30:53.021857+0000","last_became_active":"2023-09-23T18:30:57.759899+0000","last_became_peered":"2023-09-23T18:30:57.759899+0000","last_unstale":"2023-09-23T18:31:29.146715+0000","last_undegraded":"2023-09-23T18:30:57.745402+0000","last_fullsized":"2023-09-23T18:30:57.745244+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:26.821613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":61,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.150001+0000","last_change":"2023-09-23T18:15:09.125623+0000","last_active":"2023-09-23T18:31:29.150001+0000","last_peered":"2023-09-23T18:31:29.150001+0000","last_clean":"2023-09-23T18:31:29.150001+0000","last_became_active":"2023-09-23T18:15:09.125018+0000","last_became_peered":"2023-09-23T18:15:09.125018+0000","last_unstale":"2023-09-23T18:31:29.150001+0000","last_undegraded":"2023-09-23T18:31:29.150001+0000","last_fullsized":"2023-09-23T18:31:29.150001+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"111'2200","reported_seq":2210,"reported_epoch":111,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:31:46.305160+0000","last_change":"2023-09-23T18:31:31.601373+0000","last_active":"2023-09-23T18:31:46.305160+0000","last_peered":"2023-09-23T18:31:46.305160+0000","last_clean":"2023-09-23T18:31:10.759555+0000","last_became_active":"2023-09-23T18:31:11.795405+0000","last_became_peered":"2023-09-23T18:31:11.795405+0000","last_unstale":"2023-09-23T18:31:46.305160+0000","last_undegraded":"2023-09-23T18:31:31.594507+0000","last_fullsized":"2023-09-23T18:31:11.782086+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":2200,"log_dups_size":0,"ondisk_log_size":2200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:30.717583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144179200,"num_objects":2200,"num_object_clones":0,"num_object_copies":6600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2200,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2200,"num_write_kb":140800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":2200}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":49,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T18:31:31.790112+0000","last_change":"2023-09-23T18:26:34.994176+0000","last_active":"2023-09-23T18:31:31.790112+0000","last_peered":"2023-09-23T18:31:31.790112+0000","last_clean":"2023-09-23T18:31:31.790112+0000","last_became_active":"2023-09-23T18:26:34.993829+0000","last_became_peered":"2023-09-23T18:26:34.993829+0000","last_unstale":"2023-09-23T18:31:31.790112+0000","last_undegraded":"2023-09-23T18:31:31.790112+0000","last_fullsized":"2023-09-23T18:31:31.790112+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:27.111857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":105,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:29.149299+0000","last_change":"2023-09-23T18:30:57.750699+0000","last_active":"2023-09-23T18:31:29.149299+0000","last_peered":"2023-09-23T18:31:29.149299+0000","last_clean":"2023-09-23T18:30:01.592430+0000","last_became_active":"2023-09-23T18:30:57.750699+0000","last_became_peered":"2023-09-23T18:30:57.750699+0000","last_unstale":"2023-09-23T18:31:29.149299+0000","last_undegraded":"2023-09-23T18:31:29.149299+0000","last_fullsized":"2023-09-23T18:30:57.747336+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:59.891827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":85,"reported_epoch":110,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:31:29.153310+0000","last_change":"2023-09-23T18:30:57.760848+0000","last_active":"2023-09-23T18:31:29.153310+0000","last_peered":"2023-09-23T18:31:29.153310+0000","last_clean":"2023-09-23T18:28:03.290321+0000","last_became_active":"2023-09-23T18:30:57.760848+0000","last_became_peered":"2023-09-23T18:30:57.760848+0000","last_unstale":"2023-09-23T18:31:29.153310+0000","last_undegraded":"2023-09-23T18:30:57.754803+0000","last_fullsized":"2023-09-23T18:30:57.754669+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":97,"reported_epoch":110,"state":"active+undersized","last_fresh":"2023-09-23T18:31:29.150846+0000","last_change":"2023-09-23T18:30:57.765166+0000","last_active":"2023-09-23T18:31:29.150846+0000","last_peered":"2023-09-23T18:31:29.150846+0000","last_clean":"2023-09-23T18:28:03.301070+0000","last_became_active":"2023-09-23T18:30:57.765166+0000","last_became_peered":"2023-09-23T18:30:57.765166+0000","last_unstale":"2023-09-23T18:31:29.150846+0000","last_undegraded":"2023-09-23T18:31:29.150846+0000","last_fullsized":"2023-09-23T18:30:57.757210+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.154118+0000","last_change":"2023-09-23T18:15:08.902664+0000","last_active":"2023-09-23T18:31:29.154118+0000","last_peered":"2023-09-23T18:31:29.154118+0000","last_clean":"2023-09-23T18:31:29.154118+0000","last_became_active":"2023-09-23T18:15:08.901252+0000","last_became_peered":"2023-09-23T18:15:08.901252+0000","last_unstale":"2023-09-23T18:31:29.154118+0000","last_undegraded":"2023-09-23T18:31:29.154118+0000","last_fullsized":"2023-09-23T18:31:29.154118+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":91,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.150381+0000","last_change":"2023-09-23T18:28:03.302261+0000","last_active":"2023-09-23T18:31:29.150381+0000","last_peered":"2023-09-23T18:31:29.150381+0000","last_clean":"2023-09-23T18:31:29.150381+0000","last_became_active":"2023-09-23T18:28:03.301911+0000","last_became_peered":"2023-09-23T18:28:03.301911+0000","last_unstale":"2023-09-23T18:31:29.150381+0000","last_undegraded":"2023-09-23T18:31:29.150381+0000","last_fullsized":"2023-09-23T18:31:29.150381+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:28.757766+0000","last_change":"2023-09-23T18:28:03.296850+0000","last_active":"2023-09-23T18:31:28.757766+0000","last_peered":"2023-09-23T18:31:28.757766+0000","last_clean":"2023-09-23T18:31:28.757766+0000","last_became_active":"2023-09-23T18:28:03.295842+0000","last_became_peered":"2023-09-23T18:28:03.295842+0000","last_unstale":"2023-09-23T18:31:28.757766+0000","last_undegraded":"2023-09-23T18:31:28.757766+0000","last_fullsized":"2023-09-23T18:31:28.757766+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:53.048579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":98,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:28.756786+0000","last_change":"2023-09-23T18:28:03.299520+0000","last_active":"2023-09-23T18:31:28.756786+0000","last_peered":"2023-09-23T18:31:28.756786+0000","last_clean":"2023-09-23T18:31:28.756786+0000","last_became_active":"2023-09-23T18:28:03.299158+0000","last_became_peered":"2023-09-23T18:28:03.299158+0000","last_unstale":"2023-09-23T18:31:28.756786+0000","last_undegraded":"2023-09-23T18:31:28.756786+0000","last_fullsized":"2023-09-23T18:31:28.756786+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:26.247064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.146828+0000","last_change":"2023-09-23T18:27:33.583935+0000","last_active":"2023-09-23T18:31:29.146828+0000","last_peered":"2023-09-23T18:31:29.146828+0000","last_clean":"2023-09-23T18:31:29.146828+0000","last_became_active":"2023-09-23T18:27:33.582561+0000","last_became_peered":"2023-09-23T18:27:33.582561+0000","last_unstale":"2023-09-23T18:31:29.146828+0000","last_undegraded":"2023-09-23T18:31:29.146828+0000","last_fullsized":"2023-09-23T18:31:29.146828+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:43.589363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T18:31:29.149519+0000","last_change":"2023-09-23T18:15:43.682873+0000","last_active":"2023-09-23T18:31:29.149519+0000","last_peered":"2023-09-23T18:31:29.149519+0000","last_clean":"2023-09-23T18:31:29.149519+0000","last_became_active":"2023-09-23T18:15:43.682465+0000","last_became_peered":"2023-09-23T18:15:43.682465+0000","last_unstale":"2023-09-23T18:31:29.149519+0000","last_undegraded":"2023-09-23T18:31:29.149519+0000","last_fullsized":"2023-09-23T18:31:29.149519+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":2251751424,"num_objects":34359,"num_object_clones":0,"num_object_copies":103077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10876,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":34359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":34359,"num_write_kb":2198976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6040977408,"data_stored":6040977408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":34359,"ondisk_log_size":34359,"up":43,"acting":43,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":79,"acting":79,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26886,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":85,"acting":85,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5621,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":20,"acting":20,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430447,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":979752,"kb_used_data":674336,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92765400,"statfs":{"total":95995035648,"available":94991769600,"internally_reserved":0,"allocated":690520064,"data_stored":690008542,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:30:44 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":0.92800000000000005,"15min":0.93000000000000005},"min":{"1min":0.57799999999999996,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.284,"5min":1.4670000000000001,"15min":1.4670000000000001},"last":0.252},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.93300000000000005,"15min":0.95699999999999996},"min":{"1min":0.56699999999999995,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.359,"5min":1.359,"15min":1.359},"last":0.36399999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:30:07 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.91100000000000003,"15min":0.91500000000000004},"min":{"1min":0.58599999999999997,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.4039999999999999,"5min":1.4039999999999999,"15min":1.4039999999999999},"last":1.1819999999999999},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.97499999999999998,"15min":0.96699999999999997},"min":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.254,"5min":1.306,"15min":1.306},"last":1.3340000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:31:38 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":0.96599999999999997,"15min":0.97299999999999998},"min":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":1.25,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":1.587},{"interface":"front","average":{"1min":1.0780000000000001,"5min":0.97699999999999998,"15min":0.96499999999999997},"min":{"1min":0.65200000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":2.0339999999999998,"5min":2.0339999999999998,"15min":2.0339999999999998},"last":0.38300000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:31:04 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.98599999999999999,"15min":0.99399999999999999},"min":{"1min":0.69099999999999995,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.3460000000000001,"5min":1.383,"15min":1.383},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.93999999999999995,"15min":0.94799999999999995},"min":{"1min":0.621,"5min":0.621,"15min":0.621},"max":{"1min":1.3200000000000001,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":1.869}]},{"osd":4,"last update":"Sat Sep 23 18:30:53 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.92900000000000005,"15min":0.89000000000000001},"min":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.371},"max":{"1min":1.264,"5min":1.264,"15min":1.3839999999999999},"last":0.29399999999999998},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.0309999999999999,"15min":1.024},"min":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.50700000000000001},"max":{"1min":1.609,"5min":1.609,"15min":1.609},"last":0.79400000000000004}]},{"osd":5,"last update":"Sat Sep 23 18:30:53 2023","interfaces":[{"interface":"back","average":{"1min":0.91800000000000004,"5min":0.92700000000000005,"15min":0.94799999999999995},"min":{"1min":0.72499999999999998,"5min":0.60699999999999998,"15min":0.505},"max":{"1min":1.4330000000000001,"5min":1.4330000000000001,"15min":1.4330000000000001},"last":4.0780000000000003},{"interface":"front","average":{"1min":1.103,"5min":1.0600000000000001,"15min":1.016},"min":{"1min":0.72899999999999998,"5min":0.68600000000000005,"15min":0.40899999999999997},"max":{"1min":1.4850000000000001,"5min":1.4850000000000001,"15min":1.4850000000000001},"last":0.75700000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:30:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.109,"15min":1.054},"min":{"1min":0.81599999999999995,"5min":0.74099999999999999,"15min":0.442},"max":{"1min":1.556,"5min":1.556,"15min":1.556},"last":0.71999999999999997},{"interface":"front","average":{"1min":1.1990000000000001,"5min":1.137,"15min":1.129},"min":{"1min":0.81999999999999995,"5min":0.63600000000000001,"15min":0.53200000000000003},"max":{"1min":1.5329999999999999,"5min":1.548,"15min":1.548},"last":0.45200000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626679,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":832348,"kb_used_data":535124,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92912804,"statfs":{"total":95995035648,"available":95142711296,"internally_reserved":0,"allocated":547966976,"data_stored":547467300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:30:42 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.91900000000000004,"15min":0.91100000000000003},"min":{"1min":0.57899999999999996,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.3260000000000001,"5min":1.3260000000000001,"15min":1.3260000000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.044,"15min":1.046},"min":{"1min":0.80500000000000005,"5min":0.70999999999999996,"15min":0.70999999999999996},"max":{"1min":1.3140000000000001,"5min":1.3540000000000001,"15min":1.3540000000000001},"last":0.38600000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:30:10 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90400000000000003},"min":{"1min":0.70699999999999996,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.1140000000000001,"5min":1.2210000000000001,"15min":1.2210000000000001},"last":0.91900000000000004},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.93100000000000005,"15min":0.92500000000000004},"min":{"1min":0.61599999999999999,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.403,"5min":1.403,"15min":1.403},"last":0.77200000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:31:38 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.97399999999999998,"15min":0.996},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.2529999999999999,"5min":1.2949999999999999,"15min":1.2949999999999999},"last":1.357},{"interface":"front","average":{"1min":0.81499999999999995,"5min":0.93200000000000005,"15min":0.94999999999999996},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.157,"5min":1.196,"15min":1.196},"last":1.6299999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:31:05 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":0.99299999999999999,"15min":0.98599999999999999},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.2809999999999999,"5min":1.3260000000000001,"15min":1.3260000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.99099999999999999,"5min":0.97799999999999998,"15min":0.98399999999999999},"min":{"1min":0.81899999999999995,"5min":0.64800000000000002,"15min":0.64800000000000002},"max":{"1min":1.1000000000000001,"5min":1.288,"15min":1.288},"last":0.503}]},{"osd":4,"last update":"Sat Sep 23 18:30:56 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":0.93000000000000005,"15min":0.92400000000000004},"min":{"1min":0.65400000000000003,"5min":0.61699999999999999,"15min":0.44800000000000001},"max":{"1min":1.421,"5min":1.421,"15min":1.4399999999999999},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.0149999999999999,"15min":0.98099999999999998},"min":{"1min":0.64500000000000002,"5min":0.56499999999999995,"15min":0.41199999999999998},"max":{"1min":1.3799999999999999,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.49099999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:30:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":1.0269999999999999,"15min":1.02},"min":{"1min":0.71699999999999997,"5min":0.58999999999999997,"15min":0.36699999999999999},"max":{"1min":1.415,"5min":1.431,"15min":1.5469999999999999},"last":0.57199999999999995},{"interface":"front","average":{"1min":1.1499999999999999,"5min":1.0840000000000001,"15min":1.0629999999999999},"min":{"1min":0.70299999999999996,"5min":0.63400000000000001,"15min":0.59199999999999997},"max":{"1min":1.4870000000000001,"5min":1.504,"15min":1.5700000000000001},"last":0.42399999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:31:38 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":1,"15min":1.0620000000000001},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":1.1919999999999999,"5min":1.401,"15min":1.764},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.198,"5min":1.165,"15min":1.137},"min":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.499},"max":{"1min":3.415,"5min":3.415,"15min":3.415},"last":3.137}]}]},{"osd":1,"up_from":93,"seq":399431958576,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32648,"kb_used_data":1088,"kb_used_omap":7,"kb_used_meta":31544,"kb_avail":93712504,"statfs":{"total":95995035648,"available":95961604096,"internally_reserved":0,"allocated":1114112,"data_stored":586888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":32301838},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:30:13 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.66100000000000003,"15min":0.67400000000000004},"min":{"1min":0.44500000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.216,"5min":1.216,"15min":1.216},"last":0.77500000000000002},{"interface":"front","average":{"1min":0.748,"5min":0.72399999999999998,"15min":0.73499999999999999},"min":{"1min":0.46800000000000003,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.252,"5min":1.252,"15min":1.252},"last":0.73899999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:30:43 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.622,"15min":0.60599999999999998},"min":{"1min":0.50800000000000001,"5min":0.315,"15min":0.315},"max":{"1min":1.0720000000000001,"5min":1.1000000000000001,"15min":1.1000000000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.84499999999999997,"5min":0.73899999999999999,"15min":0.72799999999999998},"min":{"1min":0.68799999999999994,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.151,"5min":1.151,"15min":1.151},"last":1.1100000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:30:03 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.745,"15min":0.75},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.179,"5min":1.179,"15min":1.179},"last":0.95599999999999996},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.70099999999999996,"15min":0.71499999999999997},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.88,"5min":1.294,"15min":1.294},"last":0.69599999999999995}]},{"osd":4,"last update":"Sat Sep 23 18:30:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":0.94599999999999995,"15min":0.92800000000000005},"min":{"1min":0.70899999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.89000000000000001},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.97999999999999998,"15min":1.0249999999999999},"min":{"1min":0.67800000000000005,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.0269999999999999,"5min":1.4950000000000001,"15min":1.4950000000000001},"last":0.90700000000000003}]},{"osd":5,"last update":"Sat Sep 23 18:30:13 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.96099999999999997,"15min":0.99199999999999999},"min":{"1min":0.624,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.2170000000000001,"5min":1.5149999999999999,"15min":1.5149999999999999},"last":0.875},{"interface":"front","average":{"1min":1.012,"5min":0.98299999999999998,"15min":0.98599999999999999},"min":{"1min":0.73099999999999998,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.296,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":1.0449999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:30:13 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.99199999999999999,"15min":1.022},"min":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.2230000000000001,"5min":1.6100000000000001,"15min":1.6100000000000001},"last":0.98899999999999999},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.0589999999999999,"15min":1.0720000000000001},"min":{"1min":0.68899999999999995,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.3839999999999999,"5min":1.4419999999999999,"15min":1.4419999999999999},"last":1.169}]},{"osd":7,"last update":"Sat Sep 23 18:30:13 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.0860000000000001,"15min":1.1060000000000001},"min":{"1min":0.70499999999999996,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.3580000000000001,"5min":1.55,"15min":1.55},"last":1.2350000000000001},{"interface":"front","average":{"1min":1.006,"5min":1.0840000000000001,"15min":1.1240000000000001},"min":{"1min":0.79600000000000004,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.268,"5min":1.575,"15min":1.575},"last":1.014}]}]},{"osd":0,"up_from":88,"seq":377957122113,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1210020,"kb_used_data":1152092,"kb_used_omap":8,"kb_used_meta":57911,"kb_avail":92535132,"statfs":{"total":95995035648,"available":94755975168,"internally_reserved":0,"allocated":1179742208,"data_stored":1179179064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":59301758},"hb_peers":[2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:30:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.63600000000000001,"15min":0.64000000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.77500000000000002,"5min":1.2170000000000001,"15min":1.2170000000000001},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.77900000000000003,"15min":0.79400000000000004},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.97899999999999998,"5min":1.3480000000000001,"15min":1.3480000000000001},"last":0.91300000000000003}]},{"osd":2,"last update":"Sat Sep 23 18:31:36 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.75700000000000001,"15min":0.80800000000000005},"min":{"1min":0.36599999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.415,"5min":1.415,"15min":1.415},"last":0.21199999999999999},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.67100000000000004,"15min":0.68300000000000005},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.1659999999999999,"5min":1.1659999999999999,"15min":1.1659999999999999},"last":0.86699999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:31:04 2023","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":0.76500000000000001,"15min":0.78300000000000003},"min":{"1min":0.44800000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.1910000000000001,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.79200000000000004,"5min":0.82799999999999996,"15min":0.86399999999999999},"min":{"1min":0.46300000000000002,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.1639999999999999,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.83199999999999996}]},{"osd":4,"last update":"Sat Sep 23 18:31:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":0.98199999999999998,"15min":0.97699999999999998},"min":{"1min":0.63300000000000001,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":2.0800000000000001,"5min":2.0800000000000001,"15min":2.0800000000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":1.236,"5min":1.026,"15min":1.0229999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":3.9910000000000001,"5min":3.9910000000000001,"15min":3.9910000000000001},"last":0.56799999999999995}]},{"osd":5,"last update":"Sat Sep 23 18:31:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.004,"15min":1.0189999999999999},"min":{"1min":0.67900000000000005,"5min":0.67200000000000004,"15min":0.67200000000000004},"max":{"1min":2.0699999999999998,"5min":2.0699999999999998,"15min":2.0699999999999998},"last":0.76400000000000001},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.0289999999999999,"15min":1.04},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":3.4060000000000001,"5min":3.4060000000000001,"15min":3.4060000000000001},"last":0.89500000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:31:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":1.0580000000000001,"15min":1.0720000000000001},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":2.8639999999999999,"5min":2.8639999999999999,"15min":2.8639999999999999},"last":0.79800000000000004},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.0129999999999999,"15min":1.04},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":2.4649999999999999,"5min":2.4649999999999999,"15min":2.4649999999999999},"last":0.91900000000000004}]},{"osd":7,"last update":"Sat Sep 23 18:31:46 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.004,"15min":1.044},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":1.4510000000000001,"5min":1.619,"15min":1.619},"last":1.454},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.036,"15min":1.0740000000000001},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":2.7869999999999999,"5min":2.7869999999999999,"15min":2.7869999999999999},"last":0.51300000000000001}]}]},{"osd":2,"up_from":98,"seq":420906795061,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":865804,"kb_used_data":825988,"kb_used_omap":11,"kb_used_meta":39796,"kb_avail":92879348,"statfs":{"total":95995035648,"available":95108452352,"internally_reserved":0,"allocated":845811712,"data_stored":845269974,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11949,"internal_metadata":40751443},"hb_peers":[0,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:31:38 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.67800000000000005,"15min":0.68000000000000005},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":1.702,"5min":1.702,"15min":1.702},"last":1.0509999999999999},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.67400000000000004,"15min":0.68100000000000005},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.85099999999999998,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":1.1719999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:30:36 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":0.67100000000000004,"15min":0.65800000000000003},"min":{"1min":0.51400000000000001,"5min":0.31,"15min":0.31},"max":{"1min":1.0209999999999999,"5min":1.0209999999999999,"15min":1.0209999999999999},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.79100000000000004,"5min":0.749,"15min":0.752},"min":{"1min":0.501,"5min":0.252,"15min":0.252},"max":{"1min":1.016,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.74299999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:31:11 2023","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":0.71999999999999997,"15min":0.72599999999999998},"min":{"1min":0.54200000000000004,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.94199999999999995,"5min":1.3080000000000001,"15min":1.3080000000000001},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.76600000000000001,"5min":0.71799999999999997,"15min":0.72099999999999997},"min":{"1min":0.46100000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.242,"5min":1.5189999999999999,"15min":1.5189999999999999},"last":0.51100000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:31:38 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.95499999999999996,"15min":0.93899999999999995},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":1.393,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":0.85899999999999999},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.014,"15min":0.99399999999999999},"min":{"1min":0.64800000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":3.7040000000000002,"5min":3.7040000000000002,"15min":3.7040000000000002},"last":0.80000000000000004}]},{"osd":5,"last update":"Sat Sep 23 18:31:38 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.97799999999999998,"15min":1.0109999999999999},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.421,"5min":1.6990000000000001,"15min":1.6990000000000001},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.997,"15min":0.99399999999999999},"min":{"1min":0.69999999999999996,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.2849999999999999,"5min":1.488,"15min":1.488},"last":1.119}]},{"osd":6,"last update":"Sat Sep 23 18:31:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.0269999999999999,"15min":1.0369999999999999},"min":{"1min":0.67900000000000005,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.5580000000000001,"5min":1.5580000000000001,"15min":1.5580000000000001},"last":1.5800000000000001},{"interface":"front","average":{"1min":1.1559999999999999,"5min":1.103,"15min":1.085},"min":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":3.4820000000000002,"5min":3.4820000000000002,"15min":3.4820000000000002},"last":0.83699999999999997}]},{"osd":7,"last update":"Sat Sep 23 18:31:38 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.0600000000000001,"15min":1.0680000000000001},"min":{"1min":0.53400000000000003,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":2.0720000000000001,"5min":2.0720000000000001,"15min":2.0720000000000001},"last":0.82299999999999995},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.087,"15min":1.1140000000000001},"min":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.3600000000000001,"5min":1.7609999999999999,"15min":1.7609999999999999},"last":1.5209999999999999}]}]},{"osd":3,"up_from":103,"seq":442381631535,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":436016,"kb_used_data":400360,"kb_used_omap":4,"kb_used_meta":35643,"kb_avail":93309136,"statfs":{"total":95995035648,"available":95548555264,"internally_reserved":0,"allocated":409968640,"data_stored":409450814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4826,"internal_metadata":36498726},"hb_peers":[0,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:31:04 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":0.67700000000000005,"15min":0.68999999999999995},"min":{"1min":0.501,"5min":0.249,"15min":0.249},"max":{"1min":1.157,"5min":1.157,"15min":1.157},"last":0.82699999999999996},{"interface":"front","average":{"1min":0.746,"5min":0.72499999999999998,"15min":0.74099999999999999},"min":{"1min":0.41599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.153,"5min":1.153,"15min":1.153},"last":0.46100000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:30:04 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.70299999999999996,"15min":0.71499999999999997},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":1.022,"5min":1.022,"15min":1.022},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.69599999999999995,"5min":0.76400000000000001,"15min":0.77600000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.002,"5min":1.1439999999999999,"15min":1.1439999999999999},"last":0.55800000000000005}]},{"osd":2,"last update":"Sat Sep 23 18:31:04 2023","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":0.72499999999999998,"15min":0.74099999999999999},"min":{"1min":0.40899999999999997,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.24,"5min":1.24,"15min":1.24},"last":3.427},{"interface":"front","average":{"1min":0.73499999999999999,"5min":0.77800000000000002,"15min":0.80100000000000005},"min":{"1min":0.46100000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.125,"5min":1.1719999999999999,"15min":1.1719999999999999},"last":2.5430000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:31:04 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":0.95499999999999996,"15min":0.96499999999999997},"min":{"1min":0.79400000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.1799999999999999,"5min":1.3260000000000001,"15min":1.3260000000000001},"last":0.433},{"interface":"front","average":{"1min":0.96899999999999997,"5min":1.0069999999999999,"15min":1.0289999999999999},"min":{"1min":0.73299999999999998,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.3240000000000001,"5min":1.431,"15min":1.431},"last":0.58799999999999997}]},{"osd":5,"last update":"Sat Sep 23 18:31:04 2023","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":1.042,"15min":1.073},"min":{"1min":0.755,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":2.157,"5min":2.157,"15min":2.157},"last":0.97499999999999998},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.0660000000000001,"15min":1.121},"min":{"1min":0.746,"5min":0.623,"15min":0.623},"max":{"1min":1.304,"5min":1.5409999999999999,"15min":1.5409999999999999},"last":0.54200000000000004}]},{"osd":6,"last update":"Sat Sep 23 18:31:04 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":1.073,"15min":1.1180000000000001},"min":{"1min":0.70099999999999996,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.383,"5min":1.492,"15min":1.492},"last":0.496},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.0589999999999999,"15min":1.085},"min":{"1min":0.78400000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"max":{"1min":1.2470000000000001,"5min":1.425,"15min":1.425},"last":0.47399999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:31:04 2023","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":1.091,"15min":1.1220000000000001},"min":{"1min":0.86599999999999999,"5min":0.63500000000000001,"15min":0.63500000000000001},"max":{"1min":1.2390000000000001,"5min":1.5529999999999999,"15min":1.5529999999999999},"last":2.8260000000000001},{"interface":"front","average":{"1min":1.032,"5min":1.115,"15min":1.151},"min":{"1min":0.72399999999999998,"5min":0.71699999999999997,"15min":0.71699999999999997},"max":{"1min":1.2769999999999999,"5min":1.4530000000000001,"15min":1.4530000000000001},"last":0.78700000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019145,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1327520,"kb_used_data":1030296,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92417632,"statfs":{"total":95995035648,"available":94635655168,"internally_reserved":0,"allocated":1055023104,"data_stored":1054519160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":0.89900000000000002,"15min":0.92000000000000004},"min":{"1min":0.54400000000000004,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.6930000000000001,"5min":1.6930000000000001,"15min":1.6930000000000001},"last":1.6319999999999999},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.012,"15min":1.028},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.5110000000000001,"5min":4.5110000000000001,"15min":4.5110000000000001},"last":0.23799999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:30:08 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.86499999999999999,"15min":0.84699999999999998},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.2110000000000001,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.874},{"interface":"front","average":{"1min":0.88700000000000001,"5min":0.88500000000000001,"15min":0.88600000000000001},"min":{"1min":0.63300000000000001,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.103,"5min":1.224,"15min":1.224},"last":0.89700000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:31:42 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.03,"15min":1.0429999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":4.5700000000000003,"5min":4.5700000000000003,"15min":4.5700000000000003},"last":0.442},{"interface":"front","average":{"1min":0.90500000000000003,"5min":0.97399999999999998,"15min":1},"min":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"max":{"1min":1.736,"5min":1.736,"15min":1.736},"last":1.5389999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:31:13 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.90200000000000002,"15min":0.90600000000000003},"min":{"1min":0.65000000000000002,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.571,"5min":1.571,"15min":1.571},"last":0.32900000000000001},{"interface":"front","average":{"1min":1.006,"5min":1.022,"15min":1.0389999999999999},"min":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":1.3140000000000001,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":0.35299999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:31:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2150000000000001,"5min":1.0269999999999999,"15min":0.99199999999999999},"min":{"1min":0.72299999999999998,"5min":0.60699999999999998,"15min":0.51600000000000001},"max":{"1min":5.3010000000000002,"5min":5.3010000000000002,"15min":5.3010000000000002},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.98299999999999998,"5min":0.96099999999999997,"15min":0.97099999999999997},"min":{"1min":0.63800000000000001,"5min":0.58999999999999997,"15min":0.48199999999999998},"max":{"1min":2.5299999999999998,"5min":2.5299999999999998,"15min":2.5299999999999998},"last":1.514}]},{"osd":6,"last update":"Sat Sep 23 18:30:58 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":1.036,"15min":1.0309999999999999},"min":{"1min":0.88300000000000001,"5min":0.66300000000000003,"15min":0.57699999999999996},"max":{"1min":1.647,"5min":1.647,"15min":2.351},"last":0.38},{"interface":"front","average":{"1min":1.048,"5min":1.0509999999999999,"15min":1.0189999999999999},"min":{"1min":0.68799999999999994,"5min":0.66800000000000004,"15min":0.42899999999999999},"max":{"1min":1.4590000000000001,"5min":1.4590000000000001,"15min":2.621},"last":0.26100000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:31:42 2023","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":1.147,"15min":1.0940000000000001},"min":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.50800000000000001},"max":{"1min":4.5449999999999999,"5min":4.5449999999999999,"15min":4.5449999999999999},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.3,"5min":1.194,"15min":1.179},"min":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":5.3339999999999996,"5min":5.3339999999999996,"15min":5.3339999999999996},"last":0.41899999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822913,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1491340,"kb_used_data":1190020,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92253812,"statfs":{"total":95995035648,"available":94467903488,"internally_reserved":0,"allocated":1218580480,"data_stored":1218092149,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:30:44 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"min":{"1min":0.61499999999999999,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.1599999999999999,"5min":1.244,"15min":1.244},"last":2.6339999999999999},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.874,"15min":0.875},"min":{"1min":0.69499999999999995,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.099,"5min":1.1930000000000001,"15min":1.1930000000000001},"last":0.28399999999999997}]},{"osd":1,"last update":"Sat Sep 23 18:30:10 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.87,"15min":0.85599999999999998},"min":{"1min":0.64200000000000002,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.256,"5min":1.256,"15min":1.256},"last":1.139},{"interface":"front","average":{"1min":0.82499999999999996,"5min":0.89600000000000002,"15min":0.92600000000000005},"min":{"1min":0.54600000000000004,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.081,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":0.73799999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:31:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":0.93899999999999995,"15min":0.92200000000000004},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":3.8660000000000001,"5min":3.8660000000000001,"15min":3.8660000000000001},"last":0.64400000000000002},{"interface":"front","average":{"1min":1.054,"5min":0.99199999999999999,"15min":0.96899999999999997},"min":{"1min":0.63800000000000001,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":2.7000000000000002,"5min":2.7000000000000002,"15min":2.7000000000000002},"last":2.7000000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:31:07 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.95699999999999996,"15min":0.94699999999999995},"min":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"max":{"1min":1.218,"5min":1.218,"15min":1.218},"last":2.6720000000000002},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.91500000000000004,"15min":0.90700000000000003},"min":{"1min":0.76500000000000001,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.1919999999999999,"5min":1.1919999999999999,"15min":1.1919999999999999},"last":0.61199999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:31:12 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":0.90300000000000002,"15min":0.89200000000000002},"min":{"1min":0.59699999999999998,"5min":0.55100000000000005,"15min":0.39600000000000002},"max":{"1min":1.236,"5min":1.3879999999999999,"15min":2.0259999999999998},"last":0.33800000000000002},{"interface":"front","average":{"1min":1.004,"5min":0.94099999999999995,"15min":0.93200000000000005},"min":{"1min":0.69499999999999995,"5min":0.60399999999999998,"15min":0.36699999999999999},"max":{"1min":1.3049999999999999,"5min":1.319,"15min":1.73},"last":2.0600000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:31:07 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.0069999999999999,"15min":0.99299999999999999},"min":{"1min":0.67200000000000004,"5min":0.64700000000000002,"15min":0.47399999999999998},"max":{"1min":1.331,"5min":1.464,"15min":2.137},"last":0.317},{"interface":"front","average":{"1min":1.119,"5min":1.089,"15min":1.0680000000000001},"min":{"1min":0.94599999999999995,"5min":0.79200000000000004,"15min":0.51100000000000001},"max":{"1min":1.4139999999999999,"5min":1.4199999999999999,"15min":1.8779999999999999},"last":0.41699999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:30:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.056,"15min":1.0509999999999999},"min":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.53500000000000003},"max":{"1min":1.347,"5min":1.5589999999999999,"15min":2.1549999999999998},"last":10.135},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.083,"15min":1.081},"min":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.54300000000000004},"max":{"1min":1.3919999999999999,"5min":1.6990000000000001,"15min":2.2189999999999999},"last":0.434}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4224,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7725,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3429,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1231486976,"data_stored":1231486976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":859766784,"data_stored":859766784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":422313984,"data_stored":422313984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1078722560,"data_stored":1078722560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1215889408,"data_stored":1215889408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":546963456,"data_stored":546963456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":685834240,"data_stored":685834240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:31:51.902 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 1 2023-09-23T18:31:51.945 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:51.971 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:51.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:51.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:52.075 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:52.114 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:52.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:52.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:52.220 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:52.248 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:52.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:52.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:52.354 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:52.369 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:31:52.387 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:52.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:52.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:52.493 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:52.525 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:52.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:52.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:52.592 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 21 16 45188 45172 134.406 99.5625 0.0182984 0.00743423 2023-09-23T18:31:52.631 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:52.654 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:52.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:52.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:52.758 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:52.791 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:52.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:52.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:52.895 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:52.924 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:52.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:52.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:52 smithi173 bash[59422]: cluster 2023-09-23T18:31:51.676736+0000 mgr.y (mgr.26054) 397 : cluster 0 pgmap v337: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 2.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s; 13031/125814 objects degraded (10.357%) 2023-09-23T18:31:53.028 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:53.055 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:53.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:53.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:53.158 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:53.166 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:52 smithi098 bash[79516]: cluster 2023-09-23T18:31:51.676736+0000 mgr.y (mgr.26054) 397 : cluster 0 pgmap v337: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 2.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s; 13031/125814 objects degraded (10.357%) 2023-09-23T18:31:53.166 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:52 smithi098 bash[76817]: cluster 2023-09-23T18:31:51.676736+0000 mgr.y (mgr.26054) 397 : cluster 0 pgmap v337: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 2.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s; 13031/125814 objects degraded (10.357%) 2023-09-23T18:31:53.183 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:53.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:53.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:53.286 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:53.315 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:53.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:53.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:53.420 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:53.464 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:53.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:53.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:53.569 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:53.592 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 22 16 46814 46798 132.916 101.625 0.0130645 0.00751698 2023-09-23T18:31:53.595 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:53.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:53.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:53.699 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:53.723 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:53.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:53.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:53.772 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:31:53.828 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:53.857 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:53.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:53.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:53.876 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:31:53.962 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:53.990 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:53.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:53.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:54.094 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:54.130 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:54.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:54.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:54.233 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:54.258 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:54.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:54.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:54.364 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:54.387 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:54.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:54.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:54.492 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:54.520 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:54.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:54.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:54.592 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 23 16 48285 48269 131.134 91.9375 0.00638839 0.00762045 2023-09-23T18:31:54.624 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:54.657 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:54.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:54.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:54.762 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:54.794 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:54.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:54.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:54.899 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:54.939 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:54.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:54.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:55.045 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:55.055 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:54 smithi098 bash[79516]: cluster 2023-09-23T18:31:53.677223+0000 mgr.y (mgr.26054) 398 : cluster 0 pgmap v338: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 2.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s; 13565/128865 objects degraded (10.527%) 2023-09-23T18:31:55.055 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:54 smithi098 bash[79516]: cluster 2023-09-23T18:31:54.418505+0000 mon.a (mon.0) 559 : cluster 3 Health check update: Degraded data redundancy: 13565/128865 objects degraded (10.527%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T18:31:55.055 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:54 smithi098 bash[79516]: audit 2023-09-23T18:31:54.579461+0000 mon.a (mon.0) 560 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:31:55.055 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:54 smithi098 bash[76817]: cluster 2023-09-23T18:31:53.677223+0000 mgr.y (mgr.26054) 398 : cluster 0 pgmap v338: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 2.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s; 13565/128865 objects degraded (10.527%) 2023-09-23T18:31:55.055 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:54 smithi098 bash[76817]: cluster 2023-09-23T18:31:54.418505+0000 mon.a (mon.0) 559 : cluster 3 Health check update: Degraded data redundancy: 13565/128865 objects degraded (10.527%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T18:31:55.056 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:54 smithi098 bash[76817]: audit 2023-09-23T18:31:54.579461+0000 mon.a (mon.0) 560 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:31:55.071 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:55.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:55.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:55.174 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:55.182 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:54 smithi173 bash[59422]: cluster 2023-09-23T18:31:53.677223+0000 mgr.y (mgr.26054) 398 : cluster 0 pgmap v338: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 2.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s; 13565/128865 objects degraded (10.527%) 2023-09-23T18:31:55.182 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:54 smithi173 bash[59422]: cluster 2023-09-23T18:31:54.418505+0000 mon.a (mon.0) 559 : cluster 3 Health check update: Degraded data redundancy: 13565/128865 objects degraded (10.527%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T18:31:55.182 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:54 smithi173 bash[59422]: audit 2023-09-23T18:31:54.579461+0000 mon.a (mon.0) 560 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:31:55.211 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:55.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:55.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:55.314 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:55.341 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:55.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:55.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:55.447 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:55.471 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:55.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:55.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:55.576 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:55.592 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 24 16 49823 49807 129.676 96.125 0.00530109 0.00770535 2023-09-23T18:31:55.607 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:55.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:55.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:55.711 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:55.737 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:55.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:55.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:55.843 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:55.876 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:55.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:55.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:55.983 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:56.009 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:56.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:56.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:56.113 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:56.142 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:56.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:56.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:56.245 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:56.272 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:56.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:56.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:56.374 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:56.384 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:31:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:31:56] "GET /metrics HTTP/1.1" 200 36517 "" "Prometheus/2.33.4" 2023-09-23T18:31:56.402 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:56.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:56.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:56.433 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:31:56.508 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:56.538 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:56.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:56.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:56.592 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 25 16 51296 51280 128.171 92.0625 0.00439348 0.0077971 2023-09-23T18:31:56.646 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:56.671 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:56.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:56.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:56.777 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:56.804 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:56.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:56.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:56.909 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:56.937 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:56.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:56.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:57.041 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:57.050 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:56 smithi098 bash[76817]: cluster 2023-09-23T18:31:55.678050+0000 mgr.y (mgr.26054) 399 : cluster 0 pgmap v339: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 2.9 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s; 14517/140973 objects degraded (10.298%) 2023-09-23T18:31:57.050 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:56 smithi098 bash[79516]: cluster 2023-09-23T18:31:55.678050+0000 mgr.y (mgr.26054) 399 : cluster 0 pgmap v339: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 2.9 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s; 14517/140973 objects degraded (10.298%) 2023-09-23T18:31:57.071 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:57.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:57.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:57.174 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:57.183 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:56 smithi173 bash[59422]: cluster 2023-09-23T18:31:55.678050+0000 mgr.y (mgr.26054) 399 : cluster 0 pgmap v339: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 2.9 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s; 14517/140973 objects degraded (10.298%) 2023-09-23T18:31:57.203 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:57.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:57.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:57.263 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:31:57.264 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:31:57.307 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:57.334 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:57.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:57.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:57.437 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:57.462 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:31:57.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:57.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:31:57.567 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:57.592 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 26 16 52758 52742 126.755 91.375 0.00809855 0.00788313 2023-09-23T18:31:57.594 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:57.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:57.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:57.700 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:57.722 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:57.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:57.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:57.825 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:57.849 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:57.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:57.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:57.953 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:57.979 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:57.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:57.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:58.082 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:58.121 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:58.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:58.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:58.227 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:58.257 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:58.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:58.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:58.364 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:58.390 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:58.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:58.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:58.495 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:58.524 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:58.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:58.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:58.593 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 27 16 54082 54066 125.125 82.75 0.00501193 0.00797334 2023-09-23T18:31:58.628 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:58.653 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:31:58.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:58.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:31:58.756 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:58.787 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:31:58.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:58.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:31:58.892 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:58.900 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:58 smithi098 bash[76817]: cluster 2023-09-23T18:31:57.678710+0000 mgr.y (mgr.26054) 400 : cluster 0 pgmap v340: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 3.0 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s; 15462/149592 objects degraded (10.336%) 2023-09-23T18:31:58.901 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:58 smithi098 bash[79516]: cluster 2023-09-23T18:31:57.678710+0000 mgr.y (mgr.26054) 400 : cluster 0 pgmap v340: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 3.0 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s; 15462/149592 objects degraded (10.336%) 2023-09-23T18:31:58.917 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:31:58.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:58.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:31:59.021 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:59.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:58 smithi173 bash[59422]: cluster 2023-09-23T18:31:57.678710+0000 mgr.y (mgr.26054) 400 : cluster 0 pgmap v340: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 3.0 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s; 15462/149592 objects degraded (10.336%) 2023-09-23T18:31:59.044 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:59.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:59.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:59.148 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:59.176 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:59.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:59.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:59.279 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:59.311 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:59.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:59.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:59.387 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T18:31:59.415 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:59.474 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:59.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:59.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:59.578 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:59.593 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 28 16 55422 55406 123.647 83.75 0.011158 0.00808266 2023-09-23T18:31:59.614 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:31:59.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:59.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:31:59.718 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:59.746 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:31:59.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:59.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:31:59.851 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:59.853 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:31:59.853 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:31:59.874 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:31:59.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:31:59.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:31:59.980 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:00.013 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:00.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:00.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:00.119 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:00.129 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:31:59 smithi098 bash[79516]: cluster 2023-09-23T18:31:59.420082+0000 mon.a (mon.0) 561 : cluster 3 Health check update: Degraded data redundancy: 15462/149592 objects degraded (10.336%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T18:32:00.129 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:31:59 smithi098 bash[76817]: cluster 2023-09-23T18:31:59.420082+0000 mon.a (mon.0) 561 : cluster 3 Health check update: Degraded data redundancy: 15462/149592 objects degraded (10.336%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T18:32:00.146 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:00.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:00.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:00.249 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:00.259 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:31:59 smithi173 bash[59422]: cluster 2023-09-23T18:31:59.420082+0000 mon.a (mon.0) 561 : cluster 3 Health check update: Degraded data redundancy: 15462/149592 objects degraded (10.336%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T18:32:00.280 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:00.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:00.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:00.386 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:00.408 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:00.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:00.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:00.512 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:00.537 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:00.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:00.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:00.593 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 29 16 56834 56818 122.427 88.25 0.00746176 0.00816225 2023-09-23T18:32:00.641 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:00.668 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:00.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:00.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:00.772 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:00.802 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:00.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:00.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:00.908 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:00.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:00 smithi098 bash[79516]: cluster 2023-09-23T18:31:59.679431+0000 mgr.y (mgr.26054) 401 : cluster 0 pgmap v341: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 3.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s; 15894/152316 objects degraded (10.435%) 2023-09-23T18:32:00.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:00 smithi098 bash[76817]: cluster 2023-09-23T18:31:59.679431+0000 mgr.y (mgr.26054) 401 : cluster 0 pgmap v341: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 3.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s; 15894/152316 objects degraded (10.435%) 2023-09-23T18:32:00.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:00.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:00.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:01.049 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:01.058 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:00 smithi173 bash[59422]: cluster 2023-09-23T18:31:59.679431+0000 mgr.y (mgr.26054) 401 : cluster 0 pgmap v341: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 3.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s; 15894/152316 objects degraded (10.435%) 2023-09-23T18:32:01.081 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:01.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:01.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:01.186 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:01.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:01.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:01.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:01.330 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:01.365 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:01.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:01.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:01.469 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:01.492 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:01.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:01.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:01.593 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 30 16 58193 58177 121.177 84.9375 0.00938857 0.00824735 2023-09-23T18:32:01.595 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:01.620 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:01.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:01.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:01.725 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:01.754 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:01.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:01.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:01.858 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:01.892 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:01.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:01.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:01.998 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:02.025 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:02.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:02.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:02.130 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:02.152 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:02.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:02.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:02.255 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:02.285 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:02.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:02.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:02.388 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:02.420 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:02.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:02.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:02.523 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:02.559 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:02.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:02.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:02.593 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 31 16 59491 59475 119.885 81.125 0.00712543 0.0083364 2023-09-23T18:32:02.662 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:02.687 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:02.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:02.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:02.793 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:02.819 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:02.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:02.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:02.922 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:02.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:02 smithi098 bash[79516]: cluster 2023-09-23T18:32:01.680609+0000 mgr.y (mgr.26054) 402 : cluster 0 pgmap v342: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 3.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s; 17542/170583 objects degraded (10.284%) 2023-09-23T18:32:02.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:02 smithi098 bash[76817]: cluster 2023-09-23T18:32:01.680609+0000 mgr.y (mgr.26054) 402 : cluster 0 pgmap v342: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 3.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s; 17542/170583 objects degraded (10.284%) 2023-09-23T18:32:02.950 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:02.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:02.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:03.055 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:03.064 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:02 smithi173 bash[59422]: cluster 2023-09-23T18:32:01.680609+0000 mgr.y (mgr.26054) 402 : cluster 0 pgmap v342: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 3.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s; 17542/170583 objects degraded (10.284%) 2023-09-23T18:32:03.077 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:03.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:03.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:03.183 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:03.209 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:03.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:03.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:03.312 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:03.353 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:03.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:03.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:03.459 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:03.484 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:03.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:03.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:03.587 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:03.598 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 32 16 60832 60816 118.758 83.8125 0.0301171 0.00841486 2023-09-23T18:32:03.612 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:03.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:03.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:03.716 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:03.749 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:03.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:03.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:03.853 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:03.878 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:03.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:03.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:03.982 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:04.009 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:04.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:04.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:04.113 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:04.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:04.149 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:04.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:04.229 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:32:04.250 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:04.281 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:04.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:04.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:04.385 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:04.409 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:04.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:04.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:04.513 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:04.539 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:04.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:04.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:04.593 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 33 16 62047 62031 117.46 75.9375 0.0113429 0.00850825 2023-09-23T18:32:04.643 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:04.672 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:04.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:04.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:04.723 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-09-23T18:32:04.777 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:04.808 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:04.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:04.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:04.911 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:04.943 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:04.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:04.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:05.046 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:05.054 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:04 smithi098 bash[76817]: cluster 2023-09-23T18:32:03.681481+0000 mgr.y (mgr.26054) 403 : cluster 0 pgmap v343: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 17974/173169 objects degraded (10.379%) 2023-09-23T18:32:05.054 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:04 smithi098 bash[76817]: cluster 2023-09-23T18:32:04.421885+0000 mon.a (mon.0) 562 : cluster 3 Health check update: Degraded data redundancy: 17974/173169 objects degraded (10.379%), 10 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:05.054 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:04 smithi098 bash[76817]: audit 2023-09-23T18:32:04.716610+0000 mon.c (mon.2) 54 : audit 1 from='client.? 172.21.15.98:0/3114565193' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T18:32:05.055 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:04 smithi098 bash[76817]: audit 2023-09-23T18:32:04.717642+0000 mon.a (mon.0) 563 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T18:32:05.055 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:04 smithi098 bash[79516]: cluster 2023-09-23T18:32:03.681481+0000 mgr.y (mgr.26054) 403 : cluster 0 pgmap v343: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 17974/173169 objects degraded (10.379%) 2023-09-23T18:32:05.055 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:04 smithi098 bash[79516]: cluster 2023-09-23T18:32:04.421885+0000 mon.a (mon.0) 562 : cluster 3 Health check update: Degraded data redundancy: 17974/173169 objects degraded (10.379%), 10 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:05.055 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:04 smithi098 bash[79516]: audit 2023-09-23T18:32:04.716610+0000 mon.c (mon.2) 54 : audit 1 from='client.? 172.21.15.98:0/3114565193' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T18:32:05.055 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:04 smithi098 bash[79516]: audit 2023-09-23T18:32:04.717642+0000 mon.a (mon.0) 563 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T18:32:05.070 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:05.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:05.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:05.176 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:05.185 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:04 smithi173 bash[59422]: cluster 2023-09-23T18:32:03.681481+0000 mgr.y (mgr.26054) 403 : cluster 0 pgmap v343: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 17974/173169 objects degraded (10.379%) 2023-09-23T18:32:05.186 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:04 smithi173 bash[59422]: cluster 2023-09-23T18:32:04.421885+0000 mon.a (mon.0) 562 : cluster 3 Health check update: Degraded data redundancy: 17974/173169 objects degraded (10.379%), 10 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:05.186 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:04 smithi173 bash[59422]: audit 2023-09-23T18:32:04.716610+0000 mon.c (mon.2) 54 : audit 1 from='client.? 172.21.15.98:0/3114565193' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T18:32:05.186 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:04 smithi173 bash[59422]: audit 2023-09-23T18:32:04.717642+0000 mon.a (mon.0) 563 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T18:32:05.206 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:05.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:05.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:05.311 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:05.346 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:05.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:05.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:05.449 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:05.473 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:05.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:05.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:05.577 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:05.593 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 34 16 63382 63366 116.459 83.4375 0.01166 0.00858127 2023-09-23T18:32:05.614 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:05.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:05.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:05.718 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:05.749 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:05.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:05.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:05.852 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:05.884 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:05.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:05.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:05.988 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:06.011 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:06.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:06.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:06.116 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:06.143 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:06.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:06.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:06.249 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:06.274 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:06.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:06.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:06.379 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:06.388 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:32:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:32:06] "GET /metrics HTTP/1.1" 200 36517 "" "Prometheus/2.33.4" 2023-09-23T18:32:06.403 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:06.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:06.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:06.507 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:06.535 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:06.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:06.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:06.593 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 35 16 64502 64486 115.132 70 0.00467987 0.00867932 2023-09-23T18:32:06.641 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:06.706 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:06.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:06.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:06.812 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:06.846 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:06.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:06.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:06.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:06 smithi098 bash[76817]: cluster 2023-09-23T18:32:05.682372+0000 mgr.y (mgr.26054) 404 : cluster 0 pgmap v344: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 3.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 18868/182919 objects degraded (10.315%) 2023-09-23T18:32:06.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:06 smithi098 bash[79516]: cluster 2023-09-23T18:32:05.682372+0000 mgr.y (mgr.26054) 404 : cluster 0 pgmap v344: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 3.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 18868/182919 objects degraded (10.315%) 2023-09-23T18:32:06.950 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:06.977 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:06.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:06.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:07.081 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:07.090 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:06 smithi173 bash[59422]: cluster 2023-09-23T18:32:05.682372+0000 mgr.y (mgr.26054) 404 : cluster 0 pgmap v344: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 3.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 18868/182919 objects degraded (10.315%) 2023-09-23T18:32:07.106 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:07.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:07.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:07.211 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:07.229 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:07.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:07.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:07.332 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:07.356 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:07.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:07.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:07.460 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:07.490 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:07.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:07.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:07.587 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-09-23T18:32:07.593 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 36 16 65752 65736 114.104 78.125 0.00424309 0.00875705 2023-09-23T18:32:07.599 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:07.626 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:07.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:07.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:07.730 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:07.764 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:07.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:07.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:07.871 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:07.896 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:07.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:07.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:08.001 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:08.029 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:08.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:08.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:08.132 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:08.156 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:08.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:08.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:08.260 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:08.286 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:08.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:08.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:08.391 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:08.429 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:08.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:08.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:08.533 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:08.558 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:08.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:08.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:08.593 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 37 16 67020 67004 113.161 79.25 0.00973323 0.00883117 2023-09-23T18:32:08.663 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:08.691 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:08.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:08.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:08.795 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:08.820 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:08.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:08.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:08.926 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:08.951 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:08.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:08.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:09.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:08 smithi173 bash[59422]: cluster 2023-09-23T18:32:07.683199+0000 mgr.y (mgr.26054) 405 : cluster 0 pgmap v345: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 3.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 19675/190053 objects degraded (10.352%) 2023-09-23T18:32:09.058 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:09.099 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:09.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:09.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:09.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:08 smithi098 bash[79516]: cluster 2023-09-23T18:32:07.683199+0000 mgr.y (mgr.26054) 405 : cluster 0 pgmap v345: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 3.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 19675/190053 objects degraded (10.352%) 2023-09-23T18:32:09.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:08 smithi098 bash[76817]: cluster 2023-09-23T18:32:07.683199+0000 mgr.y (mgr.26054) 405 : cluster 0 pgmap v345: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 3.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 19675/190053 objects degraded (10.352%) 2023-09-23T18:32:09.204 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:09.244 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:09.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:09.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:09.347 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:09.373 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:09.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:09.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:09.478 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:09.510 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:09.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:09.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:09.594 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 38 16 68248 68232 112.203 76.75 0.0148997 0.0089065 2023-09-23T18:32:09.615 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:09.641 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:09.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:09.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:09.744 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:09.773 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:09.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:09.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:09.878 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:09.901 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:09.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:09.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:10.004 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:10.012 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:09 smithi098 bash[79516]: cluster 2023-09-23T18:32:09.423162+0000 mon.a (mon.0) 564 : cluster 3 Health check update: Degraded data redundancy: 19675/190053 objects degraded (10.352%), 10 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:10.012 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:09 smithi098 bash[79516]: audit 2023-09-23T18:32:09.578881+0000 mon.a (mon.0) 565 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:32:10.012 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:09 smithi098 bash[76817]: cluster 2023-09-23T18:32:09.423162+0000 mon.a (mon.0) 564 : cluster 3 Health check update: Degraded data redundancy: 19675/190053 objects degraded (10.352%), 10 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:10.012 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:09 smithi098 bash[76817]: audit 2023-09-23T18:32:09.578881+0000 mon.a (mon.0) 565 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:32:10.025 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:10.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:10.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:10.129 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:10.139 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:09 smithi173 bash[59422]: cluster 2023-09-23T18:32:09.423162+0000 mon.a (mon.0) 564 : cluster 3 Health check update: Degraded data redundancy: 19675/190053 objects degraded (10.352%), 10 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:10.139 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:09 smithi173 bash[59422]: audit 2023-09-23T18:32:09.578881+0000 mon.a (mon.0) 565 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:32:10.154 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:10.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:10.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:10.259 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:10.287 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:10.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:10.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:10.390 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:10.422 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:10.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:10.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:10.525 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:10.594 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 39 16 69509 69493 111.347 78.8125 0.00977543 0.00897611 2023-09-23T18:32:10.606 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:10.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:10.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:10.709 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:10.740 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:10.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:10.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:10.846 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:10.872 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:10.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:10.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:10.883 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:10 smithi173 bash[59422]: cluster 2023-09-23T18:32:09.684137+0000 mgr.y (mgr.26054) 406 : cluster 0 pgmap v346: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 20069/192399 objects degraded (10.431%) 2023-09-23T18:32:10.977 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:11.003 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:11.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:11.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:11.106 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:11.139 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:11.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:11.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:11.245 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:11.260 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:10 smithi098 bash[76817]: cluster 2023-09-23T18:32:09.684137+0000 mgr.y (mgr.26054) 406 : cluster 0 pgmap v346: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 20069/192399 objects degraded (10.431%) 2023-09-23T18:32:11.260 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:10 smithi098 bash[79516]: cluster 2023-09-23T18:32:09.684137+0000 mgr.y (mgr.26054) 406 : cluster 0 pgmap v346: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 20069/192399 objects degraded (10.431%) 2023-09-23T18:32:11.273 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:11.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:11.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:11.377 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:11.402 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:11.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:11.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:11.505 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:11.533 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:11.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:11.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:11.594 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:2023-09-23T18:32:11.593870+0000 min lat: 0.00116463 max lat: 0.161494 avg lat: 0.00905697 2023-09-23T18:32:11.594 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:32:11.594 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 40 16 70650 70634 110.346 71.3125 0.0129873 0.00905697 2023-09-23T18:32:11.637 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:11.667 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:11.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:11.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:11.769 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:11.799 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:11.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:11.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:11.890 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:11 smithi098 bash[79516]: cluster 2023-09-23T18:32:11.685737+0000 mgr.y (mgr.26054) 407 : cluster 0 pgmap v347: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 4.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s; 21619/208509 objects degraded (10.368%) 2023-09-23T18:32:11.890 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:11 smithi098 bash[76817]: cluster 2023-09-23T18:32:11.685737+0000 mgr.y (mgr.26054) 407 : cluster 0 pgmap v347: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 4.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s; 21619/208509 objects degraded (10.368%) 2023-09-23T18:32:11.901 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:11.909 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:11 smithi173 bash[59422]: cluster 2023-09-23T18:32:11.685737+0000 mgr.y (mgr.26054) 407 : cluster 0 pgmap v347: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 4.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s; 21619/208509 objects degraded (10.368%) 2023-09-23T18:32:11.938 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:11.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:11.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:12.043 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:12.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:12.074 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:12.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:12.175 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:12.216 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:12.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:12.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:12.319 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:12.341 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:12.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:12.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:12.444 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:12.448 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:32:12.469 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:12.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:12.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:12.489 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:32:12.528 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:32:12.564 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:32:12.574 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:12.588 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T18:32:12.588 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:32:12.588 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T18:32:12.588 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd out 5 2023-09-23T18:32:12.594 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 41 16 71519 71503 108.979 54.3125 0.0174708 0.00917063 2023-09-23T18:32:12.602 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:32:12.605 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:12.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:12.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:12.642 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:32:12.678 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:32:12.679 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:32:12.679 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T18:32:12.679 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:32:12.679 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:32:12.679 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:32:12.679 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:32:12.679 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:32:12.680 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:32:12.711 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:12.737 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:12.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:12.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:12.840 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:12.868 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:12.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:12.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:12.906 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is unset 2023-09-23T18:32:12.972 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:12.980 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:12 smithi173 bash[59422]: audit 2023-09-23T18:32:12.303163+0000 mon.a (mon.0) 566 : audit 0 from='client.? 172.21.15.98:0/1546268533' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:32:12.981 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:12 smithi173 bash[59422]: audit 2023-09-23T18:32:12.667446+0000 mon.a (mon.0) 567 : audit 1 from='client.? 172.21.15.98:0/3268777753' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:32:12.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:12.997 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:12.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:13.098 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:13.106 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:12 smithi098 bash[76817]: audit 2023-09-23T18:32:12.303163+0000 mon.a (mon.0) 566 : audit 0 from='client.? 172.21.15.98:0/1546268533' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:32:13.107 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:12 smithi098 bash[76817]: audit 2023-09-23T18:32:12.667446+0000 mon.a (mon.0) 567 : audit 1 from='client.? 172.21.15.98:0/3268777753' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:32:13.107 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:12 smithi098 bash[79516]: audit 2023-09-23T18:32:12.303163+0000 mon.a (mon.0) 566 : audit 0 from='client.? 172.21.15.98:0/1546268533' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:32:13.107 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:12 smithi098 bash[79516]: audit 2023-09-23T18:32:12.667446+0000 mon.a (mon.0) 567 : audit 1 from='client.? 172.21.15.98:0/3268777753' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:32:13.125 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:13.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:13.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:13.229 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:13.255 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:13.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:13.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:13.358 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:13.386 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:13.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:13.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:13.490 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:13.520 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:13.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:13.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:13.594 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 42 16 72647 72631 108.063 70.5 0.00644765 0.00924165 2023-09-23T18:32:13.626 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:13.649 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:13.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:13.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:13.752 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:13.781 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:13.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:13.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:13.890 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:13.913 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:13.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:13.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:13.921 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:13 smithi098 bash[76817]: cluster 2023-09-23T18:32:12.888612+0000 mon.a (mon.0) 568 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:32:13.921 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:13 smithi098 bash[76817]: audit 2023-09-23T18:32:12.901931+0000 mon.a (mon.0) 569 : audit 1 from='client.? 172.21.15.98:0/3268777753' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:32:13.921 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:13 smithi098 bash[76817]: cluster 2023-09-23T18:32:12.902004+0000 mon.a (mon.0) 570 : cluster 0 osdmap e113: 8 total, 7 up, 8 in 2023-09-23T18:32:13.921 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:13 smithi098 bash[76817]: cluster 2023-09-23T18:32:12.904229+0000 osd.4 (osd.4) 44 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T18:32:13.922 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:13 smithi098 bash[76817]: cluster 2023-09-23T18:32:12.906410+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T18:32:13.922 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:13 smithi098 bash[76817]: cluster 2023-09-23T18:32:12.910561+0000 osd.7 (osd.7) 27 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T18:32:13.922 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:13 smithi098 bash[76817]: cluster 2023-09-23T18:32:12.914154+0000 osd.6 (osd.6) 28 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T18:32:14.016 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:14.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:13 smithi173 bash[59422]: cluster 2023-09-23T18:32:12.888612+0000 mon.a (mon.0) 568 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:32:14.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:13 smithi173 bash[59422]: audit 2023-09-23T18:32:12.901931+0000 mon.a (mon.0) 569 : audit 1 from='client.? 172.21.15.98:0/3268777753' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:32:14.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:13 smithi173 bash[59422]: cluster 2023-09-23T18:32:12.902004+0000 mon.a (mon.0) 570 : cluster 0 osdmap e113: 8 total, 7 up, 8 in 2023-09-23T18:32:14.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:13 smithi173 bash[59422]: cluster 2023-09-23T18:32:12.904229+0000 osd.4 (osd.4) 44 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T18:32:14.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:13 smithi173 bash[59422]: cluster 2023-09-23T18:32:12.906410+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T18:32:14.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:13 smithi173 bash[59422]: cluster 2023-09-23T18:32:12.910561+0000 osd.7 (osd.7) 27 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T18:32:14.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:13 smithi173 bash[59422]: cluster 2023-09-23T18:32:12.914154+0000 osd.6 (osd.6) 28 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T18:32:14.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:13 smithi173 bash[59422]: cluster 2023-09-23T18:32:13.686646+0000 mgr.y (mgr.26054) 408 : cluster 0 pgmap v349: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 21950/210540 objects degraded (10.426%) 2023-09-23T18:32:14.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:14.050 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:14.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:14.152 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:14.176 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:14.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:14.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:14.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:13 smithi098 bash[76817]: cluster 2023-09-23T18:32:13.686646+0000 mgr.y (mgr.26054) 408 : cluster 0 pgmap v349: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 21950/210540 objects degraded (10.426%) 2023-09-23T18:32:14.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:13 smithi098 bash[79516]: cluster 2023-09-23T18:32:12.888612+0000 mon.a (mon.0) 568 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:32:14.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:13 smithi098 bash[79516]: audit 2023-09-23T18:32:12.901931+0000 mon.a (mon.0) 569 : audit 1 from='client.? 172.21.15.98:0/3268777753' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:32:14.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:13 smithi098 bash[79516]: cluster 2023-09-23T18:32:12.902004+0000 mon.a (mon.0) 570 : cluster 0 osdmap e113: 8 total, 7 up, 8 in 2023-09-23T18:32:14.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:13 smithi098 bash[79516]: cluster 2023-09-23T18:32:12.904229+0000 osd.4 (osd.4) 44 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T18:32:14.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:13 smithi098 bash[79516]: cluster 2023-09-23T18:32:12.906410+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T18:32:14.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:13 smithi098 bash[79516]: cluster 2023-09-23T18:32:12.910561+0000 osd.7 (osd.7) 27 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T18:32:14.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:13 smithi098 bash[79516]: cluster 2023-09-23T18:32:12.914154+0000 osd.6 (osd.6) 28 : cluster 3 failed to encode map e113 with expected crc 2023-09-23T18:32:14.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:13 smithi098 bash[79516]: cluster 2023-09-23T18:32:13.686646+0000 mgr.y (mgr.26054) 408 : cluster 0 pgmap v349: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 21950/210540 objects degraded (10.426%) 2023-09-23T18:32:14.281 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:14.310 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:14.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:14.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:14.413 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:14.447 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:14.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:14.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:14.552 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:14.579 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:14.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:14.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:14.594 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 43 16 73701 73685 107.082 65.875 0.0209676 0.00933359 2023-09-23T18:32:14.687 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:14.724 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:14.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:14.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:14.828 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:14.852 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:14.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:14.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:14.957 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:14.980 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:14.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:14.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:15.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:14 smithi173 bash[59422]: cluster 2023-09-23T18:32:14.424332+0000 mon.a (mon.0) 571 : cluster 3 Health check update: Degraded data redundancy: 21950/210540 objects degraded (10.426%), 10 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:15.084 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:15.115 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:15.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:15.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:15.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:14 smithi098 bash[76817]: cluster 2023-09-23T18:32:14.424332+0000 mon.a (mon.0) 571 : cluster 3 Health check update: Degraded data redundancy: 21950/210540 objects degraded (10.426%), 10 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:15.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:14 smithi098 bash[79516]: cluster 2023-09-23T18:32:14.424332+0000 mon.a (mon.0) 571 : cluster 3 Health check update: Degraded data redundancy: 21950/210540 objects degraded (10.426%), 10 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:15.221 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:15.254 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:15.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:15.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:15.361 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:15.389 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:15.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:15.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:15.494 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:15.543 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:15.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:15.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:15.553 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:32:15.553 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:32:15.594 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 44 16 74867 74851 106.304 72.875 0.0117664 0.009402 2023-09-23T18:32:15.652 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:15.680 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:15.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:15.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:15.785 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:15.818 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:15.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:15.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:15.870 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:32:15.870 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [], 2023-09-23T18:32:15.870 INFO:teuthology.orchestra.run.smithi098.stdout: "complaint_time": 30, 2023-09-23T18:32:15.870 INFO:teuthology.orchestra.run.smithi098.stdout: "num_blocked_ops": 0 2023-09-23T18:32:15.870 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:32:15.910 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:15 smithi098 bash[76817]: cluster 2023-09-23T18:32:15.687448+0000 mgr.y (mgr.26054) 409 : cluster 0 pgmap v350: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 4.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s; 22668/218778 objects degraded (10.361%) 2023-09-23T18:32:15.912 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:15 smithi173 bash[59422]: cluster 2023-09-23T18:32:15.687448+0000 mgr.y (mgr.26054) 409 : cluster 0 pgmap v350: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 4.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s; 22668/218778 objects degraded (10.361%) 2023-09-23T18:32:15.925 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:15.956 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:15.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:15.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:16.061 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:16.090 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:16.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:16.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:16.190 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:32:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:32:16] "GET /metrics HTTP/1.1" 200 36523 "" "Prometheus/2.33.4" 2023-09-23T18:32:16.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:15 smithi098 bash[79516]: cluster 2023-09-23T18:32:15.687448+0000 mgr.y (mgr.26054) 409 : cluster 0 pgmap v350: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 4.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s; 22668/218778 objects degraded (10.361%) 2023-09-23T18:32:16.196 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:16.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:16.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:16.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:16.332 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:16.359 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:16.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:16.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:16.462 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:16.493 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:16.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:16.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:16.594 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 45 16 76036 76020 105.565 73.0625 0.011164 0.00946723 2023-09-23T18:32:16.598 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:16.626 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:16.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:16.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:16.729 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:16.757 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:16.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:16.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:16.862 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:16.895 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:16.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:16.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:16.966 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T18:32:17.003 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:17.027 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:17.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:17.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:17.132 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:17.157 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:17.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:17.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:17.263 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:17.287 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:17.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:17.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:17.399 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:17.419 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:17.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:17.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:17.527 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:17.554 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:32:17.558 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:17.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:17.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:17.562 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:32:17.594 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 46 16 77121 77105 104.745 67.8125 0.00598513 0.00954166 2023-09-23T18:32:17.662 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:17.688 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:17.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:17.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:17.791 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:17.819 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:17.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:17.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:17.925 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:17.951 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:17.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:17.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:18.056 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:18.085 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:18.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:18.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:18.189 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:18.215 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:18.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:18.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:18.319 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:18.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:18.349 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:18.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:18.406 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:32:18.450 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:18.483 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:18.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:18.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:18.586 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:18.594 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 47 16 78227 78211 103.987 69.125 0.0108934 0.00961041 2023-09-23T18:32:18.620 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:18.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:18.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:18.724 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:18.753 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:18.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:18.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:18.856 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:18.865 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:18 smithi173 bash[59422]: cluster 2023-09-23T18:32:17.693530+0000 mgr.y (mgr.26054) 410 : cluster 0 pgmap v351: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 4.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s; 23355/224778 objects degraded (10.390%) 2023-09-23T18:32:18.884 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:18.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:18.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:18.988 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:19.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:19.020 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:19.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:19.121 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:19.129 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:18 smithi098 bash[79516]: cluster 2023-09-23T18:32:17.693530+0000 mgr.y (mgr.26054) 410 : cluster 0 pgmap v351: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 4.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s; 23355/224778 objects degraded (10.390%) 2023-09-23T18:32:19.129 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:18 smithi098 bash[76817]: cluster 2023-09-23T18:32:17.693530+0000 mgr.y (mgr.26054) 410 : cluster 0 pgmap v351: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 4.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s; 23355/224778 objects degraded (10.390%) 2023-09-23T18:32:19.145 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:19.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:19.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:19.249 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:19.283 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:19.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:19.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:19.388 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:19.418 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:19.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:19.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:19.525 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:19.551 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:19.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:19.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:19.594 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 48 16 79342 79326 103.272 69.6875 0.0279448 0.00967751 2023-09-23T18:32:19.656 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:19.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:19.685 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:19.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:19.786 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:19.817 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:19.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:19.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:19.919 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:19.927 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:19 smithi173 bash[59422]: cluster 2023-09-23T18:32:19.426364+0000 mon.a (mon.0) 572 : cluster 3 Health check update: Degraded data redundancy: 23355/224778 objects degraded (10.390%), 10 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:19.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:19.949 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:19.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:20.008 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:19 smithi098 bash[76817]: cluster 2023-09-23T18:32:19.426364+0000 mon.a (mon.0) 572 : cluster 3 Health check update: Degraded data redundancy: 23355/224778 objects degraded (10.390%), 10 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:20.009 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:19 smithi098 bash[79516]: cluster 2023-09-23T18:32:19.426364+0000 mon.a (mon.0) 572 : cluster 3 Health check update: Degraded data redundancy: 23355/224778 objects degraded (10.390%), 10 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:20.050 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:20.076 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:20.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:20.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:20.181 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:20.219 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:20.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:20.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:20.324 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:20.357 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:20.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:20.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:20.463 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:20.492 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:20.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:20.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:20.595 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 49 16 80468 80452 102.601 70.375 0.00887192 0.00974123 2023-09-23T18:32:20.598 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:20.626 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:20.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:20.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:20.730 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:20.759 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:20.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:20.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:20.863 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:20.888 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:20.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:20.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:20.993 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:21.002 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:20 smithi173 bash[59422]: cluster 2023-09-23T18:32:19.694447+0000 mgr.y (mgr.26054) 411 : cluster 0 pgmap v352: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.14k op/s; 23679/226776 objects degraded (10.442%) 2023-09-23T18:32:21.026 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:21.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:21.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:21.130 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:21.144 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:20 smithi098 bash[76817]: cluster 2023-09-23T18:32:19.694447+0000 mgr.y (mgr.26054) 411 : cluster 0 pgmap v352: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.14k op/s; 23679/226776 objects degraded (10.442%) 2023-09-23T18:32:21.144 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:20 smithi098 bash[79516]: cluster 2023-09-23T18:32:19.694447+0000 mgr.y (mgr.26054) 411 : cluster 0 pgmap v352: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.14k op/s; 23679/226776 objects degraded (10.442%) 2023-09-23T18:32:21.160 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:21.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:21.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:21.263 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:21.296 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:21.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:21.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:21.400 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:21.426 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:21.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:21.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:21.531 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:21.555 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:21.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:21.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:21.594 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 50 16 81560 81544 101.914 68.25 0.0118309 0.00980624 2023-09-23T18:32:21.616 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:32:21.618 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:32:21.622 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:32:21.659 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:21.684 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:21.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:21.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:21.790 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:21.815 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:21.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:21.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:21.919 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:21.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:21.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:21.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:22.047 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:22.076 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:22.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:22.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:22.181 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:22.215 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:22.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:22.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:22.322 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:22.353 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:22.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:22.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:22.456 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:22.483 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:22.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:22.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:22.586 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:22.595 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 51 16 82622 82606 101.217 66.375 0.0361399 0.00987106 2023-09-23T18:32:22.618 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:22.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:22.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:22.722 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:22.746 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:22.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:22.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:22.850 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:22.876 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:22.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:22.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:22.982 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:22.990 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:22 smithi098 bash[79516]: cluster 2023-09-23T18:32:21.695441+0000 mgr.y (mgr.26054) 412 : cluster 0 pgmap v353: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 4.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 25109/241482 objects degraded (10.398%) 2023-09-23T18:32:22.990 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:22 smithi098 bash[76817]: cluster 2023-09-23T18:32:21.695441+0000 mgr.y (mgr.26054) 412 : cluster 0 pgmap v353: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 4.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 25109/241482 objects degraded (10.398%) 2023-09-23T18:32:23.009 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:23.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:23.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:23.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:22 smithi173 bash[59422]: cluster 2023-09-23T18:32:21.695441+0000 mgr.y (mgr.26054) 412 : cluster 0 pgmap v353: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 4.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 25109/241482 objects degraded (10.398%) 2023-09-23T18:32:23.112 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:23.136 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:23.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:23.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:23.239 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:23.265 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:23.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:23.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:23.367 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:23.395 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:23.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:23.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:23.499 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:23.527 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:23.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:23.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:23.595 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 52 16 83597 83581 100.442 60.9375 0.00407555 0.00994233 2023-09-23T18:32:23.630 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:23.661 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:23.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:23.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:23.765 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:23.794 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:23.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:23.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:23.900 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:23.933 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:23.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:23.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:24.040 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:24.063 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:24.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:24.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:24.167 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:24.195 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:24.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:24.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:24.299 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:24.346 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:24.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:24.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:24.451 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:24.475 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:24.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:24.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:24.578 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:24.595 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 53 16 84495 84479 99.6057 56.125 0.0170555 0.0100339 2023-09-23T18:32:24.611 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:24.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:24.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:24.716 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:24.750 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:24.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:24.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:24.857 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:24.884 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:24.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:24.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:24.988 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:24.996 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:24 smithi173 bash[59422]: cluster 2023-09-23T18:32:23.696142+0000 mgr.y (mgr.26054) 413 : cluster 0 pgmap v354: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 5.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 25466/243543 objects degraded (10.456%) 2023-09-23T18:32:24.996 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:24 smithi173 bash[59422]: cluster 2023-09-23T18:32:24.427774+0000 mon.a (mon.0) 573 : cluster 3 Health check update: Degraded data redundancy: 25466/243543 objects degraded (10.456%), 10 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:24.997 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:24 smithi173 bash[59422]: audit 2023-09-23T18:32:24.583093+0000 mon.a (mon.0) 574 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:32:24.997 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:24 smithi173 bash[59422]: audit 2023-09-23T18:32:24.584124+0000 mon.a (mon.0) 575 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:32:25.020 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:25.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:25.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:25.123 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:25.152 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:25.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:25.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:25.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:24 smithi098 bash[76817]: cluster 2023-09-23T18:32:23.696142+0000 mgr.y (mgr.26054) 413 : cluster 0 pgmap v354: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 5.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 25466/243543 objects degraded (10.456%) 2023-09-23T18:32:25.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:24 smithi098 bash[76817]: cluster 2023-09-23T18:32:24.427774+0000 mon.a (mon.0) 573 : cluster 3 Health check update: Degraded data redundancy: 25466/243543 objects degraded (10.456%), 10 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:25.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:24 smithi098 bash[76817]: audit 2023-09-23T18:32:24.583093+0000 mon.a (mon.0) 574 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:32:25.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:24 smithi098 bash[76817]: audit 2023-09-23T18:32:24.584124+0000 mon.a (mon.0) 575 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:32:25.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:24 smithi098 bash[79516]: cluster 2023-09-23T18:32:23.696142+0000 mgr.y (mgr.26054) 413 : cluster 0 pgmap v354: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 5.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 25466/243543 objects degraded (10.456%) 2023-09-23T18:32:25.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:24 smithi098 bash[79516]: cluster 2023-09-23T18:32:24.427774+0000 mon.a (mon.0) 573 : cluster 3 Health check update: Degraded data redundancy: 25466/243543 objects degraded (10.456%), 10 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:25.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:24 smithi098 bash[79516]: audit 2023-09-23T18:32:24.583093+0000 mon.a (mon.0) 574 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:32:25.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:24 smithi098 bash[79516]: audit 2023-09-23T18:32:24.584124+0000 mon.a (mon.0) 575 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:32:25.256 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:25.297 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:25.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:25.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:25.407 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:25.432 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:25.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:25.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:25.535 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:25.568 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:25.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:25.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:25.595 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 54 16 85552 85536 98.9845 66.0625 0.0214761 0.0100976 2023-09-23T18:32:25.619 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:25 smithi098 bash[76817]: debug 2023-09-23T18:32:25.384+0000 7fc8a2a01700 -1 mon.a@0(leader).osd e113 definitely_dead 0 2023-09-23T18:32:25.674 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:25.700 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:25.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:25.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:25.785 INFO:teuthology.orchestra.run.smithi098.stderr:marked out osd.5. 2023-09-23T18:32:25.806 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:25.828 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:25.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:25.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:25.876 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:25 smithi098 bash[76817]: audit 2023-09-23T18:32:25.387143+0000 mon.c (mon.2) 55 : audit 1 from='client.? 172.21.15.98:0/3249001447' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T18:32:25.876 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:25 smithi098 bash[76817]: audit 2023-09-23T18:32:25.387871+0000 mon.a (mon.0) 576 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T18:32:25.876 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:25 smithi098 bash[76817]: cluster 2023-09-23T18:32:25.388022+0000 mon.a (mon.0) 577 : cluster 1 Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T18:32:25.876 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:25 smithi098 bash[79516]: audit 2023-09-23T18:32:25.387143+0000 mon.c (mon.2) 55 : audit 1 from='client.? 172.21.15.98:0/3249001447' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T18:32:25.876 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:25 smithi098 bash[79516]: audit 2023-09-23T18:32:25.387871+0000 mon.a (mon.0) 576 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T18:32:25.876 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:25 smithi098 bash[79516]: cluster 2023-09-23T18:32:25.388022+0000 mon.a (mon.0) 577 : cluster 1 Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T18:32:25.932 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:25.945 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:25 smithi173 bash[59422]: audit 2023-09-23T18:32:25.387143+0000 mon.c (mon.2) 55 : audit 1 from='client.? 172.21.15.98:0/3249001447' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T18:32:25.945 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:25 smithi173 bash[59422]: audit 2023-09-23T18:32:25.387871+0000 mon.a (mon.0) 576 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T18:32:25.945 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:25 smithi173 bash[59422]: cluster 2023-09-23T18:32:25.388022+0000 mon.a (mon.0) 577 : cluster 1 Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T18:32:25.967 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:25.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:25.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:26.070 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:26.099 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:26.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:26.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:26.190 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:32:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:32:26] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.33.4" 2023-09-23T18:32:26.202 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:26.232 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:26.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:26.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:26.336 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:26.364 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:26.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:26.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:26.468 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:26.498 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:26.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:26.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:26.595 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 55 16 85752 85736 97.4121 12.5 0.00668101 0.0101107 2023-09-23T18:32:26.603 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:26.629 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:26.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:26.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:26.732 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:26.758 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:26.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:26.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:26.861 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:26.891 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:26.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:26.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:26.995 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:27.008 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:26 smithi098 bash[79516]: cluster 2023-09-23T18:32:25.697992+0000 mgr.y (mgr.26054) 414 : cluster 0 pgmap v355: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 5.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 26101/251211 objects degraded (10.390%) 2023-09-23T18:32:27.008 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:26 smithi098 bash[79516]: cluster 2023-09-23T18:32:25.774936+0000 osd.6 (osd.6) 29 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T18:32:27.008 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:26 smithi098 bash[79516]: cluster 2023-09-23T18:32:25.774975+0000 osd.7 (osd.7) 28 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T18:32:27.008 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:26 smithi098 bash[79516]: audit 2023-09-23T18:32:25.776817+0000 mon.a (mon.0) 578 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T18:32:27.008 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:26 smithi098 bash[79516]: cluster 2023-09-23T18:32:25.777030+0000 mon.a (mon.0) 579 : cluster 0 osdmap e114: 8 total, 7 up, 7 in 2023-09-23T18:32:27.008 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:26 smithi098 bash[76817]: cluster 2023-09-23T18:32:25.697992+0000 mgr.y (mgr.26054) 414 : cluster 0 pgmap v355: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 5.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 26101/251211 objects degraded (10.390%) 2023-09-23T18:32:27.008 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:26 smithi098 bash[76817]: cluster 2023-09-23T18:32:25.774936+0000 osd.6 (osd.6) 29 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T18:32:27.008 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:26 smithi098 bash[76817]: cluster 2023-09-23T18:32:25.774975+0000 osd.7 (osd.7) 28 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T18:32:27.009 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:26 smithi098 bash[76817]: audit 2023-09-23T18:32:25.776817+0000 mon.a (mon.0) 578 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T18:32:27.009 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:26 smithi098 bash[76817]: cluster 2023-09-23T18:32:25.777030+0000 mon.a (mon.0) 579 : cluster 0 osdmap e114: 8 total, 7 up, 7 in 2023-09-23T18:32:27.021 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:27.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:27.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:27.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:26 smithi173 bash[59422]: cluster 2023-09-23T18:32:25.697992+0000 mgr.y (mgr.26054) 414 : cluster 0 pgmap v355: 89 pgs: 10 active+undersized+degraded, 27 active+undersized, 52 active+clean; 5.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 26101/251211 objects degraded (10.390%) 2023-09-23T18:32:27.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:26 smithi173 bash[59422]: cluster 2023-09-23T18:32:25.774936+0000 osd.6 (osd.6) 29 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T18:32:27.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:26 smithi173 bash[59422]: cluster 2023-09-23T18:32:25.774975+0000 osd.7 (osd.7) 28 : cluster 3 failed to encode map e114 with expected crc 2023-09-23T18:32:27.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:26 smithi173 bash[59422]: audit 2023-09-23T18:32:25.776817+0000 mon.a (mon.0) 578 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T18:32:27.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:26 smithi173 bash[59422]: cluster 2023-09-23T18:32:25.777030+0000 mon.a (mon.0) 579 : cluster 0 osdmap e114: 8 total, 7 up, 7 in 2023-09-23T18:32:27.125 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:27.156 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:27.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:27.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:27.260 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:27.285 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:27.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:27.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:27.388 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:27.421 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:27.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:27.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:27.525 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:27.547 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:27.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:27.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:27.595 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 56 16 85752 85736 95.6726 0 - 0.0101107 2023-09-23T18:32:27.651 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:27.674 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:27.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:27.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:27.777 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:27.803 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:27.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:27.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:27.880 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:27 smithi098 bash[79516]: cluster 2023-09-23T18:32:26.778868+0000 osd.6 (osd.6) 30 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T18:32:27.880 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:27 smithi098 bash[79516]: cluster 2023-09-23T18:32:26.780519+0000 mon.a (mon.0) 580 : cluster 0 osdmap e115: 8 total, 7 up, 7 in 2023-09-23T18:32:27.880 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:27 smithi098 bash[79516]: cluster 2023-09-23T18:32:26.781815+0000 osd.7 (osd.7) 29 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T18:32:27.880 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:27 smithi098 bash[79516]: cluster 2023-09-23T18:32:26.782079+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T18:32:27.881 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:27 smithi098 bash[79516]: cluster 2023-09-23T18:32:26.782350+0000 osd.6 (osd.6) 31 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T18:32:27.881 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:27 smithi098 bash[79516]: cluster 2023-09-23T18:32:26.782551+0000 osd.4 (osd.4) 45 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T18:32:27.881 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:27 smithi098 bash[79516]: cluster 2023-09-23T18:32:26.783526+0000 osd.7 (osd.7) 30 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T18:32:27.881 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:27 smithi098 bash[79516]: cluster 2023-09-23T18:32:26.785323+0000 osd.6 (osd.6) 32 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T18:32:27.881 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:27 smithi098 bash[76817]: cluster 2023-09-23T18:32:26.778868+0000 osd.6 (osd.6) 30 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T18:32:27.881 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:27 smithi098 bash[76817]: cluster 2023-09-23T18:32:26.780519+0000 mon.a (mon.0) 580 : cluster 0 osdmap e115: 8 total, 7 up, 7 in 2023-09-23T18:32:27.881 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:27 smithi098 bash[76817]: cluster 2023-09-23T18:32:26.781815+0000 osd.7 (osd.7) 29 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T18:32:27.881 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:27 smithi098 bash[76817]: cluster 2023-09-23T18:32:26.782079+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T18:32:27.882 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:27 smithi098 bash[76817]: cluster 2023-09-23T18:32:26.782350+0000 osd.6 (osd.6) 31 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T18:32:27.882 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:27 smithi098 bash[76817]: cluster 2023-09-23T18:32:26.782551+0000 osd.4 (osd.4) 45 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T18:32:27.882 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:27 smithi098 bash[76817]: cluster 2023-09-23T18:32:26.783526+0000 osd.7 (osd.7) 30 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T18:32:27.882 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:27 smithi098 bash[76817]: cluster 2023-09-23T18:32:26.785323+0000 osd.6 (osd.6) 32 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T18:32:27.918 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:27.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:27.950 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:27.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:28.051 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:28.058 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:27 smithi173 bash[59422]: cluster 2023-09-23T18:32:26.778868+0000 osd.6 (osd.6) 30 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T18:32:28.058 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:27 smithi173 bash[59422]: cluster 2023-09-23T18:32:26.780519+0000 mon.a (mon.0) 580 : cluster 0 osdmap e115: 8 total, 7 up, 7 in 2023-09-23T18:32:28.058 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:27 smithi173 bash[59422]: cluster 2023-09-23T18:32:26.781815+0000 osd.7 (osd.7) 29 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T18:32:28.058 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:27 smithi173 bash[59422]: cluster 2023-09-23T18:32:26.782079+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T18:32:28.059 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:27 smithi173 bash[59422]: cluster 2023-09-23T18:32:26.782350+0000 osd.6 (osd.6) 31 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T18:32:28.059 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:27 smithi173 bash[59422]: cluster 2023-09-23T18:32:26.782551+0000 osd.4 (osd.4) 45 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T18:32:28.059 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:27 smithi173 bash[59422]: cluster 2023-09-23T18:32:26.783526+0000 osd.7 (osd.7) 30 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T18:32:28.059 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:27 smithi173 bash[59422]: cluster 2023-09-23T18:32:26.785323+0000 osd.6 (osd.6) 32 : cluster 3 failed to encode map e115 with expected crc 2023-09-23T18:32:28.076 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:28.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:28.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:28.179 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:28.207 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:28.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:28.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:28.310 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:28.351 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:28.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:28.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:28.455 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:28.476 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:28.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:28.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:28.579 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:28.595 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 57 16 86050 86034 94.3209 9.3125 0.00546122 0.0105932 2023-09-23T18:32:28.603 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:28.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:28.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:28.708 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:28.735 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:28.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:28.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:28.841 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:28.858 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:28.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:28.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:28.961 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:28.997 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:29.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:29.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:29.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:28 smithi173 bash[59422]: cluster 2023-09-23T18:32:27.698838+0000 mgr.y (mgr.26054) 415 : cluster 0 pgmap v358: 89 pgs: 2 unknown, 6 peering, 10 active+undersized+degraded, 26 active+undersized, 45 active+clean; 5.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s; 26636/255951 objects degraded (10.407%) 2023-09-23T18:32:29.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:28 smithi173 bash[59422]: cluster 2023-09-23T18:32:27.776602+0000 mon.a (mon.0) 581 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:32:29.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:28 smithi173 bash[59422]: cluster 2023-09-23T18:32:27.796564+0000 mon.a (mon.0) 582 : cluster 0 osdmap e116: 8 total, 7 up, 7 in 2023-09-23T18:32:29.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:28 smithi173 bash[59422]: cluster 2023-09-23T18:32:27.797979+0000 osd.7 (osd.7) 31 : cluster 3 failed to encode map e116 with expected crc 2023-09-23T18:32:29.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:28 smithi173 bash[59422]: cluster 2023-09-23T18:32:27.798095+0000 osd.4 (osd.4) 46 : cluster 3 failed to encode map e116 with expected crc 2023-09-23T18:32:29.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:28 smithi173 bash[59422]: cluster 2023-09-23T18:32:27.798490+0000 osd.6 (osd.6) 33 : cluster 3 failed to encode map e116 with expected crc 2023-09-23T18:32:29.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:28 smithi173 bash[59422]: cluster 2023-09-23T18:32:27.800080+0000 osd.4 (osd.4) 47 : cluster 3 failed to encode map e116 with expected crc 2023-09-23T18:32:29.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:28 smithi173 bash[59422]: cluster 2023-09-23T18:32:27.800730+0000 osd.6 (osd.6) 34 : cluster 3 failed to encode map e116 with expected crc 2023-09-23T18:32:29.102 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:29.110 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:28 smithi098 bash[76817]: cluster 2023-09-23T18:32:27.698838+0000 mgr.y (mgr.26054) 415 : cluster 0 pgmap v358: 89 pgs: 2 unknown, 6 peering, 10 active+undersized+degraded, 26 active+undersized, 45 active+clean; 5.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s; 26636/255951 objects degraded (10.407%) 2023-09-23T18:32:29.111 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:28 smithi098 bash[76817]: cluster 2023-09-23T18:32:27.776602+0000 mon.a (mon.0) 581 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:32:29.111 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:28 smithi098 bash[76817]: cluster 2023-09-23T18:32:27.796564+0000 mon.a (mon.0) 582 : cluster 0 osdmap e116: 8 total, 7 up, 7 in 2023-09-23T18:32:29.111 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:28 smithi098 bash[76817]: cluster 2023-09-23T18:32:27.797979+0000 osd.7 (osd.7) 31 : cluster 3 failed to encode map e116 with expected crc 2023-09-23T18:32:29.111 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:28 smithi098 bash[76817]: cluster 2023-09-23T18:32:27.798095+0000 osd.4 (osd.4) 46 : cluster 3 failed to encode map e116 with expected crc 2023-09-23T18:32:29.111 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:28 smithi098 bash[76817]: cluster 2023-09-23T18:32:27.798490+0000 osd.6 (osd.6) 33 : cluster 3 failed to encode map e116 with expected crc 2023-09-23T18:32:29.111 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:28 smithi098 bash[76817]: cluster 2023-09-23T18:32:27.800080+0000 osd.4 (osd.4) 47 : cluster 3 failed to encode map e116 with expected crc 2023-09-23T18:32:29.111 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:28 smithi098 bash[76817]: cluster 2023-09-23T18:32:27.800730+0000 osd.6 (osd.6) 34 : cluster 3 failed to encode map e116 with expected crc 2023-09-23T18:32:29.111 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:28 smithi098 bash[79516]: cluster 2023-09-23T18:32:27.698838+0000 mgr.y (mgr.26054) 415 : cluster 0 pgmap v358: 89 pgs: 2 unknown, 6 peering, 10 active+undersized+degraded, 26 active+undersized, 45 active+clean; 5.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s; 26636/255951 objects degraded (10.407%) 2023-09-23T18:32:29.111 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:28 smithi098 bash[79516]: cluster 2023-09-23T18:32:27.776602+0000 mon.a (mon.0) 581 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:32:29.112 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:28 smithi098 bash[79516]: cluster 2023-09-23T18:32:27.796564+0000 mon.a (mon.0) 582 : cluster 0 osdmap e116: 8 total, 7 up, 7 in 2023-09-23T18:32:29.112 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:28 smithi098 bash[79516]: cluster 2023-09-23T18:32:27.797979+0000 osd.7 (osd.7) 31 : cluster 3 failed to encode map e116 with expected crc 2023-09-23T18:32:29.112 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:28 smithi098 bash[79516]: cluster 2023-09-23T18:32:27.798095+0000 osd.4 (osd.4) 46 : cluster 3 failed to encode map e116 with expected crc 2023-09-23T18:32:29.112 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:28 smithi098 bash[79516]: cluster 2023-09-23T18:32:27.798490+0000 osd.6 (osd.6) 33 : cluster 3 failed to encode map e116 with expected crc 2023-09-23T18:32:29.112 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:28 smithi098 bash[79516]: cluster 2023-09-23T18:32:27.800080+0000 osd.4 (osd.4) 47 : cluster 3 failed to encode map e116 with expected crc 2023-09-23T18:32:29.112 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:28 smithi098 bash[79516]: cluster 2023-09-23T18:32:27.800730+0000 osd.6 (osd.6) 34 : cluster 3 failed to encode map e116 with expected crc 2023-09-23T18:32:29.128 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:29.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:29.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:29.232 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:29.258 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:29.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:29.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:29.361 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:29.386 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:29.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:29.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:29.491 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:29.518 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:29.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:29.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:29.595 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 58 16 86691 86675 93.3854 40.0625 0.0682097 0.0107023 2023-09-23T18:32:29.626 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:29.651 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:29.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:29.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:29.756 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:29.780 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:29.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:29.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:29.883 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:29.924 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:29.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:29.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:30.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:29 smithi173 bash[59422]: cluster 2023-09-23T18:32:28.801231+0000 osd.7 (osd.7) 32 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T18:32:30.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:29 smithi173 bash[59422]: cluster 2023-09-23T18:32:28.802175+0000 osd.4 (osd.4) 48 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T18:32:30.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:29 smithi173 bash[59422]: cluster 2023-09-23T18:32:28.802806+0000 mon.a (mon.0) 583 : cluster 0 osdmap e117: 8 total, 7 up, 7 in 2023-09-23T18:32:30.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:29 smithi173 bash[59422]: cluster 2023-09-23T18:32:28.814096+0000 osd.6 (osd.6) 35 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T18:32:30.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:29 smithi173 bash[59422]: cluster 2023-09-23T18:32:28.816273+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T18:32:30.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:29 smithi173 bash[59422]: cluster 2023-09-23T18:32:29.428676+0000 mon.a (mon.0) 584 : cluster 3 Health check update: Degraded data redundancy: 26636/255951 objects degraded (10.407%), 10 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:30.028 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:30.050 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:30.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:30.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:30.154 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:30.178 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:30.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:30.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:30.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:29 smithi098 bash[79516]: cluster 2023-09-23T18:32:28.801231+0000 osd.7 (osd.7) 32 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T18:32:30.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:29 smithi098 bash[79516]: cluster 2023-09-23T18:32:28.802175+0000 osd.4 (osd.4) 48 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T18:32:30.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:29 smithi098 bash[79516]: cluster 2023-09-23T18:32:28.802806+0000 mon.a (mon.0) 583 : cluster 0 osdmap e117: 8 total, 7 up, 7 in 2023-09-23T18:32:30.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:29 smithi098 bash[79516]: cluster 2023-09-23T18:32:28.814096+0000 osd.6 (osd.6) 35 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T18:32:30.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:29 smithi098 bash[79516]: cluster 2023-09-23T18:32:28.816273+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T18:32:30.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:29 smithi098 bash[79516]: cluster 2023-09-23T18:32:29.428676+0000 mon.a (mon.0) 584 : cluster 3 Health check update: Degraded data redundancy: 26636/255951 objects degraded (10.407%), 10 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:30.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:29 smithi098 bash[76817]: cluster 2023-09-23T18:32:28.801231+0000 osd.7 (osd.7) 32 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T18:32:30.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:29 smithi098 bash[76817]: cluster 2023-09-23T18:32:28.802175+0000 osd.4 (osd.4) 48 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T18:32:30.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:29 smithi098 bash[76817]: cluster 2023-09-23T18:32:28.802806+0000 mon.a (mon.0) 583 : cluster 0 osdmap e117: 8 total, 7 up, 7 in 2023-09-23T18:32:30.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:29 smithi098 bash[76817]: cluster 2023-09-23T18:32:28.814096+0000 osd.6 (osd.6) 35 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T18:32:30.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:29 smithi098 bash[76817]: cluster 2023-09-23T18:32:28.816273+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e117 with expected crc 2023-09-23T18:32:30.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:29 smithi098 bash[76817]: cluster 2023-09-23T18:32:29.428676+0000 mon.a (mon.0) 584 : cluster 3 Health check update: Degraded data redundancy: 26636/255951 objects degraded (10.407%), 10 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:30.283 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:30.308 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:30.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:30.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:30.412 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:30.440 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:30.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:30.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:30.545 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:30.569 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:30.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:30.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:30.595 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 59 16 87346 87330 92.4965 40.9375 0.0146306 0.0108043 2023-09-23T18:32:30.673 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:30.707 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:30.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:30.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:30.810 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:30.840 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:30.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:30.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:30.944 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:30.984 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:30.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:30.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:31.088 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:31.099 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:30 smithi173 bash[59422]: cluster 2023-09-23T18:32:29.699593+0000 mgr.y (mgr.26054) 416 : cluster 0 pgmap v361: 89 pgs: 1 activating+undersized+degraded+remapped, 43 active+clean, 25 active+undersized, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 6 peering, 2 unknown, 2 active+clean+remapped; 5.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 30 MiB/s wr, 481 op/s; 26807/256986 objects degraded (10.431%); 10704/256986 objects misplaced (4.165%); 138 B/s, 0 objects/s recovering 2023-09-23T18:32:31.099 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:30 smithi173 bash[59422]: cluster 2023-09-23T18:32:29.808207+0000 mon.a (mon.0) 585 : cluster 0 osdmap e118: 8 total, 7 up, 7 in 2023-09-23T18:32:31.099 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:30 smithi173 bash[59422]: cluster 2023-09-23T18:32:29.809278+0000 osd.6 (osd.6) 36 : cluster 3 failed to encode map e118 with expected crc 2023-09-23T18:32:31.099 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:30 smithi173 bash[59422]: cluster 2023-09-23T18:32:29.813978+0000 osd.7 (osd.7) 33 : cluster 3 failed to encode map e118 with expected crc 2023-09-23T18:32:31.099 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:30 smithi173 bash[59422]: cluster 2023-09-23T18:32:29.820699+0000 osd.4 (osd.4) 49 : cluster 3 failed to encode map e118 with expected crc 2023-09-23T18:32:31.099 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:30 smithi173 bash[59422]: cluster 2023-09-23T18:32:29.855943+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e118 with expected crc 2023-09-23T18:32:31.112 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:31.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:31.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:31.129 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:30 smithi098 bash[76817]: cluster 2023-09-23T18:32:29.699593+0000 mgr.y (mgr.26054) 416 : cluster 0 pgmap v361: 89 pgs: 1 activating+undersized+degraded+remapped, 43 active+clean, 25 active+undersized, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 6 peering, 2 unknown, 2 active+clean+remapped; 5.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 30 MiB/s wr, 481 op/s; 26807/256986 objects degraded (10.431%); 10704/256986 objects misplaced (4.165%); 138 B/s, 0 objects/s recovering 2023-09-23T18:32:31.129 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:30 smithi098 bash[76817]: cluster 2023-09-23T18:32:29.808207+0000 mon.a (mon.0) 585 : cluster 0 osdmap e118: 8 total, 7 up, 7 in 2023-09-23T18:32:31.129 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:30 smithi098 bash[76817]: cluster 2023-09-23T18:32:29.809278+0000 osd.6 (osd.6) 36 : cluster 3 failed to encode map e118 with expected crc 2023-09-23T18:32:31.129 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:30 smithi098 bash[76817]: cluster 2023-09-23T18:32:29.813978+0000 osd.7 (osd.7) 33 : cluster 3 failed to encode map e118 with expected crc 2023-09-23T18:32:31.129 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:30 smithi098 bash[76817]: cluster 2023-09-23T18:32:29.820699+0000 osd.4 (osd.4) 49 : cluster 3 failed to encode map e118 with expected crc 2023-09-23T18:32:31.130 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:30 smithi098 bash[76817]: cluster 2023-09-23T18:32:29.855943+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e118 with expected crc 2023-09-23T18:32:31.130 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:30 smithi098 bash[79516]: cluster 2023-09-23T18:32:29.699593+0000 mgr.y (mgr.26054) 416 : cluster 0 pgmap v361: 89 pgs: 1 activating+undersized+degraded+remapped, 43 active+clean, 25 active+undersized, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 6 peering, 2 unknown, 2 active+clean+remapped; 5.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 30 MiB/s wr, 481 op/s; 26807/256986 objects degraded (10.431%); 10704/256986 objects misplaced (4.165%); 138 B/s, 0 objects/s recovering 2023-09-23T18:32:31.130 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:30 smithi098 bash[79516]: cluster 2023-09-23T18:32:29.808207+0000 mon.a (mon.0) 585 : cluster 0 osdmap e118: 8 total, 7 up, 7 in 2023-09-23T18:32:31.130 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:30 smithi098 bash[79516]: cluster 2023-09-23T18:32:29.809278+0000 osd.6 (osd.6) 36 : cluster 3 failed to encode map e118 with expected crc 2023-09-23T18:32:31.130 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:30 smithi098 bash[79516]: cluster 2023-09-23T18:32:29.813978+0000 osd.7 (osd.7) 33 : cluster 3 failed to encode map e118 with expected crc 2023-09-23T18:32:31.130 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:30 smithi098 bash[79516]: cluster 2023-09-23T18:32:29.820699+0000 osd.4 (osd.4) 49 : cluster 3 failed to encode map e118 with expected crc 2023-09-23T18:32:31.130 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:30 smithi098 bash[79516]: cluster 2023-09-23T18:32:29.855943+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e118 with expected crc 2023-09-23T18:32:31.215 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:31.244 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:31.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:31.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:31.348 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:31.373 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:31.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:31.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:31.479 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:31.511 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:31.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:31.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:31.596 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:2023-09-23T18:32:31.595795+0000 min lat: 0.00116463 max lat: 2.46834 avg lat: 0.0109128 2023-09-23T18:32:31.596 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:32:31.596 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 60 16 87936 87920 91.5695 36.875 0.00806999 0.0109128 2023-09-23T18:32:31.616 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:31.644 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:31.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:31.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:31.749 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:31.778 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:31.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:31.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:31.886 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:31.895 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:31 smithi173 bash[59422]: cluster 2023-09-23T18:32:31.806888+0000 mon.a (mon.0) 586 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 1 pg peering) 2023-09-23T18:32:31.912 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:31.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:31.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:32.016 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:32.049 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:32.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:32.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:32.156 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:32.164 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:31 smithi098 bash[79516]: cluster 2023-09-23T18:32:31.806888+0000 mon.a (mon.0) 586 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 1 pg peering) 2023-09-23T18:32:32.164 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:31 smithi098 bash[76817]: cluster 2023-09-23T18:32:31.806888+0000 mon.a (mon.0) 586 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 1 pg peering) 2023-09-23T18:32:32.187 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:32.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:32.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:32.290 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:32.321 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:32.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:32.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:32.425 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:32.450 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:32.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:32.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:32.553 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:32.591 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:32.596 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 61 16 88615 88599 90.764 42.4375 0.0105429 0.0110105 2023-09-23T18:32:32.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:32.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:32.700 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:32.727 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:32.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:32.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:32.830 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:32.863 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:32.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:32.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:32.970 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:32.993 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:32.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:32.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:32 smithi173 bash[59422]: cluster 2023-09-23T18:32:31.700617+0000 mgr.y (mgr.26054) 417 : cluster 0 pgmap v363: 89 pgs: 1 activating+undersized+degraded+remapped, 40 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 25 MiB/s wr, 396 op/s; 59318/261813 objects degraded (22.657%); 16003/261813 objects misplaced (6.112%); 1.8 MiB/s, 20 objects/s recovering 2023-09-23T18:32:33.095 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:33.121 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:33.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:33.128 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:32 smithi098 bash[76817]: cluster 2023-09-23T18:32:31.700617+0000 mgr.y (mgr.26054) 417 : cluster 0 pgmap v363: 89 pgs: 1 activating+undersized+degraded+remapped, 40 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 25 MiB/s wr, 396 op/s; 59318/261813 objects degraded (22.657%); 16003/261813 objects misplaced (6.112%); 1.8 MiB/s, 20 objects/s recovering 2023-09-23T18:32:33.128 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:32 smithi098 bash[79516]: cluster 2023-09-23T18:32:31.700617+0000 mgr.y (mgr.26054) 417 : cluster 0 pgmap v363: 89 pgs: 1 activating+undersized+degraded+remapped, 40 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 25 MiB/s wr, 396 op/s; 59318/261813 objects degraded (22.657%); 16003/261813 objects misplaced (6.112%); 1.8 MiB/s, 20 objects/s recovering 2023-09-23T18:32:33.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:33.230 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:33.260 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:33.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:33.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:33.363 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:33.413 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:33.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:33.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:33.520 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:33.550 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:33.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:33.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:33.596 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 62 16 89273 89257 89.9633 41.125 0.0147553 0.0111079 2023-09-23T18:32:33.654 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:33.684 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:33.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:33.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:33.790 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:33.827 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:33.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:33.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:33.935 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:33.963 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:33.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:33.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:34.066 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:34.094 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:34.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:34.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:34.199 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:34.224 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:34.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:34.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:34.332 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:34.335 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T18:32:34.335 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:32:34.335 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-09-23T18:32:34.336 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:32:34.359 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:34.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:34.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:34.465 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:34.523 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:34.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:34.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:34.596 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 63 16 89844 89828 89.1018 35.6875 0.0524762 0.0112119 2023-09-23T18:32:34.630 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:34.686 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:34.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:34.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:34.713 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:32:34.751 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:32:34.752 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:32:34.752 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:32:34.790 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:34.793 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:32:34.824 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:34.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:34.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:34.833 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:32:34.874 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:32:34.907 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:32:34.930 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:34.938 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:34 smithi098 bash[76817]: cluster 2023-09-23T18:32:33.701571+0000 mgr.y (mgr.26054) 418 : cluster 0 pgmap v364: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 27 MiB/s wr, 425 op/s; 59515/263610 objects degraded (22.577%); 16060/263610 objects misplaced (6.092%); 2.8 MiB/s, 39 objects/s recovering 2023-09-23T18:32:34.938 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:34 smithi098 bash[76817]: cluster 2023-09-23T18:32:34.430597+0000 mon.a (mon.0) 587 : cluster 3 Health check update: Degraded data redundancy: 59515/263610 objects degraded (22.577%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:34.938 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:34 smithi098 bash[76817]: audit 2023-09-23T18:32:34.644337+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.98:0/1106521954' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:32:34.939 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:34 smithi098 bash[76817]: audit 2023-09-23T18:32:34.794608+0000 mon.c (mon.2) 57 : audit 1 from='client.? 172.21.15.98:0/3798538783' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:32:34.939 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:34 smithi098 bash[76817]: audit 2023-09-23T18:32:34.795097+0000 mon.a (mon.0) 588 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:32:34.939 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:34 smithi098 bash[79516]: cluster 2023-09-23T18:32:33.701571+0000 mgr.y (mgr.26054) 418 : cluster 0 pgmap v364: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 27 MiB/s wr, 425 op/s; 59515/263610 objects degraded (22.577%); 16060/263610 objects misplaced (6.092%); 2.8 MiB/s, 39 objects/s recovering 2023-09-23T18:32:34.939 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:34 smithi098 bash[79516]: cluster 2023-09-23T18:32:34.430597+0000 mon.a (mon.0) 587 : cluster 3 Health check update: Degraded data redundancy: 59515/263610 objects degraded (22.577%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:34.939 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:34 smithi098 bash[79516]: audit 2023-09-23T18:32:34.644337+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.98:0/1106521954' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:32:34.939 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:34 smithi098 bash[79516]: audit 2023-09-23T18:32:34.794608+0000 mon.c (mon.2) 57 : audit 1 from='client.? 172.21.15.98:0/3798538783' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:32:34.939 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:34 smithi098 bash[79516]: audit 2023-09-23T18:32:34.795097+0000 mon.a (mon.0) 588 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:32:34.941 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:32:34.941 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:32:34.941 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T18:32:34.941 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:32:34.941 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:32:34.942 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:32:34.942 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:32:34.942 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:32:34.942 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:32:34.967 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:34.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:34.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:35.075 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:35.108 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:35.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:35.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:35.210 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:35.220 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:34 smithi173 bash[59422]: cluster 2023-09-23T18:32:33.701571+0000 mgr.y (mgr.26054) 418 : cluster 0 pgmap v364: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 27 MiB/s wr, 425 op/s; 59515/263610 objects degraded (22.577%); 16060/263610 objects misplaced (6.092%); 2.8 MiB/s, 39 objects/s recovering 2023-09-23T18:32:35.220 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:34 smithi173 bash[59422]: cluster 2023-09-23T18:32:34.430597+0000 mon.a (mon.0) 587 : cluster 3 Health check update: Degraded data redundancy: 59515/263610 objects degraded (22.577%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:35.220 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:34 smithi173 bash[59422]: audit 2023-09-23T18:32:34.644337+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.98:0/1106521954' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:32:35.221 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:34 smithi173 bash[59422]: audit 2023-09-23T18:32:34.794608+0000 mon.c (mon.2) 57 : audit 1 from='client.? 172.21.15.98:0/3798538783' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:32:35.221 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:34 smithi173 bash[59422]: audit 2023-09-23T18:32:34.795097+0000 mon.a (mon.0) 588 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:32:35.241 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:35.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:35.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:35.344 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:35.380 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:35.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:35.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:35.452 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is set 2023-09-23T18:32:35.491 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:35.521 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:35.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:35.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:35.596 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 64 16 90430 90414 88.2818 36.625 0.00899922 0.011319 2023-09-23T18:32:35.627 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:35.663 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:35.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:35.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:35.769 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:35.795 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:35.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:35.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:35.899 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:35.925 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:35.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:35.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:36.028 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:36.054 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:36.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:36.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:36.085 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:35 smithi098 bash[79516]: cluster 2023-09-23T18:32:35.433659+0000 mon.a (mon.0) 589 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:32:36.085 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:35 smithi098 bash[79516]: cluster 2023-09-23T18:32:35.437359+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e119 with expected crc 2023-09-23T18:32:36.085 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:35 smithi098 bash[79516]: audit 2023-09-23T18:32:35.439152+0000 mon.a (mon.0) 590 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:32:36.086 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:35 smithi098 bash[79516]: cluster 2023-09-23T18:32:35.439235+0000 mon.a (mon.0) 591 : cluster 0 osdmap e119: 8 total, 7 up, 7 in 2023-09-23T18:32:36.086 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:32:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:32:36] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.33.4" 2023-09-23T18:32:36.086 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:35 smithi098 bash[76817]: cluster 2023-09-23T18:32:35.433659+0000 mon.a (mon.0) 589 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:32:36.086 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:35 smithi098 bash[76817]: cluster 2023-09-23T18:32:35.437359+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e119 with expected crc 2023-09-23T18:32:36.086 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:35 smithi098 bash[76817]: audit 2023-09-23T18:32:35.439152+0000 mon.a (mon.0) 590 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:32:36.086 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:35 smithi098 bash[76817]: cluster 2023-09-23T18:32:35.439235+0000 mon.a (mon.0) 591 : cluster 0 osdmap e119: 8 total, 7 up, 7 in 2023-09-23T18:32:36.158 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:36.202 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:36.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:36.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:36.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:35 smithi173 bash[59422]: cluster 2023-09-23T18:32:35.433659+0000 mon.a (mon.0) 589 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:32:36.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:35 smithi173 bash[59422]: cluster 2023-09-23T18:32:35.437359+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e119 with expected crc 2023-09-23T18:32:36.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:35 smithi173 bash[59422]: audit 2023-09-23T18:32:35.439152+0000 mon.a (mon.0) 590 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:32:36.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:35 smithi173 bash[59422]: cluster 2023-09-23T18:32:35.439235+0000 mon.a (mon.0) 591 : cluster 0 osdmap e119: 8 total, 7 up, 7 in 2023-09-23T18:32:36.309 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:36.350 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:36.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:36.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:36.457 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:36.481 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:36.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:36.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:36.587 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:36.596 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 65 16 91080 91064 87.5486 40.625 0.00963804 0.011415 2023-09-23T18:32:36.617 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:36.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:36.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:36.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:36.744 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:36.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:36.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:36.849 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:36.886 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:36.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:36.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:36.992 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:37.001 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:36 smithi098 bash[79516]: cluster 2023-09-23T18:32:35.437342+0000 osd.4 (osd.4) 50 : cluster 3 failed to encode map e119 with expected crc 2023-09-23T18:32:37.001 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:36 smithi098 bash[79516]: cluster 2023-09-23T18:32:35.702603+0000 mgr.y (mgr.26054) 419 : cluster 0 pgmap v366: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 32 MiB/s wr, 517 op/s; 59707/267693 objects degraded (22.304%); 16060/267693 objects misplaced (5.999%); 2.5 MiB/s, 34 objects/s recovering 2023-09-23T18:32:37.003 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:36 smithi098 bash[76817]: cluster 2023-09-23T18:32:35.437342+0000 osd.4 (osd.4) 50 : cluster 3 failed to encode map e119 with expected crc 2023-09-23T18:32:37.003 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:36 smithi098 bash[76817]: cluster 2023-09-23T18:32:35.702603+0000 mgr.y (mgr.26054) 419 : cluster 0 pgmap v366: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 32 MiB/s wr, 517 op/s; 59707/267693 objects degraded (22.304%); 16060/267693 objects misplaced (5.999%); 2.5 MiB/s, 34 objects/s recovering 2023-09-23T18:32:37.030 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:37.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:37.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:37.136 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:37.179 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:37.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:37.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:37.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:36 smithi173 bash[59422]: cluster 2023-09-23T18:32:35.437342+0000 osd.4 (osd.4) 50 : cluster 3 failed to encode map e119 with expected crc 2023-09-23T18:32:37.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:36 smithi173 bash[59422]: cluster 2023-09-23T18:32:35.702603+0000 mgr.y (mgr.26054) 419 : cluster 0 pgmap v366: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 32 MiB/s wr, 517 op/s; 59707/267693 objects degraded (22.304%); 16060/267693 objects misplaced (5.999%); 2.5 MiB/s, 34 objects/s recovering 2023-09-23T18:32:37.286 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:37.329 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:37.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:37.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:37.433 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:37.461 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:37.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:37.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:37.568 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:37.596 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 66 15 91752 91737 86.8594 42.0625 0.00727232 0.0115055 2023-09-23T18:32:37.600 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:37.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:37.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:37.707 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:37.732 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:37.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:37.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:37.836 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:37.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:37.875 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:37.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:37.976 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:38.009 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:38.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:38.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:38.118 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:38.144 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:38.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:38.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:38.250 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:38.275 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:38.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:38.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:38.378 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:38.411 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:38.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:38.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:38.514 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:38.540 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:38.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:38.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:38.596 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 67 16 92340 92324 86.1106 36.6875 0.0189983 0.0116061 2023-09-23T18:32:38.647 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:38.675 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:38.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:38.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:38.784 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:38.808 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:38.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:38.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:38.903 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T18:32:38.913 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:38.981 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:38.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:38.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:39.032 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:32:39.033 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:32:39.086 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:39.095 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:38 smithi098 bash[76817]: cluster 2023-09-23T18:32:37.703492+0000 mgr.y (mgr.26054) 420 : cluster 0 pgmap v367: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 37 MiB/s wr, 594 op/s; 60079/271263 objects degraded (22.148%); 15946/271263 objects misplaced (5.878%); 3.0 MiB/s, 43 objects/s recovering 2023-09-23T18:32:39.095 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:38 smithi098 bash[76817]: audit 2023-09-23T18:32:37.894701+0000 mon.a (mon.0) 592 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:32:39.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:38 smithi098 bash[79516]: cluster 2023-09-23T18:32:37.703492+0000 mgr.y (mgr.26054) 420 : cluster 0 pgmap v367: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 37 MiB/s wr, 594 op/s; 60079/271263 objects degraded (22.148%); 15946/271263 objects misplaced (5.878%); 3.0 MiB/s, 43 objects/s recovering 2023-09-23T18:32:39.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:38 smithi098 bash[79516]: audit 2023-09-23T18:32:37.894701+0000 mon.a (mon.0) 592 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:32:39.126 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:39.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:39.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:39.231 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:39.240 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:38 smithi173 bash[59422]: cluster 2023-09-23T18:32:37.703492+0000 mgr.y (mgr.26054) 420 : cluster 0 pgmap v367: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 37 MiB/s wr, 594 op/s; 60079/271263 objects degraded (22.148%); 15946/271263 objects misplaced (5.878%); 3.0 MiB/s, 43 objects/s recovering 2023-09-23T18:32:39.241 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:38 smithi173 bash[59422]: audit 2023-09-23T18:32:37.894701+0000 mon.a (mon.0) 592 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:32:39.266 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:39.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:39.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:39.371 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:39.399 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:39.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:39.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:39.506 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:39.540 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:39.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:39.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:39.596 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 68 16 92966 92950 85.4196 39.125 0.0425109 0.0117003 2023-09-23T18:32:39.647 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:39.682 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:39.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:39.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:39.788 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:39.812 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:39.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:39.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:39.884 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:39 smithi098 bash[76817]: cluster 2023-09-23T18:32:39.432421+0000 mon.a (mon.0) 593 : cluster 3 Health check update: Degraded data redundancy: 60079/271263 objects degraded (22.148%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:39.884 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:39 smithi098 bash[76817]: audit 2023-09-23T18:32:39.584576+0000 mon.a (mon.0) 594 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:32:39.884 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:39 smithi098 bash[76817]: audit 2023-09-23T18:32:39.587850+0000 mon.a (mon.0) 595 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:32:39.915 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:39.937 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:39.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:39.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:40.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:39 smithi173 bash[59422]: cluster 2023-09-23T18:32:39.432421+0000 mon.a (mon.0) 593 : cluster 3 Health check update: Degraded data redundancy: 60079/271263 objects degraded (22.148%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:40.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:39 smithi173 bash[59422]: audit 2023-09-23T18:32:39.584576+0000 mon.a (mon.0) 594 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:32:40.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:39 smithi173 bash[59422]: audit 2023-09-23T18:32:39.587850+0000 mon.a (mon.0) 595 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:32:40.040 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:40.066 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:40.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:40.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:40.175 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:40.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:39 smithi098 bash[79516]: cluster 2023-09-23T18:32:39.432421+0000 mon.a (mon.0) 593 : cluster 3 Health check update: Degraded data redundancy: 60079/271263 objects degraded (22.148%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:40.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:39 smithi098 bash[79516]: audit 2023-09-23T18:32:39.584576+0000 mon.a (mon.0) 594 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:32:40.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:39 smithi098 bash[79516]: audit 2023-09-23T18:32:39.587850+0000 mon.a (mon.0) 595 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:32:40.202 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:40.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:40.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:40.309 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:40.343 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:40.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:40.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:40.447 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:40.474 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:40.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:40.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:40.578 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:40.596 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 69 16 93633 93617 84.7857 41.6875 0.0070386 0.0117873 2023-09-23T18:32:40.616 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:40.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:40.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:40.724 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:40.753 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:40.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:40.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:40.859 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:40.886 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:40 smithi098 bash[76817]: cluster 2023-09-23T18:32:39.704273+0000 mgr.y (mgr.26054) 421 : cluster 0 pgmap v368: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 34 MiB/s wr, 542 op/s; 60271/273105 objects degraded (22.069%); 16040/273105 objects misplaced (5.873%); 3.3 MiB/s, 48 objects/s recovering 2023-09-23T18:32:40.891 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:40.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:40.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:40.994 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:41.003 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:40 smithi173 bash[59422]: cluster 2023-09-23T18:32:39.704273+0000 mgr.y (mgr.26054) 421 : cluster 0 pgmap v368: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 34 MiB/s wr, 542 op/s; 60271/273105 objects degraded (22.069%); 16040/273105 objects misplaced (5.873%); 3.3 MiB/s, 48 objects/s recovering 2023-09-23T18:32:41.020 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:32:41.024 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:41.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:41.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:41.034 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:32:41.127 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:41.153 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:41.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:41.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:41.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:40 smithi098 bash[79516]: cluster 2023-09-23T18:32:39.704273+0000 mgr.y (mgr.26054) 421 : cluster 0 pgmap v368: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 34 MiB/s wr, 542 op/s; 60271/273105 objects degraded (22.069%); 16040/273105 objects misplaced (5.873%); 3.3 MiB/s, 48 objects/s recovering 2023-09-23T18:32:41.256 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:41.280 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:41.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:41.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:41.384 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:41.420 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:41.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:41.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:41.523 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:41.558 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:41.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:41.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:41.598 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 70 16 94233 94217 84.1083 37.5 0.0200045 0.0118822 2023-09-23T18:32:41.663 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:41.698 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:41.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:41.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:41.803 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:41.837 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:41.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:41.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:41.942 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:41.978 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:41.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:41.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:42.081 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:42.120 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:42.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:42.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:42.227 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:42.256 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:42.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:42.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:42.360 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:42.392 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:42.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:42.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:42.495 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:42.534 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:42.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:42.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:42.556 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:32:42.602 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 71 16 94862 94846 83.4729 39.3125 0.0198443 0.0119735 2023-09-23T18:32:42.637 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:42.679 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:42.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:42.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:42.783 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:42.813 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:42.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:42.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:42.891 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:42 smithi098 bash[76817]: cluster 2023-09-23T18:32:41.705267+0000 mgr.y (mgr.26054) 422 : cluster 0 pgmap v369: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 33 MiB/s wr, 533 op/s; 60685/277818 objects degraded (21.843%); 16040/277818 objects misplaced (5.774%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T18:32:42.918 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:42.986 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:42.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:42.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:43.090 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:43.123 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:43.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:43.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:43.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:42 smithi098 bash[79516]: cluster 2023-09-23T18:32:41.705267+0000 mgr.y (mgr.26054) 422 : cluster 0 pgmap v369: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 33 MiB/s wr, 533 op/s; 60685/277818 objects degraded (21.843%); 16040/277818 objects misplaced (5.774%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T18:32:43.230 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:43.272 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:43.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:43.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:43.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:42 smithi173 bash[59422]: cluster 2023-09-23T18:32:41.705267+0000 mgr.y (mgr.26054) 422 : cluster 0 pgmap v369: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 33 MiB/s wr, 533 op/s; 60685/277818 objects degraded (21.843%); 16040/277818 objects misplaced (5.774%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T18:32:43.376 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:43.412 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:43.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:43.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:43.518 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:43.550 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:43.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:43.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:43.577 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:32:43.602 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 72 16 95448 95432 82.8222 36.625 0.0650512 0.0120643 2023-09-23T18:32:43.656 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:43.688 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:43.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:43.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:43.792 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:43.839 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:43.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:43.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:43.896 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:43 smithi098 bash[76817]: cluster 2023-09-23T18:32:43.706299+0000 mgr.y (mgr.26054) 423 : cluster 0 pgmap v370: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 40 MiB/s wr, 634 op/s; 61109/282645 objects degraded (21.620%); 15982/282645 objects misplaced (5.654%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T18:32:43.948 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:44.000 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:44.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:44.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:44.103 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:44.154 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:44.157 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:43 smithi173 bash[59422]: cluster 2023-09-23T18:32:43.706299+0000 mgr.y (mgr.26054) 423 : cluster 0 pgmap v370: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 40 MiB/s wr, 634 op/s; 61109/282645 objects degraded (21.620%); 15982/282645 objects misplaced (5.654%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T18:32:44.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:44.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:44.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:43 smithi098 bash[79516]: cluster 2023-09-23T18:32:43.706299+0000 mgr.y (mgr.26054) 423 : cluster 0 pgmap v370: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 40 MiB/s wr, 634 op/s; 61109/282645 objects degraded (21.620%); 15982/282645 objects misplaced (5.654%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T18:32:44.259 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:44.284 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:44.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:44.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:44.388 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:44.422 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:44.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:44.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:44.525 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:44.554 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:44.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:44.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:44.603 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 73 16 95984 95968 82.1458 33.5 0.0163286 0.0121672 2023-09-23T18:32:44.661 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:44.698 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:44.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:44.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:44.802 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:44.850 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:44.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:44.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:44.955 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:44.983 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:44.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:44.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:45.089 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:45.117 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:45.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:45.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:45.220 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:45.281 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:45.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:45.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:45.385 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:45.416 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:45.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:45.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:45.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:45 smithi098 bash[76817]: audit 2023-09-23T18:32:44.319960+0000 mon.a (mon.0) 596 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:32:45.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:45 smithi098 bash[76817]: audit 2023-09-23T18:32:44.333407+0000 mon.a (mon.0) 597 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:32:45.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:45 smithi098 bash[76817]: cluster 2023-09-23T18:32:44.433423+0000 mon.a (mon.0) 598 : cluster 3 Health check update: Degraded data redundancy: 61109/282645 objects degraded (21.620%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:45.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:45 smithi098 bash[76817]: audit 2023-09-23T18:32:45.108278+0000 mon.a (mon.0) 599 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:32:45.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:45 smithi098 bash[76817]: audit 2023-09-23T18:32:45.116346+0000 mon.a (mon.0) 600 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:32:45.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:45 smithi098 bash[79516]: audit 2023-09-23T18:32:44.319960+0000 mon.a (mon.0) 596 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:32:45.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:45 smithi098 bash[79516]: audit 2023-09-23T18:32:44.333407+0000 mon.a (mon.0) 597 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:32:45.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:45 smithi098 bash[79516]: cluster 2023-09-23T18:32:44.433423+0000 mon.a (mon.0) 598 : cluster 3 Health check update: Degraded data redundancy: 61109/282645 objects degraded (21.620%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:45.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:45 smithi098 bash[79516]: audit 2023-09-23T18:32:45.108278+0000 mon.a (mon.0) 599 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:32:45.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:45 smithi098 bash[79516]: audit 2023-09-23T18:32:45.116346+0000 mon.a (mon.0) 600 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:32:45.519 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:45.554 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:45.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:45.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:45.603 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 74 16 96422 96406 81.4058 27.375 0.00579945 0.0122769 2023-09-23T18:32:45.619 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:32:45.619 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:32:45.658 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:45.665 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:45 smithi173 bash[59422]: audit 2023-09-23T18:32:44.319960+0000 mon.a (mon.0) 596 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:32:45.666 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:45 smithi173 bash[59422]: audit 2023-09-23T18:32:44.333407+0000 mon.a (mon.0) 597 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:32:45.666 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:45 smithi173 bash[59422]: cluster 2023-09-23T18:32:44.433423+0000 mon.a (mon.0) 598 : cluster 3 Health check update: Degraded data redundancy: 61109/282645 objects degraded (21.620%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:45.666 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:45 smithi173 bash[59422]: audit 2023-09-23T18:32:45.108278+0000 mon.a (mon.0) 599 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:32:45.666 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:45 smithi173 bash[59422]: audit 2023-09-23T18:32:45.116346+0000 mon.a (mon.0) 600 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:32:45.687 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:45.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:45.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:45.791 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:45.822 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:45.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:45.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:45.927 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:45.963 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:45.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:45.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:46.069 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:46.096 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:46.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:46.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:46.202 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:46.232 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:46.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:46.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:46.337 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:46.340 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:32:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:32:46] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.33.4" 2023-09-23T18:32:46.340 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:46 smithi098 bash[79516]: audit 2023-09-23T18:32:45.597977+0000 mon.a (mon.0) 601 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:32:46.340 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:46 smithi098 bash[79516]: audit 2023-09-23T18:32:45.599284+0000 mon.a (mon.0) 602 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:32:46.340 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:46 smithi098 bash[79516]: audit 2023-09-23T18:32:45.609572+0000 mon.a (mon.0) 603 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:32:46.340 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:46 smithi098 bash[79516]: cluster 2023-09-23T18:32:45.707267+0000 mgr.y (mgr.26054) 424 : cluster 0 pgmap v371: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 46 MiB/s wr, 735 op/s; 61257/286260 objects degraded (21.399%); 15982/286260 objects misplaced (5.583%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:32:46.341 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:46 smithi098 bash[76817]: audit 2023-09-23T18:32:45.597977+0000 mon.a (mon.0) 601 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:32:46.341 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:46 smithi098 bash[76817]: audit 2023-09-23T18:32:45.599284+0000 mon.a (mon.0) 602 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:32:46.341 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:46 smithi098 bash[76817]: audit 2023-09-23T18:32:45.609572+0000 mon.a (mon.0) 603 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:32:46.341 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:46 smithi098 bash[76817]: cluster 2023-09-23T18:32:45.707267+0000 mgr.y (mgr.26054) 424 : cluster 0 pgmap v371: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 46 MiB/s wr, 735 op/s; 61257/286260 objects degraded (21.399%); 15982/286260 objects misplaced (5.583%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:32:46.368 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:46.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:46.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:46.477 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:46.500 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:46.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:46.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:46.603 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:46.604 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 75 16 97051 97035 80.8432 39.3125 0.0288747 0.012363 2023-09-23T18:32:46.636 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:46.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:46.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:46.743 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:46.773 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:46.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:46.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:46.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:46 smithi173 bash[59422]: audit 2023-09-23T18:32:45.597977+0000 mon.a (mon.0) 601 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:32:46.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:46 smithi173 bash[59422]: audit 2023-09-23T18:32:45.599284+0000 mon.a (mon.0) 602 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:32:46.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:46 smithi173 bash[59422]: audit 2023-09-23T18:32:45.609572+0000 mon.a (mon.0) 603 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:32:46.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:46 smithi173 bash[59422]: cluster 2023-09-23T18:32:45.707267+0000 mgr.y (mgr.26054) 424 : cluster 0 pgmap v371: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 46 MiB/s wr, 735 op/s; 61257/286260 objects degraded (21.399%); 15982/286260 objects misplaced (5.583%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:32:46.877 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:46.910 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:46.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:46.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:47.015 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:47.048 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:47.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:47.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:47.153 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:47.177 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:47.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:47.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:47.281 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:47.308 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:47.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:47.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:47.415 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:47.450 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:47.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:47.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:47.552 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:47.579 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:47.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:47.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:47.605 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 76 16 97704 97688 80.3165 40.8125 0.0328755 0.0124406 2023-09-23T18:32:47.681 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:47.712 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:47.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:47.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:47.815 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:47.840 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:47.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:47.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:47.946 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:47.970 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:47.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:47.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:48.076 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:48.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:48.124 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:48.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:48.225 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:48.253 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:48.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:48.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:48.360 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:48.388 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:48.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:48.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:48.491 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:48.517 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:48.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:48.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:48.605 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 77 16 98296 98280 79.7539 37 0.0128492 0.012531 2023-09-23T18:32:48.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:48.647 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:48.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:48.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:48.751 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:48.784 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:48.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:48.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:48.888 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:48.896 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:48 smithi173 bash[59422]: cluster 2023-09-23T18:32:47.708137+0000 mgr.y (mgr.26054) 425 : cluster 0 pgmap v372: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 38 MiB/s wr, 604 op/s; 61614/289458 objects degraded (21.286%); 15861/289458 objects misplaced (5.480%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T18:32:48.917 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:48.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:48.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:49.023 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:49.053 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:49.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:49.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:49.156 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:49.183 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:49.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:49.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:49.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:48 smithi098 bash[79516]: cluster 2023-09-23T18:32:47.708137+0000 mgr.y (mgr.26054) 425 : cluster 0 pgmap v372: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 38 MiB/s wr, 604 op/s; 61614/289458 objects degraded (21.286%); 15861/289458 objects misplaced (5.480%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T18:32:49.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:48 smithi098 bash[76817]: cluster 2023-09-23T18:32:47.708137+0000 mgr.y (mgr.26054) 425 : cluster 0 pgmap v372: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 38 MiB/s wr, 604 op/s; 61614/289458 objects degraded (21.286%); 15861/289458 objects misplaced (5.480%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T18:32:49.290 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:49.328 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:49.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:49.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:49.436 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:49.458 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:49.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:49.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:49.564 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:49.599 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:49.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:49.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:49.605 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 78 16 98925 98909 79.2355 39.3125 0.0117651 0.0126127 2023-09-23T18:32:49.704 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:49.734 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:49.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:49.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:49.839 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:49.863 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:49.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:49.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:49.897 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:49 smithi098 bash[79516]: cluster 2023-09-23T18:32:49.434652+0000 mon.a (mon.0) 604 : cluster 3 Health check update: Degraded data redundancy: 61614/289458 objects degraded (21.286%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:49.897 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:49 smithi098 bash[76817]: cluster 2023-09-23T18:32:49.434652+0000 mon.a (mon.0) 604 : cluster 3 Health check update: Degraded data redundancy: 61614/289458 objects degraded (21.286%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:49.968 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:49.976 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:49 smithi173 bash[59422]: cluster 2023-09-23T18:32:49.434652+0000 mon.a (mon.0) 604 : cluster 3 Health check update: Degraded data redundancy: 61614/289458 objects degraded (21.286%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:50.008 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:50.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:50.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:50.110 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:50.141 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:50.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:50.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:50.245 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:50.271 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:50.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:50.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:50.376 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:50.404 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:50.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:50.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:50.508 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:50.528 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:50.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:50.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:50.605 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 79 16 99501 99485 78.6882 36 0.00670133 0.0126854 2023-09-23T18:32:50.630 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:50.658 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:50.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:50.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:50.763 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:50.776 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:50 smithi173 bash[59422]: cluster 2023-09-23T18:32:49.709083+0000 mgr.y (mgr.26054) 426 : cluster 0 pgmap v373: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 34 MiB/s wr, 549 op/s; 61796/291051 objects degraded (21.232%); 15922/291051 objects misplaced (5.471%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:32:50.784 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:50.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:50.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:50.887 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:50.915 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:50.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:50.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:50.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:50 smithi098 bash[76817]: cluster 2023-09-23T18:32:49.709083+0000 mgr.y (mgr.26054) 426 : cluster 0 pgmap v373: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 34 MiB/s wr, 549 op/s; 61796/291051 objects degraded (21.232%); 15922/291051 objects misplaced (5.471%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:32:50.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:50 smithi098 bash[79516]: cluster 2023-09-23T18:32:49.709083+0000 mgr.y (mgr.26054) 426 : cluster 0 pgmap v373: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 34 MiB/s wr, 549 op/s; 61796/291051 objects degraded (21.232%); 15922/291051 objects misplaced (5.471%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:32:51.020 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:51.043 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:51.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:51.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:51.148 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:51.170 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:51.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:51.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:51.276 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:51.304 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:51.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:51.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:51.410 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:51.439 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:51.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:51.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:51.547 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:51.573 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:51.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:51.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:51.605 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:2023-09-23T18:32:51.605120+0000 min lat: 0.00116463 max lat: 2.46834 avg lat: 0.0127855 2023-09-23T18:32:51.605 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:32:51.605 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 80 16 100079 100063 78.1562 36.125 0.0899869 0.0127855 2023-09-23T18:32:51.677 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:51.717 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:51.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:51.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:51.827 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:51.860 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:51.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:51.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:51.965 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:52.000 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:52.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:52.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:52.105 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:52.133 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:52.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:52.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:52.237 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:52.261 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:52.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:52.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:52.366 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:52.396 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:52.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:52.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:52.502 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:52.527 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:52.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:52.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:52.605 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 81 16 100713 100697 77.6805 39.625 0.0138608 0.0128618 2023-09-23T18:32:52.631 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:52.661 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:52.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:52.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:52.764 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:52.773 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:52 smithi098 bash[76817]: cluster 2023-09-23T18:32:51.710079+0000 mgr.y (mgr.26054) 427 : cluster 0 pgmap v374: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 39 MiB/s wr, 628 op/s; 62192/295731 objects degraded (21.030%); 15922/295731 objects misplaced (5.384%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:32:52.774 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:52 smithi098 bash[79516]: cluster 2023-09-23T18:32:51.710079+0000 mgr.y (mgr.26054) 427 : cluster 0 pgmap v374: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 39 MiB/s wr, 628 op/s; 62192/295731 objects degraded (21.030%); 15922/295731 objects misplaced (5.384%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:32:52.775 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:52 smithi173 bash[59422]: cluster 2023-09-23T18:32:51.710079+0000 mgr.y (mgr.26054) 427 : cluster 0 pgmap v374: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 39 MiB/s wr, 628 op/s; 62192/295731 objects degraded (21.030%); 15922/295731 objects misplaced (5.384%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:32:52.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:52.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:52.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:52.894 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:52.922 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:52.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:52.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:53.027 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:53.058 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:53.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:53.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:53.162 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:53.204 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:53.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:53.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:53.309 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:53.336 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:53.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:53.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:53.443 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:53.486 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:53.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:53.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:53.588 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:53.605 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 82 16 101378 101362 77.2401 41.5625 0.0360652 0.0129402 2023-09-23T18:32:53.617 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:53.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:53.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:53.720 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:53.748 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:53.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:53.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:53.851 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:53.874 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:53.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:53.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:53.979 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:54.006 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:54.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:54.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:54.109 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:54.147 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:54.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:54.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:54.250 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:54.278 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:54.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:54.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:54.383 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:54.411 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:54.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:54.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:54.516 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:54.546 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:54.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:54.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:54.605 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 83 16 102022 102006 76.7944 40.25 0.0124593 0.0130145 2023-09-23T18:32:54.649 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:54.679 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:54.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:54.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:54.784 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:54.811 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:54.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:54.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:54.919 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:54.929 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:54 smithi173 bash[59422]: cluster 2023-09-23T18:32:53.710899+0000 mgr.y (mgr.26054) 428 : cluster 0 pgmap v375: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 39 MiB/s wr, 624 op/s; 62584/300303 objects degraded (20.840%); 15838/300303 objects misplaced (5.274%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T18:32:54.929 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:54 smithi173 bash[59422]: cluster 2023-09-23T18:32:54.436096+0000 mon.a (mon.0) 605 : cluster 3 Health check update: Degraded data redundancy: 62584/300303 objects degraded (20.840%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:54.929 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:54 smithi173 bash[59422]: audit 2023-09-23T18:32:54.581048+0000 mon.a (mon.0) 606 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:32:54.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:54 smithi098 bash[79516]: cluster 2023-09-23T18:32:53.710899+0000 mgr.y (mgr.26054) 428 : cluster 0 pgmap v375: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 39 MiB/s wr, 624 op/s; 62584/300303 objects degraded (20.840%); 15838/300303 objects misplaced (5.274%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T18:32:54.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:54 smithi098 bash[79516]: cluster 2023-09-23T18:32:54.436096+0000 mon.a (mon.0) 605 : cluster 3 Health check update: Degraded data redundancy: 62584/300303 objects degraded (20.840%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:54.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:54 smithi098 bash[79516]: audit 2023-09-23T18:32:54.581048+0000 mon.a (mon.0) 606 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:32:54.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:54 smithi098 bash[76817]: cluster 2023-09-23T18:32:53.710899+0000 mgr.y (mgr.26054) 428 : cluster 0 pgmap v375: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 39 MiB/s wr, 624 op/s; 62584/300303 objects degraded (20.840%); 15838/300303 objects misplaced (5.274%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T18:32:54.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:54 smithi098 bash[76817]: cluster 2023-09-23T18:32:54.436096+0000 mon.a (mon.0) 605 : cluster 3 Health check update: Degraded data redundancy: 62584/300303 objects degraded (20.840%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:32:54.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:54 smithi098 bash[76817]: audit 2023-09-23T18:32:54.581048+0000 mon.a (mon.0) 606 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:32:54.953 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:54.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:54.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:55.056 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:55.089 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:55.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:55.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:55.195 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:55.220 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:55.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:55.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:55.325 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:55.349 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:55.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:55.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:55.456 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:55.492 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:55.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:55.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:55.599 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:55.605 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 84 16 102650 102634 76.3475 39.25 0.0132951 0.0130914 2023-09-23T18:32:55.628 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:55.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:55.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:55.736 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:55.768 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:55.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:55.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:55.871 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:55.896 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:55.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:55.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:55.999 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:56.026 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:56.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:56.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:56.129 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:56.152 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:56.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:56.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:56.255 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:56.280 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:56.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:56.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:56.383 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:56.421 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:56.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:56.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:56.440 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:32:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:32:56] "GET /metrics HTTP/1.1" 200 36544 "" "Prometheus/2.33.4" 2023-09-23T18:32:56.524 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:56.554 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:56.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:56.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:56.606 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 85 16 103245 103229 75.8868 37.1875 0.00526996 0.0131708 2023-09-23T18:32:56.659 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:56.693 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:56.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:56.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:56.797 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:56.805 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:56 smithi173 bash[59422]: cluster 2023-09-23T18:32:55.711776+0000 mgr.y (mgr.26054) 429 : cluster 0 pgmap v376: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 38 MiB/s wr, 602 op/s; 62787/304359 objects degraded (20.629%); 15838/304359 objects misplaced (5.204%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T18:32:56.824 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:56.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:56.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:56.927 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:56.961 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:56.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:56.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:57.044 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:56 smithi098 bash[76817]: cluster 2023-09-23T18:32:55.711776+0000 mgr.y (mgr.26054) 429 : cluster 0 pgmap v376: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 38 MiB/s wr, 602 op/s; 62787/304359 objects degraded (20.629%); 15838/304359 objects misplaced (5.204%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T18:32:57.044 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:56 smithi098 bash[79516]: cluster 2023-09-23T18:32:55.711776+0000 mgr.y (mgr.26054) 429 : cluster 0 pgmap v376: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 38 MiB/s wr, 602 op/s; 62787/304359 objects degraded (20.629%); 15838/304359 objects misplaced (5.204%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T18:32:57.064 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:57.096 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:57.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:57.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:57.199 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:57.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:57.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:57.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:57.329 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:57.360 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:57.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:57.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:57.463 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:57.496 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:57.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:57.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:57.600 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:57.606 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 86 16 103884 103868 75.4688 39.9375 0.00829938 0.0132433 2023-09-23T18:32:57.629 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:32:57.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:57.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:32:57.732 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:57.762 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:57.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:57.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:57.870 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:57.904 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:57.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:57.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:58.010 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:58.034 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:58.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:58.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:58.145 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:58.171 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:58.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:58.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:58.275 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:58.310 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:58.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:58.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:58.416 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:58.453 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:58.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:58.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:58.558 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:58.590 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:58.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:58.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:58.606 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 87 16 104531 104515 75.0661 40.4375 0.0365337 0.0133155 2023-09-23T18:32:58.697 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:58.727 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:58.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:58.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:58.829 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:58.857 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:58.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:58.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:58.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:58 smithi098 bash[79516]: cluster 2023-09-23T18:32:57.712610+0000 mgr.y (mgr.26054) 430 : cluster 0 pgmap v377: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 38 MiB/s wr, 602 op/s; 63205/307944 objects degraded (20.525%); 15722/307944 objects misplaced (5.105%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:32:58.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:58 smithi098 bash[76817]: cluster 2023-09-23T18:32:57.712610+0000 mgr.y (mgr.26054) 430 : cluster 0 pgmap v377: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 38 MiB/s wr, 602 op/s; 63205/307944 objects degraded (20.525%); 15722/307944 objects misplaced (5.105%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:32:58.963 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:58.986 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:58.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:58.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:59.089 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:59.116 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:32:59.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:59.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:32:59.224 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:59.232 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:58 smithi173 bash[59422]: cluster 2023-09-23T18:32:57.712610+0000 mgr.y (mgr.26054) 430 : cluster 0 pgmap v377: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 38 MiB/s wr, 602 op/s; 63205/307944 objects degraded (20.525%); 15722/307944 objects misplaced (5.105%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:32:59.251 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:59.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:59.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:32:59.355 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:59.381 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:32:59.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:59.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:32:59.486 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:59.516 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:32:59.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:59.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:32:59.606 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 88 16 105178 105162 74.6726 40.4375 0.0227942 0.0133856 2023-09-23T18:32:59.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:59.648 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:32:59.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:59.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:32:59.751 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:59.776 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:32:59.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:59.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:32:59.880 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:59.906 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:32:59.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:32:59.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:00.009 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:00.017 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:32:59 smithi173 bash[59422]: cluster 2023-09-23T18:32:59.437092+0000 mon.a (mon.0) 607 : cluster 3 Health check update: Degraded data redundancy: 63205/307944 objects degraded (20.525%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:00.037 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:00.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:00.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:00.045 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:32:59 smithi098 bash[79516]: cluster 2023-09-23T18:32:59.437092+0000 mon.a (mon.0) 607 : cluster 3 Health check update: Degraded data redundancy: 63205/307944 objects degraded (20.525%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:00.045 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:32:59 smithi098 bash[76817]: cluster 2023-09-23T18:32:59.437092+0000 mon.a (mon.0) 607 : cluster 3 Health check update: Degraded data redundancy: 63205/307944 objects degraded (20.525%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:00.141 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:00.179 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:00.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:00.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:00.285 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:00.308 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:00.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:00.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:00.320 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:33:00.354 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:33:00.354 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:33:00.377 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:33:00.412 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:00.438 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:33:00.455 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:00.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:00.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:00.509 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:33:00.563 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:00.564 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:33:00.594 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:00.601 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:33:00.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:00.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:00.606 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 89 16 105697 105681 74.1981 32.4375 0.0103698 0.0134686 2023-09-23T18:33:00.649 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:33:00.650 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:33:00.650 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T18:33:00.650 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:33:00.650 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:33:00.650 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:33:00.650 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:33:00.650 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:33:00.650 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:33:00.706 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:33:00.707 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:00.709 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-23T18:33:00.742 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:00.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:00.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:00.848 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:00.876 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:00.880 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:00 smithi098 bash[79516]: cluster 2023-09-23T18:32:59.713447+0000 mgr.y (mgr.26054) 431 : cluster 0 pgmap v378: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 35 MiB/s wr, 561 op/s; 63419/309678 objects degraded (20.479%); 15756/309678 objects misplaced (5.088%); 3.4 MiB/s, 55 objects/s recovering 2023-09-23T18:33:00.880 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:00 smithi098 bash[79516]: audit 2023-09-23T18:33:00.262061+0000 mon.c (mon.2) 58 : audit 0 from='client.? 172.21.15.98:0/1497780713' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:33:00.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:00.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:00.881 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:00 smithi098 bash[76817]: cluster 2023-09-23T18:32:59.713447+0000 mgr.y (mgr.26054) 431 : cluster 0 pgmap v378: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 35 MiB/s wr, 561 op/s; 63419/309678 objects degraded (20.479%); 15756/309678 objects misplaced (5.088%); 3.4 MiB/s, 55 objects/s recovering 2023-09-23T18:33:00.881 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:00 smithi098 bash[76817]: audit 2023-09-23T18:33:00.262061+0000 mon.c (mon.2) 58 : audit 0 from='client.? 172.21.15.98:0/1497780713' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:33:00.981 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:00.991 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:00 smithi173 bash[59422]: cluster 2023-09-23T18:32:59.713447+0000 mgr.y (mgr.26054) 431 : cluster 0 pgmap v378: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 35 MiB/s wr, 561 op/s; 63419/309678 objects degraded (20.479%); 15756/309678 objects misplaced (5.088%); 3.4 MiB/s, 55 objects/s recovering 2023-09-23T18:33:00.991 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:00 smithi173 bash[59422]: audit 2023-09-23T18:33:00.262061+0000 mon.c (mon.2) 58 : audit 0 from='client.? 172.21.15.98:0/1497780713' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:33:01.021 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:01.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:01.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:01.128 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:01.158 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:01.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:01.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:01.264 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:01.290 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:01.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:01.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:01.394 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:01.421 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:01.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:01.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:01.527 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:01.564 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:01.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:01.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:01.606 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 90 1 106282 106281 73.7904 37.5 0.0297387 0.0135459 2023-09-23T18:33:01.671 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:01.703 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:01.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:01.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:01.807 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:01.813 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is set 2023-09-23T18:33:01.835 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:01.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:01.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:01.939 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:01.958 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:01.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:01.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:02.060 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:02.073 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:01 smithi173 bash[59422]: audit 2023-09-23T18:33:00.674435+0000 mgr.y (mgr.26054) 432 : audit 0 from='client.44750 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:33:02.073 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:01 smithi173 bash[59422]: audit 2023-09-23T18:33:00.892216+0000 mon.c (mon.2) 59 : audit 1 from='client.? 172.21.15.98:0/2242851681' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:33:02.074 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:01 smithi173 bash[59422]: audit 2023-09-23T18:33:00.893443+0000 mon.a (mon.0) 608 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:33:02.091 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:02.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:02.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:02.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:01 smithi098 bash[76817]: audit 2023-09-23T18:33:00.674435+0000 mgr.y (mgr.26054) 432 : audit 0 from='client.44750 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:33:02.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:01 smithi098 bash[76817]: audit 2023-09-23T18:33:00.892216+0000 mon.c (mon.2) 59 : audit 1 from='client.? 172.21.15.98:0/2242851681' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:33:02.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:01 smithi098 bash[76817]: audit 2023-09-23T18:33:00.893443+0000 mon.a (mon.0) 608 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:33:02.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:01 smithi098 bash[79516]: audit 2023-09-23T18:33:00.674435+0000 mgr.y (mgr.26054) 432 : audit 0 from='client.44750 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:33:02.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:01 smithi098 bash[79516]: audit 2023-09-23T18:33:00.892216+0000 mon.c (mon.2) 59 : audit 1 from='client.? 172.21.15.98:0/2242851681' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:33:02.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:01 smithi098 bash[79516]: audit 2023-09-23T18:33:00.893443+0000 mon.a (mon.0) 608 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:33:02.196 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:02.214 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:02.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:02.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:02.316 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:02.344 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:02.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:02.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:02.448 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:02.480 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:02.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:02.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:02.583 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:02.606 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Total time run: 90.021 2023-09-23T18:33:02.606 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Total writes made: 106282 2023-09-23T18:33:02.606 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Write size: 65536 2023-09-23T18:33:02.606 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Object size: 65536 2023-09-23T18:33:02.606 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Bandwidth (MB/sec): 73.7897 2023-09-23T18:33:02.607 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Stddev Bandwidth: 41.7117 2023-09-23T18:33:02.607 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Max bandwidth (MB/sec): 200.125 2023-09-23T18:33:02.607 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Min bandwidth (MB/sec): 0 2023-09-23T18:33:02.607 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Average IOPS: 1180 2023-09-23T18:33:02.607 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Stddev IOPS: 667.388 2023-09-23T18:33:02.607 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Max IOPS: 3202 2023-09-23T18:33:02.607 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Min IOPS: 0 2023-09-23T18:33:02.607 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Average Latency(s): 0.0135461 2023-09-23T18:33:02.607 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Stddev Latency(s): 0.0317327 2023-09-23T18:33:02.607 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Max latency(s): 2.46834 2023-09-23T18:33:02.607 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Min latency(s): 0.00116463 2023-09-23T18:33:02.613 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:02.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:02.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:02.620 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T18:33:02.717 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:02.746 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:02.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:02.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:02.849 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:02.896 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:02.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:02.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:03.002 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:03.013 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:02 smithi173 bash[59422]: cluster 2023-09-23T18:33:01.714921+0000 mgr.y (mgr.26054) 433 : cluster 0 pgmap v379: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 40 MiB/s wr, 645 op/s; 63806/314295 objects degraded (20.301%); 15756/314295 objects misplaced (5.013%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T18:33:03.013 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:02 smithi173 bash[59422]: cluster 2023-09-23T18:33:01.790751+0000 mon.a (mon.0) 609 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:33:03.013 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:02 smithi173 bash[59422]: cluster 2023-09-23T18:33:01.801669+0000 osd.7 (osd.7) 34 : cluster 3 failed to encode map e120 with expected crc 2023-09-23T18:33:03.013 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:02 smithi173 bash[59422]: audit 2023-09-23T18:33:01.807512+0000 mon.a (mon.0) 610 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:33:03.013 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:02 smithi173 bash[59422]: cluster 2023-09-23T18:33:01.807668+0000 mon.a (mon.0) 611 : cluster 0 osdmap e120: 8 total, 7 up, 7 in 2023-09-23T18:33:03.031 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:03.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:03.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:03.137 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:03.147 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:02 smithi098 bash[76817]: cluster 2023-09-23T18:33:01.714921+0000 mgr.y (mgr.26054) 433 : cluster 0 pgmap v379: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 40 MiB/s wr, 645 op/s; 63806/314295 objects degraded (20.301%); 15756/314295 objects misplaced (5.013%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T18:33:03.147 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:02 smithi098 bash[76817]: cluster 2023-09-23T18:33:01.790751+0000 mon.a (mon.0) 609 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:33:03.147 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:02 smithi098 bash[76817]: cluster 2023-09-23T18:33:01.801669+0000 osd.7 (osd.7) 34 : cluster 3 failed to encode map e120 with expected crc 2023-09-23T18:33:03.147 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:02 smithi098 bash[76817]: audit 2023-09-23T18:33:01.807512+0000 mon.a (mon.0) 610 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:33:03.148 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:02 smithi098 bash[76817]: cluster 2023-09-23T18:33:01.807668+0000 mon.a (mon.0) 611 : cluster 0 osdmap e120: 8 total, 7 up, 7 in 2023-09-23T18:33:03.148 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:02 smithi098 bash[79516]: cluster 2023-09-23T18:33:01.714921+0000 mgr.y (mgr.26054) 433 : cluster 0 pgmap v379: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 40 MiB/s wr, 645 op/s; 63806/314295 objects degraded (20.301%); 15756/314295 objects misplaced (5.013%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T18:33:03.148 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:02 smithi098 bash[79516]: cluster 2023-09-23T18:33:01.790751+0000 mon.a (mon.0) 609 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:33:03.148 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:02 smithi098 bash[79516]: cluster 2023-09-23T18:33:01.801669+0000 osd.7 (osd.7) 34 : cluster 3 failed to encode map e120 with expected crc 2023-09-23T18:33:03.148 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:02 smithi098 bash[79516]: audit 2023-09-23T18:33:01.807512+0000 mon.a (mon.0) 610 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:33:03.148 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:02 smithi098 bash[79516]: cluster 2023-09-23T18:33:01.807668+0000 mon.a (mon.0) 611 : cluster 0 osdmap e120: 8 total, 7 up, 7 in 2023-09-23T18:33:03.167 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:03.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:03.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:03.273 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:03.302 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:03.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:03.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:03.405 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:03.435 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:03.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:03.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:03.538 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:03.564 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:03.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:03.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:03.668 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:03.692 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:03.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:03.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:03.756 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T18:33:03.797 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:03.826 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:03.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:03.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:03.930 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:03.962 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:03.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:03.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:04.069 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:04.111 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:04.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:04.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:04.218 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:04.250 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:04.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:04.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:04.357 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:04.384 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:04.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:04.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:04.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:04.514 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:04.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:04.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:04.619 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:04.650 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:04.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:04.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:04.754 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:04.783 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:04.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:04.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:04.890 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:04.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:04.929 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:04.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:05.030 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:05.038 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:04 smithi098 bash[79516]: cluster 2023-09-23T18:33:03.715835+0000 mgr.y (mgr.26054) 434 : cluster 0 pgmap v381: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 37 MiB/s wr, 597 op/s; 64098/317820 objects degraded (20.168%); 15542/317820 objects misplaced (4.890%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T18:33:05.038 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:04 smithi098 bash[79516]: cluster 2023-09-23T18:33:04.438349+0000 mon.a (mon.0) 612 : cluster 3 Health check update: Degraded data redundancy: 64098/317820 objects degraded (20.168%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:05.039 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:04 smithi098 bash[76817]: cluster 2023-09-23T18:33:03.715835+0000 mgr.y (mgr.26054) 434 : cluster 0 pgmap v381: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 37 MiB/s wr, 597 op/s; 64098/317820 objects degraded (20.168%); 15542/317820 objects misplaced (4.890%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T18:33:05.039 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:04 smithi098 bash[76817]: cluster 2023-09-23T18:33:04.438349+0000 mon.a (mon.0) 612 : cluster 3 Health check update: Degraded data redundancy: 64098/317820 objects degraded (20.168%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:05.073 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:05.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:05.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:05.180 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:05.190 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:04 smithi173 bash[59422]: cluster 2023-09-23T18:33:03.715835+0000 mgr.y (mgr.26054) 434 : cluster 0 pgmap v381: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 37 MiB/s wr, 597 op/s; 64098/317820 objects degraded (20.168%); 15542/317820 objects misplaced (4.890%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T18:33:05.190 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:04 smithi173 bash[59422]: cluster 2023-09-23T18:33:04.438349+0000 mon.a (mon.0) 612 : cluster 3 Health check update: Degraded data redundancy: 64098/317820 objects degraded (20.168%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:05.209 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:05.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:05.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:05.315 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:05.348 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:05.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:05.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:05.451 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:05.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:05.482 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:05.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:05.582 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:05.612 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:05.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:05.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:05.716 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:05.748 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:05.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:05.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:05.855 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:05.881 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:05.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:05.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:05.983 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:06.007 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:06.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:06.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:06.033 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:33:06.034 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:33:06.083 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":378,"stamp":"2023-09-23T18:32:59.712795+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6767974323,"num_objects":103226,"num_object_clones":0,"num_object_copies":309678,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":63419,"num_objects_misplaced":15756,"num_objects_unfound":0,"num_objects_dirty":103226,"num_whiteouts":0,"num_read":6245,"num_read_kb":13702,"num_write":104304,"num_write_kb":6619831,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1512,"num_bytes_recovered":101450371,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104368,"ondisk_log_size":104368,"up":226,"acting":229,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":258,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":15992600,"kb_used_data":14729248,"kb_used_omap":31,"kb_used_meta":1263264,"kb_avail":640223464,"statfs":{"total":671965249536,"available":655588827136,"internally_reserved":0,"allocated":15082749952,"data_stored":15078881372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32507,"internal_metadata":1293582597},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":441712640,"num_objects":6740,"num_object_clones":0,"num_object_copies":20220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1805,"num_objects_misplaced":-105,"num_objects_unfound":0,"num_objects_dirty":6740,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6740,"num_write_kb":431360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":661,"num_bytes_recovered":43319296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005338"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":107,"reported_epoch":118,"state":"active+undersized","last_fresh":"2023-09-23T18:32:29.834051+0000","last_change":"2023-09-23T18:30:57.765424+0000","last_active":"2023-09-23T18:32:29.834051+0000","last_peered":"2023-09-23T18:32:29.834051+0000","last_clean":"2023-09-23T18:28:03.292943+0000","last_became_active":"2023-09-23T18:30:57.765424+0000","last_became_peered":"2023-09-23T18:30:57.765424+0000","last_unstale":"2023-09-23T18:32:29.834051+0000","last_undegraded":"2023-09-23T18:32:29.834051+0000","last_fullsized":"2023-09-23T18:30:57.757189+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":93,"reported_epoch":118,"state":"active+undersized","last_fresh":"2023-09-23T18:32:29.807967+0000","last_change":"2023-09-23T18:30:57.761478+0000","last_active":"2023-09-23T18:32:29.807967+0000","last_peered":"2023-09-23T18:32:29.807967+0000","last_clean":"2023-09-23T18:30:31.307538+0000","last_became_active":"2023-09-23T18:30:57.761478+0000","last_became_peered":"2023-09-23T18:30:57.761478+0000","last_unstale":"2023-09-23T18:32:29.807967+0000","last_undegraded":"2023-09-23T18:32:29.807967+0000","last_fullsized":"2023-09-23T18:30:57.742716+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:23.036992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":60,"reported_epoch":118,"state":"active+undersized","last_fresh":"2023-09-23T18:32:29.807030+0000","last_change":"2023-09-23T18:30:57.756090+0000","last_active":"2023-09-23T18:32:29.807030+0000","last_peered":"2023-09-23T18:32:29.807030+0000","last_clean":"2023-09-23T18:27:33.568258+0000","last_became_active":"2023-09-23T18:30:57.756090+0000","last_became_peered":"2023-09-23T18:30:57.756090+0000","last_unstale":"2023-09-23T18:32:29.807030+0000","last_undegraded":"2023-09-23T18:32:29.807030+0000","last_fullsized":"2023-09-23T18:30:57.742018+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:14.861981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":70,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:33.089213+0000","last_change":"2023-09-23T18:28:03.289500+0000","last_active":"2023-09-23T18:32:33.089213+0000","last_peered":"2023-09-23T18:32:33.089213+0000","last_clean":"2023-09-23T18:32:33.089213+0000","last_became_active":"2023-09-23T18:28:03.289122+0000","last_became_peered":"2023-09-23T18:28:03.289122+0000","last_unstale":"2023-09-23T18:32:33.089213+0000","last_undegraded":"2023-09-23T18:32:33.089213+0000","last_fullsized":"2023-09-23T18:32:33.089213+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:22:26.904660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":110,"reported_epoch":119,"state":"active+undersized","last_fresh":"2023-09-23T18:32:35.460122+0000","last_change":"2023-09-23T18:30:57.745533+0000","last_active":"2023-09-23T18:32:35.460122+0000","last_peered":"2023-09-23T18:32:35.460122+0000","last_clean":"2023-09-23T18:30:26.306231+0000","last_became_active":"2023-09-23T18:30:57.745533+0000","last_became_peered":"2023-09-23T18:30:57.745533+0000","last_unstale":"2023-09-23T18:32:35.460122+0000","last_undegraded":"2023-09-23T18:32:35.460122+0000","last_fullsized":"2023-09-23T18:30:57.741719+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:13.887774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":100,"reported_epoch":118,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:32:29.817176+0000","last_change":"2023-09-23T18:32:27.841917+0000","last_active":"2023-09-23T18:32:29.817176+0000","last_peered":"2023-09-23T18:32:29.817176+0000","last_clean":"2023-09-23T18:32:29.817176+0000","last_became_active":"2023-09-23T18:32:27.841317+0000","last_became_peered":"2023-09-23T18:32:27.841317+0000","last_unstale":"2023-09-23T18:32:29.817176+0000","last_undegraded":"2023-09-23T18:32:29.817176+0000","last_fullsized":"2023-09-23T18:32:29.817176+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0,5],"avail_no_missing":["6","0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":77,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.822694+0000","last_change":"2023-09-23T18:32:26.797582+0000","last_active":"2023-09-23T18:32:29.822694+0000","last_peered":"2023-09-23T18:32:29.822694+0000","last_clean":"2023-09-23T18:32:29.822694+0000","last_became_active":"2023-09-23T18:32:26.797107+0000","last_became_peered":"2023-09-23T18:32:26.797107+0000","last_unstale":"2023-09-23T18:32:29.822694+0000","last_undegraded":"2023-09-23T18:32:29.822694+0000","last_fullsized":"2023-09-23T18:32:29.822694+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:56.973116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":94,"reported_epoch":119,"state":"active+undersized","last_fresh":"2023-09-23T18:32:35.460015+0000","last_change":"2023-09-23T18:30:57.755753+0000","last_active":"2023-09-23T18:32:35.460015+0000","last_peered":"2023-09-23T18:32:35.460015+0000","last_clean":"2023-09-23T18:30:21.304949+0000","last_became_active":"2023-09-23T18:30:57.755753+0000","last_became_peered":"2023-09-23T18:30:57.755753+0000","last_unstale":"2023-09-23T18:32:35.460015+0000","last_undegraded":"2023-09-23T18:32:35.460015+0000","last_fullsized":"2023-09-23T18:30:57.741711+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:47.235882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":89,"reported_epoch":118,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:32:29.817635+0000","last_change":"2023-09-23T18:32:27.816671+0000","last_active":"2023-09-23T18:32:29.817635+0000","last_peered":"2023-09-23T18:32:29.817635+0000","last_clean":"2023-09-23T18:32:29.817635+0000","last_became_active":"2023-09-23T18:32:27.814984+0000","last_became_peered":"2023-09-23T18:32:27.814984+0000","last_unstale":"2023-09-23T18:32:29.817635+0000","last_undegraded":"2023-09-23T18:32:29.817635+0000","last_fullsized":"2023-09-23T18:32:29.817635+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3,5],"avail_no_missing":["7","3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":78,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-09-23T18:32:35.458685+0000","last_change":"2023-09-23T18:28:03.298214+0000","last_active":"2023-09-23T18:32:35.458685+0000","last_peered":"2023-09-23T18:32:35.458685+0000","last_clean":"2023-09-23T18:32:35.458685+0000","last_became_active":"2023-09-23T18:28:03.297852+0000","last_became_peered":"2023-09-23T18:28:03.297852+0000","last_unstale":"2023-09-23T18:32:35.458685+0000","last_undegraded":"2023-09-23T18:32:35.458685+0000","last_fullsized":"2023-09-23T18:32:35.458685+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.146541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":88,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.817891+0000","last_change":"2023-09-23T18:28:03.296913+0000","last_active":"2023-09-23T18:32:29.817891+0000","last_peered":"2023-09-23T18:32:29.817891+0000","last_clean":"2023-09-23T18:32:29.817891+0000","last_became_active":"2023-09-23T18:28:03.296499+0000","last_became_peered":"2023-09-23T18:28:03.296499+0000","last_unstale":"2023-09-23T18:32:29.817891+0000","last_undegraded":"2023-09-23T18:32:29.817891+0000","last_fullsized":"2023-09-23T18:32:29.817891+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":55,"reported_epoch":118,"state":"active+undersized","last_fresh":"2023-09-23T18:32:29.816747+0000","last_change":"2023-09-23T18:30:57.765407+0000","last_active":"2023-09-23T18:32:29.816747+0000","last_peered":"2023-09-23T18:32:29.816747+0000","last_clean":"2023-09-23T18:27:04.257256+0000","last_became_active":"2023-09-23T18:30:57.765407+0000","last_became_peered":"2023-09-23T18:30:57.765407+0000","last_unstale":"2023-09-23T18:32:29.816747+0000","last_undegraded":"2023-09-23T18:32:29.816747+0000","last_fullsized":"2023-09-23T18:30:57.754597+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":92,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.834294+0000","last_change":"2023-09-23T18:15:08.909339+0000","last_active":"2023-09-23T18:32:29.834294+0000","last_peered":"2023-09-23T18:32:29.834294+0000","last_clean":"2023-09-23T18:32:29.834294+0000","last_became_active":"2023-09-23T18:15:08.908912+0000","last_became_peered":"2023-09-23T18:15:08.908912+0000","last_unstale":"2023-09-23T18:32:29.834294+0000","last_undegraded":"2023-09-23T18:32:29.834294+0000","last_fullsized":"2023-09-23T18:32:29.834294+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":90,"reported_epoch":118,"state":"active+undersized","last_fresh":"2023-09-23T18:32:29.833725+0000","last_change":"2023-09-23T18:30:57.762665+0000","last_active":"2023-09-23T18:32:29.833725+0000","last_peered":"2023-09-23T18:32:29.833725+0000","last_clean":"2023-09-23T18:28:03.293385+0000","last_became_active":"2023-09-23T18:30:57.762665+0000","last_became_peered":"2023-09-23T18:30:57.762665+0000","last_unstale":"2023-09-23T18:32:29.833725+0000","last_undegraded":"2023-09-23T18:32:29.833725+0000","last_fullsized":"2023-09-23T18:30:57.757694+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":87,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.820296+0000","last_change":"2023-09-23T18:27:33.585121+0000","last_active":"2023-09-23T18:32:29.820296+0000","last_peered":"2023-09-23T18:32:29.820296+0000","last_clean":"2023-09-23T18:32:29.820296+0000","last_became_active":"2023-09-23T18:27:33.584620+0000","last_became_peered":"2023-09-23T18:27:33.584620+0000","last_unstale":"2023-09-23T18:32:29.820296+0000","last_undegraded":"2023-09-23T18:32:29.820296+0000","last_fullsized":"2023-09-23T18:32:29.820296+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":94,"reported_epoch":119,"state":"active+undersized","last_fresh":"2023-09-23T18:32:35.459037+0000","last_change":"2023-09-23T18:30:57.755267+0000","last_active":"2023-09-23T18:32:35.459037+0000","last_peered":"2023-09-23T18:32:35.459037+0000","last_clean":"2023-09-23T18:30:21.305279+0000","last_became_active":"2023-09-23T18:30:57.755267+0000","last_became_peered":"2023-09-23T18:30:57.755267+0000","last_unstale":"2023-09-23T18:32:35.459037+0000","last_undegraded":"2023-09-23T18:32:35.459037+0000","last_fullsized":"2023-09-23T18:30:57.743249+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:25.575440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":68,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.821218+0000","last_change":"2023-09-23T18:28:03.290316+0000","last_active":"2023-09-23T18:32:29.821218+0000","last_peered":"2023-09-23T18:32:29.821218+0000","last_clean":"2023-09-23T18:32:29.821218+0000","last_became_active":"2023-09-23T18:28:03.289698+0000","last_became_peered":"2023-09-23T18:28:03.289698+0000","last_unstale":"2023-09-23T18:32:29.821218+0000","last_undegraded":"2023-09-23T18:32:29.821218+0000","last_fullsized":"2023-09-23T18:32:29.821218+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:00.779542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":76,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.817430+0000","last_change":"2023-09-23T18:32:26.808237+0000","last_active":"2023-09-23T18:32:29.817430+0000","last_peered":"2023-09-23T18:32:29.817430+0000","last_clean":"2023-09-23T18:32:29.817430+0000","last_became_active":"2023-09-23T18:32:26.807843+0000","last_became_peered":"2023-09-23T18:32:26.807843+0000","last_unstale":"2023-09-23T18:32:29.817430+0000","last_undegraded":"2023-09-23T18:32:29.817430+0000","last_fullsized":"2023-09-23T18:32:29.817430+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":70,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.818211+0000","last_change":"2023-09-23T18:15:08.916658+0000","last_active":"2023-09-23T18:32:29.818211+0000","last_peered":"2023-09-23T18:32:29.818211+0000","last_clean":"2023-09-23T18:32:29.818211+0000","last_became_active":"2023-09-23T18:15:08.916301+0000","last_became_peered":"2023-09-23T18:15:08.916301+0000","last_unstale":"2023-09-23T18:32:29.818211+0000","last_undegraded":"2023-09-23T18:32:29.818211+0000","last_fullsized":"2023-09-23T18:32:29.818211+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":69,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.821243+0000","last_change":"2023-09-23T18:28:03.290217+0000","last_active":"2023-09-23T18:32:29.821243+0000","last_peered":"2023-09-23T18:32:29.821243+0000","last_clean":"2023-09-23T18:32:29.821243+0000","last_became_active":"2023-09-23T18:28:03.289888+0000","last_became_peered":"2023-09-23T18:28:03.289888+0000","last_unstale":"2023-09-23T18:32:29.821243+0000","last_undegraded":"2023-09-23T18:32:29.821243+0000","last_fullsized":"2023-09-23T18:32:29.821243+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:51:45.400847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":112,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.816340+0000","last_change":"2023-09-23T18:32:26.810836+0000","last_active":"2023-09-23T18:32:29.816340+0000","last_peered":"2023-09-23T18:32:29.816340+0000","last_clean":"2023-09-23T18:32:29.816340+0000","last_became_active":"2023-09-23T18:32:26.795378+0000","last_became_peered":"2023-09-23T18:32:26.795378+0000","last_unstale":"2023-09-23T18:32:29.816340+0000","last_undegraded":"2023-09-23T18:32:29.816340+0000","last_fullsized":"2023-09-23T18:32:29.816340+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":94,"reported_epoch":119,"state":"active+undersized","last_fresh":"2023-09-23T18:32:35.459574+0000","last_change":"2023-09-23T18:30:57.754720+0000","last_active":"2023-09-23T18:32:35.459574+0000","last_peered":"2023-09-23T18:32:35.459574+0000","last_clean":"2023-09-23T18:30:26.306588+0000","last_became_active":"2023-09-23T18:30:57.754720+0000","last_became_peered":"2023-09-23T18:30:57.754720+0000","last_unstale":"2023-09-23T18:32:35.459574+0000","last_undegraded":"2023-09-23T18:32:35.459574+0000","last_fullsized":"2023-09-23T18:30:57.742671+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:46.816007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"119'6529","reported_seq":6549,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-09-23T18:32:55.552856+0000","last_change":"2023-09-23T18:31:11.799201+0000","last_active":"2023-09-23T18:32:55.552856+0000","last_peered":"2023-09-23T18:32:55.552856+0000","last_clean":"2023-09-23T18:32:55.552856+0000","last_became_active":"2023-09-23T18:31:11.798833+0000","last_became_peered":"2023-09-23T18:31:11.798833+0000","last_unstale":"2023-09-23T18:32:55.552856+0000","last_undegraded":"2023-09-23T18:32:55.552856+0000","last_fullsized":"2023-09-23T18:32:55.552856+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":6529,"log_dups_size":0,"ondisk_log_size":6529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":427884544,"num_objects":6529,"num_object_clones":0,"num_object_copies":19587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6529,"num_write_kb":417856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":86,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.817849+0000","last_change":"2023-09-23T18:26:34.999684+0000","last_active":"2023-09-23T18:32:29.817849+0000","last_peered":"2023-09-23T18:32:29.817849+0000","last_clean":"2023-09-23T18:32:29.817849+0000","last_became_active":"2023-09-23T18:26:34.999076+0000","last_became_peered":"2023-09-23T18:26:34.999076+0000","last_unstale":"2023-09-23T18:32:29.817849+0000","last_undegraded":"2023-09-23T18:32:29.817849+0000","last_fullsized":"2023-09-23T18:32:29.817849+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":17,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.816856+0000","last_change":"2023-09-23T18:32:26.806328+0000","last_active":"2023-09-23T18:32:29.816856+0000","last_peered":"2023-09-23T18:32:29.816856+0000","last_clean":"2023-09-23T18:32:29.816856+0000","last_became_active":"2023-09-23T18:32:26.803969+0000","last_became_peered":"2023-09-23T18:32:26.803969+0000","last_unstale":"2023-09-23T18:32:29.816856+0000","last_undegraded":"2023-09-23T18:32:29.816856+0000","last_fullsized":"2023-09-23T18:32:29.816856+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.8","version":"119'6500","reported_seq":6518,"reported_epoch":119,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:32:56.768003+0000","last_change":"2023-09-23T18:31:31.595872+0000","last_active":"2023-09-23T18:32:56.768003+0000","last_peered":"2023-09-23T18:32:56.768003+0000","last_clean":"2023-09-23T18:31:10.759555+0000","last_became_active":"2023-09-23T18:31:11.789789+0000","last_became_peered":"2023-09-23T18:31:11.789789+0000","last_unstale":"2023-09-23T18:32:56.768003+0000","last_undegraded":"2023-09-23T18:31:31.591874+0000","last_fullsized":"2023-09-23T18:31:11.784114+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":6500,"log_dups_size":0,"ondisk_log_size":6500,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:02.167863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":425984000,"num_objects":6500,"num_object_clones":0,"num_object_copies":19500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6500,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6500,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":6500}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":93,"reported_epoch":118,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:32:29.817783+0000","last_change":"2023-09-23T18:30:57.760718+0000","last_active":"2023-09-23T18:32:29.817783+0000","last_peered":"2023-09-23T18:32:29.817783+0000","last_clean":"2023-09-23T18:28:03.287970+0000","last_became_active":"2023-09-23T18:30:57.760718+0000","last_became_peered":"2023-09-23T18:30:57.760718+0000","last_unstale":"2023-09-23T18:32:29.817783+0000","last_undegraded":"2023-09-23T18:30:57.754075+0000","last_fullsized":"2023-09-23T18:30:57.753412+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":105,"reported_epoch":118,"state":"active+undersized","last_fresh":"2023-09-23T18:32:29.827405+0000","last_change":"2023-09-23T18:30:57.759991+0000","last_active":"2023-09-23T18:32:29.827405+0000","last_peered":"2023-09-23T18:32:29.827405+0000","last_clean":"2023-09-23T18:28:03.302737+0000","last_became_active":"2023-09-23T18:30:57.759991+0000","last_became_peered":"2023-09-23T18:30:57.759991+0000","last_unstale":"2023-09-23T18:32:29.827405+0000","last_undegraded":"2023-09-23T18:32:29.827405+0000","last_fullsized":"2023-09-23T18:30:57.755928+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"119'6483","reported_seq":7862,"reported_epoch":119,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T18:32:56.873505+0000","last_change":"2023-09-23T18:32:28.380669+0000","last_active":"2023-09-23T18:32:56.873505+0000","last_peered":"2023-09-23T18:32:56.873505+0000","last_clean":"2023-09-23T18:32:25.776126+0000","last_became_active":"2023-09-23T18:32:28.013755+0000","last_became_peered":"2023-09-23T18:32:28.013755+0000","last_unstale":"2023-09-23T18:32:56.873505+0000","last_undegraded":"2023-09-23T18:32:56.873505+0000","last_fullsized":"2023-09-23T18:32:27.798678+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":6483,"log_dups_size":0,"ondisk_log_size":6483,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:16.098262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":424869888,"num_objects":6483,"num_object_clones":0,"num_object_copies":19449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":6483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6483,"num_write_kb":414912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":675,"num_bytes_recovered":44236800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0],"avail_no_missing":["2","0","5"],"object_location_counts":[{"shards":"0,2","objects":1787},{"shards":"0,2,5","objects":4696}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":84,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.817977+0000","last_change":"2023-09-23T18:32:26.854615+0000","last_active":"2023-09-23T18:32:29.817977+0000","last_peered":"2023-09-23T18:32:29.817977+0000","last_clean":"2023-09-23T18:32:29.817977+0000","last_became_active":"2023-09-23T18:32:26.806771+0000","last_became_peered":"2023-09-23T18:32:26.806771+0000","last_unstale":"2023-09-23T18:32:29.817977+0000","last_undegraded":"2023-09-23T18:32:29.817977+0000","last_fullsized":"2023-09-23T18:32:29.817977+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":93,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.817461+0000","last_change":"2023-09-23T18:27:33.589116+0000","last_active":"2023-09-23T18:32:29.817461+0000","last_peered":"2023-09-23T18:32:29.817461+0000","last_clean":"2023-09-23T18:32:29.817461+0000","last_became_active":"2023-09-23T18:27:33.588809+0000","last_became_peered":"2023-09-23T18:27:33.588809+0000","last_unstale":"2023-09-23T18:32:29.817461+0000","last_undegraded":"2023-09-23T18:32:29.817461+0000","last_fullsized":"2023-09-23T18:32:29.817461+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"119'6304","reported_seq":6325,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-09-23T18:32:56.790470+0000","last_change":"2023-09-23T18:31:11.796358+0000","last_active":"2023-09-23T18:32:56.790470+0000","last_peered":"2023-09-23T18:32:56.790470+0000","last_clean":"2023-09-23T18:32:56.790470+0000","last_became_active":"2023-09-23T18:31:11.795962+0000","last_became_peered":"2023-09-23T18:31:11.795962+0000","last_unstale":"2023-09-23T18:32:56.790470+0000","last_undegraded":"2023-09-23T18:32:56.790470+0000","last_fullsized":"2023-09-23T18:32:56.790470+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":6304,"log_dups_size":0,"ondisk_log_size":6304,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:29.481931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":413138944,"num_objects":6304,"num_object_clones":0,"num_object_copies":18912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6304,"num_write_kb":403456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":78,"reported_epoch":118,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:32:29.806841+0000","last_change":"2023-09-23T18:32:27.820211+0000","last_active":"2023-09-23T18:32:29.806841+0000","last_peered":"2023-09-23T18:32:29.806841+0000","last_clean":"2023-09-23T18:32:29.806841+0000","last_became_active":"2023-09-23T18:32:27.819760+0000","last_became_peered":"2023-09-23T18:32:27.819760+0000","last_unstale":"2023-09-23T18:32:29.806841+0000","last_undegraded":"2023-09-23T18:32:29.806841+0000","last_fullsized":"2023-09-23T18:32:29.806841+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:42.052114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6,5],"avail_no_missing":["3","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":81,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.828127+0000","last_change":"2023-09-23T18:27:33.582033+0000","last_active":"2023-09-23T18:32:29.828127+0000","last_peered":"2023-09-23T18:32:29.828127+0000","last_clean":"2023-09-23T18:32:29.828127+0000","last_became_active":"2023-09-23T18:27:33.581667+0000","last_became_peered":"2023-09-23T18:27:33.581667+0000","last_unstale":"2023-09-23T18:32:29.828127+0000","last_undegraded":"2023-09-23T18:32:29.828127+0000","last_fullsized":"2023-09-23T18:32:29.828127+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:36.039922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"119'6456","reported_seq":6485,"reported_epoch":119,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:32:56.839651+0000","last_change":"2023-09-23T18:32:28.082814+0000","last_active":"2023-09-23T18:32:56.839651+0000","last_peered":"2023-09-23T18:32:56.839651+0000","last_clean":"2023-09-23T18:32:25.736744+0000","last_became_active":"2023-09-23T18:32:28.065620+0000","last_became_peered":"2023-09-23T18:32:28.065620+0000","last_unstale":"2023-09-23T18:32:56.839651+0000","last_undegraded":"2023-09-23T18:32:27.851494+0000","last_fullsized":"2023-09-23T18:32:27.798797+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":6456,"log_dups_size":0,"ondisk_log_size":6456,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:49.026074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":423100416,"num_objects":6456,"num_object_clones":0,"num_object_copies":19368,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5354,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6456,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6456,"num_write_kb":413184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":5354},{"shards":"2,7","objects":1102}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":87,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:31.334296+0000","last_change":"2023-09-23T18:28:03.297628+0000","last_active":"2023-09-23T18:32:31.334296+0000","last_peered":"2023-09-23T18:32:31.334296+0000","last_clean":"2023-09-23T18:32:31.334296+0000","last_became_active":"2023-09-23T18:28:03.296694+0000","last_became_peered":"2023-09-23T18:28:03.296694+0000","last_unstale":"2023-09-23T18:32:31.334296+0000","last_undegraded":"2023-09-23T18:32:31.334296+0000","last_fullsized":"2023-09-23T18:32:31.334296+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:43.375652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":84,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.833941+0000","last_change":"2023-09-23T18:28:03.302326+0000","last_active":"2023-09-23T18:32:29.833941+0000","last_peered":"2023-09-23T18:32:29.833941+0000","last_clean":"2023-09-23T18:32:29.833941+0000","last_became_active":"2023-09-23T18:28:03.301970+0000","last_became_peered":"2023-09-23T18:28:03.301970+0000","last_unstale":"2023-09-23T18:32:29.833941+0000","last_undegraded":"2023-09-23T18:32:29.833941+0000","last_fullsized":"2023-09-23T18:32:29.833941+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"119'6488","reported_seq":6505,"reported_epoch":119,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:32:55.507816+0000","last_change":"2023-09-23T18:31:31.595358+0000","last_active":"2023-09-23T18:32:55.507816+0000","last_peered":"2023-09-23T18:32:55.507816+0000","last_clean":"2023-09-23T18:31:10.759555+0000","last_became_active":"2023-09-23T18:31:11.793471+0000","last_became_peered":"2023-09-23T18:31:11.793471+0000","last_unstale":"2023-09-23T18:32:55.507816+0000","last_undegraded":"2023-09-23T18:31:31.595206+0000","last_fullsized":"2023-09-23T18:31:11.790449+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":6488,"log_dups_size":0,"ondisk_log_size":6488,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":425197568,"num_objects":6488,"num_object_clones":0,"num_object_copies":19464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6488,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6488,"num_write_kb":415232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":6488}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":91,"reported_epoch":118,"state":"active+undersized","last_fresh":"2023-09-23T18:32:29.817733+0000","last_change":"2023-09-23T18:30:57.763833+0000","last_active":"2023-09-23T18:32:29.817733+0000","last_peered":"2023-09-23T18:32:29.817733+0000","last_clean":"2023-09-23T18:28:03.294831+0000","last_became_active":"2023-09-23T18:30:57.763833+0000","last_became_peered":"2023-09-23T18:30:57.763833+0000","last_unstale":"2023-09-23T18:32:29.817733+0000","last_undegraded":"2023-09-23T18:32:29.817733+0000","last_fullsized":"2023-09-23T18:30:57.755260+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":105,"reported_epoch":118,"state":"active+undersized","last_fresh":"2023-09-23T18:32:29.833875+0000","last_change":"2023-09-23T18:30:57.765518+0000","last_active":"2023-09-23T18:32:29.833875+0000","last_peered":"2023-09-23T18:32:29.833875+0000","last_clean":"2023-09-23T18:28:03.294665+0000","last_became_active":"2023-09-23T18:30:57.765518+0000","last_became_peered":"2023-09-23T18:30:57.765518+0000","last_unstale":"2023-09-23T18:32:29.833875+0000","last_undegraded":"2023-09-23T18:32:29.833875+0000","last_fullsized":"2023-09-23T18:30:57.757909+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"119'6297","reported_seq":6323,"reported_epoch":119,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:32:55.561059+0000","last_change":"2023-09-23T18:32:28.156547+0000","last_active":"2023-09-23T18:32:55.561059+0000","last_peered":"2023-09-23T18:32:55.561059+0000","last_clean":"2023-09-23T18:32:25.747134+0000","last_became_active":"2023-09-23T18:32:28.139799+0000","last_became_peered":"2023-09-23T18:32:28.139799+0000","last_unstale":"2023-09-23T18:32:55.561059+0000","last_undegraded":"2023-09-23T18:32:27.846258+0000","last_fullsized":"2023-09-23T18:32:27.805907+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":6297,"log_dups_size":0,"ondisk_log_size":6297,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":412680192,"num_objects":6297,"num_object_clones":0,"num_object_copies":18891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5297,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6297,"num_write_kb":403008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"4,5,6","objects":5297},{"shards":"4,6","objects":1000}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":85,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.827129+0000","last_change":"2023-09-23T18:27:33.585714+0000","last_active":"2023-09-23T18:32:29.827129+0000","last_peered":"2023-09-23T18:32:29.827129+0000","last_clean":"2023-09-23T18:32:29.827129+0000","last_became_active":"2023-09-23T18:27:33.584641+0000","last_became_peered":"2023-09-23T18:27:33.584641+0000","last_unstale":"2023-09-23T18:32:29.827129+0000","last_undegraded":"2023-09-23T18:32:29.827129+0000","last_fullsized":"2023-09-23T18:32:29.827129+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":90,"reported_epoch":118,"state":"active+undersized","last_fresh":"2023-09-23T18:32:29.834426+0000","last_change":"2023-09-23T18:30:57.762661+0000","last_active":"2023-09-23T18:32:29.834426+0000","last_peered":"2023-09-23T18:32:29.834426+0000","last_clean":"2023-09-23T18:28:03.292763+0000","last_became_active":"2023-09-23T18:30:57.762661+0000","last_became_peered":"2023-09-23T18:30:57.762661+0000","last_unstale":"2023-09-23T18:32:29.834426+0000","last_undegraded":"2023-09-23T18:32:29.834426+0000","last_fullsized":"2023-09-23T18:30:57.760181+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"119'6463","reported_seq":6488,"reported_epoch":119,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:32:55.569363+0000","last_change":"2023-09-23T18:32:28.237717+0000","last_active":"2023-09-23T18:32:55.569363+0000","last_peered":"2023-09-23T18:32:55.569363+0000","last_clean":"2023-09-23T18:32:25.757964+0000","last_became_active":"2023-09-23T18:32:28.219874+0000","last_became_peered":"2023-09-23T18:32:28.219874+0000","last_unstale":"2023-09-23T18:32:55.569363+0000","last_undegraded":"2023-09-23T18:32:27.848749+0000","last_fullsized":"2023-09-23T18:32:27.803577+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":6463,"log_dups_size":0,"ondisk_log_size":6463,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":423559168,"num_objects":6463,"num_object_clones":0,"num_object_copies":19389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5355,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6463,"num_write_kb":413632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0],"avail_no_missing":["4","0","5"],"object_location_counts":[{"shards":"0,4","objects":1108},{"shards":"0,4,5","objects":5355}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":95,"reported_epoch":118,"state":"active+undersized","last_fresh":"2023-09-23T18:32:29.807080+0000","last_change":"2023-09-23T18:30:57.758546+0000","last_active":"2023-09-23T18:32:29.807080+0000","last_peered":"2023-09-23T18:32:29.807080+0000","last_clean":"2023-09-23T18:30:11.303354+0000","last_became_active":"2023-09-23T18:30:57.758546+0000","last_became_peered":"2023-09-23T18:30:57.758546+0000","last_unstale":"2023-09-23T18:32:29.807080+0000","last_undegraded":"2023-09-23T18:32:29.807080+0000","last_fullsized":"2023-09-23T18:30:57.742833+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:09:29.647331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":60,"reported_epoch":118,"state":"active+undersized","last_fresh":"2023-09-23T18:32:29.807655+0000","last_change":"2023-09-23T18:30:57.759447+0000","last_active":"2023-09-23T18:32:29.807655+0000","last_peered":"2023-09-23T18:32:29.807655+0000","last_clean":"2023-09-23T18:27:33.569231+0000","last_became_active":"2023-09-23T18:30:57.759447+0000","last_became_peered":"2023-09-23T18:30:57.759447+0000","last_unstale":"2023-09-23T18:32:29.807655+0000","last_undegraded":"2023-09-23T18:32:29.807655+0000","last_fullsized":"2023-09-23T18:30:57.743500+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:37.916625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"77'130","reported_seq":561,"reported_epoch":118,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:32:29.829302+0000","last_change":"2023-09-23T18:30:57.754736+0000","last_active":"2023-09-23T18:32:29.829302+0000","last_peered":"2023-09-23T18:32:29.829302+0000","last_clean":"2023-09-23T18:17:47.121124+0000","last_became_active":"2023-09-23T18:30:57.754736+0000","last_became_peered":"2023-09-23T18:30:57.754736+0000","last_unstale":"2023-09-23T18:32:29.829302+0000","last_undegraded":"2023-09-23T18:30:57.744730+0000","last_fullsized":"2023-09-23T18:30:57.744572+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:31.511177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":772,"num_read_kb":1712,"num_write":126,"num_write_kb":2126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":92,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.806937+0000","last_change":"2023-09-23T18:32:26.793356+0000","last_active":"2023-09-23T18:32:29.806937+0000","last_peered":"2023-09-23T18:32:29.806937+0000","last_clean":"2023-09-23T18:32:29.806937+0000","last_became_active":"2023-09-23T18:32:26.792868+0000","last_became_peered":"2023-09-23T18:32:26.792868+0000","last_unstale":"2023-09-23T18:32:29.806937+0000","last_undegraded":"2023-09-23T18:32:29.806937+0000","last_fullsized":"2023-09-23T18:32:29.806937+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:40:21.193604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":118,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:32:29.833913+0000","last_change":"2023-09-23T18:32:27.816903+0000","last_active":"2023-09-23T18:32:29.833913+0000","last_peered":"2023-09-23T18:32:29.833913+0000","last_clean":"2023-09-23T18:32:29.833913+0000","last_became_active":"2023-09-23T18:32:27.816423+0000","last_became_peered":"2023-09-23T18:32:27.816423+0000","last_unstale":"2023-09-23T18:32:29.833913+0000","last_undegraded":"2023-09-23T18:32:29.833913+0000","last_fullsized":"2023-09-23T18:32:29.833913+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0,5],"avail_no_missing":["4","0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.3","version":"119'6507","reported_seq":8154,"reported_epoch":119,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T18:32:58.212351+0000","last_change":"2023-09-23T18:32:28.079871+0000","last_active":"2023-09-23T18:32:58.212351+0000","last_peered":"2023-09-23T18:32:58.212351+0000","last_clean":"2023-09-23T18:32:25.776318+0000","last_became_active":"2023-09-23T18:32:28.011068+0000","last_became_peered":"2023-09-23T18:32:28.011068+0000","last_unstale":"2023-09-23T18:32:58.212351+0000","last_undegraded":"2023-09-23T18:32:58.212351+0000","last_fullsized":"2023-09-23T18:32:27.796343+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":6507,"log_dups_size":0,"ondisk_log_size":6507,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:03.564903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":426442752,"num_objects":6507,"num_object_clones":0,"num_object_copies":19521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4531,"num_objects_unfound":0,"num_objects_dirty":6507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6507,"num_write_kb":416448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":809,"num_bytes_recovered":53018624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":4531},{"shards":"0,6","objects":1976}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"119'6525","reported_seq":6543,"reported_epoch":119,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:32:58.160250+0000","last_change":"2023-09-23T18:31:31.595287+0000","last_active":"2023-09-23T18:32:58.160250+0000","last_peered":"2023-09-23T18:32:58.160250+0000","last_clean":"2023-09-23T18:31:10.759555+0000","last_became_active":"2023-09-23T18:31:11.792758+0000","last_became_peered":"2023-09-23T18:31:11.792758+0000","last_unstale":"2023-09-23T18:32:58.160250+0000","last_undegraded":"2023-09-23T18:31:31.594926+0000","last_fullsized":"2023-09-23T18:31:11.788267+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":6525,"log_dups_size":0,"ondisk_log_size":6525,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:34:46.104736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":427622400,"num_objects":6525,"num_object_clones":0,"num_object_copies":19575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6525,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6525,"num_write_kb":417600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":6525}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'96","reported_seq":556,"reported_epoch":118,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:32:29.817565+0000","last_change":"2023-09-23T18:32:28.339396+0000","last_active":"2023-09-23T18:32:29.817565+0000","last_peered":"2023-09-23T18:32:29.817565+0000","last_clean":"2023-09-23T18:32:29.817565+0000","last_became_active":"2023-09-23T18:32:27.871434+0000","last_became_peered":"2023-09-23T18:32:27.871434+0000","last_unstale":"2023-09-23T18:32:29.817565+0000","last_undegraded":"2023-09-23T18:32:29.817565+0000","last_fullsized":"2023-09-23T18:32:29.817565+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":96,"log_dups_size":0,"ondisk_log_size":96,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":728,"num_read_kb":1234,"num_write":87,"num_write_kb":338,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0,5],"avail_no_missing":["6","0","5"],"object_location_counts":[{"shards":"0,5,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":85,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.833740+0000","last_change":"2023-09-23T18:26:34.992535+0000","last_active":"2023-09-23T18:32:29.833740+0000","last_peered":"2023-09-23T18:32:29.833740+0000","last_clean":"2023-09-23T18:32:29.833740+0000","last_became_active":"2023-09-23T18:26:34.992177+0000","last_became_peered":"2023-09-23T18:26:34.992177+0000","last_unstale":"2023-09-23T18:32:29.833740+0000","last_undegraded":"2023-09-23T18:32:29.833740+0000","last_fullsized":"2023-09-23T18:32:29.833740+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":75,"reported_epoch":118,"state":"active+undersized","last_fresh":"2023-09-23T18:32:29.811163+0000","last_change":"2023-09-23T18:30:57.750780+0000","last_active":"2023-09-23T18:32:29.811163+0000","last_peered":"2023-09-23T18:32:29.811163+0000","last_clean":"2023-09-23T18:27:33.585799+0000","last_became_active":"2023-09-23T18:30:57.750780+0000","last_became_peered":"2023-09-23T18:30:57.750780+0000","last_unstale":"2023-09-23T18:32:29.811163+0000","last_undegraded":"2023-09-23T18:32:29.811163+0000","last_fullsized":"2023-09-23T18:30:57.747747+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:02.927953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.6","version":"119'6515","reported_seq":6543,"reported_epoch":119,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:32:56.768924+0000","last_change":"2023-09-23T18:32:28.232018+0000","last_active":"2023-09-23T18:32:56.768924+0000","last_peered":"2023-09-23T18:32:56.768924+0000","last_clean":"2023-09-23T18:32:25.754559+0000","last_became_active":"2023-09-23T18:32:28.220390+0000","last_became_peered":"2023-09-23T18:32:28.220390+0000","last_unstale":"2023-09-23T18:32:56.768924+0000","last_undegraded":"2023-09-23T18:32:27.868829+0000","last_fullsized":"2023-09-23T18:32:27.798864+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":6515,"log_dups_size":0,"ondisk_log_size":6515,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:07.398730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":426967040,"num_objects":6515,"num_object_clones":0,"num_object_copies":19545,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6515,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6515,"num_write_kb":416960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":5398},{"shards":"2,7","objects":1117}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"77'112","reported_seq":555,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.810780+0000","last_change":"2023-09-23T18:28:03.297540+0000","last_active":"2023-09-23T18:32:29.810780+0000","last_peered":"2023-09-23T18:32:29.810780+0000","last_clean":"2023-09-23T18:32:29.810780+0000","last_became_active":"2023-09-23T18:28:03.296649+0000","last_became_peered":"2023-09-23T18:28:03.296649+0000","last_unstale":"2023-09-23T18:32:29.810780+0000","last_undegraded":"2023-09-23T18:32:29.810780+0000","last_fullsized":"2023-09-23T18:32:29.810780+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:32.531775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":774,"num_read_kb":596,"num_write":102,"num_write_kb":52,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":55,"reported_epoch":118,"state":"active+undersized","last_fresh":"2023-09-23T18:32:29.810943+0000","last_change":"2023-09-23T18:30:57.760762+0000","last_active":"2023-09-23T18:32:29.810943+0000","last_peered":"2023-09-23T18:32:29.810943+0000","last_clean":"2023-09-23T18:27:04.256214+0000","last_became_active":"2023-09-23T18:30:57.760762+0000","last_became_peered":"2023-09-23T18:30:57.760762+0000","last_unstale":"2023-09-23T18:32:29.810943+0000","last_undegraded":"2023-09-23T18:32:29.810943+0000","last_fullsized":"2023-09-23T18:30:57.747320+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:22.564483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":78,"reported_epoch":118,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:32:29.822585+0000","last_change":"2023-09-23T18:32:27.839980+0000","last_active":"2023-09-23T18:32:29.822585+0000","last_peered":"2023-09-23T18:32:29.822585+0000","last_clean":"2023-09-23T18:32:29.822585+0000","last_became_active":"2023-09-23T18:32:27.839511+0000","last_became_peered":"2023-09-23T18:32:27.839511+0000","last_unstale":"2023-09-23T18:32:29.822585+0000","last_undegraded":"2023-09-23T18:32:29.822585+0000","last_fullsized":"2023-09-23T18:32:29.822585+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:03.298537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7,5],"avail_no_missing":["0","5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"77'147","reported_seq":504,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.851530+0000","last_change":"2023-09-23T18:32:29.845404+0000","last_active":"2023-09-23T18:32:29.851530+0000","last_peered":"2023-09-23T18:32:29.851530+0000","last_clean":"2023-09-23T18:32:29.851530+0000","last_became_active":"2023-09-23T18:32:29.844545+0000","last_became_peered":"2023-09-23T18:32:29.844545+0000","last_unstale":"2023-09-23T18:32:29.851530+0000","last_undegraded":"2023-09-23T18:32:29.851530+0000","last_fullsized":"2023-09-23T18:32:29.851530+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:44.834482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":624,"num_read_kb":1868,"num_write":143,"num_write_kb":5368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":321,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.817093+0000","last_change":"2023-09-23T18:26:34.999402+0000","last_active":"2023-09-23T18:32:29.817093+0000","last_peered":"2023-09-23T18:32:29.817093+0000","last_clean":"2023-09-23T18:32:29.817093+0000","last_became_active":"2023-09-23T18:26:34.998828+0000","last_became_peered":"2023-09-23T18:26:34.998828+0000","last_unstale":"2023-09-23T18:32:29.817093+0000","last_undegraded":"2023-09-23T18:32:29.817093+0000","last_fullsized":"2023-09-23T18:32:29.817093+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_clean_scrub_stamp":"2023-09-23T18:08:35.701374+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":88,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.807627+0000","last_change":"2023-09-23T18:32:26.800920+0000","last_active":"2023-09-23T18:32:29.807627+0000","last_peered":"2023-09-23T18:32:29.807627+0000","last_clean":"2023-09-23T18:32:29.807627+0000","last_became_active":"2023-09-23T18:32:26.800528+0000","last_became_peered":"2023-09-23T18:32:26.800528+0000","last_unstale":"2023-09-23T18:32:29.807627+0000","last_undegraded":"2023-09-23T18:32:29.807627+0000","last_fullsized":"2023-09-23T18:32:29.807627+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:20:50.939219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":69,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.816896+0000","last_change":"2023-09-23T18:15:08.921629+0000","last_active":"2023-09-23T18:32:29.816896+0000","last_peered":"2023-09-23T18:32:29.816896+0000","last_clean":"2023-09-23T18:32:29.816896+0000","last_became_active":"2023-09-23T18:15:08.921283+0000","last_became_peered":"2023-09-23T18:15:08.921283+0000","last_unstale":"2023-09-23T18:32:29.816896+0000","last_undegraded":"2023-09-23T18:32:29.816896+0000","last_fullsized":"2023-09-23T18:32:29.816896+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"119'6398","reported_seq":6425,"reported_epoch":119,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:32:55.505642+0000","last_change":"2023-09-23T18:32:28.109050+0000","last_active":"2023-09-23T18:32:55.505642+0000","last_peered":"2023-09-23T18:32:55.505642+0000","last_clean":"2023-09-23T18:32:25.776893+0000","last_became_active":"2023-09-23T18:32:28.091250+0000","last_became_peered":"2023-09-23T18:32:28.091250+0000","last_unstale":"2023-09-23T18:32:55.505642+0000","last_undegraded":"2023-09-23T18:32:27.847048+0000","last_fullsized":"2023-09-23T18:32:27.801418+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":6398,"log_dups_size":0,"ondisk_log_size":6398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":419299328,"num_objects":6398,"num_object_clones":0,"num_object_copies":19194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5355,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6398,"num_write_kb":409472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2],"avail_no_missing":["7","2","5"],"object_location_counts":[{"shards":"2,5,7","objects":5355},{"shards":"2,7","objects":1043}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"119'6526","reported_seq":6550,"reported_epoch":119,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:32:58.184434+0000","last_change":"2023-09-23T18:32:28.121228+0000","last_active":"2023-09-23T18:32:58.184434+0000","last_peered":"2023-09-23T18:32:58.184434+0000","last_clean":"2023-09-23T18:31:10.759555+0000","last_became_active":"2023-09-23T18:32:28.096141+0000","last_became_peered":"2023-09-23T18:32:28.096141+0000","last_unstale":"2023-09-23T18:32:58.184434+0000","last_undegraded":"2023-09-23T18:32:27.835816+0000","last_fullsized":"2023-09-23T18:32:27.795136+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":6526,"log_dups_size":0,"ondisk_log_size":6526,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:44:10.405124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":427687936,"num_objects":6526,"num_object_clones":0,"num_object_copies":19578,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5364,"num_objects_misplaced":6526,"num_objects_unfound":0,"num_objects_dirty":6526,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6526,"num_write_kb":417664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":1162}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'120","reported_seq":571,"reported_epoch":118,"state":"active+undersized","last_fresh":"2023-09-23T18:32:29.817174+0000","last_change":"2023-09-23T18:30:57.756612+0000","last_active":"2023-09-23T18:32:29.817174+0000","last_peered":"2023-09-23T18:32:29.817174+0000","last_clean":"2023-09-23T18:28:03.286680+0000","last_became_active":"2023-09-23T18:30:57.756612+0000","last_became_peered":"2023-09-23T18:30:57.756612+0000","last_unstale":"2023-09-23T18:32:29.817174+0000","last_undegraded":"2023-09-23T18:32:29.817174+0000","last_fullsized":"2023-09-23T18:30:57.751463+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":120,"log_dups_size":0,"ondisk_log_size":120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":809,"num_read_kb":2749,"num_write":98,"num_write_kb":2087,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":69,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-09-23T18:32:35.443459+0000","last_change":"2023-09-23T18:28:03.299508+0000","last_active":"2023-09-23T18:32:35.443459+0000","last_peered":"2023-09-23T18:32:35.443459+0000","last_clean":"2023-09-23T18:32:35.443459+0000","last_became_active":"2023-09-23T18:28:03.298682+0000","last_became_peered":"2023-09-23T18:28:03.298682+0000","last_unstale":"2023-09-23T18:32:35.443459+0000","last_undegraded":"2023-09-23T18:32:35.443459+0000","last_fullsized":"2023-09-23T18:32:35.443459+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:08.398302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":90,"reported_epoch":118,"state":"active+undersized","last_fresh":"2023-09-23T18:32:29.821515+0000","last_change":"2023-09-23T18:30:57.748793+0000","last_active":"2023-09-23T18:32:29.821515+0000","last_peered":"2023-09-23T18:32:29.821515+0000","last_clean":"2023-09-23T18:30:53.021930+0000","last_became_active":"2023-09-23T18:30:57.748793+0000","last_became_peered":"2023-09-23T18:30:57.748793+0000","last_unstale":"2023-09-23T18:32:29.821515+0000","last_undegraded":"2023-09-23T18:32:29.821515+0000","last_fullsized":"2023-09-23T18:30:57.745371+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:43.511022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'78","reported_seq":355,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.818002+0000","last_change":"2023-09-23T18:26:34.992345+0000","last_active":"2023-09-23T18:32:29.818002+0000","last_peered":"2023-09-23T18:32:29.818002+0000","last_clean":"2023-09-23T18:32:29.818002+0000","last_became_active":"2023-09-23T18:26:34.991774+0000","last_became_peered":"2023-09-23T18:26:34.991774+0000","last_unstale":"2023-09-23T18:32:29.818002+0000","last_undegraded":"2023-09-23T18:32:29.818002+0000","last_fullsized":"2023-09-23T18:32:29.818002+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":78,"log_dups_size":0,"ondisk_log_size":78,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":428,"num_read_kb":1125,"num_write":73,"num_write_kb":431,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":65,"reported_epoch":118,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:32:29.811285+0000","last_change":"2023-09-23T18:32:27.812251+0000","last_active":"2023-09-23T18:32:29.811285+0000","last_peered":"2023-09-23T18:32:29.811285+0000","last_clean":"2023-09-23T18:32:29.811285+0000","last_became_active":"2023-09-23T18:32:27.811530+0000","last_became_peered":"2023-09-23T18:32:27.811530+0000","last_unstale":"2023-09-23T18:32:29.811285+0000","last_undegraded":"2023-09-23T18:32:29.811285+0000","last_fullsized":"2023-09-23T18:32:29.811285+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:16.046570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0,5],"avail_no_missing":["2","0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":65,"reported_epoch":119,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:32:35.451105+0000","last_change":"2023-09-23T18:32:27.811131+0000","last_active":"2023-09-23T18:32:35.451105+0000","last_peered":"2023-09-23T18:32:35.451105+0000","last_clean":"2023-09-23T18:32:35.451105+0000","last_became_active":"2023-09-23T18:32:27.809942+0000","last_became_peered":"2023-09-23T18:32:27.809942+0000","last_unstale":"2023-09-23T18:32:35.451105+0000","last_undegraded":"2023-09-23T18:32:35.451105+0000","last_fullsized":"2023-09-23T18:32:35.451105+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:06.930015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7,5],"avail_no_missing":["0","5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"119'6422","reported_seq":6449,"reported_epoch":119,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:32:55.529475+0000","last_change":"2023-09-23T18:32:28.155678+0000","last_active":"2023-09-23T18:32:55.529475+0000","last_peered":"2023-09-23T18:32:55.529475+0000","last_clean":"2023-09-23T18:32:25.776264+0000","last_became_active":"2023-09-23T18:32:28.136844+0000","last_became_peered":"2023-09-23T18:32:28.136844+0000","last_unstale":"2023-09-23T18:32:55.529475+0000","last_undegraded":"2023-09-23T18:32:27.856830+0000","last_fullsized":"2023-09-23T18:32:27.810366+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":6422,"log_dups_size":0,"ondisk_log_size":6422,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":420872192,"num_objects":6422,"num_object_clones":0,"num_object_copies":19266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5380,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6422,"num_write_kb":411008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0],"avail_no_missing":["6","0","5"],"object_location_counts":[{"shards":"0,5,6","objects":5380},{"shards":"0,6","objects":1042}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":87,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.807563+0000","last_change":"2023-09-23T18:28:03.298805+0000","last_active":"2023-09-23T18:32:29.807563+0000","last_peered":"2023-09-23T18:32:29.807563+0000","last_clean":"2023-09-23T18:32:29.807563+0000","last_became_active":"2023-09-23T18:28:03.298437+0000","last_became_peered":"2023-09-23T18:28:03.298437+0000","last_unstale":"2023-09-23T18:32:29.807563+0000","last_undegraded":"2023-09-23T18:32:29.807563+0000","last_fullsized":"2023-09-23T18:32:29.807563+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:27.831669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'181","reported_seq":590,"reported_epoch":118,"state":"active+undersized","last_fresh":"2023-09-23T18:32:29.816837+0000","last_change":"2023-09-23T18:30:57.764331+0000","last_active":"2023-09-23T18:32:29.816837+0000","last_peered":"2023-09-23T18:32:29.816837+0000","last_clean":"2023-09-23T18:17:46.151116+0000","last_became_active":"2023-09-23T18:30:57.764331+0000","last_became_peered":"2023-09-23T18:30:57.764331+0000","last_unstale":"2023-09-23T18:32:29.816837+0000","last_undegraded":"2023-09-23T18:32:29.816837+0000","last_fullsized":"2023-09-23T18:30:57.758120+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":181,"log_dups_size":0,"ondisk_log_size":181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":909,"num_read_kb":734,"num_write":148,"num_write_kb":75,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":84,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.806985+0000","last_change":"2023-09-23T18:28:03.292295+0000","last_active":"2023-09-23T18:32:29.806985+0000","last_peered":"2023-09-23T18:32:29.806985+0000","last_clean":"2023-09-23T18:32:29.806985+0000","last_became_active":"2023-09-23T18:28:03.291902+0000","last_became_peered":"2023-09-23T18:28:03.291902+0000","last_unstale":"2023-09-23T18:32:29.806985+0000","last_undegraded":"2023-09-23T18:32:29.806985+0000","last_fullsized":"2023-09-23T18:32:29.806985+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:38.415256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"119'6385","reported_seq":6405,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-09-23T18:32:55.553231+0000","last_change":"2023-09-23T18:31:11.796383+0000","last_active":"2023-09-23T18:32:55.553231+0000","last_peered":"2023-09-23T18:32:55.553231+0000","last_clean":"2023-09-23T18:32:55.553231+0000","last_became_active":"2023-09-23T18:31:11.796024+0000","last_became_peered":"2023-09-23T18:31:11.796024+0000","last_unstale":"2023-09-23T18:32:55.553231+0000","last_undegraded":"2023-09-23T18:32:55.553231+0000","last_fullsized":"2023-09-23T18:32:55.553231+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":6385,"log_dups_size":0,"ondisk_log_size":6385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":418447360,"num_objects":6385,"num_object_clones":0,"num_object_copies":19155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6385,"num_write_kb":408640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'112","reported_seq":609,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.817057+0000","last_change":"2023-09-23T18:27:33.587478+0000","last_active":"2023-09-23T18:32:29.817057+0000","last_peered":"2023-09-23T18:32:29.817057+0000","last_clean":"2023-09-23T18:32:29.817057+0000","last_became_active":"2023-09-23T18:27:33.586958+0000","last_became_peered":"2023-09-23T18:27:33.586958+0000","last_unstale":"2023-09-23T18:32:29.817057+0000","last_undegraded":"2023-09-23T18:32:29.817057+0000","last_fullsized":"2023-09-23T18:32:29.817057+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":893,"num_read_kb":3172,"num_write":69,"num_write_kb":2237,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":78,"reported_epoch":118,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:32:29.821375+0000","last_change":"2023-09-23T18:30:57.759899+0000","last_active":"2023-09-23T18:32:29.821375+0000","last_peered":"2023-09-23T18:32:29.821375+0000","last_clean":"2023-09-23T18:30:53.021857+0000","last_became_active":"2023-09-23T18:30:57.759899+0000","last_became_peered":"2023-09-23T18:30:57.759899+0000","last_unstale":"2023-09-23T18:32:29.821375+0000","last_undegraded":"2023-09-23T18:30:57.745402+0000","last_fullsized":"2023-09-23T18:30:57.745244+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:26.821613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":78,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.827294+0000","last_change":"2023-09-23T18:32:26.805472+0000","last_active":"2023-09-23T18:32:29.827294+0000","last_peered":"2023-09-23T18:32:29.827294+0000","last_clean":"2023-09-23T18:32:29.827294+0000","last_became_active":"2023-09-23T18:32:26.805078+0000","last_became_peered":"2023-09-23T18:32:26.805078+0000","last_unstale":"2023-09-23T18:32:29.827294+0000","last_undegraded":"2023-09-23T18:32:29.827294+0000","last_fullsized":"2023-09-23T18:32:29.827294+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"119'6400","reported_seq":6418,"reported_epoch":119,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:32:56.233538+0000","last_change":"2023-09-23T18:31:31.601373+0000","last_active":"2023-09-23T18:32:56.233538+0000","last_peered":"2023-09-23T18:32:56.233538+0000","last_clean":"2023-09-23T18:31:10.759555+0000","last_became_active":"2023-09-23T18:31:11.795405+0000","last_became_peered":"2023-09-23T18:31:11.795405+0000","last_unstale":"2023-09-23T18:32:56.233538+0000","last_undegraded":"2023-09-23T18:31:31.594507+0000","last_fullsized":"2023-09-23T18:31:11.782086+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":6400,"log_dups_size":0,"ondisk_log_size":6400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:30.717583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":419430400,"num_objects":6400,"num_object_clones":0,"num_object_copies":19200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6400,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6400,"num_write_kb":409600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":6400}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":57,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.828422+0000","last_change":"2023-09-23T18:26:34.994176+0000","last_active":"2023-09-23T18:32:29.828422+0000","last_peered":"2023-09-23T18:32:29.828422+0000","last_clean":"2023-09-23T18:32:29.828422+0000","last_became_active":"2023-09-23T18:26:34.993829+0000","last_became_peered":"2023-09-23T18:26:34.993829+0000","last_unstale":"2023-09-23T18:32:29.828422+0000","last_undegraded":"2023-09-23T18:32:29.828422+0000","last_fullsized":"2023-09-23T18:32:29.828422+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:27.111857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":114,"reported_epoch":118,"state":"active+undersized","last_fresh":"2023-09-23T18:32:31.662422+0000","last_change":"2023-09-23T18:30:57.750699+0000","last_active":"2023-09-23T18:32:31.662422+0000","last_peered":"2023-09-23T18:32:31.662422+0000","last_clean":"2023-09-23T18:30:01.592430+0000","last_became_active":"2023-09-23T18:30:57.750699+0000","last_became_peered":"2023-09-23T18:30:57.750699+0000","last_unstale":"2023-09-23T18:32:31.662422+0000","last_undegraded":"2023-09-23T18:32:31.662422+0000","last_fullsized":"2023-09-23T18:30:57.747336+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:59.891827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":32,"reported_epoch":118,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:32:29.833721+0000","last_change":"2023-09-23T18:32:27.838933+0000","last_active":"2023-09-23T18:32:29.833721+0000","last_peered":"2023-09-23T18:32:29.833721+0000","last_clean":"2023-09-23T18:32:29.833721+0000","last_became_active":"2023-09-23T18:32:27.818662+0000","last_became_peered":"2023-09-23T18:32:27.818662+0000","last_unstale":"2023-09-23T18:32:29.833721+0000","last_undegraded":"2023-09-23T18:32:29.833721+0000","last_fullsized":"2023-09-23T18:32:29.833721+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6,5],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":105,"reported_epoch":118,"state":"active+undersized","last_fresh":"2023-09-23T18:32:29.833861+0000","last_change":"2023-09-23T18:30:57.765166+0000","last_active":"2023-09-23T18:32:29.833861+0000","last_peered":"2023-09-23T18:32:29.833861+0000","last_clean":"2023-09-23T18:28:03.301070+0000","last_became_active":"2023-09-23T18:30:57.765166+0000","last_became_peered":"2023-09-23T18:30:57.765166+0000","last_unstale":"2023-09-23T18:32:29.833861+0000","last_undegraded":"2023-09-23T18:32:29.833861+0000","last_fullsized":"2023-09-23T18:30:57.757210+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":28,"reported_epoch":118,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:32:29.833970+0000","last_change":"2023-09-23T18:32:27.817095+0000","last_active":"2023-09-23T18:32:29.833970+0000","last_peered":"2023-09-23T18:32:29.833970+0000","last_clean":"2023-09-23T18:32:29.833970+0000","last_became_active":"2023-09-23T18:32:27.816422+0000","last_became_peered":"2023-09-23T18:32:27.816422+0000","last_unstale":"2023-09-23T18:32:29.833970+0000","last_undegraded":"2023-09-23T18:32:29.833970+0000","last_fullsized":"2023-09-23T18:32:29.833970+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6,5],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":99,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.833711+0000","last_change":"2023-09-23T18:28:03.302261+0000","last_active":"2023-09-23T18:32:29.833711+0000","last_peered":"2023-09-23T18:32:29.833711+0000","last_clean":"2023-09-23T18:32:29.833711+0000","last_became_active":"2023-09-23T18:28:03.301911+0000","last_became_peered":"2023-09-23T18:28:03.301911+0000","last_unstale":"2023-09-23T18:32:29.833711+0000","last_undegraded":"2023-09-23T18:32:29.833711+0000","last_fullsized":"2023-09-23T18:32:29.833711+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":94,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T18:32:29.807194+0000","last_change":"2023-09-23T18:28:03.296850+0000","last_active":"2023-09-23T18:32:29.807194+0000","last_peered":"2023-09-23T18:32:29.807194+0000","last_clean":"2023-09-23T18:32:29.807194+0000","last_became_active":"2023-09-23T18:28:03.295842+0000","last_became_peered":"2023-09-23T18:28:03.295842+0000","last_unstale":"2023-09-23T18:32:29.807194+0000","last_undegraded":"2023-09-23T18:32:29.807194+0000","last_fullsized":"2023-09-23T18:32:29.807194+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:53.048579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":107,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-09-23T18:32:35.458490+0000","last_change":"2023-09-23T18:28:03.299520+0000","last_active":"2023-09-23T18:32:35.458490+0000","last_peered":"2023-09-23T18:32:35.458490+0000","last_clean":"2023-09-23T18:32:35.458490+0000","last_became_active":"2023-09-23T18:28:03.299158+0000","last_became_peered":"2023-09-23T18:28:03.299158+0000","last_unstale":"2023-09-23T18:32:35.458490+0000","last_undegraded":"2023-09-23T18:32:35.458490+0000","last_fullsized":"2023-09-23T18:32:35.458490+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:26.247064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":88,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-09-23T18:32:35.444609+0000","last_change":"2023-09-23T18:32:26.805676+0000","last_active":"2023-09-23T18:32:35.444609+0000","last_peered":"2023-09-23T18:32:35.444609+0000","last_clean":"2023-09-23T18:32:35.444609+0000","last_became_active":"2023-09-23T18:32:26.797129+0000","last_became_peered":"2023-09-23T18:32:26.797129+0000","last_unstale":"2023-09-23T18:32:35.444609+0000","last_undegraded":"2023-09-23T18:32:35.444609+0000","last_fullsized":"2023-09-23T18:32:35.444609+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:07.414550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":88,"reported_epoch":118,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:32:29.826701+0000","last_change":"2023-09-23T18:32:27.816921+0000","last_active":"2023-09-23T18:32:29.826701+0000","last_peered":"2023-09-23T18:32:29.826701+0000","last_clean":"2023-09-23T18:32:29.826701+0000","last_became_active":"2023-09-23T18:32:27.815619+0000","last_became_peered":"2023-09-23T18:32:27.815619+0000","last_unstale":"2023-09-23T18:32:29.826701+0000","last_undegraded":"2023-09-23T18:32:29.826701+0000","last_fullsized":"2023-09-23T18:32:29.826701+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6,5],"avail_no_missing":["4","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":6763184128,"num_objects":103198,"num_object_clones":0,"num_object_copies":309594,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":63416,"num_objects_misplaced":15753,"num_objects_unfound":0,"num_objects_dirty":103198,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":103198,"num_write_kb":6604672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1484,"num_bytes_recovered":97255424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18336448512,"data_stored":18336448512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":103198,"ondisk_log_size":103198,"up":43,"acting":35,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":77,"acting":82,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":624,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26886,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":83,"acting":88,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5937,"num_read_kb":13190,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5621,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":20,"acting":21,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430461,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2480316,"kb_used_data":2170804,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91264836,"statfs":{"total":95995035648,"available":93455192064,"internally_reserved":0,"allocated":2222903296,"data_stored":2222370762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:32:50 2023","interfaces":[{"interface":"back","average":{"1min":3.9350000000000001,"5min":1.917,"15min":1.2589999999999999},"min":{"1min":0.27100000000000002,"5min":0.252,"15min":0.252},"max":{"1min":57.049999999999997,"5min":57.049999999999997,"15min":57.049999999999997},"last":1.8919999999999999},{"interface":"front","average":{"1min":3.9060000000000001,"5min":1.857,"15min":1.2649999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":56.773000000000003,"5min":56.773000000000003,"15min":56.773000000000003},"last":0.39000000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:30:07 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.91100000000000003,"15min":0.91500000000000004},"min":{"1min":0.58599999999999997,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.4039999999999999,"5min":1.4039999999999999,"15min":1.4039999999999999},"last":1.1819999999999999},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.97499999999999998,"15min":0.96699999999999997},"min":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.254,"5min":1.306,"15min":1.306},"last":1.3340000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:32:42 2023","interfaces":[{"interface":"back","average":{"1min":3.3660000000000001,"5min":1.4430000000000001,"15min":1.1319999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":40.091999999999999,"5min":40.091999999999999,"15min":40.091999999999999},"last":0.34799999999999998},{"interface":"front","average":{"1min":2.8620000000000001,"5min":1.3580000000000001,"15min":1.0920000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":38.261000000000003,"5min":38.261000000000003,"15min":38.261000000000003},"last":1.8520000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:32:05 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.94399999999999995,"15min":0.97999999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.0750000000000002,"5min":2.0750000000000002,"15min":2.0750000000000002},"last":1.496},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.93999999999999995,"15min":0.94799999999999995},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.7229999999999999,"5min":3.7229999999999999,"15min":3.7229999999999999},"last":2.7469999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:31:54 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":0.96299999999999997,"15min":0.89900000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":4.0170000000000003,"5min":4.0170000000000003,"15min":4.0170000000000003},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.1759999999999999,"5min":1.0589999999999999,"15min":1.0309999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":3.8540000000000001,"5min":3.8540000000000001,"15min":3.8540000000000001},"last":3.8929999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:31:54 2023","interfaces":[{"interface":"back","average":{"1min":1.349,"5min":1,"15min":0.97199999999999998},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":4.0780000000000003,"5min":4.0780000000000003,"15min":4.0780000000000003},"last":2.847},{"interface":"front","average":{"1min":0.997,"5min":1.05,"15min":1.016},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.40899999999999997},"max":{"1min":2.8420000000000001,"5min":2.8420000000000001,"15min":2.8420000000000001},"last":0.45300000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:31:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.1040000000000001,"15min":1.0529999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":3.254,"5min":3.254,"15min":3.254},"last":0.42499999999999999},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.117,"15min":1.125},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":2.093,"5min":2.093,"15min":2.093},"last":1.4470000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626693,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1940852,"kb_used_data":1643628,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91804300,"statfs":{"total":95995035648,"available":94007603200,"internally_reserved":0,"allocated":1683075072,"data_stored":1682550842,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:32:50 2023","interfaces":[{"interface":"back","average":{"1min":10.121,"5min":2.7599999999999998,"15min":1.5249999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":182.655,"5min":182.655,"15min":182.655},"last":2.149},{"interface":"front","average":{"1min":10.241,"5min":2.8730000000000002,"15min":1.655},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":182.58799999999999,"5min":182.58799999999999,"15min":182.58799999999999},"last":2.2170000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:30:10 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90400000000000003},"min":{"1min":0.70699999999999996,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.1140000000000001,"5min":1.2210000000000001,"15min":1.2210000000000001},"last":0.91900000000000004},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.93100000000000005,"15min":0.92500000000000004},"min":{"1min":0.61599999999999999,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.403,"5min":1.403,"15min":1.403},"last":0.77200000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:32:42 2023","interfaces":[{"interface":"back","average":{"1min":3.8980000000000001,"5min":1.552,"15min":1.1890000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":51.728000000000002,"5min":51.728000000000002,"15min":51.728000000000002},"last":0.48599999999999999},{"interface":"front","average":{"1min":4.016,"5min":1.5429999999999999,"15min":1.1539999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":51.820999999999998,"5min":51.820999999999998,"15min":51.820999999999998},"last":0.68300000000000005}]},{"osd":3,"last update":"Sat Sep 23 18:32:05 2023","interfaces":[{"interface":"back","average":{"1min":1.772,"5min":1.151,"15min":1.0389999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":21.282,"5min":21.282,"15min":21.282},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.8400000000000001,"5min":1.1479999999999999,"15min":1.0409999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":21.260000000000002,"5min":21.260000000000002,"15min":21.260000000000002},"last":0.72599999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:31:58 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.94199999999999995,"15min":0.92300000000000004},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.1239999999999997,"5min":4.1239999999999997,"15min":4.1239999999999997},"last":2.1850000000000001},{"interface":"front","average":{"1min":0.93899999999999995,"5min":1.014,"15min":0.97799999999999998},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":2.7879999999999998,"5min":2.7879999999999998,"15min":2.7879999999999998},"last":0.90100000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:31:58 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":1.036,"15min":1.0169999999999999},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.36699999999999999},"max":{"1min":4.2359999999999998,"5min":4.2359999999999998,"15min":4.2359999999999998},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.998,"5min":1.0800000000000001,"15min":1.056},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":2.8540000000000001,"5min":2.8540000000000001,"15min":2.8540000000000001},"last":0.44600000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:32:42 2023","interfaces":[{"interface":"back","average":{"1min":1.3340000000000001,"5min":1.0569999999999999,"15min":1.0740000000000001},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":4.2699999999999996,"5min":4.2699999999999996,"15min":4.2699999999999996},"last":0.75800000000000001},{"interface":"front","average":{"1min":2.1230000000000002,"5min":1.3600000000000001,"15min":1.198},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":9.1129999999999995,"5min":9.1129999999999995,"15min":9.1129999999999995},"last":0.64300000000000002}]}]},{"osd":1,"up_from":93,"seq":399431958576,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32648,"kb_used_data":1088,"kb_used_omap":7,"kb_used_meta":31544,"kb_avail":93712504,"statfs":{"total":95995035648,"available":95961604096,"internally_reserved":0,"allocated":1114112,"data_stored":586888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":32301838},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:30:13 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.66100000000000003,"15min":0.67400000000000004},"min":{"1min":0.44500000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.216,"5min":1.216,"15min":1.216},"last":0.77500000000000002},{"interface":"front","average":{"1min":0.748,"5min":0.72399999999999998,"15min":0.73499999999999999},"min":{"1min":0.46800000000000003,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.252,"5min":1.252,"15min":1.252},"last":0.73899999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:30:43 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.622,"15min":0.60599999999999998},"min":{"1min":0.50800000000000001,"5min":0.315,"15min":0.315},"max":{"1min":1.0720000000000001,"5min":1.1000000000000001,"15min":1.1000000000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.84499999999999997,"5min":0.73899999999999999,"15min":0.72799999999999998},"min":{"1min":0.68799999999999994,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.151,"5min":1.151,"15min":1.151},"last":1.1100000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:30:03 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.745,"15min":0.75},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.179,"5min":1.179,"15min":1.179},"last":0.95599999999999996},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.70099999999999996,"15min":0.71499999999999997},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.88,"5min":1.294,"15min":1.294},"last":0.69599999999999995}]},{"osd":4,"last update":"Sat Sep 23 18:30:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":0.94599999999999995,"15min":0.92800000000000005},"min":{"1min":0.70899999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.89000000000000001},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.97999999999999998,"15min":1.0249999999999999},"min":{"1min":0.67800000000000005,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.0269999999999999,"5min":1.4950000000000001,"15min":1.4950000000000001},"last":0.90700000000000003}]},{"osd":5,"last update":"Sat Sep 23 18:30:13 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.96099999999999997,"15min":0.99199999999999999},"min":{"1min":0.624,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.2170000000000001,"5min":1.5149999999999999,"15min":1.5149999999999999},"last":0.875},{"interface":"front","average":{"1min":1.012,"5min":0.98299999999999998,"15min":0.98599999999999999},"min":{"1min":0.73099999999999998,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.296,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":1.0449999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:30:13 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.99199999999999999,"15min":1.022},"min":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.2230000000000001,"5min":1.6100000000000001,"15min":1.6100000000000001},"last":0.98899999999999999},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.0589999999999999,"15min":1.0720000000000001},"min":{"1min":0.68899999999999995,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.3839999999999999,"5min":1.4419999999999999,"15min":1.4419999999999999},"last":1.169}]},{"osd":7,"last update":"Sat Sep 23 18:30:13 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.0860000000000001,"15min":1.1060000000000001},"min":{"1min":0.70499999999999996,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.3580000000000001,"5min":1.55,"15min":1.55},"last":1.2350000000000001},{"interface":"front","average":{"1min":1.006,"5min":1.0840000000000001,"15min":1.1240000000000001},"min":{"1min":0.79600000000000004,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.268,"5min":1.575,"15min":1.575},"last":1.014}]}]},{"osd":0,"up_from":88,"seq":377957122127,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3645392,"kb_used_data":3497224,"kb_used_omap":8,"kb_used_meta":148151,"kb_avail":90099760,"statfs":{"total":95995035648,"available":92262154240,"internally_reserved":0,"allocated":3581157376,"data_stored":3580550988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":151707518},"hb_peers":[2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:30:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.63600000000000001,"15min":0.64000000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.77500000000000002,"5min":1.2170000000000001,"15min":1.2170000000000001},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.77900000000000003,"15min":0.79400000000000004},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.97899999999999998,"5min":1.3480000000000001,"15min":1.3480000000000001},"last":0.91300000000000003}]},{"osd":2,"last update":"Sat Sep 23 18:32:38 2023","interfaces":[{"interface":"back","average":{"1min":1.478,"5min":0.88500000000000001,"15min":0.85099999999999998},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":10.644,"5min":10.644,"15min":10.644},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.72799999999999998,"15min":0.70199999999999996},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":4.968,"5min":4.968,"15min":4.968},"last":5.6980000000000004}]},{"osd":3,"last update":"Sat Sep 23 18:32:05 2023","interfaces":[{"interface":"back","average":{"1min":2.194,"5min":1.046,"15min":0.876},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":26.513999999999999,"5min":26.513999999999999,"15min":26.513999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":2.052,"5min":1.0620000000000001,"15min":0.94199999999999995},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":26.401,"5min":26.401,"15min":26.401},"last":5.75}]},{"osd":4,"last update":"Sat Sep 23 18:32:46 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":1.044,"15min":0.998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.1479999999999997,"5min":6.1479999999999997,"15min":6.1479999999999997},"last":0.63500000000000001},{"interface":"front","average":{"1min":1.6599999999999999,"5min":1.1539999999999999,"15min":1.0660000000000001},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":4.4749999999999996,"5min":4.4749999999999996,"15min":4.4749999999999996},"last":0.39600000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:32:46 2023","interfaces":[{"interface":"back","average":{"1min":1.3959999999999999,"5min":1.0780000000000001,"15min":1.044},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":4.0039999999999996,"5min":4.0039999999999996,"15min":4.0039999999999996},"last":0.498},{"interface":"front","average":{"1min":1.389,"5min":1.097,"15min":1.0629999999999999},"min":{"1min":0.38100000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":4.1479999999999997,"5min":4.1479999999999997,"15min":4.1479999999999997},"last":5.8899999999999997}]},{"osd":6,"last update":"Sat Sep 23 18:32:46 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":1.1379999999999999,"15min":1.0980000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":4.0220000000000002,"5min":4.0220000000000002,"15min":4.0220000000000002},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.2350000000000001,"5min":1.05,"15min":1.052},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":4.093,"5min":4.093,"15min":4.093},"last":5.8550000000000004}]},{"osd":7,"last update":"Sat Sep 23 18:32:46 2023","interfaces":[{"interface":"back","average":{"1min":1.351,"5min":1.0609999999999999,"15min":1.0629999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":5.6849999999999996,"5min":5.6849999999999996,"15min":5.6849999999999996},"last":5.819},{"interface":"front","average":{"1min":1.151,"5min":1.048,"15min":1.0780000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":6.3129999999999997,"5min":6.3129999999999997,"15min":6.3129999999999997},"last":0.68200000000000005}]}]},{"osd":2,"up_from":98,"seq":420906795075,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2782904,"kb_used_data":2679088,"kb_used_omap":11,"kb_used_meta":103796,"kb_avail":90962248,"statfs":{"total":95995035648,"available":93145341952,"internally_reserved":0,"allocated":2743386112,"data_stored":2742801442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11949,"internal_metadata":106287443},"hb_peers":[0,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:32:38 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":0.76900000000000002,"15min":0.71099999999999997},"min":{"1min":0.25900000000000001,"5min":0.214,"15min":0.214},"max":{"1min":7.9569999999999999,"5min":7.9569999999999999,"15min":7.9569999999999999},"last":0.29999999999999999},{"interface":"front","average":{"1min":2.1259999999999999,"5min":0.96199999999999997,"15min":0.77700000000000002},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":6.649,"5min":6.649,"15min":6.649},"last":0.22900000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:30:36 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":0.67100000000000004,"15min":0.65800000000000003},"min":{"1min":0.51400000000000001,"5min":0.31,"15min":0.31},"max":{"1min":1.0209999999999999,"5min":1.0209999999999999,"15min":1.0209999999999999},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.79100000000000004,"5min":0.749,"15min":0.752},"min":{"1min":0.501,"5min":0.252,"15min":0.252},"max":{"1min":1.016,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.74299999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:32:13 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.753,"15min":0.73699999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.052,"5min":3.052,"15min":3.052},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.68500000000000005,"15min":0.70999999999999996},"min":{"1min":0.30599999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.1719999999999999,"5min":1.5189999999999999,"15min":1.5189999999999999},"last":0.373}]},{"osd":4,"last update":"Sat Sep 23 18:32:38 2023","interfaces":[{"interface":"back","average":{"1min":1.7989999999999999,"5min":1.129,"15min":0.997},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":3.8439999999999999,"5min":3.8439999999999999,"15min":3.8439999999999999},"last":1.6799999999999999},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.0289999999999999,"15min":0.999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":3.9060000000000001,"5min":3.9060000000000001,"15min":3.9060000000000001},"last":1.1970000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:32:38 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":1.05,"15min":1.0349999999999999},"min":{"1min":0.47599999999999998,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":4.4329999999999998,"5min":4.4329999999999998,"15min":4.4329999999999998},"last":0.52500000000000002},{"interface":"front","average":{"1min":2.206,"5min":1.24,"15min":1.075},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":6.6040000000000001,"5min":6.6040000000000001,"15min":6.6040000000000001},"last":0.46300000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:32:38 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":1.052,"15min":1.0449999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":3.7930000000000001,"5min":3.7930000000000001,"15min":3.7930000000000001},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.0940000000000001,"15min":1.0820000000000001},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":3.5499999999999998,"5min":3.5499999999999998,"15min":3.5499999999999998},"last":1.2310000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:32:38 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.075,"15min":1.073},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":3.8260000000000001,"5min":3.8260000000000001,"15min":3.8260000000000001},"last":0.42499999999999999},{"interface":"front","average":{"1min":2.1779999999999999,"5min":1.2969999999999999,"15min":1.1839999999999999},"min":{"1min":0.53700000000000003,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":8.0009999999999994,"5min":8.0009999999999994,"15min":8.0009999999999994},"last":0.496}]}]},{"osd":3,"up_from":103,"seq":442381631549,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1400740,"kb_used_data":1337180,"kb_used_omap":4,"kb_used_meta":63547,"kb_avail":92344412,"statfs":{"total":95995035648,"available":94560677888,"internally_reserved":0,"allocated":1369272320,"data_stored":1368703742,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4826,"internal_metadata":65072422},"hb_peers":[0,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:32:06 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":0.69399999999999995,"15min":0.69599999999999995},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":6.8869999999999996,"5min":6.8869999999999996,"15min":6.8869999999999996},"last":1.734},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.69399999999999995,"15min":0.72999999999999998},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":1.9590000000000001,"5min":1.9590000000000001,"15min":1.9590000000000001},"last":0.56200000000000006}]},{"osd":1,"last update":"Sat Sep 23 18:30:04 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.70299999999999996,"15min":0.71499999999999997},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":1.022,"5min":1.022,"15min":1.022},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.69599999999999995,"5min":0.76400000000000001,"15min":0.77600000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.002,"5min":1.1439999999999999,"15min":1.1439999999999999},"last":0.55800000000000005}]},{"osd":2,"last update":"Sat Sep 23 18:32:06 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":0.73699999999999999,"15min":0.745},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.427,"5min":3.427,"15min":3.427},"last":0.25700000000000001},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.749,"15min":0.79100000000000004},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.5430000000000001,"5min":2.5430000000000001,"15min":2.5430000000000001},"last":3.798}]},{"osd":4,"last update":"Sat Sep 23 18:32:06 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":0.91700000000000004,"15min":0.95199999999999996},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.897,"5min":1.897,"15min":1.897},"last":4.2809999999999997},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.0149999999999999,"15min":1.0309999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":5.5179999999999998,"5min":5.5179999999999998,"15min":5.5179999999999998},"last":1.8080000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:32:06 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":0.97699999999999998,"15min":1.0509999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.8919999999999999,"5min":2.157,"15min":2.157},"last":1.8340000000000001},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.0580000000000001,"15min":1.1180000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.5790000000000002,"5min":3.5790000000000002,"15min":3.5790000000000002},"last":0.39000000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:32:06 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":1.024,"15min":1.1020000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.0920000000000001,"5min":2.0920000000000001,"15min":2.0920000000000001},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.78300000000000003,"5min":0.996,"15min":1.0640000000000001},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.9430000000000001,"5min":1.9430000000000001,"15min":1.9430000000000001},"last":3.9809999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:32:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.073,"15min":1.1160000000000001},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":2.8260000000000001,"5min":2.8260000000000001,"15min":2.8260000000000001},"last":4.0220000000000002},{"interface":"front","average":{"1min":0.79500000000000004,"5min":1.04,"15min":1.1259999999999999},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.603,"5min":1.603,"15min":1.603},"last":1.857}]}]},{"osd":4,"up_from":30,"seq":128849019159,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3709748,"kb_used_data":3400236,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":90035404,"statfs":{"total":95995035648,"available":92196253696,"internally_reserved":0,"allocated":3481841664,"data_stored":3481316708,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:32:43 2023","interfaces":[{"interface":"back","average":{"1min":1.6000000000000001,"5min":1.0329999999999999,"15min":0.96499999999999997},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.1609999999999996,"5min":5.1609999999999996,"15min":5.1609999999999996},"last":1.4139999999999999},{"interface":"front","average":{"1min":1.04,"5min":1.0129999999999999,"15min":1.028},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.992,"5min":4.5110000000000001,"15min":4.5110000000000001},"last":1.4219999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:30:08 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.86499999999999999,"15min":0.84699999999999998},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.2110000000000001,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.874},{"interface":"front","average":{"1min":0.88700000000000001,"5min":0.88500000000000001,"15min":0.88600000000000001},"min":{"1min":0.63300000000000001,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.103,"5min":1.224,"15min":1.224},"last":0.89700000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:32:43 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.0209999999999999,"15min":1.04},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.27,"5min":4.5700000000000003,"15min":4.5700000000000003},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.6040000000000001,"5min":1.093,"15min":1.0389999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":5.2839999999999998,"5min":5.2839999999999998,"15min":5.2839999999999998},"last":0.64900000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:32:14 2023","interfaces":[{"interface":"back","average":{"1min":2.1120000000000001,"5min":1.1419999999999999,"15min":0.98599999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":20.850000000000001,"5min":20.850000000000001,"15min":20.850000000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":2.1819999999999999,"5min":1.2490000000000001,"15min":1.115},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":20.792999999999999,"5min":20.792999999999999,"15min":20.792999999999999},"last":0.63300000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:32:34 2023","interfaces":[{"interface":"back","average":{"1min":1.6819999999999999,"5min":1.175,"15min":1.0389999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":7.819,"5min":7.819,"15min":7.819},"last":0.55600000000000005},{"interface":"front","average":{"1min":1.6970000000000001,"5min":1.113,"15min":1.018},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.4649999999999999,"5min":5.4649999999999999,"15min":5.4649999999999999},"last":0.61299999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:32:00 2023","interfaces":[{"interface":"back","average":{"1min":1.3520000000000001,"5min":1.107,"15min":1.052},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":5.258,"5min":5.258,"15min":5.258},"last":0.57799999999999996},{"interface":"front","average":{"1min":1.268,"5min":1.0960000000000001,"15min":1.0389999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.4779999999999998,"5min":4.4779999999999998,"15min":4.4779999999999998},"last":0.47599999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:32:43 2023","interfaces":[{"interface":"back","average":{"1min":1.6100000000000001,"5min":1.248,"15min":1.1299999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":7.7389999999999999,"5min":7.7389999999999999,"15min":7.7389999999999999},"last":0.54800000000000004},{"interface":"front","average":{"1min":1.6379999999999999,"5min":1.298,"15min":1.206},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":9.1199999999999992,"5min":9.1199999999999992,"15min":9.1199999999999992},"last":0.60599999999999998}]}]},{"osd":5,"up_from":0,"seq":154618822927,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4224,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7725,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3429,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3580887040,"data_stored":3580887040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2753953792,"data_stored":2753953792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1372389376,"data_stored":1372389376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3496804352,"data_stored":3496804352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3228631040,"data_stored":3228631040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1681457152,"data_stored":1681457152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2222325760,"data_stored":2222325760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:33:06.089 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T18:33:06.090 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 11 2023-09-23T18:33:06.110 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:06.136 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:06.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:06.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:06.243 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:06.264 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:06.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:06.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:06.368 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:06.384 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:33:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:33:06] "GET /metrics HTTP/1.1" 200 36544 "" "Prometheus/2.33.4" 2023-09-23T18:33:06.421 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:06.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:06.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:06.527 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:06.553 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:06.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:06.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:06.662 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:06.686 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:06.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:06.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:06.795 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:06.823 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:06.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:06.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:06.927 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:06.956 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:06.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:06.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:07.063 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:07.076 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:06 smithi098 bash[76817]: cluster 2023-09-23T18:33:05.716690+0000 mgr.y (mgr.26054) 435 : cluster 0 pgmap v382: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 30 MiB/s wr, 569 op/s; 64010/316374 objects degraded (20.232%); 15542/316374 objects misplaced (4.913%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T18:33:07.076 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:06 smithi098 bash[79516]: cluster 2023-09-23T18:33:05.716690+0000 mgr.y (mgr.26054) 435 : cluster 0 pgmap v382: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 30 MiB/s wr, 569 op/s; 64010/316374 objects degraded (20.232%); 15542/316374 objects misplaced (4.913%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T18:33:07.089 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:07.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:07.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:07.197 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:07.205 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:06 smithi173 bash[59422]: cluster 2023-09-23T18:33:05.716690+0000 mgr.y (mgr.26054) 435 : cluster 0 pgmap v382: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 30 MiB/s wr, 569 op/s; 64010/316374 objects degraded (20.232%); 15542/316374 objects misplaced (4.913%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T18:33:07.245 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:07.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:07.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:07.349 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:07.378 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:07.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:07.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:07.481 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:07.512 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:07.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:07.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:07.617 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:07.654 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:07.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:07.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:07.757 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:07.793 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:07.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:07.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:07.899 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:07.924 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:07.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:07.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:08.028 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:08.074 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:33:08.074 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:33:08.119 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:08.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:08.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:08.228 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:08.255 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:08.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:08.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:08.359 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:08.383 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:08.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:08.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:08.486 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:08.519 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:08.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:08.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:08.610 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:33:08.624 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:08.666 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:08.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:08.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:08.769 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:08.800 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:08.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:08.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:08.908 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:08.917 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:08 smithi098 bash[76817]: cluster 2023-09-23T18:33:07.717535+0000 mgr.y (mgr.26054) 436 : cluster 0 pgmap v383: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 102 B/s rd, 23 MiB/s wr, 546 op/s; 63724/313515 objects degraded (20.326%); 15382/313515 objects misplaced (4.906%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T18:33:08.917 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:08 smithi098 bash[79516]: cluster 2023-09-23T18:33:07.717535+0000 mgr.y (mgr.26054) 436 : cluster 0 pgmap v383: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 102 B/s rd, 23 MiB/s wr, 546 op/s; 63724/313515 objects degraded (20.326%); 15382/313515 objects misplaced (4.906%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T18:33:08.938 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:08.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:08.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:09.047 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:09.086 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:09.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:09.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:09.190 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:09.200 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:08 smithi173 bash[59422]: cluster 2023-09-23T18:33:07.717535+0000 mgr.y (mgr.26054) 436 : cluster 0 pgmap v383: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 102 B/s rd, 23 MiB/s wr, 546 op/s; 63724/313515 objects degraded (20.326%); 15382/313515 objects misplaced (4.906%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T18:33:09.219 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:09.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:09.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:09.327 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:09.353 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:09.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:09.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:09.455 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:09.489 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:09.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:09.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:09.593 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:09.617 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:09.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:09.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:09.722 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:09.751 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:09.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:09.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:09.860 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:09.889 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:09.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:09.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:09.992 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:10.002 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:09 smithi173 bash[59422]: cluster 2023-09-23T18:33:09.439235+0000 mon.a (mon.0) 613 : cluster 3 Health check update: Degraded data redundancy: 63724/313515 objects degraded (20.326%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:10.002 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:09 smithi173 bash[59422]: audit 2023-09-23T18:33:09.580956+0000 mon.a (mon.0) 614 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:33:10.024 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:10.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:10.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:10.130 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:10.156 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:10.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:10.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:10.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:09 smithi098 bash[79516]: cluster 2023-09-23T18:33:09.439235+0000 mon.a (mon.0) 613 : cluster 3 Health check update: Degraded data redundancy: 63724/313515 objects degraded (20.326%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:10.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:09 smithi098 bash[79516]: audit 2023-09-23T18:33:09.580956+0000 mon.a (mon.0) 614 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:33:10.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:09 smithi098 bash[76817]: cluster 2023-09-23T18:33:09.439235+0000 mon.a (mon.0) 613 : cluster 3 Health check update: Degraded data redundancy: 63724/313515 objects degraded (20.326%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:10.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:09 smithi098 bash[76817]: audit 2023-09-23T18:33:09.580956+0000 mon.a (mon.0) 614 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:33:10.260 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:10.292 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:10.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:10.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:10.397 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:10.435 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:10.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:10.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:10.541 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:10.572 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:10.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:10.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:10.648 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:33:10.678 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:10.701 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:10.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:10.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:10.806 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:10.831 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:10.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:10.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:10.934 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:10.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:10 smithi098 bash[76817]: cluster 2023-09-23T18:33:09.718364+0000 mgr.y (mgr.26054) 437 : cluster 0 pgmap v384: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 102 B/s rd, 19 MiB/s wr, 549 op/s; 63540/311706 objects degraded (20.385%); 15029/311706 objects misplaced (4.822%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:33:10.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:10 smithi098 bash[79516]: cluster 2023-09-23T18:33:09.718364+0000 mgr.y (mgr.26054) 437 : cluster 0 pgmap v384: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 102 B/s rd, 19 MiB/s wr, 549 op/s; 63540/311706 objects degraded (20.385%); 15029/311706 objects misplaced (4.822%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:33:10.967 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:10.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:10.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:11.073 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:11.109 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:11.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:11.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:11.217 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:11.242 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:11.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:11.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:11.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:10 smithi173 bash[59422]: cluster 2023-09-23T18:33:09.718364+0000 mgr.y (mgr.26054) 437 : cluster 0 pgmap v384: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 102 B/s rd, 19 MiB/s wr, 549 op/s; 63540/311706 objects degraded (20.385%); 15029/311706 objects misplaced (4.822%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:33:11.345 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:11.369 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:11.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:11.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:11.475 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:11.505 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:11.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:11.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:11.611 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:11.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:11.638 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:11.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:11.739 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:11.782 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:11.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:11.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:11.887 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:11.916 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:11.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:11.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:12.020 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:12.045 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:12.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:12.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:12.151 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:12.182 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:12.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:12.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:12.287 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:12.316 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:12.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:12.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:12.421 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:12.449 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:12.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:12.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:12.553 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:12.576 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:12.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:12.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:12.680 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:33:12.683 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:12.684 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:33:12.732 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:12.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:12.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:12.830 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:12 smithi098 bash[76817]: cluster 2023-09-23T18:33:11.719142+0000 mgr.y (mgr.26054) 438 : cluster 0 pgmap v385: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 102 B/s rd, 9.7 MiB/s wr, 552 op/s; 63135/306990 objects degraded (20.566%); 15029/306990 objects misplaced (4.896%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:33:12.839 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:12.862 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:12.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:12.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:12.965 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:12.990 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:12.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:12.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:13.092 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:13.100 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:12 smithi098 bash[79516]: cluster 2023-09-23T18:33:11.719142+0000 mgr.y (mgr.26054) 438 : cluster 0 pgmap v385: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 102 B/s rd, 9.7 MiB/s wr, 552 op/s; 63135/306990 objects degraded (20.566%); 15029/306990 objects misplaced (4.896%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:33:13.125 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:13.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:13.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:13.229 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:13.243 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:12 smithi173 bash[59422]: cluster 2023-09-23T18:33:11.719142+0000 mgr.y (mgr.26054) 438 : cluster 0 pgmap v385: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 102 B/s rd, 9.7 MiB/s wr, 552 op/s; 63135/306990 objects degraded (20.566%); 15029/306990 objects misplaced (4.896%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:33:13.260 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:13.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:13.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:13.365 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:13.408 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:13.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:13.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:13.514 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:13.547 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:13.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:13.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:13.653 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:13.677 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:13.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:13.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:13.780 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:13.808 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:13.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:13.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:13.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:13.942 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:13.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:13.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:14.047 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:14.077 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:14.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:14.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:14.180 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:14.215 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:14.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:14.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:14.321 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:14.348 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:14.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:14.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:14.454 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:14.504 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:14.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:14.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:14.607 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:14.627 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:14.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:14.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:14.736 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:14.763 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:14.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:14.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:14.871 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:14.903 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:14.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:14.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:15.006 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:15.014 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:14 smithi173 bash[59422]: cluster 2023-09-23T18:33:13.719987+0000 mgr.y (mgr.26054) 439 : cluster 0 pgmap v386: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 85 B/s rd, 8.1 MiB/s wr, 590 op/s; 62736/302448 objects degraded (20.743%); 14553/302448 objects misplaced (4.812%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T18:33:15.014 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:14 smithi173 bash[59422]: cluster 2023-09-23T18:33:14.440397+0000 mon.a (mon.0) 615 : cluster 3 Health check update: Degraded data redundancy: 62736/302448 objects degraded (20.743%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:15.048 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:15.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:15.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:15.151 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:15.180 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:15.181 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:14 smithi098 bash[79516]: cluster 2023-09-23T18:33:13.719987+0000 mgr.y (mgr.26054) 439 : cluster 0 pgmap v386: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 85 B/s rd, 8.1 MiB/s wr, 590 op/s; 62736/302448 objects degraded (20.743%); 14553/302448 objects misplaced (4.812%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T18:33:15.181 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:14 smithi098 bash[79516]: cluster 2023-09-23T18:33:14.440397+0000 mon.a (mon.0) 615 : cluster 3 Health check update: Degraded data redundancy: 62736/302448 objects degraded (20.743%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:15.181 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:14 smithi098 bash[76817]: cluster 2023-09-23T18:33:13.719987+0000 mgr.y (mgr.26054) 439 : cluster 0 pgmap v386: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 85 B/s rd, 8.1 MiB/s wr, 590 op/s; 62736/302448 objects degraded (20.743%); 14553/302448 objects misplaced (4.812%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T18:33:15.181 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:14 smithi098 bash[76817]: cluster 2023-09-23T18:33:14.440397+0000 mon.a (mon.0) 615 : cluster 3 Health check update: Degraded data redundancy: 62736/302448 objects degraded (20.743%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:15.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:15.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:15.283 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:15.319 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:15.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:15.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:15.428 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:15.461 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:15.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:15.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:15.568 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:15.600 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:15.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:15.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:15.706 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:15.732 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:15.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:15.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:15.836 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:15.864 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:15.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:15.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:15.967 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:16.002 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:16.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:16.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:16.109 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:16.143 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:16.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:16.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:16.190 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:33:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:33:16] "GET /metrics HTTP/1.1" 200 36548 "" "Prometheus/2.33.4" 2023-09-23T18:33:16.249 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:16.292 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:16.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:16.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:16.400 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:16.436 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:16.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:16.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:16.541 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:16.566 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:16.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:16.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:16.671 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:16.707 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:16.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:16.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:16.814 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:16.843 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:16.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:16.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:16.947 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:16.982 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:16.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:16.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:17.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:16 smithi173 bash[59422]: cluster 2023-09-23T18:33:15.720983+0000 mgr.y (mgr.26054) 440 : cluster 0 pgmap v387: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 85 B/s rd, 1.6 MiB/s wr, 586 op/s; 62551/298497 objects degraded (20.955%); 14553/298497 objects misplaced (4.875%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T18:33:17.090 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:17.115 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:17.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:17.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:17.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:16 smithi098 bash[79516]: cluster 2023-09-23T18:33:15.720983+0000 mgr.y (mgr.26054) 440 : cluster 0 pgmap v387: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 85 B/s rd, 1.6 MiB/s wr, 586 op/s; 62551/298497 objects degraded (20.955%); 14553/298497 objects misplaced (4.875%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T18:33:17.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:16 smithi098 bash[76817]: cluster 2023-09-23T18:33:15.720983+0000 mgr.y (mgr.26054) 440 : cluster 0 pgmap v387: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 85 B/s rd, 1.6 MiB/s wr, 586 op/s; 62551/298497 objects degraded (20.955%); 14553/298497 objects misplaced (4.875%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T18:33:17.219 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:17.246 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:17.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:17.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:17.349 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:17.397 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:17.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:17.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:17.500 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:17.531 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:17.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:17.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:17.635 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:17.661 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:17.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:17.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:17.764 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:17.794 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:17.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:17.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:17.900 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:17.925 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:17.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:17.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:18.029 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:18.061 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:18.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:18.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:18.166 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:18.191 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:18.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:18.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:18.295 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:18.340 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:18.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:18.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:18.444 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:18.465 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:18.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:18.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:18.569 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:18.598 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:18.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:18.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:18.704 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:18.732 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:18.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:18.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:18.838 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:18.846 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:18 smithi098 bash[79516]: cluster 2023-09-23T18:33:17.721679+0000 mgr.y (mgr.26054) 441 : cluster 0 pgmap v388: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 85 B/s rd, 43 KiB/s wr, 592 op/s; 62163/295080 objects degraded (21.066%); 14400/295080 objects misplaced (4.880%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:33:18.847 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:18 smithi098 bash[76817]: cluster 2023-09-23T18:33:17.721679+0000 mgr.y (mgr.26054) 441 : cluster 0 pgmap v388: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 85 B/s rd, 43 KiB/s wr, 592 op/s; 62163/295080 objects degraded (21.066%); 14400/295080 objects misplaced (4.880%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:33:18.877 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:18.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:18.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:18.985 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:19.016 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:19.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:19.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:19.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:18 smithi173 bash[59422]: cluster 2023-09-23T18:33:17.721679+0000 mgr.y (mgr.26054) 441 : cluster 0 pgmap v388: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 85 B/s rd, 43 KiB/s wr, 592 op/s; 62163/295080 objects degraded (21.066%); 14400/295080 objects misplaced (4.880%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:33:19.120 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:19.143 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:19.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:19.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:19.250 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:19.286 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:19.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:19.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:19.392 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:19.426 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:19.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:19.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:19.536 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:19.557 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:19.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:19.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:19.660 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:19.690 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:19.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:19.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:19.795 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:19.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:19.827 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:19.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:19.847 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:19 smithi173 bash[59422]: cluster 2023-09-23T18:33:19.441534+0000 mon.a (mon.0) 616 : cluster 3 Health check update: Degraded data redundancy: 62163/295080 objects degraded (21.066%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:19.929 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:19.961 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:19.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:19.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:20.064 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:20.075 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:19 smithi098 bash[76817]: cluster 2023-09-23T18:33:19.441534+0000 mon.a (mon.0) 616 : cluster 3 Health check update: Degraded data redundancy: 62163/295080 objects degraded (21.066%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:20.075 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:19 smithi098 bash[79516]: cluster 2023-09-23T18:33:19.441534+0000 mon.a (mon.0) 616 : cluster 3 Health check update: Degraded data redundancy: 62163/295080 objects degraded (21.066%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:20.104 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:20.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:20.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:20.208 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:20.242 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:20.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:20.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:20.345 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:20.368 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:20.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:20.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:20.473 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:20.504 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:20.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:20.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:20.609 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:20.629 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:20.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:20.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:20.732 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:20.763 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:20.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:20.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:20.866 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:20.900 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:20.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:20.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:20.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:20 smithi098 bash[79516]: cluster 2023-09-23T18:33:19.722522+0000 mgr.y (mgr.26054) 442 : cluster 0 pgmap v389: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 61960/293247 objects degraded (21.129%); 14055/293247 objects misplaced (4.793%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T18:33:20.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:20 smithi098 bash[76817]: cluster 2023-09-23T18:33:19.722522+0000 mgr.y (mgr.26054) 442 : cluster 0 pgmap v389: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 61960/293247 objects degraded (21.129%); 14055/293247 objects misplaced (4.793%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T18:33:21.004 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:21.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:20 smithi173 bash[59422]: cluster 2023-09-23T18:33:19.722522+0000 mgr.y (mgr.26054) 442 : cluster 0 pgmap v389: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 6.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 61960/293247 objects degraded (21.129%); 14055/293247 objects misplaced (4.793%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T18:33:21.027 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:21.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:21.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:21.135 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:21.170 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:21.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:21.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:21.275 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:21.300 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:21.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:21.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:21.405 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:21.428 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:21.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:21.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:21.533 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:21.563 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:21.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:21.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:21.667 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:21.695 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:21.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:21.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:21.798 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:21.831 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:21.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:21.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:21.935 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:21.961 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:21.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:21.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:22.064 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:22.090 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:22.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:22.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:22.195 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:22.219 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:22.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:22.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:22.323 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:22.352 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:22.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:22.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:22.455 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:22.492 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:22.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:22.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:22.595 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:22.627 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:22.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:22.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:22.733 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:22.763 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:22.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:22.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:22.871 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:22.902 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:22.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:22.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:23.012 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:23.020 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:22 smithi173 bash[59422]: cluster 2023-09-23T18:33:21.725179+0000 mgr.y (mgr.26054) 443 : cluster 0 pgmap v390: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 61555/288528 objects degraded (21.334%); 14055/288528 objects misplaced (4.871%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T18:33:23.045 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:23.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:23.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:23.149 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:23.179 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:23.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:23.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:23.200 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:22 smithi098 bash[76817]: cluster 2023-09-23T18:33:21.725179+0000 mgr.y (mgr.26054) 443 : cluster 0 pgmap v390: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 61555/288528 objects degraded (21.334%); 14055/288528 objects misplaced (4.871%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T18:33:23.200 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:22 smithi098 bash[79516]: cluster 2023-09-23T18:33:21.725179+0000 mgr.y (mgr.26054) 443 : cluster 0 pgmap v390: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 61555/288528 objects degraded (21.334%); 14055/288528 objects misplaced (4.871%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T18:33:23.283 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:23.306 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:23.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:23.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:23.409 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:23.430 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:23.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:23.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:23.533 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:23.557 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:23.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:23.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:23.660 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:23.683 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:23.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:23.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:23.791 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:23.814 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:23.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:23.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:23.917 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:23.940 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:23.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:23.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:24.043 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:24.067 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:24.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:24.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:24.170 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:24.197 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:24.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:24.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:24.301 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:24.334 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:24.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:24.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:24.440 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:24.465 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:24.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:24.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:24.569 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:24.599 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:24.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:24.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:24.703 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:24.730 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:24.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:24.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:24.838 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:24.864 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:24.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:24.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:24.967 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:24.975 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:24 smithi098 bash[76817]: cluster 2023-09-23T18:33:23.725987+0000 mgr.y (mgr.26054) 444 : cluster 0 pgmap v391: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 61189/284073 objects degraded (21.540%); 13606/284073 objects misplaced (4.790%); 4.6 MiB/s, 72 objects/s recovering 2023-09-23T18:33:24.975 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:24 smithi098 bash[76817]: cluster 2023-09-23T18:33:24.442811+0000 mon.a (mon.0) 617 : cluster 3 Health check update: Degraded data redundancy: 61189/284073 objects degraded (21.540%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:24.975 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:24 smithi098 bash[76817]: audit 2023-09-23T18:33:24.582523+0000 mon.a (mon.0) 618 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:33:24.975 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:24 smithi098 bash[79516]: cluster 2023-09-23T18:33:23.725987+0000 mgr.y (mgr.26054) 444 : cluster 0 pgmap v391: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 61189/284073 objects degraded (21.540%); 13606/284073 objects misplaced (4.790%); 4.6 MiB/s, 72 objects/s recovering 2023-09-23T18:33:24.976 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:24 smithi098 bash[79516]: cluster 2023-09-23T18:33:24.442811+0000 mon.a (mon.0) 617 : cluster 3 Health check update: Degraded data redundancy: 61189/284073 objects degraded (21.540%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:24.976 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:24 smithi098 bash[79516]: audit 2023-09-23T18:33:24.582523+0000 mon.a (mon.0) 618 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:33:25.004 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:25.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:25.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:25.108 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:25.126 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:25.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:25.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:25.228 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:25.255 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:25.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:25.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:25.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:24 smithi173 bash[59422]: cluster 2023-09-23T18:33:23.725987+0000 mgr.y (mgr.26054) 444 : cluster 0 pgmap v391: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 61189/284073 objects degraded (21.540%); 13606/284073 objects misplaced (4.790%); 4.6 MiB/s, 72 objects/s recovering 2023-09-23T18:33:25.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:24 smithi173 bash[59422]: cluster 2023-09-23T18:33:24.442811+0000 mon.a (mon.0) 617 : cluster 3 Health check update: Degraded data redundancy: 61189/284073 objects degraded (21.540%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:25.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:24 smithi173 bash[59422]: audit 2023-09-23T18:33:24.582523+0000 mon.a (mon.0) 618 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:33:25.361 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:25.386 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:25.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:25.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:25.490 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:25.521 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:25.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:25.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:25.625 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:25.651 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:25.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:25.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:25.755 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:25.791 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:25.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:25.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:25.899 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:25.922 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:25.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:25.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:26.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:26.059 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:26.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:26.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:26.086 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:33:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:33:26] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.33.4" 2023-09-23T18:33:26.163 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:26.194 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:26.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:26.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:26.299 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:26.323 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:26.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:26.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:26.426 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:26.449 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:26.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:26.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:26.553 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:26.576 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:26.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:26.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:26.679 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:26.708 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:26.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:26.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:26.811 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:26.841 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:26.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:26.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:26.956 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:26.984 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:26.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:26.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:27.087 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:27.115 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:27.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:27.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:27.203 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:26 smithi098 bash[79516]: cluster 2023-09-23T18:33:25.727454+0000 mgr.y (mgr.26054) 445 : cluster 0 pgmap v392: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 61024/280200 objects degraded (21.779%); 13606/280200 objects misplaced (4.856%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T18:33:27.203 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:26 smithi098 bash[76817]: cluster 2023-09-23T18:33:25.727454+0000 mgr.y (mgr.26054) 445 : cluster 0 pgmap v392: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 61024/280200 objects degraded (21.779%); 13606/280200 objects misplaced (4.856%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T18:33:27.222 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:27.248 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:27.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:27.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:27.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:26 smithi173 bash[59422]: cluster 2023-09-23T18:33:25.727454+0000 mgr.y (mgr.26054) 445 : cluster 0 pgmap v392: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 61024/280200 objects degraded (21.779%); 13606/280200 objects misplaced (4.856%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T18:33:27.356 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:27.381 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:27.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:27.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:27.483 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:27.515 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:27.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:27.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:27.525 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:33:27.526 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [], 2023-09-23T18:33:27.526 INFO:teuthology.orchestra.run.smithi098.stdout: "complaint_time": 30, 2023-09-23T18:33:27.526 INFO:teuthology.orchestra.run.smithi098.stdout: "num_blocked_ops": 0 2023-09-23T18:33:27.526 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:33:27.621 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:27.654 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:27.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:27.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:27.760 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:27.803 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:27.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:27.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:27.905 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:27.939 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:27.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:27.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:28.043 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:28.084 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:28.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:28.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:28.187 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:28.214 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:28.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:28.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:28.323 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:28.358 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:28.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:28.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:28.461 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:28.486 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:28.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:28.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:28.591 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:28.616 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:28.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:28.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:28.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:28.753 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:28.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:28.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:28.771 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-09-23T18:33:28.860 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:28.892 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:28.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:28.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:28.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:28 smithi098 bash[76817]: cluster 2023-09-23T18:33:27.728333+0000 mgr.y (mgr.26054) 446 : cluster 0 pgmap v393: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 60674/277068 objects degraded (21.899%); 13468/277068 objects misplaced (4.861%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T18:33:28.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:28 smithi098 bash[76817]: audit 2023-09-23T18:33:28.762864+0000 mon.c (mon.2) 60 : audit 1 from='client.? 172.21.15.98:0/386564643' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T18:33:28.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:28 smithi098 bash[76817]: audit 2023-09-23T18:33:28.763342+0000 mon.a (mon.0) 619 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T18:33:28.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:28 smithi098 bash[79516]: cluster 2023-09-23T18:33:27.728333+0000 mgr.y (mgr.26054) 446 : cluster 0 pgmap v393: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 60674/277068 objects degraded (21.899%); 13468/277068 objects misplaced (4.861%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T18:33:28.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:28 smithi098 bash[79516]: audit 2023-09-23T18:33:28.762864+0000 mon.c (mon.2) 60 : audit 1 from='client.? 172.21.15.98:0/386564643' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T18:33:28.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:28 smithi098 bash[79516]: audit 2023-09-23T18:33:28.763342+0000 mon.a (mon.0) 619 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T18:33:28.996 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:29.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:28 smithi173 bash[59422]: cluster 2023-09-23T18:33:27.728333+0000 mgr.y (mgr.26054) 446 : cluster 0 pgmap v393: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 60674/277068 objects degraded (21.899%); 13468/277068 objects misplaced (4.861%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T18:33:29.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:28 smithi173 bash[59422]: audit 2023-09-23T18:33:28.762864+0000 mon.c (mon.2) 60 : audit 1 from='client.? 172.21.15.98:0/386564643' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T18:33:29.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:28 smithi173 bash[59422]: audit 2023-09-23T18:33:28.763342+0000 mon.a (mon.0) 619 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T18:33:29.035 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:29.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:29.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:29.140 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:29.170 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:29.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:29.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:29.276 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:29.310 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:29.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:29.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:29.413 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:29.438 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:29.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:29.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:29.541 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:29.570 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:29.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:29.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:29.673 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:29.709 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:29.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:29.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:29.814 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:29.841 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:29.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:29.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:29.946 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:29.973 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:29.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:29.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:30.079 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:30.109 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:30.115 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:29 smithi173 bash[59422]: cluster 2023-09-23T18:33:29.444857+0000 mon.a (mon.0) 620 : cluster 3 Health check update: Degraded data redundancy: 60674/277068 objects degraded (21.899%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:30.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:30.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:30.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:29 smithi098 bash[76817]: cluster 2023-09-23T18:33:29.444857+0000 mon.a (mon.0) 620 : cluster 3 Health check update: Degraded data redundancy: 60674/277068 objects degraded (21.899%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:30.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:29 smithi098 bash[79516]: cluster 2023-09-23T18:33:29.444857+0000 mon.a (mon.0) 620 : cluster 3 Health check update: Degraded data redundancy: 60674/277068 objects degraded (21.899%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:30.216 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:30.245 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:30.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:30.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:30.349 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:30.390 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:30.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:30.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:30.493 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:30.519 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:30.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:30.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:30.625 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:30.655 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:30.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:30.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:30.758 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:30.782 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:30.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:30.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:30.885 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:30.920 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:30.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:30.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:31.023 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:31.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:30 smithi173 bash[59422]: cluster 2023-09-23T18:33:29.729218+0000 mgr.y (mgr.26054) 447 : cluster 0 pgmap v394: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 60505/275481 objects degraded (21.963%); 13145/275481 objects misplaced (4.772%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T18:33:31.049 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:31.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:31.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:31.153 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:31.161 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:30 smithi098 bash[76817]: cluster 2023-09-23T18:33:29.729218+0000 mgr.y (mgr.26054) 447 : cluster 0 pgmap v394: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 60505/275481 objects degraded (21.963%); 13145/275481 objects misplaced (4.772%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T18:33:31.162 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:30 smithi098 bash[79516]: cluster 2023-09-23T18:33:29.729218+0000 mgr.y (mgr.26054) 447 : cluster 0 pgmap v394: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 60505/275481 objects degraded (21.963%); 13145/275481 objects misplaced (4.772%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T18:33:31.198 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:31.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:31.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:31.303 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:31.337 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:31.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:31.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:31.442 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:31.466 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:31.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:31.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:31.571 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:31.611 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:31.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:31.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:31.719 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:31.744 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:31.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:31.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:31.848 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:31.880 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:31.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:31.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:31.983 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:31.993 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T18:33:32.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:32.024 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:32.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:32.096 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-09-23T18:33:32.125 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:32.149 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:32.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:32.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:32.255 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:32.297 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:32.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:32.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:32.402 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:32.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:32.439 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:32.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:32.539 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:32.568 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:32.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:32.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:32.674 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:32.705 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:32.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:32.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:32.809 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:32.828 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:33:32.832 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:32.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:32.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:32.868 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:32 smithi098 bash[76817]: cluster 2023-09-23T18:33:31.730256+0000 mgr.y (mgr.26054) 448 : cluster 0 pgmap v395: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 60119/271080 objects degraded (22.178%); 13145/271080 objects misplaced (4.849%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T18:33:32.868 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:32 smithi098 bash[76817]: audit 2023-09-23T18:33:32.777753+0000 mon.b (mon.1) 3 : audit 0 from='client.? 172.21.15.98:0/833336918' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:33:32.869 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:33:32.928 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:33:32.936 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:32.969 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:33:32.975 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:32.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:32.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:33.007 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:33:33.043 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:33:33.081 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:33.086 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:33:33.086 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:33:33.086 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T18:33:33.086 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:33:33.086 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:33:33.087 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:33:33.087 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:33:33.087 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:33:33.087 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:33:33.089 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:32 smithi173 bash[59422]: cluster 2023-09-23T18:33:31.730256+0000 mgr.y (mgr.26054) 448 : cluster 0 pgmap v395: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 60119/271080 objects degraded (22.178%); 13145/271080 objects misplaced (4.849%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T18:33:33.089 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:32 smithi173 bash[59422]: audit 2023-09-23T18:33:32.777753+0000 mon.b (mon.1) 3 : audit 0 from='client.? 172.21.15.98:0/833336918' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:33:33.112 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:33.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:33.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:33.124 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:32 smithi098 bash[79516]: cluster 2023-09-23T18:33:31.730256+0000 mgr.y (mgr.26054) 448 : cluster 0 pgmap v395: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 60119/271080 objects degraded (22.178%); 13145/271080 objects misplaced (4.849%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T18:33:33.125 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:32 smithi098 bash[79516]: audit 2023-09-23T18:33:32.777753+0000 mon.b (mon.1) 3 : audit 0 from='client.? 172.21.15.98:0/833336918' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:33:33.218 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:33.252 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:33.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:33.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:33.357 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:33.382 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:33.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:33.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:33.486 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:33.524 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:33.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:33.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:33.629 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:33.660 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:33.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:33.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:33.762 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:33.788 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:33.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:33.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:33.891 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:33.895 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is unset 2023-09-23T18:33:33.900 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:33 smithi173 bash[59422]: audit 2023-09-23T18:33:33.183454+0000 mon.c (mon.2) 61 : audit 1 from='client.? 172.21.15.98:0/3825947636' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:33:33.900 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:33 smithi173 bash[59422]: audit 2023-09-23T18:33:33.184638+0000 mon.a (mon.0) 621 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:33:33.920 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:33.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:33.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:34.025 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:34.065 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:34.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:34.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:34.169 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:34.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:33 smithi098 bash[79516]: audit 2023-09-23T18:33:33.183454+0000 mon.c (mon.2) 61 : audit 1 from='client.? 172.21.15.98:0/3825947636' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:33:34.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:33 smithi098 bash[79516]: audit 2023-09-23T18:33:33.184638+0000 mon.a (mon.0) 621 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:33:34.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:33 smithi098 bash[76817]: audit 2023-09-23T18:33:33.183454+0000 mon.c (mon.2) 61 : audit 1 from='client.? 172.21.15.98:0/3825947636' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:33:34.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:33 smithi098 bash[76817]: audit 2023-09-23T18:33:33.184638+0000 mon.a (mon.0) 621 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:33:34.197 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:34.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:34.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:34.303 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:34.326 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:34.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:34.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:34.430 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:34.458 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:34.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:34.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:34.565 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:34.591 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:34.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:34.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:34.693 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:34.717 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:34.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:34.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:34.828 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:34.856 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:34.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:34.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:34.964 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:34.987 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:34.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:34.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:35.090 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:35.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:34 smithi098 bash[79516]: cluster 2023-09-23T18:33:33.731273+0000 mgr.y (mgr.26054) 449 : cluster 0 pgmap v396: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 59755/266472 objects degraded (22.424%); 12681/266472 objects misplaced (4.759%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T18:33:35.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:34 smithi098 bash[79516]: cluster 2023-09-23T18:33:33.867090+0000 mon.a (mon.0) 622 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:33:35.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:34 smithi098 bash[79516]: audit 2023-09-23T18:33:33.881703+0000 mon.a (mon.0) 623 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:33:35.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:34 smithi098 bash[79516]: cluster 2023-09-23T18:33:33.881895+0000 mon.a (mon.0) 624 : cluster 0 osdmap e121: 8 total, 7 up, 7 in 2023-09-23T18:33:35.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:34 smithi098 bash[79516]: cluster 2023-09-23T18:33:33.889352+0000 osd.6 (osd.6) 37 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T18:33:35.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:34 smithi098 bash[79516]: cluster 2023-09-23T18:33:33.891784+0000 osd.4 (osd.4) 51 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T18:33:35.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:34 smithi098 bash[79516]: cluster 2023-09-23T18:33:33.899343+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T18:33:35.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:34 smithi098 bash[79516]: cluster 2023-09-23T18:33:33.912690+0000 osd.7 (osd.7) 35 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T18:33:35.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:34 smithi098 bash[79516]: cluster 2023-09-23T18:33:34.448781+0000 mon.a (mon.0) 625 : cluster 3 Health check update: Degraded data redundancy: 59755/266472 objects degraded (22.424%), 14 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:35.099 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:34 smithi098 bash[76817]: cluster 2023-09-23T18:33:33.731273+0000 mgr.y (mgr.26054) 449 : cluster 0 pgmap v396: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 59755/266472 objects degraded (22.424%); 12681/266472 objects misplaced (4.759%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T18:33:35.099 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:34 smithi098 bash[76817]: cluster 2023-09-23T18:33:33.867090+0000 mon.a (mon.0) 622 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:33:35.099 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:34 smithi098 bash[76817]: audit 2023-09-23T18:33:33.881703+0000 mon.a (mon.0) 623 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:33:35.099 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:34 smithi098 bash[76817]: cluster 2023-09-23T18:33:33.881895+0000 mon.a (mon.0) 624 : cluster 0 osdmap e121: 8 total, 7 up, 7 in 2023-09-23T18:33:35.099 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:34 smithi098 bash[76817]: cluster 2023-09-23T18:33:33.889352+0000 osd.6 (osd.6) 37 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T18:33:35.099 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:34 smithi098 bash[76817]: cluster 2023-09-23T18:33:33.891784+0000 osd.4 (osd.4) 51 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T18:33:35.099 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:34 smithi098 bash[76817]: cluster 2023-09-23T18:33:33.899343+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T18:33:35.099 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:34 smithi098 bash[76817]: cluster 2023-09-23T18:33:33.912690+0000 osd.7 (osd.7) 35 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T18:33:35.099 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:34 smithi098 bash[76817]: cluster 2023-09-23T18:33:34.448781+0000 mon.a (mon.0) 625 : cluster 3 Health check update: Degraded data redundancy: 59755/266472 objects degraded (22.424%), 14 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:35.119 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:35.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:35.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:35.226 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:35.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:34 smithi173 bash[59422]: cluster 2023-09-23T18:33:33.731273+0000 mgr.y (mgr.26054) 449 : cluster 0 pgmap v396: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 59755/266472 objects degraded (22.424%); 12681/266472 objects misplaced (4.759%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T18:33:35.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:34 smithi173 bash[59422]: cluster 2023-09-23T18:33:33.867090+0000 mon.a (mon.0) 622 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:33:35.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:34 smithi173 bash[59422]: audit 2023-09-23T18:33:33.881703+0000 mon.a (mon.0) 623 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:33:35.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:34 smithi173 bash[59422]: cluster 2023-09-23T18:33:33.881895+0000 mon.a (mon.0) 624 : cluster 0 osdmap e121: 8 total, 7 up, 7 in 2023-09-23T18:33:35.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:34 smithi173 bash[59422]: cluster 2023-09-23T18:33:33.889352+0000 osd.6 (osd.6) 37 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T18:33:35.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:34 smithi173 bash[59422]: cluster 2023-09-23T18:33:33.891784+0000 osd.4 (osd.4) 51 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T18:33:35.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:34 smithi173 bash[59422]: cluster 2023-09-23T18:33:33.899343+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T18:33:35.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:34 smithi173 bash[59422]: cluster 2023-09-23T18:33:33.912690+0000 osd.7 (osd.7) 35 : cluster 3 failed to encode map e121 with expected crc 2023-09-23T18:33:35.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:34 smithi173 bash[59422]: cluster 2023-09-23T18:33:34.448781+0000 mon.a (mon.0) 625 : cluster 3 Health check update: Degraded data redundancy: 59755/266472 objects degraded (22.424%), 14 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:35.260 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:35.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:35.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:35.364 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:35.393 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:35.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:35.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:35.498 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:35.526 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:35.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:35.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:35.631 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:35.654 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:35.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:35.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:35.729 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:33:35.729 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:33:35.757 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:35.788 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:35.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:35.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:35.892 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:35.926 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:35.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:35.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:36.029 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:36.061 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:36.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:36.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:36.165 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:36.190 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:33:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:33:36] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.33.4" 2023-09-23T18:33:36.198 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:36.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:36.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:36.301 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:36.328 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:36.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:36.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:36.431 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:36.464 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:36.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:36.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:36.569 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:36.596 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:36.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:36.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:36.699 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:36.729 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:36.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:36.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:36.832 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:36.858 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:36.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:36.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:36.963 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:37.010 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:37.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:37.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:37.097 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T18:33:37.098 INFO:tasks.thrashosds.thrasher:Reviving osd 1 2023-09-23T18:33:37.098 INFO:tasks.cephadm.osd.1:Restarting osd.1 (starting--it wasn't running)... 2023-09-23T18:33:37.098 DEBUG:teuthology.orchestra.run.smithi098:osd.1> sudo journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.1.service 2023-09-23T18:33:37.100 DEBUG:teuthology.orchestra.run.smithi098:> sudo systemctl start ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T18:33:37.112 INFO:journalctl@ceph.osd.1.smithi098.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:33:37.113 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:37.127 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T18:33:37.177 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:36 smithi098 bash[79516]: cluster 2023-09-23T18:33:35.732264+0000 mgr.y (mgr.26054) 450 : cluster 0 pgmap v398: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 59584/262557 objects degraded (22.694%); 12681/262557 objects misplaced (4.830%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T18:33:37.177 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:33:37 smithi098 systemd[1]: Started Ceph osd.1 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T18:33:37.181 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:36 smithi098 bash[76817]: cluster 2023-09-23T18:33:35.732264+0000 mgr.y (mgr.26054) 450 : cluster 0 pgmap v398: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 59584/262557 objects degraded (22.694%); 12681/262557 objects misplaced (4.830%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T18:33:37.229 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:37.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:37.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:37.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:36 smithi173 bash[59422]: cluster 2023-09-23T18:33:35.732264+0000 mgr.y (mgr.26054) 450 : cluster 0 pgmap v398: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 59584/262557 objects degraded (22.694%); 12681/262557 objects misplaced (4.830%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T18:33:37.335 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:37.373 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:37.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:37.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:37.481 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:37.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:37.522 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:37.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:37.623 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:37.652 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:37.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:37.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:37.731 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:33:37.756 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:37.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:37.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:37.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:37.891 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:37.930 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:37.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:37.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:38.033 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:38.058 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:38.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:38.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:38.169 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:38.198 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:38.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:38.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:38.303 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:38.343 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:38.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:38.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:38.449 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:38.472 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:38.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:38.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:38.577 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:38.606 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:38.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:38.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:38.613 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:33:38.713 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:38.737 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:38.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:38.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:38.844 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:38.880 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:38.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:38.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:38.983 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:39.010 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:38 smithi173 bash[59422]: cluster 2023-09-23T18:33:37.733268+0000 mgr.y (mgr.26054) 451 : cluster 0 pgmap v399: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 597 op/s; 59215/259146 objects degraded (22.850%); 12539/259146 objects misplaced (4.839%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T18:33:39.038 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:39.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:39.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:39.147 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:39.160 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:38 smithi098 bash[76817]: cluster 2023-09-23T18:33:37.733268+0000 mgr.y (mgr.26054) 451 : cluster 0 pgmap v399: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 597 op/s; 59215/259146 objects degraded (22.850%); 12539/259146 objects misplaced (4.839%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T18:33:39.160 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:38 smithi098 bash[79516]: cluster 2023-09-23T18:33:37.733268+0000 mgr.y (mgr.26054) 451 : cluster 0 pgmap v399: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 597 op/s; 59215/259146 objects degraded (22.850%); 12539/259146 objects misplaced (4.839%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T18:33:39.190 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:39.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:39.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:39.300 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:39.331 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:39.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:39.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:39.436 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:39.464 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:39.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:39.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:39.572 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:39.604 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:39.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:39.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:39.711 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:39.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:39.759 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:39.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:39.860 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:39.892 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:39.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:39.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:39.999 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:40.037 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:40.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:40.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:40.143 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:40.155 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:39 smithi173 bash[59422]: cluster 2023-09-23T18:33:39.449909+0000 mon.a (mon.0) 626 : cluster 3 Health check update: Degraded data redundancy: 59215/259146 objects degraded (22.850%), 14 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:40.155 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:39 smithi173 bash[59422]: audit 2023-09-23T18:33:39.582596+0000 mon.a (mon.0) 627 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:33:40.170 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:40.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:40.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:40.181 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:39 smithi098 bash[76817]: cluster 2023-09-23T18:33:39.449909+0000 mon.a (mon.0) 626 : cluster 3 Health check update: Degraded data redundancy: 59215/259146 objects degraded (22.850%), 14 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:40.181 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:39 smithi098 bash[76817]: audit 2023-09-23T18:33:39.582596+0000 mon.a (mon.0) 627 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:33:40.182 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:39 smithi098 bash[79516]: cluster 2023-09-23T18:33:39.449909+0000 mon.a (mon.0) 626 : cluster 3 Health check update: Degraded data redundancy: 59215/259146 objects degraded (22.850%), 14 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:40.182 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:39 smithi098 bash[79516]: audit 2023-09-23T18:33:39.582596+0000 mon.a (mon.0) 627 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:33:40.182 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:33:39 smithi098 bash[114230]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T18:33:40.182 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:33:39 smithi098 bash[114230]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T18:33:40.182 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:33:39 smithi098 bash[114230]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-09-23T18:33:40.182 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:33:39 smithi098 bash[114230]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-09-23T18:33:40.182 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:33:39 smithi098 bash[114230]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-09-23T18:33:40.182 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:33:39 smithi098 bash[114230]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-09-23T18:33:40.182 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:33:39 smithi098 bash[114230]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T18:33:40.183 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:33:39 smithi098 bash[114230]: --> ceph-volume lvm activate successful for osd ID: 1 2023-09-23T18:33:40.273 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:40.299 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:40.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:40.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:40.403 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:40.429 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:40.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:40.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:40.532 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:40.563 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:40.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:40.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:40.668 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:40.697 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:40.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:40.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:40.804 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:40.834 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:40.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:40.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:40.940 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:40.967 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:40.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:40.971 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:40 smithi098 bash[76817]: cluster 2023-09-23T18:33:39.734128+0000 mgr.y (mgr.26054) 452 : cluster 0 pgmap v400: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 58974/257430 objects degraded (22.909%); 12237/257430 objects misplaced (4.754%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T18:33:40.971 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:40 smithi098 bash[79516]: cluster 2023-09-23T18:33:39.734128+0000 mgr.y (mgr.26054) 452 : cluster 0 pgmap v400: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 58974/257430 objects degraded (22.909%); 12237/257430 objects misplaced (4.754%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T18:33:40.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:41.072 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:41.082 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:40 smithi173 bash[59422]: cluster 2023-09-23T18:33:39.734128+0000 mgr.y (mgr.26054) 452 : cluster 0 pgmap v400: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 58974/257430 objects degraded (22.909%); 12237/257430 objects misplaced (4.754%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T18:33:41.098 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:41.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:41.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:41.205 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:41.247 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:41.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:41.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:41.350 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:41.375 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:41.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:41.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:41.478 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:41.513 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:41.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:41.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:41.619 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:41.645 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:41.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:41.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:41.751 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:41.786 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:41.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:41.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:41.893 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:41.916 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:41.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:41.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:41.929 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:33:41.996 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:33:42.019 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:42.045 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:42.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:42.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:42.150 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:42.179 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:42.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:42.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:42.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:41 smithi173 bash[59422]: cluster 2023-09-23T18:33:41.735201+0000 mgr.y (mgr.26054) 453 : cluster 0 pgmap v401: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 58609/252915 objects degraded (23.173%); 12237/252915 objects misplaced (4.838%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T18:33:42.290 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:42.311 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:42.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:42.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:42.322 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:41 smithi098 bash[79516]: cluster 2023-09-23T18:33:41.735201+0000 mgr.y (mgr.26054) 453 : cluster 0 pgmap v401: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 58609/252915 objects degraded (23.173%); 12237/252915 objects misplaced (4.838%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T18:33:42.323 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:41 smithi098 bash[76817]: cluster 2023-09-23T18:33:41.735201+0000 mgr.y (mgr.26054) 453 : cluster 0 pgmap v401: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 58609/252915 objects degraded (23.173%); 12237/252915 objects misplaced (4.838%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T18:33:42.413 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:42.437 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:42.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:42.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:42.544 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:42.567 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:42.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:42.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:42.669 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:42.697 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:42.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:42.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:42.801 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:42.829 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:42.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:42.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:42.934 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:42.968 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:42.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:42.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:43.071 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:43.095 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:43.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:43.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:43.201 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:43.224 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:43.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:43.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:43.328 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:43.364 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:43.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:43.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:43.470 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:43.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:43.499 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:43.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:43.600 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:43.629 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:43.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:43.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:43.735 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:43.775 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:43.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:43.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:43.883 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:43.916 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:43.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:43.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:44.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:44.026 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:33:44.027 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:33:44.046 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:44.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:44.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:44.151 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:44.178 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:44.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:44.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:44.284 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:44.315 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:44.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:44.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:44.420 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:44.446 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:44.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:44.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:44.551 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:44.580 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:44.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:44.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:44.685 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:44.715 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:44.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:44.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:44.823 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:44.832 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:44 smithi098 bash[79516]: cluster 2023-09-23T18:33:43.736059+0000 mgr.y (mgr.26054) 454 : cluster 0 pgmap v402: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 604 op/s; 58051/248343 objects degraded (23.375%); 11938/248343 objects misplaced (4.807%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T18:33:44.832 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:44 smithi098 bash[79516]: cluster 2023-09-23T18:33:44.452512+0000 mon.a (mon.0) 628 : cluster 3 Health check update: Degraded data redundancy: 58051/248343 objects degraded (23.375%), 14 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:44.832 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:44 smithi098 bash[76817]: cluster 2023-09-23T18:33:43.736059+0000 mgr.y (mgr.26054) 454 : cluster 0 pgmap v402: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 604 op/s; 58051/248343 objects degraded (23.375%); 11938/248343 objects misplaced (4.807%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T18:33:44.833 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:44 smithi098 bash[76817]: cluster 2023-09-23T18:33:44.452512+0000 mon.a (mon.0) 628 : cluster 3 Health check update: Degraded data redundancy: 58051/248343 objects degraded (23.375%), 14 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:44.865 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:44.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:44.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:44.970 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:44.986 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:44.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:44.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:45.089 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:45.137 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:45.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:45.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:45.247 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:45.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:44 smithi173 bash[59422]: cluster 2023-09-23T18:33:43.736059+0000 mgr.y (mgr.26054) 454 : cluster 0 pgmap v402: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 604 op/s; 58051/248343 objects degraded (23.375%); 11938/248343 objects misplaced (4.807%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T18:33:45.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:44 smithi173 bash[59422]: cluster 2023-09-23T18:33:44.452512+0000 mon.a (mon.0) 628 : cluster 3 Health check update: Degraded data redundancy: 58051/248343 objects degraded (23.375%), 14 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:45.288 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:45.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:45.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:45.391 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:45.436 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:45.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:45.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:45.540 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:45.571 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:45.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:45.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:45.675 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:45.708 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:45.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:45.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:45.817 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:45.875 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:45.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:45.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:45.984 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:46.024 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:46.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:46.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:46.028 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:45 smithi098 bash[76817]: audit 2023-09-23T18:33:45.723890+0000 mon.a (mon.0) 629 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:33:46.029 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:45 smithi098 bash[79516]: audit 2023-09-23T18:33:45.723890+0000 mon.a (mon.0) 629 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:33:46.127 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:46.138 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:45 smithi173 bash[59422]: audit 2023-09-23T18:33:45.723890+0000 mon.a (mon.0) 629 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:33:46.161 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:46.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:46.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:46.264 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:46.303 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:46.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:46.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:46.407 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:46.438 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:46.440 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:33:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:33:46] "GET /metrics HTTP/1.1" 200 36548 "" "Prometheus/2.33.4" 2023-09-23T18:33:46.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:46.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:46.542 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:46.571 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:46.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:46.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:46.676 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:46.715 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:46.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:46.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:46.820 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:46.842 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:46.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:46.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:46.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:46 smithi098 bash[79516]: cluster 2023-09-23T18:33:45.737009+0000 mgr.y (mgr.26054) 455 : cluster 0 pgmap v403: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 57859/244368 objects degraded (23.677%); 11938/244368 objects misplaced (4.885%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T18:33:46.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:46 smithi098 bash[76817]: cluster 2023-09-23T18:33:45.737009+0000 mgr.y (mgr.26054) 455 : cluster 0 pgmap v403: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 57859/244368 objects degraded (23.677%); 11938/244368 objects misplaced (4.885%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T18:33:46.947 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:46.954 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:46 smithi173 bash[59422]: cluster 2023-09-23T18:33:45.737009+0000 mgr.y (mgr.26054) 455 : cluster 0 pgmap v403: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 5.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 57859/244368 objects degraded (23.677%); 11938/244368 objects misplaced (4.885%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T18:33:46.970 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:46.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:46.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:47.075 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:47.104 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:47.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:47.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:47.208 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:47.234 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:47.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:47.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:47.340 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:47.378 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:47.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:47.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:47.481 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:47.507 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:47.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:47.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:47.611 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:47.642 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:47.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:47.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:47.747 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:47.778 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:47.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:47.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:47.881 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:47.911 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:47.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:47.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:48.015 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:48.048 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:48.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:48.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:48.155 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:48.187 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:48.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:48.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:48.291 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:48.317 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:33:48 smithi098 bash[114693]: debug 2023-09-23T18:33:48.199+0000 7f3a0a0e2640 -1 Falling back to public interface 2023-09-23T18:33:48.329 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:48.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:48.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:48.434 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:48.467 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:48.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:48.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:48.571 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:48.596 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:48.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:48.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:48.701 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:48.736 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:48.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:48.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:48.841 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:48.848 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:48 smithi098 bash[76817]: cluster 2023-09-23T18:33:47.737755+0000 mgr.y (mgr.26054) 456 : cluster 0 pgmap v404: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 4.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 57473/241002 objects degraded (23.848%); 11789/241002 objects misplaced (4.892%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:33:48.849 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:48 smithi098 bash[79516]: cluster 2023-09-23T18:33:47.737755+0000 mgr.y (mgr.26054) 456 : cluster 0 pgmap v404: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 4.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 57473/241002 objects degraded (23.848%); 11789/241002 objects misplaced (4.892%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:33:48.867 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:48.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:48.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:48.972 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:48.980 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:48 smithi173 bash[59422]: cluster 2023-09-23T18:33:47.737755+0000 mgr.y (mgr.26054) 456 : cluster 0 pgmap v404: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 4.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 57473/241002 objects degraded (23.848%); 11789/241002 objects misplaced (4.892%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:33:49.003 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:49.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:49.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:49.106 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:49.134 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:49.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:49.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:49.238 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:49.282 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:49.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:49.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:49.390 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:49.420 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:49.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:49.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:49.528 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:49.553 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:49.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:49.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:49.660 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:49.685 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:49.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:49.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:49.788 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:49.808 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:49 smithi098 bash[79516]: cluster 2023-09-23T18:33:49.453644+0000 mon.a (mon.0) 630 : cluster 3 Health check update: Degraded data redundancy: 57473/241002 objects degraded (23.848%), 14 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:49.812 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:49.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:49.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:49.919 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:49.926 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:49 smithi173 bash[59422]: cluster 2023-09-23T18:33:49.453644+0000 mon.a (mon.0) 630 : cluster 3 Health check update: Degraded data redundancy: 57473/241002 objects degraded (23.848%), 14 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:49.941 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:49.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:49.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:50.044 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:50.074 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:50.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:50.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:50.177 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:50.185 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:49 smithi098 bash[76817]: cluster 2023-09-23T18:33:49.453644+0000 mon.a (mon.0) 630 : cluster 3 Health check update: Degraded data redundancy: 57473/241002 objects degraded (23.848%), 14 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:50.218 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:50.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:50.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:50.322 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:50.347 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:50.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:50.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:50.453 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:50.488 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:50.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:50.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:50.592 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:50.625 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:50.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:50.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:50.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:50.765 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:50.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:50.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:50.872 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:50.881 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:50 smithi173 bash[59422]: cluster 2023-09-23T18:33:49.738631+0000 mgr.y (mgr.26054) 457 : cluster 0 pgmap v405: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 4.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 57110/239397 objects degraded (23.856%); 11610/239397 objects misplaced (4.850%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T18:33:50.909 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:50.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:50.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:51.013 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:51.039 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:51.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:51.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:51.062 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:50 smithi098 bash[79516]: cluster 2023-09-23T18:33:49.738631+0000 mgr.y (mgr.26054) 457 : cluster 0 pgmap v405: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 4.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 57110/239397 objects degraded (23.856%); 11610/239397 objects misplaced (4.850%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T18:33:51.062 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:50 smithi098 bash[76817]: cluster 2023-09-23T18:33:49.738631+0000 mgr.y (mgr.26054) 457 : cluster 0 pgmap v405: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 4.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 57110/239397 objects degraded (23.856%); 11610/239397 objects misplaced (4.850%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T18:33:51.145 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:51.192 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:51.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:51.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:51.298 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:51.340 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:51.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:51.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:51.445 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:51.473 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:51.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:51.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:51.579 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:51.609 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:51.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:51.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:51.690 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:33:51 smithi098 bash[114693]: debug 2023-09-23T18:33:51.326+0000 7f3a0a0e2640 -1 osd.1 104 log_to_monitors true 2023-09-23T18:33:51.712 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:51.735 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:51.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:51.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:51.838 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:51.848 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:51 smithi173 bash[59422]: audit 2023-09-23T18:33:51.360937+0000 mon.a (mon.0) 631 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T18:33:51.848 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:51 smithi173 bash[59422]: audit 2023-09-23T18:33:51.361039+0000 mon.b (mon.1) 4 : audit 1 from='osd.1 [v2:172.21.15.98:6810/1500977610,v1:172.21.15.98:6811/1500977610]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T18:33:51.871 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:51.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:51.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:51.976 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:51.984 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:51 smithi098 bash[79516]: audit 2023-09-23T18:33:51.360937+0000 mon.a (mon.0) 631 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T18:33:51.985 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:51 smithi098 bash[79516]: audit 2023-09-23T18:33:51.361039+0000 mon.b (mon.1) 4 : audit 1 from='osd.1 [v2:172.21.15.98:6810/1500977610,v1:172.21.15.98:6811/1500977610]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T18:33:51.985 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 18:33:51 smithi098 bash[114693]: debug 2023-09-23T18:33:51.854+0000 7f39f7d48700 -1 osd.1 104 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T18:33:51.985 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:51 smithi098 bash[76817]: audit 2023-09-23T18:33:51.360937+0000 mon.a (mon.0) 631 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T18:33:51.985 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:51 smithi098 bash[76817]: audit 2023-09-23T18:33:51.361039+0000 mon.b (mon.1) 4 : audit 1 from='osd.1 [v2:172.21.15.98:6810/1500977610,v1:172.21.15.98:6811/1500977610]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T18:33:52.009 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:52.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:52.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:52.114 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:52.158 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:52.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:52.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:52.264 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:52.288 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:52.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:52.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:52.394 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:52.426 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:52.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:52.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:52.532 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:52.564 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:52.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:52.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:52.668 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:52.722 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:52.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:52.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:52.828 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:52.861 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:52.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:52.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:52.964 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:52.972 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:52 smithi173 bash[59422]: cluster 2023-09-23T18:33:51.739886+0000 mgr.y (mgr.26054) 458 : cluster 0 pgmap v406: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 4.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 56736/235089 objects degraded (24.134%); 11610/235089 objects misplaced (4.939%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:33:52.973 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:52 smithi173 bash[59422]: audit 2023-09-23T18:33:51.817564+0000 mon.a (mon.0) 632 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T18:33:52.973 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:52 smithi173 bash[59422]: cluster 2023-09-23T18:33:51.817636+0000 mon.a (mon.0) 633 : cluster 0 osdmap e122: 8 total, 7 up, 7 in 2023-09-23T18:33:52.973 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:52 smithi173 bash[59422]: cluster 2023-09-23T18:33:51.823143+0000 osd.4 (osd.4) 52 : cluster 3 failed to encode map e122 with expected crc 2023-09-23T18:33:52.973 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:52 smithi173 bash[59422]: audit 2023-09-23T18:33:51.823926+0000 mon.a (mon.0) 634 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:33:52.973 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:52 smithi173 bash[59422]: audit 2023-09-23T18:33:51.824043+0000 mon.b (mon.1) 5 : audit 1 from='osd.1 [v2:172.21.15.98:6810/1500977610,v1:172.21.15.98:6811/1500977610]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:33:52.973 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:52 smithi173 bash[59422]: cluster 2023-09-23T18:33:51.882404+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e122 with expected crc 2023-09-23T18:33:52.973 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:52 smithi173 bash[59422]: audit 2023-09-23T18:33:52.189322+0000 mon.a (mon.0) 635 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:33:52.973 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:52 smithi173 bash[59422]: audit 2023-09-23T18:33:52.201226+0000 mon.a (mon.0) 636 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:33:52.990 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:52.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:52.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:53.094 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:53.101 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:52 smithi098 bash[76817]: cluster 2023-09-23T18:33:51.739886+0000 mgr.y (mgr.26054) 458 : cluster 0 pgmap v406: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 4.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 56736/235089 objects degraded (24.134%); 11610/235089 objects misplaced (4.939%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:33:53.101 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:52 smithi098 bash[76817]: audit 2023-09-23T18:33:51.817564+0000 mon.a (mon.0) 632 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T18:33:53.101 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:52 smithi098 bash[76817]: cluster 2023-09-23T18:33:51.817636+0000 mon.a (mon.0) 633 : cluster 0 osdmap e122: 8 total, 7 up, 7 in 2023-09-23T18:33:53.101 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:52 smithi098 bash[76817]: cluster 2023-09-23T18:33:51.823143+0000 osd.4 (osd.4) 52 : cluster 3 failed to encode map e122 with expected crc 2023-09-23T18:33:53.101 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:52 smithi098 bash[76817]: audit 2023-09-23T18:33:51.823926+0000 mon.a (mon.0) 634 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:33:53.101 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:52 smithi098 bash[76817]: audit 2023-09-23T18:33:51.824043+0000 mon.b (mon.1) 5 : audit 1 from='osd.1 [v2:172.21.15.98:6810/1500977610,v1:172.21.15.98:6811/1500977610]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:33:53.101 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:52 smithi098 bash[76817]: cluster 2023-09-23T18:33:51.882404+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e122 with expected crc 2023-09-23T18:33:53.101 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:52 smithi098 bash[76817]: audit 2023-09-23T18:33:52.189322+0000 mon.a (mon.0) 635 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:33:53.102 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:52 smithi098 bash[76817]: audit 2023-09-23T18:33:52.201226+0000 mon.a (mon.0) 636 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:33:53.102 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:52 smithi098 bash[79516]: cluster 2023-09-23T18:33:51.739886+0000 mgr.y (mgr.26054) 458 : cluster 0 pgmap v406: 89 pgs: 40 active+clean, 2 active+recovering+undersized+remapped, 22 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 11 active+clean+remapped; 4.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 56736/235089 objects degraded (24.134%); 11610/235089 objects misplaced (4.939%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:33:53.102 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:52 smithi098 bash[79516]: audit 2023-09-23T18:33:51.817564+0000 mon.a (mon.0) 632 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T18:33:53.102 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:52 smithi098 bash[79516]: cluster 2023-09-23T18:33:51.817636+0000 mon.a (mon.0) 633 : cluster 0 osdmap e122: 8 total, 7 up, 7 in 2023-09-23T18:33:53.102 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:52 smithi098 bash[79516]: cluster 2023-09-23T18:33:51.823143+0000 osd.4 (osd.4) 52 : cluster 3 failed to encode map e122 with expected crc 2023-09-23T18:33:53.102 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:52 smithi098 bash[79516]: audit 2023-09-23T18:33:51.823926+0000 mon.a (mon.0) 634 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:33:53.102 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:52 smithi098 bash[79516]: audit 2023-09-23T18:33:51.824043+0000 mon.b (mon.1) 5 : audit 1 from='osd.1 [v2:172.21.15.98:6810/1500977610,v1:172.21.15.98:6811/1500977610]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi098", "root=default"]}]: dispatch 2023-09-23T18:33:53.102 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:52 smithi098 bash[79516]: cluster 2023-09-23T18:33:51.882404+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e122 with expected crc 2023-09-23T18:33:53.102 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:52 smithi098 bash[79516]: audit 2023-09-23T18:33:52.189322+0000 mon.a (mon.0) 635 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:33:53.103 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:52 smithi098 bash[79516]: audit 2023-09-23T18:33:52.201226+0000 mon.a (mon.0) 636 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:33:53.112 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:53.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:53.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:53.218 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:53.249 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:53.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:53.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:53.353 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:53.376 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:53.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:53.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:53.479 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:53.509 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:53.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:53.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:53.613 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:53.636 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:53.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:53.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:53.739 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:53.772 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:53.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:53.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:53.875 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:53.905 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:53.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:53.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:53.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:53 smithi098 bash[79516]: cluster 2023-09-23T18:33:52.814383+0000 mon.a (mon.0) 637 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:33:53.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:53 smithi098 bash[79516]: cluster 2023-09-23T18:33:52.821024+0000 mon.a (mon.0) 638 : cluster 1 osd.1 [v2:172.21.15.98:6810/1500977610,v1:172.21.15.98:6811/1500977610] boot 2023-09-23T18:33:53.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:53 smithi098 bash[79516]: cluster 2023-09-23T18:33:52.821270+0000 mon.a (mon.0) 639 : cluster 0 osdmap e123: 8 total, 8 up, 7 in 2023-09-23T18:33:53.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:53 smithi098 bash[79516]: audit 2023-09-23T18:33:52.822708+0000 mon.a (mon.0) 640 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:33:53.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:53 smithi098 bash[79516]: cluster 2023-09-23T18:33:52.825516+0000 osd.4 (osd.4) 53 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T18:33:53.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:53 smithi098 bash[79516]: cluster 2023-09-23T18:33:52.826721+0000 osd.7 (osd.7) 36 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T18:33:53.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:53 smithi098 bash[79516]: cluster 2023-09-23T18:33:52.827152+0000 osd.6 (osd.6) 38 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T18:33:53.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:53 smithi098 bash[79516]: cluster 2023-09-23T18:33:52.831387+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T18:33:53.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:53 smithi098 bash[79516]: cluster 2023-09-23T18:33:53.822374+0000 mon.a (mon.0) 641 : cluster 0 osdmap e124: 8 total, 8 up, 7 in 2023-09-23T18:33:53.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:53 smithi098 bash[76817]: cluster 2023-09-23T18:33:52.814383+0000 mon.a (mon.0) 637 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:33:53.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:53 smithi098 bash[76817]: cluster 2023-09-23T18:33:52.821024+0000 mon.a (mon.0) 638 : cluster 1 osd.1 [v2:172.21.15.98:6810/1500977610,v1:172.21.15.98:6811/1500977610] boot 2023-09-23T18:33:53.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:53 smithi098 bash[76817]: cluster 2023-09-23T18:33:52.821270+0000 mon.a (mon.0) 639 : cluster 0 osdmap e123: 8 total, 8 up, 7 in 2023-09-23T18:33:53.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:53 smithi098 bash[76817]: audit 2023-09-23T18:33:52.822708+0000 mon.a (mon.0) 640 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:33:53.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:53 smithi098 bash[76817]: cluster 2023-09-23T18:33:52.825516+0000 osd.4 (osd.4) 53 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T18:33:53.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:53 smithi098 bash[76817]: cluster 2023-09-23T18:33:52.826721+0000 osd.7 (osd.7) 36 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T18:33:53.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:53 smithi098 bash[76817]: cluster 2023-09-23T18:33:52.827152+0000 osd.6 (osd.6) 38 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T18:33:53.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:53 smithi098 bash[76817]: cluster 2023-09-23T18:33:52.831387+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T18:33:53.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:53 smithi098 bash[76817]: cluster 2023-09-23T18:33:53.822374+0000 mon.a (mon.0) 641 : cluster 0 osdmap e124: 8 total, 8 up, 7 in 2023-09-23T18:33:54.008 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:54.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:53 smithi173 bash[59422]: cluster 2023-09-23T18:33:52.814383+0000 mon.a (mon.0) 637 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:33:54.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:53 smithi173 bash[59422]: cluster 2023-09-23T18:33:52.821024+0000 mon.a (mon.0) 638 : cluster 1 osd.1 [v2:172.21.15.98:6810/1500977610,v1:172.21.15.98:6811/1500977610] boot 2023-09-23T18:33:54.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:53 smithi173 bash[59422]: cluster 2023-09-23T18:33:52.821270+0000 mon.a (mon.0) 639 : cluster 0 osdmap e123: 8 total, 8 up, 7 in 2023-09-23T18:33:54.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:53 smithi173 bash[59422]: audit 2023-09-23T18:33:52.822708+0000 mon.a (mon.0) 640 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:33:54.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:53 smithi173 bash[59422]: cluster 2023-09-23T18:33:52.825516+0000 osd.4 (osd.4) 53 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T18:33:54.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:53 smithi173 bash[59422]: cluster 2023-09-23T18:33:52.826721+0000 osd.7 (osd.7) 36 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T18:33:54.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:53 smithi173 bash[59422]: cluster 2023-09-23T18:33:52.827152+0000 osd.6 (osd.6) 38 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T18:33:54.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:53 smithi173 bash[59422]: cluster 2023-09-23T18:33:52.831387+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e123 with expected crc 2023-09-23T18:33:54.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:53 smithi173 bash[59422]: cluster 2023-09-23T18:33:53.822374+0000 mon.a (mon.0) 641 : cluster 0 osdmap e124: 8 total, 8 up, 7 in 2023-09-23T18:33:54.031 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:54.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:54.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:54.133 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:54.153 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:54.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:54.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:54.256 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:54.278 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:54.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:54.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:54.381 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:54.402 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:54.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:54.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:54.505 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:54.535 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:54.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:54.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:54.638 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:54.671 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:54.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:54.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:54.775 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:54.801 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:54.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:54.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:54.849 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:54 smithi098 bash[76817]: cluster 2023-09-23T18:33:53.741018+0000 mgr.y (mgr.26054) 459 : cluster 0 pgmap v409: 89 pgs: 3 remapped+peering, 3 peering, 40 active+clean, 2 active+recovering+undersized+remapped, 21 active+undersized, 6 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 9 active+clean+remapped; 4.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 46675/231225 objects degraded (20.186%); 5853/231225 objects misplaced (2.531%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T18:33:54.849 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:54 smithi098 bash[76817]: cluster 2023-09-23T18:33:53.821333+0000 osd.4 (osd.4) 54 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T18:33:54.849 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:54 smithi098 bash[76817]: cluster 2023-09-23T18:33:53.823109+0000 osd.7 (osd.7) 37 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T18:33:54.849 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:54 smithi098 bash[76817]: cluster 2023-09-23T18:33:53.824050+0000 osd.4 (osd.4) 55 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T18:33:54.850 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:54 smithi098 bash[76817]: cluster 2023-09-23T18:33:53.832430+0000 osd.6 (osd.6) 39 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T18:33:54.850 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:54 smithi098 bash[76817]: cluster 2023-09-23T18:33:53.835543+0000 osd.6 (osd.6) 40 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T18:33:54.850 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:54 smithi098 bash[76817]: cluster 2023-09-23T18:33:53.837382+0000 osd.6 (osd.6) 41 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T18:33:54.850 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:54 smithi098 bash[76817]: cluster 2023-09-23T18:33:54.454856+0000 mon.a (mon.0) 642 : cluster 3 Health check update: Degraded data redundancy: 46675/231225 objects degraded (20.186%), 11 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:54.850 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:54 smithi098 bash[76817]: audit 2023-09-23T18:33:54.586719+0000 mon.a (mon.0) 643 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:33:54.850 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:54 smithi098 bash[76817]: audit 2023-09-23T18:33:54.587993+0000 mon.a (mon.0) 644 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:33:54.850 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:54 smithi098 bash[76817]: cluster 2023-09-23T18:33:54.827332+0000 mon.a (mon.0) 645 : cluster 0 osdmap e125: 8 total, 8 up, 7 in 2023-09-23T18:33:54.904 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:54.934 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:54.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:54.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:55.037 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:55.068 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:55.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:55.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:55.177 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:55.188 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:54 smithi098 bash[79516]: cluster 2023-09-23T18:33:53.741018+0000 mgr.y (mgr.26054) 459 : cluster 0 pgmap v409: 89 pgs: 3 remapped+peering, 3 peering, 40 active+clean, 2 active+recovering+undersized+remapped, 21 active+undersized, 6 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 9 active+clean+remapped; 4.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 46675/231225 objects degraded (20.186%); 5853/231225 objects misplaced (2.531%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T18:33:55.188 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:54 smithi098 bash[79516]: cluster 2023-09-23T18:33:53.821333+0000 osd.4 (osd.4) 54 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T18:33:55.188 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:54 smithi098 bash[79516]: cluster 2023-09-23T18:33:53.823109+0000 osd.7 (osd.7) 37 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T18:33:55.188 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:54 smithi098 bash[79516]: cluster 2023-09-23T18:33:53.824050+0000 osd.4 (osd.4) 55 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T18:33:55.189 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:54 smithi098 bash[79516]: cluster 2023-09-23T18:33:53.832430+0000 osd.6 (osd.6) 39 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T18:33:55.189 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:54 smithi098 bash[79516]: cluster 2023-09-23T18:33:53.835543+0000 osd.6 (osd.6) 40 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T18:33:55.189 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:54 smithi098 bash[79516]: cluster 2023-09-23T18:33:53.837382+0000 osd.6 (osd.6) 41 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T18:33:55.189 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:54 smithi098 bash[79516]: cluster 2023-09-23T18:33:54.454856+0000 mon.a (mon.0) 642 : cluster 3 Health check update: Degraded data redundancy: 46675/231225 objects degraded (20.186%), 11 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:55.189 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:54 smithi098 bash[79516]: audit 2023-09-23T18:33:54.586719+0000 mon.a (mon.0) 643 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:33:55.189 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:54 smithi098 bash[79516]: audit 2023-09-23T18:33:54.587993+0000 mon.a (mon.0) 644 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:33:55.189 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:54 smithi098 bash[79516]: cluster 2023-09-23T18:33:54.827332+0000 mon.a (mon.0) 645 : cluster 0 osdmap e125: 8 total, 8 up, 7 in 2023-09-23T18:33:55.236 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:55.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:55.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:55.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:54 smithi173 bash[59422]: cluster 2023-09-23T18:33:53.741018+0000 mgr.y (mgr.26054) 459 : cluster 0 pgmap v409: 89 pgs: 3 remapped+peering, 3 peering, 40 active+clean, 2 active+recovering+undersized+remapped, 21 active+undersized, 6 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 9 active+clean+remapped; 4.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 46675/231225 objects degraded (20.186%); 5853/231225 objects misplaced (2.531%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T18:33:55.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:54 smithi173 bash[59422]: cluster 2023-09-23T18:33:53.821333+0000 osd.4 (osd.4) 54 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T18:33:55.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:54 smithi173 bash[59422]: cluster 2023-09-23T18:33:53.823109+0000 osd.7 (osd.7) 37 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T18:33:55.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:54 smithi173 bash[59422]: cluster 2023-09-23T18:33:53.824050+0000 osd.4 (osd.4) 55 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T18:33:55.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:54 smithi173 bash[59422]: cluster 2023-09-23T18:33:53.832430+0000 osd.6 (osd.6) 39 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T18:33:55.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:54 smithi173 bash[59422]: cluster 2023-09-23T18:33:53.835543+0000 osd.6 (osd.6) 40 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T18:33:55.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:54 smithi173 bash[59422]: cluster 2023-09-23T18:33:53.837382+0000 osd.6 (osd.6) 41 : cluster 3 failed to encode map e124 with expected crc 2023-09-23T18:33:55.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:54 smithi173 bash[59422]: cluster 2023-09-23T18:33:54.454856+0000 mon.a (mon.0) 642 : cluster 3 Health check update: Degraded data redundancy: 46675/231225 objects degraded (20.186%), 11 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2023-09-23T18:33:55.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:54 smithi173 bash[59422]: audit 2023-09-23T18:33:54.586719+0000 mon.a (mon.0) 643 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:33:55.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:54 smithi173 bash[59422]: audit 2023-09-23T18:33:54.587993+0000 mon.a (mon.0) 644 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:33:55.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:54 smithi173 bash[59422]: cluster 2023-09-23T18:33:54.827332+0000 mon.a (mon.0) 645 : cluster 0 osdmap e125: 8 total, 8 up, 7 in 2023-09-23T18:33:55.343 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:55.373 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:55.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:55.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:55.480 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:55.518 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:55.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:55.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:55.622 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:55.662 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:55.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:55.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:55.768 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:55.789 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:55.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:55.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:55.893 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:55.925 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:55.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:55.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:55.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:55 smithi098 bash[76817]: cluster 2023-09-23T18:33:54.820896+0000 osd.5 (osd.5) 36 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T18:33:55.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:55 smithi098 bash[76817]: cluster 2023-09-23T18:33:54.828494+0000 osd.7 (osd.7) 38 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T18:33:55.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:55 smithi098 bash[76817]: cluster 2023-09-23T18:33:54.829003+0000 osd.6 (osd.6) 42 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T18:33:55.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:55 smithi098 bash[76817]: cluster 2023-09-23T18:33:54.829259+0000 osd.4 (osd.4) 56 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T18:33:55.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:55 smithi098 bash[76817]: cluster 2023-09-23T18:33:54.830562+0000 osd.3 (osd.3) 1 : cluster 0 5.0 starting backfill to osd.1 from (0'0,0'0] MAX to 122'8480 2023-09-23T18:33:55.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:55 smithi098 bash[76817]: cluster 2023-09-23T18:33:54.832050+0000 osd.0 (osd.0) 1 : cluster 0 5.c starting backfill to osd.1 from (0'0,0'0] MAX to 122'8590 2023-09-23T18:33:55.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:55 smithi098 bash[76817]: cluster 2023-09-23T18:33:54.832072+0000 osd.4 (osd.4) 57 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T18:33:55.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:55 smithi098 bash[76817]: cluster 2023-09-23T18:33:54.832771+0000 osd.6 (osd.6) 43 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T18:33:55.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:55 smithi098 bash[76817]: cluster 2023-09-23T18:33:54.833218+0000 osd.0 (osd.0) 2 : cluster 0 5.5 starting backfill to osd.1 from (0'0,0'0] MAX to 122'8498 2023-09-23T18:33:55.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:55 smithi098 bash[76817]: cluster 2023-09-23T18:33:54.835639+0000 osd.0 (osd.0) 3 : cluster 0 5.7 starting backfill to osd.1 from (0'0,0'0] MAX to 122'8526 2023-09-23T18:33:55.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:55 smithi098 bash[79516]: cluster 2023-09-23T18:33:54.820896+0000 osd.5 (osd.5) 36 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T18:33:55.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:55 smithi098 bash[79516]: cluster 2023-09-23T18:33:54.828494+0000 osd.7 (osd.7) 38 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T18:33:55.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:55 smithi098 bash[79516]: cluster 2023-09-23T18:33:54.829003+0000 osd.6 (osd.6) 42 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T18:33:55.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:55 smithi098 bash[79516]: cluster 2023-09-23T18:33:54.829259+0000 osd.4 (osd.4) 56 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T18:33:55.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:55 smithi098 bash[79516]: cluster 2023-09-23T18:33:54.830562+0000 osd.3 (osd.3) 1 : cluster 0 5.0 starting backfill to osd.1 from (0'0,0'0] MAX to 122'8480 2023-09-23T18:33:55.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:55 smithi098 bash[79516]: cluster 2023-09-23T18:33:54.832050+0000 osd.0 (osd.0) 1 : cluster 0 5.c starting backfill to osd.1 from (0'0,0'0] MAX to 122'8590 2023-09-23T18:33:55.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:55 smithi098 bash[79516]: cluster 2023-09-23T18:33:54.832072+0000 osd.4 (osd.4) 57 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T18:33:55.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:55 smithi098 bash[79516]: cluster 2023-09-23T18:33:54.832771+0000 osd.6 (osd.6) 43 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T18:33:55.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:55 smithi098 bash[79516]: cluster 2023-09-23T18:33:54.833218+0000 osd.0 (osd.0) 2 : cluster 0 5.5 starting backfill to osd.1 from (0'0,0'0] MAX to 122'8498 2023-09-23T18:33:55.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:55 smithi098 bash[79516]: cluster 2023-09-23T18:33:54.835639+0000 osd.0 (osd.0) 3 : cluster 0 5.7 starting backfill to osd.1 from (0'0,0'0] MAX to 122'8526 2023-09-23T18:33:56.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:55 smithi173 bash[59422]: cluster 2023-09-23T18:33:54.820896+0000 osd.5 (osd.5) 36 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T18:33:56.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:55 smithi173 bash[59422]: cluster 2023-09-23T18:33:54.828494+0000 osd.7 (osd.7) 38 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T18:33:56.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:55 smithi173 bash[59422]: cluster 2023-09-23T18:33:54.829003+0000 osd.6 (osd.6) 42 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T18:33:56.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:55 smithi173 bash[59422]: cluster 2023-09-23T18:33:54.829259+0000 osd.4 (osd.4) 56 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T18:33:56.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:55 smithi173 bash[59422]: cluster 2023-09-23T18:33:54.830562+0000 osd.3 (osd.3) 1 : cluster 0 5.0 starting backfill to osd.1 from (0'0,0'0] MAX to 122'8480 2023-09-23T18:33:56.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:55 smithi173 bash[59422]: cluster 2023-09-23T18:33:54.832050+0000 osd.0 (osd.0) 1 : cluster 0 5.c starting backfill to osd.1 from (0'0,0'0] MAX to 122'8590 2023-09-23T18:33:56.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:55 smithi173 bash[59422]: cluster 2023-09-23T18:33:54.832072+0000 osd.4 (osd.4) 57 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T18:33:56.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:55 smithi173 bash[59422]: cluster 2023-09-23T18:33:54.832771+0000 osd.6 (osd.6) 43 : cluster 3 failed to encode map e125 with expected crc 2023-09-23T18:33:56.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:55 smithi173 bash[59422]: cluster 2023-09-23T18:33:54.833218+0000 osd.0 (osd.0) 2 : cluster 0 5.5 starting backfill to osd.1 from (0'0,0'0] MAX to 122'8498 2023-09-23T18:33:56.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:55 smithi173 bash[59422]: cluster 2023-09-23T18:33:54.835639+0000 osd.0 (osd.0) 3 : cluster 0 5.7 starting backfill to osd.1 from (0'0,0'0] MAX to 122'8526 2023-09-23T18:33:56.032 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:56.069 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:56.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:56.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:56.176 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:56.206 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:56.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:56.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:56.311 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:56.320 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:33:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:33:56] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.33.4" 2023-09-23T18:33:56.339 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:56.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:56.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:56.443 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:56.475 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:56.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:56.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:56.578 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:56.607 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:56.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:56.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:56.711 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:56.752 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:56.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:56.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:56.855 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:56.889 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:56.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:56.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:56.994 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:57.021 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:57.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:57.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:57.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:56 smithi173 bash[59422]: cluster 2023-09-23T18:33:55.741895+0000 mgr.y (mgr.26054) 460 : cluster 0 pgmap v412: 89 pgs: 3 remapped+peering, 3 peering, 54 active+clean, 2 active+recovering+undersized+remapped, 13 active+undersized, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 4 active+clean+remapped; 4.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 456 op/s; 43265/229611 objects degraded (18.843%); 9233/229611 objects misplaced (4.021%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T18:33:57.124 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:57.145 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:57.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:57.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:57.164 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:56 smithi098 bash[76817]: cluster 2023-09-23T18:33:55.741895+0000 mgr.y (mgr.26054) 460 : cluster 0 pgmap v412: 89 pgs: 3 remapped+peering, 3 peering, 54 active+clean, 2 active+recovering+undersized+remapped, 13 active+undersized, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 4 active+clean+remapped; 4.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 456 op/s; 43265/229611 objects degraded (18.843%); 9233/229611 objects misplaced (4.021%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T18:33:57.165 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:56 smithi098 bash[79516]: cluster 2023-09-23T18:33:55.741895+0000 mgr.y (mgr.26054) 460 : cluster 0 pgmap v412: 89 pgs: 3 remapped+peering, 3 peering, 54 active+clean, 2 active+recovering+undersized+remapped, 13 active+undersized, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 4 active+clean+remapped; 4.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 456 op/s; 43265/229611 objects degraded (18.843%); 9233/229611 objects misplaced (4.021%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T18:33:57.249 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:57.277 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:57.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:57.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:57.380 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:57.416 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:57.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:57.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:57.520 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:57.541 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:57.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:57.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:57.643 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:57.692 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:57.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:57.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:57.797 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:57.817 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:57.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:57.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:57.924 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:57.958 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:33:57.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:57.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:33:58.062 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:58.087 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:58.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:58.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:58.190 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:58.212 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:58.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:58.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:58.320 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:58.343 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:58.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:58.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:58.448 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:58.486 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:58.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:58.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:58.589 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:58.663 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:33:58.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:58.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:33:58.767 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:58.830 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:58.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:58.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:58.935 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:58.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:33:58 smithi098 bash[79516]: cluster 2023-09-23T18:33:57.742529+0000 mgr.y (mgr.26054) 461 : cluster 0 pgmap v413: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 remapped+peering, 3 peering, 73 active+clean, 2 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded; 4.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 390 op/s; 44897/228144 objects degraded (19.679%); 8985/228144 objects misplaced (3.938%); 8.3 MiB/s, 131 objects/s recovering 2023-09-23T18:33:58.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:33:58 smithi098 bash[76817]: cluster 2023-09-23T18:33:57.742529+0000 mgr.y (mgr.26054) 461 : cluster 0 pgmap v413: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 remapped+peering, 3 peering, 73 active+clean, 2 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded; 4.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 390 op/s; 44897/228144 objects degraded (19.679%); 8985/228144 objects misplaced (3.938%); 8.3 MiB/s, 131 objects/s recovering 2023-09-23T18:33:58.959 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:58.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:58.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:59.062 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:59.097 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:59.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:59.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:59.201 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:59.250 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:59.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:59.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:59.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:33:58 smithi173 bash[59422]: cluster 2023-09-23T18:33:57.742529+0000 mgr.y (mgr.26054) 461 : cluster 0 pgmap v413: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 remapped+peering, 3 peering, 73 active+clean, 2 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded; 4.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 390 op/s; 44897/228144 objects degraded (19.679%); 8985/228144 objects misplaced (3.938%); 8.3 MiB/s, 131 objects/s recovering 2023-09-23T18:33:59.360 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:59.381 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:33:59.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:59.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:33:59.485 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:59.516 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:33:59.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:59.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:33:59.624 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:59.645 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:33:59.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:59.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:33:59.748 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:59.772 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:33:59.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:59.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:33:59.876 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:59.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:33:59.917 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:33:59.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:33:59.959 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:33:59.959 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:34:00.018 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:00.048 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:00.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:00.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:00.154 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:00.183 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:00.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:00.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:00.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:00 smithi173 bash[59422]: audit 2023-09-23T18:33:59.133369+0000 mon.a (mon.0) 646 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:34:00.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:00 smithi173 bash[59422]: audit 2023-09-23T18:33:59.149719+0000 mon.a (mon.0) 647 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:34:00.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:00 smithi173 bash[59422]: cluster 2023-09-23T18:33:59.455876+0000 mon.a (mon.0) 648 : cluster 3 Health check update: Degraded data redundancy: 44897/228144 objects degraded (19.679%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:00.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:00 smithi173 bash[59422]: audit 2023-09-23T18:33:59.734792+0000 mon.a (mon.0) 649 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:34:00.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:00 smithi173 bash[59422]: audit 2023-09-23T18:33:59.738840+0000 mon.a (mon.0) 650 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:34:00.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:00 smithi173 bash[59422]: cluster 2023-09-23T18:33:59.743178+0000 mgr.y (mgr.26054) 462 : cluster 0 pgmap v414: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 280 op/s; 56743/226728 objects degraded (25.027%); 11527/226728 objects misplaced (5.084%); 7.8 MiB/s, 123 objects/s recovering 2023-09-23T18:34:00.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:00 smithi173 bash[59422]: audit 2023-09-23T18:33:59.745459+0000 mon.a (mon.0) 651 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:34:00.292 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:00.327 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:00.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:00.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:00.430 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:00.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:00 smithi098 bash[79516]: audit 2023-09-23T18:33:59.133369+0000 mon.a (mon.0) 646 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:34:00.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:00 smithi098 bash[79516]: audit 2023-09-23T18:33:59.149719+0000 mon.a (mon.0) 647 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:34:00.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:00 smithi098 bash[79516]: cluster 2023-09-23T18:33:59.455876+0000 mon.a (mon.0) 648 : cluster 3 Health check update: Degraded data redundancy: 44897/228144 objects degraded (19.679%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:00.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:00 smithi098 bash[79516]: audit 2023-09-23T18:33:59.734792+0000 mon.a (mon.0) 649 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:34:00.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:00 smithi098 bash[79516]: audit 2023-09-23T18:33:59.738840+0000 mon.a (mon.0) 650 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:34:00.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:00 smithi098 bash[79516]: cluster 2023-09-23T18:33:59.743178+0000 mgr.y (mgr.26054) 462 : cluster 0 pgmap v414: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 280 op/s; 56743/226728 objects degraded (25.027%); 11527/226728 objects misplaced (5.084%); 7.8 MiB/s, 123 objects/s recovering 2023-09-23T18:34:00.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:00 smithi098 bash[79516]: audit 2023-09-23T18:33:59.745459+0000 mon.a (mon.0) 651 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:34:00.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:00 smithi098 bash[76817]: audit 2023-09-23T18:33:59.133369+0000 mon.a (mon.0) 646 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:34:00.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:00 smithi098 bash[76817]: audit 2023-09-23T18:33:59.149719+0000 mon.a (mon.0) 647 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:34:00.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:00 smithi098 bash[76817]: cluster 2023-09-23T18:33:59.455876+0000 mon.a (mon.0) 648 : cluster 3 Health check update: Degraded data redundancy: 44897/228144 objects degraded (19.679%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:00.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:00 smithi098 bash[76817]: audit 2023-09-23T18:33:59.734792+0000 mon.a (mon.0) 649 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:34:00.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:00 smithi098 bash[76817]: audit 2023-09-23T18:33:59.738840+0000 mon.a (mon.0) 650 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:34:00.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:00 smithi098 bash[76817]: cluster 2023-09-23T18:33:59.743178+0000 mgr.y (mgr.26054) 462 : cluster 0 pgmap v414: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 280 op/s; 56743/226728 objects degraded (25.027%); 11527/226728 objects misplaced (5.084%); 7.8 MiB/s, 123 objects/s recovering 2023-09-23T18:34:00.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:00 smithi098 bash[76817]: audit 2023-09-23T18:33:59.745459+0000 mon.a (mon.0) 651 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:34:00.458 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:00.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:00.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:00.563 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:00.603 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:00.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:00.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:00.710 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:00.733 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:00.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:00.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:00.839 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:00.864 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:00.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:00.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:00.968 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:01.004 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:01.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:01.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:01.109 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:01.151 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:01.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:01.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:01.254 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:01.277 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:01.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:01.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:01.381 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:01.407 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:01.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:01.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:01.457 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:34:01.499 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:34:01.514 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:01.540 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:01.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:01.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:01.546 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:34:01.589 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:34:01.631 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:34:01.646 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:01.675 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:01.677 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:34:01.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:01.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:01.705 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:34:01.750 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:34:01.752 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:34:01.753 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:34:01.753 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:34:01.753 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:34:01.753 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:34:01.753 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:34:01.753 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:34:01.753 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:34:01.776 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:01 smithi173 bash[59422]: audit 2023-09-23T18:34:01.376144+0000 mon.c (mon.2) 62 : audit 0 from='client.? 172.21.15.98:0/948209317' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:34:01.783 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:01.805 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:01.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:01.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:01.908 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:01.935 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:01.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:01.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:02.038 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:02.046 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:01 smithi098 bash[76817]: audit 2023-09-23T18:34:01.376144+0000 mon.c (mon.2) 62 : audit 0 from='client.? 172.21.15.98:0/948209317' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:34:02.047 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:01 smithi098 bash[79516]: audit 2023-09-23T18:34:01.376144+0000 mon.c (mon.2) 62 : audit 0 from='client.? 172.21.15.98:0/948209317' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:34:02.071 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:02.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:02.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:02.177 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:02.204 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:02.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:02.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:02.312 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:02.337 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:02.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:02.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:02.440 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:02.474 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:02.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:02.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:02.581 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:02.605 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:02.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:02.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:02.712 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:02.734 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:02.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:02.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:02.839 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:02.862 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:02.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:02.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:02.968 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:02.994 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:03.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:03.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:03.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:02 smithi173 bash[59422]: cluster 2023-09-23T18:34:01.743975+0000 mgr.y (mgr.26054) 463 : cluster 0 pgmap v415: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 312 op/s; 55975/223728 objects degraded (25.019%); 11395/223728 objects misplaced (5.093%); 11 MiB/s, 170 objects/s recovering 2023-09-23T18:34:03.028 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:34:03.028 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [], 2023-09-23T18:34:03.028 INFO:teuthology.orchestra.run.smithi098.stdout: "num_ops": 0 2023-09-23T18:34:03.028 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:34:03.102 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:03.106 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:02 smithi098 bash[79516]: cluster 2023-09-23T18:34:01.743975+0000 mgr.y (mgr.26054) 463 : cluster 0 pgmap v415: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 312 op/s; 55975/223728 objects degraded (25.019%); 11395/223728 objects misplaced (5.093%); 11 MiB/s, 170 objects/s recovering 2023-09-23T18:34:03.106 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:02 smithi098 bash[76817]: cluster 2023-09-23T18:34:01.743975+0000 mgr.y (mgr.26054) 463 : cluster 0 pgmap v415: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 312 op/s; 55975/223728 objects degraded (25.019%); 11395/223728 objects misplaced (5.093%); 11 MiB/s, 170 objects/s recovering 2023-09-23T18:34:03.127 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:03.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:03.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:03.240 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:03.267 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:03.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:03.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:03.370 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:03.396 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:03.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:03.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:03.499 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:03.530 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:03.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:03.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:03.635 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:03.668 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:03.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:03.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:03.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:03.801 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:03.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:03.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:03.841 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is unset 2023-09-23T18:34:03.843 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T18:34:03.904 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:03.913 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:03 smithi173 bash[59422]: audit 2023-09-23T18:34:03.685078+0000 mon.c (mon.2) 63 : audit 1 from='client.? 172.21.15.98:0/1299462610' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:34:03.914 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:03 smithi173 bash[59422]: audit 2023-09-23T18:34:03.685604+0000 mon.a (mon.0) 652 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:34:03.942 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:03.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:03.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:04.045 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:04.073 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:04.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:04.076 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:03 smithi098 bash[76817]: audit 2023-09-23T18:34:03.685078+0000 mon.c (mon.2) 63 : audit 1 from='client.? 172.21.15.98:0/1299462610' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:34:04.076 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:03 smithi098 bash[76817]: audit 2023-09-23T18:34:03.685604+0000 mon.a (mon.0) 652 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:34:04.077 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:03 smithi098 bash[79516]: audit 2023-09-23T18:34:03.685078+0000 mon.c (mon.2) 63 : audit 1 from='client.? 172.21.15.98:0/1299462610' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:34:04.077 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:03 smithi098 bash[79516]: audit 2023-09-23T18:34:03.685604+0000 mon.a (mon.0) 652 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:34:04.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:04.178 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:04.202 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:04.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:04.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:04.311 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:04.333 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:04.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:04.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:04.437 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:04.462 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:04.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:04.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:04.564 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:04.592 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:04.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:04.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:04.696 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:04.719 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:04.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:04.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:04.822 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:04.844 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:04.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:04.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:04.947 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:04.982 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:04.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:04.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:05.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:04 smithi173 bash[59422]: cluster 2023-09-23T18:34:03.744633+0000 mgr.y (mgr.26054) 464 : cluster 0 pgmap v416: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 381 op/s; 55542/219861 objects degraded (25.262%); 11101/219861 objects misplaced (5.049%); 10 MiB/s, 166 objects/s recovering 2023-09-23T18:34:05.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:04 smithi173 bash[59422]: cluster 2023-09-23T18:34:03.804610+0000 mon.a (mon.0) 653 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:34:05.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:04 smithi173 bash[59422]: cluster 2023-09-23T18:34:03.825170+0000 osd.5 (osd.5) 37 : cluster 3 failed to encode map e126 with expected crc 2023-09-23T18:34:05.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:04 smithi173 bash[59422]: cluster 2023-09-23T18:34:03.825422+0000 osd.4 (osd.4) 58 : cluster 3 failed to encode map e126 with expected crc 2023-09-23T18:34:05.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:04 smithi173 bash[59422]: audit 2023-09-23T18:34:03.825791+0000 mon.a (mon.0) 654 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:34:05.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:04 smithi173 bash[59422]: cluster 2023-09-23T18:34:03.826031+0000 mon.a (mon.0) 655 : cluster 0 osdmap e126: 8 total, 8 up, 7 in 2023-09-23T18:34:05.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:04 smithi173 bash[59422]: cluster 2023-09-23T18:34:04.457147+0000 mon.a (mon.0) 656 : cluster 3 Health check update: Degraded data redundancy: 55542/219861 objects degraded (25.262%), 12 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:05.086 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:05.094 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:04 smithi098 bash[79516]: cluster 2023-09-23T18:34:03.744633+0000 mgr.y (mgr.26054) 464 : cluster 0 pgmap v416: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 381 op/s; 55542/219861 objects degraded (25.262%); 11101/219861 objects misplaced (5.049%); 10 MiB/s, 166 objects/s recovering 2023-09-23T18:34:05.094 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:04 smithi098 bash[79516]: cluster 2023-09-23T18:34:03.804610+0000 mon.a (mon.0) 653 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:34:05.094 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:04 smithi098 bash[79516]: cluster 2023-09-23T18:34:03.825170+0000 osd.5 (osd.5) 37 : cluster 3 failed to encode map e126 with expected crc 2023-09-23T18:34:05.094 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:04 smithi098 bash[79516]: cluster 2023-09-23T18:34:03.825422+0000 osd.4 (osd.4) 58 : cluster 3 failed to encode map e126 with expected crc 2023-09-23T18:34:05.094 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:04 smithi098 bash[79516]: audit 2023-09-23T18:34:03.825791+0000 mon.a (mon.0) 654 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:34:05.094 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:04 smithi098 bash[79516]: cluster 2023-09-23T18:34:03.826031+0000 mon.a (mon.0) 655 : cluster 0 osdmap e126: 8 total, 8 up, 7 in 2023-09-23T18:34:05.094 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:04 smithi098 bash[79516]: cluster 2023-09-23T18:34:04.457147+0000 mon.a (mon.0) 656 : cluster 3 Health check update: Degraded data redundancy: 55542/219861 objects degraded (25.262%), 12 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:05.094 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:04 smithi098 bash[76817]: cluster 2023-09-23T18:34:03.744633+0000 mgr.y (mgr.26054) 464 : cluster 0 pgmap v416: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 381 op/s; 55542/219861 objects degraded (25.262%); 11101/219861 objects misplaced (5.049%); 10 MiB/s, 166 objects/s recovering 2023-09-23T18:34:05.095 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:04 smithi098 bash[76817]: cluster 2023-09-23T18:34:03.804610+0000 mon.a (mon.0) 653 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:34:05.095 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:04 smithi098 bash[76817]: cluster 2023-09-23T18:34:03.825170+0000 osd.5 (osd.5) 37 : cluster 3 failed to encode map e126 with expected crc 2023-09-23T18:34:05.095 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:04 smithi098 bash[76817]: cluster 2023-09-23T18:34:03.825422+0000 osd.4 (osd.4) 58 : cluster 3 failed to encode map e126 with expected crc 2023-09-23T18:34:05.095 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:04 smithi098 bash[76817]: audit 2023-09-23T18:34:03.825791+0000 mon.a (mon.0) 654 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:34:05.095 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:04 smithi098 bash[76817]: cluster 2023-09-23T18:34:03.826031+0000 mon.a (mon.0) 655 : cluster 0 osdmap e126: 8 total, 8 up, 7 in 2023-09-23T18:34:05.095 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:04 smithi098 bash[76817]: cluster 2023-09-23T18:34:04.457147+0000 mon.a (mon.0) 656 : cluster 3 Health check update: Degraded data redundancy: 55542/219861 objects degraded (25.262%), 12 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:05.113 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:05.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:05.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:05.216 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:05.257 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:05.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:05.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:05.362 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:05.393 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:05.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:05.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:05.496 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:05.528 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:05.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:05.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:05.631 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:05.655 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:05.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:05.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:05.758 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:05.818 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:05.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:05.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:05.924 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:05.947 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:05.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:05.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:06.051 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:06.079 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:06.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:06.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:06.085 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:34:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:34:06] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.33.4" 2023-09-23T18:34:06.181 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:06.215 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:06.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:06.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:06.321 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:06.341 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:06.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:06.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:06.445 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:06.479 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:06.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:06.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:06.584 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:06.620 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:06.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:06.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:06.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:06.751 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:06.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:06.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:06.855 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:06.881 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:06.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:06.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:06.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:06 smithi098 bash[79516]: cluster 2023-09-23T18:34:05.746023+0000 mgr.y (mgr.26054) 465 : cluster 0 pgmap v418: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 412 op/s; 55557/217239 objects degraded (25.574%); 10955/217239 objects misplaced (5.043%); 11 MiB/s, 175 objects/s recovering 2023-09-23T18:34:06.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:06 smithi098 bash[76817]: cluster 2023-09-23T18:34:05.746023+0000 mgr.y (mgr.26054) 465 : cluster 0 pgmap v418: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 412 op/s; 55557/217239 objects degraded (25.574%); 10955/217239 objects misplaced (5.043%); 11 MiB/s, 175 objects/s recovering 2023-09-23T18:34:06.990 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:07.002 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:06 smithi173 bash[59422]: cluster 2023-09-23T18:34:05.746023+0000 mgr.y (mgr.26054) 465 : cluster 0 pgmap v418: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 412 op/s; 55557/217239 objects degraded (25.574%); 10955/217239 objects misplaced (5.043%); 11 MiB/s, 175 objects/s recovering 2023-09-23T18:34:07.014 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:07.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:07.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:07.119 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:07.152 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:07.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:07.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:07.237 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T18:34:07.255 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:07.286 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:07.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:07.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:07.392 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:07.429 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:07.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:07.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:07.532 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:07.556 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:07.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:07.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:07.658 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:07.685 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:07.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:07.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:07.794 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:07.831 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:07.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:07.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:07.940 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:07.973 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:07.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:07.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:08.079 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:08.110 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:08.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:08.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:08.214 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:08.240 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:08.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:08.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:08.343 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:08.372 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:08.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:08.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:08.476 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:08.501 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:08.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:08.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:08.603 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:34:08.605 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:08.641 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:08.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:08.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:08.665 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:34:08.752 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:08.795 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:08.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:08.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:08.905 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:08.932 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:08.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:08.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:08.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:08 smithi098 bash[79516]: cluster 2023-09-23T18:34:07.746887+0000 mgr.y (mgr.26054) 466 : cluster 0 pgmap v419: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 446 op/s; 54709/214758 objects degraded (25.475%); 10787/214758 objects misplaced (5.023%); 15 MiB/s, 232 objects/s recovering 2023-09-23T18:34:08.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:08 smithi098 bash[76817]: cluster 2023-09-23T18:34:07.746887+0000 mgr.y (mgr.26054) 466 : cluster 0 pgmap v419: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 446 op/s; 54709/214758 objects degraded (25.475%); 10787/214758 objects misplaced (5.023%); 15 MiB/s, 232 objects/s recovering 2023-09-23T18:34:09.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:08 smithi173 bash[59422]: cluster 2023-09-23T18:34:07.746887+0000 mgr.y (mgr.26054) 466 : cluster 0 pgmap v419: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 446 op/s; 54709/214758 objects degraded (25.475%); 10787/214758 objects misplaced (5.023%); 15 MiB/s, 232 objects/s recovering 2023-09-23T18:34:09.035 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:09.070 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:09.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:09.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:09.175 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:09.204 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:09.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:09.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:09.258 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:34:09.307 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:09.341 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:09.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:09.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:09.445 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:09.483 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:09.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:09.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:09.589 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:09.615 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:09.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:09.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:09.719 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:09.745 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:09.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:09.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:09.848 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:09.879 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:09.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:09.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:09.982 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:09.991 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:09 smithi173 bash[59422]: cluster 2023-09-23T18:34:09.458557+0000 mon.a (mon.0) 657 : cluster 3 Health check update: Degraded data redundancy: 54709/214758 objects degraded (25.475%), 12 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:09.991 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:09 smithi173 bash[59422]: audit 2023-09-23T18:34:09.586353+0000 mon.a (mon.0) 658 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:34:09.991 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:09 smithi173 bash[59422]: audit 2023-09-23T18:34:09.587564+0000 mon.a (mon.0) 659 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:34:10.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:10.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:10.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:10.121 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:10.136 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:09 smithi098 bash[79516]: cluster 2023-09-23T18:34:09.458557+0000 mon.a (mon.0) 657 : cluster 3 Health check update: Degraded data redundancy: 54709/214758 objects degraded (25.475%), 12 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:10.136 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:09 smithi098 bash[79516]: audit 2023-09-23T18:34:09.586353+0000 mon.a (mon.0) 658 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:34:10.136 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:09 smithi098 bash[79516]: audit 2023-09-23T18:34:09.587564+0000 mon.a (mon.0) 659 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:34:10.137 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:09 smithi098 bash[76817]: cluster 2023-09-23T18:34:09.458557+0000 mon.a (mon.0) 657 : cluster 3 Health check update: Degraded data redundancy: 54709/214758 objects degraded (25.475%), 12 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:10.138 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:09 smithi098 bash[76817]: audit 2023-09-23T18:34:09.586353+0000 mon.a (mon.0) 658 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:34:10.138 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:09 smithi098 bash[76817]: audit 2023-09-23T18:34:09.587564+0000 mon.a (mon.0) 659 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:34:10.155 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:10.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:10.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:10.260 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:10.302 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:10.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:10.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:10.406 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:10.428 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:10.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:10.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:10.530 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:10.567 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:10.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:10.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:10.672 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:10.696 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:10.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:10.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:10.798 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:10.823 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:10 smithi098 bash[76817]: cluster 2023-09-23T18:34:09.747598+0000 mgr.y (mgr.26054) 467 : cluster 0 pgmap v420: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 463 op/s; 54235/212808 objects degraded (25.485%); 10648/212808 objects misplaced (5.004%); 14 MiB/s, 224 objects/s recovering 2023-09-23T18:34:10.829 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:10.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:10.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:10.932 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:10.968 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:10.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:10.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:11.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:10 smithi173 bash[59422]: cluster 2023-09-23T18:34:09.747598+0000 mgr.y (mgr.26054) 467 : cluster 0 pgmap v420: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 463 op/s; 54235/212808 objects degraded (25.485%); 10648/212808 objects misplaced (5.004%); 14 MiB/s, 224 objects/s recovering 2023-09-23T18:34:11.074 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:11.098 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:11.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:11.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:11.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:10 smithi098 bash[79516]: cluster 2023-09-23T18:34:09.747598+0000 mgr.y (mgr.26054) 467 : cluster 0 pgmap v420: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 463 op/s; 54235/212808 objects degraded (25.485%); 10648/212808 objects misplaced (5.004%); 14 MiB/s, 224 objects/s recovering 2023-09-23T18:34:11.200 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:11.240 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:11.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:11.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:11.346 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:11.373 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:11.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:11.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:11.478 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:11.503 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:11.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:11.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:11.612 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:11.643 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:11.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:11.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:11.748 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:11.781 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:11.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:11.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:11.886 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:11.927 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:11.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:11.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:12.036 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:12.062 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:12.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:12.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:12.086 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:34:12.165 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:12.194 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:12.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:12.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:12.297 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:12.331 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:12.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:12.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:12.438 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:12.460 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:12.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:12.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:12.564 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:12.587 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:12.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:12.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:12.692 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:12.718 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:12.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:12.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:12.822 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:12.851 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:12.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:12.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:12.956 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:12.964 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:12 smithi098 bash[79516]: cluster 2023-09-23T18:34:11.748287+0000 mgr.y (mgr.26054) 468 : cluster 0 pgmap v421: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 469 op/s; 53536/209640 objects degraded (25.537%); 10507/209640 objects misplaced (5.012%); 14 MiB/s, 222 objects/s recovering 2023-09-23T18:34:12.965 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:12 smithi098 bash[76817]: cluster 2023-09-23T18:34:11.748287+0000 mgr.y (mgr.26054) 468 : cluster 0 pgmap v421: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 469 op/s; 53536/209640 objects degraded (25.537%); 10507/209640 objects misplaced (5.012%); 14 MiB/s, 222 objects/s recovering 2023-09-23T18:34:12.986 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:12.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:12.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:13.092 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:13.100 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:12 smithi173 bash[59422]: cluster 2023-09-23T18:34:11.748287+0000 mgr.y (mgr.26054) 468 : cluster 0 pgmap v421: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 469 op/s; 53536/209640 objects degraded (25.537%); 10507/209640 objects misplaced (5.012%); 14 MiB/s, 222 objects/s recovering 2023-09-23T18:34:13.112 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:34:13.113 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:34:13.119 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:13.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:13.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:13.226 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:13.269 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:13.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:13.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:13.376 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:13.406 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:13.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:13.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:13.509 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:13.534 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:13.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:13.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:13.640 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:13.671 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:13.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:13.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:13.776 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:13.805 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:13.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:13.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:13.908 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:13.936 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:13.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:13.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:14.041 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:14.064 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:14.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:14.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:14.166 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:14.218 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:14.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:14.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:14.323 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:14.351 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:14.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:14.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:14.455 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:14.493 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:14.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:14.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:14.596 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:14.632 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:14.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:14.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:14.735 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:14.760 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:14.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:14.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:14.863 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:14.902 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:14.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:14.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:15.008 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:15.017 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:14 smithi098 bash[79516]: cluster 2023-09-23T18:34:13.748885+0000 mgr.y (mgr.26054) 469 : cluster 0 pgmap v422: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 414 op/s; 53536/207414 objects degraded (25.811%); 10344/207414 objects misplaced (4.987%); 13 MiB/s, 209 objects/s recovering 2023-09-23T18:34:15.017 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:14 smithi098 bash[79516]: cluster 2023-09-23T18:34:14.460102+0000 mon.a (mon.0) 660 : cluster 3 Health check update: Degraded data redundancy: 53536/207414 objects degraded (25.811%), 12 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:15.017 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:14 smithi098 bash[76817]: cluster 2023-09-23T18:34:13.748885+0000 mgr.y (mgr.26054) 469 : cluster 0 pgmap v422: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 414 op/s; 53536/207414 objects degraded (25.811%); 10344/207414 objects misplaced (4.987%); 13 MiB/s, 209 objects/s recovering 2023-09-23T18:34:15.018 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:14 smithi098 bash[76817]: cluster 2023-09-23T18:34:14.460102+0000 mon.a (mon.0) 660 : cluster 3 Health check update: Degraded data redundancy: 53536/207414 objects degraded (25.811%), 12 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:15.033 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:15.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:15.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:15.119 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:34:15.120 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-09-23T18:34:15.120 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.120 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:145600 5.f 5:fb30f577:::benchmark_data_smithi098_105788_object39315:head [delete] snapc 0=[] ondisk+write+known_if_redirected e126)", 2023-09-23T18:34:15.120 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:34:15.101299+0000", 2023-09-23T18:34:15.120 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.016909587, 2023-09-23T18:34:15.120 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.016942195, 2023-09-23T18:34:15.120 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:34:15.120 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:34:15.120 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:34:15.121 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:34:15.121 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:34:15.121 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 145600 2023-09-23T18:34:15.121 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.121 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:34:15.121 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.121 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:34:15.121 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.101299+0000", 2023-09-23T18:34:15.121 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:34:15.121 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.121 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.121 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:34:15.122 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.101299+0000", 2023-09-23T18:34:15.122 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:34:15.122 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.122 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.122 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:34:15.122 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.101297+0000", 2023-09-23T18:34:15.122 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.7260000000000008e-06 2023-09-23T18:34:15.122 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.122 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.122 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:34:15.122 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.101305+0000", 2023-09-23T18:34:15.122 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6626999999999999e-05 2023-09-23T18:34:15.122 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.123 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.123 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:34:15.123 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.101322+0000", 2023-09-23T18:34:15.123 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.7800000000000003e-06 2023-09-23T18:34:15.123 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.123 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.123 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:34:15.123 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.101329+0000", 2023-09-23T18:34:15.123 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0038348420000000002 2023-09-23T18:34:15.123 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.123 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.123 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:34:15.123 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.105164+0000", 2023-09-23T18:34:15.124 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000145964 2023-09-23T18:34:15.124 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.124 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.124 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:34:15.124 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.105310+0000", 2023-09-23T18:34:15.124 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00017680699999999999 2023-09-23T18:34:15.124 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.124 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.124 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,6", 2023-09-23T18:34:15.124 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.105487+0000", 2023-09-23T18:34:15.124 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.005481367 2023-09-23T18:34:15.124 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.125 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.125 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:34:15.125 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.110968+0000", 2023-09-23T18:34:15.125 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1912e-05 2023-09-23T18:34:15.125 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.125 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.125 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:34:15.125 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.110980+0000", 2023-09-23T18:34:15.125 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0096812059999999995 2023-09-23T18:34:15.125 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:34:15.125 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:34:15.125 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:34:15.125 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.126 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.126 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:145606 5.1 5:8e459c2a:::benchmark_data_smithi098_105788_object39321:head [delete] snapc 0=[] ondisk+write+known_if_redirected e126)", 2023-09-23T18:34:15.126 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:34:15.116811+0000", 2023-09-23T18:34:15.126 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.001396935, 2023-09-23T18:34:15.126 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001527221, 2023-09-23T18:34:15.126 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:34:15.126 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:34:15.126 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:34:15.126 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:34:15.126 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:34:15.126 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 145606 2023-09-23T18:34:15.126 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.126 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:34:15.127 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.127 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:34:15.127 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.116811+0000", 2023-09-23T18:34:15.127 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:34:15.127 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.127 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.127 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:34:15.127 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.116811+0000", 2023-09-23T18:34:15.127 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:34:15.127 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.127 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.127 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:34:15.127 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.116809+0000", 2023-09-23T18:34:15.128 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.1060000000000006e-06 2023-09-23T18:34:15.128 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.128 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.128 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:34:15.128 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.116818+0000", 2023-09-23T18:34:15.128 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4388e-05 2023-09-23T18:34:15.128 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.128 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.128 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:34:15.128 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.116833+0000", 2023-09-23T18:34:15.128 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.0089999999999996e-06 2023-09-23T18:34:15.128 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.128 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.129 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:34:15.129 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.116842+0000", 2023-09-23T18:34:15.129 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6851000000000003e-05 2023-09-23T18:34:15.129 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.129 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.129 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:34:15.129 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.116889+0000", 2023-09-23T18:34:15.129 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014892200000000001 2023-09-23T18:34:15.129 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.129 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.129 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:34:15.129 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.117037+0000", 2023-09-23T18:34:15.129 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00017634000000000001 2023-09-23T18:34:15.130 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.130 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.130 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 3,7", 2023-09-23T18:34:15.130 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.117214+0000", 2023-09-23T18:34:15.130 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000402598 2023-09-23T18:34:15.130 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:34:15.130 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:34:15.130 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:34:15.130 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.130 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.130 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:145583 5.e 5:76747408:::benchmark_data_smithi098_105788_object39298:head [delete] snapc 0=[] ondisk+write+known_if_redirected e126)", 2023-09-23T18:34:15.130 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:34:15.057050+0000", 2023-09-23T18:34:15.130 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.061158268000000002, 2023-09-23T18:34:15.131 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.061365295, 2023-09-23T18:34:15.131 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:34:15.131 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:34:15.131 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:34:15.131 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:34:15.131 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:34:15.131 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 145583 2023-09-23T18:34:15.131 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.131 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:34:15.131 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.131 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:34:15.131 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.057050+0000", 2023-09-23T18:34:15.131 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:34:15.132 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.132 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.132 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:34:15.132 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.057050+0000", 2023-09-23T18:34:15.132 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:34:15.132 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.132 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.132 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:34:15.132 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.057048+0000", 2023-09-23T18:34:15.132 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.1240000000000005e-06 2023-09-23T18:34:15.132 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.132 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.132 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:34:15.133 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.057056+0000", 2023-09-23T18:34:15.133 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2359e-05 2023-09-23T18:34:15.133 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.133 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.133 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:34:15.133 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.057068+0000", 2023-09-23T18:34:15.133 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.7680000000000003e-06 2023-09-23T18:34:15.133 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.133 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.133 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:34:15.133 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.057075+0000", 2023-09-23T18:34:15.133 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002139185 2023-09-23T18:34:15.134 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.134 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.134 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:34:15.134 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.059214+0000", 2023-09-23T18:34:15.134 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000188889 2023-09-23T18:34:15.134 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.134 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.134 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:34:15.134 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.059403+0000", 2023-09-23T18:34:15.134 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00022024499999999999 2023-09-23T18:34:15.134 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.134 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.134 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:34:15.135 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.059623+0000", 2023-09-23T18:34:15.135 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0052810069999999999 2023-09-23T18:34:15.135 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.135 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.135 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:34:15.135 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.064904+0000", 2023-09-23T18:34:15.135 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0020631239999999999 2023-09-23T18:34:15.135 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.135 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.135 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:34:15.135 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.066967+0000", 2023-09-23T18:34:15.135 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.009917571 2023-09-23T18:34:15.136 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:34:15.136 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:34:15.136 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:34:15.136 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.136 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.136 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:145586 5.e 5:7fcd7d6a:::benchmark_data_smithi098_105788_object39301:head [delete] snapc 0=[] ondisk+write+known_if_redirected e126)", 2023-09-23T18:34:15.136 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:34:15.067077+0000", 2023-09-23T18:34:15.136 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.051131134000000002, 2023-09-23T18:34:15.136 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.051426000999999999, 2023-09-23T18:34:15.136 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:34:15.136 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:34:15.136 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:34:15.137 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:34:15.137 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:34:15.137 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 145586 2023-09-23T18:34:15.137 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.137 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:34:15.137 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.137 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:34:15.137 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.067077+0000", 2023-09-23T18:34:15.137 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:34:15.137 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.137 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.137 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:34:15.138 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.067077+0000", 2023-09-23T18:34:15.138 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:34:15.138 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.138 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.138 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:34:15.138 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.067076+0000", 2023-09-23T18:34:15.138 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.8900000000000004e-06 2023-09-23T18:34:15.138 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.138 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.138 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:34:15.138 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.067082+0000", 2023-09-23T18:34:15.138 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.0299999999999994e-06 2023-09-23T18:34:15.138 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.139 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.139 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:34:15.139 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.067090+0000", 2023-09-23T18:34:15.139 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.3320000000000002e-06 2023-09-23T18:34:15.139 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.139 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.139 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:34:15.139 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.067094+0000", 2023-09-23T18:34:15.139 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.0187e-05 2023-09-23T18:34:15.139 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.139 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.139 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:34:15.140 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.067134+0000", 2023-09-23T18:34:15.140 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013615099999999999 2023-09-23T18:34:15.140 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.140 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.140 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:34:15.140 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.067270+0000", 2023-09-23T18:34:15.140 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013087400000000001 2023-09-23T18:34:15.140 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.140 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.140 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:34:15.140 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.067401+0000", 2023-09-23T18:34:15.140 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.006051959 2023-09-23T18:34:15.140 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.141 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.141 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:34:15.141 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.073453+0000", 2023-09-23T18:34:15.141 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0095573029999999996 2023-09-23T18:34:15.141 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.141 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.141 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:34:15.141 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.083011+0000", 2023-09-23T18:34:15.141 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.015933569000000002 2023-09-23T18:34:15.141 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:34:15.141 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:34:15.141 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:34:15.142 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.142 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.142 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:145588 5.e 5:73342a24:::benchmark_data_smithi098_105788_object39303:head [delete] snapc 0=[] ondisk+write+known_if_redirected e126)", 2023-09-23T18:34:15.142 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:34:15.067108+0000", 2023-09-23T18:34:15.142 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.051100072000000003, 2023-09-23T18:34:15.142 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.051480769000000003, 2023-09-23T18:34:15.142 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:34:15.142 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:34:15.142 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:34:15.142 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:34:15.142 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:34:15.143 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 145588 2023-09-23T18:34:15.143 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.143 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:34:15.143 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.143 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:34:15.143 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.067108+0000", 2023-09-23T18:34:15.143 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:34:15.143 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.143 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.143 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:34:15.143 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.067108+0000", 2023-09-23T18:34:15.143 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-09-23T18:34:15.144 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.144 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.144 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:34:15.144 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.067108+0000", 2023-09-23T18:34:15.144 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.2559999999999998e-06 2023-09-23T18:34:15.144 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.144 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.144 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:34:15.144 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.067111+0000", 2023-09-23T18:34:15.144 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.1640000000000004e-06 2023-09-23T18:34:15.144 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.144 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.144 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:34:15.145 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.067118+0000", 2023-09-23T18:34:15.145 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.903e-06 2023-09-23T18:34:15.145 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.145 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.145 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:34:15.145 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.067121+0000", 2023-09-23T18:34:15.145 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0032041299999999999 2023-09-23T18:34:15.145 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.145 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.145 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:34:15.145 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.070325+0000", 2023-09-23T18:34:15.145 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000121122 2023-09-23T18:34:15.146 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.146 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.146 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:34:15.146 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.070446+0000", 2023-09-23T18:34:15.146 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014544800000000001 2023-09-23T18:34:15.146 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.146 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.146 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:34:15.146 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.070592+0000", 2023-09-23T18:34:15.146 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0030960789999999998 2023-09-23T18:34:15.146 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.146 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.146 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:34:15.147 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.073688+0000", 2023-09-23T18:34:15.147 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0093786679999999997 2023-09-23T18:34:15.147 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:15.147 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:15.147 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:34:15.147 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:15.083066+0000", 2023-09-23T18:34:15.147 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.015958231 2023-09-23T18:34:15.147 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:34:15.147 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:34:15.147 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:34:15.147 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:34:15.147 INFO:teuthology.orchestra.run.smithi173.stdout: ], 2023-09-23T18:34:15.148 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 5 2023-09-23T18:34:15.148 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:34:15.148 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:15.148 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:14 smithi173 bash[59422]: cluster 2023-09-23T18:34:13.748885+0000 mgr.y (mgr.26054) 469 : cluster 0 pgmap v422: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 414 op/s; 53536/207414 objects degraded (25.811%); 10344/207414 objects misplaced (4.987%); 13 MiB/s, 209 objects/s recovering 2023-09-23T18:34:15.149 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:14 smithi173 bash[59422]: cluster 2023-09-23T18:34:14.460102+0000 mon.a (mon.0) 660 : cluster 3 Health check update: Degraded data redundancy: 53536/207414 objects degraded (25.811%), 12 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:15.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:15.186 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:15.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:15.287 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:15.318 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:15.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:15.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:15.423 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:15.454 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:15.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:15.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:15.562 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:15.606 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:15.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:15.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:15.710 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:15.736 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:15.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:15.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:15.842 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:15.865 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:15.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:15.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:15.969 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:16.007 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:16.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:16.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:16.113 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:16.122 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:34:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:34:16] "GET /metrics HTTP/1.1" 200 36554 "" "Prometheus/2.33.4" 2023-09-23T18:34:16.165 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:16.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:16.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:16.253 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T18:34:16.275 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:16.325 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:16.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:16.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:16.439 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:16.470 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:16.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:16.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:16.576 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:16.603 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:16.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:16.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:16.708 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:16.752 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:16.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:16.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:16.856 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:16.885 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:16.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:16.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:16.989 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:17.010 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:17.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:17.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:17.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:16 smithi173 bash[59422]: cluster 2023-09-23T18:34:15.749669+0000 mgr.y (mgr.26054) 470 : cluster 0 pgmap v423: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 441 op/s; 53062/204060 objects degraded (26.003%); 10198/204060 objects misplaced (4.998%); 12 MiB/s, 188 objects/s recovering 2023-09-23T18:34:17.115 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:17.132 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:16 smithi098 bash[76817]: cluster 2023-09-23T18:34:15.749669+0000 mgr.y (mgr.26054) 470 : cluster 0 pgmap v423: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 441 op/s; 53062/204060 objects degraded (26.003%); 10198/204060 objects misplaced (4.998%); 12 MiB/s, 188 objects/s recovering 2023-09-23T18:34:17.132 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:16 smithi098 bash[79516]: cluster 2023-09-23T18:34:15.749669+0000 mgr.y (mgr.26054) 470 : cluster 0 pgmap v423: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 441 op/s; 53062/204060 objects degraded (26.003%); 10198/204060 objects misplaced (4.998%); 12 MiB/s, 188 objects/s recovering 2023-09-23T18:34:17.159 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:17.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:17.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:17.266 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:17.291 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:17.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:17.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:17.397 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:17.419 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:17.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:17.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:17.522 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:17.547 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:17.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:17.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:17.650 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:17.673 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:17.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:17.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:17.777 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:17.800 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:17.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:17.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:17.904 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:17.941 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:17.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:17.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:18.045 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:18.068 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:18.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:18.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:18.173 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:18.198 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:18.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:18.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:18.304 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:18.340 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:18.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:18.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:18.445 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:18.482 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:18.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:18.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:18.589 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:18.617 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:18.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:18.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:18.723 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:18.758 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:18.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:18.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:18.861 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:18.889 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:18.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:18.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:18.992 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:19.001 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:18 smithi173 bash[59422]: cluster 2023-09-23T18:34:17.750529+0000 mgr.y (mgr.26054) 471 : cluster 0 pgmap v424: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 52347/199971 objects degraded (26.177%); 9872/199971 objects misplaced (4.937%); 16 MiB/s, 258 objects/s recovering 2023-09-23T18:34:19.025 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:19.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:19.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:19.133 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:19.141 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:18 smithi098 bash[76817]: cluster 2023-09-23T18:34:17.750529+0000 mgr.y (mgr.26054) 471 : cluster 0 pgmap v424: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 52347/199971 objects degraded (26.177%); 9872/199971 objects misplaced (4.937%); 16 MiB/s, 258 objects/s recovering 2023-09-23T18:34:19.142 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:18 smithi098 bash[79516]: cluster 2023-09-23T18:34:17.750529+0000 mgr.y (mgr.26054) 471 : cluster 0 pgmap v424: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 52347/199971 objects degraded (26.177%); 9872/199971 objects misplaced (4.937%); 16 MiB/s, 258 objects/s recovering 2023-09-23T18:34:19.162 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:19.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:19.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:19.268 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:19.294 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:19.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:19.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:19.403 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:19.424 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:19.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:19.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:19.527 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:19.554 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:19.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:19.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:19.657 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:19.682 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:19.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:19.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:19.784 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:19.814 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:19.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:19.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:19.919 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:19.945 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:19.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:19.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:20.052 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:20.081 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:20.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:20.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:20.186 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:20.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:19 smithi098 bash[79516]: cluster 2023-09-23T18:34:19.461351+0000 mon.a (mon.0) 661 : cluster 3 Health check update: Degraded data redundancy: 52347/199971 objects degraded (26.177%), 12 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:20.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:19 smithi098 bash[76817]: cluster 2023-09-23T18:34:19.461351+0000 mon.a (mon.0) 661 : cluster 3 Health check update: Degraded data redundancy: 52347/199971 objects degraded (26.177%), 12 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:20.193 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:19 smithi173 bash[59422]: cluster 2023-09-23T18:34:19.461351+0000 mon.a (mon.0) 661 : cluster 3 Health check update: Degraded data redundancy: 52347/199971 objects degraded (26.177%), 12 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:20.207 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:20.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:20.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:20.313 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:20.343 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:20.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:20.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:20.446 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:20.472 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:20.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:20.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:20.576 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:20.601 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:20.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:20.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:20.707 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:20.750 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:20.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:20.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:20.857 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:20.893 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:20.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:20.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:20.997 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:21.034 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:21.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:21.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:21.065 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:34:21.139 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:21.148 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:20 smithi098 bash[79516]: cluster 2023-09-23T18:34:19.751361+0000 mgr.y (mgr.26054) 472 : cluster 0 pgmap v425: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 463 op/s; 51858/198075 objects degraded (26.181%); 9709/198075 objects misplaced (4.902%); 12 MiB/s, 195 objects/s recovering 2023-09-23T18:34:21.148 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:20 smithi098 bash[76817]: cluster 2023-09-23T18:34:19.751361+0000 mgr.y (mgr.26054) 472 : cluster 0 pgmap v425: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 463 op/s; 51858/198075 objects degraded (26.181%); 9709/198075 objects misplaced (4.902%); 12 MiB/s, 195 objects/s recovering 2023-09-23T18:34:21.171 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:21.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:21.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:21.276 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:21.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:20 smithi173 bash[59422]: cluster 2023-09-23T18:34:19.751361+0000 mgr.y (mgr.26054) 472 : cluster 0 pgmap v425: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 463 op/s; 51858/198075 objects degraded (26.181%); 9709/198075 objects misplaced (4.902%); 12 MiB/s, 195 objects/s recovering 2023-09-23T18:34:21.316 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:21.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:21.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:21.422 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:21.446 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:21.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:21.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:21.550 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:21.601 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:21.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:21.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:21.707 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:21.745 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:21.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:21.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:21.848 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:21.874 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:21.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:21.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:21.979 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:22.005 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:22.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:22.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:22.108 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:22.136 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:22.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:22.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:22.239 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:22.260 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:22.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:22.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:22.367 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:22.396 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:22.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:22.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:22.499 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:22.521 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:22.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:22.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:22.629 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:22.654 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:22.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:22.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:22.757 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:22.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:22.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:22.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:22.890 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:22.919 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:22.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:22.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:23.023 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:23.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:22 smithi173 bash[59422]: cluster 2023-09-23T18:34:21.752806+0000 mgr.y (mgr.26054) 473 : cluster 0 pgmap v426: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 51154/194844 objects degraded (26.254%); 9565/194844 objects misplaced (4.909%); 16 MiB/s, 250 objects/s recovering 2023-09-23T18:34:23.055 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:23.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:23.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:23.158 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:23.167 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:22 smithi098 bash[76817]: cluster 2023-09-23T18:34:21.752806+0000 mgr.y (mgr.26054) 473 : cluster 0 pgmap v426: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 51154/194844 objects degraded (26.254%); 9565/194844 objects misplaced (4.909%); 16 MiB/s, 250 objects/s recovering 2023-09-23T18:34:23.167 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:22 smithi098 bash[79516]: cluster 2023-09-23T18:34:21.752806+0000 mgr.y (mgr.26054) 473 : cluster 0 pgmap v426: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 51154/194844 objects degraded (26.254%); 9565/194844 objects misplaced (4.909%); 16 MiB/s, 250 objects/s recovering 2023-09-23T18:34:23.195 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:23.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:23.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:23.302 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:23.333 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:23.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:23.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:23.436 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:23.466 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:23.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:23.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:23.571 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:23.600 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:23.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:23.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:23.704 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:23.730 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:23.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:23.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:23.838 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:23.859 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:23.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:23.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:23.963 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:23.999 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:24.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:24.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:24.102 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:24.135 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:24.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:24.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:24.239 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:24.261 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:24.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:24.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:24.365 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:24.387 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:24.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:24.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:24.489 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:24.538 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:24.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:24.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:24.644 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:24.683 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:24.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:24.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:24.723 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:34:24.774 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:34:24.789 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:24.832 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:24.833 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:34:24.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:24.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:24.889 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:34:24.937 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:34:24.937 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:24.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:24 smithi098 bash[79516]: cluster 2023-09-23T18:34:23.753524+0000 mgr.y (mgr.26054) 474 : cluster 0 pgmap v427: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 51154/192540 objects degraded (26.568%); 9394/192540 objects misplaced (4.879%); 13 MiB/s, 202 objects/s recovering 2023-09-23T18:34:24.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:24 smithi098 bash[79516]: cluster 2023-09-23T18:34:24.462495+0000 mon.a (mon.0) 662 : cluster 3 Health check update: Degraded data redundancy: 51154/192540 objects degraded (26.568%), 12 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:24.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:24 smithi098 bash[79516]: audit 2023-09-23T18:34:24.582571+0000 mon.a (mon.0) 663 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:34:24.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:24 smithi098 bash[79516]: audit 2023-09-23T18:34:24.674610+0000 mon.c (mon.2) 64 : audit 0 from='client.? 172.21.15.98:0/1223848428' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:34:24.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:24 smithi098 bash[76817]: cluster 2023-09-23T18:34:23.753524+0000 mgr.y (mgr.26054) 474 : cluster 0 pgmap v427: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 51154/192540 objects degraded (26.568%); 9394/192540 objects misplaced (4.879%); 13 MiB/s, 202 objects/s recovering 2023-09-23T18:34:24.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:24 smithi098 bash[76817]: cluster 2023-09-23T18:34:24.462495+0000 mon.a (mon.0) 662 : cluster 3 Health check update: Degraded data redundancy: 51154/192540 objects degraded (26.568%), 12 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:24.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:24 smithi098 bash[76817]: audit 2023-09-23T18:34:24.582571+0000 mon.a (mon.0) 663 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:34:24.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:24 smithi098 bash[76817]: audit 2023-09-23T18:34:24.674610+0000 mon.c (mon.2) 64 : audit 0 from='client.? 172.21.15.98:0/1223848428' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:34:24.972 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:34:25.001 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:25.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:25.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:25.018 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:34:25.058 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:34:25.063 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:34:25.063 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:34:25.063 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:34:25.064 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:34:25.064 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:34:25.064 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:34:25.064 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:34:25.064 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:34:25.110 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:25.154 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:25.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:25.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:25.258 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:25.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:24 smithi173 bash[59422]: cluster 2023-09-23T18:34:23.753524+0000 mgr.y (mgr.26054) 474 : cluster 0 pgmap v427: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 51154/192540 objects degraded (26.568%); 9394/192540 objects misplaced (4.879%); 13 MiB/s, 202 objects/s recovering 2023-09-23T18:34:25.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:24 smithi173 bash[59422]: cluster 2023-09-23T18:34:24.462495+0000 mon.a (mon.0) 662 : cluster 3 Health check update: Degraded data redundancy: 51154/192540 objects degraded (26.568%), 12 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:25.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:24 smithi173 bash[59422]: audit 2023-09-23T18:34:24.582571+0000 mon.a (mon.0) 663 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:34:25.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:24 smithi173 bash[59422]: audit 2023-09-23T18:34:24.674610+0000 mon.c (mon.2) 64 : audit 0 from='client.? 172.21.15.98:0/1223848428' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:34:25.286 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:25.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:25.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:25.388 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:25.414 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:25.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:25.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:25.521 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:25.545 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:25.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:25.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:25.651 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:25.673 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:25.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:25.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:25.782 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:25.817 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:25.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:25.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:25.869 INFO:teuthology.orchestra.run.smithi098.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T18:34:25.872 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is set 2023-09-23T18:34:25.910 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:25 smithi098 bash[79516]: audit 2023-09-23T18:34:25.074688+0000 mon.c (mon.2) 65 : audit 1 from='client.? 172.21.15.98:0/832916528' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:34:25.910 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:25 smithi098 bash[79516]: audit 2023-09-23T18:34:25.075815+0000 mon.a (mon.0) 664 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:34:25.910 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:25 smithi098 bash[79516]: audit 2023-09-23T18:34:25.136124+0000 mon.a (mon.0) 665 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:34:25.911 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:25 smithi098 bash[79516]: audit 2023-09-23T18:34:25.136271+0000 mon.b (mon.1) 6 : audit 1 from='client.? 172.21.15.98:0/1379687615' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:34:25.911 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:25 smithi098 bash[76817]: audit 2023-09-23T18:34:25.074688+0000 mon.c (mon.2) 65 : audit 1 from='client.? 172.21.15.98:0/832916528' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:34:25.911 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:25 smithi098 bash[76817]: audit 2023-09-23T18:34:25.075815+0000 mon.a (mon.0) 664 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:34:25.911 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:25 smithi098 bash[76817]: audit 2023-09-23T18:34:25.136124+0000 mon.a (mon.0) 665 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:34:25.911 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:25 smithi098 bash[76817]: audit 2023-09-23T18:34:25.136271+0000 mon.b (mon.1) 6 : audit 1 from='client.? 172.21.15.98:0/1379687615' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:34:25.921 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:25.952 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:25.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:25.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:26.060 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:26.087 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:26.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:26.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:26.135 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:25 smithi173 bash[59422]: audit 2023-09-23T18:34:25.074688+0000 mon.c (mon.2) 65 : audit 1 from='client.? 172.21.15.98:0/832916528' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:34:26.135 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:25 smithi173 bash[59422]: audit 2023-09-23T18:34:25.075815+0000 mon.a (mon.0) 664 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:34:26.135 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:25 smithi173 bash[59422]: audit 2023-09-23T18:34:25.136124+0000 mon.a (mon.0) 665 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:34:26.136 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:25 smithi173 bash[59422]: audit 2023-09-23T18:34:25.136271+0000 mon.b (mon.1) 6 : audit 1 from='client.? 172.21.15.98:0/1379687615' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:34:26.190 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:34:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:34:26] "GET /metrics HTTP/1.1" 200 36547 "" "Prometheus/2.33.4" 2023-09-23T18:34:26.192 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:26.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:26.226 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:26.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:26.327 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:26.374 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:26.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:26.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:26.390 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T18:34:26.390 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:34:26.479 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:26.519 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:26.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:26.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:26.623 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:26.661 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:26.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:26.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:26.769 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:26.805 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:26.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:26.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:26.878 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:26 smithi173 bash[59422]: cluster 2023-09-23T18:34:25.754212+0000 mgr.y (mgr.26054) 475 : cluster 0 pgmap v428: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 50685/189351 objects degraded (26.768%); 9245/189351 objects misplaced (4.882%); 13 MiB/s, 201 objects/s recovering 2023-09-23T18:34:26.878 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:26 smithi173 bash[59422]: cluster 2023-09-23T18:34:25.853010+0000 mon.a (mon.0) 666 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:34:26.909 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:26.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:26 smithi098 bash[79516]: cluster 2023-09-23T18:34:25.754212+0000 mgr.y (mgr.26054) 475 : cluster 0 pgmap v428: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 50685/189351 objects degraded (26.768%); 9245/189351 objects misplaced (4.882%); 13 MiB/s, 201 objects/s recovering 2023-09-23T18:34:26.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:26 smithi098 bash[79516]: cluster 2023-09-23T18:34:25.853010+0000 mon.a (mon.0) 666 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:34:26.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:26 smithi098 bash[79516]: cluster 2023-09-23T18:34:25.855977+0000 osd.5 (osd.5) 38 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T18:34:26.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:26 smithi098 bash[79516]: audit 2023-09-23T18:34:25.858656+0000 mon.a (mon.0) 667 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T18:34:26.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:26 smithi098 bash[79516]: audit 2023-09-23T18:34:25.858938+0000 mon.a (mon.0) 668 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:34:26.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:26 smithi098 bash[79516]: cluster 2023-09-23T18:34:25.859000+0000 osd.6 (osd.6) 44 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T18:34:26.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:26 smithi098 bash[79516]: cluster 2023-09-23T18:34:25.859318+0000 mon.a (mon.0) 669 : cluster 0 osdmap e127: 8 total, 8 up, 7 in 2023-09-23T18:34:26.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:26 smithi098 bash[76817]: cluster 2023-09-23T18:34:25.754212+0000 mgr.y (mgr.26054) 475 : cluster 0 pgmap v428: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 50685/189351 objects degraded (26.768%); 9245/189351 objects misplaced (4.882%); 13 MiB/s, 201 objects/s recovering 2023-09-23T18:34:26.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:26 smithi098 bash[76817]: cluster 2023-09-23T18:34:25.853010+0000 mon.a (mon.0) 666 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:34:26.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:26 smithi098 bash[76817]: cluster 2023-09-23T18:34:25.855977+0000 osd.5 (osd.5) 38 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T18:34:26.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:26 smithi098 bash[76817]: audit 2023-09-23T18:34:25.858656+0000 mon.a (mon.0) 667 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T18:34:26.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:26 smithi098 bash[76817]: audit 2023-09-23T18:34:25.858938+0000 mon.a (mon.0) 668 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:34:26.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:26 smithi098 bash[76817]: cluster 2023-09-23T18:34:25.859000+0000 osd.6 (osd.6) 44 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T18:34:26.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:26 smithi098 bash[76817]: cluster 2023-09-23T18:34:25.859318+0000 mon.a (mon.0) 669 : cluster 0 osdmap e127: 8 total, 8 up, 7 in 2023-09-23T18:34:26.949 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:26.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:26.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:27.055 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:27.074 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:27.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:27.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:27.176 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:27.186 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:26 smithi173 bash[59422]: cluster 2023-09-23T18:34:25.855977+0000 osd.5 (osd.5) 38 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T18:34:27.187 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:26 smithi173 bash[59422]: audit 2023-09-23T18:34:25.858656+0000 mon.a (mon.0) 667 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T18:34:27.187 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:26 smithi173 bash[59422]: audit 2023-09-23T18:34:25.858938+0000 mon.a (mon.0) 668 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:34:27.187 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:26 smithi173 bash[59422]: cluster 2023-09-23T18:34:25.859000+0000 osd.6 (osd.6) 44 : cluster 3 failed to encode map e127 with expected crc 2023-09-23T18:34:27.187 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:26 smithi173 bash[59422]: cluster 2023-09-23T18:34:25.859318+0000 mon.a (mon.0) 669 : cluster 0 osdmap e127: 8 total, 8 up, 7 in 2023-09-23T18:34:27.204 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:27.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:27.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:27.311 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:27.336 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:27.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:27.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:27.441 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:27.466 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:27.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:27.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:27.571 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:27.577 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T18:34:27.597 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:27.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:27.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:27.700 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:27.723 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:27.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:27.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:27.826 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:27.853 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:27.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:27.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:27.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:27 smithi098 bash[79516]: cluster 2023-09-23T18:34:26.898698+0000 osd.7 (osd.7) 39 : cluster 3 failed to encode map e128 with expected crc 2023-09-23T18:34:27.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:27 smithi098 bash[79516]: cluster 2023-09-23T18:34:26.902403+0000 mon.a (mon.0) 670 : cluster 0 osdmap e128: 8 total, 8 up, 7 in 2023-09-23T18:34:27.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:27 smithi098 bash[79516]: cluster 2023-09-23T18:34:26.908518+0000 osd.4 (osd.4) 59 : cluster 3 failed to encode map e128 with expected crc 2023-09-23T18:34:27.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:27 smithi098 bash[79516]: cluster 2023-09-23T18:34:26.921374+0000 osd.5 (osd.5) 39 : cluster 3 failed to encode map e128 with expected crc 2023-09-23T18:34:27.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:27 smithi098 bash[76817]: cluster 2023-09-23T18:34:26.898698+0000 osd.7 (osd.7) 39 : cluster 3 failed to encode map e128 with expected crc 2023-09-23T18:34:27.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:27 smithi098 bash[76817]: cluster 2023-09-23T18:34:26.902403+0000 mon.a (mon.0) 670 : cluster 0 osdmap e128: 8 total, 8 up, 7 in 2023-09-23T18:34:27.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:27 smithi098 bash[76817]: cluster 2023-09-23T18:34:26.908518+0000 osd.4 (osd.4) 59 : cluster 3 failed to encode map e128 with expected crc 2023-09-23T18:34:27.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:27 smithi098 bash[76817]: cluster 2023-09-23T18:34:26.921374+0000 osd.5 (osd.5) 39 : cluster 3 failed to encode map e128 with expected crc 2023-09-23T18:34:27.957 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:27.984 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:27.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:27.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:28.091 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:28.126 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:28.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:28.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:28.232 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:28.277 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:28.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:27 smithi173 bash[59422]: cluster 2023-09-23T18:34:26.898698+0000 osd.7 (osd.7) 39 : cluster 3 failed to encode map e128 with expected crc 2023-09-23T18:34:28.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:27 smithi173 bash[59422]: cluster 2023-09-23T18:34:26.902403+0000 mon.a (mon.0) 670 : cluster 0 osdmap e128: 8 total, 8 up, 7 in 2023-09-23T18:34:28.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:27 smithi173 bash[59422]: cluster 2023-09-23T18:34:26.908518+0000 osd.4 (osd.4) 59 : cluster 3 failed to encode map e128 with expected crc 2023-09-23T18:34:28.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:27 smithi173 bash[59422]: cluster 2023-09-23T18:34:26.921374+0000 osd.5 (osd.5) 39 : cluster 3 failed to encode map e128 with expected crc 2023-09-23T18:34:28.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:28.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:28.386 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:28.412 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:28.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:28.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:28.522 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:28.553 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:28.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:28.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:28.665 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:28.689 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:28.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:28.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:28.792 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:28.824 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:28.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:28.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:28.899 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:28 smithi098 bash[76817]: cluster 2023-09-23T18:34:27.754785+0000 mgr.y (mgr.26054) 476 : cluster 0 pgmap v431: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 50036/185523 objects degraded (26.970%); 8884/185523 objects misplaced (4.789%); 17 MiB/s, 264 objects/s recovering 2023-09-23T18:34:28.899 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:28 smithi098 bash[76817]: cluster 2023-09-23T18:34:27.898441+0000 osd.6 (osd.6) 45 : cluster 3 failed to encode map e129 with expected crc 2023-09-23T18:34:28.899 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:28 smithi098 bash[76817]: cluster 2023-09-23T18:34:27.900085+0000 mon.a (mon.0) 671 : cluster 0 osdmap e129: 8 total, 8 up, 7 in 2023-09-23T18:34:28.899 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:28 smithi098 bash[76817]: cluster 2023-09-23T18:34:27.905402+0000 osd.4 (osd.4) 60 : cluster 3 failed to encode map e129 with expected crc 2023-09-23T18:34:28.900 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:28 smithi098 bash[79516]: cluster 2023-09-23T18:34:27.754785+0000 mgr.y (mgr.26054) 476 : cluster 0 pgmap v431: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 50036/185523 objects degraded (26.970%); 8884/185523 objects misplaced (4.789%); 17 MiB/s, 264 objects/s recovering 2023-09-23T18:34:28.900 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:28 smithi098 bash[79516]: cluster 2023-09-23T18:34:27.898441+0000 osd.6 (osd.6) 45 : cluster 3 failed to encode map e129 with expected crc 2023-09-23T18:34:28.900 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:28 smithi098 bash[79516]: cluster 2023-09-23T18:34:27.900085+0000 mon.a (mon.0) 671 : cluster 0 osdmap e129: 8 total, 8 up, 7 in 2023-09-23T18:34:28.927 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:28.958 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:28.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:28.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:29.061 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:29.074 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:28 smithi173 bash[59422]: cluster 2023-09-23T18:34:27.754785+0000 mgr.y (mgr.26054) 476 : cluster 0 pgmap v431: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 50036/185523 objects degraded (26.970%); 8884/185523 objects misplaced (4.789%); 17 MiB/s, 264 objects/s recovering 2023-09-23T18:34:29.075 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:28 smithi173 bash[59422]: cluster 2023-09-23T18:34:27.898441+0000 osd.6 (osd.6) 45 : cluster 3 failed to encode map e129 with expected crc 2023-09-23T18:34:29.075 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:28 smithi173 bash[59422]: cluster 2023-09-23T18:34:27.900085+0000 mon.a (mon.0) 671 : cluster 0 osdmap e129: 8 total, 8 up, 7 in 2023-09-23T18:34:29.075 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:28 smithi173 bash[59422]: cluster 2023-09-23T18:34:27.905402+0000 osd.4 (osd.4) 60 : cluster 3 failed to encode map e129 with expected crc 2023-09-23T18:34:29.075 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:28 smithi173 bash[59422]: cluster 2023-09-23T18:34:28.046323+0000 osd.5 (osd.5) 40 : cluster 3 failed to encode map e129 with expected crc 2023-09-23T18:34:29.097 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:29.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:29.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:29.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:28 smithi098 bash[79516]: cluster 2023-09-23T18:34:27.905402+0000 osd.4 (osd.4) 60 : cluster 3 failed to encode map e129 with expected crc 2023-09-23T18:34:29.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:28 smithi098 bash[79516]: cluster 2023-09-23T18:34:28.046323+0000 osd.5 (osd.5) 40 : cluster 3 failed to encode map e129 with expected crc 2023-09-23T18:34:29.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:28 smithi098 bash[76817]: cluster 2023-09-23T18:34:28.046323+0000 osd.5 (osd.5) 40 : cluster 3 failed to encode map e129 with expected crc 2023-09-23T18:34:29.199 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:29.234 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:29.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:29.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:29.299 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T18:34:29.338 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:29.369 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:29.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:29.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:29.474 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:29.505 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:29.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:29.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:29.611 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:29.644 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:29.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:29.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:29.670 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:34:29.747 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:29.774 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:29.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:29.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:29.880 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:29.920 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:29.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:29.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:30.022 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:30.051 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:30.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:30.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:30.156 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:30.165 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:29 smithi173 bash[59422]: cluster 2023-09-23T18:34:29.463735+0000 mon.a (mon.0) 672 : cluster 3 Health check update: Degraded data redundancy: 50036/185523 objects degraded (26.970%), 12 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:30.180 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:30.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:30.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:30.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:29 smithi098 bash[76817]: cluster 2023-09-23T18:34:29.463735+0000 mon.a (mon.0) 672 : cluster 3 Health check update: Degraded data redundancy: 50036/185523 objects degraded (26.970%), 12 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:30.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:29 smithi098 bash[79516]: cluster 2023-09-23T18:34:29.463735+0000 mon.a (mon.0) 672 : cluster 3 Health check update: Degraded data redundancy: 50036/185523 objects degraded (26.970%), 12 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:30.290 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:30.324 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:30.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:30.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:30.430 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:30.469 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:30.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:30.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:30.575 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:30.603 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:30.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:30.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:30.705 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:30.737 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:30.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:30.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:30.844 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:30.869 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:30.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:30.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:30.975 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:30.982 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:30 smithi098 bash[79516]: cluster 2023-09-23T18:34:29.755374+0000 mgr.y (mgr.26054) 477 : cluster 0 pgmap v433: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 49655/184032 objects degraded (26.982%); 8755/184032 objects misplaced (4.757%); 13 MiB/s, 211 objects/s recovering 2023-09-23T18:34:30.982 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:30 smithi098 bash[76817]: cluster 2023-09-23T18:34:29.755374+0000 mgr.y (mgr.26054) 477 : cluster 0 pgmap v433: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 49655/184032 objects degraded (26.982%); 8755/184032 objects misplaced (4.757%); 13 MiB/s, 211 objects/s recovering 2023-09-23T18:34:30.999 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:31.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:31.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:31.103 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:31.111 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:30 smithi173 bash[59422]: cluster 2023-09-23T18:34:29.755374+0000 mgr.y (mgr.26054) 477 : cluster 0 pgmap v433: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 49655/184032 objects degraded (26.982%); 8755/184032 objects misplaced (4.757%); 13 MiB/s, 211 objects/s recovering 2023-09-23T18:34:31.126 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:31.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:31.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:31.232 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:31.248 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:34:31.258 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:31.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:31.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:31.363 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:31.389 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:31.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:31.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:31.496 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:31.521 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:31.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:31.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:31.628 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:31.664 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:31.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:31.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:31.772 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:31.798 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:31.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:31.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:31.904 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:31.927 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:31.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:31.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:32.031 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:32.073 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:32.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:32.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:32.177 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:32.189 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:31 smithi173 bash[59422]: cluster 2023-09-23T18:34:31.756192+0000 mgr.y (mgr.26054) 478 : cluster 0 pgmap v434: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 466 op/s; 49567/180957 objects degraded (27.392%); 8594/180957 objects misplaced (4.749%); 14 MiB/s, 228 objects/s recovering 2023-09-23T18:34:32.214 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:32.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:32.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:32.324 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:32.351 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:32.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:32.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:32.373 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:34:32.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:31 smithi098 bash[76817]: cluster 2023-09-23T18:34:31.756192+0000 mgr.y (mgr.26054) 478 : cluster 0 pgmap v434: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 466 op/s; 49567/180957 objects degraded (27.392%); 8594/180957 objects misplaced (4.749%); 14 MiB/s, 228 objects/s recovering 2023-09-23T18:34:32.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:31 smithi098 bash[79516]: cluster 2023-09-23T18:34:31.756192+0000 mgr.y (mgr.26054) 478 : cluster 0 pgmap v434: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 466 op/s; 49567/180957 objects degraded (27.392%); 8594/180957 objects misplaced (4.749%); 14 MiB/s, 228 objects/s recovering 2023-09-23T18:34:32.458 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:32.483 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:32.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:32.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:32.586 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:32.614 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:32.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:32.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:32.722 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:32.757 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:32.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:32.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:32.862 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:32.889 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:32.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:32.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:32.992 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:33.020 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:33.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:33.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:33.127 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:33.161 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:33.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:33.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:33.267 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:33.302 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:33.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:33.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:33.410 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:33.439 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:33.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:33.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:33.546 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:33.572 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:33.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:33.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:33.678 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:33.710 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:33.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:33.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:33.815 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:33.836 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:33.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:33.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:33.940 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:33.962 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:33.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:33.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:34.065 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:34.118 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:34.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:34.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:34.160 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:34:34.160 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:34:34.222 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:34.250 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:34.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:34.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:34.355 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:34.385 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:34.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:34.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:34.490 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:34.543 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:34.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:34.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:34.649 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:34.674 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:34.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:34.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:34.777 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:34.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:34.823 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:34.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:34.924 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:34.926 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:34 smithi173 bash[59422]: cluster 2023-09-23T18:34:33.757001+0000 mgr.y (mgr.26054) 479 : cluster 0 pgmap v435: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 49567/178725 objects degraded (27.734%); 8368/178725 objects misplaced (4.682%); 12 MiB/s, 185 objects/s recovering 2023-09-23T18:34:34.926 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:34 smithi173 bash[59422]: cluster 2023-09-23T18:34:34.464888+0000 mon.a (mon.0) 673 : cluster 3 Health check update: Degraded data redundancy: 49567/178725 objects degraded (27.734%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:34.965 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:34.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:34.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:35.070 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:35.093 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:35.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:35.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:35.180 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:34:35.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:34 smithi098 bash[76817]: cluster 2023-09-23T18:34:33.757001+0000 mgr.y (mgr.26054) 479 : cluster 0 pgmap v435: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 49567/178725 objects degraded (27.734%); 8368/178725 objects misplaced (4.682%); 12 MiB/s, 185 objects/s recovering 2023-09-23T18:34:35.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:34 smithi098 bash[76817]: cluster 2023-09-23T18:34:34.464888+0000 mon.a (mon.0) 673 : cluster 3 Health check update: Degraded data redundancy: 49567/178725 objects degraded (27.734%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:35.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:34 smithi098 bash[79516]: cluster 2023-09-23T18:34:33.757001+0000 mgr.y (mgr.26054) 479 : cluster 0 pgmap v435: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 49567/178725 objects degraded (27.734%); 8368/178725 objects misplaced (4.682%); 12 MiB/s, 185 objects/s recovering 2023-09-23T18:34:35.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:34 smithi098 bash[79516]: cluster 2023-09-23T18:34:34.464888+0000 mon.a (mon.0) 673 : cluster 3 Health check update: Degraded data redundancy: 49567/178725 objects degraded (27.734%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:35.198 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:35.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:35.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:35.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:35.329 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:35.361 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:35.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:35.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:35.464 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:35.501 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:35.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:35.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:35.604 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:35.628 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:35.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:35.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:35.730 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:35.752 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:35.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:35.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:35.855 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:35.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:35.895 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:35.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:35.996 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:36.017 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:36.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:36.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:36.128 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:36.158 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:36.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:36.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:36.221 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:34:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:34:36] "GET /metrics HTTP/1.1" 200 36547 "" "Prometheus/2.33.4" 2023-09-23T18:34:36.261 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:36.286 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:36.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:36.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:36.388 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:36.414 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:36.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:36.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:36.520 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:36.546 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:36.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:36.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:36.650 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:36.689 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:36.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:36.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:36.796 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:36.819 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:36 smithi098 bash[79516]: cluster 2023-09-23T18:34:35.757808+0000 mgr.y (mgr.26054) 480 : cluster 0 pgmap v436: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 457 op/s; 49122/175413 objects degraded (28.004%); 8216/175413 objects misplaced (4.684%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T18:34:36.820 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:36 smithi098 bash[76817]: cluster 2023-09-23T18:34:35.757808+0000 mgr.y (mgr.26054) 480 : cluster 0 pgmap v436: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 457 op/s; 49122/175413 objects degraded (28.004%); 8216/175413 objects misplaced (4.684%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T18:34:36.831 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:36.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:36.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:36.933 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:36.959 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:36.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:36.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:37.065 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:37.117 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:37.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:37.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:37.221 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:37.228 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:36 smithi173 bash[59422]: cluster 2023-09-23T18:34:35.757808+0000 mgr.y (mgr.26054) 480 : cluster 0 pgmap v436: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 457 op/s; 49122/175413 objects degraded (28.004%); 8216/175413 objects misplaced (4.684%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T18:34:37.257 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:37.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:37.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:37.363 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:37.387 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:37.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:37.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:37.492 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:37.524 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:37.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:37.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:37.629 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:37.659 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:37.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:37.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:37.723 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T18:34:37.723 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:34:37.762 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:37.789 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:37.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:37.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:37.892 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:37.923 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:37.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:37.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:38.027 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:38.062 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:38.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:38.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:38.166 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:38.195 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:38.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:38.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:38.301 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:38.325 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:38.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:38.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:38.428 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:38.455 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:38.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:38.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:38.558 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:38.592 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:38.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:38.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:38.695 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:38.719 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:38.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:38.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:38.827 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:38.852 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:38.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:38.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:38.910 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T18:34:38.960 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:38.987 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:38.989 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:38 smithi098 bash[76817]: cluster 2023-09-23T18:34:37.758606+0000 mgr.y (mgr.26054) 481 : cluster 0 pgmap v437: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 482 op/s; 49122/171042 objects degraded (28.719%); 7879/171042 objects misplaced (4.606%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T18:34:38.989 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:38 smithi098 bash[79516]: cluster 2023-09-23T18:34:37.758606+0000 mgr.y (mgr.26054) 481 : cluster 0 pgmap v437: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 482 op/s; 49122/171042 objects degraded (28.719%); 7879/171042 objects misplaced (4.606%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T18:34:38.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:38.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:39.092 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:39.101 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:38 smithi173 bash[59422]: cluster 2023-09-23T18:34:37.758606+0000 mgr.y (mgr.26054) 481 : cluster 0 pgmap v437: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 482 op/s; 49122/171042 objects degraded (28.719%); 7879/171042 objects misplaced (4.606%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T18:34:39.128 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:39.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:39.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:39.234 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:39.280 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:39.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:39.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:39.386 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:39.408 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:39.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:39.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:39.518 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:39.549 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:39.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:39.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:39.655 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:39.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:39.689 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:39.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:39.790 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:39.815 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:39.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:39.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:39.919 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:39.927 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:39 smithi173 bash[59422]: cluster 2023-09-23T18:34:39.466592+0000 mon.a (mon.0) 674 : cluster 3 Health check update: Degraded data redundancy: 49122/171042 objects degraded (28.719%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:39.927 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:39 smithi173 bash[59422]: audit 2023-09-23T18:34:39.587279+0000 mon.a (mon.0) 675 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:34:39.927 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:39 smithi173 bash[59422]: audit 2023-09-23T18:34:39.589039+0000 mon.a (mon.0) 676 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:34:39.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:39 smithi098 bash[76817]: cluster 2023-09-23T18:34:39.466592+0000 mon.a (mon.0) 674 : cluster 3 Health check update: Degraded data redundancy: 49122/171042 objects degraded (28.719%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:39.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:39 smithi098 bash[76817]: audit 2023-09-23T18:34:39.587279+0000 mon.a (mon.0) 675 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:34:39.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:39 smithi098 bash[76817]: audit 2023-09-23T18:34:39.589039+0000 mon.a (mon.0) 676 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:34:39.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:39 smithi098 bash[79516]: cluster 2023-09-23T18:34:39.466592+0000 mon.a (mon.0) 674 : cluster 3 Health check update: Degraded data redundancy: 49122/171042 objects degraded (28.719%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:39.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:39 smithi098 bash[79516]: audit 2023-09-23T18:34:39.587279+0000 mon.a (mon.0) 675 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:34:39.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:39 smithi098 bash[79516]: audit 2023-09-23T18:34:39.589039+0000 mon.a (mon.0) 676 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:34:39.941 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:39.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:39.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:40.048 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:40.071 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:40.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:40.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:40.174 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:40.211 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:40.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:40.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:40.317 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:40.344 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:40.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:40.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:40.448 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:40.474 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:40.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:40.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:40.577 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:40.607 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:40.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:40.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:40.710 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:40.735 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:40.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:40.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:40.838 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:40.850 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:40 smithi098 bash[76817]: cluster 2023-09-23T18:34:39.759264+0000 mgr.y (mgr.26054) 482 : cluster 0 pgmap v438: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 458 op/s; 48679/169218 objects degraded (28.767%); 7739/169218 objects misplaced (4.573%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T18:34:40.873 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:40.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:40.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:40.977 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:41.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:41.011 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:41.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:41.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:40 smithi173 bash[59422]: cluster 2023-09-23T18:34:39.759264+0000 mgr.y (mgr.26054) 482 : cluster 0 pgmap v438: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 458 op/s; 48679/169218 objects degraded (28.767%); 7739/169218 objects misplaced (4.573%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T18:34:41.112 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:41.139 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:41.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:41.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:41.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:40 smithi098 bash[79516]: cluster 2023-09-23T18:34:39.759264+0000 mgr.y (mgr.26054) 482 : cluster 0 pgmap v438: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 458 op/s; 48679/169218 objects degraded (28.767%); 7739/169218 objects misplaced (4.573%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T18:34:41.242 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:41.268 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:41.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:41.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:41.371 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:41.406 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:41.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:41.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:41.511 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:41.549 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:41.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:41.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:41.657 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:41.692 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:41.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:41.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:41.796 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:41.820 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:41.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:41.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:41.928 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:41.950 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:41.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:41.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:42.054 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:42.077 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:42.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:42.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:42.185 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:42.221 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:42.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:42.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:42.325 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:42.350 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:42.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:42.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:42.453 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:42.496 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:42.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:42.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:42.606 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:42.632 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:42.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:42.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:42.740 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:42.763 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:42.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:42.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:42.842 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:42 smithi098 bash[76817]: cluster 2023-09-23T18:34:41.760065+0000 mgr.y (mgr.26054) 483 : cluster 0 pgmap v439: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 48691/165798 objects degraded (29.368%); 7584/165798 objects misplaced (4.574%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T18:34:42.870 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:42.878 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:42 smithi173 bash[59422]: cluster 2023-09-23T18:34:41.760065+0000 mgr.y (mgr.26054) 483 : cluster 0 pgmap v439: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 48691/165798 objects degraded (29.368%); 7584/165798 objects misplaced (4.574%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T18:34:42.908 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:42.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:42.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:43.012 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:43.034 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:43.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:43.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:43.137 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:43.146 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:42 smithi098 bash[79516]: cluster 2023-09-23T18:34:41.760065+0000 mgr.y (mgr.26054) 483 : cluster 0 pgmap v439: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 48691/165798 objects degraded (29.368%); 7584/165798 objects misplaced (4.574%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T18:34:43.160 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:43.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:43.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:43.267 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:43.301 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:43.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:43.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:43.406 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:43.431 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:43.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:43.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:43.541 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:43.581 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:43.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:43.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:43.686 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:43.715 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:43.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:43.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:43.731 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:34:43.821 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:43.849 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:43.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:43.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:43.953 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:43.992 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:43.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:43.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:44.094 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:44.117 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:44.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:44.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:44.221 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:44.251 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:44.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:44.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:44.358 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:44.388 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:44.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:44.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:44.492 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:44.526 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:44.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:44.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:44.630 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:44.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:44.671 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:44.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:44.772 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:44.799 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:44.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:44.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:44.847 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:44 smithi098 bash[76817]: cluster 2023-09-23T18:34:43.760747+0000 mgr.y (mgr.26054) 484 : cluster 0 pgmap v440: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 48691/163332 objects degraded (29.811%); 7366/163332 objects misplaced (4.510%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T18:34:44.848 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:44 smithi098 bash[76817]: cluster 2023-09-23T18:34:44.467814+0000 mon.a (mon.0) 677 : cluster 3 Health check update: Degraded data redundancy: 48691/163332 objects degraded (29.811%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:44.903 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:44.942 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:44.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:44.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:45.047 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:45.075 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:45.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:45.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:45.170 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:44 smithi098 bash[79516]: cluster 2023-09-23T18:34:43.760747+0000 mgr.y (mgr.26054) 484 : cluster 0 pgmap v440: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 48691/163332 objects degraded (29.811%); 7366/163332 objects misplaced (4.510%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T18:34:45.170 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:44 smithi098 bash[79516]: cluster 2023-09-23T18:34:44.467814+0000 mon.a (mon.0) 677 : cluster 3 Health check update: Degraded data redundancy: 48691/163332 objects degraded (29.811%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:45.179 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:45.190 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:44 smithi173 bash[59422]: cluster 2023-09-23T18:34:43.760747+0000 mgr.y (mgr.26054) 484 : cluster 0 pgmap v440: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 48691/163332 objects degraded (29.811%); 7366/163332 objects misplaced (4.510%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T18:34:45.190 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:44 smithi173 bash[59422]: cluster 2023-09-23T18:34:44.467814+0000 mon.a (mon.0) 677 : cluster 3 Health check update: Degraded data redundancy: 48691/163332 objects degraded (29.811%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:45.209 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:45.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:45.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:45.316 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:45.341 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:45.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:45.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:45.444 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:45.481 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:45.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:45.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:45.587 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:45.613 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:45.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:45.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:45.657 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:34:45.715 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:45.718 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:34:45.753 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:34:45.764 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:45.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:45.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:45.810 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:34:45.871 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:45.872 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:34:45.879 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:45 smithi173 bash[59422]: audit 2023-09-23T18:34:45.610720+0000 mon.a (mon.0) 678 : audit 0 from='client.? 172.21.15.98:0/3102854079' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:34:45.892 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:45.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:45.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:45.914 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:34:45.961 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:34:45.995 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:45.998 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:34:46.002 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:34:46.002 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:34:46.002 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:34:46.002 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:34:46.002 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:34:46.002 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:34:46.002 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:34:46.002 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:34:46.020 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:46.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:46.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:46.040 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:45 smithi098 bash[76817]: audit 2023-09-23T18:34:45.610720+0000 mon.a (mon.0) 678 : audit 0 from='client.? 172.21.15.98:0/3102854079' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:34:46.040 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:45 smithi098 bash[79516]: audit 2023-09-23T18:34:45.610720+0000 mon.a (mon.0) 678 : audit 0 from='client.? 172.21.15.98:0/3102854079' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:34:46.123 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:46.149 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:46.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:46.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:46.256 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:46.310 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:46.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:46.315 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:34:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:34:46] "GET /metrics HTTP/1.1" 200 36543 "" "Prometheus/2.33.4" 2023-09-23T18:34:46.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:46.416 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:46.453 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:46.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:46.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:46.558 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:46.582 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:46.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:46.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:46.690 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:46.720 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:46.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:46.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:46.824 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:46.853 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:46.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:46.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:46.861 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:46 smithi098 bash[76817]: cluster 2023-09-23T18:34:45.761509+0000 mgr.y (mgr.26054) 485 : cluster 0 pgmap v441: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 48205/159798 objects degraded (30.166%); 7207/159798 objects misplaced (4.510%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T18:34:46.881 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is set 2023-09-23T18:34:46.886 INFO:teuthology.orchestra.run.smithi098.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T18:34:46.957 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:46.994 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:46.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:46.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:47.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:46 smithi173 bash[59422]: cluster 2023-09-23T18:34:45.761509+0000 mgr.y (mgr.26054) 485 : cluster 0 pgmap v441: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 48205/159798 objects degraded (30.166%); 7207/159798 objects misplaced (4.510%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T18:34:47.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:46 smithi173 bash[59422]: audit 2023-09-23T18:34:45.891944+0000 mon.a (mon.0) 679 : audit 1 from='client.? 172.21.15.98:0/4064424898' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:34:47.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:46 smithi173 bash[59422]: audit 2023-09-23T18:34:45.947162+0000 mon.c (mon.2) 66 : audit 1 from='client.? 172.21.15.98:0/3377245060' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:34:47.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:46 smithi173 bash[59422]: audit 2023-09-23T18:34:45.947983+0000 mon.a (mon.0) 680 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:34:47.098 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:47.128 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:47.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:47.132 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:46 smithi098 bash[76817]: audit 2023-09-23T18:34:45.891944+0000 mon.a (mon.0) 679 : audit 1 from='client.? 172.21.15.98:0/4064424898' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:34:47.132 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:46 smithi098 bash[76817]: audit 2023-09-23T18:34:45.947162+0000 mon.c (mon.2) 66 : audit 1 from='client.? 172.21.15.98:0/3377245060' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:34:47.132 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:46 smithi098 bash[76817]: audit 2023-09-23T18:34:45.947983+0000 mon.a (mon.0) 680 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:34:47.133 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:46 smithi098 bash[79516]: cluster 2023-09-23T18:34:45.761509+0000 mgr.y (mgr.26054) 485 : cluster 0 pgmap v441: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 48205/159798 objects degraded (30.166%); 7207/159798 objects misplaced (4.510%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T18:34:47.133 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:46 smithi098 bash[79516]: audit 2023-09-23T18:34:45.891944+0000 mon.a (mon.0) 679 : audit 1 from='client.? 172.21.15.98:0/4064424898' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:34:47.133 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:46 smithi098 bash[79516]: audit 2023-09-23T18:34:45.947162+0000 mon.c (mon.2) 66 : audit 1 from='client.? 172.21.15.98:0/3377245060' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:34:47.133 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:46 smithi098 bash[79516]: audit 2023-09-23T18:34:45.947983+0000 mon.a (mon.0) 680 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:34:47.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:47.234 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:47.253 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:47.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:47.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:47.357 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:47.393 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:47.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:47.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:47.500 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:47.524 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:47.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:47.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:47.630 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:47.672 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:47.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:47.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:47.780 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:47.817 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:47.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:47.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:47.924 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:47.935 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:47 smithi098 bash[79516]: cluster 2023-09-23T18:34:46.857053+0000 mon.a (mon.0) 681 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:34:47.935 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:47 smithi098 bash[79516]: cluster 2023-09-23T18:34:46.871031+0000 osd.4 (osd.4) 61 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T18:34:47.935 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:47 smithi098 bash[79516]: audit 2023-09-23T18:34:46.872127+0000 mon.a (mon.0) 682 : audit 1 from='client.? 172.21.15.98:0/4064424898' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:34:47.935 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:47 smithi098 bash[79516]: audit 2023-09-23T18:34:46.872352+0000 mon.a (mon.0) 683 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T18:34:47.935 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:47 smithi098 bash[79516]: cluster 2023-09-23T18:34:46.872579+0000 mon.a (mon.0) 684 : cluster 0 osdmap e130: 8 total, 8 up, 7 in 2023-09-23T18:34:47.935 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:47 smithi098 bash[79516]: cluster 2023-09-23T18:34:46.874827+0000 osd.4 (osd.4) 62 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T18:34:47.935 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:47 smithi098 bash[79516]: cluster 2023-09-23T18:34:46.895628+0000 osd.6 (osd.6) 46 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T18:34:47.935 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:47 smithi098 bash[79516]: cluster 2023-09-23T18:34:46.897625+0000 osd.6 (osd.6) 47 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T18:34:47.935 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:47 smithi098 bash[79516]: cluster 2023-09-23T18:34:46.914882+0000 osd.7 (osd.7) 40 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T18:34:47.935 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:47 smithi098 bash[79516]: cluster 2023-09-23T18:34:46.947008+0000 osd.5 (osd.5) 41 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T18:34:47.936 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:47 smithi098 bash[76817]: cluster 2023-09-23T18:34:46.857053+0000 mon.a (mon.0) 681 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:34:47.936 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:47 smithi098 bash[76817]: cluster 2023-09-23T18:34:46.871031+0000 osd.4 (osd.4) 61 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T18:34:47.936 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:47 smithi098 bash[76817]: audit 2023-09-23T18:34:46.872127+0000 mon.a (mon.0) 682 : audit 1 from='client.? 172.21.15.98:0/4064424898' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:34:47.936 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:47 smithi098 bash[76817]: audit 2023-09-23T18:34:46.872352+0000 mon.a (mon.0) 683 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T18:34:47.936 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:47 smithi098 bash[76817]: cluster 2023-09-23T18:34:46.872579+0000 mon.a (mon.0) 684 : cluster 0 osdmap e130: 8 total, 8 up, 7 in 2023-09-23T18:34:47.936 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:47 smithi098 bash[76817]: cluster 2023-09-23T18:34:46.874827+0000 osd.4 (osd.4) 62 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T18:34:47.937 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:47 smithi098 bash[76817]: cluster 2023-09-23T18:34:46.895628+0000 osd.6 (osd.6) 46 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T18:34:47.937 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:47 smithi098 bash[76817]: cluster 2023-09-23T18:34:46.897625+0000 osd.6 (osd.6) 47 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T18:34:47.937 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:47 smithi098 bash[76817]: cluster 2023-09-23T18:34:46.914882+0000 osd.7 (osd.7) 40 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T18:34:47.937 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:47 smithi098 bash[76817]: cluster 2023-09-23T18:34:46.947008+0000 osd.5 (osd.5) 41 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T18:34:47.951 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:47.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:47.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:48.059 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:48.088 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:48.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:48.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:48.197 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:48.228 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:48.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:48.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:48.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:47 smithi173 bash[59422]: cluster 2023-09-23T18:34:46.857053+0000 mon.a (mon.0) 681 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:34:48.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:47 smithi173 bash[59422]: cluster 2023-09-23T18:34:46.871031+0000 osd.4 (osd.4) 61 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T18:34:48.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:47 smithi173 bash[59422]: audit 2023-09-23T18:34:46.872127+0000 mon.a (mon.0) 682 : audit 1 from='client.? 172.21.15.98:0/4064424898' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:34:48.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:47 smithi173 bash[59422]: audit 2023-09-23T18:34:46.872352+0000 mon.a (mon.0) 683 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T18:34:48.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:47 smithi173 bash[59422]: cluster 2023-09-23T18:34:46.872579+0000 mon.a (mon.0) 684 : cluster 0 osdmap e130: 8 total, 8 up, 7 in 2023-09-23T18:34:48.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:47 smithi173 bash[59422]: cluster 2023-09-23T18:34:46.874827+0000 osd.4 (osd.4) 62 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T18:34:48.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:47 smithi173 bash[59422]: cluster 2023-09-23T18:34:46.895628+0000 osd.6 (osd.6) 46 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T18:34:48.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:47 smithi173 bash[59422]: cluster 2023-09-23T18:34:46.897625+0000 osd.6 (osd.6) 47 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T18:34:48.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:47 smithi173 bash[59422]: cluster 2023-09-23T18:34:46.914882+0000 osd.7 (osd.7) 40 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T18:34:48.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:47 smithi173 bash[59422]: cluster 2023-09-23T18:34:46.947008+0000 osd.5 (osd.5) 41 : cluster 3 failed to encode map e130 with expected crc 2023-09-23T18:34:48.334 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:48.372 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:48.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:48.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:48.480 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:48.506 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:48.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:48.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:48.609 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:48.633 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:48.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:48.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:48.740 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:48.765 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:48.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:48.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:48.867 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:48.895 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:48.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:48.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:48.999 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:49.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:48 smithi173 bash[59422]: cluster 2023-09-23T18:34:47.762181+0000 mgr.y (mgr.26054) 486 : cluster 0 pgmap v443: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 48226/155325 objects degraded (31.048%); 6821/155325 objects misplaced (4.391%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T18:34:49.034 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:49.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:49.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:49.107 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T18:34:49.137 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:49.161 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:48 smithi098 bash[76817]: cluster 2023-09-23T18:34:47.762181+0000 mgr.y (mgr.26054) 486 : cluster 0 pgmap v443: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 48226/155325 objects degraded (31.048%); 6821/155325 objects misplaced (4.391%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T18:34:49.162 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:48 smithi098 bash[79516]: cluster 2023-09-23T18:34:47.762181+0000 mgr.y (mgr.26054) 486 : cluster 0 pgmap v443: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 48226/155325 objects degraded (31.048%); 6821/155325 objects misplaced (4.391%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T18:34:49.202 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:49.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:49.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:49.306 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:49.341 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:49.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:49.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:49.451 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:49.477 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:34:49.477 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-09-23T18:34:49.477 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:49.477 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.44693.0:162599 5.7 e130/124 5:e38ee74d:::benchmark_data_smithi098_105788_object56314:head v 130'10117, mlcod=112'3451)", 2023-09-23T18:34:49.477 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:34:49.474461+0000", 2023-09-23T18:34:49.477 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.0020250110000000002, 2023-09-23T18:34:49.478 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0020589839999999998, 2023-09-23T18:34:49.478 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:34:49.478 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "started", 2023-09-23T18:34:49.478 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:34:49.478 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:49.478 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:34:49.478 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:49.474461+0000", 2023-09-23T18:34:49.478 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:34:49.478 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:49.478 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:49.478 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:34:49.479 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:49.474461+0000", 2023-09-23T18:34:49.479 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:34:49.479 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:49.479 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:49.479 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:34:49.479 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:49.474460+0000", 2023-09-23T18:34:49.479 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2785e-05 2023-09-23T18:34:49.479 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:49.479 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:49.479 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:34:49.479 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:49.474473+0000", 2023-09-23T18:34:49.479 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3845e-05 2023-09-23T18:34:49.480 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:49.480 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:49.480 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:34:49.480 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:49.474496+0000", 2023-09-23T18:34:49.480 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.7220000000000005e-06 2023-09-23T18:34:49.480 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:49.480 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:49.480 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:34:49.480 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:49.474505+0000", 2023-09-23T18:34:49.480 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.0903000000000003e-05 2023-09-23T18:34:49.480 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:49.480 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:49.480 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:34:49.481 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:49.474566+0000", 2023-09-23T18:34:49.481 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.1288000000000003e-05 2023-09-23T18:34:49.481 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:34:49.481 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:34:49.481 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:34:49.481 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:49.474597+0000", 2023-09-23T18:34:49.481 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013621299999999999 2023-09-23T18:34:49.481 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:34:49.481 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:34:49.481 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:34:49.481 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:34:49.481 INFO:teuthology.orchestra.run.smithi173.stdout: ], 2023-09-23T18:34:49.482 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 1 2023-09-23T18:34:49.482 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:34:49.492 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:49.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:49.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:49.597 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:49.636 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:49.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:49.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:49.742 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:49.770 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:49.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:49.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:49.792 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:34:49.879 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:49.901 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:49.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:49.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:50.007 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:50.015 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:49 smithi173 bash[59422]: cluster 2023-09-23T18:34:49.468862+0000 mon.a (mon.0) 685 : cluster 3 Health check update: Degraded data redundancy: 48226/155325 objects degraded (31.048%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:50.031 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:50.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:50.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:50.138 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:50.168 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:50.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:50.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:50.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:49 smithi098 bash[76817]: cluster 2023-09-23T18:34:49.468862+0000 mon.a (mon.0) 685 : cluster 3 Health check update: Degraded data redundancy: 48226/155325 objects degraded (31.048%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:50.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:49 smithi098 bash[79516]: cluster 2023-09-23T18:34:49.468862+0000 mon.a (mon.0) 685 : cluster 3 Health check update: Degraded data redundancy: 48226/155325 objects degraded (31.048%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:50.271 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:50.296 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:50.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:50.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:50.400 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:50.435 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:50.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:50.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:50.541 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:50.564 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:50.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:50.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:50.643 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T18:34:50.668 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:50.718 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:50.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:50.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:50.821 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:50.850 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:50.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:50.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:50.953 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:50.962 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:50 smithi173 bash[59422]: cluster 2023-09-23T18:34:49.762854+0000 mgr.y (mgr.26054) 487 : cluster 0 pgmap v444: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 47743/153435 objects degraded (31.116%); 6672/153435 objects misplaced (4.348%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T18:34:50.979 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:50.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:50.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:51.085 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:51.090 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:34:51.094 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:50 smithi098 bash[76817]: cluster 2023-09-23T18:34:49.762854+0000 mgr.y (mgr.26054) 487 : cluster 0 pgmap v444: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 47743/153435 objects degraded (31.116%); 6672/153435 objects misplaced (4.348%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T18:34:51.095 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:50 smithi098 bash[79516]: cluster 2023-09-23T18:34:49.762854+0000 mgr.y (mgr.26054) 487 : cluster 0 pgmap v444: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 47743/153435 objects degraded (31.116%); 6672/153435 objects misplaced (4.348%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T18:34:51.115 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:51.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:51.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:51.225 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:51.279 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:51.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:51.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:51.385 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:51.410 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:51.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:51.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:51.515 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:51.540 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:51.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:51.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:51.643 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:51.672 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:51.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:51.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:51.775 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:51.816 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:51.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:51.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:51.923 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:51.949 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:51.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:51.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:52.057 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:52.088 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:52.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:52.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:52.191 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:52.218 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:52.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:52.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:52.324 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:52.360 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:52.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:52.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:52.464 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:52.496 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:52.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:52.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:52.603 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:52.630 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:52.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:52.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:52.735 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:52.773 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:52.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:52.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:52.880 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:52.886 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:52 smithi173 bash[59422]: cluster 2023-09-23T18:34:51.763652+0000 mgr.y (mgr.26054) 488 : cluster 0 pgmap v445: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 47743/150093 objects degraded (31.809%); 6513/150093 objects misplaced (4.339%); 6.6 MiB/s, 104 objects/s recovering 2023-09-23T18:34:52.887 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:52 smithi098 bash[79516]: cluster 2023-09-23T18:34:51.763652+0000 mgr.y (mgr.26054) 488 : cluster 0 pgmap v445: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 47743/150093 objects degraded (31.809%); 6513/150093 objects misplaced (4.339%); 6.6 MiB/s, 104 objects/s recovering 2023-09-23T18:34:52.887 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:52 smithi098 bash[76817]: cluster 2023-09-23T18:34:51.763652+0000 mgr.y (mgr.26054) 488 : cluster 0 pgmap v445: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 47743/150093 objects degraded (31.809%); 6513/150093 objects misplaced (4.339%); 6.6 MiB/s, 104 objects/s recovering 2023-09-23T18:34:52.905 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:52.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:52.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:53.011 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:53.046 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:53.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:53.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:53.154 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:53.187 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:53.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:53.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:53.295 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:53.329 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:53.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:53.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:53.436 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:53.477 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:53.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:53.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:53.581 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:53.604 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:53.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:53.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:53.709 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:53.732 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:53.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:53.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:53.836 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:53.861 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:53.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:53.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:53.968 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:54.015 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:54.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:54.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:54.018 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:34:54.025 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:34:54.119 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:54.145 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:54.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:54.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:54.256 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:54.279 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:54.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:54.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:54.385 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:54.422 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:54.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:54.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:54.527 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:54.564 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:54.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:54.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:54.669 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:54.709 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:54.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:54.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:54.812 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:54.837 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:54.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:54.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:54.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:54 smithi098 bash[79516]: cluster 2023-09-23T18:34:53.764346+0000 mgr.y (mgr.26054) 489 : cluster 0 pgmap v446: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 47743/147663 objects degraded (32.332%); 6263/147663 objects misplaced (4.241%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T18:34:54.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:54 smithi098 bash[79516]: cluster 2023-09-23T18:34:54.469884+0000 mon.a (mon.0) 686 : cluster 3 Health check update: Degraded data redundancy: 47743/147663 objects degraded (32.332%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:54.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:54 smithi098 bash[79516]: audit 2023-09-23T18:34:54.583514+0000 mon.a (mon.0) 687 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:34:54.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:54 smithi098 bash[76817]: cluster 2023-09-23T18:34:53.764346+0000 mgr.y (mgr.26054) 489 : cluster 0 pgmap v446: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 47743/147663 objects degraded (32.332%); 6263/147663 objects misplaced (4.241%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T18:34:54.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:54 smithi098 bash[76817]: cluster 2023-09-23T18:34:54.469884+0000 mon.a (mon.0) 686 : cluster 3 Health check update: Degraded data redundancy: 47743/147663 objects degraded (32.332%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:54.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:54 smithi098 bash[76817]: audit 2023-09-23T18:34:54.583514+0000 mon.a (mon.0) 687 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:34:54.941 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:54.974 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:54.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:54.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:55.039 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:34:55.083 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:55.092 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:54 smithi173 bash[59422]: cluster 2023-09-23T18:34:53.764346+0000 mgr.y (mgr.26054) 489 : cluster 0 pgmap v446: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 47743/147663 objects degraded (32.332%); 6263/147663 objects misplaced (4.241%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T18:34:55.092 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:54 smithi173 bash[59422]: cluster 2023-09-23T18:34:54.469884+0000 mon.a (mon.0) 686 : cluster 3 Health check update: Degraded data redundancy: 47743/147663 objects degraded (32.332%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:55.092 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:54 smithi173 bash[59422]: audit 2023-09-23T18:34:54.583514+0000 mon.a (mon.0) 687 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:34:55.110 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:55.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:55.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:55.213 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:55.243 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:55.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:55.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:55.347 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:55.376 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:55.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:55.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:55.465 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:34:55.484 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:55.512 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:55.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:55.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:55.618 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:55.650 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:55.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:55.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:55.756 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:55.789 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:55.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:55.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:55.894 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:55.922 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:55.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:55.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:56.027 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:56.054 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:56.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:56.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:56.085 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:34:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:34:56] "GET /metrics HTTP/1.1" 200 36539 "" "Prometheus/2.33.4" 2023-09-23T18:34:56.160 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:56.191 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:56.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:56.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:56.294 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:56.319 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:56.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:56.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:56.424 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:56.451 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:56.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:56.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:56.555 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:56.586 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:56.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:56.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:56.689 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:56.718 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:56.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:56.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:56.820 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:56.848 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:56.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:56.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:56.953 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:56.962 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:56 smithi173 bash[59422]: cluster 2023-09-23T18:34:55.765110+0000 mgr.y (mgr.26054) 490 : cluster 0 pgmap v447: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 47267/144321 objects degraded (32.751%); 6084/144321 objects misplaced (4.216%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T18:34:56.980 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:56.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:56.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:57.087 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:57.128 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:57.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:57.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:57.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:56 smithi098 bash[79516]: cluster 2023-09-23T18:34:55.765110+0000 mgr.y (mgr.26054) 490 : cluster 0 pgmap v447: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 47267/144321 objects degraded (32.751%); 6084/144321 objects misplaced (4.216%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T18:34:57.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:56 smithi098 bash[76817]: cluster 2023-09-23T18:34:55.765110+0000 mgr.y (mgr.26054) 490 : cluster 0 pgmap v447: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 47267/144321 objects degraded (32.751%); 6084/144321 objects misplaced (4.216%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T18:34:57.231 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:57.267 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:57.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:57.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:57.373 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:57.407 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:57.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:57.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:57.513 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:57.539 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:57.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:57.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:57.642 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:57.670 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:57.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:57.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:57.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:57.811 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:57.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:57.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:57.916 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:57.951 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:34:57.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:57.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:34:58.057 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:58.083 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:58.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:58.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:58.190 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:58.213 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:58.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:58.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:58.315 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:58.347 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:34:58.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:58.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:34:58.455 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:58.481 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:58.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:58.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:58.585 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:58.612 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:58.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:58.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:58.719 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:58.750 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:34:58.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:58.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:34:58.856 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:58.884 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:58.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:58.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:34:58.898 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:58 smithi098 bash[76817]: cluster 2023-09-23T18:34:57.765896+0000 mgr.y (mgr.26054) 491 : cluster 0 pgmap v448: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 470 op/s; 47267/142494 objects degraded (33.171%); 5929/142494 objects misplaced (4.161%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T18:34:58.989 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:59.023 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:59.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:59.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:59.027 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:58 smithi098 bash[79516]: cluster 2023-09-23T18:34:57.765896+0000 mgr.y (mgr.26054) 491 : cluster 0 pgmap v448: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 470 op/s; 47267/142494 objects degraded (33.171%); 5929/142494 objects misplaced (4.161%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T18:34:59.127 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:59.155 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:59.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:59.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:59.262 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:59.268 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:58 smithi173 bash[59422]: cluster 2023-09-23T18:34:57.765896+0000 mgr.y (mgr.26054) 491 : cluster 0 pgmap v448: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 470 op/s; 47267/142494 objects degraded (33.171%); 5929/142494 objects misplaced (4.161%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T18:34:59.283 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:34:59.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:59.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:34:59.391 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:59.420 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:59.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:59.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:59.524 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:59.562 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:34:59.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:59.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:34:59.672 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:59.695 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:34:59.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:59.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:34:59.801 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:59.832 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:34:59.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:59.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:34:59.935 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:59.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:59 smithi098 bash[79516]: cluster 2023-09-23T18:34:59.470921+0000 mon.a (mon.0) 688 : cluster 3 Health check update: Degraded data redundancy: 47267/142494 objects degraded (33.171%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:59.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:34:59 smithi098 bash[79516]: audit 2023-09-23T18:34:59.815073+0000 mon.a (mon.0) 689 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:34:59.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:59 smithi098 bash[76817]: cluster 2023-09-23T18:34:59.470921+0000 mon.a (mon.0) 688 : cluster 3 Health check update: Degraded data redundancy: 47267/142494 objects degraded (33.171%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T18:34:59.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:34:59 smithi098 bash[76817]: audit 2023-09-23T18:34:59.815073+0000 mon.a (mon.0) 689 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:34:59.956 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:34:59.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:34:59.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:00.059 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:00.097 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:00.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:00.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:00.206 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:00.216 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:59 smithi173 bash[59422]: cluster 2023-09-23T18:34:59.470921+0000 mon.a (mon.0) 688 : cluster 3 Health check update: Degraded data redundancy: 47267/142494 objects degraded (33.171%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:00.216 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:34:59 smithi173 bash[59422]: audit 2023-09-23T18:34:59.815073+0000 mon.a (mon.0) 689 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:35:00.234 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:00.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:00.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:00.337 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:00.377 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:00.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:00.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:00.481 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:00.514 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:00.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:00.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:00.560 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:35:00.560 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-09-23T18:35:00.560 INFO:teuthology.orchestra.run.smithi173.stdout: "complaint_time": 30, 2023-09-23T18:35:00.560 INFO:teuthology.orchestra.run.smithi173.stdout: "num_blocked_ops": 0 2023-09-23T18:35:00.561 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:35:00.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:00.643 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:00.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:00.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:00.747 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:00.782 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:00.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:00.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:00.889 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:00.911 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:00.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:00.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:01.017 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:01.030 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:00 smithi173 bash[59422]: cluster 2023-09-23T18:34:59.766563+0000 mgr.y (mgr.26054) 492 : cluster 0 pgmap v449: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 46751/137976 objects degraded (33.883%); 5477/137976 objects misplaced (3.970%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T18:35:01.044 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:01.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:01.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:01.147 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:01.154 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:00 smithi098 bash[79516]: cluster 2023-09-23T18:34:59.766563+0000 mgr.y (mgr.26054) 492 : cluster 0 pgmap v449: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 46751/137976 objects degraded (33.883%); 5477/137976 objects misplaced (3.970%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T18:35:01.154 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:00 smithi098 bash[76817]: cluster 2023-09-23T18:34:59.766563+0000 mgr.y (mgr.26054) 492 : cluster 0 pgmap v449: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 46751/137976 objects degraded (33.883%); 5477/137976 objects misplaced (3.970%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T18:35:01.167 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:01.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:01.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:01.270 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:01.310 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:01.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:01.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:01.417 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:01.445 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:01.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:01.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:01.550 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:01.590 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:01.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:01.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:01.635 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T18:35:01.694 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:01.736 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:01.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:01.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:01.842 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:01.890 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:01.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:01.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:01.997 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:02.025 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:02.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:02.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:02.130 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:02.159 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:02.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:02.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:02.262 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:02.285 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:02.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:02.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:02.390 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:02.438 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:02.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:02.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:02.540 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:02.571 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:02.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:02.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:02.676 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:02.698 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:02.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:02.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:02.800 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:02.835 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:02.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:02.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:02.940 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:02.968 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:02.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:02.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:03.074 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:03.102 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:03.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:03.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:03.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:02 smithi098 bash[76817]: cluster 2023-09-23T18:35:01.767361+0000 mgr.y (mgr.26054) 493 : cluster 0 pgmap v450: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 46751/134844 objects degraded (34.670%); 5312/134844 objects misplaced (3.939%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T18:35:03.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:02 smithi098 bash[79516]: cluster 2023-09-23T18:35:01.767361+0000 mgr.y (mgr.26054) 493 : cluster 0 pgmap v450: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 46751/134844 objects degraded (34.670%); 5312/134844 objects misplaced (3.939%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T18:35:03.210 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:03.218 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:02 smithi173 bash[59422]: cluster 2023-09-23T18:35:01.767361+0000 mgr.y (mgr.26054) 493 : cluster 0 pgmap v450: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 46751/134844 objects degraded (34.670%); 5312/134844 objects misplaced (3.939%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T18:35:03.237 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:03.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:03.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:03.344 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:03.389 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:03.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:03.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:03.494 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:03.521 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:03.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:03.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:03.627 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:03.652 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:03.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:03.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:03.758 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:03.784 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:03.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:03.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:03.887 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:03.915 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:03.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:03.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:04.025 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:04.052 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:04.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:04.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:04.162 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:04.185 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:04.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:04.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:04.288 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:04.320 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:04.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:04.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:04.425 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:04.449 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:04.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:04.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:04.552 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:04.580 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:04.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:04.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:04.683 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:04.710 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:04.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:04.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:04.816 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:04.840 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:04.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:04.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:04.946 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:04.974 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:04.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:04.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:05.077 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:05.088 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:04 smithi173 bash[59422]: cluster 2023-09-23T18:35:03.768178+0000 mgr.y (mgr.26054) 494 : cluster 0 pgmap v451: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 46751/132447 objects degraded (35.298%); 4973/132447 objects misplaced (3.755%); 8.0 MiB/s, 128 objects/s recovering 2023-09-23T18:35:05.088 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:04 smithi173 bash[59422]: cluster 2023-09-23T18:35:04.471949+0000 mon.a (mon.0) 690 : cluster 3 Health check update: Degraded data redundancy: 46751/132447 objects degraded (35.298%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:05.088 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:04 smithi173 bash[59422]: cluster 2023-09-23T18:35:04.479198+0000 mon.a (mon.0) 691 : cluster 0 osdmap e131: 8 total, 8 up, 7 in 2023-09-23T18:35:05.088 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:04 smithi173 bash[59422]: cluster 2023-09-23T18:35:04.480894+0000 osd.6 (osd.6) 48 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T18:35:05.088 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:04 smithi173 bash[59422]: cluster 2023-09-23T18:35:04.490705+0000 osd.5 (osd.5) 42 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T18:35:05.102 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:05.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:05.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:05.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:04 smithi098 bash[76817]: cluster 2023-09-23T18:35:03.768178+0000 mgr.y (mgr.26054) 494 : cluster 0 pgmap v451: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 46751/132447 objects degraded (35.298%); 4973/132447 objects misplaced (3.755%); 8.0 MiB/s, 128 objects/s recovering 2023-09-23T18:35:05.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:04 smithi098 bash[76817]: cluster 2023-09-23T18:35:04.471949+0000 mon.a (mon.0) 690 : cluster 3 Health check update: Degraded data redundancy: 46751/132447 objects degraded (35.298%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:05.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:04 smithi098 bash[76817]: cluster 2023-09-23T18:35:04.479198+0000 mon.a (mon.0) 691 : cluster 0 osdmap e131: 8 total, 8 up, 7 in 2023-09-23T18:35:05.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:04 smithi098 bash[76817]: cluster 2023-09-23T18:35:04.480894+0000 osd.6 (osd.6) 48 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T18:35:05.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:04 smithi098 bash[76817]: cluster 2023-09-23T18:35:04.490705+0000 osd.5 (osd.5) 42 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T18:35:05.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:04 smithi098 bash[79516]: cluster 2023-09-23T18:35:03.768178+0000 mgr.y (mgr.26054) 494 : cluster 0 pgmap v451: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 46751/132447 objects degraded (35.298%); 4973/132447 objects misplaced (3.755%); 8.0 MiB/s, 128 objects/s recovering 2023-09-23T18:35:05.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:04 smithi098 bash[79516]: cluster 2023-09-23T18:35:04.471949+0000 mon.a (mon.0) 690 : cluster 3 Health check update: Degraded data redundancy: 46751/132447 objects degraded (35.298%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:05.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:04 smithi098 bash[79516]: cluster 2023-09-23T18:35:04.479198+0000 mon.a (mon.0) 691 : cluster 0 osdmap e131: 8 total, 8 up, 7 in 2023-09-23T18:35:05.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:04 smithi098 bash[79516]: cluster 2023-09-23T18:35:04.480894+0000 osd.6 (osd.6) 48 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T18:35:05.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:04 smithi098 bash[79516]: cluster 2023-09-23T18:35:04.490705+0000 osd.5 (osd.5) 42 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T18:35:05.206 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:05.236 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:05.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:05.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:05.340 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:05.362 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:05.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:05.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:05.465 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:05.487 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:05.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:05.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:05.591 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:05.630 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:05.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:05.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:05.738 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:05.766 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:05.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:05.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:05.869 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:05.897 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:05.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:05.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:06.003 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:06.036 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:06.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:06.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:06.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:05 smithi098 bash[76817]: cluster 2023-09-23T18:35:04.476105+0000 osd.7 (osd.7) 41 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T18:35:06.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:05 smithi098 bash[76817]: cluster 2023-09-23T18:35:04.490477+0000 osd.4 (osd.4) 63 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T18:35:06.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:05 smithi098 bash[76817]: cluster 2023-09-23T18:35:04.492456+0000 osd.4 (osd.4) 64 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T18:35:06.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:05 smithi098 bash[76817]: cluster 2023-09-23T18:35:05.490114+0000 osd.5 (osd.5) 43 : cluster 3 failed to encode map e132 with expected crc 2023-09-23T18:35:06.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:05 smithi098 bash[76817]: cluster 2023-09-23T18:35:05.491485+0000 mon.a (mon.0) 692 : cluster 0 osdmap e132: 8 total, 8 up, 7 in 2023-09-23T18:35:06.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:05 smithi098 bash[79516]: cluster 2023-09-23T18:35:04.476105+0000 osd.7 (osd.7) 41 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T18:35:06.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:05 smithi098 bash[79516]: cluster 2023-09-23T18:35:04.490477+0000 osd.4 (osd.4) 63 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T18:35:06.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:05 smithi098 bash[79516]: cluster 2023-09-23T18:35:04.492456+0000 osd.4 (osd.4) 64 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T18:35:06.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:05 smithi098 bash[79516]: cluster 2023-09-23T18:35:05.490114+0000 osd.5 (osd.5) 43 : cluster 3 failed to encode map e132 with expected crc 2023-09-23T18:35:06.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:05 smithi098 bash[79516]: cluster 2023-09-23T18:35:05.491485+0000 mon.a (mon.0) 692 : cluster 0 osdmap e132: 8 total, 8 up, 7 in 2023-09-23T18:35:06.088 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:35:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:35:06] "GET /metrics HTTP/1.1" 200 36539 "" "Prometheus/2.33.4" 2023-09-23T18:35:06.164 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:06.186 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:35:06.207 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:06.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:06.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:06.219 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:05 smithi173 bash[59422]: cluster 2023-09-23T18:35:04.476105+0000 osd.7 (osd.7) 41 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T18:35:06.219 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:05 smithi173 bash[59422]: cluster 2023-09-23T18:35:04.490477+0000 osd.4 (osd.4) 63 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T18:35:06.219 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:05 smithi173 bash[59422]: cluster 2023-09-23T18:35:04.492456+0000 osd.4 (osd.4) 64 : cluster 3 failed to encode map e131 with expected crc 2023-09-23T18:35:06.219 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:05 smithi173 bash[59422]: cluster 2023-09-23T18:35:05.490114+0000 osd.5 (osd.5) 43 : cluster 3 failed to encode map e132 with expected crc 2023-09-23T18:35:06.219 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:05 smithi173 bash[59422]: cluster 2023-09-23T18:35:05.491485+0000 mon.a (mon.0) 692 : cluster 0 osdmap e132: 8 total, 8 up, 7 in 2023-09-23T18:35:06.313 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:06.342 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:06.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:06.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:06.448 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:06.475 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:06.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:06.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:06.583 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:06.616 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:06.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:06.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:06.722 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:06.765 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:06.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:06.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:06.869 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:06.887 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:35:06.903 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:06.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:06.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:06.953 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:35:07.006 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:07.008 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:35:07.017 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:06 smithi173 bash[59422]: cluster 2023-09-23T18:35:05.494597+0000 osd.7 (osd.7) 42 : cluster 3 failed to encode map e132 with expected crc 2023-09-23T18:35:07.018 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:06 smithi173 bash[59422]: cluster 2023-09-23T18:35:05.511652+0000 osd.4 (osd.4) 65 : cluster 3 failed to encode map e132 with expected crc 2023-09-23T18:35:07.018 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:06 smithi173 bash[59422]: cluster 2023-09-23T18:35:05.768921+0000 mgr.y (mgr.26054) 495 : cluster 0 pgmap v454: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 46239/129177 objects degraded (35.795%); 4793/129177 objects misplaced (3.710%); 8.9 MiB/s, 141 objects/s recovering 2023-09-23T18:35:07.018 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:06 smithi173 bash[59422]: audit 2023-09-23T18:35:06.620626+0000 mon.a (mon.0) 693 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:35:07.018 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:06 smithi173 bash[59422]: audit 2023-09-23T18:35:06.636797+0000 mon.a (mon.0) 694 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:35:07.018 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:06 smithi173 bash[59422]: audit 2023-09-23T18:35:06.821807+0000 mon.b (mon.1) 7 : audit 0 from='client.? 172.21.15.98:0/692488083' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:35:07.034 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:07.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:07.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:07.054 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:35:07.113 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:35:07.140 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:07.162 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:07.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:07.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:07.168 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:35:07.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:06 smithi098 bash[76817]: cluster 2023-09-23T18:35:05.494597+0000 osd.7 (osd.7) 42 : cluster 3 failed to encode map e132 with expected crc 2023-09-23T18:35:07.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:06 smithi098 bash[76817]: cluster 2023-09-23T18:35:05.511652+0000 osd.4 (osd.4) 65 : cluster 3 failed to encode map e132 with expected crc 2023-09-23T18:35:07.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:06 smithi098 bash[76817]: cluster 2023-09-23T18:35:05.768921+0000 mgr.y (mgr.26054) 495 : cluster 0 pgmap v454: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 46239/129177 objects degraded (35.795%); 4793/129177 objects misplaced (3.710%); 8.9 MiB/s, 141 objects/s recovering 2023-09-23T18:35:07.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:06 smithi098 bash[76817]: audit 2023-09-23T18:35:06.620626+0000 mon.a (mon.0) 693 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:35:07.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:06 smithi098 bash[76817]: audit 2023-09-23T18:35:06.636797+0000 mon.a (mon.0) 694 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:35:07.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:06 smithi098 bash[76817]: audit 2023-09-23T18:35:06.821807+0000 mon.b (mon.1) 7 : audit 0 from='client.? 172.21.15.98:0/692488083' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:35:07.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:06 smithi098 bash[79516]: cluster 2023-09-23T18:35:05.494597+0000 osd.7 (osd.7) 42 : cluster 3 failed to encode map e132 with expected crc 2023-09-23T18:35:07.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:06 smithi098 bash[79516]: cluster 2023-09-23T18:35:05.511652+0000 osd.4 (osd.4) 65 : cluster 3 failed to encode map e132 with expected crc 2023-09-23T18:35:07.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:06 smithi098 bash[79516]: cluster 2023-09-23T18:35:05.768921+0000 mgr.y (mgr.26054) 495 : cluster 0 pgmap v454: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 46239/129177 objects degraded (35.795%); 4793/129177 objects misplaced (3.710%); 8.9 MiB/s, 141 objects/s recovering 2023-09-23T18:35:07.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:06 smithi098 bash[79516]: audit 2023-09-23T18:35:06.620626+0000 mon.a (mon.0) 693 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:35:07.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:06 smithi098 bash[79516]: audit 2023-09-23T18:35:06.636797+0000 mon.a (mon.0) 694 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:35:07.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:06 smithi098 bash[79516]: audit 2023-09-23T18:35:06.821807+0000 mon.b (mon.1) 7 : audit 0 from='client.? 172.21.15.98:0/692488083' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:35:07.212 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:35:07.256 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:35:07.264 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:35:07.264 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:35:07.264 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:35:07.264 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:35:07.264 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:35:07.265 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:35:07.265 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:35:07.265 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:35:07.266 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:07.318 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:07.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:07.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:07.425 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:07.447 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:07.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:07.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:07.550 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:07.574 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:07.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:07.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:07.678 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:07.706 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:07.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:07.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:07.811 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:07.840 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:07.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:07.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:07.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:07 smithi098 bash[76817]: audit 2023-09-23T18:35:07.471026+0000 mon.a (mon.0) 695 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:35:07.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:07 smithi098 bash[76817]: audit 2023-09-23T18:35:07.471069+0000 mon.b (mon.1) 8 : audit 1 from='client.? 172.21.15.98:0/3983811517' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:35:07.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:07 smithi098 bash[76817]: audit 2023-09-23T18:35:07.578052+0000 mon.c (mon.2) 67 : audit 1 from='client.? 172.21.15.98:0/1697995455' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T18:35:07.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:07 smithi098 bash[76817]: audit 2023-09-23T18:35:07.580643+0000 mon.a (mon.0) 696 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T18:35:07.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:07 smithi098 bash[79516]: audit 2023-09-23T18:35:07.471026+0000 mon.a (mon.0) 695 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:35:07.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:07 smithi098 bash[79516]: audit 2023-09-23T18:35:07.471069+0000 mon.b (mon.1) 8 : audit 1 from='client.? 172.21.15.98:0/3983811517' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:35:07.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:07 smithi098 bash[79516]: audit 2023-09-23T18:35:07.578052+0000 mon.c (mon.2) 67 : audit 1 from='client.? 172.21.15.98:0/1697995455' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T18:35:07.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:07 smithi098 bash[79516]: audit 2023-09-23T18:35:07.580643+0000 mon.a (mon.0) 696 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T18:35:07.943 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:07.945 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is unset 2023-09-23T18:35:07.954 INFO:teuthology.orchestra.run.smithi098.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T18:35:07.975 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:07.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:07.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:08.078 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:08.104 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:08.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:08.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:08.209 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:08.238 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:08.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:08.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:08.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:07 smithi173 bash[59422]: audit 2023-09-23T18:35:07.471026+0000 mon.a (mon.0) 695 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:35:08.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:07 smithi173 bash[59422]: audit 2023-09-23T18:35:07.471069+0000 mon.b (mon.1) 8 : audit 1 from='client.? 172.21.15.98:0/3983811517' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:35:08.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:07 smithi173 bash[59422]: audit 2023-09-23T18:35:07.578052+0000 mon.c (mon.2) 67 : audit 1 from='client.? 172.21.15.98:0/1697995455' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T18:35:08.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:07 smithi173 bash[59422]: audit 2023-09-23T18:35:07.580643+0000 mon.a (mon.0) 696 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T18:35:08.346 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:08.378 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:08.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:08.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:08.487 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:08.511 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:08.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:08.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:08.614 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:08.637 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:08.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:08.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:08.744 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:08.782 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:08.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:08.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:08.886 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:08.911 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:08.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:08.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:08.934 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:08 smithi098 bash[76817]: cluster 2023-09-23T18:35:07.769850+0000 mgr.y (mgr.26054) 496 : cluster 0 pgmap v455: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 439 op/s; 46239/127416 objects degraded (36.290%); 4617/127416 objects misplaced (3.624%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T18:35:08.934 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:08 smithi098 bash[76817]: cluster 2023-09-23T18:35:07.923170+0000 mon.a (mon.0) 697 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:35:08.934 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:08 smithi098 bash[76817]: audit 2023-09-23T18:35:07.936217+0000 mon.a (mon.0) 698 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:35:08.934 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:08 smithi098 bash[76817]: audit 2023-09-23T18:35:07.936293+0000 mon.a (mon.0) 699 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T18:35:08.934 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:08 smithi098 bash[76817]: cluster 2023-09-23T18:35:07.936347+0000 mon.a (mon.0) 700 : cluster 0 osdmap e133: 8 total, 8 up, 7 in 2023-09-23T18:35:08.935 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:08 smithi098 bash[76817]: cluster 2023-09-23T18:35:07.942884+0000 osd.4 (osd.4) 66 : cluster 3 failed to encode map e133 with expected crc 2023-09-23T18:35:08.935 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:08 smithi098 bash[76817]: cluster 2023-09-23T18:35:07.943871+0000 osd.7 (osd.7) 43 : cluster 3 failed to encode map e133 with expected crc 2023-09-23T18:35:08.935 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:08 smithi098 bash[76817]: cluster 2023-09-23T18:35:07.944175+0000 osd.5 (osd.5) 44 : cluster 3 failed to encode map e133 with expected crc 2023-09-23T18:35:08.935 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:08 smithi098 bash[76817]: cluster 2023-09-23T18:35:07.945096+0000 osd.4 (osd.4) 67 : cluster 3 failed to encode map e133 with expected crc 2023-09-23T18:35:08.935 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:08 smithi098 bash[76817]: cluster 2023-09-23T18:35:07.954398+0000 osd.6 (osd.6) 49 : cluster 3 failed to encode map e133 with expected crc 2023-09-23T18:35:08.935 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:08 smithi098 bash[79516]: cluster 2023-09-23T18:35:07.769850+0000 mgr.y (mgr.26054) 496 : cluster 0 pgmap v455: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 439 op/s; 46239/127416 objects degraded (36.290%); 4617/127416 objects misplaced (3.624%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T18:35:08.935 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:08 smithi098 bash[79516]: cluster 2023-09-23T18:35:07.923170+0000 mon.a (mon.0) 697 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:35:08.935 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:08 smithi098 bash[79516]: audit 2023-09-23T18:35:07.936217+0000 mon.a (mon.0) 698 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:35:08.935 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:08 smithi098 bash[79516]: audit 2023-09-23T18:35:07.936293+0000 mon.a (mon.0) 699 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T18:35:08.935 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:08 smithi098 bash[79516]: cluster 2023-09-23T18:35:07.936347+0000 mon.a (mon.0) 700 : cluster 0 osdmap e133: 8 total, 8 up, 7 in 2023-09-23T18:35:08.936 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:08 smithi098 bash[79516]: cluster 2023-09-23T18:35:07.942884+0000 osd.4 (osd.4) 66 : cluster 3 failed to encode map e133 with expected crc 2023-09-23T18:35:08.936 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:08 smithi098 bash[79516]: cluster 2023-09-23T18:35:07.943871+0000 osd.7 (osd.7) 43 : cluster 3 failed to encode map e133 with expected crc 2023-09-23T18:35:08.936 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:08 smithi098 bash[79516]: cluster 2023-09-23T18:35:07.944175+0000 osd.5 (osd.5) 44 : cluster 3 failed to encode map e133 with expected crc 2023-09-23T18:35:09.017 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:09.020 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:08 smithi173 bash[59422]: cluster 2023-09-23T18:35:07.769850+0000 mgr.y (mgr.26054) 496 : cluster 0 pgmap v455: 89 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 439 op/s; 46239/127416 objects degraded (36.290%); 4617/127416 objects misplaced (3.624%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T18:35:09.020 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:08 smithi173 bash[59422]: cluster 2023-09-23T18:35:07.923170+0000 mon.a (mon.0) 697 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:35:09.020 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:08 smithi173 bash[59422]: audit 2023-09-23T18:35:07.936217+0000 mon.a (mon.0) 698 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:35:09.020 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:08 smithi173 bash[59422]: audit 2023-09-23T18:35:07.936293+0000 mon.a (mon.0) 699 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T18:35:09.020 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:08 smithi173 bash[59422]: cluster 2023-09-23T18:35:07.936347+0000 mon.a (mon.0) 700 : cluster 0 osdmap e133: 8 total, 8 up, 7 in 2023-09-23T18:35:09.020 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:08 smithi173 bash[59422]: cluster 2023-09-23T18:35:07.942884+0000 osd.4 (osd.4) 66 : cluster 3 failed to encode map e133 with expected crc 2023-09-23T18:35:09.020 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:08 smithi173 bash[59422]: cluster 2023-09-23T18:35:07.943871+0000 osd.7 (osd.7) 43 : cluster 3 failed to encode map e133 with expected crc 2023-09-23T18:35:09.020 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:08 smithi173 bash[59422]: cluster 2023-09-23T18:35:07.944175+0000 osd.5 (osd.5) 44 : cluster 3 failed to encode map e133 with expected crc 2023-09-23T18:35:09.021 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:08 smithi173 bash[59422]: cluster 2023-09-23T18:35:07.945096+0000 osd.4 (osd.4) 67 : cluster 3 failed to encode map e133 with expected crc 2023-09-23T18:35:09.021 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:08 smithi173 bash[59422]: cluster 2023-09-23T18:35:07.954398+0000 osd.6 (osd.6) 49 : cluster 3 failed to encode map e133 with expected crc 2023-09-23T18:35:09.048 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:09.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:09.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:09.156 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:09.179 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:09.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:09.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:09.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:08 smithi098 bash[79516]: cluster 2023-09-23T18:35:07.945096+0000 osd.4 (osd.4) 67 : cluster 3 failed to encode map e133 with expected crc 2023-09-23T18:35:09.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:08 smithi098 bash[79516]: cluster 2023-09-23T18:35:07.954398+0000 osd.6 (osd.6) 49 : cluster 3 failed to encode map e133 with expected crc 2023-09-23T18:35:09.288 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:09.311 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:09.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:09.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:09.414 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:09.440 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:09.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:09.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:09.544 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:09.569 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:09.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:09.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:09.675 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:09.699 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:09.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:09.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:09.804 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:09.830 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:09.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:09.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:09.936 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:09 smithi173 bash[59422]: cluster 2023-09-23T18:35:09.473058+0000 mon.a (mon.0) 701 : cluster 3 Health check update: Degraded data redundancy: 46239/127416 objects degraded (36.290%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:09.936 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:09 smithi173 bash[59422]: audit 2023-09-23T18:35:09.583203+0000 mon.a (mon.0) 702 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:35:09.936 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:09 smithi173 bash[59422]: cluster 2023-09-23T18:35:09.770520+0000 mgr.y (mgr.26054) 497 : cluster 0 pgmap v457: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 45808/123522 objects degraded (37.085%); 4365/123522 objects misplaced (3.534%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T18:35:09.940 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:09.976 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:09.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:09.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:10.083 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:10.091 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T18:35:10.107 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:10.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:10.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:10.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:09 smithi098 bash[76817]: cluster 2023-09-23T18:35:09.473058+0000 mon.a (mon.0) 701 : cluster 3 Health check update: Degraded data redundancy: 46239/127416 objects degraded (36.290%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:10.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:09 smithi098 bash[76817]: audit 2023-09-23T18:35:09.583203+0000 mon.a (mon.0) 702 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:35:10.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:09 smithi098 bash[76817]: cluster 2023-09-23T18:35:09.770520+0000 mgr.y (mgr.26054) 497 : cluster 0 pgmap v457: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 45808/123522 objects degraded (37.085%); 4365/123522 objects misplaced (3.534%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T18:35:10.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:09 smithi098 bash[79516]: cluster 2023-09-23T18:35:09.473058+0000 mon.a (mon.0) 701 : cluster 3 Health check update: Degraded data redundancy: 46239/127416 objects degraded (36.290%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:10.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:09 smithi098 bash[79516]: audit 2023-09-23T18:35:09.583203+0000 mon.a (mon.0) 702 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:35:10.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:09 smithi098 bash[79516]: cluster 2023-09-23T18:35:09.770520+0000 mgr.y (mgr.26054) 497 : cluster 0 pgmap v457: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 45808/123522 objects degraded (37.085%); 4365/123522 objects misplaced (3.534%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T18:35:10.213 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:10.244 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:10.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:10.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:10.363 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:10.402 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:10.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:10.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:10.508 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:10.538 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:10.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:10.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:10.641 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:10.665 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:35:10.682 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:10.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:10.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:10.787 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:10.839 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:10.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:10.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:10.943 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:10.965 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:10.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:10.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:11.068 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:11.105 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:11.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:11.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:11.210 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:11.235 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:11.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:11.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:11.317 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T18:35:11.317 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:35:11.340 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:11.365 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:11.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:11.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:11.468 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:11.546 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:11.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:11.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:11.651 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:11.680 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:11.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:11.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:11.784 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:11.810 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:11.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:11.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:11.913 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:11.955 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:11.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:11.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:12.064 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:12.072 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:35:12.091 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:12.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:12.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:12.193 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:12.216 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:12.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:12.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:12.320 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:12.342 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:12.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:12.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:12.438 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T18:35:12.454 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:12.497 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:12.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:12.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:12.602 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:12.627 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:12.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:12.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:12.730 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:12.768 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:12.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:12.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:12.875 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:12.884 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:12 smithi173 bash[59422]: cluster 2023-09-23T18:35:11.771278+0000 mgr.y (mgr.26054) 498 : cluster 0 pgmap v458: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 463 op/s; 45808/120297 objects degraded (38.079%); 4219/120297 objects misplaced (3.507%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T18:35:12.885 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:12 smithi173 bash[59422]: audit 2023-09-23T18:35:12.000447+0000 mon.a (mon.0) 703 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:35:12.885 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:12 smithi173 bash[59422]: audit 2023-09-23T18:35:12.008975+0000 mon.a (mon.0) 704 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:35:12.885 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:12 smithi173 bash[59422]: audit 2023-09-23T18:35:12.644715+0000 mon.a (mon.0) 705 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:35:12.885 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:12 smithi173 bash[59422]: audit 2023-09-23T18:35:12.646025+0000 mon.a (mon.0) 706 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:35:12.885 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:12 smithi173 bash[59422]: audit 2023-09-23T18:35:12.655945+0000 mon.a (mon.0) 707 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:35:12.911 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:12.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:12.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:13.016 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:13.040 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:13.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:13.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:13.142 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:13.150 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:12 smithi098 bash[79516]: cluster 2023-09-23T18:35:11.771278+0000 mgr.y (mgr.26054) 498 : cluster 0 pgmap v458: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 463 op/s; 45808/120297 objects degraded (38.079%); 4219/120297 objects misplaced (3.507%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T18:35:13.151 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:12 smithi098 bash[79516]: audit 2023-09-23T18:35:12.000447+0000 mon.a (mon.0) 703 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:35:13.151 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:12 smithi098 bash[79516]: audit 2023-09-23T18:35:12.008975+0000 mon.a (mon.0) 704 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:35:13.151 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:12 smithi098 bash[79516]: audit 2023-09-23T18:35:12.644715+0000 mon.a (mon.0) 705 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:35:13.151 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:12 smithi098 bash[79516]: audit 2023-09-23T18:35:12.646025+0000 mon.a (mon.0) 706 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:35:13.151 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:12 smithi098 bash[79516]: audit 2023-09-23T18:35:12.655945+0000 mon.a (mon.0) 707 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:35:13.151 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:12 smithi098 bash[76817]: cluster 2023-09-23T18:35:11.771278+0000 mgr.y (mgr.26054) 498 : cluster 0 pgmap v458: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 463 op/s; 45808/120297 objects degraded (38.079%); 4219/120297 objects misplaced (3.507%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T18:35:13.151 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:12 smithi098 bash[76817]: audit 2023-09-23T18:35:12.000447+0000 mon.a (mon.0) 703 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:35:13.151 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:12 smithi098 bash[76817]: audit 2023-09-23T18:35:12.008975+0000 mon.a (mon.0) 704 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:35:13.152 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:12 smithi098 bash[76817]: audit 2023-09-23T18:35:12.644715+0000 mon.a (mon.0) 705 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:35:13.152 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:12 smithi098 bash[76817]: audit 2023-09-23T18:35:12.646025+0000 mon.a (mon.0) 706 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:35:13.152 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:12 smithi098 bash[76817]: audit 2023-09-23T18:35:12.655945+0000 mon.a (mon.0) 707 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:35:13.174 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:13.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:13.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:13.279 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:13.301 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:13.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:13.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:13.403 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:13.435 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:13.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:13.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:13.541 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:13.565 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:13.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:13.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:13.668 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:13.701 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:13.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:13.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:13.808 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:13.836 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:13.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:13.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:13.939 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:13.967 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:13.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:13.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:14.071 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:14.152 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:14.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:14.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:14.258 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:14.286 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:14.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:14.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:14.392 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:14.414 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:14.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:14.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:14.517 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:14.546 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:14.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:14.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:14.630 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:35:14.636 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:35:14.649 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:14.670 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:14.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:14.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:14.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:14.799 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:14.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:14.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:14.902 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:14.911 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:14 smithi173 bash[59422]: cluster 2023-09-23T18:35:13.771948+0000 mgr.y (mgr.26054) 499 : cluster 0 pgmap v459: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 45808/117858 objects degraded (38.867%); 4219/117858 objects misplaced (3.580%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T18:35:14.911 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:14 smithi173 bash[59422]: cluster 2023-09-23T18:35:14.474099+0000 mon.a (mon.0) 708 : cluster 3 Health check update: Degraded data redundancy: 45808/117858 objects degraded (38.867%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:14.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:14 smithi098 bash[79516]: cluster 2023-09-23T18:35:13.771948+0000 mgr.y (mgr.26054) 499 : cluster 0 pgmap v459: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 45808/117858 objects degraded (38.867%); 4219/117858 objects misplaced (3.580%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T18:35:14.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:14 smithi098 bash[79516]: cluster 2023-09-23T18:35:14.474099+0000 mon.a (mon.0) 708 : cluster 3 Health check update: Degraded data redundancy: 45808/117858 objects degraded (38.867%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:14.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:14 smithi098 bash[76817]: cluster 2023-09-23T18:35:13.771948+0000 mgr.y (mgr.26054) 499 : cluster 0 pgmap v459: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 45808/117858 objects degraded (38.867%); 4219/117858 objects misplaced (3.580%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T18:35:14.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:14 smithi098 bash[76817]: cluster 2023-09-23T18:35:14.474099+0000 mon.a (mon.0) 708 : cluster 3 Health check update: Degraded data redundancy: 45808/117858 objects degraded (38.867%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:14.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:14.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:14.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:15.048 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:15.071 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:15.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:15.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:15.175 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:15.210 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:15.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:15.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:15.316 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:15.359 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:15.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:15.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:15.465 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:15.487 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:15.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:15.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:15.590 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:15.612 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:15.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:15.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:15.715 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:15.743 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:15.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:15.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:15.846 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:15.870 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:15.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:15.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:15.975 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:16.002 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:16.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:16.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:16.107 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:16.130 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:16.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:16.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:16.233 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:16.260 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:16.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:16.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:16.364 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:16.390 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:16.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:16.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:16.440 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:35:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:35:16] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.33.4" 2023-09-23T18:35:16.496 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:16.519 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:16.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:16.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:16.624 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:16.653 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:16.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:16.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:16.674 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:35:16.756 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:16.782 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:16.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:16.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:16.832 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:16 smithi098 bash[76817]: cluster 2023-09-23T18:35:15.772726+0000 mgr.y (mgr.26054) 500 : cluster 0 pgmap v460: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 45323/114519 objects degraded (39.577%); 4056/114519 objects misplaced (3.542%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T18:35:16.885 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:16.908 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:16.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:16.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:17.013 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:17.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:16 smithi173 bash[59422]: cluster 2023-09-23T18:35:15.772726+0000 mgr.y (mgr.26054) 500 : cluster 0 pgmap v460: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 45323/114519 objects degraded (39.577%); 4056/114519 objects misplaced (3.542%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T18:35:17.041 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:17.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:17.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:17.146 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:17.154 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:16 smithi098 bash[79516]: cluster 2023-09-23T18:35:15.772726+0000 mgr.y (mgr.26054) 500 : cluster 0 pgmap v460: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 45323/114519 objects degraded (39.577%); 4056/114519 objects misplaced (3.542%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T18:35:17.170 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:17.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:17.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:17.260 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:35:17.275 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:17.315 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:17.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:17.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:17.421 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:17.442 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:17.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:17.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:17.546 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:17.569 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:17.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:17.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:17.672 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:17.705 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:17.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:17.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:17.813 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:17.842 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:17.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:17.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:17.946 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:17.976 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:17.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:17.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:18.080 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:18.284 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:18.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:18.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:18.388 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:18.418 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:18.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:18.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:18.523 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:18.545 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:18.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:18.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:18.650 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:18.679 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:18.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:18.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:18.782 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:18.817 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:18.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:18.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:18.921 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:18.930 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:18 smithi173 bash[59422]: cluster 2023-09-23T18:35:17.773640+0000 mgr.y (mgr.26054) 501 : cluster 0 pgmap v461: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 487 op/s; 45323/112773 objects degraded (40.190%); 3865/112773 objects misplaced (3.427%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T18:35:18.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:18.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:18.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:19.020 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:18 smithi098 bash[79516]: cluster 2023-09-23T18:35:17.773640+0000 mgr.y (mgr.26054) 501 : cluster 0 pgmap v461: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 487 op/s; 45323/112773 objects degraded (40.190%); 3865/112773 objects misplaced (3.427%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T18:35:19.020 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:18 smithi098 bash[76817]: cluster 2023-09-23T18:35:17.773640+0000 mgr.y (mgr.26054) 501 : cluster 0 pgmap v461: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 487 op/s; 45323/112773 objects degraded (40.190%); 3865/112773 objects misplaced (3.427%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T18:35:19.047 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:19.077 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:19.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:19.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:19.188 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:19.210 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:19.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:19.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:19.313 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:19.340 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:19.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:19.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:19.443 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:19.468 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:19.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:19.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:19.575 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:19.604 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:19.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:19.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:19.708 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:19.732 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:19.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:19.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:19.835 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:19.869 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:19.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:19.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:19.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:19 smithi098 bash[79516]: cluster 2023-09-23T18:35:19.475316+0000 mon.a (mon.0) 709 : cluster 3 Health check update: Degraded data redundancy: 45323/112773 objects degraded (40.190%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:19.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:19 smithi098 bash[76817]: cluster 2023-09-23T18:35:19.475316+0000 mon.a (mon.0) 709 : cluster 3 Health check update: Degraded data redundancy: 45323/112773 objects degraded (40.190%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:19.974 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:20.009 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:19 smithi173 bash[59422]: cluster 2023-09-23T18:35:19.475316+0000 mon.a (mon.0) 709 : cluster 3 Health check update: Degraded data redundancy: 45323/112773 objects degraded (40.190%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:20.016 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:20.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:20.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:20.122 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:20.168 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:20.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:20.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:20.275 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:20.307 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:20.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:20.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:20.412 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:20.451 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:20.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:20.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:20.556 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:20.585 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:20.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:20.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:20.692 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:20.723 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:20.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:20.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:20.833 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:20.854 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:20.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:20.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:20.957 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:20.981 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:20.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:20.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:21.088 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:21.135 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:21.136 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:20 smithi098 bash[79516]: cluster 2023-09-23T18:35:19.774421+0000 mgr.y (mgr.26054) 502 : cluster 0 pgmap v462: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 44849/108471 objects degraded (41.347%); 3698/108471 objects misplaced (3.409%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T18:35:21.136 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:20 smithi098 bash[76817]: cluster 2023-09-23T18:35:19.774421+0000 mgr.y (mgr.26054) 502 : cluster 0 pgmap v462: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 44849/108471 objects degraded (41.347%); 3698/108471 objects misplaced (3.409%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T18:35:21.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:21.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:21.238 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:21.252 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:20 smithi173 bash[59422]: cluster 2023-09-23T18:35:19.774421+0000 mgr.y (mgr.26054) 502 : cluster 0 pgmap v462: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 44849/108471 objects degraded (41.347%); 3698/108471 objects misplaced (3.409%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T18:35:21.268 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:21.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:21.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:21.377 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:21.404 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:21.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:21.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:21.509 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:21.558 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:21.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:21.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:21.668 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:21.694 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:21.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:21.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:21.798 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:21.838 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:21.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:21.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:21.943 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:21.970 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:22.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:22.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:22.176 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:22.200 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:22.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:22.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:22.302 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:22.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:22.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:22.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:22.432 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:22.460 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:22.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:22.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:22.563 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:22.595 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:22.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:22.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:22.702 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:22.733 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:22.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:22.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:22.845 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:22 smithi098 bash[76817]: cluster 2023-09-23T18:35:21.775987+0000 mgr.y (mgr.26054) 503 : cluster 0 pgmap v463: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 44849/104949 objects degraded (42.734%); 3524/104949 objects misplaced (3.358%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:35:22.847 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:22.881 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:22.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:22.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:22.985 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:22.990 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T18:35:22.990 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:35:22.996 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:22 smithi173 bash[59422]: cluster 2023-09-23T18:35:21.775987+0000 mgr.y (mgr.26054) 503 : cluster 0 pgmap v463: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 44849/104949 objects degraded (42.734%); 3524/104949 objects misplaced (3.358%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:35:23.010 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:23.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:23.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:23.118 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:23.127 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:22 smithi098 bash[79516]: cluster 2023-09-23T18:35:21.775987+0000 mgr.y (mgr.26054) 503 : cluster 0 pgmap v463: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 44849/104949 objects degraded (42.734%); 3524/104949 objects misplaced (3.358%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:35:23.147 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:23.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:23.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:23.251 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:23.287 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:23.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:23.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:23.390 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:23.414 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:23.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:23.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:23.517 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:23.543 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:23.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:23.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:23.646 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:23.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:23.678 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:23.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:23.779 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:23.815 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:23.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:23.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:23.920 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:23.948 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:23.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:23.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:24.053 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:24.079 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:24.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:24.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:24.183 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T18:35:24.185 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:24.209 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:24.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:24.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:24.312 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:24.345 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:24.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:24.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:24.452 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:24.483 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:24.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:24.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:24.590 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:24.615 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:24.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:24.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:24.719 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:24.751 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:24.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:24.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:24.858 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:24.881 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:24.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:24.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:24.984 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:25.013 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:25.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:25.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:25.119 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:25.126 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:24 smithi173 bash[59422]: cluster 2023-09-23T18:35:23.776505+0000 mgr.y (mgr.26054) 504 : cluster 0 pgmap v464: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 44849/102534 objects degraded (43.741%); 3524/102534 objects misplaced (3.437%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T18:35:25.127 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:24 smithi173 bash[59422]: cluster 2023-09-23T18:35:24.476491+0000 mon.a (mon.0) 710 : cluster 3 Health check update: Degraded data redundancy: 44849/102534 objects degraded (43.741%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:25.127 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:24 smithi173 bash[59422]: audit 2023-09-23T18:35:24.584009+0000 mon.a (mon.0) 711 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:35:25.146 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:25.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:25.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:25.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:24 smithi098 bash[76817]: cluster 2023-09-23T18:35:23.776505+0000 mgr.y (mgr.26054) 504 : cluster 0 pgmap v464: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 44849/102534 objects degraded (43.741%); 3524/102534 objects misplaced (3.437%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T18:35:25.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:24 smithi098 bash[76817]: cluster 2023-09-23T18:35:24.476491+0000 mon.a (mon.0) 710 : cluster 3 Health check update: Degraded data redundancy: 44849/102534 objects degraded (43.741%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:25.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:24 smithi098 bash[76817]: audit 2023-09-23T18:35:24.584009+0000 mon.a (mon.0) 711 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:35:25.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:24 smithi098 bash[79516]: cluster 2023-09-23T18:35:23.776505+0000 mgr.y (mgr.26054) 504 : cluster 0 pgmap v464: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 44849/102534 objects degraded (43.741%); 3524/102534 objects misplaced (3.437%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T18:35:25.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:24 smithi098 bash[79516]: cluster 2023-09-23T18:35:24.476491+0000 mon.a (mon.0) 710 : cluster 3 Health check update: Degraded data redundancy: 44849/102534 objects degraded (43.741%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:25.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:24 smithi098 bash[79516]: audit 2023-09-23T18:35:24.584009+0000 mon.a (mon.0) 711 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:35:25.253 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:25.286 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:25.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:25.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:25.393 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:25.421 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:25.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:25.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:25.525 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:25.554 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:25.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:25.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:25.658 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:25.692 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:25.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:25.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:25.802 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:25.825 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:25.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:25.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:25.928 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:25.964 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:25.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:25.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:26.068 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:26.099 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:26.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:26.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:26.190 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:35:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:35:26] "GET /metrics HTTP/1.1" 200 36541 "" "Prometheus/2.33.4" 2023-09-23T18:35:26.201 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:26.224 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:26.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:26.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:26.328 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:26.358 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:26.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:26.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:26.462 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:26.492 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:26.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:26.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:26.598 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:26.646 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:26.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:26.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:26.752 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:26.765 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:35:26.784 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:26.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:26.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:26.819 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:35:26.862 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:26 smithi098 bash[76817]: cluster 2023-09-23T18:35:25.777242+0000 mgr.y (mgr.26054) 505 : cluster 0 pgmap v465: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 526 op/s; 44308/98904 objects degraded (44.799%); 3342/98904 objects misplaced (3.379%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T18:35:26.876 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:35:26.887 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:26.912 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:26.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:26.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:26.916 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:35:26.954 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:35:27.003 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:35:27.015 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:27.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:26 smithi173 bash[59422]: cluster 2023-09-23T18:35:25.777242+0000 mgr.y (mgr.26054) 505 : cluster 0 pgmap v465: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 526 op/s; 44308/98904 objects degraded (44.799%); 3342/98904 objects misplaced (3.379%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T18:35:27.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:26 smithi173 bash[59422]: audit 2023-09-23T18:35:26.693613+0000 mon.c (mon.2) 68 : audit 0 from='client.? 172.21.15.98:0/151904519' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:35:27.039 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:35:27.053 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:27.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:27.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:27.090 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:35:27.095 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:35:27.095 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:35:27.095 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:35:27.095 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:35:27.095 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:35:27.095 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:35:27.095 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:35:27.095 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:35:27.121 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:26 smithi098 bash[76817]: audit 2023-09-23T18:35:26.693613+0000 mon.c (mon.2) 68 : audit 0 from='client.? 172.21.15.98:0/151904519' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:35:27.122 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:26 smithi098 bash[79516]: cluster 2023-09-23T18:35:25.777242+0000 mgr.y (mgr.26054) 505 : cluster 0 pgmap v465: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 526 op/s; 44308/98904 objects degraded (44.799%); 3342/98904 objects misplaced (3.379%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T18:35:27.122 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:26 smithi098 bash[79516]: audit 2023-09-23T18:35:26.693613+0000 mon.c (mon.2) 68 : audit 0 from='client.? 172.21.15.98:0/151904519' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:35:27.156 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:27.210 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:27.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:27.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:27.318 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:27.342 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:27.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:27.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:27.445 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:27.472 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:27.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:27.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:27.575 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:27.603 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:27.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:27.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:27.707 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:27.746 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:27.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:27.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:27.850 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:27.882 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:27.883 INFO:teuthology.orchestra.run.smithi098.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T18:35:27.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:27.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:27.892 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is unset 2023-09-23T18:35:27.977 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:27 smithi098 bash[76817]: audit 2023-09-23T18:35:26.943400+0000 mon.a (mon.0) 712 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:35:27.977 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:27 smithi098 bash[76817]: audit 2023-09-23T18:35:26.943413+0000 mon.b (mon.1) 9 : audit 1 from='client.? 172.21.15.98:0/3118962046' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:35:27.977 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:27 smithi098 bash[76817]: audit 2023-09-23T18:35:27.113534+0000 mon.c (mon.2) 69 : audit 1 from='client.? 172.21.15.98:0/2617091183' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:35:27.977 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:27 smithi098 bash[76817]: audit 2023-09-23T18:35:27.114143+0000 mon.a (mon.0) 713 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:35:27.987 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:28.030 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:28.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:28.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:28.136 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:28.160 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:28.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:28.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:28.262 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:28.271 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:27 smithi098 bash[79516]: audit 2023-09-23T18:35:26.943400+0000 mon.a (mon.0) 712 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:35:28.271 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:27 smithi098 bash[79516]: audit 2023-09-23T18:35:26.943413+0000 mon.b (mon.1) 9 : audit 1 from='client.? 172.21.15.98:0/3118962046' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:35:28.271 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:27 smithi098 bash[79516]: audit 2023-09-23T18:35:27.113534+0000 mon.c (mon.2) 69 : audit 1 from='client.? 172.21.15.98:0/2617091183' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:35:28.271 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:27 smithi098 bash[79516]: audit 2023-09-23T18:35:27.114143+0000 mon.a (mon.0) 713 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:35:28.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:27 smithi173 bash[59422]: audit 2023-09-23T18:35:26.943400+0000 mon.a (mon.0) 712 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:35:28.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:27 smithi173 bash[59422]: audit 2023-09-23T18:35:26.943413+0000 mon.b (mon.1) 9 : audit 1 from='client.? 172.21.15.98:0/3118962046' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:35:28.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:27 smithi173 bash[59422]: audit 2023-09-23T18:35:27.113534+0000 mon.c (mon.2) 69 : audit 1 from='client.? 172.21.15.98:0/2617091183' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:35:28.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:27 smithi173 bash[59422]: audit 2023-09-23T18:35:27.114143+0000 mon.a (mon.0) 713 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:35:28.304 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:28.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:28.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:28.411 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:28.443 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:28.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:28.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:28.547 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:28.581 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:28.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:28.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:28.684 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:28.711 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:28.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:28.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:28.814 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:28.842 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:28.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:28.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:28.889 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:28 smithi098 bash[76817]: cluster 2023-09-23T18:35:27.777983+0000 mgr.y (mgr.26054) 506 : cluster 0 pgmap v466: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 486 op/s; 44308/96987 objects degraded (45.684%); 3162/96987 objects misplaced (3.260%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T18:35:28.949 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:28.976 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:28.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:28.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:29.002 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:35:29.079 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:29.105 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:29.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:29.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:29.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:28 smithi098 bash[76817]: cluster 2023-09-23T18:35:27.860763+0000 mon.a (mon.0) 714 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:35:29.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:28 smithi098 bash[76817]: audit 2023-09-23T18:35:27.870415+0000 mon.a (mon.0) 715 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T18:35:29.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:28 smithi098 bash[76817]: audit 2023-09-23T18:35:27.870572+0000 mon.a (mon.0) 716 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:35:29.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:28 smithi098 bash[76817]: cluster 2023-09-23T18:35:27.870857+0000 mon.a (mon.0) 717 : cluster 0 osdmap e134: 8 total, 8 up, 7 in 2023-09-23T18:35:29.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:28 smithi098 bash[76817]: cluster 2023-09-23T18:35:27.876266+0000 osd.4 (osd.4) 68 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T18:35:29.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:28 smithi098 bash[76817]: cluster 2023-09-23T18:35:27.887875+0000 osd.6 (osd.6) 50 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T18:35:29.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:28 smithi098 bash[76817]: cluster 2023-09-23T18:35:27.896826+0000 osd.7 (osd.7) 44 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T18:35:29.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:28 smithi098 bash[76817]: cluster 2023-09-23T18:35:27.937760+0000 osd.5 (osd.5) 45 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T18:35:29.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:28 smithi098 bash[79516]: cluster 2023-09-23T18:35:27.777983+0000 mgr.y (mgr.26054) 506 : cluster 0 pgmap v466: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 486 op/s; 44308/96987 objects degraded (45.684%); 3162/96987 objects misplaced (3.260%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T18:35:29.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:28 smithi098 bash[79516]: cluster 2023-09-23T18:35:27.860763+0000 mon.a (mon.0) 714 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:35:29.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:28 smithi098 bash[79516]: audit 2023-09-23T18:35:27.870415+0000 mon.a (mon.0) 715 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T18:35:29.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:28 smithi098 bash[79516]: audit 2023-09-23T18:35:27.870572+0000 mon.a (mon.0) 716 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:35:29.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:28 smithi098 bash[79516]: cluster 2023-09-23T18:35:27.870857+0000 mon.a (mon.0) 717 : cluster 0 osdmap e134: 8 total, 8 up, 7 in 2023-09-23T18:35:29.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:28 smithi098 bash[79516]: cluster 2023-09-23T18:35:27.876266+0000 osd.4 (osd.4) 68 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T18:35:29.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:28 smithi098 bash[79516]: cluster 2023-09-23T18:35:27.887875+0000 osd.6 (osd.6) 50 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T18:35:29.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:28 smithi098 bash[79516]: cluster 2023-09-23T18:35:27.896826+0000 osd.7 (osd.7) 44 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T18:35:29.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:28 smithi098 bash[79516]: cluster 2023-09-23T18:35:27.937760+0000 osd.5 (osd.5) 45 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T18:35:29.213 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:29.243 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:29.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:29.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:29.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:28 smithi173 bash[59422]: cluster 2023-09-23T18:35:27.777983+0000 mgr.y (mgr.26054) 506 : cluster 0 pgmap v466: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 486 op/s; 44308/96987 objects degraded (45.684%); 3162/96987 objects misplaced (3.260%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T18:35:29.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:28 smithi173 bash[59422]: cluster 2023-09-23T18:35:27.860763+0000 mon.a (mon.0) 714 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:35:29.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:28 smithi173 bash[59422]: audit 2023-09-23T18:35:27.870415+0000 mon.a (mon.0) 715 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T18:35:29.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:28 smithi173 bash[59422]: audit 2023-09-23T18:35:27.870572+0000 mon.a (mon.0) 716 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:35:29.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:28 smithi173 bash[59422]: cluster 2023-09-23T18:35:27.870857+0000 mon.a (mon.0) 717 : cluster 0 osdmap e134: 8 total, 8 up, 7 in 2023-09-23T18:35:29.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:28 smithi173 bash[59422]: cluster 2023-09-23T18:35:27.876266+0000 osd.4 (osd.4) 68 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T18:35:29.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:28 smithi173 bash[59422]: cluster 2023-09-23T18:35:27.887875+0000 osd.6 (osd.6) 50 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T18:35:29.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:28 smithi173 bash[59422]: cluster 2023-09-23T18:35:27.896826+0000 osd.7 (osd.7) 44 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T18:35:29.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:28 smithi173 bash[59422]: cluster 2023-09-23T18:35:27.937760+0000 osd.5 (osd.5) 45 : cluster 3 failed to encode map e134 with expected crc 2023-09-23T18:35:29.349 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:29.376 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:29.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:29.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:29.479 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:29.502 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:29.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:29.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:29.608 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:29.645 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:29.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:29.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:29.749 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:29.775 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:29.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:29.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:29.881 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:29.902 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:29.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:29.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:30.004 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:30.012 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:29 smithi098 bash[76817]: cluster 2023-09-23T18:35:29.477571+0000 mon.a (mon.0) 718 : cluster 3 Health check update: Degraded data redundancy: 44308/96987 objects degraded (45.684%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:30.012 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:29 smithi098 bash[79516]: cluster 2023-09-23T18:35:29.477571+0000 mon.a (mon.0) 718 : cluster 3 Health check update: Degraded data redundancy: 44308/96987 objects degraded (45.684%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:30.041 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:30.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:30.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:30.147 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T18:35:30.150 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:30.158 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:29 smithi173 bash[59422]: cluster 2023-09-23T18:35:29.477571+0000 mon.a (mon.0) 718 : cluster 3 Health check update: Degraded data redundancy: 44308/96987 objects degraded (45.684%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:30.174 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:30.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:30.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:30.277 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:30.312 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:30.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:30.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:30.417 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:30.465 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:30.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:30.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:30.570 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:30.597 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:30.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:30.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:30.618 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:35:30.702 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:30.732 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:30.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:30.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:30.836 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:30.863 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:30.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:30.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:30.971 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:30.995 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:30.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:30.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:31.100 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:31.109 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:30 smithi173 bash[59422]: cluster 2023-09-23T18:35:29.779584+0000 mgr.y (mgr.26054) 507 : cluster 0 pgmap v468: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 43837/92568 objects degraded (47.357%); 3012/92568 objects misplaced (3.254%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T18:35:31.124 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:31.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:31.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:31.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:30 smithi098 bash[79516]: cluster 2023-09-23T18:35:29.779584+0000 mgr.y (mgr.26054) 507 : cluster 0 pgmap v468: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 43837/92568 objects degraded (47.357%); 3012/92568 objects misplaced (3.254%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T18:35:31.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:30 smithi098 bash[76817]: cluster 2023-09-23T18:35:29.779584+0000 mgr.y (mgr.26054) 507 : cluster 0 pgmap v468: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 43837/92568 objects degraded (47.357%); 3012/92568 objects misplaced (3.254%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T18:35:31.226 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:31.248 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:31.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:31.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:31.352 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:31.385 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:31.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:31.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:31.489 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:31.517 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:31.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:31.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:31.623 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:31.646 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:31.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:31.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:31.751 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:31.781 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:31.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:31.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:31.886 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:31.914 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:31.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:31.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:32.018 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:32.051 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:32.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:32.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:32.131 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:35:32.154 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:32.217 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:32.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:32.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:32.326 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:32.351 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:32.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:32.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:32.460 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:32.486 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:32.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:32.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:32.592 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:32.634 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:32.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:32.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:32.737 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:32.770 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:32.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:32.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:32.875 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:32.915 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:32.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:32.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:32.971 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:32 smithi098 bash[76817]: cluster 2023-09-23T18:35:31.782350+0000 mgr.y (mgr.26054) 508 : cluster 0 pgmap v469: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 526 op/s; 43837/89130 objects degraded (49.183%); 2823/89130 objects misplaced (3.167%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:35:32.971 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:32 smithi098 bash[79516]: cluster 2023-09-23T18:35:31.782350+0000 mgr.y (mgr.26054) 508 : cluster 0 pgmap v469: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 526 op/s; 43837/89130 objects degraded (49.183%); 2823/89130 objects misplaced (3.167%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:35:33.021 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:33.051 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:33.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:33.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:33.160 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:33.189 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:33.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:33.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:33.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:32 smithi173 bash[59422]: cluster 2023-09-23T18:35:31.782350+0000 mgr.y (mgr.26054) 508 : cluster 0 pgmap v469: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 526 op/s; 43837/89130 objects degraded (49.183%); 2823/89130 objects misplaced (3.167%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:35:33.295 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:33.325 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:33.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:33.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:33.429 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:33.451 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:33.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:33.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:33.555 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:33.590 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:33.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:33.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:33.693 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:33.728 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:33.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:33.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:33.834 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:33.873 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:33.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:33.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:33.978 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:34.027 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:34.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:34.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:34.134 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:34.160 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:34.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:34.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:34.265 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:34.303 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:34.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:34.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:34.354 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:35:34.354 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-09-23T18:35:34.354 INFO:teuthology.orchestra.run.smithi173.stdout: "complaint_time": 30, 2023-09-23T18:35:34.354 INFO:teuthology.orchestra.run.smithi173.stdout: "num_blocked_ops": 0 2023-09-23T18:35:34.354 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:35:34.411 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:34.443 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:34.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:34.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:34.546 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:34.574 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:34.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:34.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:34.677 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:34.703 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:34.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:34.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:34.808 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:34.839 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:34.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:34.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:34.944 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:34.951 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:34 smithi173 bash[59422]: cluster 2023-09-23T18:35:33.783055+0000 mgr.y (mgr.26054) 509 : cluster 0 pgmap v470: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 43837/86592 objects degraded (50.625%); 2823/86592 objects misplaced (3.260%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:35:34.951 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:34 smithi173 bash[59422]: cluster 2023-09-23T18:35:34.478794+0000 mon.a (mon.0) 719 : cluster 3 Health check update: Degraded data redundancy: 43837/86592 objects degraded (50.625%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:34.967 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:34.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:34.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:35.052 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:35:35.053 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:35:35.071 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:35.086 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:34 smithi098 bash[76817]: cluster 2023-09-23T18:35:33.783055+0000 mgr.y (mgr.26054) 509 : cluster 0 pgmap v470: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 43837/86592 objects degraded (50.625%); 2823/86592 objects misplaced (3.260%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:35:35.086 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:34 smithi098 bash[76817]: cluster 2023-09-23T18:35:34.478794+0000 mon.a (mon.0) 719 : cluster 3 Health check update: Degraded data redundancy: 43837/86592 objects degraded (50.625%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:35.086 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:34 smithi098 bash[79516]: cluster 2023-09-23T18:35:33.783055+0000 mgr.y (mgr.26054) 509 : cluster 0 pgmap v470: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 43837/86592 objects degraded (50.625%); 2823/86592 objects misplaced (3.260%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:35:35.086 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:34 smithi098 bash[79516]: cluster 2023-09-23T18:35:34.478794+0000 mon.a (mon.0) 719 : cluster 3 Health check update: Degraded data redundancy: 43837/86592 objects degraded (50.625%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:35.102 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:35.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:35.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:35.206 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:35.232 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:35.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:35.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:35.336 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:35.361 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:35.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:35.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:35.397 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T18:35:35.464 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:35.499 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:35.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:35.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:35.606 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:35.647 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:35.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:35.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:35.754 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:35.776 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:35.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:35.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:35.880 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:35.915 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:35.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:35.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:36.019 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:36.046 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:36.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:36.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:36.072 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:35:36.152 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:36.175 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:36.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:36.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:36.193 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:35:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:35:36] "GET /metrics HTTP/1.1" 200 36541 "" "Prometheus/2.33.4" 2023-09-23T18:35:36.277 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:36.314 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:36.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:36.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:36.419 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:36.442 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:36.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:36.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:36.545 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:36.581 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:36.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:36.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:36.687 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:36.711 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:36.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:36.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:36.814 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:36.857 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:36.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:36.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:36.960 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:36.968 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:36 smithi098 bash[76817]: cluster 2023-09-23T18:35:35.786380+0000 mgr.y (mgr.26054) 510 : cluster 0 pgmap v471: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 43302/83118 objects degraded (52.097%); 2642/83118 objects misplaced (3.179%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T18:35:36.969 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:36 smithi098 bash[79516]: cluster 2023-09-23T18:35:35.786380+0000 mgr.y (mgr.26054) 510 : cluster 0 pgmap v471: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 43302/83118 objects degraded (52.097%); 2642/83118 objects misplaced (3.179%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T18:35:36.988 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:36.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:36.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:37.096 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:37.121 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:37.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:37.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:37.228 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:37.237 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:36 smithi173 bash[59422]: cluster 2023-09-23T18:35:35.786380+0000 mgr.y (mgr.26054) 510 : cluster 0 pgmap v471: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 43302/83118 objects degraded (52.097%); 2642/83118 objects misplaced (3.179%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T18:35:37.261 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:37.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:37.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:37.365 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:37.396 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:37.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:37.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:37.499 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:37.541 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:37.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:37.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:37.645 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:37.676 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:37.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:37.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:37.781 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:37.838 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:37.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:37.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:37.944 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:37.969 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:37.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:37.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:38.075 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:38.114 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:38.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:38.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:38.219 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:38.248 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:38.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:38.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:38.355 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:38.389 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:38.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:38.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:38.493 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:38.527 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:38.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:38.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:38.633 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:38.659 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:38.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:38.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:38.764 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:38.795 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:38.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:38.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:38.900 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:38.934 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:38.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:38.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:39.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:38 smithi173 bash[59422]: cluster 2023-09-23T18:35:37.787209+0000 mgr.y (mgr.26054) 511 : cluster 0 pgmap v472: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 43302/81090 objects degraded (53.400%); 2457/81090 objects misplaced (3.030%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:35:39.039 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:39.047 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:38 smithi098 bash[79516]: cluster 2023-09-23T18:35:37.787209+0000 mgr.y (mgr.26054) 511 : cluster 0 pgmap v472: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 43302/81090 objects degraded (53.400%); 2457/81090 objects misplaced (3.030%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:35:39.048 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:38 smithi098 bash[76817]: cluster 2023-09-23T18:35:37.787209+0000 mgr.y (mgr.26054) 511 : cluster 0 pgmap v472: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 43302/81090 objects degraded (53.400%); 2457/81090 objects misplaced (3.030%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:35:39.061 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:39.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:39.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:39.167 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:39.196 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:39.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:39.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:39.300 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:39.323 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:39.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:39.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:39.427 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:39.451 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:39.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:39.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:39.554 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:39.578 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:39.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:39.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:39.683 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:39.725 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:39.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:39.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:39.831 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:39.859 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:39.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:39.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:39.964 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:39.999 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:40.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:40.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:40.102 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:40.111 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:39 smithi173 bash[59422]: cluster 2023-09-23T18:35:39.482507+0000 mon.a (mon.0) 720 : cluster 3 Health check update: Degraded data redundancy: 43302/81090 objects degraded (53.400%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:40.111 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:39 smithi173 bash[59422]: audit 2023-09-23T18:35:39.592420+0000 mon.a (mon.0) 721 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:35:40.111 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:39 smithi173 bash[59422]: audit 2023-09-23T18:35:39.593559+0000 mon.a (mon.0) 722 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:35:40.127 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:40.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:40.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:40.198 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:35:40.208 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:39 smithi098 bash[76817]: cluster 2023-09-23T18:35:39.482507+0000 mon.a (mon.0) 720 : cluster 3 Health check update: Degraded data redundancy: 43302/81090 objects degraded (53.400%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:40.208 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:39 smithi098 bash[76817]: audit 2023-09-23T18:35:39.592420+0000 mon.a (mon.0) 721 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:35:40.209 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:39 smithi098 bash[76817]: audit 2023-09-23T18:35:39.593559+0000 mon.a (mon.0) 722 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:35:40.209 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:39 smithi098 bash[79516]: cluster 2023-09-23T18:35:39.482507+0000 mon.a (mon.0) 720 : cluster 3 Health check update: Degraded data redundancy: 43302/81090 objects degraded (53.400%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:40.209 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:39 smithi098 bash[79516]: audit 2023-09-23T18:35:39.592420+0000 mon.a (mon.0) 721 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:35:40.209 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:39 smithi098 bash[79516]: audit 2023-09-23T18:35:39.593559+0000 mon.a (mon.0) 722 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:35:40.233 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:40.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:40.275 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:40.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:40.376 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:40.401 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:40.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:40.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:40.505 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:40.530 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:40.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:40.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:40.638 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:40.664 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:40.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:40.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:40.769 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:40.796 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:40.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:40.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:40.903 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:40.931 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:40.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:40.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:41.036 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:41.045 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:40 smithi098 bash[76817]: cluster 2023-09-23T18:35:39.787884+0000 mgr.y (mgr.26054) 512 : cluster 0 pgmap v473: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 42764/76443 objects degraded (55.942%); 2275/76443 objects misplaced (2.976%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T18:35:41.045 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:40 smithi098 bash[79516]: cluster 2023-09-23T18:35:39.787884+0000 mgr.y (mgr.26054) 512 : cluster 0 pgmap v473: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 42764/76443 objects degraded (55.942%); 2275/76443 objects misplaced (2.976%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T18:35:41.065 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:41.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:41.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:41.169 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:41.192 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:41.194 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:40 smithi173 bash[59422]: cluster 2023-09-23T18:35:39.787884+0000 mgr.y (mgr.26054) 512 : cluster 0 pgmap v473: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 42764/76443 objects degraded (55.942%); 2275/76443 objects misplaced (2.976%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T18:35:41.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:41.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:41.295 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:41.316 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:41.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:41.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:41.419 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:41.445 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:41.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:41.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:41.551 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:41.585 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:41.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:41.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:41.689 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:41.721 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:41.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:41.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:41.825 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:41.857 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:41.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:41.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:41.962 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:41.983 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:41.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:41.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:42.087 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:42.117 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:42.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:42.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:42.224 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:42.249 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:42.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:42.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:42.356 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:42.380 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:42.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:42.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:42.487 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:42.511 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:42.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:42.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:42.616 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:42.645 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:42.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:42.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:42.749 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:42.772 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:42.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:42.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:42.874 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:42.908 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:42.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:42.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:43.015 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:43.024 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:42 smithi098 bash[76817]: cluster 2023-09-23T18:35:41.788597+0000 mgr.y (mgr.26054) 513 : cluster 0 pgmap v474: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 42734/72849 objects degraded (58.661%); 2093/72849 objects misplaced (2.873%); 4.8 MiB/s, 79 objects/s recovering 2023-09-23T18:35:43.024 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:42 smithi098 bash[79516]: cluster 2023-09-23T18:35:41.788597+0000 mgr.y (mgr.26054) 513 : cluster 0 pgmap v474: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 42734/72849 objects degraded (58.661%); 2093/72849 objects misplaced (2.873%); 4.8 MiB/s, 79 objects/s recovering 2023-09-23T18:35:43.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:43.048 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:43.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:43.149 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:43.175 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:43.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:43.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:43.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:42 smithi173 bash[59422]: cluster 2023-09-23T18:35:41.788597+0000 mgr.y (mgr.26054) 513 : cluster 0 pgmap v474: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 42734/72849 objects degraded (58.661%); 2093/72849 objects misplaced (2.873%); 4.8 MiB/s, 79 objects/s recovering 2023-09-23T18:35:43.278 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:43.312 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:43.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:43.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:43.418 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:43.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:43.449 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:43.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:43.551 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:43.573 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:43.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:43.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:43.678 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:43.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:43.710 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:43.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:43.811 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:43.836 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:43.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:43.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:43.939 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:43.958 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:43.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:43.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:44.061 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:44.096 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:44.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:44.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:44.201 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:44.226 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:44.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:44.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:44.329 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:44.360 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:44.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:44.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:44.465 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:44.495 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:44.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:44.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:44.600 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:44.628 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:44.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:44.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:44.735 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:44.756 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:44.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:44.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:44.858 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:44.890 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:44.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:44.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:44.994 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:45.035 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:45.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:45.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:45.139 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:44 smithi173 bash[59422]: cluster 2023-09-23T18:35:43.789248+0000 mgr.y (mgr.26054) 514 : cluster 0 pgmap v475: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 42734/70275 objects degraded (60.810%); 2093/70275 objects misplaced (2.978%); 3.8 MiB/s, 63 objects/s recovering 2023-09-23T18:35:45.139 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:44 smithi173 bash[59422]: cluster 2023-09-23T18:35:44.484206+0000 mon.a (mon.0) 723 : cluster 3 Health check update: Degraded data redundancy: 42734/70275 objects degraded (60.810%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:45.141 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:44 smithi098 bash[76817]: cluster 2023-09-23T18:35:43.789248+0000 mgr.y (mgr.26054) 514 : cluster 0 pgmap v475: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 42734/70275 objects degraded (60.810%); 2093/70275 objects misplaced (2.978%); 3.8 MiB/s, 63 objects/s recovering 2023-09-23T18:35:45.141 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:44 smithi098 bash[76817]: cluster 2023-09-23T18:35:44.484206+0000 mon.a (mon.0) 723 : cluster 3 Health check update: Degraded data redundancy: 42734/70275 objects degraded (60.810%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:45.141 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:44 smithi098 bash[79516]: cluster 2023-09-23T18:35:43.789248+0000 mgr.y (mgr.26054) 514 : cluster 0 pgmap v475: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 42734/70275 objects degraded (60.810%); 2093/70275 objects misplaced (2.978%); 3.8 MiB/s, 63 objects/s recovering 2023-09-23T18:35:45.141 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:44 smithi098 bash[79516]: cluster 2023-09-23T18:35:44.484206+0000 mon.a (mon.0) 723 : cluster 3 Health check update: Degraded data redundancy: 42734/70275 objects degraded (60.810%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:45.142 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:45.173 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:45.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:45.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:45.280 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:45.319 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:45.323 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:35:45.323 INFO:teuthology.orchestra.run.smithi173.stdout: "size": 20, 2023-09-23T18:35:45.323 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 600, 2023-09-23T18:35:45.323 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-09-23T18:35:45.323 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.324 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:83583 5.2 5:4f612270:::benchmark_data_smithi098_105788_object83582:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e113)", 2023-09-23T18:35:45.324 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:32:23.542077+0000", 2023-09-23T18:35:45.324 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 201.776942504, 2023-09-23T18:35:45.324 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.167920757, 2023-09-23T18:35:45.324 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:35:45.324 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:35:45.325 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:35:45.325 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:35:45.325 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:35:45.325 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 83583 2023-09-23T18:35:45.325 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.325 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:35:45.326 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.326 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:35:45.326 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.542077+0000", 2023-09-23T18:35:45.326 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:35:45.326 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.326 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.327 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:35:45.327 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.542077+0000", 2023-09-23T18:35:45.327 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:35:45.327 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.327 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.327 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:35:45.328 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.542075+0000", 2023-09-23T18:35:45.328 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.8695e-05 2023-09-23T18:35:45.328 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.328 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.328 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:35:45.328 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.542124+0000", 2023-09-23T18:35:45.328 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2961e-05 2023-09-23T18:35:45.329 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.329 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.329 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:35:45.329 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.542137+0000", 2023-09-23T18:35:45.329 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1519999999999997e-06 2023-09-23T18:35:45.329 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.330 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.330 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:35:45.330 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.542143+0000", 2023-09-23T18:35:45.330 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.0191999999999995e-05 2023-09-23T18:35:45.330 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.330 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.331 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:35:45.331 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.542213+0000", 2023-09-23T18:35:45.331 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012926499999999999 2023-09-23T18:35:45.331 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.331 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.331 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:35:45.332 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.542343+0000", 2023-09-23T18:35:45.332 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015423600000000001 2023-09-23T18:35:45.332 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.332 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.332 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:35:45.332 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.542497+0000", 2023-09-23T18:35:45.332 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0088861640000000002 2023-09-23T18:35:45.333 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.333 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.333 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:35:45.333 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.551383+0000", 2023-09-23T18:35:45.333 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00072143299999999995 2023-09-23T18:35:45.333 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.334 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.334 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.334 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.552105+0000", 2023-09-23T18:35:45.334 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15767466499999999 2023-09-23T18:35:45.334 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.334 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.335 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.335 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.709779+0000", 2023-09-23T18:35:45.335 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000143604 2023-09-23T18:35:45.335 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.335 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.335 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:35:45.336 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.709923+0000", 2023-09-23T18:35:45.336 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.5131999999999998e-05 2023-09-23T18:35:45.336 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.336 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.336 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:35:45.336 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.709998+0000", 2023-09-23T18:35:45.337 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.167920757 2023-09-23T18:35:45.337 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.337 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:35:45.337 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.337 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.337 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.337 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:83585 5.2 5:43f1b378:::benchmark_data_smithi098_105788_object83584:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e113)", 2023-09-23T18:35:45.338 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:32:23.545485+0000", 2023-09-23T18:35:45.338 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 201.773534541, 2023-09-23T18:35:45.338 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.171475918, 2023-09-23T18:35:45.338 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:35:45.338 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:35:45.338 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:35:45.338 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:35:45.339 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:35:45.339 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 83585 2023-09-23T18:35:45.339 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.339 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:35:45.339 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.339 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:35:45.339 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.545485+0000", 2023-09-23T18:35:45.340 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:35:45.340 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.340 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.340 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:35:45.340 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.545485+0000", 2023-09-23T18:35:45.340 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:35:45.340 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.341 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.341 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:35:45.341 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.545483+0000", 2023-09-23T18:35:45.341 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7383000000000003e-05 2023-09-23T18:35:45.341 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.341 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.342 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:35:45.342 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.545541+0000", 2023-09-23T18:35:45.342 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4318000000000001e-05 2023-09-23T18:35:45.342 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.342 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.342 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:35:45.342 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.545555+0000", 2023-09-23T18:35:45.343 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4354000000000001e-05 2023-09-23T18:35:45.343 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.343 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.343 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:35:45.343 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.545579+0000", 2023-09-23T18:35:45.343 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000114533 2023-09-23T18:35:45.343 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.344 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.344 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:35:45.344 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.545694+0000", 2023-09-23T18:35:45.344 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002670618 2023-09-23T18:35:45.344 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.345 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.345 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:35:45.345 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.548365+0000", 2023-09-23T18:35:45.345 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000208777 2023-09-23T18:35:45.345 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.345 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.346 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:35:45.346 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.548573+0000", 2023-09-23T18:35:45.346 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0035598959999999999 2023-09-23T18:35:45.346 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.346 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.346 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.346 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.552133+0000", 2023-09-23T18:35:45.347 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7229000000000001e-05 2023-09-23T18:35:45.347 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.347 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.347 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:35:45.347 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.552150+0000", 2023-09-23T18:35:45.347 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.16441280599999999 2023-09-23T18:35:45.347 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.348 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.348 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.348 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.716563+0000", 2023-09-23T18:35:45.348 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000271721 2023-09-23T18:35:45.348 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.348 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.349 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:35:45.349 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.716835+0000", 2023-09-23T18:35:45.349 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000126074 2023-09-23T18:35:45.349 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.349 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.349 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:35:45.349 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:23.716961+0000", 2023-09-23T18:35:45.349 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.171475918 2023-09-23T18:35:45.349 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.349 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:35:45.349 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.350 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.350 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.350 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:85734 5.2 5:4beeb65f:::benchmark_data_smithi098_105788_object85733:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e113)", 2023-09-23T18:35:45.350 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:32:25.777565+0000", 2023-09-23T18:35:45.350 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 199.54145495500001, 2023-09-23T18:35:45.350 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3695080439999998, 2023-09-23T18:35:45.350 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:35:45.350 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T18:35:45.350 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:35:45.350 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:35:45.350 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:35:45.350 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 85734 2023-09-23T18:35:45.351 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.351 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:35:45.351 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.351 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:35:45.351 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.777565+0000", 2023-09-23T18:35:45.351 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:35:45.351 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.351 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.351 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:35:45.351 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.777565+0000", 2023-09-23T18:35:45.351 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:35:45.351 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.352 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.352 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:35:45.352 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.777564+0000", 2023-09-23T18:35:45.352 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.0037999999999999e-05 2023-09-23T18:35:45.352 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.352 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.352 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:35:45.352 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.777594+0000", 2023-09-23T18:35:45.352 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4399000000000001e-05 2023-09-23T18:35:45.352 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.352 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.352 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:35:45.353 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.777608+0000", 2023-09-23T18:35:45.353 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.2580000000000003e-06 2023-09-23T18:35:45.353 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.353 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.353 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:35:45.353 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.777613+0000", 2023-09-23T18:35:45.353 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.9107999999999999e-05 2023-09-23T18:35:45.353 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.353 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.353 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:35:45.353 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.777652+0000", 2023-09-23T18:35:45.353 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.8166999999999997e-05 2023-09-23T18:35:45.354 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.354 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.354 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:35:45.354 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.777751+0000", 2023-09-23T18:35:45.354 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00023226599999999999 2023-09-23T18:35:45.354 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.354 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.354 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:35:45.354 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.777983+0000", 2023-09-23T18:35:45.354 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0034416630000000002 2023-09-23T18:35:45.354 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.355 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.355 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:35:45.355 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.781424+0000", 2023-09-23T18:35:45.355 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.013460193000000001 2023-09-23T18:35:45.355 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.355 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.355 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:35:45.355 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.794885+0000", 2023-09-23T18:35:45.355 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3978999999999999e-05 2023-09-23T18:35:45.355 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.355 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.355 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:35:45.355 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.794899+0000", 2023-09-23T18:35:45.356 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0039201280000001 2023-09-23T18:35:45.356 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.356 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.356 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:35:45.356 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:26.798819+0000", 2023-09-23T18:35:45.356 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5233e-05 2023-09-23T18:35:45.356 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.356 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.356 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:35:45.356 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:26.798834+0000", 2023-09-23T18:35:45.356 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3481352040000001 2023-09-23T18:35:45.356 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.356 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.357 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:35:45.357 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.146969+0000", 2023-09-23T18:35:45.357 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00010358599999999999 2023-09-23T18:35:45.357 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.357 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.357 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:35:45.357 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.147073+0000", 2023-09-23T18:35:45.357 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3695080439999998 2023-09-23T18:35:45.357 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.357 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:35:45.357 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.357 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.358 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.358 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:85752 5.2 5:4b539e47:::benchmark_data_smithi098_105788_object85751:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e114)", 2023-09-23T18:35:45.358 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:32:25.818150+0000", 2023-09-23T18:35:45.358 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 199.50086941500001, 2023-09-23T18:35:45.358 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3573986690000002, 2023-09-23T18:35:45.358 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:35:45.358 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:35:45.359 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:35:45.359 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:35:45.359 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:35:45.359 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 85752 2023-09-23T18:35:45.359 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.359 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:35:45.359 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.359 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:35:45.360 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.818150+0000", 2023-09-23T18:35:45.360 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:35:45.360 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.360 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.360 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:35:45.360 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.818150+0000", 2023-09-23T18:35:45.360 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999971 2023-09-23T18:35:45.360 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.360 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.361 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:35:45.361 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.818147+0000", 2023-09-23T18:35:45.361 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6697999999999998e-05 2023-09-23T18:35:45.361 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.361 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.361 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:35:45.361 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.818204+0000", 2023-09-23T18:35:45.362 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2957999999999999e-05 2023-09-23T18:35:45.362 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.362 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.362 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:35:45.362 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.818227+0000", 2023-09-23T18:35:45.362 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.4380000000000001e-06 2023-09-23T18:35:45.362 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.363 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.363 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:35:45.363 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.818236+0000", 2023-09-23T18:35:45.363 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.5144e-05 2023-09-23T18:35:45.363 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.363 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.363 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:35:45.364 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.818281+0000", 2023-09-23T18:35:45.364 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.6709999999999994e-06 2023-09-23T18:35:45.364 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.364 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.364 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:35:45.364 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.818291+0000", 2023-09-23T18:35:45.364 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.98059981799999996 2023-09-23T18:35:45.364 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.365 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.365 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:35:45.365 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:26.798891+0000", 2023-09-23T18:35:45.365 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0361e-05 2023-09-23T18:35:45.365 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.365 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.365 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:35:45.366 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:26.798901+0000", 2023-09-23T18:35:45.366 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3482000059999999 2023-09-23T18:35:45.366 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.366 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.366 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:35:45.366 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.147101+0000", 2023-09-23T18:35:45.366 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013684100000000001 2023-09-23T18:35:45.367 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.367 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.367 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:35:45.367 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.147238+0000", 2023-09-23T18:35:45.367 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00019616999999999999 2023-09-23T18:35:45.367 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.367 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.368 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:35:45.368 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.147434+0000", 2023-09-23T18:35:45.368 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00815777 2023-09-23T18:35:45.368 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.368 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.368 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:35:45.368 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.155592+0000", 2023-09-23T18:35:45.368 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0031753749999999998 2023-09-23T18:35:45.369 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.369 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.369 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.369 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.158768+0000", 2023-09-23T18:35:45.369 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.014097788 2023-09-23T18:35:45.369 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.369 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.370 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.370 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.172865+0000", 2023-09-23T18:35:45.370 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.8650000000000002e-05 2023-09-23T18:35:45.370 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.370 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.370 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:35:45.370 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.172934+0000", 2023-09-23T18:35:45.371 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0026150309999999999 2023-09-23T18:35:45.371 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.371 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.371 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:35:45.371 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.175549+0000", 2023-09-23T18:35:45.371 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3573986690000002 2023-09-23T18:35:45.371 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.372 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:35:45.372 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.372 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.372 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.372 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:116965 5.2 5:4c8257a6:::benchmark_data_smithi098_105788_object10680:head [delete] snapc 0=[] ondisk+write+known_if_redirected e120)", 2023-09-23T18:35:45.372 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:33:19.935893+0000", 2023-09-23T18:35:45.372 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 145.383126989, 2023-09-23T18:35:45.373 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.17261057399999999, 2023-09-23T18:35:45.373 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:35:45.373 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:35:45.373 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:35:45.373 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:35:45.373 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:35:45.373 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 116965 2023-09-23T18:35:45.374 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.374 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:35:45.374 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.374 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:35:45.374 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.935893+0000", 2023-09-23T18:35:45.374 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:35:45.374 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.375 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.375 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:35:45.375 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.935893+0000", 2023-09-23T18:35:45.375 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-09-23T18:35:45.375 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.375 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.375 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:35:45.375 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.935890+0000", 2023-09-23T18:35:45.375 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2772000000000001e-05 2023-09-23T18:35:45.375 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.375 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.375 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:35:45.375 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.935903+0000", 2023-09-23T18:35:45.376 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.7263e-05 2023-09-23T18:35:45.376 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.376 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.376 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:35:45.376 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.935930+0000", 2023-09-23T18:35:45.376 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.3899999999999999e-06 2023-09-23T18:35:45.376 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.376 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.376 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:35:45.376 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.935940+0000", 2023-09-23T18:35:45.376 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.9933999999999996e-05 2023-09-23T18:35:45.376 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.376 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.377 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:35:45.377 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.936040+0000", 2023-09-23T18:35:45.377 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00025766 2023-09-23T18:35:45.377 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.377 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.377 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:35:45.377 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.936297+0000", 2023-09-23T18:35:45.377 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00029380200000000002 2023-09-23T18:35:45.377 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.377 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.377 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:35:45.377 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.936591+0000", 2023-09-23T18:35:45.378 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0045989660000000003 2023-09-23T18:35:45.378 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.378 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.378 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:35:45.378 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.941190+0000", 2023-09-23T18:35:45.378 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0022080759999999998 2023-09-23T18:35:45.378 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.378 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.378 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.378 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.943398+0000", 2023-09-23T18:35:45.378 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.16213697899999999 2023-09-23T18:35:45.378 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.379 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.379 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.379 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.105535+0000", 2023-09-23T18:35:45.379 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.3869999999999997e-05 2023-09-23T18:35:45.379 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.379 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.379 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:35:45.379 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.105599+0000", 2023-09-23T18:35:45.379 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0029042040000000001 2023-09-23T18:35:45.379 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.379 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.379 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:35:45.379 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.108503+0000", 2023-09-23T18:35:45.380 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.17261057399999999 2023-09-23T18:35:45.380 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.380 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:35:45.380 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.380 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.380 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.380 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:116972 5.2 5:495a5b1d:::benchmark_data_smithi098_105788_object10687:head [delete] snapc 0=[] ondisk+write+known_if_redirected e120)", 2023-09-23T18:35:45.380 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:33:19.948932+0000", 2023-09-23T18:35:45.380 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 145.37008813599999, 2023-09-23T18:35:45.380 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.166610383, 2023-09-23T18:35:45.380 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:35:45.380 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:35:45.380 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:35:45.381 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:35:45.381 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:35:45.381 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 116972 2023-09-23T18:35:45.381 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.381 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:35:45.381 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.381 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:35:45.381 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.948932+0000", 2023-09-23T18:35:45.381 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:35:45.381 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.381 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.381 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:35:45.381 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.948932+0000", 2023-09-23T18:35:45.381 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:35:45.382 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.382 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.382 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:35:45.382 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.948930+0000", 2023-09-23T18:35:45.382 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2485e-05 2023-09-23T18:35:45.382 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.382 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.382 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:35:45.382 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.948942+0000", 2023-09-23T18:35:45.382 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.1444e-05 2023-09-23T18:35:45.382 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.382 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.383 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:35:45.383 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.948964+0000", 2023-09-23T18:35:45.383 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.5780000000000005e-06 2023-09-23T18:35:45.383 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.383 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.383 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:35:45.383 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.948973+0000", 2023-09-23T18:35:45.383 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.7155000000000002e-05 2023-09-23T18:35:45.383 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.383 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.383 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:35:45.383 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.949060+0000", 2023-09-23T18:35:45.383 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000206379 2023-09-23T18:35:45.384 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.384 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.384 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:35:45.384 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.949267+0000", 2023-09-23T18:35:45.384 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00025964 2023-09-23T18:35:45.384 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.384 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.384 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:35:45.384 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.949526+0000", 2023-09-23T18:35:45.384 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0072054739999999999 2023-09-23T18:35:45.384 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.384 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.384 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:35:45.385 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.956732+0000", 2023-09-23T18:35:45.385 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0036638370000000001 2023-09-23T18:35:45.385 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.385 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.385 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.385 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.960396+0000", 2023-09-23T18:35:45.385 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15191782200000001 2023-09-23T18:35:45.385 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.385 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.385 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.385 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.112313+0000", 2023-09-23T18:35:45.385 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.4170000000000002e-05 2023-09-23T18:35:45.385 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.386 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.386 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:35:45.386 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.112398+0000", 2023-09-23T18:35:45.386 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.003144314 2023-09-23T18:35:45.386 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.386 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.386 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:35:45.386 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.115542+0000", 2023-09-23T18:35:45.386 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.166610383 2023-09-23T18:35:45.386 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.386 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:35:45.386 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.386 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.387 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.387 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:116987 5.2 5:4657143b:::benchmark_data_smithi098_105788_object10702:head [delete] snapc 0=[] ondisk+write+known_if_redirected e120)", 2023-09-23T18:35:45.387 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:33:19.968234+0000", 2023-09-23T18:35:45.387 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 145.35078599600001, 2023-09-23T18:35:45.387 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.153279417, 2023-09-23T18:35:45.387 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:35:45.387 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:35:45.387 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:35:45.387 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:35:45.387 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:35:45.387 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 116987 2023-09-23T18:35:45.387 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.388 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:35:45.388 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.388 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:35:45.388 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.968234+0000", 2023-09-23T18:35:45.388 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:35:45.388 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.388 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.388 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:35:45.388 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.968234+0000", 2023-09-23T18:35:45.388 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-09-23T18:35:45.388 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.388 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.389 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:35:45.389 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.968233+0000", 2023-09-23T18:35:45.389 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.6940000000000001e-06 2023-09-23T18:35:45.389 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.389 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.389 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:35:45.389 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.968237+0000", 2023-09-23T18:35:45.389 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.2479999999999996e-06 2023-09-23T18:35:45.389 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.389 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.389 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:35:45.389 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.968245+0000", 2023-09-23T18:35:45.389 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4099999999999999e-06 2023-09-23T18:35:45.390 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.390 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.390 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:35:45.390 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.968250+0000", 2023-09-23T18:35:45.390 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0031772509999999999 2023-09-23T18:35:45.390 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.390 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.390 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:35:45.390 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.971428+0000", 2023-09-23T18:35:45.390 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000119566 2023-09-23T18:35:45.390 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.390 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.390 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:35:45.391 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.971547+0000", 2023-09-23T18:35:45.391 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000113443 2023-09-23T18:35:45.391 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.391 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.391 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:35:45.391 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.971661+0000", 2023-09-23T18:35:45.391 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0026122110000000001 2023-09-23T18:35:45.391 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.391 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.391 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:35:45.391 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.974273+0000", 2023-09-23T18:35:45.391 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.12489196499999999 2023-09-23T18:35:45.391 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.392 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.392 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.392 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.099165+0000", 2023-09-23T18:35:45.392 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.019529633000000001 2023-09-23T18:35:45.392 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.392 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.392 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.392 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.118694+0000", 2023-09-23T18:35:45.392 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.7205000000000005e-05 2023-09-23T18:35:45.392 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.392 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.392 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:35:45.393 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.118762+0000", 2023-09-23T18:35:45.393 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0027514950000000001 2023-09-23T18:35:45.393 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.393 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.393 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:35:45.393 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.121513+0000", 2023-09-23T18:35:45.393 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.153279417 2023-09-23T18:35:45.393 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.393 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:35:45.393 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.393 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.393 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.393 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:119246 5.2 5:46b0c113:::benchmark_data_smithi098_105788_object12961:head [delete] snapc 0=[] ondisk+write+known_if_redirected e120)", 2023-09-23T18:35:45.394 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:33:23.920937+0000", 2023-09-23T18:35:45.394 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 141.398083001, 2023-09-23T18:35:45.394 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.162449866, 2023-09-23T18:35:45.394 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:35:45.394 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:35:45.394 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:35:45.394 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:35:45.394 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:35:45.394 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 119246 2023-09-23T18:35:45.394 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.394 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:35:45.394 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.394 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:35:45.395 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.920937+0000", 2023-09-23T18:35:45.395 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:35:45.395 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.395 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.395 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:35:45.395 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.920937+0000", 2023-09-23T18:35:45.395 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-09-23T18:35:45.395 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.395 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.395 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:35:45.395 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.920936+0000", 2023-09-23T18:35:45.395 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.2930000000000001e-06 2023-09-23T18:35:45.395 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.396 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.396 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:35:45.396 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.920939+0000", 2023-09-23T18:35:45.396 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.5979999999999999e-06 2023-09-23T18:35:45.396 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.396 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.396 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:35:45.396 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.920947+0000", 2023-09-23T18:35:45.396 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.2860000000000001e-06 2023-09-23T18:35:45.396 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.396 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.396 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:35:45.397 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.920951+0000", 2023-09-23T18:35:45.397 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0036218240000000001 2023-09-23T18:35:45.397 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.397 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.397 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:35:45.397 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.924573+0000", 2023-09-23T18:35:45.397 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000181948 2023-09-23T18:35:45.397 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.397 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.397 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:35:45.397 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.924755+0000", 2023-09-23T18:35:45.397 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015142700000000001 2023-09-23T18:35:45.397 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.398 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.398 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:35:45.398 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.924907+0000", 2023-09-23T18:35:45.398 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0080970309999999993 2023-09-23T18:35:45.398 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.398 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.398 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:35:45.398 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.933004+0000", 2023-09-23T18:35:45.398 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0058044610000000003 2023-09-23T18:35:45.398 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.398 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.398 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.398 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.938808+0000", 2023-09-23T18:35:45.399 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.13206895799999999 2023-09-23T18:35:45.399 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.399 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.399 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.399 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:24.070877+0000", 2023-09-23T18:35:45.399 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00011163300000000001 2023-09-23T18:35:45.399 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.399 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.399 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:35:45.399 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:24.070989+0000", 2023-09-23T18:35:45.399 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.012397956 2023-09-23T18:35:45.399 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.399 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.400 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:35:45.400 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:24.083387+0000", 2023-09-23T18:35:45.400 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.162449866 2023-09-23T18:35:45.400 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.400 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:35:45.400 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.400 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.400 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.400 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:119247 5.2 5:4efebf4d:::benchmark_data_smithi098_105788_object12962:head [delete] snapc 0=[] ondisk+write+known_if_redirected e120)", 2023-09-23T18:35:45.400 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:33:23.920969+0000", 2023-09-23T18:35:45.400 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 141.39805100000001, 2023-09-23T18:35:45.400 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.16533740399999999, 2023-09-23T18:35:45.401 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:35:45.401 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:35:45.401 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:35:45.401 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:35:45.401 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:35:45.401 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 119247 2023-09-23T18:35:45.401 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.401 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:35:45.401 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.401 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:35:45.401 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.920969+0000", 2023-09-23T18:35:45.401 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:35:45.401 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.402 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.402 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:35:45.402 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.920969+0000", 2023-09-23T18:35:45.402 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-09-23T18:35:45.402 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.402 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.402 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:35:45.402 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.920968+0000", 2023-09-23T18:35:45.402 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.4809999999999998e-06 2023-09-23T18:35:45.402 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.402 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.402 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:35:45.402 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.920972+0000", 2023-09-23T18:35:45.403 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.7470000000000005e-06 2023-09-23T18:35:45.403 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.403 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.403 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:35:45.403 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.920979+0000", 2023-09-23T18:35:45.403 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.1849999999999997e-06 2023-09-23T18:35:45.403 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.403 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.403 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:35:45.403 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.920984+0000", 2023-09-23T18:35:45.403 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.008043573 2023-09-23T18:35:45.403 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.404 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.404 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:35:45.404 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.929027+0000", 2023-09-23T18:35:45.404 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015935799999999999 2023-09-23T18:35:45.404 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.404 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.404 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:35:45.404 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.929187+0000", 2023-09-23T18:35:45.404 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000176713 2023-09-23T18:35:45.404 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.404 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.404 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:35:45.404 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.929363+0000", 2023-09-23T18:35:45.405 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0037531109999999999 2023-09-23T18:35:45.405 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.405 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.405 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:35:45.405 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.933116+0000", 2023-09-23T18:35:45.405 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0058008180000000001 2023-09-23T18:35:45.405 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.405 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.405 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.405 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.938917+0000", 2023-09-23T18:35:45.405 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.14452568599999999 2023-09-23T18:35:45.405 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.405 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.406 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.406 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:24.083443+0000", 2023-09-23T18:35:45.406 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4880999999999999e-05 2023-09-23T18:35:45.406 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.406 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.406 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:35:45.406 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:24.083498+0000", 2023-09-23T18:35:45.406 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0028083579999999999 2023-09-23T18:35:45.406 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.406 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.406 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:35:45.406 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:24.086306+0000", 2023-09-23T18:35:45.407 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.16533740399999999 2023-09-23T18:35:45.407 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.407 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:35:45.407 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.407 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.407 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.407 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:119255 5.2 5:46c31915:::benchmark_data_smithi098_105788_object12970:head [delete] snapc 0=[] ondisk+write+known_if_redirected e120)", 2023-09-23T18:35:45.407 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:33:23.938758+0000", 2023-09-23T18:35:45.407 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 141.38026153499999, 2023-09-23T18:35:45.407 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15723609799999999, 2023-09-23T18:35:45.407 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:35:45.407 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:35:45.407 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:35:45.408 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:35:45.408 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:35:45.408 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 119255 2023-09-23T18:35:45.408 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.408 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:35:45.408 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.408 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:35:45.408 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.938758+0000", 2023-09-23T18:35:45.408 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:35:45.408 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.408 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.408 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:35:45.409 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.938758+0000", 2023-09-23T18:35:45.409 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-09-23T18:35:45.409 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.409 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.409 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:35:45.409 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.938758+0000", 2023-09-23T18:35:45.409 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3080000000000002e-06 2023-09-23T18:35:45.409 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.409 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.409 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:35:45.409 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.938760+0000", 2023-09-23T18:35:45.409 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.1659999999999996e-06 2023-09-23T18:35:45.409 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.410 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.410 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:35:45.410 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.938764+0000", 2023-09-23T18:35:45.410 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.349e-06 2023-09-23T18:35:45.410 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.410 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.410 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:35:45.410 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.938768+0000", 2023-09-23T18:35:45.410 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0032056810000000002 2023-09-23T18:35:45.410 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.410 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.410 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:35:45.410 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.941973+0000", 2023-09-23T18:35:45.411 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00011773300000000001 2023-09-23T18:35:45.411 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.411 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.411 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:35:45.411 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.942091+0000", 2023-09-23T18:35:45.411 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000115524 2023-09-23T18:35:45.411 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.411 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.411 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:35:45.411 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.942206+0000", 2023-09-23T18:35:45.411 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0025328769999999998 2023-09-23T18:35:45.411 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.411 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.412 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:35:45.412 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:23.944739+0000", 2023-09-23T18:35:45.412 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.102239629 2023-09-23T18:35:45.412 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.412 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.412 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.412 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:24.046979+0000", 2023-09-23T18:35:45.412 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0423626 2023-09-23T18:35:45.412 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.412 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.412 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.412 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:24.089342+0000", 2023-09-23T18:35:45.412 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.5853999999999997e-05 2023-09-23T18:35:45.413 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.413 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.413 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:35:45.413 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:24.089407+0000", 2023-09-23T18:35:45.413 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0065868430000000002 2023-09-23T18:35:45.413 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.413 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.413 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:35:45.413 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:24.095994+0000", 2023-09-23T18:35:45.413 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15723609799999999 2023-09-23T18:35:45.413 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.413 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:35:45.413 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.414 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.414 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.414 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:139721 5.2 5:462096f1:::benchmark_data_smithi098_105788_object33436:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-09-23T18:35:45.414 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:34:02.791292+0000", 2023-09-23T18:35:45.414 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 102.527727638, 2023-09-23T18:35:45.414 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.17631037999999999, 2023-09-23T18:35:45.414 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:35:45.414 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:35:45.414 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:35:45.414 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:35:45.414 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:35:45.414 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 139721 2023-09-23T18:35:45.415 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.415 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:35:45.415 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.415 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:35:45.415 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.791292+0000", 2023-09-23T18:35:45.415 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:35:45.415 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.415 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.415 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:35:45.415 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.791292+0000", 2023-09-23T18:35:45.415 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-09-23T18:35:45.415 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.416 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.416 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:35:45.416 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.791290+0000", 2023-09-23T18:35:45.416 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1404e-05 2023-09-23T18:35:45.416 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.416 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.416 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:35:45.416 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.791301+0000", 2023-09-23T18:35:45.416 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3362000000000001e-05 2023-09-23T18:35:45.416 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.416 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.416 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:35:45.416 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.791325+0000", 2023-09-23T18:35:45.417 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.3819999999999993e-06 2023-09-23T18:35:45.417 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.417 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.417 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:35:45.417 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.791334+0000", 2023-09-23T18:35:45.417 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0058104439999999997 2023-09-23T18:35:45.417 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.417 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.417 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:35:45.417 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.797145+0000", 2023-09-23T18:35:45.417 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00020031500000000001 2023-09-23T18:35:45.417 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.418 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.418 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:35:45.418 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.797345+0000", 2023-09-23T18:35:45.418 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00023774899999999999 2023-09-23T18:35:45.418 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.418 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.418 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:35:45.418 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.797583+0000", 2023-09-23T18:35:45.418 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.007917983 2023-09-23T18:35:45.418 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.418 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.418 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:35:45.418 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.805501+0000", 2023-09-23T18:35:45.419 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.015444232 2023-09-23T18:35:45.419 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.419 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.419 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.419 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.820945+0000", 2023-09-23T18:35:45.419 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.14236015699999999 2023-09-23T18:35:45.419 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.419 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.419 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.419 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.963305+0000", 2023-09-23T18:35:45.419 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013193600000000001 2023-09-23T18:35:45.419 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.419 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.420 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:35:45.420 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.963437+0000", 2023-09-23T18:35:45.420 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0041655759999999998 2023-09-23T18:35:45.420 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.420 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.420 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:35:45.420 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.967602+0000", 2023-09-23T18:35:45.420 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.17631037999999999 2023-09-23T18:35:45.420 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.420 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:35:45.420 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.420 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.421 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.421 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:139722 5.2 5:4c16fb5c:::benchmark_data_smithi098_105788_object33437:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-09-23T18:35:45.421 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:34:02.791795+0000", 2023-09-23T18:35:45.421 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 102.527225007, 2023-09-23T18:35:45.421 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.19616816400000001, 2023-09-23T18:35:45.421 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:35:45.421 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:35:45.421 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:35:45.421 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:35:45.421 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:35:45.421 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 139722 2023-09-23T18:35:45.421 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.421 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:35:45.422 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.422 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:35:45.422 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.791795+0000", 2023-09-23T18:35:45.422 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:35:45.422 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.422 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.422 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:35:45.422 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.791795+0000", 2023-09-23T18:35:45.422 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:35:45.422 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.422 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.422 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:35:45.423 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.791793+0000", 2023-09-23T18:35:45.423 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.0199999999999994e-06 2023-09-23T18:35:45.423 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.423 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.423 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:35:45.423 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.791801+0000", 2023-09-23T18:35:45.423 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9655000000000001e-05 2023-09-23T18:35:45.423 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.423 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.423 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:35:45.423 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.791821+0000", 2023-09-23T18:35:45.423 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.7870000000000002e-06 2023-09-23T18:35:45.423 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.424 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.424 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:35:45.424 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.791829+0000", 2023-09-23T18:35:45.424 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0096525499999999993 2023-09-23T18:35:45.424 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.424 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.424 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:35:45.424 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.801481+0000", 2023-09-23T18:35:45.424 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00020985700000000001 2023-09-23T18:35:45.424 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.424 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.424 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:35:45.424 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.801691+0000", 2023-09-23T18:35:45.425 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000245641 2023-09-23T18:35:45.425 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.425 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.425 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:35:45.425 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.801937+0000", 2023-09-23T18:35:45.425 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.008302261 2023-09-23T18:35:45.425 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.425 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.425 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:35:45.425 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.810239+0000", 2023-09-23T18:35:45.425 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010737754 2023-09-23T18:35:45.425 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.425 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.426 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.426 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.820977+0000", 2023-09-23T18:35:45.426 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.163549676 2023-09-23T18:35:45.426 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.426 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.426 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.426 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.984527+0000", 2023-09-23T18:35:45.426 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.1148000000000003e-05 2023-09-23T18:35:45.426 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.426 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.426 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:35:45.426 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.984608+0000", 2023-09-23T18:35:45.426 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0033550810000000002 2023-09-23T18:35:45.427 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.427 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.427 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:35:45.427 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.987963+0000", 2023-09-23T18:35:45.427 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.19616816400000001 2023-09-23T18:35:45.427 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.427 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:35:45.427 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.427 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.427 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.427 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:139725 5.2 5:4e83e029:::benchmark_data_smithi098_105788_object33440:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-09-23T18:35:45.427 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:34:02.805396+0000", 2023-09-23T18:35:45.428 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 102.513623286, 2023-09-23T18:35:45.428 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.18493595700000001, 2023-09-23T18:35:45.428 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:35:45.428 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:35:45.428 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:35:45.428 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:35:45.428 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:35:45.428 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 139725 2023-09-23T18:35:45.428 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.428 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:35:45.428 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.428 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:35:45.428 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.805396+0000", 2023-09-23T18:35:45.429 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:35:45.429 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.429 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.429 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:35:45.429 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.805396+0000", 2023-09-23T18:35:45.429 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-09-23T18:35:45.429 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.429 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.429 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:35:45.429 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.805394+0000", 2023-09-23T18:35:45.429 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0699e-05 2023-09-23T18:35:45.429 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.429 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.430 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:35:45.430 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.805405+0000", 2023-09-23T18:35:45.430 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.6327000000000001e-05 2023-09-23T18:35:45.430 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.430 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.430 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:35:45.430 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.805431+0000", 2023-09-23T18:35:45.430 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2500000000000001e-05 2023-09-23T18:35:45.430 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.430 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.430 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:35:45.430 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.805454+0000", 2023-09-23T18:35:45.430 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.0171000000000002e-05 2023-09-23T18:35:45.431 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.431 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.431 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:35:45.431 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.805534+0000", 2023-09-23T18:35:45.431 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00019968000000000001 2023-09-23T18:35:45.431 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.431 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.431 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:35:45.431 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.805734+0000", 2023-09-23T18:35:45.431 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00025578500000000002 2023-09-23T18:35:45.431 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.431 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.431 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:35:45.432 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.805989+0000", 2023-09-23T18:35:45.432 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0044395679999999996 2023-09-23T18:35:45.432 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.432 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.432 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:35:45.432 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.810429+0000", 2023-09-23T18:35:45.432 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.014223835000000001 2023-09-23T18:35:45.432 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.432 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.432 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.432 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.824653+0000", 2023-09-23T18:35:45.432 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.16338195 2023-09-23T18:35:45.432 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.433 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.433 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.433 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.988035+0000", 2023-09-23T18:35:45.433 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.8834e-05 2023-09-23T18:35:45.433 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.433 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.433 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:35:45.433 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.988084+0000", 2023-09-23T18:35:45.433 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002248782 2023-09-23T18:35:45.433 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.433 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.433 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:35:45.434 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.990332+0000", 2023-09-23T18:35:45.434 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.18493595700000001 2023-09-23T18:35:45.434 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.434 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:35:45.434 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.434 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.434 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.434 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:139732 5.2 5:44d08adf:::benchmark_data_smithi098_105788_object33447:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-09-23T18:35:45.434 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:34:02.826433+0000", 2023-09-23T18:35:45.434 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 102.492586938, 2023-09-23T18:35:45.434 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.166270323, 2023-09-23T18:35:45.434 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:35:45.435 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:35:45.435 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:35:45.435 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:35:45.435 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:35:45.435 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 139732 2023-09-23T18:35:45.435 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.435 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:35:45.435 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.435 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:35:45.435 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.826433+0000", 2023-09-23T18:35:45.435 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:35:45.435 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.435 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.436 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:35:45.436 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.826433+0000", 2023-09-23T18:35:45.436 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:35:45.436 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.436 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.436 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:35:45.436 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.826431+0000", 2023-09-23T18:35:45.436 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.2239999999999995e-06 2023-09-23T18:35:45.436 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.436 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.436 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:35:45.436 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.826439+0000", 2023-09-23T18:35:45.436 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3893000000000001e-05 2023-09-23T18:35:45.437 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.437 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.437 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:35:45.437 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.826453+0000", 2023-09-23T18:35:45.437 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.212e-06 2023-09-23T18:35:45.437 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.437 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.437 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:35:45.437 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.826460+0000", 2023-09-23T18:35:45.437 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1130000000000006e-05 2023-09-23T18:35:45.437 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.437 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.438 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:35:45.438 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.826521+0000", 2023-09-23T18:35:45.438 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018960099999999999 2023-09-23T18:35:45.438 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.438 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.438 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:35:45.438 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.826710+0000", 2023-09-23T18:35:45.438 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00020196799999999999 2023-09-23T18:35:45.438 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.438 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.438 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:35:45.438 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.826912+0000", 2023-09-23T18:35:45.438 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.012916732 2023-09-23T18:35:45.439 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.439 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.439 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:35:45.439 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.839829+0000", 2023-09-23T18:35:45.439 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0071496600000000004 2023-09-23T18:35:45.439 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.439 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.439 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.439 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.846979+0000", 2023-09-23T18:35:45.439 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.14341531699999999 2023-09-23T18:35:45.439 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.439 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.439 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.440 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.990394+0000", 2023-09-23T18:35:45.440 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6372999999999998e-05 2023-09-23T18:35:45.440 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.440 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.440 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:35:45.440 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.990440+0000", 2023-09-23T18:35:45.440 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0022627839999999999 2023-09-23T18:35:45.440 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.440 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.440 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:35:45.440 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:02.992703+0000", 2023-09-23T18:35:45.440 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.166270323 2023-09-23T18:35:45.441 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.441 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:35:45.441 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.441 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.441 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.441 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:141165 5.2 5:419e434b:::benchmark_data_smithi098_105788_object34880:head [delete] snapc 0=[] ondisk+write+known_if_redirected e126)", 2023-09-23T18:35:45.441 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:34:05.947551+0000", 2023-09-23T18:35:45.441 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 99.371468254000007, 2023-09-23T18:35:45.441 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.180648737, 2023-09-23T18:35:45.441 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:35:45.441 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:35:45.441 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:35:45.441 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:35:45.442 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:35:45.442 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 141165 2023-09-23T18:35:45.442 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.442 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:35:45.442 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.442 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:35:45.442 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.947551+0000", 2023-09-23T18:35:45.442 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:35:45.442 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.442 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.442 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:35:45.442 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.947551+0000", 2023-09-23T18:35:45.442 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:35:45.443 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.443 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.443 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:35:45.443 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.947550+0000", 2023-09-23T18:35:45.443 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.9279999999999993e-06 2023-09-23T18:35:45.443 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.443 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.443 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:35:45.443 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.947557+0000", 2023-09-23T18:35:45.443 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6005e-05 2023-09-23T18:35:45.443 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.443 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.443 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:35:45.444 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.947573+0000", 2023-09-23T18:35:45.444 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.0250000000000002e-06 2023-09-23T18:35:45.444 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.444 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.444 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:35:45.444 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.947582+0000", 2023-09-23T18:35:45.444 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6159000000000002e-05 2023-09-23T18:35:45.444 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.444 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.444 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:35:45.444 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.947638+0000", 2023-09-23T18:35:45.444 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000153497 2023-09-23T18:35:45.445 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.445 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.445 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:35:45.445 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.947791+0000", 2023-09-23T18:35:45.445 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015809999999999999 2023-09-23T18:35:45.445 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.445 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.445 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:35:45.445 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.947949+0000", 2023-09-23T18:35:45.445 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0043631119999999997 2023-09-23T18:35:45.445 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.445 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.445 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:35:45.446 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.952312+0000", 2023-09-23T18:35:45.446 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0022775109999999999 2023-09-23T18:35:45.446 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.446 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.446 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.446 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.954590+0000", 2023-09-23T18:35:45.446 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.169698935 2023-09-23T18:35:45.446 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.446 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.446 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.446 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:06.124289+0000", 2023-09-23T18:35:45.446 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000107703 2023-09-23T18:35:45.447 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.447 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.447 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:35:45.447 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:06.124397+0000", 2023-09-23T18:35:45.447 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0038036580000000001 2023-09-23T18:35:45.447 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.447 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.447 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:35:45.447 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:06.128200+0000", 2023-09-23T18:35:45.447 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.180648737 2023-09-23T18:35:45.447 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.447 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:35:45.447 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.448 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.448 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.448 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:141167 5.2 5:45f27961:::benchmark_data_smithi098_105788_object34882:head [delete] snapc 0=[] ondisk+write+known_if_redirected e126)", 2023-09-23T18:35:45.448 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:34:05.960025+0000", 2023-09-23T18:35:45.448 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 99.358995127, 2023-09-23T18:35:45.448 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.17102857699999999, 2023-09-23T18:35:45.448 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:35:45.448 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:35:45.448 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:35:45.448 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:35:45.448 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:35:45.448 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 141167 2023-09-23T18:35:45.449 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.449 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:35:45.449 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.449 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:35:45.449 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.960025+0000", 2023-09-23T18:35:45.449 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:35:45.449 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.449 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.449 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:35:45.449 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.960025+0000", 2023-09-23T18:35:45.449 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-09-23T18:35:45.449 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.449 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.450 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:35:45.450 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.960022+0000", 2023-09-23T18:35:45.450 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1389999999999999e-05 2023-09-23T18:35:45.450 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.450 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.450 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:35:45.450 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.960034+0000", 2023-09-23T18:35:45.450 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2459e-05 2023-09-23T18:35:45.450 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.450 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.450 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:35:45.450 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.960056+0000", 2023-09-23T18:35:45.451 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.3360000000000001e-06 2023-09-23T18:35:45.451 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.451 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.451 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:35:45.451 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.960065+0000", 2023-09-23T18:35:45.451 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.6058999999999999e-05 2023-09-23T18:35:45.451 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.451 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.451 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:35:45.451 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.960132+0000", 2023-09-23T18:35:45.451 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00020369600000000001 2023-09-23T18:35:45.451 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.451 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.452 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:35:45.452 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.960335+0000", 2023-09-23T18:35:45.452 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00024368400000000001 2023-09-23T18:35:45.452 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.452 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.452 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:35:45.452 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.960579+0000", 2023-09-23T18:35:45.452 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0035917610000000002 2023-09-23T18:35:45.452 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.452 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.452 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:35:45.452 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.964171+0000", 2023-09-23T18:35:45.453 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0047557399999999996 2023-09-23T18:35:45.453 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.453 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.453 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.453 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.968926+0000", 2023-09-23T18:35:45.453 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15936303199999999 2023-09-23T18:35:45.453 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.453 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.453 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.453 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:06.128289+0000", 2023-09-23T18:35:45.453 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.5396999999999996e-05 2023-09-23T18:35:45.453 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.453 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.454 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:35:45.454 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:06.128375+0000", 2023-09-23T18:35:45.454 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0026783459999999999 2023-09-23T18:35:45.454 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.454 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.454 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:35:45.454 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:06.131053+0000", 2023-09-23T18:35:45.454 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.17102857699999999 2023-09-23T18:35:45.454 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.454 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:35:45.454 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.454 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.454 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.455 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:141168 5.2 5:42d15c31:::benchmark_data_smithi098_105788_object34883:head [delete] snapc 0=[] ondisk+write+known_if_redirected e126)", 2023-09-23T18:35:45.455 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:34:05.970712+0000", 2023-09-23T18:35:45.455 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 99.348308066000001, 2023-09-23T18:35:45.455 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.162972641, 2023-09-23T18:35:45.455 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:35:45.455 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:35:45.455 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:35:45.455 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:35:45.455 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:35:45.455 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 141168 2023-09-23T18:35:45.455 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.455 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:35:45.456 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.456 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:35:45.456 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.970712+0000", 2023-09-23T18:35:45.456 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:35:45.456 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.456 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.456 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:35:45.456 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.970712+0000", 2023-09-23T18:35:45.456 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-09-23T18:35:45.456 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.456 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.456 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:35:45.456 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.970709+0000", 2023-09-23T18:35:45.457 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3759e-05 2023-09-23T18:35:45.457 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.457 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.457 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:35:45.457 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.970723+0000", 2023-09-23T18:35:45.457 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.7863000000000001e-05 2023-09-23T18:35:45.457 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.457 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.457 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:35:45.457 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.970751+0000", 2023-09-23T18:35:45.457 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1922e-05 2023-09-23T18:35:45.457 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.457 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.458 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:35:45.458 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.970763+0000", 2023-09-23T18:35:45.458 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.8431999999999996e-05 2023-09-23T18:35:45.458 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.458 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.458 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:35:45.458 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.970851+0000", 2023-09-23T18:35:45.458 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00029291899999999999 2023-09-23T18:35:45.458 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.458 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.458 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:35:45.458 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.971144+0000", 2023-09-23T18:35:45.459 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00037022900000000002 2023-09-23T18:35:45.459 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.459 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.459 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:35:45.459 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.971514+0000", 2023-09-23T18:35:45.459 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0044879389999999998 2023-09-23T18:35:45.459 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.459 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.459 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:35:45.459 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.976002+0000", 2023-09-23T18:35:45.459 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0024777950000000001 2023-09-23T18:35:45.459 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.459 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.460 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.460 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.978480+0000", 2023-09-23T18:35:45.460 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15264909800000001 2023-09-23T18:35:45.460 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.460 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.460 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.460 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:06.131129+0000", 2023-09-23T18:35:45.460 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.6336999999999999e-05 2023-09-23T18:35:45.460 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.460 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.460 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:35:45.460 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:06.131195+0000", 2023-09-23T18:35:45.461 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0024888589999999999 2023-09-23T18:35:45.461 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.461 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.461 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:35:45.461 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:06.133684+0000", 2023-09-23T18:35:45.461 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.162972641 2023-09-23T18:35:45.461 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.461 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:35:45.461 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.461 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.461 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.461 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:168485 5.2 5:4f6d7df8:::benchmark_data_smithi098_105788_object62200:head [delete] snapc 0=[] ondisk+write+known_if_redirected e130)", 2023-09-23T18:35:45.461 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:35:01.094759+0000", 2023-09-23T18:35:45.462 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 44.224260698000002, 2023-09-23T18:35:45.462 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.18820453700000001, 2023-09-23T18:35:45.462 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:35:45.462 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:35:45.462 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:35:45.462 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:35:45.462 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:35:45.462 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 168485 2023-09-23T18:35:45.462 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.462 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:35:45.462 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.462 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:35:45.462 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.094759+0000", 2023-09-23T18:35:45.463 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:35:45.463 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.463 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.463 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:35:45.463 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.094759+0000", 2023-09-23T18:35:45.463 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:35:45.463 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.463 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.463 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:35:45.463 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.094757+0000", 2023-09-23T18:35:45.463 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.2390000000000002e-06 2023-09-23T18:35:45.463 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.464 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.464 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:35:45.464 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.094766+0000", 2023-09-23T18:35:45.464 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2832e-05 2023-09-23T18:35:45.464 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.464 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.464 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:35:45.464 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.094779+0000", 2023-09-23T18:35:45.464 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1059999999999996e-06 2023-09-23T18:35:45.464 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.464 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.464 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:35:45.465 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.094785+0000", 2023-09-23T18:35:45.465 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1035000000000001e-05 2023-09-23T18:35:45.465 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.465 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.465 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:35:45.465 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.094846+0000", 2023-09-23T18:35:45.465 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018189200000000001 2023-09-23T18:35:45.465 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.465 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.465 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:35:45.465 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.095028+0000", 2023-09-23T18:35:45.465 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00020466700000000001 2023-09-23T18:35:45.465 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.466 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.466 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:35:45.466 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.095232+0000", 2023-09-23T18:35:45.466 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00530146 2023-09-23T18:35:45.466 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.466 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.466 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:35:45.466 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.100534+0000", 2023-09-23T18:35:45.466 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0080723789999999993 2023-09-23T18:35:45.466 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.466 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.466 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.467 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.108606+0000", 2023-09-23T18:35:45.467 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.171796375 2023-09-23T18:35:45.467 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.467 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.467 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.467 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.280402+0000", 2023-09-23T18:35:45.467 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.2284e-05 2023-09-23T18:35:45.467 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.467 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.467 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:35:45.467 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.280485+0000", 2023-09-23T18:35:45.467 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002478811 2023-09-23T18:35:45.467 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.468 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.468 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:35:45.468 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.282964+0000", 2023-09-23T18:35:45.468 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.18820453700000001 2023-09-23T18:35:45.468 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.468 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:35:45.468 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.468 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.468 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.468 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:168493 5.2 5:4eeb7bcf:::benchmark_data_smithi098_105788_object62208:head [delete] snapc 0=[] ondisk+write+known_if_redirected e130)", 2023-09-23T18:35:45.468 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:35:01.104204+0000", 2023-09-23T18:35:45.468 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 44.214816155000001, 2023-09-23T18:35:45.469 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.19987765499999999, 2023-09-23T18:35:45.469 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:35:45.469 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:35:45.469 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:35:45.469 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:35:45.469 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:35:45.469 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 168493 2023-09-23T18:35:45.469 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.469 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:35:45.469 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.469 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:35:45.469 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.104204+0000", 2023-09-23T18:35:45.469 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:35:45.470 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.470 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.470 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:35:45.470 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.104204+0000", 2023-09-23T18:35:45.470 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:35:45.470 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.470 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.470 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:35:45.470 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.104202+0000", 2023-09-23T18:35:45.470 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0049e-05 2023-09-23T18:35:45.470 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.470 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.471 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:35:45.471 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.104212+0000", 2023-09-23T18:35:45.471 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9366999999999999e-05 2023-09-23T18:35:45.471 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.471 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.471 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:35:45.471 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.104231+0000", 2023-09-23T18:35:45.471 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.1999999999999994e-06 2023-09-23T18:35:45.471 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.471 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.471 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:35:45.471 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.104239+0000", 2023-09-23T18:35:45.472 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.9466000000000003e-05 2023-09-23T18:35:45.472 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.472 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.472 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:35:45.472 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.104299+0000", 2023-09-23T18:35:45.472 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00017616499999999999 2023-09-23T18:35:45.472 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.472 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.472 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:35:45.472 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.104475+0000", 2023-09-23T18:35:45.472 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00024692300000000002 2023-09-23T18:35:45.472 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.472 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.473 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:35:45.473 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.104722+0000", 2023-09-23T18:35:45.473 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0040117720000000003 2023-09-23T18:35:45.473 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.473 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.473 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:35:45.473 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.108733+0000", 2023-09-23T18:35:45.473 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00540837 2023-09-23T18:35:45.473 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.473 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.473 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.473 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.114142+0000", 2023-09-23T18:35:45.473 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.18729225899999999 2023-09-23T18:35:45.474 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.474 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.474 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.474 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.301434+0000", 2023-09-23T18:35:45.474 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.5202000000000003e-05 2023-09-23T18:35:45.474 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.474 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.474 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:35:45.474 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.301509+0000", 2023-09-23T18:35:45.474 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0025718889999999999 2023-09-23T18:35:45.474 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.474 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.474 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:35:45.475 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.304081+0000", 2023-09-23T18:35:45.475 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.19987765499999999 2023-09-23T18:35:45.475 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.475 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:35:45.475 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.475 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.475 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.475 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:168506 5.2 5:4d771d12:::benchmark_data_smithi098_105788_object62221:head [delete] snapc 0=[] ondisk+write+known_if_redirected e130)", 2023-09-23T18:35:45.475 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:35:01.148585+0000", 2023-09-23T18:35:45.475 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 44.170434946999997, 2023-09-23T18:35:45.475 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15788432499999999, 2023-09-23T18:35:45.475 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:35:45.475 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:35:45.476 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:35:45.476 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:35:45.476 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:35:45.476 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 168506 2023-09-23T18:35:45.476 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.476 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:35:45.476 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.476 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:35:45.476 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.148585+0000", 2023-09-23T18:35:45.476 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:35:45.476 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.476 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.476 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:35:45.477 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.148585+0000", 2023-09-23T18:35:45.477 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:35:45.477 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.477 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.477 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:35:45.477 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.148583+0000", 2023-09-23T18:35:45.477 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.2010000000000002e-06 2023-09-23T18:35:45.477 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.477 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.477 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:35:45.477 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.148591+0000", 2023-09-23T18:35:45.477 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2913e-05 2023-09-23T18:35:45.478 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.478 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.478 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:35:45.478 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.148603+0000", 2023-09-23T18:35:45.478 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.9699999999999996e-06 2023-09-23T18:35:45.478 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.478 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.478 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:35:45.478 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.148609+0000", 2023-09-23T18:35:45.478 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1845000000000003e-05 2023-09-23T18:35:45.478 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.478 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.478 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:35:45.479 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.148661+0000", 2023-09-23T18:35:45.479 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000131798 2023-09-23T18:35:45.479 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.479 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.479 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:35:45.479 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.148793+0000", 2023-09-23T18:35:45.479 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014041499999999999 2023-09-23T18:35:45.479 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.479 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.479 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:35:45.479 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.148934+0000", 2023-09-23T18:35:45.479 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0034199009999999999 2023-09-23T18:35:45.479 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.480 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.480 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:35:45.480 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.152353+0000", 2023-09-23T18:35:45.480 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00064117999999999998 2023-09-23T18:35:45.480 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.480 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.480 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.480 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.152995+0000", 2023-09-23T18:35:45.480 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15117538999999999 2023-09-23T18:35:45.480 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.480 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.480 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:35:45.480 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.304170+0000", 2023-09-23T18:35:45.481 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.7803000000000001e-05 2023-09-23T18:35:45.481 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.481 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.481 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:35:45.481 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.304238+0000", 2023-09-23T18:35:45.481 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002231317 2023-09-23T18:35:45.481 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:35:45.481 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:35:45.481 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:35:45.481 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:01.306469+0000", 2023-09-23T18:35:45.481 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15788432499999999 2023-09-23T18:35:45.481 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.482 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:35:45.482 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.482 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:35:45.482 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:35:45.482 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:35:45.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:45.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:45.520 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:35:45.561 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:35:45.584 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:45.622 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:45.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:45.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:45.632 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:35:45.670 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:35:45.703 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:35:45.728 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:45.740 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:35:45.753 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:45.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:45.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:45.777 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:35:45.817 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:35:45.831 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:35:45.831 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:35:45.831 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:35:45.831 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:35:45.831 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:35:45.831 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:35:45.831 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:35:45.831 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:35:45.856 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:45.878 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:45.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:45.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:45.955 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is set 2023-09-23T18:35:45.981 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:46.018 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:46.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:46.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:46.125 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:46.139 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:45 smithi098 bash[76817]: audit 2023-09-23T18:35:45.470063+0000 mon.b (mon.1) 10 : audit 0 from='client.? 172.21.15.98:0/1691290966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:35:46.139 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:45 smithi098 bash[76817]: audit 2023-09-23T18:35:45.857091+0000 mon.a (mon.0) 724 : audit 1 from='client.? 172.21.15.98:0/2890960231' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:35:46.139 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:35:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:35:46] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.33.4" 2023-09-23T18:35:46.139 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:45 smithi098 bash[79516]: audit 2023-09-23T18:35:45.470063+0000 mon.b (mon.1) 10 : audit 0 from='client.? 172.21.15.98:0/1691290966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:35:46.140 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:45 smithi098 bash[79516]: audit 2023-09-23T18:35:45.857091+0000 mon.a (mon.0) 724 : audit 1 from='client.? 172.21.15.98:0/2890960231' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:35:46.161 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:46.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:46.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:46.266 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:46.275 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:45 smithi173 bash[59422]: audit 2023-09-23T18:35:45.470063+0000 mon.b (mon.1) 10 : audit 0 from='client.? 172.21.15.98:0/1691290966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:35:46.275 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:45 smithi173 bash[59422]: audit 2023-09-23T18:35:45.857091+0000 mon.a (mon.0) 724 : audit 1 from='client.? 172.21.15.98:0/2890960231' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:35:46.291 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:46.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:46.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:46.394 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:46.429 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:46.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:46.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:46.499 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T18:35:46.533 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:46.579 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:46.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:46.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:46.682 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:46.707 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:46.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:46.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:46.816 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:46.841 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:46.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:46.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:46.945 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:46.959 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:46 smithi173 bash[59422]: cluster 2023-09-23T18:35:45.790008+0000 mgr.y (mgr.26054) 515 : cluster 0 pgmap v476: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 42226/66702 objects degraded (63.305%); 1930/66702 objects misplaced (2.893%); 4.6 MiB/s, 76 objects/s recovering 2023-09-23T18:35:46.960 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:46 smithi173 bash[59422]: cluster 2023-09-23T18:35:45.934118+0000 mon.a (mon.0) 725 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:35:46.960 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:46 smithi173 bash[59422]: cluster 2023-09-23T18:35:45.943554+0000 osd.5 (osd.5) 46 : cluster 3 failed to encode map e135 with expected crc 2023-09-23T18:35:46.960 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:46 smithi173 bash[59422]: audit 2023-09-23T18:35:45.947016+0000 mon.a (mon.0) 726 : audit 1 from='client.? 172.21.15.98:0/2890960231' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:35:46.960 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:46 smithi173 bash[59422]: cluster 2023-09-23T18:35:45.947766+0000 mon.a (mon.0) 727 : cluster 0 osdmap e135: 8 total, 8 up, 7 in 2023-09-23T18:35:46.960 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:46 smithi173 bash[59422]: cluster 2023-09-23T18:35:45.950418+0000 osd.4 (osd.4) 69 : cluster 3 failed to encode map e135 with expected crc 2023-09-23T18:35:46.961 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:46 smithi173 bash[59422]: cluster 2023-09-23T18:35:45.954871+0000 osd.4 (osd.4) 70 : cluster 3 failed to encode map e135 with expected crc 2023-09-23T18:35:46.961 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:46 smithi173 bash[59422]: cluster 2023-09-23T18:35:45.959717+0000 osd.7 (osd.7) 45 : cluster 3 failed to encode map e135 with expected crc 2023-09-23T18:35:46.961 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:46 smithi173 bash[59422]: cluster 2023-09-23T18:35:45.965550+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e135 with expected crc 2023-09-23T18:35:46.961 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:46 smithi173 bash[59422]: audit 2023-09-23T18:35:45.969495+0000 mon.a (mon.0) 728 : audit 1 from='client.? 172.21.15.98:0/2860119845' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T18:35:46.973 INFO:teuthology.orchestra.run.smithi098.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T18:35:47.000 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:47.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:47.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:47.105 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:47.136 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:47.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:47.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:47.238 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:47.278 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:47.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:47.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:47.380 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:47.405 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:47.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:47.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:47.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:46 smithi098 bash[79516]: cluster 2023-09-23T18:35:45.790008+0000 mgr.y (mgr.26054) 515 : cluster 0 pgmap v476: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 42226/66702 objects degraded (63.305%); 1930/66702 objects misplaced (2.893%); 4.6 MiB/s, 76 objects/s recovering 2023-09-23T18:35:47.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:46 smithi098 bash[79516]: cluster 2023-09-23T18:35:45.934118+0000 mon.a (mon.0) 725 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:35:47.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:46 smithi098 bash[79516]: cluster 2023-09-23T18:35:45.943554+0000 osd.5 (osd.5) 46 : cluster 3 failed to encode map e135 with expected crc 2023-09-23T18:35:47.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:46 smithi098 bash[79516]: audit 2023-09-23T18:35:45.947016+0000 mon.a (mon.0) 726 : audit 1 from='client.? 172.21.15.98:0/2890960231' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:35:47.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:46 smithi098 bash[79516]: cluster 2023-09-23T18:35:45.947766+0000 mon.a (mon.0) 727 : cluster 0 osdmap e135: 8 total, 8 up, 7 in 2023-09-23T18:35:47.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:46 smithi098 bash[79516]: cluster 2023-09-23T18:35:45.950418+0000 osd.4 (osd.4) 69 : cluster 3 failed to encode map e135 with expected crc 2023-09-23T18:35:47.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:46 smithi098 bash[79516]: cluster 2023-09-23T18:35:45.954871+0000 osd.4 (osd.4) 70 : cluster 3 failed to encode map e135 with expected crc 2023-09-23T18:35:47.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:46 smithi098 bash[79516]: cluster 2023-09-23T18:35:45.959717+0000 osd.7 (osd.7) 45 : cluster 3 failed to encode map e135 with expected crc 2023-09-23T18:35:47.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:46 smithi098 bash[79516]: cluster 2023-09-23T18:35:45.965550+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e135 with expected crc 2023-09-23T18:35:47.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:46 smithi098 bash[79516]: audit 2023-09-23T18:35:45.969495+0000 mon.a (mon.0) 728 : audit 1 from='client.? 172.21.15.98:0/2860119845' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T18:35:47.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:46 smithi098 bash[76817]: cluster 2023-09-23T18:35:45.790008+0000 mgr.y (mgr.26054) 515 : cluster 0 pgmap v476: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 42226/66702 objects degraded (63.305%); 1930/66702 objects misplaced (2.893%); 4.6 MiB/s, 76 objects/s recovering 2023-09-23T18:35:47.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:46 smithi098 bash[76817]: cluster 2023-09-23T18:35:45.934118+0000 mon.a (mon.0) 725 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:35:47.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:46 smithi098 bash[76817]: cluster 2023-09-23T18:35:45.943554+0000 osd.5 (osd.5) 46 : cluster 3 failed to encode map e135 with expected crc 2023-09-23T18:35:47.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:46 smithi098 bash[76817]: audit 2023-09-23T18:35:45.947016+0000 mon.a (mon.0) 726 : audit 1 from='client.? 172.21.15.98:0/2890960231' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:35:47.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:46 smithi098 bash[76817]: cluster 2023-09-23T18:35:45.947766+0000 mon.a (mon.0) 727 : cluster 0 osdmap e135: 8 total, 8 up, 7 in 2023-09-23T18:35:47.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:46 smithi098 bash[76817]: cluster 2023-09-23T18:35:45.950418+0000 osd.4 (osd.4) 69 : cluster 3 failed to encode map e135 with expected crc 2023-09-23T18:35:47.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:46 smithi098 bash[76817]: cluster 2023-09-23T18:35:45.954871+0000 osd.4 (osd.4) 70 : cluster 3 failed to encode map e135 with expected crc 2023-09-23T18:35:47.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:46 smithi098 bash[76817]: cluster 2023-09-23T18:35:45.959717+0000 osd.7 (osd.7) 45 : cluster 3 failed to encode map e135 with expected crc 2023-09-23T18:35:47.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:46 smithi098 bash[76817]: cluster 2023-09-23T18:35:45.965550+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e135 with expected crc 2023-09-23T18:35:47.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:46 smithi098 bash[76817]: audit 2023-09-23T18:35:45.969495+0000 mon.a (mon.0) 728 : audit 1 from='client.? 172.21.15.98:0/2860119845' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T18:35:47.508 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:47.535 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:47.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:47.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:47.640 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:47.673 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:47.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:47.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:47.781 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:47.812 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:47.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:47.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:47.917 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:47.945 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:47.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:47.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:48.052 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:48.093 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:48.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:48.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:48.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:47 smithi098 bash[76817]: cluster 2023-09-23T18:35:46.945801+0000 osd.7 (osd.7) 46 : cluster 3 failed to encode map e136 with expected crc 2023-09-23T18:35:48.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:47 smithi098 bash[76817]: cluster 2023-09-23T18:35:46.949693+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e136 with expected crc 2023-09-23T18:35:48.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:47 smithi098 bash[76817]: audit 2023-09-23T18:35:46.950511+0000 mon.a (mon.0) 729 : audit 1 from='client.? 172.21.15.98:0/2860119845' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T18:35:48.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:47 smithi098 bash[76817]: cluster 2023-09-23T18:35:46.950637+0000 mon.a (mon.0) 730 : cluster 0 osdmap e136: 8 total, 8 up, 7 in 2023-09-23T18:35:48.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:47 smithi098 bash[76817]: cluster 2023-09-23T18:35:47.790694+0000 mgr.y (mgr.26054) 516 : cluster 0 pgmap v479: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 42000/64704 objects degraded (64.911%); 1930/64704 objects misplaced (2.983%); 2.7 MiB/s, 75 objects/s recovering 2023-09-23T18:35:48.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:47 smithi098 bash[79516]: cluster 2023-09-23T18:35:46.945801+0000 osd.7 (osd.7) 46 : cluster 3 failed to encode map e136 with expected crc 2023-09-23T18:35:48.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:47 smithi098 bash[79516]: cluster 2023-09-23T18:35:46.949693+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e136 with expected crc 2023-09-23T18:35:48.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:47 smithi098 bash[79516]: audit 2023-09-23T18:35:46.950511+0000 mon.a (mon.0) 729 : audit 1 from='client.? 172.21.15.98:0/2860119845' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T18:35:48.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:47 smithi098 bash[79516]: cluster 2023-09-23T18:35:46.950637+0000 mon.a (mon.0) 730 : cluster 0 osdmap e136: 8 total, 8 up, 7 in 2023-09-23T18:35:48.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:47 smithi098 bash[79516]: cluster 2023-09-23T18:35:47.790694+0000 mgr.y (mgr.26054) 516 : cluster 0 pgmap v479: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 42000/64704 objects degraded (64.911%); 1930/64704 objects misplaced (2.983%); 2.7 MiB/s, 75 objects/s recovering 2023-09-23T18:35:48.196 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:48.208 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:47 smithi173 bash[59422]: cluster 2023-09-23T18:35:46.945801+0000 osd.7 (osd.7) 46 : cluster 3 failed to encode map e136 with expected crc 2023-09-23T18:35:48.209 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:47 smithi173 bash[59422]: cluster 2023-09-23T18:35:46.949693+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e136 with expected crc 2023-09-23T18:35:48.209 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:47 smithi173 bash[59422]: audit 2023-09-23T18:35:46.950511+0000 mon.a (mon.0) 729 : audit 1 from='client.? 172.21.15.98:0/2860119845' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T18:35:48.209 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:47 smithi173 bash[59422]: cluster 2023-09-23T18:35:46.950637+0000 mon.a (mon.0) 730 : cluster 0 osdmap e136: 8 total, 8 up, 7 in 2023-09-23T18:35:48.210 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:47 smithi173 bash[59422]: cluster 2023-09-23T18:35:47.790694+0000 mgr.y (mgr.26054) 516 : cluster 0 pgmap v479: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 42000/64704 objects degraded (64.911%); 1930/64704 objects misplaced (2.983%); 2.7 MiB/s, 75 objects/s recovering 2023-09-23T18:35:48.227 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:48.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:48.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:48.330 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:48.364 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:48.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:48.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:48.468 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:48.492 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:48.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:48.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:48.597 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:48.632 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:48.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:48.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:48.738 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:48.787 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:48.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:48.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:48.890 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:48.912 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T18:35:48.919 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:48.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:48.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:49.030 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:49.066 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:49.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:49.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:49.171 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:49.206 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:49.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:49.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:49.314 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:49.354 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:49.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:49.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:49.457 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:49.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:49.484 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:49.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:49.585 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:49.594 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:49 smithi098 bash[76817]: cluster 2023-09-23T18:35:49.485202+0000 mon.a (mon.0) 731 : cluster 3 Health check update: Degraded data redundancy: 42000/64704 objects degraded (64.911%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:49.594 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:49 smithi098 bash[79516]: cluster 2023-09-23T18:35:49.485202+0000 mon.a (mon.0) 731 : cluster 3 Health check update: Degraded data redundancy: 42000/64704 objects degraded (64.911%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:49.609 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:49.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:49.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:49.716 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:49.725 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:49 smithi173 bash[59422]: cluster 2023-09-23T18:35:49.485202+0000 mon.a (mon.0) 731 : cluster 3 Health check update: Degraded data redundancy: 42000/64704 objects degraded (64.911%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:49.743 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:49.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:49.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:49.854 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:49.881 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:49.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:49.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:49.987 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:50.020 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:50.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:50.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:50.129 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:50.154 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:50.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:50.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:50.258 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:50.287 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:50.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:50.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:50.391 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:50.425 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:50.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:50.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:50.531 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:50.554 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:50.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:50.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:50.662 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:50.693 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:35:50.696 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:50.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:50.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:50.704 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:35:50.776 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:50 smithi173 bash[59422]: cluster 2023-09-23T18:35:49.791400+0000 mgr.y (mgr.26054) 517 : cluster 0 pgmap v480: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 41513/60261 objects degraded (68.889%); 1860/60261 objects misplaced (3.087%); 1.8 MiB/s, 57 objects/s recovering 2023-09-23T18:35:50.804 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:50.818 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:50 smithi098 bash[76817]: cluster 2023-09-23T18:35:49.791400+0000 mgr.y (mgr.26054) 517 : cluster 0 pgmap v480: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 41513/60261 objects degraded (68.889%); 1860/60261 objects misplaced (3.087%); 1.8 MiB/s, 57 objects/s recovering 2023-09-23T18:35:50.819 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:50 smithi098 bash[79516]: cluster 2023-09-23T18:35:49.791400+0000 mgr.y (mgr.26054) 517 : cluster 0 pgmap v480: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 41513/60261 objects degraded (68.889%); 1860/60261 objects misplaced (3.087%); 1.8 MiB/s, 57 objects/s recovering 2023-09-23T18:35:50.844 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:50.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:50.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:50.950 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:50.976 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:50.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:50.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:51.079 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:51.130 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:51.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:51.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:51.234 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:51.269 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:51.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:51.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:51.290 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:35:51.376 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:51.399 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:51.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:51.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:51.505 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:51.533 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:51.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:51.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:51.639 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:51.664 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:51.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:51.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:51.769 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:51.800 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:51.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:51.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:51.905 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:51.941 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:51.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:51.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:52.049 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:52.101 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:52.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:52.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:52.209 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:52.271 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:52.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:52.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:52.376 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:52.396 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:52.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:52.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:52.501 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:52.536 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:52.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:52.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:52.638 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:52.687 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:52.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:52.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:52.791 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:52.825 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:52.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:52.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:52.927 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:52.936 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:52 smithi098 bash[79516]: cluster 2023-09-23T18:35:51.792163+0000 mgr.y (mgr.26054) 518 : cluster 0 pgmap v481: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 41273/56619 objects degraded (72.896%); 1860/56619 objects misplaced (3.285%); 1.8 MiB/s, 87 objects/s recovering 2023-09-23T18:35:52.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:52 smithi098 bash[76817]: cluster 2023-09-23T18:35:51.792163+0000 mgr.y (mgr.26054) 518 : cluster 0 pgmap v481: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 41273/56619 objects degraded (72.896%); 1860/56619 objects misplaced (3.285%); 1.8 MiB/s, 87 objects/s recovering 2023-09-23T18:35:52.954 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:52.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:52.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:53.057 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:53.064 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:52 smithi173 bash[59422]: cluster 2023-09-23T18:35:51.792163+0000 mgr.y (mgr.26054) 518 : cluster 0 pgmap v481: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 41273/56619 objects degraded (72.896%); 1860/56619 objects misplaced (3.285%); 1.8 MiB/s, 87 objects/s recovering 2023-09-23T18:35:53.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:53.087 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:53.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:53.188 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:53.218 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:53.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:53.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:53.325 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:53.352 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:53.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:53.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:53.455 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:53.483 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:53.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:53.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:53.589 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:53.619 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:53.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:53.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:53.726 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:53.753 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:53.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:53.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:53.761 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:35:53.857 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:53.881 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:53.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:53.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:53.985 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:54.017 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:54.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:54.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:54.125 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:54.153 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:54.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:54.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:54.259 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:54.301 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:54.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:54.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:54.405 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:54.429 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:54.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:54.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:54.533 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:54.563 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:54.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:54.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:54.668 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:54.692 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:54.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:54.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:54.784 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:35:54.785 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:35:54.796 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:54.825 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:54.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:54.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:54.930 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:54.938 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:54 smithi173 bash[59422]: cluster 2023-09-23T18:35:53.792865+0000 mgr.y (mgr.26054) 519 : cluster 0 pgmap v482: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 41273/53985 objects degraded (76.453%); 1860/53985 objects misplaced (3.445%); 560 KiB/s, 66 objects/s recovering 2023-09-23T18:35:54.938 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:54 smithi173 bash[59422]: cluster 2023-09-23T18:35:54.486364+0000 mon.a (mon.0) 732 : cluster 3 Health check update: Degraded data redundancy: 41273/53985 objects degraded (76.453%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:54.938 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:54 smithi173 bash[59422]: audit 2023-09-23T18:35:54.589624+0000 mon.a (mon.0) 733 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:35:54.938 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:54 smithi173 bash[59422]: audit 2023-09-23T18:35:54.590766+0000 mon.a (mon.0) 734 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:35:54.966 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:54.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:54.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:55.072 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:55.108 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:55.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:55.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:55.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:54 smithi098 bash[76817]: cluster 2023-09-23T18:35:53.792865+0000 mgr.y (mgr.26054) 519 : cluster 0 pgmap v482: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 41273/53985 objects degraded (76.453%); 1860/53985 objects misplaced (3.445%); 560 KiB/s, 66 objects/s recovering 2023-09-23T18:35:55.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:54 smithi098 bash[76817]: cluster 2023-09-23T18:35:54.486364+0000 mon.a (mon.0) 732 : cluster 3 Health check update: Degraded data redundancy: 41273/53985 objects degraded (76.453%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:55.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:54 smithi098 bash[76817]: audit 2023-09-23T18:35:54.589624+0000 mon.a (mon.0) 733 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:35:55.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:54 smithi098 bash[76817]: audit 2023-09-23T18:35:54.590766+0000 mon.a (mon.0) 734 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:35:55.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:54 smithi098 bash[79516]: cluster 2023-09-23T18:35:53.792865+0000 mgr.y (mgr.26054) 519 : cluster 0 pgmap v482: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 41273/53985 objects degraded (76.453%); 1860/53985 objects misplaced (3.445%); 560 KiB/s, 66 objects/s recovering 2023-09-23T18:35:55.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:54 smithi098 bash[79516]: cluster 2023-09-23T18:35:54.486364+0000 mon.a (mon.0) 732 : cluster 3 Health check update: Degraded data redundancy: 41273/53985 objects degraded (76.453%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:55.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:54 smithi098 bash[79516]: audit 2023-09-23T18:35:54.589624+0000 mon.a (mon.0) 733 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:35:55.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:54 smithi098 bash[79516]: audit 2023-09-23T18:35:54.590766+0000 mon.a (mon.0) 734 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:35:55.213 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:55.245 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:55.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:55.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:55.352 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:55.376 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:55.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:55.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:55.480 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:55.509 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:55.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:55.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:55.613 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:55.634 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:55.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:55.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:55.738 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:55.761 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:55.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:55.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:55.866 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:55.907 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:55.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:55.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:56.012 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:56.037 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:56.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:56.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:56.140 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:56.172 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:56.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:56.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:56.193 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T18:35:56.193 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:35:56.280 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:56.312 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:56.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:56.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:56.417 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:56.426 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:35:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:35:56] "GET /metrics HTTP/1.1" 200 36537 "" "Prometheus/2.33.4" 2023-09-23T18:35:56.445 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:56.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:56.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:56.549 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:56.581 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:56.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:56.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:56.685 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:56.712 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:56.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:56.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:56.821 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:56.846 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:56.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:56.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:56.951 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:56.976 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:56.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:56.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:57.084 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:57.107 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:57.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:57.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:57.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:56 smithi098 bash[79516]: cluster 2023-09-23T18:35:55.793647+0000 mgr.y (mgr.26054) 520 : cluster 0 pgmap v483: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 40748/50529 objects degraded (80.643%); 1860/50529 objects misplaced (3.681%); 455 KiB/s, 54 objects/s recovering 2023-09-23T18:35:57.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:56 smithi098 bash[76817]: cluster 2023-09-23T18:35:55.793647+0000 mgr.y (mgr.26054) 520 : cluster 0 pgmap v483: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 40748/50529 objects degraded (80.643%); 1860/50529 objects misplaced (3.681%); 455 KiB/s, 54 objects/s recovering 2023-09-23T18:35:57.210 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:57.249 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:57.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:57.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:57.254 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T18:35:57.255 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:56 smithi173 bash[59422]: cluster 2023-09-23T18:35:55.793647+0000 mgr.y (mgr.26054) 520 : cluster 0 pgmap v483: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 40748/50529 objects degraded (80.643%); 1860/50529 objects misplaced (3.681%); 455 KiB/s, 54 objects/s recovering 2023-09-23T18:35:57.354 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:57.375 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:57.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:57.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:57.478 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:57.509 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:57.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:57.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:57.617 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:57.649 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:57.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:57.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:57.751 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:57.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:57.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:57.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:35:57.891 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:57.936 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:35:57.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:57.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:35:58.040 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:58.065 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:58.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:58.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:58.168 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:58.206 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:35:58.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:58.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:35:58.311 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:58.341 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:58.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:58.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:58.448 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:58.488 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:58.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:58.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:58.591 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:58.633 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:58.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:58.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:58.737 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:58.765 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:58.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:58.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:58.868 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:58.908 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:58.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:58.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:58.915 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:58 smithi098 bash[76817]: cluster 2023-09-23T18:35:57.794653+0000 mgr.y (mgr.26054) 521 : cluster 0 pgmap v484: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1011 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 40482/48393 objects degraded (83.653%); 1860/48393 objects misplaced (3.844%); 413 KiB/s, 53 objects/s recovering 2023-09-23T18:35:58.916 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:58 smithi098 bash[79516]: cluster 2023-09-23T18:35:57.794653+0000 mgr.y (mgr.26054) 521 : cluster 0 pgmap v484: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1011 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 40482/48393 objects degraded (83.653%); 1860/48393 objects misplaced (3.844%); 413 KiB/s, 53 objects/s recovering 2023-09-23T18:35:59.015 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:59.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:58 smithi173 bash[59422]: cluster 2023-09-23T18:35:57.794653+0000 mgr.y (mgr.26054) 521 : cluster 0 pgmap v484: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 1011 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 40482/48393 objects degraded (83.653%); 1860/48393 objects misplaced (3.844%); 413 KiB/s, 53 objects/s recovering 2023-09-23T18:35:59.053 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:35:59.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:59.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:35:59.161 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:59.184 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:35:59.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:59.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:35:59.292 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:59.321 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:59.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:59.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:59.428 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:59.453 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:35:59.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:59.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:35:59.559 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:59.592 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:35:59.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:59.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:35:59.696 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:59.721 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:59.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:59.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:59.827 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:59.861 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:35:59.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:59.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:35:59.965 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:59.973 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:35:59 smithi173 bash[59422]: cluster 2023-09-23T18:35:59.487351+0000 mon.a (mon.0) 735 : cluster 3 Health check update: Degraded data redundancy: 40482/48393 objects degraded (83.653%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:35:59.988 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:35:59.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:35:59.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:00.092 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:00.099 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:35:59 smithi098 bash[76817]: cluster 2023-09-23T18:35:59.487351+0000 mon.a (mon.0) 735 : cluster 3 Health check update: Degraded data redundancy: 40482/48393 objects degraded (83.653%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:00.099 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:35:59 smithi098 bash[79516]: cluster 2023-09-23T18:35:59.487351+0000 mon.a (mon.0) 735 : cluster 3 Health check update: Degraded data redundancy: 40482/48393 objects degraded (83.653%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:00.124 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:00.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:00.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:00.227 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:00.253 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:00.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:00.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:00.359 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:00.389 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:00.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:00.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:00.491 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:00.526 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:00.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:00.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:00.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:00.662 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:00.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:00.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:00.765 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:00.791 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:00.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:00.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:00.899 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:00.934 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:00.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:00.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:01.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:00 smithi173 bash[59422]: cluster 2023-09-23T18:35:59.795373+0000 mgr.y (mgr.26054) 522 : cluster 0 pgmap v485: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 913 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 39959/43704 objects degraded (91.431%); 1860/43704 objects misplaced (4.256%); 373 KiB/s, 47 objects/s recovering 2023-09-23T18:36:01.042 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:01.050 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:00 smithi098 bash[79516]: cluster 2023-09-23T18:35:59.795373+0000 mgr.y (mgr.26054) 522 : cluster 0 pgmap v485: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 913 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 39959/43704 objects degraded (91.431%); 1860/43704 objects misplaced (4.256%); 373 KiB/s, 47 objects/s recovering 2023-09-23T18:36:01.050 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:00 smithi098 bash[76817]: cluster 2023-09-23T18:35:59.795373+0000 mgr.y (mgr.26054) 522 : cluster 0 pgmap v485: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 913 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 39959/43704 objects degraded (91.431%); 1860/43704 objects misplaced (4.256%); 373 KiB/s, 47 objects/s recovering 2023-09-23T18:36:01.071 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:01.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:01.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:01.175 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:01.198 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:01.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:01.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:01.302 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:01.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:01.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:01.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:01.429 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:01.453 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:01.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:01.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:01.557 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:01.582 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:01.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:01.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:01.685 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:01.708 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:01.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:01.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:01.814 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:01.849 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:01.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:01.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:01.953 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:01.983 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:01.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:01.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:02.028 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:36:02.088 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:02.117 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:02.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:02.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:02.222 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:02.253 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:02.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:02.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:02.357 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:02.381 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:02.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:02.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:02.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:02.514 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:02.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:02.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:02.618 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:02.658 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:02.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:02.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:02.765 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:02.787 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:02.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:02.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:02.896 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:02.927 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:02.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:02.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:03.034 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:03.084 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:03.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:03.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:03.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:02 smithi098 bash[79516]: cluster 2023-09-23T18:36:01.796142+0000 mgr.y (mgr.26054) 523 : cluster 0 pgmap v486: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 841 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 39695/40245 objects degraded (98.633%); 1860/40245 objects misplaced (4.622%); 0 B/s, 64 objects/s recovering 2023-09-23T18:36:03.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:02 smithi098 bash[76817]: cluster 2023-09-23T18:36:01.796142+0000 mgr.y (mgr.26054) 523 : cluster 0 pgmap v486: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 841 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 39695/40245 objects degraded (98.633%); 1860/40245 objects misplaced (4.622%); 0 B/s, 64 objects/s recovering 2023-09-23T18:36:03.191 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:03.213 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:03.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:03.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:03.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:02 smithi173 bash[59422]: cluster 2023-09-23T18:36:01.796142+0000 mgr.y (mgr.26054) 523 : cluster 0 pgmap v486: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 841 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 39695/40245 objects degraded (98.633%); 1860/40245 objects misplaced (4.622%); 0 B/s, 64 objects/s recovering 2023-09-23T18:36:03.317 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:03.346 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:03.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:03.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:03.450 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:03.472 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:03.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:03.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:03.574 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:03.603 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:03.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:03.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:03.709 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:03.730 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:03.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:03.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:03.836 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:03.865 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:03.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:03.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:03.968 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:04.003 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:04.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:04.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:04.109 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:04.138 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:04.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:04.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:04.241 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:04.267 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:04.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:04.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:04.372 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:04.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:04.403 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:04.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:04.504 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:04.536 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:04.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:04.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:04.641 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:04.676 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:04.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:04.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:04.782 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:04.809 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:04.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:04.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:04.912 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:04.939 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:04.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:04.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:04.978 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:04 smithi098 bash[79516]: cluster 2023-09-23T18:36:03.796820+0000 mgr.y (mgr.26054) 524 : cluster 0 pgmap v487: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 787 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 526 op/s; 39695/37662 objects degraded (105.398%); 1860/37662 objects misplaced (4.939%); 0 B/s, 44 objects/s recovering 2023-09-23T18:36:04.979 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:04 smithi098 bash[79516]: cluster 2023-09-23T18:36:04.488521+0000 mon.a (mon.0) 736 : cluster 3 Health check update: Degraded data redundancy: 39695/37662 objects degraded (105.398%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:04.979 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:04 smithi098 bash[76817]: cluster 2023-09-23T18:36:03.796820+0000 mgr.y (mgr.26054) 524 : cluster 0 pgmap v487: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 787 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 526 op/s; 39695/37662 objects degraded (105.398%); 1860/37662 objects misplaced (4.939%); 0 B/s, 44 objects/s recovering 2023-09-23T18:36:04.979 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:04 smithi098 bash[76817]: cluster 2023-09-23T18:36:04.488521+0000 mon.a (mon.0) 736 : cluster 3 Health check update: Degraded data redundancy: 39695/37662 objects degraded (105.398%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:05.044 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:05.051 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:04 smithi173 bash[59422]: cluster 2023-09-23T18:36:03.796820+0000 mgr.y (mgr.26054) 524 : cluster 0 pgmap v487: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 787 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 526 op/s; 39695/37662 objects degraded (105.398%); 1860/37662 objects misplaced (4.939%); 0 B/s, 44 objects/s recovering 2023-09-23T18:36:05.051 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:04 smithi173 bash[59422]: cluster 2023-09-23T18:36:04.488521+0000 mon.a (mon.0) 736 : cluster 3 Health check update: Degraded data redundancy: 39695/37662 objects degraded (105.398%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:05.064 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:05.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:05.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:05.167 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:05.194 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:05.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:05.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:05.304 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:05.328 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:05.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:05.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:05.433 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:05.457 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:05.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:05.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:05.560 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:05.609 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:05.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:05.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:05.713 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:05.759 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:05.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:05.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:05.868 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:05.915 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:05.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:05.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:06.023 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:06.051 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:06.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:06.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:06.156 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:06.181 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:06.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:06.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:06.190 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:36:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:36:06] "GET /metrics HTTP/1.1" 200 36537 "" "Prometheus/2.33.4" 2023-09-23T18:36:06.283 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:06.311 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:06.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:06.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:06.416 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:06.441 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:06.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:06.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:06.544 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:06.582 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:06.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:06.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:06.644 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:36:06.688 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:06.695 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:36:06.727 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:06.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:06.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:06.738 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T18:36:06.738 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:36:06.747 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:36:06.791 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:36:06.824 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:36:06.835 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:06.860 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:06.860 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:36:06.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:06.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:06.899 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:36:06.903 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:06 smithi098 bash[76817]: cluster 2023-09-23T18:36:05.797521+0000 mgr.y (mgr.26054) 525 : cluster 0 pgmap v488: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 710 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 39182/33933 objects degraded (115.469%); 1860/33933 objects misplaced (5.481%); 0 B/s, 44 objects/s recovering 2023-09-23T18:36:06.903 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:06 smithi098 bash[76817]: audit 2023-09-23T18:36:05.940156+0000 mon.c (mon.2) 70 : audit 1 from='client.? 172.21.15.98:0/1006267648' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:36:06.903 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:06 smithi098 bash[76817]: audit 2023-09-23T18:36:05.941143+0000 mon.a (mon.0) 737 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:36:06.903 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:06 smithi098 bash[76817]: audit 2023-09-23T18:36:06.602846+0000 mon.b (mon.1) 11 : audit 0 from='client.? 172.21.15.98:0/2053576272' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:36:06.904 INFO:teuthology.orchestra.run.smithi098.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T18:36:06.933 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:36:06.951 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:36:06.952 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:36:06.952 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:36:06.952 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:36:06.952 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:36:06.952 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:36:06.952 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:36:06.952 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:36:06.963 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:06.997 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:07.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:07.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:07.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:06 smithi173 bash[59422]: cluster 2023-09-23T18:36:05.797521+0000 mgr.y (mgr.26054) 525 : cluster 0 pgmap v488: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 710 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 39182/33933 objects degraded (115.469%); 1860/33933 objects misplaced (5.481%); 0 B/s, 44 objects/s recovering 2023-09-23T18:36:07.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:06 smithi173 bash[59422]: audit 2023-09-23T18:36:05.940156+0000 mon.c (mon.2) 70 : audit 1 from='client.? 172.21.15.98:0/1006267648' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:36:07.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:06 smithi173 bash[59422]: audit 2023-09-23T18:36:05.941143+0000 mon.a (mon.0) 737 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:36:07.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:06 smithi173 bash[59422]: audit 2023-09-23T18:36:06.602846+0000 mon.b (mon.1) 11 : audit 0 from='client.? 172.21.15.98:0/2053576272' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:36:07.104 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:07.132 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:07.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:07.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:07.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:06 smithi098 bash[79516]: cluster 2023-09-23T18:36:05.797521+0000 mgr.y (mgr.26054) 525 : cluster 0 pgmap v488: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 710 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 39182/33933 objects degraded (115.469%); 1860/33933 objects misplaced (5.481%); 0 B/s, 44 objects/s recovering 2023-09-23T18:36:07.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:06 smithi098 bash[79516]: audit 2023-09-23T18:36:05.940156+0000 mon.c (mon.2) 70 : audit 1 from='client.? 172.21.15.98:0/1006267648' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:36:07.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:06 smithi098 bash[79516]: audit 2023-09-23T18:36:05.941143+0000 mon.a (mon.0) 737 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:36:07.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:06 smithi098 bash[79516]: audit 2023-09-23T18:36:06.602846+0000 mon.b (mon.1) 11 : audit 0 from='client.? 172.21.15.98:0/2053576272' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:36:07.235 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:07.260 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:07.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:07.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:07.367 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:07.410 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:07.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:07.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:07.514 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:07.545 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:07.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:07.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:07.649 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:07.680 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:07.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:07.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:07.694 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T18:36:07.785 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:07.825 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:07.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:07.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:07.916 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:07 smithi098 bash[76817]: audit 2023-09-23T18:36:06.888894+0000 mon.a (mon.0) 738 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T18:36:07.916 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:07 smithi098 bash[76817]: cluster 2023-09-23T18:36:06.889080+0000 mon.a (mon.0) 739 : cluster 0 osdmap e137: 8 total, 8 up, 7 in 2023-09-23T18:36:07.929 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is set 2023-09-23T18:36:07.932 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:07.960 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:07.961 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:07 smithi173 bash[59422]: audit 2023-09-23T18:36:06.888894+0000 mon.a (mon.0) 738 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T18:36:07.961 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:07 smithi173 bash[59422]: cluster 2023-09-23T18:36:06.889080+0000 mon.a (mon.0) 739 : cluster 0 osdmap e137: 8 total, 8 up, 7 in 2023-09-23T18:36:07.961 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:07 smithi173 bash[59422]: cluster 2023-09-23T18:36:06.895494+0000 osd.4 (osd.4) 71 : cluster 3 failed to encode map e137 with expected crc 2023-09-23T18:36:07.961 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:07 smithi173 bash[59422]: cluster 2023-09-23T18:36:06.910163+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e137 with expected crc 2023-09-23T18:36:07.962 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:07 smithi173 bash[59422]: cluster 2023-09-23T18:36:06.926484+0000 osd.5 (osd.5) 47 : cluster 3 failed to encode map e137 with expected crc 2023-09-23T18:36:07.962 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:07 smithi173 bash[59422]: audit 2023-09-23T18:36:07.064240+0000 mon.a (mon.0) 740 : audit 1 from='client.? 172.21.15.98:0/3831011701' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:36:07.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:07.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:08.063 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:08.095 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:08.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:08.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:08.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:07 smithi098 bash[76817]: cluster 2023-09-23T18:36:06.895494+0000 osd.4 (osd.4) 71 : cluster 3 failed to encode map e137 with expected crc 2023-09-23T18:36:08.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:07 smithi098 bash[76817]: cluster 2023-09-23T18:36:06.910163+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e137 with expected crc 2023-09-23T18:36:08.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:07 smithi098 bash[76817]: cluster 2023-09-23T18:36:06.926484+0000 osd.5 (osd.5) 47 : cluster 3 failed to encode map e137 with expected crc 2023-09-23T18:36:08.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:07 smithi098 bash[76817]: audit 2023-09-23T18:36:07.064240+0000 mon.a (mon.0) 740 : audit 1 from='client.? 172.21.15.98:0/3831011701' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:36:08.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:07 smithi098 bash[79516]: audit 2023-09-23T18:36:06.888894+0000 mon.a (mon.0) 738 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T18:36:08.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:07 smithi098 bash[79516]: cluster 2023-09-23T18:36:06.889080+0000 mon.a (mon.0) 739 : cluster 0 osdmap e137: 8 total, 8 up, 7 in 2023-09-23T18:36:08.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:07 smithi098 bash[79516]: cluster 2023-09-23T18:36:06.895494+0000 osd.4 (osd.4) 71 : cluster 3 failed to encode map e137 with expected crc 2023-09-23T18:36:08.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:07 smithi098 bash[79516]: cluster 2023-09-23T18:36:06.910163+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e137 with expected crc 2023-09-23T18:36:08.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:07 smithi098 bash[79516]: cluster 2023-09-23T18:36:06.926484+0000 osd.5 (osd.5) 47 : cluster 3 failed to encode map e137 with expected crc 2023-09-23T18:36:08.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:07 smithi098 bash[79516]: audit 2023-09-23T18:36:07.064240+0000 mon.a (mon.0) 740 : audit 1 from='client.? 172.21.15.98:0/3831011701' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:36:08.199 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:08.224 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:08.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:08.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:08.327 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:08.347 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:08.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:08.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:08.450 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:08.481 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:08.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:08.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:08.590 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:08.616 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:08.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:08.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:08.719 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:08.752 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:08.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:08.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:08.855 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:08.879 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:08.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:08.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:08.982 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:09.004 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:09.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:09.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:09.107 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:09.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:09.142 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:09.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:09.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:08 smithi098 bash[79516]: cluster 2023-09-23T18:36:07.798110+0000 mgr.y (mgr.26054) 526 : cluster 0 pgmap v490: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 665 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 38909/31797 objects degraded (122.367%); 1860/31797 objects misplaced (5.850%); 0 B/s, 53 objects/s recovering 2023-09-23T18:36:09.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:08 smithi098 bash[79516]: cluster 2023-09-23T18:36:07.901852+0000 mon.a (mon.0) 741 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:36:09.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:08 smithi098 bash[79516]: cluster 2023-09-23T18:36:07.915444+0000 osd.5 (osd.5) 48 : cluster 3 failed to encode map e138 with expected crc 2023-09-23T18:36:09.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:08 smithi098 bash[79516]: audit 2023-09-23T18:36:07.921395+0000 mon.a (mon.0) 742 : audit 1 from='client.? 172.21.15.98:0/3831011701' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:36:09.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:08 smithi098 bash[79516]: cluster 2023-09-23T18:36:07.921478+0000 mon.a (mon.0) 743 : cluster 0 osdmap e138: 8 total, 8 up, 7 in 2023-09-23T18:36:09.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:08 smithi098 bash[79516]: cluster 2023-09-23T18:36:07.927583+0000 osd.7 (osd.7) 47 : cluster 3 failed to encode map e138 with expected crc 2023-09-23T18:36:09.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:08 smithi098 bash[79516]: cluster 2023-09-23T18:36:07.930009+0000 osd.4 (osd.4) 72 : cluster 3 failed to encode map e138 with expected crc 2023-09-23T18:36:09.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:08 smithi098 bash[79516]: cluster 2023-09-23T18:36:07.934236+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e138 with expected crc 2023-09-23T18:36:09.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:08 smithi098 bash[76817]: cluster 2023-09-23T18:36:07.798110+0000 mgr.y (mgr.26054) 526 : cluster 0 pgmap v490: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 665 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 38909/31797 objects degraded (122.367%); 1860/31797 objects misplaced (5.850%); 0 B/s, 53 objects/s recovering 2023-09-23T18:36:09.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:08 smithi098 bash[76817]: cluster 2023-09-23T18:36:07.901852+0000 mon.a (mon.0) 741 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:36:09.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:08 smithi098 bash[76817]: cluster 2023-09-23T18:36:07.915444+0000 osd.5 (osd.5) 48 : cluster 3 failed to encode map e138 with expected crc 2023-09-23T18:36:09.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:08 smithi098 bash[76817]: audit 2023-09-23T18:36:07.921395+0000 mon.a (mon.0) 742 : audit 1 from='client.? 172.21.15.98:0/3831011701' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:36:09.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:08 smithi098 bash[76817]: cluster 2023-09-23T18:36:07.921478+0000 mon.a (mon.0) 743 : cluster 0 osdmap e138: 8 total, 8 up, 7 in 2023-09-23T18:36:09.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:08 smithi098 bash[76817]: cluster 2023-09-23T18:36:07.927583+0000 osd.7 (osd.7) 47 : cluster 3 failed to encode map e138 with expected crc 2023-09-23T18:36:09.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:08 smithi098 bash[76817]: cluster 2023-09-23T18:36:07.930009+0000 osd.4 (osd.4) 72 : cluster 3 failed to encode map e138 with expected crc 2023-09-23T18:36:09.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:08 smithi098 bash[76817]: cluster 2023-09-23T18:36:07.934236+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e138 with expected crc 2023-09-23T18:36:09.243 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:09.251 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:08 smithi173 bash[59422]: cluster 2023-09-23T18:36:07.798110+0000 mgr.y (mgr.26054) 526 : cluster 0 pgmap v490: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 665 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 38909/31797 objects degraded (122.367%); 1860/31797 objects misplaced (5.850%); 0 B/s, 53 objects/s recovering 2023-09-23T18:36:09.251 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:08 smithi173 bash[59422]: cluster 2023-09-23T18:36:07.901852+0000 mon.a (mon.0) 741 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:36:09.251 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:08 smithi173 bash[59422]: cluster 2023-09-23T18:36:07.915444+0000 osd.5 (osd.5) 48 : cluster 3 failed to encode map e138 with expected crc 2023-09-23T18:36:09.251 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:08 smithi173 bash[59422]: audit 2023-09-23T18:36:07.921395+0000 mon.a (mon.0) 742 : audit 1 from='client.? 172.21.15.98:0/3831011701' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:36:09.252 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:08 smithi173 bash[59422]: cluster 2023-09-23T18:36:07.921478+0000 mon.a (mon.0) 743 : cluster 0 osdmap e138: 8 total, 8 up, 7 in 2023-09-23T18:36:09.252 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:08 smithi173 bash[59422]: cluster 2023-09-23T18:36:07.927583+0000 osd.7 (osd.7) 47 : cluster 3 failed to encode map e138 with expected crc 2023-09-23T18:36:09.252 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:08 smithi173 bash[59422]: cluster 2023-09-23T18:36:07.930009+0000 osd.4 (osd.4) 72 : cluster 3 failed to encode map e138 with expected crc 2023-09-23T18:36:09.252 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:08 smithi173 bash[59422]: cluster 2023-09-23T18:36:07.934236+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e138 with expected crc 2023-09-23T18:36:09.268 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:09.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:09.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:09.370 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:09.394 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:09.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:09.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:09.500 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:09.530 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:09.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:09.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:09.636 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:09.662 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:09.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:09.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:09.769 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:09.794 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:09.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:09.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:09.900 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:09.922 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:09.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:09.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:10.027 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:10.046 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T18:36:10.080 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:10.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:10.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:10.184 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:10.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:09 smithi098 bash[79516]: cluster 2023-09-23T18:36:09.490475+0000 mon.a (mon.0) 744 : cluster 3 Health check update: Degraded data redundancy: 38909/31797 objects degraded (122.367%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:10.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:09 smithi098 bash[79516]: audit 2023-09-23T18:36:09.584999+0000 mon.a (mon.0) 745 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:36:10.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:09 smithi098 bash[76817]: cluster 2023-09-23T18:36:09.490475+0000 mon.a (mon.0) 744 : cluster 3 Health check update: Degraded data redundancy: 38909/31797 objects degraded (122.367%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:10.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:09 smithi098 bash[76817]: audit 2023-09-23T18:36:09.584999+0000 mon.a (mon.0) 745 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:36:10.194 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:09 smithi173 bash[59422]: cluster 2023-09-23T18:36:09.490475+0000 mon.a (mon.0) 744 : cluster 3 Health check update: Degraded data redundancy: 38909/31797 objects degraded (122.367%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:10.194 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:09 smithi173 bash[59422]: audit 2023-09-23T18:36:09.584999+0000 mon.a (mon.0) 745 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:36:10.212 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:10.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:10.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:10.320 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:10.359 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:10.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:10.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:10.468 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:10.494 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:10.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:10.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:10.601 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:10.626 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:10.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:10.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:10.733 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:10.764 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:10.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:10.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:10.873 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:10.908 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:10.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:10.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:11.013 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:11.021 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:10 smithi173 bash[59422]: cluster 2023-09-23T18:36:09.798822+0000 mgr.y (mgr.26054) 527 : cluster 0 pgmap v492: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 571 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 539 op/s; 38396/27282 objects degraded (140.737%); 1860/27282 objects misplaced (6.818%); 0 B/s, 34 objects/s recovering 2023-09-23T18:36:11.036 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:11.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:11.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:11.140 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:11.176 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:10 smithi098 bash[79516]: cluster 2023-09-23T18:36:09.798822+0000 mgr.y (mgr.26054) 527 : cluster 0 pgmap v492: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 571 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 539 op/s; 38396/27282 objects degraded (140.737%); 1860/27282 objects misplaced (6.818%); 0 B/s, 34 objects/s recovering 2023-09-23T18:36:11.177 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:10 smithi098 bash[76817]: cluster 2023-09-23T18:36:09.798822+0000 mgr.y (mgr.26054) 527 : cluster 0 pgmap v492: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 571 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 539 op/s; 38396/27282 objects degraded (140.737%); 1860/27282 objects misplaced (6.818%); 0 B/s, 34 objects/s recovering 2023-09-23T18:36:11.205 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:11.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:11.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:11.311 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:11.335 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:11.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:11.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:11.440 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:11.466 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:11.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:11.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:11.572 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:11.602 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:11.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:11.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:11.705 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:11.727 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:11.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:11.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:11.830 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:11.858 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:11.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:11.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:11.962 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:11.991 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:11.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:11.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:12.058 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:36:12.096 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:12.145 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:12.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:12.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:12.248 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:12.253 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:36:12.272 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:12.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:12.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:12.375 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:12.402 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:12.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:12.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:12.505 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:12.519 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:36:12.555 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:12.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:12.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:12.660 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:12.698 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:12.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:12.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:12.801 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:12.834 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:12.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:12.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:12.927 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:12 smithi098 bash[76817]: cluster 2023-09-23T18:36:11.799614+0000 mgr.y (mgr.26054) 528 : cluster 0 pgmap v493: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 499 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 38110/23841 objects degraded (159.851%); 1860/23841 objects misplaced (7.802%); 0 B/s, 69 objects/s recovering 2023-09-23T18:36:12.938 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:13.000 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:13.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:13.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:13.106 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:13.118 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:12 smithi173 bash[59422]: cluster 2023-09-23T18:36:11.799614+0000 mgr.y (mgr.26054) 528 : cluster 0 pgmap v493: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 499 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 38110/23841 objects degraded (159.851%); 1860/23841 objects misplaced (7.802%); 0 B/s, 69 objects/s recovering 2023-09-23T18:36:13.118 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:12 smithi173 bash[59422]: audit 2023-09-23T18:36:12.736483+0000 mon.a (mon.0) 746 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:36:13.141 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:13.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:13.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:13.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:12 smithi098 bash[76817]: audit 2023-09-23T18:36:12.736483+0000 mon.a (mon.0) 746 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:36:13.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:12 smithi098 bash[79516]: cluster 2023-09-23T18:36:11.799614+0000 mgr.y (mgr.26054) 528 : cluster 0 pgmap v493: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 499 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 38110/23841 objects degraded (159.851%); 1860/23841 objects misplaced (7.802%); 0 B/s, 69 objects/s recovering 2023-09-23T18:36:13.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:12 smithi098 bash[79516]: audit 2023-09-23T18:36:12.736483+0000 mon.a (mon.0) 746 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:36:13.251 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:13.285 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:13.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:13.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:13.391 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:13.416 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:13.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:13.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:13.523 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:13.548 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:13.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:13.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:13.654 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:13.679 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:13.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:13.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:13.782 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:13.808 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:13.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:13.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:13.913 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:13.937 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:13.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:13.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:14.040 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:14.063 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:14.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:14.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:14.166 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:14.202 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:14.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:14.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:14.308 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:14.344 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:14.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:14.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:14.450 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:14.486 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:14.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:14.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:14.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:14 smithi173 bash[59422]: audit 2023-09-23T18:36:13.280014+0000 mon.a (mon.0) 747 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:14.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:14 smithi173 bash[59422]: audit 2023-09-23T18:36:13.298625+0000 mon.a (mon.0) 748 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:14.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:14 smithi173 bash[59422]: cluster 2023-09-23T18:36:13.800405+0000 mgr.y (mgr.26054) 529 : cluster 0 pgmap v494: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 449 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 38110/21405 objects degraded (178.043%); 1860/21405 objects misplaced (8.690%); 0 B/s, 69 objects/s recovering 2023-09-23T18:36:14.590 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:14.600 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:14 smithi098 bash[76817]: audit 2023-09-23T18:36:13.280014+0000 mon.a (mon.0) 747 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:14.600 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:14 smithi098 bash[76817]: audit 2023-09-23T18:36:13.298625+0000 mon.a (mon.0) 748 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:14.600 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:14 smithi098 bash[76817]: cluster 2023-09-23T18:36:13.800405+0000 mgr.y (mgr.26054) 529 : cluster 0 pgmap v494: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 449 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 38110/21405 objects degraded (178.043%); 1860/21405 objects misplaced (8.690%); 0 B/s, 69 objects/s recovering 2023-09-23T18:36:14.601 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:14 smithi098 bash[79516]: audit 2023-09-23T18:36:13.280014+0000 mon.a (mon.0) 747 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:14.601 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:14 smithi098 bash[79516]: audit 2023-09-23T18:36:13.298625+0000 mon.a (mon.0) 748 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:14.601 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:14 smithi098 bash[79516]: cluster 2023-09-23T18:36:13.800405+0000 mgr.y (mgr.26054) 529 : cluster 0 pgmap v494: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 449 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 38110/21405 objects degraded (178.043%); 1860/21405 objects misplaced (8.690%); 0 B/s, 69 objects/s recovering 2023-09-23T18:36:14.620 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:14.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:14.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:14.725 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:14.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:14.770 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:14.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:14.871 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:14.905 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:14.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:14.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:14.932 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:36:15.009 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:15.041 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:15.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:15.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:15.148 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:15.173 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:15.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:15.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:15.276 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:15.305 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:15.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:15.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:15.409 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:15.418 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:15 smithi173 bash[59422]: cluster 2023-09-23T18:36:14.492160+0000 mon.a (mon.0) 749 : cluster 3 Health check update: Degraded data redundancy: 38110/21405 objects degraded (178.043%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:15.435 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:15.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:15.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:15.538 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:15.565 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:15.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:15.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:15.669 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:15.678 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:15 smithi098 bash[76817]: cluster 2023-09-23T18:36:14.492160+0000 mon.a (mon.0) 749 : cluster 3 Health check update: Degraded data redundancy: 38110/21405 objects degraded (178.043%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:15.678 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:15 smithi098 bash[79516]: cluster 2023-09-23T18:36:14.492160+0000 mon.a (mon.0) 749 : cluster 3 Health check update: Degraded data redundancy: 38110/21405 objects degraded (178.043%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:15.696 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:15.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:15.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:15.799 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:15.829 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:15.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:15.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:15.937 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:15.958 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:15.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:15.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:16.062 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:16.105 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:16.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:16.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:16.215 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:16.237 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:16.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:16.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:16.313 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:36:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:36:16] "GET /metrics HTTP/1.1" 200 36544 "" "Prometheus/2.33.4" 2023-09-23T18:36:16.313 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:16 smithi098 bash[76817]: cluster 2023-09-23T18:36:15.801150+0000 mgr.y (mgr.26054) 530 : cluster 0 pgmap v495: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 378 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 37620/18024 objects degraded (208.722%); 1860/18024 objects misplaced (10.320%); 0 B/s, 32 objects/s recovering 2023-09-23T18:36:16.339 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:16.362 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:16.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:16.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:16.466 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:16.508 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:16.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:16.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:16.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:16 smithi173 bash[59422]: cluster 2023-09-23T18:36:15.801150+0000 mgr.y (mgr.26054) 530 : cluster 0 pgmap v495: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 378 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 37620/18024 objects degraded (208.722%); 1860/18024 objects misplaced (10.320%); 0 B/s, 32 objects/s recovering 2023-09-23T18:36:16.614 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:16.628 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:16 smithi098 bash[79516]: cluster 2023-09-23T18:36:15.801150+0000 mgr.y (mgr.26054) 530 : cluster 0 pgmap v495: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 378 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 37620/18024 objects degraded (208.722%); 1860/18024 objects misplaced (10.320%); 0 B/s, 32 objects/s recovering 2023-09-23T18:36:16.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:16.650 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:16.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:16.752 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:16.779 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:16.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:16.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:16.883 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:16.915 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:16.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:16.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:16.979 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:36:16.979 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:36:17.021 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:17.054 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:17.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:17.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:17.075 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:36:17.076 INFO:teuthology.orchestra.run.smithi173.stdout: "size": 20, 2023-09-23T18:36:17.076 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 600, 2023-09-23T18:36:17.076 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-09-23T18:36:17.076 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.076 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:85733 5.4 5:21983db8:::benchmark_data_smithi098_105788_object85732:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e113)", 2023-09-23T18:36:17.076 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:32:25.776533+0000", 2023-09-23T18:36:17.077 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 231.29371175099999, 2023-09-23T18:36:17.077 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3230933189999998, 2023-09-23T18:36:17.077 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:36:17.077 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T18:36:17.077 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:36:17.077 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:36:17.077 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:36:17.077 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 85733 2023-09-23T18:36:17.077 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.078 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:36:17.078 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.078 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:36:17.078 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.776533+0000", 2023-09-23T18:36:17.078 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:36:17.078 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.078 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.079 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:36:17.079 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.776533+0000", 2023-09-23T18:36:17.079 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:36:17.079 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.079 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.079 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:36:17.080 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.776531+0000", 2023-09-23T18:36:17.080 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.1046e-05 2023-09-23T18:36:17.080 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.080 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.080 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:36:17.080 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.776572+0000", 2023-09-23T18:36:17.080 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.43e-05 2023-09-23T18:36:17.080 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.080 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.080 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:36:17.081 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.776587+0000", 2023-09-23T18:36:17.081 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.5960000000000004e-06 2023-09-23T18:36:17.081 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.081 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.081 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:36:17.081 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.776595+0000", 2023-09-23T18:36:17.081 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00033732599999999999 2023-09-23T18:36:17.081 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.082 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.082 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.082 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.776933+0000", 2023-09-23T18:36:17.082 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00010141000000000001 2023-09-23T18:36:17.082 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.082 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.082 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:36:17.083 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.777034+0000", 2023-09-23T18:36:17.083 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013951 2023-09-23T18:36:17.083 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.083 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.083 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,5", 2023-09-23T18:36:17.083 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.777174+0000", 2023-09-23T18:36:17.083 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0098390119999999994 2023-09-23T18:36:17.083 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.084 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.084 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.084 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.787013+0000", 2023-09-23T18:36:17.084 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0042685359999999999 2023-09-23T18:36:17.084 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.084 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.084 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.085 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.791281+0000", 2023-09-23T18:36:17.085 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5006e-05 2023-09-23T18:36:17.085 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.085 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.085 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:36:17.085 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.791296+0000", 2023-09-23T18:36:17.085 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0076715810000001 2023-09-23T18:36:17.085 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.086 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.086 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.086 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:26.798968+0000", 2023-09-23T18:36:17.086 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5865000000000001e-05 2023-09-23T18:36:17.086 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.086 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.086 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:36:17.086 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:26.798984+0000", 2023-09-23T18:36:17.086 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3004405000000001 2023-09-23T18:36:17.086 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.087 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.087 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.087 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.099424+0000", 2023-09-23T18:36:17.087 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000202136 2023-09-23T18:36:17.087 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.087 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.087 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:36:17.087 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.099626+0000", 2023-09-23T18:36:17.088 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3230933189999998 2023-09-23T18:36:17.088 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.088 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:36:17.088 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.088 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.088 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.088 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:85741 5.4 5:285e6c2f:::benchmark_data_smithi098_105788_object85740:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e113)", 2023-09-23T18:36:17.088 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:32:25.782548+0000", 2023-09-23T18:36:17.089 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 231.287697092, 2023-09-23T18:36:17.089 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3344347230000002, 2023-09-23T18:36:17.089 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:36:17.089 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:36:17.089 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:36:17.089 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:36:17.089 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:36:17.090 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 85741 2023-09-23T18:36:17.090 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.090 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:36:17.090 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.090 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:36:17.090 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.782548+0000", 2023-09-23T18:36:17.090 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:36:17.090 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.091 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.091 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:36:17.091 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.782548+0000", 2023-09-23T18:36:17.091 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:36:17.091 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.091 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.091 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:36:17.091 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.782546+0000", 2023-09-23T18:36:17.091 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.8912000000000003e-05 2023-09-23T18:36:17.092 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.092 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.092 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:36:17.092 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.782585+0000", 2023-09-23T18:36:17.092 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0910999999999999e-05 2023-09-23T18:36:17.092 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.092 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.092 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:36:17.092 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.782596+0000", 2023-09-23T18:36:17.093 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4882000000000001e-05 2023-09-23T18:36:17.093 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.093 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.093 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:36:17.093 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.782611+0000", 2023-09-23T18:36:17.093 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0089575179999999994 2023-09-23T18:36:17.093 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.093 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.093 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.093 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.791568+0000", 2023-09-23T18:36:17.094 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1286e-05 2023-09-23T18:36:17.094 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.094 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.094 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:36:17.094 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.791579+0000", 2023-09-23T18:36:17.094 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0074678480000001 2023-09-23T18:36:17.094 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.094 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.095 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.095 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:26.799047+0000", 2023-09-23T18:36:17.095 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3213999999999999e-05 2023-09-23T18:36:17.095 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.095 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.095 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:36:17.095 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:26.799060+0000", 2023-09-23T18:36:17.095 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3005803890000001 2023-09-23T18:36:17.095 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.096 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.096 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.096 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.099641+0000", 2023-09-23T18:36:17.096 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000124549 2023-09-23T18:36:17.096 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.096 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.096 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:36:17.096 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.099765+0000", 2023-09-23T18:36:17.096 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014228400000000001 2023-09-23T18:36:17.096 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.096 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.097 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:36:17.097 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.099908+0000", 2023-09-23T18:36:17.097 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0072214219999999999 2023-09-23T18:36:17.097 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.097 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.097 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:36:17.097 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.107129+0000", 2023-09-23T18:36:17.097 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0044447540000000004 2023-09-23T18:36:17.097 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.097 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.098 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.098 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.111574+0000", 2023-09-23T18:36:17.098 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.003284361 2023-09-23T18:36:17.098 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.098 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.098 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.098 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.114858+0000", 2023-09-23T18:36:17.098 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.5772999999999997e-05 2023-09-23T18:36:17.098 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.099 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.099 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:36:17.099 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.114904+0000", 2023-09-23T18:36:17.099 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0020783870000000001 2023-09-23T18:36:17.099 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.099 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.099 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:36:17.099 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.116982+0000", 2023-09-23T18:36:17.099 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3344347230000002 2023-09-23T18:36:17.100 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.100 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:36:17.100 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.100 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.100 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.100 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:85744 5.4 5:2a3b56f3:::benchmark_data_smithi098_105788_object85743:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e114)", 2023-09-23T18:36:17.100 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:32:25.796811+0000", 2023-09-23T18:36:17.100 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 231.27343419900001, 2023-09-23T18:36:17.100 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3322170939999998, 2023-09-23T18:36:17.100 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:36:17.101 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:36:17.101 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:36:17.101 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:36:17.101 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:36:17.101 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 85744 2023-09-23T18:36:17.101 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.101 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:36:17.101 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.101 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:36:17.102 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.796811+0000", 2023-09-23T18:36:17.102 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:36:17.102 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.102 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.102 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:36:17.102 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.796811+0000", 2023-09-23T18:36:17.102 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:36:17.102 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.102 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.102 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:36:17.103 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.796809+0000", 2023-09-23T18:36:17.103 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.2477e-05 2023-09-23T18:36:17.103 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.103 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.103 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:36:17.103 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.796851+0000", 2023-09-23T18:36:17.103 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.6353999999999994e-05 2023-09-23T18:36:17.103 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.103 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.104 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:36:17.104 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.796928+0000", 2023-09-23T18:36:17.104 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.2309999999999996e-06 2023-09-23T18:36:17.104 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.104 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.104 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:36:17.104 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.796936+0000", 2023-09-23T18:36:17.104 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000110574 2023-09-23T18:36:17.104 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.104 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.105 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.105 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.797047+0000", 2023-09-23T18:36:17.105 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5038000000000001e-05 2023-09-23T18:36:17.105 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.105 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.105 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:36:17.105 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.797062+0000", 2023-09-23T18:36:17.105 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0020488910000001 2023-09-23T18:36:17.105 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.106 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.106 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.106 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:26.799110+0000", 2023-09-23T18:36:17.106 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0626e-05 2023-09-23T18:36:17.106 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.106 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.106 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:36:17.106 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:26.799121+0000", 2023-09-23T18:36:17.106 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.303324597 2023-09-23T18:36:17.107 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.107 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.107 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.107 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.102446+0000", 2023-09-23T18:36:17.107 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000120023 2023-09-23T18:36:17.107 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.107 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.107 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:36:17.108 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.102566+0000", 2023-09-23T18:36:17.108 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014001899999999999 2023-09-23T18:36:17.108 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.108 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.108 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:36:17.108 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.102706+0000", 2023-09-23T18:36:17.108 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0061260589999999997 2023-09-23T18:36:17.108 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.108 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.108 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:36:17.109 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.108832+0000", 2023-09-23T18:36:17.109 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0041907029999999996 2023-09-23T18:36:17.109 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.109 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.109 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.109 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.113022+0000", 2023-09-23T18:36:17.109 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0080179829999999994 2023-09-23T18:36:17.109 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.109 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.109 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.110 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.121040+0000", 2023-09-23T18:36:17.110 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015620200000000001 2023-09-23T18:36:17.110 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.110 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.110 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:36:17.110 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.121197+0000", 2023-09-23T18:36:17.110 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0078309369999999996 2023-09-23T18:36:17.110 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.110 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.111 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:36:17.111 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.129028+0000", 2023-09-23T18:36:17.111 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3322170939999998 2023-09-23T18:36:17.111 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.111 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:36:17.111 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.111 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.111 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.112 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:85745 5.4 5:23f3b8ef:::benchmark_data_smithi098_105788_object85744:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e114)", 2023-09-23T18:36:17.112 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:32:25.797003+0000", 2023-09-23T18:36:17.112 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 231.27324120700001, 2023-09-23T18:36:17.112 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3392420280000001, 2023-09-23T18:36:17.112 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:36:17.112 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:36:17.112 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:36:17.112 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:36:17.112 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:36:17.113 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 85745 2023-09-23T18:36:17.113 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.113 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:36:17.113 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.113 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:36:17.113 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.797003+0000", 2023-09-23T18:36:17.113 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:36:17.113 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.114 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.114 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:36:17.114 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.797003+0000", 2023-09-23T18:36:17.114 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-09-23T18:36:17.114 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.114 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.114 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:36:17.114 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.797003+0000", 2023-09-23T18:36:17.114 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.8083000000000001e-05 2023-09-23T18:36:17.114 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.115 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.115 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:36:17.115 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.797041+0000", 2023-09-23T18:36:17.115 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.8179999999999998e-06 2023-09-23T18:36:17.115 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.115 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.115 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:36:17.115 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.797048+0000", 2023-09-23T18:36:17.115 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.7160000000000001e-06 2023-09-23T18:36:17.116 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.116 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.116 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:36:17.116 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.797051+0000", 2023-09-23T18:36:17.116 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0040549089999999998 2023-09-23T18:36:17.116 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.116 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.116 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.117 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.801106+0000", 2023-09-23T18:36:17.117 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3244000000000001e-05 2023-09-23T18:36:17.117 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.117 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.117 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:36:17.117 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.801120+0000", 2023-09-23T18:36:17.117 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.99805806799999996 2023-09-23T18:36:17.117 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.117 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.117 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.118 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:26.799178+0000", 2023-09-23T18:36:17.118 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2016e-05 2023-09-23T18:36:17.118 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.118 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.118 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:36:17.118 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:26.799190+0000", 2023-09-23T18:36:17.119 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.305693446 2023-09-23T18:36:17.119 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.119 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.119 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.119 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.104883+0000", 2023-09-23T18:36:17.119 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.9204999999999996e-05 2023-09-23T18:36:17.119 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.119 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.119 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:36:17.119 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.104982+0000", 2023-09-23T18:36:17.119 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012820600000000001 2023-09-23T18:36:17.120 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.120 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.120 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:36:17.120 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.105111+0000", 2023-09-23T18:36:17.120 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0063297479999999996 2023-09-23T18:36:17.120 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.120 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.120 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:36:17.120 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.111440+0000", 2023-09-23T18:36:17.121 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0016809399999999999 2023-09-23T18:36:17.121 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.121 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.121 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.121 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.113121+0000", 2023-09-23T18:36:17.121 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.019258497999999999 2023-09-23T18:36:17.121 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.121 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.121 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.121 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.132380+0000", 2023-09-23T18:36:17.121 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.9589000000000001e-05 2023-09-23T18:36:17.122 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.122 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.122 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:36:17.122 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.132449+0000", 2023-09-23T18:36:17.122 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0037961819999999999 2023-09-23T18:36:17.122 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.122 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.122 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:36:17.122 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.136246+0000", 2023-09-23T18:36:17.122 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3392420280000001 2023-09-23T18:36:17.123 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.123 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:36:17.123 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.123 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.123 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.123 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:85748 5.4 5:2abb4f8d:::benchmark_data_smithi098_105788_object85747:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e114)", 2023-09-23T18:36:17.123 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:32:25.797105+0000", 2023-09-23T18:36:17.123 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 231.273139821, 2023-09-23T18:36:17.123 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3432868190000002, 2023-09-23T18:36:17.123 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:36:17.124 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:36:17.124 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:36:17.124 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:36:17.124 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:36:17.124 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 85748 2023-09-23T18:36:17.124 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.124 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:36:17.124 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.124 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:36:17.124 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.797105+0000", 2023-09-23T18:36:17.124 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:36:17.125 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.125 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.125 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:36:17.125 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.797105+0000", 2023-09-23T18:36:17.125 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-09-23T18:36:17.125 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.125 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.125 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:36:17.125 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.797105+0000", 2023-09-23T18:36:17.125 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2184e-05 2023-09-23T18:36:17.125 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.126 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.126 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:36:17.126 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.797127+0000", 2023-09-23T18:36:17.126 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.1269999999999996e-06 2023-09-23T18:36:17.126 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.126 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.126 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:36:17.126 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.797131+0000", 2023-09-23T18:36:17.127 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.6850000000000001e-06 2023-09-23T18:36:17.127 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.127 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.127 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:36:17.127 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.797134+0000", 2023-09-23T18:36:17.127 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.004013483 2023-09-23T18:36:17.127 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.127 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.127 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.128 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.801147+0000", 2023-09-23T18:36:17.128 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0095000000000001e-05 2023-09-23T18:36:17.128 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.128 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.128 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:36:17.128 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:25.801157+0000", 2023-09-23T18:36:17.128 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.99807198200000002 2023-09-23T18:36:17.128 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.128 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.128 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.129 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:26.799229+0000", 2023-09-23T18:36:17.129 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0438999999999999e-05 2023-09-23T18:36:17.129 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.129 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.129 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:36:17.129 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:26.799240+0000", 2023-09-23T18:36:17.129 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.307908697 2023-09-23T18:36:17.129 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.129 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.129 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.129 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.107148+0000", 2023-09-23T18:36:17.130 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.1509999999999996e-05 2023-09-23T18:36:17.130 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.130 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.130 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:36:17.130 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.107240+0000", 2023-09-23T18:36:17.130 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000129863 2023-09-23T18:36:17.130 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.130 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.130 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:36:17.130 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.107370+0000", 2023-09-23T18:36:17.130 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0040774510000000002 2023-09-23T18:36:17.131 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.131 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.131 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:36:17.131 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.111447+0000", 2023-09-23T18:36:17.131 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0056637040000000003 2023-09-23T18:36:17.131 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.131 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.131 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.132 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.117111+0000", 2023-09-23T18:36:17.132 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.019233655999999998 2023-09-23T18:36:17.132 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.132 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.132 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.132 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.136344+0000", 2023-09-23T18:36:17.132 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000100518 2023-09-23T18:36:17.132 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.132 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.132 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:36:17.133 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.136445+0000", 2023-09-23T18:36:17.133 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0039467929999999997 2023-09-23T18:36:17.133 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.133 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.133 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:36:17.133 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:28.140392+0000", 2023-09-23T18:36:17.133 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3432868190000002 2023-09-23T18:36:17.133 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.134 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:36:17.134 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.134 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.134 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.134 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:87467 5.4 5:2f645398:::benchmark_data_smithi098_105788_object87466:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e118)", 2023-09-23T18:36:17.134 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:32:30.762174+0000", 2023-09-23T18:36:17.135 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 226.30807100999999, 2023-09-23T18:36:17.135 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.14274504599999999, 2023-09-23T18:36:17.135 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:36:17.135 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:36:17.135 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:36:17.135 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:36:17.135 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:36:17.135 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 87467 2023-09-23T18:36:17.135 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.135 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:36:17.136 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.136 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:36:17.136 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:30.762174+0000", 2023-09-23T18:36:17.136 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:36:17.136 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.136 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.136 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:36:17.136 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:30.762174+0000", 2023-09-23T18:36:17.137 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:36:17.137 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.137 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.137 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:36:17.137 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:30.762172+0000", 2023-09-23T18:36:17.137 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4846999999999999e-05 2023-09-23T18:36:17.137 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.138 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.138 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:36:17.138 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:30.762226+0000", 2023-09-23T18:36:17.138 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4173999999999999e-05 2023-09-23T18:36:17.138 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.138 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.138 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:36:17.139 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:30.762251+0000", 2023-09-23T18:36:17.139 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0558e-05 2023-09-23T18:36:17.139 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.139 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.139 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:36:17.139 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:30.762261+0000", 2023-09-23T18:36:17.140 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.1983000000000004e-05 2023-09-23T18:36:17.140 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.140 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.140 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.140 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:30.762343+0000", 2023-09-23T18:36:17.140 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018553600000000001 2023-09-23T18:36:17.140 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.141 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.141 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:36:17.141 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:30.762529+0000", 2023-09-23T18:36:17.141 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00024527499999999999 2023-09-23T18:36:17.141 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.141 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.141 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:36:17.141 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:30.762774+0000", 2023-09-23T18:36:17.142 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0033256750000000002 2023-09-23T18:36:17.142 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.142 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.142 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:36:17.142 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:30.766100+0000", 2023-09-23T18:36:17.142 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.128602987 2023-09-23T18:36:17.143 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.143 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.143 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.143 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:30.894703+0000", 2023-09-23T18:36:17.143 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0052427790000000004 2023-09-23T18:36:17.143 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.143 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.144 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.144 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:30.899945+0000", 2023-09-23T18:36:17.144 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000104127 2023-09-23T18:36:17.144 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.144 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.144 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:36:17.145 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:30.900049+0000", 2023-09-23T18:36:17.145 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.004869249 2023-09-23T18:36:17.145 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.145 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.145 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:36:17.145 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:32:30.904919+0000", 2023-09-23T18:36:17.145 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.14274504599999999 2023-09-23T18:36:17.146 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.146 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:36:17.146 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.146 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.146 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.146 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:106855 5.4 5:2e5d210d:::benchmark_data_smithi098_105788_object570:head [delete] snapc 0=[] ondisk+write+known_if_redirected e120)", 2023-09-23T18:36:17.146 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:33:03.509483+0000", 2023-09-23T18:36:17.146 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 193.56076168800001, 2023-09-23T18:36:17.147 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.14064786700000001, 2023-09-23T18:36:17.147 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:36:17.147 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:36:17.147 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:36:17.147 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:36:17.147 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:36:17.147 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 106855 2023-09-23T18:36:17.147 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.148 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:36:17.148 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.148 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:36:17.148 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.509483+0000", 2023-09-23T18:36:17.148 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:36:17.148 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.149 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.149 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:36:17.149 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.509483+0000", 2023-09-23T18:36:17.149 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:36:17.149 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.149 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.149 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:36:17.150 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.509481+0000", 2023-09-23T18:36:17.150 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1055e-05 2023-09-23T18:36:17.150 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.150 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.150 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:36:17.150 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.509492+0000", 2023-09-23T18:36:17.150 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7033000000000001e-05 2023-09-23T18:36:17.151 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.151 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.151 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:36:17.151 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.509509+0000", 2023-09-23T18:36:17.151 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.0309999999999998e-06 2023-09-23T18:36:17.151 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.151 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.151 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:36:17.152 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.509517+0000", 2023-09-23T18:36:17.152 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0039937710000000001 2023-09-23T18:36:17.152 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.152 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.152 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.152 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.513511+0000", 2023-09-23T18:36:17.152 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015685300000000001 2023-09-23T18:36:17.153 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.153 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.153 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:36:17.153 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.513668+0000", 2023-09-23T18:36:17.153 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013039799999999999 2023-09-23T18:36:17.153 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.153 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.154 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:36:17.154 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.513798+0000", 2023-09-23T18:36:17.154 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0031808330000000001 2023-09-23T18:36:17.154 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.154 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.154 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:36:17.154 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.516979+0000", 2023-09-23T18:36:17.155 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0017439300000000001 2023-09-23T18:36:17.155 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.155 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.155 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.155 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.518723+0000", 2023-09-23T18:36:17.155 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.12753933000000001 2023-09-23T18:36:17.156 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.156 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.156 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.156 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.646262+0000", 2023-09-23T18:36:17.156 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.7323999999999994e-05 2023-09-23T18:36:17.156 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.156 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.156 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:36:17.157 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.646330+0000", 2023-09-23T18:36:17.157 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0038012050000000002 2023-09-23T18:36:17.157 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.157 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.157 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:36:17.157 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.650131+0000", 2023-09-23T18:36:17.157 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.14064786700000001 2023-09-23T18:36:17.157 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.158 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:36:17.158 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.158 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.158 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.158 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:106859 5.4 5:23dbd7b7:::benchmark_data_smithi098_105788_object574:head [delete] snapc 0=[] ondisk+write+known_if_redirected e120)", 2023-09-23T18:36:17.158 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:33:03.518785+0000", 2023-09-23T18:36:17.158 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 193.55145969, 2023-09-23T18:36:17.159 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.13905083400000001, 2023-09-23T18:36:17.159 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:36:17.159 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:36:17.159 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:36:17.159 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:36:17.159 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:36:17.159 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 106859 2023-09-23T18:36:17.160 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.160 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:36:17.160 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.160 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:36:17.160 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.518785+0000", 2023-09-23T18:36:17.160 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:36:17.160 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.160 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.160 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:36:17.161 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.518785+0000", 2023-09-23T18:36:17.161 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:36:17.161 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.161 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.161 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:36:17.161 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.518783+0000", 2023-09-23T18:36:17.161 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.0650000000000001e-06 2023-09-23T18:36:17.161 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.162 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.162 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:36:17.162 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.518790+0000", 2023-09-23T18:36:17.162 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3964e-05 2023-09-23T18:36:17.162 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.162 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.162 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:36:17.162 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.518804+0000", 2023-09-23T18:36:17.162 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.5859999999999999e-06 2023-09-23T18:36:17.163 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.163 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.163 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:36:17.163 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.518812+0000", 2023-09-23T18:36:17.163 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.4783999999999997e-05 2023-09-23T18:36:17.163 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.164 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.164 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.164 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.518856+0000", 2023-09-23T18:36:17.164 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015621799999999999 2023-09-23T18:36:17.164 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.164 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.164 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:36:17.165 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.519013+0000", 2023-09-23T18:36:17.165 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015937199999999999 2023-09-23T18:36:17.165 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.165 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.165 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:36:17.165 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.519172+0000", 2023-09-23T18:36:17.166 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.003435876 2023-09-23T18:36:17.166 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.166 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.166 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.166 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.522608+0000", 2023-09-23T18:36:17.166 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013112900000000001 2023-09-23T18:36:17.166 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.167 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.167 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:36:17.167 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.522739+0000", 2023-09-23T18:36:17.167 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.127492511 2023-09-23T18:36:17.167 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.167 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.168 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.168 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.650232+0000", 2023-09-23T18:36:17.168 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.8851999999999998e-05 2023-09-23T18:36:17.168 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.168 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.168 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:36:17.168 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.650290+0000", 2023-09-23T18:36:17.169 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0075454679999999996 2023-09-23T18:36:17.169 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.169 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.169 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:36:17.169 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:03.657836+0000", 2023-09-23T18:36:17.169 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.13905083400000001 2023-09-23T18:36:17.169 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.170 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:36:17.170 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.170 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.170 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.170 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:113764 5.4 5:2c99164f:::benchmark_data_smithi098_105788_object7479:head [delete] snapc 0=[] ondisk+write+known_if_redirected e120)", 2023-09-23T18:36:17.170 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:33:14.793886+0000", 2023-09-23T18:36:17.171 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 182.276358888, 2023-09-23T18:36:17.171 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.14111499599999999, 2023-09-23T18:36:17.171 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:36:17.171 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:36:17.171 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:36:17.171 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:36:17.171 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:36:17.171 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 113764 2023-09-23T18:36:17.171 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.171 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:36:17.172 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.172 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:36:17.172 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.793886+0000", 2023-09-23T18:36:17.172 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:36:17.172 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.172 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.172 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:36:17.172 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.793886+0000", 2023-09-23T18:36:17.173 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:36:17.173 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.173 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.173 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:36:17.173 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.793884+0000", 2023-09-23T18:36:17.173 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.5830000000000001e-06 2023-09-23T18:36:17.173 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.173 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.173 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:36:17.174 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.793891+0000", 2023-09-23T18:36:17.174 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0016e-05 2023-09-23T18:36:17.174 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.174 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.174 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:36:17.174 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.793901+0000", 2023-09-23T18:36:17.174 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1650000000000003e-06 2023-09-23T18:36:17.174 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.175 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.175 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:36:17.175 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.793908+0000", 2023-09-23T18:36:17.175 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.011364852 2023-09-23T18:36:17.175 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.175 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.175 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.175 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.805272+0000", 2023-09-23T18:36:17.176 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012991200000000001 2023-09-23T18:36:17.176 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.176 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.176 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:36:17.176 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.805402+0000", 2023-09-23T18:36:17.176 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000170154 2023-09-23T18:36:17.177 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.177 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.177 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:36:17.177 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.805572+0000", 2023-09-23T18:36:17.177 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0078262990000000001 2023-09-23T18:36:17.177 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.177 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.178 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:36:17.178 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.813399+0000", 2023-09-23T18:36:17.178 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001341079 2023-09-23T18:36:17.178 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.178 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.178 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.178 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.814740+0000", 2023-09-23T18:36:17.178 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.11575038999999999 2023-09-23T18:36:17.179 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.179 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.179 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.179 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.930490+0000", 2023-09-23T18:36:17.179 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000160748 2023-09-23T18:36:17.179 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.179 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.180 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:36:17.180 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.930651+0000", 2023-09-23T18:36:17.180 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0043498410000000001 2023-09-23T18:36:17.180 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.180 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.180 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:36:17.180 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.935001+0000", 2023-09-23T18:36:17.180 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.14111499599999999 2023-09-23T18:36:17.180 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.181 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:36:17.181 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.181 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.181 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.181 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:113770 5.4 5:2b857679:::benchmark_data_smithi098_105788_object7485:head [delete] snapc 0=[] ondisk+write+known_if_redirected e120)", 2023-09-23T18:36:17.181 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:33:14.804388+0000", 2023-09-23T18:36:17.182 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 182.26585684700001, 2023-09-23T18:36:17.182 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.13607188100000001, 2023-09-23T18:36:17.182 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:36:17.182 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:36:17.182 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:36:17.182 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:36:17.183 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:36:17.183 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 113770 2023-09-23T18:36:17.183 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.183 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:36:17.183 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.183 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:36:17.183 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.804388+0000", 2023-09-23T18:36:17.183 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:36:17.184 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.184 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.184 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:36:17.184 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.804388+0000", 2023-09-23T18:36:17.184 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:36:17.184 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.184 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.185 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:36:17.185 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.804386+0000", 2023-09-23T18:36:17.185 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.4760000000000003e-06 2023-09-23T18:36:17.185 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.185 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.185 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:36:17.185 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.804393+0000", 2023-09-23T18:36:17.185 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4484000000000001e-05 2023-09-23T18:36:17.185 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.185 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.186 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:36:17.186 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.804407+0000", 2023-09-23T18:36:17.186 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.9319999999999999e-06 2023-09-23T18:36:17.186 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.186 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.186 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:36:17.187 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.804414+0000", 2023-09-23T18:36:17.187 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.005104356 2023-09-23T18:36:17.187 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.187 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.187 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.187 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.809518+0000", 2023-09-23T18:36:17.187 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000174181 2023-09-23T18:36:17.187 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.187 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.187 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:36:17.188 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.809693+0000", 2023-09-23T18:36:17.188 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00019645200000000001 2023-09-23T18:36:17.188 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.188 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.188 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:36:17.188 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.809889+0000", 2023-09-23T18:36:17.189 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.003757748 2023-09-23T18:36:17.189 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.189 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.189 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:36:17.189 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.813647+0000", 2023-09-23T18:36:17.189 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001208329 2023-09-23T18:36:17.189 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.189 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.189 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.190 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.814855+0000", 2023-09-23T18:36:17.190 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.120310323 2023-09-23T18:36:17.190 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.190 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.190 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.190 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.935165+0000", 2023-09-23T18:36:17.190 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000182707 2023-09-23T18:36:17.190 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.191 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.191 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:36:17.191 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.935348+0000", 2023-09-23T18:36:17.191 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0051116130000000001 2023-09-23T18:36:17.191 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.191 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.192 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:36:17.192 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:14.940460+0000", 2023-09-23T18:36:17.192 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.13607188100000001 2023-09-23T18:36:17.192 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.192 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:36:17.192 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.192 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.193 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.193 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:116968 5.4 5:204e9dd7:::benchmark_data_smithi098_105788_object10683:head [delete] snapc 0=[] ondisk+write+known_if_redirected e120)", 2023-09-23T18:36:17.193 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:33:19.940301+0000", 2023-09-23T18:36:17.193 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 177.129944137, 2023-09-23T18:36:17.193 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.13882898399999999, 2023-09-23T18:36:17.193 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:36:17.193 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:36:17.193 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:36:17.194 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:36:17.194 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:36:17.194 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 116968 2023-09-23T18:36:17.194 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.194 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:36:17.194 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.194 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:36:17.195 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.940301+0000", 2023-09-23T18:36:17.195 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:36:17.195 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.195 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.195 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:36:17.195 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.940301+0000", 2023-09-23T18:36:17.195 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999971 2023-09-23T18:36:17.196 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.196 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.196 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:36:17.196 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.940298+0000", 2023-09-23T18:36:17.196 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3281999999999999e-05 2023-09-23T18:36:17.196 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.196 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.196 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:36:17.197 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.940311+0000", 2023-09-23T18:36:17.197 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7631000000000001e-05 2023-09-23T18:36:17.197 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.197 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.197 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:36:17.197 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.940329+0000", 2023-09-23T18:36:17.198 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.8719999999999997e-06 2023-09-23T18:36:17.198 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.198 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.198 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:36:17.198 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.940339+0000", 2023-09-23T18:36:17.198 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.0098000000000005e-05 2023-09-23T18:36:17.198 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.199 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.199 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.199 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.940419+0000", 2023-09-23T18:36:17.199 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00021370699999999999 2023-09-23T18:36:17.199 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.199 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.199 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:36:17.200 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.940632+0000", 2023-09-23T18:36:17.200 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000198049 2023-09-23T18:36:17.200 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.200 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.200 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:36:17.200 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.940831+0000", 2023-09-23T18:36:17.200 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0044774530000000002 2023-09-23T18:36:17.200 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.201 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.201 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:36:17.201 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.945308+0000", 2023-09-23T18:36:17.201 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.020783445000000001 2023-09-23T18:36:17.201 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.201 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.201 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.202 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.966091+0000", 2023-09-23T18:36:17.202 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.10900921399999999 2023-09-23T18:36:17.202 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.202 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.202 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.202 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.075101+0000", 2023-09-23T18:36:17.203 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000108261 2023-09-23T18:36:17.203 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.203 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.203 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:36:17.203 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.075209+0000", 2023-09-23T18:36:17.203 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0039206390000000001 2023-09-23T18:36:17.203 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.203 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.203 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:36:17.204 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.079130+0000", 2023-09-23T18:36:17.204 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.13882898399999999 2023-09-23T18:36:17.204 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.204 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:36:17.204 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.204 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.204 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.204 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:116976 5.4 5:26dbd1c2:::benchmark_data_smithi098_105788_object10691:head [delete] snapc 0=[] ondisk+write+known_if_redirected e120)", 2023-09-23T18:36:17.205 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:33:19.953103+0000", 2023-09-23T18:36:17.205 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 177.117141578, 2023-09-23T18:36:17.205 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.13615623499999999, 2023-09-23T18:36:17.205 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:36:17.205 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:36:17.205 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:36:17.205 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:36:17.206 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:36:17.206 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 116976 2023-09-23T18:36:17.206 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.206 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:36:17.206 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.206 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:36:17.206 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.953103+0000", 2023-09-23T18:36:17.206 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:36:17.207 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.207 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.207 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:36:17.207 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.953103+0000", 2023-09-23T18:36:17.207 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-09-23T18:36:17.207 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.207 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.207 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:36:17.207 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.953101+0000", 2023-09-23T18:36:17.207 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.294e-05 2023-09-23T18:36:17.208 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.208 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.208 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:36:17.208 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.953114+0000", 2023-09-23T18:36:17.208 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.1202000000000001e-05 2023-09-23T18:36:17.208 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.208 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.208 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:36:17.208 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.953135+0000", 2023-09-23T18:36:17.209 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.3409999999999992e-06 2023-09-23T18:36:17.209 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.209 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.209 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:36:17.209 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.953144+0000", 2023-09-23T18:36:17.209 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.7523999999999998e-05 2023-09-23T18:36:17.209 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.210 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.210 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.210 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.953222+0000", 2023-09-23T18:36:17.210 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00024049299999999999 2023-09-23T18:36:17.210 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.210 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.211 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:36:17.211 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.953462+0000", 2023-09-23T18:36:17.211 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00025103400000000002 2023-09-23T18:36:17.211 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.211 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.211 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:36:17.211 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.953713+0000", 2023-09-23T18:36:17.212 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0041140819999999998 2023-09-23T18:36:17.212 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.212 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.212 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:36:17.212 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.957827+0000", 2023-09-23T18:36:17.212 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0083336609999999992 2023-09-23T18:36:17.212 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.213 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.213 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.213 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.966161+0000", 2023-09-23T18:36:17.213 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.113583226 2023-09-23T18:36:17.213 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.213 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.213 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.213 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.079744+0000", 2023-09-23T18:36:17.213 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000183556 2023-09-23T18:36:17.214 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.214 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.214 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:36:17.214 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.079928+0000", 2023-09-23T18:36:17.214 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0093314290000000005 2023-09-23T18:36:17.214 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.215 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.215 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:36:17.215 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.089259+0000", 2023-09-23T18:36:17.215 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.13615623499999999 2023-09-23T18:36:17.215 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.215 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:36:17.215 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.215 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.215 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.216 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:116986 5.4 5:24fbd4ef:::benchmark_data_smithi098_105788_object10701:head [delete] snapc 0=[] ondisk+write+known_if_redirected e120)", 2023-09-23T18:36:17.216 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:33:19.968230+0000", 2023-09-23T18:36:17.216 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 177.10201432400001, 2023-09-23T18:36:17.216 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.127672701, 2023-09-23T18:36:17.216 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:36:17.216 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:36:17.216 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:36:17.217 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:36:17.217 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:36:17.217 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 116986 2023-09-23T18:36:17.217 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.217 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:36:17.217 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.217 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:36:17.218 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.968230+0000", 2023-09-23T18:36:17.218 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:36:17.218 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.218 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.218 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:36:17.218 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.968230+0000", 2023-09-23T18:36:17.219 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:36:17.219 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.219 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.219 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:36:17.219 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.968229+0000", 2023-09-23T18:36:17.219 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.2840000000000001e-06 2023-09-23T18:36:17.219 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.220 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.220 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:36:17.220 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.968236+0000", 2023-09-23T18:36:17.220 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3736e-05 2023-09-23T18:36:17.220 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.220 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.220 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:36:17.221 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.968250+0000", 2023-09-23T18:36:17.221 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7629999999999997e-06 2023-09-23T18:36:17.221 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.221 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.221 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:36:17.221 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.968256+0000", 2023-09-23T18:36:17.221 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.3230000000000001e-05 2023-09-23T18:36:17.222 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.222 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.222 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.222 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.968339+0000", 2023-09-23T18:36:17.222 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00028778400000000002 2023-09-23T18:36:17.222 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.222 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.222 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:36:17.223 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.968627+0000", 2023-09-23T18:36:17.223 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00033981499999999998 2023-09-23T18:36:17.223 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.223 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.223 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:36:17.223 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.968966+0000", 2023-09-23T18:36:17.223 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.003793238 2023-09-23T18:36:17.223 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.223 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.223 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:36:17.224 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:19.972760+0000", 2023-09-23T18:36:17.224 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.11658663499999999 2023-09-23T18:36:17.224 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.224 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.224 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.224 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.089346+0000", 2023-09-23T18:36:17.224 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0023779970000000002 2023-09-23T18:36:17.224 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.225 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.225 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.225 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.091724+0000", 2023-09-23T18:36:17.225 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.4165000000000007e-05 2023-09-23T18:36:17.225 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.225 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.225 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:36:17.226 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.091788+0000", 2023-09-23T18:36:17.226 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0041146480000000003 2023-09-23T18:36:17.226 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.226 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.226 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:36:17.226 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.095903+0000", 2023-09-23T18:36:17.226 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.127672701 2023-09-23T18:36:17.227 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.227 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:36:17.227 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.227 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.227 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.227 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:117300 5.4 5:2554f453:::benchmark_data_smithi098_105788_object11015:head [delete] snapc 0=[] ondisk+write+known_if_redirected e120)", 2023-09-23T18:36:17.227 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:33:20.541983+0000", 2023-09-23T18:36:17.228 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 176.528261224, 2023-09-23T18:36:17.228 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.13867464099999999, 2023-09-23T18:36:17.228 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:36:17.228 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:36:17.228 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:36:17.228 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:36:17.228 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:36:17.228 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 117300 2023-09-23T18:36:17.229 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.229 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:36:17.229 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.229 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:36:17.229 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.541983+0000", 2023-09-23T18:36:17.229 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:36:17.229 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.230 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.230 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:36:17.230 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.541983+0000", 2023-09-23T18:36:17.230 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:36:17.230 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.230 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.231 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:36:17.231 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.541982+0000", 2023-09-23T18:36:17.231 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.553e-06 2023-09-23T18:36:17.231 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.231 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.231 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:36:17.231 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.541987+0000", 2023-09-23T18:36:17.232 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.7370000000000001e-06 2023-09-23T18:36:17.232 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.232 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.232 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:36:17.232 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.541997+0000", 2023-09-23T18:36:17.232 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.0769999999999996e-06 2023-09-23T18:36:17.232 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.233 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.233 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:36:17.233 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.542003+0000", 2023-09-23T18:36:17.233 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.004690311 2023-09-23T18:36:17.233 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.233 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.233 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.233 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.546693+0000", 2023-09-23T18:36:17.233 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015774499999999999 2023-09-23T18:36:17.234 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.234 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.234 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:36:17.234 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.546851+0000", 2023-09-23T18:36:17.234 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00017292800000000001 2023-09-23T18:36:17.234 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.234 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.235 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:36:17.235 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.547024+0000", 2023-09-23T18:36:17.235 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0070876680000000001 2023-09-23T18:36:17.235 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.235 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.235 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:36:17.235 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.554112+0000", 2023-09-23T18:36:17.235 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.084024440000000006 2023-09-23T18:36:17.236 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.236 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.236 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.236 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.638136+0000", 2023-09-23T18:36:17.236 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.038673649999999997 2023-09-23T18:36:17.236 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.237 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.237 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.237 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.676810+0000", 2023-09-23T18:36:17.237 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000110423 2023-09-23T18:36:17.237 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.237 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.237 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:36:17.237 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.676920+0000", 2023-09-23T18:36:17.237 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0037377959999999998 2023-09-23T18:36:17.238 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.238 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.238 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:36:17.238 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.680658+0000", 2023-09-23T18:36:17.238 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.13867464099999999 2023-09-23T18:36:17.238 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.238 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:36:17.239 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.239 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.239 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.239 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:117308 5.4 5:281d42ea:::benchmark_data_smithi098_105788_object11023:head [delete] snapc 0=[] ondisk+write+known_if_redirected e120)", 2023-09-23T18:36:17.239 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:33:20.550547+0000", 2023-09-23T18:36:17.239 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 176.51969724200001, 2023-09-23T18:36:17.239 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.161906404, 2023-09-23T18:36:17.240 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:36:17.240 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:36:17.240 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:36:17.240 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:36:17.240 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:36:17.240 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 117308 2023-09-23T18:36:17.240 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.241 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:36:17.241 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.241 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:36:17.241 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.550547+0000", 2023-09-23T18:36:17.241 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:36:17.241 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.241 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.241 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:36:17.242 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.550547+0000", 2023-09-23T18:36:17.242 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:36:17.242 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.242 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.242 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:36:17.242 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.550546+0000", 2023-09-23T18:36:17.242 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.9089999999999999e-06 2023-09-23T18:36:17.242 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.243 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.243 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:36:17.243 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.550551+0000", 2023-09-23T18:36:17.243 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.7829999999999993e-06 2023-09-23T18:36:17.243 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.243 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.243 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:36:17.243 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.550561+0000", 2023-09-23T18:36:17.244 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.4139999999999996e-06 2023-09-23T18:36:17.244 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.244 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.244 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:36:17.244 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.550565+0000", 2023-09-23T18:36:17.244 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00023562400000000001 2023-09-23T18:36:17.244 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.245 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.245 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.245 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.550801+0000", 2023-09-23T18:36:17.245 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00019932399999999999 2023-09-23T18:36:17.245 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.245 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.245 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:36:17.245 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.551000+0000", 2023-09-23T18:36:17.246 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000221036 2023-09-23T18:36:17.246 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.246 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.246 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:36:17.246 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.551221+0000", 2023-09-23T18:36:17.246 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0030459100000000002 2023-09-23T18:36:17.247 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.247 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.247 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:36:17.247 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.554267+0000", 2023-09-23T18:36:17.247 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.084150295999999999 2023-09-23T18:36:17.247 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.247 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.248 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.248 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.638418+0000", 2023-09-23T18:36:17.248 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.066508380000000006 2023-09-23T18:36:17.248 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.248 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.248 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.248 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.704926+0000", 2023-09-23T18:36:17.248 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.8237000000000004e-05 2023-09-23T18:36:17.248 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.249 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.249 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:36:17.249 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.704994+0000", 2023-09-23T18:36:17.249 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0074596489999999996 2023-09-23T18:36:17.249 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.249 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.249 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:36:17.250 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:20.712454+0000", 2023-09-23T18:36:17.250 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.161906404 2023-09-23T18:36:17.250 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.250 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:36:17.250 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.250 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.250 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.251 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:119894 5.4 5:2893eca1:::benchmark_data_smithi098_105788_object13609:head [delete] snapc 0=[] ondisk+write+known_if_redirected e120)", 2023-09-23T18:36:17.251 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:33:25.048924+0000", 2023-09-23T18:36:17.251 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 172.021320594, 2023-09-23T18:36:17.251 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.13210042699999999, 2023-09-23T18:36:17.251 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:36:17.251 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:36:17.251 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:36:17.251 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:36:17.251 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:36:17.252 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 119894 2023-09-23T18:36:17.252 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.252 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:36:17.252 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.252 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:36:17.252 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:25.048924+0000", 2023-09-23T18:36:17.252 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:36:17.252 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.252 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.253 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:36:17.253 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:25.048924+0000", 2023-09-23T18:36:17.253 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999948 2023-09-23T18:36:17.253 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.253 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.253 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:36:17.253 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:25.048919+0000", 2023-09-23T18:36:17.253 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2873e-05 2023-09-23T18:36:17.254 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.254 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.254 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:36:17.254 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:25.048932+0000", 2023-09-23T18:36:17.254 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3851e-05 2023-09-23T18:36:17.254 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.254 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.255 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:36:17.255 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:25.048946+0000", 2023-09-23T18:36:17.255 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.1720000000000001e-06 2023-09-23T18:36:17.255 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.255 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.255 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:36:17.255 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:25.048953+0000", 2023-09-23T18:36:17.255 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.4689000000000004e-05 2023-09-23T18:36:17.255 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.256 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.256 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.256 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:25.049017+0000", 2023-09-23T18:36:17.256 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016595600000000001 2023-09-23T18:36:17.256 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.256 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.256 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:36:17.256 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:25.049183+0000", 2023-09-23T18:36:17.256 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018898600000000001 2023-09-23T18:36:17.256 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.257 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.257 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:36:17.257 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:25.049372+0000", 2023-09-23T18:36:17.257 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0041925729999999998 2023-09-23T18:36:17.257 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.257 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.257 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:36:17.257 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:25.053565+0000", 2023-09-23T18:36:17.257 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002388828 2023-09-23T18:36:17.258 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.258 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.258 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.258 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:25.055954+0000", 2023-09-23T18:36:17.259 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.11789614399999999 2023-09-23T18:36:17.259 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.259 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.259 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.259 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:25.173850+0000", 2023-09-23T18:36:17.260 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.2694000000000005e-05 2023-09-23T18:36:17.260 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.260 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.260 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:36:17.261 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:25.173933+0000", 2023-09-23T18:36:17.261 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0070919049999999999 2023-09-23T18:36:17.261 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.261 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.261 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:36:17.262 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:33:25.181025+0000", 2023-09-23T18:36:17.262 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.13210042699999999 2023-09-23T18:36:17.262 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.262 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:36:17.262 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.263 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.263 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.263 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:141152 5.4 5:2d4e4e56:::benchmark_data_smithi098_105788_object34867:head [delete] snapc 0=[] ondisk+write+known_if_redirected e126)", 2023-09-23T18:36:17.263 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:34:05.914172+0000", 2023-09-23T18:36:17.263 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 131.15607246499999, 2023-09-23T18:36:17.263 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.22119594200000001, 2023-09-23T18:36:17.263 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:36:17.263 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:36:17.264 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:36:17.264 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:36:17.264 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:36:17.264 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 141152 2023-09-23T18:36:17.264 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.264 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:36:17.265 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.265 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:36:17.265 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.914172+0000", 2023-09-23T18:36:17.265 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:36:17.265 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.265 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.265 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:36:17.266 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.914172+0000", 2023-09-23T18:36:17.266 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:36:17.266 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.266 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.266 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:36:17.266 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.914171+0000", 2023-09-23T18:36:17.267 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.1450000000000008e-06 2023-09-23T18:36:17.267 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.267 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.267 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:36:17.267 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.914179+0000", 2023-09-23T18:36:17.267 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4491999999999999e-05 2023-09-23T18:36:17.268 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.268 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.268 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:36:17.268 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.914193+0000", 2023-09-23T18:36:17.268 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.3280000000000006e-06 2023-09-23T18:36:17.268 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.268 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.268 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:36:17.269 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.914202+0000", 2023-09-23T18:36:17.269 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.3544e-05 2023-09-23T18:36:17.269 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.269 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.269 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.269 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.914265+0000", 2023-09-23T18:36:17.269 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013129099999999999 2023-09-23T18:36:17.270 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.270 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.270 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:36:17.270 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.914397+0000", 2023-09-23T18:36:17.270 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016110899999999999 2023-09-23T18:36:17.270 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.271 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.271 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:36:17.271 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.914558+0000", 2023-09-23T18:36:17.271 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0079424709999999996 2023-09-23T18:36:17.271 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.271 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.272 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:36:17.272 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.922500+0000", 2023-09-23T18:36:17.272 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6165000000000003e-05 2023-09-23T18:36:17.272 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.272 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.272 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.272 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.922546+0000", 2023-09-23T18:36:17.272 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.20922811499999999 2023-09-23T18:36:17.273 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.273 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.273 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.273 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:06.131774+0000", 2023-09-23T18:36:17.273 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1870999999999999e-05 2023-09-23T18:36:17.273 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.273 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.274 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:36:17.274 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:06.131826+0000", 2023-09-23T18:36:17.274 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0035419179999999998 2023-09-23T18:36:17.274 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.274 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.275 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:36:17.275 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:06.135368+0000", 2023-09-23T18:36:17.275 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.22119594200000001 2023-09-23T18:36:17.275 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.275 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:36:17.275 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.275 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.276 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.276 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:141155 5.4 5:2abc0c34:::benchmark_data_smithi098_105788_object34870:head [delete] snapc 0=[] ondisk+write+known_if_redirected e126)", 2023-09-23T18:36:17.276 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:34:05.914730+0000", 2023-09-23T18:36:17.276 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 131.15551450199999, 2023-09-23T18:36:17.276 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.22832068799999999, 2023-09-23T18:36:17.276 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:36:17.276 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:36:17.276 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:36:17.276 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:36:17.277 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:36:17.277 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 141155 2023-09-23T18:36:17.277 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.277 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:36:17.277 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.277 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:36:17.277 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.914730+0000", 2023-09-23T18:36:17.277 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:36:17.277 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.277 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.277 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:36:17.277 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.914730+0000", 2023-09-23T18:36:17.278 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:36:17.278 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.278 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.278 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:36:17.278 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.914729+0000", 2023-09-23T18:36:17.278 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.904e-06 2023-09-23T18:36:17.278 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.278 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.278 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:36:17.278 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.914734+0000", 2023-09-23T18:36:17.279 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0647e-05 2023-09-23T18:36:17.279 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.279 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.279 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:36:17.279 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.914745+0000", 2023-09-23T18:36:17.279 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.1729999999999998e-06 2023-09-23T18:36:17.279 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.279 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.280 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:36:17.280 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.914752+0000", 2023-09-23T18:36:17.280 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0039518469999999997 2023-09-23T18:36:17.280 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.280 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.280 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.280 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.918704+0000", 2023-09-23T18:36:17.280 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014673999999999999 2023-09-23T18:36:17.281 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.281 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.281 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:36:17.281 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.918851+0000", 2023-09-23T18:36:17.281 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015404500000000001 2023-09-23T18:36:17.281 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.281 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.281 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:36:17.281 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.919005+0000", 2023-09-23T18:36:17.281 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0036214630000000001 2023-09-23T18:36:17.281 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.282 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.282 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:36:17.282 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.922626+0000", 2023-09-23T18:36:17.282 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000484162 2023-09-23T18:36:17.282 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.282 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.282 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.282 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:05.923110+0000", 2023-09-23T18:36:17.283 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.21232636999999999 2023-09-23T18:36:17.283 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.283 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.283 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.283 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:06.135437+0000", 2023-09-23T18:36:17.283 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5340000000000002e-05 2023-09-23T18:36:17.283 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.283 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.283 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:36:17.284 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:06.135492+0000", 2023-09-23T18:36:17.284 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0075587809999999997 2023-09-23T18:36:17.284 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.284 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.284 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:36:17.284 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:06.143051+0000", 2023-09-23T18:36:17.284 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.22832068799999999 2023-09-23T18:36:17.284 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.284 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:36:17.285 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.285 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.285 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.285 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:166746 5.4 5:2622f516:::benchmark_data_smithi098_105788_object60461:head [delete] snapc 0=[] ondisk+write+known_if_redirected e130)", 2023-09-23T18:36:17.285 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:34:57.533927+0000", 2023-09-23T18:36:17.285 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 79.536317308999998, 2023-09-23T18:36:17.285 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.12846432299999999, 2023-09-23T18:36:17.286 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:36:17.286 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:36:17.286 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:36:17.286 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:36:17.286 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:36:17.286 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 166746 2023-09-23T18:36:17.286 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.286 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:36:17.287 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.287 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:36:17.287 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:57.533927+0000", 2023-09-23T18:36:17.287 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:36:17.287 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.287 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.287 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:36:17.287 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:57.533927+0000", 2023-09-23T18:36:17.287 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:36:17.287 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.288 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.288 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:36:17.288 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:57.533926+0000", 2023-09-23T18:36:17.288 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1633e-05 2023-09-23T18:36:17.288 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.288 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.288 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:36:17.288 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:57.533937+0000", 2023-09-23T18:36:17.289 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2106999999999999e-05 2023-09-23T18:36:17.289 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.289 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.289 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:36:17.289 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:57.533950+0000", 2023-09-23T18:36:17.289 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1340000000000001e-06 2023-09-23T18:36:17.289 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.289 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.289 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:36:17.290 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:57.533956+0000", 2023-09-23T18:36:17.290 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013846799999999999 2023-09-23T18:36:17.290 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.290 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.290 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.290 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:57.534094+0000", 2023-09-23T18:36:17.290 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0037200839999999998 2023-09-23T18:36:17.290 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.291 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.291 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:36:17.291 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:57.537814+0000", 2023-09-23T18:36:17.291 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000208228 2023-09-23T18:36:17.291 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.291 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.291 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:36:17.291 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:57.538022+0000", 2023-09-23T18:36:17.292 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.003667071 2023-09-23T18:36:17.292 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.292 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.292 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:36:17.292 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:57.541689+0000", 2023-09-23T18:36:17.292 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010749396 2023-09-23T18:36:17.292 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.292 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.292 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.292 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:57.552439+0000", 2023-09-23T18:36:17.293 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.10659856500000001 2023-09-23T18:36:17.293 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.293 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.293 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.293 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:57.659037+0000", 2023-09-23T18:36:17.293 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.5908999999999995e-05 2023-09-23T18:36:17.293 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.293 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.293 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:36:17.293 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:57.659103+0000", 2023-09-23T18:36:17.294 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.003288352 2023-09-23T18:36:17.294 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.294 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.294 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:36:17.294 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:34:57.662392+0000", 2023-09-23T18:36:17.294 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.12846432299999999 2023-09-23T18:36:17.294 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.294 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:36:17.295 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.295 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.295 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.295 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44693.0:182139 5.4 5:2bc68c8d:::benchmark_data_smithi098_105788_object75854:head [delete] snapc 0=[] ondisk+write+known_if_redirected e134)", 2023-09-23T18:36:17.295 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:35:28.117509+0000", 2023-09-23T18:36:17.295 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 48.952735431999997, 2023-09-23T18:36:17.295 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.13468022900000001, 2023-09-23T18:36:17.296 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:36:17.296 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:36:17.296 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:36:17.296 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44693", 2023-09-23T18:36:17.296 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3630081638", 2023-09-23T18:36:17.296 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 182139 2023-09-23T18:36:17.296 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.296 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:36:17.296 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.296 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:36:17.297 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:28.117509+0000", 2023-09-23T18:36:17.297 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:36:17.297 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.297 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.297 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:36:17.297 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:28.117509+0000", 2023-09-23T18:36:17.297 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:36:17.297 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.297 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.297 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:36:17.298 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:28.117507+0000", 2023-09-23T18:36:17.298 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0414000000000001e-05 2023-09-23T18:36:17.298 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.298 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.298 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:36:17.298 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:28.117518+0000", 2023-09-23T18:36:17.298 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7997999999999999e-05 2023-09-23T18:36:17.298 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.299 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.299 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:36:17.299 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:28.117536+0000", 2023-09-23T18:36:17.299 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.2760000000000003e-06 2023-09-23T18:36:17.299 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.299 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.299 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:36:17.300 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:28.117543+0000", 2023-09-23T18:36:17.300 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.012847074 2023-09-23T18:36:17.300 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.300 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.300 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:36:17.300 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:28.130390+0000", 2023-09-23T18:36:17.300 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018814600000000001 2023-09-23T18:36:17.300 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.300 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.300 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:36:17.301 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:28.130578+0000", 2023-09-23T18:36:17.301 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000219194 2023-09-23T18:36:17.301 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.301 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.301 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:36:17.301 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:28.130798+0000", 2023-09-23T18:36:17.301 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0040332900000000001 2023-09-23T18:36:17.302 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.302 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.302 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:36:17.302 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:28.134831+0000", 2023-09-23T18:36:17.302 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.095403640999999997 2023-09-23T18:36:17.302 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.302 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.303 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.303 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:28.230234+0000", 2023-09-23T18:36:17.303 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.019082066000000002 2023-09-23T18:36:17.303 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.303 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.303 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:36:17.303 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:28.249317+0000", 2023-09-23T18:36:17.303 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014919 2023-09-23T18:36:17.303 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.304 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.304 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:36:17.304 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:28.249466+0000", 2023-09-23T18:36:17.304 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0027237810000000002 2023-09-23T18:36:17.304 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:36:17.304 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:36:17.304 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:36:17.304 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:35:28.252189+0000", 2023-09-23T18:36:17.304 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.13468022900000001 2023-09-23T18:36:17.305 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.305 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:36:17.305 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.305 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:36:17.305 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:36:17.305 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:36:17.305 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:17.332 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:17.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:17.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:17.435 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:17.461 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:17.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:17.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:17.571 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:17.591 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:17.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:17.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:17.696 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:17.726 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:17.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:17.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:17.830 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:17.870 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:17.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:17.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:17.976 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:18.000 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:18.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:18.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:18.028 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T18:36:18.105 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:18.133 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:18.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:18.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:18.236 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:18.259 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:18.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:18.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:18.363 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:18.387 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:18.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:18.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:18.490 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:18.517 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:18.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:18.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:18.621 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:18.644 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:18.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:18.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:18.747 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:18.784 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:18.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:18.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:18.885 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:18 smithi098 bash[76817]: cluster 2023-09-23T18:36:17.582367+0000 osd.5 (osd.5) 49 : cluster 3 failed to encode map e139 with expected crc 2023-09-23T18:36:18.885 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:18 smithi098 bash[76817]: cluster 2023-09-23T18:36:17.583378+0000 mon.a (mon.0) 750 : cluster 0 osdmap e139: 8 total, 8 up, 7 in 2023-09-23T18:36:18.885 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:18 smithi098 bash[76817]: cluster 2023-09-23T18:36:17.585719+0000 osd.4 (osd.4) 73 : cluster 3 failed to encode map e139 with expected crc 2023-09-23T18:36:18.885 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:18 smithi098 bash[76817]: cluster 2023-09-23T18:36:17.621015+0000 osd.7 (osd.7) 48 : cluster 3 failed to encode map e139 with expected crc 2023-09-23T18:36:18.885 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:18 smithi098 bash[76817]: cluster 2023-09-23T18:36:17.624706+0000 osd.7 (osd.7) 49 : cluster 3 failed to encode map e139 with expected crc 2023-09-23T18:36:18.885 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:18 smithi098 bash[76817]: cluster 2023-09-23T18:36:17.801789+0000 mgr.y (mgr.26054) 531 : cluster 0 pgmap v497: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 337 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 37315/16020 objects degraded (232.928%); 1860/16020 objects misplaced (11.610%); 0 B/s, 59 objects/s recovering 2023-09-23T18:36:18.886 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:18 smithi098 bash[79516]: cluster 2023-09-23T18:36:17.582367+0000 osd.5 (osd.5) 49 : cluster 3 failed to encode map e139 with expected crc 2023-09-23T18:36:18.886 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:18 smithi098 bash[79516]: cluster 2023-09-23T18:36:17.583378+0000 mon.a (mon.0) 750 : cluster 0 osdmap e139: 8 total, 8 up, 7 in 2023-09-23T18:36:18.886 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:18 smithi098 bash[79516]: cluster 2023-09-23T18:36:17.585719+0000 osd.4 (osd.4) 73 : cluster 3 failed to encode map e139 with expected crc 2023-09-23T18:36:18.886 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:18 smithi098 bash[79516]: cluster 2023-09-23T18:36:17.621015+0000 osd.7 (osd.7) 48 : cluster 3 failed to encode map e139 with expected crc 2023-09-23T18:36:18.886 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:18 smithi098 bash[79516]: cluster 2023-09-23T18:36:17.624706+0000 osd.7 (osd.7) 49 : cluster 3 failed to encode map e139 with expected crc 2023-09-23T18:36:18.886 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:18 smithi098 bash[79516]: cluster 2023-09-23T18:36:17.801789+0000 mgr.y (mgr.26054) 531 : cluster 0 pgmap v497: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 337 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 37315/16020 objects degraded (232.928%); 1860/16020 objects misplaced (11.610%); 0 B/s, 59 objects/s recovering 2023-09-23T18:36:18.889 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:18.900 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:18 smithi173 bash[59422]: cluster 2023-09-23T18:36:17.582367+0000 osd.5 (osd.5) 49 : cluster 3 failed to encode map e139 with expected crc 2023-09-23T18:36:18.900 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:18 smithi173 bash[59422]: cluster 2023-09-23T18:36:17.583378+0000 mon.a (mon.0) 750 : cluster 0 osdmap e139: 8 total, 8 up, 7 in 2023-09-23T18:36:18.900 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:18 smithi173 bash[59422]: cluster 2023-09-23T18:36:17.585719+0000 osd.4 (osd.4) 73 : cluster 3 failed to encode map e139 with expected crc 2023-09-23T18:36:18.900 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:18 smithi173 bash[59422]: cluster 2023-09-23T18:36:17.621015+0000 osd.7 (osd.7) 48 : cluster 3 failed to encode map e139 with expected crc 2023-09-23T18:36:18.900 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:18 smithi173 bash[59422]: cluster 2023-09-23T18:36:17.624706+0000 osd.7 (osd.7) 49 : cluster 3 failed to encode map e139 with expected crc 2023-09-23T18:36:18.900 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:18 smithi173 bash[59422]: cluster 2023-09-23T18:36:17.801789+0000 mgr.y (mgr.26054) 531 : cluster 0 pgmap v497: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 337 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 37315/16020 objects degraded (232.928%); 1860/16020 objects misplaced (11.610%); 0 B/s, 59 objects/s recovering 2023-09-23T18:36:18.918 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:18.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:18.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:19.022 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:19.073 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:19.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:19.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:19.178 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:19.204 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:19.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:19.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:19.307 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:19.354 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:19.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:19.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:19.459 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:19.514 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:19.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:19.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:19.618 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:19.650 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:19.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:19.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:19.765 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:19.790 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:19.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:19.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:19.897 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:19.913 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:19 smithi173 bash[59422]: cluster 2023-09-23T18:36:18.591478+0000 mon.a (mon.0) 751 : cluster 0 osdmap e140: 8 total, 8 up, 7 in 2023-09-23T18:36:19.913 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:19 smithi173 bash[59422]: cluster 2023-09-23T18:36:18.593713+0000 osd.4 (osd.4) 74 : cluster 3 failed to encode map e140 with expected crc 2023-09-23T18:36:19.913 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:19 smithi173 bash[59422]: cluster 2023-09-23T18:36:18.594939+0000 osd.4 (osd.4) 75 : cluster 3 failed to encode map e140 with expected crc 2023-09-23T18:36:19.913 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:19 smithi173 bash[59422]: cluster 2023-09-23T18:36:18.616449+0000 osd.7 (osd.7) 50 : cluster 3 failed to encode map e140 with expected crc 2023-09-23T18:36:19.913 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:19 smithi173 bash[59422]: cluster 2023-09-23T18:36:18.674564+0000 osd.5 (osd.5) 50 : cluster 3 failed to encode map e140 with expected crc 2023-09-23T18:36:19.913 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:19 smithi173 bash[59422]: audit 2023-09-23T18:36:19.216545+0000 mon.a (mon.0) 752 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:19.913 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:19 smithi173 bash[59422]: audit 2023-09-23T18:36:19.225619+0000 mon.a (mon.0) 753 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:19.913 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:19 smithi173 bash[59422]: cluster 2023-09-23T18:36:19.494164+0000 mon.a (mon.0) 754 : cluster 3 Health check update: Degraded data redundancy: 37315/16020 objects degraded (232.928%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:19.920 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:19 smithi098 bash[79516]: cluster 2023-09-23T18:36:18.591478+0000 mon.a (mon.0) 751 : cluster 0 osdmap e140: 8 total, 8 up, 7 in 2023-09-23T18:36:19.920 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:19 smithi098 bash[79516]: cluster 2023-09-23T18:36:18.593713+0000 osd.4 (osd.4) 74 : cluster 3 failed to encode map e140 with expected crc 2023-09-23T18:36:19.920 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:19 smithi098 bash[79516]: cluster 2023-09-23T18:36:18.594939+0000 osd.4 (osd.4) 75 : cluster 3 failed to encode map e140 with expected crc 2023-09-23T18:36:19.920 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:19 smithi098 bash[79516]: cluster 2023-09-23T18:36:18.616449+0000 osd.7 (osd.7) 50 : cluster 3 failed to encode map e140 with expected crc 2023-09-23T18:36:19.921 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:19 smithi098 bash[79516]: cluster 2023-09-23T18:36:18.674564+0000 osd.5 (osd.5) 50 : cluster 3 failed to encode map e140 with expected crc 2023-09-23T18:36:19.921 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:19 smithi098 bash[79516]: audit 2023-09-23T18:36:19.216545+0000 mon.a (mon.0) 752 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:19.921 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:19 smithi098 bash[79516]: audit 2023-09-23T18:36:19.225619+0000 mon.a (mon.0) 753 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:19.921 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:19 smithi098 bash[79516]: cluster 2023-09-23T18:36:19.494164+0000 mon.a (mon.0) 754 : cluster 3 Health check update: Degraded data redundancy: 37315/16020 objects degraded (232.928%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:19.921 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:19 smithi098 bash[76817]: cluster 2023-09-23T18:36:18.591478+0000 mon.a (mon.0) 751 : cluster 0 osdmap e140: 8 total, 8 up, 7 in 2023-09-23T18:36:19.921 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:19 smithi098 bash[76817]: cluster 2023-09-23T18:36:18.593713+0000 osd.4 (osd.4) 74 : cluster 3 failed to encode map e140 with expected crc 2023-09-23T18:36:19.921 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:19 smithi098 bash[76817]: cluster 2023-09-23T18:36:18.594939+0000 osd.4 (osd.4) 75 : cluster 3 failed to encode map e140 with expected crc 2023-09-23T18:36:19.921 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:19 smithi098 bash[76817]: cluster 2023-09-23T18:36:18.616449+0000 osd.7 (osd.7) 50 : cluster 3 failed to encode map e140 with expected crc 2023-09-23T18:36:19.922 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:19 smithi098 bash[76817]: cluster 2023-09-23T18:36:18.674564+0000 osd.5 (osd.5) 50 : cluster 3 failed to encode map e140 with expected crc 2023-09-23T18:36:19.922 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:19 smithi098 bash[76817]: audit 2023-09-23T18:36:19.216545+0000 mon.a (mon.0) 752 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:19.922 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:19 smithi098 bash[76817]: audit 2023-09-23T18:36:19.225619+0000 mon.a (mon.0) 753 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:19.922 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:19 smithi098 bash[76817]: cluster 2023-09-23T18:36:19.494164+0000 mon.a (mon.0) 754 : cluster 3 Health check update: Degraded data redundancy: 37315/16020 objects degraded (232.928%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:19.932 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:19.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:19.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:20.038 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:20.063 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:20.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:20.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:20.167 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:20.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:20.197 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:20.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:20.298 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:20.329 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:20.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:20.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:20.436 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:20.463 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:20.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:20.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:20.566 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:20.592 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:20.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:20.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:20.695 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:20.739 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:20.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:20.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:20.844 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:20.853 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:20 smithi098 bash[76817]: cluster 2023-09-23T18:36:19.802685+0000 mgr.y (mgr.26054) 532 : cluster 0 pgmap v499: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 253 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 36750/12021 objects degraded (305.715%); 1860/12021 objects misplaced (15.473%); 0 B/s, 54 objects/s recovering 2023-09-23T18:36:20.854 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:20 smithi098 bash[79516]: cluster 2023-09-23T18:36:19.802685+0000 mgr.y (mgr.26054) 532 : cluster 0 pgmap v499: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 253 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 36750/12021 objects degraded (305.715%); 1860/12021 objects misplaced (15.473%); 0 B/s, 54 objects/s recovering 2023-09-23T18:36:20.873 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:20.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:20.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:20.978 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:21.007 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:21.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:21.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:21.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:20 smithi173 bash[59422]: cluster 2023-09-23T18:36:19.802685+0000 mgr.y (mgr.26054) 532 : cluster 0 pgmap v499: 89 pgs: 1 active+recovery_wait+degraded+remapped, 78 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 253 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 36750/12021 objects degraded (305.715%); 1860/12021 objects misplaced (15.473%); 0 B/s, 54 objects/s recovering 2023-09-23T18:36:21.112 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:21.138 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:21.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:21.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:21.248 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:21.282 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:21.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:21.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:21.389 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:21.424 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:21.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:21.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:21.527 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:21.549 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:21.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:21.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:21.655 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:21.688 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:21.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:21.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:21.796 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:21.834 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:21.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:21.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:21.941 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:21.972 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:21.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:21.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:22.075 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:22.098 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:22.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:22.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:22.203 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:22.214 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:21 smithi173 bash[59422]: audit 2023-09-23T18:36:20.964882+0000 mon.a (mon.0) 755 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:22.214 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:21 smithi173 bash[59422]: audit 2023-09-23T18:36:20.974925+0000 mon.a (mon.0) 756 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:22.214 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:21 smithi173 bash[59422]: audit 2023-09-23T18:36:21.564673+0000 mon.a (mon.0) 757 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:36:22.214 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:21 smithi173 bash[59422]: audit 2023-09-23T18:36:21.565904+0000 mon.a (mon.0) 758 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:36:22.215 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:21 smithi173 bash[59422]: audit 2023-09-23T18:36:21.584590+0000 mon.a (mon.0) 759 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:22.215 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:21 smithi173 bash[59422]: cluster 2023-09-23T18:36:21.803490+0000 mgr.y (mgr.26054) 533 : cluster 0 pgmap v500: 89 pgs: 1 active+recovery_wait+degraded+remapped, 79 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 191 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 36588/9024 objects degraded (405.452%); 1860/9024 objects misplaced (20.612%); 0 B/s, 74 objects/s recovering 2023-09-23T18:36:22.232 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:22.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:22.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:22.336 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:22.344 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:21 smithi098 bash[76817]: audit 2023-09-23T18:36:20.964882+0000 mon.a (mon.0) 755 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:22.344 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:21 smithi098 bash[76817]: audit 2023-09-23T18:36:20.974925+0000 mon.a (mon.0) 756 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:22.345 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:21 smithi098 bash[76817]: audit 2023-09-23T18:36:21.564673+0000 mon.a (mon.0) 757 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:36:22.345 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:21 smithi098 bash[76817]: audit 2023-09-23T18:36:21.565904+0000 mon.a (mon.0) 758 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:36:22.345 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:21 smithi098 bash[76817]: audit 2023-09-23T18:36:21.584590+0000 mon.a (mon.0) 759 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:22.345 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:21 smithi098 bash[76817]: cluster 2023-09-23T18:36:21.803490+0000 mgr.y (mgr.26054) 533 : cluster 0 pgmap v500: 89 pgs: 1 active+recovery_wait+degraded+remapped, 79 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 191 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 36588/9024 objects degraded (405.452%); 1860/9024 objects misplaced (20.612%); 0 B/s, 74 objects/s recovering 2023-09-23T18:36:22.346 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:21 smithi098 bash[79516]: audit 2023-09-23T18:36:20.964882+0000 mon.a (mon.0) 755 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:22.349 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:21 smithi098 bash[79516]: audit 2023-09-23T18:36:20.974925+0000 mon.a (mon.0) 756 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:22.349 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:21 smithi098 bash[79516]: audit 2023-09-23T18:36:21.564673+0000 mon.a (mon.0) 757 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:36:22.349 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:21 smithi098 bash[79516]: audit 2023-09-23T18:36:21.565904+0000 mon.a (mon.0) 758 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:36:22.349 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:21 smithi098 bash[79516]: audit 2023-09-23T18:36:21.584590+0000 mon.a (mon.0) 759 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:22.349 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:21 smithi098 bash[79516]: cluster 2023-09-23T18:36:21.803490+0000 mgr.y (mgr.26054) 533 : cluster 0 pgmap v500: 89 pgs: 1 active+recovery_wait+degraded+remapped, 79 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 191 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 36588/9024 objects degraded (405.452%); 1860/9024 objects misplaced (20.612%); 0 B/s, 74 objects/s recovering 2023-09-23T18:36:22.361 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:22.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:22.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:22.468 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:22.497 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:22.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:22.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:22.599 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:22.625 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:22.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:22.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:22.729 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:22.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:22.758 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:22.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:22.859 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:22.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:22.884 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:22.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:22.985 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:23.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:23.011 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:23.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:23.112 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:23.135 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:23.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:23.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:23.240 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:23.267 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:23.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:23.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:23.370 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:23.394 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:23.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:23.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:23.497 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:23.521 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:23.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:23.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:23.625 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:23.659 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:23.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:23.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:23.764 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:23.790 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:23.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:23.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:23.898 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:23.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:23.939 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:23.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:24.040 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:24.071 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:24.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:24.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:24.174 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:24.203 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:24.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:24.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:24.311 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:24.337 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:24.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:24.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:24.444 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:24.478 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:24.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:24.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:24.586 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:24.612 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:24.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:24.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:24.714 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:24.741 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:24.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:24.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:24.844 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:24.864 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:24 smithi098 bash[76817]: cluster 2023-09-23T18:36:23.804221+0000 mgr.y (mgr.26054) 534 : cluster 0 pgmap v501: 89 pgs: 1 active+recovery_wait+degraded+remapped, 79 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 145 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 465 op/s; 36588/6837 objects degraded (535.147%); 1860/6837 objects misplaced (27.205%); 0 B/s, 74 objects/s recovering 2023-09-23T18:36:24.864 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:24 smithi098 bash[76817]: cluster 2023-09-23T18:36:24.512342+0000 mon.a (mon.0) 760 : cluster 3 Health check update: Degraded data redundancy: 36588/6837 objects degraded (535.147%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:24.871 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:24.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:24.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:24.974 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:25.005 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:25.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:25.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:25.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:24 smithi173 bash[59422]: cluster 2023-09-23T18:36:23.804221+0000 mgr.y (mgr.26054) 534 : cluster 0 pgmap v501: 89 pgs: 1 active+recovery_wait+degraded+remapped, 79 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 145 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 465 op/s; 36588/6837 objects degraded (535.147%); 1860/6837 objects misplaced (27.205%); 0 B/s, 74 objects/s recovering 2023-09-23T18:36:25.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:24 smithi173 bash[59422]: cluster 2023-09-23T18:36:24.512342+0000 mon.a (mon.0) 760 : cluster 3 Health check update: Degraded data redundancy: 36588/6837 objects degraded (535.147%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:25.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:24 smithi173 bash[59422]: audit 2023-09-23T18:36:24.591874+0000 mon.a (mon.0) 761 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:36:25.112 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:25.120 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:24 smithi098 bash[79516]: cluster 2023-09-23T18:36:23.804221+0000 mgr.y (mgr.26054) 534 : cluster 0 pgmap v501: 89 pgs: 1 active+recovery_wait+degraded+remapped, 79 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 145 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 465 op/s; 36588/6837 objects degraded (535.147%); 1860/6837 objects misplaced (27.205%); 0 B/s, 74 objects/s recovering 2023-09-23T18:36:25.120 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:24 smithi098 bash[79516]: cluster 2023-09-23T18:36:24.512342+0000 mon.a (mon.0) 760 : cluster 3 Health check update: Degraded data redundancy: 36588/6837 objects degraded (535.147%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:25.121 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:24 smithi098 bash[79516]: audit 2023-09-23T18:36:24.591874+0000 mon.a (mon.0) 761 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:36:25.124 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:24 smithi098 bash[76817]: audit 2023-09-23T18:36:24.591874+0000 mon.a (mon.0) 761 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:36:25.139 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:25.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:25.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:25.243 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:25.268 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:25.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:25.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:25.374 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:25.413 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:25.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:25.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:25.517 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:25.533 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:25.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:25.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:25.636 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:25.663 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:25.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:25.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:25.720 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Removed 106282 objects 2023-09-23T18:36:25.720 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Clean up completed and total clean up time :203.099 2023-09-23T18:36:25.767 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:25.785 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:25.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:25.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:25.889 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:25.909 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:25.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:25.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:26.011 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:26.034 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:26.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:26.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:26.139 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:26.170 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:26.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:26.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:26.190 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:36:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:36:26] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.33.4" 2023-09-23T18:36:26.273 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:26.291 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:26.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:26.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:26.394 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:26.424 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:26.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:26.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:26.527 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:26.552 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:26.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:26.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:26.655 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:26.686 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:26.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:26.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:26.789 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:26.818 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:26.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:26.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:26.922 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:26.933 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:26 smithi173 bash[59422]: cluster 2023-09-23T18:36:25.805352+0000 mgr.y (mgr.26054) 535 : cluster 0 pgmap v502: 89 pgs: 1 active+recovery_wait+degraded+remapped, 79 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 72 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 35793/3309 objects degraded (1081.686%); 1860/3309 objects misplaced (56.210%); 0 B/s, 69 objects/s recovering 2023-09-23T18:36:26.948 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:26.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:26.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:27.051 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:27.071 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:27.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:27.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:27.173 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:27.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:26 smithi098 bash[79516]: cluster 2023-09-23T18:36:25.805352+0000 mgr.y (mgr.26054) 535 : cluster 0 pgmap v502: 89 pgs: 1 active+recovery_wait+degraded+remapped, 79 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 72 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 35793/3309 objects degraded (1081.686%); 1860/3309 objects misplaced (56.210%); 0 B/s, 69 objects/s recovering 2023-09-23T18:36:27.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:26 smithi098 bash[76817]: cluster 2023-09-23T18:36:25.805352+0000 mgr.y (mgr.26054) 535 : cluster 0 pgmap v502: 89 pgs: 1 active+recovery_wait+degraded+remapped, 79 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 72 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 35793/3309 objects degraded (1081.686%); 1860/3309 objects misplaced (56.210%); 0 B/s, 69 objects/s recovering 2023-09-23T18:36:27.206 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:27.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:27.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:27.310 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:27.334 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:27.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:27.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:27.438 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:27.465 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:27.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:27.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:27.567 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:27.593 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:27.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:27.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:27.696 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:27.723 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:27.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:27.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:27.826 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:27.847 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:27.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:27.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:27.884 INFO:teuthology.orchestra.run.smithi098.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T18:36:27.950 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:27.970 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:27.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:27.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:28.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:27 smithi173 bash[59422]: audit 2023-09-23T18:36:26.980446+0000 mon.c (mon.2) 71 : audit 1 from='client.? 172.21.15.98:0/3264132000' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T18:36:28.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:27 smithi173 bash[59422]: audit 2023-09-23T18:36:26.980765+0000 mon.a (mon.0) 762 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T18:36:28.074 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:28.096 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:27 smithi098 bash[79516]: audit 2023-09-23T18:36:26.980446+0000 mon.c (mon.2) 71 : audit 1 from='client.? 172.21.15.98:0/3264132000' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T18:36:28.096 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:27 smithi098 bash[79516]: audit 2023-09-23T18:36:26.980765+0000 mon.a (mon.0) 762 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T18:36:28.097 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:27 smithi098 bash[76817]: audit 2023-09-23T18:36:26.980446+0000 mon.c (mon.2) 71 : audit 1 from='client.? 172.21.15.98:0/3264132000' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T18:36:28.097 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:27 smithi098 bash[76817]: audit 2023-09-23T18:36:26.980765+0000 mon.a (mon.0) 762 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T18:36:28.097 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:28.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:28.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:28.199 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:28.230 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:28.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:28.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:28.333 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:28.360 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:28.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:28.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:28.463 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:28.487 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:28.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:28.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:28.591 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:28.619 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:28.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:28.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:28.723 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:28.750 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:28.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:28.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:28.853 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:28.880 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:28.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:28.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:28.984 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:29.006 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:29.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:29.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:29.110 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:29.134 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:29.136 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:28 smithi098 bash[79516]: cluster 2023-09-23T18:36:27.805991+0000 mgr.y (mgr.26054) 536 : cluster 0 pgmap v503: 89 pgs: 1 active+recovery_wait+degraded+remapped, 79 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 36 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 35793/1575 objects degraded (2272.571%); 1860/1575 objects misplaced (118.095%); 0 B/s, 56 objects/s recovering 2023-09-23T18:36:29.136 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:28 smithi098 bash[79516]: cluster 2023-09-23T18:36:27.877340+0000 osd.4 (osd.4) 76 : cluster 3 failed to encode map e141 with expected crc 2023-09-23T18:36:29.136 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:28 smithi098 bash[79516]: cluster 2023-09-23T18:36:27.877345+0000 osd.7 (osd.7) 51 : cluster 3 failed to encode map e141 with expected crc 2023-09-23T18:36:29.136 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:28 smithi098 bash[79516]: audit 2023-09-23T18:36:27.878938+0000 mon.a (mon.0) 763 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T18:36:29.136 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:28 smithi098 bash[79516]: cluster 2023-09-23T18:36:27.879079+0000 mon.a (mon.0) 764 : cluster 0 osdmap e141: 8 total, 8 up, 7 in 2023-09-23T18:36:29.136 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:28 smithi098 bash[79516]: cluster 2023-09-23T18:36:28.219853+0000 osd.6 (osd.6) 55 : cluster 3 failed to encode map e141 with expected crc 2023-09-23T18:36:29.136 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:28 smithi098 bash[79516]: cluster 2023-09-23T18:36:28.261351+0000 osd.5 (osd.5) 51 : cluster 3 failed to encode map e141 with expected crc 2023-09-23T18:36:29.137 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:28 smithi098 bash[76817]: cluster 2023-09-23T18:36:27.805991+0000 mgr.y (mgr.26054) 536 : cluster 0 pgmap v503: 89 pgs: 1 active+recovery_wait+degraded+remapped, 79 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 36 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 35793/1575 objects degraded (2272.571%); 1860/1575 objects misplaced (118.095%); 0 B/s, 56 objects/s recovering 2023-09-23T18:36:29.137 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:28 smithi098 bash[76817]: cluster 2023-09-23T18:36:27.877340+0000 osd.4 (osd.4) 76 : cluster 3 failed to encode map e141 with expected crc 2023-09-23T18:36:29.137 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:28 smithi098 bash[76817]: cluster 2023-09-23T18:36:27.877345+0000 osd.7 (osd.7) 51 : cluster 3 failed to encode map e141 with expected crc 2023-09-23T18:36:29.137 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:28 smithi098 bash[76817]: audit 2023-09-23T18:36:27.878938+0000 mon.a (mon.0) 763 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T18:36:29.137 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:28 smithi098 bash[76817]: cluster 2023-09-23T18:36:27.879079+0000 mon.a (mon.0) 764 : cluster 0 osdmap e141: 8 total, 8 up, 7 in 2023-09-23T18:36:29.137 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:28 smithi098 bash[76817]: cluster 2023-09-23T18:36:28.219853+0000 osd.6 (osd.6) 55 : cluster 3 failed to encode map e141 with expected crc 2023-09-23T18:36:29.137 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:28 smithi098 bash[76817]: cluster 2023-09-23T18:36:28.261351+0000 osd.5 (osd.5) 51 : cluster 3 failed to encode map e141 with expected crc 2023-09-23T18:36:29.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:29.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:29.239 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:29.250 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:28 smithi173 bash[59422]: cluster 2023-09-23T18:36:27.805991+0000 mgr.y (mgr.26054) 536 : cluster 0 pgmap v503: 89 pgs: 1 active+recovery_wait+degraded+remapped, 79 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 36 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 35793/1575 objects degraded (2272.571%); 1860/1575 objects misplaced (118.095%); 0 B/s, 56 objects/s recovering 2023-09-23T18:36:29.250 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:28 smithi173 bash[59422]: cluster 2023-09-23T18:36:27.877340+0000 osd.4 (osd.4) 76 : cluster 3 failed to encode map e141 with expected crc 2023-09-23T18:36:29.250 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:28 smithi173 bash[59422]: cluster 2023-09-23T18:36:27.877345+0000 osd.7 (osd.7) 51 : cluster 3 failed to encode map e141 with expected crc 2023-09-23T18:36:29.250 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:28 smithi173 bash[59422]: audit 2023-09-23T18:36:27.878938+0000 mon.a (mon.0) 763 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T18:36:29.250 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:28 smithi173 bash[59422]: cluster 2023-09-23T18:36:27.879079+0000 mon.a (mon.0) 764 : cluster 0 osdmap e141: 8 total, 8 up, 7 in 2023-09-23T18:36:29.250 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:28 smithi173 bash[59422]: cluster 2023-09-23T18:36:28.219853+0000 osd.6 (osd.6) 55 : cluster 3 failed to encode map e141 with expected crc 2023-09-23T18:36:29.250 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:28 smithi173 bash[59422]: cluster 2023-09-23T18:36:28.261351+0000 osd.5 (osd.5) 51 : cluster 3 failed to encode map e141 with expected crc 2023-09-23T18:36:29.267 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:29.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:29.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:29.370 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:29.402 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:29.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:29.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:29.505 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:29.533 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:29.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:29.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:29.635 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:29.657 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:29.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:29.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:29.760 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:29.783 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:29.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:29.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:29.886 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:29.907 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:29.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:29.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:30.010 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:30.021 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:29 smithi098 bash[79516]: cluster 2023-09-23T18:36:29.513549+0000 mon.a (mon.0) 765 : cluster 3 Health check update: Degraded data redundancy: 35793/1575 objects degraded (2272.571%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:30.021 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:29 smithi098 bash[76817]: cluster 2023-09-23T18:36:29.513549+0000 mon.a (mon.0) 765 : cluster 3 Health check update: Degraded data redundancy: 35793/1575 objects degraded (2272.571%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:30.034 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:30.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:30.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:30.137 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:30.158 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:30.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:30.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:30.261 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:30.272 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:29 smithi173 bash[59422]: cluster 2023-09-23T18:36:29.513549+0000 mon.a (mon.0) 765 : cluster 3 Health check update: Degraded data redundancy: 35793/1575 objects degraded (2272.571%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:30.286 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:30.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:30.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:30.389 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:30.419 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:30.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:30.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:30.505 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-09-23T18:36:30.505 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-09-23T18:36:30.523 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:30.574 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:30.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:30.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:30.677 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:30.699 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:30.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:30.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:30.802 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:30.830 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:30.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:30.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:30.934 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:30.966 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:30.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:30.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:31.069 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:31.082 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:30 smithi173 bash[59422]: cluster 2023-09-23T18:36:29.806581+0000 mgr.y (mgr.26054) 537 : cluster 0 pgmap v505: 89 pgs: 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 79 active+clean, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 397 op/s; 34912/87 objects degraded (40128.736%); 1860/87 objects misplaced (2137.931%); 0 B/s, 119 objects/s recovering 2023-09-23T18:36:31.101 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:31.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:31.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:31.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:30 smithi098 bash[79516]: cluster 2023-09-23T18:36:29.806581+0000 mgr.y (mgr.26054) 537 : cluster 0 pgmap v505: 89 pgs: 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 79 active+clean, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 397 op/s; 34912/87 objects degraded (40128.736%); 1860/87 objects misplaced (2137.931%); 0 B/s, 119 objects/s recovering 2023-09-23T18:36:31.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:30 smithi098 bash[76817]: cluster 2023-09-23T18:36:29.806581+0000 mgr.y (mgr.26054) 537 : cluster 0 pgmap v505: 89 pgs: 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 79 active+clean, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 397 op/s; 34912/87 objects degraded (40128.736%); 1860/87 objects misplaced (2137.931%); 0 B/s, 119 objects/s recovering 2023-09-23T18:36:31.204 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:31.227 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:31.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:31.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:31.330 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:31.349 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:31.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:31.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:31.452 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:31.484 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:31.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:31.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:31.587 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:31.614 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:31.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:31.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:31.718 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:31.745 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:31.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:31.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:31.848 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:31.872 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:31.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:31.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:31.918 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T18:36:31.918 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:36:31.975 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:32.003 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:36:32.007 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:32.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:32.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:32.111 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:32.136 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:32.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:32.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:32.227 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:36:32.240 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:32.252 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:36:32.259 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:32.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:32.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:32.276 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:36:32.301 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:36:32.323 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:36:32.344 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:36:32.362 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:32.367 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:36:32.388 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:36:32.391 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:36:32.391 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:36:32.391 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:36:32.391 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:36:32.391 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:36:32.391 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:36:32.392 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:36:32.392 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:36:32.392 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:32.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:32.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:32.496 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:32.517 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:32.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:32.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:32.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:32.652 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:32.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:32.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:32.756 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:32.780 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:32.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:32.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:32.883 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:32.884 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:32 smithi098 bash[76817]: cluster 2023-09-23T18:36:31.807589+0000 mgr.y (mgr.26054) 538 : cluster 0 pgmap v506: 89 pgs: 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 79 active+clean, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 297 op/s; 34796/84 objects degraded (41423.810%); 1860/84 objects misplaced (2214.286%); 0 B/s, 114 objects/s recovering 2023-09-23T18:36:32.884 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:32 smithi098 bash[76817]: audit 2023-09-23T18:36:32.201304+0000 mon.a (mon.0) 766 : audit 0 from='client.? 172.21.15.98:0/1053645509' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:36:32.908 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:32.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:32.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:33.011 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:33.037 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:33.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:33.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:33.140 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:33.154 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:32 smithi098 bash[79516]: cluster 2023-09-23T18:36:31.807589+0000 mgr.y (mgr.26054) 538 : cluster 0 pgmap v506: 89 pgs: 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 79 active+clean, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 297 op/s; 34796/84 objects degraded (41423.810%); 1860/84 objects misplaced (2214.286%); 0 B/s, 114 objects/s recovering 2023-09-23T18:36:33.154 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:32 smithi098 bash[79516]: audit 2023-09-23T18:36:32.201304+0000 mon.a (mon.0) 766 : audit 0 from='client.? 172.21.15.98:0/1053645509' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:36:33.160 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:33.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:33.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:33.263 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:33.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:32 smithi173 bash[59422]: cluster 2023-09-23T18:36:31.807589+0000 mgr.y (mgr.26054) 538 : cluster 0 pgmap v506: 89 pgs: 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 79 active+clean, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 297 op/s; 34796/84 objects degraded (41423.810%); 1860/84 objects misplaced (2214.286%); 0 B/s, 114 objects/s recovering 2023-09-23T18:36:33.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:32 smithi173 bash[59422]: audit 2023-09-23T18:36:32.201304+0000 mon.a (mon.0) 766 : audit 0 from='client.? 172.21.15.98:0/1053645509' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:36:33.292 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:33.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:33.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:33.395 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:33.425 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:33.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:33.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:33.528 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:33.549 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:33.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:33.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:33.652 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:33.673 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:33.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:33.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:33.776 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:33.802 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:33.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:33.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:33.905 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:33.930 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:33.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:33.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:34.033 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:34.065 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:34.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:34.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:34.168 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:34.189 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:34.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:34.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:34.292 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:34.312 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:34.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:34.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:34.415 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:34.441 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:34.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:34.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:34.545 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:34.576 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:34.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:34.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:34.680 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:34.708 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:34.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:34.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:34.813 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:34.839 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:34.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:34.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:34.900 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is unset 2023-09-23T18:36:34.942 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:34.965 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:34.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:34.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:35.068 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:35.092 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:35.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:35.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:35.108 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:34 smithi098 bash[79516]: cluster 2023-09-23T18:36:33.808295+0000 mgr.y (mgr.26054) 539 : cluster 0 pgmap v507: 89 pgs: 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 79 active+clean, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 225 op/s; 34796/84 objects degraded (41423.810%); 1860/84 objects misplaced (2214.286%); 0 B/s, 114 objects/s recovering 2023-09-23T18:36:35.109 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:34 smithi098 bash[79516]: audit 2023-09-23T18:36:34.514488+0000 mon.c (mon.2) 72 : audit 1 from='client.? 172.21.15.98:0/4154183175' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:36:35.109 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:34 smithi098 bash[79516]: audit 2023-09-23T18:36:34.514894+0000 mon.a (mon.0) 767 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:36:35.109 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:34 smithi098 bash[79516]: cluster 2023-09-23T18:36:34.515353+0000 mon.a (mon.0) 768 : cluster 3 Health check update: Degraded data redundancy: 34796/84 objects degraded (41423.810%), 7 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:35.109 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:34 smithi098 bash[76817]: cluster 2023-09-23T18:36:33.808295+0000 mgr.y (mgr.26054) 539 : cluster 0 pgmap v507: 89 pgs: 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 79 active+clean, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 225 op/s; 34796/84 objects degraded (41423.810%); 1860/84 objects misplaced (2214.286%); 0 B/s, 114 objects/s recovering 2023-09-23T18:36:35.109 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:34 smithi098 bash[76817]: audit 2023-09-23T18:36:34.514488+0000 mon.c (mon.2) 72 : audit 1 from='client.? 172.21.15.98:0/4154183175' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:36:35.109 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:34 smithi098 bash[76817]: audit 2023-09-23T18:36:34.514894+0000 mon.a (mon.0) 767 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:36:35.109 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:34 smithi098 bash[76817]: cluster 2023-09-23T18:36:34.515353+0000 mon.a (mon.0) 768 : cluster 3 Health check update: Degraded data redundancy: 34796/84 objects degraded (41423.810%), 7 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:35.196 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:35.218 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:35.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:35.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:35.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:34 smithi173 bash[59422]: cluster 2023-09-23T18:36:33.808295+0000 mgr.y (mgr.26054) 539 : cluster 0 pgmap v507: 89 pgs: 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 79 active+clean, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 225 op/s; 34796/84 objects degraded (41423.810%); 1860/84 objects misplaced (2214.286%); 0 B/s, 114 objects/s recovering 2023-09-23T18:36:35.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:34 smithi173 bash[59422]: audit 2023-09-23T18:36:34.514488+0000 mon.c (mon.2) 72 : audit 1 from='client.? 172.21.15.98:0/4154183175' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:36:35.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:34 smithi173 bash[59422]: audit 2023-09-23T18:36:34.514894+0000 mon.a (mon.0) 767 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:36:35.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:34 smithi173 bash[59422]: cluster 2023-09-23T18:36:34.515353+0000 mon.a (mon.0) 768 : cluster 3 Health check update: Degraded data redundancy: 34796/84 objects degraded (41423.810%), 7 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:35.322 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:35.347 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:35.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:35.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:35.451 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:35.475 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:35.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:35.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:35.578 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:35.600 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:35.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:35.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:35.702 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:35.727 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:35.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:35.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:35.831 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:35.860 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:35.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:35.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:35.964 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:35.984 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:35.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:35.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:35 smithi173 bash[59422]: cluster 2023-09-23T18:36:34.884347+0000 mon.a (mon.0) 769 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:36:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:35 smithi173 bash[59422]: cluster 2023-09-23T18:36:34.888498+0000 osd.4 (osd.4) 77 : cluster 3 failed to encode map e142 with expected crc 2023-09-23T18:36:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:35 smithi173 bash[59422]: audit 2023-09-23T18:36:34.892452+0000 mon.a (mon.0) 770 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:36:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:35 smithi173 bash[59422]: cluster 2023-09-23T18:36:34.892629+0000 mon.a (mon.0) 771 : cluster 0 osdmap e142: 8 total, 8 up, 7 in 2023-09-23T18:36:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:35 smithi173 bash[59422]: cluster 2023-09-23T18:36:34.902774+0000 osd.7 (osd.7) 52 : cluster 3 failed to encode map e142 with expected crc 2023-09-23T18:36:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:35 smithi173 bash[59422]: cluster 2023-09-23T18:36:35.133148+0000 osd.6 (osd.6) 56 : cluster 3 failed to encode map e142 with expected crc 2023-09-23T18:36:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:35 smithi173 bash[59422]: cluster 2023-09-23T18:36:35.133195+0000 osd.5 (osd.5) 52 : cluster 3 failed to encode map e142 with expected crc 2023-09-23T18:36:36.087 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:36.114 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:36.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:36.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:36.190 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:36:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:36:36] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.33.4" 2023-09-23T18:36:36.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:35 smithi098 bash[79516]: cluster 2023-09-23T18:36:34.884347+0000 mon.a (mon.0) 769 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:36:36.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:35 smithi098 bash[79516]: cluster 2023-09-23T18:36:34.888498+0000 osd.4 (osd.4) 77 : cluster 3 failed to encode map e142 with expected crc 2023-09-23T18:36:36.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:35 smithi098 bash[79516]: audit 2023-09-23T18:36:34.892452+0000 mon.a (mon.0) 770 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:36:36.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:35 smithi098 bash[79516]: cluster 2023-09-23T18:36:34.892629+0000 mon.a (mon.0) 771 : cluster 0 osdmap e142: 8 total, 8 up, 7 in 2023-09-23T18:36:36.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:35 smithi098 bash[79516]: cluster 2023-09-23T18:36:34.902774+0000 osd.7 (osd.7) 52 : cluster 3 failed to encode map e142 with expected crc 2023-09-23T18:36:36.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:35 smithi098 bash[79516]: cluster 2023-09-23T18:36:35.133148+0000 osd.6 (osd.6) 56 : cluster 3 failed to encode map e142 with expected crc 2023-09-23T18:36:36.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:35 smithi098 bash[79516]: cluster 2023-09-23T18:36:35.133195+0000 osd.5 (osd.5) 52 : cluster 3 failed to encode map e142 with expected crc 2023-09-23T18:36:36.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:35 smithi098 bash[76817]: cluster 2023-09-23T18:36:34.884347+0000 mon.a (mon.0) 769 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:36:36.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:35 smithi098 bash[76817]: cluster 2023-09-23T18:36:34.888498+0000 osd.4 (osd.4) 77 : cluster 3 failed to encode map e142 with expected crc 2023-09-23T18:36:36.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:35 smithi098 bash[76817]: audit 2023-09-23T18:36:34.892452+0000 mon.a (mon.0) 770 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:36:36.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:35 smithi098 bash[76817]: cluster 2023-09-23T18:36:34.892629+0000 mon.a (mon.0) 771 : cluster 0 osdmap e142: 8 total, 8 up, 7 in 2023-09-23T18:36:36.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:35 smithi098 bash[76817]: cluster 2023-09-23T18:36:34.902774+0000 osd.7 (osd.7) 52 : cluster 3 failed to encode map e142 with expected crc 2023-09-23T18:36:36.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:35 smithi098 bash[76817]: cluster 2023-09-23T18:36:35.133148+0000 osd.6 (osd.6) 56 : cluster 3 failed to encode map e142 with expected crc 2023-09-23T18:36:36.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:35 smithi098 bash[76817]: cluster 2023-09-23T18:36:35.133195+0000 osd.5 (osd.5) 52 : cluster 3 failed to encode map e142 with expected crc 2023-09-23T18:36:36.217 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:36.242 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:36.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:36.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:36.346 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:36.374 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:36.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:36.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:36.477 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:36.502 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:36.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:36.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:36.605 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:36.633 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:36.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:36.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:36.735 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:36.761 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:36.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:36.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:36.852 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:36:36.865 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:36.926 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:36.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:36.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:37.030 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:37.055 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:37.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:37.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:37.159 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:37.166 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:36 smithi098 bash[76817]: cluster 2023-09-23T18:36:35.808995+0000 mgr.y (mgr.26054) 540 : cluster 0 pgmap v509: 89 pgs: 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 79 active+clean, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 62 op/s; 33709/84 objects degraded (40129.762%); 1860/84 objects misplaced (2214.286%); 0 B/s, 244 objects/s recovering 2023-09-23T18:36:37.166 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:36 smithi098 bash[79516]: cluster 2023-09-23T18:36:35.808995+0000 mgr.y (mgr.26054) 540 : cluster 0 pgmap v509: 89 pgs: 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 79 active+clean, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 62 op/s; 33709/84 objects degraded (40129.762%); 1860/84 objects misplaced (2214.286%); 0 B/s, 244 objects/s recovering 2023-09-23T18:36:37.180 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:37.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:37.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:37.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:36 smithi173 bash[59422]: cluster 2023-09-23T18:36:35.808995+0000 mgr.y (mgr.26054) 540 : cluster 0 pgmap v509: 89 pgs: 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 79 active+clean, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 62 op/s; 33709/84 objects degraded (40129.762%); 1860/84 objects misplaced (2214.286%); 0 B/s, 244 objects/s recovering 2023-09-23T18:36:37.283 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:37.308 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:37.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:37.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:37.412 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:37.439 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:37.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:37.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:37.542 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:37.574 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:37.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:37.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:37.677 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:37.701 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:37.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:37.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:37.804 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:37.832 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:37.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:37.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:37.937 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:37.957 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:37.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:37.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:38.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:37 smithi173 bash[59422]: cluster 2023-09-23T18:36:36.912519+0000 osd.5 (osd.5) 53 : cluster 3 failed to encode map e143 with expected crc 2023-09-23T18:36:38.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:37 smithi173 bash[59422]: cluster 2023-09-23T18:36:36.914586+0000 mon.a (mon.0) 772 : cluster 0 osdmap e143: 8 total, 8 up, 7 in 2023-09-23T18:36:38.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:37 smithi173 bash[59422]: cluster 2023-09-23T18:36:36.926916+0000 osd.6 (osd.6) 57 : cluster 3 failed to encode map e143 with expected crc 2023-09-23T18:36:38.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:37 smithi173 bash[59422]: cluster 2023-09-23T18:36:36.927425+0000 osd.7 (osd.7) 53 : cluster 3 failed to encode map e143 with expected crc 2023-09-23T18:36:38.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:37 smithi173 bash[59422]: cluster 2023-09-23T18:36:36.936143+0000 osd.4 (osd.4) 78 : cluster 3 failed to encode map e143 with expected crc 2023-09-23T18:36:38.061 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:38.090 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:38.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:38.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:38.194 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:38.222 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:38.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:38.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:38.326 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:38.357 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:38.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:38.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:38.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:38 smithi098 bash[79516]: cluster 2023-09-23T18:36:36.912519+0000 osd.5 (osd.5) 53 : cluster 3 failed to encode map e143 with expected crc 2023-09-23T18:36:38.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:38 smithi098 bash[79516]: cluster 2023-09-23T18:36:36.914586+0000 mon.a (mon.0) 772 : cluster 0 osdmap e143: 8 total, 8 up, 7 in 2023-09-23T18:36:38.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:38 smithi098 bash[79516]: cluster 2023-09-23T18:36:36.926916+0000 osd.6 (osd.6) 57 : cluster 3 failed to encode map e143 with expected crc 2023-09-23T18:36:38.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:38 smithi098 bash[79516]: cluster 2023-09-23T18:36:36.927425+0000 osd.7 (osd.7) 53 : cluster 3 failed to encode map e143 with expected crc 2023-09-23T18:36:38.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:38 smithi098 bash[79516]: cluster 2023-09-23T18:36:36.936143+0000 osd.4 (osd.4) 78 : cluster 3 failed to encode map e143 with expected crc 2023-09-23T18:36:38.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:38 smithi098 bash[76817]: cluster 2023-09-23T18:36:36.912519+0000 osd.5 (osd.5) 53 : cluster 3 failed to encode map e143 with expected crc 2023-09-23T18:36:38.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:38 smithi098 bash[76817]: cluster 2023-09-23T18:36:36.914586+0000 mon.a (mon.0) 772 : cluster 0 osdmap e143: 8 total, 8 up, 7 in 2023-09-23T18:36:38.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:38 smithi098 bash[76817]: cluster 2023-09-23T18:36:36.926916+0000 osd.6 (osd.6) 57 : cluster 3 failed to encode map e143 with expected crc 2023-09-23T18:36:38.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:38 smithi098 bash[76817]: cluster 2023-09-23T18:36:36.927425+0000 osd.7 (osd.7) 53 : cluster 3 failed to encode map e143 with expected crc 2023-09-23T18:36:38.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:38 smithi098 bash[76817]: cluster 2023-09-23T18:36:36.936143+0000 osd.4 (osd.4) 78 : cluster 3 failed to encode map e143 with expected crc 2023-09-23T18:36:38.460 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:38.482 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:38.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:38.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:38.585 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:38.612 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:38.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:38.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:38.715 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:38.740 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:38.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:38.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:38.843 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:38.874 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:38.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:38.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:38.977 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:39.007 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:39.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:39.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:39.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:38 smithi173 bash[59422]: cluster 2023-09-23T18:36:37.809983+0000 mgr.y (mgr.26054) 541 : cluster 0 pgmap v511: 89 pgs: 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 79 active+clean, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 33709/84 objects degraded (40129.762%); 1860/84 objects misplaced (2214.286%); 0 B/s, 150 objects/s recovering 2023-09-23T18:36:39.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:38 smithi173 bash[59422]: cluster 2023-09-23T18:36:37.924840+0000 osd.4 (osd.4) 79 : cluster 3 failed to encode map e144 with expected crc 2023-09-23T18:36:39.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:38 smithi173 bash[59422]: cluster 2023-09-23T18:36:37.928443+0000 mon.a (mon.0) 773 : cluster 0 osdmap e144: 8 total, 8 up, 7 in 2023-09-23T18:36:39.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:38 smithi173 bash[59422]: cluster 2023-09-23T18:36:37.930442+0000 osd.6 (osd.6) 58 : cluster 3 failed to encode map e144 with expected crc 2023-09-23T18:36:39.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:38 smithi173 bash[59422]: cluster 2023-09-23T18:36:37.953579+0000 osd.7 (osd.7) 54 : cluster 3 failed to encode map e144 with expected crc 2023-09-23T18:36:39.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:38 smithi173 bash[59422]: cluster 2023-09-23T18:36:38.599545+0000 osd.5 (osd.5) 54 : cluster 3 failed to encode map e144 with expected crc 2023-09-23T18:36:39.111 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:39.142 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:39.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:39.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:39.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:38 smithi098 bash[76817]: cluster 2023-09-23T18:36:37.809983+0000 mgr.y (mgr.26054) 541 : cluster 0 pgmap v511: 89 pgs: 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 79 active+clean, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 33709/84 objects degraded (40129.762%); 1860/84 objects misplaced (2214.286%); 0 B/s, 150 objects/s recovering 2023-09-23T18:36:39.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:38 smithi098 bash[76817]: cluster 2023-09-23T18:36:37.924840+0000 osd.4 (osd.4) 79 : cluster 3 failed to encode map e144 with expected crc 2023-09-23T18:36:39.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:38 smithi098 bash[76817]: cluster 2023-09-23T18:36:37.928443+0000 mon.a (mon.0) 773 : cluster 0 osdmap e144: 8 total, 8 up, 7 in 2023-09-23T18:36:39.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:38 smithi098 bash[76817]: cluster 2023-09-23T18:36:37.930442+0000 osd.6 (osd.6) 58 : cluster 3 failed to encode map e144 with expected crc 2023-09-23T18:36:39.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:38 smithi098 bash[76817]: cluster 2023-09-23T18:36:37.953579+0000 osd.7 (osd.7) 54 : cluster 3 failed to encode map e144 with expected crc 2023-09-23T18:36:39.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:38 smithi098 bash[76817]: cluster 2023-09-23T18:36:38.599545+0000 osd.5 (osd.5) 54 : cluster 3 failed to encode map e144 with expected crc 2023-09-23T18:36:39.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:38 smithi098 bash[79516]: cluster 2023-09-23T18:36:37.809983+0000 mgr.y (mgr.26054) 541 : cluster 0 pgmap v511: 89 pgs: 1 active+recovery_wait+remapped, 2 active+undersized+remapped+backfill_wait, 79 active+clean, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 33709/84 objects degraded (40129.762%); 1860/84 objects misplaced (2214.286%); 0 B/s, 150 objects/s recovering 2023-09-23T18:36:39.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:38 smithi098 bash[79516]: cluster 2023-09-23T18:36:37.924840+0000 osd.4 (osd.4) 79 : cluster 3 failed to encode map e144 with expected crc 2023-09-23T18:36:39.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:38 smithi098 bash[79516]: cluster 2023-09-23T18:36:37.928443+0000 mon.a (mon.0) 773 : cluster 0 osdmap e144: 8 total, 8 up, 7 in 2023-09-23T18:36:39.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:38 smithi098 bash[79516]: cluster 2023-09-23T18:36:37.930442+0000 osd.6 (osd.6) 58 : cluster 3 failed to encode map e144 with expected crc 2023-09-23T18:36:39.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:38 smithi098 bash[79516]: cluster 2023-09-23T18:36:37.953579+0000 osd.7 (osd.7) 54 : cluster 3 failed to encode map e144 with expected crc 2023-09-23T18:36:39.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:38 smithi098 bash[79516]: cluster 2023-09-23T18:36:38.599545+0000 osd.5 (osd.5) 54 : cluster 3 failed to encode map e144 with expected crc 2023-09-23T18:36:39.246 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:39.271 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:39.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:39.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:39.325 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:36:39.374 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:39.397 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:39.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:39.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:39.501 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:39.532 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:39.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:39.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:39.602 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:36:39.602 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:36:39.635 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:39.658 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:39.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:39.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:39.761 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:39.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:39.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:39.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:39.825 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:36:39.828 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:36:39.889 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:39.912 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:39.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:39.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:40.016 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:40.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:39 smithi173 bash[59422]: cluster 2023-09-23T18:36:39.516497+0000 mon.a (mon.0) 774 : cluster 3 Health check update: Degraded data redundancy: 33709/84 objects degraded (40129.762%), 7 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:40.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:39 smithi173 bash[59422]: audit 2023-09-23T18:36:39.592504+0000 mon.a (mon.0) 775 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:36:40.040 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:40.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:40.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:40.143 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:40.155 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:39 smithi098 bash[79516]: cluster 2023-09-23T18:36:39.516497+0000 mon.a (mon.0) 774 : cluster 3 Health check update: Degraded data redundancy: 33709/84 objects degraded (40129.762%), 7 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:40.155 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:39 smithi098 bash[79516]: audit 2023-09-23T18:36:39.592504+0000 mon.a (mon.0) 775 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:36:40.155 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:39 smithi098 bash[76817]: cluster 2023-09-23T18:36:39.516497+0000 mon.a (mon.0) 774 : cluster 3 Health check update: Degraded data redundancy: 33709/84 objects degraded (40129.762%), 7 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:40.156 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:39 smithi098 bash[76817]: audit 2023-09-23T18:36:39.592504+0000 mon.a (mon.0) 775 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:36:40.169 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:40.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:40.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:40.272 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:40.298 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:40.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:40.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:40.401 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:40.428 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:40.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:40.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:40.531 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:40.560 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:40.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:40.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:40.662 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:40.687 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:40.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:40.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:40.790 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:40.816 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:40.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:40.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:40.919 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:40.941 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:40.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:40.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:41.044 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:41.069 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:41.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:41.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:41.173 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:41.181 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:40 smithi098 bash[76817]: cluster 2023-09-23T18:36:39.810790+0000 mgr.y (mgr.26054) 542 : cluster 0 pgmap v513: 89 pgs: 1 active+recovering+remapped, 2 active+undersized+remapped+backfill_wait, 80 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 33160/84 objects degraded (39476.190%); 0 B/s, 421 objects/s recovering 2023-09-23T18:36:41.181 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:40 smithi098 bash[79516]: cluster 2023-09-23T18:36:39.810790+0000 mgr.y (mgr.26054) 542 : cluster 0 pgmap v513: 89 pgs: 1 active+recovering+remapped, 2 active+undersized+remapped+backfill_wait, 80 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 33160/84 objects degraded (39476.190%); 0 B/s, 421 objects/s recovering 2023-09-23T18:36:41.197 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:41.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:41.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:41.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:40 smithi173 bash[59422]: cluster 2023-09-23T18:36:39.810790+0000 mgr.y (mgr.26054) 542 : cluster 0 pgmap v513: 89 pgs: 1 active+recovering+remapped, 2 active+undersized+remapped+backfill_wait, 80 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 33160/84 objects degraded (39476.190%); 0 B/s, 421 objects/s recovering 2023-09-23T18:36:41.302 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:41.327 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:41.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:41.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:41.430 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:41.458 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:41.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:41.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:41.561 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:41.589 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:41.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:41.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:41.691 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:41.719 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:41.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:41.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:41.822 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:41.849 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:41.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:41.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:41.952 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:41.983 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:41.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:41.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:42.086 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:42.111 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:42.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:42.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:42.203 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T18:36:42.215 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:42.243 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:42.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:42.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:42.395 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:42.427 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:42.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:42.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:42.530 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:42.552 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:42.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:42.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:42.656 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:42.682 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:42.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:42.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:42.785 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:42.810 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:42.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:42.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:42.913 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:42.939 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:42.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:42.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:43.042 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:43.052 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:42 smithi173 bash[59422]: cluster 2023-09-23T18:36:41.811544+0000 mgr.y (mgr.26054) 543 : cluster 0 pgmap v514: 89 pgs: 1 active+recovering+remapped, 2 active+undersized+remapped+backfill_wait, 80 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 33044/84 objects degraded (39338.095%); 0 B/s, 241 objects/s recovering 2023-09-23T18:36:43.069 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:43.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:43.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:43.156 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:42 smithi098 bash[79516]: cluster 2023-09-23T18:36:41.811544+0000 mgr.y (mgr.26054) 543 : cluster 0 pgmap v514: 89 pgs: 1 active+recovering+remapped, 2 active+undersized+remapped+backfill_wait, 80 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 33044/84 objects degraded (39338.095%); 0 B/s, 241 objects/s recovering 2023-09-23T18:36:43.156 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:42 smithi098 bash[76817]: cluster 2023-09-23T18:36:41.811544+0000 mgr.y (mgr.26054) 543 : cluster 0 pgmap v514: 89 pgs: 1 active+recovering+remapped, 2 active+undersized+remapped+backfill_wait, 80 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 33044/84 objects degraded (39338.095%); 0 B/s, 241 objects/s recovering 2023-09-23T18:36:43.172 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:43.201 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:43.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:43.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:43.304 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:43.337 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:43.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:43.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:43.440 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:43.468 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:43.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:43.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:43.572 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:43.594 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:43.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:43.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:43.697 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:43.726 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:43.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:43.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:43.830 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:43.860 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:43.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:43.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:43.964 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:43.975 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:43 smithi173 bash[59422]: cluster 2023-09-23T18:36:42.932079+0000 mon.a (mon.0) 776 : cluster 0 osdmap e145: 8 total, 8 up, 7 in 2023-09-23T18:36:43.975 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:43 smithi173 bash[59422]: cluster 2023-09-23T18:36:42.934540+0000 osd.4 (osd.4) 80 : cluster 3 failed to encode map e145 with expected crc 2023-09-23T18:36:43.975 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:43 smithi173 bash[59422]: cluster 2023-09-23T18:36:42.935560+0000 osd.5 (osd.5) 55 : cluster 3 failed to encode map e145 with expected crc 2023-09-23T18:36:43.975 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:43 smithi173 bash[59422]: cluster 2023-09-23T18:36:42.958731+0000 osd.7 (osd.7) 55 : cluster 3 failed to encode map e145 with expected crc 2023-09-23T18:36:43.975 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:43 smithi173 bash[59422]: cluster 2023-09-23T18:36:43.931892+0000 mon.a (mon.0) 777 : cluster 0 osdmap e146: 8 total, 8 up, 7 in 2023-09-23T18:36:43.992 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:43.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:43.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:44.096 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:44.127 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:44.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:44.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:44.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:43 smithi098 bash[76817]: cluster 2023-09-23T18:36:42.932079+0000 mon.a (mon.0) 776 : cluster 0 osdmap e145: 8 total, 8 up, 7 in 2023-09-23T18:36:44.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:43 smithi098 bash[76817]: cluster 2023-09-23T18:36:42.934540+0000 osd.4 (osd.4) 80 : cluster 3 failed to encode map e145 with expected crc 2023-09-23T18:36:44.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:43 smithi098 bash[76817]: cluster 2023-09-23T18:36:42.935560+0000 osd.5 (osd.5) 55 : cluster 3 failed to encode map e145 with expected crc 2023-09-23T18:36:44.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:43 smithi098 bash[76817]: cluster 2023-09-23T18:36:42.958731+0000 osd.7 (osd.7) 55 : cluster 3 failed to encode map e145 with expected crc 2023-09-23T18:36:44.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:43 smithi098 bash[76817]: cluster 2023-09-23T18:36:43.931892+0000 mon.a (mon.0) 777 : cluster 0 osdmap e146: 8 total, 8 up, 7 in 2023-09-23T18:36:44.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:43 smithi098 bash[79516]: cluster 2023-09-23T18:36:42.932079+0000 mon.a (mon.0) 776 : cluster 0 osdmap e145: 8 total, 8 up, 7 in 2023-09-23T18:36:44.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:43 smithi098 bash[79516]: cluster 2023-09-23T18:36:42.934540+0000 osd.4 (osd.4) 80 : cluster 3 failed to encode map e145 with expected crc 2023-09-23T18:36:44.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:43 smithi098 bash[79516]: cluster 2023-09-23T18:36:42.935560+0000 osd.5 (osd.5) 55 : cluster 3 failed to encode map e145 with expected crc 2023-09-23T18:36:44.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:43 smithi098 bash[79516]: cluster 2023-09-23T18:36:42.958731+0000 osd.7 (osd.7) 55 : cluster 3 failed to encode map e145 with expected crc 2023-09-23T18:36:44.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:43 smithi098 bash[79516]: cluster 2023-09-23T18:36:43.931892+0000 mon.a (mon.0) 777 : cluster 0 osdmap e146: 8 total, 8 up, 7 in 2023-09-23T18:36:44.230 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:44.255 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:44.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:44.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:44.359 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:44.385 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:44.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:44.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:44.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:44.517 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:44.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:44.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:44.621 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:44.644 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:44.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:44.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:44.747 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:44.779 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:44.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:44.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:44.882 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:44.908 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:44.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:44.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:45.011 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:45.022 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:44 smithi173 bash[59422]: cluster 2023-09-23T18:36:43.812389+0000 mgr.y (mgr.26054) 544 : cluster 0 pgmap v516: 89 pgs: 1 active+recovering+remapped, 2 active+undersized+remapped+backfill_wait, 80 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 33044/84 objects degraded (39338.095%); 0 B/s, 226 objects/s recovering 2023-09-23T18:36:45.022 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:44 smithi173 bash[59422]: cluster 2023-09-23T18:36:43.929686+0000 osd.5 (osd.5) 56 : cluster 3 failed to encode map e146 with expected crc 2023-09-23T18:36:45.022 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:44 smithi173 bash[59422]: cluster 2023-09-23T18:36:43.930246+0000 osd.6 (osd.6) 59 : cluster 3 failed to encode map e146 with expected crc 2023-09-23T18:36:45.022 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:44 smithi173 bash[59422]: cluster 2023-09-23T18:36:43.933738+0000 osd.4 (osd.4) 81 : cluster 3 failed to encode map e146 with expected crc 2023-09-23T18:36:45.022 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:44 smithi173 bash[59422]: cluster 2023-09-23T18:36:43.937065+0000 osd.4 (osd.4) 82 : cluster 3 failed to encode map e146 with expected crc 2023-09-23T18:36:45.022 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:44 smithi173 bash[59422]: cluster 2023-09-23T18:36:43.970418+0000 osd.7 (osd.7) 56 : cluster 3 failed to encode map e146 with expected crc 2023-09-23T18:36:45.022 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:44 smithi173 bash[59422]: cluster 2023-09-23T18:36:44.517499+0000 mon.a (mon.0) 778 : cluster 3 Health check update: Degraded data redundancy: 33044/84 objects degraded (39338.095%), 6 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:45.039 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:45.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:45.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:45.142 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:45.168 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:45.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:45.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:45.272 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:45.296 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:36:45.300 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:45.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:45.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:45.404 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:45.431 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:45.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:45.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:45.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:45 smithi098 bash[79516]: cluster 2023-09-23T18:36:43.812389+0000 mgr.y (mgr.26054) 544 : cluster 0 pgmap v516: 89 pgs: 1 active+recovering+remapped, 2 active+undersized+remapped+backfill_wait, 80 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 33044/84 objects degraded (39338.095%); 0 B/s, 226 objects/s recovering 2023-09-23T18:36:45.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:45 smithi098 bash[79516]: cluster 2023-09-23T18:36:43.929686+0000 osd.5 (osd.5) 56 : cluster 3 failed to encode map e146 with expected crc 2023-09-23T18:36:45.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:45 smithi098 bash[79516]: cluster 2023-09-23T18:36:43.930246+0000 osd.6 (osd.6) 59 : cluster 3 failed to encode map e146 with expected crc 2023-09-23T18:36:45.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:45 smithi098 bash[79516]: cluster 2023-09-23T18:36:43.933738+0000 osd.4 (osd.4) 81 : cluster 3 failed to encode map e146 with expected crc 2023-09-23T18:36:45.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:45 smithi098 bash[79516]: cluster 2023-09-23T18:36:43.937065+0000 osd.4 (osd.4) 82 : cluster 3 failed to encode map e146 with expected crc 2023-09-23T18:36:45.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:45 smithi098 bash[79516]: cluster 2023-09-23T18:36:43.970418+0000 osd.7 (osd.7) 56 : cluster 3 failed to encode map e146 with expected crc 2023-09-23T18:36:45.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:45 smithi098 bash[79516]: cluster 2023-09-23T18:36:44.517499+0000 mon.a (mon.0) 778 : cluster 3 Health check update: Degraded data redundancy: 33044/84 objects degraded (39338.095%), 6 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:45.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:45 smithi098 bash[76817]: cluster 2023-09-23T18:36:43.812389+0000 mgr.y (mgr.26054) 544 : cluster 0 pgmap v516: 89 pgs: 1 active+recovering+remapped, 2 active+undersized+remapped+backfill_wait, 80 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 33044/84 objects degraded (39338.095%); 0 B/s, 226 objects/s recovering 2023-09-23T18:36:45.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:45 smithi098 bash[76817]: cluster 2023-09-23T18:36:43.929686+0000 osd.5 (osd.5) 56 : cluster 3 failed to encode map e146 with expected crc 2023-09-23T18:36:45.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:45 smithi098 bash[76817]: cluster 2023-09-23T18:36:43.930246+0000 osd.6 (osd.6) 59 : cluster 3 failed to encode map e146 with expected crc 2023-09-23T18:36:45.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:45 smithi098 bash[76817]: cluster 2023-09-23T18:36:43.933738+0000 osd.4 (osd.4) 81 : cluster 3 failed to encode map e146 with expected crc 2023-09-23T18:36:45.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:45 smithi098 bash[76817]: cluster 2023-09-23T18:36:43.937065+0000 osd.4 (osd.4) 82 : cluster 3 failed to encode map e146 with expected crc 2023-09-23T18:36:45.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:45 smithi098 bash[76817]: cluster 2023-09-23T18:36:43.970418+0000 osd.7 (osd.7) 56 : cluster 3 failed to encode map e146 with expected crc 2023-09-23T18:36:45.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:45 smithi098 bash[76817]: cluster 2023-09-23T18:36:44.517499+0000 mon.a (mon.0) 778 : cluster 3 Health check update: Degraded data redundancy: 33044/84 objects degraded (39338.095%), 6 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:45.534 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:45.560 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:45.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:45.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:45.663 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:45.690 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:45.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:45.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:45.793 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:45.824 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:45.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:45.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:45.928 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:45.960 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:45.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:45.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:46.063 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:46.088 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:46.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:46.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:46.192 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:46.222 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:46.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:46.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:46.325 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:46.355 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:46.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:46.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:46.440 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:36:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:36:46] "GET /metrics HTTP/1.1" 200 36517 "" "Prometheus/2.33.4" 2023-09-23T18:36:46.458 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:46.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:46.494 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:46.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:46.596 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:46.625 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:46.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:46.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:46.728 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:46.753 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:46.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:46.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:46.857 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:46.885 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:46.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:46.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:46.988 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:47.018 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:47.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:47.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:47.121 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:47.154 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:47.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:47.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:47.258 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:47.269 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:46 smithi173 bash[59422]: cluster 2023-09-23T18:36:45.813467+0000 mgr.y (mgr.26054) 545 : cluster 0 pgmap v518: 89 pgs: 1 active+recovering+remapped, 1 active+undersized+remapped+backfill_wait, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 32927/84 objects degraded (39198.810%); 0 B/s, 212 objects/s recovering 2023-09-23T18:36:47.281 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:47.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:47.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:47.385 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:47.393 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:46 smithi098 bash[76817]: cluster 2023-09-23T18:36:45.813467+0000 mgr.y (mgr.26054) 545 : cluster 0 pgmap v518: 89 pgs: 1 active+recovering+remapped, 1 active+undersized+remapped+backfill_wait, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 32927/84 objects degraded (39198.810%); 0 B/s, 212 objects/s recovering 2023-09-23T18:36:47.393 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:46 smithi098 bash[79516]: cluster 2023-09-23T18:36:45.813467+0000 mgr.y (mgr.26054) 545 : cluster 0 pgmap v518: 89 pgs: 1 active+recovering+remapped, 1 active+undersized+remapped+backfill_wait, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 32927/84 objects degraded (39198.810%); 0 B/s, 212 objects/s recovering 2023-09-23T18:36:47.410 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:47.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:47.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:47.514 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:47.537 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:47.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:47.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:47.641 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:47.663 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:47.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:47.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:47.767 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:47.795 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:47.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:47.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:47.860 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:36:47.861 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:36:47.899 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:47.926 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:47.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:47.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:47.939 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:36:47.941 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-23T18:36:48.029 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:48.054 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:48.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:48.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:48.158 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:48.179 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:48.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:48.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:48.283 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:48.304 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:48.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:48.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:48.408 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:48.432 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:48.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:48.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:48.536 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:48.558 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:48.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:48.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:48.661 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:48.682 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:48.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:48.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:48.785 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:48.812 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:48.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:48.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:48.916 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:48.941 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:48.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:48.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:48.958 INFO:teuthology.orchestra.run.smithi098.stderr:pool 'unique_pool_0' removed 2023-09-23T18:36:49.045 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:49.051 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:48 smithi098 bash[79516]: cluster 2023-09-23T18:36:47.814131+0000 mgr.y (mgr.26054) 546 : cluster 0 pgmap v519: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 32927/84 objects degraded (39198.810%); 0 B/s, 149 objects/s recovering 2023-09-23T18:36:49.051 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:48 smithi098 bash[79516]: audit 2023-09-23T18:36:47.924356+0000 mgr.y (mgr.26054) 547 : audit 0 from='client.34971 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:36:49.052 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:48 smithi098 bash[79516]: audit 2023-09-23T18:36:47.971959+0000 mon.c (mon.2) 73 : audit 1 from='client.? 172.21.15.98:0/154588580' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:36:49.052 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:48 smithi098 bash[79516]: audit 2023-09-23T18:36:47.972360+0000 mon.a (mon.0) 779 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:36:49.052 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:48 smithi098 bash[76817]: cluster 2023-09-23T18:36:47.814131+0000 mgr.y (mgr.26054) 546 : cluster 0 pgmap v519: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 32927/84 objects degraded (39198.810%); 0 B/s, 149 objects/s recovering 2023-09-23T18:36:49.052 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:48 smithi098 bash[76817]: audit 2023-09-23T18:36:47.924356+0000 mgr.y (mgr.26054) 547 : audit 0 from='client.34971 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:36:49.052 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:48 smithi098 bash[76817]: audit 2023-09-23T18:36:47.971959+0000 mon.c (mon.2) 73 : audit 1 from='client.? 172.21.15.98:0/154588580' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:36:49.052 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:48 smithi098 bash[76817]: audit 2023-09-23T18:36:47.972360+0000 mon.a (mon.0) 779 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:36:49.069 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:49.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:49.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:49.174 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:49.199 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:49.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:49.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:49.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:48 smithi173 bash[59422]: cluster 2023-09-23T18:36:47.814131+0000 mgr.y (mgr.26054) 546 : cluster 0 pgmap v519: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 32927/84 objects degraded (39198.810%); 0 B/s, 149 objects/s recovering 2023-09-23T18:36:49.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:48 smithi173 bash[59422]: audit 2023-09-23T18:36:47.924356+0000 mgr.y (mgr.26054) 547 : audit 0 from='client.34971 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:36:49.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:48 smithi173 bash[59422]: audit 2023-09-23T18:36:47.971959+0000 mon.c (mon.2) 73 : audit 1 from='client.? 172.21.15.98:0/154588580' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:36:49.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:48 smithi173 bash[59422]: audit 2023-09-23T18:36:47.972360+0000 mon.a (mon.0) 779 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:36:49.302 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:49.323 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:49.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:49.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:49.427 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:49.457 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:49.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:49.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:49.560 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:49.581 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:49.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:49.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:49.684 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:49.702 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:49.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:49.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:49.805 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:49.824 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:49.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:49.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:49.927 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:49.949 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:49.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:49.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:50.052 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:50.069 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:50.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:50.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:50.172 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:50.191 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:50.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:50.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:50.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:49 smithi173 bash[59422]: cluster 2023-09-23T18:36:48.950156+0000 osd.5 (osd.5) 57 : cluster 3 failed to encode map e147 with expected crc 2023-09-23T18:36:50.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:49 smithi173 bash[59422]: cluster 2023-09-23T18:36:48.950353+0000 osd.4 (osd.4) 83 : cluster 3 failed to encode map e147 with expected crc 2023-09-23T18:36:50.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:49 smithi173 bash[59422]: audit 2023-09-23T18:36:48.952787+0000 mon.a (mon.0) 780 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:36:50.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:49 smithi173 bash[59422]: cluster 2023-09-23T18:36:48.952962+0000 mon.a (mon.0) 781 : cluster 0 osdmap e147: 8 total, 8 up, 7 in 2023-09-23T18:36:50.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:49 smithi173 bash[59422]: cluster 2023-09-23T18:36:49.148060+0000 osd.6 (osd.6) 60 : cluster 3 failed to encode map e147 with expected crc 2023-09-23T18:36:50.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:49 smithi173 bash[59422]: cluster 2023-09-23T18:36:49.518725+0000 mon.a (mon.0) 782 : cluster 3 Health check update: Degraded data redundancy: 32927/84 objects degraded (39198.810%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:50.294 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:50.329 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:50.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:50.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:50.432 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:50.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:50 smithi098 bash[76817]: cluster 2023-09-23T18:36:48.950156+0000 osd.5 (osd.5) 57 : cluster 3 failed to encode map e147 with expected crc 2023-09-23T18:36:50.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:50 smithi098 bash[76817]: cluster 2023-09-23T18:36:48.950353+0000 osd.4 (osd.4) 83 : cluster 3 failed to encode map e147 with expected crc 2023-09-23T18:36:50.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:50 smithi098 bash[76817]: audit 2023-09-23T18:36:48.952787+0000 mon.a (mon.0) 780 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:36:50.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:50 smithi098 bash[76817]: cluster 2023-09-23T18:36:48.952962+0000 mon.a (mon.0) 781 : cluster 0 osdmap e147: 8 total, 8 up, 7 in 2023-09-23T18:36:50.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:50 smithi098 bash[76817]: cluster 2023-09-23T18:36:49.148060+0000 osd.6 (osd.6) 60 : cluster 3 failed to encode map e147 with expected crc 2023-09-23T18:36:50.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:50 smithi098 bash[76817]: cluster 2023-09-23T18:36:49.518725+0000 mon.a (mon.0) 782 : cluster 3 Health check update: Degraded data redundancy: 32927/84 objects degraded (39198.810%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:50.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:50 smithi098 bash[79516]: cluster 2023-09-23T18:36:48.950156+0000 osd.5 (osd.5) 57 : cluster 3 failed to encode map e147 with expected crc 2023-09-23T18:36:50.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:50 smithi098 bash[79516]: cluster 2023-09-23T18:36:48.950353+0000 osd.4 (osd.4) 83 : cluster 3 failed to encode map e147 with expected crc 2023-09-23T18:36:50.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:50 smithi098 bash[79516]: audit 2023-09-23T18:36:48.952787+0000 mon.a (mon.0) 780 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:36:50.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:50 smithi098 bash[79516]: cluster 2023-09-23T18:36:48.952962+0000 mon.a (mon.0) 781 : cluster 0 osdmap e147: 8 total, 8 up, 7 in 2023-09-23T18:36:50.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:50 smithi098 bash[79516]: cluster 2023-09-23T18:36:49.148060+0000 osd.6 (osd.6) 60 : cluster 3 failed to encode map e147 with expected crc 2023-09-23T18:36:50.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:50 smithi098 bash[79516]: cluster 2023-09-23T18:36:49.518725+0000 mon.a (mon.0) 782 : cluster 3 Health check update: Degraded data redundancy: 32927/84 objects degraded (39198.810%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:36:50.461 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:50.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:50.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:50.564 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:50.593 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:50.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:50.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:50.696 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:50.721 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:50.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:50.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:50.824 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:50.851 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:50.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:50.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:50.955 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:50.977 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:50.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:50.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:51.079 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:51.114 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:51.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:51.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:51.217 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:51.226 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:50 smithi098 bash[79516]: cluster 2023-09-23T18:36:49.814587+0000 mgr.y (mgr.26054) 548 : cluster 0 pgmap v521: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:36:51.227 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:50 smithi098 bash[79516]: cluster 2023-09-23T18:36:50.520880+0000 mon.a (mon.0) 783 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32927/84 objects degraded (39198.810%), 6 pgs degraded, 6 pgs undersized) 2023-09-23T18:36:51.227 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:50 smithi098 bash[76817]: cluster 2023-09-23T18:36:49.814587+0000 mgr.y (mgr.26054) 548 : cluster 0 pgmap v521: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:36:51.227 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:50 smithi098 bash[76817]: cluster 2023-09-23T18:36:50.520880+0000 mon.a (mon.0) 783 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32927/84 objects degraded (39198.810%), 6 pgs degraded, 6 pgs undersized) 2023-09-23T18:36:51.243 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:51.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:51.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:51.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:50 smithi173 bash[59422]: cluster 2023-09-23T18:36:49.814587+0000 mgr.y (mgr.26054) 548 : cluster 0 pgmap v521: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:36:51.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:50 smithi173 bash[59422]: cluster 2023-09-23T18:36:50.520880+0000 mon.a (mon.0) 783 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32927/84 objects degraded (39198.810%), 6 pgs degraded, 6 pgs undersized) 2023-09-23T18:36:51.345 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:51.374 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:51.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:51.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:51.477 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:51.503 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:51.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:51.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:51.606 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:51.635 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:51.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:51.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:51.738 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:51.765 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:51.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:51.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:51.869 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:51.896 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:51.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:51.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:52.001 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:52.028 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:52.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:52.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:52.132 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:52.143 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:51 smithi098 bash[76817]: cluster 2023-09-23T18:36:51.815126+0000 mgr.y (mgr.26054) 549 : cluster 0 pgmap v522: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:36:52.143 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:51 smithi098 bash[79516]: cluster 2023-09-23T18:36:51.815126+0000 mgr.y (mgr.26054) 549 : cluster 0 pgmap v522: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:36:52.157 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:52.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:52.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:52.203 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":519,"stamp":"2023-09-23T18:36:47.813708+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":32927,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6246,"num_read_kb":13703,"num_write":213672,"num_write_kb":6817208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26194,"num_bytes_recovered":1256784496,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133536,"ondisk_log_size":133536,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":1639144,"kb_used_data":24688,"kb_used_omap":31,"kb_used_meta":1614368,"kb_avail":654576920,"statfs":{"total":671965249536,"available":670286766080,"internally_reserved":0,"allocated":25280512,"data_stored":20852397,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32507,"internal_metadata":1653113093},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-233,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1198,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003490"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":143,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.941898+0000","last_change":"2023-09-23T18:33:53.842027+0000","last_active":"2023-09-23T18:36:42.941898+0000","last_peered":"2023-09-23T18:36:42.941898+0000","last_clean":"2023-09-23T18:36:42.941898+0000","last_became_active":"2023-09-23T18:33:53.841496+0000","last_became_peered":"2023-09-23T18:33:53.841496+0000","last_unstale":"2023-09-23T18:36:42.941898+0000","last_undegraded":"2023-09-23T18:36:42.941898+0000","last_fullsized":"2023-09-23T18:36:42.941898+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":131,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.929338+0000","last_change":"2023-09-23T18:33:53.841529+0000","last_active":"2023-09-23T18:36:42.929338+0000","last_peered":"2023-09-23T18:36:42.929338+0000","last_clean":"2023-09-23T18:36:42.929338+0000","last_became_active":"2023-09-23T18:33:53.841290+0000","last_became_peered":"2023-09-23T18:33:53.841290+0000","last_unstale":"2023-09-23T18:36:42.929338+0000","last_undegraded":"2023-09-23T18:36:42.929338+0000","last_fullsized":"2023-09-23T18:36:42.929338+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:27.552815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":82,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.931039+0000","last_change":"2023-09-23T18:33:53.848655+0000","last_active":"2023-09-23T18:36:42.931039+0000","last_peered":"2023-09-23T18:36:42.931039+0000","last_clean":"2023-09-23T18:36:42.931039+0000","last_became_active":"2023-09-23T18:33:53.848279+0000","last_became_peered":"2023-09-23T18:33:53.848279+0000","last_unstale":"2023-09-23T18:36:42.931039+0000","last_undegraded":"2023-09-23T18:36:42.931039+0000","last_fullsized":"2023-09-23T18:36:42.931039+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:39.223612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":99,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.956452+0000","last_change":"2023-09-23T18:28:03.289500+0000","last_active":"2023-09-23T18:36:42.956452+0000","last_peered":"2023-09-23T18:36:42.956452+0000","last_clean":"2023-09-23T18:36:42.956452+0000","last_became_active":"2023-09-23T18:28:03.289122+0000","last_became_peered":"2023-09-23T18:28:03.289122+0000","last_unstale":"2023-09-23T18:36:42.956452+0000","last_undegraded":"2023-09-23T18:36:42.956452+0000","last_fullsized":"2023-09-23T18:36:42.956452+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:22:26.904660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":147,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.928555+0000","last_change":"2023-09-23T18:33:53.842010+0000","last_active":"2023-09-23T18:36:42.928555+0000","last_peered":"2023-09-23T18:36:42.928555+0000","last_clean":"2023-09-23T18:36:42.928555+0000","last_became_active":"2023-09-23T18:33:53.841790+0000","last_became_peered":"2023-09-23T18:33:53.841790+0000","last_unstale":"2023-09-23T18:36:42.928555+0000","last_undegraded":"2023-09-23T18:36:42.928555+0000","last_fullsized":"2023-09-23T18:36:42.928555+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:46.540440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":137,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.975645+0000","last_change":"2023-09-23T18:33:55.084557+0000","last_active":"2023-09-23T18:36:42.975645+0000","last_peered":"2023-09-23T18:36:42.975645+0000","last_clean":"2023-09-23T18:36:42.975645+0000","last_became_active":"2023-09-23T18:33:55.083987+0000","last_became_peered":"2023-09-23T18:33:55.083987+0000","last_unstale":"2023-09-23T18:36:42.975645+0000","last_undegraded":"2023-09-23T18:36:42.975645+0000","last_fullsized":"2023-09-23T18:36:42.975645+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":106,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.956796+0000","last_change":"2023-09-23T18:32:26.797582+0000","last_active":"2023-09-23T18:36:42.956796+0000","last_peered":"2023-09-23T18:36:42.956796+0000","last_clean":"2023-09-23T18:36:42.956796+0000","last_became_active":"2023-09-23T18:32:26.797107+0000","last_became_peered":"2023-09-23T18:32:26.797107+0000","last_unstale":"2023-09-23T18:36:42.956796+0000","last_undegraded":"2023-09-23T18:36:42.956796+0000","last_fullsized":"2023-09-23T18:36:42.956796+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:56.973116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":131,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.928495+0000","last_change":"2023-09-23T18:33:53.844742+0000","last_active":"2023-09-23T18:36:42.928495+0000","last_peered":"2023-09-23T18:36:42.928495+0000","last_clean":"2023-09-23T18:36:42.928495+0000","last_became_active":"2023-09-23T18:33:53.844425+0000","last_became_peered":"2023-09-23T18:33:53.844425+0000","last_unstale":"2023-09-23T18:36:42.928495+0000","last_undegraded":"2023-09-23T18:36:42.928495+0000","last_fullsized":"2023-09-23T18:36:42.928495+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:42.044108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":126,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.964281+0000","last_change":"2023-09-23T18:33:54.847051+0000","last_active":"2023-09-23T18:36:42.964281+0000","last_peered":"2023-09-23T18:36:42.964281+0000","last_clean":"2023-09-23T18:36:42.964281+0000","last_became_active":"2023-09-23T18:33:54.845357+0000","last_became_peered":"2023-09-23T18:33:54.845357+0000","last_unstale":"2023-09-23T18:36:42.964281+0000","last_undegraded":"2023-09-23T18:36:42.964281+0000","last_fullsized":"2023-09-23T18:36:42.964281+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":106,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.928550+0000","last_change":"2023-09-23T18:28:03.298214+0000","last_active":"2023-09-23T18:36:42.928550+0000","last_peered":"2023-09-23T18:36:42.928550+0000","last_clean":"2023-09-23T18:36:42.928550+0000","last_became_active":"2023-09-23T18:28:03.297852+0000","last_became_peered":"2023-09-23T18:28:03.297852+0000","last_unstale":"2023-09-23T18:36:42.928550+0000","last_undegraded":"2023-09-23T18:36:42.928550+0000","last_fullsized":"2023-09-23T18:36:42.928550+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.146541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":115,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.964558+0000","last_change":"2023-09-23T18:28:03.296913+0000","last_active":"2023-09-23T18:36:42.964558+0000","last_peered":"2023-09-23T18:36:42.964558+0000","last_clean":"2023-09-23T18:36:42.964558+0000","last_became_active":"2023-09-23T18:28:03.296499+0000","last_became_peered":"2023-09-23T18:28:03.296499+0000","last_unstale":"2023-09-23T18:36:42.964558+0000","last_undegraded":"2023-09-23T18:36:42.964558+0000","last_fullsized":"2023-09-23T18:36:42.964558+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":78,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.931497+0000","last_change":"2023-09-23T18:33:53.850096+0000","last_active":"2023-09-23T18:36:42.931497+0000","last_peered":"2023-09-23T18:36:42.931497+0000","last_clean":"2023-09-23T18:36:42.931497+0000","last_became_active":"2023-09-23T18:33:53.849811+0000","last_became_peered":"2023-09-23T18:33:53.849811+0000","last_unstale":"2023-09-23T18:36:42.931497+0000","last_undegraded":"2023-09-23T18:36:42.931497+0000","last_fullsized":"2023-09-23T18:36:42.931497+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:44.938072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":119,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.942249+0000","last_change":"2023-09-23T18:15:08.909339+0000","last_active":"2023-09-23T18:36:42.942249+0000","last_peered":"2023-09-23T18:36:42.942249+0000","last_clean":"2023-09-23T18:36:42.942249+0000","last_became_active":"2023-09-23T18:15:08.908912+0000","last_became_peered":"2023-09-23T18:15:08.908912+0000","last_unstale":"2023-09-23T18:36:42.942249+0000","last_undegraded":"2023-09-23T18:36:42.942249+0000","last_fullsized":"2023-09-23T18:36:42.942249+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":126,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.941828+0000","last_change":"2023-09-23T18:33:53.847489+0000","last_active":"2023-09-23T18:36:42.941828+0000","last_peered":"2023-09-23T18:36:42.941828+0000","last_clean":"2023-09-23T18:36:42.941828+0000","last_became_active":"2023-09-23T18:33:53.846935+0000","last_became_peered":"2023-09-23T18:33:53.846935+0000","last_unstale":"2023-09-23T18:36:42.941828+0000","last_undegraded":"2023-09-23T18:36:42.941828+0000","last_fullsized":"2023-09-23T18:36:42.941828+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":114,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.964063+0000","last_change":"2023-09-23T18:27:33.585121+0000","last_active":"2023-09-23T18:36:42.964063+0000","last_peered":"2023-09-23T18:36:42.964063+0000","last_clean":"2023-09-23T18:36:42.964063+0000","last_became_active":"2023-09-23T18:27:33.584620+0000","last_became_peered":"2023-09-23T18:27:33.584620+0000","last_unstale":"2023-09-23T18:36:42.964063+0000","last_undegraded":"2023-09-23T18:36:42.964063+0000","last_fullsized":"2023-09-23T18:36:42.964063+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":131,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.929781+0000","last_change":"2023-09-23T18:33:53.852933+0000","last_active":"2023-09-23T18:36:42.929781+0000","last_peered":"2023-09-23T18:36:42.929781+0000","last_clean":"2023-09-23T18:36:42.929781+0000","last_became_active":"2023-09-23T18:33:53.852626+0000","last_became_peered":"2023-09-23T18:33:53.852626+0000","last_unstale":"2023-09-23T18:36:42.929781+0000","last_undegraded":"2023-09-23T18:36:42.929781+0000","last_fullsized":"2023-09-23T18:36:42.929781+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:00.174536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":97,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.937707+0000","last_change":"2023-09-23T18:28:03.290316+0000","last_active":"2023-09-23T18:36:42.937707+0000","last_peered":"2023-09-23T18:36:42.937707+0000","last_clean":"2023-09-23T18:36:42.937707+0000","last_became_active":"2023-09-23T18:28:03.289698+0000","last_became_peered":"2023-09-23T18:28:03.289698+0000","last_unstale":"2023-09-23T18:36:42.937707+0000","last_undegraded":"2023-09-23T18:36:42.937707+0000","last_fullsized":"2023-09-23T18:36:42.937707+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:00.779542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":103,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.964222+0000","last_change":"2023-09-23T18:32:26.808237+0000","last_active":"2023-09-23T18:36:42.964222+0000","last_peered":"2023-09-23T18:36:42.964222+0000","last_clean":"2023-09-23T18:36:42.964222+0000","last_became_active":"2023-09-23T18:32:26.807843+0000","last_became_peered":"2023-09-23T18:32:26.807843+0000","last_unstale":"2023-09-23T18:36:42.964222+0000","last_undegraded":"2023-09-23T18:36:42.964222+0000","last_fullsized":"2023-09-23T18:36:42.964222+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":97,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.963637+0000","last_change":"2023-09-23T18:15:08.916658+0000","last_active":"2023-09-23T18:36:42.963637+0000","last_peered":"2023-09-23T18:36:42.963637+0000","last_clean":"2023-09-23T18:36:42.963637+0000","last_became_active":"2023-09-23T18:15:08.916301+0000","last_became_peered":"2023-09-23T18:15:08.916301+0000","last_unstale":"2023-09-23T18:36:42.963637+0000","last_undegraded":"2023-09-23T18:36:42.963637+0000","last_fullsized":"2023-09-23T18:36:42.963637+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":98,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.956452+0000","last_change":"2023-09-23T18:28:03.290217+0000","last_active":"2023-09-23T18:36:42.956452+0000","last_peered":"2023-09-23T18:36:42.956452+0000","last_clean":"2023-09-23T18:36:42.956452+0000","last_became_active":"2023-09-23T18:28:03.289888+0000","last_became_peered":"2023-09-23T18:28:03.289888+0000","last_unstale":"2023-09-23T18:36:42.956452+0000","last_undegraded":"2023-09-23T18:36:42.956452+0000","last_fullsized":"2023-09-23T18:36:42.956452+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:51:45.400847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":139,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.974817+0000","last_change":"2023-09-23T18:32:26.810836+0000","last_active":"2023-09-23T18:36:42.974817+0000","last_peered":"2023-09-23T18:36:42.974817+0000","last_clean":"2023-09-23T18:36:42.974817+0000","last_became_active":"2023-09-23T18:32:26.795378+0000","last_became_peered":"2023-09-23T18:32:26.795378+0000","last_unstale":"2023-09-23T18:36:42.974817+0000","last_undegraded":"2023-09-23T18:36:42.974817+0000","last_fullsized":"2023-09-23T18:36:42.974817+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":131,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.928876+0000","last_change":"2023-09-23T18:33:53.852173+0000","last_active":"2023-09-23T18:36:42.928876+0000","last_peered":"2023-09-23T18:36:42.928876+0000","last_clean":"2023-09-23T18:36:42.928876+0000","last_became_active":"2023-09-23T18:33:53.851814+0000","last_became_peered":"2023-09-23T18:33:53.851814+0000","last_unstale":"2023-09-23T18:36:42.928876+0000","last_undegraded":"2023-09-23T18:36:42.928876+0000","last_fullsized":"2023-09-23T18:36:42.928876+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:15.845259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"140'13552","reported_seq":13599,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.963244+0000","last_change":"2023-09-23T18:31:11.799201+0000","last_active":"2023-09-23T18:36:42.963244+0000","last_peered":"2023-09-23T18:36:42.963244+0000","last_clean":"2023-09-23T18:36:42.963244+0000","last_became_active":"2023-09-23T18:31:11.798833+0000","last_became_peered":"2023-09-23T18:31:11.798833+0000","last_unstale":"2023-09-23T18:36:42.963244+0000","last_undegraded":"2023-09-23T18:36:42.963244+0000","last_fullsized":"2023-09-23T18:36:42.963244+0000","mapping_epoch":107,"log_start":"113'5400","ondisk_log_start":"113'5400","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":8152,"log_dups_size":0,"ondisk_log_size":8152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13552,"num_write_kb":433664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":113,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.964645+0000","last_change":"2023-09-23T18:26:34.999684+0000","last_active":"2023-09-23T18:36:42.964645+0000","last_peered":"2023-09-23T18:36:42.964645+0000","last_clean":"2023-09-23T18:36:42.964645+0000","last_became_active":"2023-09-23T18:26:34.999076+0000","last_became_peered":"2023-09-23T18:26:34.999076+0000","last_unstale":"2023-09-23T18:36:42.964645+0000","last_undegraded":"2023-09-23T18:36:42.964645+0000","last_fullsized":"2023-09-23T18:36:42.964645+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":44,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.963124+0000","last_change":"2023-09-23T18:32:26.806328+0000","last_active":"2023-09-23T18:36:42.963124+0000","last_peered":"2023-09-23T18:36:42.963124+0000","last_clean":"2023-09-23T18:36:42.963124+0000","last_became_active":"2023-09-23T18:32:26.803969+0000","last_became_peered":"2023-09-23T18:32:26.803969+0000","last_unstale":"2023-09-23T18:36:42.963124+0000","last_undegraded":"2023-09-23T18:36:42.963124+0000","last_fullsized":"2023-09-23T18:36:42.963124+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.8","version":"140'13382","reported_seq":13436,"reported_epoch":144,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:36:38.019684+0000","last_change":"2023-09-23T18:33:55.189377+0000","last_active":"2023-09-23T18:36:38.019684+0000","last_peered":"2023-09-23T18:36:38.019684+0000","last_clean":"2023-09-23T18:31:10.759555+0000","last_became_active":"2023-09-23T18:33:55.126552+0000","last_became_peered":"2023-09-23T18:33:55.126552+0000","last_unstale":"2023-09-23T18:36:38.019684+0000","last_undegraded":"2023-09-23T18:33:54.873754+0000","last_fullsized":"2023-09-23T18:33:54.821501+0000","mapping_epoch":124,"log_start":"112'3300","ondisk_log_start":"112'3300","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":10082,"log_dups_size":0,"ondisk_log_size":10082,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:02.564184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6691,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13382,"num_write_kb":428224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":-4819}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":129,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.964407+0000","last_change":"2023-09-23T18:33:53.847817+0000","last_active":"2023-09-23T18:36:42.964407+0000","last_peered":"2023-09-23T18:36:42.964407+0000","last_clean":"2023-09-23T18:36:42.964407+0000","last_became_active":"2023-09-23T18:33:53.847342+0000","last_became_peered":"2023-09-23T18:33:53.847342+0000","last_unstale":"2023-09-23T18:36:42.964407+0000","last_undegraded":"2023-09-23T18:36:42.964407+0000","last_fullsized":"2023-09-23T18:36:42.964407+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":141,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.941862+0000","last_change":"2023-09-23T18:33:53.841434+0000","last_active":"2023-09-23T18:36:42.941862+0000","last_peered":"2023-09-23T18:36:42.941862+0000","last_clean":"2023-09-23T18:36:42.941862+0000","last_became_active":"2023-09-23T18:33:53.840923+0000","last_became_peered":"2023-09-23T18:33:53.840923+0000","last_unstale":"2023-09-23T18:36:42.941862+0000","last_undegraded":"2023-09-23T18:36:42.941862+0000","last_fullsized":"2023-09-23T18:36:42.941862+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"140'13276","reported_seq":24086,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T18:36:38.019271+0000","last_change":"2023-09-23T18:35:05.527467+0000","last_active":"2023-09-23T18:36:38.019271+0000","last_peered":"2023-09-23T18:36:38.019271+0000","last_clean":"2023-09-23T18:36:38.019271+0000","last_became_active":"2023-09-23T18:35:05.501984+0000","last_became_peered":"2023-09-23T18:35:05.501984+0000","last_unstale":"2023-09-23T18:36:38.019271+0000","last_undegraded":"2023-09-23T18:36:38.019271+0000","last_fullsized":"2023-09-23T18:36:38.019271+0000","mapping_epoch":131,"log_start":"112'3200","ondisk_log_start":"112'3200","created":107,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":10076,"log_dups_size":0,"ondisk_log_size":10076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:52:01.127998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13276,"num_write_kb":424832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5371,"num_bytes_recovered":351993856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":111,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.964762+0000","last_change":"2023-09-23T18:32:26.854615+0000","last_active":"2023-09-23T18:36:42.964762+0000","last_peered":"2023-09-23T18:36:42.964762+0000","last_clean":"2023-09-23T18:36:42.964762+0000","last_became_active":"2023-09-23T18:32:26.806771+0000","last_became_peered":"2023-09-23T18:32:26.806771+0000","last_unstale":"2023-09-23T18:36:42.964762+0000","last_undegraded":"2023-09-23T18:36:42.964762+0000","last_fullsized":"2023-09-23T18:36:42.964762+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":120,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.974980+0000","last_change":"2023-09-23T18:27:33.589116+0000","last_active":"2023-09-23T18:36:42.974980+0000","last_peered":"2023-09-23T18:36:42.974980+0000","last_clean":"2023-09-23T18:36:42.974980+0000","last_became_active":"2023-09-23T18:27:33.588809+0000","last_became_peered":"2023-09-23T18:27:33.588809+0000","last_unstale":"2023-09-23T18:36:42.974980+0000","last_undegraded":"2023-09-23T18:36:42.974980+0000","last_fullsized":"2023-09-23T18:36:42.974980+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"140'12964","reported_seq":13013,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T18:36:38.019408+0000","last_change":"2023-09-23T18:31:11.796358+0000","last_active":"2023-09-23T18:36:38.019408+0000","last_peered":"2023-09-23T18:36:38.019408+0000","last_clean":"2023-09-23T18:36:38.019408+0000","last_became_active":"2023-09-23T18:31:11.795962+0000","last_became_peered":"2023-09-23T18:31:11.795962+0000","last_unstale":"2023-09-23T18:36:38.019408+0000","last_undegraded":"2023-09-23T18:36:38.019408+0000","last_fullsized":"2023-09-23T18:36:38.019408+0000","mapping_epoch":107,"log_start":"112'2900","ondisk_log_start":"112'2900","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":10064,"log_dups_size":0,"ondisk_log_size":10064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:29.481931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12964,"num_write_kb":414848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":116,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.929337+0000","last_change":"2023-09-23T18:33:54.848776+0000","last_active":"2023-09-23T18:36:42.929337+0000","last_peered":"2023-09-23T18:36:42.929337+0000","last_clean":"2023-09-23T18:36:42.929337+0000","last_became_active":"2023-09-23T18:33:54.848380+0000","last_became_peered":"2023-09-23T18:33:54.848380+0000","last_unstale":"2023-09-23T18:36:42.929337+0000","last_undegraded":"2023-09-23T18:36:42.929337+0000","last_fullsized":"2023-09-23T18:36:42.929337+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:17:17.093993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":110,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.937967+0000","last_change":"2023-09-23T18:27:33.582033+0000","last_active":"2023-09-23T18:36:42.937967+0000","last_peered":"2023-09-23T18:36:42.937967+0000","last_clean":"2023-09-23T18:36:42.937967+0000","last_became_active":"2023-09-23T18:27:33.581667+0000","last_became_peered":"2023-09-23T18:27:33.581667+0000","last_unstale":"2023-09-23T18:36:42.937967+0000","last_undegraded":"2023-09-23T18:36:42.937967+0000","last_fullsized":"2023-09-23T18:36:42.937967+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:36.039922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"140'13202","reported_seq":13260,"reported_epoch":144,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:36:38.020145+0000","last_change":"2023-09-23T18:32:28.082814+0000","last_active":"2023-09-23T18:36:38.020145+0000","last_peered":"2023-09-23T18:36:38.020145+0000","last_clean":"2023-09-23T18:32:25.736744+0000","last_became_active":"2023-09-23T18:32:28.065620+0000","last_became_peered":"2023-09-23T18:32:28.065620+0000","last_unstale":"2023-09-23T18:36:38.020145+0000","last_undegraded":"2023-09-23T18:32:27.851494+0000","last_fullsized":"2023-09-23T18:32:27.798797+0000","mapping_epoch":115,"log_start":"112'3200","ondisk_log_start":"112'3200","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":10002,"log_dups_size":0,"ondisk_log_size":10002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:49.026074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5354,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13202,"num_write_kb":422401,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,7","objects":-5354}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":115,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.929511+0000","last_change":"2023-09-23T18:28:03.297628+0000","last_active":"2023-09-23T18:36:42.929511+0000","last_peered":"2023-09-23T18:36:42.929511+0000","last_clean":"2023-09-23T18:36:42.929511+0000","last_became_active":"2023-09-23T18:28:03.296694+0000","last_became_peered":"2023-09-23T18:28:03.296694+0000","last_unstale":"2023-09-23T18:36:42.929511+0000","last_undegraded":"2023-09-23T18:36:42.929511+0000","last_fullsized":"2023-09-23T18:36:42.929511+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:43.375652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":111,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.942072+0000","last_change":"2023-09-23T18:28:03.302326+0000","last_active":"2023-09-23T18:36:42.942072+0000","last_peered":"2023-09-23T18:36:42.942072+0000","last_clean":"2023-09-23T18:36:42.942072+0000","last_became_active":"2023-09-23T18:28:03.301970+0000","last_became_peered":"2023-09-23T18:28:03.301970+0000","last_unstale":"2023-09-23T18:36:42.942072+0000","last_undegraded":"2023-09-23T18:36:42.942072+0000","last_fullsized":"2023-09-23T18:36:42.942072+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"140'13472","reported_seq":13530,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T18:36:44.000756+0000","last_change":"2023-09-23T18:36:44.000457+0000","last_active":"2023-09-23T18:36:44.000756+0000","last_peered":"2023-09-23T18:36:44.000756+0000","last_clean":"2023-09-23T18:36:44.000756+0000","last_became_active":"2023-09-23T18:36:43.946893+0000","last_became_peered":"2023-09-23T18:36:43.946893+0000","last_unstale":"2023-09-23T18:36:44.000756+0000","last_undegraded":"2023-09-23T18:36:44.000756+0000","last_fullsized":"2023-09-23T18:36:44.000756+0000","mapping_epoch":145,"log_start":"119'6100","ondisk_log_start":"119'6100","created":107,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":7372,"log_dups_size":0,"ondisk_log_size":7372,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:18.674323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13472,"num_write_kb":431104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":127,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.975122+0000","last_change":"2023-09-23T18:33:53.855371+0000","last_active":"2023-09-23T18:36:42.975122+0000","last_peered":"2023-09-23T18:36:42.975122+0000","last_clean":"2023-09-23T18:36:42.975122+0000","last_became_active":"2023-09-23T18:33:53.854996+0000","last_became_peered":"2023-09-23T18:33:53.854996+0000","last_unstale":"2023-09-23T18:36:42.975122+0000","last_undegraded":"2023-09-23T18:36:42.975122+0000","last_fullsized":"2023-09-23T18:36:42.975122+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":141,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.942390+0000","last_change":"2023-09-23T18:33:53.845253+0000","last_active":"2023-09-23T18:36:42.942390+0000","last_peered":"2023-09-23T18:36:42.942390+0000","last_clean":"2023-09-23T18:36:42.942390+0000","last_became_active":"2023-09-23T18:33:53.844843+0000","last_became_peered":"2023-09-23T18:33:53.844843+0000","last_unstale":"2023-09-23T18:36:42.942390+0000","last_undegraded":"2023-09-23T18:36:42.942390+0000","last_fullsized":"2023-09-23T18:36:42.942390+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"140'13112","reported_seq":23770,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.942515+0000","last_change":"2023-09-23T18:36:18.645981+0000","last_active":"2023-09-23T18:36:42.942515+0000","last_peered":"2023-09-23T18:36:42.942515+0000","last_clean":"2023-09-23T18:36:42.942515+0000","last_became_active":"2023-09-23T18:36:18.602592+0000","last_became_peered":"2023-09-23T18:36:18.602592+0000","last_unstale":"2023-09-23T18:36:42.942515+0000","last_undegraded":"2023-09-23T18:36:42.942515+0000","last_fullsized":"2023-09-23T18:36:42.942515+0000","mapping_epoch":139,"log_start":"119'6400","ondisk_log_start":"119'6400","created":107,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":6712,"log_dups_size":0,"ondisk_log_size":6712,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13112,"num_write_kb":419584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5297,"num_bytes_recovered":216727552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":112,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.941730+0000","last_change":"2023-09-23T18:27:33.585714+0000","last_active":"2023-09-23T18:36:42.941730+0000","last_peered":"2023-09-23T18:36:42.941730+0000","last_clean":"2023-09-23T18:36:42.941730+0000","last_became_active":"2023-09-23T18:27:33.584641+0000","last_became_peered":"2023-09-23T18:27:33.584641+0000","last_unstale":"2023-09-23T18:36:42.941730+0000","last_undegraded":"2023-09-23T18:36:42.941730+0000","last_fullsized":"2023-09-23T18:36:42.941730+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":126,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.942505+0000","last_change":"2023-09-23T18:33:53.851266+0000","last_active":"2023-09-23T18:36:42.942505+0000","last_peered":"2023-09-23T18:36:42.942505+0000","last_clean":"2023-09-23T18:36:42.942505+0000","last_became_active":"2023-09-23T18:33:53.850926+0000","last_became_peered":"2023-09-23T18:33:53.850926+0000","last_unstale":"2023-09-23T18:36:42.942505+0000","last_undegraded":"2023-09-23T18:36:42.942505+0000","last_fullsized":"2023-09-23T18:36:42.942505+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"140'13352","reported_seq":13404,"reported_epoch":145,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:36:42.943065+0000","last_change":"2023-09-23T18:32:28.237717+0000","last_active":"2023-09-23T18:36:42.943065+0000","last_peered":"2023-09-23T18:36:42.943065+0000","last_clean":"2023-09-23T18:32:25.757964+0000","last_became_active":"2023-09-23T18:32:28.219874+0000","last_became_peered":"2023-09-23T18:32:28.219874+0000","last_unstale":"2023-09-23T18:36:42.943065+0000","last_undegraded":"2023-09-23T18:32:27.848749+0000","last_fullsized":"2023-09-23T18:32:27.803577+0000","mapping_epoch":115,"log_start":"119'6600","ondisk_log_start":"119'6600","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":6752,"log_dups_size":0,"ondisk_log_size":6752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5355,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13352,"num_write_kb":427264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0],"avail_no_missing":["4","0","5"],"object_location_counts":[{"shards":"0,4","objects":-5355}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":133,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.928732+0000","last_change":"2023-09-23T18:33:53.835731+0000","last_active":"2023-09-23T18:36:42.928732+0000","last_peered":"2023-09-23T18:36:42.928732+0000","last_clean":"2023-09-23T18:36:42.928732+0000","last_became_active":"2023-09-23T18:33:53.835238+0000","last_became_peered":"2023-09-23T18:33:53.835238+0000","last_unstale":"2023-09-23T18:36:42.928732+0000","last_undegraded":"2023-09-23T18:36:42.928732+0000","last_fullsized":"2023-09-23T18:36:42.928732+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:07:18.870358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":82,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.932193+0000","last_change":"2023-09-23T18:33:53.855430+0000","last_active":"2023-09-23T18:36:42.932193+0000","last_peered":"2023-09-23T18:36:42.932193+0000","last_clean":"2023-09-23T18:36:42.932193+0000","last_became_active":"2023-09-23T18:33:53.854935+0000","last_became_peered":"2023-09-23T18:33:53.854935+0000","last_unstale":"2023-09-23T18:36:42.932193+0000","last_undegraded":"2023-09-23T18:36:42.932193+0000","last_fullsized":"2023-09-23T18:36:42.932193+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:36.078411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"77'130","reported_seq":608,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.932003+0000","last_change":"2023-09-23T18:33:53.850713+0000","last_active":"2023-09-23T18:36:42.932003+0000","last_peered":"2023-09-23T18:36:42.932003+0000","last_clean":"2023-09-23T18:36:42.932003+0000","last_became_active":"2023-09-23T18:33:53.850011+0000","last_became_peered":"2023-09-23T18:33:53.850011+0000","last_unstale":"2023-09-23T18:36:42.932003+0000","last_undegraded":"2023-09-23T18:36:42.932003+0000","last_fullsized":"2023-09-23T18:36:42.932003+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:58.943556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":772,"num_read_kb":1712,"num_write":126,"num_write_kb":2126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":121,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.929202+0000","last_change":"2023-09-23T18:32:26.793356+0000","last_active":"2023-09-23T18:36:42.929202+0000","last_peered":"2023-09-23T18:36:42.929202+0000","last_clean":"2023-09-23T18:36:42.929202+0000","last_became_active":"2023-09-23T18:32:26.792868+0000","last_became_peered":"2023-09-23T18:32:26.792868+0000","last_unstale":"2023-09-23T18:36:42.929202+0000","last_undegraded":"2023-09-23T18:36:42.929202+0000","last_fullsized":"2023-09-23T18:36:42.929202+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:40:21.193604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":56,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.931617+0000","last_change":"2023-09-23T18:33:54.843246+0000","last_active":"2023-09-23T18:36:42.931617+0000","last_peered":"2023-09-23T18:36:42.931617+0000","last_clean":"2023-09-23T18:36:42.931617+0000","last_became_active":"2023-09-23T18:33:54.842933+0000","last_became_peered":"2023-09-23T18:33:54.842933+0000","last_unstale":"2023-09-23T18:36:42.931617+0000","last_undegraded":"2023-09-23T18:36:42.931617+0000","last_fullsized":"2023-09-23T18:36:42.931617+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:43.716707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"140'13264","reported_seq":24015,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.938841+0000","last_change":"2023-09-23T18:36:37.973421+0000","last_active":"2023-09-23T18:36:42.938841+0000","last_peered":"2023-09-23T18:36:42.938841+0000","last_clean":"2023-09-23T18:36:42.938841+0000","last_became_active":"2023-09-23T18:36:37.934280+0000","last_became_peered":"2023-09-23T18:36:37.934280+0000","last_unstale":"2023-09-23T18:36:42.938841+0000","last_undegraded":"2023-09-23T18:36:42.938841+0000","last_fullsized":"2023-09-23T18:36:42.938841+0000","mapping_epoch":143,"log_start":"119'5900","ondisk_log_start":"119'5900","created":107,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":7364,"log_dups_size":0,"ondisk_log_size":7364,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:01.761472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13264,"num_write_kb":424448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5340,"num_bytes_recovered":179306496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"140'13298","reported_seq":13366,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T18:36:44.003177+0000","last_change":"2023-09-23T18:36:44.003177+0000","last_active":"2023-09-23T18:36:44.003177+0000","last_peered":"2023-09-23T18:36:44.003177+0000","last_clean":"2023-09-23T18:36:44.003177+0000","last_became_active":"2023-09-23T18:36:43.941109+0000","last_became_peered":"2023-09-23T18:36:43.941109+0000","last_unstale":"2023-09-23T18:36:44.003177+0000","last_undegraded":"2023-09-23T18:36:44.003177+0000","last_fullsized":"2023-09-23T18:36:44.003177+0000","mapping_epoch":145,"log_start":"119'5900","ondisk_log_start":"119'5900","created":107,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":7398,"log_dups_size":0,"ondisk_log_size":7398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:21.814699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13298,"num_write_kb":425536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'96","reported_seq":552,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.931187+0000","last_change":"2023-09-23T18:33:54.842857+0000","last_active":"2023-09-23T18:36:42.931187+0000","last_peered":"2023-09-23T18:36:42.931187+0000","last_clean":"2023-09-23T18:36:42.931187+0000","last_became_active":"2023-09-23T18:33:54.842134+0000","last_became_peered":"2023-09-23T18:33:54.842134+0000","last_unstale":"2023-09-23T18:36:42.931187+0000","last_undegraded":"2023-09-23T18:36:42.931187+0000","last_fullsized":"2023-09-23T18:36:42.931187+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":96,"log_dups_size":0,"ondisk_log_size":96,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:48.960158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":728,"num_read_kb":1234,"num_write":87,"num_write_kb":338,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":112,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.941928+0000","last_change":"2023-09-23T18:26:34.992535+0000","last_active":"2023-09-23T18:36:42.941928+0000","last_peered":"2023-09-23T18:36:42.941928+0000","last_clean":"2023-09-23T18:36:42.941928+0000","last_became_active":"2023-09-23T18:26:34.992177+0000","last_became_peered":"2023-09-23T18:26:34.992177+0000","last_unstale":"2023-09-23T18:36:42.941928+0000","last_undegraded":"2023-09-23T18:36:42.941928+0000","last_fullsized":"2023-09-23T18:36:42.941928+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":97,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.931870+0000","last_change":"2023-09-23T18:33:53.841776+0000","last_active":"2023-09-23T18:36:42.931870+0000","last_peered":"2023-09-23T18:36:42.931870+0000","last_clean":"2023-09-23T18:36:42.931870+0000","last_became_active":"2023-09-23T18:33:53.841436+0000","last_became_peered":"2023-09-23T18:33:53.841436+0000","last_unstale":"2023-09-23T18:36:42.931870+0000","last_undegraded":"2023-09-23T18:36:42.931870+0000","last_fullsized":"2023-09-23T18:36:42.931870+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:27.198179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"140'13408","reported_seq":13464,"reported_epoch":144,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:36:38.020287+0000","last_change":"2023-09-23T18:32:28.232018+0000","last_active":"2023-09-23T18:36:38.020287+0000","last_peered":"2023-09-23T18:36:38.020287+0000","last_clean":"2023-09-23T18:32:25.754559+0000","last_became_active":"2023-09-23T18:32:28.220390+0000","last_became_peered":"2023-09-23T18:32:28.220390+0000","last_unstale":"2023-09-23T18:36:38.020287+0000","last_undegraded":"2023-09-23T18:32:27.868829+0000","last_fullsized":"2023-09-23T18:32:27.798864+0000","mapping_epoch":115,"log_start":"112'3400","ondisk_log_start":"112'3400","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":10008,"log_dups_size":0,"ondisk_log_size":10008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:07.398730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13408,"num_write_kb":429056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,7","objects":-5398}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"77'112","reported_seq":583,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T18:36:38.019235+0000","last_change":"2023-09-23T18:28:03.297540+0000","last_active":"2023-09-23T18:36:38.019235+0000","last_peered":"2023-09-23T18:36:38.019235+0000","last_clean":"2023-09-23T18:36:38.019235+0000","last_became_active":"2023-09-23T18:28:03.296649+0000","last_became_peered":"2023-09-23T18:28:03.296649+0000","last_unstale":"2023-09-23T18:36:38.019235+0000","last_undegraded":"2023-09-23T18:36:38.019235+0000","last_fullsized":"2023-09-23T18:36:38.019235+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:32.531775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":774,"num_read_kb":596,"num_write":102,"num_write_kb":52,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":78,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.931072+0000","last_change":"2023-09-23T18:33:53.849084+0000","last_active":"2023-09-23T18:36:42.931072+0000","last_peered":"2023-09-23T18:36:42.931072+0000","last_clean":"2023-09-23T18:36:42.931072+0000","last_became_active":"2023-09-23T18:33:53.848757+0000","last_became_peered":"2023-09-23T18:33:53.848757+0000","last_unstale":"2023-09-23T18:36:42.931072+0000","last_undegraded":"2023-09-23T18:36:42.931072+0000","last_fullsized":"2023-09-23T18:36:42.931072+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:15.267742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":101,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.932300+0000","last_change":"2023-09-23T18:33:55.087304+0000","last_active":"2023-09-23T18:36:42.932300+0000","last_peered":"2023-09-23T18:36:42.932300+0000","last_clean":"2023-09-23T18:36:42.932300+0000","last_became_active":"2023-09-23T18:33:55.086832+0000","last_became_peered":"2023-09-23T18:33:55.086832+0000","last_unstale":"2023-09-23T18:36:42.932300+0000","last_undegraded":"2023-09-23T18:36:42.932300+0000","last_fullsized":"2023-09-23T18:36:42.932300+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:55.457664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'147","reported_seq":532,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T18:36:38.492595+0000","last_change":"2023-09-23T18:32:29.845404+0000","last_active":"2023-09-23T18:36:38.492595+0000","last_peered":"2023-09-23T18:36:38.492595+0000","last_clean":"2023-09-23T18:36:38.492595+0000","last_became_active":"2023-09-23T18:32:29.844545+0000","last_became_peered":"2023-09-23T18:32:29.844545+0000","last_unstale":"2023-09-23T18:36:38.492595+0000","last_undegraded":"2023-09-23T18:36:38.492595+0000","last_fullsized":"2023-09-23T18:36:38.492595+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:44.834482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":624,"num_read_kb":1868,"num_write":143,"num_write_kb":5368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":348,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.963403+0000","last_change":"2023-09-23T18:26:34.999402+0000","last_active":"2023-09-23T18:36:42.963403+0000","last_peered":"2023-09-23T18:36:42.963403+0000","last_clean":"2023-09-23T18:36:42.963403+0000","last_became_active":"2023-09-23T18:26:34.998828+0000","last_became_peered":"2023-09-23T18:26:34.998828+0000","last_unstale":"2023-09-23T18:36:42.963403+0000","last_undegraded":"2023-09-23T18:36:42.963403+0000","last_fullsized":"2023-09-23T18:36:42.963403+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_clean_scrub_stamp":"2023-09-23T18:08:35.701374+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":117,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.928931+0000","last_change":"2023-09-23T18:32:26.800920+0000","last_active":"2023-09-23T18:36:42.928931+0000","last_peered":"2023-09-23T18:36:42.928931+0000","last_clean":"2023-09-23T18:36:42.928931+0000","last_became_active":"2023-09-23T18:32:26.800528+0000","last_became_peered":"2023-09-23T18:32:26.800528+0000","last_unstale":"2023-09-23T18:36:42.928931+0000","last_undegraded":"2023-09-23T18:36:42.928931+0000","last_fullsized":"2023-09-23T18:36:42.928931+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:20:50.939219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":96,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.975034+0000","last_change":"2023-09-23T18:15:08.921629+0000","last_active":"2023-09-23T18:36:42.975034+0000","last_peered":"2023-09-23T18:36:42.975034+0000","last_clean":"2023-09-23T18:36:42.975034+0000","last_became_active":"2023-09-23T18:15:08.921283+0000","last_became_peered":"2023-09-23T18:15:08.921283+0000","last_unstale":"2023-09-23T18:36:42.975034+0000","last_undegraded":"2023-09-23T18:36:42.975034+0000","last_fullsized":"2023-09-23T18:36:42.975034+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"140'13294","reported_seq":14561,"reported_epoch":146,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:36:45.737670+0000","last_change":"2023-09-23T18:36:17.529180+0000","last_active":"2023-09-23T18:36:45.737670+0000","last_peered":"2023-09-23T18:36:45.737670+0000","last_clean":"2023-09-23T18:32:25.776893+0000","last_became_active":"2023-09-23T18:32:28.091250+0000","last_became_peered":"2023-09-23T18:32:28.091250+0000","last_unstale":"2023-09-23T18:36:45.737670+0000","last_undegraded":"2023-09-23T18:32:27.847048+0000","last_fullsized":"2023-09-23T18:32:27.801418+0000","mapping_epoch":115,"log_start":"113'5100","ondisk_log_start":"113'5100","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":8194,"log_dups_size":0,"ondisk_log_size":8194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4749,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13294,"num_write_kb":425408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":606,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2],"avail_no_missing":["7","2","5"],"object_location_counts":[{"shards":"2,7","objects":-4749}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"140'13330","reported_seq":24133,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T18:36:44.009586+0000","last_change":"2023-09-23T18:36:44.009240+0000","last_active":"2023-09-23T18:36:44.009586+0000","last_peered":"2023-09-23T18:36:44.009586+0000","last_clean":"2023-09-23T18:36:44.009586+0000","last_became_active":"2023-09-23T18:36:43.941227+0000","last_became_peered":"2023-09-23T18:36:43.941227+0000","last_unstale":"2023-09-23T18:36:44.009586+0000","last_undegraded":"2023-09-23T18:36:44.009586+0000","last_fullsized":"2023-09-23T18:36:44.009586+0000","mapping_epoch":145,"log_start":"119'6000","ondisk_log_start":"119'6000","created":107,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":7330,"log_dups_size":0,"ondisk_log_size":7330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:06.323270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13330,"num_write_kb":426560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5364,"num_bytes_recovered":229572608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"77'120","reported_seq":607,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.963710+0000","last_change":"2023-09-23T18:33:53.852697+0000","last_active":"2023-09-23T18:36:42.963710+0000","last_peered":"2023-09-23T18:36:42.963710+0000","last_clean":"2023-09-23T18:36:42.963710+0000","last_became_active":"2023-09-23T18:33:53.852210+0000","last_became_peered":"2023-09-23T18:33:53.852210+0000","last_unstale":"2023-09-23T18:36:42.963710+0000","last_undegraded":"2023-09-23T18:36:42.963710+0000","last_fullsized":"2023-09-23T18:36:42.963710+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":120,"log_dups_size":0,"ondisk_log_size":120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":809,"num_read_kb":2749,"num_write":98,"num_write_kb":2087,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":97,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.937036+0000","last_change":"2023-09-23T18:28:03.299508+0000","last_active":"2023-09-23T18:36:42.937036+0000","last_peered":"2023-09-23T18:36:42.937036+0000","last_clean":"2023-09-23T18:36:42.937036+0000","last_became_active":"2023-09-23T18:28:03.298682+0000","last_became_peered":"2023-09-23T18:28:03.298682+0000","last_unstale":"2023-09-23T18:36:42.937036+0000","last_undegraded":"2023-09-23T18:36:42.937036+0000","last_fullsized":"2023-09-23T18:36:42.937036+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:08.398302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":128,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.956682+0000","last_change":"2023-09-23T18:33:53.846904+0000","last_active":"2023-09-23T18:36:42.956682+0000","last_peered":"2023-09-23T18:36:42.956682+0000","last_clean":"2023-09-23T18:36:42.956682+0000","last_became_active":"2023-09-23T18:33:53.846654+0000","last_became_peered":"2023-09-23T18:33:53.846654+0000","last_unstale":"2023-09-23T18:36:42.956682+0000","last_undegraded":"2023-09-23T18:36:42.956682+0000","last_fullsized":"2023-09-23T18:36:42.956682+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:55.898016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'78","reported_seq":382,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.964539+0000","last_change":"2023-09-23T18:26:34.992345+0000","last_active":"2023-09-23T18:36:42.964539+0000","last_peered":"2023-09-23T18:36:42.964539+0000","last_clean":"2023-09-23T18:36:42.964539+0000","last_became_active":"2023-09-23T18:26:34.991774+0000","last_became_peered":"2023-09-23T18:26:34.991774+0000","last_unstale":"2023-09-23T18:36:42.964539+0000","last_undegraded":"2023-09-23T18:36:42.964539+0000","last_fullsized":"2023-09-23T18:36:42.964539+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":78,"log_dups_size":0,"ondisk_log_size":78,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":428,"num_read_kb":1125,"num_write":73,"num_write_kb":431,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":76,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.932128+0000","last_change":"2023-09-23T18:33:54.897293+0000","last_active":"2023-09-23T18:36:42.932128+0000","last_peered":"2023-09-23T18:36:42.932128+0000","last_clean":"2023-09-23T18:36:42.932128+0000","last_became_active":"2023-09-23T18:33:54.896491+0000","last_became_peered":"2023-09-23T18:33:54.896491+0000","last_unstale":"2023-09-23T18:36:42.932128+0000","last_undegraded":"2023-09-23T18:36:42.932128+0000","last_fullsized":"2023-09-23T18:36:42.932128+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:03.636617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":103,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.938752+0000","last_change":"2023-09-23T18:33:54.852309+0000","last_active":"2023-09-23T18:36:42.938752+0000","last_peered":"2023-09-23T18:36:42.938752+0000","last_clean":"2023-09-23T18:36:42.938752+0000","last_became_active":"2023-09-23T18:33:54.850255+0000","last_became_peered":"2023-09-23T18:33:54.850255+0000","last_unstale":"2023-09-23T18:36:42.938752+0000","last_undegraded":"2023-09-23T18:36:42.938752+0000","last_fullsized":"2023-09-23T18:36:42.938752+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:54.148992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"140'13288","reported_seq":13342,"reported_epoch":145,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:36:42.975337+0000","last_change":"2023-09-23T18:32:28.155678+0000","last_active":"2023-09-23T18:36:42.975337+0000","last_peered":"2023-09-23T18:36:42.975337+0000","last_clean":"2023-09-23T18:32:25.776264+0000","last_became_active":"2023-09-23T18:32:28.136844+0000","last_became_peered":"2023-09-23T18:32:28.136844+0000","last_unstale":"2023-09-23T18:36:42.975337+0000","last_undegraded":"2023-09-23T18:32:27.856830+0000","last_fullsized":"2023-09-23T18:32:27.810366+0000","mapping_epoch":115,"log_start":"113'4900","ondisk_log_start":"113'4900","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":8388,"log_dups_size":0,"ondisk_log_size":8388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5380,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13288,"num_write_kb":425216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0],"avail_no_missing":["6","0","5"],"object_location_counts":[{"shards":"0,6","objects":-5380}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":116,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.929499+0000","last_change":"2023-09-23T18:28:03.298805+0000","last_active":"2023-09-23T18:36:42.929499+0000","last_peered":"2023-09-23T18:36:42.929499+0000","last_clean":"2023-09-23T18:36:42.929499+0000","last_became_active":"2023-09-23T18:28:03.298437+0000","last_became_peered":"2023-09-23T18:28:03.298437+0000","last_unstale":"2023-09-23T18:36:42.929499+0000","last_undegraded":"2023-09-23T18:36:42.929499+0000","last_fullsized":"2023-09-23T18:36:42.929499+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:27.831669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'181","reported_seq":616,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.931542+0000","last_change":"2023-09-23T18:33:53.854528+0000","last_active":"2023-09-23T18:36:42.931542+0000","last_peered":"2023-09-23T18:36:42.931542+0000","last_clean":"2023-09-23T18:36:42.931542+0000","last_became_active":"2023-09-23T18:33:53.851883+0000","last_became_peered":"2023-09-23T18:33:53.851883+0000","last_unstale":"2023-09-23T18:36:42.931542+0000","last_undegraded":"2023-09-23T18:36:42.931542+0000","last_fullsized":"2023-09-23T18:36:42.931542+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":181,"log_dups_size":0,"ondisk_log_size":181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:43.888844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":909,"num_read_kb":734,"num_write":148,"num_write_kb":75,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":113,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.928669+0000","last_change":"2023-09-23T18:28:03.292295+0000","last_active":"2023-09-23T18:36:42.928669+0000","last_peered":"2023-09-23T18:36:42.928669+0000","last_clean":"2023-09-23T18:36:42.928669+0000","last_became_active":"2023-09-23T18:28:03.291902+0000","last_became_peered":"2023-09-23T18:28:03.291902+0000","last_unstale":"2023-09-23T18:36:42.928669+0000","last_undegraded":"2023-09-23T18:36:42.928669+0000","last_fullsized":"2023-09-23T18:36:42.928669+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:38.415256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"140'13196","reported_seq":13243,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.941505+0000","last_change":"2023-09-23T18:31:11.796383+0000","last_active":"2023-09-23T18:36:42.941505+0000","last_peered":"2023-09-23T18:36:42.941505+0000","last_clean":"2023-09-23T18:36:42.941505+0000","last_became_active":"2023-09-23T18:31:11.796024+0000","last_became_peered":"2023-09-23T18:31:11.796024+0000","last_unstale":"2023-09-23T18:36:42.941505+0000","last_undegraded":"2023-09-23T18:36:42.941505+0000","last_fullsized":"2023-09-23T18:36:42.941505+0000","mapping_epoch":107,"log_start":"119'6500","ondisk_log_start":"119'6500","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":6696,"log_dups_size":0,"ondisk_log_size":6696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13196,"num_write_kb":422272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'112","reported_seq":636,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.975576+0000","last_change":"2023-09-23T18:27:33.587478+0000","last_active":"2023-09-23T18:36:42.975576+0000","last_peered":"2023-09-23T18:36:42.975576+0000","last_clean":"2023-09-23T18:36:42.975576+0000","last_became_active":"2023-09-23T18:27:33.586958+0000","last_became_peered":"2023-09-23T18:27:33.586958+0000","last_unstale":"2023-09-23T18:36:42.975576+0000","last_undegraded":"2023-09-23T18:36:42.975576+0000","last_fullsized":"2023-09-23T18:36:42.975576+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":893,"num_read_kb":3172,"num_write":69,"num_write_kb":2237,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":116,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.956683+0000","last_change":"2023-09-23T18:33:53.851421+0000","last_active":"2023-09-23T18:36:42.956683+0000","last_peered":"2023-09-23T18:36:42.956683+0000","last_clean":"2023-09-23T18:36:42.956683+0000","last_became_active":"2023-09-23T18:33:53.851012+0000","last_became_peered":"2023-09-23T18:33:53.851012+0000","last_unstale":"2023-09-23T18:36:42.956683+0000","last_undegraded":"2023-09-23T18:36:42.956683+0000","last_fullsized":"2023-09-23T18:36:42.956683+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:03.971054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":105,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.941748+0000","last_change":"2023-09-23T18:32:26.805472+0000","last_active":"2023-09-23T18:36:42.941748+0000","last_peered":"2023-09-23T18:36:42.941748+0000","last_clean":"2023-09-23T18:36:42.941748+0000","last_became_active":"2023-09-23T18:32:26.805078+0000","last_became_peered":"2023-09-23T18:32:26.805078+0000","last_unstale":"2023-09-23T18:36:42.941748+0000","last_undegraded":"2023-09-23T18:36:42.941748+0000","last_fullsized":"2023-09-23T18:36:42.941748+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"140'13176","reported_seq":21640,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.928353+0000","last_change":"2023-09-23T18:34:27.989044+0000","last_active":"2023-09-23T18:36:42.928353+0000","last_peered":"2023-09-23T18:36:42.928353+0000","last_clean":"2023-09-23T18:36:42.928353+0000","last_became_active":"2023-09-23T18:34:27.912388+0000","last_became_peered":"2023-09-23T18:34:27.912388+0000","last_unstale":"2023-09-23T18:36:42.928353+0000","last_undegraded":"2023-09-23T18:36:42.928353+0000","last_fullsized":"2023-09-23T18:36:42.928353+0000","mapping_epoch":128,"log_start":"118'5400","ondisk_log_start":"118'5400","created":107,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:31:10.759555+0000","last_clean_scrub_stamp":"2023-09-23T18:31:10.759555+0000","objects_scrubbed":0,"log_size":7776,"log_dups_size":0,"ondisk_log_size":7776,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:19:56.961859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13176,"num_write_kb":421632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4197,"num_bytes_recovered":274989056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":86,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.938271+0000","last_change":"2023-09-23T18:26:34.994176+0000","last_active":"2023-09-23T18:36:42.938271+0000","last_peered":"2023-09-23T18:36:42.938271+0000","last_clean":"2023-09-23T18:36:42.938271+0000","last_became_active":"2023-09-23T18:26:34.993829+0000","last_became_peered":"2023-09-23T18:26:34.993829+0000","last_unstale":"2023-09-23T18:36:42.938271+0000","last_undegraded":"2023-09-23T18:36:42.938271+0000","last_fullsized":"2023-09-23T18:36:42.938271+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:27.111857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":150,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T18:36:38.019554+0000","last_change":"2023-09-23T18:33:53.840519+0000","last_active":"2023-09-23T18:36:38.019554+0000","last_peered":"2023-09-23T18:36:38.019554+0000","last_clean":"2023-09-23T18:36:38.019554+0000","last_became_active":"2023-09-23T18:33:53.840181+0000","last_became_peered":"2023-09-23T18:33:53.840181+0000","last_unstale":"2023-09-23T18:36:38.019554+0000","last_undegraded":"2023-09-23T18:36:38.019554+0000","last_fullsized":"2023-09-23T18:36:38.019554+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:41.293009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":69,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.942017+0000","last_change":"2023-09-23T18:33:54.847172+0000","last_active":"2023-09-23T18:36:42.942017+0000","last_peered":"2023-09-23T18:36:42.942017+0000","last_clean":"2023-09-23T18:36:42.942017+0000","last_became_active":"2023-09-23T18:33:54.846751+0000","last_became_peered":"2023-09-23T18:33:54.846751+0000","last_unstale":"2023-09-23T18:36:42.942017+0000","last_undegraded":"2023-09-23T18:36:42.942017+0000","last_fullsized":"2023-09-23T18:36:42.942017+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":141,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.942016+0000","last_change":"2023-09-23T18:33:53.841232+0000","last_active":"2023-09-23T18:36:42.942016+0000","last_peered":"2023-09-23T18:36:42.942016+0000","last_clean":"2023-09-23T18:36:42.942016+0000","last_became_active":"2023-09-23T18:33:53.840749+0000","last_became_peered":"2023-09-23T18:33:53.840749+0000","last_unstale":"2023-09-23T18:36:42.942016+0000","last_undegraded":"2023-09-23T18:36:42.942016+0000","last_fullsized":"2023-09-23T18:36:42.942016+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":69,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.942505+0000","last_change":"2023-09-23T18:33:54.858971+0000","last_active":"2023-09-23T18:36:42.942505+0000","last_peered":"2023-09-23T18:36:42.942505+0000","last_clean":"2023-09-23T18:36:42.942505+0000","last_became_active":"2023-09-23T18:33:54.845921+0000","last_became_peered":"2023-09-23T18:33:54.845921+0000","last_unstale":"2023-09-23T18:36:42.942505+0000","last_undegraded":"2023-09-23T18:36:42.942505+0000","last_fullsized":"2023-09-23T18:36:42.942505+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":126,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.941806+0000","last_change":"2023-09-23T18:28:03.302261+0000","last_active":"2023-09-23T18:36:42.941806+0000","last_peered":"2023-09-23T18:36:42.941806+0000","last_clean":"2023-09-23T18:36:42.941806+0000","last_became_active":"2023-09-23T18:28:03.301911+0000","last_became_peered":"2023-09-23T18:28:03.301911+0000","last_unstale":"2023-09-23T18:36:42.941806+0000","last_undegraded":"2023-09-23T18:36:42.941806+0000","last_fullsized":"2023-09-23T18:36:42.941806+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":123,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.929088+0000","last_change":"2023-09-23T18:28:03.296850+0000","last_active":"2023-09-23T18:36:42.929088+0000","last_peered":"2023-09-23T18:36:42.929088+0000","last_clean":"2023-09-23T18:36:42.929088+0000","last_became_active":"2023-09-23T18:28:03.295842+0000","last_became_peered":"2023-09-23T18:28:03.295842+0000","last_unstale":"2023-09-23T18:36:42.929088+0000","last_undegraded":"2023-09-23T18:36:42.929088+0000","last_fullsized":"2023-09-23T18:36:42.929088+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:53.048579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":135,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.928472+0000","last_change":"2023-09-23T18:28:03.299520+0000","last_active":"2023-09-23T18:36:42.928472+0000","last_peered":"2023-09-23T18:36:42.928472+0000","last_clean":"2023-09-23T18:36:42.928472+0000","last_became_active":"2023-09-23T18:28:03.299158+0000","last_became_peered":"2023-09-23T18:28:03.299158+0000","last_unstale":"2023-09-23T18:36:42.928472+0000","last_undegraded":"2023-09-23T18:36:42.928472+0000","last_fullsized":"2023-09-23T18:36:42.928472+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:26.247064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":116,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.956868+0000","last_change":"2023-09-23T18:32:26.805676+0000","last_active":"2023-09-23T18:36:42.956868+0000","last_peered":"2023-09-23T18:36:42.956868+0000","last_clean":"2023-09-23T18:36:42.956868+0000","last_became_active":"2023-09-23T18:32:26.797129+0000","last_became_peered":"2023-09-23T18:32:26.797129+0000","last_unstale":"2023-09-23T18:36:42.956868+0000","last_undegraded":"2023-09-23T18:36:42.956868+0000","last_fullsized":"2023-09-23T18:36:42.956868+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:07.414550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":125,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T18:36:42.941411+0000","last_change":"2023-09-23T18:33:54.847156+0000","last_active":"2023-09-23T18:36:42.941411+0000","last_peered":"2023-09-23T18:36:42.941411+0000","last_clean":"2023-09-23T18:36:42.941411+0000","last_became_active":"2023-09-23T18:33:54.846765+0000","last_became_peered":"2023-09-23T18:33:54.846765+0000","last_unstale":"2023-09-23T18:36:42.941411+0000","last_undegraded":"2023-09-23T18:36:42.941411+0000","last_fullsized":"2023-09-23T18:36:42.941411+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":32927,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":212566,"num_write_kb":6802049,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26175,"num_bytes_recovered":1252589568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2544959488,"data_stored":2544959488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":132366,"ondisk_log_size":132366,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":624,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26886,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5937,"num_read_kb":13190,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5621,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430507,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":315620,"kb_used_data":6108,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93429532,"statfs":{"total":95995035648,"available":95671840768,"internally_reserved":0,"allocated":6254592,"data_stored":5702671,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:35:56 2023","interfaces":[{"interface":"back","average":{"1min":2.1339999999999999,"5min":2.3359999999999999,"15min":1.3979999999999999},"min":{"1min":0.30299999999999999,"5min":0.252,"15min":0.252},"max":{"1min":14.653,"5min":57.049999999999997,"15min":57.049999999999997},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.9550000000000001,"5min":2.298,"15min":1.3999999999999999},"min":{"1min":0.33700000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":9.9939999999999998,"5min":56.773000000000003,"15min":56.773000000000003},"last":1.081}]},{"osd":1,"last update":"Sat Sep 23 18:35:56 2023","interfaces":[{"interface":"back","average":{"1min":1.5509999999999999,"5min":1.206,"15min":1.1479999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":9.2349999999999994,"5min":9.2349999999999994,"15min":9.2349999999999994},"last":1.109},{"interface":"front","average":{"1min":1.643,"5min":1.446,"15min":1.413},"min":{"1min":0.35799999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.984,"5min":4.984,"15min":4.984},"last":0.92700000000000005}]},{"osd":2,"last update":"Sat Sep 23 18:35:48 2023","interfaces":[{"interface":"back","average":{"1min":1.8440000000000001,"5min":1.8480000000000001,"15min":1.268},"min":{"1min":0.312,"5min":0.254,"15min":0.254},"max":{"1min":6.9080000000000004,"5min":40.091999999999999,"15min":40.091999999999999},"last":1.1499999999999999},{"interface":"front","average":{"1min":1.6919999999999999,"5min":1.6100000000000001,"15min":1.1739999999999999},"min":{"1min":0.32000000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":9.2669999999999995,"5min":38.261000000000003,"15min":38.261000000000003},"last":0.77700000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:36:14 2023","interfaces":[{"interface":"back","average":{"1min":1.397,"5min":0.98899999999999999,"15min":0.99099999999999999},"min":{"1min":0.39000000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":5.4029999999999996,"5min":5.8920000000000003,"15min":5.8920000000000003},"last":0.84899999999999998},{"interface":"front","average":{"1min":1.02,"5min":1.056,"15min":0.98299999999999998},"min":{"1min":0.35699999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":4.0010000000000003,"5min":4.6929999999999996,"15min":4.6929999999999996},"last":0.80400000000000005}]},{"osd":4,"last update":"Sat Sep 23 18:36:00 2023","interfaces":[{"interface":"back","average":{"1min":1.675,"5min":1.2190000000000001,"15min":1.0009999999999999},"min":{"1min":0.39500000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.3529999999999998,"5min":6.3529999999999998,"15min":6.3529999999999998},"last":1.222},{"interface":"front","average":{"1min":1.167,"5min":1.3180000000000001,"15min":1.121},"min":{"1min":0.32500000000000001,"5min":0.309,"15min":0.309},"max":{"1min":4.3680000000000003,"5min":11.119,"15min":11.119},"last":1.2450000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:36:00 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":1.1779999999999999,"15min":1.0189999999999999},"min":{"1min":0.33800000000000002,"5min":0.315,"15min":0.315},"max":{"1min":4.6559999999999997,"5min":5.8899999999999997,"15min":5.8899999999999997},"last":0.877},{"interface":"front","average":{"1min":1.659,"5min":1.1910000000000001,"15min":1.083},"min":{"1min":0.48499999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.0380000000000003,"5min":7.46,"15min":7.46},"last":1.3080000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:36:00 2023","interfaces":[{"interface":"back","average":{"1min":1.5149999999999999,"5min":1.0660000000000001,"15min":1.0669999999999999},"min":{"1min":0.39700000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":5.5129999999999999,"5min":5.5129999999999999,"15min":5.5129999999999999},"last":1.357},{"interface":"front","average":{"1min":1.4199999999999999,"5min":1.103,"15min":1.1240000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":9.1850000000000005,"5min":9.1850000000000005,"15min":9.1850000000000005},"last":1.2829999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626739,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311516,"kb_used_data":2004,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93433636,"statfs":{"total":95995035648,"available":95676043264,"internally_reserved":0,"allocated":2052096,"data_stored":1508479,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:35:55 2023","interfaces":[{"interface":"back","average":{"1min":1.819,"5min":3.0819999999999999,"15min":1.6359999999999999},"min":{"1min":0.29199999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.2539999999999996,"5min":182.655,"15min":182.655},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.8839999999999999,"5min":3.1659999999999999,"15min":1.752},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.2910000000000004,"5min":182.58799999999999,"15min":182.58799999999999},"last":0.60799999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:35:55 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.74399999999999999,"15min":0.76100000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":3.3340000000000001,"5min":3.3340000000000001,"15min":3.3340000000000001},"last":0.77200000000000002},{"interface":"front","average":{"1min":0.92100000000000004,"5min":1.0720000000000001,"15min":1.097},"min":{"1min":0.34799999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.8359999999999999,"5min":3.8359999999999999,"15min":3.8359999999999999},"last":0.58199999999999996}]},{"osd":2,"last update":"Sat Sep 23 18:35:50 2023","interfaces":[{"interface":"back","average":{"1min":1.369,"5min":1.9239999999999999,"15min":1.3149999999999999},"min":{"1min":0.40699999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.2119999999999997,"5min":51.728000000000002,"15min":51.728000000000002},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.3100000000000001,"5min":1.9430000000000001,"15min":1.288},"min":{"1min":0.34699999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.2359999999999998,"5min":51.820999999999998,"15min":51.820999999999998},"last":0.64200000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:36:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":0.95599999999999996,"15min":0.97699999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.6020000000000003,"5min":21.282,"15min":21.282},"last":0.81799999999999995},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.1220000000000001,"15min":1.0289999999999999},"min":{"1min":0.38100000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":4.0529999999999999,"5min":21.260000000000002,"15min":21.260000000000002},"last":0.70699999999999996}]},{"osd":4,"last update":"Sat Sep 23 18:36:09 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":1.0009999999999999,"15min":0.94499999999999995},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.7229999999999999,"5min":8.0169999999999995,"15min":8.0169999999999995},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.878,"5min":0.879,"15min":0.95199999999999996},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":4.1260000000000003,"5min":4.1260000000000003,"15min":4.1260000000000003},"last":0.86399999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:36:09 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.92900000000000005,"15min":0.98599999999999999},"min":{"1min":0.377,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.83199999999999996,"5min":7.9240000000000004,"15min":7.9240000000000004},"last":0.501},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.80700000000000005,"15min":0.97899999999999998},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.91300000000000003,"5min":4.5270000000000001,"15min":4.5270000000000001},"last":0.72099999999999997}]},{"osd":7,"last update":"Sat Sep 23 18:35:50 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":1.0369999999999999,"15min":1.0549999999999999},"min":{"1min":0.44900000000000001,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":4.1589999999999998,"5min":4.6120000000000001,"15min":4.6120000000000001},"last":0.78400000000000003},{"interface":"front","average":{"1min":0.873,"5min":1.3759999999999999,"15min":1.2170000000000001},"min":{"1min":0.39400000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":4.5049999999999999,"5min":9.1129999999999995,"15min":9.1129999999999995},"last":0.55800000000000005}]}]},{"osd":1,"up_from":123,"seq":528280977444,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":92004,"kb_used_data":1628,"kb_used_omap":7,"kb_used_meta":90360,"kb_avail":93653148,"statfs":{"total":95995035648,"available":95900823552,"internally_reserved":0,"allocated":1667072,"data_stored":942579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":92529422},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:36:01 2023","interfaces":[{"interface":"back","average":{"1min":11.238,"5min":9.593,"15min":9.3190000000000008},"min":{"1min":0.36199999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":165.19999999999999,"5min":173.41800000000001,"15min":173.41800000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":10.677,"5min":8.9809999999999999,"15min":8.6989999999999998},"min":{"1min":0.29899999999999999,"5min":0.249,"15min":0.249},"max":{"1min":165.13300000000001,"5min":166.42599999999999,"15min":166.42599999999999},"last":0.437}]},{"osd":2,"last update":"Sat Sep 23 18:36:01 2023","interfaces":[{"interface":"back","average":{"1min":2.1539999999999999,"5min":2.633,"15min":2.7130000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":7.1449999999999996,"5min":13.811,"15min":13.811},"last":0.71299999999999997},{"interface":"front","average":{"1min":2.3769999999999998,"5min":2.3210000000000002,"15min":2.3109999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.0380000000000003,"5min":13.515000000000001,"15min":13.515000000000001},"last":0.61299999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:36:01 2023","interfaces":[{"interface":"back","average":{"1min":1.2949999999999999,"5min":1.073,"15min":1.036},"min":{"1min":0.34100000000000003,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":4.766,"5min":4.766,"15min":4.766},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.2090000000000001,"5min":1.1970000000000001,"15min":1.1950000000000001},"min":{"1min":0.30599999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.4119999999999999,"5min":4.4119999999999999,"15min":4.4119999999999999},"last":0.64800000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:36:01 2023","interfaces":[{"interface":"back","average":{"1min":1.26,"5min":1.1990000000000001,"15min":1.1890000000000001},"min":{"1min":0.40899999999999997,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":4.4699999999999998,"5min":4.4699999999999998,"15min":4.4699999999999998},"last":0.83999999999999997},{"interface":"front","average":{"1min":1.286,"5min":1.4670000000000001,"15min":1.4970000000000001},"min":{"1min":0.36399999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.6260000000000003,"5min":6.3129999999999997,"15min":6.3129999999999997},"last":0.873}]},{"osd":5,"last update":"Sat Sep 23 18:36:01 2023","interfaces":[{"interface":"back","average":{"1min":1.331,"5min":0.83799999999999997,"15min":0.75600000000000001},"min":{"1min":0.41099999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.7889999999999997,"5min":4.7889999999999997,"15min":4.7889999999999997},"last":0.73999999999999999},{"interface":"front","average":{"1min":1.151,"5min":1.3839999999999999,"15min":1.423},"min":{"1min":0.46100000000000002,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":4.5270000000000001,"5min":6.5250000000000004,"15min":6.5250000000000004},"last":0.76100000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:36:01 2023","interfaces":[{"interface":"back","average":{"1min":1.5089999999999999,"5min":1.579,"15min":1.591},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":4.6109999999999998,"5min":6.4859999999999998,"15min":6.4859999999999998},"last":1.016},{"interface":"front","average":{"1min":1.1759999999999999,"5min":1.6679999999999999,"15min":1.75},"min":{"1min":0.374,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":4.6470000000000002,"5min":6.1109999999999998,"15min":6.1109999999999998},"last":0.83299999999999996}]},{"osd":7,"last update":"Sat Sep 23 18:36:01 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.7929999999999999,"15min":1.901},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":4.5049999999999999,"5min":8.1940000000000008,"15min":8.1940000000000008},"last":0.91600000000000004},{"interface":"front","average":{"1min":1.5109999999999999,"5min":1.7589999999999999,"15min":1.8009999999999999},"min":{"1min":0.433,"5min":0.432,"15min":0.432},"max":{"1min":4.8179999999999996,"5min":6.3899999999999997,"15min":6.3899999999999997},"last":0.80100000000000005}]}]},{"osd":0,"up_from":88,"seq":377957122172,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":277200,"kb_used_data":2184,"kb_used_omap":8,"kb_used_meta":274999,"kb_avail":93467952,"statfs":{"total":95995035648,"available":95711182848,"internally_reserved":0,"allocated":2236416,"data_stored":1529836,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":281599870},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:35:59 2023","interfaces":[{"interface":"back","average":{"1min":1.494,"5min":1.8260000000000001,"15min":1.881},"min":{"1min":0.17100000000000001,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":6.4299999999999997,"5min":8.1140000000000008,"15min":8.1140000000000008},"last":0.80900000000000005},{"interface":"front","average":{"1min":1.8160000000000001,"5min":2.3639999999999999,"15min":2.4550000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":6.0529999999999999,"5min":7.6040000000000001,"15min":7.6040000000000001},"last":0.92800000000000005}]},{"osd":2,"last update":"Sat Sep 23 18:35:45 2023","interfaces":[{"interface":"back","average":{"1min":2.677,"5min":2.3650000000000002,"15min":1.3280000000000001},"min":{"1min":0.25700000000000001,"5min":0.192,"15min":0.192},"max":{"1min":10.468,"5min":40.340000000000003,"15min":40.340000000000003},"last":0.60399999999999998},{"interface":"front","average":{"1min":3.0750000000000002,"5min":2.3839999999999999,"15min":1.2509999999999999},"min":{"1min":0.27800000000000002,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":6.8010000000000002,"5min":42.119999999999997,"15min":42.119999999999997},"last":1.109}]},{"osd":3,"last update":"Sat Sep 23 18:36:13 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":2.1419999999999999,"15min":1.2330000000000001},"min":{"1min":0.27600000000000002,"5min":0.216,"15min":0.216},"max":{"1min":7.9349999999999996,"5min":34.942,"15min":34.942},"last":0.83099999999999996},{"interface":"front","average":{"1min":2.0499999999999998,"5min":2.0790000000000002,"15min":1.2629999999999999},"min":{"1min":0.29799999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.6959999999999997,"5min":34.722000000000001,"15min":34.722000000000001},"last":0.94699999999999995}]},{"osd":4,"last update":"Sat Sep 23 18:35:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":1.52,"15min":1.1519999999999999},"min":{"1min":0.29899999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.165,"5min":6.6879999999999997,"15min":6.6879999999999997},"last":0.89200000000000002},{"interface":"front","average":{"1min":3.3570000000000002,"5min":1.98,"15min":1.3280000000000001},"min":{"1min":0.437,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":16.486999999999998,"5min":16.486999999999998,"15min":16.486999999999998},"last":0.56799999999999995}]},{"osd":5,"last update":"Sat Sep 23 18:35:56 2023","interfaces":[{"interface":"back","average":{"1min":2.4060000000000001,"5min":1.6619999999999999,"15min":1.226},"min":{"1min":0.40200000000000002,"5min":0.371,"15min":0.371},"max":{"1min":8.0589999999999993,"5min":8.0589999999999993,"15min":8.0589999999999993},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.9930000000000001,"5min":1.5669999999999999,"15min":1.21},"min":{"1min":0.42799999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":6.7530000000000001,"5min":8.9250000000000007,"15min":8.9250000000000007},"last":0.84399999999999997}]},{"osd":6,"last update":"Sat Sep 23 18:35:56 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":1.7410000000000001,"15min":1.29},"min":{"1min":0.432,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":8.0329999999999995,"5min":8.0329999999999995,"15min":8.0329999999999995},"last":0.753},{"interface":"front","average":{"1min":2.0059999999999998,"5min":1.6339999999999999,"15min":1.2390000000000001},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.7279999999999998,"5min":8.9740000000000002,"15min":8.9740000000000002},"last":1.0049999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:35:56 2023","interfaces":[{"interface":"back","average":{"1min":2.165,"5min":1.786,"15min":1.2949999999999999},"min":{"1min":0.44600000000000001,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":6.774,"5min":9.0600000000000005,"15min":9.0600000000000005},"last":1.0589999999999999},{"interface":"front","average":{"1min":1.8,"5min":1.5449999999999999,"15min":1.23},"min":{"1min":0.42299999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":5.9969999999999999,"5min":6.7229999999999999,"15min":6.7229999999999999},"last":0.97199999999999998}]}]},{"osd":2,"up_from":98,"seq":420906795120,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":222452,"kb_used_data":5676,"kb_used_omap":11,"kb_used_meta":216756,"kb_avail":93522700,"statfs":{"total":95995035648,"available":95767244800,"internally_reserved":0,"allocated":5812224,"data_stored":5123849,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11949,"internal_metadata":221958483},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:35:41 2023","interfaces":[{"interface":"back","average":{"1min":2.5750000000000002,"5min":1.75,"15min":1.0369999999999999},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":9.0660000000000007,"5min":19.431999999999999,"15min":19.431999999999999},"last":0.64300000000000002},{"interface":"front","average":{"1min":1.897,"5min":1.696,"15min":1.0229999999999999},"min":{"1min":0.22500000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":6.7560000000000002,"5min":8.8629999999999995,"15min":8.8629999999999995},"last":0.39600000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":2.2269999999999999,"15min":2.4590000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.0089999999999999,"5min":19.361999999999998,"15min":19.361999999999998},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.724,"5min":2.2789999999999999,"15min":2.371},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":6.6459999999999999,"5min":8.8070000000000004,"15min":8.8070000000000004},"last":0.56599999999999995}]},{"osd":3,"last update":"Sat Sep 23 18:36:23 2023","interfaces":[{"interface":"back","average":{"1min":1.9870000000000001,"5min":1.5289999999999999,"15min":0.99299999999999999},"min":{"1min":0.28199999999999997,"5min":0.218,"15min":0.218},"max":{"1min":5.3949999999999996,"5min":7.7859999999999996,"15min":7.7859999999999996},"last":0.55500000000000005},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.264,"15min":0.90100000000000002},"min":{"1min":0.28199999999999997,"5min":0.249,"15min":0.249},"max":{"1min":4.9390000000000001,"5min":19.407,"15min":19.407},"last":0.72899999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:35:42 2023","interfaces":[{"interface":"back","average":{"1min":1.6000000000000001,"5min":1.599,"15min":1.1579999999999999},"min":{"1min":0.42499999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":4.6710000000000003,"5min":7.8220000000000001,"15min":7.8220000000000001},"last":0.91000000000000003},{"interface":"front","average":{"1min":2.5230000000000001,"5min":1.6910000000000001,"15min":1.222},"min":{"1min":0.51200000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":17.282,"5min":17.282,"15min":17.282},"last":0.84199999999999997}]},{"osd":5,"last update":"Sat Sep 23 18:35:42 2023","interfaces":[{"interface":"back","average":{"1min":2,"5min":1.3959999999999999,"15min":1.141},"min":{"1min":0.49099999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":17.353999999999999,"5min":17.353999999999999,"15min":17.353999999999999},"last":0.76800000000000002},{"interface":"front","average":{"1min":1.6779999999999999,"5min":1.583,"15min":1.194},"min":{"1min":0.48299999999999998,"5min":0.316,"15min":0.316},"max":{"1min":6.6849999999999996,"5min":6.6849999999999996,"15min":6.6849999999999996},"last":0.67800000000000005}]},{"osd":6,"last update":"Sat Sep 23 18:35:42 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":1.4630000000000001,"15min":1.179},"min":{"1min":0.38300000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":4.6449999999999996,"5min":7.867,"15min":7.867},"last":0.69699999999999995},{"interface":"front","average":{"1min":2.0699999999999998,"5min":1.4550000000000001,"15min":1.206},"min":{"1min":0.442,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":17.379999999999999,"5min":17.379999999999999,"15min":17.379999999999999},"last":0.80000000000000004}]},{"osd":7,"last update":"Sat Sep 23 18:35:42 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":1.583,"15min":1.24},"min":{"1min":0.56200000000000006,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.6360000000000001,"5min":7.8460000000000001,"15min":7.8460000000000001},"last":0.747},{"interface":"front","average":{"1min":3.194,"5min":1.9850000000000001,"15min":1.409},"min":{"1min":0.57299999999999995,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":17.326000000000001,"5min":17.326000000000001,"15min":17.326000000000001},"last":0.94299999999999995}]}]},{"osd":3,"up_from":103,"seq":442381631595,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":113508,"kb_used_data":5660,"kb_used_omap":4,"kb_used_meta":107835,"kb_avail":93631644,"statfs":{"total":95995035648,"available":95878803456,"internally_reserved":0,"allocated":5795840,"data_stored":5127054,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4826,"internal_metadata":110423334},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:36:15 2023","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":1.5800000000000001,"15min":0.98499999999999999},"min":{"1min":0.217,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":4.5110000000000001,"5min":14.231,"15min":14.231},"last":0.69699999999999995},{"interface":"front","average":{"1min":1.4790000000000001,"5min":1.179,"15min":0.88400000000000001},"min":{"1min":0.28499999999999998,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":5.7969999999999997,"5min":8.4659999999999993,"15min":8.4659999999999993},"last":0.752}]},{"osd":1,"last update":"Sat Sep 23 18:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":1.1439999999999999,"15min":1.2050000000000001},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":1.768,"5min":3.3730000000000002,"15min":3.3730000000000002},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.91200000000000003,"5min":1.288,"15min":1.351},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.6560000000000001,"5min":4.6459999999999999,"15min":4.6459999999999999},"last":0.95999999999999996}]},{"osd":2,"last update":"Sat Sep 23 18:36:15 2023","interfaces":[{"interface":"back","average":{"1min":5.0700000000000003,"5min":4.2370000000000001,"15min":1.9039999999999999},"min":{"1min":0.25700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":41.625,"5min":187.40000000000001,"15min":187.40000000000001},"last":0.85999999999999999},{"interface":"front","average":{"1min":3.7799999999999998,"5min":3.6699999999999999,"15min":1.754},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":41.523000000000003,"5min":187.71700000000001,"15min":187.71700000000001},"last":0.65700000000000003}]},{"osd":4,"last update":"Sat Sep 23 18:36:15 2023","interfaces":[{"interface":"back","average":{"1min":0.81200000000000006,"5min":1.097,"15min":1.0069999999999999},"min":{"1min":0.38500000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.899,"5min":8.3689999999999998,"15min":8.3689999999999998},"last":0.88700000000000001},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.0489999999999999,"15min":1.032},"min":{"1min":0.33500000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.6989999999999998,"5min":5.9429999999999996,"15min":5.9429999999999996},"last":1.143}]},{"osd":5,"last update":"Sat Sep 23 18:36:15 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":1.0489999999999999,"15min":1.0600000000000001},"min":{"1min":0.40100000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":3.7269999999999999,"5min":5.9930000000000003,"15min":5.9930000000000003},"last":1.2350000000000001},{"interface":"front","average":{"1min":1.127,"5min":1.022,"15min":1.079},"min":{"1min":0.36299999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.774,"5min":4.1479999999999997,"15min":4.1479999999999997},"last":1.214}]},{"osd":6,"last update":"Sat Sep 23 18:36:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.042,"15min":1.085},"min":{"1min":0.41099999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":3.738,"5min":5.3070000000000004,"15min":5.3070000000000004},"last":1.357},{"interface":"front","average":{"1min":0.81699999999999995,"5min":1.03,"15min":1.0620000000000001},"min":{"1min":0.41699999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.9219999999999999,"5min":8.4139999999999997,"15min":8.4139999999999997},"last":1.1779999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:36:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.216,"15min":1.1479999999999999},"min":{"1min":0.44800000000000001,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":3.3580000000000001,"5min":8.4459999999999997,"15min":8.4459999999999997},"last":1.262},{"interface":"front","average":{"1min":1.1399999999999999,"5min":1.1579999999999999,"15min":1.147},"min":{"1min":0.45500000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":3.8279999999999998,"5min":6.0259999999999998,"15min":6.0259999999999998},"last":1.319}]}]},{"osd":4,"up_from":30,"seq":128849019205,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306844,"kb_used_data":1428,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93438308,"statfs":{"total":95995035648,"available":95680827392,"internally_reserved":0,"allocated":1462272,"data_stored":917929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.8100000000000001,"5min":2.5640000000000001,"15min":1.47},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":6.3789999999999996,"5min":82.623000000000005,"15min":82.623000000000005},"last":0.88300000000000001},{"interface":"front","average":{"1min":1.4850000000000001,"5min":2.2919999999999998,"15min":1.446},"min":{"1min":0.27600000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":5.8520000000000003,"5min":82.638000000000005,"15min":82.638000000000005},"last":1.123}]},{"osd":1,"last update":"Sat Sep 23 18:35:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3440000000000001,"5min":0.90700000000000003,"15min":0.83399999999999996},"min":{"1min":0.35799999999999998,"5min":0.31,"15min":0.31},"max":{"1min":6.0069999999999997,"5min":6.0069999999999997,"15min":6.0069999999999997},"last":1.0049999999999999},{"interface":"front","average":{"1min":1.341,"5min":1.157,"15min":1.1259999999999999},"min":{"1min":0.33400000000000002,"5min":0.313,"15min":0.313},"max":{"1min":6.1429999999999998,"5min":6.1429999999999998,"15min":6.1429999999999998},"last":1.0649999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":1.3700000000000001,"15min":1.151},"min":{"1min":0.375,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.0880000000000001,"5min":12.552,"15min":12.552},"last":0.73099999999999998},{"interface":"front","average":{"1min":2.3919999999999999,"5min":1.694,"15min":1.234},"min":{"1min":0.443,"5min":0.309,"15min":0.309},"max":{"1min":8.391,"5min":12.221,"15min":12.221},"last":1.1060000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:36:21 2023","interfaces":[{"interface":"back","average":{"1min":1.456,"5min":1.575,"15min":1.1279999999999999},"min":{"1min":0.36699999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":6.3390000000000004,"5min":20.850000000000001,"15min":20.850000000000001},"last":0.90100000000000002},{"interface":"front","average":{"1min":1.411,"5min":1.728,"15min":1.266},"min":{"1min":0.436,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":6.3959999999999999,"5min":20.792999999999999,"15min":20.792999999999999},"last":1.081}]},{"osd":5,"last update":"Sat Sep 23 18:36:44 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":1.331,"15min":1.111},"min":{"1min":0.40000000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.532,"5min":7.819,"15min":7.819},"last":0.70499999999999996},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.2649999999999999,"15min":1.073},"min":{"1min":0.46700000000000003,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":4.1660000000000004,"5min":6.4779999999999998,"15min":6.4779999999999998},"last":1.0289999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:36:08 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":1.292,"15min":1.117},"min":{"1min":0.40100000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":6.3680000000000003,"5min":7.8419999999999996,"15min":7.8419999999999996},"last":1.1539999999999999},{"interface":"front","average":{"1min":1.3879999999999999,"5min":1.091,"15min":1.054},"min":{"1min":0.40000000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.9059999999999997,"5min":5.9059999999999997,"15min":5.9059999999999997},"last":1.2310000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:35:45 2023","interfaces":[{"interface":"back","average":{"1min":2.1880000000000002,"5min":1.575,"15min":1.2549999999999999},"min":{"1min":0.41199999999999998,"5min":0.311,"15min":0.311},"max":{"1min":13.491,"5min":13.491,"15min":13.491},"last":1.1870000000000001},{"interface":"front","average":{"1min":2.6480000000000001,"5min":1.768,"15min":1.3700000000000001},"min":{"1min":0.45600000000000002,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":17.062999999999999,"5min":17.062999999999999,"15min":17.062999999999999},"last":1.2}]}]},{"osd":5,"up_from":0,"seq":154618822973,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4224,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7725,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3429,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2544959488,"data_stored":2544959488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:36:52.209 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:36:52.260 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:52.272 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:51 smithi173 bash[59422]: cluster 2023-09-23T18:36:51.815126+0000 mgr.y (mgr.26054) 549 : cluster 0 pgmap v522: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:36:52.290 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:52.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:52.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:52.394 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:52.412 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:52.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:52.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:52.516 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:52.541 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:52.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:52.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:52.644 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:52.680 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:52.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:52.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:52.783 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:52.813 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:52.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:52.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:52.918 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:52.953 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:52.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:52.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:53.057 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:53.088 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:53.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:53.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:53.191 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:53.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:53.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:53.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:53.329 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:53.362 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:53.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:53.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:53.448 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T18:36:53.448 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T18:36:53.449 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-09-23T18:36:53.449 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-09-23T18:36:53.465 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:53.498 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:53.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:53.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:53.602 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:53.631 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:53.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:53.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:53.734 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:53.768 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:53.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:53.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:53.873 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:53.907 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:53.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:53.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:54.011 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:54.042 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:54.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:54.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:54.146 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:54.174 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:54.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:54.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:54.278 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:54.298 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:54.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:54.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:54.327 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:36:54.350 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:36:54.374 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:36:54.397 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:36:54.401 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:54.419 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:36:54.436 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:54.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:54.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:54.446 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:36:54.477 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:36:54.499 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:36:54.501 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:36:54.501 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:36:54.501 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:36:54.501 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:36:54.501 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:36:54.502 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:36:54.502 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:36:54.502 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:36:54.541 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:54.575 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:54.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:54.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:54.678 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:54.705 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:54.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:54.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:54.808 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:54.839 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:54.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:54.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:54.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:54 smithi098 bash[76817]: cluster 2023-09-23T18:36:53.815906+0000 mgr.y (mgr.26054) 550 : cluster 0 pgmap v523: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:36:54.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:54 smithi098 bash[76817]: audit 2023-09-23T18:36:54.304326+0000 mon.c (mon.2) 74 : audit 0 from='client.? 172.21.15.98:0/1789497815' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:36:54.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:54 smithi098 bash[76817]: audit 2023-09-23T18:36:54.597275+0000 mon.a (mon.0) 784 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:54.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:54 smithi098 bash[76817]: audit 2023-09-23T18:36:54.598260+0000 mon.a (mon.0) 785 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:36:54.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:54 smithi098 bash[79516]: cluster 2023-09-23T18:36:53.815906+0000 mgr.y (mgr.26054) 550 : cluster 0 pgmap v523: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:36:54.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:54 smithi098 bash[79516]: audit 2023-09-23T18:36:54.304326+0000 mon.c (mon.2) 74 : audit 0 from='client.? 172.21.15.98:0/1789497815' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:36:54.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:54 smithi098 bash[79516]: audit 2023-09-23T18:36:54.597275+0000 mon.a (mon.0) 784 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:54.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:54 smithi098 bash[79516]: audit 2023-09-23T18:36:54.598260+0000 mon.a (mon.0) 785 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:36:54.943 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:54.970 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:54.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:54.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:55.073 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:55.086 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:54 smithi173 bash[59422]: cluster 2023-09-23T18:36:53.815906+0000 mgr.y (mgr.26054) 550 : cluster 0 pgmap v523: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:36:55.086 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:54 smithi173 bash[59422]: audit 2023-09-23T18:36:54.304326+0000 mon.c (mon.2) 74 : audit 0 from='client.? 172.21.15.98:0/1789497815' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:36:55.086 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:54 smithi173 bash[59422]: audit 2023-09-23T18:36:54.597275+0000 mon.a (mon.0) 784 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:55.086 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:54 smithi173 bash[59422]: audit 2023-09-23T18:36:54.598260+0000 mon.a (mon.0) 785 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:36:55.105 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:55.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:55.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:55.209 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:55.243 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:55.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:55.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:55.347 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:55.373 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:55.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:55.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:55.476 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:55.509 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:55.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:55.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:55.613 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:55.638 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:55.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:55.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:55.741 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:55.767 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:55.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:55.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:55.871 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:55.907 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:55.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:55.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:56.011 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:56.035 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:56.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:56.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:56.138 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:56.167 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:56.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:56.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:56.270 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:56.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:56 smithi173 bash[59422]: audit 2023-09-23T18:36:55.007982+0000 mon.a (mon.0) 786 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:56.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:56 smithi173 bash[59422]: cluster 2023-09-23T18:36:55.816532+0000 mgr.y (mgr.26054) 551 : cluster 0 pgmap v524: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:36:56.303 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:56.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:56.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:36:56.407 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:56.416 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:56 smithi098 bash[76817]: audit 2023-09-23T18:36:55.007982+0000 mon.a (mon.0) 786 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:56.417 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:56 smithi098 bash[76817]: cluster 2023-09-23T18:36:55.816532+0000 mgr.y (mgr.26054) 551 : cluster 0 pgmap v524: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:36:56.417 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:36:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:36:56] "GET /metrics HTTP/1.1" 200 34161 "" "Prometheus/2.33.4" 2023-09-23T18:36:56.417 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:56 smithi098 bash[79516]: audit 2023-09-23T18:36:55.007982+0000 mon.a (mon.0) 786 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:36:56.417 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:56 smithi098 bash[79516]: cluster 2023-09-23T18:36:55.816532+0000 mgr.y (mgr.26054) 551 : cluster 0 pgmap v524: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:36:56.430 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:56.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:56.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:56.534 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:56.565 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:56.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:56.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:56.669 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:56.701 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:56.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:56.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:56.805 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:56.832 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:56.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:56.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:56.935 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:56.966 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:56.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:56.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:57.070 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:57.102 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:57.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:57.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:57.206 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:57.232 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:57.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:57.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:57.335 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:57.364 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:57.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:57.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:57.468 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:57.495 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:57.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:57.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:57.598 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:57.624 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:57.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:57.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:57.728 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:57.758 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:57.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:57.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:57.861 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:57.887 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:57.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:57.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:57.990 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:58.024 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:58.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:58.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:58.127 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:58.149 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:58.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:58.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:58.252 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:58.275 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:58.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:58.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:58.378 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:58.411 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:58.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:58.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:58.516 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:58.540 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:58.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:58.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:58.644 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:58.667 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:36:58.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:58.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:36:58.772 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:58.800 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:36:58.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:58.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:36:58.887 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is unset 2023-09-23T18:36:58.904 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:58.936 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:36:58.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:58.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:36:59.039 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:59.051 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:58 smithi098 bash[79516]: cluster 2023-09-23T18:36:57.817051+0000 mgr.y (mgr.26054) 552 : cluster 0 pgmap v525: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:36:59.051 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:58 smithi098 bash[79516]: audit 2023-09-23T18:36:58.822227+0000 mon.c (mon.2) 75 : audit 1 from='client.? 172.21.15.98:0/2684926991' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:36:59.052 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:58 smithi098 bash[79516]: audit 2023-09-23T18:36:58.822953+0000 mon.a (mon.0) 787 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:36:59.052 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:58 smithi098 bash[76817]: cluster 2023-09-23T18:36:57.817051+0000 mgr.y (mgr.26054) 552 : cluster 0 pgmap v525: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:36:59.052 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:58 smithi098 bash[76817]: audit 2023-09-23T18:36:58.822227+0000 mon.c (mon.2) 75 : audit 1 from='client.? 172.21.15.98:0/2684926991' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:36:59.052 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:58 smithi098 bash[76817]: audit 2023-09-23T18:36:58.822953+0000 mon.a (mon.0) 787 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:36:59.064 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:59.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:59.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:59.146 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:36:59.146 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [], 2023-09-23T18:36:59.146 INFO:teuthology.orchestra.run.smithi098.stdout: "complaint_time": 30, 2023-09-23T18:36:59.146 INFO:teuthology.orchestra.run.smithi098.stdout: "num_blocked_ops": 0 2023-09-23T18:36:59.146 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:36:59.168 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:59.180 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:58 smithi173 bash[59422]: cluster 2023-09-23T18:36:57.817051+0000 mgr.y (mgr.26054) 552 : cluster 0 pgmap v525: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:36:59.180 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:58 smithi173 bash[59422]: audit 2023-09-23T18:36:58.822227+0000 mon.c (mon.2) 75 : audit 1 from='client.? 172.21.15.98:0/2684926991' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:36:59.180 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:58 smithi173 bash[59422]: audit 2023-09-23T18:36:58.822953+0000 mon.a (mon.0) 787 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:36:59.199 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:36:59.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:59.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:36:59.204 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:36:59.304 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:59.326 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:36:59.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:59.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:36:59.429 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:59.459 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:59.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:59.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:59.563 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:59.592 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:59.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:59.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:59.698 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:59.729 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:36:59.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:59.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:36:59.833 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:59.862 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:36:59.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:36:59.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:36:59.966 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:59.977 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:59 smithi173 bash[59422]: cluster 2023-09-23T18:36:58.870399+0000 mon.a (mon.0) 788 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:36:59.977 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:59 smithi173 bash[59422]: cluster 2023-09-23T18:36:58.870468+0000 mon.a (mon.0) 789 : cluster 1 Cluster is now healthy 2023-09-23T18:36:59.977 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:59 smithi173 bash[59422]: cluster 2023-09-23T18:36:58.880923+0000 osd.7 (osd.7) 57 : cluster 3 failed to encode map e148 with expected crc 2023-09-23T18:36:59.978 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:59 smithi173 bash[59422]: cluster 2023-09-23T18:36:58.881089+0000 osd.6 (osd.6) 61 : cluster 3 failed to encode map e148 with expected crc 2023-09-23T18:36:59.978 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:59 smithi173 bash[59422]: audit 2023-09-23T18:36:58.882043+0000 mon.a (mon.0) 790 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:36:59.978 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:59 smithi173 bash[59422]: cluster 2023-09-23T18:36:58.882099+0000 mon.a (mon.0) 791 : cluster 0 osdmap e148: 8 total, 8 up, 7 in 2023-09-23T18:36:59.978 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:36:59 smithi173 bash[59422]: cluster 2023-09-23T18:36:59.667306+0000 osd.5 (osd.5) 58 : cluster 3 failed to encode map e148 with expected crc 2023-09-23T18:36:59.998 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:36:59.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:00.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:00.100 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:00.132 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:00.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:00.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:00.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:59 smithi098 bash[79516]: cluster 2023-09-23T18:36:58.870399+0000 mon.a (mon.0) 788 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:37:00.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:59 smithi098 bash[79516]: cluster 2023-09-23T18:36:58.870468+0000 mon.a (mon.0) 789 : cluster 1 Cluster is now healthy 2023-09-23T18:37:00.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:59 smithi098 bash[79516]: cluster 2023-09-23T18:36:58.880923+0000 osd.7 (osd.7) 57 : cluster 3 failed to encode map e148 with expected crc 2023-09-23T18:37:00.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:59 smithi098 bash[79516]: cluster 2023-09-23T18:36:58.881089+0000 osd.6 (osd.6) 61 : cluster 3 failed to encode map e148 with expected crc 2023-09-23T18:37:00.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:59 smithi098 bash[79516]: audit 2023-09-23T18:36:58.882043+0000 mon.a (mon.0) 790 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:37:00.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:59 smithi098 bash[79516]: cluster 2023-09-23T18:36:58.882099+0000 mon.a (mon.0) 791 : cluster 0 osdmap e148: 8 total, 8 up, 7 in 2023-09-23T18:37:00.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:36:59 smithi098 bash[79516]: cluster 2023-09-23T18:36:59.667306+0000 osd.5 (osd.5) 58 : cluster 3 failed to encode map e148 with expected crc 2023-09-23T18:37:00.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:59 smithi098 bash[76817]: cluster 2023-09-23T18:36:58.870399+0000 mon.a (mon.0) 788 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:37:00.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:59 smithi098 bash[76817]: cluster 2023-09-23T18:36:58.870468+0000 mon.a (mon.0) 789 : cluster 1 Cluster is now healthy 2023-09-23T18:37:00.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:59 smithi098 bash[76817]: cluster 2023-09-23T18:36:58.880923+0000 osd.7 (osd.7) 57 : cluster 3 failed to encode map e148 with expected crc 2023-09-23T18:37:00.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:59 smithi098 bash[76817]: cluster 2023-09-23T18:36:58.881089+0000 osd.6 (osd.6) 61 : cluster 3 failed to encode map e148 with expected crc 2023-09-23T18:37:00.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:59 smithi098 bash[76817]: audit 2023-09-23T18:36:58.882043+0000 mon.a (mon.0) 790 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:37:00.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:59 smithi098 bash[76817]: cluster 2023-09-23T18:36:58.882099+0000 mon.a (mon.0) 791 : cluster 0 osdmap e148: 8 total, 8 up, 7 in 2023-09-23T18:37:00.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:36:59 smithi098 bash[76817]: cluster 2023-09-23T18:36:59.667306+0000 osd.5 (osd.5) 58 : cluster 3 failed to encode map e148 with expected crc 2023-09-23T18:37:00.236 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:00.268 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:00.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:00.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:00.372 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:00.400 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:00.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:00.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:00.503 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:00.532 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:00.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:00.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:00.635 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T18:37:00.635 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:00.668 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:00.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:00.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:00.772 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:00.802 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:00.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:00.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:00.906 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:00.918 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:00 smithi098 bash[79516]: cluster 2023-09-23T18:36:59.667321+0000 osd.4 (osd.4) 84 : cluster 3 failed to encode map e148 with expected crc 2023-09-23T18:37:00.918 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:00 smithi098 bash[79516]: cluster 2023-09-23T18:36:59.817754+0000 mgr.y (mgr.26054) 553 : cluster 0 pgmap v527: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:00.918 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:00 smithi098 bash[76817]: cluster 2023-09-23T18:36:59.667321+0000 osd.4 (osd.4) 84 : cluster 3 failed to encode map e148 with expected crc 2023-09-23T18:37:00.918 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:00 smithi098 bash[76817]: cluster 2023-09-23T18:36:59.817754+0000 mgr.y (mgr.26054) 553 : cluster 0 pgmap v527: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:00.935 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:00.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:00.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:01.030 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:00 smithi173 bash[59422]: cluster 2023-09-23T18:36:59.667321+0000 osd.4 (osd.4) 84 : cluster 3 failed to encode map e148 with expected crc 2023-09-23T18:37:01.030 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:00 smithi173 bash[59422]: cluster 2023-09-23T18:36:59.817754+0000 mgr.y (mgr.26054) 553 : cluster 0 pgmap v527: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:01.040 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:01.070 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:01.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:01.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:01.132 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:37:01.132 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:37:01.175 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:01.201 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:01.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:01.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:01.304 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:01.337 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:01.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:01.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:01.441 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:01.466 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:01.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:01.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:01.569 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:01.594 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:01.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:01.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:01.698 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:01.730 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:01.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:01.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:01.833 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:01.864 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:01.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:01.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:01.969 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:02.000 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:02.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:02.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:02.103 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:02.131 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:02.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:02.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:02.234 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:02.268 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:02.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:02.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:02.372 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:02.407 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:02.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:02.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:02.510 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:02.538 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:02.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:02.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:02.641 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:02.655 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:37:02.673 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:02.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:02.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:02.777 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:02.799 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:02.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:02.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:02.902 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:02.913 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:02 smithi098 bash[76817]: cluster 2023-09-23T18:37:01.818470+0000 mgr.y (mgr.26054) 554 : cluster 0 pgmap v528: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:02.914 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:02 smithi098 bash[79516]: cluster 2023-09-23T18:37:01.818470+0000 mgr.y (mgr.26054) 554 : cluster 0 pgmap v528: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:02.934 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:02.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:02.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:03.038 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:03.066 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:03.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:03.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:03.170 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:03.182 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:02 smithi173 bash[59422]: cluster 2023-09-23T18:37:01.818470+0000 mgr.y (mgr.26054) 554 : cluster 0 pgmap v528: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:03.205 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:03.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:03.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:03.309 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:03.342 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:03.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:03.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:03.445 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:03.480 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:03.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:03.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:03.585 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:03.614 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:03.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:03.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:03.718 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:03.739 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:03.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:03.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:03.843 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:03.872 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:03.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:03.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:03.976 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:04.002 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:04.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:04.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:04.106 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:04.138 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:04.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:04.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:04.242 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:04.273 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:04.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:04.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:04.377 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:04.412 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:04.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:04.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:04.516 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:04.547 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:04.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:04.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:04.650 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:04.677 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:04.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:04.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:04.782 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:04.812 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:04.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:04.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:04.916 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:04.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:04 smithi098 bash[76817]: cluster 2023-09-23T18:37:03.819129+0000 mgr.y (mgr.26054) 555 : cluster 0 pgmap v529: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:04.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:04 smithi098 bash[79516]: cluster 2023-09-23T18:37:03.819129+0000 mgr.y (mgr.26054) 555 : cluster 0 pgmap v529: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:04.951 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:04.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:04.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:05.055 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:05.088 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:05.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:05.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:05.191 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:05.203 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:04 smithi173 bash[59422]: cluster 2023-09-23T18:37:03.819129+0000 mgr.y (mgr.26054) 555 : cluster 0 pgmap v529: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:05.219 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:05.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:05.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:05.322 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:05.354 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:05.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:05.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:05.459 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:05.459 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:37:05.460 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:37:05.488 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:05.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:05.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:05.592 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:05.619 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:05.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:05.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:05.722 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:05.749 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:05.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:05.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:05.853 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:05.882 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:05.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:05.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:05.985 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:06.013 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:06.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:06.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:06.116 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:06.149 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:06.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:06.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:06.253 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:06.286 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:06.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:06.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:06.390 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:06.424 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:06.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:06.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:06.440 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:37:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:37:06] "GET /metrics HTTP/1.1" 200 34161 "" "Prometheus/2.33.4" 2023-09-23T18:37:06.528 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:06.561 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:06.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:06.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:06.665 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:06.688 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:06.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:06.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:06.792 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:06.821 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:06.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:06.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:06.924 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:06.935 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:06 smithi098 bash[79516]: cluster 2023-09-23T18:37:05.819781+0000 mgr.y (mgr.26054) 556 : cluster 0 pgmap v530: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:06.935 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:06 smithi098 bash[76817]: cluster 2023-09-23T18:37:05.819781+0000 mgr.y (mgr.26054) 556 : cluster 0 pgmap v530: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:06.955 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:06.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:06.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:07.059 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:07.090 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:07.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:07.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:07.194 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:07.221 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:07.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:07.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:07.226 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:37:07.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:06 smithi173 bash[59422]: cluster 2023-09-23T18:37:05.819781+0000 mgr.y (mgr.26054) 556 : cluster 0 pgmap v530: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:07.324 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:07.350 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:07.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:07.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:07.452 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:07.484 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:07.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:07.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:07.587 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:07.608 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:07.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:07.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:07.712 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:07.737 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:07.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:07.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:07.840 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:07.874 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:07.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:07.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:07.979 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:08.005 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:08.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:08.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:08.108 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:08.141 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:08.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:08.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:08.246 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:08.267 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:08.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:08.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:08.371 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:08.400 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:08.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:08.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:08.503 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:08.537 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:08.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:08.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:08.642 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:08.679 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:08.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:08.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:08.783 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:08.806 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:08.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:08.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:08.910 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:08.922 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:08 smithi098 bash[76817]: cluster 2023-09-23T18:37:07.820375+0000 mgr.y (mgr.26054) 557 : cluster 0 pgmap v531: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:08.922 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:08 smithi098 bash[79516]: cluster 2023-09-23T18:37:07.820375+0000 mgr.y (mgr.26054) 557 : cluster 0 pgmap v531: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:08.938 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:08.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:08.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:09.041 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:09.068 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:09.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:09.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:09.172 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:09.185 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:08 smithi173 bash[59422]: cluster 2023-09-23T18:37:07.820375+0000 mgr.y (mgr.26054) 557 : cluster 0 pgmap v531: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:09.206 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:09.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:09.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:09.310 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:09.340 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:09.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:09.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:09.444 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:09.477 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:09.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:09.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:09.580 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:09.601 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:09.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:09.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:09.704 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:09.734 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:09.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:09.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:09.838 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:09.863 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:09.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:09.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:09.915 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:37:09.916 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-23T18:37:09.968 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:09.993 INFO:teuthology.orchestra.run.smithi098.stderr:pool 'unique_pool_1' created 2023-09-23T18:37:10.002 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:10.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:10.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:10.108 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:10.134 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:10.136 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:09 smithi098 bash[76817]: audit 2023-09-23T18:37:09.457778+0000 mon.a (mon.0) 792 : audit 1 from='client.? 172.21.15.98:0/1613216662' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T18:37:10.136 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:09 smithi098 bash[76817]: cluster 2023-09-23T18:37:09.821061+0000 mgr.y (mgr.26054) 558 : cluster 0 pgmap v532: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:10.136 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:09 smithi098 bash[76817]: audit 2023-09-23T18:37:09.903430+0000 mgr.y (mgr.26054) 559 : audit 0 from='client.44927 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:37:10.136 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:09 smithi098 bash[76817]: audit 2023-09-23T18:37:09.933502+0000 mon.a (mon.0) 793 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:10.136 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:09 smithi098 bash[76817]: audit 2023-09-23T18:37:09.934803+0000 mon.a (mon.0) 794 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:37:10.136 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:09 smithi098 bash[79516]: audit 2023-09-23T18:37:09.457778+0000 mon.a (mon.0) 792 : audit 1 from='client.? 172.21.15.98:0/1613216662' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T18:37:10.136 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:09 smithi098 bash[79516]: cluster 2023-09-23T18:37:09.821061+0000 mgr.y (mgr.26054) 558 : cluster 0 pgmap v532: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:10.136 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:09 smithi098 bash[79516]: audit 2023-09-23T18:37:09.903430+0000 mgr.y (mgr.26054) 559 : audit 0 from='client.44927 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:37:10.136 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:09 smithi098 bash[79516]: audit 2023-09-23T18:37:09.933502+0000 mon.a (mon.0) 793 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:10.137 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:09 smithi098 bash[79516]: audit 2023-09-23T18:37:09.934803+0000 mon.a (mon.0) 794 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:37:10.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:10.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:10.238 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:10.261 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:10.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:10.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:10.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:09 smithi173 bash[59422]: audit 2023-09-23T18:37:09.457778+0000 mon.a (mon.0) 792 : audit 1 from='client.? 172.21.15.98:0/1613216662' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T18:37:10.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:09 smithi173 bash[59422]: cluster 2023-09-23T18:37:09.821061+0000 mgr.y (mgr.26054) 558 : cluster 0 pgmap v532: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:10.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:09 smithi173 bash[59422]: audit 2023-09-23T18:37:09.903430+0000 mgr.y (mgr.26054) 559 : audit 0 from='client.44927 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:37:10.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:09 smithi173 bash[59422]: audit 2023-09-23T18:37:09.933502+0000 mon.a (mon.0) 793 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:10.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:09 smithi173 bash[59422]: audit 2023-09-23T18:37:09.934803+0000 mon.a (mon.0) 794 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:37:10.365 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:10.394 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:10.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:10.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:10.498 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:10.523 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:10.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:10.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:10.626 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:10.657 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:10.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:10.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:10.761 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:10.793 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:10.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:10.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:10.897 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:10.919 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:10.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:10.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:11.023 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:11.029 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:10 smithi098 bash[79516]: cluster 2023-09-23T18:37:09.977629+0000 osd.4 (osd.4) 85 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T18:37:11.030 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:10 smithi098 bash[79516]: cluster 2023-09-23T18:37:09.980351+0000 osd.6 (osd.6) 62 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T18:37:11.030 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:10 smithi098 bash[79516]: audit 2023-09-23T18:37:09.982068+0000 mon.a (mon.0) 795 : audit 1 from='client.? 172.21.15.98:0/1613216662' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T18:37:11.030 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:10 smithi098 bash[79516]: cluster 2023-09-23T18:37:09.982220+0000 mon.a (mon.0) 796 : cluster 0 osdmap e149: 8 total, 8 up, 7 in 2023-09-23T18:37:11.030 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:10 smithi098 bash[79516]: cluster 2023-09-23T18:37:09.982653+0000 osd.4 (osd.4) 86 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T18:37:11.030 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:10 smithi098 bash[79516]: cluster 2023-09-23T18:37:09.984126+0000 osd.7 (osd.7) 58 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T18:37:11.030 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:10 smithi098 bash[79516]: cluster 2023-09-23T18:37:09.985465+0000 osd.7 (osd.7) 59 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T18:37:11.030 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:10 smithi098 bash[79516]: cluster 2023-09-23T18:37:09.986734+0000 osd.7 (osd.7) 60 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T18:37:11.030 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:10 smithi098 bash[79516]: cluster 2023-09-23T18:37:10.006357+0000 osd.5 (osd.5) 59 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T18:37:11.030 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:10 smithi098 bash[79516]: cluster 2023-09-23T18:37:10.008678+0000 osd.5 (osd.5) 60 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T18:37:11.030 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:10 smithi098 bash[79516]: cluster 2023-09-23T18:37:10.980871+0000 mon.a (mon.0) 797 : cluster 0 osdmap e150: 8 total, 8 up, 7 in 2023-09-23T18:37:11.031 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:10 smithi098 bash[76817]: cluster 2023-09-23T18:37:09.977629+0000 osd.4 (osd.4) 85 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T18:37:11.031 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:10 smithi098 bash[76817]: cluster 2023-09-23T18:37:09.980351+0000 osd.6 (osd.6) 62 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T18:37:11.031 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:10 smithi098 bash[76817]: audit 2023-09-23T18:37:09.982068+0000 mon.a (mon.0) 795 : audit 1 from='client.? 172.21.15.98:0/1613216662' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T18:37:11.031 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:10 smithi098 bash[76817]: cluster 2023-09-23T18:37:09.982220+0000 mon.a (mon.0) 796 : cluster 0 osdmap e149: 8 total, 8 up, 7 in 2023-09-23T18:37:11.031 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:10 smithi098 bash[76817]: cluster 2023-09-23T18:37:09.982653+0000 osd.4 (osd.4) 86 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T18:37:11.031 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:10 smithi098 bash[76817]: cluster 2023-09-23T18:37:09.984126+0000 osd.7 (osd.7) 58 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T18:37:11.031 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:10 smithi098 bash[76817]: cluster 2023-09-23T18:37:09.985465+0000 osd.7 (osd.7) 59 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T18:37:11.031 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:10 smithi098 bash[76817]: cluster 2023-09-23T18:37:09.986734+0000 osd.7 (osd.7) 60 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T18:37:11.031 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:10 smithi098 bash[76817]: cluster 2023-09-23T18:37:10.006357+0000 osd.5 (osd.5) 59 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T18:37:11.031 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:10 smithi098 bash[76817]: cluster 2023-09-23T18:37:10.008678+0000 osd.5 (osd.5) 60 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T18:37:11.031 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:10 smithi098 bash[76817]: cluster 2023-09-23T18:37:10.980871+0000 mon.a (mon.0) 797 : cluster 0 osdmap e150: 8 total, 8 up, 7 in 2023-09-23T18:37:11.046 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:11.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:11.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:11.149 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:11.183 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:11.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:11.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:11.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:10 smithi173 bash[59422]: cluster 2023-09-23T18:37:09.977629+0000 osd.4 (osd.4) 85 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T18:37:11.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:10 smithi173 bash[59422]: cluster 2023-09-23T18:37:09.980351+0000 osd.6 (osd.6) 62 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T18:37:11.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:10 smithi173 bash[59422]: audit 2023-09-23T18:37:09.982068+0000 mon.a (mon.0) 795 : audit 1 from='client.? 172.21.15.98:0/1613216662' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T18:37:11.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:10 smithi173 bash[59422]: cluster 2023-09-23T18:37:09.982220+0000 mon.a (mon.0) 796 : cluster 0 osdmap e149: 8 total, 8 up, 7 in 2023-09-23T18:37:11.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:10 smithi173 bash[59422]: cluster 2023-09-23T18:37:09.982653+0000 osd.4 (osd.4) 86 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T18:37:11.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:10 smithi173 bash[59422]: cluster 2023-09-23T18:37:09.984126+0000 osd.7 (osd.7) 58 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T18:37:11.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:10 smithi173 bash[59422]: cluster 2023-09-23T18:37:09.985465+0000 osd.7 (osd.7) 59 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T18:37:11.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:10 smithi173 bash[59422]: cluster 2023-09-23T18:37:09.986734+0000 osd.7 (osd.7) 60 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T18:37:11.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:10 smithi173 bash[59422]: cluster 2023-09-23T18:37:10.006357+0000 osd.5 (osd.5) 59 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T18:37:11.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:10 smithi173 bash[59422]: cluster 2023-09-23T18:37:10.008678+0000 osd.5 (osd.5) 60 : cluster 3 failed to encode map e149 with expected crc 2023-09-23T18:37:11.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:10 smithi173 bash[59422]: cluster 2023-09-23T18:37:10.980871+0000 mon.a (mon.0) 797 : cluster 0 osdmap e150: 8 total, 8 up, 7 in 2023-09-23T18:37:11.287 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:11.317 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:11.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:11.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:11.421 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:11.443 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:11.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:11.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:11.546 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:11.568 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:11.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:11.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:11.671 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:11.703 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:11.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:11.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:11.806 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:11.838 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:11.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:11.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:11.941 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:11.965 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:11.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:11.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:12.069 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:12.076 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:11 smithi098 bash[76817]: cluster 2023-09-23T18:37:10.982563+0000 osd.7 (osd.7) 61 : cluster 3 failed to encode map e150 with expected crc 2023-09-23T18:37:12.076 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:11 smithi098 bash[76817]: cluster 2023-09-23T18:37:10.984110+0000 osd.6 (osd.6) 63 : cluster 3 failed to encode map e150 with expected crc 2023-09-23T18:37:12.077 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:11 smithi098 bash[76817]: cluster 2023-09-23T18:37:10.985371+0000 osd.4 (osd.4) 87 : cluster 3 failed to encode map e150 with expected crc 2023-09-23T18:37:12.077 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:11 smithi098 bash[76817]: cluster 2023-09-23T18:37:10.987545+0000 osd.6 (osd.6) 64 : cluster 3 failed to encode map e150 with expected crc 2023-09-23T18:37:12.077 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:11 smithi098 bash[76817]: cluster 2023-09-23T18:37:11.409713+0000 osd.5 (osd.5) 61 : cluster 3 failed to encode map e150 with expected crc 2023-09-23T18:37:12.077 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:11 smithi098 bash[76817]: cluster 2023-09-23T18:37:11.822324+0000 mgr.y (mgr.26054) 560 : cluster 0 pgmap v535: 89 pgs: 4 creating+peering, 6 unknown, 79 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:12.077 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:11 smithi098 bash[79516]: cluster 2023-09-23T18:37:10.982563+0000 osd.7 (osd.7) 61 : cluster 3 failed to encode map e150 with expected crc 2023-09-23T18:37:12.077 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:11 smithi098 bash[79516]: cluster 2023-09-23T18:37:10.984110+0000 osd.6 (osd.6) 63 : cluster 3 failed to encode map e150 with expected crc 2023-09-23T18:37:12.077 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:11 smithi098 bash[79516]: cluster 2023-09-23T18:37:10.985371+0000 osd.4 (osd.4) 87 : cluster 3 failed to encode map e150 with expected crc 2023-09-23T18:37:12.077 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:11 smithi098 bash[79516]: cluster 2023-09-23T18:37:10.987545+0000 osd.6 (osd.6) 64 : cluster 3 failed to encode map e150 with expected crc 2023-09-23T18:37:12.077 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:11 smithi098 bash[79516]: cluster 2023-09-23T18:37:11.409713+0000 osd.5 (osd.5) 61 : cluster 3 failed to encode map e150 with expected crc 2023-09-23T18:37:12.077 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:11 smithi098 bash[79516]: cluster 2023-09-23T18:37:11.822324+0000 mgr.y (mgr.26054) 560 : cluster 0 pgmap v535: 89 pgs: 4 creating+peering, 6 unknown, 79 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:12.092 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:12.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:12.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:12.196 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:12.224 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:12.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:12.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:12.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:11 smithi173 bash[59422]: cluster 2023-09-23T18:37:10.982563+0000 osd.7 (osd.7) 61 : cluster 3 failed to encode map e150 with expected crc 2023-09-23T18:37:12.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:11 smithi173 bash[59422]: cluster 2023-09-23T18:37:10.984110+0000 osd.6 (osd.6) 63 : cluster 3 failed to encode map e150 with expected crc 2023-09-23T18:37:12.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:11 smithi173 bash[59422]: cluster 2023-09-23T18:37:10.985371+0000 osd.4 (osd.4) 87 : cluster 3 failed to encode map e150 with expected crc 2023-09-23T18:37:12.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:11 smithi173 bash[59422]: cluster 2023-09-23T18:37:10.987545+0000 osd.6 (osd.6) 64 : cluster 3 failed to encode map e150 with expected crc 2023-09-23T18:37:12.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:11 smithi173 bash[59422]: cluster 2023-09-23T18:37:11.409713+0000 osd.5 (osd.5) 61 : cluster 3 failed to encode map e150 with expected crc 2023-09-23T18:37:12.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:11 smithi173 bash[59422]: cluster 2023-09-23T18:37:11.822324+0000 mgr.y (mgr.26054) 560 : cluster 0 pgmap v535: 89 pgs: 4 creating+peering, 6 unknown, 79 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:12.327 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:12.361 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:12.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:12.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:12.465 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:12.498 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:12.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:12.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:12.602 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:12.634 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:12.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:12.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:12.738 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:12.772 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:12.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:12.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:12.875 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:12.899 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:12.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:12.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:13.003 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:13.023 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:13.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:13.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:13.126 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:13.154 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:13.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:13.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:13.258 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:13.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:12 smithi173 bash[59422]: cluster 2023-09-23T18:37:11.997922+0000 osd.6 (osd.6) 65 : cluster 3 failed to encode map e151 with expected crc 2023-09-23T18:37:13.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:12 smithi173 bash[59422]: cluster 2023-09-23T18:37:11.998826+0000 osd.7 (osd.7) 62 : cluster 3 failed to encode map e151 with expected crc 2023-09-23T18:37:13.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:12 smithi173 bash[59422]: cluster 2023-09-23T18:37:12.001892+0000 mon.a (mon.0) 798 : cluster 0 osdmap e151: 8 total, 8 up, 7 in 2023-09-23T18:37:13.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:12 smithi173 bash[59422]: cluster 2023-09-23T18:37:12.395676+0000 osd.5 (osd.5) 62 : cluster 3 failed to encode map e151 with expected crc 2023-09-23T18:37:13.286 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:13.287 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:13 smithi098 bash[76817]: cluster 2023-09-23T18:37:11.997922+0000 osd.6 (osd.6) 65 : cluster 3 failed to encode map e151 with expected crc 2023-09-23T18:37:13.288 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:13 smithi098 bash[76817]: cluster 2023-09-23T18:37:11.998826+0000 osd.7 (osd.7) 62 : cluster 3 failed to encode map e151 with expected crc 2023-09-23T18:37:13.288 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:13 smithi098 bash[76817]: cluster 2023-09-23T18:37:12.001892+0000 mon.a (mon.0) 798 : cluster 0 osdmap e151: 8 total, 8 up, 7 in 2023-09-23T18:37:13.288 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:13 smithi098 bash[76817]: cluster 2023-09-23T18:37:12.395676+0000 osd.5 (osd.5) 62 : cluster 3 failed to encode map e151 with expected crc 2023-09-23T18:37:13.288 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:13 smithi098 bash[79516]: cluster 2023-09-23T18:37:11.997922+0000 osd.6 (osd.6) 65 : cluster 3 failed to encode map e151 with expected crc 2023-09-23T18:37:13.288 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:13 smithi098 bash[79516]: cluster 2023-09-23T18:37:11.998826+0000 osd.7 (osd.7) 62 : cluster 3 failed to encode map e151 with expected crc 2023-09-23T18:37:13.288 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:13 smithi098 bash[79516]: cluster 2023-09-23T18:37:12.001892+0000 mon.a (mon.0) 798 : cluster 0 osdmap e151: 8 total, 8 up, 7 in 2023-09-23T18:37:13.288 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:13 smithi098 bash[79516]: cluster 2023-09-23T18:37:12.395676+0000 osd.5 (osd.5) 62 : cluster 3 failed to encode map e151 with expected crc 2023-09-23T18:37:13.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:13.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:13.390 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:13.413 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:13.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:13.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:13.517 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:13.547 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:13.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:13.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:13.651 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:13.683 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:13.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:13.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:13.787 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:13.821 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:13.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:13.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:13.924 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:13.955 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:13.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:13.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:14.060 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:14.081 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:14.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:14.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:14.184 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:14.197 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:14 smithi173 bash[59422]: cluster 2023-09-23T18:37:12.395655+0000 osd.4 (osd.4) 88 : cluster 3 failed to encode map e151 with expected crc 2023-09-23T18:37:14.198 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:14 smithi173 bash[59422]: cluster 2023-09-23T18:37:13.823174+0000 mgr.y (mgr.26054) 561 : cluster 0 pgmap v537: 89 pgs: 4 creating+peering, 2 unknown, 83 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:14.218 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:14.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:14.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:14.322 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:14.354 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:14.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:14.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:14.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:14 smithi098 bash[76817]: cluster 2023-09-23T18:37:12.395655+0000 osd.4 (osd.4) 88 : cluster 3 failed to encode map e151 with expected crc 2023-09-23T18:37:14.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:14 smithi098 bash[76817]: cluster 2023-09-23T18:37:13.823174+0000 mgr.y (mgr.26054) 561 : cluster 0 pgmap v537: 89 pgs: 4 creating+peering, 2 unknown, 83 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:14.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:14 smithi098 bash[79516]: cluster 2023-09-23T18:37:12.395655+0000 osd.4 (osd.4) 88 : cluster 3 failed to encode map e151 with expected crc 2023-09-23T18:37:14.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:14 smithi098 bash[79516]: cluster 2023-09-23T18:37:13.823174+0000 mgr.y (mgr.26054) 561 : cluster 0 pgmap v537: 89 pgs: 4 creating+peering, 2 unknown, 83 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:14.458 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:14.486 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:14.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:14.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:14.590 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:14.626 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:14.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:14.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:14.730 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:14.760 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:14.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:14.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:14.865 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:14.889 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:14.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:14.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:14.933 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":532,"stamp":"2023-09-23T18:37:09.820598+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6245,"num_read_kb":13702,"num_write":1106,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":4194928,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":1675412,"kb_used_data":24860,"kb_used_omap":31,"kb_used_meta":1650464,"kb_avail":654540652,"statfs":{"total":671965249536,"available":670249627648,"internally_reserved":0,"allocated":25456640,"data_stored":20967911,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32507,"internal_metadata":1690075397},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.940901"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":145,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.968315+0000","last_change":"2023-09-23T18:33:53.842027+0000","last_active":"2023-09-23T18:36:48.968315+0000","last_peered":"2023-09-23T18:36:48.968315+0000","last_clean":"2023-09-23T18:36:48.968315+0000","last_became_active":"2023-09-23T18:33:53.841496+0000","last_became_peered":"2023-09-23T18:33:53.841496+0000","last_unstale":"2023-09-23T18:36:48.968315+0000","last_undegraded":"2023-09-23T18:36:48.968315+0000","last_fullsized":"2023-09-23T18:36:48.968315+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":133,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.235839+0000","last_change":"2023-09-23T18:33:53.841529+0000","last_active":"2023-09-23T18:36:49.235839+0000","last_peered":"2023-09-23T18:36:49.235839+0000","last_clean":"2023-09-23T18:36:49.235839+0000","last_became_active":"2023-09-23T18:33:53.841290+0000","last_became_peered":"2023-09-23T18:33:53.841290+0000","last_unstale":"2023-09-23T18:36:49.235839+0000","last_undegraded":"2023-09-23T18:36:49.235839+0000","last_fullsized":"2023-09-23T18:36:49.235839+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:27.552815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":84,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.951455+0000","last_change":"2023-09-23T18:33:53.848655+0000","last_active":"2023-09-23T18:36:48.951455+0000","last_peered":"2023-09-23T18:36:48.951455+0000","last_clean":"2023-09-23T18:36:48.951455+0000","last_became_active":"2023-09-23T18:33:53.848279+0000","last_became_peered":"2023-09-23T18:33:53.848279+0000","last_unstale":"2023-09-23T18:36:48.951455+0000","last_undegraded":"2023-09-23T18:36:48.951455+0000","last_fullsized":"2023-09-23T18:36:48.951455+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:39.223612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":101,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.505885+0000","last_change":"2023-09-23T18:28:03.289500+0000","last_active":"2023-09-23T18:36:49.505885+0000","last_peered":"2023-09-23T18:36:49.505885+0000","last_clean":"2023-09-23T18:36:49.505885+0000","last_became_active":"2023-09-23T18:28:03.289122+0000","last_became_peered":"2023-09-23T18:28:03.289122+0000","last_unstale":"2023-09-23T18:36:49.505885+0000","last_undegraded":"2023-09-23T18:36:49.505885+0000","last_fullsized":"2023-09-23T18:36:49.505885+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:22:26.904660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":149,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.235169+0000","last_change":"2023-09-23T18:33:53.842010+0000","last_active":"2023-09-23T18:36:49.235169+0000","last_peered":"2023-09-23T18:36:49.235169+0000","last_clean":"2023-09-23T18:36:49.235169+0000","last_became_active":"2023-09-23T18:33:53.841790+0000","last_became_peered":"2023-09-23T18:33:53.841790+0000","last_unstale":"2023-09-23T18:36:49.235169+0000","last_undegraded":"2023-09-23T18:36:49.235169+0000","last_fullsized":"2023-09-23T18:36:49.235169+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:46.540440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":139,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.152230+0000","last_change":"2023-09-23T18:33:55.084557+0000","last_active":"2023-09-23T18:36:49.152230+0000","last_peered":"2023-09-23T18:36:49.152230+0000","last_clean":"2023-09-23T18:36:49.152230+0000","last_became_active":"2023-09-23T18:33:55.083987+0000","last_became_peered":"2023-09-23T18:33:55.083987+0000","last_unstale":"2023-09-23T18:36:49.152230+0000","last_undegraded":"2023-09-23T18:36:49.152230+0000","last_fullsized":"2023-09-23T18:36:49.152230+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":108,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.505443+0000","last_change":"2023-09-23T18:32:26.797582+0000","last_active":"2023-09-23T18:36:49.505443+0000","last_peered":"2023-09-23T18:36:49.505443+0000","last_clean":"2023-09-23T18:36:49.505443+0000","last_became_active":"2023-09-23T18:32:26.797107+0000","last_became_peered":"2023-09-23T18:32:26.797107+0000","last_unstale":"2023-09-23T18:36:49.505443+0000","last_undegraded":"2023-09-23T18:36:49.505443+0000","last_fullsized":"2023-09-23T18:36:49.505443+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:56.973116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":133,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.235027+0000","last_change":"2023-09-23T18:33:53.844742+0000","last_active":"2023-09-23T18:36:49.235027+0000","last_peered":"2023-09-23T18:36:49.235027+0000","last_clean":"2023-09-23T18:36:49.235027+0000","last_became_active":"2023-09-23T18:33:53.844425+0000","last_became_peered":"2023-09-23T18:33:53.844425+0000","last_unstale":"2023-09-23T18:36:49.235027+0000","last_undegraded":"2023-09-23T18:36:49.235027+0000","last_fullsized":"2023-09-23T18:36:49.235027+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:42.044108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":128,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.508456+0000","last_change":"2023-09-23T18:33:54.847051+0000","last_active":"2023-09-23T18:36:49.508456+0000","last_peered":"2023-09-23T18:36:49.508456+0000","last_clean":"2023-09-23T18:36:49.508456+0000","last_became_active":"2023-09-23T18:33:54.845357+0000","last_became_peered":"2023-09-23T18:33:54.845357+0000","last_unstale":"2023-09-23T18:36:49.508456+0000","last_undegraded":"2023-09-23T18:36:49.508456+0000","last_fullsized":"2023-09-23T18:36:49.508456+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":108,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.235120+0000","last_change":"2023-09-23T18:28:03.298214+0000","last_active":"2023-09-23T18:36:49.235120+0000","last_peered":"2023-09-23T18:36:49.235120+0000","last_clean":"2023-09-23T18:36:49.235120+0000","last_became_active":"2023-09-23T18:28:03.297852+0000","last_became_peered":"2023-09-23T18:28:03.297852+0000","last_unstale":"2023-09-23T18:36:49.235120+0000","last_undegraded":"2023-09-23T18:36:49.235120+0000","last_fullsized":"2023-09-23T18:36:49.235120+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.146541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":117,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.505617+0000","last_change":"2023-09-23T18:28:03.296913+0000","last_active":"2023-09-23T18:36:49.505617+0000","last_peered":"2023-09-23T18:36:49.505617+0000","last_clean":"2023-09-23T18:36:49.505617+0000","last_became_active":"2023-09-23T18:28:03.296499+0000","last_became_peered":"2023-09-23T18:28:03.296499+0000","last_unstale":"2023-09-23T18:36:49.505617+0000","last_undegraded":"2023-09-23T18:36:49.505617+0000","last_fullsized":"2023-09-23T18:36:49.505617+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":80,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.952063+0000","last_change":"2023-09-23T18:33:53.850096+0000","last_active":"2023-09-23T18:36:48.952063+0000","last_peered":"2023-09-23T18:36:48.952063+0000","last_clean":"2023-09-23T18:36:48.952063+0000","last_became_active":"2023-09-23T18:33:53.849811+0000","last_became_peered":"2023-09-23T18:33:53.849811+0000","last_unstale":"2023-09-23T18:36:48.952063+0000","last_undegraded":"2023-09-23T18:36:48.952063+0000","last_fullsized":"2023-09-23T18:36:48.952063+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:44.938072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":121,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.969044+0000","last_change":"2023-09-23T18:15:08.909339+0000","last_active":"2023-09-23T18:36:48.969044+0000","last_peered":"2023-09-23T18:36:48.969044+0000","last_clean":"2023-09-23T18:36:48.969044+0000","last_became_active":"2023-09-23T18:15:08.908912+0000","last_became_peered":"2023-09-23T18:15:08.908912+0000","last_unstale":"2023-09-23T18:36:48.969044+0000","last_undegraded":"2023-09-23T18:36:48.969044+0000","last_fullsized":"2023-09-23T18:36:48.969044+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":128,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.968172+0000","last_change":"2023-09-23T18:33:53.847489+0000","last_active":"2023-09-23T18:36:48.968172+0000","last_peered":"2023-09-23T18:36:48.968172+0000","last_clean":"2023-09-23T18:36:48.968172+0000","last_became_active":"2023-09-23T18:33:53.846935+0000","last_became_peered":"2023-09-23T18:33:53.846935+0000","last_unstale":"2023-09-23T18:36:48.968172+0000","last_undegraded":"2023-09-23T18:36:48.968172+0000","last_fullsized":"2023-09-23T18:36:48.968172+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":116,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.504819+0000","last_change":"2023-09-23T18:27:33.585121+0000","last_active":"2023-09-23T18:36:49.504819+0000","last_peered":"2023-09-23T18:36:49.504819+0000","last_clean":"2023-09-23T18:36:49.504819+0000","last_became_active":"2023-09-23T18:27:33.584620+0000","last_became_peered":"2023-09-23T18:27:33.584620+0000","last_unstale":"2023-09-23T18:36:49.504819+0000","last_undegraded":"2023-09-23T18:36:49.504819+0000","last_fullsized":"2023-09-23T18:36:49.504819+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":133,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.235583+0000","last_change":"2023-09-23T18:33:53.852933+0000","last_active":"2023-09-23T18:36:49.235583+0000","last_peered":"2023-09-23T18:36:49.235583+0000","last_clean":"2023-09-23T18:36:49.235583+0000","last_became_active":"2023-09-23T18:33:53.852626+0000","last_became_peered":"2023-09-23T18:33:53.852626+0000","last_unstale":"2023-09-23T18:36:49.235583+0000","last_undegraded":"2023-09-23T18:36:49.235583+0000","last_fullsized":"2023-09-23T18:36:49.235583+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:00.174536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":99,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.503987+0000","last_change":"2023-09-23T18:28:03.290316+0000","last_active":"2023-09-23T18:36:49.503987+0000","last_peered":"2023-09-23T18:36:49.503987+0000","last_clean":"2023-09-23T18:36:49.503987+0000","last_became_active":"2023-09-23T18:28:03.289698+0000","last_became_peered":"2023-09-23T18:28:03.289698+0000","last_unstale":"2023-09-23T18:36:49.503987+0000","last_undegraded":"2023-09-23T18:36:49.503987+0000","last_fullsized":"2023-09-23T18:36:49.503987+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:00.779542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":105,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.504981+0000","last_change":"2023-09-23T18:32:26.808237+0000","last_active":"2023-09-23T18:36:49.504981+0000","last_peered":"2023-09-23T18:36:49.504981+0000","last_clean":"2023-09-23T18:36:49.504981+0000","last_became_active":"2023-09-23T18:32:26.807843+0000","last_became_peered":"2023-09-23T18:32:26.807843+0000","last_unstale":"2023-09-23T18:36:49.504981+0000","last_undegraded":"2023-09-23T18:36:49.504981+0000","last_fullsized":"2023-09-23T18:36:49.504981+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":99,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.505346+0000","last_change":"2023-09-23T18:15:08.916658+0000","last_active":"2023-09-23T18:36:49.505346+0000","last_peered":"2023-09-23T18:36:49.505346+0000","last_clean":"2023-09-23T18:36:49.505346+0000","last_became_active":"2023-09-23T18:15:08.916301+0000","last_became_peered":"2023-09-23T18:15:08.916301+0000","last_unstale":"2023-09-23T18:36:49.505346+0000","last_undegraded":"2023-09-23T18:36:49.505346+0000","last_fullsized":"2023-09-23T18:36:49.505346+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":100,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.505725+0000","last_change":"2023-09-23T18:28:03.290217+0000","last_active":"2023-09-23T18:36:49.505725+0000","last_peered":"2023-09-23T18:36:49.505725+0000","last_clean":"2023-09-23T18:36:49.505725+0000","last_became_active":"2023-09-23T18:28:03.289888+0000","last_became_peered":"2023-09-23T18:28:03.289888+0000","last_unstale":"2023-09-23T18:36:49.505725+0000","last_undegraded":"2023-09-23T18:36:49.505725+0000","last_fullsized":"2023-09-23T18:36:49.505725+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:51:45.400847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":141,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.151486+0000","last_change":"2023-09-23T18:32:26.810836+0000","last_active":"2023-09-23T18:36:49.151486+0000","last_peered":"2023-09-23T18:36:49.151486+0000","last_clean":"2023-09-23T18:36:49.151486+0000","last_became_active":"2023-09-23T18:32:26.795378+0000","last_became_peered":"2023-09-23T18:32:26.795378+0000","last_unstale":"2023-09-23T18:36:49.151486+0000","last_undegraded":"2023-09-23T18:36:49.151486+0000","last_fullsized":"2023-09-23T18:36:49.151486+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":133,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.235845+0000","last_change":"2023-09-23T18:33:53.852173+0000","last_active":"2023-09-23T18:36:49.235845+0000","last_peered":"2023-09-23T18:36:49.235845+0000","last_clean":"2023-09-23T18:36:49.235845+0000","last_became_active":"2023-09-23T18:33:53.851814+0000","last_became_peered":"2023-09-23T18:33:53.851814+0000","last_unstale":"2023-09-23T18:36:49.235845+0000","last_undegraded":"2023-09-23T18:36:49.235845+0000","last_fullsized":"2023-09-23T18:36:49.235845+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:15.845259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":115,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.505873+0000","last_change":"2023-09-23T18:26:34.999684+0000","last_active":"2023-09-23T18:36:49.505873+0000","last_peered":"2023-09-23T18:36:49.505873+0000","last_clean":"2023-09-23T18:36:49.505873+0000","last_became_active":"2023-09-23T18:26:34.999076+0000","last_became_peered":"2023-09-23T18:26:34.999076+0000","last_unstale":"2023-09-23T18:36:49.505873+0000","last_undegraded":"2023-09-23T18:36:49.505873+0000","last_fullsized":"2023-09-23T18:36:49.505873+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":46,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.504220+0000","last_change":"2023-09-23T18:32:26.806328+0000","last_active":"2023-09-23T18:36:49.504220+0000","last_peered":"2023-09-23T18:36:49.504220+0000","last_clean":"2023-09-23T18:36:49.504220+0000","last_became_active":"2023-09-23T18:32:26.803969+0000","last_became_peered":"2023-09-23T18:32:26.803969+0000","last_unstale":"2023-09-23T18:36:49.504220+0000","last_undegraded":"2023-09-23T18:36:49.504220+0000","last_fullsized":"2023-09-23T18:36:49.504220+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":131,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.508589+0000","last_change":"2023-09-23T18:33:53.847817+0000","last_active":"2023-09-23T18:36:49.508589+0000","last_peered":"2023-09-23T18:36:49.508589+0000","last_clean":"2023-09-23T18:36:49.508589+0000","last_became_active":"2023-09-23T18:33:53.847342+0000","last_became_peered":"2023-09-23T18:33:53.847342+0000","last_unstale":"2023-09-23T18:36:49.508589+0000","last_undegraded":"2023-09-23T18:36:49.508589+0000","last_fullsized":"2023-09-23T18:36:49.508589+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":143,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.968721+0000","last_change":"2023-09-23T18:33:53.841434+0000","last_active":"2023-09-23T18:36:48.968721+0000","last_peered":"2023-09-23T18:36:48.968721+0000","last_clean":"2023-09-23T18:36:48.968721+0000","last_became_active":"2023-09-23T18:33:53.840923+0000","last_became_peered":"2023-09-23T18:33:53.840923+0000","last_unstale":"2023-09-23T18:36:48.968721+0000","last_undegraded":"2023-09-23T18:36:48.968721+0000","last_fullsized":"2023-09-23T18:36:48.968721+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":113,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.505792+0000","last_change":"2023-09-23T18:32:26.854615+0000","last_active":"2023-09-23T18:36:49.505792+0000","last_peered":"2023-09-23T18:36:49.505792+0000","last_clean":"2023-09-23T18:36:49.505792+0000","last_became_active":"2023-09-23T18:32:26.806771+0000","last_became_peered":"2023-09-23T18:32:26.806771+0000","last_unstale":"2023-09-23T18:36:49.505792+0000","last_undegraded":"2023-09-23T18:36:49.505792+0000","last_fullsized":"2023-09-23T18:36:49.505792+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":122,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.152319+0000","last_change":"2023-09-23T18:27:33.589116+0000","last_active":"2023-09-23T18:36:49.152319+0000","last_peered":"2023-09-23T18:36:49.152319+0000","last_clean":"2023-09-23T18:36:49.152319+0000","last_became_active":"2023-09-23T18:27:33.588809+0000","last_became_peered":"2023-09-23T18:27:33.588809+0000","last_unstale":"2023-09-23T18:36:49.152319+0000","last_undegraded":"2023-09-23T18:36:49.152319+0000","last_fullsized":"2023-09-23T18:36:49.152319+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":119,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:56.514492+0000","last_change":"2023-09-23T18:33:54.848776+0000","last_active":"2023-09-23T18:36:56.514492+0000","last_peered":"2023-09-23T18:36:56.514492+0000","last_clean":"2023-09-23T18:36:56.514492+0000","last_became_active":"2023-09-23T18:33:54.848380+0000","last_became_peered":"2023-09-23T18:33:54.848380+0000","last_unstale":"2023-09-23T18:36:56.514492+0000","last_undegraded":"2023-09-23T18:36:56.514492+0000","last_fullsized":"2023-09-23T18:36:56.514492+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:17:17.093993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":112,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.504085+0000","last_change":"2023-09-23T18:27:33.582033+0000","last_active":"2023-09-23T18:36:49.504085+0000","last_peered":"2023-09-23T18:36:49.504085+0000","last_clean":"2023-09-23T18:36:49.504085+0000","last_became_active":"2023-09-23T18:27:33.581667+0000","last_became_peered":"2023-09-23T18:27:33.581667+0000","last_unstale":"2023-09-23T18:36:49.504085+0000","last_undegraded":"2023-09-23T18:36:49.504085+0000","last_fullsized":"2023-09-23T18:36:49.504085+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:36.039922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":118,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T18:36:59.667744+0000","last_change":"2023-09-23T18:28:03.297628+0000","last_active":"2023-09-23T18:36:59.667744+0000","last_peered":"2023-09-23T18:36:59.667744+0000","last_clean":"2023-09-23T18:36:59.667744+0000","last_became_active":"2023-09-23T18:28:03.296694+0000","last_became_peered":"2023-09-23T18:28:03.296694+0000","last_unstale":"2023-09-23T18:36:59.667744+0000","last_undegraded":"2023-09-23T18:36:59.667744+0000","last_fullsized":"2023-09-23T18:36:59.667744+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:43.375652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":113,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.968428+0000","last_change":"2023-09-23T18:28:03.302326+0000","last_active":"2023-09-23T18:36:48.968428+0000","last_peered":"2023-09-23T18:36:48.968428+0000","last_clean":"2023-09-23T18:36:48.968428+0000","last_became_active":"2023-09-23T18:28:03.301970+0000","last_became_peered":"2023-09-23T18:28:03.301970+0000","last_unstale":"2023-09-23T18:36:48.968428+0000","last_undegraded":"2023-09-23T18:36:48.968428+0000","last_fullsized":"2023-09-23T18:36:48.968428+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":129,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.152846+0000","last_change":"2023-09-23T18:33:53.855371+0000","last_active":"2023-09-23T18:36:49.152846+0000","last_peered":"2023-09-23T18:36:49.152846+0000","last_clean":"2023-09-23T18:36:49.152846+0000","last_became_active":"2023-09-23T18:33:53.854996+0000","last_became_peered":"2023-09-23T18:33:53.854996+0000","last_unstale":"2023-09-23T18:36:49.152846+0000","last_undegraded":"2023-09-23T18:36:49.152846+0000","last_fullsized":"2023-09-23T18:36:49.152846+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":143,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.969733+0000","last_change":"2023-09-23T18:33:53.845253+0000","last_active":"2023-09-23T18:36:48.969733+0000","last_peered":"2023-09-23T18:36:48.969733+0000","last_clean":"2023-09-23T18:36:48.969733+0000","last_became_active":"2023-09-23T18:33:53.844843+0000","last_became_peered":"2023-09-23T18:33:53.844843+0000","last_unstale":"2023-09-23T18:36:48.969733+0000","last_undegraded":"2023-09-23T18:36:48.969733+0000","last_fullsized":"2023-09-23T18:36:48.969733+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":114,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.968594+0000","last_change":"2023-09-23T18:27:33.585714+0000","last_active":"2023-09-23T18:36:48.968594+0000","last_peered":"2023-09-23T18:36:48.968594+0000","last_clean":"2023-09-23T18:36:48.968594+0000","last_became_active":"2023-09-23T18:27:33.584641+0000","last_became_peered":"2023-09-23T18:27:33.584641+0000","last_unstale":"2023-09-23T18:36:48.968594+0000","last_undegraded":"2023-09-23T18:36:48.968594+0000","last_fullsized":"2023-09-23T18:36:48.968594+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":128,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.970249+0000","last_change":"2023-09-23T18:33:53.851266+0000","last_active":"2023-09-23T18:36:48.970249+0000","last_peered":"2023-09-23T18:36:48.970249+0000","last_clean":"2023-09-23T18:36:48.970249+0000","last_became_active":"2023-09-23T18:33:53.850926+0000","last_became_peered":"2023-09-23T18:33:53.850926+0000","last_unstale":"2023-09-23T18:36:48.970249+0000","last_undegraded":"2023-09-23T18:36:48.970249+0000","last_fullsized":"2023-09-23T18:36:48.970249+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":135,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.235357+0000","last_change":"2023-09-23T18:33:53.835731+0000","last_active":"2023-09-23T18:36:49.235357+0000","last_peered":"2023-09-23T18:36:49.235357+0000","last_clean":"2023-09-23T18:36:49.235357+0000","last_became_active":"2023-09-23T18:33:53.835238+0000","last_became_peered":"2023-09-23T18:33:53.835238+0000","last_unstale":"2023-09-23T18:36:49.235357+0000","last_undegraded":"2023-09-23T18:36:49.235357+0000","last_fullsized":"2023-09-23T18:36:49.235357+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:07:18.870358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":84,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.954559+0000","last_change":"2023-09-23T18:33:53.855430+0000","last_active":"2023-09-23T18:36:48.954559+0000","last_peered":"2023-09-23T18:36:48.954559+0000","last_clean":"2023-09-23T18:36:48.954559+0000","last_became_active":"2023-09-23T18:33:53.854935+0000","last_became_peered":"2023-09-23T18:33:53.854935+0000","last_unstale":"2023-09-23T18:36:48.954559+0000","last_undegraded":"2023-09-23T18:36:48.954559+0000","last_fullsized":"2023-09-23T18:36:48.954559+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:36.078411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"77'130","reported_seq":610,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.951845+0000","last_change":"2023-09-23T18:33:53.850713+0000","last_active":"2023-09-23T18:36:48.951845+0000","last_peered":"2023-09-23T18:36:48.951845+0000","last_clean":"2023-09-23T18:36:48.951845+0000","last_became_active":"2023-09-23T18:33:53.850011+0000","last_became_peered":"2023-09-23T18:33:53.850011+0000","last_unstale":"2023-09-23T18:36:48.951845+0000","last_undegraded":"2023-09-23T18:36:48.951845+0000","last_fullsized":"2023-09-23T18:36:48.951845+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:58.943556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":772,"num_read_kb":1712,"num_write":126,"num_write_kb":2126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":123,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.235623+0000","last_change":"2023-09-23T18:32:26.793356+0000","last_active":"2023-09-23T18:36:49.235623+0000","last_peered":"2023-09-23T18:36:49.235623+0000","last_clean":"2023-09-23T18:36:49.235623+0000","last_became_active":"2023-09-23T18:32:26.792868+0000","last_became_peered":"2023-09-23T18:32:26.792868+0000","last_unstale":"2023-09-23T18:36:49.235623+0000","last_undegraded":"2023-09-23T18:36:49.235623+0000","last_fullsized":"2023-09-23T18:36:49.235623+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:40:21.193604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":58,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.952067+0000","last_change":"2023-09-23T18:33:54.843246+0000","last_active":"2023-09-23T18:36:48.952067+0000","last_peered":"2023-09-23T18:36:48.952067+0000","last_clean":"2023-09-23T18:36:48.952067+0000","last_became_active":"2023-09-23T18:33:54.842933+0000","last_became_peered":"2023-09-23T18:33:54.842933+0000","last_unstale":"2023-09-23T18:36:48.952067+0000","last_undegraded":"2023-09-23T18:36:48.952067+0000","last_fullsized":"2023-09-23T18:36:48.952067+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:43.716707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"77'96","reported_seq":554,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.951895+0000","last_change":"2023-09-23T18:33:54.842857+0000","last_active":"2023-09-23T18:36:48.951895+0000","last_peered":"2023-09-23T18:36:48.951895+0000","last_clean":"2023-09-23T18:36:48.951895+0000","last_became_active":"2023-09-23T18:33:54.842134+0000","last_became_peered":"2023-09-23T18:33:54.842134+0000","last_unstale":"2023-09-23T18:36:48.951895+0000","last_undegraded":"2023-09-23T18:36:48.951895+0000","last_fullsized":"2023-09-23T18:36:48.951895+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":96,"log_dups_size":0,"ondisk_log_size":96,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:48.960158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":728,"num_read_kb":1234,"num_write":87,"num_write_kb":338,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":114,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.968267+0000","last_change":"2023-09-23T18:26:34.992535+0000","last_active":"2023-09-23T18:36:48.968267+0000","last_peered":"2023-09-23T18:36:48.968267+0000","last_clean":"2023-09-23T18:36:48.968267+0000","last_became_active":"2023-09-23T18:26:34.992177+0000","last_became_peered":"2023-09-23T18:26:34.992177+0000","last_unstale":"2023-09-23T18:36:48.968267+0000","last_undegraded":"2023-09-23T18:36:48.968267+0000","last_fullsized":"2023-09-23T18:36:48.968267+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":99,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.951731+0000","last_change":"2023-09-23T18:33:53.841776+0000","last_active":"2023-09-23T18:36:48.951731+0000","last_peered":"2023-09-23T18:36:48.951731+0000","last_clean":"2023-09-23T18:36:48.951731+0000","last_became_active":"2023-09-23T18:33:53.841436+0000","last_became_peered":"2023-09-23T18:33:53.841436+0000","last_unstale":"2023-09-23T18:36:48.951731+0000","last_undegraded":"2023-09-23T18:36:48.951731+0000","last_fullsized":"2023-09-23T18:36:48.951731+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:27.198179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'112","reported_seq":586,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.002148+0000","last_change":"2023-09-23T18:28:03.297540+0000","last_active":"2023-09-23T18:36:49.002148+0000","last_peered":"2023-09-23T18:36:49.002148+0000","last_clean":"2023-09-23T18:36:49.002148+0000","last_became_active":"2023-09-23T18:28:03.296649+0000","last_became_peered":"2023-09-23T18:28:03.296649+0000","last_unstale":"2023-09-23T18:36:49.002148+0000","last_undegraded":"2023-09-23T18:36:49.002148+0000","last_fullsized":"2023-09-23T18:36:49.002148+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:32.531775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":774,"num_read_kb":596,"num_write":102,"num_write_kb":52,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":80,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.952831+0000","last_change":"2023-09-23T18:33:53.849084+0000","last_active":"2023-09-23T18:36:48.952831+0000","last_peered":"2023-09-23T18:36:48.952831+0000","last_clean":"2023-09-23T18:36:48.952831+0000","last_became_active":"2023-09-23T18:33:53.848757+0000","last_became_peered":"2023-09-23T18:33:53.848757+0000","last_unstale":"2023-09-23T18:36:48.952831+0000","last_undegraded":"2023-09-23T18:36:48.952831+0000","last_fullsized":"2023-09-23T18:36:48.952831+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:15.267742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":103,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.954648+0000","last_change":"2023-09-23T18:33:55.087304+0000","last_active":"2023-09-23T18:36:48.954648+0000","last_peered":"2023-09-23T18:36:48.954648+0000","last_clean":"2023-09-23T18:36:48.954648+0000","last_became_active":"2023-09-23T18:33:55.086832+0000","last_became_peered":"2023-09-23T18:33:55.086832+0000","last_unstale":"2023-09-23T18:36:48.954648+0000","last_undegraded":"2023-09-23T18:36:48.954648+0000","last_fullsized":"2023-09-23T18:36:48.954648+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:55.457664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'147","reported_seq":535,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.005141+0000","last_change":"2023-09-23T18:32:29.845404+0000","last_active":"2023-09-23T18:36:49.005141+0000","last_peered":"2023-09-23T18:36:49.005141+0000","last_clean":"2023-09-23T18:36:49.005141+0000","last_became_active":"2023-09-23T18:32:29.844545+0000","last_became_peered":"2023-09-23T18:32:29.844545+0000","last_unstale":"2023-09-23T18:36:49.005141+0000","last_undegraded":"2023-09-23T18:36:49.005141+0000","last_fullsized":"2023-09-23T18:36:49.005141+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:44.834482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":624,"num_read_kb":1868,"num_write":143,"num_write_kb":5368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":350,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.504314+0000","last_change":"2023-09-23T18:26:34.999402+0000","last_active":"2023-09-23T18:36:49.504314+0000","last_peered":"2023-09-23T18:36:49.504314+0000","last_clean":"2023-09-23T18:36:49.504314+0000","last_became_active":"2023-09-23T18:26:34.998828+0000","last_became_peered":"2023-09-23T18:26:34.998828+0000","last_unstale":"2023-09-23T18:36:49.504314+0000","last_undegraded":"2023-09-23T18:36:49.504314+0000","last_fullsized":"2023-09-23T18:36:49.504314+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_clean_scrub_stamp":"2023-09-23T18:08:35.701374+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":119,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.235162+0000","last_change":"2023-09-23T18:32:26.800920+0000","last_active":"2023-09-23T18:36:49.235162+0000","last_peered":"2023-09-23T18:36:49.235162+0000","last_clean":"2023-09-23T18:36:49.235162+0000","last_became_active":"2023-09-23T18:32:26.800528+0000","last_became_peered":"2023-09-23T18:32:26.800528+0000","last_unstale":"2023-09-23T18:36:49.235162+0000","last_undegraded":"2023-09-23T18:36:49.235162+0000","last_fullsized":"2023-09-23T18:36:49.235162+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:20:50.939219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":98,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.151637+0000","last_change":"2023-09-23T18:15:08.921629+0000","last_active":"2023-09-23T18:36:49.151637+0000","last_peered":"2023-09-23T18:36:49.151637+0000","last_clean":"2023-09-23T18:36:49.151637+0000","last_became_active":"2023-09-23T18:15:08.921283+0000","last_became_peered":"2023-09-23T18:15:08.921283+0000","last_unstale":"2023-09-23T18:36:49.151637+0000","last_undegraded":"2023-09-23T18:36:49.151637+0000","last_fullsized":"2023-09-23T18:36:49.151637+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"77'120","reported_seq":609,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.504347+0000","last_change":"2023-09-23T18:33:53.852697+0000","last_active":"2023-09-23T18:36:49.504347+0000","last_peered":"2023-09-23T18:36:49.504347+0000","last_clean":"2023-09-23T18:36:49.504347+0000","last_became_active":"2023-09-23T18:33:53.852210+0000","last_became_peered":"2023-09-23T18:33:53.852210+0000","last_unstale":"2023-09-23T18:36:49.504347+0000","last_undegraded":"2023-09-23T18:36:49.504347+0000","last_fullsized":"2023-09-23T18:36:49.504347+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":120,"log_dups_size":0,"ondisk_log_size":120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":809,"num_read_kb":2749,"num_write":98,"num_write_kb":2087,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":100,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T18:36:58.989482+0000","last_change":"2023-09-23T18:28:03.299508+0000","last_active":"2023-09-23T18:36:58.989482+0000","last_peered":"2023-09-23T18:36:58.989482+0000","last_clean":"2023-09-23T18:36:58.989482+0000","last_became_active":"2023-09-23T18:28:03.298682+0000","last_became_peered":"2023-09-23T18:28:03.298682+0000","last_unstale":"2023-09-23T18:36:58.989482+0000","last_undegraded":"2023-09-23T18:36:58.989482+0000","last_fullsized":"2023-09-23T18:36:58.989482+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:08.398302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":130,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.506040+0000","last_change":"2023-09-23T18:33:53.846904+0000","last_active":"2023-09-23T18:36:49.506040+0000","last_peered":"2023-09-23T18:36:49.506040+0000","last_clean":"2023-09-23T18:36:49.506040+0000","last_became_active":"2023-09-23T18:33:53.846654+0000","last_became_peered":"2023-09-23T18:33:53.846654+0000","last_unstale":"2023-09-23T18:36:49.506040+0000","last_undegraded":"2023-09-23T18:36:49.506040+0000","last_fullsized":"2023-09-23T18:36:49.506040+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:55.898016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'78","reported_seq":384,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.505700+0000","last_change":"2023-09-23T18:26:34.992345+0000","last_active":"2023-09-23T18:36:49.505700+0000","last_peered":"2023-09-23T18:36:49.505700+0000","last_clean":"2023-09-23T18:36:49.505700+0000","last_became_active":"2023-09-23T18:26:34.991774+0000","last_became_peered":"2023-09-23T18:26:34.991774+0000","last_unstale":"2023-09-23T18:36:49.505700+0000","last_undegraded":"2023-09-23T18:36:49.505700+0000","last_fullsized":"2023-09-23T18:36:49.505700+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":78,"log_dups_size":0,"ondisk_log_size":78,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":428,"num_read_kb":1125,"num_write":73,"num_write_kb":431,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":78,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.951954+0000","last_change":"2023-09-23T18:33:54.897293+0000","last_active":"2023-09-23T18:36:48.951954+0000","last_peered":"2023-09-23T18:36:48.951954+0000","last_clean":"2023-09-23T18:36:48.951954+0000","last_became_active":"2023-09-23T18:33:54.896491+0000","last_became_peered":"2023-09-23T18:33:54.896491+0000","last_unstale":"2023-09-23T18:36:48.951954+0000","last_undegraded":"2023-09-23T18:36:48.951954+0000","last_fullsized":"2023-09-23T18:36:48.951954+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:03.636617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":106,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T18:36:58.989073+0000","last_change":"2023-09-23T18:33:54.852309+0000","last_active":"2023-09-23T18:36:58.989073+0000","last_peered":"2023-09-23T18:36:58.989073+0000","last_clean":"2023-09-23T18:36:58.989073+0000","last_became_active":"2023-09-23T18:33:54.850255+0000","last_became_peered":"2023-09-23T18:33:54.850255+0000","last_unstale":"2023-09-23T18:36:58.989073+0000","last_undegraded":"2023-09-23T18:36:58.989073+0000","last_fullsized":"2023-09-23T18:36:58.989073+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:54.148992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":118,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.235921+0000","last_change":"2023-09-23T18:28:03.298805+0000","last_active":"2023-09-23T18:36:49.235921+0000","last_peered":"2023-09-23T18:36:49.235921+0000","last_clean":"2023-09-23T18:36:49.235921+0000","last_became_active":"2023-09-23T18:28:03.298437+0000","last_became_peered":"2023-09-23T18:28:03.298437+0000","last_unstale":"2023-09-23T18:36:49.235921+0000","last_undegraded":"2023-09-23T18:36:49.235921+0000","last_fullsized":"2023-09-23T18:36:49.235921+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:27.831669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'181","reported_seq":618,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.954059+0000","last_change":"2023-09-23T18:33:53.854528+0000","last_active":"2023-09-23T18:36:48.954059+0000","last_peered":"2023-09-23T18:36:48.954059+0000","last_clean":"2023-09-23T18:36:48.954059+0000","last_became_active":"2023-09-23T18:33:53.851883+0000","last_became_peered":"2023-09-23T18:33:53.851883+0000","last_unstale":"2023-09-23T18:36:48.954059+0000","last_undegraded":"2023-09-23T18:36:48.954059+0000","last_fullsized":"2023-09-23T18:36:48.954059+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":181,"log_dups_size":0,"ondisk_log_size":181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:43.888844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":909,"num_read_kb":734,"num_write":148,"num_write_kb":75,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":115,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.235270+0000","last_change":"2023-09-23T18:28:03.292295+0000","last_active":"2023-09-23T18:36:49.235270+0000","last_peered":"2023-09-23T18:36:49.235270+0000","last_clean":"2023-09-23T18:36:49.235270+0000","last_became_active":"2023-09-23T18:28:03.291902+0000","last_became_peered":"2023-09-23T18:28:03.291902+0000","last_unstale":"2023-09-23T18:36:49.235270+0000","last_undegraded":"2023-09-23T18:36:49.235270+0000","last_fullsized":"2023-09-23T18:36:49.235270+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:38.415256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"77'112","reported_seq":638,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.152070+0000","last_change":"2023-09-23T18:27:33.587478+0000","last_active":"2023-09-23T18:36:49.152070+0000","last_peered":"2023-09-23T18:36:49.152070+0000","last_clean":"2023-09-23T18:36:49.152070+0000","last_became_active":"2023-09-23T18:27:33.586958+0000","last_became_peered":"2023-09-23T18:27:33.586958+0000","last_unstale":"2023-09-23T18:36:49.152070+0000","last_undegraded":"2023-09-23T18:36:49.152070+0000","last_fullsized":"2023-09-23T18:36:49.152070+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":893,"num_read_kb":3172,"num_write":69,"num_write_kb":2237,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":118,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.505985+0000","last_change":"2023-09-23T18:33:53.851421+0000","last_active":"2023-09-23T18:36:49.505985+0000","last_peered":"2023-09-23T18:36:49.505985+0000","last_clean":"2023-09-23T18:36:49.505985+0000","last_became_active":"2023-09-23T18:33:53.851012+0000","last_became_peered":"2023-09-23T18:33:53.851012+0000","last_unstale":"2023-09-23T18:36:49.505985+0000","last_undegraded":"2023-09-23T18:36:49.505985+0000","last_fullsized":"2023-09-23T18:36:49.505985+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:03.971054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":107,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.968677+0000","last_change":"2023-09-23T18:32:26.805472+0000","last_active":"2023-09-23T18:36:48.968677+0000","last_peered":"2023-09-23T18:36:48.968677+0000","last_clean":"2023-09-23T18:36:48.968677+0000","last_became_active":"2023-09-23T18:32:26.805078+0000","last_became_peered":"2023-09-23T18:32:26.805078+0000","last_unstale":"2023-09-23T18:36:48.968677+0000","last_undegraded":"2023-09-23T18:36:48.968677+0000","last_fullsized":"2023-09-23T18:36:48.968677+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":88,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.504295+0000","last_change":"2023-09-23T18:26:34.994176+0000","last_active":"2023-09-23T18:36:49.504295+0000","last_peered":"2023-09-23T18:36:49.504295+0000","last_clean":"2023-09-23T18:36:49.504295+0000","last_became_active":"2023-09-23T18:26:34.993829+0000","last_became_peered":"2023-09-23T18:26:34.993829+0000","last_unstale":"2023-09-23T18:36:49.504295+0000","last_undegraded":"2023-09-23T18:36:49.504295+0000","last_fullsized":"2023-09-23T18:36:49.504295+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:27.111857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":153,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.003251+0000","last_change":"2023-09-23T18:33:53.840519+0000","last_active":"2023-09-23T18:36:49.003251+0000","last_peered":"2023-09-23T18:36:49.003251+0000","last_clean":"2023-09-23T18:36:49.003251+0000","last_became_active":"2023-09-23T18:33:53.840181+0000","last_became_peered":"2023-09-23T18:33:53.840181+0000","last_unstale":"2023-09-23T18:36:49.003251+0000","last_undegraded":"2023-09-23T18:36:49.003251+0000","last_fullsized":"2023-09-23T18:36:49.003251+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:41.293009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":71,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.968295+0000","last_change":"2023-09-23T18:33:54.847172+0000","last_active":"2023-09-23T18:36:48.968295+0000","last_peered":"2023-09-23T18:36:48.968295+0000","last_clean":"2023-09-23T18:36:48.968295+0000","last_became_active":"2023-09-23T18:33:54.846751+0000","last_became_peered":"2023-09-23T18:33:54.846751+0000","last_unstale":"2023-09-23T18:36:48.968295+0000","last_undegraded":"2023-09-23T18:36:48.968295+0000","last_fullsized":"2023-09-23T18:36:48.968295+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":143,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.968833+0000","last_change":"2023-09-23T18:33:53.841232+0000","last_active":"2023-09-23T18:36:48.968833+0000","last_peered":"2023-09-23T18:36:48.968833+0000","last_clean":"2023-09-23T18:36:48.968833+0000","last_became_active":"2023-09-23T18:33:53.840749+0000","last_became_peered":"2023-09-23T18:33:53.840749+0000","last_unstale":"2023-09-23T18:36:48.968833+0000","last_undegraded":"2023-09-23T18:36:48.968833+0000","last_fullsized":"2023-09-23T18:36:48.968833+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":71,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.969865+0000","last_change":"2023-09-23T18:33:54.858971+0000","last_active":"2023-09-23T18:36:48.969865+0000","last_peered":"2023-09-23T18:36:48.969865+0000","last_clean":"2023-09-23T18:36:48.969865+0000","last_became_active":"2023-09-23T18:33:54.845921+0000","last_became_peered":"2023-09-23T18:33:54.845921+0000","last_unstale":"2023-09-23T18:36:48.969865+0000","last_undegraded":"2023-09-23T18:36:48.969865+0000","last_fullsized":"2023-09-23T18:36:48.969865+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":128,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.968179+0000","last_change":"2023-09-23T18:28:03.302261+0000","last_active":"2023-09-23T18:36:48.968179+0000","last_peered":"2023-09-23T18:36:48.968179+0000","last_clean":"2023-09-23T18:36:48.968179+0000","last_became_active":"2023-09-23T18:28:03.301911+0000","last_became_peered":"2023-09-23T18:28:03.301911+0000","last_unstale":"2023-09-23T18:36:48.968179+0000","last_undegraded":"2023-09-23T18:36:48.968179+0000","last_fullsized":"2023-09-23T18:36:48.968179+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":125,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.236119+0000","last_change":"2023-09-23T18:28:03.296850+0000","last_active":"2023-09-23T18:36:49.236119+0000","last_peered":"2023-09-23T18:36:49.236119+0000","last_clean":"2023-09-23T18:36:49.236119+0000","last_became_active":"2023-09-23T18:28:03.295842+0000","last_became_peered":"2023-09-23T18:28:03.295842+0000","last_unstale":"2023-09-23T18:36:49.236119+0000","last_undegraded":"2023-09-23T18:36:49.236119+0000","last_fullsized":"2023-09-23T18:36:49.236119+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:53.048579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":137,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:49.235027+0000","last_change":"2023-09-23T18:28:03.299520+0000","last_active":"2023-09-23T18:36:49.235027+0000","last_peered":"2023-09-23T18:36:49.235027+0000","last_clean":"2023-09-23T18:36:49.235027+0000","last_became_active":"2023-09-23T18:28:03.299158+0000","last_became_peered":"2023-09-23T18:28:03.299158+0000","last_unstale":"2023-09-23T18:36:49.235027+0000","last_undegraded":"2023-09-23T18:36:49.235027+0000","last_fullsized":"2023-09-23T18:36:49.235027+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:26.247064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":119,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T18:36:58.989174+0000","last_change":"2023-09-23T18:32:26.805676+0000","last_active":"2023-09-23T18:36:58.989174+0000","last_peered":"2023-09-23T18:36:58.989174+0000","last_clean":"2023-09-23T18:36:58.989174+0000","last_became_active":"2023-09-23T18:32:26.797129+0000","last_became_peered":"2023-09-23T18:32:26.797129+0000","last_unstale":"2023-09-23T18:36:58.989174+0000","last_undegraded":"2023-09-23T18:36:58.989174+0000","last_fullsized":"2023-09-23T18:36:58.989174+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:07.414550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":127,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T18:36:48.967749+0000","last_change":"2023-09-23T18:33:54.847156+0000","last_active":"2023-09-23T18:36:48.967749+0000","last_peered":"2023-09-23T18:36:48.967749+0000","last_clean":"2023-09-23T18:36:48.967749+0000","last_became_active":"2023-09-23T18:33:54.846765+0000","last_became_peered":"2023-09-23T18:33:54.846765+0000","last_unstale":"2023-09-23T18:36:48.967749+0000","last_undegraded":"2023-09-23T18:36:48.967749+0000","last_fullsized":"2023-09-23T18:36:48.967749+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":624,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26886,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5937,"num_read_kb":13190,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5621,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430511,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":315640,"kb_used_data":6128,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93429512,"statfs":{"total":95995035648,"available":95671820288,"internally_reserved":0,"allocated":6275072,"data_stored":5717748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:36:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.9750000000000001,"15min":1.407},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.252},"max":{"1min":3.3929999999999998,"5min":57.049999999999997,"15min":57.049999999999997},"last":0.79000000000000004},{"interface":"front","average":{"1min":1.375,"5min":2.0419999999999998,"15min":1.427},"min":{"1min":0.378,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.7880000000000003,"5min":56.773000000000003,"15min":56.773000000000003},"last":0.93300000000000005}]},{"osd":1,"last update":"Sat Sep 23 18:36:59 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.202,"15min":1.147},"min":{"1min":0.32500000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.4420000000000002,"5min":9.2349999999999994,"15min":9.2349999999999994},"last":1.208},{"interface":"front","average":{"1min":0.84999999999999998,"5min":1.3360000000000001,"15min":1.3759999999999999},"min":{"1min":0.44,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":2.8809999999999998,"5min":4.984,"15min":4.984},"last":1.016}]},{"osd":2,"last update":"Sat Sep 23 18:36:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":1.871,"15min":1.2709999999999999},"min":{"1min":0.28899999999999998,"5min":0.254,"15min":0.254},"max":{"1min":4.1500000000000004,"5min":40.091999999999999,"15min":40.091999999999999},"last":1.0649999999999999},{"interface":"front","average":{"1min":1.423,"5min":1.679,"15min":1.2050000000000001},"min":{"1min":0.34100000000000003,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.0170000000000003,"5min":38.261000000000003,"15min":38.261000000000003},"last":0.96899999999999997}]},{"osd":3,"last update":"Sat Sep 23 18:36:14 2023","interfaces":[{"interface":"back","average":{"1min":1.397,"5min":0.98899999999999999,"15min":0.99099999999999999},"min":{"1min":0.39000000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":5.4029999999999996,"5min":5.8920000000000003,"15min":5.8920000000000003},"last":0.88600000000000001},{"interface":"front","average":{"1min":1.02,"5min":1.056,"15min":0.98299999999999998},"min":{"1min":0.35699999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":4.0010000000000003,"5min":4.6929999999999996,"15min":4.6929999999999996},"last":0.83299999999999996}]},{"osd":4,"last update":"Sat Sep 23 18:37:02 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.171,"15min":1.0049999999999999},"min":{"1min":0.378,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.76,"5min":6.3529999999999998,"15min":6.3529999999999998},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.89800000000000002,"5min":1.2629999999999999,"15min":1.123},"min":{"1min":0.40400000000000003,"5min":0.32100000000000001,"15min":0.309},"max":{"1min":1.6220000000000001,"5min":11.119,"15min":11.119},"last":1.097}]},{"osd":5,"last update":"Sat Sep 23 18:37:02 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":1.0620000000000001,"15min":1.0109999999999999},"min":{"1min":0.318,"5min":0.315,"15min":0.315},"max":{"1min":1.274,"5min":5.8899999999999997,"15min":5.8899999999999997},"last":1.165},{"interface":"front","average":{"1min":1.004,"5min":1.1919999999999999,"15min":1.0900000000000001},"min":{"1min":0.41699999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":2.8079999999999998,"5min":7.46,"15min":7.46},"last":1.141}]},{"osd":6,"last update":"Sat Sep 23 18:37:02 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":1.0389999999999999,"15min":1.0680000000000001},"min":{"1min":0.53800000000000003,"5min":0.39400000000000002,"15min":0.35299999999999998},"max":{"1min":1.855,"5min":5.5129999999999999,"15min":5.5129999999999999},"last":1.226},{"interface":"front","average":{"1min":0.96999999999999997,"5min":1.077,"15min":1.121},"min":{"1min":0.435,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.6699999999999999,"5min":9.1850000000000005,"15min":9.1850000000000005},"last":1.2709999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626743,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311536,"kb_used_data":2024,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93433616,"statfs":{"total":95995035648,"available":95676022784,"internally_reserved":0,"allocated":2072576,"data_stored":1523556,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:36:57 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":3.0590000000000002,"15min":1.6279999999999999},"min":{"1min":0.29899999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.9180000000000001,"5min":182.655,"15min":182.655},"last":1.0469999999999999},{"interface":"front","average":{"1min":0.89500000000000002,"5min":3.145,"15min":1.742},"min":{"1min":0.44,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":3.9569999999999999,"5min":182.58799999999999,"15min":182.58799999999999},"last":0.88900000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:36:57 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":0.73899999999999999,"15min":0.75900000000000001},"min":{"1min":0.374,"5min":0.317,"15min":0.317},"max":{"1min":2.246,"5min":3.3340000000000001,"15min":3.3340000000000001},"last":0.79600000000000004},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.97199999999999998,"15min":1.0640000000000001},"min":{"1min":0.32700000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.95799999999999996,"5min":3.8359999999999999,"15min":3.8359999999999999},"last":1.2729999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:36:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":2.02,"15min":1.3340000000000001},"min":{"1min":0.48299999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.3380000000000001,"5min":51.728000000000002,"15min":51.728000000000002},"last":0.92000000000000004},{"interface":"front","average":{"1min":1.321,"5min":2.044,"15min":1.3120000000000001},"min":{"1min":0.443,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.3230000000000004,"5min":51.820999999999998,"15min":51.820999999999998},"last":1.095}]},{"osd":3,"last update":"Sat Sep 23 18:36:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":0.95599999999999996,"15min":0.97699999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.6020000000000003,"5min":21.282,"15min":21.282},"last":0.94999999999999996},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.1220000000000001,"15min":1.0289999999999999},"min":{"1min":0.38100000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":4.0529999999999999,"5min":21.260000000000002,"15min":21.260000000000002},"last":1.1890000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:36:09 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":1.0009999999999999,"15min":0.94499999999999995},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.7229999999999999,"5min":8.0169999999999995,"15min":8.0169999999999995},"last":1.1379999999999999},{"interface":"front","average":{"1min":0.878,"5min":0.879,"15min":0.95199999999999996},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":4.1260000000000003,"5min":4.1260000000000003,"15min":4.1260000000000003},"last":1.1599999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:36:09 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.92900000000000005,"15min":0.98599999999999999},"min":{"1min":0.377,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.83199999999999996,"5min":7.9240000000000004,"15min":7.9240000000000004},"last":1.014},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.80700000000000005,"15min":0.97899999999999998},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.91300000000000003,"5min":4.5270000000000001,"15min":4.5270000000000001},"last":1.248}]},{"osd":7,"last update":"Sat Sep 23 18:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":1.0229999999999999,"15min":1.044},"min":{"1min":0.46100000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.516,"5min":4.6120000000000001,"15min":4.6120000000000001},"last":1.3220000000000001},{"interface":"front","average":{"1min":0.98699999999999999,"5min":1.3340000000000001,"15min":1.208},"min":{"1min":0.42899999999999999,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":2.1360000000000001,"5min":9.1129999999999995,"15min":9.1129999999999995},"last":1.2190000000000001}]}]},{"osd":1,"up_from":123,"seq":528280977448,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":92032,"kb_used_data":1656,"kb_used_omap":7,"kb_used_meta":90360,"kb_avail":93653120,"statfs":{"total":95995035648,"available":95900794880,"internally_reserved":0,"allocated":1695744,"data_stored":958394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":92529422},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:37:01 2023","interfaces":[{"interface":"back","average":{"1min":2.488,"5min":8.2539999999999996,"15min":8.8719999999999999},"min":{"1min":0.28299999999999997,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":23.280000000000001,"5min":173.41800000000001,"15min":173.41800000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":2.8140000000000001,"5min":7.8330000000000002,"15min":8.3160000000000007},"min":{"1min":0.35099999999999998,"5min":0.249,"15min":0.249},"max":{"1min":22.93,"5min":166.42599999999999,"15min":166.42599999999999},"last":0.375}]},{"osd":2,"last update":"Sat Sep 23 18:37:01 2023","interfaces":[{"interface":"back","average":{"1min":2.464,"5min":2.5750000000000002,"15min":2.6930000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":23.074999999999999,"5min":23.074999999999999,"15min":23.074999999999999},"last":0.505},{"interface":"front","average":{"1min":2.984,"5min":2.456,"15min":2.3559999999999999},"min":{"1min":0.311,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":23.129000000000001,"5min":23.129000000000001,"15min":23.129000000000001},"last":0.42599999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:37:01 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":1.0569999999999999,"15min":1.0309999999999999},"min":{"1min":0.32500000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.798,"5min":4.766,"15min":4.766},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.77900000000000003,"5min":1.1140000000000001,"15min":1.1679999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.1680000000000001,"5min":4.4119999999999999,"15min":4.4119999999999999},"last":0.70999999999999996}]},{"osd":4,"last update":"Sat Sep 23 18:37:01 2023","interfaces":[{"interface":"back","average":{"1min":1.361,"5min":1.234,"15min":1.2},"min":{"1min":0.45100000000000001,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":5.5510000000000002,"5min":5.5510000000000002,"15min":5.5510000000000002},"last":0.82399999999999995},{"interface":"front","average":{"1min":0.98499999999999999,"5min":1.3620000000000001,"15min":1.462},"min":{"1min":0.35899999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.4109999999999996,"5min":6.3129999999999997,"15min":6.3129999999999997},"last":0.65200000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:37:01 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":0.97999999999999998,"15min":0.80300000000000005},"min":{"1min":0.38,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":6.1200000000000001,"5min":6.1200000000000001,"15min":6.1200000000000001},"last":0.72599999999999998},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.3149999999999999,"15min":1.3999999999999999},"min":{"1min":0.495,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":4.0259999999999998,"5min":6.5250000000000004,"15min":6.5250000000000004},"last":0.86099999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:37:01 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.4810000000000001,"15min":1.5580000000000001},"min":{"1min":0.51700000000000002,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":4.1029999999999998,"5min":6.4859999999999998,"15min":6.4859999999999998},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.115,"5min":1.5329999999999999,"15min":1.7050000000000001},"min":{"1min":0.39100000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":4.3179999999999996,"5min":6.1109999999999998,"15min":6.1109999999999998},"last":0.80600000000000005}]},{"osd":7,"last update":"Sat Sep 23 18:37:01 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":1.6819999999999999,"15min":1.8640000000000001},"min":{"1min":0.46000000000000002,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":5.3319999999999999,"5min":8.1940000000000008,"15min":8.1940000000000008},"last":1.028},{"interface":"front","average":{"1min":1.401,"5min":1.675,"15min":1.7729999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":5.3029999999999999,"5min":6.3899999999999997,"15min":6.3899999999999997},"last":0.69799999999999995}]}]},{"osd":0,"up_from":88,"seq":377957122177,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":277224,"kb_used_data":2208,"kb_used_omap":8,"kb_used_meta":274999,"kb_avail":93467928,"statfs":{"total":95995035648,"available":95711158272,"internally_reserved":0,"allocated":2260992,"data_stored":1545215,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":281599870},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:37:01 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":1.613,"15min":1.8100000000000001},"min":{"1min":0.27100000000000002,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":4.0030000000000001,"5min":8.1140000000000008,"15min":8.1140000000000008},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.85499999999999998,"5min":2.0339999999999998,"15min":2.3450000000000002},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":5.7699999999999996,"5min":7.6040000000000001,"15min":7.6040000000000001},"last":0.51700000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:36:47 2023","interfaces":[{"interface":"back","average":{"1min":1.621,"5min":2.5529999999999999,"15min":1.381},"min":{"1min":0.34699999999999998,"5min":0.192,"15min":0.192},"max":{"1min":11.698,"5min":40.340000000000003,"15min":40.340000000000003},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.6180000000000001,"5min":2.577,"15min":1.3120000000000001},"min":{"1min":0.26000000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":7.6699999999999999,"5min":42.119999999999997,"15min":42.119999999999997},"last":0.32200000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:36:13 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":2.1419999999999999,"15min":1.2330000000000001},"min":{"1min":0.27600000000000002,"5min":0.216,"15min":0.216},"max":{"1min":7.9349999999999996,"5min":34.942,"15min":34.942},"last":0.47399999999999998},{"interface":"front","average":{"1min":2.0499999999999998,"5min":2.0790000000000002,"15min":1.2629999999999999},"min":{"1min":0.29799999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.6959999999999997,"5min":34.722000000000001,"15min":34.722000000000001},"last":0.36799999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:36:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":1.546,"15min":1.167},"min":{"1min":0.32700000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.6829999999999998,"5min":6.6879999999999997,"15min":6.6879999999999997},"last":0.70199999999999996},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.9450000000000001,"15min":1.331},"min":{"1min":0.34599999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":4.7069999999999999,"5min":16.486999999999998,"15min":16.486999999999998},"last":1.0109999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:36:59 2023","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":1.617,"15min":1.216},"min":{"1min":0.38600000000000001,"5min":0.371,"15min":0.371},"max":{"1min":3.2610000000000001,"5min":8.0589999999999993,"15min":8.0589999999999993},"last":0.93999999999999995},{"interface":"front","average":{"1min":1.369,"5min":1.621,"15min":1.232},"min":{"1min":0.36799999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.8170000000000002,"5min":8.9250000000000007,"15min":8.9250000000000007},"last":0.66300000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:36:59 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":1.706,"15min":1.2809999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":3.3330000000000002,"5min":8.0329999999999995,"15min":8.0329999999999995},"last":0.79500000000000004},{"interface":"front","average":{"1min":1.3879999999999999,"5min":1.7170000000000001,"15min":1.2609999999999999},"min":{"1min":0.45900000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":4.4569999999999999,"5min":8.9740000000000002,"15min":8.9740000000000002},"last":0.82699999999999996}]},{"osd":7,"last update":"Sat Sep 23 18:36:59 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":1.9099999999999999,"15min":1.3260000000000001},"min":{"1min":0.43099999999999999,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":4.4279999999999999,"5min":9.0600000000000005,"15min":9.0600000000000005},"last":0.64100000000000001},{"interface":"front","average":{"1min":1.204,"5min":1.5840000000000001,"15min":1.238},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":5.7009999999999996,"5min":6.7229999999999999,"15min":6.7229999999999999},"last":0.89200000000000002}]}]},{"osd":2,"up_from":98,"seq":420906795125,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":240532,"kb_used_data":5708,"kb_used_omap":11,"kb_used_meta":234804,"kb_avail":93504620,"statfs":{"total":95995035648,"available":95748730880,"internally_reserved":0,"allocated":5844992,"data_stored":5147123,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11949,"internal_metadata":240439635},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:36:46 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.821,"15min":1.0600000000000001},"min":{"1min":0.27100000000000002,"5min":0.192,"15min":0.192},"max":{"1min":3.9100000000000001,"5min":19.431999999999999,"15min":19.431999999999999},"last":0.59099999999999997},{"interface":"front","average":{"1min":1.762,"5min":1.9279999999999999,"15min":1.095},"min":{"1min":0.21099999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":9.423,"5min":9.423,"15min":9.423},"last":0.84599999999999997}]},{"osd":1,"last update":"Sat Sep 23 18:37:00 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":1.871,"15min":2.3410000000000002},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.7080000000000002,"5min":19.361999999999998,"15min":19.361999999999998},"last":0.502},{"interface":"front","average":{"1min":1.417,"5min":2.0790000000000002,"15min":2.3050000000000002},"min":{"1min":0.254,"5min":0.253,"15min":0.253},"max":{"1min":9.468,"5min":9.468,"15min":9.468},"last":0.77000000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:36:23 2023","interfaces":[{"interface":"back","average":{"1min":1.9870000000000001,"5min":1.5289999999999999,"15min":0.99299999999999999},"min":{"1min":0.28199999999999997,"5min":0.218,"15min":0.218},"max":{"1min":5.3949999999999996,"5min":7.7859999999999996,"15min":7.7859999999999996},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.264,"15min":0.90100000000000002},"min":{"1min":0.28199999999999997,"5min":0.249,"15min":0.249},"max":{"1min":4.9390000000000001,"5min":19.407,"15min":19.407},"last":0.54400000000000004}]},{"osd":4,"last update":"Sat Sep 23 18:36:46 2023","interfaces":[{"interface":"back","average":{"1min":1.714,"5min":1.744,"15min":1.21},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.5209999999999999,"5min":7.8220000000000001,"15min":7.8220000000000001},"last":0.79700000000000004},{"interface":"front","average":{"1min":1.5329999999999999,"5min":1.776,"15min":1.258},"min":{"1min":0.34300000000000003,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":11.050000000000001,"5min":17.282,"15min":17.282},"last":0.72599999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:36:46 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":1.407,"15min":1.1379999999999999},"min":{"1min":0.36499999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":2.649,"5min":17.353999999999999,"15min":17.353999999999999},"last":0.96099999999999997},{"interface":"front","average":{"1min":2.0880000000000001,"5min":1.8120000000000001,"15min":1.2669999999999999},"min":{"1min":0.41199999999999998,"5min":0.316,"15min":0.316},"max":{"1min":10.218999999999999,"5min":10.218999999999999,"15min":10.218999999999999},"last":0.90700000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:36:46 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":1.6339999999999999,"15min":1.234},"min":{"1min":0.46500000000000002,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":5.6680000000000001,"5min":7.867,"15min":7.867},"last":0.89300000000000002},{"interface":"front","average":{"1min":1.044,"5min":1.4330000000000001,"15min":1.204},"min":{"1min":0.41699999999999998,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":2.5920000000000001,"5min":17.379999999999999,"15min":17.379999999999999},"last":0.81299999999999994}]},{"osd":7,"last update":"Sat Sep 23 18:36:46 2023","interfaces":[{"interface":"back","average":{"1min":2.016,"5min":1.776,"15min":1.3029999999999999},"min":{"1min":0.45800000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":5.7439999999999998,"5min":7.8460000000000001,"15min":7.8460000000000001},"last":0.74199999999999999},{"interface":"front","average":{"1min":1.022,"5min":1.9910000000000001,"15min":1.4019999999999999},"min":{"1min":0.47099999999999997,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":2.7250000000000001,"5min":17.326000000000001,"15min":17.326000000000001},"last":0.87}]}]},{"osd":3,"up_from":103,"seq":442381631599,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":131584,"kb_used_data":5688,"kb_used_omap":4,"kb_used_meta":125883,"kb_avail":93613568,"statfs":{"total":95995035648,"available":95860293632,"internally_reserved":0,"allocated":5824512,"data_stored":5142869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4826,"internal_metadata":128904486},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:36:15 2023","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":1.5800000000000001,"15min":0.98499999999999999},"min":{"1min":0.217,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":4.5110000000000001,"5min":14.231,"15min":14.231},"last":0.66100000000000003},{"interface":"front","average":{"1min":1.4790000000000001,"5min":1.179,"15min":0.88400000000000001},"min":{"1min":0.28499999999999998,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":5.7969999999999997,"5min":8.4659999999999993,"15min":8.4659999999999993},"last":0.77200000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:37:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":1.018,"15min":1.163},"min":{"1min":0.246,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":1.165,"5min":3.3730000000000002,"15min":3.3730000000000002},"last":0.84199999999999997},{"interface":"front","average":{"1min":0.80800000000000005,"5min":1.173,"15min":1.3129999999999999},"min":{"1min":0.33500000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.0299999999999998,"5min":4.6459999999999999,"15min":4.6459999999999999},"last":0.90500000000000003}]},{"osd":2,"last update":"Sat Sep 23 18:36:15 2023","interfaces":[{"interface":"back","average":{"1min":5.0700000000000003,"5min":4.2370000000000001,"15min":1.9039999999999999},"min":{"1min":0.25700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":41.625,"5min":187.40000000000001,"15min":187.40000000000001},"last":0.46899999999999997},{"interface":"front","average":{"1min":3.7799999999999998,"5min":3.6699999999999999,"15min":1.754},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":41.523000000000003,"5min":187.71700000000001,"15min":187.71700000000001},"last":0.73699999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:36:15 2023","interfaces":[{"interface":"back","average":{"1min":0.81200000000000006,"5min":1.097,"15min":1.0069999999999999},"min":{"1min":0.38500000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.899,"5min":8.3689999999999998,"15min":8.3689999999999998},"last":1},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.0489999999999999,"15min":1.032},"min":{"1min":0.33500000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.6989999999999998,"5min":5.9429999999999996,"15min":5.9429999999999996},"last":0.98699999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:36:15 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":1.0489999999999999,"15min":1.0600000000000001},"min":{"1min":0.40100000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":3.7269999999999999,"5min":5.9930000000000003,"15min":5.9930000000000003},"last":0.81999999999999995},{"interface":"front","average":{"1min":1.127,"5min":1.022,"15min":1.079},"min":{"1min":0.36299999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.774,"5min":4.1479999999999997,"15min":4.1479999999999997},"last":0.86599999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:36:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.042,"15min":1.085},"min":{"1min":0.41099999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":3.738,"5min":5.3070000000000004,"15min":5.3070000000000004},"last":0.96599999999999997},{"interface":"front","average":{"1min":0.81699999999999995,"5min":1.03,"15min":1.0620000000000001},"min":{"1min":0.41699999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.9219999999999999,"5min":8.4139999999999997,"15min":8.4139999999999997},"last":0.80800000000000005}]},{"osd":7,"last update":"Sat Sep 23 18:36:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.216,"15min":1.1479999999999999},"min":{"1min":0.44800000000000001,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":3.3580000000000001,"5min":8.4459999999999997,"15min":8.4459999999999997},"last":0.94799999999999995},{"interface":"front","average":{"1min":1.1399999999999999,"5min":1.1579999999999999,"15min":1.147},"min":{"1min":0.45500000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":3.8279999999999998,"5min":6.0259999999999998,"15min":6.0259999999999998},"last":1.0309999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019209,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306864,"kb_used_data":1448,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93438288,"statfs":{"total":95995035648,"available":95680806912,"internally_reserved":0,"allocated":1482752,"data_stored":933006,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:36:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7030000000000001,"5min":2.734,"15min":1.5209999999999999},"min":{"1min":0.30299999999999999,"5min":0.253,"15min":0.253},"max":{"1min":7.5179999999999998,"5min":82.623000000000005,"15min":82.623000000000005},"last":0.92800000000000005},{"interface":"front","average":{"1min":1.3089999999999999,"5min":2.3450000000000002,"15min":1.464},"min":{"1min":0.35099999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":7.5869999999999997,"5min":82.638000000000005,"15min":82.638000000000005},"last":0.85799999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:36:56 2023","interfaces":[{"interface":"back","average":{"1min":1.171,"5min":0.98099999999999998,"15min":0.85899999999999999},"min":{"1min":0.33800000000000002,"5min":0.31,"15min":0.31},"max":{"1min":4.6589999999999998,"5min":6.0069999999999997,"15min":6.0069999999999997},"last":0.98599999999999999},{"interface":"front","average":{"1min":1.5329999999999999,"5min":1.2410000000000001,"15min":1.1539999999999999},"min":{"1min":0.54200000000000004,"5min":0.313,"15min":0.313},"max":{"1min":4.6779999999999999,"5min":6.1429999999999998,"15min":6.1429999999999998},"last":0.71899999999999997}]},{"osd":2,"last update":"Sat Sep 23 18:36:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4199999999999999,"5min":1.448,"15min":1.1759999999999999},"min":{"1min":0.33200000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.1239999999999997,"5min":12.552,"15min":12.552},"last":1.1240000000000001},{"interface":"front","average":{"1min":1.702,"5min":1.8540000000000001,"15min":1.28},"min":{"1min":0.40200000000000002,"5min":0.309,"15min":0.309},"max":{"1min":5.234,"5min":12.221,"15min":12.221},"last":1.0980000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:36:21 2023","interfaces":[{"interface":"back","average":{"1min":1.456,"5min":1.575,"15min":1.1279999999999999},"min":{"1min":0.36699999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":6.3390000000000004,"5min":20.850000000000001,"15min":20.850000000000001},"last":1.2090000000000001},{"interface":"front","average":{"1min":1.411,"5min":1.728,"15min":1.266},"min":{"1min":0.436,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":6.3959999999999999,"5min":20.792999999999999,"15min":20.792999999999999},"last":1.258}]},{"osd":5,"last update":"Sat Sep 23 18:36:44 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":1.331,"15min":1.111},"min":{"1min":0.40000000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.532,"5min":7.819,"15min":7.819},"last":1.0469999999999999},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.2649999999999999,"15min":1.073},"min":{"1min":0.46700000000000003,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":4.1660000000000004,"5min":6.4779999999999998,"15min":6.4779999999999998},"last":1.155}]},{"osd":6,"last update":"Sat Sep 23 18:36:08 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":1.292,"15min":1.117},"min":{"1min":0.40100000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":6.3680000000000003,"5min":7.8419999999999996,"15min":7.8419999999999996},"last":1.145},{"interface":"front","average":{"1min":1.3879999999999999,"5min":1.091,"15min":1.054},"min":{"1min":0.40000000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.9059999999999997,"5min":5.9059999999999997,"15min":5.9059999999999997},"last":1.1859999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:36:48 2023","interfaces":[{"interface":"back","average":{"1min":1.216,"5min":1.5780000000000001,"15min":1.268},"min":{"1min":0.45000000000000001,"5min":0.311,"15min":0.311},"max":{"1min":5.7279999999999998,"5min":13.491,"15min":13.491},"last":1.242},{"interface":"front","average":{"1min":1.6020000000000001,"5min":1.829,"15min":1.403},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":5.5789999999999997,"5min":17.062999999999999,"15min":17.062999999999999},"last":1.3109999999999999}]}]},{"osd":5,"up_from":0,"seq":154618822977,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4224,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7725,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3429,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:37:14.940 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T18:37:14.940 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:37:14.941 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-09-23T18:37:14.993 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:15.025 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:15.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:15.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:15.129 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:15.157 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:15.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:15.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:15.261 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:15.291 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:15.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:15.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:15.396 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:15.424 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:15.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:15.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:15.528 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:15.551 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:15.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:15.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:15.654 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:15.679 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:15.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:15.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:15.782 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:15.810 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:15.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:15.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:15.914 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:15.942 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:15.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:15.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:16.045 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:16.071 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:16.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:16.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:16.175 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:16.203 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:16.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:16.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:16.306 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:16.313 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:37:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:37:16] "GET /metrics HTTP/1.1" 200 34158 "" "Prometheus/2.33.4" 2023-09-23T18:37:16.329 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:16.330 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:37:16.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:16.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:16.351 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:37:16.372 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:37:16.397 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:37:16.422 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:37:16.433 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:16.448 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:37:16.459 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:16.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:16.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:16.474 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:37:16.500 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:37:16.502 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:37:16.502 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:37:16.502 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:37:16.502 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:37:16.502 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:37:16.502 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:37:16.503 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:37:16.503 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:37:16.563 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:16.594 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:16.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:16.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:16.609 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:37:16.609 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:37:16.697 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:16.728 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:16.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:16.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:16.832 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:16.862 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:16.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:16.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:16.963 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is set 2023-09-23T18:37:16.965 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:16.975 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:16 smithi173 bash[59422]: cluster 2023-09-23T18:37:15.823781+0000 mgr.y (mgr.26054) 562 : cluster 0 pgmap v538: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:16.975 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:16 smithi173 bash[59422]: audit 2023-09-23T18:37:16.302960+0000 mon.a (mon.0) 799 : audit 0 from='client.? 172.21.15.98:0/1800603386' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:37:16.975 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:16 smithi173 bash[59422]: audit 2023-09-23T18:37:16.919522+0000 mon.c (mon.2) 76 : audit 1 from='client.? 172.21.15.98:0/1685067880' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:37:16.975 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:16 smithi173 bash[59422]: audit 2023-09-23T18:37:16.920045+0000 mon.a (mon.0) 800 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:37:16.993 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:16.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:16.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:17.097 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:17.129 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:17.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:17.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:17.152 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:16 smithi098 bash[79516]: cluster 2023-09-23T18:37:15.823781+0000 mgr.y (mgr.26054) 562 : cluster 0 pgmap v538: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:17.152 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:16 smithi098 bash[79516]: audit 2023-09-23T18:37:16.302960+0000 mon.a (mon.0) 799 : audit 0 from='client.? 172.21.15.98:0/1800603386' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:37:17.153 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:16 smithi098 bash[79516]: audit 2023-09-23T18:37:16.919522+0000 mon.c (mon.2) 76 : audit 1 from='client.? 172.21.15.98:0/1685067880' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:37:17.153 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:16 smithi098 bash[79516]: audit 2023-09-23T18:37:16.920045+0000 mon.a (mon.0) 800 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:37:17.153 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:16 smithi098 bash[76817]: cluster 2023-09-23T18:37:15.823781+0000 mgr.y (mgr.26054) 562 : cluster 0 pgmap v538: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:17.153 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:16 smithi098 bash[76817]: audit 2023-09-23T18:37:16.302960+0000 mon.a (mon.0) 799 : audit 0 from='client.? 172.21.15.98:0/1800603386' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:37:17.153 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:16 smithi098 bash[76817]: audit 2023-09-23T18:37:16.919522+0000 mon.c (mon.2) 76 : audit 1 from='client.? 172.21.15.98:0/1685067880' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:37:17.153 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:16 smithi098 bash[76817]: audit 2023-09-23T18:37:16.920045+0000 mon.a (mon.0) 800 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:37:17.233 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:17.259 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:17.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:17.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:17.362 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:17.384 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:17.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:17.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:17.487 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:17.519 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:17.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:17.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:17.623 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:17.653 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:17.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:17.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:17.757 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:17.789 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:17.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:17.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:17.892 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:17.921 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:17.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:17.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:18.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:18.053 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:18.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:18.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:18.156 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:18.188 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:18.190 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:17 smithi173 bash[59422]: cluster 2023-09-23T18:37:16.935980+0000 mon.a (mon.0) 801 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:37:18.191 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:17 smithi173 bash[59422]: cluster 2023-09-23T18:37:16.955016+0000 osd.4 (osd.4) 89 : cluster 3 failed to encode map e152 with expected crc 2023-09-23T18:37:18.191 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:17 smithi173 bash[59422]: audit 2023-09-23T18:37:16.957674+0000 mon.a (mon.0) 802 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:37:18.191 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:17 smithi173 bash[59422]: cluster 2023-09-23T18:37:16.957801+0000 mon.a (mon.0) 803 : cluster 0 osdmap e152: 8 total, 8 up, 7 in 2023-09-23T18:37:18.191 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:17 smithi173 bash[59422]: cluster 2023-09-23T18:37:17.082911+0000 osd.7 (osd.7) 63 : cluster 3 failed to encode map e152 with expected crc 2023-09-23T18:37:18.191 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:17 smithi173 bash[59422]: cluster 2023-09-23T18:37:17.223946+0000 osd.6 (osd.6) 66 : cluster 3 failed to encode map e152 with expected crc 2023-09-23T18:37:18.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:17 smithi098 bash[76817]: cluster 2023-09-23T18:37:16.935980+0000 mon.a (mon.0) 801 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:37:18.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:17 smithi098 bash[76817]: cluster 2023-09-23T18:37:16.955016+0000 osd.4 (osd.4) 89 : cluster 3 failed to encode map e152 with expected crc 2023-09-23T18:37:18.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:17 smithi098 bash[76817]: audit 2023-09-23T18:37:16.957674+0000 mon.a (mon.0) 802 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:37:18.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:17 smithi098 bash[76817]: cluster 2023-09-23T18:37:16.957801+0000 mon.a (mon.0) 803 : cluster 0 osdmap e152: 8 total, 8 up, 7 in 2023-09-23T18:37:18.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:17 smithi098 bash[76817]: cluster 2023-09-23T18:37:17.082911+0000 osd.7 (osd.7) 63 : cluster 3 failed to encode map e152 with expected crc 2023-09-23T18:37:18.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:17 smithi098 bash[76817]: cluster 2023-09-23T18:37:17.223946+0000 osd.6 (osd.6) 66 : cluster 3 failed to encode map e152 with expected crc 2023-09-23T18:37:18.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:17 smithi098 bash[79516]: cluster 2023-09-23T18:37:16.935980+0000 mon.a (mon.0) 801 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:37:18.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:17 smithi098 bash[79516]: cluster 2023-09-23T18:37:16.955016+0000 osd.4 (osd.4) 89 : cluster 3 failed to encode map e152 with expected crc 2023-09-23T18:37:18.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:17 smithi098 bash[79516]: audit 2023-09-23T18:37:16.957674+0000 mon.a (mon.0) 802 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:37:18.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:17 smithi098 bash[79516]: cluster 2023-09-23T18:37:16.957801+0000 mon.a (mon.0) 803 : cluster 0 osdmap e152: 8 total, 8 up, 7 in 2023-09-23T18:37:18.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:17 smithi098 bash[79516]: cluster 2023-09-23T18:37:17.082911+0000 osd.7 (osd.7) 63 : cluster 3 failed to encode map e152 with expected crc 2023-09-23T18:37:18.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:17 smithi098 bash[79516]: cluster 2023-09-23T18:37:17.223946+0000 osd.6 (osd.6) 66 : cluster 3 failed to encode map e152 with expected crc 2023-09-23T18:37:18.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:18.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:18.294 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:18.324 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:18.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:18.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:18.427 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:18.453 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:18.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:18.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:18.556 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:18.584 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:18.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:18.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:18.689 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:18.715 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:18.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:18.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:18.795 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T18:37:18.818 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:18.874 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:18.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:18.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:18.978 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:19.011 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:19.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:19.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:19.115 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:19.144 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:19.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:19.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:19.248 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:19.259 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:18 smithi098 bash[76817]: cluster 2023-09-23T18:37:17.824309+0000 mgr.y (mgr.26054) 563 : cluster 0 pgmap v540: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:19.259 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:18 smithi098 bash[79516]: cluster 2023-09-23T18:37:17.824309+0000 mgr.y (mgr.26054) 563 : cluster 0 pgmap v540: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:19.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:18 smithi173 bash[59422]: cluster 2023-09-23T18:37:17.824309+0000 mgr.y (mgr.26054) 563 : cluster 0 pgmap v540: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:19.281 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:19.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:19.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:19.385 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:19.419 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:19.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:19.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:19.523 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:19.555 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:19.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:19.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:19.660 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:19.692 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:19.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:19.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:19.796 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:19.824 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:19.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:19.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:19.928 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:19.958 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:19.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:19.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:20.062 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:20.094 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:20.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:20.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:20.199 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:20.230 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:20.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:20.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:20.334 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:20.369 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:20.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:20.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:20.472 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:20.505 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:20.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:20.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:20.609 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:20.641 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:20.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:20.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:20.658 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:37:20.746 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:20.777 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:20.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:20.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:20.786 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:37:20.881 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:20.913 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:20.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:20.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:21.017 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:21.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:20 smithi173 bash[59422]: cluster 2023-09-23T18:37:19.825019+0000 mgr.y (mgr.26054) 564 : cluster 0 pgmap v541: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:21.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:20 smithi173 bash[59422]: audit 2023-09-23T18:37:20.026300+0000 mon.a (mon.0) 804 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:21.049 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:21.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:21.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:21.152 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:21.189 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:21.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:21.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:21.292 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:21.304 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:20 smithi098 bash[76817]: cluster 2023-09-23T18:37:19.825019+0000 mgr.y (mgr.26054) 564 : cluster 0 pgmap v541: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:21.304 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:20 smithi098 bash[76817]: audit 2023-09-23T18:37:20.026300+0000 mon.a (mon.0) 804 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:21.304 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:20 smithi098 bash[79516]: cluster 2023-09-23T18:37:19.825019+0000 mgr.y (mgr.26054) 564 : cluster 0 pgmap v541: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:21.304 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:20 smithi098 bash[79516]: audit 2023-09-23T18:37:20.026300+0000 mon.a (mon.0) 804 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:21.311 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:37:21.321 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:21.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:21.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:21.425 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:21.455 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:21.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:21.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:21.558 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:21.589 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:21.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:21.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:21.694 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:21.721 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:21.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:21.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:21.824 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:21.856 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:21.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:21.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:21.961 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:21.987 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:21.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:21.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:22.090 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:22.120 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:22.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:22.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:22.224 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:22.236 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:21 smithi098 bash[76817]: audit 2023-09-23T18:37:21.679590+0000 mon.a (mon.0) 805 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:37:22.236 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:21 smithi098 bash[79516]: audit 2023-09-23T18:37:21.679590+0000 mon.a (mon.0) 805 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:37:22.254 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:22.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:22.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:22.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:21 smithi173 bash[59422]: audit 2023-09-23T18:37:21.679590+0000 mon.a (mon.0) 805 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:37:22.358 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:22.391 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:22.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:22.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:22.495 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:22.527 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:22.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:22.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:22.630 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:22.661 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:22.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:22.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:22.765 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:22.792 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:22.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:22.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:22.896 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:22.927 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:22.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:22.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:23.030 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:23.062 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:23.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:23.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:23.165 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:23.178 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:22 smithi173 bash[59422]: cluster 2023-09-23T18:37:21.825736+0000 mgr.y (mgr.26054) 565 : cluster 0 pgmap v542: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:23.178 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:22 smithi173 bash[59422]: audit 2023-09-23T18:37:22.055421+0000 mon.a (mon.0) 806 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:23.178 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:22 smithi173 bash[59422]: audit 2023-09-23T18:37:22.062137+0000 mon.a (mon.0) 807 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:23.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:22 smithi098 bash[76817]: cluster 2023-09-23T18:37:21.825736+0000 mgr.y (mgr.26054) 565 : cluster 0 pgmap v542: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:23.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:22 smithi098 bash[76817]: audit 2023-09-23T18:37:22.055421+0000 mon.a (mon.0) 806 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:23.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:22 smithi098 bash[76817]: audit 2023-09-23T18:37:22.062137+0000 mon.a (mon.0) 807 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:23.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:22 smithi098 bash[79516]: cluster 2023-09-23T18:37:21.825736+0000 mgr.y (mgr.26054) 565 : cluster 0 pgmap v542: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:23.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:22 smithi098 bash[79516]: audit 2023-09-23T18:37:22.055421+0000 mon.a (mon.0) 806 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:23.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:22 smithi098 bash[79516]: audit 2023-09-23T18:37:22.062137+0000 mon.a (mon.0) 807 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:23.198 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:23.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:23.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:23.302 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:23.338 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:23.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:23.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:23.442 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:23.473 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:23.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:23.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:23.576 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:23.609 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:23.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:23.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:23.712 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:23.748 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:23.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:23.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:23.851 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:23.877 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:23.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:23.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:23.980 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:24.007 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:24.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:24.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:24.066 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:37:24.110 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:24.121 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:23 smithi098 bash[79516]: cluster 2023-09-23T18:37:23.826395+0000 mgr.y (mgr.26054) 566 : cluster 0 pgmap v543: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:24.122 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:23 smithi098 bash[76817]: cluster 2023-09-23T18:37:23.826395+0000 mgr.y (mgr.26054) 566 : cluster 0 pgmap v543: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:24.135 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:24.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:24.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:24.239 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:24.264 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:24.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:24.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:24.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:23 smithi173 bash[59422]: cluster 2023-09-23T18:37:23.826395+0000 mgr.y (mgr.26054) 566 : cluster 0 pgmap v543: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:24.367 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:24.402 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:24.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:24.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:24.505 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:24.531 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:24.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:24.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:24.635 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:24.666 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:24.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:24.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:24.770 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:24.799 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:24.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:24.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:24.902 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:24.928 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:24.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:24.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:25.032 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:25.058 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:25.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:25.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:25.161 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:25.189 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:25.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:25.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:25.293 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:25.329 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:25.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:25.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:25.433 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:25.459 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:25.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:25.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:25.562 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:25.589 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:25.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:25.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:25.692 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:25.703 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:25 smithi098 bash[76817]: audit 2023-09-23T18:37:24.598177+0000 mon.a (mon.0) 808 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:25.703 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:25 smithi098 bash[76817]: audit 2023-09-23T18:37:24.599701+0000 mon.a (mon.0) 809 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:37:25.704 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:25 smithi098 bash[79516]: audit 2023-09-23T18:37:24.598177+0000 mon.a (mon.0) 808 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:25.704 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:25 smithi098 bash[79516]: audit 2023-09-23T18:37:24.599701+0000 mon.a (mon.0) 809 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:37:25.723 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:25.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:25.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:25.829 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:25.857 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:25.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:25.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:25.961 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:25.974 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:25 smithi173 bash[59422]: audit 2023-09-23T18:37:24.598177+0000 mon.a (mon.0) 808 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:25.974 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:25 smithi173 bash[59422]: audit 2023-09-23T18:37:24.599701+0000 mon.a (mon.0) 809 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:37:25.997 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:25.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:26.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:26.100 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:26.132 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:26.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:26.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:26.237 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:26.270 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:26.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:26.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:26.375 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:26.385 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:37:26.386 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:37:26.409 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:26.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:26.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:26.412 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:37:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:37:26] "GET /metrics HTTP/1.1" 200 36446 "" "Prometheus/2.33.4" 2023-09-23T18:37:26.513 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:26.533 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:26.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:26.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:26.614 INFO:teuthology.orchestra.run.smithi098.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-09-23T18:37:26.636 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:26.660 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:26.661 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:26 smithi098 bash[79516]: cluster 2023-09-23T18:37:25.827415+0000 mgr.y (mgr.26054) 567 : cluster 0 pgmap v544: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:26.661 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:26 smithi098 bash[79516]: audit 2023-09-23T18:37:26.334045+0000 mon.a (mon.0) 810 : audit 1 from='client.? 172.21.15.98:0/2554739582' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:37:26.661 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:26 smithi098 bash[76817]: cluster 2023-09-23T18:37:25.827415+0000 mgr.y (mgr.26054) 567 : cluster 0 pgmap v544: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:26.661 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:26 smithi098 bash[76817]: audit 2023-09-23T18:37:26.334045+0000 mon.a (mon.0) 810 : audit 1 from='client.? 172.21.15.98:0/2554739582' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:37:26.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:26.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:26.763 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:26.782 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:26.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:26.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:26.885 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:26.906 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:26.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:26.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:27.009 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:27.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:26 smithi173 bash[59422]: cluster 2023-09-23T18:37:25.827415+0000 mgr.y (mgr.26054) 567 : cluster 0 pgmap v544: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:27.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:26 smithi173 bash[59422]: audit 2023-09-23T18:37:26.334045+0000 mon.a (mon.0) 810 : audit 1 from='client.? 172.21.15.98:0/2554739582' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:37:27.033 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:27.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:27.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:27.136 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:27.181 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:27.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:27.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:27.284 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:27.303 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:27.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:27.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:27.406 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:27.437 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:27.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:27.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:27.540 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:27.569 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:27.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:27.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:27.673 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:27.696 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:27.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:27.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:27.799 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:27.810 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:27 smithi173 bash[59422]: cluster 2023-09-23T18:37:26.606801+0000 osd.4 (osd.4) 90 : cluster 3 failed to encode map e153 with expected crc 2023-09-23T18:37:27.810 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:27 smithi173 bash[59422]: audit 2023-09-23T18:37:26.608912+0000 mon.a (mon.0) 811 : audit 1 from='client.? 172.21.15.98:0/2554739582' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:37:27.811 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:27 smithi173 bash[59422]: cluster 2023-09-23T18:37:26.608965+0000 mon.a (mon.0) 812 : cluster 0 osdmap e153: 8 total, 8 up, 7 in 2023-09-23T18:37:27.811 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:27 smithi173 bash[59422]: audit 2023-09-23T18:37:27.371256+0000 mon.a (mon.0) 813 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:27.811 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:27 smithi173 bash[59422]: audit 2023-09-23T18:37:27.378835+0000 mon.a (mon.0) 814 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:27.826 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:27.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:27.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:27.930 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:27.953 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:27.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:27.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:27.975 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:27 smithi098 bash[76817]: cluster 2023-09-23T18:37:26.606801+0000 osd.4 (osd.4) 90 : cluster 3 failed to encode map e153 with expected crc 2023-09-23T18:37:27.975 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:27 smithi098 bash[76817]: audit 2023-09-23T18:37:26.608912+0000 mon.a (mon.0) 811 : audit 1 from='client.? 172.21.15.98:0/2554739582' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:37:27.975 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:27 smithi098 bash[76817]: cluster 2023-09-23T18:37:26.608965+0000 mon.a (mon.0) 812 : cluster 0 osdmap e153: 8 total, 8 up, 7 in 2023-09-23T18:37:27.975 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:27 smithi098 bash[76817]: audit 2023-09-23T18:37:27.371256+0000 mon.a (mon.0) 813 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:27.975 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:27 smithi098 bash[76817]: audit 2023-09-23T18:37:27.378835+0000 mon.a (mon.0) 814 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:27.975 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:27 smithi098 bash[79516]: cluster 2023-09-23T18:37:26.606801+0000 osd.4 (osd.4) 90 : cluster 3 failed to encode map e153 with expected crc 2023-09-23T18:37:27.975 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:27 smithi098 bash[79516]: audit 2023-09-23T18:37:26.608912+0000 mon.a (mon.0) 811 : audit 1 from='client.? 172.21.15.98:0/2554739582' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:37:27.975 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:27 smithi098 bash[79516]: cluster 2023-09-23T18:37:26.608965+0000 mon.a (mon.0) 812 : cluster 0 osdmap e153: 8 total, 8 up, 7 in 2023-09-23T18:37:27.976 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:27 smithi098 bash[79516]: audit 2023-09-23T18:37:27.371256+0000 mon.a (mon.0) 813 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:27.976 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:27 smithi098 bash[79516]: audit 2023-09-23T18:37:27.378835+0000 mon.a (mon.0) 814 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:28.056 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:28.082 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:28.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:28.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:28.185 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:28.218 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:28.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:28.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:28.323 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:28.356 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:28.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:28.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:28.460 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:28.481 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:28.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:28.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:28.584 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:28.616 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:28.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:28.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:28.720 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:28.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:28 smithi098 bash[79516]: cluster 2023-09-23T18:37:27.827913+0000 mgr.y (mgr.26054) 568 : cluster 0 pgmap v546: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:28.722 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:28 smithi098 bash[79516]: audit 2023-09-23T18:37:28.352986+0000 mon.a (mon.0) 815 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:28.723 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:28 smithi098 bash[79516]: audit 2023-09-23T18:37:28.360466+0000 mon.a (mon.0) 816 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:28.723 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:28 smithi098 bash[76817]: cluster 2023-09-23T18:37:27.827913+0000 mgr.y (mgr.26054) 568 : cluster 0 pgmap v546: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:28.723 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:28 smithi098 bash[76817]: audit 2023-09-23T18:37:28.352986+0000 mon.a (mon.0) 815 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:28.723 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:28 smithi098 bash[76817]: audit 2023-09-23T18:37:28.360466+0000 mon.a (mon.0) 816 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:28.753 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:28.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:28.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:28.856 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:28.868 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:28 smithi173 bash[59422]: cluster 2023-09-23T18:37:27.827913+0000 mgr.y (mgr.26054) 568 : cluster 0 pgmap v546: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:28.868 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:28 smithi173 bash[59422]: audit 2023-09-23T18:37:28.352986+0000 mon.a (mon.0) 815 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:28.868 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:28 smithi173 bash[59422]: audit 2023-09-23T18:37:28.360466+0000 mon.a (mon.0) 816 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:28.883 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:28.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:28.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:28.986 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:29.008 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:29.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:29.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:29.110 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:29.143 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:29.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:29.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:29.246 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:29.276 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:29.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:29.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:29.378 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:29.407 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:29.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:29.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:29.511 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:29.533 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:29.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:29.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:29.636 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:29.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:29 smithi098 bash[79516]: audit 2023-09-23T18:37:28.836550+0000 mon.a (mon.0) 817 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:37:29.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:29 smithi098 bash[79516]: audit 2023-09-23T18:37:28.837292+0000 mon.a (mon.0) 818 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:37:29.644 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:29 smithi098 bash[79516]: audit 2023-09-23T18:37:28.842707+0000 mon.a (mon.0) 819 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:29.645 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:29 smithi098 bash[76817]: audit 2023-09-23T18:37:28.836550+0000 mon.a (mon.0) 817 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:37:29.645 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:29 smithi098 bash[76817]: audit 2023-09-23T18:37:28.837292+0000 mon.a (mon.0) 818 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:37:29.645 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:29 smithi098 bash[76817]: audit 2023-09-23T18:37:28.842707+0000 mon.a (mon.0) 819 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:29.661 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:29.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:29.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:29.683 DEBUG:teuthology.orchestra.run.smithi098:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-09-23T18:37:29.684 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-09-23T18:37:29.684 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:37:29.712 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T18:37:29.712 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T18:37:29.747 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:hints = 1 2023-09-23T18:37:29.748 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T18:37:29.748 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Object prefix: benchmark_data_smithi098_131983 2023-09-23T18:37:29.748 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:37:29.748 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 0 0 0 0 0 0 - 0 2023-09-23T18:37:29.764 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:29.814 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:29.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:29.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:29.917 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:29.965 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:29.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:29.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:30.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:29 smithi173 bash[59422]: audit 2023-09-23T18:37:28.836550+0000 mon.a (mon.0) 817 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:37:30.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:29 smithi173 bash[59422]: audit 2023-09-23T18:37:28.837292+0000 mon.a (mon.0) 818 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:37:30.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:29 smithi173 bash[59422]: audit 2023-09-23T18:37:28.842707+0000 mon.a (mon.0) 819 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:30.071 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:30.100 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:30.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:30.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:30.204 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:30.239 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:30.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:30.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:30.345 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:30.372 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:30.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:30.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:30.476 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:30.505 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:30.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:30.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:30.611 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:30.638 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:30.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:30.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:30.742 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:30.748 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 1 16 2430 2414 150.865 150.875 0.00676127 0.00660069 2023-09-23T18:37:30.774 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:30.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:30.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:30.879 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:30.922 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:30.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:30.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:31.025 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:31.034 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:30 smithi098 bash[76817]: cluster 2023-09-23T18:37:29.829086+0000 mgr.y (mgr.26054) 569 : cluster 0 pgmap v547: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:31.034 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:30 smithi098 bash[79516]: cluster 2023-09-23T18:37:29.829086+0000 mgr.y (mgr.26054) 569 : cluster 0 pgmap v547: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:31.052 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:31.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:31.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:31.157 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:31.187 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:37:31.187 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:37:31.190 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:31.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:31.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:31.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:30 smithi173 bash[59422]: cluster 2023-09-23T18:37:29.829086+0000 mgr.y (mgr.26054) 569 : cluster 0 pgmap v547: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:37:31.295 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:31.326 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:31.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:31.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:31.431 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:31.459 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:31.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:31.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:31.562 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:31.595 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:31.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:31.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:31.698 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:31.723 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:31.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:31.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:31.748 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 2 16 4841 4825 150.769 150.688 0.00293871 0.00661765 2023-09-23T18:37:31.827 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:31.850 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:31.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:31.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:31.954 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:31.983 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:31.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:31.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:32.087 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:32.114 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:32.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:32.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:32.217 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:32.243 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:32.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:32.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:32.347 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:32.372 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:32.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:32.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:32.477 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:32.504 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:32.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:32.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:32.612 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:32.639 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:32.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:32.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:32.742 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:32.748 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 3 16 7573 7557 157.424 170.75 0.00411196 0.00634231 2023-09-23T18:37:32.766 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:32.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:32.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:32.871 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:32.900 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:32.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:32.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:33.005 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:32 smithi173 bash[59422]: cluster 2023-09-23T18:37:31.830003+0000 mgr.y (mgr.26054) 570 : cluster 0 pgmap v548: 89 pgs: 89 active+clean; 129 MiB data, 1.8 GiB used, 624 GiB / 626 GiB avail; 12 MiB/s wr, 199 op/s 2023-09-23T18:37:33.033 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:33.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:33.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:33.139 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:33.148 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:32 smithi098 bash[79516]: cluster 2023-09-23T18:37:31.830003+0000 mgr.y (mgr.26054) 570 : cluster 0 pgmap v548: 89 pgs: 89 active+clean; 129 MiB data, 1.8 GiB used, 624 GiB / 626 GiB avail; 12 MiB/s wr, 199 op/s 2023-09-23T18:37:33.149 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:32 smithi098 bash[76817]: cluster 2023-09-23T18:37:31.830003+0000 mgr.y (mgr.26054) 570 : cluster 0 pgmap v548: 89 pgs: 89 active+clean; 129 MiB data, 1.8 GiB used, 624 GiB / 626 GiB avail; 12 MiB/s wr, 199 op/s 2023-09-23T18:37:33.163 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:33.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:33.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:33.266 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:33.294 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:33.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:33.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:33.397 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:33.422 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:33.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:33.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:33.527 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:33.552 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:33.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:33.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:33.657 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:33.686 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:33.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:33.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:33.748 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 4 16 10118 10102 157.83 159.062 0.00335888 0.00632648 2023-09-23T18:37:33.791 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:33.819 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:33.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:33.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:33.923 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:33.948 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:33.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:33.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:34.054 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:34.079 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:34.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:34.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:34.185 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:34.213 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:34.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:34.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:34.316 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:34.348 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:34.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:34.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:34.454 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:34.476 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:34.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:34.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:34.579 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:34.608 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:34.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:34.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:34.712 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:34.742 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:34.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:34.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:34.748 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 5 16 12502 12486 156.061 149 0.00574771 0.00639898 2023-09-23T18:37:34.846 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:34.877 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:34.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:34.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:34.981 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:35.009 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:35.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:35.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:35.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:34 smithi173 bash[59422]: cluster 2023-09-23T18:37:33.830759+0000 mgr.y (mgr.26054) 571 : cluster 0 pgmap v549: 89 pgs: 89 active+clean; 233 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 23 MiB/s wr, 365 op/s 2023-09-23T18:37:35.113 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:35.140 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:35.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:35.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:35.244 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:35.253 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:34 smithi098 bash[79516]: cluster 2023-09-23T18:37:33.830759+0000 mgr.y (mgr.26054) 571 : cluster 0 pgmap v549: 89 pgs: 89 active+clean; 233 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 23 MiB/s wr, 365 op/s 2023-09-23T18:37:35.253 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:34 smithi098 bash[76817]: cluster 2023-09-23T18:37:33.830759+0000 mgr.y (mgr.26054) 571 : cluster 0 pgmap v549: 89 pgs: 89 active+clean; 233 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 23 MiB/s wr, 365 op/s 2023-09-23T18:37:35.270 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:35.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:35.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:35.373 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:35.402 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:35.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:35.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:35.506 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:35.530 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:35.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:35.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:35.633 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:35.659 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T18:37:35.663 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:35.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:35.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:35.749 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 6 16 14702 14686 152.966 137.5 0.00851671 0.0065303 2023-09-23T18:37:35.770 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:35.802 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:35.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:35.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:35.906 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:35.932 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:35.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:35.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:36.036 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:36.060 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:36.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:36.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:36.164 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:36.175 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:37:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:37:36] "GET /metrics HTTP/1.1" 200 36446 "" "Prometheus/2.33.4" 2023-09-23T18:37:36.198 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:36.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:36.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:36.302 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:36.329 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:36.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:36.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:36.433 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:36.460 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:36.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:36.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:36.563 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:36.587 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:36.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:36.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:36.690 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:36.716 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:36.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:36.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:36.749 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 7 16 16673 16657 148.71 123.188 0.0127248 0.00671514 2023-09-23T18:37:36.820 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:36.847 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:36.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:36.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:36.950 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:36.985 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:36.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:36.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:37.088 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:37.118 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:37.120 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:36 smithi098 bash[79516]: cluster 2023-09-23T18:37:35.831204+0000 mgr.y (mgr.26054) 572 : cluster 0 pgmap v550: 89 pgs: 89 active+clean; 519 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 51 MiB/s wr, 823 op/s 2023-09-23T18:37:37.120 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:36 smithi098 bash[76817]: cluster 2023-09-23T18:37:35.831204+0000 mgr.y (mgr.26054) 572 : cluster 0 pgmap v550: 89 pgs: 89 active+clean; 519 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 51 MiB/s wr, 823 op/s 2023-09-23T18:37:37.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:37.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:37.222 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:37.247 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:37.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:37.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:37.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:36 smithi173 bash[59422]: cluster 2023-09-23T18:37:35.831204+0000 mgr.y (mgr.26054) 572 : cluster 0 pgmap v550: 89 pgs: 89 active+clean; 519 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 51 MiB/s wr, 823 op/s 2023-09-23T18:37:37.350 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:37.379 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:37.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:37.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:37.484 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:37.514 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:37.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:37.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:37.619 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:37.650 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:37.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:37.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:37.749 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 8 16 18832 18816 146.987 134.938 0.00535907 0.0067958 2023-09-23T18:37:37.752 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:37.787 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:37.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:37.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:37.891 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:37.914 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:37.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:37.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:38.018 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:38.047 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:38.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:38.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:38.149 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:38.158 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:37:38.174 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:38.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:38.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:38.277 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:38.304 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:38.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:38.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:38.408 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:38.440 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:38.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:38.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:38.545 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:38.571 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:38.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:38.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:38.674 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:38.699 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:38.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:38.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:38.749 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 9 16 20606 20590 142.973 110.875 0.00454416 0.00698391 2023-09-23T18:37:38.802 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:38.820 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:38.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:38.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:38.923 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:38.953 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:38.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:38.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:39.057 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:39.086 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:39.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:39.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:39.191 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:39.200 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:38 smithi173 bash[59422]: cluster 2023-09-23T18:37:37.831823+0000 mgr.y (mgr.26054) 573 : cluster 0 pgmap v551: 89 pgs: 89 active+clean; 936 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-09-23T18:37:39.218 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:39.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:39.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:39.322 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:39.349 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:39.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:39.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:39.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:39 smithi098 bash[79516]: cluster 2023-09-23T18:37:37.831823+0000 mgr.y (mgr.26054) 573 : cluster 0 pgmap v551: 89 pgs: 89 active+clean; 936 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-09-23T18:37:39.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:38 smithi098 bash[76817]: cluster 2023-09-23T18:37:37.831823+0000 mgr.y (mgr.26054) 573 : cluster 0 pgmap v551: 89 pgs: 89 active+clean; 936 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-09-23T18:37:39.452 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:39.489 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:39.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:39.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:39.593 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:39.622 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:39.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:39.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:39.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:39.749 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 10 16 22271 22255 139.081 104.062 0.00193585 0.00710138 2023-09-23T18:37:39.760 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:39.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:39.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:39.864 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:39.892 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:39.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:39.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:39.997 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:40.025 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:40.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:40.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:40.128 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:40.137 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:39 smithi098 bash[76817]: audit 2023-09-23T18:37:39.542475+0000 mon.a (mon.0) 820 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]: dispatch 2023-09-23T18:37:40.137 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:39 smithi098 bash[76817]: audit 2023-09-23T18:37:39.543241+0000 mon.a (mon.0) 821 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 0]}]: dispatch 2023-09-23T18:37:40.137 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:39 smithi098 bash[76817]: audit 2023-09-23T18:37:39.544025+0000 mon.a (mon.0) 822 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-09-23T18:37:40.137 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:39 smithi098 bash[76817]: audit 2023-09-23T18:37:39.593478+0000 mon.a (mon.0) 823 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:37:40.137 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:39 smithi098 bash[79516]: audit 2023-09-23T18:37:39.542475+0000 mon.a (mon.0) 820 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]: dispatch 2023-09-23T18:37:40.138 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:39 smithi098 bash[79516]: audit 2023-09-23T18:37:39.543241+0000 mon.a (mon.0) 821 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 0]}]: dispatch 2023-09-23T18:37:40.138 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:39 smithi098 bash[79516]: audit 2023-09-23T18:37:39.544025+0000 mon.a (mon.0) 822 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-09-23T18:37:40.138 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:39 smithi098 bash[79516]: audit 2023-09-23T18:37:39.593478+0000 mon.a (mon.0) 823 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:37:40.153 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:40.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:40.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:40.186 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:37:40.256 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:40.266 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:39 smithi173 bash[59422]: audit 2023-09-23T18:37:39.542475+0000 mon.a (mon.0) 820 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]: dispatch 2023-09-23T18:37:40.266 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:39 smithi173 bash[59422]: audit 2023-09-23T18:37:39.543241+0000 mon.a (mon.0) 821 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 0]}]: dispatch 2023-09-23T18:37:40.266 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:39 smithi173 bash[59422]: audit 2023-09-23T18:37:39.544025+0000 mon.a (mon.0) 822 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-09-23T18:37:40.266 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:39 smithi173 bash[59422]: audit 2023-09-23T18:37:39.593478+0000 mon.a (mon.0) 823 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:37:40.284 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:40.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:40.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:40.387 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:40.412 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:40.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:40.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:40.515 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:40.543 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:40.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:40.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:40.648 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:40.676 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:40.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:40.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:40.749 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 11 16 23949 23933 135.971 104.875 0.007432 0.00734651 2023-09-23T18:37:40.781 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:40.808 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:40.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:40.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:40.912 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:40.941 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:40.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:40.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:41.044 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:41.068 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:41.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:41.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:41.172 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:41.195 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:41.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:41.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:41.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:40 smithi173 bash[59422]: cluster 2023-09-23T18:37:39.832399+0000 mgr.y (mgr.26054) 574 : cluster 0 pgmap v552: 89 pgs: 89 active+clean; 1.0 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-09-23T18:37:41.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:40 smithi173 bash[59422]: cluster 2023-09-23T18:37:39.971737+0000 osd.7 (osd.7) 64 : cluster 3 failed to encode map e154 with expected crc 2023-09-23T18:37:41.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:40 smithi173 bash[59422]: audit 2023-09-23T18:37:39.976938+0000 mon.a (mon.0) 824 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]': finished 2023-09-23T18:37:41.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:40 smithi173 bash[59422]: audit 2023-09-23T18:37:39.976995+0000 mon.a (mon.0) 825 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 0]}]': finished 2023-09-23T18:37:41.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:40 smithi173 bash[59422]: audit 2023-09-23T18:37:39.977027+0000 mon.a (mon.0) 826 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2023-09-23T18:37:41.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:40 smithi173 bash[59422]: cluster 2023-09-23T18:37:39.977065+0000 mon.a (mon.0) 827 : cluster 0 osdmap e154: 8 total, 8 up, 7 in 2023-09-23T18:37:41.298 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:41.305 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:40 smithi098 bash[76817]: cluster 2023-09-23T18:37:39.832399+0000 mgr.y (mgr.26054) 574 : cluster 0 pgmap v552: 89 pgs: 89 active+clean; 1.0 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-09-23T18:37:41.305 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:40 smithi098 bash[76817]: cluster 2023-09-23T18:37:39.971737+0000 osd.7 (osd.7) 64 : cluster 3 failed to encode map e154 with expected crc 2023-09-23T18:37:41.305 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:40 smithi098 bash[76817]: audit 2023-09-23T18:37:39.976938+0000 mon.a (mon.0) 824 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]': finished 2023-09-23T18:37:41.305 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:40 smithi098 bash[76817]: audit 2023-09-23T18:37:39.976995+0000 mon.a (mon.0) 825 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 0]}]': finished 2023-09-23T18:37:41.305 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:40 smithi098 bash[76817]: audit 2023-09-23T18:37:39.977027+0000 mon.a (mon.0) 826 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2023-09-23T18:37:41.305 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:40 smithi098 bash[76817]: cluster 2023-09-23T18:37:39.977065+0000 mon.a (mon.0) 827 : cluster 0 osdmap e154: 8 total, 8 up, 7 in 2023-09-23T18:37:41.306 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:40 smithi098 bash[79516]: cluster 2023-09-23T18:37:39.832399+0000 mgr.y (mgr.26054) 574 : cluster 0 pgmap v552: 89 pgs: 89 active+clean; 1.0 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-09-23T18:37:41.306 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:40 smithi098 bash[79516]: cluster 2023-09-23T18:37:39.971737+0000 osd.7 (osd.7) 64 : cluster 3 failed to encode map e154 with expected crc 2023-09-23T18:37:41.306 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:40 smithi098 bash[79516]: audit 2023-09-23T18:37:39.976938+0000 mon.a (mon.0) 824 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]': finished 2023-09-23T18:37:41.306 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:40 smithi098 bash[79516]: audit 2023-09-23T18:37:39.976995+0000 mon.a (mon.0) 825 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 0]}]': finished 2023-09-23T18:37:41.306 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:40 smithi098 bash[79516]: audit 2023-09-23T18:37:39.977027+0000 mon.a (mon.0) 826 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2023-09-23T18:37:41.306 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:40 smithi098 bash[79516]: cluster 2023-09-23T18:37:39.977065+0000 mon.a (mon.0) 827 : cluster 0 osdmap e154: 8 total, 8 up, 7 in 2023-09-23T18:37:41.323 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:41.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:41.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:41.429 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:41.450 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:41.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:41.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:41.553 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:41.579 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:41.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:41.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:41.684 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:41.706 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:41.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:41.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:41.749 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 12 16 25640 25624 133.446 105.688 0.00229429 0.00745539 2023-09-23T18:37:41.809 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:41.826 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:41.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:41.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:41.928 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:41.958 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:41.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:41.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:42.062 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:42.088 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:42.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:42.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:42.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:41 smithi098 bash[76817]: cluster 2023-09-23T18:37:40.969965+0000 osd.5 (osd.5) 63 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T18:37:42.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:41 smithi098 bash[76817]: cluster 2023-09-23T18:37:40.972745+0000 mon.a (mon.0) 828 : cluster 0 osdmap e155: 8 total, 8 up, 7 in 2023-09-23T18:37:42.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:41 smithi098 bash[76817]: cluster 2023-09-23T18:37:40.973502+0000 osd.7 (osd.7) 65 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T18:37:42.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:41 smithi098 bash[76817]: cluster 2023-09-23T18:37:40.973934+0000 osd.6 (osd.6) 67 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T18:37:42.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:41 smithi098 bash[76817]: cluster 2023-09-23T18:37:40.977432+0000 osd.6 (osd.6) 68 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T18:37:42.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:41 smithi098 bash[76817]: cluster 2023-09-23T18:37:40.977753+0000 osd.4 (osd.4) 91 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T18:37:42.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:41 smithi098 bash[76817]: cluster 2023-09-23T18:37:41.832853+0000 mgr.y (mgr.26054) 575 : cluster 0 pgmap v555: 89 pgs: 1 peering, 88 active+clean; 1.4 GiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 148 MiB/s wr, 2.37k op/s 2023-09-23T18:37:42.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:41 smithi098 bash[79516]: cluster 2023-09-23T18:37:40.969965+0000 osd.5 (osd.5) 63 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T18:37:42.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:41 smithi098 bash[79516]: cluster 2023-09-23T18:37:40.972745+0000 mon.a (mon.0) 828 : cluster 0 osdmap e155: 8 total, 8 up, 7 in 2023-09-23T18:37:42.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:41 smithi098 bash[79516]: cluster 2023-09-23T18:37:40.973502+0000 osd.7 (osd.7) 65 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T18:37:42.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:41 smithi098 bash[79516]: cluster 2023-09-23T18:37:40.973934+0000 osd.6 (osd.6) 67 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T18:37:42.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:41 smithi098 bash[79516]: cluster 2023-09-23T18:37:40.977432+0000 osd.6 (osd.6) 68 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T18:37:42.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:41 smithi098 bash[79516]: cluster 2023-09-23T18:37:40.977753+0000 osd.4 (osd.4) 91 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T18:37:42.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:41 smithi098 bash[79516]: cluster 2023-09-23T18:37:41.832853+0000 mgr.y (mgr.26054) 575 : cluster 0 pgmap v555: 89 pgs: 1 peering, 88 active+clean; 1.4 GiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 148 MiB/s wr, 2.37k op/s 2023-09-23T18:37:42.194 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:42.222 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:42.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:42.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:42.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:41 smithi173 bash[59422]: cluster 2023-09-23T18:37:40.969965+0000 osd.5 (osd.5) 63 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T18:37:42.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:41 smithi173 bash[59422]: cluster 2023-09-23T18:37:40.972745+0000 mon.a (mon.0) 828 : cluster 0 osdmap e155: 8 total, 8 up, 7 in 2023-09-23T18:37:42.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:41 smithi173 bash[59422]: cluster 2023-09-23T18:37:40.973502+0000 osd.7 (osd.7) 65 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T18:37:42.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:41 smithi173 bash[59422]: cluster 2023-09-23T18:37:40.973934+0000 osd.6 (osd.6) 67 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T18:37:42.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:41 smithi173 bash[59422]: cluster 2023-09-23T18:37:40.977432+0000 osd.6 (osd.6) 68 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T18:37:42.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:41 smithi173 bash[59422]: cluster 2023-09-23T18:37:40.977753+0000 osd.4 (osd.4) 91 : cluster 3 failed to encode map e155 with expected crc 2023-09-23T18:37:42.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:41 smithi173 bash[59422]: cluster 2023-09-23T18:37:41.832853+0000 mgr.y (mgr.26054) 575 : cluster 0 pgmap v555: 89 pgs: 1 peering, 88 active+clean; 1.4 GiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 148 MiB/s wr, 2.37k op/s 2023-09-23T18:37:42.327 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:42.347 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:42.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:42.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:42.451 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:42.482 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:42.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:42.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:42.586 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:42.609 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:42.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:42.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:42.712 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:42.738 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:42.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:42.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:42.749 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 13 16 26928 26912 129.373 80.5 0.00465355 0.00772373 2023-09-23T18:37:42.842 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:42.867 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:42.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:42.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:42.971 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:42.998 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:42.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:42.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:43.100 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:43.127 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:43.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:43.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:43.233 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:43.267 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:43.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:43.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:43.370 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:43.398 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:43.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:43.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:43.498 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:37:43.501 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:43.527 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:43.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:43.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:43.537 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:37:43.569 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:37:43.632 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:43.646 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:43 smithi098 bash[79516]: audit 2023-09-23T18:37:43.404835+0000 mon.a (mon.0) 829 : audit 0 from='client.? 172.21.15.98:0/4264572619' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:37:43.646 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:43 smithi098 bash[76817]: audit 2023-09-23T18:37:43.404835+0000 mon.a (mon.0) 829 : audit 0 from='client.? 172.21.15.98:0/4264572619' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:37:43.653 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:43.654 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:37:43.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:43.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:43.692 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:37:43.732 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:37:43.749 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 14 16 28382 28366 126.622 90.875 0.0130827 0.00789094 2023-09-23T18:37:43.757 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:43.767 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:37:43.776 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:43 smithi173 bash[59422]: audit 2023-09-23T18:37:43.404835+0000 mon.a (mon.0) 829 : audit 0 from='client.? 172.21.15.98:0/4264572619' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:37:43.782 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:43.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:43.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:43.801 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:37:43.804 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:37:43.804 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:37:43.804 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:37:43.804 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:37:43.804 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:37:43.804 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:37:43.805 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:37:43.805 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:37:43.885 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:43.916 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:43.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:43.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:44.019 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:44.056 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:44.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:44.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:44.164 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:44.189 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:44.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:44.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:44.367 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:44.397 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:44.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:44.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:44.502 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:44.528 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:44.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:44.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:44.633 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:44.668 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:44.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:44.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:44.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:44 smithi098 bash[79516]: cluster 2023-09-23T18:37:43.833388+0000 mgr.y (mgr.26054) 576 : cluster 0 pgmap v556: 89 pgs: 1 peering, 88 active+clean; 1.5 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-09-23T18:37:44.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:44 smithi098 bash[76817]: cluster 2023-09-23T18:37:43.833388+0000 mgr.y (mgr.26054) 576 : cluster 0 pgmap v556: 89 pgs: 1 peering, 88 active+clean; 1.5 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-09-23T18:37:44.750 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 15 16 29989 29973 124.876 100.438 0.00952519 0.00800299 2023-09-23T18:37:44.771 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:44.776 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:44 smithi173 bash[59422]: cluster 2023-09-23T18:37:43.833388+0000 mgr.y (mgr.26054) 576 : cluster 0 pgmap v556: 89 pgs: 1 peering, 88 active+clean; 1.5 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-09-23T18:37:44.796 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:44.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:44.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:44.899 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:44.928 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:44.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:44.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:45.032 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:45.059 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:45.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:45.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:45.164 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:45.187 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:45.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:45.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:45.291 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:45.316 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:45.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:45.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:45.419 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:45.456 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:45.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:45.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:45.493 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is set 2023-09-23T18:37:45.560 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:45.567 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:45 smithi098 bash[79516]: audit 2023-09-23T18:37:44.764506+0000 mon.a (mon.0) 830 : audit 1 from='client.? 172.21.15.98:0/2775179678' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:37:45.568 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:45 smithi098 bash[76817]: audit 2023-09-23T18:37:44.764506+0000 mon.a (mon.0) 830 : audit 1 from='client.? 172.21.15.98:0/2775179678' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:37:45.585 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:45.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:45.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:45.689 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:45.699 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:45 smithi173 bash[59422]: audit 2023-09-23T18:37:44.764506+0000 mon.a (mon.0) 830 : audit 1 from='client.? 172.21.15.98:0/2775179678' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:37:45.718 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:45.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:45.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:45.750 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 16 16 31506 31490 122.996 94.8125 0.0186701 0.00812362 2023-09-23T18:37:45.820 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:45.846 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:45.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:45.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:45.949 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:45.972 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:45.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:45.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:46.075 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:46.100 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:46.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:46.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:46.203 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:46.232 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:46.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:46.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:46.339 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:46.348 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:37:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:37:46] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.33.4" 2023-09-23T18:37:46.371 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:46.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:46.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:46.476 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:46.493 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:46.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:46.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:46.596 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:46.604 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:46 smithi098 bash[76817]: cluster 2023-09-23T18:37:45.467329+0000 mon.a (mon.0) 831 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:37:46.604 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:46 smithi098 bash[76817]: cluster 2023-09-23T18:37:45.476495+0000 osd.7 (osd.7) 66 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T18:37:46.604 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:46 smithi098 bash[76817]: cluster 2023-09-23T18:37:45.478146+0000 osd.4 (osd.4) 92 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T18:37:46.604 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:46 smithi098 bash[76817]: audit 2023-09-23T18:37:45.482534+0000 mon.a (mon.0) 832 : audit 1 from='client.? 172.21.15.98:0/2775179678' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:37:46.604 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:46 smithi098 bash[76817]: cluster 2023-09-23T18:37:45.482697+0000 mon.a (mon.0) 833 : cluster 0 osdmap e156: 8 total, 8 up, 7 in 2023-09-23T18:37:46.604 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:46 smithi098 bash[76817]: cluster 2023-09-23T18:37:45.486260+0000 osd.6 (osd.6) 69 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T18:37:46.604 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:46 smithi098 bash[76817]: cluster 2023-09-23T18:37:45.486351+0000 osd.7 (osd.7) 67 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T18:37:46.605 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:46 smithi098 bash[76817]: cluster 2023-09-23T18:37:45.833928+0000 mgr.y (mgr.26054) 577 : cluster 0 pgmap v558: 89 pgs: 89 active+clean; 1.7 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 111 MiB/s wr, 1.78k op/s; 3 B/s, 0 objects/s recovering 2023-09-23T18:37:46.605 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:46 smithi098 bash[79516]: cluster 2023-09-23T18:37:45.467329+0000 mon.a (mon.0) 831 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:37:46.605 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:46 smithi098 bash[79516]: cluster 2023-09-23T18:37:45.476495+0000 osd.7 (osd.7) 66 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T18:37:46.605 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:46 smithi098 bash[79516]: cluster 2023-09-23T18:37:45.478146+0000 osd.4 (osd.4) 92 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T18:37:46.605 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:46 smithi098 bash[79516]: audit 2023-09-23T18:37:45.482534+0000 mon.a (mon.0) 832 : audit 1 from='client.? 172.21.15.98:0/2775179678' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:37:46.605 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:46 smithi098 bash[79516]: cluster 2023-09-23T18:37:45.482697+0000 mon.a (mon.0) 833 : cluster 0 osdmap e156: 8 total, 8 up, 7 in 2023-09-23T18:37:46.605 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:46 smithi098 bash[79516]: cluster 2023-09-23T18:37:45.486260+0000 osd.6 (osd.6) 69 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T18:37:46.605 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:46 smithi098 bash[79516]: cluster 2023-09-23T18:37:45.486351+0000 osd.7 (osd.7) 67 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T18:37:46.605 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:46 smithi098 bash[79516]: cluster 2023-09-23T18:37:45.833928+0000 mgr.y (mgr.26054) 577 : cluster 0 pgmap v558: 89 pgs: 89 active+clean; 1.7 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 111 MiB/s wr, 1.78k op/s; 3 B/s, 0 objects/s recovering 2023-09-23T18:37:46.620 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:46.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:46.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:46.726 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:46.737 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:46 smithi173 bash[59422]: cluster 2023-09-23T18:37:45.467329+0000 mon.a (mon.0) 831 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:37:46.737 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:46 smithi173 bash[59422]: cluster 2023-09-23T18:37:45.476495+0000 osd.7 (osd.7) 66 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T18:37:46.738 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:46 smithi173 bash[59422]: cluster 2023-09-23T18:37:45.478146+0000 osd.4 (osd.4) 92 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T18:37:46.738 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:46 smithi173 bash[59422]: audit 2023-09-23T18:37:45.482534+0000 mon.a (mon.0) 832 : audit 1 from='client.? 172.21.15.98:0/2775179678' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:37:46.738 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:46 smithi173 bash[59422]: cluster 2023-09-23T18:37:45.482697+0000 mon.a (mon.0) 833 : cluster 0 osdmap e156: 8 total, 8 up, 7 in 2023-09-23T18:37:46.738 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:46 smithi173 bash[59422]: cluster 2023-09-23T18:37:45.486260+0000 osd.6 (osd.6) 69 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T18:37:46.738 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:46 smithi173 bash[59422]: cluster 2023-09-23T18:37:45.486351+0000 osd.7 (osd.7) 67 : cluster 3 failed to encode map e156 with expected crc 2023-09-23T18:37:46.738 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:46 smithi173 bash[59422]: cluster 2023-09-23T18:37:45.833928+0000 mgr.y (mgr.26054) 577 : cluster 0 pgmap v558: 89 pgs: 89 active+clean; 1.7 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 111 MiB/s wr, 1.78k op/s; 3 B/s, 0 objects/s recovering 2023-09-23T18:37:46.750 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 17 16 33113 33097 121.669 100.438 0.0138958 0.00821307 2023-09-23T18:37:46.751 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:46.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:46.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:46.854 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:46.886 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:46.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:46.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:46.990 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:47.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:47.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:47.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:47.119 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:47.151 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:47.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:47.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:47.256 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:47.284 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:47.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:47.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:47.389 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:47.418 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:47.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:47.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:47.523 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:47.549 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:47.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:47.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:47.656 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:47.679 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:47.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:47.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:47.750 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 18 16 34404 34388 119.392 80.6875 0.00703418 0.00837002 2023-09-23T18:37:47.782 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:47.808 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:47.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:47.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:47.913 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:47.942 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:47.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:47.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:48.047 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:48.078 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:48.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:48.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:48.182 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:48.206 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:48.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:48.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:48.310 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:48.335 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:48.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:48.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:48.439 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:48.469 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:48.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:48.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:48.572 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:48.601 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:48.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:48.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:48.705 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:48.740 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:48.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:48.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:48.750 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 19 16 35705 35689 117.387 81.3125 0.00711446 0.00851256 2023-09-23T18:37:48.844 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:48.866 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:48.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:48.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:48.936 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:37:48.948 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:48 smithi098 bash[79516]: cluster 2023-09-23T18:37:47.834555+0000 mgr.y (mgr.26054) 578 : cluster 0 pgmap v559: 89 pgs: 89 active+clean; 1.9 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s; 2 B/s, 0 objects/s recovering 2023-09-23T18:37:48.949 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:48 smithi098 bash[76817]: cluster 2023-09-23T18:37:47.834555+0000 mgr.y (mgr.26054) 578 : cluster 0 pgmap v559: 89 pgs: 89 active+clean; 1.9 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s; 2 B/s, 0 objects/s recovering 2023-09-23T18:37:48.969 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:48.997 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:49.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:49.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:49.101 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:49.128 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:49.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:49.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:49.231 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:49.239 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:48 smithi173 bash[59422]: cluster 2023-09-23T18:37:47.834555+0000 mgr.y (mgr.26054) 578 : cluster 0 pgmap v559: 89 pgs: 89 active+clean; 1.9 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s; 2 B/s, 0 objects/s recovering 2023-09-23T18:37:49.257 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:49.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:49.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:49.361 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:49.378 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:49.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:49.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:49.482 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:49.506 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:49.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:49.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:49.610 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:49.635 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:49.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:49.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:49.741 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:49.750 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:2023-09-23T18:37:49.749931+0000 min lat: 0.00144566 max lat: 0.187 avg lat: 0.00862773 2023-09-23T18:37:49.750 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:37:49.750 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 20 16 37081 37065 115.817 86 0.0192119 0.00862773 2023-09-23T18:37:49.771 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:49.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:49.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:49.877 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:49.901 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:49.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:49.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:50.007 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:50.036 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:50.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:50.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:50.139 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:50.163 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:50.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:50.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:50.267 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:50.297 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:50.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:50.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:50.400 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:50.433 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:50.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:50.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:50.537 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:50.562 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:50.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:50.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:50.669 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:50.694 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:50.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:50.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:50.697 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:37:50.750 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 21 16 38523 38507 114.594 90.125 0.00394253 0.00872053 2023-09-23T18:37:50.797 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:50.824 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:50.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:50.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:50.929 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:50.954 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:50.955 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:50 smithi098 bash[79516]: cluster 2023-09-23T18:37:49.835109+0000 mgr.y (mgr.26054) 579 : cluster 0 pgmap v560: 89 pgs: 89 active+clean; 2.0 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 2 B/s, 0 objects/s recovering 2023-09-23T18:37:50.956 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:50 smithi098 bash[76817]: cluster 2023-09-23T18:37:49.835109+0000 mgr.y (mgr.26054) 579 : cluster 0 pgmap v560: 89 pgs: 89 active+clean; 2.0 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 2 B/s, 0 objects/s recovering 2023-09-23T18:37:50.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:50.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:51.058 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:51.067 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:50 smithi173 bash[59422]: cluster 2023-09-23T18:37:49.835109+0000 mgr.y (mgr.26054) 579 : cluster 0 pgmap v560: 89 pgs: 89 active+clean; 2.0 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 2 B/s, 0 objects/s recovering 2023-09-23T18:37:51.085 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:51.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:51.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:51.190 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:51.216 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:51.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:51.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:51.321 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:51.362 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:51.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:51.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:51.468 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:51.495 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:51.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:51.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:51.600 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:51.634 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:51.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:51.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:51.738 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:51.750 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 22 16 40000 39984 113.58 92.3125 0.0118682 0.00879694 2023-09-23T18:37:51.763 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:51.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:51.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:51.866 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:51.893 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:51.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:51.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:51.999 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:52.024 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:52.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:52.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:52.130 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:52.156 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:52.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:52.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:52.259 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:52.286 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:52.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:52.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:52.390 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:52.417 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:52.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:52.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:52.520 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:52.545 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:52.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:52.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:52.648 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:52.677 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:52.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:52.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:52.750 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 23 16 41367 41351 112.356 85.4375 0.0111323 0.00889438 2023-09-23T18:37:52.782 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:52.810 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:52.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:52.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:52.913 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:52.945 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:52.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:52.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:53.050 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:53.079 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:53.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:53.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:53.183 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:53.209 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:53.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:53.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:53.213 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:52 smithi098 bash[76817]: cluster 2023-09-23T18:37:51.835779+0000 mgr.y (mgr.26054) 580 : cluster 0 pgmap v561: 89 pgs: 89 active+clean; 2.3 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s; 2 B/s, 0 objects/s recovering 2023-09-23T18:37:53.214 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:52 smithi098 bash[79516]: cluster 2023-09-23T18:37:51.835779+0000 mgr.y (mgr.26054) 580 : cluster 0 pgmap v561: 89 pgs: 89 active+clean; 2.3 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s; 2 B/s, 0 objects/s recovering 2023-09-23T18:37:53.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:52 smithi173 bash[59422]: cluster 2023-09-23T18:37:51.835779+0000 mgr.y (mgr.26054) 580 : cluster 0 pgmap v561: 89 pgs: 89 active+clean; 2.3 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s; 2 B/s, 0 objects/s recovering 2023-09-23T18:37:53.314 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:53.341 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:53.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:53.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:53.446 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:53.482 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:53.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:53.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:53.587 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:53.626 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:53.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:53.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:53.729 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:53.750 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 24 16 42731 42715 111.226 85.25 0.0106692 0.00898418 2023-09-23T18:37:53.757 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:53.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:53.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:53.776 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:37:53.861 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:53.893 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:53.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:53.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:53.995 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:54.027 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:54.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:54.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:54.033 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:37:54.033 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [], 2023-09-23T18:37:54.033 INFO:teuthology.orchestra.run.smithi098.stdout: "complaint_time": 30, 2023-09-23T18:37:54.033 INFO:teuthology.orchestra.run.smithi098.stdout: "num_blocked_ops": 0 2023-09-23T18:37:54.033 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:37:54.129 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:54.159 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:54.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:54.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:54.265 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:54.288 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:54.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:54.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:54.392 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:54.416 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:54.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:54.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:54.520 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:54.556 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:54.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:54.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:54.659 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:54.687 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:54.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:54.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:54.751 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 25 16 43786 43770 109.414 65.9375 0.00624525 0.00913116 2023-09-23T18:37:54.790 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:54.820 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:54.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:54.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:54.924 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:54.926 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:37:54.935 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-23T18:37:54.952 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:54.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:54.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:55.056 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:55.085 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:55.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:55.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:55.194 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:55.219 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:55.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:55.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:55.271 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:54 smithi098 bash[79516]: cluster 2023-09-23T18:37:53.837676+0000 mgr.y (mgr.26054) 581 : cluster 0 pgmap v562: 89 pgs: 89 active+clean; 2.4 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 90 MiB/s wr, 1.45k op/s; 2 B/s, 0 objects/s recovering 2023-09-23T18:37:55.271 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:54 smithi098 bash[79516]: audit 2023-09-23T18:37:54.595566+0000 mon.a (mon.0) 834 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:37:55.272 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:54 smithi098 bash[76817]: cluster 2023-09-23T18:37:53.837676+0000 mgr.y (mgr.26054) 581 : cluster 0 pgmap v562: 89 pgs: 89 active+clean; 2.4 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 90 MiB/s wr, 1.45k op/s; 2 B/s, 0 objects/s recovering 2023-09-23T18:37:55.272 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:54 smithi098 bash[76817]: audit 2023-09-23T18:37:54.595566+0000 mon.a (mon.0) 834 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:37:55.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:54 smithi173 bash[59422]: cluster 2023-09-23T18:37:53.837676+0000 mgr.y (mgr.26054) 581 : cluster 0 pgmap v562: 89 pgs: 89 active+clean; 2.4 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 90 MiB/s wr, 1.45k op/s; 2 B/s, 0 objects/s recovering 2023-09-23T18:37:55.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:54 smithi173 bash[59422]: audit 2023-09-23T18:37:54.595566+0000 mon.a (mon.0) 834 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:37:55.322 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:55.356 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:55.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:55.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:55.458 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:55.489 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:55.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:55.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:55.592 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:55.622 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:55.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:55.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:55.726 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:55.749 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:55.751 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 26 16 44903 44887 107.891 69.8125 0.0225099 0.00926308 2023-09-23T18:37:55.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:55.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:55.852 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:55.885 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:55.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:55.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:55.991 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:56.018 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:55 smithi098 bash[76817]: audit 2023-09-23T18:37:54.909062+0000 mgr.y (mgr.26054) 582 : audit 0 from='client.44972 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:37:56.019 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:55 smithi098 bash[76817]: audit 2023-09-23T18:37:55.045190+0000 mon.a (mon.0) 835 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:56.022 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:56.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:56.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:56.125 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:56.149 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:56.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:56.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:56.258 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:56.268 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:56 smithi098 bash[79516]: audit 2023-09-23T18:37:54.909062+0000 mgr.y (mgr.26054) 582 : audit 0 from='client.44972 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:37:56.269 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:56 smithi098 bash[79516]: audit 2023-09-23T18:37:55.045190+0000 mon.a (mon.0) 835 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:56.269 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:37:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:37:56] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.33.4" 2023-09-23T18:37:56.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:55 smithi173 bash[59422]: audit 2023-09-23T18:37:54.909062+0000 mgr.y (mgr.26054) 582 : audit 0 from='client.44972 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:37:56.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:55 smithi173 bash[59422]: audit 2023-09-23T18:37:55.045190+0000 mon.a (mon.0) 835 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:37:56.282 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:56.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:56.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:56.385 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:56.404 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:56.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:56.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:56.509 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:56.533 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:56.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:56.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:56.637 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:56.664 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:56.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:56.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:56.751 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 27 16 46104 46088 106.675 75.0625 0.00886633 0.00936759 2023-09-23T18:37:56.767 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:56.792 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:56.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:56.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:56.899 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:56.926 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:56.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:56.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:57.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:56 smithi173 bash[59422]: cluster 2023-09-23T18:37:55.838765+0000 mgr.y (mgr.26054) 583 : cluster 0 pgmap v563: 89 pgs: 89 active+clean; 2.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s; 2 B/s, 0 objects/s recovering 2023-09-23T18:37:57.031 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:57.058 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:57.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:57.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:57.162 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:57.192 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:57.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:57.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:57.296 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:57.305 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:56 smithi098 bash[76817]: cluster 2023-09-23T18:37:55.838765+0000 mgr.y (mgr.26054) 583 : cluster 0 pgmap v563: 89 pgs: 89 active+clean; 2.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s; 2 B/s, 0 objects/s recovering 2023-09-23T18:37:57.305 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:56 smithi098 bash[79516]: cluster 2023-09-23T18:37:55.838765+0000 mgr.y (mgr.26054) 583 : cluster 0 pgmap v563: 89 pgs: 89 active+clean; 2.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s; 2 B/s, 0 objects/s recovering 2023-09-23T18:37:57.322 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:57.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:57.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:57.426 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:57.455 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:57.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:57.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:57.559 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:57.584 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:57.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:57.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:57.665 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T18:37:57.687 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:57.740 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":562,"stamp":"2023-09-23T18:37:53.837319+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2572949427,"num_objects":39215,"num_object_clones":0,"num_object_copies":117645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39215,"num_whiteouts":0,"num_read":6245,"num_read_kb":13702,"num_write":40293,"num_write_kb":2523127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":4194950,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":40357,"ondisk_log_size":40357,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":9177992,"kb_used_data":7489680,"kb_used_omap":31,"kb_used_meta":1688224,"kb_avail":647038072,"statfs":{"total":671965249536,"available":662566985728,"internally_reserved":0,"allocated":7669432320,"data_stored":7664813465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32507,"internal_metadata":1728741637},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,3,5,8,6,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":948109312,"num_objects":14467,"num_object_clones":0,"num_object_copies":43401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14467,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14467,"num_write_kb":925888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004276"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":12,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:44.478684+0000","last_change":"2023-09-23T18:37:40.992912+0000","last_active":"2023-09-23T18:37:44.478684+0000","last_peered":"2023-09-23T18:37:44.478684+0000","last_clean":"2023-09-23T18:37:44.478684+0000","last_became_active":"2023-09-23T18:37:40.992453+0000","last_became_peered":"2023-09-23T18:37:40.992453+0000","last_unstale":"2023-09-23T18:37:44.478684+0000","last_undegraded":"2023-09-23T18:37:44.478684+0000","last_fullsized":"2023-09-23T18:37:44.478684+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:49.539251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":141,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.976990+0000","last_change":"2023-09-23T18:33:53.841529+0000","last_active":"2023-09-23T18:37:40.976990+0000","last_peered":"2023-09-23T18:37:40.976990+0000","last_clean":"2023-09-23T18:37:40.976990+0000","last_became_active":"2023-09-23T18:33:53.841290+0000","last_became_peered":"2023-09-23T18:33:53.841290+0000","last_unstale":"2023-09-23T18:37:40.976990+0000","last_undegraded":"2023-09-23T18:37:40.976990+0000","last_fullsized":"2023-09-23T18:37:40.976990+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:27.552815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":92,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.977076+0000","last_change":"2023-09-23T18:33:53.848655+0000","last_active":"2023-09-23T18:37:40.977076+0000","last_peered":"2023-09-23T18:37:40.977076+0000","last_clean":"2023-09-23T18:37:40.977076+0000","last_became_active":"2023-09-23T18:33:53.848279+0000","last_became_peered":"2023-09-23T18:33:53.848279+0000","last_unstale":"2023-09-23T18:37:40.977076+0000","last_undegraded":"2023-09-23T18:37:40.977076+0000","last_fullsized":"2023-09-23T18:37:40.977076+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:39.223612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":109,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.977782+0000","last_change":"2023-09-23T18:28:03.289500+0000","last_active":"2023-09-23T18:37:40.977782+0000","last_peered":"2023-09-23T18:37:40.977782+0000","last_clean":"2023-09-23T18:37:40.977782+0000","last_became_active":"2023-09-23T18:28:03.289122+0000","last_became_peered":"2023-09-23T18:28:03.289122+0000","last_unstale":"2023-09-23T18:37:40.977782+0000","last_undegraded":"2023-09-23T18:37:40.977782+0000","last_fullsized":"2023-09-23T18:37:40.977782+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:22:26.904660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":158,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:41.528338+0000","last_change":"2023-09-23T18:33:53.842010+0000","last_active":"2023-09-23T18:37:41.528338+0000","last_peered":"2023-09-23T18:37:41.528338+0000","last_clean":"2023-09-23T18:37:41.528338+0000","last_became_active":"2023-09-23T18:33:53.841790+0000","last_became_peered":"2023-09-23T18:33:53.841790+0000","last_unstale":"2023-09-23T18:37:41.528338+0000","last_undegraded":"2023-09-23T18:37:41.528338+0000","last_fullsized":"2023-09-23T18:37:41.528338+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:46.540440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":147,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.982173+0000","last_change":"2023-09-23T18:33:55.084557+0000","last_active":"2023-09-23T18:37:40.982173+0000","last_peered":"2023-09-23T18:37:40.982173+0000","last_clean":"2023-09-23T18:37:40.982173+0000","last_became_active":"2023-09-23T18:33:55.083987+0000","last_became_peered":"2023-09-23T18:33:55.083987+0000","last_unstale":"2023-09-23T18:37:40.982173+0000","last_undegraded":"2023-09-23T18:37:40.982173+0000","last_fullsized":"2023-09-23T18:37:40.982173+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":116,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.978118+0000","last_change":"2023-09-23T18:32:26.797582+0000","last_active":"2023-09-23T18:37:40.978118+0000","last_peered":"2023-09-23T18:37:40.978118+0000","last_clean":"2023-09-23T18:37:40.978118+0000","last_became_active":"2023-09-23T18:32:26.797107+0000","last_became_peered":"2023-09-23T18:32:26.797107+0000","last_unstale":"2023-09-23T18:37:40.978118+0000","last_undegraded":"2023-09-23T18:37:40.978118+0000","last_fullsized":"2023-09-23T18:37:40.978118+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:56.973116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":151,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.998793+0000","last_change":"2023-09-23T18:37:40.998526+0000","last_active":"2023-09-23T18:37:40.998793+0000","last_peered":"2023-09-23T18:37:40.998793+0000","last_clean":"2023-09-23T18:37:40.998793+0000","last_became_active":"2023-09-23T18:37:40.993211+0000","last_became_peered":"2023-09-23T18:37:40.993211+0000","last_unstale":"2023-09-23T18:37:40.998793+0000","last_undegraded":"2023-09-23T18:37:40.998793+0000","last_fullsized":"2023-09-23T18:37:40.998793+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:40.463496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":136,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.980258+0000","last_change":"2023-09-23T18:33:54.847051+0000","last_active":"2023-09-23T18:37:40.980258+0000","last_peered":"2023-09-23T18:37:40.980258+0000","last_clean":"2023-09-23T18:37:40.980258+0000","last_became_active":"2023-09-23T18:33:54.845357+0000","last_became_peered":"2023-09-23T18:33:54.845357+0000","last_unstale":"2023-09-23T18:37:40.980258+0000","last_undegraded":"2023-09-23T18:37:40.980258+0000","last_fullsized":"2023-09-23T18:37:40.980258+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":117,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.975824+0000","last_change":"2023-09-23T18:28:03.298214+0000","last_active":"2023-09-23T18:37:40.975824+0000","last_peered":"2023-09-23T18:37:40.975824+0000","last_clean":"2023-09-23T18:37:40.975824+0000","last_became_active":"2023-09-23T18:28:03.297852+0000","last_became_peered":"2023-09-23T18:28:03.297852+0000","last_unstale":"2023-09-23T18:37:40.975824+0000","last_undegraded":"2023-09-23T18:37:40.975824+0000","last_fullsized":"2023-09-23T18:37:40.975824+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.146541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":125,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.980394+0000","last_change":"2023-09-23T18:28:03.296913+0000","last_active":"2023-09-23T18:37:40.980394+0000","last_peered":"2023-09-23T18:37:40.980394+0000","last_clean":"2023-09-23T18:37:40.980394+0000","last_became_active":"2023-09-23T18:28:03.296499+0000","last_became_peered":"2023-09-23T18:28:03.296499+0000","last_unstale":"2023-09-23T18:37:40.980394+0000","last_undegraded":"2023-09-23T18:37:40.980394+0000","last_fullsized":"2023-09-23T18:37:40.980394+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":88,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.973710+0000","last_change":"2023-09-23T18:33:53.850096+0000","last_active":"2023-09-23T18:37:40.973710+0000","last_peered":"2023-09-23T18:37:40.973710+0000","last_clean":"2023-09-23T18:37:40.973710+0000","last_became_active":"2023-09-23T18:33:53.849811+0000","last_became_peered":"2023-09-23T18:33:53.849811+0000","last_unstale":"2023-09-23T18:37:40.973710+0000","last_undegraded":"2023-09-23T18:37:40.973710+0000","last_fullsized":"2023-09-23T18:37:40.973710+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:44.938072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":129,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.984713+0000","last_change":"2023-09-23T18:15:08.909339+0000","last_active":"2023-09-23T18:37:40.984713+0000","last_peered":"2023-09-23T18:37:40.984713+0000","last_clean":"2023-09-23T18:37:40.984713+0000","last_became_active":"2023-09-23T18:15:08.908912+0000","last_became_peered":"2023-09-23T18:15:08.908912+0000","last_unstale":"2023-09-23T18:37:40.984713+0000","last_undegraded":"2023-09-23T18:37:40.984713+0000","last_fullsized":"2023-09-23T18:37:40.984713+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":136,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.985729+0000","last_change":"2023-09-23T18:33:53.847489+0000","last_active":"2023-09-23T18:37:40.985729+0000","last_peered":"2023-09-23T18:37:40.985729+0000","last_clean":"2023-09-23T18:37:40.985729+0000","last_became_active":"2023-09-23T18:33:53.846935+0000","last_became_peered":"2023-09-23T18:33:53.846935+0000","last_unstale":"2023-09-23T18:37:40.985729+0000","last_undegraded":"2023-09-23T18:37:40.985729+0000","last_fullsized":"2023-09-23T18:37:40.985729+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":124,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.982594+0000","last_change":"2023-09-23T18:27:33.585121+0000","last_active":"2023-09-23T18:37:40.982594+0000","last_peered":"2023-09-23T18:37:40.982594+0000","last_clean":"2023-09-23T18:37:40.982594+0000","last_became_active":"2023-09-23T18:27:33.584620+0000","last_became_peered":"2023-09-23T18:27:33.584620+0000","last_unstale":"2023-09-23T18:37:40.982594+0000","last_undegraded":"2023-09-23T18:37:40.982594+0000","last_fullsized":"2023-09-23T18:37:40.982594+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":142,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.979163+0000","last_change":"2023-09-23T18:33:53.852933+0000","last_active":"2023-09-23T18:37:40.979163+0000","last_peered":"2023-09-23T18:37:40.979163+0000","last_clean":"2023-09-23T18:37:40.979163+0000","last_became_active":"2023-09-23T18:33:53.852626+0000","last_became_peered":"2023-09-23T18:33:53.852626+0000","last_unstale":"2023-09-23T18:37:40.979163+0000","last_undegraded":"2023-09-23T18:37:40.979163+0000","last_fullsized":"2023-09-23T18:37:40.979163+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:00.174536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":107,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.978728+0000","last_change":"2023-09-23T18:28:03.290316+0000","last_active":"2023-09-23T18:37:40.978728+0000","last_peered":"2023-09-23T18:37:40.978728+0000","last_clean":"2023-09-23T18:37:40.978728+0000","last_became_active":"2023-09-23T18:28:03.289698+0000","last_became_peered":"2023-09-23T18:28:03.289698+0000","last_unstale":"2023-09-23T18:37:40.978728+0000","last_undegraded":"2023-09-23T18:37:40.978728+0000","last_fullsized":"2023-09-23T18:37:40.978728+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:00.779542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":113,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.980287+0000","last_change":"2023-09-23T18:32:26.808237+0000","last_active":"2023-09-23T18:37:40.980287+0000","last_peered":"2023-09-23T18:37:40.980287+0000","last_clean":"2023-09-23T18:37:40.980287+0000","last_became_active":"2023-09-23T18:32:26.807843+0000","last_became_peered":"2023-09-23T18:32:26.807843+0000","last_unstale":"2023-09-23T18:37:40.980287+0000","last_undegraded":"2023-09-23T18:37:40.980287+0000","last_fullsized":"2023-09-23T18:37:40.980287+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":107,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.980018+0000","last_change":"2023-09-23T18:15:08.916658+0000","last_active":"2023-09-23T18:37:40.980018+0000","last_peered":"2023-09-23T18:37:40.980018+0000","last_clean":"2023-09-23T18:37:40.980018+0000","last_became_active":"2023-09-23T18:15:08.916301+0000","last_became_peered":"2023-09-23T18:15:08.916301+0000","last_unstale":"2023-09-23T18:37:40.980018+0000","last_undegraded":"2023-09-23T18:37:40.980018+0000","last_fullsized":"2023-09-23T18:37:40.980018+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":108,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.977708+0000","last_change":"2023-09-23T18:28:03.290217+0000","last_active":"2023-09-23T18:37:40.977708+0000","last_peered":"2023-09-23T18:37:40.977708+0000","last_clean":"2023-09-23T18:37:40.977708+0000","last_became_active":"2023-09-23T18:28:03.289888+0000","last_became_peered":"2023-09-23T18:28:03.289888+0000","last_unstale":"2023-09-23T18:37:40.977708+0000","last_undegraded":"2023-09-23T18:37:40.977708+0000","last_fullsized":"2023-09-23T18:37:40.977708+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:51:45.400847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":149,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.982346+0000","last_change":"2023-09-23T18:32:26.810836+0000","last_active":"2023-09-23T18:37:40.982346+0000","last_peered":"2023-09-23T18:37:40.982346+0000","last_clean":"2023-09-23T18:37:40.982346+0000","last_became_active":"2023-09-23T18:32:26.795378+0000","last_became_peered":"2023-09-23T18:32:26.795378+0000","last_unstale":"2023-09-23T18:37:40.982346+0000","last_undegraded":"2023-09-23T18:37:40.982346+0000","last_fullsized":"2023-09-23T18:37:40.982346+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":142,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:41.528498+0000","last_change":"2023-09-23T18:33:53.852173+0000","last_active":"2023-09-23T18:37:41.528498+0000","last_peered":"2023-09-23T18:37:41.528498+0000","last_clean":"2023-09-23T18:37:41.528498+0000","last_became_active":"2023-09-23T18:33:53.851814+0000","last_became_peered":"2023-09-23T18:33:53.851814+0000","last_unstale":"2023-09-23T18:37:41.528498+0000","last_undegraded":"2023-09-23T18:37:41.528498+0000","last_fullsized":"2023-09-23T18:37:41.528498+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:15.845259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"156'2490","reported_seq":2505,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-09-23T18:37:50.612013+0000","last_change":"2023-09-23T18:37:11.000750+0000","last_active":"2023-09-23T18:37:50.612013+0000","last_peered":"2023-09-23T18:37:50.612013+0000","last_clean":"2023-09-23T18:37:50.612013+0000","last_became_active":"2023-09-23T18:37:11.000441+0000","last_became_peered":"2023-09-23T18:37:11.000441+0000","last_unstale":"2023-09-23T18:37:50.612013+0000","last_undegraded":"2023-09-23T18:37:50.612013+0000","last_fullsized":"2023-09-23T18:37:50.612013+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":2490,"log_dups_size":0,"ondisk_log_size":2490,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163184640,"num_objects":2490,"num_object_clones":0,"num_object_copies":7470,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2490,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2490,"num_write_kb":159360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":136,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:41.034998+0000","last_change":"2023-09-23T18:37:41.034732+0000","last_active":"2023-09-23T18:37:41.034998+0000","last_peered":"2023-09-23T18:37:41.034998+0000","last_clean":"2023-09-23T18:37:41.034998+0000","last_became_active":"2023-09-23T18:37:40.985865+0000","last_became_peered":"2023-09-23T18:37:40.985865+0000","last_unstale":"2023-09-23T18:37:41.034998+0000","last_undegraded":"2023-09-23T18:37:41.034998+0000","last_fullsized":"2023-09-23T18:37:41.034998+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":54,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.979733+0000","last_change":"2023-09-23T18:32:26.806328+0000","last_active":"2023-09-23T18:37:40.979733+0000","last_peered":"2023-09-23T18:37:40.979733+0000","last_clean":"2023-09-23T18:37:40.979733+0000","last_became_active":"2023-09-23T18:32:26.803969+0000","last_became_peered":"2023-09-23T18:32:26.803969+0000","last_unstale":"2023-09-23T18:37:40.979733+0000","last_undegraded":"2023-09-23T18:37:40.979733+0000","last_fullsized":"2023-09-23T18:37:40.979733+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.b","version":"156'2469","reported_seq":2484,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-09-23T18:37:51.541636+0000","last_change":"2023-09-23T18:37:10.997233+0000","last_active":"2023-09-23T18:37:51.541636+0000","last_peered":"2023-09-23T18:37:51.541636+0000","last_clean":"2023-09-23T18:37:51.541636+0000","last_became_active":"2023-09-23T18:37:10.996865+0000","last_became_peered":"2023-09-23T18:37:10.996865+0000","last_unstale":"2023-09-23T18:37:51.541636+0000","last_undegraded":"2023-09-23T18:37:51.541636+0000","last_fullsized":"2023-09-23T18:37:51.541636+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":2469,"log_dups_size":0,"ondisk_log_size":2469,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:06.628611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161808384,"num_objects":2469,"num_object_clones":0,"num_object_copies":7407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2469,"num_write_kb":158016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":139,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.980365+0000","last_change":"2023-09-23T18:33:53.847817+0000","last_active":"2023-09-23T18:37:40.980365+0000","last_peered":"2023-09-23T18:37:40.980365+0000","last_clean":"2023-09-23T18:37:40.980365+0000","last_became_active":"2023-09-23T18:33:53.847342+0000","last_became_peered":"2023-09-23T18:33:53.847342+0000","last_unstale":"2023-09-23T18:37:40.980365+0000","last_undegraded":"2023-09-23T18:37:40.980365+0000","last_fullsized":"2023-09-23T18:37:40.980365+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":151,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.985001+0000","last_change":"2023-09-23T18:33:53.841434+0000","last_active":"2023-09-23T18:37:40.985001+0000","last_peered":"2023-09-23T18:37:40.985001+0000","last_clean":"2023-09-23T18:37:40.985001+0000","last_became_active":"2023-09-23T18:33:53.840923+0000","last_became_peered":"2023-09-23T18:33:53.840923+0000","last_unstale":"2023-09-23T18:37:40.985001+0000","last_undegraded":"2023-09-23T18:37:40.985001+0000","last_fullsized":"2023-09-23T18:37:40.985001+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"156'2450","reported_seq":2465,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-09-23T18:37:50.747065+0000","last_change":"2023-09-23T18:37:10.998596+0000","last_active":"2023-09-23T18:37:50.747065+0000","last_peered":"2023-09-23T18:37:50.747065+0000","last_clean":"2023-09-23T18:37:50.747065+0000","last_became_active":"2023-09-23T18:37:10.997807+0000","last_became_peered":"2023-09-23T18:37:10.997807+0000","last_unstale":"2023-09-23T18:37:50.747065+0000","last_undegraded":"2023-09-23T18:37:50.747065+0000","last_fullsized":"2023-09-23T18:37:50.747065+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":2450,"log_dups_size":0,"ondisk_log_size":2450,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160563200,"num_objects":2450,"num_object_clones":0,"num_object_copies":7350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2450,"num_write_kb":156800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":121,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.980527+0000","last_change":"2023-09-23T18:32:26.854615+0000","last_active":"2023-09-23T18:37:40.980527+0000","last_peered":"2023-09-23T18:37:40.980527+0000","last_clean":"2023-09-23T18:37:40.980527+0000","last_became_active":"2023-09-23T18:32:26.806771+0000","last_became_peered":"2023-09-23T18:32:26.806771+0000","last_unstale":"2023-09-23T18:37:40.980527+0000","last_undegraded":"2023-09-23T18:37:40.980527+0000","last_fullsized":"2023-09-23T18:37:40.980527+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":130,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.982576+0000","last_change":"2023-09-23T18:27:33.589116+0000","last_active":"2023-09-23T18:37:40.982576+0000","last_peered":"2023-09-23T18:37:40.982576+0000","last_clean":"2023-09-23T18:37:40.982576+0000","last_became_active":"2023-09-23T18:27:33.588809+0000","last_became_peered":"2023-09-23T18:27:33.588809+0000","last_unstale":"2023-09-23T18:37:40.982576+0000","last_undegraded":"2023-09-23T18:37:40.982576+0000","last_fullsized":"2023-09-23T18:37:40.982576+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"156'2257","reported_seq":2273,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-09-23T18:37:49.467550+0000","last_change":"2023-09-23T18:37:10.994972+0000","last_active":"2023-09-23T18:37:49.467550+0000","last_peered":"2023-09-23T18:37:49.467550+0000","last_clean":"2023-09-23T18:37:49.467550+0000","last_became_active":"2023-09-23T18:37:10.994640+0000","last_became_peered":"2023-09-23T18:37:10.994640+0000","last_unstale":"2023-09-23T18:37:49.467550+0000","last_undegraded":"2023-09-23T18:37:49.467550+0000","last_fullsized":"2023-09-23T18:37:49.467550+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":2257,"log_dups_size":0,"ondisk_log_size":2257,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:05.096016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147914752,"num_objects":2257,"num_object_clones":0,"num_object_copies":6771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2257,"num_write_kb":144448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":127,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.979421+0000","last_change":"2023-09-23T18:33:54.848776+0000","last_active":"2023-09-23T18:37:40.979421+0000","last_peered":"2023-09-23T18:37:40.979421+0000","last_clean":"2023-09-23T18:37:40.979421+0000","last_became_active":"2023-09-23T18:33:54.848380+0000","last_became_peered":"2023-09-23T18:33:54.848380+0000","last_unstale":"2023-09-23T18:37:40.979421+0000","last_undegraded":"2023-09-23T18:37:40.979421+0000","last_fullsized":"2023-09-23T18:37:40.979421+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:17:17.093993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":121,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-09-23T18:37:45.483402+0000","last_change":"2023-09-23T18:27:33.582033+0000","last_active":"2023-09-23T18:37:45.483402+0000","last_peered":"2023-09-23T18:37:45.483402+0000","last_clean":"2023-09-23T18:37:45.483402+0000","last_became_active":"2023-09-23T18:27:33.581667+0000","last_became_peered":"2023-09-23T18:27:33.581667+0000","last_unstale":"2023-09-23T18:37:45.483402+0000","last_undegraded":"2023-09-23T18:37:45.483402+0000","last_fullsized":"2023-09-23T18:37:45.483402+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:36.039922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"156'2469","reported_seq":2481,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-09-23T18:37:51.043144+0000","last_change":"2023-09-23T18:37:10.999332+0000","last_active":"2023-09-23T18:37:51.043144+0000","last_peered":"2023-09-23T18:37:51.043144+0000","last_clean":"2023-09-23T18:37:51.043144+0000","last_became_active":"2023-09-23T18:37:10.998962+0000","last_became_peered":"2023-09-23T18:37:10.998962+0000","last_unstale":"2023-09-23T18:37:51.043144+0000","last_undegraded":"2023-09-23T18:37:51.043144+0000","last_fullsized":"2023-09-23T18:37:51.043144+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":2469,"log_dups_size":0,"ondisk_log_size":2469,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161808384,"num_objects":2469,"num_object_clones":0,"num_object_copies":7407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2469,"num_write_kb":158016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":126,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.975981+0000","last_change":"2023-09-23T18:28:03.297628+0000","last_active":"2023-09-23T18:37:40.975981+0000","last_peered":"2023-09-23T18:37:40.975981+0000","last_clean":"2023-09-23T18:37:40.975981+0000","last_became_active":"2023-09-23T18:28:03.296694+0000","last_became_peered":"2023-09-23T18:28:03.296694+0000","last_unstale":"2023-09-23T18:37:40.975981+0000","last_undegraded":"2023-09-23T18:37:40.975981+0000","last_fullsized":"2023-09-23T18:37:40.975981+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:43.375652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":121,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.986644+0000","last_change":"2023-09-23T18:28:03.302326+0000","last_active":"2023-09-23T18:37:40.986644+0000","last_peered":"2023-09-23T18:37:40.986644+0000","last_clean":"2023-09-23T18:37:40.986644+0000","last_became_active":"2023-09-23T18:28:03.301970+0000","last_became_peered":"2023-09-23T18:28:03.301970+0000","last_unstale":"2023-09-23T18:37:40.986644+0000","last_undegraded":"2023-09-23T18:37:40.986644+0000","last_fullsized":"2023-09-23T18:37:40.986644+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"156'2639","reported_seq":2655,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-09-23T18:37:53.481364+0000","last_change":"2023-09-23T18:37:10.997349+0000","last_active":"2023-09-23T18:37:53.481364+0000","last_peered":"2023-09-23T18:37:53.481364+0000","last_clean":"2023-09-23T18:37:53.481364+0000","last_became_active":"2023-09-23T18:37:10.997090+0000","last_became_peered":"2023-09-23T18:37:10.997090+0000","last_unstale":"2023-09-23T18:37:53.481364+0000","last_undegraded":"2023-09-23T18:37:53.481364+0000","last_fullsized":"2023-09-23T18:37:53.481364+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":2639,"log_dups_size":0,"ondisk_log_size":2639,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:54.279665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172949504,"num_objects":2639,"num_object_clones":0,"num_object_copies":7917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2639,"num_write_kb":168896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":137,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.982392+0000","last_change":"2023-09-23T18:33:53.855371+0000","last_active":"2023-09-23T18:37:40.982392+0000","last_peered":"2023-09-23T18:37:40.982392+0000","last_clean":"2023-09-23T18:37:40.982392+0000","last_became_active":"2023-09-23T18:33:53.854996+0000","last_became_peered":"2023-09-23T18:33:53.854996+0000","last_unstale":"2023-09-23T18:37:40.982392+0000","last_undegraded":"2023-09-23T18:37:40.982392+0000","last_fullsized":"2023-09-23T18:37:40.982392+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":151,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.986219+0000","last_change":"2023-09-23T18:33:53.845253+0000","last_active":"2023-09-23T18:37:40.986219+0000","last_peered":"2023-09-23T18:37:40.986219+0000","last_clean":"2023-09-23T18:37:40.986219+0000","last_became_active":"2023-09-23T18:33:53.844843+0000","last_became_peered":"2023-09-23T18:33:53.844843+0000","last_unstale":"2023-09-23T18:37:40.986219+0000","last_undegraded":"2023-09-23T18:37:40.986219+0000","last_fullsized":"2023-09-23T18:37:40.986219+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"156'2479","reported_seq":2494,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-09-23T18:37:51.539298+0000","last_change":"2023-09-23T18:37:11.000452+0000","last_active":"2023-09-23T18:37:51.539298+0000","last_peered":"2023-09-23T18:37:51.539298+0000","last_clean":"2023-09-23T18:37:51.539298+0000","last_became_active":"2023-09-23T18:37:11.000056+0000","last_became_peered":"2023-09-23T18:37:11.000056+0000","last_unstale":"2023-09-23T18:37:51.539298+0000","last_undegraded":"2023-09-23T18:37:51.539298+0000","last_fullsized":"2023-09-23T18:37:51.539298+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":2479,"log_dups_size":0,"ondisk_log_size":2479,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:53.369250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162463744,"num_objects":2479,"num_object_clones":0,"num_object_copies":7437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2479,"num_write_kb":158656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":122,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.984815+0000","last_change":"2023-09-23T18:27:33.585714+0000","last_active":"2023-09-23T18:37:40.984815+0000","last_peered":"2023-09-23T18:37:40.984815+0000","last_clean":"2023-09-23T18:37:40.984815+0000","last_became_active":"2023-09-23T18:27:33.584641+0000","last_became_peered":"2023-09-23T18:27:33.584641+0000","last_unstale":"2023-09-23T18:37:40.984815+0000","last_undegraded":"2023-09-23T18:37:40.984815+0000","last_fullsized":"2023-09-23T18:37:40.984815+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":136,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.986316+0000","last_change":"2023-09-23T18:33:53.851266+0000","last_active":"2023-09-23T18:37:40.986316+0000","last_peered":"2023-09-23T18:37:40.986316+0000","last_clean":"2023-09-23T18:37:40.986316+0000","last_became_active":"2023-09-23T18:33:53.850926+0000","last_became_peered":"2023-09-23T18:33:53.850926+0000","last_unstale":"2023-09-23T18:37:40.986316+0000","last_undegraded":"2023-09-23T18:37:40.986316+0000","last_fullsized":"2023-09-23T18:37:40.986316+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"156'2414","reported_seq":2430,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-09-23T18:37:52.036117+0000","last_change":"2023-09-23T18:37:10.994424+0000","last_active":"2023-09-23T18:37:52.036117+0000","last_peered":"2023-09-23T18:37:52.036117+0000","last_clean":"2023-09-23T18:37:52.036117+0000","last_became_active":"2023-09-23T18:37:10.994110+0000","last_became_peered":"2023-09-23T18:37:10.994110+0000","last_unstale":"2023-09-23T18:37:52.036117+0000","last_undegraded":"2023-09-23T18:37:52.036117+0000","last_fullsized":"2023-09-23T18:37:52.036117+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":2414,"log_dups_size":0,"ondisk_log_size":2414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:05.421139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158203904,"num_objects":2414,"num_object_clones":0,"num_object_copies":7242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2414,"num_write_kb":154496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":143,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.977273+0000","last_change":"2023-09-23T18:33:53.835731+0000","last_active":"2023-09-23T18:37:40.977273+0000","last_peered":"2023-09-23T18:37:40.977273+0000","last_clean":"2023-09-23T18:37:40.977273+0000","last_became_active":"2023-09-23T18:33:53.835238+0000","last_became_peered":"2023-09-23T18:33:53.835238+0000","last_unstale":"2023-09-23T18:37:40.977273+0000","last_undegraded":"2023-09-23T18:37:40.977273+0000","last_fullsized":"2023-09-23T18:37:40.977273+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:07:18.870358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":92,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.974464+0000","last_change":"2023-09-23T18:33:53.855430+0000","last_active":"2023-09-23T18:37:40.974464+0000","last_peered":"2023-09-23T18:37:40.974464+0000","last_clean":"2023-09-23T18:37:40.974464+0000","last_became_active":"2023-09-23T18:33:53.854935+0000","last_became_peered":"2023-09-23T18:33:53.854935+0000","last_unstale":"2023-09-23T18:37:40.974464+0000","last_undegraded":"2023-09-23T18:37:40.974464+0000","last_fullsized":"2023-09-23T18:37:40.974464+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:36.078411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"156'2340","reported_seq":2356,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-09-23T18:37:49.463354+0000","last_change":"2023-09-23T18:37:10.995736+0000","last_active":"2023-09-23T18:37:49.463354+0000","last_peered":"2023-09-23T18:37:49.463354+0000","last_clean":"2023-09-23T18:37:49.463354+0000","last_became_active":"2023-09-23T18:37:10.995437+0000","last_became_peered":"2023-09-23T18:37:10.995437+0000","last_unstale":"2023-09-23T18:37:49.463354+0000","last_undegraded":"2023-09-23T18:37:49.463354+0000","last_fullsized":"2023-09-23T18:37:49.463354+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":2340,"log_dups_size":0,"ondisk_log_size":2340,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:39:17.818584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153354240,"num_objects":2340,"num_object_clones":0,"num_object_copies":7020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2340,"num_write_kb":149760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"77'130","reported_seq":618,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.977508+0000","last_change":"2023-09-23T18:33:53.850713+0000","last_active":"2023-09-23T18:37:40.977508+0000","last_peered":"2023-09-23T18:37:40.977508+0000","last_clean":"2023-09-23T18:37:40.977508+0000","last_became_active":"2023-09-23T18:33:53.850011+0000","last_became_peered":"2023-09-23T18:33:53.850011+0000","last_unstale":"2023-09-23T18:37:40.977508+0000","last_undegraded":"2023-09-23T18:37:40.977508+0000","last_fullsized":"2023-09-23T18:37:40.977508+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:58.943556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":772,"num_read_kb":1712,"num_write":126,"num_write_kb":2126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":131,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.976559+0000","last_change":"2023-09-23T18:32:26.793356+0000","last_active":"2023-09-23T18:37:40.976559+0000","last_peered":"2023-09-23T18:37:40.976559+0000","last_clean":"2023-09-23T18:37:40.976559+0000","last_became_active":"2023-09-23T18:32:26.792868+0000","last_became_peered":"2023-09-23T18:32:26.792868+0000","last_unstale":"2023-09-23T18:37:40.976559+0000","last_undegraded":"2023-09-23T18:37:40.976559+0000","last_fullsized":"2023-09-23T18:37:40.976559+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:40:21.193604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":66,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.973616+0000","last_change":"2023-09-23T18:33:54.843246+0000","last_active":"2023-09-23T18:37:40.973616+0000","last_peered":"2023-09-23T18:37:40.973616+0000","last_clean":"2023-09-23T18:37:40.973616+0000","last_became_active":"2023-09-23T18:33:54.842933+0000","last_became_peered":"2023-09-23T18:33:54.842933+0000","last_unstale":"2023-09-23T18:37:40.973616+0000","last_undegraded":"2023-09-23T18:37:40.973616+0000","last_fullsized":"2023-09-23T18:37:40.973616+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:43.716707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"156'2466","reported_seq":2482,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-09-23T18:37:51.523491+0000","last_change":"2023-09-23T18:37:11.000225+0000","last_active":"2023-09-23T18:37:51.523491+0000","last_peered":"2023-09-23T18:37:51.523491+0000","last_clean":"2023-09-23T18:37:51.523491+0000","last_became_active":"2023-09-23T18:37:10.999900+0000","last_became_peered":"2023-09-23T18:37:10.999900+0000","last_unstale":"2023-09-23T18:37:51.523491+0000","last_undegraded":"2023-09-23T18:37:51.523491+0000","last_fullsized":"2023-09-23T18:37:51.523491+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":2466,"log_dups_size":0,"ondisk_log_size":2466,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:41.493477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161611776,"num_objects":2466,"num_object_clones":0,"num_object_copies":7398,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2466,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2466,"num_write_kb":157824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"77'96","reported_seq":562,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.973785+0000","last_change":"2023-09-23T18:33:54.842857+0000","last_active":"2023-09-23T18:37:40.973785+0000","last_peered":"2023-09-23T18:37:40.973785+0000","last_clean":"2023-09-23T18:37:40.973785+0000","last_became_active":"2023-09-23T18:33:54.842134+0000","last_became_peered":"2023-09-23T18:33:54.842134+0000","last_unstale":"2023-09-23T18:37:40.973785+0000","last_undegraded":"2023-09-23T18:37:40.973785+0000","last_fullsized":"2023-09-23T18:37:40.973785+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":96,"log_dups_size":0,"ondisk_log_size":96,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:48.960158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":728,"num_read_kb":1234,"num_write":87,"num_write_kb":338,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":122,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.985758+0000","last_change":"2023-09-23T18:26:34.992535+0000","last_active":"2023-09-23T18:37:40.985758+0000","last_peered":"2023-09-23T18:37:40.985758+0000","last_clean":"2023-09-23T18:37:40.985758+0000","last_became_active":"2023-09-23T18:26:34.992177+0000","last_became_peered":"2023-09-23T18:26:34.992177+0000","last_unstale":"2023-09-23T18:37:40.985758+0000","last_undegraded":"2023-09-23T18:37:40.985758+0000","last_fullsized":"2023-09-23T18:37:40.985758+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":107,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.977425+0000","last_change":"2023-09-23T18:33:53.841776+0000","last_active":"2023-09-23T18:37:40.977425+0000","last_peered":"2023-09-23T18:37:40.977425+0000","last_clean":"2023-09-23T18:37:40.977425+0000","last_became_active":"2023-09-23T18:33:53.841436+0000","last_became_peered":"2023-09-23T18:33:53.841436+0000","last_unstale":"2023-09-23T18:37:40.977425+0000","last_undegraded":"2023-09-23T18:37:40.977425+0000","last_fullsized":"2023-09-23T18:37:40.977425+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:27.198179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"156'2322","reported_seq":2337,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-09-23T18:37:50.714971+0000","last_change":"2023-09-23T18:37:10.998898+0000","last_active":"2023-09-23T18:37:50.714971+0000","last_peered":"2023-09-23T18:37:50.714971+0000","last_clean":"2023-09-23T18:37:50.714971+0000","last_became_active":"2023-09-23T18:37:10.998064+0000","last_became_peered":"2023-09-23T18:37:10.998064+0000","last_unstale":"2023-09-23T18:37:50.714971+0000","last_undegraded":"2023-09-23T18:37:50.714971+0000","last_fullsized":"2023-09-23T18:37:50.714971+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":2322,"log_dups_size":0,"ondisk_log_size":2322,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152174592,"num_objects":2322,"num_object_clones":0,"num_object_copies":6966,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2322,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2322,"num_write_kb":148608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"77'112","reported_seq":594,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.991324+0000","last_change":"2023-09-23T18:28:03.297540+0000","last_active":"2023-09-23T18:37:40.991324+0000","last_peered":"2023-09-23T18:37:40.991324+0000","last_clean":"2023-09-23T18:37:40.991324+0000","last_became_active":"2023-09-23T18:28:03.296649+0000","last_became_peered":"2023-09-23T18:28:03.296649+0000","last_unstale":"2023-09-23T18:37:40.991324+0000","last_undegraded":"2023-09-23T18:37:40.991324+0000","last_fullsized":"2023-09-23T18:37:40.991324+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:32.531775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":774,"num_read_kb":596,"num_write":102,"num_write_kb":52,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":89,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-09-23T18:37:45.481081+0000","last_change":"2023-09-23T18:33:53.849084+0000","last_active":"2023-09-23T18:37:45.481081+0000","last_peered":"2023-09-23T18:37:45.481081+0000","last_clean":"2023-09-23T18:37:45.481081+0000","last_became_active":"2023-09-23T18:33:53.848757+0000","last_became_peered":"2023-09-23T18:33:53.848757+0000","last_unstale":"2023-09-23T18:37:45.481081+0000","last_undegraded":"2023-09-23T18:37:45.481081+0000","last_fullsized":"2023-09-23T18:37:45.481081+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:15.267742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":111,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.974533+0000","last_change":"2023-09-23T18:33:55.087304+0000","last_active":"2023-09-23T18:37:40.974533+0000","last_peered":"2023-09-23T18:37:40.974533+0000","last_clean":"2023-09-23T18:37:40.974533+0000","last_became_active":"2023-09-23T18:33:55.086832+0000","last_became_peered":"2023-09-23T18:33:55.086832+0000","last_unstale":"2023-09-23T18:37:40.974533+0000","last_undegraded":"2023-09-23T18:37:40.974533+0000","last_fullsized":"2023-09-23T18:37:40.974533+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:55.457664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"156'2508","reported_seq":2524,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-09-23T18:37:51.546103+0000","last_change":"2023-09-23T18:37:11.002449+0000","last_active":"2023-09-23T18:37:51.546103+0000","last_peered":"2023-09-23T18:37:51.546103+0000","last_clean":"2023-09-23T18:37:51.546103+0000","last_became_active":"2023-09-23T18:37:11.002022+0000","last_became_peered":"2023-09-23T18:37:11.002022+0000","last_unstale":"2023-09-23T18:37:51.546103+0000","last_undegraded":"2023-09-23T18:37:51.546103+0000","last_fullsized":"2023-09-23T18:37:51.546103+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":2508,"log_dups_size":0,"ondisk_log_size":2508,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:34.756717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164364288,"num_objects":2508,"num_object_clones":0,"num_object_copies":7524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2508,"num_write_kb":160512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"77'147","reported_seq":543,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.992526+0000","last_change":"2023-09-23T18:32:29.845404+0000","last_active":"2023-09-23T18:37:40.992526+0000","last_peered":"2023-09-23T18:37:40.992526+0000","last_clean":"2023-09-23T18:37:40.992526+0000","last_became_active":"2023-09-23T18:32:29.844545+0000","last_became_peered":"2023-09-23T18:32:29.844545+0000","last_unstale":"2023-09-23T18:37:40.992526+0000","last_undegraded":"2023-09-23T18:37:40.992526+0000","last_fullsized":"2023-09-23T18:37:40.992526+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:44.834482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":624,"num_read_kb":1868,"num_write":143,"num_write_kb":5368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":358,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.980027+0000","last_change":"2023-09-23T18:26:34.999402+0000","last_active":"2023-09-23T18:37:40.980027+0000","last_peered":"2023-09-23T18:37:40.980027+0000","last_clean":"2023-09-23T18:37:40.980027+0000","last_became_active":"2023-09-23T18:26:34.998828+0000","last_became_peered":"2023-09-23T18:26:34.998828+0000","last_unstale":"2023-09-23T18:37:40.980027+0000","last_undegraded":"2023-09-23T18:37:40.980027+0000","last_fullsized":"2023-09-23T18:37:40.980027+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_clean_scrub_stamp":"2023-09-23T18:08:35.701374+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":127,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.977141+0000","last_change":"2023-09-23T18:32:26.800920+0000","last_active":"2023-09-23T18:37:40.977141+0000","last_peered":"2023-09-23T18:37:40.977141+0000","last_clean":"2023-09-23T18:37:40.977141+0000","last_became_active":"2023-09-23T18:32:26.800528+0000","last_became_peered":"2023-09-23T18:32:26.800528+0000","last_unstale":"2023-09-23T18:37:40.977141+0000","last_undegraded":"2023-09-23T18:37:40.977141+0000","last_fullsized":"2023-09-23T18:37:40.977141+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:20:50.939219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":106,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.982005+0000","last_change":"2023-09-23T18:15:08.921629+0000","last_active":"2023-09-23T18:37:40.982005+0000","last_peered":"2023-09-23T18:37:40.982005+0000","last_clean":"2023-09-23T18:37:40.982005+0000","last_became_active":"2023-09-23T18:15:08.921283+0000","last_became_peered":"2023-09-23T18:15:08.921283+0000","last_unstale":"2023-09-23T18:37:40.982005+0000","last_undegraded":"2023-09-23T18:37:40.982005+0000","last_fullsized":"2023-09-23T18:37:40.982005+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"156'2545","reported_seq":2561,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-09-23T18:37:52.039610+0000","last_change":"2023-09-23T18:37:11.001959+0000","last_active":"2023-09-23T18:37:52.039610+0000","last_peered":"2023-09-23T18:37:52.039610+0000","last_clean":"2023-09-23T18:37:52.039610+0000","last_became_active":"2023-09-23T18:37:11.001542+0000","last_became_peered":"2023-09-23T18:37:11.001542+0000","last_unstale":"2023-09-23T18:37:52.039610+0000","last_undegraded":"2023-09-23T18:37:52.039610+0000","last_fullsized":"2023-09-23T18:37:52.039610+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":2545,"log_dups_size":0,"ondisk_log_size":2545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:27.036499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166789120,"num_objects":2545,"num_object_clones":0,"num_object_copies":7635,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2545,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2545,"num_write_kb":162880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"77'120","reported_seq":617,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.980144+0000","last_change":"2023-09-23T18:33:53.852697+0000","last_active":"2023-09-23T18:37:40.980144+0000","last_peered":"2023-09-23T18:37:40.980144+0000","last_clean":"2023-09-23T18:37:40.980144+0000","last_became_active":"2023-09-23T18:33:53.852210+0000","last_became_peered":"2023-09-23T18:33:53.852210+0000","last_unstale":"2023-09-23T18:37:40.980144+0000","last_undegraded":"2023-09-23T18:37:40.980144+0000","last_fullsized":"2023-09-23T18:37:40.980144+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":120,"log_dups_size":0,"ondisk_log_size":120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":809,"num_read_kb":2749,"num_write":98,"num_write_kb":2087,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":108,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.978075+0000","last_change":"2023-09-23T18:28:03.299508+0000","last_active":"2023-09-23T18:37:40.978075+0000","last_peered":"2023-09-23T18:37:40.978075+0000","last_clean":"2023-09-23T18:37:40.978075+0000","last_became_active":"2023-09-23T18:28:03.298682+0000","last_became_peered":"2023-09-23T18:28:03.298682+0000","last_unstale":"2023-09-23T18:37:40.978075+0000","last_undegraded":"2023-09-23T18:37:40.978075+0000","last_fullsized":"2023-09-23T18:37:40.978075+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:08.398302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":139,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.977942+0000","last_change":"2023-09-23T18:33:53.846904+0000","last_active":"2023-09-23T18:37:40.977942+0000","last_peered":"2023-09-23T18:37:40.977942+0000","last_clean":"2023-09-23T18:37:40.977942+0000","last_became_active":"2023-09-23T18:33:53.846654+0000","last_became_peered":"2023-09-23T18:33:53.846654+0000","last_unstale":"2023-09-23T18:37:40.977942+0000","last_undegraded":"2023-09-23T18:37:40.977942+0000","last_fullsized":"2023-09-23T18:37:40.977942+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:55.898016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"156'2576","reported_seq":2592,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-09-23T18:37:52.047111+0000","last_change":"2023-09-23T18:37:11.001841+0000","last_active":"2023-09-23T18:37:52.047111+0000","last_peered":"2023-09-23T18:37:52.047111+0000","last_clean":"2023-09-23T18:37:52.047111+0000","last_became_active":"2023-09-23T18:37:11.001499+0000","last_became_peered":"2023-09-23T18:37:11.001499+0000","last_unstale":"2023-09-23T18:37:52.047111+0000","last_undegraded":"2023-09-23T18:37:52.047111+0000","last_fullsized":"2023-09-23T18:37:52.047111+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":2576,"log_dups_size":0,"ondisk_log_size":2576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:32.700925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168820736,"num_objects":2576,"num_object_clones":0,"num_object_copies":7728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2576,"num_write_kb":164864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'78","reported_seq":392,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.980399+0000","last_change":"2023-09-23T18:26:34.992345+0000","last_active":"2023-09-23T18:37:40.980399+0000","last_peered":"2023-09-23T18:37:40.980399+0000","last_clean":"2023-09-23T18:37:40.980399+0000","last_became_active":"2023-09-23T18:26:34.991774+0000","last_became_peered":"2023-09-23T18:26:34.991774+0000","last_unstale":"2023-09-23T18:37:40.980399+0000","last_undegraded":"2023-09-23T18:37:40.980399+0000","last_fullsized":"2023-09-23T18:37:40.980399+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":78,"log_dups_size":0,"ondisk_log_size":78,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":428,"num_read_kb":1125,"num_write":73,"num_write_kb":431,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":86,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.977623+0000","last_change":"2023-09-23T18:33:54.897293+0000","last_active":"2023-09-23T18:37:40.977623+0000","last_peered":"2023-09-23T18:37:40.977623+0000","last_clean":"2023-09-23T18:37:40.977623+0000","last_became_active":"2023-09-23T18:33:54.896491+0000","last_became_peered":"2023-09-23T18:33:54.896491+0000","last_unstale":"2023-09-23T18:37:40.977623+0000","last_undegraded":"2023-09-23T18:37:40.977623+0000","last_fullsized":"2023-09-23T18:37:40.977623+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:03.636617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":114,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.977587+0000","last_change":"2023-09-23T18:33:54.852309+0000","last_active":"2023-09-23T18:37:40.977587+0000","last_peered":"2023-09-23T18:37:40.977587+0000","last_clean":"2023-09-23T18:37:40.977587+0000","last_became_active":"2023-09-23T18:33:54.850255+0000","last_became_peered":"2023-09-23T18:33:54.850255+0000","last_unstale":"2023-09-23T18:37:40.977587+0000","last_undegraded":"2023-09-23T18:37:40.977587+0000","last_fullsized":"2023-09-23T18:37:40.977587+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:54.148992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"156'2404","reported_seq":2419,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-09-23T18:37:51.006961+0000","last_change":"2023-09-23T18:37:11.002174+0000","last_active":"2023-09-23T18:37:51.006961+0000","last_peered":"2023-09-23T18:37:51.006961+0000","last_clean":"2023-09-23T18:37:51.006961+0000","last_became_active":"2023-09-23T18:37:11.001860+0000","last_became_peered":"2023-09-23T18:37:11.001860+0000","last_unstale":"2023-09-23T18:37:51.006961+0000","last_undegraded":"2023-09-23T18:37:51.006961+0000","last_fullsized":"2023-09-23T18:37:51.006961+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":2404,"log_dups_size":0,"ondisk_log_size":2404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157548544,"num_objects":2404,"num_object_clones":0,"num_object_copies":7212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2404,"num_write_kb":153856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":126,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.977631+0000","last_change":"2023-09-23T18:28:03.298805+0000","last_active":"2023-09-23T18:37:40.977631+0000","last_peered":"2023-09-23T18:37:40.977631+0000","last_clean":"2023-09-23T18:37:40.977631+0000","last_became_active":"2023-09-23T18:28:03.298437+0000","last_became_peered":"2023-09-23T18:28:03.298437+0000","last_unstale":"2023-09-23T18:37:40.977631+0000","last_undegraded":"2023-09-23T18:37:40.977631+0000","last_fullsized":"2023-09-23T18:37:40.977631+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:27.831669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'181","reported_seq":626,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.974036+0000","last_change":"2023-09-23T18:33:53.854528+0000","last_active":"2023-09-23T18:37:40.974036+0000","last_peered":"2023-09-23T18:37:40.974036+0000","last_clean":"2023-09-23T18:37:40.974036+0000","last_became_active":"2023-09-23T18:33:53.851883+0000","last_became_peered":"2023-09-23T18:33:53.851883+0000","last_unstale":"2023-09-23T18:37:40.974036+0000","last_undegraded":"2023-09-23T18:37:40.974036+0000","last_fullsized":"2023-09-23T18:37:40.974036+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":181,"log_dups_size":0,"ondisk_log_size":181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:43.888844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":909,"num_read_kb":734,"num_write":148,"num_write_kb":75,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":123,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.977294+0000","last_change":"2023-09-23T18:28:03.292295+0000","last_active":"2023-09-23T18:37:40.977294+0000","last_peered":"2023-09-23T18:37:40.977294+0000","last_clean":"2023-09-23T18:37:40.977294+0000","last_became_active":"2023-09-23T18:28:03.291902+0000","last_became_peered":"2023-09-23T18:28:03.291902+0000","last_unstale":"2023-09-23T18:37:40.977294+0000","last_undegraded":"2023-09-23T18:37:40.977294+0000","last_fullsized":"2023-09-23T18:37:40.977294+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:38.415256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"156'2359","reported_seq":2374,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-09-23T18:37:50.732189+0000","last_change":"2023-09-23T18:37:10.998526+0000","last_active":"2023-09-23T18:37:50.732189+0000","last_peered":"2023-09-23T18:37:50.732189+0000","last_clean":"2023-09-23T18:37:50.732189+0000","last_became_active":"2023-09-23T18:37:10.998129+0000","last_became_peered":"2023-09-23T18:37:10.998129+0000","last_unstale":"2023-09-23T18:37:50.732189+0000","last_undegraded":"2023-09-23T18:37:50.732189+0000","last_fullsized":"2023-09-23T18:37:50.732189+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":2359,"log_dups_size":0,"ondisk_log_size":2359,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154599424,"num_objects":2359,"num_object_clones":0,"num_object_copies":7077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2359,"num_write_kb":150976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"77'112","reported_seq":646,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.982077+0000","last_change":"2023-09-23T18:27:33.587478+0000","last_active":"2023-09-23T18:37:40.982077+0000","last_peered":"2023-09-23T18:37:40.982077+0000","last_clean":"2023-09-23T18:37:40.982077+0000","last_became_active":"2023-09-23T18:27:33.586958+0000","last_became_peered":"2023-09-23T18:27:33.586958+0000","last_unstale":"2023-09-23T18:37:40.982077+0000","last_undegraded":"2023-09-23T18:37:40.982077+0000","last_fullsized":"2023-09-23T18:37:40.982077+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":893,"num_read_kb":3172,"num_write":69,"num_write_kb":2237,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":127,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.977866+0000","last_change":"2023-09-23T18:33:53.851421+0000","last_active":"2023-09-23T18:37:40.977866+0000","last_peered":"2023-09-23T18:37:40.977866+0000","last_clean":"2023-09-23T18:37:40.977866+0000","last_became_active":"2023-09-23T18:33:53.851012+0000","last_became_peered":"2023-09-23T18:33:53.851012+0000","last_unstale":"2023-09-23T18:37:40.977866+0000","last_undegraded":"2023-09-23T18:37:40.977866+0000","last_fullsized":"2023-09-23T18:37:40.977866+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:03.971054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":115,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.984920+0000","last_change":"2023-09-23T18:32:26.805472+0000","last_active":"2023-09-23T18:37:40.984920+0000","last_peered":"2023-09-23T18:37:40.984920+0000","last_clean":"2023-09-23T18:37:40.984920+0000","last_became_active":"2023-09-23T18:32:26.805078+0000","last_became_peered":"2023-09-23T18:32:26.805078+0000","last_unstale":"2023-09-23T18:37:40.984920+0000","last_undegraded":"2023-09-23T18:37:40.984920+0000","last_fullsized":"2023-09-23T18:37:40.984920+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":96,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.977525+0000","last_change":"2023-09-23T18:26:34.994176+0000","last_active":"2023-09-23T18:37:40.977525+0000","last_peered":"2023-09-23T18:37:40.977525+0000","last_clean":"2023-09-23T18:37:40.977525+0000","last_became_active":"2023-09-23T18:26:34.993829+0000","last_became_peered":"2023-09-23T18:26:34.993829+0000","last_unstale":"2023-09-23T18:37:40.977525+0000","last_undegraded":"2023-09-23T18:37:40.977525+0000","last_fullsized":"2023-09-23T18:37:40.977525+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:27.111857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":162,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.992293+0000","last_change":"2023-09-23T18:33:53.840519+0000","last_active":"2023-09-23T18:37:40.992293+0000","last_peered":"2023-09-23T18:37:40.992293+0000","last_clean":"2023-09-23T18:37:40.992293+0000","last_became_active":"2023-09-23T18:33:53.840181+0000","last_became_peered":"2023-09-23T18:33:53.840181+0000","last_unstale":"2023-09-23T18:37:40.992293+0000","last_undegraded":"2023-09-23T18:37:40.992293+0000","last_fullsized":"2023-09-23T18:37:40.992293+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:41.293009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":79,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.986586+0000","last_change":"2023-09-23T18:33:54.847172+0000","last_active":"2023-09-23T18:37:40.986586+0000","last_peered":"2023-09-23T18:37:40.986586+0000","last_clean":"2023-09-23T18:37:40.986586+0000","last_became_active":"2023-09-23T18:33:54.846751+0000","last_became_peered":"2023-09-23T18:33:54.846751+0000","last_unstale":"2023-09-23T18:37:40.986586+0000","last_undegraded":"2023-09-23T18:37:40.986586+0000","last_fullsized":"2023-09-23T18:37:40.986586+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":151,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.985831+0000","last_change":"2023-09-23T18:33:53.841232+0000","last_active":"2023-09-23T18:37:40.985831+0000","last_peered":"2023-09-23T18:37:40.985831+0000","last_clean":"2023-09-23T18:37:40.985831+0000","last_became_active":"2023-09-23T18:33:53.840749+0000","last_became_peered":"2023-09-23T18:33:53.840749+0000","last_unstale":"2023-09-23T18:37:40.985831+0000","last_undegraded":"2023-09-23T18:37:40.985831+0000","last_fullsized":"2023-09-23T18:37:40.985831+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":79,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.986319+0000","last_change":"2023-09-23T18:33:54.858971+0000","last_active":"2023-09-23T18:37:40.986319+0000","last_peered":"2023-09-23T18:37:40.986319+0000","last_clean":"2023-09-23T18:37:40.986319+0000","last_became_active":"2023-09-23T18:33:54.845921+0000","last_became_peered":"2023-09-23T18:33:54.845921+0000","last_unstale":"2023-09-23T18:37:40.986319+0000","last_undegraded":"2023-09-23T18:37:40.986319+0000","last_fullsized":"2023-09-23T18:37:40.986319+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":136,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.985628+0000","last_change":"2023-09-23T18:28:03.302261+0000","last_active":"2023-09-23T18:37:40.985628+0000","last_peered":"2023-09-23T18:37:40.985628+0000","last_clean":"2023-09-23T18:37:40.985628+0000","last_became_active":"2023-09-23T18:28:03.301911+0000","last_became_peered":"2023-09-23T18:28:03.301911+0000","last_unstale":"2023-09-23T18:37:40.985628+0000","last_undegraded":"2023-09-23T18:37:40.985628+0000","last_fullsized":"2023-09-23T18:37:40.985628+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":133,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.976980+0000","last_change":"2023-09-23T18:28:03.296850+0000","last_active":"2023-09-23T18:37:40.976980+0000","last_peered":"2023-09-23T18:37:40.976980+0000","last_clean":"2023-09-23T18:37:40.976980+0000","last_became_active":"2023-09-23T18:28:03.295842+0000","last_became_peered":"2023-09-23T18:28:03.295842+0000","last_unstale":"2023-09-23T18:37:40.976980+0000","last_undegraded":"2023-09-23T18:37:40.976980+0000","last_fullsized":"2023-09-23T18:37:40.976980+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:53.048579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":146,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:41.528370+0000","last_change":"2023-09-23T18:28:03.299520+0000","last_active":"2023-09-23T18:37:41.528370+0000","last_peered":"2023-09-23T18:37:41.528370+0000","last_clean":"2023-09-23T18:37:41.528370+0000","last_became_active":"2023-09-23T18:28:03.299158+0000","last_became_peered":"2023-09-23T18:28:03.299158+0000","last_unstale":"2023-09-23T18:37:41.528370+0000","last_undegraded":"2023-09-23T18:37:41.528370+0000","last_fullsized":"2023-09-23T18:37:41.528370+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:26.247064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":127,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.978192+0000","last_change":"2023-09-23T18:32:26.805676+0000","last_active":"2023-09-23T18:37:40.978192+0000","last_peered":"2023-09-23T18:37:40.978192+0000","last_clean":"2023-09-23T18:37:40.978192+0000","last_became_active":"2023-09-23T18:32:26.797129+0000","last_became_peered":"2023-09-23T18:32:26.797129+0000","last_unstale":"2023-09-23T18:37:40.978192+0000","last_undegraded":"2023-09-23T18:37:40.978192+0000","last_fullsized":"2023-09-23T18:37:40.978192+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:07.414550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":135,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-09-23T18:37:40.984729+0000","last_change":"2023-09-23T18:33:54.847156+0000","last_active":"2023-09-23T18:37:40.984729+0000","last_peered":"2023-09-23T18:37:40.984729+0000","last_clean":"2023-09-23T18:37:40.984729+0000","last_became_active":"2023-09-23T18:33:54.846765+0000","last_became_peered":"2023-09-23T18:33:54.846765+0000","last_unstale":"2023-09-23T18:37:40.984729+0000","last_undegraded":"2023-09-23T18:37:40.984729+0000","last_fullsized":"2023-09-23T18:37:40.984729+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":2568159232,"num_objects":39187,"num_object_clones":0,"num_object_copies":117561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":39187,"num_write_kb":2507968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7800160256,"data_stored":7800160256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39187,"ondisk_log_size":39187,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":646,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26886,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5937,"num_read_kb":13190,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5621,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430520,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1340792,"kb_used_data":1039472,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92404360,"statfs":{"total":95995035648,"available":94622064640,"internally_reserved":0,"allocated":1064419328,"data_stored":1063858465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:36:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.9750000000000001,"15min":1.407},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.252},"max":{"1min":3.3929999999999998,"5min":57.049999999999997,"15min":57.049999999999997},"last":0.73499999999999999},{"interface":"front","average":{"1min":1.375,"5min":2.0419999999999998,"15min":1.427},"min":{"1min":0.378,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.7880000000000003,"5min":56.773000000000003,"15min":56.773000000000003},"last":0.53900000000000003}]},{"osd":1,"last update":"Sat Sep 23 18:36:59 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.202,"15min":1.147},"min":{"1min":0.32500000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.4420000000000002,"5min":9.2349999999999994,"15min":9.2349999999999994},"last":1.702},{"interface":"front","average":{"1min":0.84999999999999998,"5min":1.3360000000000001,"15min":1.3759999999999999},"min":{"1min":0.44,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":2.8809999999999998,"5min":4.984,"15min":4.984},"last":1.671}]},{"osd":2,"last update":"Sat Sep 23 18:36:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":1.871,"15min":1.2709999999999999},"min":{"1min":0.28899999999999998,"5min":0.254,"15min":0.254},"max":{"1min":4.1500000000000004,"5min":40.091999999999999,"15min":40.091999999999999},"last":0.37},{"interface":"front","average":{"1min":1.423,"5min":1.679,"15min":1.2050000000000001},"min":{"1min":0.34100000000000003,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.0170000000000003,"5min":38.261000000000003,"15min":38.261000000000003},"last":0.79700000000000004}]},{"osd":3,"last update":"Sat Sep 23 18:37:19 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":1.0049999999999999,"15min":0.98199999999999998},"min":{"1min":0.39800000000000002,"5min":0.34399999999999997,"15min":0.32800000000000001},"max":{"1min":1.2210000000000001,"5min":5.8920000000000003,"15min":5.8920000000000003},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.79700000000000004,"5min":1.024,"15min":0.97199999999999998},"min":{"1min":0.51000000000000001,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.2030000000000001,"5min":4.6929999999999996,"15min":4.6929999999999996},"last":1.6379999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:37:02 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.171,"15min":1.0049999999999999},"min":{"1min":0.378,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.76,"5min":6.3529999999999998,"15min":6.3529999999999998},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.89800000000000002,"5min":1.2629999999999999,"15min":1.123},"min":{"1min":0.40400000000000003,"5min":0.32100000000000001,"15min":0.309},"max":{"1min":1.6220000000000001,"5min":11.119,"15min":11.119},"last":0.57599999999999996}]},{"osd":5,"last update":"Sat Sep 23 18:37:02 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":1.0620000000000001,"15min":1.0109999999999999},"min":{"1min":0.318,"5min":0.315,"15min":0.315},"max":{"1min":1.274,"5min":5.8899999999999997,"15min":5.8899999999999997},"last":1.599},{"interface":"front","average":{"1min":1.004,"5min":1.1919999999999999,"15min":1.0900000000000001},"min":{"1min":0.41699999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":2.8079999999999998,"5min":7.46,"15min":7.46},"last":0.47199999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:37:02 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":1.0389999999999999,"15min":1.0680000000000001},"min":{"1min":0.53800000000000003,"5min":0.39400000000000002,"15min":0.35299999999999998},"max":{"1min":1.855,"5min":5.5129999999999999,"15min":5.5129999999999999},"last":0.98799999999999999},{"interface":"front","average":{"1min":0.96999999999999997,"5min":1.077,"15min":1.121},"min":{"1min":0.435,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.6699999999999999,"5min":9.1850000000000005,"15min":9.1850000000000005},"last":0.433}]}]},{"osd":6,"up_from":42,"seq":180388626752,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1205296,"kb_used_data":908072,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92539856,"statfs":{"total":95995035648,"available":94760812544,"internally_reserved":0,"allocated":929865728,"data_stored":929313169,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:36:57 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":3.0590000000000002,"15min":1.6279999999999999},"min":{"1min":0.29899999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.9180000000000001,"5min":182.655,"15min":182.655},"last":0.89700000000000002},{"interface":"front","average":{"1min":0.89500000000000002,"5min":3.145,"15min":1.742},"min":{"1min":0.44,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":3.9569999999999999,"5min":182.58799999999999,"15min":182.58799999999999},"last":0.98299999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:36:57 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":0.73899999999999999,"15min":0.75900000000000001},"min":{"1min":0.374,"5min":0.317,"15min":0.317},"max":{"1min":2.246,"5min":3.3340000000000001,"15min":3.3340000000000001},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.97199999999999998,"15min":1.0640000000000001},"min":{"1min":0.32700000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.95799999999999996,"5min":3.8359999999999999,"15min":3.8359999999999999},"last":0.65000000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:36:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":2.02,"15min":1.3340000000000001},"min":{"1min":0.48299999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.3380000000000001,"5min":51.728000000000002,"15min":51.728000000000002},"last":122.242},{"interface":"front","average":{"1min":1.321,"5min":2.044,"15min":1.3120000000000001},"min":{"1min":0.443,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.3230000000000004,"5min":51.820999999999998,"15min":51.820999999999998},"last":122.32899999999999}]},{"osd":3,"last update":"Sat Sep 23 18:37:18 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.77300000000000002,"15min":0.96899999999999997},"min":{"1min":0.42999999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.3480000000000001,"5min":4.6020000000000003,"15min":21.282},"last":123.52200000000001},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.91700000000000004,"15min":1.0169999999999999},"min":{"1min":0.47699999999999998,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.1890000000000001,"5min":6.7060000000000004,"15min":21.260000000000002},"last":123.605}]},{"osd":4,"last update":"Sat Sep 23 18:37:12 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":0.96299999999999997,"15min":0.93700000000000006},"min":{"1min":0.46500000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.1439999999999999,"5min":8.0169999999999995,"15min":8.0169999999999995},"last":0.81299999999999994},{"interface":"front","average":{"1min":1.1020000000000001,"5min":0.91100000000000003,"15min":0.96899999999999997},"min":{"1min":0.35699999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":4.2539999999999996,"5min":4.2539999999999996,"15min":4.2539999999999996},"last":1.141}]},{"osd":5,"last update":"Sat Sep 23 18:37:12 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.89900000000000002,"15min":0.97899999999999998},"min":{"1min":0.501,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.282,"5min":7.9240000000000004,"15min":7.9240000000000004},"last":1.024},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.80500000000000005,"15min":0.98099999999999998},"min":{"1min":0.64100000000000001,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.2689999999999999,"5min":4.5270000000000001,"15min":4.5270000000000001},"last":1.1990000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":1.0229999999999999,"15min":1.044},"min":{"1min":0.46100000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.516,"5min":4.6120000000000001,"15min":4.6120000000000001},"last":0.92500000000000004},{"interface":"front","average":{"1min":0.98699999999999999,"5min":1.3340000000000001,"15min":1.208},"min":{"1min":0.42899999999999999,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":2.1360000000000001,"5min":9.1129999999999995,"15min":9.1129999999999995},"last":1.109}]}]},{"osd":1,"up_from":123,"seq":528280977457,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1058208,"kb_used_data":967832,"kb_used_omap":7,"kb_used_meta":90360,"kb_avail":92686944,"statfs":{"total":95995035648,"available":94911430656,"internally_reserved":0,"allocated":991059968,"data_stored":990290087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":92529422},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:37:01 2023","interfaces":[{"interface":"back","average":{"1min":2.488,"5min":8.2539999999999996,"15min":8.8719999999999999},"min":{"1min":0.28299999999999997,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":23.280000000000001,"5min":173.41800000000001,"15min":173.41800000000001},"last":1.1539999999999999},{"interface":"front","average":{"1min":2.8140000000000001,"5min":7.8330000000000002,"15min":8.3160000000000007},"min":{"1min":0.35099999999999998,"5min":0.249,"15min":0.249},"max":{"1min":22.93,"5min":166.42599999999999,"15min":166.42599999999999},"last":1.7350000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:37:01 2023","interfaces":[{"interface":"back","average":{"1min":2.464,"5min":2.5750000000000002,"15min":2.6930000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":23.074999999999999,"5min":23.074999999999999,"15min":23.074999999999999},"last":0.20799999999999999},{"interface":"front","average":{"1min":2.984,"5min":2.456,"15min":2.3559999999999999},"min":{"1min":0.311,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":23.129000000000001,"5min":23.129000000000001,"15min":23.129000000000001},"last":0.69199999999999995}]},{"osd":3,"last update":"Sat Sep 23 18:37:01 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":1.0569999999999999,"15min":1.0309999999999999},"min":{"1min":0.32500000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.798,"5min":4.766,"15min":4.766},"last":2.1469999999999998},{"interface":"front","average":{"1min":0.77900000000000003,"5min":1.1140000000000001,"15min":1.1679999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.1680000000000001,"5min":4.4119999999999999,"15min":4.4119999999999999},"last":2.1219999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:37:01 2023","interfaces":[{"interface":"back","average":{"1min":1.361,"5min":1.234,"15min":1.2},"min":{"1min":0.45100000000000001,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":5.5510000000000002,"5min":5.5510000000000002,"15min":5.5510000000000002},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.98499999999999999,"5min":1.3620000000000001,"15min":1.462},"min":{"1min":0.35899999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.4109999999999996,"5min":6.3129999999999997,"15min":6.3129999999999997},"last":0.57399999999999995}]},{"osd":5,"last update":"Sat Sep 23 18:37:01 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":0.97999999999999998,"15min":0.80300000000000005},"min":{"1min":0.38,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":6.1200000000000001,"5min":6.1200000000000001,"15min":6.1200000000000001},"last":0.73299999999999998},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.3149999999999999,"15min":1.3999999999999999},"min":{"1min":0.495,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":4.0259999999999998,"5min":6.5250000000000004,"15min":6.5250000000000004},"last":0.41399999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:37:01 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.4810000000000001,"15min":1.5580000000000001},"min":{"1min":0.51700000000000002,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":4.1029999999999998,"5min":6.4859999999999998,"15min":6.4859999999999998},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.115,"5min":1.5329999999999999,"15min":1.7050000000000001},"min":{"1min":0.39100000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":4.3179999999999996,"5min":6.1109999999999998,"15min":6.1109999999999998},"last":0.53900000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:37:01 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":1.6819999999999999,"15min":1.8640000000000001},"min":{"1min":0.46000000000000002,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":5.3319999999999999,"5min":8.1940000000000008,"15min":8.1940000000000008},"last":0.50600000000000001},{"interface":"front","average":{"1min":1.401,"5min":1.675,"15min":1.7729999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":5.3029999999999999,"5min":6.3899999999999997,"15min":6.3899999999999997},"last":0.48899999999999999}]}]},{"osd":0,"up_from":88,"seq":377957122185,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":856004,"kb_used_data":580988,"kb_used_omap":8,"kb_used_meta":274999,"kb_avail":92889148,"statfs":{"total":95995035648,"available":95118487552,"internally_reserved":0,"allocated":594931712,"data_stored":594186858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":281599870},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:37:01 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":1.613,"15min":1.8100000000000001},"min":{"1min":0.27100000000000002,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":4.0030000000000001,"5min":8.1140000000000008,"15min":8.1140000000000008},"last":0.245},{"interface":"front","average":{"1min":0.85499999999999998,"5min":2.0339999999999998,"15min":2.3450000000000002},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":5.7699999999999996,"5min":7.6040000000000001,"15min":7.6040000000000001},"last":0.34699999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:37:48 2023","interfaces":[{"interface":"back","average":{"1min":7.468,"5min":3.7509999999999999,"15min":1.823},"min":{"1min":0.45000000000000001,"5min":0.22600000000000001,"15min":0.192},"max":{"1min":108.73999999999999,"5min":108.73999999999999,"15min":108.73999999999999},"last":108.73999999999999},{"interface":"front","average":{"1min":7.2930000000000001,"5min":3.8410000000000002,"15min":1.7529999999999999},"min":{"1min":0.32200000000000001,"5min":0.20899999999999999,"15min":0.20799999999999999},"max":{"1min":108.136,"5min":108.136,"15min":108.136},"last":108.136}]},{"osd":3,"last update":"Sat Sep 23 18:37:16 2023","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":1.847,"15min":1.228},"min":{"1min":0.32000000000000001,"5min":0.216,"15min":0.216},"max":{"1min":1.9530000000000001,"5min":34.942,"15min":34.942},"last":111.32599999999999},{"interface":"front","average":{"1min":0.92300000000000004,"5min":1.853,"15min":1.2649999999999999},"min":{"1min":0.36799999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":4.0220000000000002,"5min":34.722000000000001,"15min":34.722000000000001},"last":111.346}]},{"osd":4,"last update":"Sat Sep 23 18:36:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":1.546,"15min":1.167},"min":{"1min":0.32700000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.6829999999999998,"5min":6.6879999999999997,"15min":6.6879999999999997},"last":0.63400000000000001},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.9450000000000001,"15min":1.331},"min":{"1min":0.34599999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":4.7069999999999999,"5min":16.486999999999998,"15min":16.486999999999998},"last":0.58499999999999996}]},{"osd":5,"last update":"Sat Sep 23 18:36:59 2023","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":1.617,"15min":1.216},"min":{"1min":0.38600000000000001,"5min":0.371,"15min":0.371},"max":{"1min":3.2610000000000001,"5min":8.0589999999999993,"15min":8.0589999999999993},"last":0.55400000000000005},{"interface":"front","average":{"1min":1.369,"5min":1.621,"15min":1.232},"min":{"1min":0.36799999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.8170000000000002,"5min":8.9250000000000007,"15min":8.9250000000000007},"last":0.77300000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:36:59 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":1.706,"15min":1.2809999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":3.3330000000000002,"5min":8.0329999999999995,"15min":8.0329999999999995},"last":0.53100000000000003},{"interface":"front","average":{"1min":1.3879999999999999,"5min":1.7170000000000001,"15min":1.2609999999999999},"min":{"1min":0.45900000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":4.4569999999999999,"5min":8.9740000000000002,"15min":8.9740000000000002},"last":0.68700000000000006}]},{"osd":7,"last update":"Sat Sep 23 18:36:59 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":1.9099999999999999,"15min":1.3260000000000001},"min":{"1min":0.43099999999999999,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":4.4279999999999999,"5min":9.0600000000000005,"15min":9.0600000000000005},"last":0.81699999999999995},{"interface":"front","average":{"1min":1.204,"5min":1.5840000000000001,"15min":1.238},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":5.7009999999999996,"5min":6.7229999999999999,"15min":6.7229999999999999},"last":0.57399999999999995}]}]},{"osd":2,"up_from":98,"seq":420906795134,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1955952,"kb_used_data":1676840,"kb_used_omap":11,"kb_used_meta":279092,"kb_avail":91789200,"statfs":{"total":95995035648,"available":93992140800,"internally_reserved":0,"allocated":1717084160,"data_stored":1716356847,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11949,"internal_metadata":285790547},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:37:47 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":1.724,"15min":1.0580000000000001},"min":{"1min":0.34799999999999998,"5min":0.192,"15min":0.192},"max":{"1min":1.5129999999999999,"5min":19.431999999999999,"15min":19.431999999999999},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.77900000000000003,"5min":1.6579999999999999,"15min":1.1020000000000001},"min":{"1min":0.28599999999999998,"5min":0.21099999999999999,"15min":0.17699999999999999},"max":{"1min":2.234,"5min":9.423,"15min":9.423},"last":0.42699999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:37:00 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":1.871,"15min":2.3410000000000002},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.7080000000000002,"5min":19.361999999999998,"15min":19.361999999999998},"last":0.72999999999999998},{"interface":"front","average":{"1min":1.417,"5min":2.0790000000000002,"15min":2.3050000000000002},"min":{"1min":0.254,"5min":0.253,"15min":0.253},"max":{"1min":9.468,"5min":9.468,"15min":9.468},"last":0.80100000000000005}]},{"osd":3,"last update":"Sat Sep 23 18:37:24 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":1.4730000000000001,"15min":0.98499999999999999},"min":{"1min":0.317,"5min":0.218,"15min":0.218},"max":{"1min":1.133,"5min":7.7859999999999996,"15min":7.7859999999999996},"last":0.22600000000000001},{"interface":"front","average":{"1min":0.67300000000000004,"5min":1.2869999999999999,"15min":0.89800000000000002},"min":{"1min":0.38500000000000001,"5min":0.249,"15min":0.249},"max":{"1min":1.1839999999999999,"5min":19.407,"15min":19.407},"last":0.318}]},{"osd":4,"last update":"Sat Sep 23 18:37:47 2023","interfaces":[{"interface":"back","average":{"1min":1.3,"5min":1.645,"15min":1.2350000000000001},"min":{"1min":0.41499999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.8490000000000002,"5min":7.8220000000000001,"15min":7.8220000000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.056,"5min":1.7749999999999999,"15min":1.2629999999999999},"min":{"1min":0.624,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":2.8239999999999998,"5min":17.282,"15min":17.282},"last":0.49399999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:37:47 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":1.321,"15min":1.133},"min":{"1min":0.58499999999999996,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.7430000000000001,"5min":17.353999999999999,"15min":17.353999999999999},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.234,"5min":1.6180000000000001,"15min":1.2829999999999999},"min":{"1min":0.56799999999999995,"5min":0.316,"15min":0.316},"max":{"1min":5.3129999999999997,"5min":10.218999999999999,"15min":10.218999999999999},"last":0.78100000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:37:47 2023","interfaces":[{"interface":"back","average":{"1min":1.377,"5min":1.6759999999999999,"15min":1.256},"min":{"1min":0.52900000000000003,"5min":0.372,"15min":0.36799999999999999},"max":{"1min":4.8109999999999999,"5min":7.867,"15min":7.867},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.425,"15min":1.1990000000000001},"min":{"1min":0.64800000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.645,"5min":17.379999999999999,"15min":17.379999999999999},"last":0.57699999999999996}]},{"osd":7,"last update":"Sat Sep 23 18:37:47 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":1.8220000000000001,"15min":1.323},"min":{"1min":0.48199999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.8689999999999998,"5min":7.8460000000000001,"15min":7.8460000000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.1719999999999999,"5min":1.79,"15min":1.405},"min":{"1min":0.73899999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":3.992,"5min":17.326000000000001,"15min":17.326000000000001},"last":0.46100000000000002}]}]},{"osd":3,"up_from":103,"seq":442381631608,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1690080,"kb_used_data":1546136,"kb_used_omap":4,"kb_used_meta":143931,"kb_avail":92055072,"statfs":{"total":95995035648,"available":94264393728,"internally_reserved":0,"allocated":1583243264,"data_stored":1582529090,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4826,"internal_metadata":147385638},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,6,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:37:15 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":1.5609999999999999,"15min":0.98399999999999999},"min":{"1min":0.308,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":2.9900000000000002,"5min":14.231,"15min":14.231},"last":2.8820000000000001},{"interface":"front","average":{"1min":0.58699999999999997,"5min":1.177,"15min":0.873},"min":{"1min":0.34000000000000002,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":1.0109999999999999,"5min":8.4659999999999993,"15min":8.4659999999999993},"last":3.4900000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:37:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":1.018,"15min":1.163},"min":{"1min":0.246,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":1.165,"5min":3.3730000000000002,"15min":3.3730000000000002},"last":1.7210000000000001},{"interface":"front","average":{"1min":0.80800000000000005,"5min":1.173,"15min":1.3129999999999999},"min":{"1min":0.33500000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.0299999999999998,"5min":4.6459999999999999,"15min":4.6459999999999999},"last":2.7570000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:37:15 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":4.1890000000000001,"15min":1.8919999999999999},"min":{"1min":0.253,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":1.137,"5min":187.40000000000001,"15min":187.40000000000001},"last":2.4089999999999998},{"interface":"front","average":{"1min":0.63600000000000001,"5min":3.6629999999999998,"15min":1.742},"min":{"1min":0.316,"5min":0.25,"15min":0.25},"max":{"1min":1.571,"5min":187.71700000000001,"15min":187.71700000000001},"last":1.7709999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:37:15 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":1.1060000000000001,"15min":0.998},"min":{"1min":0.54200000000000004,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.1990000000000001,"5min":8.3689999999999998,"15min":8.3689999999999998},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.97699999999999998,"5min":1.0289999999999999,"15min":1.028},"min":{"1min":0.42799999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.1070000000000002,"5min":5.9429999999999996,"15min":5.9429999999999996},"last":2.8100000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:37:15 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":1.0960000000000001,"15min":1.054},"min":{"1min":0.40300000000000002,"5min":0.35699999999999998,"15min":0.34499999999999997},"max":{"1min":3.0840000000000001,"5min":5.9930000000000003,"15min":5.9930000000000003},"last":2.8340000000000001},{"interface":"front","average":{"1min":0.90200000000000002,"5min":0.98099999999999998,"15min":1.0620000000000001},"min":{"1min":0.442,"5min":0.36299999999999999,"15min":0.36099999999999999},"max":{"1min":1.399,"5min":4.1479999999999997,"15min":4.1479999999999997},"last":2.3860000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:37:15 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":1.0469999999999999,"15min":1.0700000000000001},"min":{"1min":0.41999999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.4119999999999999,"5min":5.3070000000000004,"15min":5.3070000000000004},"last":2.2559999999999998},{"interface":"front","average":{"1min":0.90800000000000003,"5min":1.0549999999999999,"15min":1.05},"min":{"1min":0.52000000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.4690000000000001,"5min":8.4139999999999997,"15min":8.4139999999999997},"last":0.54400000000000004}]},{"osd":7,"last update":"Sat Sep 23 18:37:15 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":1.1899999999999999,"15min":1.1339999999999999},"min":{"1min":0.59799999999999998,"5min":0.42399999999999999,"15min":0.41499999999999998},"max":{"1min":1.397,"5min":8.4459999999999997,"15min":8.4459999999999997},"last":0.57599999999999996},{"interface":"front","average":{"1min":1.038,"5min":1.206,"15min":1.1379999999999999},"min":{"1min":0.53800000000000003,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":3.0590000000000002,"5min":6.0259999999999998,"15min":6.0259999999999998},"last":2.8580000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019218,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1071660,"kb_used_data":770340,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92673492,"statfs":{"total":95995035648,"available":94897655808,"internally_reserved":0,"allocated":788828160,"data_stored":788278949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:36:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7030000000000001,"5min":2.734,"15min":1.5209999999999999},"min":{"1min":0.30299999999999999,"5min":0.253,"15min":0.253},"max":{"1min":7.5179999999999998,"5min":82.623000000000005,"15min":82.623000000000005},"last":0.68300000000000005},{"interface":"front","average":{"1min":1.3089999999999999,"5min":2.3450000000000002,"15min":1.464},"min":{"1min":0.35099999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":7.5869999999999997,"5min":82.638000000000005,"15min":82.638000000000005},"last":0.88900000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:36:56 2023","interfaces":[{"interface":"back","average":{"1min":1.171,"5min":0.98099999999999998,"15min":0.85899999999999999},"min":{"1min":0.33800000000000002,"5min":0.31,"15min":0.31},"max":{"1min":4.6589999999999998,"5min":6.0069999999999997,"15min":6.0069999999999997},"last":1.522},{"interface":"front","average":{"1min":1.5329999999999999,"5min":1.2410000000000001,"15min":1.1539999999999999},"min":{"1min":0.54200000000000004,"5min":0.313,"15min":0.313},"max":{"1min":4.6779999999999999,"5min":6.1429999999999998,"15min":6.1429999999999998},"last":1.544}]},{"osd":2,"last update":"Sat Sep 23 18:36:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4199999999999999,"5min":1.448,"15min":1.1759999999999999},"min":{"1min":0.33200000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.1239999999999997,"5min":12.552,"15min":12.552},"last":1.1140000000000001},{"interface":"front","average":{"1min":1.702,"5min":1.8540000000000001,"15min":1.28},"min":{"1min":0.40200000000000002,"5min":0.309,"15min":0.309},"max":{"1min":5.234,"5min":12.221,"15min":12.221},"last":1.0109999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:37:23 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":1.319,"15min":1.123},"min":{"1min":0.52700000000000002,"5min":0.33800000000000002,"15min":0.32900000000000001},"max":{"1min":1.2090000000000001,"5min":8.2509999999999994,"15min":20.850000000000001},"last":1.385},{"interface":"front","average":{"1min":1.163,"5min":1.524,"15min":1.2729999999999999},"min":{"1min":0.72099999999999997,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":5.1200000000000001,"5min":8.4009999999999998,"15min":20.792999999999999},"last":0.84299999999999997}]},{"osd":5,"last update":"Sat Sep 23 18:37:46 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":1.177,"15min":1.111},"min":{"1min":0.50900000000000001,"5min":0.34100000000000003,"15min":0.29899999999999999},"max":{"1min":1.2729999999999999,"5min":7.7869999999999999,"15min":7.819},"last":1.0720000000000001},{"interface":"front","average":{"1min":0.95199999999999996,"5min":1.1160000000000001,"15min":1.0740000000000001},"min":{"1min":0.60999999999999999,"5min":0.32900000000000001,"15min":0.28399999999999997},"max":{"1min":1.464,"5min":6.4779999999999998,"15min":6.4779999999999998},"last":1.284}]},{"osd":6,"last update":"Sat Sep 23 18:37:13 2023","interfaces":[{"interface":"back","average":{"1min":1.246,"5min":1.2709999999999999,"15min":1.135},"min":{"1min":0.51600000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":6.5369999999999999,"5min":7.8419999999999996,"15min":7.8419999999999996},"last":1.1899999999999999},{"interface":"front","average":{"1min":0.91800000000000004,"5min":1.0209999999999999,"15min":1.0469999999999999},"min":{"1min":0.374,"5min":0.29799999999999999,"15min":0.26100000000000001},"max":{"1min":1.339,"5min":5.9059999999999997,"15min":5.9059999999999997},"last":1.0389999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:36:48 2023","interfaces":[{"interface":"back","average":{"1min":1.216,"5min":1.5780000000000001,"15min":1.268},"min":{"1min":0.45000000000000001,"5min":0.311,"15min":0.311},"max":{"1min":5.7279999999999998,"5min":13.491,"15min":13.491},"last":1.2629999999999999},{"interface":"front","average":{"1min":1.6020000000000001,"5min":1.829,"15min":1.403},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":5.5789999999999997,"5min":17.062999999999999,"15min":17.062999999999999},"last":1.3220000000000001}]}]},{"osd":5,"up_from":0,"seq":154618822986,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4224,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7725,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3429,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":600506368,"data_stored":600506368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":994639872,"data_stored":994639872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1740963840,"data_stored":1740963840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1627586560,"data_stored":1627586560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":798359552,"data_stored":798359552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":944766976,"data_stored":944766976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1093337088,"data_stored":1093337088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:37:57.747 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 11 2023-09-23T18:37:57.751 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 28 16 47457 47441 105.885 84.5625 0.0105465 0.00943737 2023-09-23T18:37:57.751 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:57.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:57.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:57.856 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:57.882 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:37:57.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:57.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:37:57.987 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:58.011 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:58.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:58.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:58.114 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:58.147 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:37:58.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:58.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:37:58.194 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:37:58.253 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:58.278 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:58.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:58.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:58.382 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:58.410 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:37:58.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:58.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:37:58.515 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:58.539 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:58.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:58.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:58.642 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:58.676 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:58.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:58.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:58.751 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 29 16 48641 48625 104.785 74 0.00742182 0.00953767 2023-09-23T18:37:58.782 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:58.810 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:58.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:58.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:58.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:58.939 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:37:58.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:58.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:37:59.044 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:59.068 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:59.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:59.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:59.172 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:59.180 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:37:58 smithi173 bash[59422]: cluster 2023-09-23T18:37:57.839358+0000 mgr.y (mgr.26054) 584 : cluster 0 pgmap v564: 89 pgs: 89 active+clean; 2.8 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-09-23T18:37:59.196 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:37:59.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:59.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:37:59.300 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:59.331 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:59.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:59.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:59.435 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:59.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:37:58 smithi098 bash[76817]: cluster 2023-09-23T18:37:57.839358+0000 mgr.y (mgr.26054) 584 : cluster 0 pgmap v564: 89 pgs: 89 active+clean; 2.8 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-09-23T18:37:59.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:37:58 smithi098 bash[79516]: cluster 2023-09-23T18:37:57.839358+0000 mgr.y (mgr.26054) 584 : cluster 0 pgmap v564: 89 pgs: 89 active+clean; 2.8 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-09-23T18:37:59.465 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:37:59.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:59.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:37:59.568 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:59.600 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:59.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:59.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:59.708 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:59.733 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:37:59.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:59.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:37:59.751 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 30 15 49615 49600 103.323 60.9375 0.0189719 0.00967153 2023-09-23T18:37:59.838 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:59.869 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:37:59.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:37:59.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:37:59.976 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:00.003 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:00.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:00.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:00.107 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:00.130 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:00.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:00.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:00.235 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:00.270 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:00.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:00.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:00.375 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:00.406 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:00.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:00.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:00.509 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:00.538 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:00.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:00.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:00.643 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:00.669 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:00.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:00.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:00.751 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 31 16 50755 50739 102.287 71.1875 0.0146074 0.0097707 2023-09-23T18:38:00.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:00.809 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:00.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:00.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:00.912 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:00.938 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:00.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:00.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:01.042 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:01.059 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:01.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:01.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:01.162 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:01.188 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:01.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:01.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:01.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:00 smithi173 bash[59422]: cluster 2023-09-23T18:37:59.839897+0000 mgr.y (mgr.26054) 585 : cluster 0 pgmap v565: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T18:38:01.291 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:01.312 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:00 smithi098 bash[76817]: cluster 2023-09-23T18:37:59.839897+0000 mgr.y (mgr.26054) 585 : cluster 0 pgmap v565: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T18:38:01.313 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:00 smithi098 bash[79516]: cluster 2023-09-23T18:37:59.839897+0000 mgr.y (mgr.26054) 585 : cluster 0 pgmap v565: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T18:38:01.319 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:01.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:01.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:01.424 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:01.450 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:01.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:01.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:01.553 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:01.579 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:01.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:01.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:01.684 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:01.713 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:01.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:01.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:01.751 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 32 16 51892 51876 101.311 71.0625 0.0162559 0.00986517 2023-09-23T18:38:01.816 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:01.844 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:01.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:01.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:01.947 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:01.977 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:01.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:01.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:02.081 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:02.111 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:02.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:02.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:02.217 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:02.244 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:02.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:02.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:02.348 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:02.378 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:02.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:02.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:02.481 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:02.502 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:02.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:02.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:02.606 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:02.632 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:02.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:02.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:02.735 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:02.751 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 33 16 53078 53062 100.486 74.125 0.0207184 0.00994462 2023-09-23T18:38:02.761 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:02.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:02.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:02.866 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:02.892 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:02.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:02.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:02.996 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:03.004 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:02 smithi098 bash[76817]: cluster 2023-09-23T18:38:01.840456+0000 mgr.y (mgr.26054) 586 : cluster 0 pgmap v566: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T18:38:03.022 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:03.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:03.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:03.127 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:03.137 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:02 smithi173 bash[59422]: cluster 2023-09-23T18:38:01.840456+0000 mgr.y (mgr.26054) 586 : cluster 0 pgmap v566: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T18:38:03.151 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:03.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:03.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:03.255 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:03.263 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:03 smithi098 bash[79516]: cluster 2023-09-23T18:38:01.840456+0000 mgr.y (mgr.26054) 586 : cluster 0 pgmap v566: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T18:38:03.280 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:03.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:03.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:03.385 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:03.414 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:03.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:03.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:03.520 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:03.549 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:03.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:03.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:03.654 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:03.678 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:03.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:03.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:03.752 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 34 16 54270 54254 99.7216 74.5 0.00838739 0.0100224 2023-09-23T18:38:03.784 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:03.808 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:03.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:03.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:03.911 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:03.939 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:03.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:03.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:04.045 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:04.075 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:04.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:04.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:04.180 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:04.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:04 smithi098 bash[76817]: cluster 2023-09-23T18:38:03.841031+0000 mgr.y (mgr.26054) 587 : cluster 0 pgmap v567: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T18:38:04.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:04 smithi098 bash[79516]: cluster 2023-09-23T18:38:03.841031+0000 mgr.y (mgr.26054) 587 : cluster 0 pgmap v567: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T18:38:04.209 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:04.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:04.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:04.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:04 smithi173 bash[59422]: cluster 2023-09-23T18:38:03.841031+0000 mgr.y (mgr.26054) 587 : cluster 0 pgmap v567: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T18:38:04.313 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:04.338 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:04.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:04.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:04.443 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:04.477 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:38:04.482 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:04.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:04.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:04.514 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:38:04.551 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:38:04.582 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:38:04.587 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:04.615 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:04.616 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:38:04.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:04.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:04.644 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:38:04.684 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:38:04.718 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:38:04.720 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:04.724 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:38:04.724 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:38:04.724 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:38:04.724 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:38:04.724 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:38:04.724 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:38:04.724 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:38:04.724 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:38:04.743 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:04.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:04.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:04.751 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 35 16 55386 55370 98.8651 69.75 0.0130962 0.0101087 2023-09-23T18:38:04.847 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:04.877 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:04.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:04.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:04.982 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:05.016 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:05.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:05.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:05.123 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:05.131 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:05 smithi173 bash[59422]: audit 2023-09-23T18:38:04.439358+0000 mon.c (mon.2) 77 : audit 0 from='client.? 172.21.15.98:0/2626021771' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:38:05.145 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:05.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:05.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:05.248 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:05.281 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:05.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:05.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:05.297 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:05 smithi098 bash[76817]: audit 2023-09-23T18:38:04.439358+0000 mon.c (mon.2) 77 : audit 0 from='client.? 172.21.15.98:0/2626021771' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:38:05.298 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:05 smithi098 bash[79516]: audit 2023-09-23T18:38:04.439358+0000 mon.c (mon.2) 77 : audit 0 from='client.? 172.21.15.98:0/2626021771' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:38:05.387 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:05.414 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:05.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:05.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:05.518 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:05.550 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:05.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:05.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:05.655 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:05.690 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:05.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:05.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:05.752 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 36 16 56487 56471 98.0302 68.8125 0.0242487 0.0101947 2023-09-23T18:38:05.794 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:05.822 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:05.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:05.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:05.925 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:05.956 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:05.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:05.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:06.060 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:06.092 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:06.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:06.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:06.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:06 smithi098 bash[76817]: cluster 2023-09-23T18:38:05.841683+0000 mgr.y (mgr.26054) 588 : cluster 0 pgmap v568: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T18:38:06.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:06 smithi098 bash[79516]: cluster 2023-09-23T18:38:05.841683+0000 mgr.y (mgr.26054) 588 : cluster 0 pgmap v568: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T18:38:06.191 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:38:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:38:06] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.33.4" 2023-09-23T18:38:06.197 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:06.206 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:06 smithi173 bash[59422]: cluster 2023-09-23T18:38:05.841683+0000 mgr.y (mgr.26054) 588 : cluster 0 pgmap v568: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T18:38:06.223 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:06.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:06.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:06.326 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:06.353 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:06.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:06.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:06.456 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:06.482 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:06.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:06.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:06.585 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:06.617 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:06.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:06.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:06.722 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:06.752 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 37 16 57659 57643 97.3603 73.25 0.00612627 0.0102651 2023-09-23T18:38:06.754 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:06.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:06.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:06.857 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:06.880 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:06.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:06.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:06.983 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:07.008 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:07.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:07.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:07.110 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:07.137 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:07.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:07.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:07.243 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:07.270 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:07.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:07.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:07.341 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:38:07.342 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:38:07.373 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:07.417 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:07.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:07.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:07.520 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:07.563 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:07.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:07.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:07.667 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:07.695 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:07.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:07.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:07.752 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 38 16 58758 58742 96.6056 68.6875 0.00467383 0.0103457 2023-09-23T18:38:07.797 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:07.815 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:07.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:07.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:07.918 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:07.942 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:07.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:07.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:08.044 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:08.071 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:08.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:08.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:08.175 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:08.204 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:08.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:08.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:08.307 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:08.333 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:08.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:08.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:08.436 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:08.468 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:08.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:08.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:08.572 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:08.599 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:08.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:08.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:08.704 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:08.728 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:08.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:08.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:08.752 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 39 16 59675 59659 95.5979 57.3125 0.0126191 0.0104543 2023-09-23T18:38:08.836 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:08.861 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:08.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:08.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:08.968 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:08.976 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:08 smithi098 bash[76817]: cluster 2023-09-23T18:38:07.842277+0000 mgr.y (mgr.26054) 589 : cluster 0 pgmap v569: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T18:38:08.976 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:08 smithi098 bash[79516]: cluster 2023-09-23T18:38:07.842277+0000 mgr.y (mgr.26054) 589 : cluster 0 pgmap v569: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T18:38:08.996 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:09.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:09.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:09.102 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:09.129 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:09.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:09.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:09.236 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:09.261 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:09.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:09.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:09.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:08 smithi173 bash[59422]: cluster 2023-09-23T18:38:07.842277+0000 mgr.y (mgr.26054) 589 : cluster 0 pgmap v569: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T18:38:09.365 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:09.393 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:09.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:09.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:09.414 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:38:09.496 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:09.524 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:09.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:09.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:09.627 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:09.655 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:09.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:09.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:09.753 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:2023-09-23T18:38:09.753266+0000 min lat: 0.00144566 max lat: 0.21301 avg lat: 0.0105165 2023-09-23T18:38:09.754 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:38:09.754 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 40 16 60841 60825 95.0268 72.875 0.00934513 0.0105165 2023-09-23T18:38:09.763 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:09.798 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:09.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:09.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:09.902 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:09.919 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:09.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:09.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:10.021 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:10.038 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:09 smithi098 bash[76817]: audit 2023-09-23T18:38:09.594229+0000 mon.a (mon.0) 836 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:38:10.058 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:10.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:10.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:10.162 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:10.195 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:10.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:10.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:10.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:09 smithi173 bash[59422]: audit 2023-09-23T18:38:09.594229+0000 mon.a (mon.0) 836 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:38:10.301 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:10.310 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:10 smithi098 bash[79516]: audit 2023-09-23T18:38:09.594229+0000 mon.a (mon.0) 836 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:38:10.333 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:10.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:10.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:10.438 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:10.468 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:10.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:10.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:10.574 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:10.612 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:10.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:10.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:10.715 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:10.740 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:10.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:10.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:10.753 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 41 16 61702 61686 94.0214 53.8125 0.0136096 0.0106304 2023-09-23T18:38:10.844 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:10.880 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:10.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:10.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:10.968 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is unset 2023-09-23T18:38:10.985 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:11.017 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:11.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:11.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:11.121 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:11.131 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:10 smithi173 bash[59422]: cluster 2023-09-23T18:38:09.842715+0000 mgr.y (mgr.26054) 590 : cluster 0 pgmap v570: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T18:38:11.131 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:10 smithi173 bash[59422]: audit 2023-09-23T18:38:10.648067+0000 mon.c (mon.2) 78 : audit 1 from='client.? 172.21.15.98:0/2864880386' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:38:11.131 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:10 smithi173 bash[59422]: audit 2023-09-23T18:38:10.650184+0000 mon.a (mon.0) 837 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:38:11.148 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:11.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:11.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:11.247 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:10 smithi098 bash[76817]: cluster 2023-09-23T18:38:09.842715+0000 mgr.y (mgr.26054) 590 : cluster 0 pgmap v570: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T18:38:11.247 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:10 smithi098 bash[76817]: audit 2023-09-23T18:38:10.648067+0000 mon.c (mon.2) 78 : audit 1 from='client.? 172.21.15.98:0/2864880386' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:38:11.247 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:10 smithi098 bash[76817]: audit 2023-09-23T18:38:10.650184+0000 mon.a (mon.0) 837 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:38:11.248 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:10 smithi098 bash[79516]: cluster 2023-09-23T18:38:09.842715+0000 mgr.y (mgr.26054) 590 : cluster 0 pgmap v570: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T18:38:11.248 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:10 smithi098 bash[79516]: audit 2023-09-23T18:38:10.648067+0000 mon.c (mon.2) 78 : audit 1 from='client.? 172.21.15.98:0/2864880386' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:38:11.248 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:10 smithi098 bash[79516]: audit 2023-09-23T18:38:10.650184+0000 mon.a (mon.0) 837 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:38:11.251 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:11.278 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:11.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:11.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:11.382 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:11.409 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:11.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:11.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:11.516 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:11.544 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:11.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:11.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:11.650 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:11.678 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:11.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:11.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:11.754 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 42 16 62760 62744 93.3571 66.125 0.0135079 0.0107055 2023-09-23T18:38:11.782 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:11.808 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:11.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:11.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:11.914 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:11.939 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:11.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:11.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:12.044 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:12.053 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:11 smithi098 bash[79516]: cluster 2023-09-23T18:38:10.940598+0000 mon.a (mon.0) 838 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:38:12.054 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:11 smithi098 bash[79516]: cluster 2023-09-23T18:38:10.960575+0000 osd.7 (osd.7) 68 : cluster 3 failed to encode map e157 with expected crc 2023-09-23T18:38:12.054 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:11 smithi098 bash[79516]: audit 2023-09-23T18:38:10.962430+0000 mon.a (mon.0) 839 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:38:12.054 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:11 smithi098 bash[79516]: cluster 2023-09-23T18:38:10.962586+0000 mon.a (mon.0) 840 : cluster 0 osdmap e157: 8 total, 8 up, 7 in 2023-09-23T18:38:12.054 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:11 smithi098 bash[79516]: cluster 2023-09-23T18:38:10.965041+0000 osd.6 (osd.6) 70 : cluster 3 failed to encode map e157 with expected crc 2023-09-23T18:38:12.054 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:11 smithi098 bash[79516]: cluster 2023-09-23T18:38:10.967222+0000 osd.4 (osd.4) 93 : cluster 3 failed to encode map e157 with expected crc 2023-09-23T18:38:12.054 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:11 smithi098 bash[76817]: cluster 2023-09-23T18:38:10.940598+0000 mon.a (mon.0) 838 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:38:12.054 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:11 smithi098 bash[76817]: cluster 2023-09-23T18:38:10.960575+0000 osd.7 (osd.7) 68 : cluster 3 failed to encode map e157 with expected crc 2023-09-23T18:38:12.054 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:11 smithi098 bash[76817]: audit 2023-09-23T18:38:10.962430+0000 mon.a (mon.0) 839 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:38:12.055 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:11 smithi098 bash[76817]: cluster 2023-09-23T18:38:10.962586+0000 mon.a (mon.0) 840 : cluster 0 osdmap e157: 8 total, 8 up, 7 in 2023-09-23T18:38:12.055 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:11 smithi098 bash[76817]: cluster 2023-09-23T18:38:10.965041+0000 osd.6 (osd.6) 70 : cluster 3 failed to encode map e157 with expected crc 2023-09-23T18:38:12.055 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:11 smithi098 bash[76817]: cluster 2023-09-23T18:38:10.967222+0000 osd.4 (osd.4) 93 : cluster 3 failed to encode map e157 with expected crc 2023-09-23T18:38:12.077 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:12.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:12.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:12.182 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:12.213 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:12.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:12.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:12.217 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:11 smithi173 bash[59422]: cluster 2023-09-23T18:38:10.940598+0000 mon.a (mon.0) 838 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:38:12.217 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:11 smithi173 bash[59422]: cluster 2023-09-23T18:38:10.960575+0000 osd.7 (osd.7) 68 : cluster 3 failed to encode map e157 with expected crc 2023-09-23T18:38:12.217 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:11 smithi173 bash[59422]: audit 2023-09-23T18:38:10.962430+0000 mon.a (mon.0) 839 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:38:12.218 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:11 smithi173 bash[59422]: cluster 2023-09-23T18:38:10.962586+0000 mon.a (mon.0) 840 : cluster 0 osdmap e157: 8 total, 8 up, 7 in 2023-09-23T18:38:12.218 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:11 smithi173 bash[59422]: cluster 2023-09-23T18:38:10.965041+0000 osd.6 (osd.6) 70 : cluster 3 failed to encode map e157 with expected crc 2023-09-23T18:38:12.218 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:11 smithi173 bash[59422]: cluster 2023-09-23T18:38:10.967222+0000 osd.4 (osd.4) 93 : cluster 3 failed to encode map e157 with expected crc 2023-09-23T18:38:12.317 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:12.342 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:12.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:12.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:12.447 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:12.478 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:12.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:12.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:12.582 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:12.617 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:12.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:12.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:12.721 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:12.747 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:12.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:12.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:12.753 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 43 16 63888 63872 92.8254 70.5 0.012155 0.0107665 2023-09-23T18:38:12.850 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:12.877 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:12.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:12.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:12.963 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:12 smithi173 bash[59422]: cluster 2023-09-23T18:38:11.843506+0000 mgr.y (mgr.26054) 591 : cluster 0 pgmap v572: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-09-23T18:38:12.981 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:13.005 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:13.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:13.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:13.109 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:13.135 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:13.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:13.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:13.238 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:13.256 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:13.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:13.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:13.261 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:12 smithi098 bash[76817]: cluster 2023-09-23T18:38:11.843506+0000 mgr.y (mgr.26054) 591 : cluster 0 pgmap v572: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-09-23T18:38:13.261 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:12 smithi098 bash[79516]: cluster 2023-09-23T18:38:11.843506+0000 mgr.y (mgr.26054) 591 : cluster 0 pgmap v572: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-09-23T18:38:13.359 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:13.381 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:13.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:13.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:13.487 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:13.520 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:13.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:13.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:13.628 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:13.651 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:13.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:13.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:13.754 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 44 16 64916 64900 92.1758 64.25 0.0203393 0.0108433 2023-09-23T18:38:13.754 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:13.781 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:13.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:13.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:13.884 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:13.907 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:13.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:13.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:14.011 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:14.037 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:14.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:14.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:14.143 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:14.179 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:14.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:14.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:14.286 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:14.313 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:14.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:14.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:14.416 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:14.447 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:14.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:14.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:14.555 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:14.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:14.584 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:14.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:14.685 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:14.714 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:14.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:14.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:14.754 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 45 16 66033 66017 91.6787 69.8125 0.00533764 0.010902 2023-09-23T18:38:14.818 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:14.844 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:14.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:14.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:14.947 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:14.991 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:14.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:14.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:15.098 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:15.125 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:15.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:15.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:15.227 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:15.236 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:14 smithi173 bash[59422]: cluster 2023-09-23T18:38:13.844010+0000 mgr.y (mgr.26054) 592 : cluster 0 pgmap v573: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T18:38:15.255 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:15.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:15.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:15.358 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:15.367 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:14 smithi098 bash[76817]: cluster 2023-09-23T18:38:13.844010+0000 mgr.y (mgr.26054) 592 : cluster 0 pgmap v573: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T18:38:15.367 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:14 smithi098 bash[79516]: cluster 2023-09-23T18:38:13.844010+0000 mgr.y (mgr.26054) 592 : cluster 0 pgmap v573: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T18:38:15.386 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:15.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:15.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:15.433 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:38:15.490 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:15.528 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:15.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:15.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:15.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:15.664 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:15.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:15.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:15.754 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 46 16 66911 66895 90.8786 54.875 0.0119804 0.0109979 2023-09-23T18:38:15.767 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:15.798 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:15.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:15.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:15.901 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:15.937 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:15.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:15.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:16.042 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:16.066 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:16.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:16.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:16.170 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:16.196 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:16.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:16.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:16.300 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:16.325 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:16.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:16.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:16.431 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:16.440 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:38:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:38:16] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.33.4" 2023-09-23T18:38:16.457 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:16.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:16.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:16.560 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:16.587 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:16.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:16.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:16.691 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:16.720 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:16.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:16.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:16.754 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 47 16 67983 67967 90.3704 67 0.0167004 0.0110602 2023-09-23T18:38:16.823 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:16.853 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:16.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:16.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:16.958 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:16.989 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:16.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:16.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:17.048 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:17 smithi098 bash[79516]: cluster 2023-09-23T18:38:15.844602+0000 mgr.y (mgr.26054) 593 : cluster 0 pgmap v574: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-09-23T18:38:17.048 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:17 smithi098 bash[76817]: cluster 2023-09-23T18:38:15.844602+0000 mgr.y (mgr.26054) 593 : cluster 0 pgmap v574: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-09-23T18:38:17.093 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:17.122 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:17.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:17.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:17.226 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:17.254 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:17.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:17.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:17.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:16 smithi173 bash[59422]: cluster 2023-09-23T18:38:15.844602+0000 mgr.y (mgr.26054) 593 : cluster 0 pgmap v574: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-09-23T18:38:17.361 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:17.384 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:17.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:17.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:17.487 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:17.510 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:17.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:17.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:17.614 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:17.644 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:17.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:17.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:17.750 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:17.754 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 48 16 69068 69052 89.9003 67.8125 0.0115014 0.0111163 2023-09-23T18:38:17.788 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:17.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:17.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:17.892 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:17.915 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:17.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:17.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:18.018 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:18.022 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:38:18.062 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:18.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:18.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:18.167 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:18.191 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:18.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:18.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:18.298 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:18.325 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:18.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:18.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:18.430 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:18.455 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:18.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:18.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:18.559 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:18.575 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:18.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:18.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:18.678 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:18.703 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:18.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:18.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:18.754 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 49 16 69947 69931 89.1866 54.9375 0.0140132 0.0111947 2023-09-23T18:38:18.807 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:18.830 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:18.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:18.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:18.933 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:18.975 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:18.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:18.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:19.077 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:19.122 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:19.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:19.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:19.227 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:19.234 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:18 smithi173 bash[59422]: cluster 2023-09-23T18:38:17.845426+0000 mgr.y (mgr.26054) 594 : cluster 0 pgmap v575: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T18:38:19.254 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:19.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:19.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:19.357 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:19.382 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:19.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:19.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:19.412 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:38:19.412 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:38:19.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:18 smithi098 bash[76817]: cluster 2023-09-23T18:38:17.845426+0000 mgr.y (mgr.26054) 594 : cluster 0 pgmap v575: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T18:38:19.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:18 smithi098 bash[79516]: cluster 2023-09-23T18:38:17.845426+0000 mgr.y (mgr.26054) 594 : cluster 0 pgmap v575: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T18:38:19.485 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:19.519 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:19.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:19.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:19.623 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:19.648 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:19.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:19.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:19.752 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:19.755 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 50 16 70936 70920 88.6391 61.8125 0.00375744 0.0112758 2023-09-23T18:38:19.773 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:19.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:19.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:19.876 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:19.905 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:19.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:19.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:20.010 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:20.032 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:20.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:20.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:20.052 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:38:20.137 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:20.162 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:20.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:20.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:20.265 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:20.300 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:20.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:20.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:20.403 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:20.431 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:20.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:20.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:20.538 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:20.575 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:20.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:20.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:20.680 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:20.707 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:20.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:20.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:20.755 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 51 16 71642 71626 87.7662 44.125 0.0118153 0.0113884 2023-09-23T18:38:20.812 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:20.841 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:20.841 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-09-23T18:38:20.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:20.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:20.946 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:20.981 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:20.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:20.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:21.084 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:21.110 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:21.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:21.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:21.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:20 smithi098 bash[79516]: cluster 2023-09-23T18:38:19.846211+0000 mgr.y (mgr.26054) 595 : cluster 0 pgmap v576: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T18:38:21.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:20 smithi098 bash[79516]: audit 2023-09-23T18:38:20.836118+0000 mon.a (mon.0) 841 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T18:38:21.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:20 smithi098 bash[79516]: audit 2023-09-23T18:38:20.836350+0000 mon.b (mon.1) 12 : audit 1 from='client.? 172.21.15.98:0/3297157677' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T18:38:21.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:20 smithi098 bash[76817]: cluster 2023-09-23T18:38:19.846211+0000 mgr.y (mgr.26054) 595 : cluster 0 pgmap v576: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T18:38:21.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:20 smithi098 bash[76817]: audit 2023-09-23T18:38:20.836118+0000 mon.a (mon.0) 841 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T18:38:21.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:20 smithi098 bash[76817]: audit 2023-09-23T18:38:20.836350+0000 mon.b (mon.1) 12 : audit 1 from='client.? 172.21.15.98:0/3297157677' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T18:38:21.215 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:21.223 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:20 smithi173 bash[59422]: cluster 2023-09-23T18:38:19.846211+0000 mgr.y (mgr.26054) 595 : cluster 0 pgmap v576: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T18:38:21.223 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:20 smithi173 bash[59422]: audit 2023-09-23T18:38:20.836118+0000 mon.a (mon.0) 841 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T18:38:21.224 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:20 smithi173 bash[59422]: audit 2023-09-23T18:38:20.836350+0000 mon.b (mon.1) 12 : audit 1 from='client.? 172.21.15.98:0/3297157677' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T18:38:21.239 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:21.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:21.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:21.347 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:21.376 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:21.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:21.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:21.479 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:21.501 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:21.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:21.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:21.607 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:21.635 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:21.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:21.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:21.737 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:21.754 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 52 16 72666 72650 87.309 64 0.0134168 0.0114481 2023-09-23T18:38:21.763 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:21.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:21.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:21.867 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:21.895 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:21.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:21.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:21.999 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:22.031 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:22.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:22.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:22.134 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:22.164 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:22.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:22.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:22.267 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:22.291 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:22.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:22.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:22.395 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:22.420 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:22.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:22.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:22.526 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:22.555 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:22.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:22.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:22.660 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:22.697 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:22.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:22.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:22.754 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 53 16 73505 73489 86.651 52.4375 0.0113506 0.0115354 2023-09-23T18:38:22.800 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:22.822 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:22.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:22.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:22.927 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:22.969 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:22.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:22.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:23.075 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:23.112 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:23.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:23.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:23.206 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T18:38:23.217 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:23.257 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:22 smithi098 bash[76817]: cluster 2023-09-23T18:38:21.846911+0000 mgr.y (mgr.26054) 596 : cluster 0 pgmap v577: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T18:38:23.258 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:22 smithi098 bash[79516]: cluster 2023-09-23T18:38:21.846911+0000 mgr.y (mgr.26054) 596 : cluster 0 pgmap v577: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T18:38:23.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:22 smithi173 bash[59422]: cluster 2023-09-23T18:38:21.846911+0000 mgr.y (mgr.26054) 596 : cluster 0 pgmap v577: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T18:38:23.291 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:23.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:23.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:23.395 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:23.426 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:23.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:23.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:23.531 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:23.572 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:23.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:23.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:23.674 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:23.704 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:23.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:23.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:23.755 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 54 16 74409 74393 86.0926 56.5 0.0158885 0.0116096 2023-09-23T18:38:23.812 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:23.848 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:23.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:23.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:23.951 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:23.975 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:23.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:23.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:24.084 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:24.110 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:24.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:24.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:24.213 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:24.223 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:24 smithi173 bash[59422]: cluster 2023-09-23T18:38:23.848242+0000 mgr.y (mgr.26054) 597 : cluster 0 pgmap v578: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 58 MiB/s wr, 929 op/s 2023-09-23T18:38:24.240 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:24.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:24.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:24.326 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:24 smithi098 bash[76817]: cluster 2023-09-23T18:38:23.848242+0000 mgr.y (mgr.26054) 597 : cluster 0 pgmap v578: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 58 MiB/s wr, 929 op/s 2023-09-23T18:38:24.326 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:24 smithi098 bash[79516]: cluster 2023-09-23T18:38:23.848242+0000 mgr.y (mgr.26054) 597 : cluster 0 pgmap v578: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 58 MiB/s wr, 929 op/s 2023-09-23T18:38:24.327 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-09-23T18:38:24.344 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:24.367 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:24.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:24.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:24.471 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:24.502 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:24.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:24.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:24.605 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:24.629 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:24.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:24.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:24.732 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:24.755 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 55 16 75422 75406 85.6783 63.3125 0.0165739 0.011666 2023-09-23T18:38:24.760 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:24.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:24.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:24.865 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:24.894 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:24.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:24.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:25.000 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:25.031 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:25.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:25.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:25.042 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:25 smithi098 bash[76817]: audit 2023-09-23T18:38:24.595490+0000 mon.a (mon.0) 842 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:38:25.136 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:25.167 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:25.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:25.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:25.272 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:25.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:25 smithi173 bash[59422]: audit 2023-09-23T18:38:24.595490+0000 mon.a (mon.0) 842 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:38:25.297 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:25.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:25.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:25.400 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:25.409 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:25 smithi098 bash[79516]: audit 2023-09-23T18:38:24.595490+0000 mon.a (mon.0) 842 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:38:25.433 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:25.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:25.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:25.537 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:25.562 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:25.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:25.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:25.666 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:25.694 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:25.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:25.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:25.755 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 56 16 76425 76409 85.2677 62.6875 0.0106201 0.0117224 2023-09-23T18:38:25.799 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:25.821 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:25.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:25.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:25.924 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:25.953 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:25.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:25.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:26.057 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:26.081 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:26.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:26.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:26.185 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:26 smithi098 bash[79516]: cluster 2023-09-23T18:38:25.849071+0000 mgr.y (mgr.26054) 598 : cluster 0 pgmap v579: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 61 MiB/s wr, 973 op/s 2023-09-23T18:38:26.185 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:38:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:38:26] "GET /metrics HTTP/1.1" 200 36516 "" "Prometheus/2.33.4" 2023-09-23T18:38:26.186 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:26 smithi098 bash[76817]: cluster 2023-09-23T18:38:25.849071+0000 mgr.y (mgr.26054) 598 : cluster 0 pgmap v579: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 61 MiB/s wr, 973 op/s 2023-09-23T18:38:26.186 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:26.210 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:26.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:26.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:26.314 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:26.323 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:26 smithi173 bash[59422]: cluster 2023-09-23T18:38:25.849071+0000 mgr.y (mgr.26054) 598 : cluster 0 pgmap v579: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 61 MiB/s wr, 973 op/s 2023-09-23T18:38:26.339 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:26.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:26.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:26.443 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:26.471 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:26.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:26.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:26.574 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:26.600 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:26.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:26.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:26.704 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:26.740 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:26.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:26.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:26.755 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 57 16 77339 77323 84.7738 57.125 0.0334464 0.0117903 2023-09-23T18:38:26.846 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:26.872 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:26.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:26.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:26.975 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:27.000 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:27.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:27.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:27.104 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:27.128 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:27.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:27.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:27.234 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:27.260 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:27.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:27.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:27.367 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:27.392 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:27.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:27.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:27.495 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:27.526 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:27.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:27.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:27.631 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:27.665 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:27.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:27.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:27.755 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 58 16 78346 78330 84.3972 62.9375 0.0284909 0.0118426 2023-09-23T18:38:27.768 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:27.792 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:27.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:27.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:27.900 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:27.941 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:27.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:27.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:28.045 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:28.068 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:28.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:28.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:28.171 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:28.205 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:28.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:28.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:28.307 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:28.342 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:28.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:28.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:28.447 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:28.475 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:28.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:28.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:28.580 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:28.607 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:28.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:28.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:28.710 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:28.745 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:28.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:28.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:28.755 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 59 16 79187 79171 83.8576 52.5625 0.0171725 0.0119194 2023-09-23T18:38:28.852 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:28.874 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:28.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:28.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:28.976 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:29.010 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:29.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:29.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:29.115 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:29.125 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:28 smithi173 bash[59422]: cluster 2023-09-23T18:38:27.849672+0000 mgr.y (mgr.26054) 599 : cluster 0 pgmap v580: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T18:38:29.142 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:29.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:29.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:29.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:28 smithi098 bash[76817]: cluster 2023-09-23T18:38:27.849672+0000 mgr.y (mgr.26054) 599 : cluster 0 pgmap v580: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T18:38:29.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:28 smithi098 bash[79516]: cluster 2023-09-23T18:38:27.849672+0000 mgr.y (mgr.26054) 599 : cluster 0 pgmap v580: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T18:38:29.246 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:29.272 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:29.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:29.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:29.328 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T18:38:29.328 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:38:29.329 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-09-23T18:38:29.329 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:38:29.374 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:29.411 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:29.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:29.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:29.519 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:29.543 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:29.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:29.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:29.646 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:29.686 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:29.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:29.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:29.735 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:38:29.755 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:2023-09-23T18:38:29.755204+0000 min lat: 0.00144566 max lat: 0.223301 avg lat: 0.0119994 2023-09-23T18:38:29.755 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:38:29.755 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 60 16 79983 79967 83.2891 49.75 0.0278982 0.0119994 2023-09-23T18:38:29.786 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:38:29.790 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:29.830 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:29.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:29.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:29.842 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:38:29.899 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:38:29.920 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:29 smithi098 bash[76817]: audit 2023-09-23T18:38:28.909718+0000 mon.a (mon.0) 843 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:38:29.936 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:29.942 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:38:29.963 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:29.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:29.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:29.990 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:38:30.026 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:38:30.066 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:38:30.067 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:30.072 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:38:30.072 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:38:30.072 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:38:30.072 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:38:30.073 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:38:30.073 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:38:30.073 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:38:30.073 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:38:30.092 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:30.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:30.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:30.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:29 smithi098 bash[79516]: audit 2023-09-23T18:38:28.909718+0000 mon.a (mon.0) 843 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:38:30.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:29 smithi098 bash[79516]: audit 2023-09-23T18:38:29.676844+0000 mon.a (mon.0) 844 : audit 0 from='client.? 172.21.15.98:0/229298986' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:38:30.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:29 smithi098 bash[76817]: audit 2023-09-23T18:38:29.676844+0000 mon.a (mon.0) 844 : audit 0 from='client.? 172.21.15.98:0/229298986' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:38:30.195 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:30.202 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:29 smithi173 bash[59422]: audit 2023-09-23T18:38:28.909718+0000 mon.a (mon.0) 843 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:38:30.202 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:29 smithi173 bash[59422]: audit 2023-09-23T18:38:29.676844+0000 mon.a (mon.0) 844 : audit 0 from='client.? 172.21.15.98:0/229298986' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:38:30.221 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:30.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:30.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:30.324 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:30.350 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:30.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:30.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:30.456 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:30.483 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:30.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:30.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:30.588 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:30.611 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:30.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:30.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:30.717 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:30.742 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:30.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:30.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:30.755 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 61 16 80846 80830 82.8078 53.9375 0.0128337 0.0120704 2023-09-23T18:38:30.847 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:30.873 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:30.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:30.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:30.979 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:30.980 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is unset 2023-09-23T18:38:30.990 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:30 smithi173 bash[59422]: cluster 2023-09-23T18:38:29.850732+0000 mgr.y (mgr.26054) 600 : cluster 0 pgmap v581: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 54 MiB/s wr, 858 op/s 2023-09-23T18:38:30.990 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:30 smithi173 bash[59422]: audit 2023-09-23T18:38:30.297691+0000 mon.a (mon.0) 845 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:38:30.990 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:30 smithi173 bash[59422]: audit 2023-09-23T18:38:30.297828+0000 mon.b (mon.1) 13 : audit 1 from='client.? 172.21.15.98:0/1835136037' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:38:31.004 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:31.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:31.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:31.110 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:31.119 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:31 smithi098 bash[76817]: cluster 2023-09-23T18:38:29.850732+0000 mgr.y (mgr.26054) 600 : cluster 0 pgmap v581: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 54 MiB/s wr, 858 op/s 2023-09-23T18:38:31.120 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:31 smithi098 bash[76817]: audit 2023-09-23T18:38:30.297691+0000 mon.a (mon.0) 845 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:38:31.120 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:31 smithi098 bash[76817]: audit 2023-09-23T18:38:30.297828+0000 mon.b (mon.1) 13 : audit 1 from='client.? 172.21.15.98:0/1835136037' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:38:31.120 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:31 smithi098 bash[79516]: cluster 2023-09-23T18:38:29.850732+0000 mgr.y (mgr.26054) 600 : cluster 0 pgmap v581: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 54 MiB/s wr, 858 op/s 2023-09-23T18:38:31.120 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:31 smithi098 bash[79516]: audit 2023-09-23T18:38:30.297691+0000 mon.a (mon.0) 845 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:38:31.120 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:31 smithi098 bash[79516]: audit 2023-09-23T18:38:30.297828+0000 mon.b (mon.1) 13 : audit 1 from='client.? 172.21.15.98:0/1835136037' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:38:31.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:31.139 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:31.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:31.240 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:31.271 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:31.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:31.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:31.374 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:31.406 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:31.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:31.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:31.510 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:31.539 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:31.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:31.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:31.644 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:31.675 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:31.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:31.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:31.755 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 62 16 81792 81776 82.4258 59.125 0.0149448 0.0121273 2023-09-23T18:38:31.781 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:31.810 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:31.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:31.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:31.914 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:31.953 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:31.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:31.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:32.057 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:32.088 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:32.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:32.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:32.192 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:32.214 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:32.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:32.220 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:31 smithi173 bash[59422]: cluster 2023-09-23T18:38:30.947662+0000 mon.a (mon.0) 846 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:38:32.220 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:31 smithi173 bash[59422]: cluster 2023-09-23T18:38:30.947693+0000 mon.a (mon.0) 847 : cluster 1 Cluster is now healthy 2023-09-23T18:38:32.220 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:31 smithi173 bash[59422]: cluster 2023-09-23T18:38:30.964141+0000 osd.4 (osd.4) 94 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T18:38:32.220 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:31 smithi173 bash[59422]: audit 2023-09-23T18:38:30.973804+0000 mon.a (mon.0) 848 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:38:32.220 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:31 smithi173 bash[59422]: cluster 2023-09-23T18:38:30.974091+0000 mon.a (mon.0) 849 : cluster 0 osdmap e158: 8 total, 8 up, 7 in 2023-09-23T18:38:32.221 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:31 smithi173 bash[59422]: cluster 2023-09-23T18:38:30.976247+0000 osd.6 (osd.6) 71 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T18:38:32.221 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:31 smithi173 bash[59422]: cluster 2023-09-23T18:38:30.976798+0000 osd.7 (osd.7) 69 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T18:38:32.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:32.322 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:32.350 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:32.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:32.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:32.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:32 smithi098 bash[76817]: cluster 2023-09-23T18:38:30.947662+0000 mon.a (mon.0) 846 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:38:32.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:32 smithi098 bash[76817]: cluster 2023-09-23T18:38:30.947693+0000 mon.a (mon.0) 847 : cluster 1 Cluster is now healthy 2023-09-23T18:38:32.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:32 smithi098 bash[76817]: cluster 2023-09-23T18:38:30.964141+0000 osd.4 (osd.4) 94 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T18:38:32.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:32 smithi098 bash[76817]: audit 2023-09-23T18:38:30.973804+0000 mon.a (mon.0) 848 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:38:32.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:32 smithi098 bash[76817]: cluster 2023-09-23T18:38:30.974091+0000 mon.a (mon.0) 849 : cluster 0 osdmap e158: 8 total, 8 up, 7 in 2023-09-23T18:38:32.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:32 smithi098 bash[76817]: cluster 2023-09-23T18:38:30.976247+0000 osd.6 (osd.6) 71 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T18:38:32.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:32 smithi098 bash[76817]: cluster 2023-09-23T18:38:30.976798+0000 osd.7 (osd.7) 69 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T18:38:32.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:32 smithi098 bash[79516]: cluster 2023-09-23T18:38:30.947662+0000 mon.a (mon.0) 846 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:38:32.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:32 smithi098 bash[79516]: cluster 2023-09-23T18:38:30.947693+0000 mon.a (mon.0) 847 : cluster 1 Cluster is now healthy 2023-09-23T18:38:32.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:32 smithi098 bash[79516]: cluster 2023-09-23T18:38:30.964141+0000 osd.4 (osd.4) 94 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T18:38:32.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:32 smithi098 bash[79516]: audit 2023-09-23T18:38:30.973804+0000 mon.a (mon.0) 848 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:38:32.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:32 smithi098 bash[79516]: cluster 2023-09-23T18:38:30.974091+0000 mon.a (mon.0) 849 : cluster 0 osdmap e158: 8 total, 8 up, 7 in 2023-09-23T18:38:32.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:32 smithi098 bash[79516]: cluster 2023-09-23T18:38:30.976247+0000 osd.6 (osd.6) 71 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T18:38:32.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:32 smithi098 bash[79516]: cluster 2023-09-23T18:38:30.976798+0000 osd.7 (osd.7) 69 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T18:38:32.455 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:32.482 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:32.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:32.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:32.587 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:32.610 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:32.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:32.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:32.713 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:32.739 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:32.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:32.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:32.755 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 63 16 82747 82731 82.0647 59.6875 0.0113636 0.0121801 2023-09-23T18:38:32.844 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:32.870 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:32.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:32.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:32.974 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:33.001 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:33.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:33.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:33.106 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:33.130 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:33.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:33.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:33.202 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:38:33.236 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:33.244 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:32 smithi098 bash[76817]: cluster 2023-09-23T18:38:31.852014+0000 mgr.y (mgr.26054) 601 : cluster 0 pgmap v583: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 57 MiB/s wr, 918 op/s 2023-09-23T18:38:33.244 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:32 smithi098 bash[76817]: cluster 2023-09-23T18:38:31.982710+0000 osd.5 (osd.5) 64 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T18:38:33.244 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:32 smithi098 bash[79516]: cluster 2023-09-23T18:38:31.852014+0000 mgr.y (mgr.26054) 601 : cluster 0 pgmap v583: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 57 MiB/s wr, 918 op/s 2023-09-23T18:38:33.245 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:32 smithi098 bash[79516]: cluster 2023-09-23T18:38:31.982710+0000 osd.5 (osd.5) 64 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T18:38:33.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:32 smithi173 bash[59422]: cluster 2023-09-23T18:38:31.852014+0000 mgr.y (mgr.26054) 601 : cluster 0 pgmap v583: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 57 MiB/s wr, 918 op/s 2023-09-23T18:38:33.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:32 smithi173 bash[59422]: cluster 2023-09-23T18:38:31.982710+0000 osd.5 (osd.5) 64 : cluster 3 failed to encode map e158 with expected crc 2023-09-23T18:38:33.277 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:33.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:33.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:33.381 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:33.403 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:33.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:33.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:33.509 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:33.539 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:33.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:33.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:33.644 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:33.674 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:33.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:33.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:33.756 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 64 16 83635 83619 81.6495 55.5 0.0114275 0.0122412 2023-09-23T18:38:33.776 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:33.794 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:38:33.819 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:33.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:33.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:33.923 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:33.955 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:33.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:33.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:34.063 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:34.088 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:34.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:34.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:34.193 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:34.217 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:34.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:34.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:34.324 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:34.349 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:34.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:34.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:34.454 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:34.483 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:34.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:34.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:34.587 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:34.611 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:34.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:34.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:34.717 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:34.720 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:38:34.745 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:34.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:34.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:34.756 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 65 16 84482 84466 81.2071 52.9375 0.0205473 0.0123081 2023-09-23T18:38:34.849 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:34.892 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:34.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:34.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:34.997 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:35.026 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:35.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:34 smithi173 bash[59422]: cluster 2023-09-23T18:38:33.852567+0000 mgr.y (mgr.26054) 602 : cluster 0 pgmap v584: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 58 MiB/s wr, 921 op/s 2023-09-23T18:38:35.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:35.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:35.130 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:35.161 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:35.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:35.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:35.188 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:34 smithi098 bash[76817]: cluster 2023-09-23T18:38:33.852567+0000 mgr.y (mgr.26054) 602 : cluster 0 pgmap v584: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 58 MiB/s wr, 921 op/s 2023-09-23T18:38:35.188 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:34 smithi098 bash[79516]: cluster 2023-09-23T18:38:33.852567+0000 mgr.y (mgr.26054) 602 : cluster 0 pgmap v584: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 58 MiB/s wr, 921 op/s 2023-09-23T18:38:35.267 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:35.297 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:35.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:35.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:35.403 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:35.427 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:35.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:35.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:35.533 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:35.564 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:35.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:35.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:35.673 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:35.709 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:35.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:35.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:35.762 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 66 16 85254 85238 80.7006 48.25 0.0221713 0.0123832 2023-09-23T18:38:35.813 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:35.848 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:35.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:35.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:35.954 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:35.980 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:35.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:35.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:36.083 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:36.135 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:36.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:36.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:36.243 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:36.276 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:36.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:36.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:36.380 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:36.388 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:36 smithi098 bash[79516]: audit 2023-09-23T18:38:35.285874+0000 mon.a (mon.0) 850 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:38:36.388 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:36 smithi098 bash[79516]: audit 2023-09-23T18:38:35.302974+0000 mon.a (mon.0) 851 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:38:36.388 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:36 smithi098 bash[79516]: cluster 2023-09-23T18:38:35.853179+0000 mgr.y (mgr.26054) 603 : cluster 0 pgmap v585: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 57 MiB/s wr, 913 op/s 2023-09-23T18:38:36.388 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:38:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:38:36] "GET /metrics HTTP/1.1" 200 36516 "" "Prometheus/2.33.4" 2023-09-23T18:38:36.389 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:36 smithi098 bash[76817]: audit 2023-09-23T18:38:35.285874+0000 mon.a (mon.0) 850 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:38:36.389 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:36 smithi098 bash[76817]: audit 2023-09-23T18:38:35.302974+0000 mon.a (mon.0) 851 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:38:36.389 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:36 smithi098 bash[76817]: cluster 2023-09-23T18:38:35.853179+0000 mgr.y (mgr.26054) 603 : cluster 0 pgmap v585: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 57 MiB/s wr, 913 op/s 2023-09-23T18:38:36.427 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:36.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:36.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:36.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:36 smithi173 bash[59422]: audit 2023-09-23T18:38:35.285874+0000 mon.a (mon.0) 850 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:38:36.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:36 smithi173 bash[59422]: audit 2023-09-23T18:38:35.302974+0000 mon.a (mon.0) 851 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:38:36.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:36 smithi173 bash[59422]: cluster 2023-09-23T18:38:35.853179+0000 mgr.y (mgr.26054) 603 : cluster 0 pgmap v585: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 57 MiB/s wr, 913 op/s 2023-09-23T18:38:36.534 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:36.571 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:36.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:36.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:36.674 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:36.702 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:36.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:36.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:36.762 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 67 16 86048 86032 80.2368 49.625 0.0122847 0.012456 2023-09-23T18:38:36.807 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:36.829 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:36.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:36.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:36.926 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:38:36.932 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:36.955 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:36.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:36.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:37.058 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:37.082 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:37.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:37.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:37.186 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:37.208 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:37.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:37.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:37.312 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:37.338 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:37.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:37.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:37.441 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:37.462 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:37.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:37.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:37.567 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:37.601 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:37.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:37.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:37.703 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:37.737 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:37.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:37.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:37.762 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 68 16 86910 86894 79.849 53.875 0.0263742 0.0125173 2023-09-23T18:38:37.843 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:37.851 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:37 smithi098 bash[76817]: audit 2023-09-23T18:38:36.594992+0000 mon.a (mon.0) 852 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:38:37.851 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:37 smithi098 bash[76817]: audit 2023-09-23T18:38:36.608350+0000 mon.a (mon.0) 853 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:38:37.851 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:37 smithi098 bash[76817]: audit 2023-09-23T18:38:37.185723+0000 mon.a (mon.0) 854 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:38:37.852 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:37 smithi098 bash[76817]: audit 2023-09-23T18:38:37.186895+0000 mon.a (mon.0) 855 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:38:37.852 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:37 smithi098 bash[76817]: audit 2023-09-23T18:38:37.196788+0000 mon.a (mon.0) 856 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:38:37.852 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:37 smithi098 bash[79516]: audit 2023-09-23T18:38:36.594992+0000 mon.a (mon.0) 852 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:38:37.852 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:37 smithi098 bash[79516]: audit 2023-09-23T18:38:36.608350+0000 mon.a (mon.0) 853 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:38:37.852 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:37 smithi098 bash[79516]: audit 2023-09-23T18:38:37.185723+0000 mon.a (mon.0) 854 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:38:37.852 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:37 smithi098 bash[79516]: audit 2023-09-23T18:38:37.186895+0000 mon.a (mon.0) 855 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:38:37.852 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:37 smithi098 bash[79516]: audit 2023-09-23T18:38:37.196788+0000 mon.a (mon.0) 856 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:38:37.872 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:37.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:37.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:37.975 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:38.003 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:38.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:38.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:38.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:37 smithi173 bash[59422]: audit 2023-09-23T18:38:36.594992+0000 mon.a (mon.0) 852 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:38:38.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:37 smithi173 bash[59422]: audit 2023-09-23T18:38:36.608350+0000 mon.a (mon.0) 853 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:38:38.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:37 smithi173 bash[59422]: audit 2023-09-23T18:38:37.185723+0000 mon.a (mon.0) 854 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:38:38.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:37 smithi173 bash[59422]: audit 2023-09-23T18:38:37.186895+0000 mon.a (mon.0) 855 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:38:38.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:37 smithi173 bash[59422]: audit 2023-09-23T18:38:37.196788+0000 mon.a (mon.0) 856 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:38:38.108 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:38.128 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:38.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:38.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:38.231 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:38.257 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:38.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:38.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:38.360 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:38.394 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:38.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:38.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:38.497 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:38.522 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:38.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:38.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:38.628 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:38.657 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:38.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:38.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:38.762 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:38.763 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 69 16 87762 87746 79.4631 53.25 0.0195488 0.0125783 2023-09-23T18:38:38.776 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:38 smithi173 bash[59422]: cluster 2023-09-23T18:38:37.853943+0000 mgr.y (mgr.26054) 604 : cluster 0 pgmap v586: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 55 MiB/s wr, 879 op/s 2023-09-23T18:38:38.792 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:38.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:38.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:38.898 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:38.923 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:38.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:38.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:38.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:38 smithi098 bash[76817]: cluster 2023-09-23T18:38:37.853943+0000 mgr.y (mgr.26054) 604 : cluster 0 pgmap v586: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 55 MiB/s wr, 879 op/s 2023-09-23T18:38:38.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:38 smithi098 bash[79516]: cluster 2023-09-23T18:38:37.853943+0000 mgr.y (mgr.26054) 604 : cluster 0 pgmap v586: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 55 MiB/s wr, 879 op/s 2023-09-23T18:38:38.975 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:38:38.976 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [ 2023-09-23T18:38:38.976 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.976 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.44954.0:87896 6.0 e158/149 6:06b4659c:::benchmark_data_smithi098_131983_object87895:head v 158'5634, mlcod=158'5632)", 2023-09-23T18:38:38.976 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:38:38.960519+0000", 2023-09-23T18:38:38.976 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.009397397, 2023-09-23T18:38:38.976 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0094231609999999993, 2023-09-23T18:38:38.976 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:38:38.976 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T18:38:38.976 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:38:38.976 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.976 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:38:38.976 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.960519+0000", 2023-09-23T18:38:38.977 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:38:38.977 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.977 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.977 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:38:38.977 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.960519+0000", 2023-09-23T18:38:38.977 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:38:38.977 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.977 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.977 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:38:38.977 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.960518+0000", 2023-09-23T18:38:38.977 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:38:38.977 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.978 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.978 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:38:38.978 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.960553+0000", 2023-09-23T18:38:38.978 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.5154000000000003e-05 2023-09-23T18:38:38.978 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.978 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.978 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:38:38.978 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.960566+0000", 2023-09-23T18:38:38.978 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3025e-05 2023-09-23T18:38:38.978 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.978 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.978 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:38:38.979 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.960574+0000", 2023-09-23T18:38:38.979 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.3520000000000001e-06 2023-09-23T18:38:38.979 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.979 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.979 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:38:38.979 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.960713+0000", 2023-09-23T18:38:38.979 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00013947699999999999 2023-09-23T18:38:38.979 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.979 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.979 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:38:38.979 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.960760+0000", 2023-09-23T18:38:38.979 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.7194000000000002e-05 2023-09-23T18:38:38.979 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:38:38.980 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:38:38.980 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:38:38.980 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.980 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.980 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.44954.0:87894 6.8 e158/149 6:1fcd23fd:::benchmark_data_smithi098_131983_object87893:head v 158'5603, mlcod=158'5600)", 2023-09-23T18:38:38.980 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:38:38.968102+0000", 2023-09-23T18:38:38.980 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.001813811, 2023-09-23T18:38:38.980 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.001882957, 2023-09-23T18:38:38.980 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:38:38.980 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T18:38:38.980 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:38:38.980 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.981 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:38:38.981 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968102+0000", 2023-09-23T18:38:38.981 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:38:38.981 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.981 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.981 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:38:38.981 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968102+0000", 2023-09-23T18:38:38.981 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:38:38.981 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.982 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.982 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:38:38.982 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968102+0000", 2023-09-23T18:38:38.982 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:38:38.982 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.982 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.982 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:38:38.983 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968143+0000", 2023-09-23T18:38:38.983 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.1269000000000001e-05 2023-09-23T18:38:38.983 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.983 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.983 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:38:38.983 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968157+0000", 2023-09-23T18:38:38.983 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3730000000000001e-05 2023-09-23T18:38:38.983 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.984 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.984 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:38:38.984 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968165+0000", 2023-09-23T18:38:38.984 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.6510000000000007e-06 2023-09-23T18:38:38.984 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.984 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.984 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:38:38.984 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968519+0000", 2023-09-23T18:38:38.984 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000354025 2023-09-23T18:38:38.985 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.985 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.985 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:38:38.985 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968540+0000", 2023-09-23T18:38:38.985 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.0324999999999999e-05 2023-09-23T18:38:38.985 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:38:38.985 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:38:38.985 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:38:38.986 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.986 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.986 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.44954.0:87895 6.8 e158/149)", 2023-09-23T18:38:38.986 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:38:38.968176+0000", 2023-09-23T18:38:38.986 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.0017397440000000001, 2023-09-23T18:38:38.986 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0018471029999999999, 2023-09-23T18:38:38.987 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:38:38.987 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "queued for pg", 2023-09-23T18:38:38.987 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:38:38.987 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.987 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:38:38.987 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968176+0000", 2023-09-23T18:38:38.987 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:38:38.987 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.988 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.988 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:38:38.988 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968176+0000", 2023-09-23T18:38:38.988 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:38:38.988 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.988 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.988 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:38:38.988 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968176+0000", 2023-09-23T18:38:38.989 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:38:38.989 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.989 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.989 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:38:38.989 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968209+0000", 2023-09-23T18:38:38.989 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.3148000000000002e-05 2023-09-23T18:38:38.989 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.989 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.990 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:38:38.990 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968217+0000", 2023-09-23T18:38:38.990 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.272e-06 2023-09-23T18:38:38.990 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.990 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.990 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:38:38.990 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968237+0000", 2023-09-23T18:38:38.990 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.0961000000000001e-05 2023-09-23T18:38:38.991 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:38:38.991 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:38:38.991 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:38:38.991 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.991 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.991 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.44954.0:87897 6.2 e158/149)", 2023-09-23T18:38:38.991 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:38:38.968261+0000", 2023-09-23T18:38:38.991 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.001654731, 2023-09-23T18:38:38.991 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0017887280000000001, 2023-09-23T18:38:38.992 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:38:38.992 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "queued for pg", 2023-09-23T18:38:38.992 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:38:38.992 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.992 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:38:38.992 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968261+0000", 2023-09-23T18:38:38.992 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:38:38.993 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.993 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.993 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:38:38.993 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968261+0000", 2023-09-23T18:38:38.993 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:38:38.993 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.993 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.994 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:38:38.994 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968261+0000", 2023-09-23T18:38:38.994 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:38:38.994 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.994 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.994 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:38:38.994 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968293+0000", 2023-09-23T18:38:38.994 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.2378000000000001e-05 2023-09-23T18:38:38.994 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.994 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.994 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:38:38.995 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968303+0000", 2023-09-23T18:38:38.995 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.4800000000000007e-06 2023-09-23T18:38:38.995 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.995 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.995 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:38:38.995 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968309+0000", 2023-09-23T18:38:38.995 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.9800000000000003e-06 2023-09-23T18:38:38.995 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:38:38.995 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:38:38.995 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:38:38.995 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.995 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.995 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.44954.0:87898 6.1 e158/149 6:8d3812a2:::benchmark_data_smithi098_131983_object87897:head v 158'5555, mlcod=158'5553)", 2023-09-23T18:38:38.996 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:38:38.968340+0000", 2023-09-23T18:38:38.996 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.001576346, 2023-09-23T18:38:38.996 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.001742801, 2023-09-23T18:38:38.996 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:38:38.996 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T18:38:38.996 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:38:38.996 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.996 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:38:38.996 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968340+0000", 2023-09-23T18:38:38.997 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:38:38.997 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.997 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.997 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:38:38.997 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968340+0000", 2023-09-23T18:38:38.998 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:38:38.998 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.998 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.998 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:38:38.998 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968339+0000", 2023-09-23T18:38:38.998 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:38:38.998 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.998 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.998 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:38:38.998 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968364+0000", 2023-09-23T18:38:38.998 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.4805000000000001e-05 2023-09-23T18:38:38.998 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.999 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.999 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:38:38.999 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968374+0000", 2023-09-23T18:38:38.999 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0217999999999999e-05 2023-09-23T18:38:38.999 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:38.999 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:38.999 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:38:39.000 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968380+0000", 2023-09-23T18:38:39.000 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.755e-06 2023-09-23T18:38:39.000 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:39.000 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:39.000 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:38:39.000 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968421+0000", 2023-09-23T18:38:39.000 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.0741999999999997e-05 2023-09-23T18:38:39.001 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:39.001 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:39.001 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:38:39.001 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968445+0000", 2023-09-23T18:38:39.001 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.3703999999999999e-05 2023-09-23T18:38:39.001 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:38:39.001 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:38:39.001 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:38:39.002 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:39.002 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:39.002 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.44954.0:87899 6.1 e158/149)", 2023-09-23T18:38:39.002 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:38:38.968689+0000", 2023-09-23T18:38:39.002 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.001227499, 2023-09-23T18:38:39.002 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0014273549999999999, 2023-09-23T18:38:39.003 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:38:39.003 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "queued for pg", 2023-09-23T18:38:39.003 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:38:39.003 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:39.003 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:38:39.003 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968689+0000", 2023-09-23T18:38:39.003 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:38:39.003 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:39.003 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:39.003 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:38:39.003 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968689+0000", 2023-09-23T18:38:39.004 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:38:39.004 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:39.004 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:39.004 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:38:39.004 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968688+0000", 2023-09-23T18:38:39.004 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:38:39.004 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:39.004 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:39.004 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:38:39.004 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968721+0000", 2023-09-23T18:38:39.004 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.2886000000000003e-05 2023-09-23T18:38:39.004 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:39.005 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:39.005 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:38:39.005 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968732+0000", 2023-09-23T18:38:39.005 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0721e-05 2023-09-23T18:38:39.005 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:39.005 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:39.005 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:38:39.005 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.968739+0000", 2023-09-23T18:38:39.005 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.4819999999999997e-06 2023-09-23T18:38:39.005 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:38:39.005 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:38:39.005 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:38:39.005 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:39.006 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:39.006 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.44954.0:87904 6.2 e158/149)", 2023-09-23T18:38:39.006 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:38:38.969192+0000", 2023-09-23T18:38:39.006 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.00072368599999999997, 2023-09-23T18:38:39.006 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00094969499999999997, 2023-09-23T18:38:39.006 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:38:39.006 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "queued for pg", 2023-09-23T18:38:39.007 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:38:39.007 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:39.007 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:38:39.007 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.969192+0000", 2023-09-23T18:38:39.007 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:38:39.007 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:39.007 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:39.007 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:38:39.007 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.969192+0000", 2023-09-23T18:38:39.008 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:38:39.008 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:39.008 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:39.008 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:38:39.008 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.969192+0000", 2023-09-23T18:38:39.008 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:38:39.008 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:39.008 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:39.009 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:38:39.009 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.969275+0000", 2023-09-23T18:38:39.009 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.3547000000000005e-05 2023-09-23T18:38:39.009 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:39.009 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:39.009 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:38:39.009 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.969287+0000", 2023-09-23T18:38:39.009 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1781e-05 2023-09-23T18:38:39.009 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:38:39.009 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:38:39.010 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:38:39.010 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:38:38.969294+0000", 2023-09-23T18:38:39.010 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.4749999999999998e-06 2023-09-23T18:38:39.010 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:38:39.010 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:38:39.010 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:38:39.010 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:38:39.010 INFO:teuthology.orchestra.run.smithi098.stdout: ], 2023-09-23T18:38:39.010 INFO:teuthology.orchestra.run.smithi098.stdout: "num_ops": 7 2023-09-23T18:38:39.010 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:38:39.028 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:39.052 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:39.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:39.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:39.156 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:39.190 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:39.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:39.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:39.292 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:39.330 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:38:39.331 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:38:39.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:39.343 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:39.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:39.444 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:39.473 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:39.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:39.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:39.580 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:39.612 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:39.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:39.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:39.716 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:39.743 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:39.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:39.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:39.763 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 70 16 88599 88583 79.0752 52.3125 0.0160115 0.0126406 2023-09-23T18:38:39.847 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:39.881 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:39.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:39.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:39.985 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:40.012 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:40.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:40.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:40.116 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:40.142 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:40.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:40.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:40.246 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:40.272 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:40.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:40.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:40.379 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:40.420 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:40.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:40.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:40.524 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:40.547 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:40.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:40.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:40.656 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:40.684 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:40.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:40.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:40.763 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 71 16 89512 89496 78.7652 57.0625 0.00868948 0.0126909 2023-09-23T18:38:40.776 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:40 smithi173 bash[59422]: audit 2023-09-23T18:38:39.600772+0000 mon.a (mon.0) 857 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:38:40.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:40 smithi173 bash[59422]: audit 2023-09-23T18:38:39.601907+0000 mon.a (mon.0) 858 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:38:40.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:40 smithi173 bash[59422]: cluster 2023-09-23T18:38:39.855173+0000 mgr.y (mgr.26054) 605 : cluster 0 pgmap v587: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 54 MiB/s wr, 868 op/s 2023-09-23T18:38:40.790 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:40.799 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:40 smithi098 bash[79516]: audit 2023-09-23T18:38:39.600772+0000 mon.a (mon.0) 857 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:38:40.799 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:40 smithi098 bash[79516]: audit 2023-09-23T18:38:39.601907+0000 mon.a (mon.0) 858 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:38:40.799 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:40 smithi098 bash[79516]: cluster 2023-09-23T18:38:39.855173+0000 mgr.y (mgr.26054) 605 : cluster 0 pgmap v587: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 54 MiB/s wr, 868 op/s 2023-09-23T18:38:40.799 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:40 smithi098 bash[76817]: audit 2023-09-23T18:38:39.600772+0000 mon.a (mon.0) 857 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:38:40.800 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:40 smithi098 bash[76817]: audit 2023-09-23T18:38:39.601907+0000 mon.a (mon.0) 858 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:38:40.800 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:40 smithi098 bash[76817]: cluster 2023-09-23T18:38:39.855173+0000 mgr.y (mgr.26054) 605 : cluster 0 pgmap v587: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 54 MiB/s wr, 868 op/s 2023-09-23T18:38:40.818 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:40.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:40.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:40.922 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:40.954 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:40.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:40.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:41.057 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:41.088 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:41.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:41.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:41.195 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:41.224 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:41.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:41.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:41.328 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:41.359 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:41.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:41.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:41.465 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:41.497 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:41.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:41.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:41.602 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:41.636 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:41.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:41.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:41.739 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:41.763 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 72 16 90407 90391 78.4481 55.9375 0.0108144 0.0127418 2023-09-23T18:38:41.780 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:41.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:41.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:41.885 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:41.918 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:41.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:41.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:42.023 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:42.049 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:42.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:42.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:42.151 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:42.179 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:42.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:42.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:42.285 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:42.311 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:42.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:42.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:42.413 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:42.439 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:42.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:42.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:42.543 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:42.580 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:42.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:42.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:42.685 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:42.724 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:42.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:42.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:42.763 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 73 16 91304 91288 78.1414 56.0625 0.0109875 0.012791 2023-09-23T18:38:42.830 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:42.856 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:42.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:42.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:42.949 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:42 smithi098 bash[76817]: cluster 2023-09-23T18:38:41.855832+0000 mgr.y (mgr.26054) 606 : cluster 0 pgmap v588: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 59 MiB/s wr, 949 op/s 2023-09-23T18:38:42.949 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:42 smithi098 bash[79516]: cluster 2023-09-23T18:38:41.855832+0000 mgr.y (mgr.26054) 606 : cluster 0 pgmap v588: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 59 MiB/s wr, 949 op/s 2023-09-23T18:38:42.962 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:42.983 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:42.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:42.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:43.087 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:43.095 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:42 smithi173 bash[59422]: cluster 2023-09-23T18:38:41.855832+0000 mgr.y (mgr.26054) 606 : cluster 0 pgmap v588: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 59 MiB/s wr, 949 op/s 2023-09-23T18:38:43.117 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:43.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:43.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:43.215 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T18:38:43.221 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:43.249 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:43.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:43.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:43.352 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:43.379 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:43.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:43.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:43.482 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:43.529 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:43.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:43.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:43.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:43.655 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:43.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:43.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:43.757 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:43.763 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 74 16 92093 92077 77.7518 49.3125 0.027792 0.0128551 2023-09-23T18:38:43.788 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:43.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:43.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:43.893 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:43.925 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:43.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:43.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:44.029 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:44.055 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:44.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:44.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:44.158 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:44.183 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:44.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:44.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:44.289 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:44.317 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:44.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:44.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:44.422 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:44.458 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:44.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:44.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:44.563 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:44.588 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:44.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:44.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:44.691 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:44.723 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:44.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:44.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:44.763 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 75 16 92975 92959 77.4501 55.125 0.00458995 0.0129051 2023-09-23T18:38:44.829 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:44.852 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:44.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:44.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:44.955 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:44.983 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:44 smithi098 bash[76817]: cluster 2023-09-23T18:38:43.856384+0000 mgr.y (mgr.26054) 607 : cluster 0 pgmap v589: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 813 op/s 2023-09-23T18:38:44.984 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:44 smithi098 bash[79516]: cluster 2023-09-23T18:38:43.856384+0000 mgr.y (mgr.26054) 607 : cluster 0 pgmap v589: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 813 op/s 2023-09-23T18:38:44.985 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:44.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:44.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:45.089 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:45.100 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:44 smithi173 bash[59422]: cluster 2023-09-23T18:38:43.856384+0000 mgr.y (mgr.26054) 607 : cluster 0 pgmap v589: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 813 op/s 2023-09-23T18:38:45.118 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:45.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:45.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:45.221 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:45.251 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:45.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:45.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:45.357 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:45.391 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:45.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:45.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:45.497 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:45.520 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:45.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:45.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:45.623 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:45.646 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:45.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:45.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:45.751 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:45.763 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 76 16 93768 93752 77.0831 49.5625 0.0104776 0.0129669 2023-09-23T18:38:45.780 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:45.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:45.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:45.884 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:45.917 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:45.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:45.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:46.021 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:46.051 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:46.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:46.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:46.089 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:38:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:38:46] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.33.4" 2023-09-23T18:38:46.157 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:46.186 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:46.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:46.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:46.292 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:46.328 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:46.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:46.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:46.434 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:46.483 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:38:46.488 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-23T18:38:46.556 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:46.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:46.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:46.662 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:46.688 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:46.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:46.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:46.764 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 77 16 94495 94479 76.6721 45.4375 0.0287047 0.0130364 2023-09-23T18:38:46.792 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:46.823 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:46.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:46.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:46.926 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:46.959 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:46 smithi173 bash[59422]: cluster 2023-09-23T18:38:45.856845+0000 mgr.y (mgr.26054) 608 : cluster 0 pgmap v590: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 53 MiB/s wr, 852 op/s 2023-09-23T18:38:46.963 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:46.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:46.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:47.067 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:47.095 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:47.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:47.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:47.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:46 smithi098 bash[76817]: cluster 2023-09-23T18:38:45.856845+0000 mgr.y (mgr.26054) 608 : cluster 0 pgmap v590: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 53 MiB/s wr, 852 op/s 2023-09-23T18:38:47.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:46 smithi098 bash[79516]: cluster 2023-09-23T18:38:45.856845+0000 mgr.y (mgr.26054) 608 : cluster 0 pgmap v590: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 53 MiB/s wr, 852 op/s 2023-09-23T18:38:47.198 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:47.228 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:47.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:47.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:47.333 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:47.358 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:47.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:47.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:47.464 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:47.503 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:47.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:47.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:47.609 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:47.639 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:47.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:47.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:47.744 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:47.764 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 78 16 95354 95338 76.3774 53.6875 0.0173297 0.0130872 2023-09-23T18:38:47.768 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:47.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:47.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:47.875 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:47.900 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:47.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:47.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:48.006 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:48.030 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:48.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:48.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:48.135 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:48.143 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:47 smithi173 bash[59422]: audit 2023-09-23T18:38:46.433424+0000 mgr.y (mgr.26054) 609 : audit 0 from='client.45005 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:38:48.166 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:48.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:48.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:48.272 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:48.297 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:48.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:48.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:48.405 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:48.430 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:48.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:48.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:48.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:47 smithi098 bash[76817]: audit 2023-09-23T18:38:46.433424+0000 mgr.y (mgr.26054) 609 : audit 0 from='client.45005 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:38:48.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:47 smithi098 bash[79516]: audit 2023-09-23T18:38:46.433424+0000 mgr.y (mgr.26054) 609 : audit 0 from='client.45005 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:38:48.536 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:48.562 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:48.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:48.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:48.665 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:48.691 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:48.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:48.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:48.731 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":590,"stamp":"2023-09-23T18:38:45.856558+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6007691187,"num_objects":91625,"num_object_clones":0,"num_object_copies":274875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":91625,"num_whiteouts":0,"num_read":6245,"num_read_kb":13702,"num_write":92703,"num_write_kb":5877367,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":4194950,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92767,"ondisk_log_size":92767,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":19453720,"kb_used_data":17568544,"kb_used_omap":31,"kb_used_meta":1885088,"kb_avail":636762344,"statfs":{"total":671965249536,"available":652044640256,"internally_reserved":0,"allocated":17990189056,"data_stored":17985535169,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32507,"internal_metadata":1930330373},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[7,2,1,15,7,4,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":670564352,"num_objects":10232,"num_object_clones":0,"num_object_copies":30696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10232,"num_write_kb":654848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004349"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":14,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.967791+0000","last_change":"2023-09-23T18:37:40.992912+0000","last_active":"2023-09-23T18:38:10.967791+0000","last_peered":"2023-09-23T18:38:10.967791+0000","last_clean":"2023-09-23T18:38:10.967791+0000","last_became_active":"2023-09-23T18:37:40.992453+0000","last_became_peered":"2023-09-23T18:37:40.992453+0000","last_unstale":"2023-09-23T18:38:10.967791+0000","last_undegraded":"2023-09-23T18:38:10.967791+0000","last_fullsized":"2023-09-23T18:38:10.967791+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:49.539251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":144,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.976301+0000","last_change":"2023-09-23T18:33:53.841529+0000","last_active":"2023-09-23T18:38:10.976301+0000","last_peered":"2023-09-23T18:38:10.976301+0000","last_clean":"2023-09-23T18:38:10.976301+0000","last_became_active":"2023-09-23T18:33:53.841290+0000","last_became_peered":"2023-09-23T18:33:53.841290+0000","last_unstale":"2023-09-23T18:38:10.976301+0000","last_undegraded":"2023-09-23T18:38:10.976301+0000","last_fullsized":"2023-09-23T18:38:10.976301+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:27.552815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":95,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T18:38:30.971902+0000","last_change":"2023-09-23T18:33:53.848655+0000","last_active":"2023-09-23T18:38:30.971902+0000","last_peered":"2023-09-23T18:38:30.971902+0000","last_clean":"2023-09-23T18:38:30.971902+0000","last_became_active":"2023-09-23T18:33:53.848279+0000","last_became_peered":"2023-09-23T18:33:53.848279+0000","last_unstale":"2023-09-23T18:38:30.971902+0000","last_undegraded":"2023-09-23T18:38:30.971902+0000","last_fullsized":"2023-09-23T18:38:30.971902+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:39.223612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":112,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T18:38:30.982217+0000","last_change":"2023-09-23T18:28:03.289500+0000","last_active":"2023-09-23T18:38:30.982217+0000","last_peered":"2023-09-23T18:38:30.982217+0000","last_clean":"2023-09-23T18:38:30.982217+0000","last_became_active":"2023-09-23T18:28:03.289122+0000","last_became_peered":"2023-09-23T18:28:03.289122+0000","last_unstale":"2023-09-23T18:38:30.982217+0000","last_undegraded":"2023-09-23T18:38:30.982217+0000","last_fullsized":"2023-09-23T18:38:30.982217+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:22:26.904660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":160,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.976591+0000","last_change":"2023-09-23T18:33:53.842010+0000","last_active":"2023-09-23T18:38:10.976591+0000","last_peered":"2023-09-23T18:38:10.976591+0000","last_clean":"2023-09-23T18:38:10.976591+0000","last_became_active":"2023-09-23T18:33:53.841790+0000","last_became_peered":"2023-09-23T18:33:53.841790+0000","last_unstale":"2023-09-23T18:38:10.976591+0000","last_undegraded":"2023-09-23T18:38:10.976591+0000","last_fullsized":"2023-09-23T18:38:10.976591+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:46.540440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":149,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.972062+0000","last_change":"2023-09-23T18:33:55.084557+0000","last_active":"2023-09-23T18:38:10.972062+0000","last_peered":"2023-09-23T18:38:10.972062+0000","last_clean":"2023-09-23T18:38:10.972062+0000","last_became_active":"2023-09-23T18:33:55.083987+0000","last_became_peered":"2023-09-23T18:33:55.083987+0000","last_unstale":"2023-09-23T18:38:10.972062+0000","last_undegraded":"2023-09-23T18:38:10.972062+0000","last_fullsized":"2023-09-23T18:38:10.972062+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":119,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T18:38:30.982612+0000","last_change":"2023-09-23T18:32:26.797582+0000","last_active":"2023-09-23T18:38:30.982612+0000","last_peered":"2023-09-23T18:38:30.982612+0000","last_clean":"2023-09-23T18:38:30.982612+0000","last_became_active":"2023-09-23T18:32:26.797107+0000","last_became_peered":"2023-09-23T18:32:26.797107+0000","last_unstale":"2023-09-23T18:38:30.982612+0000","last_undegraded":"2023-09-23T18:38:30.982612+0000","last_fullsized":"2023-09-23T18:38:30.982612+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:56.973116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":153,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.976666+0000","last_change":"2023-09-23T18:37:40.998526+0000","last_active":"2023-09-23T18:38:10.976666+0000","last_peered":"2023-09-23T18:38:10.976666+0000","last_clean":"2023-09-23T18:38:10.976666+0000","last_became_active":"2023-09-23T18:37:40.993211+0000","last_became_peered":"2023-09-23T18:37:40.993211+0000","last_unstale":"2023-09-23T18:38:10.976666+0000","last_undegraded":"2023-09-23T18:38:10.976666+0000","last_fullsized":"2023-09-23T18:38:10.976666+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:40.463496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":138,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.969425+0000","last_change":"2023-09-23T18:33:54.847051+0000","last_active":"2023-09-23T18:38:10.969425+0000","last_peered":"2023-09-23T18:38:10.969425+0000","last_clean":"2023-09-23T18:38:10.969425+0000","last_became_active":"2023-09-23T18:33:54.845357+0000","last_became_peered":"2023-09-23T18:33:54.845357+0000","last_unstale":"2023-09-23T18:38:10.969425+0000","last_undegraded":"2023-09-23T18:38:10.969425+0000","last_fullsized":"2023-09-23T18:38:10.969425+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":119,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.964953+0000","last_change":"2023-09-23T18:28:03.298214+0000","last_active":"2023-09-23T18:38:10.964953+0000","last_peered":"2023-09-23T18:38:10.964953+0000","last_clean":"2023-09-23T18:38:10.964953+0000","last_became_active":"2023-09-23T18:28:03.297852+0000","last_became_peered":"2023-09-23T18:28:03.297852+0000","last_unstale":"2023-09-23T18:38:10.964953+0000","last_undegraded":"2023-09-23T18:38:10.964953+0000","last_fullsized":"2023-09-23T18:38:10.964953+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.146541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":127,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.969430+0000","last_change":"2023-09-23T18:28:03.296913+0000","last_active":"2023-09-23T18:38:10.969430+0000","last_peered":"2023-09-23T18:38:10.969430+0000","last_clean":"2023-09-23T18:38:10.969430+0000","last_became_active":"2023-09-23T18:28:03.296499+0000","last_became_peered":"2023-09-23T18:28:03.296499+0000","last_unstale":"2023-09-23T18:38:10.969430+0000","last_undegraded":"2023-09-23T18:38:10.969430+0000","last_fullsized":"2023-09-23T18:38:10.969430+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":91,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T18:38:30.973117+0000","last_change":"2023-09-23T18:33:53.850096+0000","last_active":"2023-09-23T18:38:30.973117+0000","last_peered":"2023-09-23T18:38:30.973117+0000","last_clean":"2023-09-23T18:38:30.973117+0000","last_became_active":"2023-09-23T18:33:53.849811+0000","last_became_peered":"2023-09-23T18:33:53.849811+0000","last_unstale":"2023-09-23T18:38:30.973117+0000","last_undegraded":"2023-09-23T18:38:30.973117+0000","last_fullsized":"2023-09-23T18:38:30.973117+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:44.938072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":131,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.973474+0000","last_change":"2023-09-23T18:15:08.909339+0000","last_active":"2023-09-23T18:38:10.973474+0000","last_peered":"2023-09-23T18:38:10.973474+0000","last_clean":"2023-09-23T18:38:10.973474+0000","last_became_active":"2023-09-23T18:15:08.908912+0000","last_became_peered":"2023-09-23T18:15:08.908912+0000","last_unstale":"2023-09-23T18:38:10.973474+0000","last_undegraded":"2023-09-23T18:38:10.973474+0000","last_fullsized":"2023-09-23T18:38:10.973474+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":138,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.973086+0000","last_change":"2023-09-23T18:33:53.847489+0000","last_active":"2023-09-23T18:38:10.973086+0000","last_peered":"2023-09-23T18:38:10.973086+0000","last_clean":"2023-09-23T18:38:10.973086+0000","last_became_active":"2023-09-23T18:33:53.846935+0000","last_became_peered":"2023-09-23T18:33:53.846935+0000","last_unstale":"2023-09-23T18:38:10.973086+0000","last_undegraded":"2023-09-23T18:38:10.973086+0000","last_fullsized":"2023-09-23T18:38:10.973086+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":126,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.970007+0000","last_change":"2023-09-23T18:27:33.585121+0000","last_active":"2023-09-23T18:38:10.970007+0000","last_peered":"2023-09-23T18:38:10.970007+0000","last_clean":"2023-09-23T18:38:10.970007+0000","last_became_active":"2023-09-23T18:27:33.584620+0000","last_became_peered":"2023-09-23T18:27:33.584620+0000","last_unstale":"2023-09-23T18:38:10.970007+0000","last_undegraded":"2023-09-23T18:38:10.970007+0000","last_fullsized":"2023-09-23T18:38:10.970007+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":144,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.978347+0000","last_change":"2023-09-23T18:33:53.852933+0000","last_active":"2023-09-23T18:38:10.978347+0000","last_peered":"2023-09-23T18:38:10.978347+0000","last_clean":"2023-09-23T18:38:10.978347+0000","last_became_active":"2023-09-23T18:33:53.852626+0000","last_became_peered":"2023-09-23T18:33:53.852626+0000","last_unstale":"2023-09-23T18:38:10.978347+0000","last_undegraded":"2023-09-23T18:38:10.978347+0000","last_fullsized":"2023-09-23T18:38:10.978347+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:00.174536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":110,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:14.584472+0000","last_change":"2023-09-23T18:28:03.290316+0000","last_active":"2023-09-23T18:38:14.584472+0000","last_peered":"2023-09-23T18:38:14.584472+0000","last_clean":"2023-09-23T18:38:14.584472+0000","last_became_active":"2023-09-23T18:28:03.289698+0000","last_became_peered":"2023-09-23T18:28:03.289698+0000","last_unstale":"2023-09-23T18:38:14.584472+0000","last_undegraded":"2023-09-23T18:38:14.584472+0000","last_fullsized":"2023-09-23T18:38:14.584472+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:00.779542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":115,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.969259+0000","last_change":"2023-09-23T18:32:26.808237+0000","last_active":"2023-09-23T18:38:10.969259+0000","last_peered":"2023-09-23T18:38:10.969259+0000","last_clean":"2023-09-23T18:38:10.969259+0000","last_became_active":"2023-09-23T18:32:26.807843+0000","last_became_peered":"2023-09-23T18:32:26.807843+0000","last_unstale":"2023-09-23T18:38:10.969259+0000","last_undegraded":"2023-09-23T18:38:10.969259+0000","last_fullsized":"2023-09-23T18:38:10.969259+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":109,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.969926+0000","last_change":"2023-09-23T18:15:08.916658+0000","last_active":"2023-09-23T18:38:10.969926+0000","last_peered":"2023-09-23T18:38:10.969926+0000","last_clean":"2023-09-23T18:38:10.969926+0000","last_became_active":"2023-09-23T18:15:08.916301+0000","last_became_peered":"2023-09-23T18:15:08.916301+0000","last_unstale":"2023-09-23T18:38:10.969926+0000","last_undegraded":"2023-09-23T18:38:10.969926+0000","last_fullsized":"2023-09-23T18:38:10.969926+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":111,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T18:38:30.982150+0000","last_change":"2023-09-23T18:28:03.290217+0000","last_active":"2023-09-23T18:38:30.982150+0000","last_peered":"2023-09-23T18:38:30.982150+0000","last_clean":"2023-09-23T18:38:30.982150+0000","last_became_active":"2023-09-23T18:28:03.289888+0000","last_became_peered":"2023-09-23T18:28:03.289888+0000","last_unstale":"2023-09-23T18:38:30.982150+0000","last_undegraded":"2023-09-23T18:38:30.982150+0000","last_fullsized":"2023-09-23T18:38:30.982150+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:51:45.400847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":151,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.968476+0000","last_change":"2023-09-23T18:32:26.810836+0000","last_active":"2023-09-23T18:38:10.968476+0000","last_peered":"2023-09-23T18:38:10.968476+0000","last_clean":"2023-09-23T18:38:10.968476+0000","last_became_active":"2023-09-23T18:32:26.795378+0000","last_became_peered":"2023-09-23T18:32:26.795378+0000","last_unstale":"2023-09-23T18:38:10.968476+0000","last_undegraded":"2023-09-23T18:38:10.968476+0000","last_fullsized":"2023-09-23T18:38:10.968476+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":144,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.981280+0000","last_change":"2023-09-23T18:33:53.852173+0000","last_active":"2023-09-23T18:38:10.981280+0000","last_peered":"2023-09-23T18:38:10.981280+0000","last_clean":"2023-09-23T18:38:10.981280+0000","last_became_active":"2023-09-23T18:33:53.851814+0000","last_became_peered":"2023-09-23T18:33:53.851814+0000","last_unstale":"2023-09-23T18:38:10.981280+0000","last_undegraded":"2023-09-23T18:38:10.981280+0000","last_fullsized":"2023-09-23T18:38:10.981280+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:15.845259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"158'5952","reported_seq":5969,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T18:38:45.602805+0000","last_change":"2023-09-23T18:37:11.000750+0000","last_active":"2023-09-23T18:38:45.602805+0000","last_peered":"2023-09-23T18:38:45.602805+0000","last_clean":"2023-09-23T18:38:45.602805+0000","last_became_active":"2023-09-23T18:37:11.000441+0000","last_became_peered":"2023-09-23T18:37:11.000441+0000","last_unstale":"2023-09-23T18:38:45.602805+0000","last_undegraded":"2023-09-23T18:38:45.602805+0000","last_fullsized":"2023-09-23T18:38:45.602805+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":5952,"log_dups_size":0,"ondisk_log_size":5952,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390070272,"num_objects":5952,"num_object_clones":0,"num_object_copies":17856,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5952,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5952,"num_write_kb":380928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":138,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.969639+0000","last_change":"2023-09-23T18:37:41.034732+0000","last_active":"2023-09-23T18:38:10.969639+0000","last_peered":"2023-09-23T18:38:10.969639+0000","last_clean":"2023-09-23T18:38:10.969639+0000","last_became_active":"2023-09-23T18:37:40.985865+0000","last_became_peered":"2023-09-23T18:37:40.985865+0000","last_unstale":"2023-09-23T18:38:10.969639+0000","last_undegraded":"2023-09-23T18:38:10.969639+0000","last_fullsized":"2023-09-23T18:38:10.969639+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":56,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.968831+0000","last_change":"2023-09-23T18:32:26.806328+0000","last_active":"2023-09-23T18:38:10.968831+0000","last_peered":"2023-09-23T18:38:10.968831+0000","last_clean":"2023-09-23T18:38:10.968831+0000","last_became_active":"2023-09-23T18:32:26.803969+0000","last_became_peered":"2023-09-23T18:32:26.803969+0000","last_unstale":"2023-09-23T18:38:10.968831+0000","last_undegraded":"2023-09-23T18:38:10.968831+0000","last_fullsized":"2023-09-23T18:38:10.968831+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.b","version":"158'5587","reported_seq":5605,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T18:38:41.694997+0000","last_change":"2023-09-23T18:37:10.997233+0000","last_active":"2023-09-23T18:38:41.694997+0000","last_peered":"2023-09-23T18:38:41.694997+0000","last_clean":"2023-09-23T18:38:41.694997+0000","last_became_active":"2023-09-23T18:37:10.996865+0000","last_became_peered":"2023-09-23T18:37:10.996865+0000","last_unstale":"2023-09-23T18:38:41.694997+0000","last_undegraded":"2023-09-23T18:38:41.694997+0000","last_fullsized":"2023-09-23T18:38:41.694997+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":5587,"log_dups_size":0,"ondisk_log_size":5587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:06.628611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366149632,"num_objects":5587,"num_object_clones":0,"num_object_copies":16761,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5587,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5587,"num_write_kb":357568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":141,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.969577+0000","last_change":"2023-09-23T18:33:53.847817+0000","last_active":"2023-09-23T18:38:10.969577+0000","last_peered":"2023-09-23T18:38:10.969577+0000","last_clean":"2023-09-23T18:38:10.969577+0000","last_became_active":"2023-09-23T18:33:53.847342+0000","last_became_peered":"2023-09-23T18:33:53.847342+0000","last_unstale":"2023-09-23T18:38:10.969577+0000","last_undegraded":"2023-09-23T18:38:10.969577+0000","last_fullsized":"2023-09-23T18:38:10.969577+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":153,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.972899+0000","last_change":"2023-09-23T18:33:53.841434+0000","last_active":"2023-09-23T18:38:10.972899+0000","last_peered":"2023-09-23T18:38:10.972899+0000","last_clean":"2023-09-23T18:38:10.972899+0000","last_became_active":"2023-09-23T18:33:53.840923+0000","last_became_peered":"2023-09-23T18:33:53.840923+0000","last_unstale":"2023-09-23T18:38:10.972899+0000","last_undegraded":"2023-09-23T18:38:10.972899+0000","last_fullsized":"2023-09-23T18:38:10.972899+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"158'5943","reported_seq":5960,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T18:38:45.751900+0000","last_change":"2023-09-23T18:37:10.998596+0000","last_active":"2023-09-23T18:38:45.751900+0000","last_peered":"2023-09-23T18:38:45.751900+0000","last_clean":"2023-09-23T18:38:45.751900+0000","last_became_active":"2023-09-23T18:37:10.997807+0000","last_became_peered":"2023-09-23T18:37:10.997807+0000","last_unstale":"2023-09-23T18:38:45.751900+0000","last_undegraded":"2023-09-23T18:38:45.751900+0000","last_fullsized":"2023-09-23T18:38:45.751900+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":5943,"log_dups_size":0,"ondisk_log_size":5943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389480448,"num_objects":5943,"num_object_clones":0,"num_object_copies":17829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5943,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5943,"num_write_kb":380352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":123,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.969694+0000","last_change":"2023-09-23T18:32:26.854615+0000","last_active":"2023-09-23T18:38:10.969694+0000","last_peered":"2023-09-23T18:38:10.969694+0000","last_clean":"2023-09-23T18:38:10.969694+0000","last_became_active":"2023-09-23T18:32:26.806771+0000","last_became_peered":"2023-09-23T18:32:26.806771+0000","last_unstale":"2023-09-23T18:38:10.969694+0000","last_undegraded":"2023-09-23T18:38:10.969694+0000","last_fullsized":"2023-09-23T18:38:10.969694+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":132,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.971782+0000","last_change":"2023-09-23T18:27:33.589116+0000","last_active":"2023-09-23T18:38:10.971782+0000","last_peered":"2023-09-23T18:38:10.971782+0000","last_clean":"2023-09-23T18:38:10.971782+0000","last_became_active":"2023-09-23T18:27:33.588809+0000","last_became_peered":"2023-09-23T18:27:33.588809+0000","last_unstale":"2023-09-23T18:38:10.971782+0000","last_undegraded":"2023-09-23T18:38:10.971782+0000","last_fullsized":"2023-09-23T18:38:10.971782+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"158'5753","reported_seq":5770,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T18:38:44.605563+0000","last_change":"2023-09-23T18:37:10.994972+0000","last_active":"2023-09-23T18:38:44.605563+0000","last_peered":"2023-09-23T18:38:44.605563+0000","last_clean":"2023-09-23T18:38:44.605563+0000","last_became_active":"2023-09-23T18:37:10.994640+0000","last_became_peered":"2023-09-23T18:37:10.994640+0000","last_unstale":"2023-09-23T18:38:44.605563+0000","last_undegraded":"2023-09-23T18:38:44.605563+0000","last_fullsized":"2023-09-23T18:38:44.605563+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":5753,"log_dups_size":0,"ondisk_log_size":5753,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:05.096016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377028608,"num_objects":5753,"num_object_clones":0,"num_object_copies":17259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5753,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5753,"num_write_kb":368192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":129,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.979088+0000","last_change":"2023-09-23T18:33:54.848776+0000","last_active":"2023-09-23T18:38:10.979088+0000","last_peered":"2023-09-23T18:38:10.979088+0000","last_clean":"2023-09-23T18:38:10.979088+0000","last_became_active":"2023-09-23T18:33:54.848380+0000","last_became_peered":"2023-09-23T18:33:54.848380+0000","last_unstale":"2023-09-23T18:38:10.979088+0000","last_undegraded":"2023-09-23T18:38:10.979088+0000","last_fullsized":"2023-09-23T18:38:10.979088+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:17:17.093993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":123,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.968102+0000","last_change":"2023-09-23T18:27:33.582033+0000","last_active":"2023-09-23T18:38:10.968102+0000","last_peered":"2023-09-23T18:38:10.968102+0000","last_clean":"2023-09-23T18:38:10.968102+0000","last_became_active":"2023-09-23T18:27:33.581667+0000","last_became_peered":"2023-09-23T18:27:33.581667+0000","last_unstale":"2023-09-23T18:38:10.968102+0000","last_undegraded":"2023-09-23T18:38:10.968102+0000","last_fullsized":"2023-09-23T18:38:10.968102+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:36.039922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"158'5699","reported_seq":5715,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T18:38:41.040451+0000","last_change":"2023-09-23T18:37:10.999332+0000","last_active":"2023-09-23T18:38:41.040451+0000","last_peered":"2023-09-23T18:38:41.040451+0000","last_clean":"2023-09-23T18:38:41.040451+0000","last_became_active":"2023-09-23T18:37:10.998962+0000","last_became_peered":"2023-09-23T18:37:10.998962+0000","last_unstale":"2023-09-23T18:38:41.040451+0000","last_undegraded":"2023-09-23T18:38:41.040451+0000","last_fullsized":"2023-09-23T18:38:41.040451+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":5699,"log_dups_size":0,"ondisk_log_size":5699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373489664,"num_objects":5699,"num_object_clones":0,"num_object_copies":17097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5699,"num_write_kb":364736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":128,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.975120+0000","last_change":"2023-09-23T18:28:03.297628+0000","last_active":"2023-09-23T18:38:10.975120+0000","last_peered":"2023-09-23T18:38:10.975120+0000","last_clean":"2023-09-23T18:38:10.975120+0000","last_became_active":"2023-09-23T18:28:03.296694+0000","last_became_peered":"2023-09-23T18:28:03.296694+0000","last_unstale":"2023-09-23T18:38:10.975120+0000","last_undegraded":"2023-09-23T18:38:10.975120+0000","last_fullsized":"2023-09-23T18:38:10.975120+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:43.375652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":123,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.973346+0000","last_change":"2023-09-23T18:28:03.302326+0000","last_active":"2023-09-23T18:38:10.973346+0000","last_peered":"2023-09-23T18:38:10.973346+0000","last_clean":"2023-09-23T18:38:10.973346+0000","last_became_active":"2023-09-23T18:28:03.301970+0000","last_became_peered":"2023-09-23T18:28:03.301970+0000","last_unstale":"2023-09-23T18:38:10.973346+0000","last_undegraded":"2023-09-23T18:38:10.973346+0000","last_fullsized":"2023-09-23T18:38:10.973346+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"158'5715","reported_seq":5733,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T18:38:43.705551+0000","last_change":"2023-09-23T18:37:10.997349+0000","last_active":"2023-09-23T18:38:43.705551+0000","last_peered":"2023-09-23T18:38:43.705551+0000","last_clean":"2023-09-23T18:38:43.705551+0000","last_became_active":"2023-09-23T18:37:10.997090+0000","last_became_peered":"2023-09-23T18:37:10.997090+0000","last_unstale":"2023-09-23T18:38:43.705551+0000","last_undegraded":"2023-09-23T18:38:43.705551+0000","last_fullsized":"2023-09-23T18:38:43.705551+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":5715,"log_dups_size":0,"ondisk_log_size":5715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:54.279665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374538240,"num_objects":5715,"num_object_clones":0,"num_object_copies":17145,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5715,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5715,"num_write_kb":365760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":139,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.972007+0000","last_change":"2023-09-23T18:33:53.855371+0000","last_active":"2023-09-23T18:38:10.972007+0000","last_peered":"2023-09-23T18:38:10.972007+0000","last_clean":"2023-09-23T18:38:10.972007+0000","last_became_active":"2023-09-23T18:33:53.854996+0000","last_became_peered":"2023-09-23T18:33:53.854996+0000","last_unstale":"2023-09-23T18:38:10.972007+0000","last_undegraded":"2023-09-23T18:38:10.972007+0000","last_fullsized":"2023-09-23T18:38:10.972007+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":153,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.973141+0000","last_change":"2023-09-23T18:33:53.845253+0000","last_active":"2023-09-23T18:38:10.973141+0000","last_peered":"2023-09-23T18:38:10.973141+0000","last_clean":"2023-09-23T18:38:10.973141+0000","last_became_active":"2023-09-23T18:33:53.844843+0000","last_became_peered":"2023-09-23T18:33:53.844843+0000","last_unstale":"2023-09-23T18:38:10.973141+0000","last_undegraded":"2023-09-23T18:38:10.973141+0000","last_fullsized":"2023-09-23T18:38:10.973141+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"158'5550","reported_seq":5568,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T18:38:41.583608+0000","last_change":"2023-09-23T18:37:11.000452+0000","last_active":"2023-09-23T18:38:41.583608+0000","last_peered":"2023-09-23T18:38:41.583608+0000","last_clean":"2023-09-23T18:38:41.583608+0000","last_became_active":"2023-09-23T18:37:11.000056+0000","last_became_peered":"2023-09-23T18:37:11.000056+0000","last_unstale":"2023-09-23T18:38:41.583608+0000","last_undegraded":"2023-09-23T18:38:41.583608+0000","last_fullsized":"2023-09-23T18:38:41.583608+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":5550,"log_dups_size":0,"ondisk_log_size":5550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:53.369250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363724800,"num_objects":5550,"num_object_clones":0,"num_object_copies":16650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5550,"num_write_kb":355200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":124,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.972781+0000","last_change":"2023-09-23T18:27:33.585714+0000","last_active":"2023-09-23T18:38:10.972781+0000","last_peered":"2023-09-23T18:38:10.972781+0000","last_clean":"2023-09-23T18:38:10.972781+0000","last_became_active":"2023-09-23T18:27:33.584641+0000","last_became_peered":"2023-09-23T18:27:33.584641+0000","last_unstale":"2023-09-23T18:38:10.972781+0000","last_undegraded":"2023-09-23T18:38:10.972781+0000","last_fullsized":"2023-09-23T18:38:10.972781+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":138,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.973581+0000","last_change":"2023-09-23T18:33:53.851266+0000","last_active":"2023-09-23T18:38:10.973581+0000","last_peered":"2023-09-23T18:38:10.973581+0000","last_clean":"2023-09-23T18:38:10.973581+0000","last_became_active":"2023-09-23T18:33:53.850926+0000","last_became_peered":"2023-09-23T18:33:53.850926+0000","last_unstale":"2023-09-23T18:38:10.973581+0000","last_undegraded":"2023-09-23T18:38:10.973581+0000","last_fullsized":"2023-09-23T18:38:10.973581+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"158'5487","reported_seq":5505,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T18:38:42.086688+0000","last_change":"2023-09-23T18:37:10.994424+0000","last_active":"2023-09-23T18:38:42.086688+0000","last_peered":"2023-09-23T18:38:42.086688+0000","last_clean":"2023-09-23T18:38:42.086688+0000","last_became_active":"2023-09-23T18:37:10.994110+0000","last_became_peered":"2023-09-23T18:37:10.994110+0000","last_unstale":"2023-09-23T18:38:42.086688+0000","last_undegraded":"2023-09-23T18:38:42.086688+0000","last_fullsized":"2023-09-23T18:38:42.086688+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":5487,"log_dups_size":0,"ondisk_log_size":5487,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:05.421139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359596032,"num_objects":5487,"num_object_clones":0,"num_object_copies":16461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5487,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5487,"num_write_kb":351168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":145,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.975232+0000","last_change":"2023-09-23T18:33:53.835731+0000","last_active":"2023-09-23T18:38:10.975232+0000","last_peered":"2023-09-23T18:38:10.975232+0000","last_clean":"2023-09-23T18:38:10.975232+0000","last_became_active":"2023-09-23T18:33:53.835238+0000","last_became_peered":"2023-09-23T18:33:53.835238+0000","last_unstale":"2023-09-23T18:38:10.975232+0000","last_undegraded":"2023-09-23T18:38:10.975232+0000","last_fullsized":"2023-09-23T18:38:10.975232+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:07:18.870358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":95,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:22.117347+0000","last_change":"2023-09-23T18:33:53.855430+0000","last_active":"2023-09-23T18:38:22.117347+0000","last_peered":"2023-09-23T18:38:22.117347+0000","last_clean":"2023-09-23T18:38:22.117347+0000","last_became_active":"2023-09-23T18:33:53.854935+0000","last_became_peered":"2023-09-23T18:33:53.854935+0000","last_unstale":"2023-09-23T18:38:22.117347+0000","last_undegraded":"2023-09-23T18:38:22.117347+0000","last_fullsized":"2023-09-23T18:38:22.117347+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:36.078411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"158'5922","reported_seq":5939,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T18:38:44.600047+0000","last_change":"2023-09-23T18:37:10.995736+0000","last_active":"2023-09-23T18:38:44.600047+0000","last_peered":"2023-09-23T18:38:44.600047+0000","last_clean":"2023-09-23T18:38:44.600047+0000","last_became_active":"2023-09-23T18:37:10.995437+0000","last_became_peered":"2023-09-23T18:37:10.995437+0000","last_unstale":"2023-09-23T18:38:44.600047+0000","last_undegraded":"2023-09-23T18:38:44.600047+0000","last_fullsized":"2023-09-23T18:38:44.600047+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":5922,"log_dups_size":0,"ondisk_log_size":5922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:39:17.818584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388104192,"num_objects":5922,"num_object_clones":0,"num_object_copies":17766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5922,"num_write_kb":379008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"77'130","reported_seq":621,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.959215+0000","last_change":"2023-09-23T18:33:53.850713+0000","last_active":"2023-09-23T18:38:10.959215+0000","last_peered":"2023-09-23T18:38:10.959215+0000","last_clean":"2023-09-23T18:38:10.959215+0000","last_became_active":"2023-09-23T18:33:53.850011+0000","last_became_peered":"2023-09-23T18:33:53.850011+0000","last_unstale":"2023-09-23T18:38:10.959215+0000","last_undegraded":"2023-09-23T18:38:10.959215+0000","last_fullsized":"2023-09-23T18:38:10.959215+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:58.943556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":772,"num_read_kb":1712,"num_write":126,"num_write_kb":2126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":134,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T18:38:30.973584+0000","last_change":"2023-09-23T18:32:26.793356+0000","last_active":"2023-09-23T18:38:30.973584+0000","last_peered":"2023-09-23T18:38:30.973584+0000","last_clean":"2023-09-23T18:38:30.973584+0000","last_became_active":"2023-09-23T18:32:26.792868+0000","last_became_peered":"2023-09-23T18:32:26.792868+0000","last_unstale":"2023-09-23T18:38:30.973584+0000","last_undegraded":"2023-09-23T18:38:30.973584+0000","last_fullsized":"2023-09-23T18:38:30.973584+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:40:21.193604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":69,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.959092+0000","last_change":"2023-09-23T18:33:54.843246+0000","last_active":"2023-09-23T18:38:10.959092+0000","last_peered":"2023-09-23T18:38:10.959092+0000","last_clean":"2023-09-23T18:38:10.959092+0000","last_became_active":"2023-09-23T18:33:54.842933+0000","last_became_peered":"2023-09-23T18:33:54.842933+0000","last_unstale":"2023-09-23T18:38:10.959092+0000","last_undegraded":"2023-09-23T18:38:10.959092+0000","last_fullsized":"2023-09-23T18:38:10.959092+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:43.716707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"158'5605","reported_seq":5623,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T18:38:41.638523+0000","last_change":"2023-09-23T18:37:11.000225+0000","last_active":"2023-09-23T18:38:41.638523+0000","last_peered":"2023-09-23T18:38:41.638523+0000","last_clean":"2023-09-23T18:38:41.638523+0000","last_became_active":"2023-09-23T18:37:10.999900+0000","last_became_peered":"2023-09-23T18:37:10.999900+0000","last_unstale":"2023-09-23T18:38:41.638523+0000","last_undegraded":"2023-09-23T18:38:41.638523+0000","last_fullsized":"2023-09-23T18:38:41.638523+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":5605,"log_dups_size":0,"ondisk_log_size":5605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:41.493477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367329280,"num_objects":5605,"num_object_clones":0,"num_object_copies":16815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5605,"num_write_kb":358720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"77'96","reported_seq":565,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.958961+0000","last_change":"2023-09-23T18:33:54.842857+0000","last_active":"2023-09-23T18:38:10.958961+0000","last_peered":"2023-09-23T18:38:10.958961+0000","last_clean":"2023-09-23T18:38:10.958961+0000","last_became_active":"2023-09-23T18:33:54.842134+0000","last_became_peered":"2023-09-23T18:33:54.842134+0000","last_unstale":"2023-09-23T18:38:10.958961+0000","last_undegraded":"2023-09-23T18:38:10.958961+0000","last_fullsized":"2023-09-23T18:38:10.958961+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":96,"log_dups_size":0,"ondisk_log_size":96,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:48.960158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":728,"num_read_kb":1234,"num_write":87,"num_write_kb":338,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":124,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.973151+0000","last_change":"2023-09-23T18:26:34.992535+0000","last_active":"2023-09-23T18:38:10.973151+0000","last_peered":"2023-09-23T18:38:10.973151+0000","last_clean":"2023-09-23T18:38:10.973151+0000","last_became_active":"2023-09-23T18:26:34.992177+0000","last_became_peered":"2023-09-23T18:26:34.992177+0000","last_unstale":"2023-09-23T18:38:10.973151+0000","last_undegraded":"2023-09-23T18:38:10.973151+0000","last_fullsized":"2023-09-23T18:38:10.973151+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":110,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.959135+0000","last_change":"2023-09-23T18:33:53.841776+0000","last_active":"2023-09-23T18:38:10.959135+0000","last_peered":"2023-09-23T18:38:10.959135+0000","last_clean":"2023-09-23T18:38:10.959135+0000","last_became_active":"2023-09-23T18:33:53.841436+0000","last_became_peered":"2023-09-23T18:33:53.841436+0000","last_unstale":"2023-09-23T18:38:10.959135+0000","last_undegraded":"2023-09-23T18:38:10.959135+0000","last_fullsized":"2023-09-23T18:38:10.959135+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:27.198179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"158'5848","reported_seq":5865,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T18:38:45.748225+0000","last_change":"2023-09-23T18:37:10.998898+0000","last_active":"2023-09-23T18:38:45.748225+0000","last_peered":"2023-09-23T18:38:45.748225+0000","last_clean":"2023-09-23T18:38:45.748225+0000","last_became_active":"2023-09-23T18:37:10.998064+0000","last_became_peered":"2023-09-23T18:37:10.998064+0000","last_unstale":"2023-09-23T18:38:45.748225+0000","last_undegraded":"2023-09-23T18:38:45.748225+0000","last_fullsized":"2023-09-23T18:38:45.748225+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":5848,"log_dups_size":0,"ondisk_log_size":5848,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383254528,"num_objects":5848,"num_object_clones":0,"num_object_copies":17544,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5848,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5848,"num_write_kb":374272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"77'112","reported_seq":597,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.965657+0000","last_change":"2023-09-23T18:28:03.297540+0000","last_active":"2023-09-23T18:38:10.965657+0000","last_peered":"2023-09-23T18:38:10.965657+0000","last_clean":"2023-09-23T18:38:10.965657+0000","last_became_active":"2023-09-23T18:28:03.296649+0000","last_became_peered":"2023-09-23T18:28:03.296649+0000","last_unstale":"2023-09-23T18:38:10.965657+0000","last_undegraded":"2023-09-23T18:38:10.965657+0000","last_fullsized":"2023-09-23T18:38:10.965657+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:32.531775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":774,"num_read_kb":596,"num_write":102,"num_write_kb":52,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":91,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.959239+0000","last_change":"2023-09-23T18:33:53.849084+0000","last_active":"2023-09-23T18:38:10.959239+0000","last_peered":"2023-09-23T18:38:10.959239+0000","last_clean":"2023-09-23T18:38:10.959239+0000","last_became_active":"2023-09-23T18:33:53.848757+0000","last_became_peered":"2023-09-23T18:33:53.848757+0000","last_unstale":"2023-09-23T18:38:10.959239+0000","last_undegraded":"2023-09-23T18:38:10.959239+0000","last_fullsized":"2023-09-23T18:38:10.959239+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:15.267742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":114,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:12.106256+0000","last_change":"2023-09-23T18:33:55.087304+0000","last_active":"2023-09-23T18:38:12.106256+0000","last_peered":"2023-09-23T18:38:12.106256+0000","last_clean":"2023-09-23T18:38:12.106256+0000","last_became_active":"2023-09-23T18:33:55.086832+0000","last_became_peered":"2023-09-23T18:33:55.086832+0000","last_unstale":"2023-09-23T18:38:12.106256+0000","last_undegraded":"2023-09-23T18:38:12.106256+0000","last_fullsized":"2023-09-23T18:38:12.106256+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:55.457664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"158'5641","reported_seq":5659,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T18:38:41.670241+0000","last_change":"2023-09-23T18:37:11.002449+0000","last_active":"2023-09-23T18:38:41.670241+0000","last_peered":"2023-09-23T18:38:41.670241+0000","last_clean":"2023-09-23T18:38:41.670241+0000","last_became_active":"2023-09-23T18:37:11.002022+0000","last_became_peered":"2023-09-23T18:37:11.002022+0000","last_unstale":"2023-09-23T18:38:41.670241+0000","last_undegraded":"2023-09-23T18:38:41.670241+0000","last_fullsized":"2023-09-23T18:38:41.670241+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":5641,"log_dups_size":0,"ondisk_log_size":5641,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:34.756717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369688576,"num_objects":5641,"num_object_clones":0,"num_object_copies":16923,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5641,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5641,"num_write_kb":361024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"77'147","reported_seq":546,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T18:38:30.978403+0000","last_change":"2023-09-23T18:32:29.845404+0000","last_active":"2023-09-23T18:38:30.978403+0000","last_peered":"2023-09-23T18:38:30.978403+0000","last_clean":"2023-09-23T18:38:30.978403+0000","last_became_active":"2023-09-23T18:32:29.844545+0000","last_became_peered":"2023-09-23T18:32:29.844545+0000","last_unstale":"2023-09-23T18:38:30.978403+0000","last_undegraded":"2023-09-23T18:38:30.978403+0000","last_fullsized":"2023-09-23T18:38:30.978403+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:44.834482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":624,"num_read_kb":1868,"num_write":143,"num_write_kb":5368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":360,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.969014+0000","last_change":"2023-09-23T18:26:34.999402+0000","last_active":"2023-09-23T18:38:10.969014+0000","last_peered":"2023-09-23T18:38:10.969014+0000","last_clean":"2023-09-23T18:38:10.969014+0000","last_became_active":"2023-09-23T18:26:34.998828+0000","last_became_peered":"2023-09-23T18:26:34.998828+0000","last_unstale":"2023-09-23T18:38:10.969014+0000","last_undegraded":"2023-09-23T18:38:10.969014+0000","last_fullsized":"2023-09-23T18:38:10.969014+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_clean_scrub_stamp":"2023-09-23T18:08:35.701374+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":130,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:26.629139+0000","last_change":"2023-09-23T18:32:26.800920+0000","last_active":"2023-09-23T18:38:26.629139+0000","last_peered":"2023-09-23T18:38:26.629139+0000","last_clean":"2023-09-23T18:38:26.629139+0000","last_became_active":"2023-09-23T18:32:26.800528+0000","last_became_peered":"2023-09-23T18:32:26.800528+0000","last_unstale":"2023-09-23T18:38:26.629139+0000","last_undegraded":"2023-09-23T18:38:26.629139+0000","last_fullsized":"2023-09-23T18:38:26.629139+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:20:50.939219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":108,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.971834+0000","last_change":"2023-09-23T18:15:08.921629+0000","last_active":"2023-09-23T18:38:10.971834+0000","last_peered":"2023-09-23T18:38:10.971834+0000","last_clean":"2023-09-23T18:38:10.971834+0000","last_became_active":"2023-09-23T18:15:08.921283+0000","last_became_peered":"2023-09-23T18:15:08.921283+0000","last_unstale":"2023-09-23T18:38:10.971834+0000","last_undegraded":"2023-09-23T18:38:10.971834+0000","last_fullsized":"2023-09-23T18:38:10.971834+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"158'5774","reported_seq":5792,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T18:38:42.131003+0000","last_change":"2023-09-23T18:37:11.001959+0000","last_active":"2023-09-23T18:38:42.131003+0000","last_peered":"2023-09-23T18:38:42.131003+0000","last_clean":"2023-09-23T18:38:42.131003+0000","last_became_active":"2023-09-23T18:37:11.001542+0000","last_became_peered":"2023-09-23T18:37:11.001542+0000","last_unstale":"2023-09-23T18:38:42.131003+0000","last_undegraded":"2023-09-23T18:38:42.131003+0000","last_fullsized":"2023-09-23T18:38:42.131003+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":5774,"log_dups_size":0,"ondisk_log_size":5774,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:27.036499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378404864,"num_objects":5774,"num_object_clones":0,"num_object_copies":17322,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5774,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5774,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"77'120","reported_seq":619,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.969051+0000","last_change":"2023-09-23T18:33:53.852697+0000","last_active":"2023-09-23T18:38:10.969051+0000","last_peered":"2023-09-23T18:38:10.969051+0000","last_clean":"2023-09-23T18:38:10.969051+0000","last_became_active":"2023-09-23T18:33:53.852210+0000","last_became_peered":"2023-09-23T18:33:53.852210+0000","last_unstale":"2023-09-23T18:38:10.969051+0000","last_undegraded":"2023-09-23T18:38:10.969051+0000","last_fullsized":"2023-09-23T18:38:10.969051+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":120,"log_dups_size":0,"ondisk_log_size":120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":809,"num_read_kb":2749,"num_write":98,"num_write_kb":2087,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":110,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.967751+0000","last_change":"2023-09-23T18:28:03.299508+0000","last_active":"2023-09-23T18:38:10.967751+0000","last_peered":"2023-09-23T18:38:10.967751+0000","last_clean":"2023-09-23T18:38:10.967751+0000","last_became_active":"2023-09-23T18:28:03.298682+0000","last_became_peered":"2023-09-23T18:28:03.298682+0000","last_unstale":"2023-09-23T18:38:10.967751+0000","last_undegraded":"2023-09-23T18:38:10.967751+0000","last_fullsized":"2023-09-23T18:38:10.967751+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:08.398302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":141,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.968074+0000","last_change":"2023-09-23T18:33:53.846904+0000","last_active":"2023-09-23T18:38:10.968074+0000","last_peered":"2023-09-23T18:38:10.968074+0000","last_clean":"2023-09-23T18:38:10.968074+0000","last_became_active":"2023-09-23T18:33:53.846654+0000","last_became_peered":"2023-09-23T18:33:53.846654+0000","last_unstale":"2023-09-23T18:38:10.968074+0000","last_undegraded":"2023-09-23T18:38:10.968074+0000","last_fullsized":"2023-09-23T18:38:10.968074+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:55.898016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"158'5743","reported_seq":5761,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T18:38:42.091698+0000","last_change":"2023-09-23T18:37:11.001841+0000","last_active":"2023-09-23T18:38:42.091698+0000","last_peered":"2023-09-23T18:38:42.091698+0000","last_clean":"2023-09-23T18:38:42.091698+0000","last_became_active":"2023-09-23T18:37:11.001499+0000","last_became_peered":"2023-09-23T18:37:11.001499+0000","last_unstale":"2023-09-23T18:38:42.091698+0000","last_undegraded":"2023-09-23T18:38:42.091698+0000","last_fullsized":"2023-09-23T18:38:42.091698+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":5743,"log_dups_size":0,"ondisk_log_size":5743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:32.700925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376373248,"num_objects":5743,"num_object_clones":0,"num_object_copies":17229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5743,"num_write_kb":367552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'78","reported_seq":394,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.969571+0000","last_change":"2023-09-23T18:26:34.992345+0000","last_active":"2023-09-23T18:38:10.969571+0000","last_peered":"2023-09-23T18:38:10.969571+0000","last_clean":"2023-09-23T18:38:10.969571+0000","last_became_active":"2023-09-23T18:26:34.991774+0000","last_became_peered":"2023-09-23T18:26:34.991774+0000","last_unstale":"2023-09-23T18:38:10.969571+0000","last_undegraded":"2023-09-23T18:38:10.969571+0000","last_fullsized":"2023-09-23T18:38:10.969571+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":78,"log_dups_size":0,"ondisk_log_size":78,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":428,"num_read_kb":1125,"num_write":73,"num_write_kb":431,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":89,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.959300+0000","last_change":"2023-09-23T18:33:54.897293+0000","last_active":"2023-09-23T18:38:10.959300+0000","last_peered":"2023-09-23T18:38:10.959300+0000","last_clean":"2023-09-23T18:38:10.959300+0000","last_became_active":"2023-09-23T18:33:54.896491+0000","last_became_peered":"2023-09-23T18:33:54.896491+0000","last_unstale":"2023-09-23T18:38:10.959300+0000","last_undegraded":"2023-09-23T18:38:10.959300+0000","last_fullsized":"2023-09-23T18:38:10.959300+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:03.636617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":116,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.968214+0000","last_change":"2023-09-23T18:33:54.852309+0000","last_active":"2023-09-23T18:38:10.968214+0000","last_peered":"2023-09-23T18:38:10.968214+0000","last_clean":"2023-09-23T18:38:10.968214+0000","last_became_active":"2023-09-23T18:33:54.850255+0000","last_became_peered":"2023-09-23T18:33:54.850255+0000","last_unstale":"2023-09-23T18:38:10.968214+0000","last_undegraded":"2023-09-23T18:38:10.968214+0000","last_fullsized":"2023-09-23T18:38:10.968214+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:54.148992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"158'5489","reported_seq":5506,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T18:38:41.045118+0000","last_change":"2023-09-23T18:37:11.002174+0000","last_active":"2023-09-23T18:38:41.045118+0000","last_peered":"2023-09-23T18:38:41.045118+0000","last_clean":"2023-09-23T18:38:41.045118+0000","last_became_active":"2023-09-23T18:37:11.001860+0000","last_became_peered":"2023-09-23T18:37:11.001860+0000","last_unstale":"2023-09-23T18:38:41.045118+0000","last_undegraded":"2023-09-23T18:38:41.045118+0000","last_fullsized":"2023-09-23T18:38:41.045118+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":5489,"log_dups_size":0,"ondisk_log_size":5489,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359727104,"num_objects":5489,"num_object_clones":0,"num_object_copies":16467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5489,"num_write_kb":351296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":128,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.981371+0000","last_change":"2023-09-23T18:28:03.298805+0000","last_active":"2023-09-23T18:38:10.981371+0000","last_peered":"2023-09-23T18:38:10.981371+0000","last_clean":"2023-09-23T18:38:10.981371+0000","last_became_active":"2023-09-23T18:28:03.298437+0000","last_became_peered":"2023-09-23T18:28:03.298437+0000","last_unstale":"2023-09-23T18:38:10.981371+0000","last_undegraded":"2023-09-23T18:38:10.981371+0000","last_fullsized":"2023-09-23T18:38:10.981371+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:27.831669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'181","reported_seq":629,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:12.106226+0000","last_change":"2023-09-23T18:33:53.854528+0000","last_active":"2023-09-23T18:38:12.106226+0000","last_peered":"2023-09-23T18:38:12.106226+0000","last_clean":"2023-09-23T18:38:12.106226+0000","last_became_active":"2023-09-23T18:33:53.851883+0000","last_became_peered":"2023-09-23T18:33:53.851883+0000","last_unstale":"2023-09-23T18:38:12.106226+0000","last_undegraded":"2023-09-23T18:38:12.106226+0000","last_fullsized":"2023-09-23T18:38:12.106226+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":181,"log_dups_size":0,"ondisk_log_size":181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:43.888844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":909,"num_read_kb":734,"num_write":148,"num_write_kb":75,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":126,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.976508+0000","last_change":"2023-09-23T18:28:03.292295+0000","last_active":"2023-09-23T18:38:10.976508+0000","last_peered":"2023-09-23T18:38:10.976508+0000","last_clean":"2023-09-23T18:38:10.976508+0000","last_became_active":"2023-09-23T18:28:03.291902+0000","last_became_peered":"2023-09-23T18:28:03.291902+0000","last_unstale":"2023-09-23T18:38:10.976508+0000","last_undegraded":"2023-09-23T18:38:10.976508+0000","last_fullsized":"2023-09-23T18:38:10.976508+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:38.415256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"158'5889","reported_seq":5906,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T18:38:45.742744+0000","last_change":"2023-09-23T18:37:10.998526+0000","last_active":"2023-09-23T18:38:45.742744+0000","last_peered":"2023-09-23T18:38:45.742744+0000","last_clean":"2023-09-23T18:38:45.742744+0000","last_became_active":"2023-09-23T18:37:10.998129+0000","last_became_peered":"2023-09-23T18:37:10.998129+0000","last_unstale":"2023-09-23T18:38:45.742744+0000","last_undegraded":"2023-09-23T18:38:45.742744+0000","last_fullsized":"2023-09-23T18:38:45.742744+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":5889,"log_dups_size":0,"ondisk_log_size":5889,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":385941504,"num_objects":5889,"num_object_clones":0,"num_object_copies":17667,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5889,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5889,"num_write_kb":376896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"77'112","reported_seq":648,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.971796+0000","last_change":"2023-09-23T18:27:33.587478+0000","last_active":"2023-09-23T18:38:10.971796+0000","last_peered":"2023-09-23T18:38:10.971796+0000","last_clean":"2023-09-23T18:38:10.971796+0000","last_became_active":"2023-09-23T18:27:33.586958+0000","last_became_peered":"2023-09-23T18:27:33.586958+0000","last_unstale":"2023-09-23T18:38:10.971796+0000","last_undegraded":"2023-09-23T18:38:10.971796+0000","last_fullsized":"2023-09-23T18:38:10.971796+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":893,"num_read_kb":3172,"num_write":69,"num_write_kb":2237,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":129,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.967987+0000","last_change":"2023-09-23T18:33:53.851421+0000","last_active":"2023-09-23T18:38:10.967987+0000","last_peered":"2023-09-23T18:38:10.967987+0000","last_clean":"2023-09-23T18:38:10.967987+0000","last_became_active":"2023-09-23T18:33:53.851012+0000","last_became_peered":"2023-09-23T18:33:53.851012+0000","last_unstale":"2023-09-23T18:38:10.967987+0000","last_undegraded":"2023-09-23T18:38:10.967987+0000","last_fullsized":"2023-09-23T18:38:10.967987+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:03.971054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":117,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.972855+0000","last_change":"2023-09-23T18:32:26.805472+0000","last_active":"2023-09-23T18:38:10.972855+0000","last_peered":"2023-09-23T18:38:10.972855+0000","last_clean":"2023-09-23T18:38:10.972855+0000","last_became_active":"2023-09-23T18:32:26.805078+0000","last_became_peered":"2023-09-23T18:32:26.805078+0000","last_unstale":"2023-09-23T18:38:10.972855+0000","last_undegraded":"2023-09-23T18:38:10.972855+0000","last_fullsized":"2023-09-23T18:38:10.972855+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":98,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.968158+0000","last_change":"2023-09-23T18:26:34.994176+0000","last_active":"2023-09-23T18:38:10.968158+0000","last_peered":"2023-09-23T18:38:10.968158+0000","last_clean":"2023-09-23T18:38:10.968158+0000","last_became_active":"2023-09-23T18:26:34.993829+0000","last_became_peered":"2023-09-23T18:26:34.993829+0000","last_unstale":"2023-09-23T18:38:10.968158+0000","last_undegraded":"2023-09-23T18:38:10.968158+0000","last_fullsized":"2023-09-23T18:38:10.968158+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:27.111857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":164,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.963487+0000","last_change":"2023-09-23T18:33:53.840519+0000","last_active":"2023-09-23T18:38:10.963487+0000","last_peered":"2023-09-23T18:38:10.963487+0000","last_clean":"2023-09-23T18:38:10.963487+0000","last_became_active":"2023-09-23T18:33:53.840181+0000","last_became_peered":"2023-09-23T18:33:53.840181+0000","last_unstale":"2023-09-23T18:38:10.963487+0000","last_undegraded":"2023-09-23T18:38:10.963487+0000","last_fullsized":"2023-09-23T18:38:10.963487+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:41.293009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":81,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.973385+0000","last_change":"2023-09-23T18:33:54.847172+0000","last_active":"2023-09-23T18:38:10.973385+0000","last_peered":"2023-09-23T18:38:10.973385+0000","last_clean":"2023-09-23T18:38:10.973385+0000","last_became_active":"2023-09-23T18:33:54.846751+0000","last_became_peered":"2023-09-23T18:33:54.846751+0000","last_unstale":"2023-09-23T18:38:10.973385+0000","last_undegraded":"2023-09-23T18:38:10.973385+0000","last_fullsized":"2023-09-23T18:38:10.973385+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":153,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.973205+0000","last_change":"2023-09-23T18:33:53.841232+0000","last_active":"2023-09-23T18:38:10.973205+0000","last_peered":"2023-09-23T18:38:10.973205+0000","last_clean":"2023-09-23T18:38:10.973205+0000","last_became_active":"2023-09-23T18:33:53.840749+0000","last_became_peered":"2023-09-23T18:33:53.840749+0000","last_unstale":"2023-09-23T18:38:10.973205+0000","last_undegraded":"2023-09-23T18:38:10.973205+0000","last_fullsized":"2023-09-23T18:38:10.973205+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":81,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.973201+0000","last_change":"2023-09-23T18:33:54.858971+0000","last_active":"2023-09-23T18:38:10.973201+0000","last_peered":"2023-09-23T18:38:10.973201+0000","last_clean":"2023-09-23T18:38:10.973201+0000","last_became_active":"2023-09-23T18:33:54.845921+0000","last_became_peered":"2023-09-23T18:33:54.845921+0000","last_unstale":"2023-09-23T18:38:10.973201+0000","last_undegraded":"2023-09-23T18:38:10.973201+0000","last_fullsized":"2023-09-23T18:38:10.973201+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":138,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.973053+0000","last_change":"2023-09-23T18:28:03.302261+0000","last_active":"2023-09-23T18:38:10.973053+0000","last_peered":"2023-09-23T18:38:10.973053+0000","last_clean":"2023-09-23T18:38:10.973053+0000","last_became_active":"2023-09-23T18:28:03.301911+0000","last_became_peered":"2023-09-23T18:28:03.301911+0000","last_unstale":"2023-09-23T18:38:10.973053+0000","last_undegraded":"2023-09-23T18:38:10.973053+0000","last_fullsized":"2023-09-23T18:38:10.973053+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":136,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.978396+0000","last_change":"2023-09-23T18:28:03.296850+0000","last_active":"2023-09-23T18:38:10.978396+0000","last_peered":"2023-09-23T18:38:10.978396+0000","last_clean":"2023-09-23T18:38:10.978396+0000","last_became_active":"2023-09-23T18:28:03.295842+0000","last_became_peered":"2023-09-23T18:28:03.295842+0000","last_unstale":"2023-09-23T18:38:10.978396+0000","last_undegraded":"2023-09-23T18:38:10.978396+0000","last_fullsized":"2023-09-23T18:38:10.978396+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:53.048579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":148,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.964871+0000","last_change":"2023-09-23T18:28:03.299520+0000","last_active":"2023-09-23T18:38:10.964871+0000","last_peered":"2023-09-23T18:38:10.964871+0000","last_clean":"2023-09-23T18:38:10.964871+0000","last_became_active":"2023-09-23T18:28:03.299158+0000","last_became_peered":"2023-09-23T18:28:03.299158+0000","last_unstale":"2023-09-23T18:38:10.964871+0000","last_undegraded":"2023-09-23T18:38:10.964871+0000","last_fullsized":"2023-09-23T18:38:10.964871+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:26.247064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":129,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.968285+0000","last_change":"2023-09-23T18:32:26.805676+0000","last_active":"2023-09-23T18:38:10.968285+0000","last_peered":"2023-09-23T18:38:10.968285+0000","last_clean":"2023-09-23T18:38:10.968285+0000","last_became_active":"2023-09-23T18:32:26.797129+0000","last_became_peered":"2023-09-23T18:32:26.797129+0000","last_unstale":"2023-09-23T18:38:10.968285+0000","last_undegraded":"2023-09-23T18:38:10.968285+0000","last_fullsized":"2023-09-23T18:38:10.968285+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:07.414550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":137,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T18:38:10.972757+0000","last_change":"2023-09-23T18:33:54.847156+0000","last_active":"2023-09-23T18:38:10.972757+0000","last_peered":"2023-09-23T18:38:10.972757+0000","last_clean":"2023-09-23T18:38:10.972757+0000","last_became_active":"2023-09-23T18:33:54.846765+0000","last_became_peered":"2023-09-23T18:33:54.846765+0000","last_unstale":"2023-09-23T18:38:10.972757+0000","last_undegraded":"2023-09-23T18:38:10.972757+0000","last_fullsized":"2023-09-23T18:38:10.972757+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":6002900992,"num_objects":91597,"num_object_clones":0,"num_object_copies":274791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":91597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":91597,"num_write_kb":5862208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18053201920,"data_stored":18053201920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91597,"ondisk_log_size":91597,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":646,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26886,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5937,"num_read_kb":13190,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5621,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430531,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2914568,"kb_used_data":2605056,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":90830584,"statfs":{"total":95995035648,"available":93010518016,"internally_reserved":0,"allocated":2667577344,"data_stored":2667015713,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,3,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:37:59 2023","interfaces":[{"interface":"back","average":{"1min":1.6830000000000001,"5min":1.5249999999999999,"15min":1.4570000000000001},"min":{"1min":0.33700000000000002,"5min":0.26200000000000001,"15min":0.252},"max":{"1min":17.530000000000001,"5min":17.530000000000001,"15min":57.049999999999997},"last":0.626},{"interface":"front","average":{"1min":1.7629999999999999,"5min":1.613,"15min":1.48},"min":{"1min":0.53900000000000003,"5min":0.28599999999999998,"15min":0.22900000000000001},"max":{"1min":14.186999999999999,"5min":14.186999999999999,"15min":56.773000000000003},"last":0.317}]},{"osd":1,"last update":"Sat Sep 23 18:37:59 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":1.302,"15min":1.1799999999999999},"min":{"1min":0.42099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":13.186,"5min":13.186,"15min":13.186},"last":0.92500000000000004},{"interface":"front","average":{"1min":1.647,"5min":1.3859999999999999,"15min":1.393},"min":{"1min":0.47499999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":13.166,"5min":13.166,"15min":13.166},"last":3.3340000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:37:54 2023","interfaces":[{"interface":"back","average":{"1min":0.77000000000000002,"5min":1.3520000000000001,"15min":1.2569999999999999},"min":{"1min":0.35799999999999998,"5min":0.28899999999999998,"15min":0.254},"max":{"1min":1.212,"5min":10.064,"15min":40.091999999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.3100000000000001,"15min":1.2090000000000001},"min":{"1min":0.59299999999999997,"5min":0.309,"15min":0.29999999999999999},"max":{"1min":2.6560000000000001,"5min":9.2669999999999995,"15min":38.261000000000003},"last":0.66800000000000004}]},{"osd":3,"last update":"Sat Sep 23 18:38:22 2023","interfaces":[{"interface":"back","average":{"1min":2.8580000000000001,"5min":1.3779999999999999,"15min":1.1060000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.32800000000000001},"max":{"1min":13.465,"5min":13.465,"15min":13.465},"last":1.77},{"interface":"front","average":{"1min":2.5339999999999998,"5min":1.302,"15min":1.0780000000000001},"min":{"1min":0.46000000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":13.465999999999999,"5min":13.465999999999999,"15min":13.465999999999999},"last":3.3900000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:38:02 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":1.0329999999999999,"15min":1.0089999999999999},"min":{"1min":0.34100000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.7829999999999999,"5min":6.3529999999999998,"15min":6.3529999999999998},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.91300000000000003,"5min":0.91700000000000004,"15min":1.1140000000000001},"min":{"1min":0.496,"5min":0.32500000000000001,"15min":0.309},"max":{"1min":1.831,"5min":4.3680000000000003,"15min":11.119},"last":0.69799999999999995}]},{"osd":5,"last update":"Sat Sep 23 18:38:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":0.95199999999999996,"15min":1.0169999999999999},"min":{"1min":0.432,"5min":0.318,"15min":0.315},"max":{"1min":2.294,"5min":4.6559999999999997,"15min":5.8899999999999997},"last":3.4580000000000002},{"interface":"front","average":{"1min":1.0609999999999999,"5min":1.1259999999999999,"15min":1.095},"min":{"1min":0.47199999999999998,"5min":0.38500000000000001,"15min":0.26800000000000002},"max":{"1min":2.8180000000000001,"5min":7.46,"15min":7.46},"last":0.80700000000000005}]},{"osd":6,"last update":"Sat Sep 23 18:38:02 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.0189999999999999,"15min":1.0760000000000001},"min":{"1min":0.55500000000000005,"5min":0.39400000000000002,"15min":0.35299999999999998},"max":{"1min":2.8759999999999999,"5min":5.5129999999999999,"15min":5.5129999999999999},"last":0.84199999999999997},{"interface":"front","average":{"1min":1.081,"5min":1.0800000000000001,"15min":1.1180000000000001},"min":{"1min":0.433,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":2.5310000000000001,"5min":9.1850000000000005,"15min":9.1850000000000005},"last":0.95499999999999996}]}]},{"osd":6,"up_from":42,"seq":180388626763,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2554048,"kb_used_data":2256824,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91191104,"statfs":{"total":95995035648,"available":93379690496,"internally_reserved":0,"allocated":2310987776,"data_stored":2310434449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:37:58 2023","interfaces":[{"interface":"back","average":{"1min":5.1719999999999997,"5min":2.069,"15min":1.9119999999999999},"min":{"1min":0.36399999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":66.599000000000004,"5min":66.599000000000004,"15min":182.655},"last":2.1000000000000001},{"interface":"front","average":{"1min":5.2880000000000003,"5min":2.1549999999999998,"15min":2.0249999999999999},"min":{"1min":0.443,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":66.527000000000001,"5min":66.527000000000001,"15min":182.58799999999999},"last":2.153}]},{"osd":1,"last update":"Sat Sep 23 18:37:58 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":0.81000000000000005,"15min":0.78300000000000003},"min":{"1min":0.36799999999999999,"5min":0.317,"15min":0.317},"max":{"1min":3.1309999999999998,"5min":3.3340000000000001,"15min":3.3340000000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.82199999999999995,"5min":0.91400000000000003,"15min":1.044},"min":{"1min":0.34699999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.73,"5min":3.8359999999999999,"15min":3.8359999999999999},"last":0.38600000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:37:53 2023","interfaces":[{"interface":"back","average":{"1min":11.901999999999999,"5min":3.621,"15min":2.0600000000000001},"min":{"1min":0.67200000000000004,"5min":0.371,"15min":0.28100000000000003},"max":{"1min":122.242,"5min":122.242,"15min":122.242},"last":0.53400000000000003},{"interface":"front","average":{"1min":11.997,"5min":3.6400000000000001,"15min":2.048},"min":{"1min":0.53900000000000003,"5min":0.34699999999999998,"15min":0.28799999999999998},"max":{"1min":122.32899999999999,"5min":122.32899999999999,"15min":122.32899999999999},"last":0.45100000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:38:19 2023","interfaces":[{"interface":"back","average":{"1min":13.074999999999999,"5min":3.274,"15min":1.7749999999999999},"min":{"1min":0.49199999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":123.52200000000001,"5min":123.52200000000001,"15min":123.52200000000001},"last":0.55400000000000005},{"interface":"front","average":{"1min":13.092000000000001,"5min":3.3460000000000001,"15min":1.8240000000000001},"min":{"1min":0.49199999999999999,"5min":0.36699999999999999,"15min":0.36099999999999999},"max":{"1min":123.605,"5min":123.605,"15min":123.605},"last":0.499}]},{"osd":4,"last update":"Sat Sep 23 18:38:16 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.84599999999999997,"15min":0.93999999999999995},"min":{"1min":0.48999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.7170000000000001,"5min":4.3940000000000001,"15min":8.0169999999999995},"last":0.69799999999999995},{"interface":"front","average":{"1min":1.0149999999999999,"5min":0.94499999999999995,"15min":0.96999999999999997},"min":{"1min":0.36299999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":2.3130000000000002,"5min":4.2539999999999996,"15min":4.2539999999999996},"last":0.48299999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:38:16 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.79800000000000004,"15min":0.96799999999999997},"min":{"1min":0.46200000000000002,"5min":0.35999999999999999,"15min":0.32000000000000001},"max":{"1min":1.5509999999999999,"5min":4.5640000000000001,"15min":7.9240000000000004},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.83599999999999997,"15min":0.96999999999999997},"min":{"1min":0.435,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":2.3679999999999999,"5min":4.5270000000000001,"15min":4.5270000000000001},"last":0.61799999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":0.94499999999999995,"15min":1.0329999999999999},"min":{"1min":0.41599999999999998,"5min":0.39800000000000002,"15min":0.39500000000000002},"max":{"1min":1.3220000000000001,"5min":4.6120000000000001,"15min":4.6120000000000001},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.262,"5min":1.1619999999999999,"15min":1.216},"min":{"1min":0.79600000000000004,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":2.7999999999999998,"5min":7.9710000000000001,"15min":9.1129999999999995},"last":0.65800000000000003}]}]},{"osd":1,"up_from":123,"seq":528280977467,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2299192,"kb_used_data":2172720,"kb_used_omap":7,"kb_used_meta":126456,"kb_avail":91445960,"statfs":{"total":95995035648,"available":93640663040,"internally_reserved":0,"allocated":2224865280,"data_stored":2224087217,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":129491726},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,5,2,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:38:02 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":6.5979999999999999,"15min":8.3200000000000003},"min":{"1min":0.22800000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":6.2220000000000004,"5min":173.41800000000001,"15min":173.41800000000001},"last":1.4119999999999999},{"interface":"front","average":{"1min":0.996,"5min":6.3200000000000003,"15min":7.8120000000000003},"min":{"1min":0.28599999999999998,"5min":0.249,"15min":0.249},"max":{"1min":3.3090000000000002,"5min":166.42599999999999,"15min":166.42599999999999},"last":3.5350000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:38:02 2023","interfaces":[{"interface":"back","average":{"1min":1.208,"5min":2.266,"15min":2.5899999999999999},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":5.9509999999999996,"5min":23.074999999999999,"15min":23.074999999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.337,"5min":2.262,"15min":2.2919999999999998},"min":{"1min":0.32200000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.3040000000000003,"5min":23.129000000000001,"15min":23.129000000000001},"last":3.972}]},{"osd":3,"last update":"Sat Sep 23 18:38:02 2023","interfaces":[{"interface":"back","average":{"1min":3.9689999999999999,"5min":1.647,"15min":1.2270000000000001},"min":{"1min":0.34399999999999997,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":55.014000000000003,"5min":55.014000000000003,"15min":55.014000000000003},"last":3.423},{"interface":"front","average":{"1min":4.1139999999999999,"5min":1.698,"15min":1.3620000000000001},"min":{"1min":0.34999999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":55.841999999999999,"5min":55.841999999999999,"15min":55.841999999999999},"last":0.51500000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:38:02 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":1.218,"15min":1.1950000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":3.3279999999999998,"5min":5.5510000000000002,"15min":5.5510000000000002},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.155,"5min":1.29,"15min":1.4379999999999999},"min":{"1min":0.439,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.0670000000000002,"5min":6.3129999999999997,"15min":6.3129999999999997},"last":0.33800000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:38:02 2023","interfaces":[{"interface":"back","average":{"1min":1.1359999999999999,"5min":1.0640000000000001,"15min":0.83099999999999996},"min":{"1min":0.46700000000000003,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":3.234,"5min":6.1200000000000001,"15min":6.1200000000000001},"last":3.5139999999999998},{"interface":"front","average":{"1min":1.155,"5min":1.2569999999999999,"15min":1.381},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":3.2530000000000001,"5min":6.5250000000000004,"15min":6.5250000000000004},"last":0.57599999999999996}]},{"osd":6,"last update":"Sat Sep 23 18:38:02 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":1.4099999999999999,"15min":1.534},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":3.2120000000000002,"5min":6.4859999999999998,"15min":6.4859999999999998},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.1919999999999999,"5min":1.413,"15min":1.665},"min":{"1min":0.45700000000000002,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":6.0919999999999996,"5min":6.1109999999999998,"15min":6.1109999999999998},"last":0.374}]},{"osd":7,"last update":"Sat Sep 23 18:38:02 2023","interfaces":[{"interface":"back","average":{"1min":1.3089999999999999,"5min":1.5529999999999999,"15min":1.821},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":3.286,"5min":8.1940000000000008,"15min":8.1940000000000008},"last":0.65500000000000003},{"interface":"front","average":{"1min":1.294,"5min":1.5700000000000001,"15min":1.738},"min":{"1min":0.48899999999999999,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":5.9980000000000002,"5min":6.3899999999999997,"15min":6.3899999999999997},"last":0.80200000000000005}]}]},{"osd":0,"up_from":88,"seq":377957122196,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1798748,"kb_used_data":1483540,"kb_used_omap":8,"kb_used_meta":315191,"kb_avail":91946404,"statfs":{"total":95995035648,"available":94153117696,"internally_reserved":0,"allocated":1519144960,"data_stored":1518391924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":322756478},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:38:04 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":1.4490000000000001,"15min":1.7549999999999999},"min":{"1min":0.245,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":4.8070000000000004,"5min":8.1140000000000008,"15min":8.1140000000000008},"last":0.89800000000000002},{"interface":"front","average":{"1min":0.95199999999999996,"5min":1.7250000000000001,"15min":2.242},"min":{"1min":0.34699999999999998,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":2.9620000000000002,"5min":7.6040000000000001,"15min":7.6040000000000001},"last":1.659}]},{"osd":2,"last update":"Sat Sep 23 18:37:48 2023","interfaces":[{"interface":"back","average":{"1min":7.468,"5min":3.7509999999999999,"15min":1.823},"min":{"1min":0.45000000000000001,"5min":0.22600000000000001,"15min":0.192},"max":{"1min":108.73999999999999,"5min":108.73999999999999,"15min":108.73999999999999},"last":0.71499999999999997},{"interface":"front","average":{"1min":7.2930000000000001,"5min":3.8410000000000002,"15min":1.7529999999999999},"min":{"1min":0.32200000000000001,"5min":0.20899999999999999,"15min":0.20799999999999999},"max":{"1min":108.136,"5min":108.136,"15min":108.136},"last":1.383}]},{"osd":3,"last update":"Sat Sep 23 18:38:17 2023","interfaces":[{"interface":"back","average":{"1min":10.784000000000001,"5min":3.8010000000000002,"15min":1.8939999999999999},"min":{"1min":0.26500000000000001,"5min":0.216,"15min":0.216},"max":{"1min":111.32599999999999,"5min":111.32599999999999,"15min":111.32599999999999},"last":0.624},{"interface":"front","average":{"1min":10.532999999999999,"5min":3.7559999999999998,"15min":1.909},"min":{"1min":0.375,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":111.346,"5min":111.346,"15min":111.346},"last":0.96599999999999997}]},{"osd":4,"last update":"Sat Sep 23 18:38:04 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":1.494,"15min":1.171},"min":{"1min":0.442,"5min":0.29899999999999999,"15min":0.28199999999999997},"max":{"1min":3.1520000000000001,"5min":6.6879999999999997,"15min":6.6879999999999997},"last":0.435},{"interface":"front","average":{"1min":1.23,"5min":1.859,"15min":1.345},"min":{"1min":0.35999999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":10.561999999999999,"5min":16.486999999999998,"15min":16.486999999999998},"last":0.48699999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:38:04 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":1.587,"15min":1.2310000000000001},"min":{"1min":0.443,"5min":0.371,"15min":0.371},"max":{"1min":10.606,"5min":10.606,"15min":10.606},"last":0.85499999999999998},{"interface":"front","average":{"1min":0.755,"5min":1.4950000000000001,"15min":1.212},"min":{"1min":0.42699999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.5600000000000001,"5min":8.9250000000000007,"15min":8.9250000000000007},"last":1.012}]},{"osd":6,"last update":"Sat Sep 23 18:38:04 2023","interfaces":[{"interface":"back","average":{"1min":1.226,"5min":1.655,"15min":1.2909999999999999},"min":{"1min":0.39200000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":10.634,"5min":10.634,"15min":10.634},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.78100000000000003,"5min":1.6259999999999999,"15min":1.2430000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.647,"5min":8.9740000000000002,"15min":8.9740000000000002},"last":1.1990000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:38:04 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":1.8080000000000001,"15min":1.3109999999999999},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.39200000000000002},"max":{"1min":1.6100000000000001,"5min":9.0600000000000005,"15min":9.0600000000000005},"last":1.127},{"interface":"front","average":{"1min":1.129,"5min":1.5800000000000001,"15min":1.24},"min":{"1min":0.5,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.46,"5min":6.7229999999999999,"15min":6.7229999999999999},"last":0.47099999999999997}]}]},{"osd":2,"up_from":98,"seq":420906795144,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4034696,"kb_used_data":3679296,"kb_used_omap":11,"kb_used_meta":355380,"kb_avail":89710456,"statfs":{"total":95995035648,"available":91863506944,"internally_reserved":0,"allocated":3767599104,"data_stored":3766863609,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11949,"internal_metadata":363909459},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:37:47 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":1.724,"15min":1.0580000000000001},"min":{"1min":0.34799999999999998,"5min":0.192,"15min":0.192},"max":{"1min":1.5129999999999999,"5min":19.431999999999999,"15min":19.431999999999999},"last":4.2880000000000003},{"interface":"front","average":{"1min":0.77900000000000003,"5min":1.6579999999999999,"15min":1.1020000000000001},"min":{"1min":0.28599999999999998,"5min":0.21099999999999999,"15min":0.17699999999999999},"max":{"1min":2.234,"5min":9.423,"15min":9.423},"last":0.56899999999999995}]},{"osd":1,"last update":"Sat Sep 23 18:38:00 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":1.6000000000000001,"15min":2.25},"min":{"1min":0.26000000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":5.4390000000000001,"5min":19.361999999999998,"15min":19.361999999999998},"last":0.86699999999999999},{"interface":"front","average":{"1min":1.238,"5min":1.843,"15min":2.226},"min":{"1min":0.27800000000000002,"5min":0.253,"15min":0.253},"max":{"1min":3.8809999999999998,"5min":9.468,"15min":9.468},"last":0.71199999999999997}]},{"osd":3,"last update":"Sat Sep 23 18:38:28 2023","interfaces":[{"interface":"back","average":{"1min":2.323,"5min":1.732,"15min":1.091},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":4.8949999999999996,"5min":7.7859999999999996,"15min":7.7859999999999996},"last":7.3449999999999998},{"interface":"front","average":{"1min":1.861,"5min":1.5549999999999999,"15min":0.97399999999999998},"min":{"1min":0.29699999999999999,"5min":0.26600000000000001,"15min":0.249},"max":{"1min":7.3600000000000003,"5min":19.407,"15min":19.407},"last":4.3319999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:37:47 2023","interfaces":[{"interface":"back","average":{"1min":1.3,"5min":1.645,"15min":1.2350000000000001},"min":{"1min":0.41499999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.8490000000000002,"5min":7.8220000000000001,"15min":7.8220000000000001},"last":4.6429999999999998},{"interface":"front","average":{"1min":1.056,"5min":1.7749999999999999,"15min":1.2629999999999999},"min":{"1min":0.624,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":2.8239999999999998,"5min":17.282,"15min":17.282},"last":0.73699999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:37:47 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":1.321,"15min":1.133},"min":{"1min":0.58499999999999996,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.7430000000000001,"5min":17.353999999999999,"15min":17.353999999999999},"last":0.83499999999999996},{"interface":"front","average":{"1min":1.234,"5min":1.6180000000000001,"15min":1.2829999999999999},"min":{"1min":0.56799999999999995,"5min":0.316,"15min":0.316},"max":{"1min":5.3129999999999997,"5min":10.218999999999999,"15min":10.218999999999999},"last":0.65600000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:37:47 2023","interfaces":[{"interface":"back","average":{"1min":1.377,"5min":1.6759999999999999,"15min":1.256},"min":{"1min":0.52900000000000003,"5min":0.372,"15min":0.36799999999999999},"max":{"1min":4.8109999999999999,"5min":7.867,"15min":7.867},"last":4.6769999999999996},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.425,"15min":1.1990000000000001},"min":{"1min":0.64800000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.645,"5min":17.379999999999999,"15min":17.379999999999999},"last":0.77200000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:37:47 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":1.8220000000000001,"15min":1.323},"min":{"1min":0.48199999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.8689999999999998,"5min":7.8460000000000001,"15min":7.8460000000000001},"last":4.6100000000000003},{"interface":"front","average":{"1min":1.1719999999999999,"5min":1.79,"15min":1.405},"min":{"1min":0.73899999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":3.992,"5min":17.326000000000001,"15min":17.326000000000001},"last":0.80400000000000005}]}]},{"osd":3,"up_from":103,"seq":442381631618,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3770552,"kb_used_data":3586416,"kb_used_omap":4,"kb_used_meta":184123,"kb_avail":89974600,"statfs":{"total":95995035648,"available":92133990400,"internally_reserved":0,"allocated":3672489984,"data_stored":3671767628,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4826,"internal_metadata":188542246},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,2,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:38:18 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":1.609,"15min":1.071},"min":{"1min":0.36699999999999999,"5min":0.20100000000000001,"15min":0.19700000000000001},"max":{"1min":8.8840000000000003,"5min":13.867000000000001,"15min":14.231},"last":2.3639999999999999},{"interface":"front","average":{"1min":2.0590000000000002,"5min":1.399,"15min":0.96099999999999997},"min":{"1min":0.35699999999999998,"5min":0.223,"15min":0.20300000000000001},"max":{"1min":6.7569999999999997,"5min":8.4659999999999993,"15min":8.4659999999999993},"last":4.3310000000000004}]},{"osd":1,"last update":"Sat Sep 23 18:38:04 2023","interfaces":[{"interface":"back","average":{"1min":1.575,"5min":1.0860000000000001,"15min":1.1859999999999999},"min":{"1min":0.39700000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":12.968999999999999,"5min":12.968999999999999,"15min":12.968999999999999},"last":4.2009999999999996},{"interface":"front","average":{"1min":1.6899999999999999,"5min":1.2350000000000001,"15min":1.333},"min":{"1min":0.42499999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":11.74,"5min":11.74,"15min":11.74},"last":4.1079999999999997}]},{"osd":2,"last update":"Sat Sep 23 18:38:18 2023","interfaces":[{"interface":"back","average":{"1min":3.0550000000000002,"5min":4.6059999999999999,"15min":2.0459999999999998},"min":{"1min":0.57299999999999995,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":14.116,"5min":187.40000000000001,"15min":187.40000000000001},"last":13.832000000000001},{"interface":"front","average":{"1min":1.841,"5min":3.8170000000000002,"15min":1.8100000000000001},"min":{"1min":0.28799999999999998,"5min":0.25,"15min":0.25},"max":{"1min":13.15,"5min":187.71700000000001,"15min":187.71700000000001},"last":1.601}]},{"osd":4,"last update":"Sat Sep 23 18:38:18 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":0.996,"15min":0.997},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.32600000000000001},"max":{"1min":4.4989999999999997,"5min":8.3689999999999998,"15min":8.3689999999999998},"last":0.66800000000000004},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.008,"15min":1.0349999999999999},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.28399999999999997},"max":{"1min":3.8639999999999999,"5min":4.4020000000000001,"15min":5.9429999999999996},"last":2.395}]},{"osd":5,"last update":"Sat Sep 23 18:38:18 2023","interfaces":[{"interface":"back","average":{"1min":1.2050000000000001,"5min":1.0680000000000001,"15min":1.0620000000000001},"min":{"1min":0.45000000000000001,"5min":0.35699999999999998,"15min":0.34499999999999997},"max":{"1min":3.839,"5min":4.923,"15min":5.9930000000000003},"last":2.4249999999999998},{"interface":"front","average":{"1min":1.964,"5min":1.2330000000000001,"15min":1.117},"min":{"1min":0.47199999999999998,"5min":0.36299999999999999,"15min":0.36099999999999999},"max":{"1min":5.1509999999999998,"5min":5.1509999999999998,"15min":5.1509999999999998},"last":4.4349999999999996}]},{"osd":6,"last update":"Sat Sep 23 18:38:18 2023","interfaces":[{"interface":"back","average":{"1min":2.012,"5min":1.2390000000000001,"15min":1.1279999999999999},"min":{"1min":0.51700000000000002,"5min":0.41099999999999998,"15min":0.35599999999999998},"max":{"1min":5.3310000000000004,"5min":5.3310000000000004,"15min":5.3310000000000004},"last":6.5510000000000002},{"interface":"front","average":{"1min":0.91600000000000004,"5min":1.0389999999999999,"15min":1.038},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.34100000000000003},"max":{"1min":4.577,"5min":8.4139999999999997,"15min":8.4139999999999997},"last":4.1639999999999997}]},{"osd":7,"last update":"Sat Sep 23 18:38:18 2023","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":1.155,"15min":1.133},"min":{"1min":0.5,"5min":0.44800000000000001,"15min":0.41499999999999998},"max":{"1min":4.601,"5min":8.4459999999999997,"15min":8.4459999999999997},"last":0.71299999999999997},{"interface":"front","average":{"1min":1.2669999999999999,"5min":1.206,"15min":1.145},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.39800000000000002},"max":{"1min":3.8109999999999999,"5min":5.0780000000000003,"15min":6.0259999999999998},"last":2.444}]}]},{"osd":4,"up_from":30,"seq":128849019228,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2081916,"kb_used_data":1784692,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91663236,"statfs":{"total":95995035648,"available":93863153664,"internally_reserved":0,"allocated":1827524608,"data_stored":1826974629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:37:51 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":2.5739999999999998,"15min":1.512},"min":{"1min":0.40400000000000003,"5min":0.253,"15min":0.253},"max":{"1min":1.1140000000000001,"5min":82.623000000000005,"15min":82.623000000000005},"last":3.9449999999999998},{"interface":"front","average":{"1min":0.86599999999999999,"5min":2.3100000000000001,"15min":1.4530000000000001},"min":{"1min":0.40000000000000002,"5min":0.27600000000000002,"15min":0.23799999999999999},"max":{"1min":1.177,"5min":82.638000000000005,"15min":82.638000000000005},"last":3.9340000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:37:57 2023","interfaces":[{"interface":"back","average":{"1min":3.5219999999999998,"5min":1.526,"15min":1.04},"min":{"1min":0.433,"5min":0.31,"15min":0.31},"max":{"1min":43.560000000000002,"5min":43.560000000000002,"15min":43.560000000000002},"last":2.645},{"interface":"front","average":{"1min":3.4470000000000001,"5min":1.708,"15min":1.3100000000000001},"min":{"1min":0.35199999999999998,"5min":0.313,"15min":0.313},"max":{"1min":43.545000000000002,"5min":43.545000000000002,"15min":43.545000000000002},"last":3.089}]},{"osd":2,"last update":"Sat Sep 23 18:37:51 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.4450000000000001,"15min":1.1719999999999999},"min":{"1min":0.54400000000000004,"5min":0.29099999999999998,"15min":0.28799999999999998},"max":{"1min":1.2969999999999999,"5min":12.552,"15min":12.552},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.93500000000000005,"5min":1.72,"15min":1.2749999999999999},"min":{"1min":0.36899999999999999,"5min":0.317,"15min":0.309},"max":{"1min":1.3340000000000001,"5min":12.221,"15min":12.221},"last":3.0409999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:38:24 2023","interfaces":[{"interface":"back","average":{"1min":6.5279999999999996,"5min":2.363,"15min":1.498},"min":{"1min":0.47499999999999998,"5min":0.33800000000000002,"15min":0.32900000000000001},"max":{"1min":72.075999999999993,"5min":72.075999999999993,"15min":72.075999999999993},"last":1.3100000000000001},{"interface":"front","average":{"1min":5.9340000000000002,"5min":2.4159999999999999,"15min":1.599},"min":{"1min":0.46500000000000002,"5min":0.42999999999999999,"15min":0.34000000000000002},"max":{"1min":71.596999999999994,"5min":71.596999999999994,"15min":71.596999999999994},"last":4.4059999999999997}]},{"osd":5,"last update":"Sat Sep 23 18:37:46 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":1.177,"15min":1.111},"min":{"1min":0.50900000000000001,"5min":0.34100000000000003,"15min":0.29899999999999999},"max":{"1min":1.2729999999999999,"5min":7.7869999999999999,"15min":7.819},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.95199999999999996,"5min":1.1160000000000001,"15min":1.0740000000000001},"min":{"1min":0.60999999999999999,"5min":0.32900000000000001,"15min":0.28399999999999997},"max":{"1min":1.464,"5min":6.4779999999999998,"15min":6.4779999999999998},"last":3.0699999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:38:13 2023","interfaces":[{"interface":"back","average":{"1min":1.099,"5min":1.298,"15min":1.145},"min":{"1min":0.42299999999999999,"5min":0.38,"15min":0.35399999999999998},"max":{"1min":6.1689999999999996,"5min":7.8419999999999996,"15min":7.8419999999999996},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.95399999999999996,"5min":1.044,"15min":1.0449999999999999},"min":{"1min":0.40999999999999998,"5min":0.374,"15min":0.26100000000000001},"max":{"1min":2.7309999999999999,"5min":5.9059999999999997,"15min":5.9059999999999997},"last":0.47499999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:37:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":1.4930000000000001,"15min":1.276},"min":{"1min":0.65400000000000003,"5min":0.38800000000000001,"15min":0.311},"max":{"1min":2.5390000000000001,"5min":13.491,"15min":13.491},"last":2.581},{"interface":"front","average":{"1min":1.1499999999999999,"5min":1.7310000000000001,"15min":1.405},"min":{"1min":0.71499999999999997,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":2.3380000000000001,"5min":17.062999999999999,"15min":17.062999999999999},"last":2.5600000000000001}]}]},{"osd":5,"up_from":0,"seq":154618822997,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4224,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7725,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3429,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1520500736,"data_stored":1520500736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2231173120,"data_stored":2231173120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3775135744,"data_stored":3775135744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3697999872,"data_stored":3697999872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1831141376,"data_stored":1831141376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2316435456,"data_stored":2316435456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2680815616,"data_stored":2680815616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:38:48.738 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T18:38:48.738 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 21 2023-09-23T18:38:48.764 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 79 16 96219 96203 76.0949 54.0625 0.00995636 0.0131349 2023-09-23T18:38:48.804 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:48.827 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:48.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:48.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:48.931 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:48.958 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:48.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:48.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:49.061 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:49.070 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:48 smithi173 bash[59422]: cluster 2023-09-23T18:38:47.857489+0000 mgr.y (mgr.26054) 610 : cluster 0 pgmap v591: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 57 MiB/s wr, 906 op/s 2023-09-23T18:38:49.087 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:49.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:49.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:49.191 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:49.219 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:49.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:49.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:49.327 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:49.336 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:48 smithi098 bash[79516]: cluster 2023-09-23T18:38:47.857489+0000 mgr.y (mgr.26054) 610 : cluster 0 pgmap v591: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 57 MiB/s wr, 906 op/s 2023-09-23T18:38:49.336 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:48 smithi098 bash[76817]: cluster 2023-09-23T18:38:47.857489+0000 mgr.y (mgr.26054) 610 : cluster 0 pgmap v591: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 57 MiB/s wr, 906 op/s 2023-09-23T18:38:49.358 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:49.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:49.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:49.465 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:49.490 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:49.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:49.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:49.593 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:49.617 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:49.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:49.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:49.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:49.752 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:49.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:49.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:49.764 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:2023-09-23T18:38:49.763829+0000 min lat: 0.00144566 max lat: 0.223301 avg lat: 0.0131888 2023-09-23T18:38:49.764 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:38:49.764 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 80 16 97046 97030 75.7898 51.6875 0.00965526 0.0131888 2023-09-23T18:38:49.856 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:49.882 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:49.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:49.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:49.988 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:50.012 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:50.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:50.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:50.120 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:50.146 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:50.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:50.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:50.225 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:38:50.249 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:50.284 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:50.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:50.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:50.389 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:50.413 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:50.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:50.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:50.518 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:50.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:50.560 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:50.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:50.661 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:50.692 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:50.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:50.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:50.764 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 81 16 97857 97841 75.4798 50.6875 0.00559141 0.0132422 2023-09-23T18:38:50.797 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:50.827 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:50.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:50.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:50.931 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:50.960 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:50.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:50.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:51.063 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:51.092 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:51.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:51.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:51.197 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:51.214 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:51.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:51.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:51.240 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:50 smithi098 bash[79516]: cluster 2023-09-23T18:38:49.857895+0000 mgr.y (mgr.26054) 611 : cluster 0 pgmap v592: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 48 MiB/s wr, 770 op/s 2023-09-23T18:38:51.240 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:50 smithi098 bash[76817]: cluster 2023-09-23T18:38:49.857895+0000 mgr.y (mgr.26054) 611 : cluster 0 pgmap v592: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 48 MiB/s wr, 770 op/s 2023-09-23T18:38:51.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:50 smithi173 bash[59422]: cluster 2023-09-23T18:38:49.857895+0000 mgr.y (mgr.26054) 611 : cluster 0 pgmap v592: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 48 MiB/s wr, 770 op/s 2023-09-23T18:38:51.316 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:51.344 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:51.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:51.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:51.448 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:51.473 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:51.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:51.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:51.577 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:51.613 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:51.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:51.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:51.716 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:51.740 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:51.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:51.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:51.764 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 82 16 98639 98623 75.1554 48.875 0.021297 0.0133005 2023-09-23T18:38:51.843 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:51.867 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:51.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:51.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:51.972 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:51.994 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:51.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:51.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:52.097 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:52.121 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:52.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:52.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:52.223 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:52.249 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:52.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:52.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:52.353 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:52.380 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:52.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:52.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:52.484 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:52.519 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:52.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:52.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:52.622 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:52.645 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:52.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:52.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:52.754 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:52.766 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 83 16 99391 99375 74.8147 47 0.0396943 0.0133596 2023-09-23T18:38:52.782 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:52.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:52.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:52.884 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:52.909 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:52.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:52.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:53.012 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:53.040 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:53.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:53.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:53.051 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:52 smithi098 bash[79516]: cluster 2023-09-23T18:38:51.858643+0000 mgr.y (mgr.26054) 612 : cluster 0 pgmap v593: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 57 MiB/s wr, 917 op/s 2023-09-23T18:38:53.051 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:52 smithi098 bash[76817]: cluster 2023-09-23T18:38:51.858643+0000 mgr.y (mgr.26054) 612 : cluster 0 pgmap v593: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 57 MiB/s wr, 917 op/s 2023-09-23T18:38:53.143 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:53.180 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:53.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:53.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:53.225 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:38:53.274 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:38:53.275 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:38:53.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:52 smithi173 bash[59422]: cluster 2023-09-23T18:38:51.858643+0000 mgr.y (mgr.26054) 612 : cluster 0 pgmap v593: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 57 MiB/s wr, 917 op/s 2023-09-23T18:38:53.284 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:53.324 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:53.326 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:38:53.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:53.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:53.379 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:38:53.413 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:38:53.432 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:53.456 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:53.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:53.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:53.460 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:38:53.506 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:38:53.545 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:38:53.550 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:38:53.550 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:38:53.550 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:38:53.550 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:38:53.550 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:38:53.550 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:38:53.550 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:38:53.550 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:38:53.560 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:53.600 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:53.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:53.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:53.706 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:53.732 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:53.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:53.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:53.766 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 84 16 100085 100069 74.4404 43.375 0.00837591 0.0134275 2023-09-23T18:38:53.835 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:53.859 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:53.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:53.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:53.963 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:53.988 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:53.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:53.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:54.012 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is set 2023-09-23T18:38:54.092 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:54.101 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:53 smithi098 bash[79516]: audit 2023-09-23T18:38:53.142521+0000 mon.a (mon.0) 859 : audit 0 from='client.? 172.21.15.98:0/1912817201' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:38:54.101 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:54 smithi098 bash[79516]: audit 2023-09-23T18:38:53.639527+0000 mon.b (mon.1) 14 : audit 1 from='client.? 172.21.15.98:0/3974714792' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:38:54.101 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:54 smithi098 bash[79516]: audit 2023-09-23T18:38:53.640894+0000 mon.a (mon.0) 860 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:38:54.101 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:53 smithi098 bash[76817]: audit 2023-09-23T18:38:53.142521+0000 mon.a (mon.0) 859 : audit 0 from='client.? 172.21.15.98:0/1912817201' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:38:54.101 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:53 smithi098 bash[76817]: audit 2023-09-23T18:38:53.639527+0000 mon.b (mon.1) 14 : audit 1 from='client.? 172.21.15.98:0/3974714792' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:38:54.101 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:53 smithi098 bash[76817]: audit 2023-09-23T18:38:53.640894+0000 mon.a (mon.0) 860 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:38:54.124 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:54.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:54.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:54.231 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:54.257 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:54.262 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:53 smithi173 bash[59422]: audit 2023-09-23T18:38:53.142521+0000 mon.a (mon.0) 859 : audit 0 from='client.? 172.21.15.98:0/1912817201' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:38:54.262 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:53 smithi173 bash[59422]: audit 2023-09-23T18:38:53.639527+0000 mon.b (mon.1) 14 : audit 1 from='client.? 172.21.15.98:0/3974714792' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:38:54.262 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:53 smithi173 bash[59422]: audit 2023-09-23T18:38:53.640894+0000 mon.a (mon.0) 860 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:38:54.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:54.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:54.363 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:54.386 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:54.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:54.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:54.492 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:54.519 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:54.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:54.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:54.624 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:54.650 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:54.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:54.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:54.754 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:54.766 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 85 16 100898 100882 74.1624 50.8125 0.0127818 0.0134771 2023-09-23T18:38:54.778 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:54.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:54.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:54.881 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:54.912 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:54.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:54.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:55.019 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:54 smithi173 bash[59422]: cluster 2023-09-23T18:38:53.859161+0000 mgr.y (mgr.26054) 613 : cluster 0 pgmap v594: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 48 MiB/s wr, 774 op/s 2023-09-23T18:38:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:54 smithi173 bash[59422]: cluster 2023-09-23T18:38:53.982442+0000 mon.a (mon.0) 861 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:38:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:54 smithi173 bash[59422]: cluster 2023-09-23T18:38:54.000013+0000 osd.7 (osd.7) 70 : cluster 3 failed to encode map e159 with expected crc 2023-09-23T18:38:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:54 smithi173 bash[59422]: cluster 2023-09-23T18:38:54.000765+0000 osd.4 (osd.4) 95 : cluster 3 failed to encode map e159 with expected crc 2023-09-23T18:38:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:54 smithi173 bash[59422]: audit 2023-09-23T18:38:54.006749+0000 mon.a (mon.0) 862 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:38:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:54 smithi173 bash[59422]: cluster 2023-09-23T18:38:54.006966+0000 mon.a (mon.0) 863 : cluster 0 osdmap e159: 8 total, 8 up, 7 in 2023-09-23T18:38:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:54 smithi173 bash[59422]: cluster 2023-09-23T18:38:54.007540+0000 osd.4 (osd.4) 96 : cluster 3 failed to encode map e159 with expected crc 2023-09-23T18:38:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:54 smithi173 bash[59422]: cluster 2023-09-23T18:38:54.011239+0000 osd.6 (osd.6) 72 : cluster 3 failed to encode map e159 with expected crc 2023-09-23T18:38:55.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:54 smithi173 bash[59422]: audit 2023-09-23T18:38:54.600846+0000 mon.a (mon.0) 864 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:38:55.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:54 smithi173 bash[59422]: audit 2023-09-23T18:38:54.604611+0000 mon.a (mon.0) 865 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:38:55.053 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:55.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:55.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:55.157 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:55.192 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:55.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:55.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:55.296 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:55.313 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:55.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:55.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:55.387 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:54 smithi098 bash[76817]: cluster 2023-09-23T18:38:53.859161+0000 mgr.y (mgr.26054) 613 : cluster 0 pgmap v594: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 48 MiB/s wr, 774 op/s 2023-09-23T18:38:55.387 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:54 smithi098 bash[76817]: cluster 2023-09-23T18:38:53.982442+0000 mon.a (mon.0) 861 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:38:55.387 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:54 smithi098 bash[76817]: cluster 2023-09-23T18:38:54.000013+0000 osd.7 (osd.7) 70 : cluster 3 failed to encode map e159 with expected crc 2023-09-23T18:38:55.387 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:54 smithi098 bash[76817]: cluster 2023-09-23T18:38:54.000765+0000 osd.4 (osd.4) 95 : cluster 3 failed to encode map e159 with expected crc 2023-09-23T18:38:55.387 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:54 smithi098 bash[76817]: audit 2023-09-23T18:38:54.006749+0000 mon.a (mon.0) 862 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:38:55.388 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:54 smithi098 bash[76817]: cluster 2023-09-23T18:38:54.006966+0000 mon.a (mon.0) 863 : cluster 0 osdmap e159: 8 total, 8 up, 7 in 2023-09-23T18:38:55.388 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:54 smithi098 bash[76817]: cluster 2023-09-23T18:38:54.007540+0000 osd.4 (osd.4) 96 : cluster 3 failed to encode map e159 with expected crc 2023-09-23T18:38:55.388 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:54 smithi098 bash[76817]: cluster 2023-09-23T18:38:54.011239+0000 osd.6 (osd.6) 72 : cluster 3 failed to encode map e159 with expected crc 2023-09-23T18:38:55.388 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:54 smithi098 bash[76817]: audit 2023-09-23T18:38:54.600846+0000 mon.a (mon.0) 864 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:38:55.388 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:54 smithi098 bash[76817]: audit 2023-09-23T18:38:54.604611+0000 mon.a (mon.0) 865 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:38:55.388 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:54 smithi098 bash[79516]: cluster 2023-09-23T18:38:53.859161+0000 mgr.y (mgr.26054) 613 : cluster 0 pgmap v594: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 48 MiB/s wr, 774 op/s 2023-09-23T18:38:55.388 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:54 smithi098 bash[79516]: cluster 2023-09-23T18:38:53.982442+0000 mon.a (mon.0) 861 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:38:55.388 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:54 smithi098 bash[79516]: cluster 2023-09-23T18:38:54.000013+0000 osd.7 (osd.7) 70 : cluster 3 failed to encode map e159 with expected crc 2023-09-23T18:38:55.388 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:54 smithi098 bash[79516]: cluster 2023-09-23T18:38:54.000765+0000 osd.4 (osd.4) 95 : cluster 3 failed to encode map e159 with expected crc 2023-09-23T18:38:55.388 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:54 smithi098 bash[79516]: audit 2023-09-23T18:38:54.006749+0000 mon.a (mon.0) 862 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:38:55.388 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:54 smithi098 bash[79516]: cluster 2023-09-23T18:38:54.006966+0000 mon.a (mon.0) 863 : cluster 0 osdmap e159: 8 total, 8 up, 7 in 2023-09-23T18:38:55.389 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:54 smithi098 bash[79516]: cluster 2023-09-23T18:38:54.007540+0000 osd.4 (osd.4) 96 : cluster 3 failed to encode map e159 with expected crc 2023-09-23T18:38:55.389 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:54 smithi098 bash[79516]: cluster 2023-09-23T18:38:54.011239+0000 osd.6 (osd.6) 72 : cluster 3 failed to encode map e159 with expected crc 2023-09-23T18:38:55.389 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:54 smithi098 bash[79516]: audit 2023-09-23T18:38:54.600846+0000 mon.a (mon.0) 864 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:38:55.389 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:54 smithi098 bash[79516]: audit 2023-09-23T18:38:54.604611+0000 mon.a (mon.0) 865 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:38:55.416 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:55.448 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:55.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:55.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:55.555 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:55.580 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:55.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:55.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:55.684 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:55.719 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:55.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:55.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:55.766 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 86 16 101564 101548 73.7841 41.625 0.0149654 0.0135471 2023-09-23T18:38:55.824 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:55.856 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:55.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:55.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:55.961 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:55.996 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:56.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:56.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:56.102 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:56.128 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:56.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:56.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:56.169 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:56 smithi098 bash[79516]: cluster 2023-09-23T18:38:55.859781+0000 mgr.y (mgr.26054) 614 : cluster 0 pgmap v596: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 50 MiB/s wr, 805 op/s 2023-09-23T18:38:56.169 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:38:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:38:56] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.33.4" 2023-09-23T18:38:56.169 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:56 smithi098 bash[76817]: cluster 2023-09-23T18:38:55.859781+0000 mgr.y (mgr.26054) 614 : cluster 0 pgmap v596: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 50 MiB/s wr, 805 op/s 2023-09-23T18:38:56.232 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:56.266 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:56.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:56.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:56.369 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:56.405 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:56.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:56.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:56.509 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:56.517 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:56 smithi173 bash[59422]: cluster 2023-09-23T18:38:55.859781+0000 mgr.y (mgr.26054) 614 : cluster 0 pgmap v596: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 50 MiB/s wr, 805 op/s 2023-09-23T18:38:56.537 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:56.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:56.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:56.640 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:56.670 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:56.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:56.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:56.766 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 87 16 102428 102412 73.5566 54 0.0232163 0.0135899 2023-09-23T18:38:56.774 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:56.801 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:56.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:56.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:56.906 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:56.935 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:56.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:56.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:57.039 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:57.092 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:57.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:57.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:57.198 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:57.224 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:57.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:57.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:57.330 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:57.358 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:57.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:57.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:57.462 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:57.486 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:57.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:57.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:57.591 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:57.618 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:57.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:57.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:57.722 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:57.758 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:57.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:57.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:57.766 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 88 16 103136 103120 73.2236 44.25 0.0202069 0.0136508 2023-09-23T18:38:57.862 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:57.892 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:57.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:57.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:57.995 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:58.022 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:58.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:58.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:58.126 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:58.155 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:58.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:58.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:58.262 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:58.287 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:58.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:58.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:58.393 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:58.421 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:58.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:58.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:58.524 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:58.549 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:58.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:58.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:58.653 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:58.670 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:38:58.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:58.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:38:58.766 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 89 16 103874 103858 72.9191 46.125 0.0080425 0.0137078 2023-09-23T18:38:58.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:58.801 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:58.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:58.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:58.907 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:58.942 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:38:58.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:58.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:38:59.045 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:59.072 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:38:59.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:59.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:38:59.177 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:59.184 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:38:58 smithi173 bash[59422]: cluster 2023-09-23T18:38:57.860387+0000 mgr.y (mgr.26054) 615 : cluster 0 pgmap v597: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 49 MiB/s wr, 789 op/s 2023-09-23T18:38:59.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:38:58 smithi098 bash[79516]: cluster 2023-09-23T18:38:57.860387+0000 mgr.y (mgr.26054) 615 : cluster 0 pgmap v597: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 49 MiB/s wr, 789 op/s 2023-09-23T18:38:59.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:38:58 smithi098 bash[76817]: cluster 2023-09-23T18:38:57.860387+0000 mgr.y (mgr.26054) 615 : cluster 0 pgmap v597: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 49 MiB/s wr, 789 op/s 2023-09-23T18:38:59.201 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:38:59.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:59.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:38:59.304 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:59.328 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:38:59.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:59.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:38:59.432 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:59.460 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:59.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:59.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:59.563 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:59.597 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:38:59.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:59.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:38:59.703 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:59.732 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:38:59.741 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:38:59.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:59.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:38:59.766 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Total time run: 90.013 2023-09-23T18:38:59.766 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Total writes made: 104666 2023-09-23T18:38:59.767 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Write size: 65536 2023-09-23T18:38:59.767 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Object size: 65536 2023-09-23T18:38:59.767 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Bandwidth (MB/sec): 72.6742 2023-09-23T18:38:59.767 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Stddev Bandwidth: 28.7218 2023-09-23T18:38:59.767 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Max bandwidth (MB/sec): 170.75 2023-09-23T18:38:59.767 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Min bandwidth (MB/sec): 41.625 2023-09-23T18:38:59.767 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Average IOPS: 1162 2023-09-23T18:38:59.767 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Stddev IOPS: 459.548 2023-09-23T18:38:59.767 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Max IOPS: 2732 2023-09-23T18:38:59.767 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Min IOPS: 666 2023-09-23T18:38:59.767 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Average Latency(s): 0.0137551 2023-09-23T18:38:59.767 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Stddev Latency(s): 0.0128494 2023-09-23T18:38:59.767 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Max latency(s): 0.227997 2023-09-23T18:38:59.768 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Min latency(s): 0.00144566 2023-09-23T18:38:59.807 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:38:59.807 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T18:38:59.846 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:59.878 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:38:59.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:38:59.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:38:59.983 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:00.019 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:00.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:00.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:00.129 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:00.162 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:00.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:00.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:00.266 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:00.301 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:00.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:00.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:00.408 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:00.436 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:00.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:00.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:00.467 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:39:00.467 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:39:00.538 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:00.559 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:00.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:00.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:00.666 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:00.693 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:00.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:00.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:00.798 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:00.831 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:00.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:00.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:00.936 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:00.980 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:00.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:00.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:01.085 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:01.112 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:01.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:01.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:01.215 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:01.222 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:00 smithi098 bash[76817]: cluster 2023-09-23T18:38:59.860963+0000 mgr.y (mgr.26054) 616 : cluster 0 pgmap v598: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 49 MiB/s wr, 784 op/s 2023-09-23T18:39:01.222 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:00 smithi098 bash[79516]: cluster 2023-09-23T18:38:59.860963+0000 mgr.y (mgr.26054) 616 : cluster 0 pgmap v598: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 49 MiB/s wr, 784 op/s 2023-09-23T18:39:01.246 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:01.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:01.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:01.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:00 smithi173 bash[59422]: cluster 2023-09-23T18:38:59.860963+0000 mgr.y (mgr.26054) 616 : cluster 0 pgmap v598: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 49 MiB/s wr, 784 op/s 2023-09-23T18:39:01.350 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:01.375 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:01.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:01.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:01.479 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:01.504 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:01.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:01.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:01.607 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:01.631 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:01.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:01.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:01.734 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:01.773 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:01.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:01.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:01.881 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:01.906 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:01.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:01.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:02.010 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:02.038 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:02.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:02.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:02.144 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:02.174 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:02.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:02.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:02.279 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:02.306 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:02.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:02.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:02.409 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:02.434 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:02.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:02.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:02.537 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:02.562 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:02.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:02.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:02.666 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:02.699 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:02.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:02.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:02.803 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:02.828 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:02.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:02.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:02.931 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:02.962 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:02 smithi173 bash[59422]: cluster 2023-09-23T18:39:01.861582+0000 mgr.y (mgr.26054) 617 : cluster 0 pgmap v599: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 44 MiB/s wr, 772 op/s 2023-09-23T18:39:02.963 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:02.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:02.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:03.068 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:03.093 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:03.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:03.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:03.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:02 smithi098 bash[76817]: cluster 2023-09-23T18:39:01.861582+0000 mgr.y (mgr.26054) 617 : cluster 0 pgmap v599: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 44 MiB/s wr, 772 op/s 2023-09-23T18:39:03.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:02 smithi098 bash[79516]: cluster 2023-09-23T18:39:01.861582+0000 mgr.y (mgr.26054) 617 : cluster 0 pgmap v599: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 44 MiB/s wr, 772 op/s 2023-09-23T18:39:03.196 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:03.221 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:03.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:03.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:03.282 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T18:39:03.324 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:03.358 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:03.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:03.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:03.463 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:03.490 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:03.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:03.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:03.595 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:03.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:03.631 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:03.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:03.732 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:03.764 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:03.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:03.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:03.869 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:03.904 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:03.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:03.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:04.008 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:04.038 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:04.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:04.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:04.142 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:04.173 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:04.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:04.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:04.225 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-09-23T18:39:04.278 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:04.299 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:04.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:04.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:04.404 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:04.432 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:04.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:04.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:04.536 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:04.560 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:04.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:04.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:04.666 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:04.698 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:04.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:04.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:04.802 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:04.829 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:04.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:04.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:04.932 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:04.960 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:04.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:04.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:05.068 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:05.077 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:04 smithi173 bash[59422]: cluster 2023-09-23T18:39:03.862090+0000 mgr.y (mgr.26054) 618 : cluster 0 pgmap v600: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 40 MiB/s wr, 768 op/s 2023-09-23T18:39:05.077 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:04 smithi173 bash[59422]: audit 2023-09-23T18:39:04.220746+0000 mon.a (mon.0) 866 : audit 1 from='client.? 172.21.15.98:0/268026912' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T18:39:05.092 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:05.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:05.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:05.195 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:05.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:05.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:05.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:05.332 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:05.341 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:05 smithi098 bash[76817]: cluster 2023-09-23T18:39:03.862090+0000 mgr.y (mgr.26054) 618 : cluster 0 pgmap v600: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 40 MiB/s wr, 768 op/s 2023-09-23T18:39:05.341 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:05 smithi098 bash[76817]: audit 2023-09-23T18:39:04.220746+0000 mon.a (mon.0) 866 : audit 1 from='client.? 172.21.15.98:0/268026912' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T18:39:05.341 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:05 smithi098 bash[79516]: cluster 2023-09-23T18:39:03.862090+0000 mgr.y (mgr.26054) 618 : cluster 0 pgmap v600: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 40 MiB/s wr, 768 op/s 2023-09-23T18:39:05.341 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:05 smithi098 bash[79516]: audit 2023-09-23T18:39:04.220746+0000 mon.a (mon.0) 866 : audit 1 from='client.? 172.21.15.98:0/268026912' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T18:39:05.355 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:05.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:05.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:05.460 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:05.493 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:05.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:05.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:05.521 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-09-23T18:39:05.598 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:05.627 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:05.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:05.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:05.731 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:05.761 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:05.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:05.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:05.864 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:05.887 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:05.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:05.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:05.989 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:06.026 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:06.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:06.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:06.131 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:06.163 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:06.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:06.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:06.272 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:06.281 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:39:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:39:06] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.33.4" 2023-09-23T18:39:06.299 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:06.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:06.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:06.405 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:06.436 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:06.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:06.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:06.542 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:06.573 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:06.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:06.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:06.676 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:06.713 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:06.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:06.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:06.818 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:06.856 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:06.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:06.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:06.965 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:06.994 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:06.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:06.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:07.025 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:39:07.029 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:39:07.097 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:07.107 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:06 smithi098 bash[76817]: cluster 2023-09-23T18:39:05.862689+0000 mgr.y (mgr.26054) 619 : cluster 0 pgmap v601: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 86 B/s rd, 34 MiB/s wr, 776 op/s 2023-09-23T18:39:07.107 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:06 smithi098 bash[79516]: cluster 2023-09-23T18:39:05.862689+0000 mgr.y (mgr.26054) 619 : cluster 0 pgmap v601: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 86 B/s rd, 34 MiB/s wr, 776 op/s 2023-09-23T18:39:07.133 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:07.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:07.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:07.237 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:07.273 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:07.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:06 smithi173 bash[59422]: cluster 2023-09-23T18:39:05.862689+0000 mgr.y (mgr.26054) 619 : cluster 0 pgmap v601: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 86 B/s rd, 34 MiB/s wr, 776 op/s 2023-09-23T18:39:07.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:07.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:07.380 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:07.416 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:07.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:07.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:07.519 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:07.553 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:07.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:07.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:07.658 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:07.711 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:07.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:07.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:07.816 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:07.841 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:07.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:07.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:07.947 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:07.970 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:07.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:07.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:08.076 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:08.104 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:08.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:08.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:08.208 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:08.237 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:08.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:08.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:08.341 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:08.365 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:08.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:08.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:08.468 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:08.491 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:08.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:08.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:08.599 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:08.622 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:08.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:08.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:08.725 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:08.751 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:08.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:08.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:08.858 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:08.884 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:08.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:08.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:08.990 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:08.998 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:08 smithi098 bash[76817]: cluster 2023-09-23T18:39:07.863326+0000 mgr.y (mgr.26054) 620 : cluster 0 pgmap v602: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 26 MiB/s wr, 840 op/s 2023-09-23T18:39:08.999 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:08 smithi098 bash[79516]: cluster 2023-09-23T18:39:07.863326+0000 mgr.y (mgr.26054) 620 : cluster 0 pgmap v602: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 26 MiB/s wr, 840 op/s 2023-09-23T18:39:09.014 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:09.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:09.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:09.062 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:39:09.119 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:09.127 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:08 smithi173 bash[59422]: cluster 2023-09-23T18:39:07.863326+0000 mgr.y (mgr.26054) 620 : cluster 0 pgmap v602: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 26 MiB/s wr, 840 op/s 2023-09-23T18:39:09.144 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:09.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:09.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:09.251 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:09.282 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:09.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:09.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:09.385 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:09.415 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:09.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:09.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:09.520 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:09.546 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:09.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:09.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:09.649 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:09.676 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:09.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:09.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:09.781 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:09.809 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:09.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:09.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:09.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:09.938 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:09.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:09.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:10.044 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:10.053 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:09 smithi098 bash[79516]: audit 2023-09-23T18:39:09.596040+0000 mon.a (mon.0) 867 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:39:10.053 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:09 smithi098 bash[76817]: audit 2023-09-23T18:39:09.596040+0000 mon.a (mon.0) 867 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:39:10.070 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:10.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:10.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:10.176 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:10.204 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:10.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:10.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:10.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:09 smithi173 bash[59422]: audit 2023-09-23T18:39:09.596040+0000 mon.a (mon.0) 867 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:39:10.309 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:10.333 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:10.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:10.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:10.437 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:10.461 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:10.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:10.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:10.522 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T18:39:10.522 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:39:10.522 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T18:39:10.523 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 3.0 2023-09-23T18:39:10.563 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:10.599 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:10.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:10.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:10.704 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:10.743 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:10.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:10.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:10.854 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:10.887 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:10.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:10.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:10.992 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:11.014 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:11.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:11.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:11.117 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:11.127 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:10 smithi098 bash[76817]: cluster 2023-09-23T18:39:09.863860+0000 mgr.y (mgr.26054) 621 : cluster 0 pgmap v603: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 15 MiB/s wr, 721 op/s 2023-09-23T18:39:11.128 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:10 smithi098 bash[79516]: cluster 2023-09-23T18:39:09.863860+0000 mgr.y (mgr.26054) 621 : cluster 0 pgmap v603: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 15 MiB/s wr, 721 op/s 2023-09-23T18:39:11.152 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:11.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:11.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:11.256 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:11.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:10 smithi173 bash[59422]: cluster 2023-09-23T18:39:09.863860+0000 mgr.y (mgr.26054) 621 : cluster 0 pgmap v603: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 15 MiB/s wr, 721 op/s 2023-09-23T18:39:11.278 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:11.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:11.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:11.381 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:11.406 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:11.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:11.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:11.512 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:11.536 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:11.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:11.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:11.640 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:11.679 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:11.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:11.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:11.783 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:11.807 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:11.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:11.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:11.910 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:11.934 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:11.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:11.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:12.039 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:12.062 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:12.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:12.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:12.165 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:12.189 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:12.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:12.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:12.292 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:12.334 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:12.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:12.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:12.440 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:12.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:12.490 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:12.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:12.592 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:12.623 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:12.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:12.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:12.726 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:12.759 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:12.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:12.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:12.864 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:12.892 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:12.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:12.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:12.995 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:13.033 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:13.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:13.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:13.138 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:13.148 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:12 smithi173 bash[59422]: cluster 2023-09-23T18:39:11.864497+0000 mgr.y (mgr.26054) 622 : cluster 0 pgmap v604: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 11 MiB/s wr, 870 op/s 2023-09-23T18:39:13.171 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:13.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:13.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:13.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:12 smithi098 bash[79516]: cluster 2023-09-23T18:39:11.864497+0000 mgr.y (mgr.26054) 622 : cluster 0 pgmap v604: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 11 MiB/s wr, 870 op/s 2023-09-23T18:39:13.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:12 smithi098 bash[76817]: cluster 2023-09-23T18:39:11.864497+0000 mgr.y (mgr.26054) 622 : cluster 0 pgmap v604: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 11 MiB/s wr, 870 op/s 2023-09-23T18:39:13.276 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:13.306 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:13.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:13.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:13.411 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:13.443 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:13.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:13.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:13.549 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:13.577 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:13.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:13.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:13.680 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:13.707 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:13.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:13.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:13.812 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:13.845 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:13.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:13.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:13.951 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:13.975 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:13.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:13.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:14.080 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:14.102 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:14.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:14.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:14.204 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:14.237 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:14.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:14.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:14.342 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:14.375 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:14.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:14.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:14.478 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:14.506 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:14.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:14.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:14.612 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:14.635 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:14.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:14.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:14.738 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:14.761 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:14.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:14.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:14.867 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:14.889 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:14.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:14.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:14.995 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:15.023 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:15.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:15.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:15.126 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:15.157 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:15.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:15.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:15.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:14 smithi098 bash[79516]: cluster 2023-09-23T18:39:13.868372+0000 mgr.y (mgr.26054) 623 : cluster 0 pgmap v605: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 2.2 MiB/s wr, 750 op/s 2023-09-23T18:39:15.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:14 smithi098 bash[76817]: cluster 2023-09-23T18:39:13.868372+0000 mgr.y (mgr.26054) 623 : cluster 0 pgmap v605: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 2.2 MiB/s wr, 750 op/s 2023-09-23T18:39:15.264 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:15.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:14 smithi173 bash[59422]: cluster 2023-09-23T18:39:13.868372+0000 mgr.y (mgr.26054) 623 : cluster 0 pgmap v605: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 2.2 MiB/s wr, 750 op/s 2023-09-23T18:39:15.286 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:15.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:15.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:15.389 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:15.423 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:15.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:15.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:15.528 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:15.551 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:15.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:15.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:15.655 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:15.685 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:15.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:15.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:15.790 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:15.821 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:15.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:15.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:15.927 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:15.954 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:15.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:15.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:16.060 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:16.083 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:16.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:16.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:16.189 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:16.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:16.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:16.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:16.331 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:16.354 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:16.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:16.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:16.440 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:39:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:39:16] "GET /metrics HTTP/1.1" 200 36516 "" "Prometheus/2.33.4" 2023-09-23T18:39:16.460 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:16.485 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:16.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:16.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:16.588 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:16.621 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:16.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:16.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:16.726 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:16.748 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:16.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:16.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:16.857 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:16.885 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:16.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:16.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:16.987 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:17.012 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:17.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:17.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:17.117 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:17.126 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:17 smithi173 bash[59422]: cluster 2023-09-23T18:39:15.868911+0000 mgr.y (mgr.26054) 624 : cluster 0 pgmap v606: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 32 KiB/s wr, 787 op/s 2023-09-23T18:39:17.143 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:17.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:17.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:17.247 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:17.275 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:17.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:17.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:17.378 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:17.387 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:17 smithi098 bash[79516]: cluster 2023-09-23T18:39:15.868911+0000 mgr.y (mgr.26054) 624 : cluster 0 pgmap v606: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 32 KiB/s wr, 787 op/s 2023-09-23T18:39:17.391 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:16 smithi098 bash[76817]: cluster 2023-09-23T18:39:15.868911+0000 mgr.y (mgr.26054) 624 : cluster 0 pgmap v606: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 32 KiB/s wr, 787 op/s 2023-09-23T18:39:17.401 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:17.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:17.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:17.505 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:17.529 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:17.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:17.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:17.635 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:17.705 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:17.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:17.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:17.812 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:17.853 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:17.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:17.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:17.871 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:39:17.958 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:17.987 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:17.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:17.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:18.020 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:39:18.021 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:39:18.094 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:18.128 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:39:18.142 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:18.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:18.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:18.177 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:39:18.235 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:39:18.254 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:18.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:18 smithi173 bash[59422]: cluster 2023-09-23T18:39:17.869528+0000 mgr.y (mgr.26054) 625 : cluster 0 pgmap v607: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 842 op/s 2023-09-23T18:39:18.280 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:39:18.290 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:18.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:18.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:18.328 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:39:18.372 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:39:18.394 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:18.418 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:39:18.419 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:18.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:18.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:18.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:18 smithi098 bash[79516]: cluster 2023-09-23T18:39:17.869528+0000 mgr.y (mgr.26054) 625 : cluster 0 pgmap v607: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 842 op/s 2023-09-23T18:39:18.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:18 smithi098 bash[76817]: cluster 2023-09-23T18:39:17.869528+0000 mgr.y (mgr.26054) 625 : cluster 0 pgmap v607: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 842 op/s 2023-09-23T18:39:18.462 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:39:18.465 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:39:18.465 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:39:18.465 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:39:18.465 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:39:18.465 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:39:18.465 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:39:18.465 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:39:18.465 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:39:18.524 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:18.550 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:18.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:18.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:18.654 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:18.682 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:18.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:18.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:18.789 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:18.817 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:18.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:18.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:18.919 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:18.942 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:18.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:18.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:19.046 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:19.067 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:19.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:19.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:19.073 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is set 2023-09-23T18:39:19.170 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:19.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:19 smithi098 bash[76817]: audit 2023-09-23T18:39:18.074504+0000 mon.c (mon.2) 79 : audit 0 from='client.? 172.21.15.98:0/6287127' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:39:19.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:19 smithi098 bash[76817]: audit 2023-09-23T18:39:18.430382+0000 mon.c (mon.2) 80 : audit 1 from='client.? 172.21.15.98:0/1862222452' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:39:19.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:19 smithi098 bash[76817]: audit 2023-09-23T18:39:18.430978+0000 mon.a (mon.0) 868 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:39:19.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:19 smithi098 bash[79516]: audit 2023-09-23T18:39:18.074504+0000 mon.c (mon.2) 79 : audit 0 from='client.? 172.21.15.98:0/6287127' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:39:19.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:19 smithi098 bash[79516]: audit 2023-09-23T18:39:18.430382+0000 mon.c (mon.2) 80 : audit 1 from='client.? 172.21.15.98:0/1862222452' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:39:19.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:19 smithi098 bash[79516]: audit 2023-09-23T18:39:18.430978+0000 mon.a (mon.0) 868 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:39:19.209 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:19.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:19.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:19.313 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:19.321 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:19 smithi173 bash[59422]: audit 2023-09-23T18:39:18.074504+0000 mon.c (mon.2) 79 : audit 0 from='client.? 172.21.15.98:0/6287127' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:39:19.321 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:19 smithi173 bash[59422]: audit 2023-09-23T18:39:18.430382+0000 mon.c (mon.2) 80 : audit 1 from='client.? 172.21.15.98:0/1862222452' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:39:19.322 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:19 smithi173 bash[59422]: audit 2023-09-23T18:39:18.430978+0000 mon.a (mon.0) 868 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:39:19.338 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:19.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:19.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:19.441 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:19.470 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:19.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:19.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:19.573 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:19.598 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:19.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:19.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:19.702 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:19.733 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:19.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:19.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:19.836 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:19.864 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:19.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:19.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:19.966 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:19.992 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:19.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:19.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:20.095 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:20.116 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:20.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:20.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:20.219 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:20.229 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:20 smithi173 bash[59422]: cluster 2023-09-23T18:39:19.063614+0000 mon.a (mon.0) 869 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:39:20.229 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:20 smithi173 bash[59422]: cluster 2023-09-23T18:39:19.067482+0000 osd.7 (osd.7) 71 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T18:39:20.229 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:20 smithi173 bash[59422]: audit 2023-09-23T18:39:19.068908+0000 mon.a (mon.0) 870 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:39:20.229 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:20 smithi173 bash[59422]: cluster 2023-09-23T18:39:19.068959+0000 mon.a (mon.0) 871 : cluster 0 osdmap e160: 8 total, 8 up, 7 in 2023-09-23T18:39:20.230 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:20 smithi173 bash[59422]: cluster 2023-09-23T18:39:19.098170+0000 osd.4 (osd.4) 97 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T18:39:20.230 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:20 smithi173 bash[59422]: cluster 2023-09-23T18:39:19.870005+0000 mgr.y (mgr.26054) 626 : cluster 0 pgmap v609: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 774 op/s 2023-09-23T18:39:20.243 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:20.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:20.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:20.346 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:20.354 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:20 smithi098 bash[76817]: cluster 2023-09-23T18:39:19.063614+0000 mon.a (mon.0) 869 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:39:20.354 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:20 smithi098 bash[76817]: cluster 2023-09-23T18:39:19.067482+0000 osd.7 (osd.7) 71 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T18:39:20.354 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:20 smithi098 bash[76817]: audit 2023-09-23T18:39:19.068908+0000 mon.a (mon.0) 870 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:39:20.354 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:20 smithi098 bash[76817]: cluster 2023-09-23T18:39:19.068959+0000 mon.a (mon.0) 871 : cluster 0 osdmap e160: 8 total, 8 up, 7 in 2023-09-23T18:39:20.355 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:20 smithi098 bash[76817]: cluster 2023-09-23T18:39:19.098170+0000 osd.4 (osd.4) 97 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T18:39:20.355 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:20 smithi098 bash[76817]: cluster 2023-09-23T18:39:19.870005+0000 mgr.y (mgr.26054) 626 : cluster 0 pgmap v609: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 774 op/s 2023-09-23T18:39:20.355 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:20 smithi098 bash[79516]: cluster 2023-09-23T18:39:19.063614+0000 mon.a (mon.0) 869 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:39:20.355 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:20 smithi098 bash[79516]: cluster 2023-09-23T18:39:19.067482+0000 osd.7 (osd.7) 71 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T18:39:20.355 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:20 smithi098 bash[79516]: audit 2023-09-23T18:39:19.068908+0000 mon.a (mon.0) 870 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:39:20.355 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:20 smithi098 bash[79516]: cluster 2023-09-23T18:39:19.068959+0000 mon.a (mon.0) 871 : cluster 0 osdmap e160: 8 total, 8 up, 7 in 2023-09-23T18:39:20.355 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:20 smithi098 bash[79516]: cluster 2023-09-23T18:39:19.098170+0000 osd.4 (osd.4) 97 : cluster 3 failed to encode map e160 with expected crc 2023-09-23T18:39:20.355 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:20 smithi098 bash[79516]: cluster 2023-09-23T18:39:19.870005+0000 mgr.y (mgr.26054) 626 : cluster 0 pgmap v609: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 774 op/s 2023-09-23T18:39:20.372 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:20.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:20.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:20.475 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:20.505 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:20.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:20.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:20.612 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:20.640 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:20.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:20.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:20.743 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:20.763 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:20.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:20.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:20.867 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:20.893 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:20.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:20.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:20.998 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:21.020 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:21.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:21.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:21.124 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:21.155 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:21.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:21.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:21.258 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:21.286 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:21.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:21.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:21.389 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:21.421 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:21.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:21.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:21.524 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:21.551 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:21.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:21.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:21.658 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:21.681 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:21.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:21.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:21.786 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:21.818 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:21.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:21.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:21.924 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:21.952 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:21.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:21.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:22.055 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:22.080 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:22.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:22.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:22.185 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:22.212 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:22.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:22.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:22.312 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T18:39:22.316 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:22.381 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:22.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:22.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:22.486 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:22.523 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:22.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:22.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:22.626 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:22.654 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:22.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:22.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:22.759 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:22.784 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:22.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:22.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:22.889 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:22.914 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:22.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:22.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:23.017 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:23.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:22 smithi173 bash[59422]: cluster 2023-09-23T18:39:21.870622+0000 mgr.y (mgr.26054) 627 : cluster 0 pgmap v610: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 751 op/s 2023-09-23T18:39:23.039 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:23.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:23.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:23.145 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:23.169 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:23.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:23.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:23.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:22 smithi098 bash[76817]: cluster 2023-09-23T18:39:21.870622+0000 mgr.y (mgr.26054) 627 : cluster 0 pgmap v610: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 751 op/s 2023-09-23T18:39:23.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:22 smithi098 bash[79516]: cluster 2023-09-23T18:39:21.870622+0000 mgr.y (mgr.26054) 627 : cluster 0 pgmap v610: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 751 op/s 2023-09-23T18:39:23.277 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:23.300 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:23.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:23.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:23.404 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:23.437 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:23.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:23.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:23.540 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:23.569 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:23.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:23.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:23.676 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:23.697 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:23.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:23.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:23.803 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:23.833 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:23.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:23.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:23.938 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:23.967 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:23.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:23.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:24.073 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:24.107 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:24.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:24.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:24.211 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:24.240 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:24.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:24.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:24.281 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:39:24.316 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:39:24.344 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:24.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:24.419 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:24.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:24.520 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:24.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:24.567 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:24.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:24.668 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:24.703 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:24.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:24.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:24.811 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:24.842 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:24.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:24.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:24.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:24.979 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:24.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:24.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:25.085 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:25.094 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:24 smithi098 bash[79516]: cluster 2023-09-23T18:39:23.871117+0000 mgr.y (mgr.26054) 628 : cluster 0 pgmap v611: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 742 op/s 2023-09-23T18:39:25.094 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:24 smithi098 bash[79516]: audit 2023-09-23T18:39:24.596480+0000 mon.a (mon.0) 872 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:39:25.094 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:24 smithi098 bash[76817]: cluster 2023-09-23T18:39:23.871117+0000 mgr.y (mgr.26054) 628 : cluster 0 pgmap v611: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 742 op/s 2023-09-23T18:39:25.094 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:24 smithi098 bash[76817]: audit 2023-09-23T18:39:24.596480+0000 mon.a (mon.0) 872 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:39:25.123 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:25.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:25.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:25.228 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:25.266 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:25.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:25.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:25.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:24 smithi173 bash[59422]: cluster 2023-09-23T18:39:23.871117+0000 mgr.y (mgr.26054) 628 : cluster 0 pgmap v611: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 742 op/s 2023-09-23T18:39:25.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:24 smithi173 bash[59422]: audit 2023-09-23T18:39:24.596480+0000 mon.a (mon.0) 872 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:39:25.374 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:25.423 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:25.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:25.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:25.528 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:25.554 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:25.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:25.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:25.657 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:25.690 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:25.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:25.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:25.726 INFO:teuthology.orchestra.run.smithi098.stdout:moved 0 / 267 (0%) 2023-09-23T18:39:25.726 INFO:teuthology.orchestra.run.smithi098.stdout:avg 38.1429 2023-09-23T18:39:25.726 INFO:teuthology.orchestra.run.smithi098.stdout:stddev 3.97954 -> 3.97954 (expected baseline 5.71786) 2023-09-23T18:39:25.727 INFO:teuthology.orchestra.run.smithi098.stdout:min osd.2 with 31 -> 31 pgs (0.812734 -> 0.812734 * mean) 2023-09-23T18:39:25.727 INFO:teuthology.orchestra.run.smithi098.stdout:max osd.3 with 45 -> 45 pgs (1.17978 -> 1.17978 * mean) 2023-09-23T18:39:25.727 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:39:25.727 INFO:teuthology.orchestra.run.smithi098.stdout:oload 1000 2023-09-23T18:39:25.727 INFO:teuthology.orchestra.run.smithi098.stdout:max_change 3 2023-09-23T18:39:25.727 INFO:teuthology.orchestra.run.smithi098.stdout:max_change_osds 4 2023-09-23T18:39:25.727 INFO:teuthology.orchestra.run.smithi098.stdout:average_utilization 0.0287 2023-09-23T18:39:25.727 INFO:teuthology.orchestra.run.smithi098.stdout:overload_utilization 0.2875 2023-09-23T18:39:25.732 INFO:teuthology.orchestra.run.smithi098.stderr:no change 2023-09-23T18:39:25.796 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:25.825 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:25.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:25.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:25.929 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:25.973 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:25.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:25.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:26.077 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:26.112 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:26.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:26.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:26.190 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:39:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:39:26] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.33.4" 2023-09-23T18:39:26.220 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:26.250 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:26.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:26.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:26.354 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:26.379 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:26.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:26.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:26.485 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:26.534 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:26.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:26.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:26.642 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:26.667 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:26.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:26.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:26.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:26.801 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:26.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:26.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:26.904 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:26.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:26.937 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:26.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:27.038 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:27.070 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:27.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:27.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:27.114 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T18:39:27.114 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 1.0 2023-09-23T18:39:27.162 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:26 smithi098 bash[79516]: audit 2023-09-23T18:39:25.723445+0000 mgr.y (mgr.26054) 629 : audit 0 from='client.54745 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:39:27.163 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:26 smithi098 bash[79516]: cluster 2023-09-23T18:39:25.871615+0000 mgr.y (mgr.26054) 630 : cluster 0 pgmap v612: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 737 op/s 2023-09-23T18:39:27.163 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:26 smithi098 bash[76817]: audit 2023-09-23T18:39:25.723445+0000 mgr.y (mgr.26054) 629 : audit 0 from='client.54745 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:39:27.163 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:26 smithi098 bash[76817]: cluster 2023-09-23T18:39:25.871615+0000 mgr.y (mgr.26054) 630 : cluster 0 pgmap v612: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 737 op/s 2023-09-23T18:39:27.174 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:27.185 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:26 smithi173 bash[59422]: audit 2023-09-23T18:39:25.723445+0000 mgr.y (mgr.26054) 629 : audit 0 from='client.54745 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:39:27.185 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:26 smithi173 bash[59422]: cluster 2023-09-23T18:39:25.871615+0000 mgr.y (mgr.26054) 630 : cluster 0 pgmap v612: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 737 op/s 2023-09-23T18:39:27.200 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:27.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:27.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:27.303 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:27.328 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:27.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:27.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:27.432 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:27.458 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:27.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:27.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:27.563 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:27.602 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:39:27.608 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:27.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:27.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:27.715 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:27.742 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:27.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:27.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:27.847 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:27.875 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:27.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:27.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:27.978 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:28.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:28.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:28.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:28.121 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:28.161 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:28.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:28.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:28.265 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:28.297 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:28.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:28.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:28.403 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:28.426 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:28.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:28.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:28.528 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:28.555 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:28.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:28.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:28.658 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:28.689 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:28.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:28.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:28.794 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:28.819 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:28.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:28.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:28.921 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:28.944 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:28.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:28.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:29.047 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:29.089 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:29.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:29.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:29.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:28 smithi098 bash[76817]: cluster 2023-09-23T18:39:27.872248+0000 mgr.y (mgr.26054) 631 : cluster 0 pgmap v613: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 724 op/s 2023-09-23T18:39:29.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:28 smithi098 bash[79516]: cluster 2023-09-23T18:39:27.872248+0000 mgr.y (mgr.26054) 631 : cluster 0 pgmap v613: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 724 op/s 2023-09-23T18:39:29.195 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:29.223 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:29.224 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:28 smithi173 bash[59422]: cluster 2023-09-23T18:39:27.872248+0000 mgr.y (mgr.26054) 631 : cluster 0 pgmap v613: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 724 op/s 2023-09-23T18:39:29.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:29.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:29.326 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:29.353 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:29.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:29.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:29.458 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:29.487 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:29.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:29.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:29.589 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:29.617 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:29.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:29.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:29.651 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:39:29.653 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:39:29.725 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:29.752 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:29.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:29.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:29.856 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:29.875 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:29.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:29.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:29.978 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:30.012 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:30.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:30.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:30.119 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:30.151 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:30.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:30.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:30.254 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:30.278 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:30.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:30.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:30.381 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:30.416 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:30.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:30.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:30.521 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:30.552 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:30.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:30.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:30.657 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:30.680 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:30.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:30.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:30.784 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:30.811 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:30.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:30.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:30.916 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:30.950 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:30.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:30.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:31.053 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:31.082 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:31.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:31.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:31.113 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:31 smithi098 bash[76817]: cluster 2023-09-23T18:39:29.872740+0000 mgr.y (mgr.26054) 632 : cluster 0 pgmap v614: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 710 op/s 2023-09-23T18:39:31.113 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:31 smithi098 bash[79516]: cluster 2023-09-23T18:39:29.872740+0000 mgr.y (mgr.26054) 632 : cluster 0 pgmap v614: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 710 op/s 2023-09-23T18:39:31.186 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:31.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:31.216 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:31.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:31.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:30 smithi173 bash[59422]: cluster 2023-09-23T18:39:29.872740+0000 mgr.y (mgr.26054) 632 : cluster 0 pgmap v614: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 710 op/s 2023-09-23T18:39:31.317 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:31.342 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:31.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:31.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:31.446 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:31.478 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:31.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:31.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:31.585 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:31.612 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:31.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:31.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:31.717 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:31.743 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:31.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:31.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:31.846 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:31.877 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:31.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:31.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:31.984 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:32.007 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:32.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:32.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:32.113 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:32.140 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:32.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:32.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:32.243 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:32.267 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:32.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:32.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:32.370 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:32.402 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:32.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:32.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:32.505 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:32.535 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:32.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:32.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:32.639 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:32.658 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:39:32.666 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:32.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:32.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:32.769 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:32.798 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:32.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:32.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:32.904 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:32.946 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:32.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:32.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:32.985 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:32 smithi173 bash[59422]: cluster 2023-09-23T18:39:31.873397+0000 mgr.y (mgr.26054) 633 : cluster 0 pgmap v615: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 792 op/s 2023-09-23T18:39:33.049 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:33.074 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:33.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:33.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:33.185 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:33.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:32 smithi098 bash[76817]: cluster 2023-09-23T18:39:31.873397+0000 mgr.y (mgr.26054) 633 : cluster 0 pgmap v615: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 792 op/s 2023-09-23T18:39:33.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:32 smithi098 bash[79516]: cluster 2023-09-23T18:39:31.873397+0000 mgr.y (mgr.26054) 633 : cluster 0 pgmap v615: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 792 op/s 2023-09-23T18:39:33.213 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:33.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:33.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:33.316 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:33.346 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:33.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:33.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:33.448 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:33.480 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:33.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:33.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:33.584 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:33.617 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:33.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:33.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:33.721 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:33.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:33.756 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:33.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:33.857 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:33.881 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:33.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:33.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:33.989 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:34.025 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:34.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:34.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:34.131 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:34.159 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:34.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:34.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:34.266 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:34.298 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:34.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:34.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:34.401 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:34.423 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:34.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:34.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:34.527 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:34.552 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:34.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:34.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:34.655 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:34.679 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:34.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:34.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:34.785 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:34.808 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:34.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:34.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:34.912 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:34.937 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:34.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:34.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:35.040 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:35.069 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:35.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:35.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:35.175 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:35.186 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:34 smithi173 bash[59422]: cluster 2023-09-23T18:39:33.873892+0000 mgr.y (mgr.26054) 634 : cluster 0 pgmap v616: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 657 op/s 2023-09-23T18:39:35.203 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:35.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:35.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:35.309 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:35.334 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:35.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:35.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:35.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:34 smithi098 bash[79516]: cluster 2023-09-23T18:39:33.873892+0000 mgr.y (mgr.26054) 634 : cluster 0 pgmap v616: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 657 op/s 2023-09-23T18:39:35.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:34 smithi098 bash[76817]: cluster 2023-09-23T18:39:33.873892+0000 mgr.y (mgr.26054) 634 : cluster 0 pgmap v616: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 657 op/s 2023-09-23T18:39:35.444 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:35.470 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:35.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:35.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:35.576 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:35.604 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:35.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:35.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:35.713 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:35.737 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:35.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:35.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:35.841 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:35.867 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:35.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:35.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:35.971 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:35.999 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:36.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:36.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:36.099 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:39:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:39:36] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.33.4" 2023-09-23T18:39:36.103 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:36.130 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:36.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:36.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:36.234 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:36.256 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:36.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:36.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:36.298 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:39:36.298 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [ 2023-09-23T18:39:36.298 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.298 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.44954.0:132240 6.4 e160/149 6:242c9892:::benchmark_data_smithi098_131983_object27571:head v 160'8405, mlcod=160'8403)", 2023-09-23T18:39:36.299 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:39:36.273869+0000", 2023-09-23T18:39:36.299 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.019992928, 2023-09-23T18:39:36.299 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.020024406000000002, 2023-09-23T18:39:36.299 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:39:36.299 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T18:39:36.299 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:39:36.299 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.299 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:39:36.299 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.273869+0000", 2023-09-23T18:39:36.299 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:39:36.299 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.299 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.300 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:39:36.300 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.273869+0000", 2023-09-23T18:39:36.300 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:39:36.300 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.300 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.300 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:39:36.300 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.273868+0000", 2023-09-23T18:39:36.300 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:39:36.300 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.300 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.300 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:39:36.301 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.273876+0000", 2023-09-23T18:39:36.301 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.2379999999999997e-06 2023-09-23T18:39:36.301 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.301 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.301 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:39:36.301 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.273886+0000", 2023-09-23T18:39:36.301 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.9879999999999999e-06 2023-09-23T18:39:36.301 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.301 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.301 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:39:36.301 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.273893+0000", 2023-09-23T18:39:36.301 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.8979999999999999e-06 2023-09-23T18:39:36.302 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.302 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.302 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:39:36.302 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.277325+0000", 2023-09-23T18:39:36.302 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0034324210000000002 2023-09-23T18:39:36.302 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.302 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.302 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:39:36.302 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.277342+0000", 2023-09-23T18:39:36.302 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.6566e-05 2023-09-23T18:39:36.302 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:39:36.302 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:39:36.303 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:39:36.303 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.303 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.303 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.44954.0:132239 6.6 6:66ce143d:::benchmark_data_smithi098_131983_object27570:head [delete] snapc 0=[] ondisk+write+known_if_redirected e160)", 2023-09-23T18:39:36.303 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:39:36.274644+0000", 2023-09-23T18:39:36.303 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.019217626000000002, 2023-09-23T18:39:36.303 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.019323503999999998, 2023-09-23T18:39:36.303 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:39:36.303 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:39:36.303 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:39:36.303 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.44954", 2023-09-23T18:39:36.303 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/343998332", 2023-09-23T18:39:36.304 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 132239 2023-09-23T18:39:36.304 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.304 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:39:36.304 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.304 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:39:36.304 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.274644+0000", 2023-09-23T18:39:36.304 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:39:36.304 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.304 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.304 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:39:36.304 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.274644+0000", 2023-09-23T18:39:36.304 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:39:36.304 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.305 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.305 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:39:36.305 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.274642+0000", 2023-09-23T18:39:36.305 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T18:39:36.305 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.305 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.305 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:39:36.305 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.274650+0000", 2023-09-23T18:39:36.305 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.7559999999999999e-06 2023-09-23T18:39:36.305 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.305 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.305 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:39:36.305 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.274663+0000", 2023-09-23T18:39:36.306 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.2497000000000001e-05 2023-09-23T18:39:36.306 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.306 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.306 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:39:36.306 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.274672+0000", 2023-09-23T18:39:36.306 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.2979999999999997e-06 2023-09-23T18:39:36.306 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.306 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.306 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:39:36.306 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.274716+0000", 2023-09-23T18:39:36.306 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.4159000000000002e-05 2023-09-23T18:39:36.306 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.307 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.307 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:39:36.307 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.274841+0000", 2023-09-23T18:39:36.307 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00012536700000000001 2023-09-23T18:39:36.307 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.307 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.307 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:39:36.307 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.274956+0000", 2023-09-23T18:39:36.307 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000114122 2023-09-23T18:39:36.307 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.307 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.307 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:39:36.307 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.279881+0000", 2023-09-23T18:39:36.308 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0049252649999999999 2023-09-23T18:39:36.308 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.308 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.308 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:39:36.308 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.281356+0000", 2023-09-23T18:39:36.308 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0014754639999999999 2023-09-23T18:39:36.308 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:39:36.308 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:39:36.308 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:39:36.308 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.308 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.308 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.44954.0:132241 6.b 6:d2c16af1:::benchmark_data_smithi098_131983_object27572:head [delete] snapc 0=[] ondisk+write+known_if_redirected e160)", 2023-09-23T18:39:36.308 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:39:36.274833+0000", 2023-09-23T18:39:36.309 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.019028943, 2023-09-23T18:39:36.309 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.019230145000000001, 2023-09-23T18:39:36.309 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:39:36.309 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:39:36.309 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:39:36.309 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.44954", 2023-09-23T18:39:36.309 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/343998332", 2023-09-23T18:39:36.309 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 132241 2023-09-23T18:39:36.309 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.309 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:39:36.309 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.309 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:39:36.309 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.274833+0000", 2023-09-23T18:39:36.310 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:39:36.310 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.310 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.310 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:39:36.310 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.274833+0000", 2023-09-23T18:39:36.310 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:39:36.310 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.310 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.310 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:39:36.310 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.274832+0000", 2023-09-23T18:39:36.310 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:39:36.311 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.311 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.311 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:39:36.311 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.274837+0000", 2023-09-23T18:39:36.311 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.6840000000000004e-06 2023-09-23T18:39:36.311 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.311 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.311 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:39:36.311 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.274847+0000", 2023-09-23T18:39:36.311 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0152e-05 2023-09-23T18:39:36.311 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.311 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.311 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:39:36.312 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.274853+0000", 2023-09-23T18:39:36.312 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.7459999999999997e-06 2023-09-23T18:39:36.312 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.312 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.312 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:39:36.312 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.274888+0000", 2023-09-23T18:39:36.312 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.5672e-05 2023-09-23T18:39:36.312 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.312 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.312 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:39:36.312 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.274961+0000", 2023-09-23T18:39:36.312 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.2948999999999996e-05 2023-09-23T18:39:36.313 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.313 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.313 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 1,7", 2023-09-23T18:39:36.313 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.275034+0000", 2023-09-23T18:39:36.313 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.3187000000000002e-05 2023-09-23T18:39:36.313 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.313 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.313 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:39:36.313 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.278521+0000", 2023-09-23T18:39:36.313 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.003486687 2023-09-23T18:39:36.313 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.313 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.314 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:39:36.314 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.293604+0000", 2023-09-23T18:39:36.314 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.015082501999999999 2023-09-23T18:39:36.314 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:39:36.314 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:39:36.314 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:39:36.314 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.314 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.314 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.44954.0:132242 6.c 6:390b77af:::benchmark_data_smithi098_131983_object27573:head [delete] snapc 0=[] ondisk+write+known_if_redirected e160)", 2023-09-23T18:39:36.314 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:39:36.275825+0000", 2023-09-23T18:39:36.314 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.018036729000000001, 2023-09-23T18:39:36.315 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.018332172000000001, 2023-09-23T18:39:36.315 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:39:36.315 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:39:36.315 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:39:36.315 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.44954", 2023-09-23T18:39:36.315 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/343998332", 2023-09-23T18:39:36.315 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 132242 2023-09-23T18:39:36.315 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.315 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:39:36.315 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.315 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:39:36.315 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.275825+0000", 2023-09-23T18:39:36.315 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:39:36.316 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.316 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.316 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:39:36.316 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.275825+0000", 2023-09-23T18:39:36.316 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:39:36.316 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.316 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.316 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:39:36.316 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.275824+0000", 2023-09-23T18:39:36.316 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:39:36.316 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.316 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.317 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:39:36.317 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.275829+0000", 2023-09-23T18:39:36.317 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.0200000000000002e-06 2023-09-23T18:39:36.317 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.317 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.317 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:39:36.317 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.275837+0000", 2023-09-23T18:39:36.317 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.2679999999999994e-06 2023-09-23T18:39:36.317 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.317 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.317 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:39:36.317 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.275842+0000", 2023-09-23T18:39:36.318 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.0320000000000003e-06 2023-09-23T18:39:36.318 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.318 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.318 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:39:36.318 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.282874+0000", 2023-09-23T18:39:36.318 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0070322359999999999 2023-09-23T18:39:36.318 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.318 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.318 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:39:36.318 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.283012+0000", 2023-09-23T18:39:36.318 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00013749199999999999 2023-09-23T18:39:36.318 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.319 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.319 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 6,7", 2023-09-23T18:39:36.319 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.283138+0000", 2023-09-23T18:39:36.319 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000126267 2023-09-23T18:39:36.319 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.319 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.319 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:39:36.319 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.287526+0000", 2023-09-23T18:39:36.319 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0043877339999999999 2023-09-23T18:39:36.319 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:39:36.319 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:39:36.319 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:39:36.320 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.320 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.320 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.44954.0:132255 6.6 6:6d3f7770:::benchmark_data_smithi098_131983_object27586:head [delete] snapc 0=[] ondisk+write+known_if_redirected e160)", 2023-09-23T18:39:36.320 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:39:36.289440+0000", 2023-09-23T18:39:36.320 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.0044216890000000003, 2023-09-23T18:39:36.320 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0048034610000000002, 2023-09-23T18:39:36.320 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:39:36.320 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:39:36.320 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:39:36.320 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.44954", 2023-09-23T18:39:36.320 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/343998332", 2023-09-23T18:39:36.320 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 132255 2023-09-23T18:39:36.320 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.321 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:39:36.321 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.321 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:39:36.321 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.289440+0000", 2023-09-23T18:39:36.321 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:39:36.321 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.321 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.321 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:39:36.321 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.289440+0000", 2023-09-23T18:39:36.321 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:39:36.321 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.321 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.322 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:39:36.322 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.289438+0000", 2023-09-23T18:39:36.322 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T18:39:36.322 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.322 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.322 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:39:36.322 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.289447+0000", 2023-09-23T18:39:36.322 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.1279999999999997e-06 2023-09-23T18:39:36.322 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.322 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.322 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:39:36.322 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.289467+0000", 2023-09-23T18:39:36.323 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.9692999999999999e-05 2023-09-23T18:39:36.323 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.323 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.323 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:39:36.323 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.289476+0000", 2023-09-23T18:39:36.323 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.1460000000000002e-06 2023-09-23T18:39:36.323 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.323 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.323 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:39:36.323 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.289522+0000", 2023-09-23T18:39:36.323 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.5532999999999999e-05 2023-09-23T18:39:36.323 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.324 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.324 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:39:36.324 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.289842+0000", 2023-09-23T18:39:36.324 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00032026500000000002 2023-09-23T18:39:36.324 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.324 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.324 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:39:36.324 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.289963+0000", 2023-09-23T18:39:36.324 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00012068799999999999 2023-09-23T18:39:36.324 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:39:36.324 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:39:36.324 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:39:36.325 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.325 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.325 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.44954.0:132244 6.0 e160/149 6:0c5044ff:::benchmark_data_smithi098_131983_object27575:head v 160'8431, mlcod=160'8429)", 2023-09-23T18:39:36.325 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:39:36.293421+0000", 2023-09-23T18:39:36.325 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.00044112200000000001, 2023-09-23T18:39:36.325 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00094893899999999999, 2023-09-23T18:39:36.325 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:39:36.325 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T18:39:36.325 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:39:36.325 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.325 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:39:36.325 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.293421+0000", 2023-09-23T18:39:36.326 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:39:36.326 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.326 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.326 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:39:36.326 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.293421+0000", 2023-09-23T18:39:36.326 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:39:36.326 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.326 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.326 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:39:36.326 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.293420+0000", 2023-09-23T18:39:36.326 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:39:36.326 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.327 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.327 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:39:36.327 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.293428+0000", 2023-09-23T18:39:36.327 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.2290000000000003e-06 2023-09-23T18:39:36.327 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.327 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.327 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:39:36.327 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.293440+0000", 2023-09-23T18:39:36.327 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1832e-05 2023-09-23T18:39:36.327 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.327 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.327 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:39:36.328 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.293448+0000", 2023-09-23T18:39:36.328 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.3410000000000003e-06 2023-09-23T18:39:36.328 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.328 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.328 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:39:36.328 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.293629+0000", 2023-09-23T18:39:36.328 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00018076499999999999 2023-09-23T18:39:36.328 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.328 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.328 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:39:36.328 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.293641+0000", 2023-09-23T18:39:36.328 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.2098e-05 2023-09-23T18:39:36.329 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:39:36.329 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:39:36.329 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:39:36.329 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.329 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.329 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.44954.0:132245 6.4 e160/149)", 2023-09-23T18:39:36.329 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:39:36.293481+0000", 2023-09-23T18:39:36.329 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.00038057500000000002, 2023-09-23T18:39:36.329 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00095061500000000003, 2023-09-23T18:39:36.329 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:39:36.329 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "queued for pg", 2023-09-23T18:39:36.329 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:39:36.330 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.330 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:39:36.330 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.293481+0000", 2023-09-23T18:39:36.330 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:39:36.330 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.330 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.330 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:39:36.330 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.293481+0000", 2023-09-23T18:39:36.330 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:39:36.330 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.330 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.330 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:39:36.330 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.293481+0000", 2023-09-23T18:39:36.331 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:39:36.331 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.331 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.331 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:39:36.332 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.293486+0000", 2023-09-23T18:39:36.332 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.1309999999999997e-06 2023-09-23T18:39:36.332 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.332 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.332 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:39:36.332 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.293493+0000", 2023-09-23T18:39:36.332 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.3719999999999999e-06 2023-09-23T18:39:36.332 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.333 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.333 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:39:36.333 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.293498+0000", 2023-09-23T18:39:36.333 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.9150000000000004e-06 2023-09-23T18:39:36.333 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:39:36.333 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:39:36.333 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:39:36.333 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.333 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.333 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop_reply(client.44954.0:132239 6.6 e160/149 ondisk, result = 0)", 2023-09-23T18:39:36.333 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:39:36.293549+0000", 2023-09-23T18:39:36.333 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.00031314100000000002, 2023-09-23T18:39:36.334 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00096023799999999998, 2023-09-23T18:39:36.334 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:39:36.334 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T18:39:36.334 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:39:36.334 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.334 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:39:36.334 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.293549+0000", 2023-09-23T18:39:36.334 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:39:36.334 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.334 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.334 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:39:36.334 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.293549+0000", 2023-09-23T18:39:36.334 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:39:36.335 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.335 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.335 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:39:36.335 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.293548+0000", 2023-09-23T18:39:36.335 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:39:36.335 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.335 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.335 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:39:36.335 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.293551+0000", 2023-09-23T18:39:36.335 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.7590000000000002e-06 2023-09-23T18:39:36.335 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.335 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.336 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:39:36.336 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.293558+0000", 2023-09-23T18:39:36.336 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.7009999999999999e-06 2023-09-23T18:39:36.336 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.336 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.336 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:39:36.336 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.293562+0000", 2023-09-23T18:39:36.336 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.4490000000000001e-06 2023-09-23T18:39:36.336 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.336 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.336 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:39:36.336 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.294413+0000", 2023-09-23T18:39:36.336 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00085066700000000005 2023-09-23T18:39:36.337 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.337 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.337 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:39:36.337 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.294421+0000", 2023-09-23T18:39:36.337 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.7409999999999992e-06 2023-09-23T18:39:36.337 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.337 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.337 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:39:36.337 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.294509+0000", 2023-09-23T18:39:36.337 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.8268000000000003e-05 2023-09-23T18:39:36.337 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:39:36.337 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:39:36.337 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:39:36.338 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.338 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.338 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.44954.0:132249 6.8 e160/149)", 2023-09-23T18:39:36.338 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:39:36.294378+0000", 2023-09-23T18:39:36.338 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 4294967295.9994841, 2023-09-23T18:39:36.338 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00024332499999999999, 2023-09-23T18:39:36.338 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:39:36.338 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "queued for pg", 2023-09-23T18:39:36.338 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:39:36.338 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.338 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:39:36.338 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.294378+0000", 2023-09-23T18:39:36.339 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:39:36.339 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.339 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.339 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:39:36.339 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.294378+0000", 2023-09-23T18:39:36.339 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:39:36.339 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.339 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.339 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:39:36.339 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.294377+0000", 2023-09-23T18:39:36.339 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:39:36.339 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.339 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.340 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:39:36.340 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.294383+0000", 2023-09-23T18:39:36.340 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.3789999999999997e-06 2023-09-23T18:39:36.340 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.340 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.340 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:39:36.340 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.294393+0000", 2023-09-23T18:39:36.340 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0063e-05 2023-09-23T18:39:36.340 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:39:36.340 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:39:36.340 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:39:36.340 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:39:36.294400+0000", 2023-09-23T18:39:36.340 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.8549999999999996e-06 2023-09-23T18:39:36.341 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:39:36.341 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:39:36.341 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:39:36.341 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:39:36.341 INFO:teuthology.orchestra.run.smithi098.stdout: ], 2023-09-23T18:39:36.341 INFO:teuthology.orchestra.run.smithi098.stdout: "num_ops": 9 2023-09-23T18:39:36.341 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:39:36.360 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:36.395 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:36.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:36.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:36.500 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:36.530 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:36.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:36.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:36.641 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:36.664 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:36.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:36.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:36.769 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:36.809 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:36.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:36.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:36.913 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:36.942 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:36.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:36.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:37.045 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:37.069 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:37.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:37.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:37.174 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:37.183 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:36 smithi098 bash[76817]: cluster 2023-09-23T18:39:35.874579+0000 mgr.y (mgr.26054) 635 : cluster 0 pgmap v617: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 721 op/s 2023-09-23T18:39:37.183 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:36 smithi098 bash[79516]: cluster 2023-09-23T18:39:35.874579+0000 mgr.y (mgr.26054) 635 : cluster 0 pgmap v617: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 721 op/s 2023-09-23T18:39:37.198 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:37.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:37.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:37.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:36 smithi173 bash[59422]: cluster 2023-09-23T18:39:35.874579+0000 mgr.y (mgr.26054) 635 : cluster 0 pgmap v617: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 721 op/s 2023-09-23T18:39:37.305 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:37.328 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:37.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:37.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:37.431 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:37.460 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:37.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:37.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:37.565 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:37.606 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:37.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:37.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:37.710 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:37.734 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:37.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:37.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:37.840 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:37.864 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:37.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:37.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:37.968 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:37.991 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:37.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:37.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:38.094 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:38.130 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:38.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:38.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:38.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:38 smithi098 bash[79516]: audit 2023-09-23T18:39:37.282812+0000 mon.a (mon.0) 873 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:39:38.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:38 smithi098 bash[76817]: audit 2023-09-23T18:39:37.282812+0000 mon.a (mon.0) 873 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:39:38.235 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:38.246 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:38 smithi173 bash[59422]: audit 2023-09-23T18:39:37.282812+0000 mon.a (mon.0) 873 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:39:38.264 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:38.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:38.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:38.367 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:38.393 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:38.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:38.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:38.498 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:38.520 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:38.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:38.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:38.622 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:38.643 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:38.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:38.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:38.746 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:38.774 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:38.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:38.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:38.881 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:38.907 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:38.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:38.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:39.010 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:39 smithi098 bash[76817]: cluster 2023-09-23T18:39:37.875172+0000 mgr.y (mgr.26054) 636 : cluster 0 pgmap v618: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 782 op/s 2023-09-23T18:39:39.012 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:39.035 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:39.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:39.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:39.140 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:39.148 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:39 smithi098 bash[79516]: cluster 2023-09-23T18:39:37.875172+0000 mgr.y (mgr.26054) 636 : cluster 0 pgmap v618: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 782 op/s 2023-09-23T18:39:39.165 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:39.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:39.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:39.268 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:39.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:39 smithi173 bash[59422]: cluster 2023-09-23T18:39:37.875172+0000 mgr.y (mgr.26054) 636 : cluster 0 pgmap v618: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 782 op/s 2023-09-23T18:39:39.294 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:39.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:39.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:39.399 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:39.420 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:39.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:39.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:39.524 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:39.554 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:39.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:39.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:39.658 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:39.682 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:39.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:39.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:39.788 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:39.813 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:39.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:39.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:39.920 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:39.945 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:39.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:39.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:40.016 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:40 smithi173 bash[59422]: audit 2023-09-23T18:39:39.596749+0000 mon.a (mon.0) 874 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:39:40.050 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:40.084 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:40.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:40.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:40.181 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:40 smithi098 bash[76817]: audit 2023-09-23T18:39:39.596749+0000 mon.a (mon.0) 874 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:39:40.185 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:40 smithi098 bash[79516]: audit 2023-09-23T18:39:39.596749+0000 mon.a (mon.0) 874 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:39:40.192 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:40.226 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:40.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:40.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:40.332 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:40.379 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:40.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:40.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:40.484 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:40.516 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:40.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:40.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:40.624 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:40.651 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:40.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:40.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:40.758 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:40.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:40.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:40.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:40.889 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:40.915 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:40.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:40.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:41.020 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:41.041 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:41.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:41.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:41.145 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:41.171 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:41.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:41.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:41.272 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:41 smithi098 bash[76817]: cluster 2023-09-23T18:39:39.875686+0000 mgr.y (mgr.26054) 637 : cluster 0 pgmap v619: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 665 op/s 2023-09-23T18:39:41.273 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:41 smithi098 bash[79516]: cluster 2023-09-23T18:39:39.875686+0000 mgr.y (mgr.26054) 637 : cluster 0 pgmap v619: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 665 op/s 2023-09-23T18:39:41.276 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:41.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:41 smithi173 bash[59422]: cluster 2023-09-23T18:39:39.875686+0000 mgr.y (mgr.26054) 637 : cluster 0 pgmap v619: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 665 op/s 2023-09-23T18:39:41.310 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:41.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:41.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:41.416 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:41.442 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:41.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:41.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:41.548 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:41.567 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T18:39:41.571 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:41.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:41.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:41.678 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:41.703 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:41.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:41.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:41.807 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:41.833 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:41.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:41.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:41.939 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:41.964 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:41.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:41.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:42.068 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:42.077 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:42 smithi173 bash[59422]: cluster 2023-09-23T18:39:41.876255+0000 mgr.y (mgr.26054) 638 : cluster 0 pgmap v620: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 719 op/s 2023-09-23T18:39:42.094 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:42.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:42.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:42.197 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:42.230 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:42.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:42.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:42.339 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:42.347 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:42 smithi098 bash[76817]: cluster 2023-09-23T18:39:41.876255+0000 mgr.y (mgr.26054) 638 : cluster 0 pgmap v620: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 719 op/s 2023-09-23T18:39:42.347 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:42 smithi098 bash[79516]: cluster 2023-09-23T18:39:41.876255+0000 mgr.y (mgr.26054) 638 : cluster 0 pgmap v620: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 719 op/s 2023-09-23T18:39:42.380 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:42.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:42.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:42.484 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:42.519 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:42.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:42.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:42.624 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:42.680 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:42.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:42.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:42.783 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:42.813 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:42.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:42.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:42.920 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:42.947 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:42.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:42.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:43.050 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:43.081 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:43.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:43.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:43.185 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:43.218 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:43.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:43.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:43.322 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:43.355 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:43.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:43.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:43.460 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:43.488 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:43.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:43.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:43.591 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:43.625 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:43.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:43.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:43.732 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:43.764 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:43.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:43.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:43.872 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:43.892 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:43.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:43.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:43.997 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:44.020 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:44.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:44.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:44.129 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:44.159 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:44.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:44.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:44.262 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:44.289 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:44.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:44.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:44.394 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:44.419 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:44.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:44.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:44.523 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:44.557 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:44.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:44.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:44.662 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:44.686 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:44.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:44.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:44.789 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:44.823 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:44.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:44.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:44.826 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:44 smithi098 bash[76817]: audit 2023-09-23T18:39:43.636030+0000 mon.a (mon.0) 875 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:39:44.827 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:44 smithi098 bash[76817]: audit 2023-09-23T18:39:43.643768+0000 mon.a (mon.0) 876 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:39:44.827 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:44 smithi098 bash[76817]: cluster 2023-09-23T18:39:43.876809+0000 mgr.y (mgr.26054) 639 : cluster 0 pgmap v621: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 659 op/s 2023-09-23T18:39:44.827 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:44 smithi098 bash[79516]: audit 2023-09-23T18:39:43.636030+0000 mon.a (mon.0) 875 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:39:44.827 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:44 smithi098 bash[79516]: audit 2023-09-23T18:39:43.643768+0000 mon.a (mon.0) 876 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:39:44.827 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:44 smithi098 bash[79516]: cluster 2023-09-23T18:39:43.876809+0000 mgr.y (mgr.26054) 639 : cluster 0 pgmap v621: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 659 op/s 2023-09-23T18:39:44.884 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:39:44.927 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:44.948 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:39:44.986 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:44.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:44.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:45.007 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:39:45.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:44 smithi173 bash[59422]: audit 2023-09-23T18:39:43.636030+0000 mon.a (mon.0) 875 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:39:45.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:44 smithi173 bash[59422]: audit 2023-09-23T18:39:43.643768+0000 mon.a (mon.0) 876 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:39:45.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:44 smithi173 bash[59422]: cluster 2023-09-23T18:39:43.876809+0000 mgr.y (mgr.26054) 639 : cluster 0 pgmap v621: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 659 op/s 2023-09-23T18:39:45.047 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:39:45.091 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:45.102 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:39:45.118 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:45.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:45.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:45.169 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:39:45.222 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:45.242 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:39:45.250 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:45.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:45.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:45.276 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:39:45.284 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:39:45.285 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:39:45.285 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:39:45.285 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:39:45.285 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:39:45.285 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:39:45.285 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:39:45.285 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:39:45.354 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:45.388 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:45.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:45.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:45.495 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:45.529 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:45.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:45.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:45.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:45.653 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:45 smithi098 bash[76817]: audit 2023-09-23T18:39:44.838698+0000 mon.c (mon.2) 81 : audit 0 from='client.? 172.21.15.98:0/896876715' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:39:45.654 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:45 smithi098 bash[76817]: audit 2023-09-23T18:39:45.126931+0000 mon.c (mon.2) 82 : audit 1 from='client.? 172.21.15.98:0/340600749' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:39:45.654 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:45 smithi098 bash[76817]: audit 2023-09-23T18:39:45.131070+0000 mon.a (mon.0) 877 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:39:45.654 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:45 smithi098 bash[76817]: audit 2023-09-23T18:39:45.239942+0000 mon.a (mon.0) 878 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'44963','3':'47656'}"}]: dispatch 2023-09-23T18:39:45.660 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:45.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:45.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:45.764 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:45.773 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:45 smithi098 bash[79516]: audit 2023-09-23T18:39:44.838698+0000 mon.c (mon.2) 81 : audit 0 from='client.? 172.21.15.98:0/896876715' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:39:45.773 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:45 smithi098 bash[79516]: audit 2023-09-23T18:39:45.126931+0000 mon.c (mon.2) 82 : audit 1 from='client.? 172.21.15.98:0/340600749' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:39:45.773 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:45 smithi098 bash[79516]: audit 2023-09-23T18:39:45.131070+0000 mon.a (mon.0) 877 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:39:45.773 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:45 smithi098 bash[79516]: audit 2023-09-23T18:39:45.239942+0000 mon.a (mon.0) 878 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'44963','3':'47656'}"}]: dispatch 2023-09-23T18:39:45.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:45 smithi173 bash[59422]: audit 2023-09-23T18:39:44.838698+0000 mon.c (mon.2) 81 : audit 0 from='client.? 172.21.15.98:0/896876715' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:39:45.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:45 smithi173 bash[59422]: audit 2023-09-23T18:39:45.126931+0000 mon.c (mon.2) 82 : audit 1 from='client.? 172.21.15.98:0/340600749' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:39:45.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:45 smithi173 bash[59422]: audit 2023-09-23T18:39:45.131070+0000 mon.a (mon.0) 877 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:39:45.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:45 smithi173 bash[59422]: audit 2023-09-23T18:39:45.239942+0000 mon.a (mon.0) 878 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'44963','3':'47656'}"}]: dispatch 2023-09-23T18:39:45.797 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:45.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:45.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:45.900 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:45.935 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:45.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:45.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:45.979 INFO:teuthology.orchestra.run.smithi098.stdout:moved 19 / 267 (7.11611%) 2023-09-23T18:39:45.979 INFO:teuthology.orchestra.run.smithi098.stdout:avg 38.1429 2023-09-23T18:39:45.980 INFO:teuthology.orchestra.run.smithi098.stdout:stddev 3.97954 -> 6.83329 (expected baseline 5.71786) 2023-09-23T18:39:45.980 INFO:teuthology.orchestra.run.smithi098.stdout:min osd.2 with 31 -> 23 pgs (0.812734 -> 0.602996 * mean) 2023-09-23T18:39:45.980 INFO:teuthology.orchestra.run.smithi098.stdout:max osd.3 with 45 -> 37 pgs (1.17978 -> 0.970037 * mean) 2023-09-23T18:39:45.980 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:39:45.980 INFO:teuthology.orchestra.run.smithi098.stdout:oload 110 2023-09-23T18:39:45.980 INFO:teuthology.orchestra.run.smithi098.stdout:max_change 1 2023-09-23T18:39:45.980 INFO:teuthology.orchestra.run.smithi098.stdout:max_change_osds 4 2023-09-23T18:39:45.980 INFO:teuthology.orchestra.run.smithi098.stdout:average_utilization 0.0248 2023-09-23T18:39:45.980 INFO:teuthology.orchestra.run.smithi098.stdout:overload_utilization 0.0273 2023-09-23T18:39:45.980 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2 weight 1.0000 -> 0.6861 2023-09-23T18:39:45.980 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3 weight 1.0000 -> 0.7272 2023-09-23T18:39:45.995 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is unset 2023-09-23T18:39:46.040 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:46.068 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:46.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:46.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:46.170 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:46.183 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:39:46] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.33.4" 2023-09-23T18:39:46.198 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:46.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:46.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:46.301 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:46.323 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:46.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:46.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:46.426 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:46.454 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:46.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:46.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:46.557 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:46.589 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:46.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:46.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:46.694 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:46.727 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:46.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:46.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:46.830 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:46.860 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:46.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:46.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:46.963 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:46.999 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:47.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:47.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:47.102 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:47.130 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:47.132 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[76817]: audit 2023-09-23T18:39:45.238887+0000 mgr.y (mgr.26054) 640 : audit 0 from='client.35097 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:39:47.132 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[76817]: cluster 2023-09-23T18:39:45.877462+0000 mgr.y (mgr.26054) 641 : cluster 0 pgmap v622: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 728 op/s 2023-09-23T18:39:47.132 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[76817]: audit 2023-09-23T18:39:45.884585+0000 mon.a (mon.0) 879 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:39:47.132 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[76817]: audit 2023-09-23T18:39:45.893660+0000 mon.a (mon.0) 880 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:39:47.132 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[76817]: cluster 2023-09-23T18:39:45.968181+0000 mon.a (mon.0) 881 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:39:47.132 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[76817]: audit 2023-09-23T18:39:45.977618+0000 mon.a (mon.0) 882 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:39:47.133 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[76817]: cluster 2023-09-23T18:39:45.977869+0000 osd.4 (osd.4) 98 : cluster 3 failed to encode map e161 with expected crc 2023-09-23T18:39:47.133 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[76817]: audit 2023-09-23T18:39:45.978173+0000 mon.a (mon.0) 883 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'44963','3':'47656'}"}]': finished 2023-09-23T18:39:47.133 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[76817]: cluster 2023-09-23T18:39:45.982570+0000 mon.a (mon.0) 884 : cluster 0 osdmap e161: 8 total, 8 up, 7 in 2023-09-23T18:39:47.133 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[76817]: cluster 2023-09-23T18:39:45.984874+0000 osd.7 (osd.7) 72 : cluster 3 failed to encode map e161 with expected crc 2023-09-23T18:39:47.133 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[76817]: audit 2023-09-23T18:39:46.378590+0000 mon.a (mon.0) 885 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:39:47.133 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[76817]: audit 2023-09-23T18:39:46.379687+0000 mon.a (mon.0) 886 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:39:47.133 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[76817]: audit 2023-09-23T18:39:46.385013+0000 mon.a (mon.0) 887 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:39:47.133 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[76817]: cluster 2023-09-23T18:39:46.502927+0000 osd.5 (osd.5) 65 : cluster 3 failed to encode map e161 with expected crc 2023-09-23T18:39:47.133 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[79516]: audit 2023-09-23T18:39:45.238887+0000 mgr.y (mgr.26054) 640 : audit 0 from='client.35097 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:39:47.134 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[79516]: cluster 2023-09-23T18:39:45.877462+0000 mgr.y (mgr.26054) 641 : cluster 0 pgmap v622: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 728 op/s 2023-09-23T18:39:47.134 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[79516]: audit 2023-09-23T18:39:45.884585+0000 mon.a (mon.0) 879 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:39:47.134 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[79516]: audit 2023-09-23T18:39:45.893660+0000 mon.a (mon.0) 880 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:39:47.134 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[79516]: cluster 2023-09-23T18:39:45.968181+0000 mon.a (mon.0) 881 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:39:47.134 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[79516]: audit 2023-09-23T18:39:45.977618+0000 mon.a (mon.0) 882 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:39:47.134 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[79516]: cluster 2023-09-23T18:39:45.977869+0000 osd.4 (osd.4) 98 : cluster 3 failed to encode map e161 with expected crc 2023-09-23T18:39:47.134 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[79516]: audit 2023-09-23T18:39:45.978173+0000 mon.a (mon.0) 883 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'44963','3':'47656'}"}]': finished 2023-09-23T18:39:47.134 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[79516]: cluster 2023-09-23T18:39:45.982570+0000 mon.a (mon.0) 884 : cluster 0 osdmap e161: 8 total, 8 up, 7 in 2023-09-23T18:39:47.134 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[79516]: cluster 2023-09-23T18:39:45.984874+0000 osd.7 (osd.7) 72 : cluster 3 failed to encode map e161 with expected crc 2023-09-23T18:39:47.134 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[79516]: audit 2023-09-23T18:39:46.378590+0000 mon.a (mon.0) 885 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:39:47.134 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[79516]: audit 2023-09-23T18:39:46.379687+0000 mon.a (mon.0) 886 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:39:47.134 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[79516]: audit 2023-09-23T18:39:46.385013+0000 mon.a (mon.0) 887 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:39:47.135 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:46 smithi098 bash[79516]: cluster 2023-09-23T18:39:46.502927+0000 osd.5 (osd.5) 65 : cluster 3 failed to encode map e161 with expected crc 2023-09-23T18:39:47.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:47.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:47.236 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:47.251 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:39:47.261 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:47.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:47.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:47.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:46 smithi173 bash[59422]: audit 2023-09-23T18:39:45.238887+0000 mgr.y (mgr.26054) 640 : audit 0 from='client.35097 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:39:47.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:46 smithi173 bash[59422]: cluster 2023-09-23T18:39:45.877462+0000 mgr.y (mgr.26054) 641 : cluster 0 pgmap v622: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 728 op/s 2023-09-23T18:39:47.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:46 smithi173 bash[59422]: audit 2023-09-23T18:39:45.884585+0000 mon.a (mon.0) 879 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:39:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:46 smithi173 bash[59422]: audit 2023-09-23T18:39:45.893660+0000 mon.a (mon.0) 880 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:39:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:46 smithi173 bash[59422]: cluster 2023-09-23T18:39:45.968181+0000 mon.a (mon.0) 881 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:39:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:46 smithi173 bash[59422]: audit 2023-09-23T18:39:45.977618+0000 mon.a (mon.0) 882 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:39:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:46 smithi173 bash[59422]: cluster 2023-09-23T18:39:45.977869+0000 osd.4 (osd.4) 98 : cluster 3 failed to encode map e161 with expected crc 2023-09-23T18:39:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:46 smithi173 bash[59422]: audit 2023-09-23T18:39:45.978173+0000 mon.a (mon.0) 883 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'44963','3':'47656'}"}]': finished 2023-09-23T18:39:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:46 smithi173 bash[59422]: cluster 2023-09-23T18:39:45.982570+0000 mon.a (mon.0) 884 : cluster 0 osdmap e161: 8 total, 8 up, 7 in 2023-09-23T18:39:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:46 smithi173 bash[59422]: cluster 2023-09-23T18:39:45.984874+0000 osd.7 (osd.7) 72 : cluster 3 failed to encode map e161 with expected crc 2023-09-23T18:39:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:46 smithi173 bash[59422]: audit 2023-09-23T18:39:46.378590+0000 mon.a (mon.0) 885 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:39:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:46 smithi173 bash[59422]: audit 2023-09-23T18:39:46.379687+0000 mon.a (mon.0) 886 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:39:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:46 smithi173 bash[59422]: audit 2023-09-23T18:39:46.385013+0000 mon.a (mon.0) 887 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:39:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:46 smithi173 bash[59422]: cluster 2023-09-23T18:39:46.502927+0000 osd.5 (osd.5) 65 : cluster 3 failed to encode map e161 with expected crc 2023-09-23T18:39:47.365 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:47.394 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:47.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:47.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:47.498 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:47.528 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:47.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:47.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:47.631 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:47.665 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:47.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:47.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:47.769 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:47.803 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:47.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:47.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:47.906 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:47.937 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:47.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:47.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:48.041 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:48.066 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:48.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:48.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:48.169 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:48.195 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:48.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:48.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:48.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:47 smithi173 bash[59422]: cluster 2023-09-23T18:39:46.974934+0000 osd.5 (osd.5) 66 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T18:39:48.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:47 smithi173 bash[59422]: cluster 2023-09-23T18:39:46.979237+0000 osd.7 (osd.7) 73 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T18:39:48.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:47 smithi173 bash[59422]: cluster 2023-09-23T18:39:46.979557+0000 mon.a (mon.0) 888 : cluster 0 osdmap e162: 8 total, 8 up, 7 in 2023-09-23T18:39:48.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:47 smithi173 bash[59422]: cluster 2023-09-23T18:39:46.980010+0000 osd.4 (osd.4) 99 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T18:39:48.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:47 smithi173 bash[59422]: cluster 2023-09-23T18:39:46.981011+0000 osd.6 (osd.6) 73 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T18:39:48.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:47 smithi173 bash[59422]: cluster 2023-09-23T18:39:46.981108+0000 osd.7 (osd.7) 74 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T18:39:48.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:47 smithi173 bash[59422]: cluster 2023-09-23T18:39:46.982473+0000 osd.4 (osd.4) 100 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T18:39:48.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:47 smithi173 bash[59422]: cluster 2023-09-23T18:39:46.984138+0000 osd.4 (osd.4) 101 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T18:39:48.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:47 smithi173 bash[59422]: cluster 2023-09-23T18:39:46.984766+0000 osd.6 (osd.6) 74 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T18:39:48.300 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:48.352 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:48.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:48.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:48.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:47 smithi098 bash[79516]: cluster 2023-09-23T18:39:46.974934+0000 osd.5 (osd.5) 66 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T18:39:48.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:47 smithi098 bash[79516]: cluster 2023-09-23T18:39:46.979237+0000 osd.7 (osd.7) 73 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T18:39:48.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:47 smithi098 bash[79516]: cluster 2023-09-23T18:39:46.979557+0000 mon.a (mon.0) 888 : cluster 0 osdmap e162: 8 total, 8 up, 7 in 2023-09-23T18:39:48.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:47 smithi098 bash[79516]: cluster 2023-09-23T18:39:46.980010+0000 osd.4 (osd.4) 99 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T18:39:48.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:47 smithi098 bash[79516]: cluster 2023-09-23T18:39:46.981011+0000 osd.6 (osd.6) 73 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T18:39:48.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:47 smithi098 bash[79516]: cluster 2023-09-23T18:39:46.981108+0000 osd.7 (osd.7) 74 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T18:39:48.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:47 smithi098 bash[79516]: cluster 2023-09-23T18:39:46.982473+0000 osd.4 (osd.4) 100 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T18:39:48.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:47 smithi098 bash[79516]: cluster 2023-09-23T18:39:46.984138+0000 osd.4 (osd.4) 101 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T18:39:48.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:47 smithi098 bash[79516]: cluster 2023-09-23T18:39:46.984766+0000 osd.6 (osd.6) 74 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T18:39:48.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:47 smithi098 bash[76817]: cluster 2023-09-23T18:39:46.974934+0000 osd.5 (osd.5) 66 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T18:39:48.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:47 smithi098 bash[76817]: cluster 2023-09-23T18:39:46.979237+0000 osd.7 (osd.7) 73 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T18:39:48.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:47 smithi098 bash[76817]: cluster 2023-09-23T18:39:46.979557+0000 mon.a (mon.0) 888 : cluster 0 osdmap e162: 8 total, 8 up, 7 in 2023-09-23T18:39:48.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:47 smithi098 bash[76817]: cluster 2023-09-23T18:39:46.980010+0000 osd.4 (osd.4) 99 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T18:39:48.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:47 smithi098 bash[76817]: cluster 2023-09-23T18:39:46.981011+0000 osd.6 (osd.6) 73 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T18:39:48.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:47 smithi098 bash[76817]: cluster 2023-09-23T18:39:46.981108+0000 osd.7 (osd.7) 74 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T18:39:48.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:47 smithi098 bash[76817]: cluster 2023-09-23T18:39:46.982473+0000 osd.4 (osd.4) 100 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T18:39:48.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:47 smithi098 bash[76817]: cluster 2023-09-23T18:39:46.984138+0000 osd.4 (osd.4) 101 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T18:39:48.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:47 smithi098 bash[76817]: cluster 2023-09-23T18:39:46.984766+0000 osd.6 (osd.6) 74 : cluster 3 failed to encode map e162 with expected crc 2023-09-23T18:39:48.494 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:48.526 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:48.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:48.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:48.631 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:48.667 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:48.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:48.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:48.771 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:48.800 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:48.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:48.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:48.903 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:48.939 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:48.941 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:39:48.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:48.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:48.947 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T18:39:48.947 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-09-23T18:39:49.044 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:49.075 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:49.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:49.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:49.179 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:49.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[79516]: cluster 2023-09-23T18:39:47.878018+0000 mgr.y (mgr.26054) 642 : cluster 0 pgmap v625: 89 pgs: 1 remapped+peering, 1 unknown, 3 peering, 84 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 752 op/s 2023-09-23T18:39:49.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[79516]: cluster 2023-09-23T18:39:47.987168+0000 osd.1 (osd.1) 1 : cluster 0 6.1 starting backfill to osd.7 from (0'0,0'0] MAX to 160'8843 2023-09-23T18:39:49.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[79516]: cluster 2023-09-23T18:39:47.989801+0000 osd.4 (osd.4) 102 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:49.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[79516]: cluster 2023-09-23T18:39:47.992267+0000 osd.4 (osd.4) 103 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:49.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[79516]: cluster 2023-09-23T18:39:47.992411+0000 osd.6 (osd.6) 75 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:49.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[79516]: cluster 2023-09-23T18:39:47.993223+0000 osd.0 (osd.0) 4 : cluster 0 6.9 starting backfill to osd.3 from (0'0,0'0] MAX to 160'8578 2023-09-23T18:39:49.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[79516]: cluster 2023-09-23T18:39:47.993932+0000 osd.0 (osd.0) 5 : cluster 0 6.0 starting backfill to osd.1 from (0'0,0'0] MAX to 160'8822 2023-09-23T18:39:49.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[79516]: cluster 2023-09-23T18:39:47.994119+0000 mon.a (mon.0) 889 : cluster 0 osdmap e163: 8 total, 8 up, 7 in 2023-09-23T18:39:49.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[79516]: cluster 2023-09-23T18:39:47.994467+0000 osd.4 (osd.4) 104 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:49.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[79516]: cluster 2023-09-23T18:39:47.994961+0000 osd.6 (osd.6) 76 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:49.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[79516]: cluster 2023-09-23T18:39:47.999338+0000 osd.6 (osd.6) 77 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:49.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[79516]: cluster 2023-09-23T18:39:47.999369+0000 osd.4 (osd.4) 105 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:49.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[79516]: cluster 2023-09-23T18:39:48.003012+0000 osd.6 (osd.6) 78 : cluster 0 6.a starting backfill to osd.1 from (0'0,0'0] MAX to 160'8850 2023-09-23T18:39:49.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[79516]: cluster 2023-09-23T18:39:48.003373+0000 osd.4 (osd.4) 106 : cluster 0 6.e starting backfill to osd.7 from (0'0,0'0] MAX to 160'8849 2023-09-23T18:39:49.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[79516]: cluster 2023-09-23T18:39:48.003592+0000 osd.4 (osd.4) 107 : cluster 0 6.2 starting backfill to osd.6 from (0'0,0'0] MAX to 160'8502 2023-09-23T18:39:49.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[79516]: cluster 2023-09-23T18:39:48.025984+0000 osd.7 (osd.7) 75 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:49.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[76817]: cluster 2023-09-23T18:39:47.878018+0000 mgr.y (mgr.26054) 642 : cluster 0 pgmap v625: 89 pgs: 1 remapped+peering, 1 unknown, 3 peering, 84 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 752 op/s 2023-09-23T18:39:49.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[76817]: cluster 2023-09-23T18:39:47.987168+0000 osd.1 (osd.1) 1 : cluster 0 6.1 starting backfill to osd.7 from (0'0,0'0] MAX to 160'8843 2023-09-23T18:39:49.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[76817]: cluster 2023-09-23T18:39:47.989801+0000 osd.4 (osd.4) 102 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:49.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[76817]: cluster 2023-09-23T18:39:47.992267+0000 osd.4 (osd.4) 103 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:49.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[76817]: cluster 2023-09-23T18:39:47.992411+0000 osd.6 (osd.6) 75 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:49.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[76817]: cluster 2023-09-23T18:39:47.993223+0000 osd.0 (osd.0) 4 : cluster 0 6.9 starting backfill to osd.3 from (0'0,0'0] MAX to 160'8578 2023-09-23T18:39:49.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[76817]: cluster 2023-09-23T18:39:47.993932+0000 osd.0 (osd.0) 5 : cluster 0 6.0 starting backfill to osd.1 from (0'0,0'0] MAX to 160'8822 2023-09-23T18:39:49.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[76817]: cluster 2023-09-23T18:39:47.994119+0000 mon.a (mon.0) 889 : cluster 0 osdmap e163: 8 total, 8 up, 7 in 2023-09-23T18:39:49.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[76817]: cluster 2023-09-23T18:39:47.994467+0000 osd.4 (osd.4) 104 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:49.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[76817]: cluster 2023-09-23T18:39:47.994961+0000 osd.6 (osd.6) 76 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:49.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[76817]: cluster 2023-09-23T18:39:47.999338+0000 osd.6 (osd.6) 77 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:49.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[76817]: cluster 2023-09-23T18:39:47.999369+0000 osd.4 (osd.4) 105 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:49.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[76817]: cluster 2023-09-23T18:39:48.003012+0000 osd.6 (osd.6) 78 : cluster 0 6.a starting backfill to osd.1 from (0'0,0'0] MAX to 160'8850 2023-09-23T18:39:49.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[76817]: cluster 2023-09-23T18:39:48.003373+0000 osd.4 (osd.4) 106 : cluster 0 6.e starting backfill to osd.7 from (0'0,0'0] MAX to 160'8849 2023-09-23T18:39:49.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[76817]: cluster 2023-09-23T18:39:48.003592+0000 osd.4 (osd.4) 107 : cluster 0 6.2 starting backfill to osd.6 from (0'0,0'0] MAX to 160'8502 2023-09-23T18:39:49.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:48 smithi098 bash[76817]: cluster 2023-09-23T18:39:48.025984+0000 osd.7 (osd.7) 75 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:49.211 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:49.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:49.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:49.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:48 smithi173 bash[59422]: cluster 2023-09-23T18:39:47.878018+0000 mgr.y (mgr.26054) 642 : cluster 0 pgmap v625: 89 pgs: 1 remapped+peering, 1 unknown, 3 peering, 84 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 752 op/s 2023-09-23T18:39:49.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:48 smithi173 bash[59422]: cluster 2023-09-23T18:39:47.987168+0000 osd.1 (osd.1) 1 : cluster 0 6.1 starting backfill to osd.7 from (0'0,0'0] MAX to 160'8843 2023-09-23T18:39:49.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:48 smithi173 bash[59422]: cluster 2023-09-23T18:39:47.989801+0000 osd.4 (osd.4) 102 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:49.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:48 smithi173 bash[59422]: cluster 2023-09-23T18:39:47.992267+0000 osd.4 (osd.4) 103 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:49.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:48 smithi173 bash[59422]: cluster 2023-09-23T18:39:47.992411+0000 osd.6 (osd.6) 75 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:49.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:48 smithi173 bash[59422]: cluster 2023-09-23T18:39:47.993223+0000 osd.0 (osd.0) 4 : cluster 0 6.9 starting backfill to osd.3 from (0'0,0'0] MAX to 160'8578 2023-09-23T18:39:49.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:48 smithi173 bash[59422]: cluster 2023-09-23T18:39:47.993932+0000 osd.0 (osd.0) 5 : cluster 0 6.0 starting backfill to osd.1 from (0'0,0'0] MAX to 160'8822 2023-09-23T18:39:49.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:48 smithi173 bash[59422]: cluster 2023-09-23T18:39:47.994119+0000 mon.a (mon.0) 889 : cluster 0 osdmap e163: 8 total, 8 up, 7 in 2023-09-23T18:39:49.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:48 smithi173 bash[59422]: cluster 2023-09-23T18:39:47.994467+0000 osd.4 (osd.4) 104 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:49.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:48 smithi173 bash[59422]: cluster 2023-09-23T18:39:47.994961+0000 osd.6 (osd.6) 76 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:49.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:48 smithi173 bash[59422]: cluster 2023-09-23T18:39:47.999338+0000 osd.6 (osd.6) 77 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:49.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:48 smithi173 bash[59422]: cluster 2023-09-23T18:39:47.999369+0000 osd.4 (osd.4) 105 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:49.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:48 smithi173 bash[59422]: cluster 2023-09-23T18:39:48.003012+0000 osd.6 (osd.6) 78 : cluster 0 6.a starting backfill to osd.1 from (0'0,0'0] MAX to 160'8850 2023-09-23T18:39:49.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:48 smithi173 bash[59422]: cluster 2023-09-23T18:39:48.003373+0000 osd.4 (osd.4) 106 : cluster 0 6.e starting backfill to osd.7 from (0'0,0'0] MAX to 160'8849 2023-09-23T18:39:49.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:48 smithi173 bash[59422]: cluster 2023-09-23T18:39:48.003592+0000 osd.4 (osd.4) 107 : cluster 0 6.2 starting backfill to osd.6 from (0'0,0'0] MAX to 160'8502 2023-09-23T18:39:49.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:48 smithi173 bash[59422]: cluster 2023-09-23T18:39:48.025984+0000 osd.7 (osd.7) 75 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:49.318 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:49.343 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:49.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:49.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:49.447 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:49.471 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:49.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:49.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:49.575 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:49.610 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:49.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:49.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:49.713 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:49.738 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:49.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:49.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:49.843 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:49.886 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:49.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:49.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:49.992 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:50.031 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:50.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:50.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:50.133 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:50 smithi098 bash[76817]: cluster 2023-09-23T18:39:49.531862+0000 osd.5 (osd.5) 67 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:50.133 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:50 smithi098 bash[76817]: cluster 2023-09-23T18:39:49.532996+0000 osd.5 (osd.5) 68 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:50.133 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:50 smithi098 bash[79516]: cluster 2023-09-23T18:39:49.531862+0000 osd.5 (osd.5) 67 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:50.134 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:50 smithi098 bash[79516]: cluster 2023-09-23T18:39:49.532996+0000 osd.5 (osd.5) 68 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:50.138 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:50.161 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:50.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:50.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:50.265 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:50.274 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:50 smithi173 bash[59422]: cluster 2023-09-23T18:39:49.531862+0000 osd.5 (osd.5) 67 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:50.274 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:50 smithi173 bash[59422]: cluster 2023-09-23T18:39:49.532996+0000 osd.5 (osd.5) 68 : cluster 3 failed to encode map e163 with expected crc 2023-09-23T18:39:50.297 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:50.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:50.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:50.400 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:50.428 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:50.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:50.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:50.531 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:50.584 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:50.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:50.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:50.691 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:50.717 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:50.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:50.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:50.822 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:50.845 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:50.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:50.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:50.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:50.981 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:50.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:50.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:51.085 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:51.102 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:51 smithi098 bash[76817]: cluster 2023-09-23T18:39:49.878538+0000 mgr.y (mgr.26054) 643 : cluster 0 pgmap v627: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 remapped+peering, 1 unknown, 3 peering, 82 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 557 op/s; 8582/211047 objects misplaced (4.066%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T18:39:51.103 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:51 smithi098 bash[79516]: cluster 2023-09-23T18:39:49.878538+0000 mgr.y (mgr.26054) 643 : cluster 0 pgmap v627: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 remapped+peering, 1 unknown, 3 peering, 82 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 557 op/s; 8582/211047 objects misplaced (4.066%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T18:39:51.115 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:51.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:51.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:51.221 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:51.245 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:51.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:51.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:51.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:51 smithi173 bash[59422]: cluster 2023-09-23T18:39:49.878538+0000 mgr.y (mgr.26054) 643 : cluster 0 pgmap v627: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 remapped+peering, 1 unknown, 3 peering, 82 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 557 op/s; 8582/211047 objects misplaced (4.066%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T18:39:51.349 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:51.382 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:51.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:51.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:51.438 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:39:51.488 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:51.546 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:51.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:51.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:51.652 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:51.688 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:51.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:51.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:51.792 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:51.830 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:51.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:51.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:51.938 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:51.974 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:51.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:51.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:52.079 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:52.104 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:52.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:52.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:52.208 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:52.236 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:52.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:52.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:52.340 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:52.382 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:52.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:52.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:52.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:52.516 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:52.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:52.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:52.618 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:52.647 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:52.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:52.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:52.754 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:52.785 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:52.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:52.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:52.888 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:52.917 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:52.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:52.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:53.024 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:53 smithi173 bash[59422]: cluster 2023-09-23T18:39:51.879051+0000 mgr.y (mgr.26054) 644 : cluster 0 pgmap v628: 89 pgs: 2 active+remapped+backfilling, 3 active+remapped+backfill_wait, 1 remapped+peering, 1 unknown, 82 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 417 op/s; 21377/209160 objects misplaced (10.220%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T18:39:53.061 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:53.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:53.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:53.167 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:53.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:53 smithi098 bash[76817]: cluster 2023-09-23T18:39:51.879051+0000 mgr.y (mgr.26054) 644 : cluster 0 pgmap v628: 89 pgs: 2 active+remapped+backfilling, 3 active+remapped+backfill_wait, 1 remapped+peering, 1 unknown, 82 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 417 op/s; 21377/209160 objects misplaced (10.220%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T18:39:53.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:53 smithi098 bash[79516]: cluster 2023-09-23T18:39:51.879051+0000 mgr.y (mgr.26054) 644 : cluster 0 pgmap v628: 89 pgs: 2 active+remapped+backfilling, 3 active+remapped+backfill_wait, 1 remapped+peering, 1 unknown, 82 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 417 op/s; 21377/209160 objects misplaced (10.220%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T18:39:53.200 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:53.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:53.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:53.305 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:53.337 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:53.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:53.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:53.443 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:53.470 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:53.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:53.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:53.573 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:53.606 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:53.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:53.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:53.714 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:53.743 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:53.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:53.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:53.846 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:53.887 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:53.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:53.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:53.998 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:54.024 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:54.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:54.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:54.129 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:54.170 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:54.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:54.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:54.273 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:54.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:54 smithi173 bash[59422]: cluster 2023-09-23T18:39:53.879643+0000 mgr.y (mgr.26054) 645 : cluster 0 pgmap v629: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 464 op/s; 81/205650 objects degraded (0.039%); 29495/205650 objects misplaced (14.342%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T18:39:54.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:54 smithi173 bash[59422]: cluster 2023-09-23T18:39:54.011173+0000 mon.a (mon.0) 890 : cluster 3 Health check failed: Degraded data redundancy: 81/205650 objects degraded (0.039%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:39:54.299 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:54.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:54.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:54.404 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:54.431 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:54.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:54.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:54.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:54 smithi098 bash[76817]: cluster 2023-09-23T18:39:53.879643+0000 mgr.y (mgr.26054) 645 : cluster 0 pgmap v629: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 464 op/s; 81/205650 objects degraded (0.039%); 29495/205650 objects misplaced (14.342%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T18:39:54.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:54 smithi098 bash[76817]: cluster 2023-09-23T18:39:54.011173+0000 mon.a (mon.0) 890 : cluster 3 Health check failed: Degraded data redundancy: 81/205650 objects degraded (0.039%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:39:54.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:54 smithi098 bash[79516]: cluster 2023-09-23T18:39:53.879643+0000 mgr.y (mgr.26054) 645 : cluster 0 pgmap v629: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 464 op/s; 81/205650 objects degraded (0.039%); 29495/205650 objects misplaced (14.342%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T18:39:54.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:54 smithi098 bash[79516]: cluster 2023-09-23T18:39:54.011173+0000 mon.a (mon.0) 890 : cluster 3 Health check failed: Degraded data redundancy: 81/205650 objects degraded (0.039%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:39:54.449 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:39:54.449 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:39:54.534 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:54.567 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:54.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:54.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:54.672 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:54.716 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:54.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:54.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:54.803 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:39:54.807 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:39:54.823 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:54.873 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:54.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:54.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:54.977 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:55.004 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:55.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:55.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:55.111 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:55.135 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:55.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:55.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:55.238 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:55.263 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:55.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:55.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:55.366 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:55.389 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:55.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:55.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:55.492 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:55.520 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:55.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:55.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:55.624 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:55.658 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:55.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:55.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:55.761 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:55.790 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:55.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:55.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:55.863 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:55 smithi098 bash[76817]: audit 2023-09-23T18:39:54.602962+0000 mon.a (mon.0) 891 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:39:55.863 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:55 smithi098 bash[76817]: audit 2023-09-23T18:39:54.604310+0000 mon.a (mon.0) 892 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:39:55.863 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:55 smithi098 bash[79516]: audit 2023-09-23T18:39:54.602962+0000 mon.a (mon.0) 891 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:39:55.863 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:55 smithi098 bash[79516]: audit 2023-09-23T18:39:54.604310+0000 mon.a (mon.0) 892 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:39:55.893 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:55.931 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:55.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:55.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:56.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:55 smithi173 bash[59422]: audit 2023-09-23T18:39:54.602962+0000 mon.a (mon.0) 891 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:39:56.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:55 smithi173 bash[59422]: audit 2023-09-23T18:39:54.604310+0000 mon.a (mon.0) 892 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:39:56.037 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:56.066 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:56.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:56.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:56.169 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:56.190 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:39:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:39:56] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.33.4" 2023-09-23T18:39:56.195 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:56.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:56.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:56.298 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:56.328 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:56.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:56.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:56.437 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:56.460 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:56.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:56.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:56.563 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:56.596 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:56.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:56.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:56.615 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:56 smithi098 bash[76817]: cluster 2023-09-23T18:39:55.880301+0000 mgr.y (mgr.26054) 646 : cluster 0 pgmap v630: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 4.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 410 op/s; 81/202143 objects degraded (0.040%); 28348/202143 objects misplaced (14.024%); 8.7 MiB/s, 138 objects/s recovering 2023-09-23T18:39:56.699 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:56.707 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:56 smithi173 bash[59422]: cluster 2023-09-23T18:39:55.880301+0000 mgr.y (mgr.26054) 646 : cluster 0 pgmap v630: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 4.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 410 op/s; 81/202143 objects degraded (0.040%); 28348/202143 objects misplaced (14.024%); 8.7 MiB/s, 138 objects/s recovering 2023-09-23T18:39:56.729 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:56.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:56.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:56.836 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:56.865 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:56.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:56.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:56.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:56 smithi098 bash[79516]: cluster 2023-09-23T18:39:55.880301+0000 mgr.y (mgr.26054) 646 : cluster 0 pgmap v630: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 4.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 410 op/s; 81/202143 objects degraded (0.040%); 28348/202143 objects misplaced (14.024%); 8.7 MiB/s, 138 objects/s recovering 2023-09-23T18:39:56.968 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:57.002 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:57.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:57.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:57.105 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:57.127 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:57.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:57.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:57.231 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:57.258 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:57.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:57.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:57.364 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:57.398 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:57.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:57.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:57.504 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:57.538 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:57.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:57.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:57.641 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:57.663 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:57.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:57.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:57.766 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:57.791 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:57.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:57.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:57.894 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:57.927 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:57.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:57.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:58.031 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:58.065 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:58.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:58.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:58.172 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:58.206 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:39:58.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:58.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:39:58.286 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T18:39:58.309 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:58.337 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:58.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:58.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:58.440 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:58.471 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:58.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:58.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:58.575 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:58.609 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:39:58.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:58.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:39:58.715 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:58.748 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:39:58.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:58.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:39:58.764 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:39:58.856 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:58.885 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:39:58.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:58.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:39:58.990 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:59.015 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:39:59.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:59.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:39:59.119 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:59.131 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:39:58 smithi098 bash[76817]: cluster 2023-09-23T18:39:57.880969+0000 mgr.y (mgr.26054) 647 : cluster 0 pgmap v631: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 194/196359 objects degraded (0.099%); 27055/196359 objects misplaced (13.778%); 12 MiB/s, 187 objects/s recovering 2023-09-23T18:39:59.131 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:39:58 smithi098 bash[79516]: cluster 2023-09-23T18:39:57.880969+0000 mgr.y (mgr.26054) 647 : cluster 0 pgmap v631: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 194/196359 objects degraded (0.099%); 27055/196359 objects misplaced (13.778%); 12 MiB/s, 187 objects/s recovering 2023-09-23T18:39:59.162 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:59.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:59.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:59.265 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:59.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:58 smithi173 bash[59422]: cluster 2023-09-23T18:39:57.880969+0000 mgr.y (mgr.26054) 647 : cluster 0 pgmap v631: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 194/196359 objects degraded (0.099%); 27055/196359 objects misplaced (13.778%); 12 MiB/s, 187 objects/s recovering 2023-09-23T18:39:59.305 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:59.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:59.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:59.408 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:59.436 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:59.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:59.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:59.543 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:59.571 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:59.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:59.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:39:59.675 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:59.717 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:39:59.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:59.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:39:59.824 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:59.849 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:39:59.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:59.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:39:59.956 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:59.978 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:39:59.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:59.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:00.081 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:00.105 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:00.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:00.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:00.211 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:00.239 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:00.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:00.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:00.267 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:00 smithi098 bash[76817]: cluster 2023-09-23T18:39:59.941485+0000 mon.a (mon.0) 893 : cluster 3 Health check update: Degraded data redundancy: 194/195246 objects degraded (0.099%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:00.267 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:00 smithi098 bash[79516]: cluster 2023-09-23T18:39:59.941485+0000 mon.a (mon.0) 893 : cluster 3 Health check update: Degraded data redundancy: 194/195246 objects degraded (0.099%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:00.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:39:59 smithi173 bash[59422]: cluster 2023-09-23T18:39:59.941485+0000 mon.a (mon.0) 893 : cluster 3 Health check update: Degraded data redundancy: 194/195246 objects degraded (0.099%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:00.345 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:00.389 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:00.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:00.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:00.491 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:00.537 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:00.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:00.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:00.643 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:00.674 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:00.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:00.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:00.780 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:00.805 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:00.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:00.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:00.912 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:00.948 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:00.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:00.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:01.052 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:01.060 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:00 smithi173 bash[59422]: cluster 2023-09-23T18:39:59.881497+0000 mgr.y (mgr.26054) 648 : cluster 0 pgmap v632: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 194/195246 objects degraded (0.099%); 26273/195246 objects misplaced (13.456%); 12 MiB/s, 195 objects/s recovering 2023-09-23T18:40:01.060 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:00 smithi173 bash[59422]: cluster 2023-09-23T18:40:00.000106+0000 mon.a (mon.0) 894 : cluster 3 Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 194/195246 objects degraded (0.099%), 1 pg degraded 2023-09-23T18:40:01.060 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:00 smithi173 bash[59422]: cluster 2023-09-23T18:40:00.000142+0000 mon.a (mon.0) 895 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T18:40:01.060 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:00 smithi173 bash[59422]: cluster 2023-09-23T18:40:00.000163+0000 mon.a (mon.0) 896 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 194/195246 objects degraded (0.099%), 1 pg degraded 2023-09-23T18:40:01.060 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:00 smithi173 bash[59422]: cluster 2023-09-23T18:40:00.000183+0000 mon.a (mon.0) 897 : cluster 3 pg 6.f is active+recovering+undersized+degraded+remapped, acting [3,4] 2023-09-23T18:40:01.076 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:01.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:01.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:01.184 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:01.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:00 smithi098 bash[76817]: cluster 2023-09-23T18:39:59.881497+0000 mgr.y (mgr.26054) 648 : cluster 0 pgmap v632: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 194/195246 objects degraded (0.099%); 26273/195246 objects misplaced (13.456%); 12 MiB/s, 195 objects/s recovering 2023-09-23T18:40:01.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:00 smithi098 bash[76817]: cluster 2023-09-23T18:40:00.000106+0000 mon.a (mon.0) 894 : cluster 3 Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 194/195246 objects degraded (0.099%), 1 pg degraded 2023-09-23T18:40:01.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:00 smithi098 bash[76817]: cluster 2023-09-23T18:40:00.000142+0000 mon.a (mon.0) 895 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T18:40:01.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:00 smithi098 bash[76817]: cluster 2023-09-23T18:40:00.000163+0000 mon.a (mon.0) 896 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 194/195246 objects degraded (0.099%), 1 pg degraded 2023-09-23T18:40:01.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:00 smithi098 bash[76817]: cluster 2023-09-23T18:40:00.000183+0000 mon.a (mon.0) 897 : cluster 3 pg 6.f is active+recovering+undersized+degraded+remapped, acting [3,4] 2023-09-23T18:40:01.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:00 smithi098 bash[79516]: cluster 2023-09-23T18:39:59.881497+0000 mgr.y (mgr.26054) 648 : cluster 0 pgmap v632: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 194/195246 objects degraded (0.099%); 26273/195246 objects misplaced (13.456%); 12 MiB/s, 195 objects/s recovering 2023-09-23T18:40:01.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:00 smithi098 bash[79516]: cluster 2023-09-23T18:40:00.000106+0000 mon.a (mon.0) 894 : cluster 3 Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 194/195246 objects degraded (0.099%), 1 pg degraded 2023-09-23T18:40:01.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:00 smithi098 bash[79516]: cluster 2023-09-23T18:40:00.000142+0000 mon.a (mon.0) 895 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T18:40:01.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:00 smithi098 bash[79516]: cluster 2023-09-23T18:40:00.000163+0000 mon.a (mon.0) 896 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 194/195246 objects degraded (0.099%), 1 pg degraded 2023-09-23T18:40:01.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:00 smithi098 bash[79516]: cluster 2023-09-23T18:40:00.000183+0000 mon.a (mon.0) 897 : cluster 3 pg 6.f is active+recovering+undersized+degraded+remapped, acting [3,4] 2023-09-23T18:40:01.229 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:01.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:01.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:01.332 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:01.358 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:01.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:01.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:01.464 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:01.490 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:01.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:01.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:01.596 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:01.627 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:01.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:01.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:01.732 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:01.765 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:01.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:01.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:01.874 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:01.915 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:01.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:01.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:02.018 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:02.053 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:02.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:02.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:02.157 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:02.191 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:02.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:02.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:02.296 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:02.330 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:02.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:02.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:02.434 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:02.457 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:02.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:02.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:02.563 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:02.597 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:02.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:02.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:02.702 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:02.729 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:02.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:02.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:02.832 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:02.867 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:02.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:02.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:02.978 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:03.003 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:03 smithi173 bash[59422]: cluster 2023-09-23T18:40:01.882025+0000 mgr.y (mgr.26054) 649 : cluster 0 pgmap v633: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 194/191682 objects degraded (0.101%); 25591/191682 objects misplaced (13.351%); 12 MiB/s, 192 objects/s recovering 2023-09-23T18:40:03.021 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:03.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:03.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:03.126 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:03.163 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:03.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:03.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:03.266 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:03.275 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:03 smithi098 bash[79516]: cluster 2023-09-23T18:40:01.882025+0000 mgr.y (mgr.26054) 649 : cluster 0 pgmap v633: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 194/191682 objects degraded (0.101%); 25591/191682 objects misplaced (13.351%); 12 MiB/s, 192 objects/s recovering 2023-09-23T18:40:03.275 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:03 smithi098 bash[76817]: cluster 2023-09-23T18:40:01.882025+0000 mgr.y (mgr.26054) 649 : cluster 0 pgmap v633: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 194/191682 objects degraded (0.101%); 25591/191682 objects misplaced (13.351%); 12 MiB/s, 192 objects/s recovering 2023-09-23T18:40:03.290 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:03.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:03.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:03.402 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:03.435 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:03.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:03.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:03.539 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:03.560 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:03.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:03.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:03.663 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:03.690 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:03.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:03.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:03.795 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:03.829 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:03.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:03.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:03.936 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:03.961 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:03.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:03.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:04.064 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:04.094 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:04.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:04.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:04.201 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:04.228 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:04.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:04.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:04.335 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:04.360 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:04.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:04.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:04.463 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:04.493 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:04.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:04.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:04.597 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:04.625 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:04.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:04.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:04.729 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:04.766 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:04.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:04.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:04.869 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:04.897 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:04.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:04.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:05.001 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:05.027 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:05.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:05.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:05.133 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:05.160 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:05.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:05.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:05.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:05 smithi098 bash[79516]: cluster 2023-09-23T18:40:03.882607+0000 mgr.y (mgr.26054) 650 : cluster 0 pgmap v634: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 276/187077 objects degraded (0.148%); 24741/187077 objects misplaced (13.225%); 15 MiB/s, 238 objects/s recovering 2023-09-23T18:40:05.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:05 smithi098 bash[79516]: cluster 2023-09-23T18:40:04.968665+0000 mon.a (mon.0) 898 : cluster 3 Health check update: Degraded data redundancy: 276/187077 objects degraded (0.148%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:05.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:05 smithi098 bash[76817]: cluster 2023-09-23T18:40:03.882607+0000 mgr.y (mgr.26054) 650 : cluster 0 pgmap v634: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 276/187077 objects degraded (0.148%); 24741/187077 objects misplaced (13.225%); 15 MiB/s, 238 objects/s recovering 2023-09-23T18:40:05.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:05 smithi098 bash[76817]: cluster 2023-09-23T18:40:04.968665+0000 mon.a (mon.0) 898 : cluster 3 Health check update: Degraded data redundancy: 276/187077 objects degraded (0.148%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:05.265 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:05.272 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:05 smithi173 bash[59422]: cluster 2023-09-23T18:40:03.882607+0000 mgr.y (mgr.26054) 650 : cluster 0 pgmap v634: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 276/187077 objects degraded (0.148%); 24741/187077 objects misplaced (13.225%); 15 MiB/s, 238 objects/s recovering 2023-09-23T18:40:05.272 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:05 smithi173 bash[59422]: cluster 2023-09-23T18:40:04.968665+0000 mon.a (mon.0) 898 : cluster 3 Health check update: Degraded data redundancy: 276/187077 objects degraded (0.148%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:05.285 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:05.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:05.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:05.387 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:05.416 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:05.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:05.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:05.522 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:05.556 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:05.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:05.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:05.666 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:05.694 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:05.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:05.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:05.797 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:05.838 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:05.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:05.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:05.941 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:05.967 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:05.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:05.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:06.073 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:06.113 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:06.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:06.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:06.220 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:06.244 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:06.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:06.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:06.346 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:06.382 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:06.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:06.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:06.440 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:40:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:40:06] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.33.4" 2023-09-23T18:40:06.487 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:06.532 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:06.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:06.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:06.637 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:06.669 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:06.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:06.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:06.776 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:06.812 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:06.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:06.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:06.916 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:06.949 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:06.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:06.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:07.052 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:07.089 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:07.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:07.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:07.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:07 smithi098 bash[79516]: cluster 2023-09-23T18:40:05.883217+0000 mgr.y (mgr.26054) 651 : cluster 0 pgmap v635: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 276/183780 objects degraded (0.150%); 23685/183780 objects misplaced (12.888%); 15 MiB/s, 246 objects/s recovering 2023-09-23T18:40:07.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:07 smithi098 bash[76817]: cluster 2023-09-23T18:40:05.883217+0000 mgr.y (mgr.26054) 651 : cluster 0 pgmap v635: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 276/183780 objects degraded (0.150%); 23685/183780 objects misplaced (12.888%); 15 MiB/s, 246 objects/s recovering 2023-09-23T18:40:07.194 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:07.202 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:07 smithi173 bash[59422]: cluster 2023-09-23T18:40:05.883217+0000 mgr.y (mgr.26054) 651 : cluster 0 pgmap v635: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 276/183780 objects degraded (0.150%); 23685/183780 objects misplaced (12.888%); 15 MiB/s, 246 objects/s recovering 2023-09-23T18:40:07.229 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:07.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:07.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:07.333 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:07.367 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:07.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:07.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:07.469 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:07.493 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:07.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:07.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:07.596 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:07.625 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:07.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:07.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:07.731 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:07.763 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:07.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:07.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:07.872 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:07.901 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:07.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:07.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:07.965 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:40:08.007 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:08.031 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:08.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:08.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:08.135 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:08.164 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:08.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:08.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:08.269 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:08.298 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:08.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:08.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:08.407 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:08.428 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:08.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:08.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:08.531 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:08.559 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:08.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:08.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:08.663 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:08.685 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:08.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:08.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:08.788 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:08.830 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:08.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:08.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:08.936 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:08.965 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:08.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:08.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:09.068 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:09.103 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:09.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:09.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:09.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:09 smithi098 bash[76817]: cluster 2023-09-23T18:40:07.883903+0000 mgr.y (mgr.26054) 652 : cluster 0 pgmap v636: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 666 op/s; 414/178125 objects degraded (0.232%); 22535/178125 objects misplaced (12.651%); 15 MiB/s, 247 objects/s recovering 2023-09-23T18:40:09.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:09 smithi098 bash[79516]: cluster 2023-09-23T18:40:07.883903+0000 mgr.y (mgr.26054) 652 : cluster 0 pgmap v636: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 666 op/s; 414/178125 objects degraded (0.232%); 22535/178125 objects misplaced (12.651%); 15 MiB/s, 247 objects/s recovering 2023-09-23T18:40:09.209 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:09.247 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:09.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:09.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:09.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:09 smithi173 bash[59422]: cluster 2023-09-23T18:40:07.883903+0000 mgr.y (mgr.26054) 652 : cluster 0 pgmap v636: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 666 op/s; 414/178125 objects degraded (0.232%); 22535/178125 objects misplaced (12.651%); 15 MiB/s, 247 objects/s recovering 2023-09-23T18:40:09.350 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:09.390 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:09.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:09.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:09.496 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:09.524 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:09.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:09.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:09.627 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:09.655 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:09.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:09.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:09.758 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:09.803 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:09.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:09.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:09.906 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:09.935 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:09.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:09.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:10.038 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:10.043 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:10 smithi098 bash[76817]: audit 2023-09-23T18:40:09.598590+0000 mon.a (mon.0) 899 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:40:10.044 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:10 smithi098 bash[76817]: cluster 2023-09-23T18:40:09.969840+0000 mon.a (mon.0) 900 : cluster 3 Health check update: Degraded data redundancy: 414/178125 objects degraded (0.232%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:10.044 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:10 smithi098 bash[79516]: audit 2023-09-23T18:40:09.598590+0000 mon.a (mon.0) 899 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:40:10.044 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:10 smithi098 bash[79516]: cluster 2023-09-23T18:40:09.969840+0000 mon.a (mon.0) 900 : cluster 3 Health check update: Degraded data redundancy: 414/178125 objects degraded (0.232%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:10.064 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:10.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:10.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:10.174 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:10.213 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:10.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:10.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:10.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:10 smithi173 bash[59422]: audit 2023-09-23T18:40:09.598590+0000 mon.a (mon.0) 899 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:40:10.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:10 smithi173 bash[59422]: cluster 2023-09-23T18:40:09.969840+0000 mon.a (mon.0) 900 : cluster 3 Health check update: Degraded data redundancy: 414/178125 objects degraded (0.232%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:10.317 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:10.345 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:10.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:10.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:10.450 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:10.477 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:10.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:10.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:10.580 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:10.611 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:10.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:10.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:10.714 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:10.743 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:10.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:10.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:10.852 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:10.880 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:10.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:10.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:10.985 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:11.010 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:11.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:11.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:11.114 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:11.143 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:11.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:11.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:11.250 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:11.261 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:11 smithi173 bash[59422]: cluster 2023-09-23T18:40:09.886726+0000 mgr.y (mgr.26054) 653 : cluster 0 pgmap v637: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 414/177066 objects degraded (0.234%); 21787/177066 objects misplaced (12.304%); 15 MiB/s, 232 objects/s recovering 2023-09-23T18:40:11.278 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:11.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:11.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:11.383 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:11.391 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:11 smithi098 bash[76817]: cluster 2023-09-23T18:40:09.886726+0000 mgr.y (mgr.26054) 653 : cluster 0 pgmap v637: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 414/177066 objects degraded (0.234%); 21787/177066 objects misplaced (12.304%); 15 MiB/s, 232 objects/s recovering 2023-09-23T18:40:11.392 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:11 smithi098 bash[79516]: cluster 2023-09-23T18:40:09.886726+0000 mgr.y (mgr.26054) 653 : cluster 0 pgmap v637: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 414/177066 objects degraded (0.234%); 21787/177066 objects misplaced (12.304%); 15 MiB/s, 232 objects/s recovering 2023-09-23T18:40:11.407 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:11.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:11.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:11.510 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:11.562 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:11.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:11.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:11.666 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:11.697 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:11.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:11.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:11.805 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:11.832 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:11.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:11.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:11.935 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:11.956 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:11.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:11.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:12.059 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:12.069 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:12 smithi173 bash[59422]: cluster 2023-09-23T18:40:11.887681+0000 mgr.y (mgr.26054) 654 : cluster 0 pgmap v638: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 414/173631 objects degraded (0.238%); 21121/173631 objects misplaced (12.164%); 13 MiB/s, 204 objects/s recovering 2023-09-23T18:40:12.088 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:12.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:12.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:12.191 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:12.199 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:12 smithi098 bash[79516]: cluster 2023-09-23T18:40:11.887681+0000 mgr.y (mgr.26054) 654 : cluster 0 pgmap v638: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 414/173631 objects degraded (0.238%); 21121/173631 objects misplaced (12.164%); 13 MiB/s, 204 objects/s recovering 2023-09-23T18:40:12.199 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:12 smithi098 bash[76817]: cluster 2023-09-23T18:40:11.887681+0000 mgr.y (mgr.26054) 654 : cluster 0 pgmap v638: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 414/173631 objects degraded (0.238%); 21121/173631 objects misplaced (12.164%); 13 MiB/s, 204 objects/s recovering 2023-09-23T18:40:12.220 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:12.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:12.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:12.317 INFO:teuthology.orchestra.run.smithi098.stdout:moved 0 / 267 (0%) 2023-09-23T18:40:12.317 INFO:teuthology.orchestra.run.smithi098.stdout:avg 38.1429 2023-09-23T18:40:12.317 INFO:teuthology.orchestra.run.smithi098.stdout:stddev 6.83329 -> 6.83329 (expected baseline 5.71786) 2023-09-23T18:40:12.317 INFO:teuthology.orchestra.run.smithi098.stdout:min osd.2 with 23 -> 23 pgs (0.602996 -> 0.602996 * mean) 2023-09-23T18:40:12.317 INFO:teuthology.orchestra.run.smithi098.stdout:max osd.1 with 46 -> 46 pgs (1.20599 -> 1.20599 * mean) 2023-09-23T18:40:12.317 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:40:12.317 INFO:teuthology.orchestra.run.smithi098.stdout:oload 1000 2023-09-23T18:40:12.317 INFO:teuthology.orchestra.run.smithi098.stdout:max_change 0.05 2023-09-23T18:40:12.317 INFO:teuthology.orchestra.run.smithi098.stdout:max_change_osds 4 2023-09-23T18:40:12.317 INFO:teuthology.orchestra.run.smithi098.stdout:average_utilization 0.0206 2023-09-23T18:40:12.318 INFO:teuthology.orchestra.run.smithi098.stdout:overload_utilization 0.2063 2023-09-23T18:40:12.322 INFO:teuthology.orchestra.run.smithi098.stderr:no change 2023-09-23T18:40:12.326 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:12.350 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:12.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:12.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:12.457 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:12.500 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:12.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:12.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:12.606 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:12.630 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:12.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:12.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:12.737 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:12.773 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:12.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:12.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:12.877 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:12.905 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:12.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:12.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:13.008 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:13.045 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:13.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:13.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:13.150 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:13.185 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:13.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:13.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:13.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:13 smithi173 bash[59422]: audit 2023-09-23T18:40:12.315156+0000 mgr.y (mgr.26054) 655 : audit 0 from='client.35106 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:40:13.294 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:13.317 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:13.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:13.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:13.426 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:13.443 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:13.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:13.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:13.484 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:13 smithi098 bash[79516]: audit 2023-09-23T18:40:12.315156+0000 mgr.y (mgr.26054) 655 : audit 0 from='client.35106 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:40:13.484 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:13 smithi098 bash[76817]: audit 2023-09-23T18:40:12.315156+0000 mgr.y (mgr.26054) 655 : audit 0 from='client.35106 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:40:13.546 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:13.585 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:13.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:13.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:13.692 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:13.721 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:13.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:13.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:13.824 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:13.858 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:13.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:13.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:13.907 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:40:13.962 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:13.969 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:40:14.012 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:14.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:14.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:14.021 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:40:14.064 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:14 smithi098 bash[76817]: audit 2023-09-23T18:40:13.865534+0000 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.98:0/1919413075' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:40:14.078 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:40:14.114 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:40:14.117 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:14.146 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:14.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:14.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:14.155 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:40:14.202 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:40:14.239 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:40:14.248 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:40:14.249 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:40:14.249 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:40:14.249 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:40:14.249 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:40:14.250 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:40:14.250 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:40:14.250 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:40:14.250 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:14.286 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:14.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:14.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:14.316 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:14 smithi098 bash[79516]: audit 2023-09-23T18:40:13.865534+0000 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.98:0/1919413075' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:40:14.316 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:14 smithi098 bash[79516]: cluster 2023-09-23T18:40:13.888331+0000 mgr.y (mgr.26054) 656 : cluster 0 pgmap v639: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 525/169245 objects degraded (0.310%); 20252/169245 objects misplaced (11.966%); 16 MiB/s, 251 objects/s recovering 2023-09-23T18:40:14.316 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:14 smithi098 bash[76817]: cluster 2023-09-23T18:40:13.888331+0000 mgr.y (mgr.26054) 656 : cluster 0 pgmap v639: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 525/169245 objects degraded (0.310%); 20252/169245 objects misplaced (11.966%); 16 MiB/s, 251 objects/s recovering 2023-09-23T18:40:14.389 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:14.399 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:14 smithi173 bash[59422]: audit 2023-09-23T18:40:13.865534+0000 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.98:0/1919413075' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:40:14.399 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:14 smithi173 bash[59422]: cluster 2023-09-23T18:40:13.888331+0000 mgr.y (mgr.26054) 656 : cluster 0 pgmap v639: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 525/169245 objects degraded (0.310%); 20252/169245 objects misplaced (11.966%); 16 MiB/s, 251 objects/s recovering 2023-09-23T18:40:14.425 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:14.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:14.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:14.532 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:14.560 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:14.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:14.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:14.666 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:14.701 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:14.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:14.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:14.806 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:14.842 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:14.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:14.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:14.946 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:14.971 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:14.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:14.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:15.067 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:15 smithi173 bash[59422]: cluster 2023-09-23T18:40:14.971071+0000 mon.a (mon.0) 901 : cluster 3 Health check update: Degraded data redundancy: 525/169245 objects degraded (0.310%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:15.076 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:15.108 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:15.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:15.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:15.215 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:15.245 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:15.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:15.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:15.352 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:15.362 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:15 smithi098 bash[76817]: cluster 2023-09-23T18:40:14.971071+0000 mon.a (mon.0) 901 : cluster 3 Health check update: Degraded data redundancy: 525/169245 objects degraded (0.310%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:15.363 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:15 smithi098 bash[79516]: cluster 2023-09-23T18:40:14.971071+0000 mon.a (mon.0) 901 : cluster 3 Health check update: Degraded data redundancy: 525/169245 objects degraded (0.310%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:15.383 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:15.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:15.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:15.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:15.513 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:15.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:15.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:15.622 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:15.652 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:15.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:15.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:15.761 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:15.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:15.793 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:15.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:15.894 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:15.928 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:15.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:15.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:16.031 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:16.064 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:16.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:16.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:16.148 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T18:40:16.148 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-09-23T18:40:16.168 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:16.244 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:16.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:16.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:16.350 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:16.360 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:16 smithi098 bash[76817]: cluster 2023-09-23T18:40:15.888966+0000 mgr.y (mgr.26054) 657 : cluster 0 pgmap v640: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 525/165831 objects degraded (0.317%); 19252/165831 objects misplaced (11.609%); 16 MiB/s, 248 objects/s recovering 2023-09-23T18:40:16.361 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:40:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:40:16] "GET /metrics HTTP/1.1" 200 36569 "" "Prometheus/2.33.4" 2023-09-23T18:40:16.361 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:16 smithi098 bash[79516]: cluster 2023-09-23T18:40:15.888966+0000 mgr.y (mgr.26054) 657 : cluster 0 pgmap v640: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 525/165831 objects degraded (0.317%); 19252/165831 objects misplaced (11.609%); 16 MiB/s, 248 objects/s recovering 2023-09-23T18:40:16.401 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:16.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:16.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:16.504 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:16.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:16 smithi173 bash[59422]: cluster 2023-09-23T18:40:15.888966+0000 mgr.y (mgr.26054) 657 : cluster 0 pgmap v640: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 525/165831 objects degraded (0.317%); 19252/165831 objects misplaced (11.609%); 16 MiB/s, 248 objects/s recovering 2023-09-23T18:40:16.540 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:16.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:16.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:16.646 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:16.673 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:16.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:16.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:16.775 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:16.799 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:16.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:16.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:16.908 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:16.940 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:16.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:16.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:17.043 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:17.089 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:17.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:17.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:17.195 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:17.229 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:17.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:17.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:17.334 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:17.355 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:17.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:17.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:17.466 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:17.488 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:17.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:17.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:17.595 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:17.624 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:17.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:17.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:17.728 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:17.771 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:17.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:17.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:17.876 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:17.906 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:17.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:17.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:18.011 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:18.033 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:18.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:18.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:18.143 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:18.169 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:18.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:18.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:18.276 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:18.301 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:18.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:18.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:18.406 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:18.450 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:18.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:18.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:18.554 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:18.579 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:18.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:18.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:18.686 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:18.715 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:18.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:18.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:18.822 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:18.850 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:18.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:18.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:18.956 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:18.959 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:18 smithi098 bash[76817]: cluster 2023-09-23T18:40:17.889620+0000 mgr.y (mgr.26054) 658 : cluster 0 pgmap v641: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 648 op/s; 632/160404 objects degraded (0.394%); 18153/160404 objects misplaced (11.317%); 16 MiB/s, 250 objects/s recovering 2023-09-23T18:40:18.974 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is unset 2023-09-23T18:40:18.978 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:18.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:18.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:19.088 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:19.111 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:19.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:19.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:19.215 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:19.225 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:18 smithi098 bash[76817]: audit 2023-09-23T18:40:18.432137+0000 mon.c (mon.2) 84 : audit 1 from='client.? 172.21.15.98:0/3758670855' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:40:19.225 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:18 smithi098 bash[76817]: audit 2023-09-23T18:40:18.432736+0000 mon.a (mon.0) 902 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:40:19.225 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:18 smithi098 bash[79516]: cluster 2023-09-23T18:40:17.889620+0000 mgr.y (mgr.26054) 658 : cluster 0 pgmap v641: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 648 op/s; 632/160404 objects degraded (0.394%); 18153/160404 objects misplaced (11.317%); 16 MiB/s, 250 objects/s recovering 2023-09-23T18:40:19.225 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:18 smithi098 bash[79516]: audit 2023-09-23T18:40:18.432137+0000 mon.c (mon.2) 84 : audit 1 from='client.? 172.21.15.98:0/3758670855' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:40:19.226 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:18 smithi098 bash[79516]: audit 2023-09-23T18:40:18.432736+0000 mon.a (mon.0) 902 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:40:19.241 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:19.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:19.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:19.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:18 smithi173 bash[59422]: cluster 2023-09-23T18:40:17.889620+0000 mgr.y (mgr.26054) 658 : cluster 0 pgmap v641: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 648 op/s; 632/160404 objects degraded (0.394%); 18153/160404 objects misplaced (11.317%); 16 MiB/s, 250 objects/s recovering 2023-09-23T18:40:19.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:18 smithi173 bash[59422]: audit 2023-09-23T18:40:18.432137+0000 mon.c (mon.2) 84 : audit 1 from='client.? 172.21.15.98:0/3758670855' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:40:19.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:18 smithi173 bash[59422]: audit 2023-09-23T18:40:18.432736+0000 mon.a (mon.0) 902 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:40:19.346 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:19.385 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:19.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:19.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:19.490 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:19.519 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:19.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:19.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:19.624 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:19.649 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:19.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:19.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:19.752 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:19.762 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:40:19.775 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:19.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:19.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:19.879 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:19.906 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:19.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:19.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:20.009 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:20.040 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:20.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:20.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:20.143 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:20.152 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:19 smithi173 bash[59422]: cluster 2023-09-23T18:40:18.944008+0000 mon.a (mon.0) 903 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:40:20.152 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:19 smithi173 bash[59422]: cluster 2023-09-23T18:40:18.947082+0000 osd.7 (osd.7) 76 : cluster 3 failed to encode map e164 with expected crc 2023-09-23T18:40:20.152 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:19 smithi173 bash[59422]: audit 2023-09-23T18:40:18.948446+0000 mon.a (mon.0) 904 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:40:20.152 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:19 smithi173 bash[59422]: cluster 2023-09-23T18:40:18.948593+0000 mon.a (mon.0) 905 : cluster 0 osdmap e164: 8 total, 8 up, 7 in 2023-09-23T18:40:20.152 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:19 smithi173 bash[59422]: cluster 2023-09-23T18:40:19.135968+0000 osd.5 (osd.5) 69 : cluster 3 failed to encode map e164 with expected crc 2023-09-23T18:40:20.171 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:20.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:20.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:20.200 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:19 smithi098 bash[76817]: cluster 2023-09-23T18:40:18.944008+0000 mon.a (mon.0) 903 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:40:20.200 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:19 smithi098 bash[76817]: cluster 2023-09-23T18:40:18.947082+0000 osd.7 (osd.7) 76 : cluster 3 failed to encode map e164 with expected crc 2023-09-23T18:40:20.200 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:19 smithi098 bash[76817]: audit 2023-09-23T18:40:18.948446+0000 mon.a (mon.0) 904 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:40:20.200 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:19 smithi098 bash[76817]: cluster 2023-09-23T18:40:18.948593+0000 mon.a (mon.0) 905 : cluster 0 osdmap e164: 8 total, 8 up, 7 in 2023-09-23T18:40:20.200 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:19 smithi098 bash[76817]: cluster 2023-09-23T18:40:19.135968+0000 osd.5 (osd.5) 69 : cluster 3 failed to encode map e164 with expected crc 2023-09-23T18:40:20.201 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:19 smithi098 bash[79516]: cluster 2023-09-23T18:40:18.944008+0000 mon.a (mon.0) 903 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:40:20.201 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:19 smithi098 bash[79516]: cluster 2023-09-23T18:40:18.947082+0000 osd.7 (osd.7) 76 : cluster 3 failed to encode map e164 with expected crc 2023-09-23T18:40:20.201 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:19 smithi098 bash[79516]: audit 2023-09-23T18:40:18.948446+0000 mon.a (mon.0) 904 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:40:20.201 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:19 smithi098 bash[79516]: cluster 2023-09-23T18:40:18.948593+0000 mon.a (mon.0) 905 : cluster 0 osdmap e164: 8 total, 8 up, 7 in 2023-09-23T18:40:20.201 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:19 smithi098 bash[79516]: cluster 2023-09-23T18:40:19.135968+0000 osd.5 (osd.5) 69 : cluster 3 failed to encode map e164 with expected crc 2023-09-23T18:40:20.279 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:20.311 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:20.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:20.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:20.414 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:20.439 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:20.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:20.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:20.543 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:20.584 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:20.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:20.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:20.689 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:20.712 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:20.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:20.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:20.815 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:20.841 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:20.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:20.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:20.944 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:20.964 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:20.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:20.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:21.067 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:21.097 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:21.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:21.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:21.200 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:21.230 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:21.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:21.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:21.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:20 smithi173 bash[59422]: cluster 2023-09-23T18:40:19.890709+0000 mgr.y (mgr.26054) 659 : cluster 0 pgmap v643: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 632/159177 objects degraded (0.397%); 17446/159177 objects misplaced (10.960%); 15 MiB/s, 240 objects/s recovering 2023-09-23T18:40:21.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:20 smithi173 bash[59422]: cluster 2023-09-23T18:40:19.973075+0000 mon.a (mon.0) 906 : cluster 3 Health check update: Degraded data redundancy: 632/159177 objects degraded (0.397%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:21.336 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:21.364 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:21.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:21.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:21.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:21 smithi098 bash[79516]: cluster 2023-09-23T18:40:19.890709+0000 mgr.y (mgr.26054) 659 : cluster 0 pgmap v643: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 632/159177 objects degraded (0.397%); 17446/159177 objects misplaced (10.960%); 15 MiB/s, 240 objects/s recovering 2023-09-23T18:40:21.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:21 smithi098 bash[79516]: cluster 2023-09-23T18:40:19.973075+0000 mon.a (mon.0) 906 : cluster 3 Health check update: Degraded data redundancy: 632/159177 objects degraded (0.397%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:21.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:20 smithi098 bash[76817]: cluster 2023-09-23T18:40:19.890709+0000 mgr.y (mgr.26054) 659 : cluster 0 pgmap v643: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 632/159177 objects degraded (0.397%); 17446/159177 objects misplaced (10.960%); 15 MiB/s, 240 objects/s recovering 2023-09-23T18:40:21.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:20 smithi098 bash[76817]: cluster 2023-09-23T18:40:19.973075+0000 mon.a (mon.0) 906 : cluster 3 Health check update: Degraded data redundancy: 632/159177 objects degraded (0.397%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:21.469 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:21.496 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:21.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:21.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:21.599 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:21.644 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:21.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:21.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:21.750 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:21.777 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:21.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:21.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:21.888 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:21.922 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:21.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:21.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:22.025 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:22.066 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:22.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:22.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:22.169 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:22.190 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:22.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:22.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:22.296 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:22.321 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:22.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:22.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:22.425 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:22.448 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:22.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:22.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:22.553 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:22.579 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:22.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:22.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:22.682 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:22.708 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:22.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:22.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:22.811 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:22.835 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:22.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:22.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:22.939 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:22.964 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:22.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:22.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:23.064 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:40:23.064 INFO:teuthology.orchestra.run.smithi098.stdout: "size": 20, 2023-09-23T18:40:23.065 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 600, 2023-09-23T18:40:23.065 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [ 2023-09-23T18:40:23.065 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.065 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.44954.0:158507 6.f 6:f5ae40ba:::benchmark_data_smithi098_131983_object53838:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-09-23T18:40:23.065 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:40:20.637728+0000", 2023-09-23T18:40:23.065 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 2.418181369, 2023-09-23T18:40:23.065 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.35604997500000002, 2023-09-23T18:40:23.065 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:40:23.065 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:40:23.065 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:40:23.065 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.44954", 2023-09-23T18:40:23.066 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/343998332", 2023-09-23T18:40:23.066 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 158507 2023-09-23T18:40:23.066 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.066 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:40:23.066 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.066 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:40:23.066 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.637728+0000", 2023-09-23T18:40:23.066 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.066 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.066 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.067 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:40:23.067 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.637728+0000", 2023-09-23T18:40:23.067 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.067 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.067 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.067 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:40:23.067 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.637727+0000", 2023-09-23T18:40:23.067 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:40:23.067 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.067 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.067 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:40:23.068 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.637734+0000", 2023-09-23T18:40:23.068 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.1029999999999996e-06 2023-09-23T18:40:23.068 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.068 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.068 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:40:23.068 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.637744+0000", 2023-09-23T18:40:23.068 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0818e-05 2023-09-23T18:40:23.068 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.068 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.068 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:40:23.068 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.637753+0000", 2023-09-23T18:40:23.068 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.1270000000000003e-06 2023-09-23T18:40:23.069 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.069 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.069 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:40:23.069 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.654782+0000", 2023-09-23T18:40:23.069 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.017029473 2023-09-23T18:40:23.069 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.069 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.069 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for rw locks", 2023-09-23T18:40:23.069 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.654892+0000", 2023-09-23T18:40:23.069 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000109611 2023-09-23T18:40:23.069 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.070 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.070 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:40:23.070 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.677856+0000", 2023-09-23T18:40:23.070 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.022964742999999999 2023-09-23T18:40:23.070 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.070 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.070 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:40:23.070 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.678000+0000", 2023-09-23T18:40:23.070 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000143366 2023-09-23T18:40:23.070 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.070 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.071 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:40:23.071 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.678145+0000", 2023-09-23T18:40:23.071 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000145181 2023-09-23T18:40:23.071 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.071 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.071 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:40:23.071 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.984846+0000", 2023-09-23T18:40:23.071 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.30670087400000001 2023-09-23T18:40:23.071 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.071 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.071 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:40:23.072 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.984910+0000", 2023-09-23T18:40:23.072 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.4610999999999995e-05 2023-09-23T18:40:23.072 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.072 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.072 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:40:23.072 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.984949+0000", 2023-09-23T18:40:23.072 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.8120000000000001e-05 2023-09-23T18:40:23.072 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.072 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.072 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:40:23.072 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.985006+0000", 2023-09-23T18:40:23.073 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.7377000000000003e-05 2023-09-23T18:40:23.073 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.073 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.073 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:40:23.073 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.993778+0000", 2023-09-23T18:40:23.073 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0087720520000000007 2023-09-23T18:40:23.073 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.073 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:40:23.073 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.073 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.073 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.074 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.44954.0:158513 6.b 6:d329e638:::benchmark_data_smithi098_131983_object53844:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-09-23T18:40:23.074 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:40:20.645181+0000", 2023-09-23T18:40:23.074 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 2.4107285520000001, 2023-09-23T18:40:23.074 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.328031721, 2023-09-23T18:40:23.074 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:40:23.074 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:40:23.074 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:40:23.074 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.44954", 2023-09-23T18:40:23.074 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/343998332", 2023-09-23T18:40:23.074 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 158513 2023-09-23T18:40:23.074 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.074 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:40:23.075 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.075 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:40:23.075 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.645181+0000", 2023-09-23T18:40:23.075 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.075 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.075 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.075 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:40:23.075 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.645181+0000", 2023-09-23T18:40:23.075 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.075 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.075 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.075 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:40:23.075 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.645180+0000", 2023-09-23T18:40:23.076 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:40:23.076 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.076 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.076 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:40:23.076 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.645185+0000", 2023-09-23T18:40:23.076 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.7060000000000003e-06 2023-09-23T18:40:23.076 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.076 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.076 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:40:23.076 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.645192+0000", 2023-09-23T18:40:23.077 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.4050000000000003e-06 2023-09-23T18:40:23.077 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.077 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.077 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:40:23.077 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.645197+0000", 2023-09-23T18:40:23.077 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.9169999999999997e-06 2023-09-23T18:40:23.077 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.077 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.077 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:40:23.077 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.645233+0000", 2023-09-23T18:40:23.077 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.6316000000000003e-05 2023-09-23T18:40:23.077 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.078 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.078 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:40:23.078 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.645626+0000", 2023-09-23T18:40:23.078 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00039319000000000002 2023-09-23T18:40:23.078 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.078 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.078 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 1,7", 2023-09-23T18:40:23.078 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.645726+0000", 2023-09-23T18:40:23.078 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.9429e-05 2023-09-23T18:40:23.078 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.078 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.078 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:40:23.078 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.652134+0000", 2023-09-23T18:40:23.079 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0064077550000000002 2023-09-23T18:40:23.079 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.079 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.079 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:40:23.079 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.656387+0000", 2023-09-23T18:40:23.079 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0042532619999999998 2023-09-23T18:40:23.079 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.079 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.079 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:40:23.079 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973126+0000", 2023-09-23T18:40:23.079 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.316739035 2023-09-23T18:40:23.079 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.080 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.080 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:40:23.080 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973172+0000", 2023-09-23T18:40:23.080 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.5911000000000001e-05 2023-09-23T18:40:23.080 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.080 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.080 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:40:23.080 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973213+0000", 2023-09-23T18:40:23.080 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.0695000000000003e-05 2023-09-23T18:40:23.080 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.080 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:40:23.080 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.080 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.081 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.081 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.44954.0:158517 6.b 6:d47756f8:::benchmark_data_smithi098_131983_object53848:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-09-23T18:40:23.081 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:40:20.653390+0000", 2023-09-23T18:40:23.081 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 2.4025190279999999, 2023-09-23T18:40:23.081 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.31993955699999999, 2023-09-23T18:40:23.081 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:40:23.081 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:40:23.081 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:40:23.081 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.44954", 2023-09-23T18:40:23.081 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/343998332", 2023-09-23T18:40:23.081 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 158517 2023-09-23T18:40:23.081 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.082 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:40:23.082 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.082 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:40:23.082 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.653390+0000", 2023-09-23T18:40:23.082 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.082 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.082 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.082 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:40:23.082 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.653390+0000", 2023-09-23T18:40:23.082 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.082 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.082 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.083 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:40:23.083 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.653389+0000", 2023-09-23T18:40:23.083 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:40:23.083 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.083 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.083 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:40:23.083 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.653395+0000", 2023-09-23T18:40:23.083 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.8030000000000002e-06 2023-09-23T18:40:23.083 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.083 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.083 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:40:23.084 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.653404+0000", 2023-09-23T18:40:23.084 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.2240000000000001e-06 2023-09-23T18:40:23.084 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.084 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.084 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:40:23.084 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.653411+0000", 2023-09-23T18:40:23.084 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.5889999999999999e-06 2023-09-23T18:40:23.084 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.084 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.084 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:40:23.084 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.656411+0000", 2023-09-23T18:40:23.084 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0030004440000000001 2023-09-23T18:40:23.085 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.085 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.085 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:40:23.085 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.656541+0000", 2023-09-23T18:40:23.085 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00012961 2023-09-23T18:40:23.085 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.085 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.085 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 1,7", 2023-09-23T18:40:23.085 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.656652+0000", 2023-09-23T18:40:23.085 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000110726 2023-09-23T18:40:23.085 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.085 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.085 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:40:23.086 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.660481+0000", 2023-09-23T18:40:23.086 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0038298889999999999 2023-09-23T18:40:23.086 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.086 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.086 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:40:23.086 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.665350+0000", 2023-09-23T18:40:23.086 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0048683779999999996 2023-09-23T18:40:23.086 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.086 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.086 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:40:23.086 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973257+0000", 2023-09-23T18:40:23.086 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.307906717 2023-09-23T18:40:23.087 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.087 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.087 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:40:23.087 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973296+0000", 2023-09-23T18:40:23.087 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.8977000000000003e-05 2023-09-23T18:40:23.087 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.087 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.087 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:40:23.087 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973330+0000", 2023-09-23T18:40:23.087 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.4400999999999998e-05 2023-09-23T18:40:23.087 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.087 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:40:23.087 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.088 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.088 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.088 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.44954.0:158522 6.7 6:eb195cf9:::benchmark_data_smithi098_131983_object53853:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-09-23T18:40:23.088 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:40:20.661994+0000", 2023-09-23T18:40:23.088 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 2.3939154199999999, 2023-09-23T18:40:23.088 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.32383111399999998, 2023-09-23T18:40:23.088 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:40:23.088 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:40:23.088 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:40:23.088 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.44954", 2023-09-23T18:40:23.088 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/343998332", 2023-09-23T18:40:23.088 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 158522 2023-09-23T18:40:23.088 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.089 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:40:23.089 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.089 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:40:23.089 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.661994+0000", 2023-09-23T18:40:23.089 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.089 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.089 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.089 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:40:23.089 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.661994+0000", 2023-09-23T18:40:23.089 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.089 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.089 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.090 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:40:23.090 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.661993+0000", 2023-09-23T18:40:23.090 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:40:23.090 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.090 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.090 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:40:23.090 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.661999+0000", 2023-09-23T18:40:23.090 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.0249999999999999e-06 2023-09-23T18:40:23.090 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.090 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.090 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:40:23.090 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.662009+0000", 2023-09-23T18:40:23.090 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0725e-05 2023-09-23T18:40:23.091 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.091 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.091 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:40:23.091 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.662015+0000", 2023-09-23T18:40:23.091 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.8000000000000004e-06 2023-09-23T18:40:23.091 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.091 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.091 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:40:23.091 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.672604+0000", 2023-09-23T18:40:23.091 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.010588548999999999 2023-09-23T18:40:23.091 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.091 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.091 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:40:23.092 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.672680+0000", 2023-09-23T18:40:23.092 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.6124000000000005e-05 2023-09-23T18:40:23.092 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.092 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.092 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:40:23.092 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.672748+0000", 2023-09-23T18:40:23.092 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.8318000000000006e-05 2023-09-23T18:40:23.092 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.092 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.092 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:40:23.092 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973175+0000", 2023-09-23T18:40:23.092 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.30042727499999999 2023-09-23T18:40:23.093 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.093 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.093 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:40:23.093 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.984860+0000", 2023-09-23T18:40:23.093 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.011684778999999999 2023-09-23T18:40:23.093 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.093 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.093 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:40:23.093 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.985661+0000", 2023-09-23T18:40:23.093 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00080057000000000001 2023-09-23T18:40:23.093 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.093 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.093 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:40:23.094 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.985782+0000", 2023-09-23T18:40:23.094 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000121409 2023-09-23T18:40:23.094 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.094 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.094 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:40:23.094 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.985825+0000", 2023-09-23T18:40:23.094 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.2839000000000001e-05 2023-09-23T18:40:23.094 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.094 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:40:23.094 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.094 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.094 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.095 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.44954.0:158524 6.7 6:eca48beb:::benchmark_data_smithi098_131983_object53855:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-09-23T18:40:23.095 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:40:20.662020+0000", 2023-09-23T18:40:23.095 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 2.3938893220000002, 2023-09-23T18:40:23.095 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.32409803500000001, 2023-09-23T18:40:23.095 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:40:23.095 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:40:23.095 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:40:23.095 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.44954", 2023-09-23T18:40:23.095 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/343998332", 2023-09-23T18:40:23.095 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 158524 2023-09-23T18:40:23.095 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.095 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:40:23.095 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.096 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:40:23.096 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.662020+0000", 2023-09-23T18:40:23.096 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.096 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.096 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.096 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:40:23.096 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.662020+0000", 2023-09-23T18:40:23.096 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.096 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.096 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.096 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:40:23.096 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.662020+0000", 2023-09-23T18:40:23.096 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:40:23.097 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.097 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.097 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:40:23.097 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.662022+0000", 2023-09-23T18:40:23.097 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.807e-06 2023-09-23T18:40:23.097 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.097 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.097 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:40:23.097 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.662025+0000", 2023-09-23T18:40:23.097 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.3210000000000001e-06 2023-09-23T18:40:23.097 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.097 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.098 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:40:23.098 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.662028+0000", 2023-09-23T18:40:23.098 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.7350000000000001e-06 2023-09-23T18:40:23.098 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.098 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.098 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:40:23.098 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.678824+0000", 2023-09-23T18:40:23.098 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.016796518999999999 2023-09-23T18:40:23.098 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.099 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.099 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:40:23.099 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.678934+0000", 2023-09-23T18:40:23.099 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000109696 2023-09-23T18:40:23.099 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.099 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.099 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:40:23.099 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.679053+0000", 2023-09-23T18:40:23.099 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000119282 2023-09-23T18:40:23.099 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.099 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.099 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:40:23.100 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.984941+0000", 2023-09-23T18:40:23.100 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.305887719 2023-09-23T18:40:23.100 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.100 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.100 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:40:23.100 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.985865+0000", 2023-09-23T18:40:23.100 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00092447500000000001 2023-09-23T18:40:23.100 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.100 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.100 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:40:23.100 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.985891+0000", 2023-09-23T18:40:23.101 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.5681999999999999e-05 2023-09-23T18:40:23.101 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.101 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.101 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:40:23.101 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.986003+0000", 2023-09-23T18:40:23.101 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000112038 2023-09-23T18:40:23.101 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.101 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.101 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:40:23.101 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.986118+0000", 2023-09-23T18:40:23.101 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00011507599999999999 2023-09-23T18:40:23.101 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.102 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:40:23.102 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.102 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.102 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.102 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.44954.0:158523 6.9 e164/162 6:9ec36027:::benchmark_data_smithi098_131983_object53854:head v 164'9796, mlcod=164'9793)", 2023-09-23T18:40:23.102 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:40:20.667866+0000", 2023-09-23T18:40:23.102 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 2.3880435360000001, 2023-09-23T18:40:23.102 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.30520835899999998, 2023-09-23T18:40:23.102 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:40:23.102 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:40:23.103 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:40:23.103 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.103 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:40:23.103 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.667866+0000", 2023-09-23T18:40:23.103 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.103 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.103 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.103 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:40:23.103 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.667866+0000", 2023-09-23T18:40:23.103 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.103 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.103 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.104 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:40:23.104 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.667865+0000", 2023-09-23T18:40:23.104 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:40:23.104 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.104 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.104 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:40:23.104 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.667872+0000", 2023-09-23T18:40:23.104 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.4240000000000004e-06 2023-09-23T18:40:23.104 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.104 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.104 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:40:23.104 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.667884+0000", 2023-09-23T18:40:23.104 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1589e-05 2023-09-23T18:40:23.105 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.105 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.105 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:40:23.105 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.667891+0000", 2023-09-23T18:40:23.105 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.2339999999999997e-06 2023-09-23T18:40:23.105 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.105 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.105 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:40:23.105 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.670489+0000", 2023-09-23T18:40:23.105 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0025979340000000001 2023-09-23T18:40:23.105 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.105 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.106 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:40:23.106 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.670512+0000", 2023-09-23T18:40:23.106 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.3326e-05 2023-09-23T18:40:23.106 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.106 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.106 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:40:23.106 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.684192+0000", 2023-09-23T18:40:23.106 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.013679926 2023-09-23T18:40:23.106 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.106 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.106 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:40:23.107 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973074+0000", 2023-09-23T18:40:23.107 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.28888180800000002 2023-09-23T18:40:23.107 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.107 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:40:23.107 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.107 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.107 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.107 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.44954.0:158526 6.f 6:fbe2bd9a:::benchmark_data_smithi098_131983_object53857:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-09-23T18:40:23.107 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:40:20.670037+0000", 2023-09-23T18:40:23.107 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 2.3858725230000002, 2023-09-23T18:40:23.107 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.335207227, 2023-09-23T18:40:23.107 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:40:23.108 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:40:23.108 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:40:23.108 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.44954", 2023-09-23T18:40:23.108 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/343998332", 2023-09-23T18:40:23.108 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 158526 2023-09-23T18:40:23.108 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.108 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:40:23.108 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.108 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:40:23.108 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.670037+0000", 2023-09-23T18:40:23.108 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.108 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.108 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.109 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:40:23.109 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.670037+0000", 2023-09-23T18:40:23.109 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.109 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.109 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.109 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:40:23.109 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.670036+0000", 2023-09-23T18:40:23.109 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:40:23.109 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.109 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.109 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:40:23.109 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.670043+0000", 2023-09-23T18:40:23.109 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.968e-06 2023-09-23T18:40:23.110 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.110 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.110 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:40:23.110 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.670053+0000", 2023-09-23T18:40:23.110 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0621e-05 2023-09-23T18:40:23.110 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.110 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.110 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:40:23.110 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.670059+0000", 2023-09-23T18:40:23.110 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.0630000000000002e-06 2023-09-23T18:40:23.111 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.111 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.111 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:40:23.111 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973235+0000", 2023-09-23T18:40:23.111 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.30317585899999999 2023-09-23T18:40:23.111 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.111 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.111 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:40:23.111 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973558+0000", 2023-09-23T18:40:23.111 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000323114 2023-09-23T18:40:23.111 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.111 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.111 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:40:23.112 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973897+0000", 2023-09-23T18:40:23.112 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00033836700000000001 2023-09-23T18:40:23.112 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.112 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.112 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:40:23.112 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.993792+0000", 2023-09-23T18:40:23.112 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.019895327000000001 2023-09-23T18:40:23.112 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.112 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.112 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:40:23.112 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.993838+0000", 2023-09-23T18:40:23.112 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.6557000000000002e-05 2023-09-23T18:40:23.113 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.113 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.113 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:40:23.113 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.997700+0000", 2023-09-23T18:40:23.113 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0038612690000000001 2023-09-23T18:40:23.113 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.113 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.113 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:40:23.113 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.997935+0000", 2023-09-23T18:40:23.114 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00023546099999999999 2023-09-23T18:40:23.114 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.114 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.114 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:40:23.114 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:21.005244+0000", 2023-09-23T18:40:23.114 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0073089289999999996 2023-09-23T18:40:23.114 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.114 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:40:23.114 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.114 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.114 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.115 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.44954.0:158527 6.9 e164/162 6:9622e290:::benchmark_data_smithi098_131983_object53858:head v 164'9797, mlcod=164'9794)", 2023-09-23T18:40:23.115 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:40:20.671591+0000", 2023-09-23T18:40:23.115 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 2.3843178759999999, 2023-09-23T18:40:23.115 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.30154203299999999, 2023-09-23T18:40:23.115 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:40:23.115 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:40:23.115 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:40:23.115 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.115 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:40:23.115 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.671591+0000", 2023-09-23T18:40:23.115 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.115 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.115 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.116 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:40:23.116 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.671591+0000", 2023-09-23T18:40:23.116 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.116 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.116 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.116 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:40:23.116 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.671591+0000", 2023-09-23T18:40:23.116 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:40:23.116 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.117 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.117 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:40:23.117 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.671597+0000", 2023-09-23T18:40:23.117 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.9540000000000001e-06 2023-09-23T18:40:23.117 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.117 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.117 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:40:23.117 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.671606+0000", 2023-09-23T18:40:23.117 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.414e-06 2023-09-23T18:40:23.117 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.117 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.117 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:40:23.117 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.671613+0000", 2023-09-23T18:40:23.118 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.6170000000000003e-06 2023-09-23T18:40:23.118 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.118 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.118 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:40:23.118 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.679192+0000", 2023-09-23T18:40:23.118 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0075788100000000001 2023-09-23T18:40:23.118 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.118 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.119 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:40:23.119 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.679217+0000", 2023-09-23T18:40:23.119 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.5468000000000001e-05 2023-09-23T18:40:23.119 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.119 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.119 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:40:23.119 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973104+0000", 2023-09-23T18:40:23.119 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.293886869 2023-09-23T18:40:23.120 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.120 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.120 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:40:23.120 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973134+0000", 2023-09-23T18:40:23.120 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.9516e-05 2023-09-23T18:40:23.120 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.120 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:40:23.120 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.120 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.120 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.120 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.44954.0:158529 6.4 e164/149 6:228038f1:::benchmark_data_smithi098_131983_object53860:head v 164'10055, mlcod=164'10052)", 2023-09-23T18:40:23.120 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:40:20.671650+0000", 2023-09-23T18:40:23.121 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 2.384258934, 2023-09-23T18:40:23.121 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.301463694, 2023-09-23T18:40:23.121 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:40:23.121 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:40:23.121 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:40:23.121 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.121 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:40:23.121 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.671650+0000", 2023-09-23T18:40:23.121 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.121 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.121 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.121 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:40:23.121 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.671650+0000", 2023-09-23T18:40:23.122 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.122 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.122 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.122 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:40:23.122 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.671650+0000", 2023-09-23T18:40:23.122 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:40:23.122 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.122 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.122 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:40:23.122 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.671655+0000", 2023-09-23T18:40:23.122 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.6110000000000004e-06 2023-09-23T18:40:23.123 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.123 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.123 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:40:23.123 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.671662+0000", 2023-09-23T18:40:23.123 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.6569999999999997e-06 2023-09-23T18:40:23.123 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.123 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.123 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:40:23.123 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.671668+0000", 2023-09-23T18:40:23.123 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.3879999999999999e-06 2023-09-23T18:40:23.123 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.123 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.123 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:40:23.124 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.680763+0000", 2023-09-23T18:40:23.124 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0090955380000000002 2023-09-23T18:40:23.124 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.124 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.124 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:40:23.124 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.680782+0000", 2023-09-23T18:40:23.124 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.9199e-05 2023-09-23T18:40:23.124 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.124 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.124 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:40:23.124 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973079+0000", 2023-09-23T18:40:23.124 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.292297113 2023-09-23T18:40:23.125 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.125 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.125 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:40:23.125 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973114+0000", 2023-09-23T18:40:23.125 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.4660999999999999e-05 2023-09-23T18:40:23.125 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.125 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:40:23.125 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.125 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.125 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.125 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.44954.0:158531 6.b 6:def419b4:::benchmark_data_smithi098_131983_object53862:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-09-23T18:40:23.125 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:40:20.674778+0000", 2023-09-23T18:40:23.125 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 2.381131109, 2023-09-23T18:40:23.126 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.29871164300000003, 2023-09-23T18:40:23.126 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:40:23.126 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:40:23.126 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:40:23.126 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.44954", 2023-09-23T18:40:23.126 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/343998332", 2023-09-23T18:40:23.126 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 158531 2023-09-23T18:40:23.126 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.126 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:40:23.126 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.126 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:40:23.126 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.674778+0000", 2023-09-23T18:40:23.127 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.127 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.127 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.127 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:40:23.127 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.674778+0000", 2023-09-23T18:40:23.127 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.127 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.127 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.127 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:40:23.127 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.674777+0000", 2023-09-23T18:40:23.127 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:40:23.127 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.127 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.128 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:40:23.128 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.674784+0000", 2023-09-23T18:40:23.128 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.1960000000000002e-06 2023-09-23T18:40:23.128 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.128 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.128 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:40:23.128 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.674798+0000", 2023-09-23T18:40:23.128 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3762e-05 2023-09-23T18:40:23.128 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.128 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.128 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:40:23.128 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.674806+0000", 2023-09-23T18:40:23.128 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.2600000000000005e-06 2023-09-23T18:40:23.129 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.129 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.129 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:40:23.129 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.674861+0000", 2023-09-23T18:40:23.129 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.4404000000000003e-05 2023-09-23T18:40:23.129 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.129 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.129 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:40:23.129 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.674979+0000", 2023-09-23T18:40:23.129 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00011845000000000001 2023-09-23T18:40:23.129 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.129 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.129 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 1,7", 2023-09-23T18:40:23.130 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.675094+0000", 2023-09-23T18:40:23.130 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000114849 2023-09-23T18:40:23.130 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.130 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.130 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:40:23.130 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.679168+0000", 2023-09-23T18:40:23.130 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0040741839999999998 2023-09-23T18:40:23.130 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.130 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.130 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:40:23.130 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973368+0000", 2023-09-23T18:40:23.130 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.294199985 2023-09-23T18:40:23.131 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.131 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.131 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:40:23.131 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973406+0000", 2023-09-23T18:40:23.131 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.7849000000000003e-05 2023-09-23T18:40:23.131 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.131 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.131 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:40:23.131 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973457+0000", 2023-09-23T18:40:23.131 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.0945000000000001e-05 2023-09-23T18:40:23.131 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.131 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.132 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:40:23.132 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973490+0000", 2023-09-23T18:40:23.132 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.3118999999999999e-05 2023-09-23T18:40:23.132 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.132 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:40:23.132 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.132 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.132 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.132 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.44954.0:158533 6.b 6:df307953:::benchmark_data_smithi098_131983_object53864:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-09-23T18:40:23.132 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:40:20.679831+0000", 2023-09-23T18:40:23.132 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 2.3760780029999999, 2023-09-23T18:40:23.132 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.29836944700000001, 2023-09-23T18:40:23.132 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:40:23.133 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:40:23.133 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:40:23.133 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.44954", 2023-09-23T18:40:23.133 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/343998332", 2023-09-23T18:40:23.133 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 158533 2023-09-23T18:40:23.133 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.133 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:40:23.133 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.133 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:40:23.133 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.679831+0000", 2023-09-23T18:40:23.134 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.134 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.134 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.134 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:40:23.134 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.679831+0000", 2023-09-23T18:40:23.134 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.134 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.134 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.134 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:40:23.134 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.679830+0000", 2023-09-23T18:40:23.134 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T18:40:23.135 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.135 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.135 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:40:23.135 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.679837+0000", 2023-09-23T18:40:23.135 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.7470000000000005e-06 2023-09-23T18:40:23.135 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.135 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.135 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:40:23.135 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.679850+0000", 2023-09-23T18:40:23.135 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.2074e-05 2023-09-23T18:40:23.135 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.135 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.135 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:40:23.136 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.679858+0000", 2023-09-23T18:40:23.136 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.1370000000000002e-06 2023-09-23T18:40:23.136 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.136 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.136 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:40:23.136 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.679902+0000", 2023-09-23T18:40:23.136 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.4128999999999997e-05 2023-09-23T18:40:23.136 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.136 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.136 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:40:23.136 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.680007+0000", 2023-09-23T18:40:23.136 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00010488399999999999 2023-09-23T18:40:23.136 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.137 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.137 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 1,7", 2023-09-23T18:40:23.137 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.680107+0000", 2023-09-23T18:40:23.137 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000100622 2023-09-23T18:40:23.137 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.137 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.137 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:40:23.137 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973525+0000", 2023-09-23T18:40:23.137 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.29341784799999998 2023-09-23T18:40:23.137 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.137 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.137 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:40:23.138 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973560+0000", 2023-09-23T18:40:23.138 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.4375999999999997e-05 2023-09-23T18:40:23.138 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.138 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.138 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:40:23.138 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.978041+0000", 2023-09-23T18:40:23.138 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0044817579999999997 2023-09-23T18:40:23.138 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.138 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.138 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:40:23.138 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.978157+0000", 2023-09-23T18:40:23.138 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000116129 2023-09-23T18:40:23.138 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.139 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.141 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:40:23.141 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.978201+0000", 2023-09-23T18:40:23.141 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.3429e-05 2023-09-23T18:40:23.141 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.141 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:40:23.141 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.141 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.141 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.141 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.44954.0:158534 6.5 e164/149 6:a5b77f08:::benchmark_data_smithi098_131983_object53865:head v 164'9859, mlcod=164'9857)", 2023-09-23T18:40:23.141 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:40:20.680445+0000", 2023-09-23T18:40:23.142 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 2.375464794, 2023-09-23T18:40:23.142 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.292692126, 2023-09-23T18:40:23.142 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:40:23.142 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:40:23.142 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:40:23.142 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.142 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:40:23.142 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.680445+0000", 2023-09-23T18:40:23.142 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.142 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.142 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.143 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:40:23.143 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.680445+0000", 2023-09-23T18:40:23.143 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.143 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.143 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.143 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:40:23.143 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.680444+0000", 2023-09-23T18:40:23.143 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:40:23.143 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.143 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.143 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:40:23.143 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.680450+0000", 2023-09-23T18:40:23.143 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.6579999999999999e-06 2023-09-23T18:40:23.144 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.144 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.144 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:40:23.144 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.680458+0000", 2023-09-23T18:40:23.144 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.439e-06 2023-09-23T18:40:23.144 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.144 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.144 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:40:23.144 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.680464+0000", 2023-09-23T18:40:23.144 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.1120000000000001e-06 2023-09-23T18:40:23.144 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.144 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.144 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:40:23.145 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.680735+0000", 2023-09-23T18:40:23.145 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00027035699999999999 2023-09-23T18:40:23.145 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.145 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.145 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:40:23.145 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.680756+0000", 2023-09-23T18:40:23.145 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.103e-05 2023-09-23T18:40:23.145 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.145 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.145 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:40:23.145 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973092+0000", 2023-09-23T18:40:23.145 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.29233672399999999 2023-09-23T18:40:23.146 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.146 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.146 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:40:23.146 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973137+0000", 2023-09-23T18:40:23.146 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.4438999999999999e-05 2023-09-23T18:40:23.146 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.146 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:40:23.146 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.146 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.146 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.146 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.44954.0:158535 6.6 6:6596f821:::benchmark_data_smithi098_131983_object53866:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-09-23T18:40:23.146 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:40:20.683614+0000", 2023-09-23T18:40:23.147 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 2.3722950389999999, 2023-09-23T18:40:23.147 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.29035354000000002, 2023-09-23T18:40:23.147 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:40:23.147 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:40:23.147 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:40:23.147 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.44954", 2023-09-23T18:40:23.147 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/343998332", 2023-09-23T18:40:23.147 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 158535 2023-09-23T18:40:23.147 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.147 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:40:23.147 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.148 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:40:23.148 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.683614+0000", 2023-09-23T18:40:23.148 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.148 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.148 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.148 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:40:23.148 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.683614+0000", 2023-09-23T18:40:23.148 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.148 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.148 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.149 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:40:23.149 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.683613+0000", 2023-09-23T18:40:23.149 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:40:23.149 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.149 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.149 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:40:23.149 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.683622+0000", 2023-09-23T18:40:23.149 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.0839999999999998e-06 2023-09-23T18:40:23.149 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.149 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.149 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:40:23.149 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.683634+0000", 2023-09-23T18:40:23.150 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.252e-05 2023-09-23T18:40:23.150 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.150 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.150 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:40:23.150 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.683644+0000", 2023-09-23T18:40:23.150 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.4409999999999999e-06 2023-09-23T18:40:23.150 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.150 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.150 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:40:23.150 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.683690+0000", 2023-09-23T18:40:23.150 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.6019000000000001e-05 2023-09-23T18:40:23.150 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.151 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.151 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:40:23.151 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.683817+0000", 2023-09-23T18:40:23.151 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000126682 2023-09-23T18:40:23.151 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.151 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.151 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:40:23.151 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.683935+0000", 2023-09-23T18:40:23.151 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000118405 2023-09-23T18:40:23.151 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.151 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.151 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:40:23.152 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973360+0000", 2023-09-23T18:40:23.152 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.28942518499999997 2023-09-23T18:40:23.152 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.152 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.152 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:40:23.152 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973422+0000", 2023-09-23T18:40:23.152 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.2345e-05 2023-09-23T18:40:23.152 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.152 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.152 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:40:23.152 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973604+0000", 2023-09-23T18:40:23.152 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00018121700000000001 2023-09-23T18:40:23.152 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.153 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.153 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:40:23.153 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973810+0000", 2023-09-23T18:40:23.153 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00020610499999999999 2023-09-23T18:40:23.153 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.153 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.153 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:40:23.153 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973968+0000", 2023-09-23T18:40:23.153 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000158078 2023-09-23T18:40:23.153 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.153 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:40:23.153 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.153 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.154 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.154 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop_reply(client.44954.0:158522 6.7 e164/149 ondisk, result = 0)", 2023-09-23T18:40:23.154 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:40:20.683813+0000", 2023-09-23T18:40:23.154 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 2.3720966099999998, 2023-09-23T18:40:23.154 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.28936817999999997, 2023-09-23T18:40:23.154 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:40:23.154 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T18:40:23.154 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:40:23.154 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.155 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:40:23.155 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.683813+0000", 2023-09-23T18:40:23.155 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.155 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.155 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.155 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:40:23.155 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.683813+0000", 2023-09-23T18:40:23.155 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.155 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.155 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.155 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:40:23.155 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.683812+0000", 2023-09-23T18:40:23.155 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:40:23.155 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.156 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.156 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:40:23.156 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.683817+0000", 2023-09-23T18:40:23.156 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.6059999999999996e-06 2023-09-23T18:40:23.156 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.156 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.156 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:40:23.156 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.683827+0000", 2023-09-23T18:40:23.156 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0084999999999999e-05 2023-09-23T18:40:23.156 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.156 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.156 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:40:23.157 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.683835+0000", 2023-09-23T18:40:23.157 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.5539999999999998e-06 2023-09-23T18:40:23.157 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.157 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.157 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:40:23.157 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973147+0000", 2023-09-23T18:40:23.157 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.289312018 2023-09-23T18:40:23.157 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.157 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.157 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:40:23.157 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973163+0000", 2023-09-23T18:40:23.157 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.5571999999999999e-05 2023-09-23T18:40:23.157 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.158 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.158 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:40:23.158 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973181+0000", 2023-09-23T18:40:23.158 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.8068e-05 2023-09-23T18:40:23.158 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.158 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:40:23.158 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.158 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.158 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.158 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop_reply(client.44954.0:158507 6.f e164/162 ondisk, result = 0)", 2023-09-23T18:40:23.158 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:40:20.683842+0000", 2023-09-23T18:40:23.158 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 2.3720678130000001, 2023-09-23T18:40:23.158 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.30995688799999999, 2023-09-23T18:40:23.159 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:40:23.159 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T18:40:23.159 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:40:23.159 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.159 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:40:23.159 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.683842+0000", 2023-09-23T18:40:23.159 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.159 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.159 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.159 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:40:23.159 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.683842+0000", 2023-09-23T18:40:23.159 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.159 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.160 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.160 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:40:23.160 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.683841+0000", 2023-09-23T18:40:23.160 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:40:23.160 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.160 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.160 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:40:23.160 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.683844+0000", 2023-09-23T18:40:23.160 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.5000000000000002e-06 2023-09-23T18:40:23.160 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.160 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.160 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:40:23.160 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.683849+0000", 2023-09-23T18:40:23.161 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.4820000000000002e-06 2023-09-23T18:40:23.161 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.161 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.161 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:40:23.161 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.683854+0000", 2023-09-23T18:40:23.161 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.5399999999999997e-06 2023-09-23T18:40:23.161 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.161 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.161 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:40:23.161 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.984893+0000", 2023-09-23T18:40:23.161 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.30103907000000002 2023-09-23T18:40:23.161 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.161 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.162 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:40:23.162 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.984904+0000", 2023-09-23T18:40:23.162 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0795000000000001e-05 2023-09-23T18:40:23.162 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.162 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.162 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:40:23.162 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.993798+0000", 2023-09-23T18:40:23.162 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0088947910000000008 2023-09-23T18:40:23.162 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.162 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:40:23.162 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.162 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.163 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.163 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop_reply(client.44954.0:158513 6.b e164/149 ondisk, result = 0)", 2023-09-23T18:40:23.163 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:40:20.684088+0000", 2023-09-23T18:40:23.163 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 2.3718216910000001, 2023-09-23T18:40:23.163 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.289134222, 2023-09-23T18:40:23.163 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:40:23.163 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T18:40:23.163 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:40:23.163 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.163 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:40:23.163 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.684088+0000", 2023-09-23T18:40:23.163 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.163 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.163 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.164 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:40:23.164 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.684088+0000", 2023-09-23T18:40:23.164 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.164 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.164 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.164 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:40:23.164 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.684087+0000", 2023-09-23T18:40:23.164 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:40:23.164 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.164 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.164 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:40:23.164 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.684091+0000", 2023-09-23T18:40:23.164 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.7890000000000001e-06 2023-09-23T18:40:23.165 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.165 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.165 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:40:23.165 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.684099+0000", 2023-09-23T18:40:23.165 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.2910000000000007e-06 2023-09-23T18:40:23.165 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.165 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.165 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:40:23.165 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.684107+0000", 2023-09-23T18:40:23.165 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.4000000000000003e-06 2023-09-23T18:40:23.165 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.165 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.166 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:40:23.166 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973110+0000", 2023-09-23T18:40:23.166 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.28900385499999998 2023-09-23T18:40:23.166 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.166 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.166 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:40:23.166 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973119+0000", 2023-09-23T18:40:23.166 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.6060000000000003e-06 2023-09-23T18:40:23.166 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.166 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.166 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:40:23.166 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973222+0000", 2023-09-23T18:40:23.166 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00010283199999999999 2023-09-23T18:40:23.167 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.167 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:40:23.167 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.167 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.167 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.167 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop_reply(client.44954.0:158517 6.b e164/149 ondisk, result = 0)", 2023-09-23T18:40:23.167 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:40:20.684129+0000", 2023-09-23T18:40:23.167 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 2.3717805059999999, 2023-09-23T18:40:23.167 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.28920520700000002, 2023-09-23T18:40:23.167 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:40:23.167 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T18:40:23.167 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:40:23.167 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.168 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:40:23.168 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.684129+0000", 2023-09-23T18:40:23.168 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.168 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.168 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.168 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:40:23.168 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.684129+0000", 2023-09-23T18:40:23.168 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.168 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.168 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.168 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:40:23.168 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.684128+0000", 2023-09-23T18:40:23.168 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:40:23.169 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.169 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.169 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:40:23.169 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.684131+0000", 2023-09-23T18:40:23.169 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.661e-06 2023-09-23T18:40:23.169 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.169 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.169 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:40:23.169 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.684138+0000", 2023-09-23T18:40:23.169 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.579e-06 2023-09-23T18:40:23.169 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.169 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.169 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:40:23.170 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.684143+0000", 2023-09-23T18:40:23.170 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.9490000000000004e-06 2023-09-23T18:40:23.170 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.170 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.170 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:40:23.170 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973244+0000", 2023-09-23T18:40:23.170 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.28910117899999999 2023-09-23T18:40:23.170 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.170 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.170 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:40:23.170 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973251+0000", 2023-09-23T18:40:23.170 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.7440000000000001e-06 2023-09-23T18:40:23.171 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.171 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.171 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:40:23.171 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973334+0000", 2023-09-23T18:40:23.171 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.3500999999999999e-05 2023-09-23T18:40:23.171 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.171 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:40:23.171 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.171 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.171 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.171 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop_reply(client.44954.0:158531 6.b e164/149 ondisk, result = 0)", 2023-09-23T18:40:23.171 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:40:20.684147+0000", 2023-09-23T18:40:23.171 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 2.3717622010000001, 2023-09-23T18:40:23.172 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.289228655, 2023-09-23T18:40:23.172 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:40:23.172 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T18:40:23.172 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:40:23.172 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.172 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:40:23.172 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.684147+0000", 2023-09-23T18:40:23.172 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.172 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.172 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.172 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:40:23.172 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.684147+0000", 2023-09-23T18:40:23.172 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.173 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.173 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.173 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:40:23.173 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.684147+0000", 2023-09-23T18:40:23.173 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:40:23.173 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.173 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.173 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:40:23.173 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.684149+0000", 2023-09-23T18:40:23.173 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.481e-06 2023-09-23T18:40:23.173 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.173 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.173 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:40:23.173 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.684154+0000", 2023-09-23T18:40:23.174 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.0660000000000003e-06 2023-09-23T18:40:23.174 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.174 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.174 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:40:23.174 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.684158+0000", 2023-09-23T18:40:23.174 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.732e-06 2023-09-23T18:40:23.174 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.174 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.174 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:40:23.174 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973356+0000", 2023-09-23T18:40:23.174 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.28919730300000002 2023-09-23T18:40:23.174 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.175 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.175 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:40:23.175 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973362+0000", 2023-09-23T18:40:23.175 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.5649999999999998e-06 2023-09-23T18:40:23.175 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.175 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.175 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:40:23.175 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973376+0000", 2023-09-23T18:40:23.175 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.376e-05 2023-09-23T18:40:23.175 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.175 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:40:23.175 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.175 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.176 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.176 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.44954.0:158536 6.9 e164/162 6:917d5dd4:::benchmark_data_smithi098_131983_object53867:head v 164'9798, mlcod=164'9794)", 2023-09-23T18:40:23.176 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:40:20.684768+0000", 2023-09-23T18:40:23.176 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 2.371141041, 2023-09-23T18:40:23.176 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.29686277, 2023-09-23T18:40:23.176 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:40:23.176 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:40:23.176 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:40:23.176 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.176 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:40:23.176 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.684768+0000", 2023-09-23T18:40:23.176 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.177 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.177 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.177 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:40:23.177 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.684768+0000", 2023-09-23T18:40:23.177 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.177 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.177 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.177 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:40:23.177 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.684768+0000", 2023-09-23T18:40:23.177 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:40:23.177 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.177 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.177 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:40:23.178 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.684772+0000", 2023-09-23T18:40:23.178 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.7890000000000002e-06 2023-09-23T18:40:23.178 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.178 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.178 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:40:23.178 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973095+0000", 2023-09-23T18:40:23.178 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.28832222099999999 2023-09-23T18:40:23.178 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.178 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.178 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:40:23.178 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973102+0000", 2023-09-23T18:40:23.178 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.3520000000000001e-06 2023-09-23T18:40:23.179 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.179 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.179 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:40:23.179 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973214+0000", 2023-09-23T18:40:23.179 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00011192 2023-09-23T18:40:23.179 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.179 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.179 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:40:23.179 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973248+0000", 2023-09-23T18:40:23.179 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.4227999999999998e-05 2023-09-23T18:40:23.179 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.179 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.179 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:40:23.180 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.981339+0000", 2023-09-23T18:40:23.180 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0080905339999999999 2023-09-23T18:40:23.180 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.180 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.180 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:40:23.180 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.981631+0000", 2023-09-23T18:40:23.180 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00029236600000000001 2023-09-23T18:40:23.180 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.180 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:40:23.180 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.180 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.180 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.180 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop_reply(client.44954.0:158507 6.f e164/162 ondisk, result = 0)", 2023-09-23T18:40:23.180 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:40:20.687625+0000", 2023-09-23T18:40:23.181 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 2.3682847119999999, 2023-09-23T18:40:23.181 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.29723713899999998, 2023-09-23T18:40:23.181 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:40:23.181 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T18:40:23.181 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:40:23.181 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.181 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:40:23.181 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.687625+0000", 2023-09-23T18:40:23.181 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.181 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.181 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.181 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:40:23.181 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.687625+0000", 2023-09-23T18:40:23.182 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:40:23.182 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.182 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.182 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:40:23.182 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.687624+0000", 2023-09-23T18:40:23.182 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:40:23.182 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.182 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.182 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:40:23.182 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.687630+0000", 2023-09-23T18:40:23.182 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.2199999999999997e-06 2023-09-23T18:40:23.182 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.182 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.183 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:40:23.183 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973048+0000", 2023-09-23T18:40:23.183 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.28541823599999999 2023-09-23T18:40:23.183 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.183 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.183 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:40:23.183 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.973057+0000", 2023-09-23T18:40:23.183 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.8559999999999996e-06 2023-09-23T18:40:23.183 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.183 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.183 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:40:23.183 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.984821+0000", 2023-09-23T18:40:23.183 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.011764037 2023-09-23T18:40:23.184 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.184 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.184 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:40:23.184 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.984834+0000", 2023-09-23T18:40:23.184 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3063e-05 2023-09-23T18:40:23.184 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:40:23.184 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:40:23.184 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:40:23.184 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:40:20.984862+0000", 2023-09-23T18:40:23.184 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.7535999999999999e-05 2023-09-23T18:40:23.184 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.184 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:40:23.185 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.185 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:40:23.185 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:40:23.185 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:40:23.185 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:23.219 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:23.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:23.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:23.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:23 smithi173 bash[59422]: cluster 2023-09-23T18:40:21.891375+0000 mgr.y (mgr.26054) 660 : cluster 0 pgmap v644: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 632/155730 objects degraded (0.406%); 16789/155730 objects misplaced (10.781%); 15 MiB/s, 240 objects/s recovering 2023-09-23T18:40:23.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:23 smithi173 bash[59422]: cluster 2023-09-23T18:40:21.991605+0000 osd.7 (osd.7) 77 : cluster 3 failed to encode map e165 with expected crc 2023-09-23T18:40:23.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:23 smithi173 bash[59422]: cluster 2023-09-23T18:40:21.992237+0000 mon.a (mon.0) 907 : cluster 0 osdmap e165: 8 total, 8 up, 7 in 2023-09-23T18:40:23.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:23 smithi173 bash[59422]: cluster 2023-09-23T18:40:21.997459+0000 osd.6 (osd.6) 79 : cluster 3 failed to encode map e165 with expected crc 2023-09-23T18:40:23.327 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:23.337 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:22 smithi098 bash[76817]: cluster 2023-09-23T18:40:21.891375+0000 mgr.y (mgr.26054) 660 : cluster 0 pgmap v644: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 632/155730 objects degraded (0.406%); 16789/155730 objects misplaced (10.781%); 15 MiB/s, 240 objects/s recovering 2023-09-23T18:40:23.337 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:23 smithi098 bash[76817]: cluster 2023-09-23T18:40:21.991605+0000 osd.7 (osd.7) 77 : cluster 3 failed to encode map e165 with expected crc 2023-09-23T18:40:23.337 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:23 smithi098 bash[76817]: cluster 2023-09-23T18:40:21.992237+0000 mon.a (mon.0) 907 : cluster 0 osdmap e165: 8 total, 8 up, 7 in 2023-09-23T18:40:23.337 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:23 smithi098 bash[76817]: cluster 2023-09-23T18:40:21.997459+0000 osd.6 (osd.6) 79 : cluster 3 failed to encode map e165 with expected crc 2023-09-23T18:40:23.337 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:23 smithi098 bash[79516]: cluster 2023-09-23T18:40:21.891375+0000 mgr.y (mgr.26054) 660 : cluster 0 pgmap v644: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 632/155730 objects degraded (0.406%); 16789/155730 objects misplaced (10.781%); 15 MiB/s, 240 objects/s recovering 2023-09-23T18:40:23.337 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:23 smithi098 bash[79516]: cluster 2023-09-23T18:40:21.991605+0000 osd.7 (osd.7) 77 : cluster 3 failed to encode map e165 with expected crc 2023-09-23T18:40:23.337 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:23 smithi098 bash[79516]: cluster 2023-09-23T18:40:21.992237+0000 mon.a (mon.0) 907 : cluster 0 osdmap e165: 8 total, 8 up, 7 in 2023-09-23T18:40:23.338 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:23 smithi098 bash[79516]: cluster 2023-09-23T18:40:21.997459+0000 osd.6 (osd.6) 79 : cluster 3 failed to encode map e165 with expected crc 2023-09-23T18:40:23.363 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:23.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:23.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:23.472 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:23.498 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:23.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:23.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:23.602 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:23.630 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:23.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:23.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:23.733 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:23.760 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:23.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:23.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:23.864 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:23.888 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:23.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:23.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:23.991 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:24.022 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:24.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:24.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:24.130 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:24.163 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:24.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:24.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:24.269 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:24.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:24 smithi173 bash[59422]: cluster 2023-09-23T18:40:22.995664+0000 osd.4 (osd.4) 108 : cluster 3 failed to encode map e166 with expected crc 2023-09-23T18:40:24.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:24 smithi173 bash[59422]: cluster 2023-09-23T18:40:22.996886+0000 mon.a (mon.0) 908 : cluster 0 osdmap e166: 8 total, 8 up, 7 in 2023-09-23T18:40:24.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:24 smithi173 bash[59422]: cluster 2023-09-23T18:40:22.997154+0000 osd.6 (osd.6) 80 : cluster 3 failed to encode map e166 with expected crc 2023-09-23T18:40:24.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:24 smithi173 bash[59422]: cluster 2023-09-23T18:40:22.997246+0000 osd.7 (osd.7) 78 : cluster 3 failed to encode map e166 with expected crc 2023-09-23T18:40:24.299 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:24.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:24.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:24.341 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:40:24.349 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:24 smithi098 bash[76817]: cluster 2023-09-23T18:40:22.995664+0000 osd.4 (osd.4) 108 : cluster 3 failed to encode map e166 with expected crc 2023-09-23T18:40:24.349 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:24 smithi098 bash[76817]: cluster 2023-09-23T18:40:22.996886+0000 mon.a (mon.0) 908 : cluster 0 osdmap e166: 8 total, 8 up, 7 in 2023-09-23T18:40:24.349 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:24 smithi098 bash[76817]: cluster 2023-09-23T18:40:22.997154+0000 osd.6 (osd.6) 80 : cluster 3 failed to encode map e166 with expected crc 2023-09-23T18:40:24.349 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:24 smithi098 bash[76817]: cluster 2023-09-23T18:40:22.997246+0000 osd.7 (osd.7) 78 : cluster 3 failed to encode map e166 with expected crc 2023-09-23T18:40:24.349 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:24 smithi098 bash[79516]: cluster 2023-09-23T18:40:22.995664+0000 osd.4 (osd.4) 108 : cluster 3 failed to encode map e166 with expected crc 2023-09-23T18:40:24.350 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:24 smithi098 bash[79516]: cluster 2023-09-23T18:40:22.996886+0000 mon.a (mon.0) 908 : cluster 0 osdmap e166: 8 total, 8 up, 7 in 2023-09-23T18:40:24.350 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:24 smithi098 bash[79516]: cluster 2023-09-23T18:40:22.997154+0000 osd.6 (osd.6) 80 : cluster 3 failed to encode map e166 with expected crc 2023-09-23T18:40:24.350 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:24 smithi098 bash[79516]: cluster 2023-09-23T18:40:22.997246+0000 osd.7 (osd.7) 78 : cluster 3 failed to encode map e166 with expected crc 2023-09-23T18:40:24.404 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:24.446 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:24.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:24.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:24.554 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:24.582 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:24.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:24.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:24.687 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:24.725 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:24.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:24.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:24.832 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:24.856 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:24.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:24.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:24.960 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:24.984 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:24.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:24.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:25.087 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:25.118 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:25.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:25.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:25.215 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T18:40:25.222 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:25.225 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:25 smithi173 bash[59422]: cluster 2023-09-23T18:40:23.893105+0000 mgr.y (mgr.26054) 661 : cluster 0 pgmap v647: 89 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 684/151605 objects degraded (0.451%); 15992/151605 objects misplaced (10.548%); 13 MiB/s, 212 objects/s recovering 2023-09-23T18:40:25.226 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:25 smithi173 bash[59422]: audit 2023-09-23T18:40:24.607261+0000 mon.a (mon.0) 909 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:40:25.226 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:25 smithi173 bash[59422]: audit 2023-09-23T18:40:24.610400+0000 mon.a (mon.0) 910 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:40:25.226 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:25 smithi173 bash[59422]: cluster 2023-09-23T18:40:24.974215+0000 mon.a (mon.0) 911 : cluster 3 Health check update: Degraded data redundancy: 684/151605 objects degraded (0.451%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:25.264 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:25.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:25.268 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:25 smithi098 bash[76817]: cluster 2023-09-23T18:40:23.893105+0000 mgr.y (mgr.26054) 661 : cluster 0 pgmap v647: 89 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 684/151605 objects degraded (0.451%); 15992/151605 objects misplaced (10.548%); 13 MiB/s, 212 objects/s recovering 2023-09-23T18:40:25.268 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:25 smithi098 bash[76817]: audit 2023-09-23T18:40:24.607261+0000 mon.a (mon.0) 909 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:40:25.268 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:25 smithi098 bash[76817]: audit 2023-09-23T18:40:24.610400+0000 mon.a (mon.0) 910 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:40:25.268 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:25 smithi098 bash[76817]: cluster 2023-09-23T18:40:24.974215+0000 mon.a (mon.0) 911 : cluster 3 Health check update: Degraded data redundancy: 684/151605 objects degraded (0.451%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:25.268 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:25 smithi098 bash[79516]: cluster 2023-09-23T18:40:23.893105+0000 mgr.y (mgr.26054) 661 : cluster 0 pgmap v647: 89 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 684/151605 objects degraded (0.451%); 15992/151605 objects misplaced (10.548%); 13 MiB/s, 212 objects/s recovering 2023-09-23T18:40:25.268 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:25 smithi098 bash[79516]: audit 2023-09-23T18:40:24.607261+0000 mon.a (mon.0) 909 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:40:25.269 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:25 smithi098 bash[79516]: audit 2023-09-23T18:40:24.610400+0000 mon.a (mon.0) 910 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:40:25.269 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:25 smithi098 bash[79516]: cluster 2023-09-23T18:40:24.974215+0000 mon.a (mon.0) 911 : cluster 3 Health check update: Degraded data redundancy: 684/151605 objects degraded (0.451%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:25.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:25.370 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:25.409 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:25.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:25.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:25.523 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:25.558 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:25.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:25.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:25.664 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:25.692 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:25.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:25.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:25.795 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:25.820 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:25.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:25.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:25.927 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:25.957 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:25.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:25.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:26.060 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:26.088 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:26.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:26.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:26.191 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:40:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:40:26] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.33.4" 2023-09-23T18:40:26.192 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:26.220 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:26.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:26.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:26.324 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:26.359 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:26.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:26.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:26.464 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:26.490 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:26.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:26.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:26.593 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:26.621 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:26.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:26.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:26.705 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:40:26.724 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:26.752 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:26.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:26.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:26.855 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:26.882 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:26.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:26.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:26.985 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:27.012 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:27.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:27.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:27.115 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:27.147 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:27.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:27.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:27.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:27 smithi098 bash[79516]: cluster 2023-09-23T18:40:25.893702+0000 mgr.y (mgr.26054) 662 : cluster 0 pgmap v648: 89 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 684/148662 objects degraded (0.460%); 14909/148662 objects misplaced (10.029%); 19 MiB/s, 298 objects/s recovering 2023-09-23T18:40:27.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:27 smithi098 bash[79516]: cluster 2023-09-23T18:40:26.029718+0000 osd.6 (osd.6) 81 : cluster 3 failed to encode map e167 with expected crc 2023-09-23T18:40:27.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:27 smithi098 bash[79516]: cluster 2023-09-23T18:40:26.030570+0000 osd.5 (osd.5) 70 : cluster 3 failed to encode map e167 with expected crc 2023-09-23T18:40:27.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:27 smithi098 bash[79516]: cluster 2023-09-23T18:40:26.033227+0000 mon.a (mon.0) 912 : cluster 0 osdmap e167: 8 total, 8 up, 7 in 2023-09-23T18:40:27.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:27 smithi098 bash[79516]: cluster 2023-09-23T18:40:26.039333+0000 osd.7 (osd.7) 79 : cluster 3 failed to encode map e167 with expected crc 2023-09-23T18:40:27.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:27 smithi098 bash[76817]: cluster 2023-09-23T18:40:25.893702+0000 mgr.y (mgr.26054) 662 : cluster 0 pgmap v648: 89 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 684/148662 objects degraded (0.460%); 14909/148662 objects misplaced (10.029%); 19 MiB/s, 298 objects/s recovering 2023-09-23T18:40:27.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:27 smithi098 bash[76817]: cluster 2023-09-23T18:40:26.029718+0000 osd.6 (osd.6) 81 : cluster 3 failed to encode map e167 with expected crc 2023-09-23T18:40:27.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:27 smithi098 bash[76817]: cluster 2023-09-23T18:40:26.030570+0000 osd.5 (osd.5) 70 : cluster 3 failed to encode map e167 with expected crc 2023-09-23T18:40:27.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:27 smithi098 bash[76817]: cluster 2023-09-23T18:40:26.033227+0000 mon.a (mon.0) 912 : cluster 0 osdmap e167: 8 total, 8 up, 7 in 2023-09-23T18:40:27.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:27 smithi098 bash[76817]: cluster 2023-09-23T18:40:26.039333+0000 osd.7 (osd.7) 79 : cluster 3 failed to encode map e167 with expected crc 2023-09-23T18:40:27.251 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:27.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:27 smithi173 bash[59422]: cluster 2023-09-23T18:40:25.893702+0000 mgr.y (mgr.26054) 662 : cluster 0 pgmap v648: 89 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 684/148662 objects degraded (0.460%); 14909/148662 objects misplaced (10.029%); 19 MiB/s, 298 objects/s recovering 2023-09-23T18:40:27.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:27 smithi173 bash[59422]: cluster 2023-09-23T18:40:26.029718+0000 osd.6 (osd.6) 81 : cluster 3 failed to encode map e167 with expected crc 2023-09-23T18:40:27.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:27 smithi173 bash[59422]: cluster 2023-09-23T18:40:26.030570+0000 osd.5 (osd.5) 70 : cluster 3 failed to encode map e167 with expected crc 2023-09-23T18:40:27.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:27 smithi173 bash[59422]: cluster 2023-09-23T18:40:26.033227+0000 mon.a (mon.0) 912 : cluster 0 osdmap e167: 8 total, 8 up, 7 in 2023-09-23T18:40:27.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:27 smithi173 bash[59422]: cluster 2023-09-23T18:40:26.039333+0000 osd.7 (osd.7) 79 : cluster 3 failed to encode map e167 with expected crc 2023-09-23T18:40:27.280 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:27.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:27.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:27.383 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:27.428 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:27.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:27.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:27.535 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:27.556 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:27.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:27.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:27.659 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:27.686 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:27.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:27.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:27.792 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:27.820 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:27.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:27.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:27.930 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:27.961 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:27.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:27.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:28.064 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:28.073 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:28 smithi098 bash[76817]: cluster 2023-09-23T18:40:27.032306+0000 mon.a (mon.0) 913 : cluster 0 osdmap e168: 8 total, 8 up, 7 in 2023-09-23T18:40:28.073 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:28 smithi098 bash[76817]: cluster 2023-09-23T18:40:27.033205+0000 osd.7 (osd.7) 80 : cluster 3 failed to encode map e168 with expected crc 2023-09-23T18:40:28.073 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:28 smithi098 bash[76817]: cluster 2023-09-23T18:40:27.054801+0000 osd.6 (osd.6) 82 : cluster 3 failed to encode map e168 with expected crc 2023-09-23T18:40:28.073 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:28 smithi098 bash[76817]: cluster 2023-09-23T18:40:27.894372+0000 mgr.y (mgr.26054) 663 : cluster 0 pgmap v651: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 83 active+clean; 2.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 701/144123 objects degraded (0.486%); 13457/144123 objects misplaced (9.337%); 28 MiB/s, 452 objects/s recovering 2023-09-23T18:40:28.074 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:28 smithi098 bash[79516]: cluster 2023-09-23T18:40:27.032306+0000 mon.a (mon.0) 913 : cluster 0 osdmap e168: 8 total, 8 up, 7 in 2023-09-23T18:40:28.074 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:28 smithi098 bash[79516]: cluster 2023-09-23T18:40:27.033205+0000 osd.7 (osd.7) 80 : cluster 3 failed to encode map e168 with expected crc 2023-09-23T18:40:28.074 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:28 smithi098 bash[79516]: cluster 2023-09-23T18:40:27.054801+0000 osd.6 (osd.6) 82 : cluster 3 failed to encode map e168 with expected crc 2023-09-23T18:40:28.074 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:28 smithi098 bash[79516]: cluster 2023-09-23T18:40:27.894372+0000 mgr.y (mgr.26054) 663 : cluster 0 pgmap v651: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 83 active+clean; 2.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 701/144123 objects degraded (0.486%); 13457/144123 objects misplaced (9.337%); 28 MiB/s, 452 objects/s recovering 2023-09-23T18:40:28.092 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:28.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:28.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:28.197 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:28.227 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:28.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:28.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:28.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:28 smithi173 bash[59422]: cluster 2023-09-23T18:40:27.032306+0000 mon.a (mon.0) 913 : cluster 0 osdmap e168: 8 total, 8 up, 7 in 2023-09-23T18:40:28.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:28 smithi173 bash[59422]: cluster 2023-09-23T18:40:27.033205+0000 osd.7 (osd.7) 80 : cluster 3 failed to encode map e168 with expected crc 2023-09-23T18:40:28.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:28 smithi173 bash[59422]: cluster 2023-09-23T18:40:27.054801+0000 osd.6 (osd.6) 82 : cluster 3 failed to encode map e168 with expected crc 2023-09-23T18:40:28.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:28 smithi173 bash[59422]: cluster 2023-09-23T18:40:27.894372+0000 mgr.y (mgr.26054) 663 : cluster 0 pgmap v651: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 83 active+clean; 2.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 701/144123 objects degraded (0.486%); 13457/144123 objects misplaced (9.337%); 28 MiB/s, 452 objects/s recovering 2023-09-23T18:40:28.336 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:28.363 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:28.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:28.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:28.467 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:28.511 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:28.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:28.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:28.618 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:28.648 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:28.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:28.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:28.745 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:40:28.754 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:28.779 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:28.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:28.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:28.884 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:28.911 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:28.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:28.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:29.014 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:29.039 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:29.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:29.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:29.143 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:29.179 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:29.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:29.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:29.285 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:29.313 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:29.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:29.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:29.424 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:29.458 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:29.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:29.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:29.562 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:29.590 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:29.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:29.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:29.697 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:29.727 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:29.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:29.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:29.762 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:40:29.830 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:29.855 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:29.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:29.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:29.959 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:29.986 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:29.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:29.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:30.029 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:40:30.089 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:30.117 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:30.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:30.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:30.222 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:30.251 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:30.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:30.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:30.356 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:30.394 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:30.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:30.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:30.498 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:30.524 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:30.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:30.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:30.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:30.662 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:30.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:30.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:30.767 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:30.794 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:30.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:30.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:30.903 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:30.936 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:30.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:30.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:31.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:30 smithi173 bash[59422]: cluster 2023-09-23T18:40:29.894847+0000 mgr.y (mgr.26054) 664 : cluster 0 pgmap v652: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 83 active+clean; 2.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 361 op/s; 701/144123 objects degraded (0.486%); 13457/144123 objects misplaced (9.337%); 18 MiB/s, 281 objects/s recovering 2023-09-23T18:40:31.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:30 smithi173 bash[59422]: cluster 2023-09-23T18:40:29.975290+0000 mon.a (mon.0) 914 : cluster 3 Health check update: Degraded data redundancy: 701/144123 objects degraded (0.486%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:31.042 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:31.083 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:31.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:31.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:31.189 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:31.213 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:31.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:31.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:31.217 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:30 smithi098 bash[76817]: cluster 2023-09-23T18:40:29.894847+0000 mgr.y (mgr.26054) 664 : cluster 0 pgmap v652: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 83 active+clean; 2.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 361 op/s; 701/144123 objects degraded (0.486%); 13457/144123 objects misplaced (9.337%); 18 MiB/s, 281 objects/s recovering 2023-09-23T18:40:31.217 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:30 smithi098 bash[76817]: cluster 2023-09-23T18:40:29.975290+0000 mon.a (mon.0) 914 : cluster 3 Health check update: Degraded data redundancy: 701/144123 objects degraded (0.486%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:31.218 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:30 smithi098 bash[79516]: cluster 2023-09-23T18:40:29.894847+0000 mgr.y (mgr.26054) 664 : cluster 0 pgmap v652: 89 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 83 active+clean; 2.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 361 op/s; 701/144123 objects degraded (0.486%); 13457/144123 objects misplaced (9.337%); 18 MiB/s, 281 objects/s recovering 2023-09-23T18:40:31.218 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:30 smithi098 bash[79516]: cluster 2023-09-23T18:40:29.975290+0000 mon.a (mon.0) 914 : cluster 3 Health check update: Degraded data redundancy: 701/144123 objects degraded (0.486%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:31.318 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:31.343 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:31.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:31.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:31.448 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:31.471 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:31.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:31.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:31.575 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:31.602 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:31.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:31.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:31.706 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:31.733 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:31.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:31.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:31.839 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:31.864 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:31.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:31.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:31.972 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:32.001 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:32.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:32.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:32.105 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:32.138 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:32.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:32.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:32.242 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:32.273 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:32.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:32.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:32.381 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:32.409 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:32.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:32.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:32.516 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:32.543 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:32.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:32.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:32.648 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:32.690 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:32.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:32.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:32.797 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:32.841 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:32.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:32.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:32.946 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:32.978 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:32.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:32.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:33.082 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:33.125 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:33.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:33.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:33.230 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:33.240 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:32 smithi173 bash[59422]: cluster 2023-09-23T18:40:31.895555+0000 mgr.y (mgr.26054) 665 : cluster 0 pgmap v653: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 701/139779 objects degraded (0.502%); 11656/139779 objects misplaced (8.339%); 25 MiB/s, 396 objects/s recovering 2023-09-23T18:40:33.259 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:33.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:33.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:33.366 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:33.397 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:33.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:33.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:33.500 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:33.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:33.535 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:33.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:33.536 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:33 smithi098 bash[76817]: cluster 2023-09-23T18:40:31.895555+0000 mgr.y (mgr.26054) 665 : cluster 0 pgmap v653: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 701/139779 objects degraded (0.502%); 11656/139779 objects misplaced (8.339%); 25 MiB/s, 396 objects/s recovering 2023-09-23T18:40:33.637 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:33.673 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:33.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:33.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:33.782 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:33.791 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:33 smithi098 bash[79516]: cluster 2023-09-23T18:40:31.895555+0000 mgr.y (mgr.26054) 665 : cluster 0 pgmap v653: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 701/139779 objects degraded (0.502%); 11656/139779 objects misplaced (8.339%); 25 MiB/s, 396 objects/s recovering 2023-09-23T18:40:33.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:33.813 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:33.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:33.914 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:33.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:33.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:33.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:34.051 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:34.077 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:34.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:34.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:34.180 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:34.207 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:34.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:34.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:34.315 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:34.344 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:34.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:34.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:34.450 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:34.484 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:34.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:34.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:34.590 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:34.627 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:34.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:34.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:34.731 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:34.762 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:34.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:34.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:34.872 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:34.897 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:34.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:34.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:35.000 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:35.021 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:35.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:34 smithi173 bash[59422]: cluster 2023-09-23T18:40:33.896187+0000 mgr.y (mgr.26054) 666 : cluster 0 pgmap v654: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 791/134928 objects degraded (0.586%); 11477/134928 objects misplaced (8.506%); 19 MiB/s, 308 objects/s recovering 2023-09-23T18:40:35.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:35.029 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:34 smithi098 bash[76817]: cluster 2023-09-23T18:40:33.896187+0000 mgr.y (mgr.26054) 666 : cluster 0 pgmap v654: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 791/134928 objects degraded (0.586%); 11477/134928 objects misplaced (8.506%); 19 MiB/s, 308 objects/s recovering 2023-09-23T18:40:35.030 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:35 smithi098 bash[79516]: cluster 2023-09-23T18:40:33.896187+0000 mgr.y (mgr.26054) 666 : cluster 0 pgmap v654: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 791/134928 objects degraded (0.586%); 11477/134928 objects misplaced (8.506%); 19 MiB/s, 308 objects/s recovering 2023-09-23T18:40:35.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:35.131 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:35.156 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:35.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:35.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:35.259 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:35.300 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:35.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:35.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:35.405 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:35.430 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:35.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:35.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:35.533 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:35.539 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:40:35.539 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-09-23T18:40:35.539 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.539 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.44954.0:166872 6.f e168/162 6:ffa3f0b9:::benchmark_data_smithi098_131983_object62203:head v 168'10400, mlcod=157'3877)", 2023-09-23T18:40:35.539 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:40:35.534187+0000", 2023-09-23T18:40:35.540 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.0046375160000000004, 2023-09-23T18:40:35.540 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0046687120000000002, 2023-09-23T18:40:35.540 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:40:35.540 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "started", 2023-09-23T18:40:35.540 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:40:35.540 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.540 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:40:35.540 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:35.534187+0000", 2023-09-23T18:40:35.540 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:40:35.540 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:40:35.540 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.540 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:40:35.541 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:35.534187+0000", 2023-09-23T18:40:35.541 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:40:35.541 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:40:35.541 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.541 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:40:35.541 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:35.534186+0000", 2023-09-23T18:40:35.541 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.057e-05 2023-09-23T18:40:35.541 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:40:35.541 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.541 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:40:35.541 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:35.534197+0000", 2023-09-23T18:40:35.541 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4576000000000001e-05 2023-09-23T18:40:35.541 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:40:35.542 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.542 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:40:35.542 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:35.534211+0000", 2023-09-23T18:40:35.542 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.9139999999999996e-06 2023-09-23T18:40:35.542 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:40:35.542 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.542 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:40:35.542 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:35.534217+0000", 2023-09-23T18:40:35.542 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.2425e-05 2023-09-23T18:40:35.542 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:40:35.542 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.542 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:40:35.543 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:35.534270+0000", 2023-09-23T18:40:35.543 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.4326000000000003e-05 2023-09-23T18:40:35.543 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:40:35.543 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.543 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:40:35.543 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:35.534304+0000", 2023-09-23T18:40:35.543 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00011698500000000001 2023-09-23T18:40:35.543 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:40:35.543 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:40:35.543 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:40:35.543 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:40:35.543 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.543 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44954.0:166883 6.2 6:47e68076:::benchmark_data_smithi098_131983_object62214:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-09-23T18:40:35.544 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:40:35.537666+0000", 2023-09-23T18:40:35.544 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.0011588550000000001, 2023-09-23T18:40:35.544 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001264491, 2023-09-23T18:40:35.544 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:40:35.544 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:40:35.544 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:40:35.544 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44954", 2023-09-23T18:40:35.544 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/343998332", 2023-09-23T18:40:35.544 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 166883 2023-09-23T18:40:35.544 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:40:35.544 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:40:35.544 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.544 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:40:35.545 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:35.537666+0000", 2023-09-23T18:40:35.545 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:40:35.545 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:40:35.545 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.545 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:40:35.545 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:35.537666+0000", 2023-09-23T18:40:35.545 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:40:35.545 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:40:35.545 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.545 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:40:35.545 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:35.537664+0000", 2023-09-23T18:40:35.545 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.2899999999999997e-06 2023-09-23T18:40:35.545 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:40:35.546 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.546 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:40:35.546 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:35.537671+0000", 2023-09-23T18:40:35.546 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4867e-05 2023-09-23T18:40:35.546 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:40:35.546 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.546 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:40:35.546 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:35.537686+0000", 2023-09-23T18:40:35.546 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5049999999999998e-06 2023-09-23T18:40:35.546 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:40:35.546 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.546 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:40:35.547 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:35.537692+0000", 2023-09-23T18:40:35.547 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.3009999999999994e-05 2023-09-23T18:40:35.547 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:40:35.547 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.547 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:40:35.547 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:35.537765+0000", 2023-09-23T18:40:35.547 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016632300000000001 2023-09-23T18:40:35.547 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:40:35.547 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.547 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:40:35.547 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:35.537931+0000", 2023-09-23T18:40:35.547 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000189335 2023-09-23T18:40:35.547 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:40:35.548 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.548 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,3,6", 2023-09-23T18:40:35.548 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:35.538120+0000", 2023-09-23T18:40:35.548 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00045464300000000002 2023-09-23T18:40:35.548 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:40:35.548 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:40:35.548 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:40:35.548 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:40:35.548 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.548 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.44954.0:166873 6.7 e168/149)", 2023-09-23T18:40:35.548 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:40:35.538330+0000", 2023-09-23T18:40:35.548 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.00049485299999999996, 2023-09-23T18:40:35.549 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000677532, 2023-09-23T18:40:35.549 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:40:35.549 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "queued for pg", 2023-09-23T18:40:35.549 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:40:35.549 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.549 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:40:35.549 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:35.538330+0000", 2023-09-23T18:40:35.549 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:40:35.549 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:40:35.549 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.549 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:40:35.549 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:35.538330+0000", 2023-09-23T18:40:35.549 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:40:35.550 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:40:35.550 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.550 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:40:35.550 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:35.538329+0000", 2023-09-23T18:40:35.550 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.5649999999999998e-06 2023-09-23T18:40:35.550 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:40:35.550 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.550 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:40:35.550 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:35.538335+0000", 2023-09-23T18:40:35.550 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.8480000000000007e-06 2023-09-23T18:40:35.550 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:40:35.550 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.551 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:40:35.551 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:35.538344+0000", 2023-09-23T18:40:35.551 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6930000000000004e-06 2023-09-23T18:40:35.551 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:40:35.551 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:40:35.551 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:40:35.551 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:35.538350+0000", 2023-09-23T18:40:35.551 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.0237999999999998e-05 2023-09-23T18:40:35.551 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:40:35.551 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:40:35.551 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:40:35.551 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:40:35.551 INFO:teuthology.orchestra.run.smithi173.stdout: ], 2023-09-23T18:40:35.552 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 3 2023-09-23T18:40:35.552 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:40:35.572 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:35.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:35.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:35.676 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:35.701 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:35.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:35.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:35.804 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:35.825 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:35.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:35.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:35.928 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:35.958 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:35.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:35.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:35 smithi173 bash[59422]: cluster 2023-09-23T18:40:34.976374+0000 mon.a (mon.0) 915 : cluster 3 Health check update: Degraded data redundancy: 791/134928 objects degraded (0.586%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:36.064 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:36.089 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:36.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:36.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:36.192 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:36.228 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:36.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:36.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:36.331 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:36.359 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:36.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:36.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:36.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:35 smithi098 bash[79516]: cluster 2023-09-23T18:40:34.976374+0000 mon.a (mon.0) 915 : cluster 3 Health check update: Degraded data redundancy: 791/134928 objects degraded (0.586%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:36.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:35 smithi098 bash[76817]: cluster 2023-09-23T18:40:34.976374+0000 mon.a (mon.0) 915 : cluster 3 Health check update: Degraded data redundancy: 791/134928 objects degraded (0.586%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:36.441 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:40:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:40:36] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.33.4" 2023-09-23T18:40:36.462 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:36.498 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:36.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:36.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:36.601 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:36.635 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:36.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:36.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:36.733 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T18:40:36.741 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:36.781 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:36.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:36.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:36.885 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:36.915 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:36.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:36.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:36.986 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:36 smithi173 bash[59422]: cluster 2023-09-23T18:40:35.896832+0000 mgr.y (mgr.26054) 667 : cluster 0 pgmap v655: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 791/131361 objects degraded (0.602%); 11014/131361 objects misplaced (8.385%); 16 MiB/s, 261 objects/s recovering 2023-09-23T18:40:37.019 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:37.041 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:37.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:37.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:37.144 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:37.168 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:37.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:37.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:37.273 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:37.300 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:37.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:37.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:37.407 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:37.439 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:37.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:37.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:36 smithi098 bash[79516]: cluster 2023-09-23T18:40:35.896832+0000 mgr.y (mgr.26054) 667 : cluster 0 pgmap v655: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 791/131361 objects degraded (0.602%); 11014/131361 objects misplaced (8.385%); 16 MiB/s, 261 objects/s recovering 2023-09-23T18:40:37.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:37.542 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:37.546 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:36 smithi098 bash[76817]: cluster 2023-09-23T18:40:35.896832+0000 mgr.y (mgr.26054) 667 : cluster 0 pgmap v655: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 791/131361 objects degraded (0.602%); 11014/131361 objects misplaced (8.385%); 16 MiB/s, 261 objects/s recovering 2023-09-23T18:40:37.563 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:37.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:37.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:37.667 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:37.692 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:37.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:37.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:37.795 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:37.813 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:37.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:37.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:37.916 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:37.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:37.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:37.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:38.047 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:38.074 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:38.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:38.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:38.180 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:38.190 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:38 smithi173 bash[59422]: cluster 2023-09-23T18:40:37.000425+0000 osd.5 (osd.5) 71 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T18:40:38.190 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:38 smithi173 bash[59422]: cluster 2023-09-23T18:40:37.002926+0000 mon.a (mon.0) 916 : cluster 0 osdmap e169: 8 total, 8 up, 7 in 2023-09-23T18:40:38.191 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:38 smithi173 bash[59422]: cluster 2023-09-23T18:40:37.005890+0000 osd.4 (osd.4) 109 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T18:40:38.191 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:38 smithi173 bash[59422]: cluster 2023-09-23T18:40:37.008508+0000 osd.4 (osd.4) 110 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T18:40:38.191 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:38 smithi173 bash[59422]: cluster 2023-09-23T18:40:37.009713+0000 osd.4 (osd.4) 111 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T18:40:38.191 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:38 smithi173 bash[59422]: cluster 2023-09-23T18:40:37.014356+0000 osd.7 (osd.7) 81 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T18:40:38.191 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:38 smithi173 bash[59422]: cluster 2023-09-23T18:40:37.014409+0000 osd.6 (osd.6) 83 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T18:40:38.191 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:38 smithi173 bash[59422]: cluster 2023-09-23T18:40:37.016040+0000 osd.6 (osd.6) 84 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T18:40:38.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:38 smithi098 bash[76817]: cluster 2023-09-23T18:40:37.000425+0000 osd.5 (osd.5) 71 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T18:40:38.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:38 smithi098 bash[76817]: cluster 2023-09-23T18:40:37.002926+0000 mon.a (mon.0) 916 : cluster 0 osdmap e169: 8 total, 8 up, 7 in 2023-09-23T18:40:38.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:38 smithi098 bash[76817]: cluster 2023-09-23T18:40:37.005890+0000 osd.4 (osd.4) 109 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T18:40:38.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:38 smithi098 bash[76817]: cluster 2023-09-23T18:40:37.008508+0000 osd.4 (osd.4) 110 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T18:40:38.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:38 smithi098 bash[76817]: cluster 2023-09-23T18:40:37.009713+0000 osd.4 (osd.4) 111 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T18:40:38.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:38 smithi098 bash[76817]: cluster 2023-09-23T18:40:37.014356+0000 osd.7 (osd.7) 81 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T18:40:38.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:38 smithi098 bash[76817]: cluster 2023-09-23T18:40:37.014409+0000 osd.6 (osd.6) 83 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T18:40:38.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:38 smithi098 bash[76817]: cluster 2023-09-23T18:40:37.016040+0000 osd.6 (osd.6) 84 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T18:40:38.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:38 smithi098 bash[79516]: cluster 2023-09-23T18:40:37.000425+0000 osd.5 (osd.5) 71 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T18:40:38.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:38 smithi098 bash[79516]: cluster 2023-09-23T18:40:37.002926+0000 mon.a (mon.0) 916 : cluster 0 osdmap e169: 8 total, 8 up, 7 in 2023-09-23T18:40:38.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:38 smithi098 bash[79516]: cluster 2023-09-23T18:40:37.005890+0000 osd.4 (osd.4) 109 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T18:40:38.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:38 smithi098 bash[79516]: cluster 2023-09-23T18:40:37.008508+0000 osd.4 (osd.4) 110 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T18:40:38.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:38 smithi098 bash[79516]: cluster 2023-09-23T18:40:37.009713+0000 osd.4 (osd.4) 111 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T18:40:38.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:38 smithi098 bash[79516]: cluster 2023-09-23T18:40:37.014356+0000 osd.7 (osd.7) 81 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T18:40:38.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:38 smithi098 bash[79516]: cluster 2023-09-23T18:40:37.014409+0000 osd.6 (osd.6) 83 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T18:40:38.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:38 smithi098 bash[79516]: cluster 2023-09-23T18:40:37.016040+0000 osd.6 (osd.6) 84 : cluster 3 failed to encode map e169 with expected crc 2023-09-23T18:40:38.211 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:38.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:38.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:38.314 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:38.339 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:38.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:38.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:38.442 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:38.470 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:38.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:38.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:38.574 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:38.608 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:38.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:38.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:38.712 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:38.744 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:38.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:38.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:38.847 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:38.873 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:38.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:38.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:38.980 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:39.013 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:39.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:39.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:39.121 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:39.129 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:39 smithi173 bash[59422]: cluster 2023-09-23T18:40:37.897322+0000 mgr.y (mgr.26054) 668 : cluster 0 pgmap v657: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 2.6 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 928/125646 objects degraded (0.739%); 9713/125646 objects misplaced (7.730%); 15 MiB/s, 236 objects/s recovering 2023-09-23T18:40:39.129 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:39 smithi173 bash[59422]: cluster 2023-09-23T18:40:38.006957+0000 osd.4 (osd.4) 112 : cluster 3 failed to encode map e170 with expected crc 2023-09-23T18:40:39.130 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:39 smithi173 bash[59422]: cluster 2023-09-23T18:40:38.008625+0000 mon.a (mon.0) 917 : cluster 0 osdmap e170: 8 total, 8 up, 7 in 2023-09-23T18:40:39.130 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:39 smithi173 bash[59422]: cluster 2023-09-23T18:40:38.008901+0000 osd.4 (osd.4) 113 : cluster 3 failed to encode map e170 with expected crc 2023-09-23T18:40:39.130 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:39 smithi173 bash[59422]: cluster 2023-09-23T18:40:38.012140+0000 osd.4 (osd.4) 114 : cluster 3 failed to encode map e170 with expected crc 2023-09-23T18:40:39.130 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:39 smithi173 bash[59422]: cluster 2023-09-23T18:40:38.021911+0000 osd.6 (osd.6) 85 : cluster 3 failed to encode map e170 with expected crc 2023-09-23T18:40:39.130 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:39 smithi173 bash[59422]: cluster 2023-09-23T18:40:38.325385+0000 osd.5 (osd.5) 72 : cluster 3 failed to encode map e170 with expected crc 2023-09-23T18:40:39.150 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:39.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:39.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:39.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:39 smithi098 bash[79516]: cluster 2023-09-23T18:40:37.897322+0000 mgr.y (mgr.26054) 668 : cluster 0 pgmap v657: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 2.6 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 928/125646 objects degraded (0.739%); 9713/125646 objects misplaced (7.730%); 15 MiB/s, 236 objects/s recovering 2023-09-23T18:40:39.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:39 smithi098 bash[79516]: cluster 2023-09-23T18:40:38.006957+0000 osd.4 (osd.4) 112 : cluster 3 failed to encode map e170 with expected crc 2023-09-23T18:40:39.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:39 smithi098 bash[79516]: cluster 2023-09-23T18:40:38.008625+0000 mon.a (mon.0) 917 : cluster 0 osdmap e170: 8 total, 8 up, 7 in 2023-09-23T18:40:39.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:39 smithi098 bash[79516]: cluster 2023-09-23T18:40:38.008901+0000 osd.4 (osd.4) 113 : cluster 3 failed to encode map e170 with expected crc 2023-09-23T18:40:39.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:39 smithi098 bash[79516]: cluster 2023-09-23T18:40:38.012140+0000 osd.4 (osd.4) 114 : cluster 3 failed to encode map e170 with expected crc 2023-09-23T18:40:39.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:39 smithi098 bash[79516]: cluster 2023-09-23T18:40:38.021911+0000 osd.6 (osd.6) 85 : cluster 3 failed to encode map e170 with expected crc 2023-09-23T18:40:39.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:39 smithi098 bash[79516]: cluster 2023-09-23T18:40:38.325385+0000 osd.5 (osd.5) 72 : cluster 3 failed to encode map e170 with expected crc 2023-09-23T18:40:39.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:39 smithi098 bash[76817]: cluster 2023-09-23T18:40:37.897322+0000 mgr.y (mgr.26054) 668 : cluster 0 pgmap v657: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 2.6 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 928/125646 objects degraded (0.739%); 9713/125646 objects misplaced (7.730%); 15 MiB/s, 236 objects/s recovering 2023-09-23T18:40:39.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:39 smithi098 bash[76817]: cluster 2023-09-23T18:40:38.006957+0000 osd.4 (osd.4) 112 : cluster 3 failed to encode map e170 with expected crc 2023-09-23T18:40:39.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:39 smithi098 bash[76817]: cluster 2023-09-23T18:40:38.008625+0000 mon.a (mon.0) 917 : cluster 0 osdmap e170: 8 total, 8 up, 7 in 2023-09-23T18:40:39.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:39 smithi098 bash[76817]: cluster 2023-09-23T18:40:38.008901+0000 osd.4 (osd.4) 113 : cluster 3 failed to encode map e170 with expected crc 2023-09-23T18:40:39.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:39 smithi098 bash[76817]: cluster 2023-09-23T18:40:38.012140+0000 osd.4 (osd.4) 114 : cluster 3 failed to encode map e170 with expected crc 2023-09-23T18:40:39.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:39 smithi098 bash[76817]: cluster 2023-09-23T18:40:38.021911+0000 osd.6 (osd.6) 85 : cluster 3 failed to encode map e170 with expected crc 2023-09-23T18:40:39.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:39 smithi098 bash[76817]: cluster 2023-09-23T18:40:38.325385+0000 osd.5 (osd.5) 72 : cluster 3 failed to encode map e170 with expected crc 2023-09-23T18:40:39.256 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:39.284 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:39.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:39.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:39.391 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:39.418 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:39.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:39.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:39.524 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:39.569 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:39.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:39.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:39.673 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:39.708 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:39.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:39.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:39.811 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:39.846 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:39.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:39.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:39.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:39.981 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:39.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:39.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:40.084 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:40.107 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:40.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:40.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:40.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:40 smithi098 bash[79516]: audit 2023-09-23T18:40:39.600185+0000 mon.a (mon.0) 918 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:40:40.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:40 smithi098 bash[79516]: cluster 2023-09-23T18:40:39.977326+0000 mon.a (mon.0) 919 : cluster 3 Health check update: Degraded data redundancy: 928/125646 objects degraded (0.739%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:40.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:40 smithi098 bash[76817]: audit 2023-09-23T18:40:39.600185+0000 mon.a (mon.0) 918 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:40:40.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:40 smithi098 bash[76817]: cluster 2023-09-23T18:40:39.977326+0000 mon.a (mon.0) 919 : cluster 3 Health check update: Degraded data redundancy: 928/125646 objects degraded (0.739%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:40.211 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:40.258 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:40.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:40.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:40.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:40 smithi173 bash[59422]: audit 2023-09-23T18:40:39.600185+0000 mon.a (mon.0) 918 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:40:40.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:40 smithi173 bash[59422]: cluster 2023-09-23T18:40:39.977326+0000 mon.a (mon.0) 919 : cluster 3 Health check update: Degraded data redundancy: 928/125646 objects degraded (0.739%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:40.362 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:40.397 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:40.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:40.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:40.502 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:40.526 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:40.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:40.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:40.630 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:40.658 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:40.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:40.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:40.763 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:40.799 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:40.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:40.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:40.905 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:40.934 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:40.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:40.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:41.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:41 smithi173 bash[59422]: cluster 2023-09-23T18:40:39.897792+0000 mgr.y (mgr.26054) 669 : cluster 0 pgmap v659: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 2.6 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 928/125646 objects degraded (0.739%); 9713/125646 objects misplaced (7.730%); 8.9 MiB/s, 141 objects/s recovering 2023-09-23T18:40:41.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:41 smithi173 bash[59422]: audit 2023-09-23T18:40:40.512086+0000 mon.a (mon.0) 920 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'32403','3':'35035','7':'55964'}"}]: dispatch 2023-09-23T18:40:41.040 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:41.055 INFO:teuthology.orchestra.run.smithi098.stdout:moved 25 / 267 (9.3633%) 2023-09-23T18:40:41.055 INFO:teuthology.orchestra.run.smithi098.stdout:avg 38.1429 2023-09-23T18:40:41.055 INFO:teuthology.orchestra.run.smithi098.stdout:stddev 6.83329 -> 11.4446 (expected baseline 5.71786) 2023-09-23T18:40:41.055 INFO:teuthology.orchestra.run.smithi098.stdout:min osd.2 with 23 -> 21 pgs (0.602996 -> 0.550562 * mean) 2023-09-23T18:40:41.055 INFO:teuthology.orchestra.run.smithi098.stdout:max osd.1 with 46 -> 54 pgs (1.20599 -> 1.41573 * mean) 2023-09-23T18:40:41.055 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:40:41.055 INFO:teuthology.orchestra.run.smithi098.stdout:oload 110 2023-09-23T18:40:41.055 INFO:teuthology.orchestra.run.smithi098.stdout:max_change 3 2023-09-23T18:40:41.055 INFO:teuthology.orchestra.run.smithi098.stdout:max_change_osds 4 2023-09-23T18:40:41.055 INFO:teuthology.orchestra.run.smithi098.stdout:average_utilization 0.0166 2023-09-23T18:40:41.056 INFO:teuthology.orchestra.run.smithi098.stdout:overload_utilization 0.0183 2023-09-23T18:40:41.056 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2 weight 0.6861 -> 0.4944 2023-09-23T18:40:41.056 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3 weight 0.7272 -> 0.5346 2023-09-23T18:40:41.056 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7 weight 1.0000 -> 0.8539 2023-09-23T18:40:41.062 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:41.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:41.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:41.165 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:41.190 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:41.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:41.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:41.293 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:41.319 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:41.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:41.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:41.422 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:41.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:41 smithi098 bash[76817]: cluster 2023-09-23T18:40:39.897792+0000 mgr.y (mgr.26054) 669 : cluster 0 pgmap v659: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 2.6 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 928/125646 objects degraded (0.739%); 9713/125646 objects misplaced (7.730%); 8.9 MiB/s, 141 objects/s recovering 2023-09-23T18:40:41.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:41 smithi098 bash[76817]: audit 2023-09-23T18:40:40.512086+0000 mon.a (mon.0) 920 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'32403','3':'35035','7':'55964'}"}]: dispatch 2023-09-23T18:40:41.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:41 smithi098 bash[79516]: cluster 2023-09-23T18:40:39.897792+0000 mgr.y (mgr.26054) 669 : cluster 0 pgmap v659: 89 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 2.6 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 928/125646 objects degraded (0.739%); 9713/125646 objects misplaced (7.730%); 8.9 MiB/s, 141 objects/s recovering 2023-09-23T18:40:41.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:41 smithi098 bash[79516]: audit 2023-09-23T18:40:40.512086+0000 mon.a (mon.0) 920 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'32403','3':'35035','7':'55964'}"}]: dispatch 2023-09-23T18:40:41.450 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:41.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:41.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:41.554 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:41.555 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:40:41.579 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:41.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:41.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:41.687 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:41.712 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:41.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:41.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:41.815 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:41.842 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:41.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:41.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:41.945 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:41.974 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:41.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:41.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:42.077 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:42.104 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:42.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:42.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:42.171 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:42 smithi098 bash[76817]: audit 2023-09-23T18:40:40.510377+0000 mgr.y (mgr.26054) 670 : audit 0 from='client.45071 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:40:42.171 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:42 smithi098 bash[76817]: audit 2023-09-23T18:40:41.029671+0000 mon.a (mon.0) 921 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'32403','3':'35035','7':'55964'}"}]': finished 2023-09-23T18:40:42.171 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:42 smithi098 bash[76817]: cluster 2023-09-23T18:40:41.029944+0000 mon.a (mon.0) 922 : cluster 0 osdmap e171: 8 total, 8 up, 7 in 2023-09-23T18:40:42.171 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:42 smithi098 bash[76817]: cluster 2023-09-23T18:40:41.030384+0000 osd.6 (osd.6) 86 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T18:40:42.171 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:42 smithi098 bash[76817]: cluster 2023-09-23T18:40:41.031930+0000 osd.7 (osd.7) 82 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T18:40:42.172 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:42 smithi098 bash[76817]: cluster 2023-09-23T18:40:41.032462+0000 osd.4 (osd.4) 115 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T18:40:42.172 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:42 smithi098 bash[76817]: cluster 2023-09-23T18:40:41.413584+0000 osd.5 (osd.5) 73 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T18:40:42.173 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:42 smithi098 bash[79516]: audit 2023-09-23T18:40:40.510377+0000 mgr.y (mgr.26054) 670 : audit 0 from='client.45071 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:40:42.173 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:42 smithi098 bash[79516]: audit 2023-09-23T18:40:41.029671+0000 mon.a (mon.0) 921 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'32403','3':'35035','7':'55964'}"}]': finished 2023-09-23T18:40:42.173 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:42 smithi098 bash[79516]: cluster 2023-09-23T18:40:41.029944+0000 mon.a (mon.0) 922 : cluster 0 osdmap e171: 8 total, 8 up, 7 in 2023-09-23T18:40:42.173 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:42 smithi098 bash[79516]: cluster 2023-09-23T18:40:41.030384+0000 osd.6 (osd.6) 86 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T18:40:42.173 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:42 smithi098 bash[79516]: cluster 2023-09-23T18:40:41.031930+0000 osd.7 (osd.7) 82 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T18:40:42.173 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:42 smithi098 bash[79516]: cluster 2023-09-23T18:40:41.032462+0000 osd.4 (osd.4) 115 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T18:40:42.173 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:42 smithi098 bash[79516]: cluster 2023-09-23T18:40:41.413584+0000 osd.5 (osd.5) 73 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T18:40:42.209 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:42.233 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:42.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:42.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:42.336 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:42.357 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:42.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:42.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:42.460 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:42.482 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:42.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:42.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:42.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:42 smithi173 bash[59422]: audit 2023-09-23T18:40:40.510377+0000 mgr.y (mgr.26054) 670 : audit 0 from='client.45071 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:40:42.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:42 smithi173 bash[59422]: audit 2023-09-23T18:40:41.029671+0000 mon.a (mon.0) 921 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'32403','3':'35035','7':'55964'}"}]': finished 2023-09-23T18:40:42.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:42 smithi173 bash[59422]: cluster 2023-09-23T18:40:41.029944+0000 mon.a (mon.0) 922 : cluster 0 osdmap e171: 8 total, 8 up, 7 in 2023-09-23T18:40:42.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:42 smithi173 bash[59422]: cluster 2023-09-23T18:40:41.030384+0000 osd.6 (osd.6) 86 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T18:40:42.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:42 smithi173 bash[59422]: cluster 2023-09-23T18:40:41.031930+0000 osd.7 (osd.7) 82 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T18:40:42.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:42 smithi173 bash[59422]: cluster 2023-09-23T18:40:41.032462+0000 osd.4 (osd.4) 115 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T18:40:42.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:42 smithi173 bash[59422]: cluster 2023-09-23T18:40:41.413584+0000 osd.5 (osd.5) 73 : cluster 3 failed to encode map e171 with expected crc 2023-09-23T18:40:42.584 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:42.612 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:42.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:42.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:42.715 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:42.734 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:42.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:42.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:42.836 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:42.865 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:42.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:42.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:42.968 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:42.988 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:42.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:42.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:43.091 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:43.114 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:43.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:43.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:43.218 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:43.244 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:43.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:43.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:43.354 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:43.365 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:43 smithi173 bash[59422]: cluster 2023-09-23T18:40:41.898650+0000 mgr.y (mgr.26054) 671 : cluster 0 pgmap v661: 89 pgs: 2 peering, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 83 active+clean; 2.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 928/120729 objects degraded (0.769%); 8969/120729 objects misplaced (7.429%); 12 MiB/s, 194 objects/s recovering 2023-09-23T18:40:43.365 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:43 smithi173 bash[59422]: cluster 2023-09-23T18:40:42.037782+0000 osd.6 (osd.6) 87 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T18:40:43.365 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:43 smithi173 bash[59422]: cluster 2023-09-23T18:40:42.038721+0000 osd.4 (osd.4) 116 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T18:40:43.365 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:43 smithi173 bash[59422]: cluster 2023-09-23T18:40:42.039789+0000 osd.7 (osd.7) 83 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T18:40:43.365 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:43 smithi173 bash[59422]: cluster 2023-09-23T18:40:42.040106+0000 mon.a (mon.0) 923 : cluster 0 osdmap e172: 8 total, 8 up, 7 in 2023-09-23T18:40:43.365 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:43 smithi173 bash[59422]: cluster 2023-09-23T18:40:42.041164+0000 osd.7 (osd.7) 84 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T18:40:43.365 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:43 smithi173 bash[59422]: cluster 2023-09-23T18:40:42.042044+0000 osd.4 (osd.4) 117 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T18:40:43.365 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:43 smithi173 bash[59422]: cluster 2023-09-23T18:40:42.044277+0000 osd.4 (osd.4) 118 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T18:40:43.399 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:43.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:43.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:43.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:43 smithi098 bash[76817]: cluster 2023-09-23T18:40:41.898650+0000 mgr.y (mgr.26054) 671 : cluster 0 pgmap v661: 89 pgs: 2 peering, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 83 active+clean; 2.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 928/120729 objects degraded (0.769%); 8969/120729 objects misplaced (7.429%); 12 MiB/s, 194 objects/s recovering 2023-09-23T18:40:43.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:43 smithi098 bash[76817]: cluster 2023-09-23T18:40:42.037782+0000 osd.6 (osd.6) 87 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T18:40:43.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:43 smithi098 bash[76817]: cluster 2023-09-23T18:40:42.038721+0000 osd.4 (osd.4) 116 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T18:40:43.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:43 smithi098 bash[76817]: cluster 2023-09-23T18:40:42.039789+0000 osd.7 (osd.7) 83 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T18:40:43.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:43 smithi098 bash[76817]: cluster 2023-09-23T18:40:42.040106+0000 mon.a (mon.0) 923 : cluster 0 osdmap e172: 8 total, 8 up, 7 in 2023-09-23T18:40:43.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:43 smithi098 bash[76817]: cluster 2023-09-23T18:40:42.041164+0000 osd.7 (osd.7) 84 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T18:40:43.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:43 smithi098 bash[76817]: cluster 2023-09-23T18:40:42.042044+0000 osd.4 (osd.4) 117 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T18:40:43.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:43 smithi098 bash[76817]: cluster 2023-09-23T18:40:42.044277+0000 osd.4 (osd.4) 118 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T18:40:43.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:43 smithi098 bash[79516]: cluster 2023-09-23T18:40:41.898650+0000 mgr.y (mgr.26054) 671 : cluster 0 pgmap v661: 89 pgs: 2 peering, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 83 active+clean; 2.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 928/120729 objects degraded (0.769%); 8969/120729 objects misplaced (7.429%); 12 MiB/s, 194 objects/s recovering 2023-09-23T18:40:43.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:43 smithi098 bash[79516]: cluster 2023-09-23T18:40:42.037782+0000 osd.6 (osd.6) 87 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T18:40:43.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:43 smithi098 bash[79516]: cluster 2023-09-23T18:40:42.038721+0000 osd.4 (osd.4) 116 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T18:40:43.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:43 smithi098 bash[79516]: cluster 2023-09-23T18:40:42.039789+0000 osd.7 (osd.7) 83 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T18:40:43.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:43 smithi098 bash[79516]: cluster 2023-09-23T18:40:42.040106+0000 mon.a (mon.0) 923 : cluster 0 osdmap e172: 8 total, 8 up, 7 in 2023-09-23T18:40:43.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:43 smithi098 bash[79516]: cluster 2023-09-23T18:40:42.041164+0000 osd.7 (osd.7) 84 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T18:40:43.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:43 smithi098 bash[79516]: cluster 2023-09-23T18:40:42.042044+0000 osd.4 (osd.4) 117 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T18:40:43.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:43 smithi098 bash[79516]: cluster 2023-09-23T18:40:42.044277+0000 osd.4 (osd.4) 118 : cluster 3 failed to encode map e172 with expected crc 2023-09-23T18:40:43.504 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:43.530 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:43.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:43.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:43.633 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:43.662 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:43.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:43.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:43.754 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T18:40:43.754 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 3.0 2023-09-23T18:40:43.766 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:43.792 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:43.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:43.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:43.895 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:43.920 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:43.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:43.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:44.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:44.039 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:40:44.049 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:44.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:44.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:44.099 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:40:44.147 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:40:44.156 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:44.165 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[76817]: cluster 2023-09-23T18:40:43.043605+0000 mon.a (mon.0) 924 : cluster 0 osdmap e173: 8 total, 8 up, 7 in 2023-09-23T18:40:44.165 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[76817]: cluster 2023-09-23T18:40:43.043876+0000 osd.1 (osd.1) 2 : cluster 0 6.4 starting backfill to osd.4 from (0'0,0'0] MAX to 170'10780 2023-09-23T18:40:44.165 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[76817]: cluster 2023-09-23T18:40:43.044723+0000 osd.4 (osd.4) 119 : cluster 3 failed to encode map e173 with expected crc 2023-09-23T18:40:44.165 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[76817]: cluster 2023-09-23T18:40:43.045102+0000 osd.6 (osd.6) 88 : cluster 3 failed to encode map e173 with expected crc 2023-09-23T18:40:44.165 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[76817]: cluster 2023-09-23T18:40:43.046398+0000 osd.4 (osd.4) 120 : cluster 3 failed to encode map e173 with expected crc 2023-09-23T18:40:44.166 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[76817]: cluster 2023-09-23T18:40:43.047947+0000 osd.6 (osd.6) 89 : cluster 3 failed to encode map e173 with expected crc 2023-09-23T18:40:44.166 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[76817]: cluster 2023-09-23T18:40:43.049358+0000 osd.4 (osd.4) 121 : cluster 0 6.f starting backfill to osd.1 from (0'0,0'0] MAX to 170'10603 2023-09-23T18:40:44.166 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[76817]: cluster 2023-09-23T18:40:43.049691+0000 osd.4 (osd.4) 122 : cluster 0 6.6 starting backfill to osd.6 from (0'0,0'0] MAX to 170'10562 2023-09-23T18:40:44.166 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[76817]: cluster 2023-09-23T18:40:43.051260+0000 osd.3 (osd.3) 2 : cluster 0 6.b starting backfill to osd.6 from (0'0,0'0] MAX to 170'10546 2023-09-23T18:40:44.166 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[76817]: cluster 2023-09-23T18:40:43.052158+0000 osd.6 (osd.6) 90 : cluster 3 failed to encode map e173 with expected crc 2023-09-23T18:40:44.166 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[76817]: cluster 2023-09-23T18:40:43.054729+0000 osd.6 (osd.6) 91 : cluster 0 6.c starting backfill to osd.1 from (0'0,0'0] MAX to 170'10520 2023-09-23T18:40:44.166 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[76817]: cluster 2023-09-23T18:40:43.113751+0000 osd.5 (osd.5) 74 : cluster 3 failed to encode map e173 with expected crc 2023-09-23T18:40:44.166 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[76817]: cluster 2023-09-23T18:40:43.899164+0000 mgr.y (mgr.26054) 672 : cluster 0 pgmap v664: 89 pgs: 2 remapped+peering, 2 peering, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 81 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 928/118716 objects degraded (0.782%); 8969/118716 objects misplaced (7.555%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T18:40:44.166 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[76817]: audit 2023-09-23T18:40:44.000951+0000 mon.a (mon.0) 925 : audit 0 from='client.? 172.21.15.98:0/877167688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:40:44.167 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[79516]: cluster 2023-09-23T18:40:43.043605+0000 mon.a (mon.0) 924 : cluster 0 osdmap e173: 8 total, 8 up, 7 in 2023-09-23T18:40:44.167 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[79516]: cluster 2023-09-23T18:40:43.043876+0000 osd.1 (osd.1) 2 : cluster 0 6.4 starting backfill to osd.4 from (0'0,0'0] MAX to 170'10780 2023-09-23T18:40:44.167 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[79516]: cluster 2023-09-23T18:40:43.044723+0000 osd.4 (osd.4) 119 : cluster 3 failed to encode map e173 with expected crc 2023-09-23T18:40:44.167 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[79516]: cluster 2023-09-23T18:40:43.045102+0000 osd.6 (osd.6) 88 : cluster 3 failed to encode map e173 with expected crc 2023-09-23T18:40:44.167 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[79516]: cluster 2023-09-23T18:40:43.046398+0000 osd.4 (osd.4) 120 : cluster 3 failed to encode map e173 with expected crc 2023-09-23T18:40:44.168 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[79516]: cluster 2023-09-23T18:40:43.047947+0000 osd.6 (osd.6) 89 : cluster 3 failed to encode map e173 with expected crc 2023-09-23T18:40:44.168 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[79516]: cluster 2023-09-23T18:40:43.049358+0000 osd.4 (osd.4) 121 : cluster 0 6.f starting backfill to osd.1 from (0'0,0'0] MAX to 170'10603 2023-09-23T18:40:44.168 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[79516]: cluster 2023-09-23T18:40:43.049691+0000 osd.4 (osd.4) 122 : cluster 0 6.6 starting backfill to osd.6 from (0'0,0'0] MAX to 170'10562 2023-09-23T18:40:44.168 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[79516]: cluster 2023-09-23T18:40:43.051260+0000 osd.3 (osd.3) 2 : cluster 0 6.b starting backfill to osd.6 from (0'0,0'0] MAX to 170'10546 2023-09-23T18:40:44.168 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[79516]: cluster 2023-09-23T18:40:43.052158+0000 osd.6 (osd.6) 90 : cluster 3 failed to encode map e173 with expected crc 2023-09-23T18:40:44.168 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[79516]: cluster 2023-09-23T18:40:43.054729+0000 osd.6 (osd.6) 91 : cluster 0 6.c starting backfill to osd.1 from (0'0,0'0] MAX to 170'10520 2023-09-23T18:40:44.168 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[79516]: cluster 2023-09-23T18:40:43.113751+0000 osd.5 (osd.5) 74 : cluster 3 failed to encode map e173 with expected crc 2023-09-23T18:40:44.168 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[79516]: cluster 2023-09-23T18:40:43.899164+0000 mgr.y (mgr.26054) 672 : cluster 0 pgmap v664: 89 pgs: 2 remapped+peering, 2 peering, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 81 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 928/118716 objects degraded (0.782%); 8969/118716 objects misplaced (7.555%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T18:40:44.168 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:44 smithi098 bash[79516]: audit 2023-09-23T18:40:44.000951+0000 mon.a (mon.0) 925 : audit 0 from='client.? 172.21.15.98:0/877167688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:40:44.196 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:44.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:44.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:44.199 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:40:44.240 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:40:44.271 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:40:44.300 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:44.313 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:44 smithi173 bash[59422]: cluster 2023-09-23T18:40:43.043605+0000 mon.a (mon.0) 924 : cluster 0 osdmap e173: 8 total, 8 up, 7 in 2023-09-23T18:40:44.313 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:44 smithi173 bash[59422]: cluster 2023-09-23T18:40:43.043876+0000 osd.1 (osd.1) 2 : cluster 0 6.4 starting backfill to osd.4 from (0'0,0'0] MAX to 170'10780 2023-09-23T18:40:44.313 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:44 smithi173 bash[59422]: cluster 2023-09-23T18:40:43.044723+0000 osd.4 (osd.4) 119 : cluster 3 failed to encode map e173 with expected crc 2023-09-23T18:40:44.313 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:44 smithi173 bash[59422]: cluster 2023-09-23T18:40:43.045102+0000 osd.6 (osd.6) 88 : cluster 3 failed to encode map e173 with expected crc 2023-09-23T18:40:44.313 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:44 smithi173 bash[59422]: cluster 2023-09-23T18:40:43.046398+0000 osd.4 (osd.4) 120 : cluster 3 failed to encode map e173 with expected crc 2023-09-23T18:40:44.313 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:44 smithi173 bash[59422]: cluster 2023-09-23T18:40:43.047947+0000 osd.6 (osd.6) 89 : cluster 3 failed to encode map e173 with expected crc 2023-09-23T18:40:44.313 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:44 smithi173 bash[59422]: cluster 2023-09-23T18:40:43.049358+0000 osd.4 (osd.4) 121 : cluster 0 6.f starting backfill to osd.1 from (0'0,0'0] MAX to 170'10603 2023-09-23T18:40:44.313 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:44 smithi173 bash[59422]: cluster 2023-09-23T18:40:43.049691+0000 osd.4 (osd.4) 122 : cluster 0 6.6 starting backfill to osd.6 from (0'0,0'0] MAX to 170'10562 2023-09-23T18:40:44.314 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:44 smithi173 bash[59422]: cluster 2023-09-23T18:40:43.051260+0000 osd.3 (osd.3) 2 : cluster 0 6.b starting backfill to osd.6 from (0'0,0'0] MAX to 170'10546 2023-09-23T18:40:44.314 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:44 smithi173 bash[59422]: cluster 2023-09-23T18:40:43.052158+0000 osd.6 (osd.6) 90 : cluster 3 failed to encode map e173 with expected crc 2023-09-23T18:40:44.314 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:44 smithi173 bash[59422]: cluster 2023-09-23T18:40:43.054729+0000 osd.6 (osd.6) 91 : cluster 0 6.c starting backfill to osd.1 from (0'0,0'0] MAX to 170'10520 2023-09-23T18:40:44.314 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:44 smithi173 bash[59422]: cluster 2023-09-23T18:40:43.113751+0000 osd.5 (osd.5) 74 : cluster 3 failed to encode map e173 with expected crc 2023-09-23T18:40:44.314 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:44 smithi173 bash[59422]: cluster 2023-09-23T18:40:43.899164+0000 mgr.y (mgr.26054) 672 : cluster 0 pgmap v664: 89 pgs: 2 remapped+peering, 2 peering, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 81 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 928/118716 objects degraded (0.782%); 8969/118716 objects misplaced (7.555%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T18:40:44.314 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:44 smithi173 bash[59422]: audit 2023-09-23T18:40:44.000951+0000 mon.a (mon.0) 925 : audit 0 from='client.? 172.21.15.98:0/877167688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:40:44.317 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:40:44.336 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:44.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:44.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:44.374 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:40:44.380 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:40:44.380 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:40:44.380 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:40:44.381 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:40:44.381 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:40:44.381 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:40:44.381 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:40:44.381 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:40:44.443 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:44.478 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:44.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:44.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:44.582 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:44.618 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:44.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:44.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:44.724 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:44.755 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:44.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:44.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:44.859 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:44.880 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:44.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:44.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:44.985 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:45.029 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:45.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:45.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:45.075 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is set 2023-09-23T18:40:45.128 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:45 smithi098 bash[76817]: cluster 2023-09-23T18:40:44.060650+0000 osd.7 (osd.7) 85 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:45.128 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:45 smithi098 bash[76817]: cluster 2023-09-23T18:40:44.065365+0000 osd.6 (osd.6) 92 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:45.128 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:45 smithi098 bash[76817]: cluster 2023-09-23T18:40:44.066576+0000 osd.7 (osd.7) 86 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:45.128 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:45 smithi098 bash[76817]: cluster 2023-09-23T18:40:44.069936+0000 osd.6 (osd.6) 93 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:45.128 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:45 smithi098 bash[76817]: cluster 2023-09-23T18:40:44.071959+0000 osd.6 (osd.6) 94 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:45.128 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:45 smithi098 bash[76817]: cluster 2023-09-23T18:40:44.074326+0000 osd.6 (osd.6) 95 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:45.128 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:45 smithi098 bash[76817]: cluster 2023-09-23T18:40:44.077814+0000 mon.a (mon.0) 926 : cluster 0 osdmap e174: 8 total, 8 up, 7 in 2023-09-23T18:40:45.128 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:45 smithi098 bash[76817]: cluster 2023-09-23T18:40:44.225250+0000 osd.5 (osd.5) 75 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:45.128 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:45 smithi098 bash[76817]: audit 2023-09-23T18:40:44.567936+0000 mon.c (mon.2) 85 : audit 1 from='client.? 172.21.15.98:0/1052039188' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:40:45.129 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:45 smithi098 bash[76817]: audit 2023-09-23T18:40:44.568812+0000 mon.a (mon.0) 927 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:40:45.129 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:45 smithi098 bash[76817]: cluster 2023-09-23T18:40:44.978178+0000 mon.a (mon.0) 928 : cluster 3 Health check update: Degraded data redundancy: 928/118716 objects degraded (0.782%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:45.129 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:45 smithi098 bash[79516]: cluster 2023-09-23T18:40:44.060650+0000 osd.7 (osd.7) 85 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:45.129 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:45 smithi098 bash[79516]: cluster 2023-09-23T18:40:44.065365+0000 osd.6 (osd.6) 92 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:45.129 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:45 smithi098 bash[79516]: cluster 2023-09-23T18:40:44.066576+0000 osd.7 (osd.7) 86 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:45.129 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:45 smithi098 bash[79516]: cluster 2023-09-23T18:40:44.069936+0000 osd.6 (osd.6) 93 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:45.129 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:45 smithi098 bash[79516]: cluster 2023-09-23T18:40:44.071959+0000 osd.6 (osd.6) 94 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:45.129 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:45 smithi098 bash[79516]: cluster 2023-09-23T18:40:44.074326+0000 osd.6 (osd.6) 95 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:45.129 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:45 smithi098 bash[79516]: cluster 2023-09-23T18:40:44.077814+0000 mon.a (mon.0) 926 : cluster 0 osdmap e174: 8 total, 8 up, 7 in 2023-09-23T18:40:45.130 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:45 smithi098 bash[79516]: cluster 2023-09-23T18:40:44.225250+0000 osd.5 (osd.5) 75 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:45.130 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:45 smithi098 bash[79516]: audit 2023-09-23T18:40:44.567936+0000 mon.c (mon.2) 85 : audit 1 from='client.? 172.21.15.98:0/1052039188' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:40:45.130 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:45 smithi098 bash[79516]: audit 2023-09-23T18:40:44.568812+0000 mon.a (mon.0) 927 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:40:45.130 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:45 smithi098 bash[79516]: cluster 2023-09-23T18:40:44.978178+0000 mon.a (mon.0) 928 : cluster 3 Health check update: Degraded data redundancy: 928/118716 objects degraded (0.782%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:45.135 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:45.168 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:45.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:45.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:45.273 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:45.296 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:45.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:45.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:45.400 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:45.443 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:45 smithi173 bash[59422]: cluster 2023-09-23T18:40:44.060650+0000 osd.7 (osd.7) 85 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:45.443 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:45 smithi173 bash[59422]: cluster 2023-09-23T18:40:44.065365+0000 osd.6 (osd.6) 92 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:45.443 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:45 smithi173 bash[59422]: cluster 2023-09-23T18:40:44.066576+0000 osd.7 (osd.7) 86 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:45.443 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:45 smithi173 bash[59422]: cluster 2023-09-23T18:40:44.069936+0000 osd.6 (osd.6) 93 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:45.443 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:45 smithi173 bash[59422]: cluster 2023-09-23T18:40:44.071959+0000 osd.6 (osd.6) 94 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:45.444 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:45 smithi173 bash[59422]: cluster 2023-09-23T18:40:44.074326+0000 osd.6 (osd.6) 95 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:45.444 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:45 smithi173 bash[59422]: cluster 2023-09-23T18:40:44.077814+0000 mon.a (mon.0) 926 : cluster 0 osdmap e174: 8 total, 8 up, 7 in 2023-09-23T18:40:45.444 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:45 smithi173 bash[59422]: cluster 2023-09-23T18:40:44.225250+0000 osd.5 (osd.5) 75 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:45.444 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:45 smithi173 bash[59422]: audit 2023-09-23T18:40:44.567936+0000 mon.c (mon.2) 85 : audit 1 from='client.? 172.21.15.98:0/1052039188' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:40:45.444 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:45 smithi173 bash[59422]: audit 2023-09-23T18:40:44.568812+0000 mon.a (mon.0) 927 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:40:45.444 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:45 smithi173 bash[59422]: cluster 2023-09-23T18:40:44.978178+0000 mon.a (mon.0) 928 : cluster 3 Health check update: Degraded data redundancy: 928/118716 objects degraded (0.782%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:40:45.444 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:45.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:45.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:45.552 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:45.586 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:45.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:45.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:45.691 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:45.734 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:45.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:45.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:45.841 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:45.869 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:45.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:45.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:45.972 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:45.996 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:45.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:45.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:46.083 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[76817]: cluster 2023-09-23T18:40:44.065358+0000 osd.4 (osd.4) 123 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:46.083 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[76817]: cluster 2023-09-23T18:40:44.067342+0000 osd.4 (osd.4) 124 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:46.084 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[76817]: cluster 2023-09-23T18:40:44.072324+0000 osd.4 (osd.4) 125 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:46.084 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[76817]: cluster 2023-09-23T18:40:45.058746+0000 mon.a (mon.0) 929 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:40:46.084 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[76817]: cluster 2023-09-23T18:40:45.061676+0000 osd.5 (osd.5) 76 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.084 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[76817]: cluster 2023-09-23T18:40:45.064444+0000 osd.6 (osd.6) 96 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.084 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[76817]: audit 2023-09-23T18:40:45.064612+0000 mon.a (mon.0) 930 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:40:46.085 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[76817]: cluster 2023-09-23T18:40:45.064739+0000 mon.a (mon.0) 931 : cluster 0 osdmap e175: 8 total, 8 up, 7 in 2023-09-23T18:40:46.085 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[76817]: cluster 2023-09-23T18:40:45.066886+0000 osd.6 (osd.6) 97 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.085 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[76817]: cluster 2023-09-23T18:40:45.073705+0000 osd.6 (osd.6) 98 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.085 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[76817]: cluster 2023-09-23T18:40:45.077772+0000 osd.6 (osd.6) 99 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.085 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[76817]: cluster 2023-09-23T18:40:45.079310+0000 osd.6 (osd.6) 100 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.085 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[76817]: cluster 2023-09-23T18:40:45.079824+0000 osd.7 (osd.7) 87 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.100 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:46.133 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:46.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:46.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:46.239 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:46.251 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:46 smithi173 bash[59422]: cluster 2023-09-23T18:40:44.065358+0000 osd.4 (osd.4) 123 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:46.252 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:46 smithi173 bash[59422]: cluster 2023-09-23T18:40:44.067342+0000 osd.4 (osd.4) 124 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:46.252 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:46 smithi173 bash[59422]: cluster 2023-09-23T18:40:44.072324+0000 osd.4 (osd.4) 125 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:46.252 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:46 smithi173 bash[59422]: cluster 2023-09-23T18:40:45.058746+0000 mon.a (mon.0) 929 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:40:46.252 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:46 smithi173 bash[59422]: cluster 2023-09-23T18:40:45.061676+0000 osd.5 (osd.5) 76 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.252 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:46 smithi173 bash[59422]: cluster 2023-09-23T18:40:45.064444+0000 osd.6 (osd.6) 96 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.252 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:46 smithi173 bash[59422]: audit 2023-09-23T18:40:45.064612+0000 mon.a (mon.0) 930 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:40:46.253 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:46 smithi173 bash[59422]: cluster 2023-09-23T18:40:45.064739+0000 mon.a (mon.0) 931 : cluster 0 osdmap e175: 8 total, 8 up, 7 in 2023-09-23T18:40:46.253 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:46 smithi173 bash[59422]: cluster 2023-09-23T18:40:45.066886+0000 osd.6 (osd.6) 97 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.253 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:46 smithi173 bash[59422]: cluster 2023-09-23T18:40:45.073705+0000 osd.6 (osd.6) 98 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.253 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:46 smithi173 bash[59422]: cluster 2023-09-23T18:40:45.077772+0000 osd.6 (osd.6) 99 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.253 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:46 smithi173 bash[59422]: cluster 2023-09-23T18:40:45.079310+0000 osd.6 (osd.6) 100 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.253 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:46 smithi173 bash[59422]: cluster 2023-09-23T18:40:45.079824+0000 osd.7 (osd.7) 87 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.253 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:46 smithi173 bash[59422]: cluster 2023-09-23T18:40:45.081668+0000 osd.4 (osd.4) 126 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.253 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:46 smithi173 bash[59422]: cluster 2023-09-23T18:40:45.082120+0000 osd.7 (osd.7) 88 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.254 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:46 smithi173 bash[59422]: cluster 2023-09-23T18:40:45.084253+0000 osd.7 (osd.7) 89 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.254 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:46 smithi173 bash[59422]: cluster 2023-09-23T18:40:45.087631+0000 osd.4 (osd.4) 127 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.254 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:46 smithi173 bash[59422]: cluster 2023-09-23T18:40:45.899768+0000 mgr.y (mgr.26054) 673 : cluster 0 pgmap v667: 89 pgs: 2 remapped+peering, 3 peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 2.4 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 397 op/s; 928/115959 objects degraded (0.800%); 10477/115959 objects misplaced (9.035%); 9.1 MiB/s, 151 objects/s recovering 2023-09-23T18:40:46.267 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:46.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:46.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:46.371 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:46.404 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:46.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:46.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:46.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[76817]: cluster 2023-09-23T18:40:45.081668+0000 osd.4 (osd.4) 126 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[76817]: cluster 2023-09-23T18:40:45.082120+0000 osd.7 (osd.7) 88 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[76817]: cluster 2023-09-23T18:40:45.084253+0000 osd.7 (osd.7) 89 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[76817]: cluster 2023-09-23T18:40:45.087631+0000 osd.4 (osd.4) 127 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[76817]: cluster 2023-09-23T18:40:45.899768+0000 mgr.y (mgr.26054) 673 : cluster 0 pgmap v667: 89 pgs: 2 remapped+peering, 3 peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 2.4 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 397 op/s; 928/115959 objects degraded (0.800%); 10477/115959 objects misplaced (9.035%); 9.1 MiB/s, 151 objects/s recovering 2023-09-23T18:40:46.442 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:40:46] "GET /metrics HTTP/1.1" 200 36566 "" "Prometheus/2.33.4" 2023-09-23T18:40:46.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[79516]: cluster 2023-09-23T18:40:44.065358+0000 osd.4 (osd.4) 123 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:46.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[79516]: cluster 2023-09-23T18:40:44.067342+0000 osd.4 (osd.4) 124 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:46.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[79516]: cluster 2023-09-23T18:40:44.072324+0000 osd.4 (osd.4) 125 : cluster 3 failed to encode map e174 with expected crc 2023-09-23T18:40:46.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[79516]: cluster 2023-09-23T18:40:45.058746+0000 mon.a (mon.0) 929 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:40:46.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[79516]: cluster 2023-09-23T18:40:45.061676+0000 osd.5 (osd.5) 76 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[79516]: cluster 2023-09-23T18:40:45.064444+0000 osd.6 (osd.6) 96 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[79516]: audit 2023-09-23T18:40:45.064612+0000 mon.a (mon.0) 930 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:40:46.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[79516]: cluster 2023-09-23T18:40:45.064739+0000 mon.a (mon.0) 931 : cluster 0 osdmap e175: 8 total, 8 up, 7 in 2023-09-23T18:40:46.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[79516]: cluster 2023-09-23T18:40:45.066886+0000 osd.6 (osd.6) 97 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[79516]: cluster 2023-09-23T18:40:45.073705+0000 osd.6 (osd.6) 98 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[79516]: cluster 2023-09-23T18:40:45.077772+0000 osd.6 (osd.6) 99 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[79516]: cluster 2023-09-23T18:40:45.079310+0000 osd.6 (osd.6) 100 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[79516]: cluster 2023-09-23T18:40:45.079824+0000 osd.7 (osd.7) 87 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[79516]: cluster 2023-09-23T18:40:45.081668+0000 osd.4 (osd.4) 126 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[79516]: cluster 2023-09-23T18:40:45.082120+0000 osd.7 (osd.7) 88 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[79516]: cluster 2023-09-23T18:40:45.084253+0000 osd.7 (osd.7) 89 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[79516]: cluster 2023-09-23T18:40:45.087631+0000 osd.4 (osd.4) 127 : cluster 3 failed to encode map e175 with expected crc 2023-09-23T18:40:46.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:46 smithi098 bash[79516]: cluster 2023-09-23T18:40:45.899768+0000 mgr.y (mgr.26054) 673 : cluster 0 pgmap v667: 89 pgs: 2 remapped+peering, 3 peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 2.4 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 397 op/s; 928/115959 objects degraded (0.800%); 10477/115959 objects misplaced (9.035%); 9.1 MiB/s, 151 objects/s recovering 2023-09-23T18:40:46.508 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:46.549 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:46.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:46.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:46.656 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:46.705 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:46.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:46.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:46.810 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:46.848 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:46.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:46.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:46.952 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:46.990 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:46.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:46.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:47.083 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:47 smithi173 bash[59422]: audit 2023-09-23T18:40:46.442486+0000 mon.a (mon.0) 932 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:40:47.094 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:47.125 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:47.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:47.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:47.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:47 smithi098 bash[79516]: audit 2023-09-23T18:40:46.442486+0000 mon.a (mon.0) 932 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:40:47.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:47 smithi098 bash[76817]: audit 2023-09-23T18:40:46.442486+0000 mon.a (mon.0) 932 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:40:47.231 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:47.260 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:47.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:47.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:47.365 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:47.392 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:47.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:47.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:47.495 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:47.530 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:47.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:47.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:47.588 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T18:40:47.589 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:40:47.633 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:47.664 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:47.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:47.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:47.770 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:47.816 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:47.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:47.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:47.921 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:47.948 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:47.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:47.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:48.052 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:48.085 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:48.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:48.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:48.092 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:48 smithi173 bash[59422]: cluster 2023-09-23T18:40:47.900445+0000 mgr.y (mgr.26054) 674 : cluster 0 pgmap v668: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 peering, 6 active+remapped+backfill_wait, 80 active+clean; 2.3 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 18890/111813 objects misplaced (16.894%); 14 MiB/s, 228 objects/s recovering 2023-09-23T18:40:48.192 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:48.227 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:48.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:48.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:48.332 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:48.352 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:48 smithi098 bash[76817]: cluster 2023-09-23T18:40:47.900445+0000 mgr.y (mgr.26054) 674 : cluster 0 pgmap v668: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 peering, 6 active+remapped+backfill_wait, 80 active+clean; 2.3 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 18890/111813 objects misplaced (16.894%); 14 MiB/s, 228 objects/s recovering 2023-09-23T18:40:48.353 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:48 smithi098 bash[79516]: cluster 2023-09-23T18:40:47.900445+0000 mgr.y (mgr.26054) 674 : cluster 0 pgmap v668: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 peering, 6 active+remapped+backfill_wait, 80 active+clean; 2.3 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 18890/111813 objects misplaced (16.894%); 14 MiB/s, 228 objects/s recovering 2023-09-23T18:40:48.372 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:48.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:48.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:48.483 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:48.517 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:48.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:48.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:48.623 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:48.649 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:40:48.664 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:48.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:48.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:48.768 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:48.807 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T18:40:48.820 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:48.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:48.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:48.924 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:48.967 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:48.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:48.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:49.070 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:49.103 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:49.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:49.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:49.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:49 smithi098 bash[79516]: cluster 2023-09-23T18:40:48.081385+0000 mon.a (mon.0) 933 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 928/115959 objects degraded (0.800%), 1 pg degraded) 2023-09-23T18:40:49.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:49 smithi098 bash[76817]: cluster 2023-09-23T18:40:48.081385+0000 mon.a (mon.0) 933 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 928/115959 objects degraded (0.800%), 1 pg degraded) 2023-09-23T18:40:49.206 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:49.238 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:49.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:49.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:49.349 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:49.364 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:40:49.378 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:49.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:49.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:49.486 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:49.496 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:49 smithi173 bash[59422]: cluster 2023-09-23T18:40:48.081385+0000 mon.a (mon.0) 933 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 928/115959 objects degraded (0.800%), 1 pg degraded) 2023-09-23T18:40:49.513 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:49.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:49.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:49.619 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:49.643 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:49.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:49.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:49.746 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:49.777 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:49.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:49.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:49.884 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:49.918 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:49.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:49.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:50.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:50.052 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:50.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:50.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:50.155 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:50.188 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:50.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:50.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:50.292 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:50.323 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:50.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:50.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:50.432 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:50.460 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:50.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:50.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:50.564 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:50.604 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:50.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:50.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:50.708 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:50.732 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:50.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:50.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:50.839 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:50.865 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:50.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:50.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:50.891 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:40:50.972 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:50.999 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:51.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:51.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:51.105 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:51.129 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:51.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:51.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:51.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:50 smithi098 bash[79516]: cluster 2023-09-23T18:40:49.907268+0000 mgr.y (mgr.26054) 675 : cluster 0 pgmap v669: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 peering, 6 active+remapped+backfill_wait, 80 active+clean; 2.3 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 335 op/s; 18890/111813 objects misplaced (16.894%); 12 MiB/s, 195 objects/s recovering 2023-09-23T18:40:51.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:50 smithi098 bash[76817]: cluster 2023-09-23T18:40:49.907268+0000 mgr.y (mgr.26054) 675 : cluster 0 pgmap v669: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 peering, 6 active+remapped+backfill_wait, 80 active+clean; 2.3 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 335 op/s; 18890/111813 objects misplaced (16.894%); 12 MiB/s, 195 objects/s recovering 2023-09-23T18:40:51.233 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:51.242 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:50 smithi173 bash[59422]: cluster 2023-09-23T18:40:49.907268+0000 mgr.y (mgr.26054) 675 : cluster 0 pgmap v669: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 peering, 6 active+remapped+backfill_wait, 80 active+clean; 2.3 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 335 op/s; 18890/111813 objects misplaced (16.894%); 12 MiB/s, 195 objects/s recovering 2023-09-23T18:40:51.260 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:51.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:51.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:51.363 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:51.401 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:51.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:51.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:51.508 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:51.539 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:51.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:51.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:51.646 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:51.677 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:51.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:51.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:51.788 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:51.816 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:51.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:51.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:51.922 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:51.964 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:51.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:51.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:52.067 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:52.132 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:52.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:52.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:52.238 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:52.260 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:52.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:52.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:52.362 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:52.400 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:52.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:52.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:52.505 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:52.547 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:52.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:52.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:52.651 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:52.677 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:52.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:52.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:52.782 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:52.817 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:52.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:52.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:52.921 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:52.933 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:40:52.949 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:52.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:52.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:53.054 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:53.067 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:52 smithi173 bash[59422]: cluster 2023-09-23T18:40:51.907929+0000 mgr.y (mgr.26054) 676 : cluster 0 pgmap v670: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 81 active+clean; 2.1 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 17595/104850 objects misplaced (16.781%); 12 MiB/s, 192 objects/s recovering 2023-09-23T18:40:53.099 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:53.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:53.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:53.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:52 smithi098 bash[76817]: cluster 2023-09-23T18:40:51.907929+0000 mgr.y (mgr.26054) 676 : cluster 0 pgmap v670: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 81 active+clean; 2.1 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 17595/104850 objects misplaced (16.781%); 12 MiB/s, 192 objects/s recovering 2023-09-23T18:40:53.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:52 smithi098 bash[79516]: cluster 2023-09-23T18:40:51.907929+0000 mgr.y (mgr.26054) 676 : cluster 0 pgmap v670: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 81 active+clean; 2.1 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 17595/104850 objects misplaced (16.781%); 12 MiB/s, 192 objects/s recovering 2023-09-23T18:40:53.202 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:53.232 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:53.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:53.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:53.293 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:40:53.336 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:53.361 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:53.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:53.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:53.464 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:53.484 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:53.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:53.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:53.587 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:53.630 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:53.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:53.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:53.736 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:53.778 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:53.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:53.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:53.882 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:53.917 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:53.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:53.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:53.950 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:40:54.021 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:54.054 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:54.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:54.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:54.164 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:54.200 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:54.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:54.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:54.306 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:54.337 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:54.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:54.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:54.440 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:54.450 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:54 smithi173 bash[59422]: audit 2023-09-23T18:40:53.311358+0000 mon.a (mon.0) 934 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:40:54.450 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:54 smithi173 bash[59422]: audit 2023-09-23T18:40:53.325635+0000 mon.a (mon.0) 935 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:40:54.450 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:54 smithi173 bash[59422]: audit 2023-09-23T18:40:53.767230+0000 mon.a (mon.0) 936 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:40:54.450 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:54 smithi173 bash[59422]: audit 2023-09-23T18:40:53.786141+0000 mon.a (mon.0) 937 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:40:54.450 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:54 smithi173 bash[59422]: cluster 2023-09-23T18:40:53.908481+0000 mgr.y (mgr.26054) 677 : cluster 0 pgmap v671: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 81 active+clean; 2.1 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 16729/101634 objects misplaced (16.460%); 13 MiB/s, 207 objects/s recovering 2023-09-23T18:40:54.473 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:54.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:54.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:54.576 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:54.600 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:54.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:54.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:54.690 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:54 smithi098 bash[79516]: audit 2023-09-23T18:40:53.311358+0000 mon.a (mon.0) 934 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:40:54.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:54 smithi098 bash[79516]: audit 2023-09-23T18:40:53.325635+0000 mon.a (mon.0) 935 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:40:54.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:54 smithi098 bash[79516]: audit 2023-09-23T18:40:53.767230+0000 mon.a (mon.0) 936 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:40:54.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:54 smithi098 bash[79516]: audit 2023-09-23T18:40:53.786141+0000 mon.a (mon.0) 937 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:40:54.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:54 smithi098 bash[79516]: cluster 2023-09-23T18:40:53.908481+0000 mgr.y (mgr.26054) 677 : cluster 0 pgmap v671: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 81 active+clean; 2.1 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 16729/101634 objects misplaced (16.460%); 13 MiB/s, 207 objects/s recovering 2023-09-23T18:40:54.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:54 smithi098 bash[76817]: audit 2023-09-23T18:40:53.311358+0000 mon.a (mon.0) 934 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:40:54.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:54 smithi098 bash[76817]: audit 2023-09-23T18:40:53.325635+0000 mon.a (mon.0) 935 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:40:54.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:54 smithi098 bash[76817]: audit 2023-09-23T18:40:53.767230+0000 mon.a (mon.0) 936 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:40:54.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:54 smithi098 bash[76817]: audit 2023-09-23T18:40:53.786141+0000 mon.a (mon.0) 937 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:40:54.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:54 smithi098 bash[76817]: cluster 2023-09-23T18:40:53.908481+0000 mgr.y (mgr.26054) 677 : cluster 0 pgmap v671: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 81 active+clean; 2.1 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 16729/101634 objects misplaced (16.460%); 13 MiB/s, 207 objects/s recovering 2023-09-23T18:40:54.703 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:54.726 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:54.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:54.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:54.830 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:54.861 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:54.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:54.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:54.966 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:54.996 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:55.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:55.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:55.101 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:55.126 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:55.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:55.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:55.229 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:55.263 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:55.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:55.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:55.365 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:55.374 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:55 smithi098 bash[76817]: audit 2023-09-23T18:40:54.498462+0000 mon.a (mon.0) 938 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:40:55.374 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:55 smithi098 bash[76817]: audit 2023-09-23T18:40:54.499482+0000 mon.a (mon.0) 939 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:40:55.374 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:55 smithi098 bash[76817]: audit 2023-09-23T18:40:54.509471+0000 mon.a (mon.0) 940 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:40:55.374 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:55 smithi098 bash[76817]: audit 2023-09-23T18:40:54.607841+0000 mon.a (mon.0) 941 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:40:55.375 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:55 smithi098 bash[76817]: audit 2023-09-23T18:40:54.610440+0000 mon.a (mon.0) 942 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:40:55.375 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:55 smithi098 bash[79516]: audit 2023-09-23T18:40:54.498462+0000 mon.a (mon.0) 938 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:40:55.375 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:55 smithi098 bash[79516]: audit 2023-09-23T18:40:54.499482+0000 mon.a (mon.0) 939 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:40:55.375 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:55 smithi098 bash[79516]: audit 2023-09-23T18:40:54.509471+0000 mon.a (mon.0) 940 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:40:55.375 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:55 smithi098 bash[79516]: audit 2023-09-23T18:40:54.607841+0000 mon.a (mon.0) 941 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:40:55.376 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:55 smithi098 bash[79516]: audit 2023-09-23T18:40:54.610440+0000 mon.a (mon.0) 942 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:40:55.394 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:55.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:55.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:55.504 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:55.541 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:55.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:55.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:55.649 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:55.658 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:55 smithi173 bash[59422]: audit 2023-09-23T18:40:54.498462+0000 mon.a (mon.0) 938 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:40:55.658 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:55 smithi173 bash[59422]: audit 2023-09-23T18:40:54.499482+0000 mon.a (mon.0) 939 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:40:55.658 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:55 smithi173 bash[59422]: audit 2023-09-23T18:40:54.509471+0000 mon.a (mon.0) 940 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:40:55.658 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:55 smithi173 bash[59422]: audit 2023-09-23T18:40:54.607841+0000 mon.a (mon.0) 941 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:40:55.658 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:55 smithi173 bash[59422]: audit 2023-09-23T18:40:54.610440+0000 mon.a (mon.0) 942 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:40:55.677 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:55.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:55.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:55.784 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:55.820 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:55.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:55.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:55.925 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:55.954 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:55.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:55.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:56.057 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:56.080 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:56.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:56.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:56.186 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:56.208 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:56.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:56.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:56.311 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:56.325 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:40:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:40:56] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.33.4" 2023-09-23T18:40:56.336 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:56.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:56.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:56.440 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:56.466 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:56.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:56.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:56.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:56 smithi173 bash[59422]: cluster 2023-09-23T18:40:55.909146+0000 mgr.y (mgr.26054) 678 : cluster 0 pgmap v672: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 81 active+clean; 2.0 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 16235/97350 objects misplaced (16.677%); 8.4 MiB/s, 136 objects/s recovering 2023-09-23T18:40:56.573 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:56.581 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:56 smithi098 bash[79516]: cluster 2023-09-23T18:40:55.909146+0000 mgr.y (mgr.26054) 678 : cluster 0 pgmap v672: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 81 active+clean; 2.0 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 16235/97350 objects misplaced (16.677%); 8.4 MiB/s, 136 objects/s recovering 2023-09-23T18:40:56.581 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:56 smithi098 bash[76817]: cluster 2023-09-23T18:40:55.909146+0000 mgr.y (mgr.26054) 678 : cluster 0 pgmap v672: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 81 active+clean; 2.0 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 16235/97350 objects misplaced (16.677%); 8.4 MiB/s, 136 objects/s recovering 2023-09-23T18:40:56.602 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:56.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:56.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:56.705 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:56.730 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:56.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:56.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:56.833 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:56.865 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:56.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:56.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:56.972 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:57.007 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:57.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:57.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:57.110 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:57.139 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:57.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:57.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:57.245 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:57.278 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:57.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:57.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:57.383 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:57.410 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:57.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:57.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:57.513 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:57.534 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:57.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:57.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:57.636 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:57.663 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:57.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:57.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:57.766 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:57.794 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:57.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:57.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:57.898 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:57.932 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:40:57.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:57.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:40:58.036 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:58.053 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:58.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:58.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:58.156 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:58.189 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:58.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:58.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:58.296 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:58.321 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:40:58.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:58.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:40:58.424 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:58.460 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:40:58.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:58.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:40:58.566 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:58.600 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:58.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:58.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:58.706 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:58.736 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:58.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:58.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:58.839 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:58.863 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:58.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:58.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:58.966 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:40:58 smithi098 bash[76817]: cluster 2023-09-23T18:40:57.909730+0000 mgr.y (mgr.26054) 679 : cluster 0 pgmap v673: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 81 active+clean; 1.9 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 668 op/s; 14701/91869 objects misplaced (16.002%); 11 MiB/s, 173 objects/s recovering 2023-09-23T18:40:58.966 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:58.993 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:58.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:58.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:59.097 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:59.120 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:59.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:59.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:59.226 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:59.234 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:40:58 smithi098 bash[79516]: cluster 2023-09-23T18:40:57.909730+0000 mgr.y (mgr.26054) 679 : cluster 0 pgmap v673: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 81 active+clean; 1.9 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 668 op/s; 14701/91869 objects misplaced (16.002%); 11 MiB/s, 173 objects/s recovering 2023-09-23T18:40:59.257 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:40:59.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:59.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:40:59.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:40:58 smithi173 bash[59422]: cluster 2023-09-23T18:40:57.909730+0000 mgr.y (mgr.26054) 679 : cluster 0 pgmap v673: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 81 active+clean; 1.9 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 668 op/s; 14701/91869 objects misplaced (16.002%); 11 MiB/s, 173 objects/s recovering 2023-09-23T18:40:59.360 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:59.379 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:40:59.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:59.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:40:59.483 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:59.509 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:59.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:59.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:40:59.613 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:59.648 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:40:59.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:59.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:40:59.752 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:59.784 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:40:59.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:59.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:40:59.889 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:59.921 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:40:59.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:40:59.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:00.025 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:00.054 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:00.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:00.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:00.156 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:00.187 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:00.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:00.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:00.294 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:00.305 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T18:41:00.305 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:41:00.320 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:00.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:00.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:00.424 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:00.445 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:00.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:00.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:00.548 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:00.572 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:00.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:00.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:00.677 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:00.710 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:00.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:00.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:00.813 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:00.847 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:00.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:00.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:00.951 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:00.983 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:00.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:00.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:01.088 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:01.096 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:01 smithi098 bash[79516]: cluster 2023-09-23T18:40:59.910217+0000 mgr.y (mgr.26054) 680 : cluster 0 pgmap v674: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 81 active+clean; 1.9 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 553 op/s; 14701/91869 objects misplaced (16.002%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T18:41:01.096 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:00 smithi098 bash[76817]: cluster 2023-09-23T18:40:59.910217+0000 mgr.y (mgr.26054) 680 : cluster 0 pgmap v674: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 81 active+clean; 1.9 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 553 op/s; 14701/91869 objects misplaced (16.002%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T18:41:01.111 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:01.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:01.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:01.218 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:01.241 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:01.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:01.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:01.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:00 smithi173 bash[59422]: cluster 2023-09-23T18:40:59.910217+0000 mgr.y (mgr.26054) 680 : cluster 0 pgmap v674: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 81 active+clean; 1.9 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 553 op/s; 14701/91869 objects misplaced (16.002%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T18:41:01.344 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:01.372 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:01.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:01.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:01.477 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:01.508 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:01.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:01.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:01.608 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T18:41:01.614 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:01.690 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:01.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:01.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:01.793 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:01.828 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:01.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:01.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:01.932 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:01.960 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:01.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:01.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:02.066 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:02.093 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:02.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:02.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:02.197 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:02.227 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:02.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:02.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:02.332 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:02.363 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:02.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:02.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:02.467 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:02.503 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:02.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:02.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:02.608 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:02.628 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:02.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:02.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:02.731 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:02.760 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:02.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:02.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:02.866 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:02.897 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:02.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:02.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:02.993 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:02 smithi098 bash[76817]: cluster 2023-09-23T18:41:01.911395+0000 mgr.y (mgr.26054) 681 : cluster 0 pgmap v675: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 81 active+clean; 1.7 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 739 op/s; 13423/85167 objects misplaced (15.761%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T18:41:03.001 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:03.037 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:03.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:03.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:03.140 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:03.177 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:03.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:03.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:03.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:02 smithi173 bash[59422]: cluster 2023-09-23T18:41:01.911395+0000 mgr.y (mgr.26054) 681 : cluster 0 pgmap v675: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 81 active+clean; 1.7 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 739 op/s; 13423/85167 objects misplaced (15.761%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T18:41:03.281 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:03.317 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:03.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:03.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:03.421 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:03.427 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:02 smithi098 bash[79516]: cluster 2023-09-23T18:41:01.911395+0000 mgr.y (mgr.26054) 681 : cluster 0 pgmap v675: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 81 active+clean; 1.7 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 739 op/s; 13423/85167 objects misplaced (15.761%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T18:41:03.453 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:03.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:03.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:03.558 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:03.596 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:03.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:03.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:03.700 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:03.725 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:03.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:03.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:03.828 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:03.858 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:03.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:03.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:03.961 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:03.987 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:03.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:03.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:04.091 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:04.115 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:04.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:04.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:04.219 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:04.242 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:04.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:04.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:04.346 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:04.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:04.378 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:04.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:04.479 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:04.507 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:04.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:04.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:04.611 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:04.651 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:04.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:04.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:04.756 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:04.790 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:04.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:04.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:04.895 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:04.925 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:04.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:04.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:05.022 INFO:teuthology.orchestra.run.smithi098.stdout:moved 0 / 267 (0%) 2023-09-23T18:41:05.023 INFO:teuthology.orchestra.run.smithi098.stdout:avg 38.1429 2023-09-23T18:41:05.023 INFO:teuthology.orchestra.run.smithi098.stdout:stddev 11.4446 -> 11.4446 (expected baseline 5.71786) 2023-09-23T18:41:05.023 INFO:teuthology.orchestra.run.smithi098.stdout:min osd.2 with 21 -> 21 pgs (0.550562 -> 0.550562 * mean) 2023-09-23T18:41:05.023 INFO:teuthology.orchestra.run.smithi098.stdout:max osd.1 with 54 -> 54 pgs (1.41573 -> 1.41573 * mean) 2023-09-23T18:41:05.024 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:41:05.024 INFO:teuthology.orchestra.run.smithi098.stdout:oload 1000 2023-09-23T18:41:05.024 INFO:teuthology.orchestra.run.smithi098.stdout:max_change 3 2023-09-23T18:41:05.024 INFO:teuthology.orchestra.run.smithi098.stdout:max_change_osds 4 2023-09-23T18:41:05.024 INFO:teuthology.orchestra.run.smithi098.stdout:average_utilization 0.0124 2023-09-23T18:41:05.024 INFO:teuthology.orchestra.run.smithi098.stdout:overload_utilization 0.1241 2023-09-23T18:41:05.031 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:05.032 INFO:teuthology.orchestra.run.smithi098.stderr:no change 2023-09-23T18:41:05.063 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:05.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:05.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:05.068 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:05 smithi098 bash[76817]: cluster 2023-09-23T18:41:03.911950+0000 mgr.y (mgr.26054) 682 : cluster 0 pgmap v676: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 81 active+clean; 1.7 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 12693/82074 objects misplaced (15.465%); 9.6 MiB/s, 153 objects/s recovering 2023-09-23T18:41:05.068 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:05 smithi098 bash[79516]: cluster 2023-09-23T18:41:03.911950+0000 mgr.y (mgr.26054) 682 : cluster 0 pgmap v676: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 81 active+clean; 1.7 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 12693/82074 objects misplaced (15.465%); 9.6 MiB/s, 153 objects/s recovering 2023-09-23T18:41:05.168 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:05.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:05.199 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:05.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:05.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:05 smithi173 bash[59422]: cluster 2023-09-23T18:41:03.911950+0000 mgr.y (mgr.26054) 682 : cluster 0 pgmap v676: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 81 active+clean; 1.7 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 12693/82074 objects misplaced (15.465%); 9.6 MiB/s, 153 objects/s recovering 2023-09-23T18:41:05.300 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:05.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:05.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:05.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:05.434 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:05.460 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:05.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:05.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:05.563 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:05.591 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:05.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:05.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:05.695 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:05.725 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:05.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:05.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:05.829 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:05.865 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:05.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:05.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:05.970 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:05.996 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:05.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:05.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:06.098 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:06.113 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:06 smithi098 bash[79516]: cluster 2023-09-23T18:41:04.987674+0000 osd.5 (osd.5) 77 : cluster 3 failed to encode map e176 with expected crc 2023-09-23T18:41:06.113 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:06 smithi098 bash[79516]: cluster 2023-09-23T18:41:04.988554+0000 mon.a (mon.0) 943 : cluster 0 osdmap e176: 8 total, 8 up, 7 in 2023-09-23T18:41:06.113 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:06 smithi098 bash[79516]: cluster 2023-09-23T18:41:04.990073+0000 osd.4 (osd.4) 128 : cluster 3 failed to encode map e176 with expected crc 2023-09-23T18:41:06.113 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:06 smithi098 bash[79516]: cluster 2023-09-23T18:41:04.997391+0000 osd.6 (osd.6) 101 : cluster 3 failed to encode map e176 with expected crc 2023-09-23T18:41:06.114 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:06 smithi098 bash[79516]: cluster 2023-09-23T18:41:04.999600+0000 osd.7 (osd.7) 90 : cluster 3 failed to encode map e176 with expected crc 2023-09-23T18:41:06.114 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:06 smithi098 bash[79516]: cluster 2023-09-23T18:41:05.001859+0000 osd.6 (osd.6) 102 : cluster 3 failed to encode map e176 with expected crc 2023-09-23T18:41:06.114 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:06 smithi098 bash[79516]: audit 2023-09-23T18:41:05.020800+0000 mgr.y (mgr.26054) 683 : audit 0 from='client.35136 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:41:06.114 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:06 smithi098 bash[76817]: cluster 2023-09-23T18:41:04.987674+0000 osd.5 (osd.5) 77 : cluster 3 failed to encode map e176 with expected crc 2023-09-23T18:41:06.114 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:06 smithi098 bash[76817]: cluster 2023-09-23T18:41:04.988554+0000 mon.a (mon.0) 943 : cluster 0 osdmap e176: 8 total, 8 up, 7 in 2023-09-23T18:41:06.114 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:06 smithi098 bash[76817]: cluster 2023-09-23T18:41:04.990073+0000 osd.4 (osd.4) 128 : cluster 3 failed to encode map e176 with expected crc 2023-09-23T18:41:06.114 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:06 smithi098 bash[76817]: cluster 2023-09-23T18:41:04.997391+0000 osd.6 (osd.6) 101 : cluster 3 failed to encode map e176 with expected crc 2023-09-23T18:41:06.114 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:06 smithi098 bash[76817]: cluster 2023-09-23T18:41:04.999600+0000 osd.7 (osd.7) 90 : cluster 3 failed to encode map e176 with expected crc 2023-09-23T18:41:06.114 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:06 smithi098 bash[76817]: cluster 2023-09-23T18:41:05.001859+0000 osd.6 (osd.6) 102 : cluster 3 failed to encode map e176 with expected crc 2023-09-23T18:41:06.115 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:06 smithi098 bash[76817]: audit 2023-09-23T18:41:05.020800+0000 mgr.y (mgr.26054) 683 : audit 0 from='client.35136 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:41:06.115 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:41:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:41:06] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.33.4" 2023-09-23T18:41:06.142 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:06.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:06.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:06.249 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:06.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:06 smithi173 bash[59422]: cluster 2023-09-23T18:41:04.987674+0000 osd.5 (osd.5) 77 : cluster 3 failed to encode map e176 with expected crc 2023-09-23T18:41:06.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:06 smithi173 bash[59422]: cluster 2023-09-23T18:41:04.988554+0000 mon.a (mon.0) 943 : cluster 0 osdmap e176: 8 total, 8 up, 7 in 2023-09-23T18:41:06.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:06 smithi173 bash[59422]: cluster 2023-09-23T18:41:04.990073+0000 osd.4 (osd.4) 128 : cluster 3 failed to encode map e176 with expected crc 2023-09-23T18:41:06.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:06 smithi173 bash[59422]: cluster 2023-09-23T18:41:04.997391+0000 osd.6 (osd.6) 101 : cluster 3 failed to encode map e176 with expected crc 2023-09-23T18:41:06.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:06 smithi173 bash[59422]: cluster 2023-09-23T18:41:04.999600+0000 osd.7 (osd.7) 90 : cluster 3 failed to encode map e176 with expected crc 2023-09-23T18:41:06.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:06 smithi173 bash[59422]: cluster 2023-09-23T18:41:05.001859+0000 osd.6 (osd.6) 102 : cluster 3 failed to encode map e176 with expected crc 2023-09-23T18:41:06.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:06 smithi173 bash[59422]: audit 2023-09-23T18:41:05.020800+0000 mgr.y (mgr.26054) 683 : audit 0 from='client.35136 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:41:06.280 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:06.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:06.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:06.388 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:06.424 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:06.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:06.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:06.433 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:41:06.527 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:06.551 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:06.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:06.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:06.655 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:06.688 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:06.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:06.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:06.791 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:06.819 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:06.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:06.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:06.922 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:06.950 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:06.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:06.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:07.053 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:07.091 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:07.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:07.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:07.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:07 smithi098 bash[79516]: cluster 2023-09-23T18:41:05.912506+0000 mgr.y (mgr.26054) 684 : cluster 0 pgmap v678: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 81 active+clean; 1.6 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 12225/77991 objects misplaced (15.675%); 8.4 MiB/s, 133 objects/s recovering 2023-09-23T18:41:07.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:07 smithi098 bash[79516]: cluster 2023-09-23T18:41:05.999747+0000 osd.5 (osd.5) 78 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T18:41:07.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:07 smithi098 bash[79516]: cluster 2023-09-23T18:41:06.001459+0000 osd.6 (osd.6) 103 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T18:41:07.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:07 smithi098 bash[79516]: cluster 2023-09-23T18:41:06.001462+0000 osd.4 (osd.4) 129 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T18:41:07.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:07 smithi098 bash[79516]: cluster 2023-09-23T18:41:06.002747+0000 mon.a (mon.0) 944 : cluster 0 osdmap e177: 8 total, 8 up, 7 in 2023-09-23T18:41:07.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:07 smithi098 bash[79516]: cluster 2023-09-23T18:41:06.006138+0000 osd.6 (osd.6) 104 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T18:41:07.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:07 smithi098 bash[79516]: cluster 2023-09-23T18:41:06.060572+0000 osd.7 (osd.7) 91 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T18:41:07.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:07 smithi098 bash[76817]: cluster 2023-09-23T18:41:05.912506+0000 mgr.y (mgr.26054) 684 : cluster 0 pgmap v678: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 81 active+clean; 1.6 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 12225/77991 objects misplaced (15.675%); 8.4 MiB/s, 133 objects/s recovering 2023-09-23T18:41:07.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:07 smithi098 bash[76817]: cluster 2023-09-23T18:41:05.999747+0000 osd.5 (osd.5) 78 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T18:41:07.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:07 smithi098 bash[76817]: cluster 2023-09-23T18:41:06.001459+0000 osd.6 (osd.6) 103 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T18:41:07.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:07 smithi098 bash[76817]: cluster 2023-09-23T18:41:06.001462+0000 osd.4 (osd.4) 129 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T18:41:07.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:07 smithi098 bash[76817]: cluster 2023-09-23T18:41:06.002747+0000 mon.a (mon.0) 944 : cluster 0 osdmap e177: 8 total, 8 up, 7 in 2023-09-23T18:41:07.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:07 smithi098 bash[76817]: cluster 2023-09-23T18:41:06.006138+0000 osd.6 (osd.6) 104 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T18:41:07.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:07 smithi098 bash[76817]: cluster 2023-09-23T18:41:06.060572+0000 osd.7 (osd.7) 91 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T18:41:07.198 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:07.207 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:07 smithi173 bash[59422]: cluster 2023-09-23T18:41:05.912506+0000 mgr.y (mgr.26054) 684 : cluster 0 pgmap v678: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 81 active+clean; 1.6 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 12225/77991 objects misplaced (15.675%); 8.4 MiB/s, 133 objects/s recovering 2023-09-23T18:41:07.209 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:07 smithi173 bash[59422]: cluster 2023-09-23T18:41:05.999747+0000 osd.5 (osd.5) 78 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T18:41:07.209 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:07 smithi173 bash[59422]: cluster 2023-09-23T18:41:06.001459+0000 osd.6 (osd.6) 103 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T18:41:07.209 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:07 smithi173 bash[59422]: cluster 2023-09-23T18:41:06.001462+0000 osd.4 (osd.4) 129 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T18:41:07.209 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:07 smithi173 bash[59422]: cluster 2023-09-23T18:41:06.002747+0000 mon.a (mon.0) 944 : cluster 0 osdmap e177: 8 total, 8 up, 7 in 2023-09-23T18:41:07.209 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:07 smithi173 bash[59422]: cluster 2023-09-23T18:41:06.006138+0000 osd.6 (osd.6) 104 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T18:41:07.209 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:07 smithi173 bash[59422]: cluster 2023-09-23T18:41:06.060572+0000 osd.7 (osd.7) 91 : cluster 3 failed to encode map e177 with expected crc 2023-09-23T18:41:07.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:07.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:07.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:07.267 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:41:07.310 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:41:07.326 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:07.358 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:41:07.368 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:07.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:07.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:07.438 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:41:07.474 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:07.477 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:41:07.514 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:07.517 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:41:07.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:07.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:07.553 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:41:07.597 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:41:07.604 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:41:07.604 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:41:07.604 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:41:07.604 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:41:07.604 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:41:07.605 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:41:07.605 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:41:07.605 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:41:07.619 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:07.658 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:07.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:07.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:07.764 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:07.796 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:07.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:07.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:07.902 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:07.935 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:07.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:07.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:08.040 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:08.054 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is set 2023-09-23T18:41:08.076 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:08.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:08.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:08.182 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:08.217 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:08.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:08.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:08.321 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:08.333 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:08 smithi173 bash[59422]: audit 2023-09-23T18:41:07.201691+0000 mon.b (mon.1) 15 : audit 0 from='client.? 172.21.15.98:0/3562405181' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:41:08.333 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:08 smithi173 bash[59422]: audit 2023-09-23T18:41:07.620558+0000 mon.a (mon.0) 945 : audit 1 from='client.? 172.21.15.98:0/2470473522' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:41:08.348 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:08.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:08.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:08.408 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:08 smithi098 bash[76817]: audit 2023-09-23T18:41:07.201691+0000 mon.b (mon.1) 15 : audit 0 from='client.? 172.21.15.98:0/3562405181' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:41:08.408 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:08 smithi098 bash[76817]: audit 2023-09-23T18:41:07.620558+0000 mon.a (mon.0) 945 : audit 1 from='client.? 172.21.15.98:0/2470473522' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:41:08.408 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:08 smithi098 bash[79516]: audit 2023-09-23T18:41:07.201691+0000 mon.b (mon.1) 15 : audit 0 from='client.? 172.21.15.98:0/3562405181' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:41:08.408 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:08 smithi098 bash[79516]: audit 2023-09-23T18:41:07.620558+0000 mon.a (mon.0) 945 : audit 1 from='client.? 172.21.15.98:0/2470473522' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:41:08.452 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:08.488 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:08.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:08.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:08.596 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:08.627 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:08.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:08.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:08.734 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:08.751 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:08.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:08.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:08.854 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:08.882 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:08.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:08.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:08.986 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:09.010 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:09.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:09.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:09.096 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:09 smithi098 bash[76817]: cluster 2023-09-23T18:41:07.913191+0000 mgr.y (mgr.26054) 685 : cluster 0 pgmap v680: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 1.5 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 780 op/s; 10989/73122 objects misplaced (15.028%); 8.9 MiB/s, 141 objects/s recovering 2023-09-23T18:41:09.097 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:09 smithi098 bash[76817]: cluster 2023-09-23T18:41:08.016312+0000 mon.a (mon.0) 946 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:41:09.097 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:09 smithi098 bash[76817]: audit 2023-09-23T18:41:08.038243+0000 mon.a (mon.0) 947 : audit 1 from='client.? 172.21.15.98:0/2470473522' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:41:09.097 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:09 smithi098 bash[76817]: cluster 2023-09-23T18:41:08.038332+0000 mon.a (mon.0) 948 : cluster 0 osdmap e178: 8 total, 8 up, 7 in 2023-09-23T18:41:09.097 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:09 smithi098 bash[76817]: cluster 2023-09-23T18:41:08.038382+0000 osd.7 (osd.7) 92 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T18:41:09.097 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:09 smithi098 bash[76817]: cluster 2023-09-23T18:41:08.040374+0000 osd.7 (osd.7) 93 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T18:41:09.097 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:09 smithi098 bash[76817]: cluster 2023-09-23T18:41:08.045934+0000 osd.6 (osd.6) 105 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T18:41:09.097 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:09 smithi098 bash[76817]: cluster 2023-09-23T18:41:08.045939+0000 osd.4 (osd.4) 130 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T18:41:09.097 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:09 smithi098 bash[76817]: cluster 2023-09-23T18:41:08.282478+0000 osd.5 (osd.5) 79 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T18:41:09.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:09 smithi098 bash[79516]: cluster 2023-09-23T18:41:07.913191+0000 mgr.y (mgr.26054) 685 : cluster 0 pgmap v680: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 1.5 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 780 op/s; 10989/73122 objects misplaced (15.028%); 8.9 MiB/s, 141 objects/s recovering 2023-09-23T18:41:09.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:09 smithi098 bash[79516]: cluster 2023-09-23T18:41:08.016312+0000 mon.a (mon.0) 946 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:41:09.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:09 smithi098 bash[79516]: audit 2023-09-23T18:41:08.038243+0000 mon.a (mon.0) 947 : audit 1 from='client.? 172.21.15.98:0/2470473522' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:41:09.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:09 smithi098 bash[79516]: cluster 2023-09-23T18:41:08.038332+0000 mon.a (mon.0) 948 : cluster 0 osdmap e178: 8 total, 8 up, 7 in 2023-09-23T18:41:09.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:09 smithi098 bash[79516]: cluster 2023-09-23T18:41:08.038382+0000 osd.7 (osd.7) 92 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T18:41:09.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:09 smithi098 bash[79516]: cluster 2023-09-23T18:41:08.040374+0000 osd.7 (osd.7) 93 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T18:41:09.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:09 smithi098 bash[79516]: cluster 2023-09-23T18:41:08.045934+0000 osd.6 (osd.6) 105 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T18:41:09.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:09 smithi098 bash[79516]: cluster 2023-09-23T18:41:08.045939+0000 osd.4 (osd.4) 130 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T18:41:09.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:09 smithi098 bash[79516]: cluster 2023-09-23T18:41:08.282478+0000 osd.5 (osd.5) 79 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T18:41:09.114 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:09.136 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:09.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:09.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:09.240 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:09.265 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:09.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:09.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:09.370 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:09.413 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:09.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:09.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:09.518 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:09.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:09 smithi173 bash[59422]: cluster 2023-09-23T18:41:07.913191+0000 mgr.y (mgr.26054) 685 : cluster 0 pgmap v680: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 1.5 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 780 op/s; 10989/73122 objects misplaced (15.028%); 8.9 MiB/s, 141 objects/s recovering 2023-09-23T18:41:09.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:09 smithi173 bash[59422]: cluster 2023-09-23T18:41:08.016312+0000 mon.a (mon.0) 946 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:41:09.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:09 smithi173 bash[59422]: audit 2023-09-23T18:41:08.038243+0000 mon.a (mon.0) 947 : audit 1 from='client.? 172.21.15.98:0/2470473522' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:41:09.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:09 smithi173 bash[59422]: cluster 2023-09-23T18:41:08.038332+0000 mon.a (mon.0) 948 : cluster 0 osdmap e178: 8 total, 8 up, 7 in 2023-09-23T18:41:09.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:09 smithi173 bash[59422]: cluster 2023-09-23T18:41:08.038382+0000 osd.7 (osd.7) 92 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T18:41:09.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:09 smithi173 bash[59422]: cluster 2023-09-23T18:41:08.040374+0000 osd.7 (osd.7) 93 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T18:41:09.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:09 smithi173 bash[59422]: cluster 2023-09-23T18:41:08.045934+0000 osd.6 (osd.6) 105 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T18:41:09.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:09 smithi173 bash[59422]: cluster 2023-09-23T18:41:08.045939+0000 osd.4 (osd.4) 130 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T18:41:09.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:09 smithi173 bash[59422]: cluster 2023-09-23T18:41:08.282478+0000 osd.5 (osd.5) 79 : cluster 3 failed to encode map e178 with expected crc 2023-09-23T18:41:09.545 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:09.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:09.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:09.651 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:09.684 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:09.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:09.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:09.788 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:09.817 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:09.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:09.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:09.921 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:09.948 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:09.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:09.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:10.050 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:10.085 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:10.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:10.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:10.189 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:10.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:10 smithi098 bash[79516]: audit 2023-09-23T18:41:09.603427+0000 mon.a (mon.0) 949 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:41:10.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:10 smithi098 bash[76817]: audit 2023-09-23T18:41:09.603427+0000 mon.a (mon.0) 949 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:41:10.198 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:10 smithi173 bash[59422]: audit 2023-09-23T18:41:09.603427+0000 mon.a (mon.0) 949 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:41:10.225 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:10.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:10.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:10.328 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:10.361 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:10.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:10.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:10.465 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:10.499 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:10.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:10.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:10.604 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:10.649 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:10.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:10.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:10.754 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:10.778 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:10.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:10.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:10.882 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:10.915 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:10.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:10.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:11.023 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:11.057 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:11.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:11.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:11.162 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:11.171 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:11 smithi173 bash[59422]: cluster 2023-09-23T18:41:09.913627+0000 mgr.y (mgr.26054) 686 : cluster 0 pgmap v682: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 1.5 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 10989/73122 objects misplaced (15.028%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T18:41:11.194 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:11.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:11.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:11.299 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:11.306 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:41:11.307 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:11 smithi098 bash[76817]: cluster 2023-09-23T18:41:09.913627+0000 mgr.y (mgr.26054) 686 : cluster 0 pgmap v682: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 1.5 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 10989/73122 objects misplaced (15.028%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T18:41:11.308 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:11 smithi098 bash[79516]: cluster 2023-09-23T18:41:09.913627+0000 mgr.y (mgr.26054) 686 : cluster 0 pgmap v682: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 1.5 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 10989/73122 objects misplaced (15.028%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T18:41:11.340 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:11.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:11.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:11.442 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:11.471 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:11.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:11.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:11.576 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:11.604 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:11.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:11.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:11.711 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:11.741 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:11.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:11.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:11.845 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:11.871 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:11.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:11.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:11.965 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:41:11.973 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:12.005 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T18:41:12.005 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:41:12.005 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-09-23T18:41:12.005 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T18:41:12.006 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T18:41:12.006 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-09-23T18:41:12.076 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:12.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:12.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:12.186 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:12.239 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:12.243 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:12 smithi098 bash[79516]: cluster 2023-09-23T18:41:11.914552+0000 mgr.y (mgr.26054) 687 : cluster 0 pgmap v683: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 1.4 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 696 op/s; 9775/66291 objects misplaced (14.746%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T18:41:12.243 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:12 smithi098 bash[76817]: cluster 2023-09-23T18:41:11.914552+0000 mgr.y (mgr.26054) 687 : cluster 0 pgmap v683: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 1.4 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 696 op/s; 9775/66291 objects misplaced (14.746%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T18:41:12.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:12.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:12.345 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:12.377 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:12.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:12.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:12.481 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:12.489 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:12 smithi173 bash[59422]: cluster 2023-09-23T18:41:11.914552+0000 mgr.y (mgr.26054) 687 : cluster 0 pgmap v683: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 1.4 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 696 op/s; 9775/66291 objects misplaced (14.746%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T18:41:12.508 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:12.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:12.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:12.616 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:12.643 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:12.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:12.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:12.746 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:12.773 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:12.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:12.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:12.876 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:12.902 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:12.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:12.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:13.005 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:13.043 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:13.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:13.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:13.146 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:13.179 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:13.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:13.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:13.282 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:13.309 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:13.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:13.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:13.414 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:13.458 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:13.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:13.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:13.554 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:41:13.554 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-09-23T18:41:13.554 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 0 2023-09-23T18:41:13.554 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:41:13.562 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:13.586 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:13.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:13.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:13.689 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:13.726 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:13.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:13.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:13.831 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:13.859 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:13.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:13.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:13.964 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:13.988 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:13.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:13.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:14.095 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:14.123 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:14.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:14.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:14.227 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:14.251 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:14.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:14.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:14.361 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:14.391 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:14.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:14.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:14.498 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:14.527 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:14.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:14.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:14.630 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:14.663 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:14.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:14.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:14.768 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:14.793 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:14.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:14.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:14.864 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T18:41:14.896 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:14.941 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:14.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:14.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:15.048 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:15.082 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:15.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:15.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:15.186 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:15.211 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:15.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:15.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:15.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:14 smithi173 bash[59422]: cluster 2023-09-23T18:41:13.915158+0000 mgr.y (mgr.26054) 688 : cluster 0 pgmap v684: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 1.3 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 9563/63045 objects misplaced (15.169%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T18:41:15.317 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:15.326 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:14 smithi098 bash[76817]: cluster 2023-09-23T18:41:13.915158+0000 mgr.y (mgr.26054) 688 : cluster 0 pgmap v684: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 1.3 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 9563/63045 objects misplaced (15.169%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T18:41:15.327 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:14 smithi098 bash[79516]: cluster 2023-09-23T18:41:13.915158+0000 mgr.y (mgr.26054) 688 : cluster 0 pgmap v684: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 1.3 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 9563/63045 objects misplaced (15.169%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T18:41:15.344 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:15.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:15.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:15.450 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:15.478 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:15.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:15.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:15.584 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:15.620 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:15.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:15.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:15.725 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:15.752 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:15.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:15.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:15.858 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:15.882 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:15.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:15.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:15.987 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:16.014 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:16.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:16.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:16.119 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:16.153 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:41:16.153 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:41:16.156 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:41:16.157 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:16.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:16.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:16.265 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:16.290 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:16.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:16.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:16.396 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:16.403 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:41:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:41:16] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.33.4" 2023-09-23T18:41:16.421 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:16.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:16.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:16.523 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:16.564 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:16.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:16.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:16.672 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:16.709 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:16.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:16.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:16.815 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:16.851 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:16.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:16.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:16.960 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:16.999 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:17.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:17.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:17.104 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:17.113 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:16 smithi098 bash[76817]: cluster 2023-09-23T18:41:15.915807+0000 mgr.y (mgr.26054) 689 : cluster 0 pgmap v685: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 1.2 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 662 op/s; 9060/58272 objects misplaced (15.548%); 3.9 MiB/s, 63 objects/s recovering 2023-09-23T18:41:17.114 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:16 smithi098 bash[79516]: cluster 2023-09-23T18:41:15.915807+0000 mgr.y (mgr.26054) 689 : cluster 0 pgmap v685: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 1.2 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 662 op/s; 9060/58272 objects misplaced (15.548%); 3.9 MiB/s, 63 objects/s recovering 2023-09-23T18:41:17.135 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:17.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:17.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:17.242 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:17.265 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:17.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:17.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:17.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:16 smithi173 bash[59422]: cluster 2023-09-23T18:41:15.915807+0000 mgr.y (mgr.26054) 689 : cluster 0 pgmap v685: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 1.2 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 662 op/s; 9060/58272 objects misplaced (15.548%); 3.9 MiB/s, 63 objects/s recovering 2023-09-23T18:41:17.368 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:17.402 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:17.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:17.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:17.504 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:17.540 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:17.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:17.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:17.644 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:17.683 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:17.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:17.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:17.790 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:17.818 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:17.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:17.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:17.924 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:17.962 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:17.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:17.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:18.066 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:18.105 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:18.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:18.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:18.210 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:18.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:18.243 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:18.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:18.344 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:18.370 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:18.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:18.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:18.475 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:18.503 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:18.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:18.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:18.608 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:18.640 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:18.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:18.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:18.744 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:18.771 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:18.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:18.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:18.874 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:18.902 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:18.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:18.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:19.002 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:19 smithi173 bash[59422]: cluster 2023-09-23T18:41:17.916440+0000 mgr.y (mgr.26054) 690 : cluster 0 pgmap v686: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 1.1 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 682 op/s; 8017/52632 objects misplaced (15.232%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T18:41:19.003 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:19 smithi098 bash[79516]: cluster 2023-09-23T18:41:17.916440+0000 mgr.y (mgr.26054) 690 : cluster 0 pgmap v686: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 1.1 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 682 op/s; 8017/52632 objects misplaced (15.232%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T18:41:19.003 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:19 smithi098 bash[76817]: cluster 2023-09-23T18:41:17.916440+0000 mgr.y (mgr.26054) 690 : cluster 0 pgmap v686: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 1.1 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 682 op/s; 8017/52632 objects misplaced (15.232%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T18:41:19.006 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:19.027 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:19.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:19.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:19.136 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:19.162 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:19.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:19.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:19.269 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:19.296 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:19.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:19.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:19.399 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:19.423 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:19.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:19.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:19.528 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:19.561 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:19.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:19.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:19.668 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:19.691 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:41:19.702 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:19.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:19.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:19.808 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:19.848 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:19.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:19.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:19.956 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:19.983 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:19.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:19.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:20.088 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:20.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:20.114 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:20.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:20.215 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:20.247 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:20.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:20.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:20.354 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:20.385 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:20.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:20.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:20.489 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:20.515 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:20.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:20.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:20.620 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:20.654 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:20.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:20.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:20.761 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:20.788 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:20.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:20.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:20.893 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:20.919 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:20.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:20.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:21.005 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:21 smithi098 bash[76817]: cluster 2023-09-23T18:41:19.916922+0000 mgr.y (mgr.26054) 691 : cluster 0 pgmap v687: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 1.1 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 8017/52632 objects misplaced (15.232%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T18:41:21.025 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:21.052 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:21.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:21.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:21.156 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:21.181 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:21.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:21.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:21.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:21 smithi173 bash[59422]: cluster 2023-09-23T18:41:19.916922+0000 mgr.y (mgr.26054) 691 : cluster 0 pgmap v687: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 1.1 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 8017/52632 objects misplaced (15.232%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T18:41:21.285 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:21.293 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:21 smithi098 bash[79516]: cluster 2023-09-23T18:41:19.916922+0000 mgr.y (mgr.26054) 691 : cluster 0 pgmap v687: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 1.1 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 8017/52632 objects misplaced (15.232%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T18:41:21.317 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:21.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:21.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:21.426 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:21.463 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:21.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:21.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:21.565 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:21.593 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:21.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:21.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:21.699 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:21.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:21.729 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:21.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:21.830 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:21.861 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:21.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:21.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:21.963 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:21.999 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:22.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:22.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:22.105 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:22.142 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:22.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:22.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:22.248 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:22.272 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:22.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:22.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:22.379 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:22.421 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:22.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:22.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:22.528 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:22.558 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:22.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:22.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:22.666 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:22.702 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:22.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:22.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:22.808 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:22.836 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:22.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:22.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:22.939 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:22.967 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:22.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:22.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:23.013 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:23 smithi173 bash[59422]: cluster 2023-09-23T18:41:21.917563+0000 mgr.y (mgr.26054) 692 : cluster 0 pgmap v688: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 956 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 759 op/s; 6631/45771 objects misplaced (14.487%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T18:41:23.072 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:23.098 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:23.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:23.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:23.202 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:23.239 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:23.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:23.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:23.345 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:23.358 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:23 smithi098 bash[79516]: cluster 2023-09-23T18:41:21.917563+0000 mgr.y (mgr.26054) 692 : cluster 0 pgmap v688: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 956 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 759 op/s; 6631/45771 objects misplaced (14.487%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T18:41:23.359 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:23 smithi098 bash[76817]: cluster 2023-09-23T18:41:21.917563+0000 mgr.y (mgr.26054) 692 : cluster 0 pgmap v688: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 956 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 759 op/s; 6631/45771 objects misplaced (14.487%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T18:41:23.380 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:23.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:23.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:23.484 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:23.529 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:23.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:23.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:23.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:23.667 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:23.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:23.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:23.769 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:23.798 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:23.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:23.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:23.901 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:23.942 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:23.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:23.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:24.053 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:24.108 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:24.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:24.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:24.213 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:24.239 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:24.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:24.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:24.347 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:24.380 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:24.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:24.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:24.486 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:24.523 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:24.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:24.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:24.629 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:24.668 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:24.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:24.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:24.771 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:24.800 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:24.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:24.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:24.904 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:24.929 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:24.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:24.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:25.033 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:25.068 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:25.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:25.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:25.172 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:25.182 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:25 smithi098 bash[79516]: cluster 2023-09-23T18:41:23.918027+0000 mgr.y (mgr.26054) 693 : cluster 0 pgmap v689: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 885 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 665 op/s; 6397/42327 objects misplaced (15.113%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:41:25.182 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:25 smithi098 bash[79516]: audit 2023-09-23T18:41:24.603450+0000 mon.a (mon.0) 950 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:41:25.182 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:25 smithi098 bash[76817]: cluster 2023-09-23T18:41:23.918027+0000 mgr.y (mgr.26054) 693 : cluster 0 pgmap v689: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 885 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 665 op/s; 6397/42327 objects misplaced (15.113%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:41:25.182 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:25 smithi098 bash[76817]: audit 2023-09-23T18:41:24.603450+0000 mon.a (mon.0) 950 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:41:25.206 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:25.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:25.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:25.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:25 smithi173 bash[59422]: cluster 2023-09-23T18:41:23.918027+0000 mgr.y (mgr.26054) 693 : cluster 0 pgmap v689: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 885 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 665 op/s; 6397/42327 objects misplaced (15.113%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:41:25.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:25 smithi173 bash[59422]: audit 2023-09-23T18:41:24.603450+0000 mon.a (mon.0) 950 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:41:25.312 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:25.346 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:25.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:25.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:25.452 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:25.478 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:25.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:25.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:25.583 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:25.614 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:25.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:25.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:25.716 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:25.742 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:25.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:25.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:25.850 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:25.880 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:25.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:25.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:25.983 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:26.011 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:41:26.011 INFO:teuthology.orchestra.run.smithi098.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T18:41:26.011 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:41:26.020 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:41:26.026 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:26.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:26.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:26.086 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:41:26.136 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:26.137 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:41:26.158 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:26.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:26.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:26.198 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:41:26.235 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:41:26.262 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:26.280 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:41:26.289 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:26.293 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:26 smithi098 bash[79516]: audit 2023-09-23T18:41:25.976246+0000 mon.c (mon.2) 86 : audit 0 from='client.? 172.21.15.98:0/1457705834' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:41:26.293 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:26 smithi098 bash[76817]: audit 2023-09-23T18:41:25.976246+0000 mon.c (mon.2) 86 : audit 0 from='client.? 172.21.15.98:0/1457705834' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:41:26.293 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:41:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:41:26] "GET /metrics HTTP/1.1" 200 36571 "" "Prometheus/2.33.4" 2023-09-23T18:41:26.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:26.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:26.318 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:41:26.356 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:41:26.359 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:41:26.359 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:41:26.359 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:41:26.359 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:41:26.359 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:41:26.360 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:41:26.360 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:41:26.360 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:41:26.395 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:26.405 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:26 smithi173 bash[59422]: audit 2023-09-23T18:41:25.976246+0000 mon.c (mon.2) 86 : audit 0 from='client.? 172.21.15.98:0/1457705834' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:41:26.436 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:26.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:26.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:26.540 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:26.573 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:26.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:26.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:26.678 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:26.704 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:26.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:26.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:26.809 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:26.839 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:26.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:26.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:26.947 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:26.973 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:26.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:26.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:27.057 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is unset 2023-09-23T18:41:27.079 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:27.085 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:27 smithi098 bash[79516]: cluster 2023-09-23T18:41:25.918834+0000 mgr.y (mgr.26054) 694 : cluster 0 pgmap v690: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 789 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 5903/37758 objects misplaced (15.634%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:41:27.086 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:27 smithi098 bash[79516]: audit 2023-09-23T18:41:26.223091+0000 mon.a (mon.0) 951 : audit 1 from='client.? 172.21.15.98:0/1751459057' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:41:27.086 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:27 smithi098 bash[76817]: cluster 2023-09-23T18:41:25.918834+0000 mgr.y (mgr.26054) 694 : cluster 0 pgmap v690: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 789 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 5903/37758 objects misplaced (15.634%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:41:27.086 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:27 smithi098 bash[76817]: audit 2023-09-23T18:41:26.223091+0000 mon.a (mon.0) 951 : audit 1 from='client.? 172.21.15.98:0/1751459057' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:41:27.113 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:27.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:27.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:27.156 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T18:41:27.156 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:41:27.220 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:27.246 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:27.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:27.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:27.349 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:27.358 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:27 smithi173 bash[59422]: cluster 2023-09-23T18:41:25.918834+0000 mgr.y (mgr.26054) 694 : cluster 0 pgmap v690: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 789 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 5903/37758 objects misplaced (15.634%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:41:27.359 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:27 smithi173 bash[59422]: audit 2023-09-23T18:41:26.223091+0000 mon.a (mon.0) 951 : audit 1 from='client.? 172.21.15.98:0/1751459057' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:41:27.377 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:27.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:27.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:27.483 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:27.515 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:27.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:27.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:27.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:27.647 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:27.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:27.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:27.750 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:27.780 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:27.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:27.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:27.885 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:27.915 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:27.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:27.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:28.023 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:28.066 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:28.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:28.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:28.111 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:28 smithi098 bash[79516]: cluster 2023-09-23T18:41:27.034462+0000 mon.a (mon.0) 952 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:41:28.112 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:28 smithi098 bash[79516]: audit 2023-09-23T18:41:27.050795+0000 mon.a (mon.0) 953 : audit 1 from='client.? 172.21.15.98:0/1751459057' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:41:28.112 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:28 smithi098 bash[79516]: cluster 2023-09-23T18:41:27.050990+0000 mon.a (mon.0) 954 : cluster 0 osdmap e179: 8 total, 8 up, 7 in 2023-09-23T18:41:28.112 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:28 smithi098 bash[79516]: cluster 2023-09-23T18:41:27.056281+0000 osd.6 (osd.6) 106 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T18:41:28.112 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:28 smithi098 bash[79516]: cluster 2023-09-23T18:41:27.056285+0000 osd.7 (osd.7) 94 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T18:41:28.112 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:28 smithi098 bash[79516]: cluster 2023-09-23T18:41:27.058319+0000 osd.6 (osd.6) 107 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T18:41:28.112 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:28 smithi098 bash[79516]: cluster 2023-09-23T18:41:27.058386+0000 osd.4 (osd.4) 131 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T18:41:28.112 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:28 smithi098 bash[79516]: cluster 2023-09-23T18:41:27.059150+0000 osd.7 (osd.7) 95 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T18:41:28.112 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:28 smithi098 bash[79516]: cluster 2023-09-23T18:41:27.099812+0000 osd.5 (osd.5) 80 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T18:41:28.113 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:28 smithi098 bash[76817]: cluster 2023-09-23T18:41:27.034462+0000 mon.a (mon.0) 952 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:41:28.113 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:28 smithi098 bash[76817]: audit 2023-09-23T18:41:27.050795+0000 mon.a (mon.0) 953 : audit 1 from='client.? 172.21.15.98:0/1751459057' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:41:28.113 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:28 smithi098 bash[76817]: cluster 2023-09-23T18:41:27.050990+0000 mon.a (mon.0) 954 : cluster 0 osdmap e179: 8 total, 8 up, 7 in 2023-09-23T18:41:28.113 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:28 smithi098 bash[76817]: cluster 2023-09-23T18:41:27.056281+0000 osd.6 (osd.6) 106 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T18:41:28.113 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:28 smithi098 bash[76817]: cluster 2023-09-23T18:41:27.056285+0000 osd.7 (osd.7) 94 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T18:41:28.113 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:28 smithi098 bash[76817]: cluster 2023-09-23T18:41:27.058319+0000 osd.6 (osd.6) 107 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T18:41:28.113 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:28 smithi098 bash[76817]: cluster 2023-09-23T18:41:27.058386+0000 osd.4 (osd.4) 131 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T18:41:28.113 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:28 smithi098 bash[76817]: cluster 2023-09-23T18:41:27.059150+0000 osd.7 (osd.7) 95 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T18:41:28.113 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:28 smithi098 bash[76817]: cluster 2023-09-23T18:41:27.099812+0000 osd.5 (osd.5) 80 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T18:41:28.173 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:28.183 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:28 smithi173 bash[59422]: cluster 2023-09-23T18:41:27.034462+0000 mon.a (mon.0) 952 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:41:28.183 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:28 smithi173 bash[59422]: audit 2023-09-23T18:41:27.050795+0000 mon.a (mon.0) 953 : audit 1 from='client.? 172.21.15.98:0/1751459057' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:41:28.183 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:28 smithi173 bash[59422]: cluster 2023-09-23T18:41:27.050990+0000 mon.a (mon.0) 954 : cluster 0 osdmap e179: 8 total, 8 up, 7 in 2023-09-23T18:41:28.183 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:28 smithi173 bash[59422]: cluster 2023-09-23T18:41:27.056281+0000 osd.6 (osd.6) 106 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T18:41:28.183 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:28 smithi173 bash[59422]: cluster 2023-09-23T18:41:27.056285+0000 osd.7 (osd.7) 94 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T18:41:28.183 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:28 smithi173 bash[59422]: cluster 2023-09-23T18:41:27.058319+0000 osd.6 (osd.6) 107 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T18:41:28.183 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:28 smithi173 bash[59422]: cluster 2023-09-23T18:41:27.058386+0000 osd.4 (osd.4) 131 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T18:41:28.183 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:28 smithi173 bash[59422]: cluster 2023-09-23T18:41:27.059150+0000 osd.7 (osd.7) 95 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T18:41:28.183 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:28 smithi173 bash[59422]: cluster 2023-09-23T18:41:27.099812+0000 osd.5 (osd.5) 80 : cluster 3 failed to encode map e179 with expected crc 2023-09-23T18:41:28.211 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:28.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:28.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:28.321 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:28.355 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:28.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:28.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:28.458 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:28.486 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:28.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:28.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:28.589 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:28.613 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:28.615 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T18:41:28.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:28.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:28.719 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:28.752 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:28.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:28.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:28.855 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:28.878 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:28.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:28.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:28.984 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:29.028 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:29.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:29.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:29.134 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:29.135 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:29 smithi098 bash[76817]: cluster 2023-09-23T18:41:27.919336+0000 mgr.y (mgr.26054) 695 : cluster 0 pgmap v692: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 668 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 4866/31941 objects misplaced (15.234%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T18:41:29.136 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:29 smithi098 bash[79516]: cluster 2023-09-23T18:41:27.919336+0000 mgr.y (mgr.26054) 695 : cluster 0 pgmap v692: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 668 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 4866/31941 objects misplaced (15.234%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T18:41:29.162 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:29.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:29.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:29.270 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:29.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:29 smithi173 bash[59422]: cluster 2023-09-23T18:41:27.919336+0000 mgr.y (mgr.26054) 695 : cluster 0 pgmap v692: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 668 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 4866/31941 objects misplaced (15.234%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T18:41:29.305 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:29.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:29.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:29.410 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:29.440 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:29.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:29.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:29.544 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:29.570 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:29.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:29.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:29.674 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:29.700 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:29.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:29.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:29.806 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:29.841 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:29.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:29.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:29.945 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:29.974 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:29.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:29.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:30.079 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:30.106 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:30.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:30.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:30.210 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:30.243 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:30.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:30.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:30.348 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:30.371 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:30.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:30.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:30.474 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:30.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:30.508 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:30.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:30.609 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:30.644 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:30.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:30.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:30.748 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:30.778 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:30.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:30.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:30.882 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:30.917 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:30.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:30.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:31.022 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:31.055 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:31.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:31.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:31.160 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:31.188 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:31.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:31.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:31.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:30 smithi098 bash[79516]: cluster 2023-09-23T18:41:29.919861+0000 mgr.y (mgr.26054) 696 : cluster 0 pgmap v693: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 668 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 4866/31941 objects misplaced (15.234%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T18:41:31.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:30 smithi098 bash[76817]: cluster 2023-09-23T18:41:29.919861+0000 mgr.y (mgr.26054) 696 : cluster 0 pgmap v693: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 668 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 4866/31941 objects misplaced (15.234%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T18:41:31.222 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:41:31.239 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:41:31.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:30 smithi173 bash[59422]: cluster 2023-09-23T18:41:29.919861+0000 mgr.y (mgr.26054) 696 : cluster 0 pgmap v693: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 668 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 4866/31941 objects misplaced (15.234%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T18:41:31.291 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:31.321 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:31.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:31.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:31.427 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:31.464 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:31.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:31.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:31.567 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:31.600 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:31.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:31.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:31.704 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:31.731 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:31.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:31.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:31.834 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:31.859 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:31.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:31.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:31.966 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:31.998 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:32.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:32.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:32.103 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:32.140 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:32.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:32.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:32.245 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:32.280 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:32.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:32.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:32.388 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:32.418 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:32.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:32.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:32.524 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:32.547 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:32.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:32.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:32.652 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:32.689 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:32.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:32.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:32.795 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:32.825 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:32.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:32.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:32.929 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:32.964 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:32.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:32.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:33.068 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:33.077 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:33 smithi098 bash[79516]: cluster 2023-09-23T18:41:31.920560+0000 mgr.y (mgr.26054) 697 : cluster 0 pgmap v694: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 522 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 694 op/s; 3487/24936 objects misplaced (13.984%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:41:33.078 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:32 smithi098 bash[76817]: cluster 2023-09-23T18:41:31.920560+0000 mgr.y (mgr.26054) 697 : cluster 0 pgmap v694: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 522 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 694 op/s; 3487/24936 objects misplaced (13.984%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:41:33.100 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:33.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:33.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:33.206 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:33.215 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:32 smithi173 bash[59422]: cluster 2023-09-23T18:41:31.920560+0000 mgr.y (mgr.26054) 697 : cluster 0 pgmap v694: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 522 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 694 op/s; 3487/24936 objects misplaced (13.984%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:41:33.239 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:33.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:33.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:33.344 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:33.370 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:33.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:33.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:33.460 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:41:33.479 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:33.513 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:33.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:33.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:33.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:33.639 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:33.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:33.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:33.741 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:33.781 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:33.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:33.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:33.883 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:33.913 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:33.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:33.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:34.018 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:34.050 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:34.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:34.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:34.136 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T18:41:34.136 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:41:34.136 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-09-23T18:41:34.136 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:41:34.156 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:34.239 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:34.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:34.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:34.342 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:34.383 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:34.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:34.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:34.497 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:34.531 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:34.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:34.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:34.636 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:34.686 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:34.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:34.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:34.792 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:34.818 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:34.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:34.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:34.923 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:34.953 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:34.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:34.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:35.058 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:35.066 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:35 smithi098 bash[76817]: cluster 2023-09-23T18:41:33.921080+0000 mgr.y (mgr.26054) 698 : cluster 0 pgmap v695: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 432 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 723 op/s; 3148/20607 objects misplaced (15.276%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:41:35.066 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:35 smithi098 bash[79516]: cluster 2023-09-23T18:41:33.921080+0000 mgr.y (mgr.26054) 698 : cluster 0 pgmap v695: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 432 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 723 op/s; 3148/20607 objects misplaced (15.276%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:41:35.099 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:35.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:35.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:35.204 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:35.240 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:35.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:35.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:35.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:35 smithi173 bash[59422]: cluster 2023-09-23T18:41:33.921080+0000 mgr.y (mgr.26054) 698 : cluster 0 pgmap v695: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 432 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 723 op/s; 3148/20607 objects misplaced (15.276%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:41:35.343 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:35.381 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:35.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:35.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:35.484 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:35.515 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:35.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:35.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:35.619 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:35.646 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:35.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:35.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:35.748 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:35.786 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:35.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:35.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:35.890 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:35.933 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:35.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:35.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:36.040 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:36.072 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:36.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:36.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:36.091 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:41:36.102 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:41:36.154 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:41:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:41:36] "GET /metrics HTTP/1.1" 200 36571 "" "Prometheus/2.33.4" 2023-09-23T18:41:36.175 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:36.201 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:36.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:36.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:36.303 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:36.328 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:36.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:36.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:36.432 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:36.461 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:36.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:36.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:36.565 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:36.593 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:36.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:36.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:36.699 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:36.728 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:36.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:36.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:36.832 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:36.868 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:36.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:36.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:36.972 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:36.995 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:36.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:36.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:37.005 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:37 smithi098 bash[76817]: cluster 2023-09-23T18:41:35.921685+0000 mgr.y (mgr.26054) 699 : cluster 0 pgmap v696: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 365 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 678 op/s; 2807/17394 objects misplaced (16.138%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:41:37.100 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:37.108 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:37 smithi173 bash[59422]: cluster 2023-09-23T18:41:35.921685+0000 mgr.y (mgr.26054) 699 : cluster 0 pgmap v696: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 365 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 678 op/s; 2807/17394 objects misplaced (16.138%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:41:37.131 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:37.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:37.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:37.237 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:37.265 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:37.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:37.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:37.267 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:37 smithi098 bash[79516]: cluster 2023-09-23T18:41:35.921685+0000 mgr.y (mgr.26054) 699 : cluster 0 pgmap v696: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 365 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 678 op/s; 2807/17394 objects misplaced (16.138%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:41:37.368 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:37.531 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:37.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:37.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:37.634 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:37.669 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:37.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:37.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:37.771 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:37.798 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:37.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:37.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:37.900 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:37.926 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:37.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:37.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:38.034 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:38.071 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:38.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:38.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:38.128 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:41:38.177 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:38.201 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:38.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:38.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:38.304 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:38.339 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:38.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:38.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:38.449 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:38.486 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:38.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:38.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:38.590 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:38.617 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:38.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:38.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:38.724 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:38.761 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:38.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:38.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:38.867 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:38.896 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:38.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:38.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:39.005 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:39.011 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:39 smithi173 bash[59422]: cluster 2023-09-23T18:41:37.922240+0000 mgr.y (mgr.26054) 700 : cluster 0 pgmap v697: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 243 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 1706/11547 objects misplaced (14.774%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T18:41:39.029 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:39.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:39.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:39.132 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:39.160 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:39.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:39.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:39.265 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:39.297 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:39.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:39.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:39.400 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:39.428 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:39.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:39.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:39.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:39 smithi098 bash[79516]: cluster 2023-09-23T18:41:37.922240+0000 mgr.y (mgr.26054) 700 : cluster 0 pgmap v697: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 243 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 1706/11547 objects misplaced (14.774%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T18:41:39.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:39 smithi098 bash[76817]: cluster 2023-09-23T18:41:37.922240+0000 mgr.y (mgr.26054) 700 : cluster 0 pgmap v697: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 243 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 1706/11547 objects misplaced (14.774%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T18:41:39.532 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:39.561 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:39.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:39.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:39.667 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:39.704 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:39.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:39.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:39.807 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:39.845 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:39.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:39.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:39.948 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:39.980 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:39.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:39.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:40.085 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:40.094 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:40 smithi098 bash[76817]: audit 2023-09-23T18:41:39.603904+0000 mon.a (mon.0) 955 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:41:40.094 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:40 smithi098 bash[79516]: audit 2023-09-23T18:41:39.603904+0000 mon.a (mon.0) 955 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:41:40.113 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:40.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:40.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:40.205 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T18:41:40.205 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:41:40.216 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:40.247 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:40 smithi173 bash[59422]: audit 2023-09-23T18:41:39.603904+0000 mon.a (mon.0) 955 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:41:40.249 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:40.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:40.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:40.354 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:40.379 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:40.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:40.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:40.484 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:40.510 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:40.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:40.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:40.614 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:40.647 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:40.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:40.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:40.754 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:40.783 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:40.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:40.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:40.887 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:40.917 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:40.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:40.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:41.022 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:41.026 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:41 smithi098 bash[76817]: cluster 2023-09-23T18:41:39.924140+0000 mgr.y (mgr.26054) 701 : cluster 0 pgmap v698: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 216 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 602 op/s; 1527/10227 objects misplaced (14.931%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T18:41:41.051 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:41.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:41.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:41.160 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:41.168 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:41 smithi173 bash[59422]: cluster 2023-09-23T18:41:39.924140+0000 mgr.y (mgr.26054) 701 : cluster 0 pgmap v698: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 216 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 602 op/s; 1527/10227 objects misplaced (14.931%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T18:41:41.181 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:41.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:41.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:41.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:41 smithi098 bash[79516]: cluster 2023-09-23T18:41:39.924140+0000 mgr.y (mgr.26054) 701 : cluster 0 pgmap v698: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 216 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 602 op/s; 1527/10227 objects misplaced (14.931%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T18:41:41.284 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:41.310 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:41.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:41.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:41.415 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:41.441 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:41.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:41.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:41.476 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Removed 104666 objects 2023-09-23T18:41:41.477 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Clean up completed and total clean up time :161.669 2023-09-23T18:41:41.546 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:41.573 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:41.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:41.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:41.612 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T18:41:41.676 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:41.701 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:41.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:41.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:41.804 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:41.830 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:41.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:41.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:41.933 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:41.965 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:41.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:41.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:42.068 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:42.098 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:42.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:42.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:42.202 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:42.229 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:42.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:42.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:42.331 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:42.360 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:42.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:42.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:42.464 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:42.499 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:42.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:42.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:42.602 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:42.628 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:42.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:42.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:42.731 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:42.762 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:42.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:42.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:42.865 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:42.894 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:42.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:42.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:42.997 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:43.032 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:43.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:43.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:43.135 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:43.145 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:43 smithi173 bash[59422]: cluster 2023-09-23T18:41:41.924952+0000 mgr.y (mgr.26054) 702 : cluster 0 pgmap v699: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 88 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 772 op/s; 346/4107 objects misplaced (8.425%); 3.4 MiB/s, 57 objects/s recovering 2023-09-23T18:41:43.165 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:43.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:43.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:43.269 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:43.298 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:43.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:43.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:43.402 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:43.413 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:43 smithi098 bash[76817]: cluster 2023-09-23T18:41:41.924952+0000 mgr.y (mgr.26054) 702 : cluster 0 pgmap v699: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 88 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 772 op/s; 346/4107 objects misplaced (8.425%); 3.4 MiB/s, 57 objects/s recovering 2023-09-23T18:41:43.414 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:43 smithi098 bash[79516]: cluster 2023-09-23T18:41:41.924952+0000 mgr.y (mgr.26054) 702 : cluster 0 pgmap v699: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 88 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 772 op/s; 346/4107 objects misplaced (8.425%); 3.4 MiB/s, 57 objects/s recovering 2023-09-23T18:41:43.430 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:43.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:43.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:43.534 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:43.559 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:43.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:43.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:43.662 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:43.689 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:43.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:43.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:43.792 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:43.824 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:43.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:43.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:43.928 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:43.954 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:43.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:43.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:44.057 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:44.078 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:44.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:44.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:44.181 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:44.211 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:44.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:44.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:44.315 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:44.339 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:44.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:44.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:44.442 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:44.470 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:44.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:44.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:44.573 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:44.603 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:44.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:44.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:44.706 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:44.733 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:44.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:44.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:44.837 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:44.867 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:44.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:44.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:44.970 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:45.000 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:45.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:45.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:45.104 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:45.135 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:45.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:45.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:45.238 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:45.239 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:45 smithi098 bash[76817]: cluster 2023-09-23T18:41:43.925953+0000 mgr.y (mgr.26054) 703 : cluster 0 pgmap v700: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 16 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 674 op/s; 68/636 objects misplaced (10.692%); 2.3 MiB/s, 39 objects/s recovering 2023-09-23T18:41:45.240 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:45 smithi098 bash[79516]: cluster 2023-09-23T18:41:43.925953+0000 mgr.y (mgr.26054) 703 : cluster 0 pgmap v700: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 16 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 674 op/s; 68/636 objects misplaced (10.692%); 2.3 MiB/s, 39 objects/s recovering 2023-09-23T18:41:45.269 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:45.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:45.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:45.372 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:45.384 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:45 smithi173 bash[59422]: cluster 2023-09-23T18:41:43.925953+0000 mgr.y (mgr.26054) 703 : cluster 0 pgmap v700: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 16 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 674 op/s; 68/636 objects misplaced (10.692%); 2.3 MiB/s, 39 objects/s recovering 2023-09-23T18:41:45.400 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:45.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:45.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:45.485 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:41:45.504 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:45.506 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:41:45.532 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:41:45.533 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:45.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:45.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:45.557 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:41:45.600 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:41:45.626 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:41:45.636 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:45.658 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:45.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:45.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:45.663 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:41:45.694 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:41:45.696 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:41:45.696 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:41:45.696 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:41:45.696 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:41:45.696 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:41:45.696 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:41:45.696 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:41:45.697 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:41:45.764 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:45.788 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:45.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:45.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:45.891 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:45.920 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:45.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:45.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:45.931 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:41:45.932 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-23T18:41:46.024 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:46.052 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:46.053 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is unset 2023-09-23T18:41:46.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:46.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:46.156 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:46.168 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:46 smithi173 bash[59422]: audit 2023-09-23T18:41:45.458845+0000 mon.c (mon.2) 87 : audit 0 from='client.? 172.21.15.98:0/3158534283' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:41:46.168 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:46 smithi173 bash[59422]: audit 2023-09-23T18:41:45.878886+0000 mon.a (mon.0) 956 : audit 1 from='client.? 172.21.15.98:0/655036210' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:41:46.184 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:46.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:46.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:46.190 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:41:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:41:46] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.33.4" 2023-09-23T18:41:46.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:46 smithi098 bash[79516]: audit 2023-09-23T18:41:45.458845+0000 mon.c (mon.2) 87 : audit 0 from='client.? 172.21.15.98:0/3158534283' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:41:46.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:46 smithi098 bash[79516]: audit 2023-09-23T18:41:45.878886+0000 mon.a (mon.0) 956 : audit 1 from='client.? 172.21.15.98:0/655036210' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:41:46.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:46 smithi098 bash[76817]: audit 2023-09-23T18:41:45.458845+0000 mon.c (mon.2) 87 : audit 0 from='client.? 172.21.15.98:0/3158534283' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:41:46.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:46 smithi098 bash[76817]: audit 2023-09-23T18:41:45.878886+0000 mon.a (mon.0) 956 : audit 1 from='client.? 172.21.15.98:0/655036210' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:41:46.288 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:46.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:46.315 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:46.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:46.385 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:41:46.416 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:46.440 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:46.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:46.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:46.543 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:46.567 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:46.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:46.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:46.670 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:46.696 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:46.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:46.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:46.799 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:46.827 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:46.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:46.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:46.930 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:46.950 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:46.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:46.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:47.052 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:47.079 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:47.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:47.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:47.183 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:47.207 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:47.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:47.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:47.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:47 smithi173 bash[59422]: audit 2023-09-23T18:41:45.915558+0000 mgr.y (mgr.26054) 704 : audit 0 from='client.45119 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:41:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:47 smithi173 bash[59422]: cluster 2023-09-23T18:41:45.926535+0000 mgr.y (mgr.26054) 705 : cluster 0 pgmap v701: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 7.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 21/228 objects misplaced (9.211%); 2.3 MiB/s, 39 objects/s recovering 2023-09-23T18:41:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:47 smithi173 bash[59422]: cluster 2023-09-23T18:41:46.034547+0000 mon.a (mon.0) 957 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:41:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:47 smithi173 bash[59422]: cluster 2023-09-23T18:41:46.034600+0000 mon.a (mon.0) 958 : cluster 1 Cluster is now healthy 2023-09-23T18:41:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:47 smithi173 bash[59422]: cluster 2023-09-23T18:41:46.046137+0000 osd.4 (osd.4) 132 : cluster 3 failed to encode map e180 with expected crc 2023-09-23T18:41:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:47 smithi173 bash[59422]: audit 2023-09-23T18:41:46.046572+0000 mon.a (mon.0) 959 : audit 1 from='client.? 172.21.15.98:0/655036210' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:41:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:47 smithi173 bash[59422]: cluster 2023-09-23T18:41:46.046833+0000 mon.a (mon.0) 960 : cluster 0 osdmap e180: 8 total, 8 up, 7 in 2023-09-23T18:41:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:47 smithi173 bash[59422]: cluster 2023-09-23T18:41:46.047445+0000 osd.5 (osd.5) 81 : cluster 3 failed to encode map e180 with expected crc 2023-09-23T18:41:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:47 smithi173 bash[59422]: cluster 2023-09-23T18:41:46.048124+0000 osd.4 (osd.4) 133 : cluster 3 failed to encode map e180 with expected crc 2023-09-23T18:41:47.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:47 smithi173 bash[59422]: cluster 2023-09-23T18:41:46.544282+0000 osd.7 (osd.7) 96 : cluster 3 failed to encode map e180 with expected crc 2023-09-23T18:41:47.310 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:47.321 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:47 smithi098 bash[79516]: audit 2023-09-23T18:41:45.915558+0000 mgr.y (mgr.26054) 704 : audit 0 from='client.45119 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:41:47.322 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:47 smithi098 bash[79516]: cluster 2023-09-23T18:41:45.926535+0000 mgr.y (mgr.26054) 705 : cluster 0 pgmap v701: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 7.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 21/228 objects misplaced (9.211%); 2.3 MiB/s, 39 objects/s recovering 2023-09-23T18:41:47.322 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:47 smithi098 bash[79516]: cluster 2023-09-23T18:41:46.034547+0000 mon.a (mon.0) 957 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:41:47.322 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:47 smithi098 bash[79516]: cluster 2023-09-23T18:41:46.034600+0000 mon.a (mon.0) 958 : cluster 1 Cluster is now healthy 2023-09-23T18:41:47.322 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:47 smithi098 bash[79516]: cluster 2023-09-23T18:41:46.046137+0000 osd.4 (osd.4) 132 : cluster 3 failed to encode map e180 with expected crc 2023-09-23T18:41:47.322 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:47 smithi098 bash[79516]: audit 2023-09-23T18:41:46.046572+0000 mon.a (mon.0) 959 : audit 1 from='client.? 172.21.15.98:0/655036210' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:41:47.322 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:47 smithi098 bash[79516]: cluster 2023-09-23T18:41:46.046833+0000 mon.a (mon.0) 960 : cluster 0 osdmap e180: 8 total, 8 up, 7 in 2023-09-23T18:41:47.322 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:47 smithi098 bash[79516]: cluster 2023-09-23T18:41:46.047445+0000 osd.5 (osd.5) 81 : cluster 3 failed to encode map e180 with expected crc 2023-09-23T18:41:47.322 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:47 smithi098 bash[79516]: cluster 2023-09-23T18:41:46.048124+0000 osd.4 (osd.4) 133 : cluster 3 failed to encode map e180 with expected crc 2023-09-23T18:41:47.322 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:47 smithi098 bash[79516]: cluster 2023-09-23T18:41:46.544282+0000 osd.7 (osd.7) 96 : cluster 3 failed to encode map e180 with expected crc 2023-09-23T18:41:47.323 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:47 smithi098 bash[76817]: audit 2023-09-23T18:41:45.915558+0000 mgr.y (mgr.26054) 704 : audit 0 from='client.45119 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:41:47.323 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:47 smithi098 bash[76817]: cluster 2023-09-23T18:41:45.926535+0000 mgr.y (mgr.26054) 705 : cluster 0 pgmap v701: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 7.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 21/228 objects misplaced (9.211%); 2.3 MiB/s, 39 objects/s recovering 2023-09-23T18:41:47.323 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:47 smithi098 bash[76817]: cluster 2023-09-23T18:41:46.034547+0000 mon.a (mon.0) 957 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:41:47.323 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:47 smithi098 bash[76817]: cluster 2023-09-23T18:41:46.034600+0000 mon.a (mon.0) 958 : cluster 1 Cluster is now healthy 2023-09-23T18:41:47.323 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:47 smithi098 bash[76817]: cluster 2023-09-23T18:41:46.046137+0000 osd.4 (osd.4) 132 : cluster 3 failed to encode map e180 with expected crc 2023-09-23T18:41:47.323 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:47 smithi098 bash[76817]: audit 2023-09-23T18:41:46.046572+0000 mon.a (mon.0) 959 : audit 1 from='client.? 172.21.15.98:0/655036210' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:41:47.323 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:47 smithi098 bash[76817]: cluster 2023-09-23T18:41:46.046833+0000 mon.a (mon.0) 960 : cluster 0 osdmap e180: 8 total, 8 up, 7 in 2023-09-23T18:41:47.323 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:47 smithi098 bash[76817]: cluster 2023-09-23T18:41:46.047445+0000 osd.5 (osd.5) 81 : cluster 3 failed to encode map e180 with expected crc 2023-09-23T18:41:47.323 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:47 smithi098 bash[76817]: cluster 2023-09-23T18:41:46.048124+0000 osd.4 (osd.4) 133 : cluster 3 failed to encode map e180 with expected crc 2023-09-23T18:41:47.323 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:47 smithi098 bash[76817]: cluster 2023-09-23T18:41:46.544282+0000 osd.7 (osd.7) 96 : cluster 3 failed to encode map e180 with expected crc 2023-09-23T18:41:47.336 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:47.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:47.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:47.439 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:47.466 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:47.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:47.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:47.570 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:47.593 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:47.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:47.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:47.696 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:47.725 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:47.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:47.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:47.829 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:47.854 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:47.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:47.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:47.957 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:47.980 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:47.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:47.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:48.083 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:48.113 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:48.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:48.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:48.217 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:48.246 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:48.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:48.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:48.285 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":700,"stamp":"2023-09-23T18:41:43.925227+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":16848819,"num_objects":212,"num_object_clones":0,"num_object_copies":636,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":68,"num_objects_unfound":0,"num_objects_dirty":212,"num_whiteouts":0,"num_read":6246,"num_read_kb":13703,"num_write":210256,"num_write_kb":6713784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17807,"num_bytes_recovered":1166410928,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117085,"ondisk_log_size":117085,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":279,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":2880792,"kb_used_data":599072,"kb_used_omap":31,"kb_used_meta":2281632,"kb_avail":653335272,"statfs":{"total":671965249536,"available":669015318528,"internally_reserved":0,"allocated":613449728,"data_stored":608498632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32507,"internal_metadata":2336391429},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-530776088,"num_objects":-8100,"num_object_clones":0,"num_object_copies":-24300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-3419,"num_objects_unfound":0,"num_objects_dirty":-8100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8100,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":470,"num_bytes_recovered":29097984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005155"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":36,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.033477+0000","last_change":"2023-09-23T18:37:40.992912+0000","last_active":"2023-09-23T18:41:08.033477+0000","last_peered":"2023-09-23T18:41:08.033477+0000","last_clean":"2023-09-23T18:41:08.033477+0000","last_became_active":"2023-09-23T18:37:40.992453+0000","last_became_peered":"2023-09-23T18:37:40.992453+0000","last_unstale":"2023-09-23T18:41:08.033477+0000","last_undegraded":"2023-09-23T18:41:08.033477+0000","last_fullsized":"2023-09-23T18:41:08.033477+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:49.539251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":118,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.039857+0000","last_change":"2023-09-23T18:40:42.051229+0000","last_active":"2023-09-23T18:41:08.039857+0000","last_peered":"2023-09-23T18:41:08.039857+0000","last_clean":"2023-09-23T18:41:08.039857+0000","last_became_active":"2023-09-23T18:40:42.050622+0000","last_became_peered":"2023-09-23T18:40:42.050622+0000","last_unstale":"2023-09-23T18:41:08.039857+0000","last_undegraded":"2023-09-23T18:41:08.039857+0000","last_fullsized":"2023-09-23T18:41:08.039857+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:17.587802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":117,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.040100+0000","last_change":"2023-09-23T18:33:53.848655+0000","last_active":"2023-09-23T18:41:08.040100+0000","last_peered":"2023-09-23T18:41:08.040100+0000","last_clean":"2023-09-23T18:41:08.040100+0000","last_became_active":"2023-09-23T18:33:53.848279+0000","last_became_peered":"2023-09-23T18:33:53.848279+0000","last_unstale":"2023-09-23T18:41:08.040100+0000","last_undegraded":"2023-09-23T18:41:08.040100+0000","last_fullsized":"2023-09-23T18:41:08.040100+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:39.223612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":152,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.034431+0000","last_change":"2023-09-23T18:40:42.052899+0000","last_active":"2023-09-23T18:41:08.034431+0000","last_peered":"2023-09-23T18:41:08.034431+0000","last_clean":"2023-09-23T18:41:08.034431+0000","last_became_active":"2023-09-23T18:40:42.052569+0000","last_became_peered":"2023-09-23T18:40:42.052569+0000","last_unstale":"2023-09-23T18:41:08.034431+0000","last_undegraded":"2023-09-23T18:41:08.034431+0000","last_fullsized":"2023-09-23T18:41:08.034431+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:46.151073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":134,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.033342+0000","last_change":"2023-09-23T18:40:42.048469+0000","last_active":"2023-09-23T18:41:08.033342+0000","last_peered":"2023-09-23T18:41:08.033342+0000","last_clean":"2023-09-23T18:41:08.033342+0000","last_became_active":"2023-09-23T18:40:42.048139+0000","last_became_peered":"2023-09-23T18:40:42.048139+0000","last_unstale":"2023-09-23T18:41:08.033342+0000","last_undegraded":"2023-09-23T18:41:08.033342+0000","last_fullsized":"2023-09-23T18:41:08.033342+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:09:09.535543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":170,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.068820+0000","last_change":"2023-09-23T18:33:55.084557+0000","last_active":"2023-09-23T18:41:08.068820+0000","last_peered":"2023-09-23T18:41:08.068820+0000","last_clean":"2023-09-23T18:41:08.068820+0000","last_became_active":"2023-09-23T18:33:55.083987+0000","last_became_peered":"2023-09-23T18:33:55.083987+0000","last_unstale":"2023-09-23T18:41:08.068820+0000","last_undegraded":"2023-09-23T18:41:08.068820+0000","last_fullsized":"2023-09-23T18:41:08.068820+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":150,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.033933+0000","last_change":"2023-09-23T18:40:42.054577+0000","last_active":"2023-09-23T18:41:08.033933+0000","last_peered":"2023-09-23T18:41:08.033933+0000","last_clean":"2023-09-23T18:41:08.033933+0000","last_became_active":"2023-09-23T18:40:42.054170+0000","last_became_peered":"2023-09-23T18:40:42.054170+0000","last_unstale":"2023-09-23T18:41:08.033933+0000","last_undegraded":"2023-09-23T18:41:08.033933+0000","last_fullsized":"2023-09-23T18:41:08.033933+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:35.503748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":175,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.043195+0000","last_change":"2023-09-23T18:37:40.998526+0000","last_active":"2023-09-23T18:41:08.043195+0000","last_peered":"2023-09-23T18:41:08.043195+0000","last_clean":"2023-09-23T18:41:08.043195+0000","last_became_active":"2023-09-23T18:37:40.993211+0000","last_became_peered":"2023-09-23T18:37:40.993211+0000","last_unstale":"2023-09-23T18:41:08.043195+0000","last_undegraded":"2023-09-23T18:41:08.043195+0000","last_fullsized":"2023-09-23T18:41:08.043195+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:40.463496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":177,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.060809+0000","last_change":"2023-09-23T18:40:42.047724+0000","last_active":"2023-09-23T18:41:08.060809+0000","last_peered":"2023-09-23T18:41:08.060809+0000","last_clean":"2023-09-23T18:41:08.060809+0000","last_became_active":"2023-09-23T18:40:42.047236+0000","last_became_peered":"2023-09-23T18:40:42.047236+0000","last_unstale":"2023-09-23T18:41:08.060809+0000","last_undegraded":"2023-09-23T18:41:08.060809+0000","last_fullsized":"2023-09-23T18:41:08.060809+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":141,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.042336+0000","last_change":"2023-09-23T18:28:03.298214+0000","last_active":"2023-09-23T18:41:08.042336+0000","last_peered":"2023-09-23T18:41:08.042336+0000","last_clean":"2023-09-23T18:41:08.042336+0000","last_became_active":"2023-09-23T18:28:03.297852+0000","last_became_peered":"2023-09-23T18:28:03.297852+0000","last_unstale":"2023-09-23T18:41:08.042336+0000","last_undegraded":"2023-09-23T18:41:08.042336+0000","last_fullsized":"2023-09-23T18:41:08.042336+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.146541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":148,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.060856+0000","last_change":"2023-09-23T18:28:03.296913+0000","last_active":"2023-09-23T18:41:08.060856+0000","last_peered":"2023-09-23T18:41:08.060856+0000","last_clean":"2023-09-23T18:41:08.060856+0000","last_became_active":"2023-09-23T18:28:03.296499+0000","last_became_peered":"2023-09-23T18:28:03.296499+0000","last_unstale":"2023-09-23T18:41:08.060856+0000","last_undegraded":"2023-09-23T18:41:08.060856+0000","last_fullsized":"2023-09-23T18:41:08.060856+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":113,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.039407+0000","last_change":"2023-09-23T18:33:53.850096+0000","last_active":"2023-09-23T18:41:08.039407+0000","last_peered":"2023-09-23T18:41:08.039407+0000","last_clean":"2023-09-23T18:41:08.039407+0000","last_became_active":"2023-09-23T18:33:53.849811+0000","last_became_peered":"2023-09-23T18:33:53.849811+0000","last_unstale":"2023-09-23T18:41:08.039407+0000","last_undegraded":"2023-09-23T18:41:08.039407+0000","last_fullsized":"2023-09-23T18:41:08.039407+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:44.938072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":152,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.058758+0000","last_change":"2023-09-23T18:15:08.909339+0000","last_active":"2023-09-23T18:41:08.058758+0000","last_peered":"2023-09-23T18:41:08.058758+0000","last_clean":"2023-09-23T18:41:08.058758+0000","last_became_active":"2023-09-23T18:15:08.908912+0000","last_became_peered":"2023-09-23T18:15:08.908912+0000","last_unstale":"2023-09-23T18:41:08.058758+0000","last_undegraded":"2023-09-23T18:41:08.058758+0000","last_fullsized":"2023-09-23T18:41:08.058758+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":159,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.050159+0000","last_change":"2023-09-23T18:33:53.847489+0000","last_active":"2023-09-23T18:41:08.050159+0000","last_peered":"2023-09-23T18:41:08.050159+0000","last_clean":"2023-09-23T18:41:08.050159+0000","last_became_active":"2023-09-23T18:33:53.846935+0000","last_became_peered":"2023-09-23T18:33:53.846935+0000","last_unstale":"2023-09-23T18:41:08.050159+0000","last_undegraded":"2023-09-23T18:41:08.050159+0000","last_fullsized":"2023-09-23T18:41:08.050159+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":147,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.042503+0000","last_change":"2023-09-23T18:27:33.585121+0000","last_active":"2023-09-23T18:41:08.042503+0000","last_peered":"2023-09-23T18:41:08.042503+0000","last_clean":"2023-09-23T18:41:08.042503+0000","last_became_active":"2023-09-23T18:27:33.584620+0000","last_became_peered":"2023-09-23T18:27:33.584620+0000","last_unstale":"2023-09-23T18:41:08.042503+0000","last_undegraded":"2023-09-23T18:41:08.042503+0000","last_fullsized":"2023-09-23T18:41:08.042503+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":166,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.042459+0000","last_change":"2023-09-23T18:33:53.852933+0000","last_active":"2023-09-23T18:41:08.042459+0000","last_peered":"2023-09-23T18:41:08.042459+0000","last_clean":"2023-09-23T18:41:08.042459+0000","last_became_active":"2023-09-23T18:33:53.852626+0000","last_became_peered":"2023-09-23T18:33:53.852626+0000","last_unstale":"2023-09-23T18:41:08.042459+0000","last_undegraded":"2023-09-23T18:41:08.042459+0000","last_fullsized":"2023-09-23T18:41:08.042459+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:00.174536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":141,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.034836+0000","last_change":"2023-09-23T18:39:46.997790+0000","last_active":"2023-09-23T18:41:08.034836+0000","last_peered":"2023-09-23T18:41:08.034836+0000","last_clean":"2023-09-23T18:41:08.034836+0000","last_became_active":"2023-09-23T18:39:46.997089+0000","last_became_peered":"2023-09-23T18:39:46.997089+0000","last_unstale":"2023-09-23T18:41:08.034836+0000","last_undegraded":"2023-09-23T18:41:08.034836+0000","last_fullsized":"2023-09-23T18:41:08.034836+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:55.378220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":136,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.060592+0000","last_change":"2023-09-23T18:32:26.808237+0000","last_active":"2023-09-23T18:41:08.060592+0000","last_peered":"2023-09-23T18:41:08.060592+0000","last_clean":"2023-09-23T18:41:08.060592+0000","last_became_active":"2023-09-23T18:32:26.807843+0000","last_became_peered":"2023-09-23T18:32:26.807843+0000","last_unstale":"2023-09-23T18:41:08.060592+0000","last_undegraded":"2023-09-23T18:41:08.060592+0000","last_fullsized":"2023-09-23T18:41:08.060592+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":130,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.042534+0000","last_change":"2023-09-23T18:15:08.916658+0000","last_active":"2023-09-23T18:41:08.042534+0000","last_peered":"2023-09-23T18:41:08.042534+0000","last_clean":"2023-09-23T18:41:08.042534+0000","last_became_active":"2023-09-23T18:15:08.916301+0000","last_became_peered":"2023-09-23T18:15:08.916301+0000","last_unstale":"2023-09-23T18:41:08.042534+0000","last_undegraded":"2023-09-23T18:41:08.042534+0000","last_fullsized":"2023-09-23T18:41:08.042534+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":142,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.034359+0000","last_change":"2023-09-23T18:40:42.049370+0000","last_active":"2023-09-23T18:41:08.034359+0000","last_peered":"2023-09-23T18:41:08.034359+0000","last_clean":"2023-09-23T18:41:08.034359+0000","last_became_active":"2023-09-23T18:40:42.049003+0000","last_became_peered":"2023-09-23T18:40:42.049003+0000","last_unstale":"2023-09-23T18:41:08.034359+0000","last_undegraded":"2023-09-23T18:41:08.034359+0000","last_fullsized":"2023-09-23T18:41:08.034359+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:15.760457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":191,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.048298+0000","last_change":"2023-09-23T18:40:42.073698+0000","last_active":"2023-09-23T18:41:08.048298+0000","last_peered":"2023-09-23T18:41:08.048298+0000","last_clean":"2023-09-23T18:41:08.048298+0000","last_became_active":"2023-09-23T18:40:42.059101+0000","last_became_peered":"2023-09-23T18:40:42.059101+0000","last_unstale":"2023-09-23T18:41:08.048298+0000","last_undegraded":"2023-09-23T18:41:08.048298+0000","last_fullsized":"2023-09-23T18:41:08.048298+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":121,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.060201+0000","last_change":"2023-09-23T18:40:42.058656+0000","last_active":"2023-09-23T18:41:08.060201+0000","last_peered":"2023-09-23T18:41:08.060201+0000","last_clean":"2023-09-23T18:41:08.060201+0000","last_became_active":"2023-09-23T18:40:42.057307+0000","last_became_peered":"2023-09-23T18:40:42.057307+0000","last_unstale":"2023-09-23T18:41:08.060201+0000","last_undegraded":"2023-09-23T18:41:08.060201+0000","last_fullsized":"2023-09-23T18:41:08.060201+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:08:45.979078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.a","version":"179'13229","reported_seq":15745,"reported_epoch":179,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:41:40.792967+0000","last_change":"2023-09-23T18:40:43.065755+0000","last_active":"2023-09-23T18:41:40.792967+0000","last_peered":"2023-09-23T18:41:40.792967+0000","last_clean":"2023-09-23T18:39:45.947537+0000","last_became_active":"2023-09-23T18:39:48.177324+0000","last_became_peered":"2023-09-23T18:39:48.177324+0000","last_unstale":"2023-09-23T18:41:40.792967+0000","last_undegraded":"2023-09-23T18:41:40.792967+0000","last_fullsized":"2023-09-23T18:41:40.792967+0000","mapping_epoch":162,"log_start":"159'7009","ondisk_log_start":"159'7009","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":6220,"log_dups_size":0,"ondisk_log_size":6220,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2162688,"num_objects":33,"num_object_clones":0,"num_object_copies":99,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":23,"num_objects_unfound":0,"num_objects_dirty":33,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13229,"num_write_kb":424384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1231,"num_bytes_recovered":80674816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,2],"avail_no_missing":["6","0","2"],"object_location_counts":[{"shards":"0,2,6","objects":33}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":159,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.060641+0000","last_change":"2023-09-23T18:37:41.034732+0000","last_active":"2023-09-23T18:41:08.060641+0000","last_peered":"2023-09-23T18:41:08.060641+0000","last_clean":"2023-09-23T18:41:08.060641+0000","last_became_active":"2023-09-23T18:37:40.985865+0000","last_became_peered":"2023-09-23T18:37:40.985865+0000","last_unstale":"2023-09-23T18:41:08.060641+0000","last_undegraded":"2023-09-23T18:41:08.060641+0000","last_fullsized":"2023-09-23T18:41:08.060641+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":77,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.041932+0000","last_change":"2023-09-23T18:32:26.806328+0000","last_active":"2023-09-23T18:41:08.041932+0000","last_peered":"2023-09-23T18:41:08.041932+0000","last_clean":"2023-09-23T18:41:08.041932+0000","last_became_active":"2023-09-23T18:32:26.803969+0000","last_became_peered":"2023-09-23T18:32:26.803969+0000","last_unstale":"2023-09-23T18:41:08.041932+0000","last_undegraded":"2023-09-23T18:41:08.041932+0000","last_fullsized":"2023-09-23T18:41:08.041932+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.b","version":"179'12970","reported_seq":13104,"reported_epoch":179,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:41:41.463955+0000","last_change":"2023-09-23T18:40:43.602362+0000","last_active":"2023-09-23T18:41:41.463955+0000","last_peered":"2023-09-23T18:41:41.463955+0000","last_clean":"2023-09-23T18:40:41.004574+0000","last_became_active":"2023-09-23T18:40:43.210705+0000","last_became_peered":"2023-09-23T18:40:43.210705+0000","last_unstale":"2023-09-23T18:41:41.463955+0000","last_undegraded":"2023-09-23T18:41:41.463955+0000","last_fullsized":"2023-09-23T18:41:41.463955+0000","mapping_epoch":172,"log_start":"156'2900","ondisk_log_start":"156'2900","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":10070,"log_dups_size":0,"ondisk_log_size":10070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:01.946732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12970,"num_write_kb":415040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38,"num_bytes_recovered":2424832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,7],"avail_no_missing":["3","1","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":162,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.060582+0000","last_change":"2023-09-23T18:33:53.847817+0000","last_active":"2023-09-23T18:41:08.060582+0000","last_peered":"2023-09-23T18:41:08.060582+0000","last_clean":"2023-09-23T18:41:08.060582+0000","last_became_active":"2023-09-23T18:33:53.847342+0000","last_became_peered":"2023-09-23T18:33:53.847342+0000","last_unstale":"2023-09-23T18:41:08.060582+0000","last_undegraded":"2023-09-23T18:41:08.060582+0000","last_fullsized":"2023-09-23T18:41:08.060582+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":183,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.049986+0000","last_change":"2023-09-23T18:39:46.995444+0000","last_active":"2023-09-23T18:41:08.049986+0000","last_peered":"2023-09-23T18:41:08.049986+0000","last_clean":"2023-09-23T18:41:08.049986+0000","last_became_active":"2023-09-23T18:39:46.995033+0000","last_became_peered":"2023-09-23T18:39:46.995033+0000","last_unstale":"2023-09-23T18:41:08.049986+0000","last_undegraded":"2023-09-23T18:41:08.049986+0000","last_fullsized":"2023-09-23T18:41:08.049986+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"179'13247","reported_seq":13285,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T18:41:40.833087+0000","last_change":"2023-09-23T18:37:10.998596+0000","last_active":"2023-09-23T18:41:40.833087+0000","last_peered":"2023-09-23T18:41:40.833087+0000","last_clean":"2023-09-23T18:41:40.833087+0000","last_became_active":"2023-09-23T18:37:10.997807+0000","last_became_peered":"2023-09-23T18:37:10.997807+0000","last_unstale":"2023-09-23T18:41:40.833087+0000","last_undegraded":"2023-09-23T18:41:40.833087+0000","last_fullsized":"2023-09-23T18:41:40.833087+0000","mapping_epoch":149,"log_start":"158'5662","ondisk_log_start":"158'5662","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":7585,"log_dups_size":0,"ondisk_log_size":7585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1638400,"num_objects":25,"num_object_clones":0,"num_object_copies":75,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13247,"num_write_kb":424704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":144,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.061128+0000","last_change":"2023-09-23T18:32:26.854615+0000","last_active":"2023-09-23T18:41:08.061128+0000","last_peered":"2023-09-23T18:41:08.061128+0000","last_clean":"2023-09-23T18:41:08.061128+0000","last_became_active":"2023-09-23T18:32:26.806771+0000","last_became_peered":"2023-09-23T18:32:26.806771+0000","last_unstale":"2023-09-23T18:41:08.061128+0000","last_undegraded":"2023-09-23T18:41:08.061128+0000","last_fullsized":"2023-09-23T18:41:08.061128+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":153,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.048701+0000","last_change":"2023-09-23T18:27:33.589116+0000","last_active":"2023-09-23T18:41:08.048701+0000","last_peered":"2023-09-23T18:41:08.048701+0000","last_clean":"2023-09-23T18:41:08.048701+0000","last_became_active":"2023-09-23T18:27:33.588809+0000","last_became_peered":"2023-09-23T18:27:33.588809+0000","last_unstale":"2023-09-23T18:41:08.048701+0000","last_undegraded":"2023-09-23T18:41:08.048701+0000","last_fullsized":"2023-09-23T18:41:08.048701+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"179'12942","reported_seq":20192,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T18:41:41.446855+0000","last_change":"2023-09-23T18:40:27.063756+0000","last_active":"2023-09-23T18:41:41.446855+0000","last_peered":"2023-09-23T18:41:41.446855+0000","last_clean":"2023-09-23T18:41:41.446855+0000","last_became_active":"2023-09-23T18:40:27.037582+0000","last_became_peered":"2023-09-23T18:40:27.037582+0000","last_unstale":"2023-09-23T18:41:41.446855+0000","last_undegraded":"2023-09-23T18:41:41.446855+0000","last_fullsized":"2023-09-23T18:41:41.446855+0000","mapping_epoch":167,"log_start":"158'5039","ondisk_log_start":"158'5039","created":149,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":7903,"log_dups_size":0,"ondisk_log_size":7903,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:57.463740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12942,"num_write_kb":414144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3593,"num_bytes_recovered":235470848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":152,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T18:41:27.048044+0000","last_change":"2023-09-23T18:33:54.848776+0000","last_active":"2023-09-23T18:41:27.048044+0000","last_peered":"2023-09-23T18:41:27.048044+0000","last_clean":"2023-09-23T18:41:27.048044+0000","last_became_active":"2023-09-23T18:33:54.848380+0000","last_became_peered":"2023-09-23T18:33:54.848380+0000","last_unstale":"2023-09-23T18:41:27.048044+0000","last_undegraded":"2023-09-23T18:41:27.048044+0000","last_fullsized":"2023-09-23T18:41:27.048044+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:17:17.093993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":145,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.033697+0000","last_change":"2023-09-23T18:27:33.582033+0000","last_active":"2023-09-23T18:41:08.033697+0000","last_peered":"2023-09-23T18:41:08.033697+0000","last_clean":"2023-09-23T18:41:08.033697+0000","last_became_active":"2023-09-23T18:27:33.581667+0000","last_became_peered":"2023-09-23T18:27:33.581667+0000","last_unstale":"2023-09-23T18:41:08.033697+0000","last_undegraded":"2023-09-23T18:41:08.033697+0000","last_fullsized":"2023-09-23T18:41:08.033697+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:36.039922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"179'13314","reported_seq":19413,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T18:41:41.212794+0000","last_change":"2023-09-23T18:40:38.050425+0000","last_active":"2023-09-23T18:41:41.212794+0000","last_peered":"2023-09-23T18:41:41.212794+0000","last_clean":"2023-09-23T18:41:41.212794+0000","last_became_active":"2023-09-23T18:40:38.019360+0000","last_became_peered":"2023-09-23T18:40:38.019360+0000","last_unstale":"2023-09-23T18:41:41.212794+0000","last_undegraded":"2023-09-23T18:41:41.212794+0000","last_fullsized":"2023-09-23T18:41:41.212794+0000","mapping_epoch":169,"log_start":"158'6292","ondisk_log_start":"158'6292","created":149,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":7022,"log_dups_size":0,"ondisk_log_size":7022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":917504,"num_objects":14,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13314,"num_write_kb":426496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3019,"num_bytes_recovered":197853184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":151,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T18:41:27.048227+0000","last_change":"2023-09-23T18:28:03.297628+0000","last_active":"2023-09-23T18:41:27.048227+0000","last_peered":"2023-09-23T18:41:27.048227+0000","last_clean":"2023-09-23T18:41:27.048227+0000","last_became_active":"2023-09-23T18:28:03.296694+0000","last_became_peered":"2023-09-23T18:28:03.296694+0000","last_unstale":"2023-09-23T18:41:27.048227+0000","last_undegraded":"2023-09-23T18:41:27.048227+0000","last_fullsized":"2023-09-23T18:41:27.048227+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:43.375652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":144,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.050274+0000","last_change":"2023-09-23T18:28:03.302326+0000","last_active":"2023-09-23T18:41:08.050274+0000","last_peered":"2023-09-23T18:41:08.050274+0000","last_clean":"2023-09-23T18:41:08.050274+0000","last_became_active":"2023-09-23T18:28:03.301970+0000","last_became_peered":"2023-09-23T18:28:03.301970+0000","last_unstale":"2023-09-23T18:41:08.050274+0000","last_undegraded":"2023-09-23T18:41:08.050274+0000","last_fullsized":"2023-09-23T18:41:08.050274+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"179'13023","reported_seq":19921,"reported_epoch":179,"state":"active+recovering+remapped","last_fresh":"2023-09-23T18:41:41.219948+0000","last_change":"2023-09-23T18:40:43.613884+0000","last_active":"2023-09-23T18:41:41.219948+0000","last_peered":"2023-09-23T18:41:41.219948+0000","last_clean":"2023-09-23T18:39:45.938193+0000","last_became_active":"2023-09-23T18:40:43.289840+0000","last_became_peered":"2023-09-23T18:40:43.289840+0000","last_unstale":"2023-09-23T18:41:41.219948+0000","last_undegraded":"2023-09-23T18:41:41.219948+0000","last_fullsized":"2023-09-23T18:41:41.219948+0000","mapping_epoch":172,"log_start":"158'6027","ondisk_log_start":"158'6027","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":6996,"log_dups_size":0,"ondisk_log_size":6996,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":851968,"num_objects":13,"num_object_clones":0,"num_object_copies":39,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13023,"num_write_kb":417152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3424,"num_bytes_recovered":222691328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,3],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"3,4,6","objects":13}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":160,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.048502+0000","last_change":"2023-09-23T18:33:53.855371+0000","last_active":"2023-09-23T18:41:08.048502+0000","last_peered":"2023-09-23T18:41:08.048502+0000","last_clean":"2023-09-23T18:41:08.048502+0000","last_became_active":"2023-09-23T18:33:53.854996+0000","last_became_peered":"2023-09-23T18:33:53.854996+0000","last_unstale":"2023-09-23T18:41:08.048502+0000","last_undegraded":"2023-09-23T18:41:08.048502+0000","last_fullsized":"2023-09-23T18:41:08.048502+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":183,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.050196+0000","last_change":"2023-09-23T18:40:42.060184+0000","last_active":"2023-09-23T18:41:08.050196+0000","last_peered":"2023-09-23T18:41:08.050196+0000","last_clean":"2023-09-23T18:41:08.050196+0000","last_became_active":"2023-09-23T18:40:42.059725+0000","last_became_peered":"2023-09-23T18:40:42.059725+0000","last_unstale":"2023-09-23T18:41:08.050196+0000","last_undegraded":"2023-09-23T18:41:08.050196+0000","last_fullsized":"2023-09-23T18:41:08.050196+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"179'12856","reported_seq":12903,"reported_epoch":179,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:41:40.808579+0000","last_change":"2023-09-23T18:40:43.333905+0000","last_active":"2023-09-23T18:41:40.808579+0000","last_peered":"2023-09-23T18:41:40.808579+0000","last_clean":"2023-09-23T18:40:40.957158+0000","last_became_active":"2023-09-23T18:40:43.247690+0000","last_became_peered":"2023-09-23T18:40:43.247690+0000","last_unstale":"2023-09-23T18:41:40.808579+0000","last_undegraded":"2023-09-23T18:41:40.808579+0000","last_fullsized":"2023-09-23T18:41:40.808579+0000","mapping_epoch":172,"log_start":"159'6698","ondisk_log_start":"159'6698","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":6158,"log_dups_size":0,"ondisk_log_size":6158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1572864,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":24,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12856,"num_write_kb":412160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,3],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":24}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":145,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.049861+0000","last_change":"2023-09-23T18:27:33.585714+0000","last_active":"2023-09-23T18:41:08.049861+0000","last_peered":"2023-09-23T18:41:08.049861+0000","last_clean":"2023-09-23T18:41:08.049861+0000","last_became_active":"2023-09-23T18:27:33.584641+0000","last_became_peered":"2023-09-23T18:27:33.584641+0000","last_unstale":"2023-09-23T18:41:08.049861+0000","last_undegraded":"2023-09-23T18:41:08.049861+0000","last_fullsized":"2023-09-23T18:41:08.049861+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":159,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.058542+0000","last_change":"2023-09-23T18:33:53.851266+0000","last_active":"2023-09-23T18:41:08.058542+0000","last_peered":"2023-09-23T18:41:08.058542+0000","last_clean":"2023-09-23T18:41:08.058542+0000","last_became_active":"2023-09-23T18:33:53.850926+0000","last_became_peered":"2023-09-23T18:33:53.850926+0000","last_unstale":"2023-09-23T18:41:08.058542+0000","last_undegraded":"2023-09-23T18:41:08.058542+0000","last_fullsized":"2023-09-23T18:41:08.058542+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"179'12706","reported_seq":12748,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T18:41:41.479681+0000","last_change":"2023-09-23T18:37:10.994424+0000","last_active":"2023-09-23T18:41:41.479681+0000","last_peered":"2023-09-23T18:41:41.479681+0000","last_clean":"2023-09-23T18:41:41.479681+0000","last_became_active":"2023-09-23T18:37:10.994110+0000","last_became_peered":"2023-09-23T18:37:10.994110+0000","last_unstale":"2023-09-23T18:41:41.479681+0000","last_undegraded":"2023-09-23T18:41:41.479681+0000","last_fullsized":"2023-09-23T18:41:41.479681+0000","mapping_epoch":149,"log_start":"159'7141","ondisk_log_start":"159'7141","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":5565,"log_dups_size":0,"ondisk_log_size":5565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:05.421139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12706,"num_write_kb":406529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":129,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.045196+0000","last_change":"2023-09-23T18:39:46.993941+0000","last_active":"2023-09-23T18:41:08.045196+0000","last_peered":"2023-09-23T18:41:08.045196+0000","last_clean":"2023-09-23T18:41:08.045196+0000","last_became_active":"2023-09-23T18:39:46.993625+0000","last_became_peered":"2023-09-23T18:39:46.993625+0000","last_unstale":"2023-09-23T18:41:08.045196+0000","last_undegraded":"2023-09-23T18:41:08.045196+0000","last_fullsized":"2023-09-23T18:41:08.045196+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:23.237229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":117,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.060036+0000","last_change":"2023-09-23T18:33:53.855430+0000","last_active":"2023-09-23T18:41:08.060036+0000","last_peered":"2023-09-23T18:41:08.060036+0000","last_clean":"2023-09-23T18:41:08.060036+0000","last_became_active":"2023-09-23T18:33:53.854935+0000","last_became_peered":"2023-09-23T18:33:53.854935+0000","last_unstale":"2023-09-23T18:41:08.060036+0000","last_undegraded":"2023-09-23T18:41:08.060036+0000","last_fullsized":"2023-09-23T18:41:08.060036+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:36.078411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"179'13282","reported_seq":14444,"reported_epoch":179,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:41:41.380148+0000","last_change":"2023-09-23T18:40:43.466170+0000","last_active":"2023-09-23T18:41:41.380148+0000","last_peered":"2023-09-23T18:41:41.380148+0000","last_clean":"2023-09-23T18:39:45.937758+0000","last_became_active":"2023-09-23T18:39:48.161207+0000","last_became_peered":"2023-09-23T18:39:48.161207+0000","last_unstale":"2023-09-23T18:41:41.380148+0000","last_undegraded":"2023-09-23T18:41:41.380148+0000","last_fullsized":"2023-09-23T18:41:41.380148+0000","mapping_epoch":162,"log_start":"158'5342","ondisk_log_start":"158'5342","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":7940,"log_dups_size":0,"ondisk_log_size":7940,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:44:20.712412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13282,"num_write_kb":425024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":551,"num_bytes_recovered":36110336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,2],"avail_no_missing":["0","1","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"77'130","reported_seq":643,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.047400+0000","last_change":"2023-09-23T18:33:53.850713+0000","last_active":"2023-09-23T18:41:08.047400+0000","last_peered":"2023-09-23T18:41:08.047400+0000","last_clean":"2023-09-23T18:41:08.047400+0000","last_became_active":"2023-09-23T18:33:53.850011+0000","last_became_peered":"2023-09-23T18:33:53.850011+0000","last_unstale":"2023-09-23T18:41:08.047400+0000","last_undegraded":"2023-09-23T18:41:08.047400+0000","last_fullsized":"2023-09-23T18:41:08.047400+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:58.943556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":772,"num_read_kb":1712,"num_write":126,"num_write_kb":2126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":98,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.042812+0000","last_change":"2023-09-23T18:40:42.058197+0000","last_active":"2023-09-23T18:41:08.042812+0000","last_peered":"2023-09-23T18:41:08.042812+0000","last_clean":"2023-09-23T18:41:08.042812+0000","last_became_active":"2023-09-23T18:40:42.057834+0000","last_became_peered":"2023-09-23T18:40:42.057834+0000","last_unstale":"2023-09-23T18:41:08.042812+0000","last_undegraded":"2023-09-23T18:41:08.042812+0000","last_fullsized":"2023-09-23T18:41:08.042812+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:28.133529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":91,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.039490+0000","last_change":"2023-09-23T18:33:54.843246+0000","last_active":"2023-09-23T18:41:08.039490+0000","last_peered":"2023-09-23T18:41:08.039490+0000","last_clean":"2023-09-23T18:41:08.039490+0000","last_became_active":"2023-09-23T18:33:54.842933+0000","last_became_peered":"2023-09-23T18:33:54.842933+0000","last_unstale":"2023-09-23T18:41:08.039490+0000","last_undegraded":"2023-09-23T18:41:08.039490+0000","last_fullsized":"2023-09-23T18:41:08.039490+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:43.716707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"179'12950","reported_seq":12999,"reported_epoch":179,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:41:41.215039+0000","last_change":"2023-09-23T18:40:43.275615+0000","last_active":"2023-09-23T18:41:41.215039+0000","last_peered":"2023-09-23T18:41:41.215039+0000","last_clean":"2023-09-23T18:40:41.005193+0000","last_became_active":"2023-09-23T18:40:43.229395+0000","last_became_peered":"2023-09-23T18:40:43.229395+0000","last_unstale":"2023-09-23T18:41:41.215039+0000","last_undegraded":"2023-09-23T18:41:41.215039+0000","last_fullsized":"2023-09-23T18:41:41.215039+0000","mapping_epoch":172,"log_start":"158'5886","ondisk_log_start":"158'5886","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":7064,"log_dups_size":0,"ondisk_log_size":7064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":786432,"num_objects":12,"num_object_clones":0,"num_object_copies":36,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":12,"num_objects_unfound":0,"num_objects_dirty":12,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12950,"num_write_kb":414784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,3],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":12}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"77'96","reported_seq":587,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.039754+0000","last_change":"2023-09-23T18:33:54.842857+0000","last_active":"2023-09-23T18:41:08.039754+0000","last_peered":"2023-09-23T18:41:08.039754+0000","last_clean":"2023-09-23T18:41:08.039754+0000","last_became_active":"2023-09-23T18:33:54.842134+0000","last_became_peered":"2023-09-23T18:33:54.842134+0000","last_unstale":"2023-09-23T18:41:08.039754+0000","last_undegraded":"2023-09-23T18:41:08.039754+0000","last_fullsized":"2023-09-23T18:41:08.039754+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":96,"log_dups_size":0,"ondisk_log_size":96,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:48.960158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":728,"num_read_kb":1234,"num_write":87,"num_write_kb":338,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":145,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.050157+0000","last_change":"2023-09-23T18:26:34.992535+0000","last_active":"2023-09-23T18:41:08.050157+0000","last_peered":"2023-09-23T18:41:08.050157+0000","last_clean":"2023-09-23T18:41:08.050157+0000","last_became_active":"2023-09-23T18:26:34.992177+0000","last_became_peered":"2023-09-23T18:26:34.992177+0000","last_unstale":"2023-09-23T18:41:08.050157+0000","last_undegraded":"2023-09-23T18:41:08.050157+0000","last_fullsized":"2023-09-23T18:41:08.050157+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":141,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.047321+0000","last_change":"2023-09-23T18:40:42.066006+0000","last_active":"2023-09-23T18:41:08.047321+0000","last_peered":"2023-09-23T18:41:08.047321+0000","last_clean":"2023-09-23T18:41:08.047321+0000","last_became_active":"2023-09-23T18:40:42.065431+0000","last_became_peered":"2023-09-23T18:40:42.065431+0000","last_unstale":"2023-09-23T18:41:08.047321+0000","last_undegraded":"2023-09-23T18:41:08.047321+0000","last_fullsized":"2023-09-23T18:41:08.047321+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:18.818629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"179'13023","reported_seq":13061,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T18:41:40.815478+0000","last_change":"2023-09-23T18:37:10.998898+0000","last_active":"2023-09-23T18:41:40.815478+0000","last_peered":"2023-09-23T18:41:40.815478+0000","last_clean":"2023-09-23T18:41:40.815478+0000","last_became_active":"2023-09-23T18:37:10.998064+0000","last_became_peered":"2023-09-23T18:37:10.998064+0000","last_unstale":"2023-09-23T18:41:40.815478+0000","last_undegraded":"2023-09-23T18:41:40.815478+0000","last_fullsized":"2023-09-23T18:41:40.815478+0000","mapping_epoch":149,"log_start":"158'5465","ondisk_log_start":"158'5465","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":7558,"log_dups_size":0,"ondisk_log_size":7558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1245184,"num_objects":19,"num_object_clones":0,"num_object_copies":57,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13023,"num_write_kb":417344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"77'112","reported_seq":528,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.035040+0000","last_change":"2023-09-23T18:40:45.106420+0000","last_active":"2023-09-23T18:41:08.035040+0000","last_peered":"2023-09-23T18:41:08.035040+0000","last_clean":"2023-09-23T18:41:08.035040+0000","last_became_active":"2023-09-23T18:40:45.104294+0000","last_became_peered":"2023-09-23T18:40:45.104294+0000","last_unstale":"2023-09-23T18:41:08.035040+0000","last_undegraded":"2023-09-23T18:41:08.035040+0000","last_fullsized":"2023-09-23T18:41:08.035040+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:45.419412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":774,"num_read_kb":596,"num_write":102,"num_write_kb":52,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":113,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.045007+0000","last_change":"2023-09-23T18:33:53.849084+0000","last_active":"2023-09-23T18:41:08.045007+0000","last_peered":"2023-09-23T18:41:08.045007+0000","last_clean":"2023-09-23T18:41:08.045007+0000","last_became_active":"2023-09-23T18:33:53.848757+0000","last_became_peered":"2023-09-23T18:33:53.848757+0000","last_unstale":"2023-09-23T18:41:08.045007+0000","last_undegraded":"2023-09-23T18:41:08.045007+0000","last_fullsized":"2023-09-23T18:41:08.045007+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:15.267742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":136,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.060157+0000","last_change":"2023-09-23T18:33:55.087304+0000","last_active":"2023-09-23T18:41:08.060157+0000","last_peered":"2023-09-23T18:41:08.060157+0000","last_clean":"2023-09-23T18:41:08.060157+0000","last_became_active":"2023-09-23T18:33:55.086832+0000","last_became_peered":"2023-09-23T18:33:55.086832+0000","last_unstale":"2023-09-23T18:41:08.060157+0000","last_undegraded":"2023-09-23T18:41:08.060157+0000","last_fullsized":"2023-09-23T18:41:08.060157+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:55.457664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"179'13098","reported_seq":13139,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T18:41:41.473194+0000","last_change":"2023-09-23T18:37:11.002449+0000","last_active":"2023-09-23T18:41:41.473194+0000","last_peered":"2023-09-23T18:41:41.473194+0000","last_clean":"2023-09-23T18:41:41.473194+0000","last_became_active":"2023-09-23T18:37:11.002022+0000","last_became_peered":"2023-09-23T18:37:11.002022+0000","last_unstale":"2023-09-23T18:41:41.473194+0000","last_undegraded":"2023-09-23T18:41:41.473194+0000","last_fullsized":"2023-09-23T18:41:41.473194+0000","mapping_epoch":149,"log_start":"156'3000","ondisk_log_start":"156'3000","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":10098,"log_dups_size":0,"ondisk_log_size":10098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:34.756717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13098,"num_write_kb":419136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"77'147","reported_seq":568,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.042985+0000","last_change":"2023-09-23T18:32:29.845404+0000","last_active":"2023-09-23T18:41:08.042985+0000","last_peered":"2023-09-23T18:41:08.042985+0000","last_clean":"2023-09-23T18:41:08.042985+0000","last_became_active":"2023-09-23T18:32:29.844545+0000","last_became_peered":"2023-09-23T18:32:29.844545+0000","last_unstale":"2023-09-23T18:41:08.042985+0000","last_undegraded":"2023-09-23T18:41:08.042985+0000","last_fullsized":"2023-09-23T18:41:08.042985+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:44.834482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":624,"num_read_kb":1868,"num_write":143,"num_write_kb":5368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":381,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.042125+0000","last_change":"2023-09-23T18:26:34.999402+0000","last_active":"2023-09-23T18:41:08.042125+0000","last_peered":"2023-09-23T18:41:08.042125+0000","last_clean":"2023-09-23T18:41:08.042125+0000","last_became_active":"2023-09-23T18:26:34.998828+0000","last_became_peered":"2023-09-23T18:26:34.998828+0000","last_unstale":"2023-09-23T18:41:08.042125+0000","last_undegraded":"2023-09-23T18:41:08.042125+0000","last_fullsized":"2023-09-23T18:41:08.042125+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_clean_scrub_stamp":"2023-09-23T18:08:35.701374+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":94,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.048712+0000","last_change":"2023-09-23T18:40:42.058559+0000","last_active":"2023-09-23T18:41:08.048712+0000","last_peered":"2023-09-23T18:41:08.048712+0000","last_clean":"2023-09-23T18:41:08.048712+0000","last_became_active":"2023-09-23T18:40:42.058253+0000","last_became_peered":"2023-09-23T18:40:42.058253+0000","last_unstale":"2023-09-23T18:41:08.048712+0000","last_undegraded":"2023-09-23T18:41:08.048712+0000","last_fullsized":"2023-09-23T18:41:08.048712+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":129,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.048728+0000","last_change":"2023-09-23T18:15:08.921629+0000","last_active":"2023-09-23T18:41:08.048728+0000","last_peered":"2023-09-23T18:41:08.048728+0000","last_clean":"2023-09-23T18:41:08.048728+0000","last_became_active":"2023-09-23T18:15:08.921283+0000","last_became_peered":"2023-09-23T18:15:08.921283+0000","last_unstale":"2023-09-23T18:41:08.048728+0000","last_undegraded":"2023-09-23T18:41:08.048728+0000","last_fullsized":"2023-09-23T18:41:08.048728+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"179'13336","reported_seq":17560,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T18:41:41.451264+0000","last_change":"2023-09-23T18:41:06.046755+0000","last_active":"2023-09-23T18:41:41.451264+0000","last_peered":"2023-09-23T18:41:41.451264+0000","last_clean":"2023-09-23T18:41:41.451264+0000","last_became_active":"2023-09-23T18:41:06.015048+0000","last_became_peered":"2023-09-23T18:41:06.015048+0000","last_unstale":"2023-09-23T18:41:41.451264+0000","last_undegraded":"2023-09-23T18:41:41.451264+0000","last_fullsized":"2023-09-23T18:41:41.451264+0000","mapping_epoch":176,"log_start":"160'7725","ondisk_log_start":"160'7725","created":149,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":5611,"log_dups_size":0,"ondisk_log_size":5611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:13.314019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13336,"num_write_kb":426752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2080,"num_bytes_recovered":136314880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"77'120","reported_seq":640,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.042136+0000","last_change":"2023-09-23T18:33:53.852697+0000","last_active":"2023-09-23T18:41:08.042136+0000","last_peered":"2023-09-23T18:41:08.042136+0000","last_clean":"2023-09-23T18:41:08.042136+0000","last_became_active":"2023-09-23T18:33:53.852210+0000","last_became_peered":"2023-09-23T18:33:53.852210+0000","last_unstale":"2023-09-23T18:41:08.042136+0000","last_undegraded":"2023-09-23T18:41:08.042136+0000","last_fullsized":"2023-09-23T18:41:08.042136+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":120,"log_dups_size":0,"ondisk_log_size":120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":809,"num_read_kb":2749,"num_write":98,"num_write_kb":2087,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":142,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:09.953449+0000","last_change":"2023-09-23T18:39:46.997952+0000","last_active":"2023-09-23T18:41:09.953449+0000","last_peered":"2023-09-23T18:41:09.953449+0000","last_clean":"2023-09-23T18:41:09.953449+0000","last_became_active":"2023-09-23T18:39:46.997589+0000","last_became_peered":"2023-09-23T18:39:46.997589+0000","last_unstale":"2023-09-23T18:41:09.953449+0000","last_undegraded":"2023-09-23T18:41:09.953449+0000","last_fullsized":"2023-09-23T18:41:09.953449+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:09.887469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":173,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T18:41:27.052258+0000","last_change":"2023-09-23T18:39:46.996973+0000","last_active":"2023-09-23T18:41:27.052258+0000","last_peered":"2023-09-23T18:41:27.052258+0000","last_clean":"2023-09-23T18:41:27.052258+0000","last_became_active":"2023-09-23T18:39:46.996633+0000","last_became_peered":"2023-09-23T18:39:46.996633+0000","last_unstale":"2023-09-23T18:41:27.052258+0000","last_undegraded":"2023-09-23T18:41:27.052258+0000","last_fullsized":"2023-09-23T18:41:27.052258+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:08:05.607474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"179'13328","reported_seq":20932,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T18:41:41.335489+0000","last_change":"2023-09-23T18:40:23.040135+0000","last_active":"2023-09-23T18:41:41.335489+0000","last_peered":"2023-09-23T18:41:41.335489+0000","last_clean":"2023-09-23T18:41:41.335489+0000","last_became_active":"2023-09-23T18:40:23.008800+0000","last_became_peered":"2023-09-23T18:40:23.008800+0000","last_unstale":"2023-09-23T18:41:41.335489+0000","last_undegraded":"2023-09-23T18:41:41.335489+0000","last_fullsized":"2023-09-23T18:41:41.335489+0000","mapping_epoch":165,"log_start":"160'7769","ondisk_log_start":"160'7769","created":149,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":5559,"log_dups_size":0,"ondisk_log_size":5559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:03.284396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13328,"num_write_kb":426496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3770,"num_bytes_recovered":247070720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'78","reported_seq":415,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.060988+0000","last_change":"2023-09-23T18:26:34.992345+0000","last_active":"2023-09-23T18:41:08.060988+0000","last_peered":"2023-09-23T18:41:08.060988+0000","last_clean":"2023-09-23T18:41:08.060988+0000","last_became_active":"2023-09-23T18:26:34.991774+0000","last_became_peered":"2023-09-23T18:26:34.991774+0000","last_unstale":"2023-09-23T18:41:08.060988+0000","last_undegraded":"2023-09-23T18:41:08.060988+0000","last_fullsized":"2023-09-23T18:41:08.060988+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":78,"log_dups_size":0,"ondisk_log_size":78,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":428,"num_read_kb":1125,"num_write":73,"num_write_kb":431,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":120,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.047484+0000","last_change":"2023-09-23T18:39:46.990343+0000","last_active":"2023-09-23T18:41:08.047484+0000","last_peered":"2023-09-23T18:41:08.047484+0000","last_clean":"2023-09-23T18:41:08.047484+0000","last_became_active":"2023-09-23T18:39:46.989904+0000","last_became_peered":"2023-09-23T18:39:46.989904+0000","last_unstale":"2023-09-23T18:41:08.047484+0000","last_undegraded":"2023-09-23T18:41:08.047484+0000","last_fullsized":"2023-09-23T18:41:08.047484+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:15.638308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":139,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:09.953524+0000","last_change":"2023-09-23T18:33:54.852309+0000","last_active":"2023-09-23T18:41:09.953524+0000","last_peered":"2023-09-23T18:41:09.953524+0000","last_clean":"2023-09-23T18:41:09.953524+0000","last_became_active":"2023-09-23T18:33:54.850255+0000","last_became_peered":"2023-09-23T18:33:54.850255+0000","last_unstale":"2023-09-23T18:41:09.953524+0000","last_undegraded":"2023-09-23T18:41:09.953524+0000","last_fullsized":"2023-09-23T18:41:09.953524+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:54.148992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"179'12763","reported_seq":12929,"reported_epoch":179,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:41:41.165971+0000","last_change":"2023-09-23T18:40:43.280592+0000","last_active":"2023-09-23T18:41:41.165971+0000","last_peered":"2023-09-23T18:41:41.165971+0000","last_clean":"2023-09-23T18:39:45.961375+0000","last_became_active":"2023-09-23T18:39:48.132711+0000","last_became_peered":"2023-09-23T18:39:48.132711+0000","last_unstale":"2023-09-23T18:41:41.165971+0000","last_undegraded":"2023-09-23T18:41:41.165971+0000","last_fullsized":"2023-09-23T18:41:41.165971+0000","mapping_epoch":162,"log_start":"158'5761","ondisk_log_start":"158'5761","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":7002,"log_dups_size":0,"ondisk_log_size":7002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":589824,"num_objects":9,"num_object_clones":0,"num_object_copies":27,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":9,"num_objects_unfound":0,"num_objects_dirty":9,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12763,"num_write_kb":408704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":3604480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,3],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"2,3,4","objects":9}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":151,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.053474+0000","last_change":"2023-09-23T18:28:03.298805+0000","last_active":"2023-09-23T18:41:08.053474+0000","last_peered":"2023-09-23T18:41:08.053474+0000","last_clean":"2023-09-23T18:41:08.053474+0000","last_became_active":"2023-09-23T18:28:03.298437+0000","last_became_peered":"2023-09-23T18:28:03.298437+0000","last_unstale":"2023-09-23T18:41:08.053474+0000","last_undegraded":"2023-09-23T18:41:08.053474+0000","last_fullsized":"2023-09-23T18:41:08.053474+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:27.831669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'181","reported_seq":651,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.040245+0000","last_change":"2023-09-23T18:33:53.854528+0000","last_active":"2023-09-23T18:41:08.040245+0000","last_peered":"2023-09-23T18:41:08.040245+0000","last_clean":"2023-09-23T18:41:08.040245+0000","last_became_active":"2023-09-23T18:33:53.851883+0000","last_became_peered":"2023-09-23T18:33:53.851883+0000","last_unstale":"2023-09-23T18:41:08.040245+0000","last_undegraded":"2023-09-23T18:41:08.040245+0000","last_fullsized":"2023-09-23T18:41:08.040245+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":181,"log_dups_size":0,"ondisk_log_size":181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:43.888844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":909,"num_read_kb":734,"num_write":148,"num_write_kb":75,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":87,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.042035+0000","last_change":"2023-09-23T18:39:46.992015+0000","last_active":"2023-09-23T18:41:08.042035+0000","last_peered":"2023-09-23T18:41:08.042035+0000","last_clean":"2023-09-23T18:41:08.042035+0000","last_became_active":"2023-09-23T18:39:46.991605+0000","last_became_peered":"2023-09-23T18:39:46.991605+0000","last_unstale":"2023-09-23T18:41:08.042035+0000","last_undegraded":"2023-09-23T18:41:08.042035+0000","last_fullsized":"2023-09-23T18:41:08.042035+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.3","version":"179'13083","reported_seq":13121,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T18:41:40.894002+0000","last_change":"2023-09-23T18:37:10.998526+0000","last_active":"2023-09-23T18:41:40.894002+0000","last_peered":"2023-09-23T18:41:40.894002+0000","last_clean":"2023-09-23T18:41:40.894002+0000","last_became_active":"2023-09-23T18:37:10.998129+0000","last_became_peered":"2023-09-23T18:37:10.998129+0000","last_unstale":"2023-09-23T18:41:40.894002+0000","last_undegraded":"2023-09-23T18:41:40.894002+0000","last_fullsized":"2023-09-23T18:41:40.894002+0000","mapping_epoch":149,"log_start":"158'5519","ondisk_log_start":"158'5519","created":149,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:37:09.971477+0000","last_clean_scrub_stamp":"2023-09-23T18:37:09.971477+0000","objects_scrubbed":0,"log_size":7564,"log_dups_size":0,"ondisk_log_size":7564,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2293760,"num_objects":35,"num_object_clones":0,"num_object_copies":105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":35,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13083,"num_write_kb":419776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"77'112","reported_seq":712,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.068346+0000","last_change":"2023-09-23T18:40:45.086639+0000","last_active":"2023-09-23T18:41:08.068346+0000","last_peered":"2023-09-23T18:41:08.068346+0000","last_clean":"2023-09-23T18:41:08.068346+0000","last_became_active":"2023-09-23T18:40:45.086119+0000","last_became_peered":"2023-09-23T18:40:45.086119+0000","last_unstale":"2023-09-23T18:41:08.068346+0000","last_undegraded":"2023-09-23T18:41:08.068346+0000","last_fullsized":"2023-09-23T18:41:08.068346+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":893,"num_read_kb":3172,"num_write":69,"num_write_kb":2237,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":152,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T18:41:27.052181+0000","last_change":"2023-09-23T18:33:53.851421+0000","last_active":"2023-09-23T18:41:27.052181+0000","last_peered":"2023-09-23T18:41:27.052181+0000","last_clean":"2023-09-23T18:41:27.052181+0000","last_became_active":"2023-09-23T18:33:53.851012+0000","last_became_peered":"2023-09-23T18:33:53.851012+0000","last_unstale":"2023-09-23T18:41:27.052181+0000","last_undegraded":"2023-09-23T18:41:27.052181+0000","last_fullsized":"2023-09-23T18:41:27.052181+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:03.971054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":138,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.049920+0000","last_change":"2023-09-23T18:32:26.805472+0000","last_active":"2023-09-23T18:41:08.049920+0000","last_peered":"2023-09-23T18:41:08.049920+0000","last_clean":"2023-09-23T18:41:08.049920+0000","last_became_active":"2023-09-23T18:32:26.805078+0000","last_became_peered":"2023-09-23T18:32:26.805078+0000","last_unstale":"2023-09-23T18:41:08.049920+0000","last_undegraded":"2023-09-23T18:41:08.049920+0000","last_fullsized":"2023-09-23T18:41:08.049920+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":121,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T18:41:27.051526+0000","last_change":"2023-09-23T18:26:34.994176+0000","last_active":"2023-09-23T18:41:27.051526+0000","last_peered":"2023-09-23T18:41:27.051526+0000","last_clean":"2023-09-23T18:41:27.051526+0000","last_became_active":"2023-09-23T18:26:34.993829+0000","last_became_peered":"2023-09-23T18:26:34.993829+0000","last_unstale":"2023-09-23T18:41:27.051526+0000","last_undegraded":"2023-09-23T18:41:27.051526+0000","last_fullsized":"2023-09-23T18:41:27.051526+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:27.111857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":157,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.047561+0000","last_change":"2023-09-23T18:40:42.053012+0000","last_active":"2023-09-23T18:41:08.047561+0000","last_peered":"2023-09-23T18:41:08.047561+0000","last_clean":"2023-09-23T18:41:08.047561+0000","last_became_active":"2023-09-23T18:40:42.052655+0000","last_became_peered":"2023-09-23T18:40:42.052655+0000","last_unstale":"2023-09-23T18:41:08.047561+0000","last_undegraded":"2023-09-23T18:41:08.047561+0000","last_fullsized":"2023-09-23T18:41:08.047561+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:33:51.009383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":102,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.050188+0000","last_change":"2023-09-23T18:33:54.847172+0000","last_active":"2023-09-23T18:41:08.050188+0000","last_peered":"2023-09-23T18:41:08.050188+0000","last_clean":"2023-09-23T18:41:08.050188+0000","last_became_active":"2023-09-23T18:33:54.846751+0000","last_became_peered":"2023-09-23T18:33:54.846751+0000","last_unstale":"2023-09-23T18:41:08.050188+0000","last_undegraded":"2023-09-23T18:41:08.050188+0000","last_fullsized":"2023-09-23T18:41:08.050188+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":174,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.050256+0000","last_change":"2023-09-23T18:33:53.841232+0000","last_active":"2023-09-23T18:41:08.050256+0000","last_peered":"2023-09-23T18:41:08.050256+0000","last_clean":"2023-09-23T18:41:08.050256+0000","last_became_active":"2023-09-23T18:33:53.840749+0000","last_became_peered":"2023-09-23T18:33:53.840749+0000","last_unstale":"2023-09-23T18:41:08.050256+0000","last_undegraded":"2023-09-23T18:41:08.050256+0000","last_fullsized":"2023-09-23T18:41:08.050256+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":102,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.050334+0000","last_change":"2023-09-23T18:33:54.858971+0000","last_active":"2023-09-23T18:41:08.050334+0000","last_peered":"2023-09-23T18:41:08.050334+0000","last_clean":"2023-09-23T18:41:08.050334+0000","last_became_active":"2023-09-23T18:33:54.845921+0000","last_became_peered":"2023-09-23T18:33:54.845921+0000","last_unstale":"2023-09-23T18:41:08.050334+0000","last_undegraded":"2023-09-23T18:41:08.050334+0000","last_fullsized":"2023-09-23T18:41:08.050334+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":159,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.050129+0000","last_change":"2023-09-23T18:28:03.302261+0000","last_active":"2023-09-23T18:41:08.050129+0000","last_peered":"2023-09-23T18:41:08.050129+0000","last_clean":"2023-09-23T18:41:08.050129+0000","last_became_active":"2023-09-23T18:28:03.301911+0000","last_became_peered":"2023-09-23T18:28:03.301911+0000","last_unstale":"2023-09-23T18:41:08.050129+0000","last_undegraded":"2023-09-23T18:41:08.050129+0000","last_fullsized":"2023-09-23T18:41:08.050129+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":27,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.048976+0000","last_change":"2023-09-23T18:40:42.049209+0000","last_active":"2023-09-23T18:41:08.048976+0000","last_peered":"2023-09-23T18:41:08.048976+0000","last_clean":"2023-09-23T18:41:08.048976+0000","last_became_active":"2023-09-23T18:40:42.048654+0000","last_became_peered":"2023-09-23T18:40:42.048654+0000","last_unstale":"2023-09-23T18:41:08.048976+0000","last_undegraded":"2023-09-23T18:41:08.048976+0000","last_fullsized":"2023-09-23T18:41:08.048976+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":100,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.043621+0000","last_change":"2023-09-23T18:40:42.051196+0000","last_active":"2023-09-23T18:41:08.043621+0000","last_peered":"2023-09-23T18:41:08.043621+0000","last_clean":"2023-09-23T18:41:08.043621+0000","last_became_active":"2023-09-23T18:40:42.050613+0000","last_became_peered":"2023-09-23T18:40:42.050613+0000","last_unstale":"2023-09-23T18:41:08.043621+0000","last_undegraded":"2023-09-23T18:41:08.043621+0000","last_fullsized":"2023-09-23T18:41:08.043621+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:21.750773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":167,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:09.953584+0000","last_change":"2023-09-23T18:39:47.006553+0000","last_active":"2023-09-23T18:41:09.953584+0000","last_peered":"2023-09-23T18:41:09.953584+0000","last_clean":"2023-09-23T18:41:09.953584+0000","last_became_active":"2023-09-23T18:39:46.996464+0000","last_became_peered":"2023-09-23T18:39:46.996464+0000","last_unstale":"2023-09-23T18:41:09.953584+0000","last_undegraded":"2023-09-23T18:41:09.953584+0000","last_fullsized":"2023-09-23T18:41:09.953584+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:02.274848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":158,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T18:41:08.049838+0000","last_change":"2023-09-23T18:33:54.847156+0000","last_active":"2023-09-23T18:41:08.049838+0000","last_peered":"2023-09-23T18:41:08.049838+0000","last_clean":"2023-09-23T18:41:08.049838+0000","last_became_active":"2023-09-23T18:33:54.846765+0000","last_became_peered":"2023-09-23T18:33:54.846765+0000","last_unstale":"2023-09-23T18:41:08.049838+0000","last_undegraded":"2023-09-23T18:41:08.049838+0000","last_fullsized":"2023-09-23T18:41:08.049838+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":12058624,"num_objects":184,"num_object_clones":0,"num_object_copies":552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":68,"num_objects_unfound":0,"num_objects_dirty":184,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":209150,"num_write_kb":6698625,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17761,"num_bytes_recovered":1162215424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":561975296,"data_stored":561971224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":115915,"ondisk_log_size":115915,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":1200,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26886,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5937,"num_read_kb":13190,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5621,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430566,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":333424,"kb_used_data":23912,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93411728,"statfs":{"total":95995035648,"available":95653609472,"internally_reserved":0,"allocated":24485888,"data_stored":23920064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:41:07 2023","interfaces":[{"interface":"back","average":{"1min":4.5940000000000003,"5min":2.0169999999999999,"15min":1.76},"min":{"1min":0.308,"5min":0.26200000000000001,"15min":0.252},"max":{"1min":61.293999999999997,"5min":61.293999999999997,"15min":61.293999999999997},"last":0.54300000000000004},{"interface":"front","average":{"1min":3.839,"5min":2.1259999999999999,"15min":1.786},"min":{"1min":0.34699999999999998,"5min":0.317,"15min":0.22900000000000001},"max":{"1min":61.368000000000002,"5min":61.368000000000002,"15min":61.368000000000002},"last":0.879}]},{"osd":1,"last update":"Sat Sep 23 18:41:07 2023","interfaces":[{"interface":"back","average":{"1min":2.161,"5min":2.0840000000000001,"15min":1.47},"min":{"1min":0.38900000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.165,"5min":41.616,"15min":41.616},"last":0.57199999999999995},{"interface":"front","average":{"1min":2.0819999999999999,"5min":2.202,"15min":1.6819999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.5250000000000004,"5min":41.654000000000003,"15min":41.654000000000003},"last":8.6890000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:40:58 2023","interfaces":[{"interface":"back","average":{"1min":1.2909999999999999,"5min":1.6579999999999999,"15min":1.4950000000000001},"min":{"1min":0.38600000000000001,"5min":0.28899999999999998,"15min":0.254},"max":{"1min":5.4980000000000002,"5min":44.392000000000003,"15min":44.392000000000003},"last":0.98299999999999998},{"interface":"front","average":{"1min":2.129,"5min":1.958,"15min":1.5069999999999999},"min":{"1min":0.379,"5min":0.34100000000000003,"15min":0.29999999999999999},"max":{"1min":7.1360000000000001,"5min":42.991999999999997,"15min":42.991999999999997},"last":0.60599999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:41:29 2023","interfaces":[{"interface":"back","average":{"1min":5.1210000000000004,"5min":3.0089999999999999,"15min":1.661},"min":{"1min":0.36199999999999999,"5min":0.34599999999999997,"15min":0.32800000000000001},"max":{"1min":70.216999999999999,"5min":70.216999999999999,"15min":70.216999999999999},"last":0.64800000000000002},{"interface":"front","average":{"1min":4.8490000000000002,"5min":3.3849999999999998,"15min":1.794},"min":{"1min":0.318,"5min":0.314,"15min":0.314},"max":{"1min":69.566000000000003,"5min":69.566000000000003,"15min":69.566000000000003},"last":8.7409999999999997}]},{"osd":4,"last update":"Sat Sep 23 18:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":0.94399999999999995,"15min":1.03},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":4.1120000000000001,"5min":5.1230000000000002,"15min":6.3529999999999998},"last":0.95599999999999996},{"interface":"front","average":{"1min":1.446,"5min":1.0009999999999999,"15min":1.117},"min":{"1min":0.40600000000000003,"5min":0.35099999999999998,"15min":0.309},"max":{"1min":7.101,"5min":7.101,"15min":11.119},"last":0.628}]},{"osd":5,"last update":"Sat Sep 23 18:41:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":1.0509999999999999,"15min":1.052},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.0899999999999999,"5min":5.2690000000000001,"15min":5.8899999999999997},"last":8.7210000000000001},{"interface":"front","average":{"1min":0.82699999999999996,"5min":1.004,"15min":1.085},"min":{"1min":0.42999999999999999,"5min":0.41699999999999998,"15min":0.26800000000000002},"max":{"1min":4.0380000000000003,"5min":5.0129999999999999,"15min":7.46},"last":0.93300000000000005}]},{"osd":6,"last update":"Sat Sep 23 18:41:07 2023","interfaces":[{"interface":"back","average":{"1min":1.4219999999999999,"5min":1.1459999999999999,"15min":1.107},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.35299999999999998},"max":{"1min":4.7770000000000001,"5min":5.0899999999999999,"15min":5.5129999999999999},"last":3.153},{"interface":"front","average":{"1min":1.623,"5min":1.119,"15min":1.1200000000000001},"min":{"1min":0.51200000000000001,"5min":0.42999999999999999,"15min":0.36499999999999999},"max":{"1min":6.9989999999999997,"5min":6.9989999999999997,"15min":9.1850000000000005},"last":0.66700000000000004}]}]},{"osd":6,"up_from":42,"seq":180388626798,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":322344,"kb_used_data":16928,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93422808,"statfs":{"total":95995035648,"available":95664955392,"internally_reserved":0,"allocated":17334272,"data_stored":16776752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:41:05 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":1.853,"15min":1.9510000000000001},"min":{"1min":0.26700000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":3.9039999999999999,"5min":66.599000000000004,"15min":182.655},"last":3.1909999999999998},{"interface":"front","average":{"1min":1.3169999999999999,"5min":1.9430000000000001,"15min":2.0510000000000002},"min":{"1min":0.39900000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.9430000000000001,"5min":66.527000000000001,"15min":182.58799999999999},"last":3.238}]},{"osd":1,"last update":"Sat Sep 23 18:41:05 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":1.4099999999999999,"15min":0.97499999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":4.4969999999999999,"5min":18.420999999999999,"15min":18.420999999999999},"last":3.3359999999999999},{"interface":"front","average":{"1min":1.8100000000000001,"5min":1.746,"15min":1.3089999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.30299999999999999},"max":{"1min":5.3120000000000003,"5min":21.193000000000001,"15min":21.193000000000001},"last":7.556}]},{"osd":2,"last update":"Sat Sep 23 18:41:01 2023","interfaces":[{"interface":"back","average":{"1min":2.484,"5min":3.9670000000000001,"15min":2.3010000000000002},"min":{"1min":0.33700000000000002,"5min":0.33500000000000002,"15min":0.28100000000000003},"max":{"1min":7.423,"5min":122.242,"15min":122.242},"last":2.8380000000000001},{"interface":"front","average":{"1min":1.8919999999999999,"5min":4.0110000000000001,"15min":2.3050000000000002},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.28799999999999998},"max":{"1min":5.7629999999999999,"5min":122.32899999999999,"15min":122.32899999999999},"last":7.6070000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:41:25 2023","interfaces":[{"interface":"back","average":{"1min":1.323,"5min":4.665,"15min":2.2050000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.7720000000000002,"5min":123.52200000000001,"15min":123.52200000000001},"last":3.4020000000000001},{"interface":"front","average":{"1min":2.657,"5min":4.9909999999999997,"15min":2.3639999999999999},"min":{"1min":0.40100000000000002,"5min":0.39800000000000002,"15min":0.36099999999999999},"max":{"1min":9.5640000000000001,"5min":123.605,"15min":123.605},"last":7.6660000000000004}]},{"osd":4,"last update":"Sat Sep 23 18:41:25 2023","interfaces":[{"interface":"back","average":{"1min":2.9100000000000001,"5min":1.3340000000000001,"15min":1.0880000000000001},"min":{"1min":0.495,"5min":0.36899999999999999,"15min":0.29999999999999999},"max":{"1min":15.496,"5min":15.496,"15min":15.496},"last":9.6349999999999998},{"interface":"front","average":{"1min":2.5289999999999999,"5min":1.24,"15min":1.0449999999999999},"min":{"1min":0.42399999999999999,"5min":0.35699999999999998,"15min":0.34899999999999998},"max":{"1min":14.875999999999999,"5min":14.875999999999999,"15min":14.875999999999999},"last":3.3639999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:41:25 2023","interfaces":[{"interface":"back","average":{"1min":1.9450000000000001,"5min":1.1259999999999999,"15min":1.0269999999999999},"min":{"1min":0.46700000000000003,"5min":0.39100000000000001,"15min":0.32000000000000001},"max":{"1min":7.4930000000000003,"5min":7.4930000000000003,"15min":7.9240000000000004},"last":2.8769999999999998},{"interface":"front","average":{"1min":0.85299999999999998,"5min":0.91400000000000003,"15min":0.93500000000000005},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":3.8220000000000001,"5min":3.8220000000000001,"15min":4.5270000000000001},"last":3.2890000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:41:01 2023","interfaces":[{"interface":"back","average":{"1min":2.1589999999999998,"5min":1.2709999999999999,"15min":1.113},"min":{"1min":0.438,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":7.9320000000000004,"5min":7.9320000000000004,"15min":7.9320000000000004},"last":7.6360000000000001},{"interface":"front","average":{"1min":2.7429999999999999,"5min":1.554,"15min":1.3600000000000001},"min":{"1min":0.46200000000000002,"5min":0.42899999999999999,"15min":0.39100000000000001},"max":{"1min":7.8209999999999997,"5min":7.8209999999999997,"15min":9.1129999999999995},"last":7.7610000000000001}]}]},{"osd":1,"up_from":123,"seq":528280977503,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":222388,"kb_used_data":2028,"kb_used_omap":7,"kb_used_meta":220344,"kb_avail":93522764,"statfs":{"total":95995035648,"available":95767310336,"internally_reserved":0,"allocated":2076672,"data_stored":1218305,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":225633038},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:41:07 2023","interfaces":[{"interface":"back","average":{"1min":2.0830000000000002,"5min":1.919,"15min":6.8979999999999997},"min":{"1min":0.20100000000000001,"5min":0.186,"15min":0.186},"max":{"1min":12.827999999999999,"5min":23.280000000000001,"15min":173.41800000000001},"last":0.78000000000000003},{"interface":"front","average":{"1min":3.234,"5min":2.4129999999999998,"15min":6.6509999999999998},"min":{"1min":0.249,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":11.773,"5min":22.93,"15min":166.42599999999999},"last":0.73399999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:41:07 2023","interfaces":[{"interface":"back","average":{"1min":2.5110000000000001,"5min":4.2910000000000004,"15min":3.2250000000000001},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":8.5939999999999994,"5min":129.25899999999999,"15min":129.25899999999999},"last":0.40999999999999998},{"interface":"front","average":{"1min":4.1180000000000003,"5min":4.6390000000000002,"15min":3.089},"min":{"1min":0.35899999999999999,"5min":0.247,"15min":0.247},"max":{"1min":16.032,"5min":129.35300000000001,"15min":129.35300000000001},"last":0.33700000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:41:07 2023","interfaces":[{"interface":"back","average":{"1min":4.6539999999999999,"5min":3.5019999999999998,"15min":1.8640000000000001},"min":{"1min":0.35399999999999998,"5min":0.25,"15min":0.25},"max":{"1min":16.123000000000001,"5min":55.014000000000003,"15min":55.014000000000003},"last":0.45900000000000002},{"interface":"front","average":{"1min":3.77,"5min":3.4630000000000001,"15min":1.952},"min":{"1min":0.25,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":15.717000000000001,"5min":55.841999999999999,"15min":55.841999999999999},"last":0.51600000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:41:07 2023","interfaces":[{"interface":"back","average":{"1min":2.399,"5min":1.8500000000000001,"15min":1.411},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":8.6170000000000009,"5min":11.763,"15min":11.763},"last":8.8670000000000009},{"interface":"front","average":{"1min":1.9059999999999999,"5min":1.5880000000000001,"15min":1.522},"min":{"1min":0.34599999999999997,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":8.2270000000000003,"5min":11.491,"15min":11.491},"last":0.67400000000000004}]},{"osd":5,"last update":"Sat Sep 23 18:41:07 2023","interfaces":[{"interface":"back","average":{"1min":2.3460000000000001,"5min":1.7989999999999999,"15min":1.117},"min":{"1min":0.38600000000000001,"5min":0.38,"15min":0.35699999999999998},"max":{"1min":9.2690000000000001,"5min":11.794,"15min":11.794},"last":0.61399999999999999},{"interface":"front","average":{"1min":2.3889999999999998,"5min":1.8029999999999999,"15min":1.5429999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":8.5190000000000001,"5min":11.733000000000001,"15min":11.733000000000001},"last":0.57499999999999996}]},{"osd":6,"last update":"Sat Sep 23 18:41:07 2023","interfaces":[{"interface":"back","average":{"1min":2.6640000000000001,"5min":1.8640000000000001,"15min":1.6799999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":8.4920000000000009,"5min":11.776999999999999,"15min":11.776999999999999},"last":0.54700000000000004},{"interface":"front","average":{"1min":3.194,"5min":1.968,"15min":1.8089999999999999},"min":{"1min":0.439,"5min":0.36099999999999999,"15min":0.35999999999999999},"max":{"1min":16.687000000000001,"5min":16.687000000000001,"15min":16.687000000000001},"last":0.63}]},{"osd":7,"last update":"Sat Sep 23 18:41:07 2023","interfaces":[{"interface":"back","average":{"1min":2.383,"5min":2.0859999999999999,"15min":1.944},"min":{"1min":0.495,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":8.4450000000000003,"5min":11.747999999999999,"15min":11.747999999999999},"last":8.9109999999999996},{"interface":"front","average":{"1min":2.1549999999999998,"5min":1.827,"15min":1.8029999999999999},"min":{"1min":0.442,"5min":0.379,"15min":0.379},"max":{"1min":8.4770000000000003,"5min":11.593,"15min":11.593},"last":0.70899999999999996}]}]},{"osd":0,"up_from":88,"seq":377957122232,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":408020,"kb_used_data":2572,"kb_used_omap":8,"kb_used_meta":405431,"kb_avail":93337132,"statfs":{"total":95995035648,"available":95577223168,"internally_reserved":0,"allocated":2633728,"data_stored":1804095,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":415162238},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:41:07 2023","interfaces":[{"interface":"back","average":{"1min":3.0649999999999999,"5min":2.4209999999999998,"15min":2.052},"min":{"1min":0.32600000000000001,"5min":0.19400000000000001,"15min":0.17100000000000001},"max":{"1min":16.355,"5min":38.545999999999999,"15min":38.545999999999999},"last":0.89500000000000002},{"interface":"front","average":{"1min":3.363,"5min":2.4340000000000002,"15min":2.4329999999999998},"min":{"1min":0.40400000000000003,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":21.535,"5min":45.904000000000003,"15min":45.904000000000003},"last":0.85599999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:40:54 2023","interfaces":[{"interface":"back","average":{"1min":2.5510000000000002,"5min":3.081,"15min":2.077},"min":{"1min":0.28799999999999998,"5min":0.23400000000000001,"15min":0.192},"max":{"1min":10.234999999999999,"5min":108.73999999999999,"15min":108.73999999999999},"last":0.749},{"interface":"front","average":{"1min":1.661,"5min":2.5499999999999998,"15min":1.871},"min":{"1min":0.23899999999999999,"5min":0.22600000000000001,"15min":0.20799999999999999},"max":{"1min":8.4600000000000009,"5min":108.136,"15min":108.136},"last":0.76900000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:41:21 2023","interfaces":[{"interface":"back","average":{"1min":12.743,"5min":6.6619999999999999,"15min":3.1899999999999999},"min":{"1min":0.27700000000000002,"5min":0.25900000000000001,"15min":0.216},"max":{"1min":177.83600000000001,"5min":177.83600000000001,"15min":177.83600000000001},"last":0.94499999999999995},{"interface":"front","average":{"1min":12.348000000000001,"5min":6.6459999999999999,"15min":3.1840000000000002},"min":{"1min":0.28199999999999997,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":181.71899999999999,"5min":181.71899999999999,"15min":181.71899999999999},"last":0.86799999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:41:07 2023","interfaces":[{"interface":"back","average":{"1min":2.278,"5min":1.429,"15min":1.3029999999999999},"min":{"1min":0.39400000000000002,"5min":0.312,"15min":0.28199999999999997},"max":{"1min":9.9450000000000003,"5min":9.9450000000000003,"15min":9.9450000000000003},"last":0.93700000000000006},{"interface":"front","average":{"1min":1.8140000000000001,"5min":1.4530000000000001,"15min":1.472},"min":{"1min":0.36499999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.6150000000000002,"5min":10.561999999999999,"15min":16.486999999999998},"last":0.80100000000000005}]},{"osd":5,"last update":"Sat Sep 23 18:41:07 2023","interfaces":[{"interface":"back","average":{"1min":1.6319999999999999,"5min":1.1859999999999999,"15min":1.2789999999999999},"min":{"1min":0.40699999999999997,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":4.5380000000000003,"5min":10.606,"15min":10.606},"last":1.054},{"interface":"front","average":{"1min":1.216,"5min":1.1910000000000001,"15min":1.2589999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.157,"5min":8.2279999999999998,"15min":8.9250000000000007},"last":0.90900000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:41:07 2023","interfaces":[{"interface":"back","average":{"1min":1.956,"5min":1.3320000000000001,"15min":1.3740000000000001},"min":{"1min":0.39900000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":4.8529999999999998,"5min":10.634,"15min":10.634},"last":1.121},{"interface":"front","average":{"1min":1.6279999999999999,"5min":1.266,"15min":1.3100000000000001},"min":{"1min":0.46999999999999997,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.2030000000000003,"5min":6.4119999999999999,"15min":8.9740000000000002},"last":1.036}]},{"osd":7,"last update":"Sat Sep 23 18:41:07 2023","interfaces":[{"interface":"back","average":{"1min":1.8400000000000001,"5min":1.3129999999999999,"15min":1.3779999999999999},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.39200000000000002},"max":{"1min":5.7060000000000004,"5min":5.7060000000000004,"15min":9.0600000000000005},"last":1.0800000000000001},{"interface":"front","average":{"1min":2.613,"5min":1.589,"15min":1.3959999999999999},"min":{"1min":0.48699999999999999,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":10.01,"5min":10.01,"15min":10.01},"last":1.1399999999999999}]}]},{"osd":2,"up_from":98,"seq":420906795179,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":907308,"kb_used_data":501860,"kb_used_omap":11,"kb_used_meta":405428,"kb_avail":92837844,"statfs":{"total":95995035648,"available":95065952256,"internally_reserved":0,"allocated":513904640,"data_stored":513110492,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11949,"internal_metadata":415158611},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:40:57 2023","interfaces":[{"interface":"back","average":{"1min":1.319,"5min":1.304,"15min":1.244},"min":{"1min":0.20799999999999999,"5min":0.184,"15min":0.184},"max":{"1min":4.8780000000000001,"5min":21.498000000000001,"15min":21.498000000000001},"last":0.23400000000000001},{"interface":"front","average":{"1min":1.4159999999999999,"5min":1.4079999999999999,"15min":1.2649999999999999},"min":{"1min":0.23899999999999999,"5min":0.20300000000000001,"15min":0.17699999999999999},"max":{"1min":6.5819999999999999,"5min":21.484999999999999,"15min":21.484999999999999},"last":0.28000000000000003}]},{"osd":1,"last update":"Sat Sep 23 18:41:06 2023","interfaces":[{"interface":"back","average":{"1min":8.1989999999999998,"5min":5.7169999999999996,"15min":3.5059999999999998},"min":{"1min":0.35199999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":94.210999999999999,"5min":184.56299999999999,"15min":184.56299999999999},"last":0.32100000000000001},{"interface":"front","average":{"1min":8.0359999999999996,"5min":5.7199999999999998,"15min":3.472},"min":{"1min":0.309,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":90.451999999999998,"5min":183.922,"15min":183.922},"last":0.33400000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:41:33 2023","interfaces":[{"interface":"back","average":{"1min":11.525,"5min":6.0830000000000002,"15min":2.7770000000000001},"min":{"1min":0.28499999999999998,"5min":0.19,"15min":0.19},"max":{"1min":161.773,"5min":161.773,"15min":161.773},"last":0.47799999999999998},{"interface":"front","average":{"1min":10.474,"5min":5.3710000000000004,"15min":2.4510000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.249},"max":{"1min":158.03700000000001,"5min":158.03700000000001,"15min":158.03700000000001},"last":0.97399999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:40:57 2023","interfaces":[{"interface":"back","average":{"1min":2.5670000000000002,"5min":2.4500000000000002,"15min":1.6639999999999999},"min":{"1min":0.37,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.0179999999999998,"5min":71.587999999999994,"15min":71.587999999999994},"last":7.0190000000000001},{"interface":"front","average":{"1min":1.3839999999999999,"5min":2.052,"15min":1.577},"min":{"1min":0.41399999999999998,"5min":0.34300000000000003,"15min":0.33500000000000002},"max":{"1min":4.9489999999999998,"5min":71.328999999999994,"15min":71.328999999999994},"last":0.35499999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:40:57 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":1.8540000000000001,"15min":1.417},"min":{"1min":0.442,"5min":0.36499999999999999,"15min":0.35999999999999999},"max":{"1min":4.9740000000000002,"5min":71.302000000000007,"15min":71.302000000000007},"last":1.0329999999999999},{"interface":"front","average":{"1min":1.2549999999999999,"5min":2.1030000000000002,"15min":1.5640000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.316},"max":{"1min":5.4340000000000002,"5min":71.543999999999997,"15min":71.543999999999997},"last":0.371}]},{"osd":6,"last update":"Sat Sep 23 18:40:57 2023","interfaces":[{"interface":"back","average":{"1min":2.7719999999999998,"5min":2.8300000000000001,"15min":1.7749999999999999},"min":{"1min":0.39700000000000002,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":12.872,"5min":71.460999999999999,"15min":71.460999999999999},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.9299999999999999,"5min":2.0459999999999998,"15min":1.53},"min":{"1min":0.45800000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":6.2759999999999998,"5min":71.271000000000001,"15min":71.271000000000001},"last":0.999}]},{"osd":7,"last update":"Sat Sep 23 18:40:57 2023","interfaces":[{"interface":"back","average":{"1min":2.2509999999999999,"5min":2.702,"15min":1.7829999999999999},"min":{"1min":0.38100000000000001,"5min":0.38,"15min":0.34799999999999998},"max":{"1min":7.9130000000000003,"5min":71.622,"15min":71.622},"last":2.242},{"interface":"front","average":{"1min":2.0760000000000001,"5min":2.1909999999999998,"15min":1.7629999999999999},"min":{"1min":0.51300000000000001,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":8.8510000000000009,"5min":71.575000000000003,"15min":71.575000000000003},"last":1.0169999999999999}]}]},{"osd":3,"up_from":103,"seq":442381631654,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":373348,"kb_used_data":39132,"kb_used_omap":4,"kb_used_meta":334203,"kb_avail":93371804,"statfs":{"total":95995035648,"available":95612727296,"internally_reserved":0,"allocated":40071168,"data_stored":39283278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4826,"internal_metadata":342224166},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:41:25 2023","interfaces":[{"interface":"back","average":{"1min":1.7450000000000001,"5min":1.9339999999999999,"15min":1.397},"min":{"1min":0.20799999999999999,"5min":0.193,"15min":0.193},"max":{"1min":12.101000000000001,"5min":12.101000000000001,"15min":14.231},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.5469999999999999,"5min":1.958,"15min":1.2869999999999999},"min":{"1min":0.224,"5min":0.219,"15min":0.20300000000000001},"max":{"1min":7.7839999999999998,"5min":10.037000000000001,"15min":10.037000000000001},"last":0.52200000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:41:14 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":1.649,"15min":1.343},"min":{"1min":0.27200000000000002,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":4.2380000000000004,"5min":12.968999999999999,"15min":12.968999999999999},"last":0.752},{"interface":"front","average":{"1min":1.9950000000000001,"5min":2.028,"15min":1.5660000000000001},"min":{"1min":0.32000000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":11.087999999999999,"5min":11.74,"15min":11.74},"last":0.54900000000000004}]},{"osd":2,"last update":"Sat Sep 23 18:41:25 2023","interfaces":[{"interface":"back","average":{"1min":2.1720000000000002,"5min":2.8380000000000001,"15min":2.6000000000000001},"min":{"1min":0.26400000000000001,"5min":0.23799999999999999,"15min":0.22700000000000001},"max":{"1min":7.7389999999999999,"5min":18.888000000000002,"15min":187.40000000000001},"last":0.63500000000000001},{"interface":"front","average":{"1min":1.4570000000000001,"5min":1.9490000000000001,"15min":2.1320000000000001},"min":{"1min":0.28899999999999998,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":4.9720000000000004,"5min":13.15,"15min":187.71700000000001},"last":0.58699999999999997}]},{"osd":4,"last update":"Sat Sep 23 18:41:25 2023","interfaces":[{"interface":"back","average":{"1min":1.6559999999999999,"5min":1.458,"15min":1.1699999999999999},"min":{"1min":0.35099999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":8.7989999999999995,"5min":8.7989999999999995,"15min":8.7989999999999995},"last":0.84199999999999997},{"interface":"front","average":{"1min":2.984,"5min":1.917,"15min":1.3240000000000001},"min":{"1min":0.36199999999999999,"5min":0.33900000000000002,"15min":0.28399999999999997},"max":{"1min":11.916,"5min":11.916,"15min":11.916},"last":0.73099999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:41:25 2023","interfaces":[{"interface":"back","average":{"1min":1.6180000000000001,"5min":1.48,"15min":1.1899999999999999},"min":{"1min":0.39700000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":7.8799999999999999,"5min":7.8799999999999999,"15min":7.8799999999999999},"last":0.66400000000000003},{"interface":"front","average":{"1min":2.1360000000000001,"5min":1.9850000000000001,"15min":1.357},"min":{"1min":0.39100000000000001,"5min":0.38800000000000001,"15min":0.36099999999999999},"max":{"1min":7.8129999999999997,"5min":10.18,"15min":10.18},"last":0.78500000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:41:25 2023","interfaces":[{"interface":"back","average":{"1min":2.9620000000000002,"5min":2.5499999999999998,"15min":1.5549999999999999},"min":{"1min":0.57599999999999996,"5min":0.41899999999999998,"15min":0.35599999999999998},"max":{"1min":8.6940000000000008,"5min":11.529,"15min":11.529},"last":0.71299999999999997},{"interface":"front","average":{"1min":2.177,"5min":1.8100000000000001,"15min":1.3},"min":{"1min":0.439,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":8.8529999999999998,"5min":11.989000000000001,"15min":11.989000000000001},"last":0.89300000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:41:25 2023","interfaces":[{"interface":"back","average":{"1min":1.5860000000000001,"5min":1.6919999999999999,"15min":1.333},"min":{"1min":0.40899999999999997,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":8.8840000000000003,"5min":9.0980000000000008,"15min":9.0980000000000008},"last":0.96699999999999997},{"interface":"front","average":{"1min":1.512,"5min":1.7210000000000001,"15min":1.331},"min":{"1min":0.45100000000000001,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":5.226,"5min":9.032,"15min":9.032},"last":0.86499999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019264,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":313960,"kb_used_data":12640,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93431192,"statfs":{"total":95995035648,"available":95673540608,"internally_reserved":0,"allocated":12943360,"data_stored":12385646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:40:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8140000000000001,"5min":1.6859999999999999,"15min":1.7210000000000001},"min":{"1min":0.32300000000000001,"5min":0.27400000000000002,"15min":0.253},"max":{"1min":7.1139999999999999,"5min":53.295000000000002,"15min":82.623000000000005},"last":0.39200000000000002},{"interface":"front","average":{"1min":1.3640000000000001,"5min":1.544,"15min":1.615},"min":{"1min":0.313,"5min":0.29899999999999999,"15min":0.23799999999999999},"max":{"1min":4.1459999999999999,"5min":53.356000000000002,"15min":82.638000000000005},"last":0.373}]},{"osd":1,"last update":"Sat Sep 23 18:41:03 2023","interfaces":[{"interface":"back","average":{"1min":2.4399999999999999,"5min":2.6440000000000001,"15min":1.4490000000000001},"min":{"1min":0.40000000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.4429999999999996,"5min":53.457000000000001,"15min":53.457000000000001},"last":2.2360000000000002},{"interface":"front","average":{"1min":2.6459999999999999,"5min":2.6890000000000001,"15min":1.6519999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":10.523,"5min":53.395000000000003,"15min":53.395000000000003},"last":2.2240000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:40:56 2023","interfaces":[{"interface":"back","average":{"1min":1.5720000000000001,"5min":2.3849999999999998,"15min":1.5960000000000001},"min":{"1min":0.36899999999999999,"5min":0.33200000000000002,"15min":0.28799999999999998},"max":{"1min":6.1900000000000004,"5min":53.249000000000002,"15min":53.249000000000002},"last":5.0309999999999997},{"interface":"front","average":{"1min":2.5960000000000001,"5min":2.7130000000000001,"15min":1.8009999999999999},"min":{"1min":0.40400000000000003,"5min":0.35499999999999998,"15min":0.309},"max":{"1min":13.188000000000001,"5min":53.220999999999997,"15min":53.220999999999997},"last":2.8479999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:41:34 2023","interfaces":[{"interface":"back","average":{"1min":2.7869999999999999,"5min":3.327,"15min":1.9339999999999999},"min":{"1min":0.39900000000000002,"5min":0.39000000000000001,"15min":0.32900000000000001},"max":{"1min":6.9870000000000001,"5min":72.075999999999993,"15min":72.075999999999993},"last":0.499},{"interface":"front","average":{"1min":2.3050000000000002,"5min":3.028,"15min":1.9259999999999999},"min":{"1min":0.432,"5min":0.36099999999999999,"15min":0.34000000000000002},"max":{"1min":6.8769999999999998,"5min":71.596999999999994,"15min":71.596999999999994},"last":6.9429999999999996}]},{"osd":5,"last update":"Sat Sep 23 18:40:50 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":0.96799999999999997,"15min":1.0920000000000001},"min":{"1min":0.39300000000000002,"5min":0.33900000000000002,"15min":0.29899999999999999},"max":{"1min":4.0789999999999997,"5min":4.0789999999999997,"15min":7.819},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.343,"5min":1.026,"15min":1.075},"min":{"1min":0.41799999999999998,"5min":0.33500000000000002,"15min":0.28399999999999997},"max":{"1min":5.7859999999999996,"5min":5.7859999999999996,"15min":6.4779999999999998},"last":0.42199999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:41:20 2023","interfaces":[{"interface":"back","average":{"1min":2.1709999999999998,"5min":1.258,"15min":1.1950000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.35399999999999998},"max":{"1min":6.9630000000000001,"5min":6.9630000000000001,"15min":7.8419999999999996},"last":0.35299999999999998},{"interface":"front","average":{"1min":1.4490000000000001,"5min":1.054,"15min":1.0649999999999999},"min":{"1min":0.44,"5min":0.32100000000000001,"15min":0.26100000000000001},"max":{"1min":6.8399999999999999,"5min":6.8399999999999999,"15min":6.8399999999999999},"last":0.51800000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:40:56 2023","interfaces":[{"interface":"back","average":{"1min":3.7770000000000001,"5min":1.74,"15min":1.476},"min":{"1min":0.39700000000000002,"5min":0.35899999999999999,"15min":0.311},"max":{"1min":12.294,"5min":12.294,"15min":13.491},"last":1.2569999999999999},{"interface":"front","average":{"1min":3.113,"5min":1.7150000000000001,"15min":1.5509999999999999},"min":{"1min":0.59699999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":10.507,"5min":10.507,"15min":17.062999999999999},"last":1.2250000000000001}]}]},{"osd":5,"up_from":0,"seq":154618823032,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4224,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7725,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3429,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":497029120,"data_stored":497025048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":31916032,"data_stored":31916032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4718592,"data_stored":4718592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":13369344,"data_stored":13369344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":14942208,"data_stored":14942208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:41:48.292 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T18:41:48.293 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 31 2023-09-23T18:41:48.349 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:48.403 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:48.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:48.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:48.506 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:48.530 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:48.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:48.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:48.633 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:48.654 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:48.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:48.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:48.757 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:48.788 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:48.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:48.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:48.891 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:48.922 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:48.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:48.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:49.026 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:49.051 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:49.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:49.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:49.154 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:49.185 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:49.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:49.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:49.289 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:49.296 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:49 smithi173 bash[59422]: cluster 2023-09-23T18:41:47.927701+0000 mgr.y (mgr.26054) 706 : cluster 0 pgmap v703: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 381 op/s; 1.2 MiB/s, 81 objects/s recovering 2023-09-23T18:41:49.314 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:49.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:49.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:49.417 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:49.429 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:49 smithi098 bash[76817]: cluster 2023-09-23T18:41:47.927701+0000 mgr.y (mgr.26054) 706 : cluster 0 pgmap v703: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 381 op/s; 1.2 MiB/s, 81 objects/s recovering 2023-09-23T18:41:49.429 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:49 smithi098 bash[79516]: cluster 2023-09-23T18:41:47.927701+0000 mgr.y (mgr.26054) 706 : cluster 0 pgmap v703: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 381 op/s; 1.2 MiB/s, 81 objects/s recovering 2023-09-23T18:41:49.446 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:49.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:49.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:49.492 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:41:49.551 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:49.576 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:49.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:49.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:49.681 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:49.709 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:49.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:49.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:49.813 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:49.839 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:49.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:49.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:49.943 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:49.967 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:49.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:49.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:50.070 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:50.094 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:50.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:50.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:50.198 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:50.228 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:50.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:50.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:50.266 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:41:50.332 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:50.363 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:50.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:50.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:50.466 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:50.488 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:50.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:50.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:50.591 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:50.618 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:50.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:50.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:50.720 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:50.752 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:50.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:50.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:50.856 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:50.885 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:50.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:50.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:50.989 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:51.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:51.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:51.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:51.119 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:51.147 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:51.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:51.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:51.251 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:51.263 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:51 smithi173 bash[59422]: cluster 2023-09-23T18:41:49.928656+0000 mgr.y (mgr.26054) 707 : cluster 0 pgmap v704: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 337 op/s; 1.2 MiB/s, 81 objects/s recovering 2023-09-23T18:41:51.280 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:51.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:51.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:51.383 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:51.412 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:51.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:51.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:51.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:51 smithi098 bash[76817]: cluster 2023-09-23T18:41:49.928656+0000 mgr.y (mgr.26054) 707 : cluster 0 pgmap v704: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 337 op/s; 1.2 MiB/s, 81 objects/s recovering 2023-09-23T18:41:51.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:51 smithi098 bash[79516]: cluster 2023-09-23T18:41:49.928656+0000 mgr.y (mgr.26054) 707 : cluster 0 pgmap v704: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 337 op/s; 1.2 MiB/s, 81 objects/s recovering 2023-09-23T18:41:51.516 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:51.548 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:51.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:51.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:51.652 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:51.677 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:51.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:51.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:51.780 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:51.811 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:51.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:51.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:51.914 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:51.937 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:51.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:51.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:52.041 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:52.074 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:52.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:52.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:52.178 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:52.191 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:52 smithi173 bash[59422]: cluster 2023-09-23T18:41:51.929668+0000 mgr.y (mgr.26054) 708 : cluster 0 pgmap v705: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 134 op/s; 0 B/s, 232 objects/s recovering 2023-09-23T18:41:52.209 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:52.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:52.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:52.312 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:52.339 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:52.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:52.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:52.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:52 smithi098 bash[76817]: cluster 2023-09-23T18:41:51.929668+0000 mgr.y (mgr.26054) 708 : cluster 0 pgmap v705: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 134 op/s; 0 B/s, 232 objects/s recovering 2023-09-23T18:41:52.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:52 smithi098 bash[79516]: cluster 2023-09-23T18:41:51.929668+0000 mgr.y (mgr.26054) 708 : cluster 0 pgmap v705: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 134 op/s; 0 B/s, 232 objects/s recovering 2023-09-23T18:41:52.441 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:52.471 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:52.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:52.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:52.574 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:52.597 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:52.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:52.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:52.700 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:52.725 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:52.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:52.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:52.829 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:52.854 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:52.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:52.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:52.958 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:52.980 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:52.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:52.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:53.083 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:53.108 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:41:53.116 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:53.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:53.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:53.220 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:53.252 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:53.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:53.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:53.355 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:53.383 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:53.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:53.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:53.486 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:53.512 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:53.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:53.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:53.615 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:53.643 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:53.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:53.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:53.746 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:53.770 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:53.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:53.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:53.873 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:53.898 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:53.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:53.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:54.002 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:54.029 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:54.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:54.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:54.127 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:41:54.128 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:41:54.132 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:54.155 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:54.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:54.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:54.258 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:54.280 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:54.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:54.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:54.384 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:54.414 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:54.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:54.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:54.518 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:54.550 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:54.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:54.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:54.653 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:54.676 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:54.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:54.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:54.779 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:54.808 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:54.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:54.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:54.910 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:54.937 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:54.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:54.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:55.040 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:55.066 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:55.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:55.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:55.075 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:41:55.075 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-09-23T18:41:55.075 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 0 2023-09-23T18:41:55.075 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:41:55.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:55 smithi173 bash[59422]: cluster 2023-09-23T18:41:53.930650+0000 mgr.y (mgr.26054) 709 : cluster 0 pgmap v706: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 18 op/s; 0 B/s, 232 objects/s recovering 2023-09-23T18:41:55.097 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:55 smithi173 bash[59422]: cluster 2023-09-23T18:41:54.011280+0000 mon.a (mon.0) 961 : cluster 0 osdmap e181: 8 total, 8 up, 7 in 2023-09-23T18:41:55.097 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:55 smithi173 bash[59422]: cluster 2023-09-23T18:41:54.012730+0000 osd.4 (osd.4) 134 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T18:41:55.097 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:55 smithi173 bash[59422]: cluster 2023-09-23T18:41:54.015767+0000 osd.4 (osd.4) 135 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T18:41:55.097 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:55 smithi173 bash[59422]: cluster 2023-09-23T18:41:54.016848+0000 osd.6 (osd.6) 108 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T18:41:55.097 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:55 smithi173 bash[59422]: cluster 2023-09-23T18:41:54.016988+0000 osd.4 (osd.4) 136 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T18:41:55.097 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:55 smithi173 bash[59422]: cluster 2023-09-23T18:41:54.017605+0000 osd.7 (osd.7) 97 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T18:41:55.097 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:55 smithi173 bash[59422]: cluster 2023-09-23T18:41:54.018825+0000 osd.6 (osd.6) 109 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T18:41:55.097 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:55 smithi173 bash[59422]: cluster 2023-09-23T18:41:54.122367+0000 osd.5 (osd.5) 82 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T18:41:55.097 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:55 smithi173 bash[59422]: audit 2023-09-23T18:41:54.581810+0000 mon.a (mon.0) 962 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:41:55.097 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:55 smithi173 bash[59422]: audit 2023-09-23T18:41:54.610437+0000 mon.a (mon.0) 963 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:41:55.097 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:55 smithi173 bash[59422]: audit 2023-09-23T18:41:54.611707+0000 mon.a (mon.0) 964 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:41:55.169 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:55.194 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:55.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:55.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:55.297 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:55.310 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:55 smithi098 bash[76817]: cluster 2023-09-23T18:41:53.930650+0000 mgr.y (mgr.26054) 709 : cluster 0 pgmap v706: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 18 op/s; 0 B/s, 232 objects/s recovering 2023-09-23T18:41:55.310 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:55 smithi098 bash[76817]: cluster 2023-09-23T18:41:54.011280+0000 mon.a (mon.0) 961 : cluster 0 osdmap e181: 8 total, 8 up, 7 in 2023-09-23T18:41:55.310 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:55 smithi098 bash[76817]: cluster 2023-09-23T18:41:54.012730+0000 osd.4 (osd.4) 134 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T18:41:55.310 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:55 smithi098 bash[76817]: cluster 2023-09-23T18:41:54.015767+0000 osd.4 (osd.4) 135 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T18:41:55.310 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:55 smithi098 bash[76817]: cluster 2023-09-23T18:41:54.016848+0000 osd.6 (osd.6) 108 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T18:41:55.310 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:55 smithi098 bash[76817]: cluster 2023-09-23T18:41:54.016988+0000 osd.4 (osd.4) 136 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T18:41:55.310 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:55 smithi098 bash[76817]: cluster 2023-09-23T18:41:54.017605+0000 osd.7 (osd.7) 97 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T18:41:55.310 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:55 smithi098 bash[76817]: cluster 2023-09-23T18:41:54.018825+0000 osd.6 (osd.6) 109 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T18:41:55.310 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:55 smithi098 bash[76817]: cluster 2023-09-23T18:41:54.122367+0000 osd.5 (osd.5) 82 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T18:41:55.311 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:55 smithi098 bash[76817]: audit 2023-09-23T18:41:54.581810+0000 mon.a (mon.0) 962 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:41:55.311 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:55 smithi098 bash[76817]: audit 2023-09-23T18:41:54.610437+0000 mon.a (mon.0) 963 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:41:55.311 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:55 smithi098 bash[76817]: audit 2023-09-23T18:41:54.611707+0000 mon.a (mon.0) 964 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:41:55.311 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:55 smithi098 bash[79516]: cluster 2023-09-23T18:41:53.930650+0000 mgr.y (mgr.26054) 709 : cluster 0 pgmap v706: 89 pgs: 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 82 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 18 op/s; 0 B/s, 232 objects/s recovering 2023-09-23T18:41:55.311 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:55 smithi098 bash[79516]: cluster 2023-09-23T18:41:54.011280+0000 mon.a (mon.0) 961 : cluster 0 osdmap e181: 8 total, 8 up, 7 in 2023-09-23T18:41:55.311 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:55 smithi098 bash[79516]: cluster 2023-09-23T18:41:54.012730+0000 osd.4 (osd.4) 134 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T18:41:55.311 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:55 smithi098 bash[79516]: cluster 2023-09-23T18:41:54.015767+0000 osd.4 (osd.4) 135 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T18:41:55.311 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:55 smithi098 bash[79516]: cluster 2023-09-23T18:41:54.016848+0000 osd.6 (osd.6) 108 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T18:41:55.311 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:55 smithi098 bash[79516]: cluster 2023-09-23T18:41:54.016988+0000 osd.4 (osd.4) 136 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T18:41:55.311 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:55 smithi098 bash[79516]: cluster 2023-09-23T18:41:54.017605+0000 osd.7 (osd.7) 97 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T18:41:55.311 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:55 smithi098 bash[79516]: cluster 2023-09-23T18:41:54.018825+0000 osd.6 (osd.6) 109 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T18:41:55.312 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:55 smithi098 bash[79516]: cluster 2023-09-23T18:41:54.122367+0000 osd.5 (osd.5) 82 : cluster 3 failed to encode map e181 with expected crc 2023-09-23T18:41:55.312 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:55 smithi098 bash[79516]: audit 2023-09-23T18:41:54.581810+0000 mon.a (mon.0) 962 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:41:55.312 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:55 smithi098 bash[79516]: audit 2023-09-23T18:41:54.610437+0000 mon.a (mon.0) 963 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:41:55.312 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:55 smithi098 bash[79516]: audit 2023-09-23T18:41:54.611707+0000 mon.a (mon.0) 964 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:41:55.329 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:55.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:55.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:55.432 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:55.463 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:55.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:55.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:55.566 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:55.593 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:55.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:55.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:55.696 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:55.721 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:55.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:55.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:55.824 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:55.850 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:55.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:55.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:55.953 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:55.979 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:55.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:55.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:55.998 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T18:41:56.083 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:56.098 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:56 smithi098 bash[76817]: cluster 2023-09-23T18:41:55.002117+0000 osd.6 (osd.6) 110 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T18:41:56.098 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:56 smithi098 bash[76817]: cluster 2023-09-23T18:41:55.005982+0000 osd.4 (osd.4) 137 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T18:41:56.098 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:56 smithi098 bash[76817]: cluster 2023-09-23T18:41:55.006029+0000 osd.6 (osd.6) 111 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T18:41:56.098 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:56 smithi098 bash[76817]: cluster 2023-09-23T18:41:55.008164+0000 osd.4 (osd.4) 138 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T18:41:56.098 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:56 smithi098 bash[76817]: cluster 2023-09-23T18:41:55.008712+0000 mon.a (mon.0) 965 : cluster 0 osdmap e182: 8 total, 8 up, 7 in 2023-09-23T18:41:56.098 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:56 smithi098 bash[76817]: cluster 2023-09-23T18:41:55.009649+0000 osd.6 (osd.6) 112 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T18:41:56.106 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:56.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:56.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:56.209 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:56.220 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:56 smithi173 bash[59422]: cluster 2023-09-23T18:41:55.002117+0000 osd.6 (osd.6) 110 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T18:41:56.220 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:56 smithi173 bash[59422]: cluster 2023-09-23T18:41:55.005982+0000 osd.4 (osd.4) 137 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T18:41:56.221 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:56 smithi173 bash[59422]: cluster 2023-09-23T18:41:55.006029+0000 osd.6 (osd.6) 111 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T18:41:56.221 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:56 smithi173 bash[59422]: cluster 2023-09-23T18:41:55.008164+0000 osd.4 (osd.4) 138 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T18:41:56.221 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:56 smithi173 bash[59422]: cluster 2023-09-23T18:41:55.008712+0000 mon.a (mon.0) 965 : cluster 0 osdmap e182: 8 total, 8 up, 7 in 2023-09-23T18:41:56.221 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:56 smithi173 bash[59422]: cluster 2023-09-23T18:41:55.009649+0000 osd.6 (osd.6) 112 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T18:41:56.237 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:56.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:56.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:56.340 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:56.348 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:56 smithi098 bash[79516]: cluster 2023-09-23T18:41:55.002117+0000 osd.6 (osd.6) 110 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T18:41:56.348 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:56 smithi098 bash[79516]: cluster 2023-09-23T18:41:55.005982+0000 osd.4 (osd.4) 137 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T18:41:56.349 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:56 smithi098 bash[79516]: cluster 2023-09-23T18:41:55.006029+0000 osd.6 (osd.6) 111 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T18:41:56.349 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:56 smithi098 bash[79516]: cluster 2023-09-23T18:41:55.008164+0000 osd.4 (osd.4) 138 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T18:41:56.349 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:56 smithi098 bash[79516]: cluster 2023-09-23T18:41:55.008712+0000 mon.a (mon.0) 965 : cluster 0 osdmap e182: 8 total, 8 up, 7 in 2023-09-23T18:41:56.349 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:56 smithi098 bash[79516]: cluster 2023-09-23T18:41:55.009649+0000 osd.6 (osd.6) 112 : cluster 3 failed to encode map e182 with expected crc 2023-09-23T18:41:56.349 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:41:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:41:56] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.33.4" 2023-09-23T18:41:56.366 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:56.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:56.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:56.469 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:56.500 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:56.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:56.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:56.603 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:56.625 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:56.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:56.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:56.729 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:56.753 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:56.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:56.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:56.856 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:56.884 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:56.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:56.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:56.988 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:57.017 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:57.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:57.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:57.120 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:57.128 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:57 smithi098 bash[79516]: cluster 2023-09-23T18:41:55.931208+0000 mgr.y (mgr.26054) 710 : cluster 0 pgmap v709: 89 pgs: 2 peering, 1 active+recovering+remapped, 4 active+remapped+backfill_wait, 82 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 216 objects/s recovering 2023-09-23T18:41:57.128 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:57 smithi098 bash[79516]: cluster 2023-09-23T18:41:56.029779+0000 osd.4 (osd.4) 139 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T18:41:57.128 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:57 smithi098 bash[79516]: cluster 2023-09-23T18:41:56.029812+0000 osd.7 (osd.7) 98 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T18:41:57.128 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:57 smithi098 bash[79516]: cluster 2023-09-23T18:41:56.033286+0000 osd.6 (osd.6) 113 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T18:41:57.128 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:57 smithi098 bash[79516]: cluster 2023-09-23T18:41:56.034016+0000 osd.4 (osd.4) 140 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T18:41:57.128 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:57 smithi098 bash[79516]: cluster 2023-09-23T18:41:56.034342+0000 mon.a (mon.0) 966 : cluster 0 osdmap e183: 8 total, 8 up, 7 in 2023-09-23T18:41:57.128 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:57 smithi098 bash[79516]: cluster 2023-09-23T18:41:56.037643+0000 osd.6 (osd.6) 114 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T18:41:57.129 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:57 smithi098 bash[76817]: cluster 2023-09-23T18:41:55.931208+0000 mgr.y (mgr.26054) 710 : cluster 0 pgmap v709: 89 pgs: 2 peering, 1 active+recovering+remapped, 4 active+remapped+backfill_wait, 82 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 216 objects/s recovering 2023-09-23T18:41:57.129 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:57 smithi098 bash[76817]: cluster 2023-09-23T18:41:56.029779+0000 osd.4 (osd.4) 139 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T18:41:57.129 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:57 smithi098 bash[76817]: cluster 2023-09-23T18:41:56.029812+0000 osd.7 (osd.7) 98 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T18:41:57.129 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:57 smithi098 bash[76817]: cluster 2023-09-23T18:41:56.033286+0000 osd.6 (osd.6) 113 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T18:41:57.129 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:57 smithi098 bash[76817]: cluster 2023-09-23T18:41:56.034016+0000 osd.4 (osd.4) 140 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T18:41:57.129 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:57 smithi098 bash[76817]: cluster 2023-09-23T18:41:56.034342+0000 mon.a (mon.0) 966 : cluster 0 osdmap e183: 8 total, 8 up, 7 in 2023-09-23T18:41:57.129 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:57 smithi098 bash[76817]: cluster 2023-09-23T18:41:56.037643+0000 osd.6 (osd.6) 114 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T18:41:57.145 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:57.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:57.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:57.248 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:57.278 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:57.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:57.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:57.381 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:57.408 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:57.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:57.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:57.511 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:57.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:57 smithi173 bash[59422]: cluster 2023-09-23T18:41:55.931208+0000 mgr.y (mgr.26054) 710 : cluster 0 pgmap v709: 89 pgs: 2 peering, 1 active+recovering+remapped, 4 active+remapped+backfill_wait, 82 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 216 objects/s recovering 2023-09-23T18:41:57.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:57 smithi173 bash[59422]: cluster 2023-09-23T18:41:56.029779+0000 osd.4 (osd.4) 139 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T18:41:57.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:57 smithi173 bash[59422]: cluster 2023-09-23T18:41:56.029812+0000 osd.7 (osd.7) 98 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T18:41:57.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:57 smithi173 bash[59422]: cluster 2023-09-23T18:41:56.033286+0000 osd.6 (osd.6) 113 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T18:41:57.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:57 smithi173 bash[59422]: cluster 2023-09-23T18:41:56.034016+0000 osd.4 (osd.4) 140 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T18:41:57.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:57 smithi173 bash[59422]: cluster 2023-09-23T18:41:56.034342+0000 mon.a (mon.0) 966 : cluster 0 osdmap e183: 8 total, 8 up, 7 in 2023-09-23T18:41:57.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:57 smithi173 bash[59422]: cluster 2023-09-23T18:41:56.037643+0000 osd.6 (osd.6) 114 : cluster 3 failed to encode map e183 with expected crc 2023-09-23T18:41:57.534 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:41:57.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:57.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:41:57.638 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:57.666 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:57.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:57.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:57.770 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:57.799 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:57.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:57.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:57.903 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:57.929 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:41:57.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:57.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:41:58.033 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:58.067 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:41:58.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:58.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:41:58.170 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:58.202 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:58.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:58.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:58.306 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:58.338 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:58.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:58.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:58.442 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:58.472 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:58.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:58.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:58.575 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:58.606 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:58.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:58.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:58.711 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:58.737 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:58.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:58.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:58.840 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:58.875 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:58.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:58.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:58.979 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:59.012 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:41:59.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:59.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:41:59.115 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:59.140 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:59.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:59.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:59.244 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:59.258 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:41:59 smithi173 bash[59422]: cluster 2023-09-23T18:41:57.931757+0000 mgr.y (mgr.26054) 711 : cluster 0 pgmap v711: 89 pgs: 2 peering, 1 active+remapped+backfill_wait, 86 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 128 objects/s recovering 2023-09-23T18:41:59.262 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:41:59 smithi098 bash[76817]: cluster 2023-09-23T18:41:57.931757+0000 mgr.y (mgr.26054) 711 : cluster 0 pgmap v711: 89 pgs: 2 peering, 1 active+remapped+backfill_wait, 86 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 128 objects/s recovering 2023-09-23T18:41:59.262 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:41:59 smithi098 bash[79516]: cluster 2023-09-23T18:41:57.931757+0000 mgr.y (mgr.26054) 711 : cluster 0 pgmap v711: 89 pgs: 2 peering, 1 active+remapped+backfill_wait, 86 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 128 objects/s recovering 2023-09-23T18:41:59.277 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:59.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:59.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:41:59.381 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:59.412 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:59.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:59.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:59.516 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:59.545 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:41:59.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:59.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:41:59.648 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:59.675 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:41:59.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:59.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:41:59.779 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:59.802 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:41:59.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:41:59.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:41:59.905 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:41:59.999 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:00.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:00.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:00.061 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-09-23T18:42:00.103 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:00.124 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:00.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:00.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:00.228 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:00.255 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:00.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:00.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:00.359 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:00.390 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:00.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:00.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:00.494 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:00.519 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:00.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:00.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:00.542 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:42:00.622 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:00.646 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:00.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:00.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:00.750 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:00.776 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:00.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:00.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:00.879 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:00.906 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:00.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:00.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:01.009 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:01.034 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:01.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:01.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:01.136 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:01.148 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:01 smithi173 bash[59422]: cluster 2023-09-23T18:41:59.932513+0000 mgr.y (mgr.26054) 712 : cluster 0 pgmap v712: 89 pgs: 2 peering, 87 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 128 objects/s recovering 2023-09-23T18:42:01.148 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:01 smithi173 bash[59422]: audit 2023-09-23T18:42:00.058318+0000 mon.c (mon.2) 88 : audit 1 from='client.? 172.21.15.98:0/2816459397' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T18:42:01.148 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:01 smithi173 bash[59422]: audit 2023-09-23T18:42:00.058636+0000 mon.a (mon.0) 967 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T18:42:01.160 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:01.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:01.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:01.263 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:01.290 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:01.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:01.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:01.329 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:01 smithi098 bash[79516]: cluster 2023-09-23T18:41:59.932513+0000 mgr.y (mgr.26054) 712 : cluster 0 pgmap v712: 89 pgs: 2 peering, 87 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 128 objects/s recovering 2023-09-23T18:42:01.330 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:01 smithi098 bash[79516]: audit 2023-09-23T18:42:00.058318+0000 mon.c (mon.2) 88 : audit 1 from='client.? 172.21.15.98:0/2816459397' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T18:42:01.330 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:01 smithi098 bash[79516]: audit 2023-09-23T18:42:00.058636+0000 mon.a (mon.0) 967 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T18:42:01.330 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:01 smithi098 bash[76817]: cluster 2023-09-23T18:41:59.932513+0000 mgr.y (mgr.26054) 712 : cluster 0 pgmap v712: 89 pgs: 2 peering, 87 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 128 objects/s recovering 2023-09-23T18:42:01.330 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:01 smithi098 bash[76817]: audit 2023-09-23T18:42:00.058318+0000 mon.c (mon.2) 88 : audit 1 from='client.? 172.21.15.98:0/2816459397' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T18:42:01.330 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:01 smithi098 bash[76817]: audit 2023-09-23T18:42:00.058636+0000 mon.a (mon.0) 967 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T18:42:01.393 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:01.426 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:01.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:01.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:01.530 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:01.558 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:01.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:01.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:01.662 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:01.692 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:01.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:01.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:01.796 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:01.826 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:01.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:01.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:01.929 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:01.956 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:01.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:01.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:02.061 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:02.091 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:02.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:02.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:02.194 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:02.222 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:02.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:02.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:02.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:02 smithi173 bash[59422]: audit 2023-09-23T18:42:01.072799+0000 mon.a (mon.0) 968 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:42:02.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:02 smithi173 bash[59422]: audit 2023-09-23T18:42:01.078482+0000 mon.a (mon.0) 969 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:42:02.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:02 smithi173 bash[59422]: cluster 2023-09-23T18:42:01.933096+0000 mgr.y (mgr.26054) 713 : cluster 0 pgmap v713: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 97 objects/s recovering 2023-09-23T18:42:02.325 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:02.352 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:02.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:02.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:02.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:02 smithi098 bash[79516]: audit 2023-09-23T18:42:01.072799+0000 mon.a (mon.0) 968 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:42:02.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:02 smithi098 bash[79516]: audit 2023-09-23T18:42:01.078482+0000 mon.a (mon.0) 969 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:42:02.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:02 smithi098 bash[79516]: cluster 2023-09-23T18:42:01.933096+0000 mgr.y (mgr.26054) 713 : cluster 0 pgmap v713: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 97 objects/s recovering 2023-09-23T18:42:02.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:02 smithi098 bash[76817]: audit 2023-09-23T18:42:01.072799+0000 mon.a (mon.0) 968 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:42:02.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:02 smithi098 bash[76817]: audit 2023-09-23T18:42:01.078482+0000 mon.a (mon.0) 969 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:42:02.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:02 smithi098 bash[76817]: cluster 2023-09-23T18:42:01.933096+0000 mgr.y (mgr.26054) 713 : cluster 0 pgmap v713: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 97 objects/s recovering 2023-09-23T18:42:02.456 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:02.487 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:02.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:02.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:02.591 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:02.616 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:02.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:02.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:02.633 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-09-23T18:42:02.634 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-09-23T18:42:02.634 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-09-23T18:42:02.720 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:02.743 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:02.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:02.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:02.846 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:02.868 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:02.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:02.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:02.971 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:02.993 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:02.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:02.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:03.097 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:03.126 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:03.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:03.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:03.229 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:03.261 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:03.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:03.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:03.365 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:03.399 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:03.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:03.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:03.502 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:03.526 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:03.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:03.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:03.630 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:03.656 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:03.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:03.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:03.759 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:03.793 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:03.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:03.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:03.898 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:03.929 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:03.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:03.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:04.033 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:04.064 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:04.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:04.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:04.168 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:04.198 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:04.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:04.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:04.280 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:42:04.280 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-09-23T18:42:04.280 INFO:teuthology.orchestra.run.smithi173.stdout: "complaint_time": 30, 2023-09-23T18:42:04.280 INFO:teuthology.orchestra.run.smithi173.stdout: "num_blocked_ops": 0 2023-09-23T18:42:04.280 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:42:04.303 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:04.330 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:04.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:04.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:04.396 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:42:04.434 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:04.435 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:42:04.466 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:42:04.466 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:04.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:04.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:04.497 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:42:04.523 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:42:04.551 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:42:04.570 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:04.589 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:42:04.600 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:04.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:04.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:04.627 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:42:04.630 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:42:04.631 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:42:04.631 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:42:04.631 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:42:04.631 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:42:04.631 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:42:04.631 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:42:04.631 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:42:04.704 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:04.727 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:04.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:04.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:04.830 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:04.855 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:04.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:04.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:04.958 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:04.985 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:04.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:04.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:05.089 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:05.124 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:05.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:05.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:05.176 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is set 2023-09-23T18:42:05.187 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T18:42:05.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:04 smithi098 bash[76817]: cluster 2023-09-23T18:42:03.933934+0000 mgr.y (mgr.26054) 714 : cluster 0 pgmap v714: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 86 objects/s recovering 2023-09-23T18:42:05.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:04 smithi098 bash[76817]: audit 2023-09-23T18:42:04.152333+0000 mon.a (mon.0) 970 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:42:05.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:04 smithi098 bash[76817]: audit 2023-09-23T18:42:04.161128+0000 mon.a (mon.0) 971 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:42:05.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:04 smithi098 bash[76817]: audit 2023-09-23T18:42:04.367221+0000 mon.c (mon.2) 89 : audit 0 from='client.? 172.21.15.98:0/2630144381' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:42:05.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:04 smithi098 bash[76817]: audit 2023-09-23T18:42:04.665828+0000 mon.c (mon.2) 90 : audit 1 from='client.? 172.21.15.98:0/1822446845' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:42:05.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:04 smithi098 bash[76817]: audit 2023-09-23T18:42:04.666266+0000 mon.a (mon.0) 972 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:42:05.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:04 smithi098 bash[76817]: audit 2023-09-23T18:42:04.682397+0000 mon.a (mon.0) 973 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:42:05.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:04 smithi098 bash[76817]: audit 2023-09-23T18:42:04.683564+0000 mon.a (mon.0) 974 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:42:05.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:04 smithi098 bash[76817]: audit 2023-09-23T18:42:04.690893+0000 mon.a (mon.0) 975 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:42:05.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:04 smithi098 bash[79516]: cluster 2023-09-23T18:42:03.933934+0000 mgr.y (mgr.26054) 714 : cluster 0 pgmap v714: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 86 objects/s recovering 2023-09-23T18:42:05.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:04 smithi098 bash[79516]: audit 2023-09-23T18:42:04.152333+0000 mon.a (mon.0) 970 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:42:05.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:04 smithi098 bash[79516]: audit 2023-09-23T18:42:04.161128+0000 mon.a (mon.0) 971 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:42:05.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:04 smithi098 bash[79516]: audit 2023-09-23T18:42:04.367221+0000 mon.c (mon.2) 89 : audit 0 from='client.? 172.21.15.98:0/2630144381' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:42:05.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:04 smithi098 bash[79516]: audit 2023-09-23T18:42:04.665828+0000 mon.c (mon.2) 90 : audit 1 from='client.? 172.21.15.98:0/1822446845' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:42:05.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:04 smithi098 bash[79516]: audit 2023-09-23T18:42:04.666266+0000 mon.a (mon.0) 972 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:42:05.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:04 smithi098 bash[79516]: audit 2023-09-23T18:42:04.682397+0000 mon.a (mon.0) 973 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:42:05.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:04 smithi098 bash[79516]: audit 2023-09-23T18:42:04.683564+0000 mon.a (mon.0) 974 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:42:05.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:04 smithi098 bash[79516]: audit 2023-09-23T18:42:04.690893+0000 mon.a (mon.0) 975 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:42:05.233 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:05.267 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:05.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:05.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:05.281 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:04 smithi173 bash[59422]: cluster 2023-09-23T18:42:03.933934+0000 mgr.y (mgr.26054) 714 : cluster 0 pgmap v714: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 86 objects/s recovering 2023-09-23T18:42:05.281 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:04 smithi173 bash[59422]: audit 2023-09-23T18:42:04.152333+0000 mon.a (mon.0) 970 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:42:05.281 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:04 smithi173 bash[59422]: audit 2023-09-23T18:42:04.161128+0000 mon.a (mon.0) 971 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:42:05.281 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:04 smithi173 bash[59422]: audit 2023-09-23T18:42:04.367221+0000 mon.c (mon.2) 89 : audit 0 from='client.? 172.21.15.98:0/2630144381' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:42:05.282 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:04 smithi173 bash[59422]: audit 2023-09-23T18:42:04.665828+0000 mon.c (mon.2) 90 : audit 1 from='client.? 172.21.15.98:0/1822446845' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:42:05.282 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:04 smithi173 bash[59422]: audit 2023-09-23T18:42:04.666266+0000 mon.a (mon.0) 972 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:42:05.282 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:04 smithi173 bash[59422]: audit 2023-09-23T18:42:04.682397+0000 mon.a (mon.0) 973 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:42:05.283 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:04 smithi173 bash[59422]: audit 2023-09-23T18:42:04.683564+0000 mon.a (mon.0) 974 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:42:05.283 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:04 smithi173 bash[59422]: audit 2023-09-23T18:42:04.690893+0000 mon.a (mon.0) 975 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:42:05.372 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:05.411 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:05.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:05.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:05.517 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:05.562 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:05.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:05.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:05.664 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:05.692 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:05.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:05.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:05.796 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:05.836 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:05.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:05.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:05.938 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:05.968 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:05.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:05.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:06.072 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:06.105 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:06.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:06.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:06.210 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:06.227 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:06 smithi173 bash[59422]: cluster 2023-09-23T18:42:05.158707+0000 mon.a (mon.0) 976 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:42:06.227 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:06 smithi173 bash[59422]: audit 2023-09-23T18:42:05.163993+0000 mon.a (mon.0) 977 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:42:06.227 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:06 smithi173 bash[59422]: cluster 2023-09-23T18:42:05.164123+0000 mon.a (mon.0) 978 : cluster 0 osdmap e184: 8 total, 8 up, 7 in 2023-09-23T18:42:06.227 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:06 smithi173 bash[59422]: cluster 2023-09-23T18:42:05.222852+0000 osd.6 (osd.6) 115 : cluster 3 failed to encode map e184 with expected crc 2023-09-23T18:42:06.227 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:06 smithi173 bash[59422]: cluster 2023-09-23T18:42:05.222870+0000 osd.5 (osd.5) 83 : cluster 3 failed to encode map e184 with expected crc 2023-09-23T18:42:06.227 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:06 smithi173 bash[59422]: cluster 2023-09-23T18:42:05.222946+0000 osd.7 (osd.7) 99 : cluster 3 failed to encode map e184 with expected crc 2023-09-23T18:42:06.243 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:06 smithi098 bash[76817]: cluster 2023-09-23T18:42:05.158707+0000 mon.a (mon.0) 976 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:42:06.243 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:06 smithi098 bash[76817]: audit 2023-09-23T18:42:05.163993+0000 mon.a (mon.0) 977 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:42:06.244 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:06 smithi098 bash[76817]: cluster 2023-09-23T18:42:05.164123+0000 mon.a (mon.0) 978 : cluster 0 osdmap e184: 8 total, 8 up, 7 in 2023-09-23T18:42:06.244 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:06 smithi098 bash[76817]: cluster 2023-09-23T18:42:05.222852+0000 osd.6 (osd.6) 115 : cluster 3 failed to encode map e184 with expected crc 2023-09-23T18:42:06.244 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:06 smithi098 bash[76817]: cluster 2023-09-23T18:42:05.222870+0000 osd.5 (osd.5) 83 : cluster 3 failed to encode map e184 with expected crc 2023-09-23T18:42:06.244 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:06 smithi098 bash[76817]: cluster 2023-09-23T18:42:05.222946+0000 osd.7 (osd.7) 99 : cluster 3 failed to encode map e184 with expected crc 2023-09-23T18:42:06.248 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:06.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:06.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:06.352 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:06.387 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:06.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:06.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:06.491 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:06.525 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:06.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:06.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:06.631 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:06.666 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:06.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:06.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:06.685 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:42:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:42:06] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.33.4" 2023-09-23T18:42:06.685 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:06 smithi098 bash[79516]: cluster 2023-09-23T18:42:05.158707+0000 mon.a (mon.0) 976 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:42:06.686 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:06 smithi098 bash[79516]: audit 2023-09-23T18:42:05.163993+0000 mon.a (mon.0) 977 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:42:06.686 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:06 smithi098 bash[79516]: cluster 2023-09-23T18:42:05.164123+0000 mon.a (mon.0) 978 : cluster 0 osdmap e184: 8 total, 8 up, 7 in 2023-09-23T18:42:06.686 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:06 smithi098 bash[79516]: cluster 2023-09-23T18:42:05.222852+0000 osd.6 (osd.6) 115 : cluster 3 failed to encode map e184 with expected crc 2023-09-23T18:42:06.686 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:06 smithi098 bash[79516]: cluster 2023-09-23T18:42:05.222870+0000 osd.5 (osd.5) 83 : cluster 3 failed to encode map e184 with expected crc 2023-09-23T18:42:06.686 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:06 smithi098 bash[79516]: cluster 2023-09-23T18:42:05.222946+0000 osd.7 (osd.7) 99 : cluster 3 failed to encode map e184 with expected crc 2023-09-23T18:42:06.769 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:06.804 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:06.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:06.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:06.908 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:06.939 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:06.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:06.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:07.043 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:07.076 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:07.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:07.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:07.180 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:07.194 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:07 smithi173 bash[59422]: cluster 2023-09-23T18:42:05.222783+0000 osd.4 (osd.4) 141 : cluster 3 failed to encode map e184 with expected crc 2023-09-23T18:42:07.194 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:07 smithi173 bash[59422]: cluster 2023-09-23T18:42:05.934533+0000 mgr.y (mgr.26054) 715 : cluster 0 pgmap v716: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 77 objects/s recovering 2023-09-23T18:42:07.214 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:07.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:07.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:07.318 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:07.331 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:07 smithi098 bash[79516]: cluster 2023-09-23T18:42:05.222783+0000 osd.4 (osd.4) 141 : cluster 3 failed to encode map e184 with expected crc 2023-09-23T18:42:07.331 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:07 smithi098 bash[79516]: cluster 2023-09-23T18:42:05.934533+0000 mgr.y (mgr.26054) 715 : cluster 0 pgmap v716: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 77 objects/s recovering 2023-09-23T18:42:07.332 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:07 smithi098 bash[76817]: cluster 2023-09-23T18:42:05.222783+0000 osd.4 (osd.4) 141 : cluster 3 failed to encode map e184 with expected crc 2023-09-23T18:42:07.332 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:07 smithi098 bash[76817]: cluster 2023-09-23T18:42:05.934533+0000 mgr.y (mgr.26054) 715 : cluster 0 pgmap v716: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 77 objects/s recovering 2023-09-23T18:42:07.349 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:07.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:07.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:07.453 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:07.484 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:07.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:07.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:07.588 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:07.622 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:07.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:07.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:07.635 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T18:42:07.635 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:42:07.635 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T18:42:07.636 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T18:42:07.636 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T18:42:07.636 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T18:42:07.726 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:07.756 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:07.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:07.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:07.860 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:07.893 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:07.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:07.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:07.988 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:42:07.997 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:08.063 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:08.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:08.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:08.166 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:08.191 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:08.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:08.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:08.294 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:08.332 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:08.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:08.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:08.435 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:08.464 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:08.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:08.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:08.568 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:08.599 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:08.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:08.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:08.687 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:42:08.704 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:08.739 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:08.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:08.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:08.843 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:08.877 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:08.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:08.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:08.981 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:09.013 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:09.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:09.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:09.118 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:09.155 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:09.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:09.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:09.191 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:42:09.258 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:09.271 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:09 smithi173 bash[59422]: cluster 2023-09-23T18:42:07.935352+0000 mgr.y (mgr.26054) 716 : cluster 0 pgmap v717: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:09.292 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:09.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:09.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:09.396 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:09.431 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:09.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:09.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:09.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:09 smithi098 bash[79516]: cluster 2023-09-23T18:42:07.935352+0000 mgr.y (mgr.26054) 716 : cluster 0 pgmap v717: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:09.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:09 smithi098 bash[76817]: cluster 2023-09-23T18:42:07.935352+0000 mgr.y (mgr.26054) 716 : cluster 0 pgmap v717: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:09.535 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:09.562 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:09.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:09.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:09.665 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:09.691 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:09.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:09.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:09.794 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:09.824 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:09.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:09.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:09.927 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:09.959 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:09.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:09.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:09.981 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:42:10.062 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:10.093 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:10.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:10.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:10.197 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:10.226 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:10.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:10.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:10.330 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:10.357 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:10.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:10.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:10.461 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:10.491 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:10.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:10.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:10.595 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:10.624 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:10.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:10.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:10.728 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:10.758 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:10.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:10.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:10.861 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:10.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:10 smithi173 bash[59422]: audit 2023-09-23T18:42:09.610269+0000 mon.a (mon.0) 979 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:42:10.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:10 smithi173 bash[59422]: audit 2023-09-23T18:42:09.611829+0000 mon.a (mon.0) 980 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:42:10.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:10 smithi173 bash[59422]: cluster 2023-09-23T18:42:09.936167+0000 mgr.y (mgr.26054) 717 : cluster 0 pgmap v718: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:10.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:10 smithi173 bash[59422]: audit 2023-09-23T18:42:10.117738+0000 mon.a (mon.0) 981 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:42:10.882 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:10 smithi098 bash[76817]: audit 2023-09-23T18:42:09.610269+0000 mon.a (mon.0) 979 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:42:10.882 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:10 smithi098 bash[76817]: audit 2023-09-23T18:42:09.611829+0000 mon.a (mon.0) 980 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:42:10.882 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:10 smithi098 bash[76817]: cluster 2023-09-23T18:42:09.936167+0000 mgr.y (mgr.26054) 717 : cluster 0 pgmap v718: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:10.882 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:10 smithi098 bash[76817]: audit 2023-09-23T18:42:10.117738+0000 mon.a (mon.0) 981 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:42:10.883 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:10 smithi098 bash[79516]: audit 2023-09-23T18:42:09.610269+0000 mon.a (mon.0) 979 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:42:10.883 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:10 smithi098 bash[79516]: audit 2023-09-23T18:42:09.611829+0000 mon.a (mon.0) 980 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:42:10.883 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:10 smithi098 bash[79516]: cluster 2023-09-23T18:42:09.936167+0000 mgr.y (mgr.26054) 717 : cluster 0 pgmap v718: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:10.883 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:10 smithi098 bash[79516]: audit 2023-09-23T18:42:10.117738+0000 mon.a (mon.0) 981 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:42:10.893 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:10.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:10.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:10.997 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:11.027 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:11.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:11.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:11.130 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:11.157 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:11.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:11.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:11.261 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:11.293 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:11.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:11.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:11.397 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:11.426 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:11.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:11.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:11.530 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:11.559 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:11.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:11.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:11.663 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:11.690 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:11.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:11.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:11.794 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:11.821 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:11.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:11.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:11.924 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:11.951 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:11.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:11.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:12.054 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:12.079 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:12.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:12.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:12.182 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:12.207 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:12.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:12.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:12.310 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:12.343 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:12.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:12.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:12.350 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:42:12.350 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:42:12.447 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:12.471 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:12.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:12.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:12.574 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:12.605 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:12.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:12.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:12.709 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:12.738 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:12.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:12.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:12.842 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:12.868 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:12.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:12.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:12.972 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:13.005 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:13.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:13.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:13.109 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:13.136 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:13.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:13.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:13.239 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:13.246 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:13 smithi098 bash[79516]: cluster 2023-09-23T18:42:11.937047+0000 mgr.y (mgr.26054) 718 : cluster 0 pgmap v719: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:13.247 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:12 smithi098 bash[76817]: cluster 2023-09-23T18:42:11.937047+0000 mgr.y (mgr.26054) 718 : cluster 0 pgmap v719: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:13.256 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:42:13.264 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:13.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:13.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:13.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:12 smithi173 bash[59422]: cluster 2023-09-23T18:42:11.937047+0000 mgr.y (mgr.26054) 718 : cluster 0 pgmap v719: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:13.368 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:13.390 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:13.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:13.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:13.493 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:13.521 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:13.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:13.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:13.625 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:13.654 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:13.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:13.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:13.757 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:13.761 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T18:42:13.761 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:42:13.784 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:13.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:13.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:13.887 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:13.917 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:13.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:13.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:14.020 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:14.050 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:14.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:14.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:14.153 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:14.179 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:14.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:14.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:14.282 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:14.313 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:14.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:14.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:14.416 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:14.449 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:14.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:14.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:14.553 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:14.585 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:14.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:14.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:14.688 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:14.710 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:14.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:14.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:14.723 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T18:42:14.814 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:14.835 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:14.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:14.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:14.938 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:14.962 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:14.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:14.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:15.066 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:15.098 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:15.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:15.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:15.201 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:15.221 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:14 smithi098 bash[79516]: cluster 2023-09-23T18:42:13.938036+0000 mgr.y (mgr.26054) 719 : cluster 0 pgmap v720: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:15.221 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:14 smithi098 bash[76817]: cluster 2023-09-23T18:42:13.938036+0000 mgr.y (mgr.26054) 719 : cluster 0 pgmap v720: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:15.234 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:15.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:15.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:15.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:14 smithi173 bash[59422]: cluster 2023-09-23T18:42:13.938036+0000 mgr.y (mgr.26054) 719 : cluster 0 pgmap v720: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:15.338 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:15.363 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:15.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:15.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:15.466 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:15.497 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:15.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:15.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:15.600 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:15.634 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:15.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:15.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:15.737 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:15.765 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:15.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:15.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:15.869 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:15.894 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:15.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:15.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:15.997 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:16.026 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:16.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:16.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:16.043 INFO:teuthology.orchestra.run.smithi098.stderr:pool 'unique_pool_1' removed 2023-09-23T18:42:16.129 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:16.149 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:16 smithi098 bash[79516]: audit 2023-09-23T18:42:15.445214+0000 mon.c (mon.2) 91 : audit 1 from='client.? 172.21.15.98:0/1117018304' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:42:16.149 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:16 smithi098 bash[79516]: audit 2023-09-23T18:42:15.445681+0000 mon.a (mon.0) 982 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:42:16.149 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:16 smithi098 bash[76817]: audit 2023-09-23T18:42:15.445214+0000 mon.c (mon.2) 91 : audit 1 from='client.? 172.21.15.98:0/1117018304' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:42:16.150 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:16 smithi098 bash[76817]: audit 2023-09-23T18:42:15.445681+0000 mon.a (mon.0) 982 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:42:16.157 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:16.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:16.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:16.261 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:16.274 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:16 smithi173 bash[59422]: audit 2023-09-23T18:42:15.445214+0000 mon.c (mon.2) 91 : audit 1 from='client.? 172.21.15.98:0/1117018304' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:42:16.274 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:16 smithi173 bash[59422]: audit 2023-09-23T18:42:15.445681+0000 mon.a (mon.0) 982 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:42:16.297 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:16.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:16.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:16.401 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:16.432 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:16.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:16.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:16.440 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:42:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:42:16] "GET /metrics HTTP/1.1" 200 36547 "" "Prometheus/2.33.4" 2023-09-23T18:42:16.536 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:16.574 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:16.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:16.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:16.679 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:16.706 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:16.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:16.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:16.809 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:16.839 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:16.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:16.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:16.942 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:16.969 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:16.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:16.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:17.073 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:17.081 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:17 smithi098 bash[76817]: cluster 2023-09-23T18:42:15.938637+0000 mgr.y (mgr.26054) 720 : cluster 0 pgmap v721: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:17.081 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:17 smithi098 bash[76817]: cluster 2023-09-23T18:42:16.018475+0000 osd.4 (osd.4) 142 : cluster 3 failed to encode map e185 with expected crc 2023-09-23T18:42:17.081 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:17 smithi098 bash[76817]: audit 2023-09-23T18:42:16.035311+0000 mon.a (mon.0) 983 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:42:17.081 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:17 smithi098 bash[76817]: cluster 2023-09-23T18:42:16.035408+0000 mon.a (mon.0) 984 : cluster 0 osdmap e185: 8 total, 8 up, 7 in 2023-09-23T18:42:17.081 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:17 smithi098 bash[76817]: cluster 2023-09-23T18:42:16.552812+0000 osd.6 (osd.6) 116 : cluster 3 failed to encode map e185 with expected crc 2023-09-23T18:42:17.082 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:17 smithi098 bash[76817]: cluster 2023-09-23T18:42:16.552824+0000 osd.5 (osd.5) 84 : cluster 3 failed to encode map e185 with expected crc 2023-09-23T18:42:17.082 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:17 smithi098 bash[76817]: cluster 2023-09-23T18:42:16.552849+0000 osd.7 (osd.7) 100 : cluster 3 failed to encode map e185 with expected crc 2023-09-23T18:42:17.082 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:17 smithi098 bash[79516]: cluster 2023-09-23T18:42:15.938637+0000 mgr.y (mgr.26054) 720 : cluster 0 pgmap v721: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:17.082 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:17 smithi098 bash[79516]: cluster 2023-09-23T18:42:16.018475+0000 osd.4 (osd.4) 142 : cluster 3 failed to encode map e185 with expected crc 2023-09-23T18:42:17.082 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:17 smithi098 bash[79516]: audit 2023-09-23T18:42:16.035311+0000 mon.a (mon.0) 983 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:42:17.082 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:17 smithi098 bash[79516]: cluster 2023-09-23T18:42:16.035408+0000 mon.a (mon.0) 984 : cluster 0 osdmap e185: 8 total, 8 up, 7 in 2023-09-23T18:42:17.082 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:17 smithi098 bash[79516]: cluster 2023-09-23T18:42:16.552812+0000 osd.6 (osd.6) 116 : cluster 3 failed to encode map e185 with expected crc 2023-09-23T18:42:17.082 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:17 smithi098 bash[79516]: cluster 2023-09-23T18:42:16.552824+0000 osd.5 (osd.5) 84 : cluster 3 failed to encode map e185 with expected crc 2023-09-23T18:42:17.082 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:17 smithi098 bash[79516]: cluster 2023-09-23T18:42:16.552849+0000 osd.7 (osd.7) 100 : cluster 3 failed to encode map e185 with expected crc 2023-09-23T18:42:17.095 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:17.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:17.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:17.198 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:17.210 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:17 smithi173 bash[59422]: cluster 2023-09-23T18:42:15.938637+0000 mgr.y (mgr.26054) 720 : cluster 0 pgmap v721: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:17.211 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:17 smithi173 bash[59422]: cluster 2023-09-23T18:42:16.018475+0000 osd.4 (osd.4) 142 : cluster 3 failed to encode map e185 with expected crc 2023-09-23T18:42:17.211 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:17 smithi173 bash[59422]: audit 2023-09-23T18:42:16.035311+0000 mon.a (mon.0) 983 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:42:17.211 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:17 smithi173 bash[59422]: cluster 2023-09-23T18:42:16.035408+0000 mon.a (mon.0) 984 : cluster 0 osdmap e185: 8 total, 8 up, 7 in 2023-09-23T18:42:17.211 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:17 smithi173 bash[59422]: cluster 2023-09-23T18:42:16.552812+0000 osd.6 (osd.6) 116 : cluster 3 failed to encode map e185 with expected crc 2023-09-23T18:42:17.211 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:17 smithi173 bash[59422]: cluster 2023-09-23T18:42:16.552824+0000 osd.5 (osd.5) 84 : cluster 3 failed to encode map e185 with expected crc 2023-09-23T18:42:17.211 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:17 smithi173 bash[59422]: cluster 2023-09-23T18:42:16.552849+0000 osd.7 (osd.7) 100 : cluster 3 failed to encode map e185 with expected crc 2023-09-23T18:42:17.228 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:17.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:17.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:17.333 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:17.366 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:17.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:17.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:17.469 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:17.492 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:17.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:17.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:17.595 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:17.629 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:17.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:17.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:17.732 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:17.767 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:17.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:17.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:17.871 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:17.895 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:17.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:17.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:17.999 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:18.028 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:18.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:18.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:18.132 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:18.157 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:18.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:18.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:18.261 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:18.290 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:18.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:18.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:18.394 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:18.426 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:18.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:18.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:18.529 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:18.560 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:18.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:18.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:18.663 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:18.685 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:18.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:18.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:18.788 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:18.811 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:18.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:18.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:18.914 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:18.941 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:18.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:18.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:19.045 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:19.067 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:19.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:19.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:19.170 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:19.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:19 smithi098 bash[76817]: cluster 2023-09-23T18:42:17.939242+0000 mgr.y (mgr.26054) 721 : cluster 0 pgmap v723: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:19.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:19 smithi098 bash[79516]: cluster 2023-09-23T18:42:17.939242+0000 mgr.y (mgr.26054) 721 : cluster 0 pgmap v723: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:19.200 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:19.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:19.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:19.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:19 smithi173 bash[59422]: cluster 2023-09-23T18:42:17.939242+0000 mgr.y (mgr.26054) 721 : cluster 0 pgmap v723: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:19.303 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:19.335 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:19.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:19.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:19.439 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:19.468 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:19.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:19.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:19.496 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T18:42:19.496 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T18:42:19.497 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-09-23T18:42:19.497 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-09-23T18:42:19.498 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:42:19.572 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:19.598 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:19.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:19.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:19.702 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:19.733 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:19.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:19.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:19.836 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:19.858 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:19.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:19.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:19.961 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:19.988 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:19.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:19.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:20.091 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:20.116 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:20.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:20.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:20.220 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:20.250 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:20.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:20.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:20.353 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:20.380 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:20.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:20.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:20.484 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:20.510 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:20.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:20.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:20.615 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:20.640 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:20.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:20.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:20.744 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:20.771 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:20.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:20.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:20.874 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:20.905 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:20.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:20.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:21.009 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:21.037 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:21.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:21.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:21.141 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:21.154 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:21 smithi173 bash[59422]: cluster 2023-09-23T18:42:19.939911+0000 mgr.y (mgr.26054) 722 : cluster 0 pgmap v724: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:21.168 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:21.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:21.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:21.271 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:21.297 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:21.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:21.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:21.347 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:21 smithi098 bash[79516]: cluster 2023-09-23T18:42:19.939911+0000 mgr.y (mgr.26054) 722 : cluster 0 pgmap v724: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:21.347 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:21 smithi098 bash[76817]: cluster 2023-09-23T18:42:19.939911+0000 mgr.y (mgr.26054) 722 : cluster 0 pgmap v724: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:21.401 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:21.427 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:21.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:21.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:21.530 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:21.552 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:21.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:21.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:21.655 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:21.673 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:42:21.687 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:21.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:21.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:21.696 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:42:21.716 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:42:21.741 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:42:21.762 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:42:21.786 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:42:21.791 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:21.811 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:21.812 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:42:21.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:21.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:21.834 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:42:21.836 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:42:21.836 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:42:21.836 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:42:21.836 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:42:21.836 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:42:21.837 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:42:21.837 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:42:21.837 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:42:21.914 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:21.938 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:21.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:21.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:22.041 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:22.066 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:22.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:22.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:22.170 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:22.184 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:22 smithi173 bash[59422]: audit 2023-09-23T18:42:21.647867+0000 mon.a (mon.0) 985 : audit 0 from='client.? 172.21.15.98:0/4104214256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:42:22.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:22 smithi098 bash[79516]: audit 2023-09-23T18:42:21.647867+0000 mon.a (mon.0) 985 : audit 0 from='client.? 172.21.15.98:0/4104214256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:42:22.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:22 smithi098 bash[76817]: audit 2023-09-23T18:42:21.647867+0000 mon.a (mon.0) 985 : audit 0 from='client.? 172.21.15.98:0/4104214256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:42:22.200 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:22.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:22.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:22.304 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:22.338 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:22.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:22.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:22.442 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:22.463 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:22.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:22.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:22.567 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:22.592 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:22.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:22.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:22.695 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:22.724 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:22.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:22.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:22.827 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:22.846 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:22.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:22.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:22.950 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:22.972 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:22.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:22.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:23.047 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is set 2023-09-23T18:42:23.075 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:23.101 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:23 smithi098 bash[79516]: cluster 2023-09-23T18:42:21.940448+0000 mgr.y (mgr.26054) 723 : cluster 0 pgmap v725: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:23.101 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:23 smithi098 bash[79516]: audit 2023-09-23T18:42:22.798786+0000 mon.a (mon.0) 986 : audit 1 from='client.? 172.21.15.98:0/310500517' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:42:23.101 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:23 smithi098 bash[76817]: cluster 2023-09-23T18:42:21.940448+0000 mgr.y (mgr.26054) 723 : cluster 0 pgmap v725: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:23.101 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:23 smithi098 bash[76817]: audit 2023-09-23T18:42:22.798786+0000 mon.a (mon.0) 986 : audit 1 from='client.? 172.21.15.98:0/310500517' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:42:23.102 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:23.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:23.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:23.210 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:23.222 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:42:23.222 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-09-23T18:42:23.223 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 0 2023-09-23T18:42:23.223 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:42:23.236 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:23.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:23.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:23.269 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:23 smithi173 bash[59422]: cluster 2023-09-23T18:42:21.940448+0000 mgr.y (mgr.26054) 723 : cluster 0 pgmap v725: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:23.269 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:23 smithi173 bash[59422]: audit 2023-09-23T18:42:22.798786+0000 mon.a (mon.0) 986 : audit 1 from='client.? 172.21.15.98:0/310500517' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:42:23.338 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:23.361 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:23.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:23.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:23.465 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:23.499 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:23.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:23.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:23.555 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:42:23.556 INFO:teuthology.orchestra.run.smithi098.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T18:42:23.556 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:42:23.602 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:23.630 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:23.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:23.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:23.733 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:23.758 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:23.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:23.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:23.861 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:23.891 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:23.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:23.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:23.994 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:24.026 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:24.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:24.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:24.129 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:24.163 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:24.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:24.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:24.213 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:24 smithi098 bash[79516]: cluster 2023-09-23T18:42:23.032666+0000 mon.a (mon.0) 987 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:42:24.213 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:24 smithi098 bash[79516]: audit 2023-09-23T18:42:23.040351+0000 mon.a (mon.0) 988 : audit 1 from='client.? 172.21.15.98:0/310500517' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:42:24.213 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:24 smithi098 bash[79516]: cluster 2023-09-23T18:42:23.040503+0000 mon.a (mon.0) 989 : cluster 0 osdmap e186: 8 total, 8 up, 7 in 2023-09-23T18:42:24.213 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:24 smithi098 bash[79516]: cluster 2023-09-23T18:42:23.127665+0000 osd.5 (osd.5) 85 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T18:42:24.213 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:24 smithi098 bash[79516]: cluster 2023-09-23T18:42:23.128088+0000 osd.7 (osd.7) 101 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T18:42:24.213 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:24 smithi098 bash[79516]: cluster 2023-09-23T18:42:23.128140+0000 osd.6 (osd.6) 117 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T18:42:24.214 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:24 smithi098 bash[76817]: cluster 2023-09-23T18:42:23.032666+0000 mon.a (mon.0) 987 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:42:24.214 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:24 smithi098 bash[76817]: audit 2023-09-23T18:42:23.040351+0000 mon.a (mon.0) 988 : audit 1 from='client.? 172.21.15.98:0/310500517' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:42:24.214 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:24 smithi098 bash[76817]: cluster 2023-09-23T18:42:23.040503+0000 mon.a (mon.0) 989 : cluster 0 osdmap e186: 8 total, 8 up, 7 in 2023-09-23T18:42:24.214 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:24 smithi098 bash[76817]: cluster 2023-09-23T18:42:23.127665+0000 osd.5 (osd.5) 85 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T18:42:24.214 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:24 smithi098 bash[76817]: cluster 2023-09-23T18:42:23.128088+0000 osd.7 (osd.7) 101 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T18:42:24.214 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:24 smithi098 bash[76817]: cluster 2023-09-23T18:42:23.128140+0000 osd.6 (osd.6) 117 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T18:42:24.255 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T18:42:24.266 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:24.267 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:24 smithi173 bash[59422]: cluster 2023-09-23T18:42:23.032666+0000 mon.a (mon.0) 987 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:42:24.267 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:24 smithi173 bash[59422]: audit 2023-09-23T18:42:23.040351+0000 mon.a (mon.0) 988 : audit 1 from='client.? 172.21.15.98:0/310500517' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:42:24.267 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:24 smithi173 bash[59422]: cluster 2023-09-23T18:42:23.040503+0000 mon.a (mon.0) 989 : cluster 0 osdmap e186: 8 total, 8 up, 7 in 2023-09-23T18:42:24.267 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:24 smithi173 bash[59422]: cluster 2023-09-23T18:42:23.127665+0000 osd.5 (osd.5) 85 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T18:42:24.267 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:24 smithi173 bash[59422]: cluster 2023-09-23T18:42:23.128088+0000 osd.7 (osd.7) 101 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T18:42:24.267 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:24 smithi173 bash[59422]: cluster 2023-09-23T18:42:23.128140+0000 osd.6 (osd.6) 117 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T18:42:24.298 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:24.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:24.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:24.403 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:24.429 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:24.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:24.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:24.532 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:24.559 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:24.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:24.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:24.663 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:24.698 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:24.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:24.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:24.802 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:24.828 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:24.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:24.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:24.932 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:24.962 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:24.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:24.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:25.065 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:25.075 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:25 smithi098 bash[79516]: cluster 2023-09-23T18:42:23.127741+0000 osd.4 (osd.4) 143 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T18:42:25.075 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:25 smithi098 bash[79516]: cluster 2023-09-23T18:42:23.941156+0000 mgr.y (mgr.26054) 724 : cluster 0 pgmap v727: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:25.075 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:25 smithi098 bash[79516]: audit 2023-09-23T18:42:24.605572+0000 mon.a (mon.0) 990 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:42:25.076 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:25 smithi098 bash[76817]: cluster 2023-09-23T18:42:23.127741+0000 osd.4 (osd.4) 143 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T18:42:25.076 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:25 smithi098 bash[76817]: cluster 2023-09-23T18:42:23.941156+0000 mgr.y (mgr.26054) 724 : cluster 0 pgmap v727: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:25.076 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:25 smithi098 bash[76817]: audit 2023-09-23T18:42:24.605572+0000 mon.a (mon.0) 990 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:42:25.095 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:25.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:25.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:25.199 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:25.229 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:25.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:25.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:25.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:25 smithi173 bash[59422]: cluster 2023-09-23T18:42:23.127741+0000 osd.4 (osd.4) 143 : cluster 3 failed to encode map e186 with expected crc 2023-09-23T18:42:25.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:25 smithi173 bash[59422]: cluster 2023-09-23T18:42:23.941156+0000 mgr.y (mgr.26054) 724 : cluster 0 pgmap v727: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:25.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:25 smithi173 bash[59422]: audit 2023-09-23T18:42:24.605572+0000 mon.a (mon.0) 990 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:42:25.333 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:25.363 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:25.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:25.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:25.441 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:42:25.467 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:25.520 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:25.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:25.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:25.624 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:25.655 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:25.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:25.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:25.759 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:25.791 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:25.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:25.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:25.895 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:25.928 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:25.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:25.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:26.032 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:26.066 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:26.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:26.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:26.170 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:26.201 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:26.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:26.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:26.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:26 smithi173 bash[59422]: cluster 2023-09-23T18:42:25.942099+0000 mgr.y (mgr.26054) 725 : cluster 0 pgmap v728: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:26.306 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:26.319 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:26 smithi098 bash[76817]: cluster 2023-09-23T18:42:25.942099+0000 mgr.y (mgr.26054) 725 : cluster 0 pgmap v728: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:26.320 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:26 smithi098 bash[79516]: cluster 2023-09-23T18:42:25.942099+0000 mgr.y (mgr.26054) 725 : cluster 0 pgmap v728: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:26.320 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:42:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:42:26] "GET /metrics HTTP/1.1" 200 34208 "" "Prometheus/2.33.4" 2023-09-23T18:42:26.337 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:26.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:26.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:26.441 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:26.473 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:26.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:26.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:26.578 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:26.613 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:26.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:26.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:26.717 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:26.748 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:26.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:26.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:26.845 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:42:26.854 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:26.885 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:26.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:26.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:26.989 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:27.021 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:27.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:27.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:27.125 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:27.156 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:27.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:27.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:27.260 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:27.294 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:27.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:27.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:27.396 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:27.427 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:27.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:27.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:27.532 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:27.562 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:27.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:27.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:27.666 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:27.696 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:27.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:27.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:27.800 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:27.831 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:27.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:27.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:27.935 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:27.966 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:27.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:27.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:28.070 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:28.104 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:28.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:28.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:28.207 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:28.237 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:28.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:28.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:28.341 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:28.371 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:42:28.374 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:28.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:28.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:28.478 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:28.510 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:28.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:28.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:28.614 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:28.644 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:28.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:28.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:28.747 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:28.773 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:28.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:28.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:28.876 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:28.909 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:28.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:28.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:29.013 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:29.024 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:28 smithi098 bash[76817]: cluster 2023-09-23T18:42:27.942906+0000 mgr.y (mgr.26054) 726 : cluster 0 pgmap v729: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:29.024 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:28 smithi098 bash[79516]: cluster 2023-09-23T18:42:27.942906+0000 mgr.y (mgr.26054) 726 : cluster 0 pgmap v729: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:29.028 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:42:29.040 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:29.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:29.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:29.144 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:29.176 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:29.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:29.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:29.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:28 smithi173 bash[59422]: cluster 2023-09-23T18:42:27.942906+0000 mgr.y (mgr.26054) 726 : cluster 0 pgmap v729: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:29.280 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:29.309 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:29.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:29.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:29.384 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:42:29.412 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:29.433 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:29.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:29.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:29.536 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:29.562 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:29.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:29.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:29.665 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:29.693 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:29.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:29.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:29.796 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:29.821 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:29.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:29.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:29.925 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:29.950 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:29.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:29.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:30.053 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:30.074 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:30.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:30.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:30.178 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:30.204 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:30.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:30.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:30.308 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:30.335 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:30.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:30.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:30.391 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T18:42:30.391 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:42:30.391 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-09-23T18:42:30.391 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T18:42:30.391 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T18:42:30.392 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-09-23T18:42:30.440 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:30.460 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:30.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:30.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:30.563 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:30.598 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:30.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:30.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:30.701 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:30.729 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:30.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:30.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:30.832 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:30.864 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:30.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:30.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:30.967 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:31.019 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:31.023 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:31 smithi098 bash[76817]: cluster 2023-09-23T18:42:29.943511+0000 mgr.y (mgr.26054) 727 : cluster 0 pgmap v730: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:31.023 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:31 smithi098 bash[79516]: cluster 2023-09-23T18:42:29.943511+0000 mgr.y (mgr.26054) 727 : cluster 0 pgmap v730: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:31.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:31.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:31.125 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:31.137 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:31 smithi173 bash[59422]: cluster 2023-09-23T18:42:29.943511+0000 mgr.y (mgr.26054) 727 : cluster 0 pgmap v730: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:31.151 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:31.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:31.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:31.255 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:31.285 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:31.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:31.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:31.388 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:31.413 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:42:31.414 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:31.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:31.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:31.517 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:31.541 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:31.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:31.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:31.644 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:31.669 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:31.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:31.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:31.772 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:31.792 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:31.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:31.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:31.895 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:31.917 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:31.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:31.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:32.020 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:32.051 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:32.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:32.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:32.154 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:32.180 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:32.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:32.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:32.285 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:32.307 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:32.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:32.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:32.411 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:32.436 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:32.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:32.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:32.538 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:32.563 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:32.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:32.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:32.666 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:32.693 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:32.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:32.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:32.791 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:42:32.791 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-09-23T18:42:32.791 INFO:teuthology.orchestra.run.smithi173.stdout: "complaint_time": 30, 2023-09-23T18:42:32.792 INFO:teuthology.orchestra.run.smithi173.stdout: "num_blocked_ops": 0 2023-09-23T18:42:32.792 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:42:32.797 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:32.829 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:32.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:32.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:32.932 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:32.958 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:32.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:32.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:33.061 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:33.088 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:33.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:33.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:33.192 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:33.205 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:33 smithi173 bash[59422]: cluster 2023-09-23T18:42:31.944143+0000 mgr.y (mgr.26054) 728 : cluster 0 pgmap v731: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:33.222 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:33.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:33.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:33.325 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:33.338 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:33 smithi098 bash[79516]: cluster 2023-09-23T18:42:31.944143+0000 mgr.y (mgr.26054) 728 : cluster 0 pgmap v731: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:33.339 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:33 smithi098 bash[76817]: cluster 2023-09-23T18:42:31.944143+0000 mgr.y (mgr.26054) 728 : cluster 0 pgmap v731: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:33.352 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:33.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:33.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:33.456 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:33.491 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:33.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:33.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:33.595 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:33.632 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:33.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:33.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:33.657 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T18:42:33.736 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:33.772 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:33.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:33.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:33.875 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:33.903 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:33.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:33.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:34.006 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:34.035 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:34.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:34.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:34.139 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:34.171 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:34.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:34.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:34.274 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:34.299 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:34.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:34.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:34.403 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:34.429 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:34.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:34.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:34.533 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:34.558 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:34.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:34.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:34.662 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:34.697 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:34.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:34.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:34.801 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:34.825 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:34.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:34.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:34.928 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:34.952 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:34.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:34.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:35.055 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:35.076 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:35.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:35.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:35.180 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:35.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:35 smithi098 bash[76817]: cluster 2023-09-23T18:42:33.944650+0000 mgr.y (mgr.26054) 729 : cluster 0 pgmap v732: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:35.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:35 smithi098 bash[79516]: cluster 2023-09-23T18:42:33.944650+0000 mgr.y (mgr.26054) 729 : cluster 0 pgmap v732: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:35.213 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:35.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:35.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:35.316 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:35.344 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:35.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:35.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:35.448 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:35.460 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:35 smithi173 bash[59422]: cluster 2023-09-23T18:42:33.944650+0000 mgr.y (mgr.26054) 729 : cluster 0 pgmap v732: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:35.479 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:35.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:35.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:35.583 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:35.612 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:35.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:35.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:35.716 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:35.746 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:35.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:35.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:35.850 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:35.882 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:35.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:35.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:35.987 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:36.021 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:36.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:36.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:36.125 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:36.153 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:36.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:36.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:36.256 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:36.283 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:36.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:36.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:36.386 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:36.421 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:36.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:36.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:36.440 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:42:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:42:36] "GET /metrics HTTP/1.1" 200 34208 "" "Prometheus/2.33.4" 2023-09-23T18:42:36.525 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:36.550 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:36.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:36.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:36.654 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:36.686 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:36.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:36.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:36.790 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:36.822 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:36.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:36.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:36.926 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:36.952 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:36.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:36.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:37.055 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:37.080 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:37.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:37.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:37.184 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:37.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:37 smithi098 bash[76817]: cluster 2023-09-23T18:42:35.945407+0000 mgr.y (mgr.26054) 730 : cluster 0 pgmap v733: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:37.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:37 smithi098 bash[79516]: cluster 2023-09-23T18:42:35.945407+0000 mgr.y (mgr.26054) 730 : cluster 0 pgmap v733: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:37.218 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:37.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:37.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:37.322 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:37.352 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:37.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:37.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:37.455 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:37.485 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:37.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:37.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:37.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:37 smithi173 bash[59422]: cluster 2023-09-23T18:42:35.945407+0000 mgr.y (mgr.26054) 730 : cluster 0 pgmap v733: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:37.588 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:37.620 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:37.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:37.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:37.725 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:37.753 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:37.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:37.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:37.857 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:37.891 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:37.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:37.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:37.995 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:38.024 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:38.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:38.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:38.127 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:38.162 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:38.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:38.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:38.266 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:38.301 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:38.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:38.309 INFO:teuthology.orchestra.run.smithi098.stderr:pool 'unique_pool_2' created 2023-09-23T18:42:38.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:38.411 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:38.433 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:38.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:38.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:38.474 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:42:38.508 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:38 smithi173 bash[59422]: audit 2023-09-23T18:42:37.797577+0000 mon.c (mon.2) 92 : audit 1 from='client.? 172.21.15.98:0/1712775240' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T18:42:38.509 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:38 smithi173 bash[59422]: audit 2023-09-23T18:42:37.797916+0000 mon.a (mon.0) 991 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T18:42:38.509 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:38 smithi173 bash[59422]: cluster 2023-09-23T18:42:37.946183+0000 mgr.y (mgr.26054) 731 : cluster 0 pgmap v734: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:38.536 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:38.562 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:38.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:38.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:38.666 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:38.691 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:38.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:38.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:38.727 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:38 smithi098 bash[79516]: audit 2023-09-23T18:42:37.797577+0000 mon.c (mon.2) 92 : audit 1 from='client.? 172.21.15.98:0/1712775240' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T18:42:38.727 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:38 smithi098 bash[79516]: audit 2023-09-23T18:42:37.797916+0000 mon.a (mon.0) 991 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T18:42:38.727 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:38 smithi098 bash[79516]: cluster 2023-09-23T18:42:37.946183+0000 mgr.y (mgr.26054) 731 : cluster 0 pgmap v734: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:38.727 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:38 smithi098 bash[76817]: audit 2023-09-23T18:42:37.797577+0000 mon.c (mon.2) 92 : audit 1 from='client.? 172.21.15.98:0/1712775240' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T18:42:38.728 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:38 smithi098 bash[76817]: audit 2023-09-23T18:42:37.797916+0000 mon.a (mon.0) 991 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T18:42:38.728 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:38 smithi098 bash[76817]: cluster 2023-09-23T18:42:37.946183+0000 mgr.y (mgr.26054) 731 : cluster 0 pgmap v734: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:38.793 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:38.820 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:38.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:38.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:38.924 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:38.946 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:38.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:38.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:39.049 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:39.072 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:39.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:39.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:39.175 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:39.200 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:39.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:39.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:39.303 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:39.329 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:39.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:39.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:39.432 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:39.455 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:39.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:39.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:39.559 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:39.572 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:39 smithi173 bash[59422]: cluster 2023-09-23T18:42:38.296452+0000 osd.4 (osd.4) 144 : cluster 3 failed to encode map e187 with expected crc 2023-09-23T18:42:39.572 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:39 smithi173 bash[59422]: cluster 2023-09-23T18:42:38.299785+0000 osd.4 (osd.4) 145 : cluster 3 failed to encode map e187 with expected crc 2023-09-23T18:42:39.572 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:39 smithi173 bash[59422]: cluster 2023-09-23T18:42:38.299816+0000 osd.6 (osd.6) 118 : cluster 3 failed to encode map e187 with expected crc 2023-09-23T18:42:39.572 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:39 smithi173 bash[59422]: audit 2023-09-23T18:42:38.300382+0000 mon.a (mon.0) 992 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T18:42:39.572 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:39 smithi173 bash[59422]: cluster 2023-09-23T18:42:38.300453+0000 mon.a (mon.0) 993 : cluster 0 osdmap e187: 8 total, 8 up, 7 in 2023-09-23T18:42:39.573 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:39 smithi173 bash[59422]: cluster 2023-09-23T18:42:38.307070+0000 osd.7 (osd.7) 102 : cluster 3 failed to encode map e187 with expected crc 2023-09-23T18:42:39.573 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:39 smithi173 bash[59422]: cluster 2023-09-23T18:42:38.345573+0000 osd.5 (osd.5) 86 : cluster 3 failed to encode map e187 with expected crc 2023-09-23T18:42:39.573 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:39 smithi173 bash[59422]: cluster 2023-09-23T18:42:39.304942+0000 mon.a (mon.0) 994 : cluster 0 osdmap e188: 8 total, 8 up, 7 in 2023-09-23T18:42:39.594 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:39.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:39.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:39.626 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:39 smithi098 bash[79516]: cluster 2023-09-23T18:42:38.296452+0000 osd.4 (osd.4) 144 : cluster 3 failed to encode map e187 with expected crc 2023-09-23T18:42:39.626 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:39 smithi098 bash[79516]: cluster 2023-09-23T18:42:38.299785+0000 osd.4 (osd.4) 145 : cluster 3 failed to encode map e187 with expected crc 2023-09-23T18:42:39.626 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:39 smithi098 bash[79516]: cluster 2023-09-23T18:42:38.299816+0000 osd.6 (osd.6) 118 : cluster 3 failed to encode map e187 with expected crc 2023-09-23T18:42:39.626 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:39 smithi098 bash[79516]: audit 2023-09-23T18:42:38.300382+0000 mon.a (mon.0) 992 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T18:42:39.626 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:39 smithi098 bash[79516]: cluster 2023-09-23T18:42:38.300453+0000 mon.a (mon.0) 993 : cluster 0 osdmap e187: 8 total, 8 up, 7 in 2023-09-23T18:42:39.627 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:39 smithi098 bash[79516]: cluster 2023-09-23T18:42:38.307070+0000 osd.7 (osd.7) 102 : cluster 3 failed to encode map e187 with expected crc 2023-09-23T18:42:39.627 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:39 smithi098 bash[79516]: cluster 2023-09-23T18:42:38.345573+0000 osd.5 (osd.5) 86 : cluster 3 failed to encode map e187 with expected crc 2023-09-23T18:42:39.627 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:39 smithi098 bash[79516]: cluster 2023-09-23T18:42:39.304942+0000 mon.a (mon.0) 994 : cluster 0 osdmap e188: 8 total, 8 up, 7 in 2023-09-23T18:42:39.627 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:39 smithi098 bash[76817]: cluster 2023-09-23T18:42:38.296452+0000 osd.4 (osd.4) 144 : cluster 3 failed to encode map e187 with expected crc 2023-09-23T18:42:39.627 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:39 smithi098 bash[76817]: cluster 2023-09-23T18:42:38.299785+0000 osd.4 (osd.4) 145 : cluster 3 failed to encode map e187 with expected crc 2023-09-23T18:42:39.627 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:39 smithi098 bash[76817]: cluster 2023-09-23T18:42:38.299816+0000 osd.6 (osd.6) 118 : cluster 3 failed to encode map e187 with expected crc 2023-09-23T18:42:39.627 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:39 smithi098 bash[76817]: audit 2023-09-23T18:42:38.300382+0000 mon.a (mon.0) 992 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T18:42:39.628 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:39 smithi098 bash[76817]: cluster 2023-09-23T18:42:38.300453+0000 mon.a (mon.0) 993 : cluster 0 osdmap e187: 8 total, 8 up, 7 in 2023-09-23T18:42:39.628 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:39 smithi098 bash[76817]: cluster 2023-09-23T18:42:38.307070+0000 osd.7 (osd.7) 102 : cluster 3 failed to encode map e187 with expected crc 2023-09-23T18:42:39.628 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:39 smithi098 bash[76817]: cluster 2023-09-23T18:42:38.345573+0000 osd.5 (osd.5) 86 : cluster 3 failed to encode map e187 with expected crc 2023-09-23T18:42:39.628 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:39 smithi098 bash[76817]: cluster 2023-09-23T18:42:39.304942+0000 mon.a (mon.0) 994 : cluster 0 osdmap e188: 8 total, 8 up, 7 in 2023-09-23T18:42:39.698 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:39.731 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:39.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:39.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:39.835 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:39.854 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:39.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:39.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:39.865 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:42:39.885 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:42:39.905 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:42:39.931 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:42:39.953 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:42:39.958 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:39.974 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:42:39.987 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:39.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:39.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:39.995 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:42:40.023 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:42:40.024 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:42:40.024 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:42:40.024 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:42:40.024 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:42:40.025 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:42:40.025 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:42:40.025 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:42:40.025 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:42:40.090 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:40.113 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:40.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:40.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:40.215 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:40.241 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:40.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:40.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:40.345 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:40.366 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:40.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:40.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:40.468 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:40.475 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:40 smithi098 bash[76817]: cluster 2023-09-23T18:42:39.305924+0000 osd.7 (osd.7) 103 : cluster 3 failed to encode map e188 with expected crc 2023-09-23T18:42:40.476 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:40 smithi098 bash[76817]: cluster 2023-09-23T18:42:39.306083+0000 osd.4 (osd.4) 146 : cluster 3 failed to encode map e188 with expected crc 2023-09-23T18:42:40.476 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:40 smithi098 bash[76817]: cluster 2023-09-23T18:42:39.306350+0000 osd.6 (osd.6) 119 : cluster 3 failed to encode map e188 with expected crc 2023-09-23T18:42:40.476 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:40 smithi098 bash[76817]: cluster 2023-09-23T18:42:39.514590+0000 osd.5 (osd.5) 87 : cluster 3 failed to encode map e188 with expected crc 2023-09-23T18:42:40.476 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:40 smithi098 bash[76817]: audit 2023-09-23T18:42:39.605910+0000 mon.a (mon.0) 995 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:42:40.476 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:40 smithi098 bash[76817]: audit 2023-09-23T18:42:39.842248+0000 mon.c (mon.2) 93 : audit 0 from='client.? 172.21.15.98:0/1298036518' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:42:40.476 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:40 smithi098 bash[76817]: cluster 2023-09-23T18:42:39.946743+0000 mgr.y (mgr.26054) 732 : cluster 0 pgmap v737: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:40.476 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:40 smithi098 bash[76817]: cluster 2023-09-23T18:42:40.300227+0000 mon.a (mon.0) 996 : cluster 0 osdmap e189: 8 total, 8 up, 7 in 2023-09-23T18:42:40.477 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:40 smithi098 bash[79516]: cluster 2023-09-23T18:42:39.305924+0000 osd.7 (osd.7) 103 : cluster 3 failed to encode map e188 with expected crc 2023-09-23T18:42:40.477 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:40 smithi098 bash[79516]: cluster 2023-09-23T18:42:39.306083+0000 osd.4 (osd.4) 146 : cluster 3 failed to encode map e188 with expected crc 2023-09-23T18:42:40.477 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:40 smithi098 bash[79516]: cluster 2023-09-23T18:42:39.306350+0000 osd.6 (osd.6) 119 : cluster 3 failed to encode map e188 with expected crc 2023-09-23T18:42:40.477 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:40 smithi098 bash[79516]: cluster 2023-09-23T18:42:39.514590+0000 osd.5 (osd.5) 87 : cluster 3 failed to encode map e188 with expected crc 2023-09-23T18:42:40.477 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:40 smithi098 bash[79516]: audit 2023-09-23T18:42:39.605910+0000 mon.a (mon.0) 995 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:42:40.477 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:40 smithi098 bash[79516]: audit 2023-09-23T18:42:39.842248+0000 mon.c (mon.2) 93 : audit 0 from='client.? 172.21.15.98:0/1298036518' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:42:40.477 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:40 smithi098 bash[79516]: cluster 2023-09-23T18:42:39.946743+0000 mgr.y (mgr.26054) 732 : cluster 0 pgmap v737: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:40.477 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:40 smithi098 bash[79516]: cluster 2023-09-23T18:42:40.300227+0000 mon.a (mon.0) 996 : cluster 0 osdmap e189: 8 total, 8 up, 7 in 2023-09-23T18:42:40.491 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:40.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:40.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:40.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:40 smithi173 bash[59422]: cluster 2023-09-23T18:42:39.305924+0000 osd.7 (osd.7) 103 : cluster 3 failed to encode map e188 with expected crc 2023-09-23T18:42:40.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:40 smithi173 bash[59422]: cluster 2023-09-23T18:42:39.306083+0000 osd.4 (osd.4) 146 : cluster 3 failed to encode map e188 with expected crc 2023-09-23T18:42:40.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:40 smithi173 bash[59422]: cluster 2023-09-23T18:42:39.306350+0000 osd.6 (osd.6) 119 : cluster 3 failed to encode map e188 with expected crc 2023-09-23T18:42:40.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:40 smithi173 bash[59422]: cluster 2023-09-23T18:42:39.514590+0000 osd.5 (osd.5) 87 : cluster 3 failed to encode map e188 with expected crc 2023-09-23T18:42:40.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:40 smithi173 bash[59422]: audit 2023-09-23T18:42:39.605910+0000 mon.a (mon.0) 995 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:42:40.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:40 smithi173 bash[59422]: audit 2023-09-23T18:42:39.842248+0000 mon.c (mon.2) 93 : audit 0 from='client.? 172.21.15.98:0/1298036518' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:42:40.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:40 smithi173 bash[59422]: cluster 2023-09-23T18:42:39.946743+0000 mgr.y (mgr.26054) 732 : cluster 0 pgmap v737: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:40.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:40 smithi173 bash[59422]: cluster 2023-09-23T18:42:40.300227+0000 mon.a (mon.0) 996 : cluster 0 osdmap e189: 8 total, 8 up, 7 in 2023-09-23T18:42:40.595 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:40.628 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:40.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:40.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:40.731 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:40.755 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:40.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:40.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:40.858 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:40.889 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:40.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:40.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:40.992 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:41.021 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:41.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:41.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:41.125 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:41.157 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:41.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:41.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:41.261 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:41.295 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:41.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:41.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:41.399 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:41.407 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:41 smithi173 bash[59422]: cluster 2023-09-23T18:42:40.459579+0000 osd.7 (osd.7) 104 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T18:42:41.407 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:41 smithi173 bash[59422]: cluster 2023-09-23T18:42:40.460603+0000 osd.7 (osd.7) 105 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T18:42:41.426 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:41.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:41.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:41.501 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:41 smithi098 bash[79516]: cluster 2023-09-23T18:42:40.459579+0000 osd.7 (osd.7) 104 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T18:42:41.502 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:41 smithi098 bash[79516]: cluster 2023-09-23T18:42:40.460603+0000 osd.7 (osd.7) 105 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T18:42:41.502 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:41 smithi098 bash[76817]: cluster 2023-09-23T18:42:40.459579+0000 osd.7 (osd.7) 104 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T18:42:41.502 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:41 smithi098 bash[76817]: cluster 2023-09-23T18:42:40.460603+0000 osd.7 (osd.7) 105 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T18:42:41.529 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:41.552 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:41.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:41.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:41.656 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:41.661 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-09-23T18:42:41.690 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:41.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:41.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:41.793 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:41.828 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:41.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:41.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:41.931 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:41.961 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:41.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:41.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:42.064 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:42.094 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:42.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:42.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:42.198 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:42.225 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:42.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:42.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:42.328 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:42.350 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:42.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:42.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:42.453 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:42.465 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:42 smithi173 bash[59422]: cluster 2023-09-23T18:42:41.756340+0000 osd.4 (osd.4) 147 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T18:42:42.465 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:42 smithi173 bash[59422]: cluster 2023-09-23T18:42:41.756425+0000 osd.5 (osd.5) 88 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T18:42:42.465 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:42 smithi173 bash[59422]: cluster 2023-09-23T18:42:41.947292+0000 mgr.y (mgr.26054) 733 : cluster 0 pgmap v739: 89 pgs: 2 creating+peering, 9 unknown, 78 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:42.489 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:42.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:42.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:42.515 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T18:42:42.516 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:42:42.592 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:42.627 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:42.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:42.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:42.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:42 smithi098 bash[79516]: cluster 2023-09-23T18:42:41.756340+0000 osd.4 (osd.4) 147 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T18:42:42.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:42 smithi098 bash[79516]: cluster 2023-09-23T18:42:41.756425+0000 osd.5 (osd.5) 88 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T18:42:42.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:42 smithi098 bash[79516]: cluster 2023-09-23T18:42:41.947292+0000 mgr.y (mgr.26054) 733 : cluster 0 pgmap v739: 89 pgs: 2 creating+peering, 9 unknown, 78 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:42.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:42 smithi098 bash[76817]: cluster 2023-09-23T18:42:41.756340+0000 osd.4 (osd.4) 147 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T18:42:42.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:42 smithi098 bash[76817]: cluster 2023-09-23T18:42:41.756425+0000 osd.5 (osd.5) 88 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T18:42:42.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:42 smithi098 bash[76817]: cluster 2023-09-23T18:42:41.947292+0000 mgr.y (mgr.26054) 733 : cluster 0 pgmap v739: 89 pgs: 2 creating+peering, 9 unknown, 78 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:42.731 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:42.758 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:42.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:42.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:42.861 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:42.877 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:42:42.897 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:42.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:42.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:43.000 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:43.032 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:43.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:43.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:43.134 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:43.162 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:43.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:43.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:43.265 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:43.294 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:43.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:43.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:43.340 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is unset 2023-09-23T18:42:43.397 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:43.444 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:43.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:43.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:43.463 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T18:42:43.548 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:43.556 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:43 smithi098 bash[79516]: cluster 2023-09-23T18:42:41.756354+0000 osd.6 (osd.6) 120 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T18:42:43.556 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:43 smithi098 bash[79516]: audit 2023-09-23T18:42:42.860977+0000 mon.a (mon.0) 997 : audit 1 from='client.? 172.21.15.98:0/3112106466' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:42:43.557 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:43 smithi098 bash[76817]: cluster 2023-09-23T18:42:41.756354+0000 osd.6 (osd.6) 120 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T18:42:43.557 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:43 smithi098 bash[76817]: audit 2023-09-23T18:42:42.860977+0000 mon.a (mon.0) 997 : audit 1 from='client.? 172.21.15.98:0/3112106466' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:42:43.569 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:43.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:43.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:43.673 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:43.686 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:43 smithi173 bash[59422]: cluster 2023-09-23T18:42:41.756354+0000 osd.6 (osd.6) 120 : cluster 3 failed to encode map e189 with expected crc 2023-09-23T18:42:43.686 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:43 smithi173 bash[59422]: audit 2023-09-23T18:42:42.860977+0000 mon.a (mon.0) 997 : audit 1 from='client.? 172.21.15.98:0/3112106466' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:42:43.704 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:43.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:43.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:43.808 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:43.842 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:43.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:43.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:43.945 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:43.970 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:43.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:43.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:44.074 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:44.100 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:44.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:44.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:44.204 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:44.236 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:44.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:44.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:44.340 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:44.363 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:44.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:44.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:44.403 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:42:44.467 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:44.494 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:44.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:44.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:44.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:44 smithi173 bash[59422]: cluster 2023-09-23T18:42:43.325531+0000 mon.a (mon.0) 998 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:42:44.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:44 smithi173 bash[59422]: cluster 2023-09-23T18:42:43.330172+0000 osd.6 (osd.6) 121 : cluster 3 failed to encode map e190 with expected crc 2023-09-23T18:42:44.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:44 smithi173 bash[59422]: audit 2023-09-23T18:42:43.331033+0000 mon.a (mon.0) 999 : audit 1 from='client.? 172.21.15.98:0/3112106466' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:42:44.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:44 smithi173 bash[59422]: cluster 2023-09-23T18:42:43.331090+0000 mon.a (mon.0) 1000 : cluster 0 osdmap e190: 8 total, 8 up, 7 in 2023-09-23T18:42:44.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:44 smithi173 bash[59422]: cluster 2023-09-23T18:42:43.457172+0000 osd.7 (osd.7) 106 : cluster 3 failed to encode map e190 with expected crc 2023-09-23T18:42:44.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:44 smithi173 bash[59422]: cluster 2023-09-23T18:42:43.457471+0000 osd.5 (osd.5) 89 : cluster 3 failed to encode map e190 with expected crc 2023-09-23T18:42:44.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:44 smithi173 bash[59422]: cluster 2023-09-23T18:42:43.457502+0000 osd.4 (osd.4) 148 : cluster 3 failed to encode map e190 with expected crc 2023-09-23T18:42:44.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:44 smithi173 bash[59422]: cluster 2023-09-23T18:42:43.947843+0000 mgr.y (mgr.26054) 734 : cluster 0 pgmap v741: 89 pgs: 2 creating+peering, 87 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:44.583 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:44 smithi098 bash[76817]: cluster 2023-09-23T18:42:43.325531+0000 mon.a (mon.0) 998 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:42:44.583 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:44 smithi098 bash[76817]: cluster 2023-09-23T18:42:43.330172+0000 osd.6 (osd.6) 121 : cluster 3 failed to encode map e190 with expected crc 2023-09-23T18:42:44.583 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:44 smithi098 bash[76817]: audit 2023-09-23T18:42:43.331033+0000 mon.a (mon.0) 999 : audit 1 from='client.? 172.21.15.98:0/3112106466' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:42:44.583 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:44 smithi098 bash[76817]: cluster 2023-09-23T18:42:43.331090+0000 mon.a (mon.0) 1000 : cluster 0 osdmap e190: 8 total, 8 up, 7 in 2023-09-23T18:42:44.583 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:44 smithi098 bash[76817]: cluster 2023-09-23T18:42:43.457172+0000 osd.7 (osd.7) 106 : cluster 3 failed to encode map e190 with expected crc 2023-09-23T18:42:44.583 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:44 smithi098 bash[76817]: cluster 2023-09-23T18:42:43.457471+0000 osd.5 (osd.5) 89 : cluster 3 failed to encode map e190 with expected crc 2023-09-23T18:42:44.584 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:44 smithi098 bash[76817]: cluster 2023-09-23T18:42:43.457502+0000 osd.4 (osd.4) 148 : cluster 3 failed to encode map e190 with expected crc 2023-09-23T18:42:44.584 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:44 smithi098 bash[76817]: cluster 2023-09-23T18:42:43.947843+0000 mgr.y (mgr.26054) 734 : cluster 0 pgmap v741: 89 pgs: 2 creating+peering, 87 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:44.584 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:44 smithi098 bash[79516]: cluster 2023-09-23T18:42:43.325531+0000 mon.a (mon.0) 998 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:42:44.584 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:44 smithi098 bash[79516]: cluster 2023-09-23T18:42:43.330172+0000 osd.6 (osd.6) 121 : cluster 3 failed to encode map e190 with expected crc 2023-09-23T18:42:44.584 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:44 smithi098 bash[79516]: audit 2023-09-23T18:42:43.331033+0000 mon.a (mon.0) 999 : audit 1 from='client.? 172.21.15.98:0/3112106466' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:42:44.584 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:44 smithi098 bash[79516]: cluster 2023-09-23T18:42:43.331090+0000 mon.a (mon.0) 1000 : cluster 0 osdmap e190: 8 total, 8 up, 7 in 2023-09-23T18:42:44.585 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:44 smithi098 bash[79516]: cluster 2023-09-23T18:42:43.457172+0000 osd.7 (osd.7) 106 : cluster 3 failed to encode map e190 with expected crc 2023-09-23T18:42:44.585 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:44 smithi098 bash[79516]: cluster 2023-09-23T18:42:43.457471+0000 osd.5 (osd.5) 89 : cluster 3 failed to encode map e190 with expected crc 2023-09-23T18:42:44.585 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:44 smithi098 bash[79516]: cluster 2023-09-23T18:42:43.457502+0000 osd.4 (osd.4) 148 : cluster 3 failed to encode map e190 with expected crc 2023-09-23T18:42:44.585 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:44 smithi098 bash[79516]: cluster 2023-09-23T18:42:43.947843+0000 mgr.y (mgr.26054) 734 : cluster 0 pgmap v741: 89 pgs: 2 creating+peering, 87 active+clean; 4.6 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:44.597 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:44.618 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:44.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:44.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:44.721 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:44.746 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:44.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:44.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:44.849 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:44.874 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:44.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:44.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:44.978 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:45.009 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:45.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:45.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:45.113 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:45.139 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:45.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:45.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:45.242 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:45.278 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:45.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:45.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:45.382 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:45.417 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:45.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:45.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:45.521 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:45.882 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:45.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:45.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:45.985 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:46.019 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:46.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:46.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:46.123 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:46.153 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:46.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:46.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:46.190 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:42:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:42:46] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.33.4" 2023-09-23T18:42:46.256 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:46.286 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:46.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:46.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:46.390 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:46.419 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:46.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:46.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:46.523 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:46.557 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:46.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:46.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:46.661 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:46.694 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:46.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:46.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:46.797 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:46.829 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:46.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:46.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:46.934 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:46.966 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:46.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:46.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:47.070 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:47.096 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:47.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:47.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:47.199 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:47.221 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:47.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:47.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:47.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:47 smithi173 bash[59422]: cluster 2023-09-23T18:42:45.948430+0000 mgr.y (mgr.26054) 735 : cluster 0 pgmap v742: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:47.324 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:47.352 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:47.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:47.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:47.404 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:42:47.415 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:47 smithi098 bash[76817]: cluster 2023-09-23T18:42:45.948430+0000 mgr.y (mgr.26054) 735 : cluster 0 pgmap v742: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:47.416 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:47 smithi098 bash[79516]: cluster 2023-09-23T18:42:45.948430+0000 mgr.y (mgr.26054) 735 : cluster 0 pgmap v742: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:47.455 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:47.486 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:47.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:47.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:47.590 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:47.621 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:47.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:47.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:47.725 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:47.755 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:47.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:47.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:47.860 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:47.888 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:47.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:47.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:47.923 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:42:47.993 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:48.029 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:48.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:48.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:48.078 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:42:48.078 INFO:teuthology.orchestra.run.smithi098.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T18:42:48.078 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:42:48.135 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:48.168 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:48.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:48.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:48.271 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:48.295 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:48.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:48.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:48.399 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:48.427 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:48.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:48.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:48.530 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:48.556 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:48.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:48.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:48.660 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:48.684 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:48.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:48.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:48.787 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:48.817 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:48.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:48.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:48.922 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:48.949 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:48.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:48.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:49.056 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:49.069 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:49 smithi173 bash[59422]: cluster 2023-09-23T18:42:47.948980+0000 mgr.y (mgr.26054) 736 : cluster 0 pgmap v743: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:49.089 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:49.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:49.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:49.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:49 smithi098 bash[79516]: cluster 2023-09-23T18:42:47.948980+0000 mgr.y (mgr.26054) 736 : cluster 0 pgmap v743: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:49.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:49 smithi098 bash[76817]: cluster 2023-09-23T18:42:47.948980+0000 mgr.y (mgr.26054) 736 : cluster 0 pgmap v743: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:49.194 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:49.223 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:49.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:49.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:49.326 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:49.360 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:49.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:49.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:49.464 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:49.491 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:49.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:49.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:49.595 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:49.617 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:49.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:49.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:49.721 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:49.748 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:49.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:49.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:49.850 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:49.878 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:49.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:49.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:49.981 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:50.004 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:50.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:50.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:50.108 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:50.134 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:50.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:50.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:50.238 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:50.270 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:50.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:50.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:50.374 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:50.408 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:50.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:50.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:50.512 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:50.543 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:50.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:50.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:50.646 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:50.678 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:50.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:50.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:50.781 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:50.812 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:50.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:50.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:50.916 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:50.950 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:50.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:50.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:50.969 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:42:50.969 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:42:51.054 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:51.066 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:51 smithi173 bash[59422]: cluster 2023-09-23T18:42:49.949624+0000 mgr.y (mgr.26054) 737 : cluster 0 pgmap v744: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:51.066 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:51 smithi173 bash[59422]: audit 2023-09-23T18:42:50.134581+0000 mon.a (mon.0) 1001 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:42:51.084 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:51.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:51.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:51.189 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:51.224 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:51.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:51.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:51.327 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:51.340 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:51 smithi098 bash[76817]: cluster 2023-09-23T18:42:49.949624+0000 mgr.y (mgr.26054) 737 : cluster 0 pgmap v744: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:51.340 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:51 smithi098 bash[76817]: audit 2023-09-23T18:42:50.134581+0000 mon.a (mon.0) 1001 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:42:51.340 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:51 smithi098 bash[79516]: cluster 2023-09-23T18:42:49.949624+0000 mgr.y (mgr.26054) 737 : cluster 0 pgmap v744: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:51.341 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:51 smithi098 bash[79516]: audit 2023-09-23T18:42:50.134581+0000 mon.a (mon.0) 1001 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:42:51.362 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:51.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:51.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:51.465 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:51.493 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:51.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:51.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:51.597 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:51.629 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:51.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:51.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:51.734 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:51.760 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:51.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:51.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:51.864 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:51.896 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:51.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:51.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:52.000 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:52.024 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:52.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:52.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:52.127 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:52.154 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:52.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:52.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:52.258 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:52.285 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:52.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:52.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:52.388 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:52.414 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:52.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:52.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:52.517 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:52.545 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:52.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:52.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:52.648 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:52.678 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:52.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:52.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:52.782 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:52.808 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:52.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:52.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:52.911 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:52.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:52.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:52.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:53.048 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:53.056 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:53 smithi098 bash[76817]: cluster 2023-09-23T18:42:51.950535+0000 mgr.y (mgr.26054) 738 : cluster 0 pgmap v745: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:53.056 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:53 smithi098 bash[79516]: cluster 2023-09-23T18:42:51.950535+0000 mgr.y (mgr.26054) 738 : cluster 0 pgmap v745: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:53.069 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:53.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:53.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:53.172 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:53.195 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:53.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:53.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:53.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:53 smithi173 bash[59422]: cluster 2023-09-23T18:42:51.950535+0000 mgr.y (mgr.26054) 738 : cluster 0 pgmap v745: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:53.299 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:53.325 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:53.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:53.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:53.429 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:53.457 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:53.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:53.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:53.544 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:42:53.560 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:53.590 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:53.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:53.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:53.694 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:53.718 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:53.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:53.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:53.821 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:53.848 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:53.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:53.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:53.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:53.988 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:53.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:53.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:54.091 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:54.121 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:54.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:54.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:54.226 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:54.252 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:54.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:54.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:54.355 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:54.377 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:54.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:54.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:54.480 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:54.512 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:54.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:54.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:54.617 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:54.648 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:54.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:54.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:54.752 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:54.785 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:54.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:54.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:54.889 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:54.916 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:54.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:54.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:55.019 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:55.052 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:55.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:55.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:55.156 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:55.189 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:55.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:55.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:55.207 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:55 smithi098 bash[79516]: cluster 2023-09-23T18:42:53.951156+0000 mgr.y (mgr.26054) 739 : cluster 0 pgmap v746: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:55.208 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:55 smithi098 bash[79516]: audit 2023-09-23T18:42:54.606391+0000 mon.a (mon.0) 1002 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:42:55.208 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:55 smithi098 bash[76817]: cluster 2023-09-23T18:42:53.951156+0000 mgr.y (mgr.26054) 739 : cluster 0 pgmap v746: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:55.208 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:55 smithi098 bash[76817]: audit 2023-09-23T18:42:54.606391+0000 mon.a (mon.0) 1002 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:42:55.293 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:55.320 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:55.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:55.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:55.423 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:55.436 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:55 smithi173 bash[59422]: cluster 2023-09-23T18:42:53.951156+0000 mgr.y (mgr.26054) 739 : cluster 0 pgmap v746: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:55.436 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:55 smithi173 bash[59422]: audit 2023-09-23T18:42:54.606391+0000 mon.a (mon.0) 1002 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:42:55.455 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:55.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:55.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:55.497 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T18:42:55.498 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:42:55.498 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-09-23T18:42:55.498 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T18:42:55.498 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T18:42:55.499 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-09-23T18:42:55.559 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:55.585 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:55.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:55.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:55.688 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:55.718 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:55.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:55.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:55.821 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:55.846 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:55.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:55.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:55.949 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:55.981 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:55.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:55.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:56.084 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:56.106 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:56.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:56.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:56.210 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:56.246 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:56.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:56.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:56.351 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:56.362 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:42:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:42:56] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.33.4" 2023-09-23T18:42:56.380 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:56.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:56.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:56.484 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:56.510 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:56.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:56.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:56.614 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:56.644 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:56.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:56.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:56.749 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:56.781 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:56.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:56.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:56.885 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:56.918 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:56.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:56.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:57.022 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:57.055 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:57.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:57.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:57.159 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:57.181 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:57.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:57.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:57.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:57 smithi173 bash[59422]: cluster 2023-09-23T18:42:55.951928+0000 mgr.y (mgr.26054) 740 : cluster 0 pgmap v747: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:57.284 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:57.291 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:57 smithi098 bash[76817]: cluster 2023-09-23T18:42:55.951928+0000 mgr.y (mgr.26054) 740 : cluster 0 pgmap v747: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:57.292 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:57 smithi098 bash[79516]: cluster 2023-09-23T18:42:55.951928+0000 mgr.y (mgr.26054) 740 : cluster 0 pgmap v747: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:57.304 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:57.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:57.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:57.407 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:57.432 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:57.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:57.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:57.535 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:57.563 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:57.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:57.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:57.666 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:57.696 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:57.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:57.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:57.800 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:57.831 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:57.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:57.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:57.935 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:57.961 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:57.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:57.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:58.063 INFO:teuthology.orchestra.run.smithi098.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-09-23T18:42:58.066 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:58.075 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:58 smithi098 bash[79516]: audit 2023-09-23T18:42:57.480867+0000 mon.c (mon.2) 94 : audit 1 from='client.? 172.21.15.98:0/143326347' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:42:58.075 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:58 smithi098 bash[79516]: audit 2023-09-23T18:42:57.481287+0000 mon.a (mon.0) 1003 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:42:58.076 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:58 smithi098 bash[76817]: audit 2023-09-23T18:42:57.480867+0000 mon.c (mon.2) 94 : audit 1 from='client.? 172.21.15.98:0/143326347' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:42:58.076 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:58 smithi098 bash[76817]: audit 2023-09-23T18:42:57.481287+0000 mon.a (mon.0) 1003 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:42:58.097 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:58.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:58.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:58.201 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:58.223 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:58.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:58.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:58.326 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:58.354 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:42:58.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:58.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:42:58.458 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:58.471 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:58 smithi173 bash[59422]: audit 2023-09-23T18:42:57.480867+0000 mon.c (mon.2) 94 : audit 1 from='client.? 172.21.15.98:0/143326347' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:42:58.471 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:58 smithi173 bash[59422]: audit 2023-09-23T18:42:57.481287+0000 mon.a (mon.0) 1003 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:42:58.488 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:58.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:58.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:58.591 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:58.618 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:58.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:58.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:58.721 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:58.750 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:58.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:58.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:58.854 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:58.885 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:42:58.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:58.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:42:58.989 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:59.023 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:42:59.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:59.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:42:59.100 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:59 smithi098 bash[76817]: cluster 2023-09-23T18:42:57.952576+0000 mgr.y (mgr.26054) 741 : cluster 0 pgmap v748: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:59.100 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:59 smithi098 bash[76817]: audit 2023-09-23T18:42:58.052459+0000 mon.a (mon.0) 1004 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:42:59.100 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:59 smithi098 bash[76817]: cluster 2023-09-23T18:42:58.052567+0000 mon.a (mon.0) 1005 : cluster 0 osdmap e191: 8 total, 8 up, 7 in 2023-09-23T18:42:59.100 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:42:59 smithi098 bash[76817]: cluster 2023-09-23T18:42:58.145527+0000 osd.7 (osd.7) 107 : cluster 3 failed to encode map e191 with expected crc 2023-09-23T18:42:59.126 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:59.157 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:42:59.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:59.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:42:59.261 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:59.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:59 smithi173 bash[59422]: cluster 2023-09-23T18:42:57.952576+0000 mgr.y (mgr.26054) 741 : cluster 0 pgmap v748: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:59.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:59 smithi173 bash[59422]: audit 2023-09-23T18:42:58.052459+0000 mon.a (mon.0) 1004 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:42:59.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:59 smithi173 bash[59422]: cluster 2023-09-23T18:42:58.052567+0000 mon.a (mon.0) 1005 : cluster 0 osdmap e191: 8 total, 8 up, 7 in 2023-09-23T18:42:59.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:42:59 smithi173 bash[59422]: cluster 2023-09-23T18:42:58.145527+0000 osd.7 (osd.7) 107 : cluster 3 failed to encode map e191 with expected crc 2023-09-23T18:42:59.286 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:59.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:59.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:42:59.306 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:59 smithi098 bash[79516]: cluster 2023-09-23T18:42:57.952576+0000 mgr.y (mgr.26054) 741 : cluster 0 pgmap v748: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:42:59.306 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:59 smithi098 bash[79516]: audit 2023-09-23T18:42:58.052459+0000 mon.a (mon.0) 1004 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:42:59.306 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:59 smithi098 bash[79516]: cluster 2023-09-23T18:42:58.052567+0000 mon.a (mon.0) 1005 : cluster 0 osdmap e191: 8 total, 8 up, 7 in 2023-09-23T18:42:59.306 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:42:59 smithi098 bash[79516]: cluster 2023-09-23T18:42:58.145527+0000 osd.7 (osd.7) 107 : cluster 3 failed to encode map e191 with expected crc 2023-09-23T18:42:59.390 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:59.420 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:42:59.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:59.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:42:59.524 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:59.555 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:59.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:59.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:59.659 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:59.689 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:42:59.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:59.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:42:59.792 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:59.829 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:42:59.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:59.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:42:59.932 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:59.962 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:42:59.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:42:59.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:00.066 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:00.090 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:00.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:00.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:00.193 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:00.223 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:00.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:00.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:00.279 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:43:00.309 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:00 smithi173 bash[59422]: cluster 2023-09-23T18:42:58.262515+0000 osd.4 (osd.4) 149 : cluster 3 failed to encode map e191 with expected crc 2023-09-23T18:43:00.310 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:00 smithi173 bash[59422]: cluster 2023-09-23T18:42:58.264392+0000 osd.4 (osd.4) 150 : cluster 3 failed to encode map e191 with expected crc 2023-09-23T18:43:00.327 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:00.355 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:00.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:00.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:00.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:00 smithi098 bash[79516]: cluster 2023-09-23T18:42:58.262515+0000 osd.4 (osd.4) 149 : cluster 3 failed to encode map e191 with expected crc 2023-09-23T18:43:00.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:00 smithi098 bash[79516]: cluster 2023-09-23T18:42:58.264392+0000 osd.4 (osd.4) 150 : cluster 3 failed to encode map e191 with expected crc 2023-09-23T18:43:00.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:00 smithi098 bash[76817]: cluster 2023-09-23T18:42:58.262515+0000 osd.4 (osd.4) 149 : cluster 3 failed to encode map e191 with expected crc 2023-09-23T18:43:00.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:00 smithi098 bash[76817]: cluster 2023-09-23T18:42:58.264392+0000 osd.4 (osd.4) 150 : cluster 3 failed to encode map e191 with expected crc 2023-09-23T18:43:00.457 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:00.490 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:00.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:00.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:00.593 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:00.623 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:00.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:00.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:00.726 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:00.756 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:00.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:00.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:00.860 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:00.885 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:00.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:00.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:00.988 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:01.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:01.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:01.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:01.120 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:01.130 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:01 smithi098 bash[79516]: cluster 2023-09-23T18:42:59.952998+0000 mgr.y (mgr.26054) 742 : cluster 0 pgmap v750: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:43:01.130 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:01 smithi098 bash[76817]: cluster 2023-09-23T18:42:59.952998+0000 mgr.y (mgr.26054) 742 : cluster 0 pgmap v750: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:43:01.145 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:01.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:01.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:01.249 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:01.270 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:01.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:01.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:01.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:01 smithi173 bash[59422]: cluster 2023-09-23T18:42:59.952998+0000 mgr.y (mgr.26054) 742 : cluster 0 pgmap v750: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:43:01.373 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:01.407 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:01.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:01.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:01.510 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:01.542 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:01.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:01.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:01.645 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:01.672 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:01.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:01.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:01.776 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:01.809 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:01.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:01.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:01.913 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:01.942 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:01.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:01.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:02.046 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:02.072 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:02.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:02.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:02.093 DEBUG:teuthology.orchestra.run.smithi098:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-09-23T18:43:02.120 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T18:43:02.120 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T18:43:02.160 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:hints = 1 2023-09-23T18:43:02.160 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T18:43:02.160 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Object prefix: benchmark_data_smithi098_155798 2023-09-23T18:43:02.161 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:43:02.161 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 0 0 0 0 0 0 - 0 2023-09-23T18:43:02.176 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:02.185 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:02 smithi098 bash[79516]: cluster 2023-09-23T18:43:01.953781+0000 mgr.y (mgr.26054) 743 : cluster 0 pgmap v751: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:43:02.186 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:02 smithi098 bash[76817]: cluster 2023-09-23T18:43:01.953781+0000 mgr.y (mgr.26054) 743 : cluster 0 pgmap v751: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:43:02.217 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:02.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:02.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:02.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:02 smithi173 bash[59422]: cluster 2023-09-23T18:43:01.953781+0000 mgr.y (mgr.26054) 743 : cluster 0 pgmap v751: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:43:02.323 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:02.351 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:02.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:02.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:02.455 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:02.481 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:02.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:02.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:02.584 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:02.615 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:02.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:02.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:02.699 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:43:02.699 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:43:02.702 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:43:02.721 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:02.764 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:43:02.772 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:02.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:02.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:02.826 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:43:02.876 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:02.893 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:43:02.900 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:02.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:02.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:02.940 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:43:02.984 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:43:03.004 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:03.026 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:03.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:03.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:03.044 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:43:03.116 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:43:03.120 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:43:03.120 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:43:03.120 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:43:03.121 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:43:03.121 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:43:03.121 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:43:03.121 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:43:03.121 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:43:03.129 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:03.139 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:03 smithi173 bash[59422]: audit 2023-09-23T18:43:02.649754+0000 mon.c (mon.2) 95 : audit 0 from='client.? 172.21.15.98:0/1098173725' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:43:03.155 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:03.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:03.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:03.161 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 1 16 2135 2119 132.43 132.438 0.00617694 0.00749653 2023-09-23T18:43:03.257 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:03.281 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:03.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:03.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:03.384 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:03.393 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:03 smithi098 bash[76817]: audit 2023-09-23T18:43:02.649754+0000 mon.c (mon.2) 95 : audit 0 from='client.? 172.21.15.98:0/1098173725' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:43:03.394 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:03 smithi098 bash[79516]: audit 2023-09-23T18:43:02.649754+0000 mon.c (mon.2) 95 : audit 0 from='client.? 172.21.15.98:0/1098173725' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:43:03.416 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:03.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:03.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:03.520 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:03.544 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:03.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:03.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:03.648 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:03.675 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:03.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:03.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:03.779 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:03.807 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:03.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:03.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:03.910 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:03.936 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:03.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:03.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:04.039 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:04.068 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:04.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:04.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:04.144 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is unset 2023-09-23T18:43:04.161 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 2 16 4778 4762 148.801 165.188 0.00672975 0.00669605 2023-09-23T18:43:04.172 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:04.197 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:04.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:04.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:04.301 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:04.330 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:04.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:04.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:04.434 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:04.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:04 smithi098 bash[79516]: audit 2023-09-23T18:43:03.377142+0000 mon.a (mon.0) 1006 : audit 1 from='client.? 172.21.15.98:0/2534450080' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:43:04.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:04 smithi098 bash[79516]: cluster 2023-09-23T18:43:03.954264+0000 mgr.y (mgr.26054) 744 : cluster 0 pgmap v752: 89 pgs: 89 active+clean; 37 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 3.3 MiB/s wr, 52 op/s 2023-09-23T18:43:04.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:04 smithi098 bash[76817]: audit 2023-09-23T18:43:03.377142+0000 mon.a (mon.0) 1006 : audit 1 from='client.? 172.21.15.98:0/2534450080' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:43:04.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:04 smithi098 bash[76817]: cluster 2023-09-23T18:43:03.954264+0000 mgr.y (mgr.26054) 744 : cluster 0 pgmap v752: 89 pgs: 89 active+clean; 37 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 3.3 MiB/s wr, 52 op/s 2023-09-23T18:43:04.445 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:04 smithi173 bash[59422]: audit 2023-09-23T18:43:03.377142+0000 mon.a (mon.0) 1006 : audit 1 from='client.? 172.21.15.98:0/2534450080' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:43:04.445 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:04 smithi173 bash[59422]: cluster 2023-09-23T18:43:03.954264+0000 mgr.y (mgr.26054) 744 : cluster 0 pgmap v752: 89 pgs: 89 active+clean; 37 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 3.3 MiB/s wr, 52 op/s 2023-09-23T18:43:04.460 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:04.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:04.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:04.563 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:04.586 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:04.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:04.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:04.689 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:04.717 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:04.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:04.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:04.820 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:04.847 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:04.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:04.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:04.949 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:04.979 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:04.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:04.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:04.999 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:43:04.999 INFO:teuthology.orchestra.run.smithi173.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T18:43:04.999 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:43:05.084 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:05.112 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:05.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:05.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:05.161 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 3 16 7215 7199 149.967 152.312 0.00836373 0.00665483 2023-09-23T18:43:05.216 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:05.242 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:05.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:05.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:05.347 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:05.357 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:05 smithi098 bash[79516]: cluster 2023-09-23T18:43:04.111766+0000 mon.a (mon.0) 1007 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:43:05.357 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:05 smithi098 bash[79516]: cluster 2023-09-23T18:43:04.111798+0000 mon.a (mon.0) 1008 : cluster 1 Cluster is now healthy 2023-09-23T18:43:05.357 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:05 smithi098 bash[79516]: cluster 2023-09-23T18:43:04.134634+0000 osd.7 (osd.7) 108 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T18:43:05.357 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:05 smithi098 bash[79516]: cluster 2023-09-23T18:43:04.135083+0000 osd.6 (osd.6) 122 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T18:43:05.357 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:05 smithi098 bash[79516]: audit 2023-09-23T18:43:04.136782+0000 mon.a (mon.0) 1009 : audit 1 from='client.? 172.21.15.98:0/2534450080' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:43:05.357 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:05 smithi098 bash[79516]: cluster 2023-09-23T18:43:04.136858+0000 mon.a (mon.0) 1010 : cluster 0 osdmap e192: 8 total, 8 up, 7 in 2023-09-23T18:43:05.358 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:05 smithi098 bash[79516]: audit 2023-09-23T18:43:04.766645+0000 mon.a (mon.0) 1011 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:43:05.358 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:05 smithi098 bash[76817]: cluster 2023-09-23T18:43:04.111766+0000 mon.a (mon.0) 1007 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:43:05.358 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:05 smithi098 bash[76817]: cluster 2023-09-23T18:43:04.111798+0000 mon.a (mon.0) 1008 : cluster 1 Cluster is now healthy 2023-09-23T18:43:05.358 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:05 smithi098 bash[76817]: cluster 2023-09-23T18:43:04.134634+0000 osd.7 (osd.7) 108 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T18:43:05.358 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:05 smithi098 bash[76817]: cluster 2023-09-23T18:43:04.135083+0000 osd.6 (osd.6) 122 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T18:43:05.358 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:05 smithi098 bash[76817]: audit 2023-09-23T18:43:04.136782+0000 mon.a (mon.0) 1009 : audit 1 from='client.? 172.21.15.98:0/2534450080' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:43:05.358 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:05 smithi098 bash[76817]: cluster 2023-09-23T18:43:04.136858+0000 mon.a (mon.0) 1010 : cluster 0 osdmap e192: 8 total, 8 up, 7 in 2023-09-23T18:43:05.358 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:05 smithi098 bash[76817]: audit 2023-09-23T18:43:04.766645+0000 mon.a (mon.0) 1011 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:43:05.376 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:05.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:05.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:05.479 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:05.490 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:05 smithi173 bash[59422]: cluster 2023-09-23T18:43:04.111766+0000 mon.a (mon.0) 1007 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:43:05.490 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:05 smithi173 bash[59422]: cluster 2023-09-23T18:43:04.111798+0000 mon.a (mon.0) 1008 : cluster 1 Cluster is now healthy 2023-09-23T18:43:05.490 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:05 smithi173 bash[59422]: cluster 2023-09-23T18:43:04.134634+0000 osd.7 (osd.7) 108 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T18:43:05.490 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:05 smithi173 bash[59422]: cluster 2023-09-23T18:43:04.135083+0000 osd.6 (osd.6) 122 : cluster 3 failed to encode map e192 with expected crc 2023-09-23T18:43:05.491 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:05 smithi173 bash[59422]: audit 2023-09-23T18:43:04.136782+0000 mon.a (mon.0) 1009 : audit 1 from='client.? 172.21.15.98:0/2534450080' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:43:05.491 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:05 smithi173 bash[59422]: cluster 2023-09-23T18:43:04.136858+0000 mon.a (mon.0) 1010 : cluster 0 osdmap e192: 8 total, 8 up, 7 in 2023-09-23T18:43:05.491 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:05 smithi173 bash[59422]: audit 2023-09-23T18:43:04.766645+0000 mon.a (mon.0) 1011 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:43:05.506 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:05.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:05.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:05.610 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:05.641 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:05.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:05.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:05.744 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:05.762 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:05.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:05.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:05.865 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:05.893 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:05.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:05.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:05.989 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T18:43:05.998 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:06.022 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:06.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:06.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:06.125 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:06.128 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:06 smithi098 bash[76817]: cluster 2023-09-23T18:43:05.954781+0000 mgr.y (mgr.26054) 745 : cluster 0 pgmap v754: 89 pgs: 89 active+clean; 214 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 26 MiB/s wr, 418 op/s 2023-09-23T18:43:06.128 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:43:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:43:06] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.33.4" 2023-09-23T18:43:06.150 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:06.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:06.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:06.161 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 4 16 9437 9421 147.191 138.875 0.00503541 0.00678395 2023-09-23T18:43:06.254 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:06.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:06 smithi173 bash[59422]: cluster 2023-09-23T18:43:05.954781+0000 mgr.y (mgr.26054) 745 : cluster 0 pgmap v754: 89 pgs: 89 active+clean; 214 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 26 MiB/s wr, 418 op/s 2023-09-23T18:43:06.281 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:06.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:06.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:06.384 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:06.413 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:06.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:06.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:06.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:06 smithi098 bash[79516]: cluster 2023-09-23T18:43:05.954781+0000 mgr.y (mgr.26054) 745 : cluster 0 pgmap v754: 89 pgs: 89 active+clean; 214 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 26 MiB/s wr, 418 op/s 2023-09-23T18:43:06.516 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:06.544 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:06.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:06.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:06.648 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:06.672 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:06.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:06.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:06.774 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:06.803 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:06.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:06.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:06.907 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:06.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:06.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:06.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:07.049 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:07.077 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:07.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:07.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:07.161 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 5 16 11447 11431 142.875 125.625 0.00713611 0.00699132 2023-09-23T18:43:07.182 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:07.210 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:07.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:07.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:07.313 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:07.344 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:07.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:07.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:07.448 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:07.474 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:07.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:07.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:07.578 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:07.603 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:07.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:07.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:07.706 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:07.734 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:07.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:07.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:07.838 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:07.864 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:07.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:07.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:07.969 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:07.996 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:07.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:07.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:08.099 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:08.129 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:08.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:08.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:08.161 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 6 16 13595 13579 141.435 134.25 0.0036597 0.00706391 2023-09-23T18:43:08.233 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:08.258 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:08.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:08.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:08.362 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:08.388 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:08.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:08.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:08.491 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:08.519 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:08.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:08.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:08.623 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:08.650 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:08.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:08.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:08.754 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:08.781 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:08.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:08.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:08.886 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:08.914 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:08.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:08.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:09.017 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:09.045 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:09.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:09.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:09.148 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:09.158 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:09 smithi098 bash[79516]: cluster 2023-09-23T18:43:07.955302+0000 mgr.y (mgr.26054) 746 : cluster 0 pgmap v755: 89 pgs: 89 active+clean; 448 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 45 MiB/s wr, 716 op/s 2023-09-23T18:43:09.159 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:09 smithi098 bash[76817]: cluster 2023-09-23T18:43:07.955302+0000 mgr.y (mgr.26054) 746 : cluster 0 pgmap v755: 89 pgs: 89 active+clean; 448 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 45 MiB/s wr, 716 op/s 2023-09-23T18:43:09.162 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 7 16 15655 15639 139.62 128.75 0.0142694 0.00715351 2023-09-23T18:43:09.176 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:09.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:09.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:09.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:09 smithi173 bash[59422]: cluster 2023-09-23T18:43:07.955302+0000 mgr.y (mgr.26054) 746 : cluster 0 pgmap v755: 89 pgs: 89 active+clean; 448 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 45 MiB/s wr, 716 op/s 2023-09-23T18:43:09.280 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:09.309 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:09.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:09.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:09.412 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:09.440 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:09.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:09.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:09.542 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:09.571 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:09.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:09.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:09.676 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:09.702 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:09.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:09.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:09.806 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:09.839 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:09.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:09.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:09.948 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:09.980 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:09.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:09.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:10.086 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:10.102 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:43:10.115 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:10.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:10.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:10.121 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:43:10.162 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 8 15 17588 17573 137.275 120.875 0.00639043 0.00727641 2023-09-23T18:43:10.220 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:10.267 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:10.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:10.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:10.371 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:10.399 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:10.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:10.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:10.502 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:10.529 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:10.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:10.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:10.633 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:10.657 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:10.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:10.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:10.766 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:10.775 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:10 smithi098 bash[76817]: audit 2023-09-23T18:43:09.617463+0000 mon.a (mon.0) 1012 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:43:10.775 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:10 smithi098 bash[76817]: audit 2023-09-23T18:43:09.619162+0000 mon.a (mon.0) 1013 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:43:10.775 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:10 smithi098 bash[76817]: cluster 2023-09-23T18:43:09.955895+0000 mgr.y (mgr.26054) 747 : cluster 0 pgmap v756: 89 pgs: 89 active+clean; 715 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T18:43:10.776 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:10 smithi098 bash[79516]: audit 2023-09-23T18:43:09.617463+0000 mon.a (mon.0) 1012 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:43:10.776 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:10 smithi098 bash[79516]: audit 2023-09-23T18:43:09.619162+0000 mon.a (mon.0) 1013 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:43:10.776 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:10 smithi098 bash[79516]: cluster 2023-09-23T18:43:09.955895+0000 mgr.y (mgr.26054) 747 : cluster 0 pgmap v756: 89 pgs: 89 active+clean; 715 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T18:43:10.788 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:43:10.801 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:10.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:10.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:10.906 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:10.939 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:10.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:10.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:11.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:10 smithi173 bash[59422]: audit 2023-09-23T18:43:09.617463+0000 mon.a (mon.0) 1012 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:43:11.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:10 smithi173 bash[59422]: audit 2023-09-23T18:43:09.619162+0000 mon.a (mon.0) 1013 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:43:11.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:10 smithi173 bash[59422]: cluster 2023-09-23T18:43:09.955895+0000 mgr.y (mgr.26054) 747 : cluster 0 pgmap v756: 89 pgs: 89 active+clean; 715 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T18:43:11.060 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:11.090 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:11.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:11.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:11.163 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 9 16 19206 19190 133.25 101.062 0.00870827 0.00749542 2023-09-23T18:43:11.195 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:11.236 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:11.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:11.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:11.340 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:11.359 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T18:43:11.359 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:43:11.360 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_2 2023-09-23T18:43:11.360 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:43:11.378 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:11.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:11.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:11.481 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:11.518 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:11.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:11.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:11.623 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:11.659 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:11.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:11.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:11.772 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:11.803 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:11.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:11.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:11.909 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:11.939 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:11.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:11.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:12.044 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:12.089 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:12.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:12.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:12.163 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 10 16 20592 20576 128.587 86.625 0.0118172 0.00776781 2023-09-23T18:43:12.192 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:12.228 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:12.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:12.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:12.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:12 smithi173 bash[59422]: audit 2023-09-23T18:43:11.173583+0000 mon.a (mon.0) 1014 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:43:12.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:12 smithi173 bash[59422]: audit 2023-09-23T18:43:11.185801+0000 mon.a (mon.0) 1015 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:43:12.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:12 smithi173 bash[59422]: cluster 2023-09-23T18:43:11.956545+0000 mgr.y (mgr.26054) 748 : cluster 0 pgmap v757: 89 pgs: 89 active+clean; 1022 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-09-23T18:43:12.331 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:12.367 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:12.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:12.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:12.451 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:12 smithi098 bash[76817]: audit 2023-09-23T18:43:11.173583+0000 mon.a (mon.0) 1014 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:43:12.452 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:12 smithi098 bash[76817]: audit 2023-09-23T18:43:11.185801+0000 mon.a (mon.0) 1015 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:43:12.452 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:12 smithi098 bash[76817]: cluster 2023-09-23T18:43:11.956545+0000 mgr.y (mgr.26054) 748 : cluster 0 pgmap v757: 89 pgs: 89 active+clean; 1022 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-09-23T18:43:12.452 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:12 smithi098 bash[79516]: audit 2023-09-23T18:43:11.173583+0000 mon.a (mon.0) 1014 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:43:12.452 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:12 smithi098 bash[79516]: audit 2023-09-23T18:43:11.185801+0000 mon.a (mon.0) 1015 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:43:12.452 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:12 smithi098 bash[79516]: cluster 2023-09-23T18:43:11.956545+0000 mgr.y (mgr.26054) 748 : cluster 0 pgmap v757: 89 pgs: 89 active+clean; 1022 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-09-23T18:43:12.473 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:12.506 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:12.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:12.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:12.609 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:12.639 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:12.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:12.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:12.743 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:12.767 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:12.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:12.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:12.870 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:12.904 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:12.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:12.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:13.010 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:13.042 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:13.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:13.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:13.144 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:13.162 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 11 16 22043 22027 125.14 90.6875 0.0122195 0.00796581 2023-09-23T18:43:13.166 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:13.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:13.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:13.270 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:13.278 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:13 smithi098 bash[79516]: audit 2023-09-23T18:43:12.246875+0000 mon.a (mon.0) 1016 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:43:13.279 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:13 smithi098 bash[79516]: audit 2023-09-23T18:43:12.254086+0000 mon.a (mon.0) 1017 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:43:13.279 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:13 smithi098 bash[79516]: audit 2023-09-23T18:43:12.895189+0000 mon.a (mon.0) 1018 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:43:13.279 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:13 smithi098 bash[79516]: audit 2023-09-23T18:43:12.896610+0000 mon.a (mon.0) 1019 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:43:13.279 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:13 smithi098 bash[79516]: audit 2023-09-23T18:43:12.905858+0000 mon.a (mon.0) 1020 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:43:13.279 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:13 smithi098 bash[76817]: audit 2023-09-23T18:43:12.246875+0000 mon.a (mon.0) 1016 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:43:13.280 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:13 smithi098 bash[76817]: audit 2023-09-23T18:43:12.254086+0000 mon.a (mon.0) 1017 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:43:13.280 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:13 smithi098 bash[76817]: audit 2023-09-23T18:43:12.895189+0000 mon.a (mon.0) 1018 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:43:13.280 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:13 smithi098 bash[76817]: audit 2023-09-23T18:43:12.896610+0000 mon.a (mon.0) 1019 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:43:13.280 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:13 smithi098 bash[76817]: audit 2023-09-23T18:43:12.905858+0000 mon.a (mon.0) 1020 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:43:13.295 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:13.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:13.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:13.400 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:13.424 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:13.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:13.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:13.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:13 smithi173 bash[59422]: audit 2023-09-23T18:43:12.246875+0000 mon.a (mon.0) 1016 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:43:13.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:13 smithi173 bash[59422]: audit 2023-09-23T18:43:12.254086+0000 mon.a (mon.0) 1017 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:43:13.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:13 smithi173 bash[59422]: audit 2023-09-23T18:43:12.895189+0000 mon.a (mon.0) 1018 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:43:13.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:13 smithi173 bash[59422]: audit 2023-09-23T18:43:12.896610+0000 mon.a (mon.0) 1019 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:43:13.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:13 smithi173 bash[59422]: audit 2023-09-23T18:43:12.905858+0000 mon.a (mon.0) 1020 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:43:13.528 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:13.554 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:13.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:13.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:13.657 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:13.683 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:13.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:13.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:13.788 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:13.816 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:13.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:13.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:13.920 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:13.947 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:13.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:13.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:14.052 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:14.081 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:14.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:14.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:14.162 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 12 16 23855 23839 124.149 113.25 0.00568407 0.00804758 2023-09-23T18:43:14.184 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:14.220 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:14.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:14.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:14.325 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:14.359 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:14.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:14.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:14.463 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:14.472 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:14 smithi098 bash[79516]: cluster 2023-09-23T18:43:13.957144+0000 mgr.y (mgr.26054) 749 : cluster 0 pgmap v758: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-09-23T18:43:14.472 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:14 smithi098 bash[76817]: cluster 2023-09-23T18:43:13.957144+0000 mgr.y (mgr.26054) 749 : cluster 0 pgmap v758: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-09-23T18:43:14.495 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:14.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:14.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:14.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:14 smithi173 bash[59422]: cluster 2023-09-23T18:43:13.957144+0000 mgr.y (mgr.26054) 749 : cluster 0 pgmap v758: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-09-23T18:43:14.599 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:14.626 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:14.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:14.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:14.729 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:14.760 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:14.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:14.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:14.860 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:43:14.863 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:14.866 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:43:14.885 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:14.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:14.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:14.988 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:15.017 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:15.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:15.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:15.120 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:15.143 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:15.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:15.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:15.162 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 13 16 25336 25320 121.718 92.5625 0.0022165 0.00817728 2023-09-23T18:43:15.246 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:15.272 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:15.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:15.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:15.375 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:15.402 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:15.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:15.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:15.506 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:15.539 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:15.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:15.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:15.644 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:15.671 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:15.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:15.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:15.775 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:15.801 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:15.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:15.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:15.907 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:15.933 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:15.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:15.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:16.038 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:16.065 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:16.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:16.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:16.162 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 14 16 26998 26982 120.443 103.875 0.00573763 0.00829619 2023-09-23T18:43:16.169 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:16.196 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:16.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:16.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:16.299 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:16.322 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:16.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:16.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:16.427 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:16.445 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:16.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:16.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:16.502 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:43:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:43:16] "GET /metrics HTTP/1.1" 200 36544 "" "Prometheus/2.33.4" 2023-09-23T18:43:16.548 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:16.574 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:16.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:16.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:16.680 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:16.710 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:16.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:16.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:16.814 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:16.841 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:16.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:16.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:16.944 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:16.974 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:16.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:16.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:17.078 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:17.103 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:17.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:17.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:17.162 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 15 16 28584 28568 119.021 99.125 0.00526442 0.00839419 2023-09-23T18:43:17.207 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:17.217 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:17 smithi173 bash[59422]: cluster 2023-09-23T18:43:15.957655+0000 mgr.y (mgr.26054) 750 : cluster 0 pgmap v759: 89 pgs: 89 active+clean; 1.4 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-09-23T18:43:17.237 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:17.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:17.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:17.340 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:17.366 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:17.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:17.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:17.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:17 smithi098 bash[79516]: cluster 2023-09-23T18:43:15.957655+0000 mgr.y (mgr.26054) 750 : cluster 0 pgmap v759: 89 pgs: 89 active+clean; 1.4 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-09-23T18:43:17.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:17 smithi098 bash[76817]: cluster 2023-09-23T18:43:15.957655+0000 mgr.y (mgr.26054) 750 : cluster 0 pgmap v759: 89 pgs: 89 active+clean; 1.4 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-09-23T18:43:17.469 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:17.496 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:17.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:17.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:17.600 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:17.631 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:17.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:17.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:17.734 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:17.761 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:17.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:17.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:17.864 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:17.890 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:17.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:17.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:17.993 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:18.019 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:18.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:18.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:18.123 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:18.147 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:18.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:18.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:18.162 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 16 16 30217 30201 117.961 102.062 0.0196712 0.00847106 2023-09-23T18:43:18.251 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:18.278 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:18.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:18.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:18.381 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:18.411 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:18.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:18.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:18.515 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:18.544 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:18.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:18.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:18.648 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:18.674 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:18.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:18.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:18.778 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:18.805 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:18.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:18.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:18.909 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:18.920 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:43:18.932 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:18.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:18.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:19.036 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:19.076 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:19.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:19.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:19.163 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 17 16 31701 31685 116.477 92.75 0.0137818 0.00857827 2023-09-23T18:43:19.182 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:19.193 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:19 smithi173 bash[59422]: cluster 2023-09-23T18:43:17.958250+0000 mgr.y (mgr.26054) 751 : cluster 0 pgmap v760: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-09-23T18:43:19.213 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:19.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:19.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:19.316 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:19.341 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:19.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:19.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:19.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:19 smithi098 bash[76817]: cluster 2023-09-23T18:43:17.958250+0000 mgr.y (mgr.26054) 751 : cluster 0 pgmap v760: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-09-23T18:43:19.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:19 smithi098 bash[79516]: cluster 2023-09-23T18:43:17.958250+0000 mgr.y (mgr.26054) 751 : cluster 0 pgmap v760: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-09-23T18:43:19.446 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:19.476 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:19.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:19.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:19.580 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:19.606 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:19.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:19.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:19.709 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:19.736 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:19.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:19.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:19.839 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:19.862 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:19.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:19.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:19.965 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:19.995 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:19.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:19.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:20.099 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:20.127 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:20.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:20.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:20.163 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 18 16 33059 33043 114.721 84.875 0.00864768 0.00871005 2023-09-23T18:43:20.231 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:20.260 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:20.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:20.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:20.363 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:20.390 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:20.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:20.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:20.494 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:20.534 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:20.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:20.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:20.637 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:20.664 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:20.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:20.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:20.770 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:20.805 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:20.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:20.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:20.908 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:20.937 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:20.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:20.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:21.044 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:21.067 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:21.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:21.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:21.163 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 19 16 34411 34395 113.13 84.5 0.0107674 0.00883257 2023-09-23T18:43:21.171 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:21.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:21 smithi098 bash[79516]: cluster 2023-09-23T18:43:19.958764+0000 mgr.y (mgr.26054) 752 : cluster 0 pgmap v761: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T18:43:21.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:21 smithi098 bash[76817]: cluster 2023-09-23T18:43:19.958764+0000 mgr.y (mgr.26054) 752 : cluster 0 pgmap v761: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T18:43:21.198 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:21.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:21.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:21.301 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:21.309 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:21 smithi173 bash[59422]: cluster 2023-09-23T18:43:19.958764+0000 mgr.y (mgr.26054) 752 : cluster 0 pgmap v761: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T18:43:21.328 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:21.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:21.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:21.431 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:21.457 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:21.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:21.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:21.561 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:21.591 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:21.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:21.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:21.694 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:21.726 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:21.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:21.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:21.829 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:21.861 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:21.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:21.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:21.963 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:21.989 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:21.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:21.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:22.092 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:22.127 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:22.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:22.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:22.163 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:2023-09-23T18:43:22.162570+0000 min lat: 0.00125843 max lat: 0.155252 avg lat: 0.00894342 2023-09-23T18:43:22.163 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:43:22.163 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 20 16 35773 35757 111.73 85.125 0.00791576 0.00894342 2023-09-23T18:43:22.232 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:22.260 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:22.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:22.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:22.363 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:22.394 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:22.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:22.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:22.497 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:22.531 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:22.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:22.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:22.636 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:22.682 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:22.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:22.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:22.788 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:22.809 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:22.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:22.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:22.913 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:22.940 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:22.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:22.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:23.044 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:23.052 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:23 smithi098 bash[76817]: cluster 2023-09-23T18:43:21.959473+0000 mgr.y (mgr.26054) 753 : cluster 0 pgmap v762: 89 pgs: 89 active+clean; 2.0 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 108 MiB/s wr, 1.74k op/s 2023-09-23T18:43:23.054 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:23 smithi098 bash[79516]: cluster 2023-09-23T18:43:21.959473+0000 mgr.y (mgr.26054) 753 : cluster 0 pgmap v762: 89 pgs: 89 active+clean; 2.0 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 108 MiB/s wr, 1.74k op/s 2023-09-23T18:43:23.069 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:23.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:23.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:23.163 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 21 16 37184 37168 110.608 88.1875 0.0151034 0.00903398 2023-09-23T18:43:23.173 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:23.208 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:23.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:23.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:23.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:23 smithi173 bash[59422]: cluster 2023-09-23T18:43:21.959473+0000 mgr.y (mgr.26054) 753 : cluster 0 pgmap v762: 89 pgs: 89 active+clean; 2.0 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 108 MiB/s wr, 1.74k op/s 2023-09-23T18:43:23.313 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:23.340 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:23.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:23.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:23.443 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:23.469 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:23.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:23.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:23.574 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:23.599 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:23.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:23.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:23.701 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:23.733 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:23.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:23.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:23.837 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:23.867 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:23.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:23.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:23.971 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:24.010 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:24.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:24.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:24.112 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:24.138 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:24.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:24.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:24.163 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 22 16 38533 38517 109.413 84.3125 0.00607322 0.0091331 2023-09-23T18:43:24.243 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:24.271 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:24.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:24.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:24.377 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:24.405 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:24.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:24.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:24.511 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:24.541 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:24.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:24.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:24.644 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:24.673 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:24.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:24.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:24.776 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:24.802 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:24.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:24.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:24.906 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:24.943 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:24.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:24.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:25.048 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:25.076 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:25.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:25.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:25.163 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 23 16 39969 39953 108.557 89.75 0.00868128 0.00920603 2023-09-23T18:43:25.181 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:25.189 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:25 smithi098 bash[79516]: cluster 2023-09-23T18:43:23.960354+0000 mgr.y (mgr.26054) 754 : cluster 0 pgmap v763: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-09-23T18:43:25.189 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:25 smithi098 bash[79516]: audit 2023-09-23T18:43:24.606436+0000 mon.a (mon.0) 1021 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:43:25.189 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:25 smithi098 bash[76817]: cluster 2023-09-23T18:43:23.960354+0000 mgr.y (mgr.26054) 754 : cluster 0 pgmap v763: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-09-23T18:43:25.189 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:25 smithi098 bash[76817]: audit 2023-09-23T18:43:24.606436+0000 mon.a (mon.0) 1021 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:43:25.216 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:25.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:25.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:25.319 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:25.346 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:25.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:25.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:25.449 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:25.460 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:25 smithi173 bash[59422]: cluster 2023-09-23T18:43:23.960354+0000 mgr.y (mgr.26054) 754 : cluster 0 pgmap v763: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-09-23T18:43:25.460 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:25 smithi173 bash[59422]: audit 2023-09-23T18:43:24.606436+0000 mon.a (mon.0) 1021 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:43:25.476 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:25.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:25.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:25.580 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:25.612 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:25.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:25.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:25.718 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:25.746 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:25.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:25.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:25.850 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:25.874 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:25.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:25.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:25.977 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:26.005 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:26.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:26.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:26.028 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:43:26.028 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [], 2023-09-23T18:43:26.028 INFO:teuthology.orchestra.run.smithi098.stdout: "complaint_time": 30, 2023-09-23T18:43:26.028 INFO:teuthology.orchestra.run.smithi098.stdout: "num_blocked_ops": 0 2023-09-23T18:43:26.028 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:43:26.108 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:26.134 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:26.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:26.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:26.163 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 24 16 41177 41161 107.18 75.5 0.00891277 0.00932364 2023-09-23T18:43:26.238 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:26.250 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:43:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:43:26] "GET /metrics HTTP/1.1" 200 36551 "" "Prometheus/2.33.4" 2023-09-23T18:43:26.265 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:26.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:26.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:26.368 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:26.391 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:26.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:26.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:26.495 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:26.530 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:26.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:26.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:26.636 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:26.664 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:26.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:26.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:26.769 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:26.800 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:26.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:26.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:26.906 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:26.932 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:26.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:26.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:27.039 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:27.066 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:27.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:27.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:27.163 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 25 16 42538 42522 106.295 85.0625 0.00518393 0.00940016 2023-09-23T18:43:27.170 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:27.178 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:27 smithi098 bash[76817]: cluster 2023-09-23T18:43:25.960900+0000 mgr.y (mgr.26054) 755 : cluster 0 pgmap v764: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-09-23T18:43:27.178 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:27 smithi098 bash[79516]: cluster 2023-09-23T18:43:25.960900+0000 mgr.y (mgr.26054) 755 : cluster 0 pgmap v764: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-09-23T18:43:27.193 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:27.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:27.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:27.296 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:27.317 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:27.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:27.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:27.420 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:27.430 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:27 smithi173 bash[59422]: cluster 2023-09-23T18:43:25.960900+0000 mgr.y (mgr.26054) 755 : cluster 0 pgmap v764: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-09-23T18:43:27.449 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:27.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:27.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:27.552 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:27.577 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:27.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:27.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:27.681 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:27.706 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:27.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:27.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:27.809 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:27.831 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:27.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:27.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:27.935 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:27.966 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:27.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:27.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:28.069 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:28.101 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:28.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:28.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:28.163 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 26 16 43915 43899 105.516 86.0625 0.0134435 0.00947135 2023-09-23T18:43:28.207 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:28.233 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:28.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:28.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:28.336 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:28.366 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:28.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:28.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:28.471 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:28.496 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:28.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:28.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:28.599 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:28.631 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:28.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:28.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:28.735 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:28.759 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:28.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:28.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:28.863 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:28.895 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:28.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:28.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:28.998 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:29.026 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:29.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:29.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:29.130 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:29.155 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:29.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:29.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:29.163 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 27 16 45222 45206 104.633 81.6875 0.00843733 0.00955033 2023-09-23T18:43:29.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:29 smithi098 bash[79516]: cluster 2023-09-23T18:43:27.961496+0000 mgr.y (mgr.26054) 756 : cluster 0 pgmap v765: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-09-23T18:43:29.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:29 smithi098 bash[76817]: cluster 2023-09-23T18:43:27.961496+0000 mgr.y (mgr.26054) 756 : cluster 0 pgmap v765: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-09-23T18:43:29.258 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:29.268 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:29 smithi173 bash[59422]: cluster 2023-09-23T18:43:27.961496+0000 mgr.y (mgr.26054) 756 : cluster 0 pgmap v765: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-09-23T18:43:29.282 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:29.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:29.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:29.385 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:29.409 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:29.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:29.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:29.512 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:29.542 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:29.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:29.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:29.647 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:29.673 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:29.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:29.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:29.776 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:29.801 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:29.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:29.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:29.905 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:29.932 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:29.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:29.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:30.035 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:30.068 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:30.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:30.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:30.164 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 28 16 46480 46464 103.704 78.625 0.00866905 0.00963556 2023-09-23T18:43:30.172 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:30.200 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:30.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:30.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:30.303 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:30.331 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:30.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:30.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:30.435 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:30.462 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:30.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:30.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:30.564 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:30.597 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:30.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:30.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:30.702 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:30.736 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:30.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:30.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:30.841 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:30.874 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:30.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:30.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:30.950 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T18:43:30.979 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:31.028 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:31.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:31.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:31.133 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:31.165 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 29 16 47687 47671 102.724 75.4375 0.0110682 0.00972656 2023-09-23T18:43:31.166 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:31.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:31.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:31.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:31 smithi098 bash[79516]: cluster 2023-09-23T18:43:29.962029+0000 mgr.y (mgr.26054) 757 : cluster 0 pgmap v766: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T18:43:31.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:31 smithi098 bash[76817]: cluster 2023-09-23T18:43:29.962029+0000 mgr.y (mgr.26054) 757 : cluster 0 pgmap v766: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T18:43:31.270 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:31.297 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:31.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:31.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:31.401 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:31.427 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:31.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:31.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:31.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:31 smithi173 bash[59422]: cluster 2023-09-23T18:43:29.962029+0000 mgr.y (mgr.26054) 757 : cluster 0 pgmap v766: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T18:43:31.532 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:31.559 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:31.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:31.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:31.663 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:31.690 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:31.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:31.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:31.793 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:31.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:31.829 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:31.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:31.930 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:31.953 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:31.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:31.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:32.056 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:32.081 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:32.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:32.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:32.165 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 30 16 48990 48974 102.015 81.4375 0.010972 0.00979694 2023-09-23T18:43:32.189 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:32.227 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:32.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:32.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:32.332 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:32.357 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:32.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:32.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:32.461 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:32.492 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:32.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:32.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:32.597 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:32.657 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:32.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:32.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:32.761 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:32.797 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:32.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:32.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:32.901 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:32.927 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:32.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:32.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:33.031 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:33.061 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:33.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:33.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:33.165 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:33.166 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 31 16 50110 50094 100.982 70 0.026062 0.00989661 2023-09-23T18:43:33.191 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:33.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:33.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:33.296 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:33.323 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:33.324 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:33 smithi173 bash[59422]: cluster 2023-09-23T18:43:31.966349+0000 mgr.y (mgr.26054) 758 : cluster 0 pgmap v767: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-09-23T18:43:33.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:33.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:33.426 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:33.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:33 smithi098 bash[76817]: cluster 2023-09-23T18:43:31.966349+0000 mgr.y (mgr.26054) 758 : cluster 0 pgmap v767: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-09-23T18:43:33.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:33 smithi098 bash[79516]: cluster 2023-09-23T18:43:31.966349+0000 mgr.y (mgr.26054) 758 : cluster 0 pgmap v767: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-09-23T18:43:33.453 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:33.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:33.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:33.557 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:33.593 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:33.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:33.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:33.699 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:33.728 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:33.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:33.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:33.832 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:33.857 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:33.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:33.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:33.960 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:33.986 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:33.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:33.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:34.089 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:34.120 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:34.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:34.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:34.165 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 32 16 51207 51191 99.9684 68.5625 0.0124436 0.00999374 2023-09-23T18:43:34.229 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:34.240 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:34 smithi173 bash[59422]: cluster 2023-09-23T18:43:33.967285+0000 mgr.y (mgr.26054) 759 : cluster 0 pgmap v768: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-09-23T18:43:34.257 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:34.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:34.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:34.311 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:43:34.360 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:34.366 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:43:34.383 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:34.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:34.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:34.402 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:43:34.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:34 smithi098 bash[76817]: cluster 2023-09-23T18:43:33.967285+0000 mgr.y (mgr.26054) 759 : cluster 0 pgmap v768: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-09-23T18:43:34.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:34 smithi098 bash[79516]: cluster 2023-09-23T18:43:33.967285+0000 mgr.y (mgr.26054) 759 : cluster 0 pgmap v768: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-09-23T18:43:34.441 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:43:34.479 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:43:34.487 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:34.510 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:43:34.514 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:34.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:34.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:34.544 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:43:34.583 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:43:34.592 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:43:34.592 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:43:34.592 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:43:34.592 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:43:34.592 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:43:34.592 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:43:34.592 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:43:34.592 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:43:34.617 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:34.642 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:34.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:34.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:34.745 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:34.781 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:34.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:34.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:34.835 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:43:34.839 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-23T18:43:34.886 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:34.912 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:34.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:34.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:35.022 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:35.050 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:35.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:35.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:35.097 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is set 2023-09-23T18:43:35.099 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:35 smithi098 bash[76817]: audit 2023-09-23T18:43:34.256021+0000 mon.c (mon.2) 96 : audit 0 from='client.? 172.21.15.98:0/2291707808' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:43:35.099 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:35 smithi098 bash[76817]: audit 2023-09-23T18:43:34.425973+0000 mon.a (mon.0) 1022 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:43:35.099 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:35 smithi098 bash[76817]: audit 2023-09-23T18:43:34.425996+0000 mon.b (mon.1) 16 : audit 1 from='client.? 172.21.15.98:0/1515622190' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:43:35.099 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:35 smithi098 bash[76817]: audit 2023-09-23T18:43:34.813219+0000 mgr.y (mgr.26054) 760 : audit 0 from='client.45218 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:43:35.151 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:35 smithi098 bash[79516]: audit 2023-09-23T18:43:34.256021+0000 mon.c (mon.2) 96 : audit 0 from='client.? 172.21.15.98:0/2291707808' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:43:35.151 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:35 smithi098 bash[79516]: audit 2023-09-23T18:43:34.425973+0000 mon.a (mon.0) 1022 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:43:35.151 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:35 smithi098 bash[79516]: audit 2023-09-23T18:43:34.425996+0000 mon.b (mon.1) 16 : audit 1 from='client.? 172.21.15.98:0/1515622190' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:43:35.151 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:35 smithi098 bash[79516]: audit 2023-09-23T18:43:34.813219+0000 mgr.y (mgr.26054) 760 : audit 0 from='client.45218 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:43:35.155 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:35.165 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 33 16 52226 52210 98.8689 63.6875 0.0164687 0.0101083 2023-09-23T18:43:35.191 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:35.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:35.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:35.294 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:35.304 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:35 smithi173 bash[59422]: audit 2023-09-23T18:43:34.256021+0000 mon.c (mon.2) 96 : audit 0 from='client.? 172.21.15.98:0/2291707808' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:43:35.304 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:35 smithi173 bash[59422]: audit 2023-09-23T18:43:34.425973+0000 mon.a (mon.0) 1022 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:43:35.304 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:35 smithi173 bash[59422]: audit 2023-09-23T18:43:34.425996+0000 mon.b (mon.1) 16 : audit 1 from='client.? 172.21.15.98:0/1515622190' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:43:35.304 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:35 smithi173 bash[59422]: audit 2023-09-23T18:43:34.813219+0000 mgr.y (mgr.26054) 760 : audit 0 from='client.45218 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:43:35.320 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:35.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:35.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:35.423 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:35.446 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:35.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:35.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:35.550 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:35.574 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:35.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:35.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:35.677 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:35.705 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:35.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:35.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:35.808 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:35.832 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:35.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:35.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:35.935 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:35.963 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:35.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:35.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:36.068 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:36.108 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:36.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:36.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:36.166 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 34 16 53415 53399 98.1464 74.3125 0.0117989 0.0101831 2023-09-23T18:43:36.190 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:43:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:43:36] "GET /metrics HTTP/1.1" 200 36551 "" "Prometheus/2.33.4" 2023-09-23T18:43:36.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:36 smithi098 bash[79516]: cluster 2023-09-23T18:43:35.079247+0000 mon.a (mon.0) 1023 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:43:36.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:36 smithi098 bash[79516]: cluster 2023-09-23T18:43:35.088365+0000 osd.5 (osd.5) 90 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T18:43:36.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:36 smithi098 bash[79516]: audit 2023-09-23T18:43:35.090931+0000 mon.a (mon.0) 1024 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:43:36.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:36 smithi098 bash[79516]: cluster 2023-09-23T18:43:35.091041+0000 mon.a (mon.0) 1025 : cluster 0 osdmap e193: 8 total, 8 up, 7 in 2023-09-23T18:43:36.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:36 smithi098 bash[79516]: cluster 2023-09-23T18:43:35.091267+0000 osd.4 (osd.4) 151 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T18:43:36.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:36 smithi098 bash[79516]: cluster 2023-09-23T18:43:35.097201+0000 osd.6 (osd.6) 123 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T18:43:36.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:36 smithi098 bash[79516]: cluster 2023-09-23T18:43:35.967796+0000 mgr.y (mgr.26054) 761 : cluster 0 pgmap v770: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T18:43:36.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:36 smithi098 bash[76817]: cluster 2023-09-23T18:43:35.079247+0000 mon.a (mon.0) 1023 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:43:36.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:36 smithi098 bash[76817]: cluster 2023-09-23T18:43:35.088365+0000 osd.5 (osd.5) 90 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T18:43:36.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:36 smithi098 bash[76817]: audit 2023-09-23T18:43:35.090931+0000 mon.a (mon.0) 1024 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:43:36.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:36 smithi098 bash[76817]: cluster 2023-09-23T18:43:35.091041+0000 mon.a (mon.0) 1025 : cluster 0 osdmap e193: 8 total, 8 up, 7 in 2023-09-23T18:43:36.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:36 smithi098 bash[76817]: cluster 2023-09-23T18:43:35.091267+0000 osd.4 (osd.4) 151 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T18:43:36.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:36 smithi098 bash[76817]: cluster 2023-09-23T18:43:35.097201+0000 osd.6 (osd.6) 123 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T18:43:36.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:36 smithi098 bash[76817]: cluster 2023-09-23T18:43:35.967796+0000 mgr.y (mgr.26054) 761 : cluster 0 pgmap v770: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T18:43:36.214 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:36.245 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:36.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:36.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:36.347 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:36.358 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:36 smithi173 bash[59422]: cluster 2023-09-23T18:43:35.079247+0000 mon.a (mon.0) 1023 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:43:36.358 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:36 smithi173 bash[59422]: cluster 2023-09-23T18:43:35.088365+0000 osd.5 (osd.5) 90 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T18:43:36.358 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:36 smithi173 bash[59422]: audit 2023-09-23T18:43:35.090931+0000 mon.a (mon.0) 1024 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:43:36.359 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:36 smithi173 bash[59422]: cluster 2023-09-23T18:43:35.091041+0000 mon.a (mon.0) 1025 : cluster 0 osdmap e193: 8 total, 8 up, 7 in 2023-09-23T18:43:36.359 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:36 smithi173 bash[59422]: cluster 2023-09-23T18:43:35.091267+0000 osd.4 (osd.4) 151 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T18:43:36.359 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:36 smithi173 bash[59422]: cluster 2023-09-23T18:43:35.097201+0000 osd.6 (osd.6) 123 : cluster 3 failed to encode map e193 with expected crc 2023-09-23T18:43:36.359 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:36 smithi173 bash[59422]: cluster 2023-09-23T18:43:35.967796+0000 mgr.y (mgr.26054) 761 : cluster 0 pgmap v770: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T18:43:36.373 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:36.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:36.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:36.476 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:36.502 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:36.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:36.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:36.608 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:36.631 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:36.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:36.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:36.734 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:36.766 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:36.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:36.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:36.869 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:36.898 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:36.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:36.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:37.003 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:37.033 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:37.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:37.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:37.137 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:37.351 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 35 16 54586 54570 97.4332 73.1875 0.0157801 0.010258 2023-09-23T18:43:37.387 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:37.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:37.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:37.494 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:37.516 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:37.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:37.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:37.619 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:37.646 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:37.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:37.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:37.748 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:37.775 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:37.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:37.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:37.879 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:37.906 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:37.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:37.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:38.009 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:38.033 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:38.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:38.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:38.137 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:38.158 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":768,"stamp":"2023-09-23T18:43:33.966962+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3153926067,"num_objects":48080,"num_object_clones":0,"num_object_copies":144240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":48080,"num_whiteouts":0,"num_read":6245,"num_read_kb":13702,"num_write":49158,"num_write_kb":3090487,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":46,"num_bytes_recovered":4195504,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":49222,"ondisk_log_size":49222,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":11655420,"kb_used_data":9215492,"kb_used_omap":31,"kb_used_meta":2439840,"kb_avail":644560644,"statfs":{"total":671965249536,"available":660030099456,"internally_reserved":0,"allocated":9436663808,"data_stored":9431490835,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32507,"internal_metadata":2498396421},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,4,10,5,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1039335424,"num_objects":15859,"num_object_clones":0,"num_object_copies":47577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15859,"num_write_kb":1014976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.007937"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":50,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.054267+0000","last_change":"2023-09-23T18:37:40.992912+0000","last_active":"2023-09-23T18:42:58.054267+0000","last_peered":"2023-09-23T18:42:58.054267+0000","last_clean":"2023-09-23T18:42:58.054267+0000","last_became_active":"2023-09-23T18:37:40.992453+0000","last_became_peered":"2023-09-23T18:37:40.992453+0000","last_unstale":"2023-09-23T18:42:58.054267+0000","last_undegraded":"2023-09-23T18:42:58.054267+0000","last_fullsized":"2023-09-23T18:42:58.054267+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:49.539251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":132,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:04.143253+0000","last_change":"2023-09-23T18:40:42.051229+0000","last_active":"2023-09-23T18:43:04.143253+0000","last_peered":"2023-09-23T18:43:04.143253+0000","last_clean":"2023-09-23T18:43:04.143253+0000","last_became_active":"2023-09-23T18:40:42.050622+0000","last_became_peered":"2023-09-23T18:40:42.050622+0000","last_unstale":"2023-09-23T18:43:04.143253+0000","last_undegraded":"2023-09-23T18:43:04.143253+0000","last_fullsized":"2023-09-23T18:43:04.143253+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:17.587802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":131,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:04.141998+0000","last_change":"2023-09-23T18:33:53.848655+0000","last_active":"2023-09-23T18:43:04.141998+0000","last_peered":"2023-09-23T18:43:04.141998+0000","last_clean":"2023-09-23T18:43:04.141998+0000","last_became_active":"2023-09-23T18:33:53.848279+0000","last_became_peered":"2023-09-23T18:33:53.848279+0000","last_unstale":"2023-09-23T18:43:04.141998+0000","last_undegraded":"2023-09-23T18:43:04.141998+0000","last_fullsized":"2023-09-23T18:43:04.141998+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:39.223612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":166,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.056636+0000","last_change":"2023-09-23T18:40:42.052899+0000","last_active":"2023-09-23T18:42:58.056636+0000","last_peered":"2023-09-23T18:42:58.056636+0000","last_clean":"2023-09-23T18:42:58.056636+0000","last_became_active":"2023-09-23T18:40:42.052569+0000","last_became_peered":"2023-09-23T18:40:42.052569+0000","last_unstale":"2023-09-23T18:42:58.056636+0000","last_undegraded":"2023-09-23T18:42:58.056636+0000","last_fullsized":"2023-09-23T18:42:58.056636+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:46.151073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":148,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:04.145739+0000","last_change":"2023-09-23T18:40:42.048469+0000","last_active":"2023-09-23T18:43:04.145739+0000","last_peered":"2023-09-23T18:43:04.145739+0000","last_clean":"2023-09-23T18:43:04.145739+0000","last_became_active":"2023-09-23T18:40:42.048139+0000","last_became_peered":"2023-09-23T18:40:42.048139+0000","last_unstale":"2023-09-23T18:43:04.145739+0000","last_undegraded":"2023-09-23T18:43:04.145739+0000","last_fullsized":"2023-09-23T18:43:04.145739+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:09:09.535543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":183,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.467884+0000","last_change":"2023-09-23T18:33:55.084557+0000","last_active":"2023-09-23T18:42:58.467884+0000","last_peered":"2023-09-23T18:42:58.467884+0000","last_clean":"2023-09-23T18:42:58.467884+0000","last_became_active":"2023-09-23T18:33:55.083987+0000","last_became_peered":"2023-09-23T18:33:55.083987+0000","last_unstale":"2023-09-23T18:42:58.467884+0000","last_undegraded":"2023-09-23T18:42:58.467884+0000","last_fullsized":"2023-09-23T18:42:58.467884+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":164,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:04.147427+0000","last_change":"2023-09-23T18:40:42.054577+0000","last_active":"2023-09-23T18:43:04.147427+0000","last_peered":"2023-09-23T18:43:04.147427+0000","last_clean":"2023-09-23T18:43:04.147427+0000","last_became_active":"2023-09-23T18:40:42.054170+0000","last_became_peered":"2023-09-23T18:40:42.054170+0000","last_unstale":"2023-09-23T18:43:04.147427+0000","last_undegraded":"2023-09-23T18:43:04.147427+0000","last_fullsized":"2023-09-23T18:43:04.147427+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:35.503748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":189,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.051092+0000","last_change":"2023-09-23T18:37:40.998526+0000","last_active":"2023-09-23T18:42:58.051092+0000","last_peered":"2023-09-23T18:42:58.051092+0000","last_clean":"2023-09-23T18:42:58.051092+0000","last_became_active":"2023-09-23T18:37:40.993211+0000","last_became_peered":"2023-09-23T18:37:40.993211+0000","last_unstale":"2023-09-23T18:42:58.051092+0000","last_undegraded":"2023-09-23T18:42:58.051092+0000","last_fullsized":"2023-09-23T18:42:58.051092+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:40.463496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":190,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.150327+0000","last_change":"2023-09-23T18:40:42.047724+0000","last_active":"2023-09-23T18:42:58.150327+0000","last_peered":"2023-09-23T18:42:58.150327+0000","last_clean":"2023-09-23T18:42:58.150327+0000","last_became_active":"2023-09-23T18:40:42.047236+0000","last_became_peered":"2023-09-23T18:40:42.047236+0000","last_unstale":"2023-09-23T18:42:58.150327+0000","last_undegraded":"2023-09-23T18:42:58.150327+0000","last_fullsized":"2023-09-23T18:42:58.150327+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":155,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.050724+0000","last_change":"2023-09-23T18:28:03.298214+0000","last_active":"2023-09-23T18:42:58.050724+0000","last_peered":"2023-09-23T18:42:58.050724+0000","last_clean":"2023-09-23T18:42:58.050724+0000","last_became_active":"2023-09-23T18:28:03.297852+0000","last_became_peered":"2023-09-23T18:28:03.297852+0000","last_unstale":"2023-09-23T18:42:58.050724+0000","last_undegraded":"2023-09-23T18:42:58.050724+0000","last_fullsized":"2023-09-23T18:42:58.050724+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.146541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":161,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.151686+0000","last_change":"2023-09-23T18:28:03.296913+0000","last_active":"2023-09-23T18:42:58.151686+0000","last_peered":"2023-09-23T18:42:58.151686+0000","last_clean":"2023-09-23T18:42:58.151686+0000","last_became_active":"2023-09-23T18:28:03.296499+0000","last_became_peered":"2023-09-23T18:28:03.296499+0000","last_unstale":"2023-09-23T18:42:58.151686+0000","last_undegraded":"2023-09-23T18:42:58.151686+0000","last_fullsized":"2023-09-23T18:42:58.151686+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":127,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:04.142672+0000","last_change":"2023-09-23T18:33:53.850096+0000","last_active":"2023-09-23T18:43:04.142672+0000","last_peered":"2023-09-23T18:43:04.142672+0000","last_clean":"2023-09-23T18:43:04.142672+0000","last_became_active":"2023-09-23T18:33:53.849811+0000","last_became_peered":"2023-09-23T18:33:53.849811+0000","last_unstale":"2023-09-23T18:43:04.142672+0000","last_undegraded":"2023-09-23T18:43:04.142672+0000","last_fullsized":"2023-09-23T18:43:04.142672+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:44.938072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":165,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.268983+0000","last_change":"2023-09-23T18:15:08.909339+0000","last_active":"2023-09-23T18:42:58.268983+0000","last_peered":"2023-09-23T18:42:58.268983+0000","last_clean":"2023-09-23T18:42:58.268983+0000","last_became_active":"2023-09-23T18:15:08.908912+0000","last_became_peered":"2023-09-23T18:15:08.908912+0000","last_unstale":"2023-09-23T18:42:58.268983+0000","last_undegraded":"2023-09-23T18:42:58.268983+0000","last_fullsized":"2023-09-23T18:42:58.268983+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":172,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.268945+0000","last_change":"2023-09-23T18:33:53.847489+0000","last_active":"2023-09-23T18:42:58.268945+0000","last_peered":"2023-09-23T18:42:58.268945+0000","last_clean":"2023-09-23T18:42:58.268945+0000","last_became_active":"2023-09-23T18:33:53.846935+0000","last_became_peered":"2023-09-23T18:33:53.846935+0000","last_unstale":"2023-09-23T18:42:58.268945+0000","last_undegraded":"2023-09-23T18:42:58.268945+0000","last_fullsized":"2023-09-23T18:42:58.268945+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":160,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.151294+0000","last_change":"2023-09-23T18:27:33.585121+0000","last_active":"2023-09-23T18:42:58.151294+0000","last_peered":"2023-09-23T18:42:58.151294+0000","last_clean":"2023-09-23T18:42:58.151294+0000","last_became_active":"2023-09-23T18:27:33.584620+0000","last_became_peered":"2023-09-23T18:27:33.584620+0000","last_unstale":"2023-09-23T18:42:58.151294+0000","last_undegraded":"2023-09-23T18:42:58.151294+0000","last_fullsized":"2023-09-23T18:42:58.151294+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":180,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.051042+0000","last_change":"2023-09-23T18:33:53.852933+0000","last_active":"2023-09-23T18:42:58.051042+0000","last_peered":"2023-09-23T18:42:58.051042+0000","last_clean":"2023-09-23T18:42:58.051042+0000","last_became_active":"2023-09-23T18:33:53.852626+0000","last_became_peered":"2023-09-23T18:33:53.852626+0000","last_unstale":"2023-09-23T18:42:58.051042+0000","last_undegraded":"2023-09-23T18:42:58.051042+0000","last_fullsized":"2023-09-23T18:42:58.051042+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:00.174536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":155,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.055527+0000","last_change":"2023-09-23T18:39:46.997790+0000","last_active":"2023-09-23T18:42:58.055527+0000","last_peered":"2023-09-23T18:42:58.055527+0000","last_clean":"2023-09-23T18:42:58.055527+0000","last_became_active":"2023-09-23T18:39:46.997089+0000","last_became_peered":"2023-09-23T18:39:46.997089+0000","last_unstale":"2023-09-23T18:42:58.055527+0000","last_undegraded":"2023-09-23T18:42:58.055527+0000","last_fullsized":"2023-09-23T18:42:58.055527+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:55.378220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":149,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.151475+0000","last_change":"2023-09-23T18:32:26.808237+0000","last_active":"2023-09-23T18:42:58.151475+0000","last_peered":"2023-09-23T18:42:58.151475+0000","last_clean":"2023-09-23T18:42:58.151475+0000","last_became_active":"2023-09-23T18:32:26.807843+0000","last_became_peered":"2023-09-23T18:32:26.807843+0000","last_unstale":"2023-09-23T18:42:58.151475+0000","last_undegraded":"2023-09-23T18:42:58.151475+0000","last_fullsized":"2023-09-23T18:42:58.151475+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":143,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.150911+0000","last_change":"2023-09-23T18:15:08.916658+0000","last_active":"2023-09-23T18:42:58.150911+0000","last_peered":"2023-09-23T18:42:58.150911+0000","last_clean":"2023-09-23T18:42:58.150911+0000","last_became_active":"2023-09-23T18:15:08.916301+0000","last_became_peered":"2023-09-23T18:15:08.916301+0000","last_unstale":"2023-09-23T18:42:58.150911+0000","last_undegraded":"2023-09-23T18:42:58.150911+0000","last_fullsized":"2023-09-23T18:42:58.150911+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":156,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:04.148199+0000","last_change":"2023-09-23T18:40:42.049370+0000","last_active":"2023-09-23T18:43:04.148199+0000","last_peered":"2023-09-23T18:43:04.148199+0000","last_clean":"2023-09-23T18:43:04.148199+0000","last_became_active":"2023-09-23T18:40:42.049003+0000","last_became_peered":"2023-09-23T18:40:42.049003+0000","last_unstale":"2023-09-23T18:43:04.148199+0000","last_undegraded":"2023-09-23T18:43:04.148199+0000","last_fullsized":"2023-09-23T18:43:04.148199+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:15.760457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":204,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.466518+0000","last_change":"2023-09-23T18:40:42.073698+0000","last_active":"2023-09-23T18:42:58.466518+0000","last_peered":"2023-09-23T18:42:58.466518+0000","last_clean":"2023-09-23T18:42:58.466518+0000","last_became_active":"2023-09-23T18:40:42.059101+0000","last_became_peered":"2023-09-23T18:40:42.059101+0000","last_unstale":"2023-09-23T18:42:58.466518+0000","last_undegraded":"2023-09-23T18:42:58.466518+0000","last_fullsized":"2023-09-23T18:42:58.466518+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":135,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:04.144827+0000","last_change":"2023-09-23T18:40:42.058656+0000","last_active":"2023-09-23T18:43:04.144827+0000","last_peered":"2023-09-23T18:43:04.144827+0000","last_clean":"2023-09-23T18:43:04.144827+0000","last_became_active":"2023-09-23T18:40:42.057307+0000","last_became_peered":"2023-09-23T18:40:42.057307+0000","last_unstale":"2023-09-23T18:43:04.144827+0000","last_undegraded":"2023-09-23T18:43:04.144827+0000","last_fullsized":"2023-09-23T18:43:04.144827+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:08:45.979078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.b","version":"192'3042","reported_seq":3056,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:32.474573+0000","last_change":"2023-09-23T18:42:39.314324+0000","last_active":"2023-09-23T18:43:32.474573+0000","last_peered":"2023-09-23T18:43:32.474573+0000","last_clean":"2023-09-23T18:43:32.474573+0000","last_became_active":"2023-09-23T18:42:39.314006+0000","last_became_peered":"2023-09-23T18:42:39.314006+0000","last_unstale":"2023-09-23T18:43:32.474573+0000","last_undegraded":"2023-09-23T18:43:32.474573+0000","last_fullsized":"2023-09-23T18:43:32.474573+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":3042,"log_dups_size":0,"ondisk_log_size":3042,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:52.663013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199360512,"num_objects":3042,"num_object_clones":0,"num_object_copies":9126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3042,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3042,"num_write_kb":194688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":172,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.152071+0000","last_change":"2023-09-23T18:37:41.034732+0000","last_active":"2023-09-23T18:42:58.152071+0000","last_peered":"2023-09-23T18:42:58.152071+0000","last_clean":"2023-09-23T18:42:58.152071+0000","last_became_active":"2023-09-23T18:37:40.985865+0000","last_became_peered":"2023-09-23T18:37:40.985865+0000","last_unstale":"2023-09-23T18:42:58.152071+0000","last_undegraded":"2023-09-23T18:42:58.152071+0000","last_fullsized":"2023-09-23T18:42:58.152071+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":90,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.150442+0000","last_change":"2023-09-23T18:32:26.806328+0000","last_active":"2023-09-23T18:42:58.150442+0000","last_peered":"2023-09-23T18:42:58.150442+0000","last_clean":"2023-09-23T18:42:58.150442+0000","last_became_active":"2023-09-23T18:32:26.803969+0000","last_became_peered":"2023-09-23T18:32:26.803969+0000","last_unstale":"2023-09-23T18:42:58.150442+0000","last_undegraded":"2023-09-23T18:42:58.150442+0000","last_fullsized":"2023-09-23T18:42:58.150442+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.a","version":"192'2916","reported_seq":2930,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:31.265340+0000","last_change":"2023-09-23T18:42:39.310080+0000","last_active":"2023-09-23T18:43:31.265340+0000","last_peered":"2023-09-23T18:43:31.265340+0000","last_clean":"2023-09-23T18:43:31.265340+0000","last_became_active":"2023-09-23T18:42:39.309414+0000","last_became_peered":"2023-09-23T18:42:39.309414+0000","last_unstale":"2023-09-23T18:43:31.265340+0000","last_undegraded":"2023-09-23T18:43:31.265340+0000","last_fullsized":"2023-09-23T18:43:31.265340+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":2916,"log_dups_size":0,"ondisk_log_size":2916,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:43.822106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191102976,"num_objects":2916,"num_object_clones":0,"num_object_copies":8748,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2916,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2916,"num_write_kb":186624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":175,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.151900+0000","last_change":"2023-09-23T18:33:53.847817+0000","last_active":"2023-09-23T18:42:58.151900+0000","last_peered":"2023-09-23T18:42:58.151900+0000","last_clean":"2023-09-23T18:42:58.151900+0000","last_became_active":"2023-09-23T18:33:53.847342+0000","last_became_peered":"2023-09-23T18:33:53.847342+0000","last_unstale":"2023-09-23T18:42:58.151900+0000","last_undegraded":"2023-09-23T18:42:58.151900+0000","last_fullsized":"2023-09-23T18:42:58.151900+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":196,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.268556+0000","last_change":"2023-09-23T18:39:46.995444+0000","last_active":"2023-09-23T18:42:58.268556+0000","last_peered":"2023-09-23T18:42:58.268556+0000","last_clean":"2023-09-23T18:42:58.268556+0000","last_became_active":"2023-09-23T18:39:46.995033+0000","last_became_peered":"2023-09-23T18:39:46.995033+0000","last_unstale":"2023-09-23T18:42:58.268556+0000","last_undegraded":"2023-09-23T18:42:58.268556+0000","last_fullsized":"2023-09-23T18:42:58.268556+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"192'3146","reported_seq":3160,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:32.944250+0000","last_change":"2023-09-23T18:42:39.318059+0000","last_active":"2023-09-23T18:43:32.944250+0000","last_peered":"2023-09-23T18:43:32.944250+0000","last_clean":"2023-09-23T18:43:32.944250+0000","last_became_active":"2023-09-23T18:42:39.317674+0000","last_became_peered":"2023-09-23T18:42:39.317674+0000","last_unstale":"2023-09-23T18:43:32.944250+0000","last_undegraded":"2023-09-23T18:43:32.944250+0000","last_fullsized":"2023-09-23T18:43:32.944250+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":3146,"log_dups_size":0,"ondisk_log_size":3146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:31:13.868307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206176256,"num_objects":3146,"num_object_clones":0,"num_object_copies":9438,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3146,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3146,"num_write_kb":201344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":157,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.151896+0000","last_change":"2023-09-23T18:32:26.854615+0000","last_active":"2023-09-23T18:42:58.151896+0000","last_peered":"2023-09-23T18:42:58.151896+0000","last_clean":"2023-09-23T18:42:58.151896+0000","last_became_active":"2023-09-23T18:32:26.806771+0000","last_became_peered":"2023-09-23T18:32:26.806771+0000","last_unstale":"2023-09-23T18:42:58.151896+0000","last_undegraded":"2023-09-23T18:42:58.151896+0000","last_fullsized":"2023-09-23T18:42:58.151896+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":166,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.467171+0000","last_change":"2023-09-23T18:27:33.589116+0000","last_active":"2023-09-23T18:42:58.467171+0000","last_peered":"2023-09-23T18:42:58.467171+0000","last_clean":"2023-09-23T18:42:58.467171+0000","last_became_active":"2023-09-23T18:27:33.588809+0000","last_became_peered":"2023-09-23T18:27:33.588809+0000","last_unstale":"2023-09-23T18:42:58.467171+0000","last_undegraded":"2023-09-23T18:42:58.467171+0000","last_fullsized":"2023-09-23T18:42:58.467171+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"192'2801","reported_seq":2815,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:29.073669+0000","last_change":"2023-09-23T18:42:39.316449+0000","last_active":"2023-09-23T18:43:29.073669+0000","last_peered":"2023-09-23T18:43:29.073669+0000","last_clean":"2023-09-23T18:43:29.073669+0000","last_became_active":"2023-09-23T18:42:39.316042+0000","last_became_peered":"2023-09-23T18:42:39.316042+0000","last_unstale":"2023-09-23T18:43:29.073669+0000","last_undegraded":"2023-09-23T18:43:29.073669+0000","last_fullsized":"2023-09-23T18:43:29.073669+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":2801,"log_dups_size":0,"ondisk_log_size":2801,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:29.847929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183566336,"num_objects":2801,"num_object_clones":0,"num_object_copies":8403,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2801,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2801,"num_write_kb":179264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":165,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.050638+0000","last_change":"2023-09-23T18:33:54.848776+0000","last_active":"2023-09-23T18:42:58.050638+0000","last_peered":"2023-09-23T18:42:58.050638+0000","last_clean":"2023-09-23T18:42:58.050638+0000","last_became_active":"2023-09-23T18:33:54.848380+0000","last_became_peered":"2023-09-23T18:33:54.848380+0000","last_unstale":"2023-09-23T18:42:58.050638+0000","last_undegraded":"2023-09-23T18:42:58.050638+0000","last_fullsized":"2023-09-23T18:42:58.050638+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:17:17.093993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":159,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.054447+0000","last_change":"2023-09-23T18:27:33.582033+0000","last_active":"2023-09-23T18:42:58.054447+0000","last_peered":"2023-09-23T18:42:58.054447+0000","last_clean":"2023-09-23T18:42:58.054447+0000","last_became_active":"2023-09-23T18:27:33.581667+0000","last_became_peered":"2023-09-23T18:27:33.581667+0000","last_unstale":"2023-09-23T18:42:58.054447+0000","last_undegraded":"2023-09-23T18:42:58.054447+0000","last_fullsized":"2023-09-23T18:42:58.054447+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:36.039922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"192'3120","reported_seq":3134,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:32.990864+0000","last_change":"2023-09-23T18:42:39.318187+0000","last_active":"2023-09-23T18:43:32.990864+0000","last_peered":"2023-09-23T18:43:32.990864+0000","last_clean":"2023-09-23T18:43:32.990864+0000","last_became_active":"2023-09-23T18:42:39.317693+0000","last_became_peered":"2023-09-23T18:42:39.317693+0000","last_unstale":"2023-09-23T18:43:32.990864+0000","last_undegraded":"2023-09-23T18:43:32.990864+0000","last_fullsized":"2023-09-23T18:43:32.990864+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":3120,"log_dups_size":0,"ondisk_log_size":3120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:53.117569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204472320,"num_objects":3120,"num_object_clones":0,"num_object_copies":9360,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3120,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3120,"num_write_kb":199680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":164,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.051213+0000","last_change":"2023-09-23T18:28:03.297628+0000","last_active":"2023-09-23T18:42:58.051213+0000","last_peered":"2023-09-23T18:42:58.051213+0000","last_clean":"2023-09-23T18:42:58.051213+0000","last_became_active":"2023-09-23T18:28:03.296694+0000","last_became_peered":"2023-09-23T18:28:03.296694+0000","last_unstale":"2023-09-23T18:42:58.051213+0000","last_undegraded":"2023-09-23T18:42:58.051213+0000","last_fullsized":"2023-09-23T18:42:58.051213+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:43.375652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":157,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.268934+0000","last_change":"2023-09-23T18:28:03.302326+0000","last_active":"2023-09-23T18:42:58.268934+0000","last_peered":"2023-09-23T18:42:58.268934+0000","last_clean":"2023-09-23T18:42:58.268934+0000","last_became_active":"2023-09-23T18:28:03.301970+0000","last_became_peered":"2023-09-23T18:28:03.301970+0000","last_unstale":"2023-09-23T18:42:58.268934+0000","last_undegraded":"2023-09-23T18:42:58.268934+0000","last_fullsized":"2023-09-23T18:42:58.268934+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"192'2835","reported_seq":2849,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:29.093132+0000","last_change":"2023-09-23T18:42:39.309814+0000","last_active":"2023-09-23T18:43:29.093132+0000","last_peered":"2023-09-23T18:43:29.093132+0000","last_clean":"2023-09-23T18:43:29.093132+0000","last_became_active":"2023-09-23T18:42:39.309499+0000","last_became_peered":"2023-09-23T18:42:39.309499+0000","last_unstale":"2023-09-23T18:43:29.093132+0000","last_undegraded":"2023-09-23T18:43:29.093132+0000","last_fullsized":"2023-09-23T18:43:29.093132+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":2835,"log_dups_size":0,"ondisk_log_size":2835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:26.106244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185794560,"num_objects":2835,"num_object_clones":0,"num_object_copies":8505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2835,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2835,"num_write_kb":181440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":173,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.466996+0000","last_change":"2023-09-23T18:33:53.855371+0000","last_active":"2023-09-23T18:42:58.466996+0000","last_peered":"2023-09-23T18:42:58.466996+0000","last_clean":"2023-09-23T18:42:58.466996+0000","last_became_active":"2023-09-23T18:33:53.854996+0000","last_became_peered":"2023-09-23T18:33:53.854996+0000","last_unstale":"2023-09-23T18:42:58.466996+0000","last_undegraded":"2023-09-23T18:42:58.466996+0000","last_fullsized":"2023-09-23T18:42:58.466996+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":196,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.268061+0000","last_change":"2023-09-23T18:40:42.060184+0000","last_active":"2023-09-23T18:42:58.268061+0000","last_peered":"2023-09-23T18:42:58.268061+0000","last_clean":"2023-09-23T18:42:58.268061+0000","last_became_active":"2023-09-23T18:40:42.059725+0000","last_became_peered":"2023-09-23T18:40:42.059725+0000","last_unstale":"2023-09-23T18:42:58.268061+0000","last_undegraded":"2023-09-23T18:42:58.268061+0000","last_fullsized":"2023-09-23T18:42:58.268061+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"192'2929","reported_seq":2942,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:30.815344+0000","last_change":"2023-09-23T18:42:39.319875+0000","last_active":"2023-09-23T18:43:30.815344+0000","last_peered":"2023-09-23T18:43:30.815344+0000","last_clean":"2023-09-23T18:43:30.815344+0000","last_became_active":"2023-09-23T18:42:39.319468+0000","last_became_peered":"2023-09-23T18:42:39.319468+0000","last_unstale":"2023-09-23T18:43:30.815344+0000","last_undegraded":"2023-09-23T18:43:30.815344+0000","last_fullsized":"2023-09-23T18:43:30.815344+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":2929,"log_dups_size":0,"ondisk_log_size":2929,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191954944,"num_objects":2929,"num_object_clones":0,"num_object_copies":8787,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2929,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2929,"num_write_kb":187456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":158,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.268309+0000","last_change":"2023-09-23T18:27:33.585714+0000","last_active":"2023-09-23T18:42:58.268309+0000","last_peered":"2023-09-23T18:42:58.268309+0000","last_clean":"2023-09-23T18:42:58.268309+0000","last_became_active":"2023-09-23T18:27:33.584641+0000","last_became_peered":"2023-09-23T18:27:33.584641+0000","last_unstale":"2023-09-23T18:42:58.268309+0000","last_undegraded":"2023-09-23T18:42:58.268309+0000","last_fullsized":"2023-09-23T18:42:58.268309+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":172,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.269641+0000","last_change":"2023-09-23T18:33:53.851266+0000","last_active":"2023-09-23T18:42:58.269641+0000","last_peered":"2023-09-23T18:42:58.269641+0000","last_clean":"2023-09-23T18:42:58.269641+0000","last_became_active":"2023-09-23T18:33:53.850926+0000","last_became_peered":"2023-09-23T18:33:53.850926+0000","last_unstale":"2023-09-23T18:42:58.269641+0000","last_undegraded":"2023-09-23T18:42:58.269641+0000","last_fullsized":"2023-09-23T18:42:58.269641+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"192'2891","reported_seq":2904,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:30.841821+0000","last_change":"2023-09-23T18:42:39.318852+0000","last_active":"2023-09-23T18:43:30.841821+0000","last_peered":"2023-09-23T18:43:30.841821+0000","last_clean":"2023-09-23T18:43:30.841821+0000","last_became_active":"2023-09-23T18:42:39.318461+0000","last_became_peered":"2023-09-23T18:42:39.318461+0000","last_unstale":"2023-09-23T18:43:30.841821+0000","last_undegraded":"2023-09-23T18:43:30.841821+0000","last_fullsized":"2023-09-23T18:43:30.841821+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":2891,"log_dups_size":0,"ondisk_log_size":2891,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189464576,"num_objects":2891,"num_object_clones":0,"num_object_copies":8673,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2891,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2891,"num_write_kb":185024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":143,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.052476+0000","last_change":"2023-09-23T18:39:46.993941+0000","last_active":"2023-09-23T18:42:58.052476+0000","last_peered":"2023-09-23T18:42:58.052476+0000","last_clean":"2023-09-23T18:42:58.052476+0000","last_became_active":"2023-09-23T18:39:46.993625+0000","last_became_peered":"2023-09-23T18:39:46.993625+0000","last_unstale":"2023-09-23T18:42:58.052476+0000","last_undegraded":"2023-09-23T18:42:58.052476+0000","last_fullsized":"2023-09-23T18:42:58.052476+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:23.237229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":131,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.056373+0000","last_change":"2023-09-23T18:33:53.855430+0000","last_active":"2023-09-23T18:42:58.056373+0000","last_peered":"2023-09-23T18:42:58.056373+0000","last_clean":"2023-09-23T18:42:58.056373+0000","last_became_active":"2023-09-23T18:33:53.854935+0000","last_became_peered":"2023-09-23T18:33:53.854935+0000","last_unstale":"2023-09-23T18:42:58.056373+0000","last_undegraded":"2023-09-23T18:42:58.056373+0000","last_fullsized":"2023-09-23T18:42:58.056373+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:36.078411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"192'3052","reported_seq":3065,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:31.233974+0000","last_change":"2023-09-23T18:42:39.317727+0000","last_active":"2023-09-23T18:43:31.233974+0000","last_peered":"2023-09-23T18:43:31.233974+0000","last_clean":"2023-09-23T18:43:31.233974+0000","last_became_active":"2023-09-23T18:42:39.317317+0000","last_became_peered":"2023-09-23T18:42:39.317317+0000","last_unstale":"2023-09-23T18:43:31.233974+0000","last_undegraded":"2023-09-23T18:43:31.233974+0000","last_fullsized":"2023-09-23T18:43:31.233974+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":3052,"log_dups_size":0,"ondisk_log_size":3052,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200015872,"num_objects":3052,"num_object_clones":0,"num_object_copies":9156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3052,"num_write_kb":195328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"77'130","reported_seq":657,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.055887+0000","last_change":"2023-09-23T18:33:53.850713+0000","last_active":"2023-09-23T18:42:58.055887+0000","last_peered":"2023-09-23T18:42:58.055887+0000","last_clean":"2023-09-23T18:42:58.055887+0000","last_became_active":"2023-09-23T18:33:53.850011+0000","last_became_peered":"2023-09-23T18:33:53.850011+0000","last_unstale":"2023-09-23T18:42:58.055887+0000","last_undegraded":"2023-09-23T18:42:58.055887+0000","last_fullsized":"2023-09-23T18:42:58.055887+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:58.943556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":772,"num_read_kb":1712,"num_write":126,"num_write_kb":2126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":112,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.466010+0000","last_change":"2023-09-23T18:40:42.058197+0000","last_active":"2023-09-23T18:42:58.466010+0000","last_peered":"2023-09-23T18:42:58.466010+0000","last_clean":"2023-09-23T18:42:58.466010+0000","last_became_active":"2023-09-23T18:40:42.057834+0000","last_became_peered":"2023-09-23T18:40:42.057834+0000","last_unstale":"2023-09-23T18:42:58.466010+0000","last_undegraded":"2023-09-23T18:42:58.466010+0000","last_fullsized":"2023-09-23T18:42:58.466010+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:28.133529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":105,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.052979+0000","last_change":"2023-09-23T18:33:54.843246+0000","last_active":"2023-09-23T18:42:58.052979+0000","last_peered":"2023-09-23T18:42:58.052979+0000","last_clean":"2023-09-23T18:42:58.052979+0000","last_became_active":"2023-09-23T18:33:54.842933+0000","last_became_peered":"2023-09-23T18:33:54.842933+0000","last_unstale":"2023-09-23T18:42:58.052979+0000","last_undegraded":"2023-09-23T18:42:58.052979+0000","last_fullsized":"2023-09-23T18:42:58.052979+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:43.716707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"192'2981","reported_seq":2994,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:30.792166+0000","last_change":"2023-09-23T18:42:39.319694+0000","last_active":"2023-09-23T18:43:30.792166+0000","last_peered":"2023-09-23T18:43:30.792166+0000","last_clean":"2023-09-23T18:43:30.792166+0000","last_became_active":"2023-09-23T18:42:39.319390+0000","last_became_peered":"2023-09-23T18:42:39.319390+0000","last_unstale":"2023-09-23T18:43:30.792166+0000","last_undegraded":"2023-09-23T18:43:30.792166+0000","last_fullsized":"2023-09-23T18:43:30.792166+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":2981,"log_dups_size":0,"ondisk_log_size":2981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195362816,"num_objects":2981,"num_object_clones":0,"num_object_copies":8943,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2981,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2981,"num_write_kb":190784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"77'96","reported_seq":601,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.053347+0000","last_change":"2023-09-23T18:33:54.842857+0000","last_active":"2023-09-23T18:42:58.053347+0000","last_peered":"2023-09-23T18:42:58.053347+0000","last_clean":"2023-09-23T18:42:58.053347+0000","last_became_active":"2023-09-23T18:33:54.842134+0000","last_became_peered":"2023-09-23T18:33:54.842134+0000","last_unstale":"2023-09-23T18:42:58.053347+0000","last_undegraded":"2023-09-23T18:42:58.053347+0000","last_fullsized":"2023-09-23T18:42:58.053347+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":96,"log_dups_size":0,"ondisk_log_size":96,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:48.960158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":728,"num_read_kb":1234,"num_write":87,"num_write_kb":338,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":158,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.268944+0000","last_change":"2023-09-23T18:26:34.992535+0000","last_active":"2023-09-23T18:42:58.268944+0000","last_peered":"2023-09-23T18:42:58.268944+0000","last_clean":"2023-09-23T18:42:58.268944+0000","last_became_active":"2023-09-23T18:26:34.992177+0000","last_became_peered":"2023-09-23T18:26:34.992177+0000","last_unstale":"2023-09-23T18:42:58.268944+0000","last_undegraded":"2023-09-23T18:42:58.268944+0000","last_fullsized":"2023-09-23T18:42:58.268944+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":155,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.055790+0000","last_change":"2023-09-23T18:40:42.066006+0000","last_active":"2023-09-23T18:42:58.055790+0000","last_peered":"2023-09-23T18:42:58.055790+0000","last_clean":"2023-09-23T18:42:58.055790+0000","last_became_active":"2023-09-23T18:40:42.065431+0000","last_became_peered":"2023-09-23T18:40:42.065431+0000","last_unstale":"2023-09-23T18:42:58.055790+0000","last_undegraded":"2023-09-23T18:42:58.055790+0000","last_fullsized":"2023-09-23T18:42:58.055790+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:18.818629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"192'3144","reported_seq":3158,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:32.929889+0000","last_change":"2023-09-23T18:42:39.317096+0000","last_active":"2023-09-23T18:43:32.929889+0000","last_peered":"2023-09-23T18:43:32.929889+0000","last_clean":"2023-09-23T18:43:32.929889+0000","last_became_active":"2023-09-23T18:42:39.316796+0000","last_became_peered":"2023-09-23T18:42:39.316796+0000","last_unstale":"2023-09-23T18:43:32.929889+0000","last_undegraded":"2023-09-23T18:43:32.929889+0000","last_fullsized":"2023-09-23T18:43:32.929889+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":3144,"log_dups_size":0,"ondisk_log_size":3144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:22.850545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206045184,"num_objects":3144,"num_object_clones":0,"num_object_copies":9432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3144,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3144,"num_write_kb":201216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"77'112","reported_seq":542,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.054121+0000","last_change":"2023-09-23T18:40:45.106420+0000","last_active":"2023-09-23T18:42:58.054121+0000","last_peered":"2023-09-23T18:42:58.054121+0000","last_clean":"2023-09-23T18:42:58.054121+0000","last_became_active":"2023-09-23T18:40:45.104294+0000","last_became_peered":"2023-09-23T18:40:45.104294+0000","last_unstale":"2023-09-23T18:42:58.054121+0000","last_undegraded":"2023-09-23T18:42:58.054121+0000","last_fullsized":"2023-09-23T18:42:58.054121+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:45.419412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":774,"num_read_kb":596,"num_write":102,"num_write_kb":52,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":127,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.052202+0000","last_change":"2023-09-23T18:33:53.849084+0000","last_active":"2023-09-23T18:42:58.052202+0000","last_peered":"2023-09-23T18:42:58.052202+0000","last_clean":"2023-09-23T18:42:58.052202+0000","last_became_active":"2023-09-23T18:33:53.848757+0000","last_became_peered":"2023-09-23T18:33:53.848757+0000","last_unstale":"2023-09-23T18:42:58.052202+0000","last_undegraded":"2023-09-23T18:42:58.052202+0000","last_fullsized":"2023-09-23T18:42:58.052202+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:15.267742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":150,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.056615+0000","last_change":"2023-09-23T18:33:55.087304+0000","last_active":"2023-09-23T18:42:58.056615+0000","last_peered":"2023-09-23T18:42:58.056615+0000","last_clean":"2023-09-23T18:42:58.056615+0000","last_became_active":"2023-09-23T18:33:55.086832+0000","last_became_peered":"2023-09-23T18:33:55.086832+0000","last_unstale":"2023-09-23T18:42:58.056615+0000","last_undegraded":"2023-09-23T18:42:58.056615+0000","last_fullsized":"2023-09-23T18:42:58.056615+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:55.457664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"192'3199","reported_seq":3213,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:32.994889+0000","last_change":"2023-09-23T18:42:39.318702+0000","last_active":"2023-09-23T18:43:32.994889+0000","last_peered":"2023-09-23T18:43:32.994889+0000","last_clean":"2023-09-23T18:43:32.994889+0000","last_became_active":"2023-09-23T18:42:39.318405+0000","last_became_peered":"2023-09-23T18:42:39.318405+0000","last_unstale":"2023-09-23T18:43:32.994889+0000","last_undegraded":"2023-09-23T18:43:32.994889+0000","last_fullsized":"2023-09-23T18:43:32.994889+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":3199,"log_dups_size":0,"ondisk_log_size":3199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:03.343758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209649664,"num_objects":3199,"num_object_clones":0,"num_object_copies":9597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3199,"num_write_kb":204736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"77'147","reported_seq":582,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.466839+0000","last_change":"2023-09-23T18:32:29.845404+0000","last_active":"2023-09-23T18:42:58.466839+0000","last_peered":"2023-09-23T18:42:58.466839+0000","last_clean":"2023-09-23T18:42:58.466839+0000","last_became_active":"2023-09-23T18:32:29.844545+0000","last_became_peered":"2023-09-23T18:32:29.844545+0000","last_unstale":"2023-09-23T18:42:58.466839+0000","last_undegraded":"2023-09-23T18:42:58.466839+0000","last_fullsized":"2023-09-23T18:42:58.466839+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:44.834482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":624,"num_read_kb":1868,"num_write":143,"num_write_kb":5368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":394,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.150903+0000","last_change":"2023-09-23T18:26:34.999402+0000","last_active":"2023-09-23T18:42:58.150903+0000","last_peered":"2023-09-23T18:42:58.150903+0000","last_clean":"2023-09-23T18:42:58.150903+0000","last_became_active":"2023-09-23T18:26:34.998828+0000","last_became_peered":"2023-09-23T18:26:34.998828+0000","last_unstale":"2023-09-23T18:42:58.150903+0000","last_undegraded":"2023-09-23T18:42:58.150903+0000","last_fullsized":"2023-09-23T18:42:58.150903+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_clean_scrub_stamp":"2023-09-23T18:08:35.701374+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":107,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.466402+0000","last_change":"2023-09-23T18:40:42.058559+0000","last_active":"2023-09-23T18:42:58.466402+0000","last_peered":"2023-09-23T18:42:58.466402+0000","last_clean":"2023-09-23T18:42:58.466402+0000","last_became_active":"2023-09-23T18:40:42.058253+0000","last_became_peered":"2023-09-23T18:40:42.058253+0000","last_unstale":"2023-09-23T18:42:58.466402+0000","last_undegraded":"2023-09-23T18:42:58.466402+0000","last_fullsized":"2023-09-23T18:42:58.466402+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":142,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.468002+0000","last_change":"2023-09-23T18:15:08.921629+0000","last_active":"2023-09-23T18:42:58.468002+0000","last_peered":"2023-09-23T18:42:58.468002+0000","last_clean":"2023-09-23T18:42:58.468002+0000","last_became_active":"2023-09-23T18:15:08.921283+0000","last_became_peered":"2023-09-23T18:15:08.921283+0000","last_unstale":"2023-09-23T18:42:58.468002+0000","last_undegraded":"2023-09-23T18:42:58.468002+0000","last_fullsized":"2023-09-23T18:42:58.468002+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"192'2942","reported_seq":2954,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:30.913891+0000","last_change":"2023-09-23T18:42:39.318947+0000","last_active":"2023-09-23T18:43:30.913891+0000","last_peered":"2023-09-23T18:43:30.913891+0000","last_clean":"2023-09-23T18:43:30.913891+0000","last_became_active":"2023-09-23T18:42:39.318579+0000","last_became_peered":"2023-09-23T18:42:39.318579+0000","last_unstale":"2023-09-23T18:43:30.913891+0000","last_undegraded":"2023-09-23T18:43:30.913891+0000","last_fullsized":"2023-09-23T18:43:30.913891+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":2942,"log_dups_size":0,"ondisk_log_size":2942,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192806912,"num_objects":2942,"num_object_clones":0,"num_object_copies":8826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2942,"num_write_kb":188288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'120","reported_seq":653,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.150959+0000","last_change":"2023-09-23T18:33:53.852697+0000","last_active":"2023-09-23T18:42:58.150959+0000","last_peered":"2023-09-23T18:42:58.150959+0000","last_clean":"2023-09-23T18:42:58.150959+0000","last_became_active":"2023-09-23T18:33:53.852210+0000","last_became_peered":"2023-09-23T18:33:53.852210+0000","last_unstale":"2023-09-23T18:42:58.150959+0000","last_undegraded":"2023-09-23T18:42:58.150959+0000","last_fullsized":"2023-09-23T18:42:58.150959+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":120,"log_dups_size":0,"ondisk_log_size":120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":809,"num_read_kb":2749,"num_write":98,"num_write_kb":2087,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":156,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:04.146783+0000","last_change":"2023-09-23T18:39:46.997952+0000","last_active":"2023-09-23T18:43:04.146783+0000","last_peered":"2023-09-23T18:43:04.146783+0000","last_clean":"2023-09-23T18:43:04.146783+0000","last_became_active":"2023-09-23T18:39:46.997589+0000","last_became_peered":"2023-09-23T18:39:46.997589+0000","last_unstale":"2023-09-23T18:43:04.146783+0000","last_undegraded":"2023-09-23T18:43:04.146783+0000","last_fullsized":"2023-09-23T18:43:04.146783+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:09.887469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":186,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.056410+0000","last_change":"2023-09-23T18:39:46.996973+0000","last_active":"2023-09-23T18:42:58.056410+0000","last_peered":"2023-09-23T18:42:58.056410+0000","last_clean":"2023-09-23T18:42:58.056410+0000","last_became_active":"2023-09-23T18:39:46.996633+0000","last_became_peered":"2023-09-23T18:39:46.996633+0000","last_unstale":"2023-09-23T18:42:58.056410+0000","last_undegraded":"2023-09-23T18:42:58.056410+0000","last_fullsized":"2023-09-23T18:42:58.056410+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:08:05.607474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"192'3075","reported_seq":3089,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:32.493250+0000","last_change":"2023-09-23T18:42:39.313363+0000","last_active":"2023-09-23T18:43:32.493250+0000","last_peered":"2023-09-23T18:43:32.493250+0000","last_clean":"2023-09-23T18:43:32.493250+0000","last_became_active":"2023-09-23T18:42:39.313043+0000","last_became_peered":"2023-09-23T18:42:39.313043+0000","last_unstale":"2023-09-23T18:43:32.493250+0000","last_undegraded":"2023-09-23T18:43:32.493250+0000","last_fullsized":"2023-09-23T18:43:32.493250+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":3075,"log_dups_size":0,"ondisk_log_size":3075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:07.928055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201523200,"num_objects":3075,"num_object_clones":0,"num_object_copies":9225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3075,"num_write_kb":196800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'78","reported_seq":428,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.151812+0000","last_change":"2023-09-23T18:26:34.992345+0000","last_active":"2023-09-23T18:42:58.151812+0000","last_peered":"2023-09-23T18:42:58.151812+0000","last_clean":"2023-09-23T18:42:58.151812+0000","last_became_active":"2023-09-23T18:26:34.991774+0000","last_became_peered":"2023-09-23T18:26:34.991774+0000","last_unstale":"2023-09-23T18:42:58.151812+0000","last_undegraded":"2023-09-23T18:42:58.151812+0000","last_fullsized":"2023-09-23T18:42:58.151812+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":78,"log_dups_size":0,"ondisk_log_size":78,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":428,"num_read_kb":1125,"num_write":73,"num_write_kb":431,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":134,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.055995+0000","last_change":"2023-09-23T18:39:46.990343+0000","last_active":"2023-09-23T18:42:58.055995+0000","last_peered":"2023-09-23T18:42:58.055995+0000","last_clean":"2023-09-23T18:42:58.055995+0000","last_became_active":"2023-09-23T18:39:46.989904+0000","last_became_peered":"2023-09-23T18:39:46.989904+0000","last_unstale":"2023-09-23T18:42:58.055995+0000","last_undegraded":"2023-09-23T18:42:58.055995+0000","last_fullsized":"2023-09-23T18:42:58.055995+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:15.638308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":153,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:04.147831+0000","last_change":"2023-09-23T18:33:54.852309+0000","last_active":"2023-09-23T18:43:04.147831+0000","last_peered":"2023-09-23T18:43:04.147831+0000","last_clean":"2023-09-23T18:43:04.147831+0000","last_became_active":"2023-09-23T18:33:54.850255+0000","last_became_peered":"2023-09-23T18:33:54.850255+0000","last_unstale":"2023-09-23T18:43:04.147831+0000","last_undegraded":"2023-09-23T18:43:04.147831+0000","last_fullsized":"2023-09-23T18:43:04.147831+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:54.148992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"192'2995","reported_seq":3007,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:31.246977+0000","last_change":"2023-09-23T18:42:39.312939+0000","last_active":"2023-09-23T18:43:31.246977+0000","last_peered":"2023-09-23T18:43:31.246977+0000","last_clean":"2023-09-23T18:43:31.246977+0000","last_became_active":"2023-09-23T18:42:39.312156+0000","last_became_peered":"2023-09-23T18:42:39.312156+0000","last_unstale":"2023-09-23T18:43:31.246977+0000","last_undegraded":"2023-09-23T18:43:31.246977+0000","last_fullsized":"2023-09-23T18:43:31.246977+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":2995,"log_dups_size":0,"ondisk_log_size":2995,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:56:04.264560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196280320,"num_objects":2995,"num_object_clones":0,"num_object_copies":8985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2995,"num_write_kb":191680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":165,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:04.148789+0000","last_change":"2023-09-23T18:28:03.298805+0000","last_active":"2023-09-23T18:43:04.148789+0000","last_peered":"2023-09-23T18:43:04.148789+0000","last_clean":"2023-09-23T18:43:04.148789+0000","last_became_active":"2023-09-23T18:28:03.298437+0000","last_became_peered":"2023-09-23T18:28:03.298437+0000","last_unstale":"2023-09-23T18:43:04.148789+0000","last_undegraded":"2023-09-23T18:43:04.148789+0000","last_fullsized":"2023-09-23T18:43:04.148789+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:27.831669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'181","reported_seq":665,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.054625+0000","last_change":"2023-09-23T18:33:53.854528+0000","last_active":"2023-09-23T18:42:58.054625+0000","last_peered":"2023-09-23T18:42:58.054625+0000","last_clean":"2023-09-23T18:42:58.054625+0000","last_became_active":"2023-09-23T18:33:53.851883+0000","last_became_peered":"2023-09-23T18:33:53.851883+0000","last_unstale":"2023-09-23T18:42:58.054625+0000","last_undegraded":"2023-09-23T18:42:58.054625+0000","last_fullsized":"2023-09-23T18:42:58.054625+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":181,"log_dups_size":0,"ondisk_log_size":181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:43.888844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":909,"num_read_kb":734,"num_write":148,"num_write_kb":75,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":100,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.150509+0000","last_change":"2023-09-23T18:39:46.992015+0000","last_active":"2023-09-23T18:42:58.150509+0000","last_peered":"2023-09-23T18:42:58.150509+0000","last_clean":"2023-09-23T18:42:58.150509+0000","last_became_active":"2023-09-23T18:39:46.991605+0000","last_became_peered":"2023-09-23T18:39:46.991605+0000","last_unstale":"2023-09-23T18:42:58.150509+0000","last_undegraded":"2023-09-23T18:42:58.150509+0000","last_fullsized":"2023-09-23T18:42:58.150509+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"192'2984","reported_seq":2998,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:31.258403+0000","last_change":"2023-09-23T18:42:39.309814+0000","last_active":"2023-09-23T18:43:31.258403+0000","last_peered":"2023-09-23T18:43:31.258403+0000","last_clean":"2023-09-23T18:43:31.258403+0000","last_became_active":"2023-09-23T18:42:39.308969+0000","last_became_peered":"2023-09-23T18:42:39.308969+0000","last_unstale":"2023-09-23T18:43:31.258403+0000","last_undegraded":"2023-09-23T18:43:31.258403+0000","last_fullsized":"2023-09-23T18:43:31.258403+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":2984,"log_dups_size":0,"ondisk_log_size":2984,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:44.419979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195559424,"num_objects":2984,"num_object_clones":0,"num_object_copies":8952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2984,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2984,"num_write_kb":190976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"77'112","reported_seq":725,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.467819+0000","last_change":"2023-09-23T18:40:45.086639+0000","last_active":"2023-09-23T18:42:58.467819+0000","last_peered":"2023-09-23T18:42:58.467819+0000","last_clean":"2023-09-23T18:42:58.467819+0000","last_became_active":"2023-09-23T18:40:45.086119+0000","last_became_peered":"2023-09-23T18:40:45.086119+0000","last_unstale":"2023-09-23T18:42:58.467819+0000","last_undegraded":"2023-09-23T18:42:58.467819+0000","last_fullsized":"2023-09-23T18:42:58.467819+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":893,"num_read_kb":3172,"num_write":69,"num_write_kb":2237,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":165,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.056547+0000","last_change":"2023-09-23T18:33:53.851421+0000","last_active":"2023-09-23T18:42:58.056547+0000","last_peered":"2023-09-23T18:42:58.056547+0000","last_clean":"2023-09-23T18:42:58.056547+0000","last_became_active":"2023-09-23T18:33:53.851012+0000","last_became_peered":"2023-09-23T18:33:53.851012+0000","last_unstale":"2023-09-23T18:42:58.056547+0000","last_undegraded":"2023-09-23T18:42:58.056547+0000","last_fullsized":"2023-09-23T18:42:58.056547+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:03.971054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":151,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.268488+0000","last_change":"2023-09-23T18:32:26.805472+0000","last_active":"2023-09-23T18:42:58.268488+0000","last_peered":"2023-09-23T18:42:58.268488+0000","last_clean":"2023-09-23T18:42:58.268488+0000","last_became_active":"2023-09-23T18:32:26.805078+0000","last_became_peered":"2023-09-23T18:32:26.805078+0000","last_unstale":"2023-09-23T18:42:58.268488+0000","last_undegraded":"2023-09-23T18:42:58.268488+0000","last_fullsized":"2023-09-23T18:42:58.268488+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":134,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.054650+0000","last_change":"2023-09-23T18:26:34.994176+0000","last_active":"2023-09-23T18:42:58.054650+0000","last_peered":"2023-09-23T18:42:58.054650+0000","last_clean":"2023-09-23T18:42:58.054650+0000","last_became_active":"2023-09-23T18:26:34.993829+0000","last_became_peered":"2023-09-23T18:26:34.993829+0000","last_unstale":"2023-09-23T18:42:58.054650+0000","last_undegraded":"2023-09-23T18:42:58.054650+0000","last_fullsized":"2023-09-23T18:42:58.054650+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:27.111857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":171,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.056091+0000","last_change":"2023-09-23T18:40:42.053012+0000","last_active":"2023-09-23T18:42:58.056091+0000","last_peered":"2023-09-23T18:42:58.056091+0000","last_clean":"2023-09-23T18:42:58.056091+0000","last_became_active":"2023-09-23T18:40:42.052655+0000","last_became_peered":"2023-09-23T18:40:42.052655+0000","last_unstale":"2023-09-23T18:42:58.056091+0000","last_undegraded":"2023-09-23T18:42:58.056091+0000","last_fullsized":"2023-09-23T18:42:58.056091+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:33:51.009383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":115,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.268836+0000","last_change":"2023-09-23T18:33:54.847172+0000","last_active":"2023-09-23T18:42:58.268836+0000","last_peered":"2023-09-23T18:42:58.268836+0000","last_clean":"2023-09-23T18:42:58.268836+0000","last_became_active":"2023-09-23T18:33:54.846751+0000","last_became_peered":"2023-09-23T18:33:54.846751+0000","last_unstale":"2023-09-23T18:42:58.268836+0000","last_undegraded":"2023-09-23T18:42:58.268836+0000","last_fullsized":"2023-09-23T18:42:58.268836+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":187,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.269201+0000","last_change":"2023-09-23T18:33:53.841232+0000","last_active":"2023-09-23T18:42:58.269201+0000","last_peered":"2023-09-23T18:42:58.269201+0000","last_clean":"2023-09-23T18:42:58.269201+0000","last_became_active":"2023-09-23T18:33:53.840749+0000","last_became_peered":"2023-09-23T18:33:53.840749+0000","last_unstale":"2023-09-23T18:42:58.269201+0000","last_undegraded":"2023-09-23T18:42:58.269201+0000","last_fullsized":"2023-09-23T18:42:58.269201+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":115,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.269194+0000","last_change":"2023-09-23T18:33:54.858971+0000","last_active":"2023-09-23T18:42:58.269194+0000","last_peered":"2023-09-23T18:42:58.269194+0000","last_clean":"2023-09-23T18:42:58.269194+0000","last_became_active":"2023-09-23T18:33:54.845921+0000","last_became_peered":"2023-09-23T18:33:54.845921+0000","last_unstale":"2023-09-23T18:42:58.269194+0000","last_undegraded":"2023-09-23T18:42:58.269194+0000","last_fullsized":"2023-09-23T18:42:58.269194+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":172,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.268645+0000","last_change":"2023-09-23T18:28:03.302261+0000","last_active":"2023-09-23T18:42:58.268645+0000","last_peered":"2023-09-23T18:42:58.268645+0000","last_clean":"2023-09-23T18:42:58.268645+0000","last_became_active":"2023-09-23T18:28:03.301911+0000","last_became_peered":"2023-09-23T18:28:03.301911+0000","last_unstale":"2023-09-23T18:42:58.268645+0000","last_undegraded":"2023-09-23T18:42:58.268645+0000","last_fullsized":"2023-09-23T18:42:58.268645+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":40,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.466494+0000","last_change":"2023-09-23T18:40:42.049209+0000","last_active":"2023-09-23T18:42:58.466494+0000","last_peered":"2023-09-23T18:42:58.466494+0000","last_clean":"2023-09-23T18:42:58.466494+0000","last_became_active":"2023-09-23T18:40:42.048654+0000","last_became_peered":"2023-09-23T18:40:42.048654+0000","last_unstale":"2023-09-23T18:42:58.466494+0000","last_undegraded":"2023-09-23T18:42:58.466494+0000","last_fullsized":"2023-09-23T18:42:58.466494+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":114,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:04.144792+0000","last_change":"2023-09-23T18:40:42.051196+0000","last_active":"2023-09-23T18:43:04.144792+0000","last_peered":"2023-09-23T18:43:04.144792+0000","last_clean":"2023-09-23T18:43:04.144792+0000","last_became_active":"2023-09-23T18:40:42.050613+0000","last_became_peered":"2023-09-23T18:40:42.050613+0000","last_unstale":"2023-09-23T18:43:04.144792+0000","last_undegraded":"2023-09-23T18:43:04.144792+0000","last_fullsized":"2023-09-23T18:43:04.144792+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:21.750773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":181,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:43:04.147522+0000","last_change":"2023-09-23T18:39:47.006553+0000","last_active":"2023-09-23T18:43:04.147522+0000","last_peered":"2023-09-23T18:43:04.147522+0000","last_clean":"2023-09-23T18:43:04.147522+0000","last_became_active":"2023-09-23T18:39:46.996464+0000","last_became_peered":"2023-09-23T18:39:46.996464+0000","last_unstale":"2023-09-23T18:43:04.147522+0000","last_undegraded":"2023-09-23T18:43:04.147522+0000","last_fullsized":"2023-09-23T18:43:04.147522+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:02.274848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":171,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T18:42:58.268185+0000","last_change":"2023-09-23T18:33:54.847156+0000","last_active":"2023-09-23T18:42:58.268185+0000","last_peered":"2023-09-23T18:42:58.268185+0000","last_clean":"2023-09-23T18:42:58.268185+0000","last_became_active":"2023-09-23T18:33:54.846765+0000","last_became_peered":"2023-09-23T18:33:54.846765+0000","last_unstale":"2023-09-23T18:42:58.268185+0000","last_undegraded":"2023-09-23T18:42:58.268185+0000","last_fullsized":"2023-09-23T18:42:58.268185+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":3149135872,"num_objects":48052,"num_object_clones":0,"num_object_copies":144156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":48052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":48052,"num_write_kb":3075328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9439215616,"data_stored":9439215616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":48052,"ondisk_log_size":48052,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":1200,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26886,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5937,"num_read_kb":13190,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5621,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430588,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1237788,"kb_used_data":936468,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92507364,"statfs":{"total":95995035648,"available":94727540736,"internally_reserved":0,"allocated":958943232,"data_stored":958368286,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:43:11 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":2.6899999999999999,"15min":2.044},"min":{"1min":0.33000000000000002,"5min":0.30199999999999999,"15min":0.252},"max":{"1min":1.8029999999999999,"5min":65.412999999999997,"15min":65.412999999999997},"last":4.335},{"interface":"front","average":{"1min":0.77600000000000002,"5min":2.6880000000000002,"15min":2.0529999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.22900000000000001},"max":{"1min":1.6000000000000001,"5min":65.379999999999995,"15min":65.379999999999995},"last":6.0590000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:43:11 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":2.028,"15min":1.4830000000000001},"min":{"1min":0.40100000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.282,"5min":41.616,"15min":41.616},"last":3.331},{"interface":"front","average":{"1min":0.86399999999999999,"5min":2.2989999999999999,"15min":1.694},"min":{"1min":0.35199999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.204,"5min":41.654000000000003,"15min":41.654000000000003},"last":3.323}]},{"osd":2,"last update":"Sat Sep 23 18:43:04 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":1.802,"15min":1.5329999999999999},"min":{"1min":0.71399999999999997,"5min":0.311,"15min":0.254},"max":{"1min":1.732,"5min":44.392000000000003,"15min":44.392000000000003},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.88800000000000001,"5min":1.8979999999999999,"15min":1.522},"min":{"1min":0.48199999999999998,"5min":0.33600000000000002,"15min":0.29999999999999999},"max":{"1min":1.2490000000000001,"5min":42.991999999999997,"15min":42.991999999999997},"last":0.94499999999999995}]},{"osd":3,"last update":"Sat Sep 23 18:42:31 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":3.044,"15min":1.6639999999999999},"min":{"1min":0.64800000000000002,"5min":0.34599999999999997,"15min":0.32800000000000001},"max":{"1min":2.6989999999999998,"5min":70.216999999999999,"15min":70.216999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":1.5009999999999999,"5min":3.5259999999999998,"15min":1.8300000000000001},"min":{"1min":0.624,"5min":0.314,"15min":0.314},"max":{"1min":8.7409999999999997,"5min":69.566000000000003,"15min":69.566000000000003},"last":0.46300000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:43:11 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":0.95799999999999996,"15min":1.0329999999999999},"min":{"1min":0.38500000000000001,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":1.3180000000000001,"5min":5.1230000000000002,"15min":6.3529999999999998},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.95799999999999996,"5min":1.143,"15min":1.147},"min":{"1min":0.52800000000000002,"5min":0.35099999999999998,"15min":0.309},"max":{"1min":1.4830000000000001,"5min":7.6029999999999998,"15min":11.119},"last":0.40500000000000003}]},{"osd":5,"last update":"Sat Sep 23 18:43:11 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":1.216,"15min":1.103},"min":{"1min":0.55700000000000005,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.3620000000000001,"5min":8.7210000000000001,"15min":8.7210000000000001},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.0129999999999999,"5min":0.995,"15min":1.081},"min":{"1min":0.45100000000000001,"5min":0.42999999999999999,"15min":0.26800000000000002},"max":{"1min":1.3180000000000001,"5min":5.0129999999999999,"15min":7.46},"last":0.41899999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:43:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":1.2170000000000001,"15min":1.1279999999999999},"min":{"1min":0.48699999999999999,"5min":0.41199999999999998,"15min":0.35299999999999998},"max":{"1min":1.4179999999999999,"5min":5.0899999999999999,"15min":5.5129999999999999},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.1020000000000001,"5min":1.256,"15min":1.151},"min":{"1min":0.55900000000000005,"5min":0.41099999999999998,"15min":0.36499999999999999},"max":{"1min":1.673,"5min":9.2850000000000001,"15min":9.2850000000000001},"last":0.48599999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626820,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1412308,"kb_used_data":1115084,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92332844,"statfs":{"total":95995035648,"available":94548832256,"internally_reserved":0,"allocated":1141846016,"data_stored":1141279374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":1.125,"15min":1.9850000000000001},"min":{"1min":0.48099999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.585,"5min":6.758,"15min":182.655},"last":2.04},{"interface":"front","average":{"1min":1.014,"5min":1.2150000000000001,"15min":2.081},"min":{"1min":0.57299999999999995,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.617,"5min":6.7930000000000001,"15min":182.58799999999999},"last":1.9630000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:43:07 2023","interfaces":[{"interface":"back","average":{"1min":1.246,"5min":1.5029999999999999,"15min":1.0269999999999999},"min":{"1min":0.51500000000000001,"5min":0.308,"15min":0.308},"max":{"1min":8.0839999999999996,"5min":18.420999999999999,"15min":18.420999999999999},"last":1.5640000000000001},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.9950000000000001,"15min":1.339},"min":{"1min":0.499,"5min":0.30599999999999999,"15min":0.30299999999999999},"max":{"1min":1.54,"5min":21.193000000000001,"15min":21.193000000000001},"last":0.77900000000000003}]},{"osd":2,"last update":"Sat Sep 23 18:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":1.7689999999999999,"15min":2.3140000000000001},"min":{"1min":0.47399999999999998,"5min":0.33500000000000002,"15min":0.28100000000000003},"max":{"1min":1.2769999999999999,"5min":10.99,"15min":122.242},"last":1.01},{"interface":"front","average":{"1min":0.88900000000000001,"5min":1.869,"15min":2.351},"min":{"1min":0.45400000000000001,"5min":0.34899999999999998,"15min":0.28799999999999998},"max":{"1min":1.165,"5min":13.959,"15min":122.32899999999999},"last":0.83699999999999997}]},{"osd":3,"last update":"Sat Sep 23 18:43:29 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":2.306,"15min":2.2160000000000002},"min":{"1min":0.502,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.476,"5min":89.897999999999996,"15min":123.52200000000001},"last":2.2749999999999999},{"interface":"front","average":{"1min":1.1120000000000001,"5min":2.7010000000000001,"15min":2.395},"min":{"1min":0.42199999999999999,"5min":0.39800000000000002,"15min":0.36099999999999999},"max":{"1min":3.339,"5min":89.933999999999997,"15min":123.605},"last":3.339}]},{"osd":4,"last update":"Sat Sep 23 18:43:29 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":1.484,"15min":1.1299999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.29999999999999999},"max":{"1min":1.53,"5min":15.496,"15min":15.496},"last":1.0900000000000001},{"interface":"front","average":{"1min":0.89300000000000002,"5min":1.306,"15min":1.075},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.34899999999999998},"max":{"1min":2.1059999999999999,"5min":14.875999999999999,"15min":14.875999999999999},"last":2.1059999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:43:29 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":1.181,"15min":1.028},"min":{"1min":0.44900000000000001,"5min":0.39100000000000001,"15min":0.32000000000000001},"max":{"1min":1.4850000000000001,"5min":7.4930000000000003,"15min":7.9240000000000004},"last":1.135},{"interface":"front","average":{"1min":0.92800000000000005,"5min":0.91900000000000004,"15min":0.92800000000000005},"min":{"1min":0.47499999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":2.1640000000000001,"5min":3.8220000000000001,"15min":4.5270000000000001},"last":2.1640000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:43:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":1.4159999999999999,"15min":1.139},"min":{"1min":0.55800000000000005,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.4419999999999999,"5min":7.9320000000000004,"15min":7.9320000000000004},"last":1.4970000000000001},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.708,"15min":1.4099999999999999},"min":{"1min":0.60999999999999999,"5min":0.438,"15min":0.39100000000000001},"max":{"1min":1.3089999999999999,"5min":7.8209999999999997,"15min":9.1129999999999995},"last":2.6829999999999998}]}]},{"osd":1,"up_from":123,"seq":528280977525,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2212436,"kb_used_data":1937932,"kb_used_omap":7,"kb_used_meta":274488,"kb_avail":91532716,"statfs":{"total":95995035648,"available":93729501184,"internally_reserved":0,"allocated":1984442368,"data_stored":1983527990,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":281076494},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:43:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8680000000000001,"5min":1.8080000000000001,"15min":5.8630000000000004},"min":{"1min":0.434,"5min":0.186,"15min":0.186},"max":{"1min":19.385000000000002,"5min":19.385000000000002,"15min":173.41800000000001},"last":1.681},{"interface":"front","average":{"1min":1.097,"5min":2.0760000000000001,"15min":5.6509999999999998},"min":{"1min":0.44900000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":7.3019999999999996,"5min":11.955,"15min":166.42599999999999},"last":1.9019999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:43:10 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":4.024,"15min":3.0139999999999998},"min":{"1min":0.312,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":2.2410000000000001,"5min":129.25899999999999,"15min":129.25899999999999},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.64500000000000002,"5min":4.1529999999999996,"15min":2.907},"min":{"1min":0.39800000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.96499999999999997,"5min":129.35300000000001,"15min":129.35300000000001},"last":1.7529999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:43:10 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":3.0059999999999998,"15min":1.8899999999999999},"min":{"1min":0.47299999999999998,"5min":0.25,"15min":0.25},"max":{"1min":1.1000000000000001,"5min":32.561,"15min":55.014000000000003},"last":1.8779999999999999},{"interface":"front","average":{"1min":0.69199999999999995,"5min":2.9590000000000001,"15min":1.9510000000000001},"min":{"1min":0.373,"5min":0.25,"15min":0.24399999999999999},"max":{"1min":1.0820000000000001,"5min":30.289999999999999,"15min":55.841999999999999},"last":0.53500000000000003}]},{"osd":4,"last update":"Sat Sep 23 18:43:10 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":1.946,"15min":1.4490000000000001},"min":{"1min":0.41399999999999998,"5min":0.308,"15min":0.308},"max":{"1min":1.3480000000000001,"5min":11.763,"15min":11.763},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.387,"5min":1.6200000000000001,"15min":1.474},"min":{"1min":0.66100000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":5.5780000000000003,"5min":11.491,"15min":11.491},"last":0.55600000000000005}]},{"osd":5,"last update":"Sat Sep 23 18:43:10 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.6799999999999999,"15min":1.153},"min":{"1min":0.60199999999999998,"5min":0.36699999999999999,"15min":0.35699999999999998},"max":{"1min":1.1819999999999999,"5min":11.794,"15min":11.794},"last":1.853},{"interface":"front","average":{"1min":0.96199999999999997,"5min":1.871,"15min":1.524},"min":{"1min":0.46500000000000002,"5min":0.371,"15min":0.371},"max":{"1min":1.4179999999999999,"5min":11.733000000000001,"15min":11.733000000000001},"last":2.5030000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:43:10 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.9590000000000001,"15min":1.655},"min":{"1min":0.53100000000000003,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.5189999999999999,"5min":11.776999999999999,"15min":11.776999999999999},"last":2.9020000000000001},{"interface":"front","average":{"1min":1.349,"5min":2.0070000000000001,"15min":1.7370000000000001},"min":{"1min":0.53100000000000003,"5min":0.36099999999999999,"15min":0.35999999999999999},"max":{"1min":5.4779999999999998,"5min":16.687000000000001,"15min":16.687000000000001},"last":0.85799999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:43:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":2.198,"15min":1.9019999999999999},"min":{"1min":0.44,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":2.286,"5min":11.747999999999999,"15min":11.747999999999999},"last":2.472},{"interface":"front","average":{"1min":1.409,"5min":1.7829999999999999,"15min":1.7250000000000001},"min":{"1min":0.64100000000000001,"5min":0.379,"15min":0.379},"max":{"1min":5.5350000000000001,"5min":11.593,"15min":11.593},"last":4.4420000000000002}]}]},{"osd":0,"up_from":88,"seq":377957122254,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2041836,"kb_used_data":1600292,"kb_used_omap":8,"kb_used_meta":441527,"kb_avail":91703316,"statfs":{"total":95995035648,"available":93904195584,"internally_reserved":0,"allocated":1638699008,"data_stored":1637820762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":452124542},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,4,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:43:14 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":2.448,"15min":1.9350000000000001},"min":{"1min":0.33200000000000002,"5min":0.19400000000000001,"15min":0.17100000000000001},"max":{"1min":1.7749999999999999,"5min":38.545999999999999,"15min":38.545999999999999},"last":1.02},{"interface":"front","average":{"1min":1.006,"5min":2.5449999999999999,"15min":2.2570000000000001},"min":{"1min":0.48199999999999998,"5min":0.20999999999999999,"15min":0.20100000000000001},"max":{"1min":6.944,"5min":45.904000000000003,"15min":45.904000000000003},"last":0.99099999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:42:55 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":1.6779999999999999,"15min":2.105},"min":{"1min":0.35799999999999998,"5min":0.23400000000000001,"15min":0.192},"max":{"1min":1.3080000000000001,"5min":10.234999999999999,"15min":108.73999999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.67600000000000005,"5min":1.0900000000000001,"15min":1.8859999999999999},"min":{"1min":0.48099999999999998,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":1.0069999999999999,"5min":8.4600000000000009,"15min":108.136},"last":0.628}]},{"osd":3,"last update":"Sat Sep 23 18:43:26 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":4.7889999999999997,"15min":3.2269999999999999},"min":{"1min":0.36399999999999999,"5min":0.25900000000000001,"15min":0.216},"max":{"1min":2.7949999999999999,"5min":177.83600000000001,"15min":177.83600000000001},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.84899999999999998,"5min":4.8109999999999999,"15min":3.2189999999999999},"min":{"1min":0.39600000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":2.8420000000000001,"5min":181.71899999999999,"15min":181.71899999999999},"last":0.69999999999999996}]},{"osd":4,"last update":"Sat Sep 23 18:43:14 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":1.417,"15min":1.3180000000000001},"min":{"1min":0.71099999999999997,"5min":0.312,"15min":0.28199999999999997},"max":{"1min":1.345,"5min":9.9450000000000003,"15min":9.9450000000000003},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.872,"5min":1.4019999999999999,"15min":1.472},"min":{"1min":0.36499999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":2.1019999999999999,"5min":6.6150000000000002,"15min":16.486999999999998},"last":0.33000000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:43:14 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":1.127,"15min":1.264},"min":{"1min":0.434,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":2.0760000000000001,"5min":5.3209999999999997,"15min":10.606},"last":0.371},{"interface":"front","average":{"1min":0.93500000000000005,"5min":1.151,"15min":1.248},"min":{"1min":0.504,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.278,"5min":8.2279999999999998,"15min":8.9250000000000007},"last":0.55800000000000005}]},{"osd":6,"last update":"Sat Sep 23 18:43:14 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":1.4710000000000001,"15min":1.421},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.35899999999999999},"max":{"1min":2.0409999999999999,"5min":11.302,"15min":11.302},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.90100000000000002,"5min":1.3460000000000001,"15min":1.341},"min":{"1min":0.45800000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.24,"5min":11.818,"15min":11.818},"last":0.47499999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:43:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":1.363,"15min":1.411},"min":{"1min":0.48099999999999998,"5min":0.40200000000000002,"15min":0.39200000000000002},"max":{"1min":1.3109999999999999,"5min":8.327,"15min":9.0600000000000005},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.96899999999999997,"5min":1.589,"15min":1.4059999999999999},"min":{"1min":0.68200000000000005,"5min":0.42599999999999999,"15min":0.35799999999999998},"max":{"1min":1.194,"5min":10.01,"15min":10.01},"last":0.83799999999999997}]}]},{"osd":2,"up_from":98,"seq":420906795201,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1812728,"kb_used_data":1354928,"kb_used_omap":11,"kb_used_meta":457780,"kb_avail":91932424,"statfs":{"total":95995035648,"available":94138802176,"internally_reserved":0,"allocated":1387446272,"data_stored":1386614541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11949,"internal_metadata":468767059},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:43:03 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":1.252,"15min":1.248},"min":{"1min":0.32000000000000001,"5min":0.184,"15min":0.184},"max":{"1min":2.4449999999999998,"5min":21.498000000000001,"15min":21.498000000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.72199999999999998,"5min":1.377,"15min":1.3320000000000001},"min":{"1min":0.31900000000000001,"5min":0.20300000000000001,"15min":0.17699999999999999},"max":{"1min":4.1900000000000004,"5min":21.484999999999999,"15min":21.484999999999999},"last":1.036}]},{"osd":1,"last update":"Sat Sep 23 18:43:09 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":5.6360000000000001,"15min":3.27},"min":{"1min":0.40100000000000002,"5min":0.314,"15min":0.23799999999999999},"max":{"1min":4.1260000000000003,"5min":184.56299999999999,"15min":184.56299999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":1.1779999999999999,"5min":5.6719999999999997,"15min":3.3109999999999999},"min":{"1min":0.52200000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":5.5789999999999997,"5min":183.922,"15min":183.922},"last":2.4569999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:42:36 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":6.0839999999999996,"15min":2.77},"min":{"1min":0.38,"5min":0.19,"15min":0.19},"max":{"1min":1.1419999999999999,"5min":161.773,"15min":161.773},"last":5.1420000000000003},{"interface":"front","average":{"1min":0.69199999999999995,"5min":5.375,"15min":2.4489999999999998},"min":{"1min":0.45500000000000002,"5min":0.25,"15min":0.249},"max":{"1min":0.97399999999999998,"5min":158.03700000000001,"15min":158.03700000000001},"last":5.1980000000000004}]},{"osd":4,"last update":"Sat Sep 23 18:43:03 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":2.3679999999999999,"15min":1.714},"min":{"1min":0.505,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.2729999999999999,"5min":71.587999999999994,"15min":71.587999999999994},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.91600000000000004,"5min":1.992,"15min":1.599},"min":{"1min":0.435,"5min":0.35499999999999998,"15min":0.33500000000000002},"max":{"1min":1.282,"5min":71.328999999999994,"15min":71.328999999999994},"last":2.548}]},{"osd":5,"last update":"Sat Sep 23 18:43:03 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":1.8740000000000001,"15min":1.415},"min":{"1min":0.46899999999999997,"5min":0.36799999999999999,"15min":0.35999999999999999},"max":{"1min":1.3540000000000001,"5min":71.302000000000007,"15min":71.302000000000007},"last":0.54800000000000004},{"interface":"front","average":{"1min":1.032,"5min":1.9770000000000001,"15min":1.611},"min":{"1min":0.56799999999999995,"5min":0.35999999999999999,"15min":0.316},"max":{"1min":3.964,"5min":71.543999999999997,"15min":71.543999999999997},"last":1.1399999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:43:03 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":2.6160000000000001,"15min":1.7809999999999999},"min":{"1min":0.69299999999999995,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.3580000000000001,"5min":71.460999999999999,"15min":71.460999999999999},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.93500000000000005,"5min":2.1269999999999998,"15min":1.5489999999999999},"min":{"1min":0.54900000000000004,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.349,"5min":71.271000000000001,"15min":71.271000000000001},"last":2.5739999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:43:03 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":2.496,"15min":1.798},"min":{"1min":0.65300000000000002,"5min":0.38,"15min":0.34799999999999998},"max":{"1min":1.3320000000000001,"5min":71.622,"15min":71.622},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.998,"5min":2.2010000000000001,"15min":1.762},"min":{"1min":0.495,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.4239999999999999,"5min":71.575000000000003,"15min":71.575000000000003},"last":0.56899999999999995}]}]},{"osd":3,"up_from":103,"seq":442381631676,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1509440,"kb_used_data":1143224,"kb_used_omap":4,"kb_used_meta":366203,"kb_avail":92235712,"statfs":{"total":95995035648,"available":94449369088,"internally_reserved":0,"allocated":1170661376,"data_stored":1169821182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4826,"internal_metadata":374992166},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:42:28 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":1.984,"15min":1.413},"min":{"1min":0.377,"5min":0.193,"15min":0.193},"max":{"1min":3.5790000000000002,"5min":12.101000000000001,"15min":14.231},"last":3.6419999999999999},{"interface":"front","average":{"1min":0.68200000000000005,"5min":1.9770000000000001,"15min":1.2829999999999999},"min":{"1min":0.26100000000000001,"5min":0.219,"15min":0.20300000000000001},"max":{"1min":1.222,"5min":10.037000000000001,"15min":10.037000000000001},"last":1.0249999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":1.607,"15min":1.3100000000000001},"min":{"1min":0.218,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":2.0910000000000002,"5min":7.9870000000000001,"15min":12.968999999999999},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.64000000000000001,"5min":1.871,"15min":1.496},"min":{"1min":0.29499999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.1850000000000001,"5min":11.087999999999999,"15min":11.74},"last":3.4390000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:42:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":2.9369999999999998,"15min":2.621},"min":{"1min":0.55900000000000005,"5min":0.23799999999999999,"15min":0.22700000000000001},"max":{"1min":3.71,"5min":18.888000000000002,"15min":187.40000000000001},"last":1.052},{"interface":"front","average":{"1min":1.0680000000000001,"5min":2.0350000000000001,"15min":2.149},"min":{"1min":0.45900000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":3.1800000000000002,"5min":13.15,"15min":187.71700000000001},"last":0.57099999999999995}]},{"osd":4,"last update":"Sat Sep 23 18:42:28 2023","interfaces":[{"interface":"back","average":{"1min":1.2,"5min":1.5309999999999999,"15min":1.1850000000000001},"min":{"1min":0.51000000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.2489999999999997,"5min":8.7989999999999995,"15min":8.7989999999999995},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.97099999999999997,"5min":1.9159999999999999,"15min":1.3200000000000001},"min":{"1min":0.40600000000000003,"5min":0.33900000000000002,"15min":0.28399999999999997},"max":{"1min":1.696,"5min":11.916,"15min":11.916},"last":3.54}]},{"osd":5,"last update":"Sat Sep 23 18:42:28 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.474,"15min":1.1819999999999999},"min":{"1min":0.437,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.5589999999999999,"5min":7.8799999999999999,"15min":7.8799999999999999},"last":3.613},{"interface":"front","average":{"1min":0.93899999999999995,"5min":1.992,"15min":1.343},"min":{"1min":0.495,"5min":0.38800000000000001,"15min":0.36099999999999999},"max":{"1min":1.4330000000000001,"5min":10.18,"15min":10.18},"last":0.99299999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:42:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":2.5840000000000001,"15min":1.552},"min":{"1min":0.59599999999999997,"5min":0.41899999999999998,"15min":0.35599999999999998},"max":{"1min":3.2480000000000002,"5min":11.529,"15min":11.529},"last":0.95199999999999996},{"interface":"front","average":{"1min":1.28,"5min":1.8839999999999999,"15min":1.3120000000000001},"min":{"1min":0.48899999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.0819999999999999,"5min":11.989000000000001,"15min":11.989000000000001},"last":0.60499999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:42:28 2023","interfaces":[{"interface":"back","average":{"1min":1.5760000000000001,"5min":1.823,"15min":1.3620000000000001},"min":{"1min":0.86199999999999999,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":8.5060000000000002,"5min":9.0980000000000008,"15min":9.0980000000000008},"last":0.621},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.72,"15min":1.3220000000000001},"min":{"1min":0.45600000000000002,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.4570000000000001,"5min":9.032,"15min":9.032},"last":3.581}]}]},{"osd":4,"up_from":30,"seq":128849019286,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1428884,"kb_used_data":1127564,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92316268,"statfs":{"total":95995035648,"available":94531858432,"internally_reserved":0,"allocated":1154625536,"data_stored":1154058700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:43:00 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":1.667,"15min":1.758},"min":{"1min":0.53300000000000003,"5min":0.27400000000000002,"15min":0.253},"max":{"1min":1.175,"5min":53.295000000000002,"15min":82.623000000000005},"last":1.125},{"interface":"front","average":{"1min":0.89500000000000002,"5min":1.5009999999999999,"15min":1.6080000000000001},"min":{"1min":0.61099999999999999,"5min":0.29899999999999999,"15min":0.23799999999999999},"max":{"1min":1.2330000000000001,"5min":53.356000000000002,"15min":82.638000000000005},"last":1.1379999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:43:04 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":2.1600000000000001,"15min":1.4950000000000001},"min":{"1min":0.68600000000000005,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.2230000000000001,"5min":53.457000000000001,"15min":53.457000000000001},"last":0.89100000000000001},{"interface":"front","average":{"1min":0.93100000000000005,"5min":2.1949999999999998,"15min":1.671},"min":{"1min":0.52200000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.2490000000000001,"5min":53.395000000000003,"15min":53.395000000000003},"last":0.88200000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:43:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":2.4079999999999999,"15min":1.6240000000000001},"min":{"1min":0.68400000000000005,"5min":0.34000000000000002,"15min":0.28799999999999998},"max":{"1min":1.3520000000000001,"5min":53.249000000000002,"15min":53.249000000000002},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.93899999999999995,"5min":2.7290000000000001,"15min":1.847},"min":{"1min":0.63200000000000001,"5min":0.34499999999999997,"15min":0.309},"max":{"1min":1.2,"5min":53.220999999999997,"15min":53.220999999999997},"last":0.53200000000000003}]},{"osd":3,"last update":"Sat Sep 23 18:42:39 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":3.3660000000000001,"15min":1.9419999999999999},"min":{"1min":0.499,"5min":0.39000000000000001,"15min":0.32900000000000001},"max":{"1min":5.101,"5min":72.075999999999993,"15min":72.075999999999993},"last":0.77000000000000002},{"interface":"front","average":{"1min":1.3320000000000001,"5min":3.0619999999999998,"15min":1.9450000000000001},"min":{"1min":0.65600000000000003,"5min":0.36099999999999999,"15min":0.34000000000000002},"max":{"1min":6.9429999999999996,"5min":71.596999999999994,"15min":71.596999999999994},"last":0.432}]},{"osd":5,"last update":"Sat Sep 23 18:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":1.0449999999999999,"15min":1.1319999999999999},"min":{"1min":0.51300000000000001,"5min":0.307,"15min":0.29899999999999999},"max":{"1min":1.2909999999999999,"5min":6.9249999999999998,"15min":7.819},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.95599999999999996,"5min":1.1299999999999999,"15min":1.1200000000000001},"min":{"1min":0.60999999999999999,"5min":0.33500000000000002,"15min":0.28399999999999997},"max":{"1min":1.224,"5min":7.1440000000000001,"15min":7.1440000000000001},"last":0.47499999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:43:22 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":1.1950000000000001,"15min":1.1970000000000001},"min":{"1min":0.41399999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.3079999999999998,"5min":6.9630000000000001,"15min":7.8419999999999996},"last":0.64800000000000002},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.1140000000000001,"15min":1.071},"min":{"1min":0.34399999999999997,"5min":0.32100000000000001,"15min":0.26100000000000001},"max":{"1min":2.4910000000000001,"5min":6.8399999999999999,"15min":6.8399999999999999},"last":0.55200000000000005}]},{"osd":7,"last update":"Sat Sep 23 18:43:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1499999999999999,"5min":1.9319999999999999,"15min":1.5580000000000001},"min":{"1min":0.745,"5min":0.35299999999999998,"15min":0.311},"max":{"1min":1.4510000000000001,"5min":12.294,"15min":13.491},"last":0.753},{"interface":"front","average":{"1min":1.109,"5min":2.085,"15min":1.7050000000000001},"min":{"1min":0.84099999999999997,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.371,"5min":10.507,"15min":17.062999999999999},"last":0.72199999999999998}]}]},{"osd":5,"up_from":0,"seq":154618823054,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4224,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7725,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3429,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1643905024,"data_stored":1643905024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2027749376,"data_stored":2027749376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1284505600,"data_stored":1284505600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1172045824,"data_stored":1172045824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1179058176,"data_stored":1179058176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1160642560,"data_stored":1160642560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":971309056,"data_stored":971309056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:43:38.163 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T18:43:38.164 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_2 pg_num 26 2023-09-23T18:43:38.166 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 36 16 55730 55714 96.7126 71.5 0.0152166 0.0103314 2023-09-23T18:43:38.183 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:38.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:38.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:38.291 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:38.328 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:38.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:38.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:38.432 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:38.466 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:38.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:38.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:38.568 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:38.597 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:38.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:38.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:38.648 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:43:38.700 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:38.736 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:38.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:38.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:38.842 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:38.869 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:38.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:38.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:38.974 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:38.998 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:39.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:39.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:39.029 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:39 smithi098 bash[76817]: cluster 2023-09-23T18:43:37.968357+0000 mgr.y (mgr.26054) 762 : cluster 0 pgmap v771: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T18:43:39.030 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:39 smithi098 bash[79516]: cluster 2023-09-23T18:43:37.968357+0000 mgr.y (mgr.26054) 762 : cluster 0 pgmap v771: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T18:43:39.101 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:39.135 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:39.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:39.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:39.166 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 37 16 56829 56813 95.9551 68.6875 0.00667879 0.0104155 2023-09-23T18:43:39.242 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:39.277 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:39.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:39 smithi173 bash[59422]: cluster 2023-09-23T18:43:37.968357+0000 mgr.y (mgr.26054) 762 : cluster 0 pgmap v771: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T18:43:39.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:39.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:39.381 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:39.407 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:39.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:39.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:39.511 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:39.541 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:39.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:39.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:39.646 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:39.671 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:39.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:39.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:39.775 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:39.813 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:39.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:39.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:39.924 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:39.955 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:39.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:39.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:40.061 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:40.088 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:40.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:40.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:40.094 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:43:40.166 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 38 16 57888 57872 95.1716 66.1875 0.0143663 0.0105 2023-09-23T18:43:40.172 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:43:40.192 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:40.272 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:40.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:40.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:40.377 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:40.407 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:40.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:40.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:40.510 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:40.550 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:40.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:40.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:40.654 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:40.664 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:40 smithi173 bash[59422]: audit 2023-09-23T18:43:39.618780+0000 mon.a (mon.0) 1026 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:43:40.664 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:40 smithi173 bash[59422]: audit 2023-09-23T18:43:39.620303+0000 mon.a (mon.0) 1027 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:43:40.664 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:40 smithi173 bash[59422]: cluster 2023-09-23T18:43:39.968897+0000 mgr.y (mgr.26054) 763 : cluster 0 pgmap v772: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-09-23T18:43:40.684 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:40.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:40.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:40.790 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:40.814 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:40.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:40.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:40.919 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:40.928 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:40 smithi098 bash[79516]: audit 2023-09-23T18:43:39.618780+0000 mon.a (mon.0) 1026 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:43:40.928 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:40 smithi098 bash[79516]: audit 2023-09-23T18:43:39.620303+0000 mon.a (mon.0) 1027 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:43:40.928 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:40 smithi098 bash[79516]: cluster 2023-09-23T18:43:39.968897+0000 mgr.y (mgr.26054) 763 : cluster 0 pgmap v772: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-09-23T18:43:40.928 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:40 smithi098 bash[76817]: audit 2023-09-23T18:43:39.618780+0000 mon.a (mon.0) 1026 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:43:40.928 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:40 smithi098 bash[76817]: audit 2023-09-23T18:43:39.620303+0000 mon.a (mon.0) 1027 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:43:40.928 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:40 smithi098 bash[76817]: cluster 2023-09-23T18:43:39.968897+0000 mgr.y (mgr.26054) 763 : cluster 0 pgmap v772: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-09-23T18:43:40.947 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:40.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:40.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:41.052 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:41.080 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:41.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:41.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:41.166 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 39 15 58926 58911 94.3962 64.9375 0.0103385 0.0105878 2023-09-23T18:43:41.183 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:41.219 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:41.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:41.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:41.324 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:41.353 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:41.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:41.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:41.457 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:41.483 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:41.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:41.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:41.589 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:41.620 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:41.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:41.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:41.728 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:41.757 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:41.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:41.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:41.862 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:41.885 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:41.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:41.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:41.988 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:42.017 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:42.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:42.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:42.121 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:42.147 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:42.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:42.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:42.166 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:2023-09-23T18:43:42.165821+0000 min lat: 0.00125843 max lat: 0.155252 avg lat: 0.0106614 2023-09-23T18:43:42.166 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:43:42.166 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 40 16 60022 60006 93.7471 68.4375 0.013432 0.0106614 2023-09-23T18:43:42.253 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:42.284 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:42.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:42.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:42.388 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:42.412 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:42.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:42.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:42.515 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:42.543 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:42.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:42.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:42.647 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:42.689 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:42.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:42.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:42.795 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:42.821 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:42.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:42.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:42.924 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:42.955 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:42.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:42.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:43.060 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:43.066 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:43:43.068 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:43 smithi173 bash[59422]: cluster 2023-09-23T18:43:41.969606+0000 mgr.y (mgr.26054) 764 : cluster 0 pgmap v773: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-09-23T18:43:43.082 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:43.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:43.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:43.166 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 41 16 61072 61056 93.0611 65.625 0.00833285 0.0107397 2023-09-23T18:43:43.185 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:43.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:43 smithi098 bash[76817]: cluster 2023-09-23T18:43:41.969606+0000 mgr.y (mgr.26054) 764 : cluster 0 pgmap v773: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-09-23T18:43:43.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:43 smithi098 bash[79516]: cluster 2023-09-23T18:43:41.969606+0000 mgr.y (mgr.26054) 764 : cluster 0 pgmap v773: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-09-23T18:43:43.210 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:43.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:43.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:43.315 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:43.349 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:43.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:43.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:43.455 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:43.484 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:43.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:43.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:43.587 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:43.610 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:43.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:43.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:43.713 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:43.752 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:43.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:43.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:43.855 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:43.881 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:43.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:43.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:43.985 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:44.011 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:44.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:44.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:44.118 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:44.142 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:44.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:44.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:44.166 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 42 16 62115 62099 92.3973 65.1875 0.00254658 0.0108104 2023-09-23T18:43:44.245 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:44.278 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:44.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:44.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:44.381 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:44.405 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:44.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:44.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:44.510 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:44.527 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:44.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:44.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:44.630 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:44.661 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:44.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:44.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:44.754 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:43:44.755 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:43:44.768 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:44.796 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:44.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:44.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:44.900 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:44.930 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:44.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:44.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:45.033 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:45.042 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:45 smithi098 bash[79516]: cluster 2023-09-23T18:43:43.970137+0000 mgr.y (mgr.26054) 765 : cluster 0 pgmap v774: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T18:43:45.042 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:45 smithi098 bash[76817]: cluster 2023-09-23T18:43:43.970137+0000 mgr.y (mgr.26054) 765 : cluster 0 pgmap v774: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T18:43:45.073 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:45.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:45.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:45.166 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 43 16 62894 62878 91.3807 48.6875 0.00409875 0.0109197 2023-09-23T18:43:45.178 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:45.209 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:45.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:45.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:45.312 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:45.320 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:45 smithi173 bash[59422]: cluster 2023-09-23T18:43:43.970137+0000 mgr.y (mgr.26054) 765 : cluster 0 pgmap v774: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T18:43:45.342 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:45.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:45.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:45.448 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:45.473 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:45.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:45.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:45.578 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:45.617 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:45.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:45.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:45.720 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:45.751 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:45.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:45.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:45.855 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:45.883 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:45.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:45.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:45.987 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:46.009 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:43:46.009 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:43:46.029 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:46.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:46.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:46.134 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:46.166 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:46.167 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 44 16 63674 63658 90.4117 48.75 0.0100912 0.0110288 2023-09-23T18:43:46.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:46.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:46.269 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:46.278 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:43:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:43:46] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.33.4" 2023-09-23T18:43:46.297 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:46.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:46.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:46.402 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:46.426 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:46.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:46.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:46.531 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:46.564 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:46.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:46.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:46.668 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:46.693 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:46.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:46.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:46.797 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:46.825 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:46.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:46.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:46.929 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:46.953 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:46.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:46.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:47.056 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:47.167 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 45 16 64570 64554 89.6468 56 0.0190547 0.011149 2023-09-23T18:43:47.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:47 smithi098 bash[79516]: cluster 2023-09-23T18:43:45.970574+0000 mgr.y (mgr.26054) 766 : cluster 0 pgmap v775: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T18:43:47.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:47 smithi098 bash[76817]: cluster 2023-09-23T18:43:45.970574+0000 mgr.y (mgr.26054) 766 : cluster 0 pgmap v775: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T18:43:47.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:47 smithi173 bash[59422]: cluster 2023-09-23T18:43:45.970574+0000 mgr.y (mgr.26054) 766 : cluster 0 pgmap v775: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T18:43:47.573 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:47.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:47.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:47.678 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:47.702 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:47.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:47.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:47.809 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:47.835 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:47.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:47.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:47.939 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:47.958 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:47.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:47.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:48.061 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:48.167 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 46 16 65547 65531 89.0253 61.0625 0.0155548 0.011226 2023-09-23T18:43:48.374 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:48.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:48.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:48.478 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:48.518 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:48.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:48.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:48.625 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:48.650 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:48.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:48.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:48.756 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:48.795 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:48.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:48.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:48.900 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:48.936 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:48.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:48.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:49.042 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:49.053 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:49 smithi098 bash[76817]: cluster 2023-09-23T18:43:47.971134+0000 mgr.y (mgr.26054) 767 : cluster 0 pgmap v776: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T18:43:49.167 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 47 16 66603 66587 88.5353 66 0.0118492 0.011289 2023-09-23T18:43:49.294 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:49.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:49.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:49.398 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:49.406 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:49 smithi098 bash[79516]: cluster 2023-09-23T18:43:47.971134+0000 mgr.y (mgr.26054) 767 : cluster 0 pgmap v776: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T18:43:49.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:49.434 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:49.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:49.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:49 smithi173 bash[59422]: cluster 2023-09-23T18:43:47.971134+0000 mgr.y (mgr.26054) 767 : cluster 0 pgmap v776: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T18:43:49.535 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:49.991 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:49.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:49.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:50.096 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:50.120 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:50.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:50.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:50.167 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 48 16 67584 67568 87.9681 61.3125 0.0140429 0.0113617 2023-09-23T18:43:50.224 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:50.252 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:50.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:50.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:50.357 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:50.480 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:50.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:50.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:50.584 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:50.610 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:50.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:50.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:50.703 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T18:43:50.715 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:50.781 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:50.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:50.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:50.886 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:50.965 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:50.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:50.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:51.059 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:51 smithi098 bash[76817]: cluster 2023-09-23T18:43:49.971696+0000 mgr.y (mgr.26054) 768 : cluster 0 pgmap v777: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T18:43:51.069 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:51.096 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:51.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:51.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:51.167 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 49 16 68573 68557 87.4341 61.8125 0.0250441 0.0114308 2023-09-23T18:43:51.199 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:51.231 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:51.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:51.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:51.335 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:51.344 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:51 smithi173 bash[59422]: cluster 2023-09-23T18:43:49.971696+0000 mgr.y (mgr.26054) 768 : cluster 0 pgmap v777: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T18:43:51.358 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:51.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:51.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:51.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:51 smithi098 bash[79516]: cluster 2023-09-23T18:43:49.971696+0000 mgr.y (mgr.26054) 768 : cluster 0 pgmap v777: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T18:43:51.461 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:51.501 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:51.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:51.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:51.605 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:51.629 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:51.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:51.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:51.732 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:51.763 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:51.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:51.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:51.868 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:51.998 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:52.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:52.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:52.101 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:52.127 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:52.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:52.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:52.167 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 50 16 69447 69431 86.7778 54.625 0.0062835 0.0115173 2023-09-23T18:43:52.231 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:52.259 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:52.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:52.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:52.503 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:52.531 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:52.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:52.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:52.635 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:52.658 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:52.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:52.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:52.760 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:52.794 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:52.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:52.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:52.902 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:53.088 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:53.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:53.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:53.192 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:53.221 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:53.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:53.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:53.242 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 51 16 70376 70360 86.2147 58.0625 0.0264395 0.011593 2023-09-23T18:43:53.324 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:53.332 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:53 smithi173 bash[59422]: cluster 2023-09-23T18:43:51.972355+0000 mgr.y (mgr.26054) 769 : cluster 0 pgmap v778: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T18:43:53.347 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:53.348 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:53 smithi098 bash[76817]: cluster 2023-09-23T18:43:51.972355+0000 mgr.y (mgr.26054) 769 : cluster 0 pgmap v778: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T18:43:53.348 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:53 smithi098 bash[79516]: cluster 2023-09-23T18:43:51.972355+0000 mgr.y (mgr.26054) 769 : cluster 0 pgmap v778: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T18:43:53.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:53.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:53.450 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:53.473 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:53.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:53.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:53.578 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:53.606 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:53.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:53.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:53.712 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:53.926 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:53.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:53.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:54.030 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:54.271 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 52 16 71383 71367 85.6451 62.9375 0.0199292 0.01167 2023-09-23T18:43:54.343 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:54.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:54.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:54.446 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:54.471 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:54.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:54.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:54.576 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:54.610 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:54.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:54.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:54.713 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:54.741 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:54.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:54.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:54.844 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:55.057 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:55.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:55.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:55.162 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:55.184 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:55.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:55.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:55.271 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 53 16 72410 72394 85.1929 64.1875 0.017083 0.0117316 2023-09-23T18:43:55.289 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:55.318 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:55.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:55.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:55.421 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:55.430 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:55 smithi173 bash[59422]: cluster 2023-09-23T18:43:53.973026+0000 mgr.y (mgr.26054) 770 : cluster 0 pgmap v779: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 60 MiB/s wr, 963 op/s 2023-09-23T18:43:55.430 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:55 smithi173 bash[59422]: audit 2023-09-23T18:43:54.607260+0000 mon.a (mon.0) 1028 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:43:55.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:55 smithi098 bash[76817]: cluster 2023-09-23T18:43:53.973026+0000 mgr.y (mgr.26054) 770 : cluster 0 pgmap v779: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 60 MiB/s wr, 963 op/s 2023-09-23T18:43:55.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:55 smithi098 bash[76817]: audit 2023-09-23T18:43:54.607260+0000 mon.a (mon.0) 1028 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:43:55.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:55 smithi098 bash[79516]: cluster 2023-09-23T18:43:53.973026+0000 mgr.y (mgr.26054) 770 : cluster 0 pgmap v779: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 60 MiB/s wr, 963 op/s 2023-09-23T18:43:55.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:55 smithi098 bash[79516]: audit 2023-09-23T18:43:54.607260+0000 mon.a (mon.0) 1028 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:43:55.447 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:55.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:55.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:55.550 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:55.578 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:55.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:55.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:55.684 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:55.714 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:55.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:55.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:55.817 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:55.846 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:55.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:55.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:55.951 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:55.978 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:55.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:55.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:56.081 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:56.100 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:56.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:56.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:56.205 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:56.240 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:56.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:56.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:56.271 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 54 16 73273 73257 84.6152 53.9375 0.016585 0.0118117 2023-09-23T18:43:56.347 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:56.357 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:56 smithi173 bash[59422]: cluster 2023-09-23T18:43:55.976149+0000 mgr.y (mgr.26054) 771 : cluster 0 pgmap v780: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 59 MiB/s wr, 945 op/s 2023-09-23T18:43:56.372 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:56.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:56.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:56.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:56 smithi098 bash[79516]: cluster 2023-09-23T18:43:55.976149+0000 mgr.y (mgr.26054) 771 : cluster 0 pgmap v780: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 59 MiB/s wr, 945 op/s 2023-09-23T18:43:56.441 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:43:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:43:56] "GET /metrics HTTP/1.1" 200 36554 "" "Prometheus/2.33.4" 2023-09-23T18:43:56.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:56 smithi098 bash[76817]: cluster 2023-09-23T18:43:55.976149+0000 mgr.y (mgr.26054) 771 : cluster 0 pgmap v780: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 59 MiB/s wr, 945 op/s 2023-09-23T18:43:56.475 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:56.503 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:56.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:56.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:56.609 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:56.634 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:56.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:56.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:56.743 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:56.775 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:56.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:56.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:56.882 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:56.913 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:56.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:56.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:57.018 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:57.051 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:57.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:57.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:57.154 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:57.199 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:57.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:57.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:57.272 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 55 16 74153 74137 84.0776 55 0.021139 0.0118876 2023-09-23T18:43:57.302 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:57.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:57.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:57.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:57.417 INFO:teuthology.orchestra.run.smithi098.stderr:set pool 7 pg_num to 26 2023-09-23T18:43:57.432 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:57.456 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:57.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:57.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:57.560 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:57.586 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:57.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:57.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:57.690 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:57.697 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:57 smithi173 bash[59422]: audit 2023-09-23T18:43:57.355425+0000 mon.a (mon.0) 1029 : audit 1 from='client.? 172.21.15.98:0/1546931029' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T18:43:57.713 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:57.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:57.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:57.728 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:57 smithi098 bash[76817]: audit 2023-09-23T18:43:57.355425+0000 mon.a (mon.0) 1029 : audit 1 from='client.? 172.21.15.98:0/1546931029' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T18:43:57.728 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:57 smithi098 bash[79516]: audit 2023-09-23T18:43:57.355425+0000 mon.a (mon.0) 1029 : audit 1 from='client.? 172.21.15.98:0/1546931029' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T18:43:57.816 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:57.852 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:57.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:57.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:57.958 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:57.981 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:57.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:57.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:58.083 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:58.114 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:58.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:58.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:58.218 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:58.244 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:58.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:58.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:58.271 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 56 16 75041 75025 83.5682 55.5 0.0130218 0.01196 2023-09-23T18:43:58.348 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:58.373 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:58.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:58.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:43:58.429 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:58 smithi098 bash[76817]: cluster 2023-09-23T18:43:57.409452+0000 osd.5 (osd.5) 91 : cluster 3 failed to encode map e194 with expected crc 2023-09-23T18:43:58.429 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:58 smithi098 bash[76817]: audit 2023-09-23T18:43:57.411169+0000 mon.a (mon.0) 1030 : audit 1 from='client.? 172.21.15.98:0/1546931029' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2023-09-23T18:43:58.429 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:58 smithi098 bash[76817]: cluster 2023-09-23T18:43:57.411288+0000 mon.a (mon.0) 1031 : cluster 0 osdmap e194: 8 total, 8 up, 7 in 2023-09-23T18:43:58.430 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:58 smithi098 bash[76817]: cluster 2023-09-23T18:43:57.417150+0000 osd.4 (osd.4) 152 : cluster 3 failed to encode map e194 with expected crc 2023-09-23T18:43:58.430 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:58 smithi098 bash[76817]: cluster 2023-09-23T18:43:57.417253+0000 osd.6 (osd.6) 124 : cluster 3 failed to encode map e194 with expected crc 2023-09-23T18:43:58.430 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:58 smithi098 bash[76817]: cluster 2023-09-23T18:43:57.976699+0000 mgr.y (mgr.26054) 772 : cluster 0 pgmap v782: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 59 MiB/s wr, 940 op/s 2023-09-23T18:43:58.430 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:58 smithi098 bash[76817]: audit 2023-09-23T18:43:57.977341+0000 mon.a (mon.0) 1032 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T18:43:58.477 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:58.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:58 smithi173 bash[59422]: cluster 2023-09-23T18:43:57.409452+0000 osd.5 (osd.5) 91 : cluster 3 failed to encode map e194 with expected crc 2023-09-23T18:43:58.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:58 smithi173 bash[59422]: audit 2023-09-23T18:43:57.411169+0000 mon.a (mon.0) 1030 : audit 1 from='client.? 172.21.15.98:0/1546931029' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2023-09-23T18:43:58.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:58 smithi173 bash[59422]: cluster 2023-09-23T18:43:57.411288+0000 mon.a (mon.0) 1031 : cluster 0 osdmap e194: 8 total, 8 up, 7 in 2023-09-23T18:43:58.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:58 smithi173 bash[59422]: cluster 2023-09-23T18:43:57.417150+0000 osd.4 (osd.4) 152 : cluster 3 failed to encode map e194 with expected crc 2023-09-23T18:43:58.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:58 smithi173 bash[59422]: cluster 2023-09-23T18:43:57.417253+0000 osd.6 (osd.6) 124 : cluster 3 failed to encode map e194 with expected crc 2023-09-23T18:43:58.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:58 smithi173 bash[59422]: cluster 2023-09-23T18:43:57.976699+0000 mgr.y (mgr.26054) 772 : cluster 0 pgmap v782: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 59 MiB/s wr, 940 op/s 2023-09-23T18:43:58.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:58 smithi173 bash[59422]: audit 2023-09-23T18:43:57.977341+0000 mon.a (mon.0) 1032 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T18:43:58.741 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:58 smithi098 bash[79516]: cluster 2023-09-23T18:43:57.409452+0000 osd.5 (osd.5) 91 : cluster 3 failed to encode map e194 with expected crc 2023-09-23T18:43:58.741 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:58 smithi098 bash[79516]: audit 2023-09-23T18:43:57.411169+0000 mon.a (mon.0) 1030 : audit 1 from='client.? 172.21.15.98:0/1546931029' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2023-09-23T18:43:58.741 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:58 smithi098 bash[79516]: cluster 2023-09-23T18:43:57.411288+0000 mon.a (mon.0) 1031 : cluster 0 osdmap e194: 8 total, 8 up, 7 in 2023-09-23T18:43:58.741 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:58 smithi098 bash[79516]: cluster 2023-09-23T18:43:57.417150+0000 osd.4 (osd.4) 152 : cluster 3 failed to encode map e194 with expected crc 2023-09-23T18:43:58.741 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:58 smithi098 bash[79516]: cluster 2023-09-23T18:43:57.417253+0000 osd.6 (osd.6) 124 : cluster 3 failed to encode map e194 with expected crc 2023-09-23T18:43:58.742 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:58 smithi098 bash[79516]: cluster 2023-09-23T18:43:57.976699+0000 mgr.y (mgr.26054) 772 : cluster 0 pgmap v782: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 59 MiB/s wr, 940 op/s 2023-09-23T18:43:58.742 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:58 smithi098 bash[79516]: audit 2023-09-23T18:43:57.977341+0000 mon.a (mon.0) 1032 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T18:43:58.931 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:43:58.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:58.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:43:59.034 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:59.067 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:43:59.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:59.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:43:59.170 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:59.196 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:59.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:59.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:59.272 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 57 16 75218 75202 82.2984 11.0625 0.00553651 0.0119878 2023-09-23T18:43:59.300 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:59.331 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:43:59.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:59.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:43:59.432 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:59 smithi098 bash[76817]: cluster 2023-09-23T18:43:58.427690+0000 osd.5 (osd.5) 92 : cluster 3 failed to encode map e195 with expected crc 2023-09-23T18:43:59.432 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:59 smithi098 bash[76817]: cluster 2023-09-23T18:43:58.429848+0000 osd.6 (osd.6) 125 : cluster 3 failed to encode map e195 with expected crc 2023-09-23T18:43:59.432 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:59 smithi098 bash[76817]: audit 2023-09-23T18:43:58.431841+0000 mon.a (mon.0) 1033 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T18:43:59.432 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:59 smithi098 bash[76817]: cluster 2023-09-23T18:43:58.432085+0000 mon.a (mon.0) 1034 : cluster 0 osdmap e195: 8 total, 8 up, 7 in 2023-09-23T18:43:59.432 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:59 smithi098 bash[76817]: cluster 2023-09-23T18:43:58.445854+0000 osd.4 (osd.4) 153 : cluster 3 failed to encode map e195 with expected crc 2023-09-23T18:43:59.432 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:43:59 smithi098 bash[76817]: cluster 2023-09-23T18:43:58.472192+0000 osd.7 (osd.7) 109 : cluster 3 failed to encode map e195 with expected crc 2023-09-23T18:43:59.435 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:59.471 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:43:59.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:59.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:43:59.572 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:59.581 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:59 smithi173 bash[59422]: cluster 2023-09-23T18:43:58.427690+0000 osd.5 (osd.5) 92 : cluster 3 failed to encode map e195 with expected crc 2023-09-23T18:43:59.582 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:59 smithi173 bash[59422]: cluster 2023-09-23T18:43:58.429848+0000 osd.6 (osd.6) 125 : cluster 3 failed to encode map e195 with expected crc 2023-09-23T18:43:59.582 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:59 smithi173 bash[59422]: audit 2023-09-23T18:43:58.431841+0000 mon.a (mon.0) 1033 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T18:43:59.582 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:59 smithi173 bash[59422]: cluster 2023-09-23T18:43:58.432085+0000 mon.a (mon.0) 1034 : cluster 0 osdmap e195: 8 total, 8 up, 7 in 2023-09-23T18:43:59.582 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:59 smithi173 bash[59422]: cluster 2023-09-23T18:43:58.445854+0000 osd.4 (osd.4) 153 : cluster 3 failed to encode map e195 with expected crc 2023-09-23T18:43:59.582 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:43:59 smithi173 bash[59422]: cluster 2023-09-23T18:43:58.472192+0000 osd.7 (osd.7) 109 : cluster 3 failed to encode map e195 with expected crc 2023-09-23T18:43:59.597 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:43:59.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:59.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:43:59.690 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:59 smithi098 bash[79516]: cluster 2023-09-23T18:43:58.427690+0000 osd.5 (osd.5) 92 : cluster 3 failed to encode map e195 with expected crc 2023-09-23T18:43:59.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:59 smithi098 bash[79516]: cluster 2023-09-23T18:43:58.429848+0000 osd.6 (osd.6) 125 : cluster 3 failed to encode map e195 with expected crc 2023-09-23T18:43:59.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:59 smithi098 bash[79516]: audit 2023-09-23T18:43:58.431841+0000 mon.a (mon.0) 1033 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T18:43:59.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:59 smithi098 bash[79516]: cluster 2023-09-23T18:43:58.432085+0000 mon.a (mon.0) 1034 : cluster 0 osdmap e195: 8 total, 8 up, 7 in 2023-09-23T18:43:59.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:59 smithi098 bash[79516]: cluster 2023-09-23T18:43:58.445854+0000 osd.4 (osd.4) 153 : cluster 3 failed to encode map e195 with expected crc 2023-09-23T18:43:59.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:43:59 smithi098 bash[79516]: cluster 2023-09-23T18:43:58.472192+0000 osd.7 (osd.7) 109 : cluster 3 failed to encode map e195 with expected crc 2023-09-23T18:43:59.701 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:59.728 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:43:59.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:59.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:43:59.833 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:59.858 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:43:59.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:59.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:43:59.961 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:59.988 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:43:59.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:43:59.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:00.093 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:00.117 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:00.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:00.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:00.222 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:00.251 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:00.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:00.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:00.272 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 58 16 76162 76146 81.8973 59 0.00778268 0.0122054 2023-09-23T18:44:00.364 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:00.401 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:00.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:00.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:00.423 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:44:00.504 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:00.514 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:00 smithi173 bash[59422]: cluster 2023-09-23T18:43:59.436843+0000 osd.5 (osd.5) 93 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T18:44:00.514 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:00 smithi173 bash[59422]: cluster 2023-09-23T18:43:59.438441+0000 mon.a (mon.0) 1035 : cluster 0 osdmap e196: 8 total, 8 up, 7 in 2023-09-23T18:44:00.514 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:00 smithi173 bash[59422]: cluster 2023-09-23T18:43:59.439587+0000 osd.7 (osd.7) 110 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T18:44:00.514 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:00 smithi173 bash[59422]: cluster 2023-09-23T18:43:59.440509+0000 osd.4 (osd.4) 154 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T18:44:00.517 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:00 smithi173 bash[59422]: cluster 2023-09-23T18:43:59.440999+0000 osd.6 (osd.6) 126 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T18:44:00.517 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:00 smithi173 bash[59422]: cluster 2023-09-23T18:43:59.442301+0000 osd.7 (osd.7) 111 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T18:44:00.517 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:00 smithi173 bash[59422]: cluster 2023-09-23T18:43:59.443647+0000 osd.4 (osd.4) 155 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T18:44:00.517 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:00 smithi173 bash[59422]: cluster 2023-09-23T18:43:59.447164+0000 osd.4 (osd.4) 156 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T18:44:00.517 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:00 smithi173 bash[59422]: cluster 2023-09-23T18:43:59.977123+0000 mgr.y (mgr.26054) 773 : cluster 0 pgmap v785: 99 pgs: 1 peering, 10 unknown, 88 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 54 MiB/s wr, 858 op/s 2023-09-23T18:44:00.530 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:00.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:00.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:00.637 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:00.671 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:00.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:00.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:00.743 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:00 smithi098 bash[76817]: cluster 2023-09-23T18:43:59.436843+0000 osd.5 (osd.5) 93 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T18:44:00.743 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:00 smithi098 bash[76817]: cluster 2023-09-23T18:43:59.438441+0000 mon.a (mon.0) 1035 : cluster 0 osdmap e196: 8 total, 8 up, 7 in 2023-09-23T18:44:00.743 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:00 smithi098 bash[76817]: cluster 2023-09-23T18:43:59.439587+0000 osd.7 (osd.7) 110 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T18:44:00.744 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:00 smithi098 bash[76817]: cluster 2023-09-23T18:43:59.440509+0000 osd.4 (osd.4) 154 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T18:44:00.744 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:00 smithi098 bash[76817]: cluster 2023-09-23T18:43:59.440999+0000 osd.6 (osd.6) 126 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T18:44:00.744 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:00 smithi098 bash[76817]: cluster 2023-09-23T18:43:59.442301+0000 osd.7 (osd.7) 111 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T18:44:00.744 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:00 smithi098 bash[76817]: cluster 2023-09-23T18:43:59.443647+0000 osd.4 (osd.4) 155 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T18:44:00.744 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:00 smithi098 bash[76817]: cluster 2023-09-23T18:43:59.447164+0000 osd.4 (osd.4) 156 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T18:44:00.744 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:00 smithi098 bash[76817]: cluster 2023-09-23T18:43:59.977123+0000 mgr.y (mgr.26054) 773 : cluster 0 pgmap v785: 99 pgs: 1 peering, 10 unknown, 88 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 54 MiB/s wr, 858 op/s 2023-09-23T18:44:00.744 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:00 smithi098 bash[79516]: cluster 2023-09-23T18:43:59.436843+0000 osd.5 (osd.5) 93 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T18:44:00.744 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:00 smithi098 bash[79516]: cluster 2023-09-23T18:43:59.438441+0000 mon.a (mon.0) 1035 : cluster 0 osdmap e196: 8 total, 8 up, 7 in 2023-09-23T18:44:00.744 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:00 smithi098 bash[79516]: cluster 2023-09-23T18:43:59.439587+0000 osd.7 (osd.7) 110 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T18:44:00.745 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:00 smithi098 bash[79516]: cluster 2023-09-23T18:43:59.440509+0000 osd.4 (osd.4) 154 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T18:44:00.745 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:00 smithi098 bash[79516]: cluster 2023-09-23T18:43:59.440999+0000 osd.6 (osd.6) 126 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T18:44:00.745 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:00 smithi098 bash[79516]: cluster 2023-09-23T18:43:59.442301+0000 osd.7 (osd.7) 111 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T18:44:00.745 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:00 smithi098 bash[79516]: cluster 2023-09-23T18:43:59.443647+0000 osd.4 (osd.4) 155 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T18:44:00.745 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:00 smithi098 bash[79516]: cluster 2023-09-23T18:43:59.447164+0000 osd.4 (osd.4) 156 : cluster 3 failed to encode map e196 with expected crc 2023-09-23T18:44:00.745 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:00 smithi098 bash[79516]: cluster 2023-09-23T18:43:59.977123+0000 mgr.y (mgr.26054) 773 : cluster 0 pgmap v785: 99 pgs: 1 peering, 10 unknown, 88 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 54 MiB/s wr, 858 op/s 2023-09-23T18:44:00.778 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:00.802 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:00.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:00.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:00.906 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:00.935 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:00.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:00.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:01.040 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:01.064 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:01.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:01.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:01.168 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:01.195 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:01.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:01.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:01.272 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 59 16 77359 77343 81.7773 74.8125 0.0221738 0.0122232 2023-09-23T18:44:01.298 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:01.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:01.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:01.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:01.430 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:01.454 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:01.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:01.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:01.557 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:01.581 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:01.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:01.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:01.684 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:01.708 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:01.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:01.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:01.811 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:01.838 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:01.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:01.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:01.943 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:01.970 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:01.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:01.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:02.021 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:02 smithi098 bash[79516]: audit 2023-09-23T18:44:01.967321+0000 mon.a (mon.0) 1036 : audit 0 from='client.? 172.21.15.98:0/2962632300' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:44:02.021 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:02 smithi098 bash[76817]: audit 2023-09-23T18:44:01.967321+0000 mon.a (mon.0) 1036 : audit 0 from='client.? 172.21.15.98:0/2962632300' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:44:02.057 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:44:02.074 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:02.100 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:02.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:02.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:02.115 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:44:02.162 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:44:02.206 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:02.211 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:44:02.239 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:02.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:02.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:02.258 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:44:02.272 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:2023-09-23T18:44:02.271659+0000 min lat: 0.00125843 max lat: 1.09398 avg lat: 0.0122923 2023-09-23T18:44:02.272 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:44:02.272 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 60 16 78223 78207 81.3151 54 0.00912409 0.0122923 2023-09-23T18:44:02.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:02 smithi173 bash[59422]: audit 2023-09-23T18:44:01.967321+0000 mon.a (mon.0) 1036 : audit 0 from='client.? 172.21.15.98:0/2962632300' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:44:02.299 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:44:02.341 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:02.342 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:44:02.379 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:02.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:02.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:02.386 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:44:02.389 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:44:02.390 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:44:02.390 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:44:02.390 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:44:02.390 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:44:02.390 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:44:02.390 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:44:02.390 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:44:02.485 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:02.516 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:02.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:02.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:02.621 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:02.646 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:02.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:02.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:02.749 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:02.979 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:02.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:02.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:03.056 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:03 smithi098 bash[76817]: cluster 2023-09-23T18:44:01.978093+0000 mgr.y (mgr.26054) 774 : cluster 0 pgmap v786: 99 pgs: 1 peering, 5 unknown, 93 active+clean; 4.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 56 MiB/s wr, 888 op/s; 21 KiB/s, 0 objects/s recovering 2023-09-23T18:44:03.056 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:03 smithi098 bash[76817]: audit 2023-09-23T18:44:02.378429+0000 mon.a (mon.0) 1037 : audit 1 from='client.? 172.21.15.98:0/2731657990' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:44:03.057 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is set 2023-09-23T18:44:03.084 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:03.111 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:03.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:03.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:03.217 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:03.241 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:03.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:03.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:03.272 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 61 16 79379 79363 81.1666 72.25 0.0142923 0.0123144 2023-09-23T18:44:03.345 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:03.353 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:03 smithi098 bash[79516]: cluster 2023-09-23T18:44:01.978093+0000 mgr.y (mgr.26054) 774 : cluster 0 pgmap v786: 99 pgs: 1 peering, 5 unknown, 93 active+clean; 4.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 56 MiB/s wr, 888 op/s; 21 KiB/s, 0 objects/s recovering 2023-09-23T18:44:03.353 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:03 smithi098 bash[79516]: audit 2023-09-23T18:44:02.378429+0000 mon.a (mon.0) 1037 : audit 1 from='client.? 172.21.15.98:0/2731657990' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:44:03.372 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:03.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:03.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:03.476 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:03.506 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:03.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:03.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:03 smithi173 bash[59422]: cluster 2023-09-23T18:44:01.978093+0000 mgr.y (mgr.26054) 774 : cluster 0 pgmap v786: 99 pgs: 1 peering, 5 unknown, 93 active+clean; 4.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 56 MiB/s wr, 888 op/s; 21 KiB/s, 0 objects/s recovering 2023-09-23T18:44:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:03 smithi173 bash[59422]: audit 2023-09-23T18:44:02.378429+0000 mon.a (mon.0) 1037 : audit 1 from='client.? 172.21.15.98:0/2731657990' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:44:03.609 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:03.646 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:03.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:03.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:03.750 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:03.776 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:03.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:03.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:03.879 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:03.905 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:03.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:03.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:04.010 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:04.040 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:04.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:04.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:04.144 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:04.173 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:04.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:04.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:04.272 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 62 16 80461 80445 80.9485 67.625 0.00504429 0.012348 2023-09-23T18:44:04.277 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:04.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:04 smithi173 bash[59422]: cluster 2023-09-23T18:44:03.041738+0000 mon.a (mon.0) 1038 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:44:04.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:04 smithi173 bash[59422]: cluster 2023-09-23T18:44:03.047084+0000 osd.5 (osd.5) 94 : cluster 3 failed to encode map e197 with expected crc 2023-09-23T18:44:04.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:04 smithi173 bash[59422]: cluster 2023-09-23T18:44:03.049208+0000 osd.4 (osd.4) 157 : cluster 3 failed to encode map e197 with expected crc 2023-09-23T18:44:04.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:04 smithi173 bash[59422]: audit 2023-09-23T18:44:03.049416+0000 mon.a (mon.0) 1039 : audit 1 from='client.? 172.21.15.98:0/2731657990' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:44:04.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:04 smithi173 bash[59422]: cluster 2023-09-23T18:44:03.049494+0000 mon.a (mon.0) 1040 : cluster 0 osdmap e197: 8 total, 8 up, 7 in 2023-09-23T18:44:04.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:04 smithi173 bash[59422]: cluster 2023-09-23T18:44:03.064456+0000 osd.6 (osd.6) 127 : cluster 3 failed to encode map e197 with expected crc 2023-09-23T18:44:04.286 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:04 smithi098 bash[79516]: cluster 2023-09-23T18:44:03.041738+0000 mon.a (mon.0) 1038 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:44:04.286 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:04 smithi098 bash[79516]: cluster 2023-09-23T18:44:03.047084+0000 osd.5 (osd.5) 94 : cluster 3 failed to encode map e197 with expected crc 2023-09-23T18:44:04.286 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:04 smithi098 bash[79516]: cluster 2023-09-23T18:44:03.049208+0000 osd.4 (osd.4) 157 : cluster 3 failed to encode map e197 with expected crc 2023-09-23T18:44:04.286 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:04 smithi098 bash[79516]: audit 2023-09-23T18:44:03.049416+0000 mon.a (mon.0) 1039 : audit 1 from='client.? 172.21.15.98:0/2731657990' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:44:04.286 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:04 smithi098 bash[79516]: cluster 2023-09-23T18:44:03.049494+0000 mon.a (mon.0) 1040 : cluster 0 osdmap e197: 8 total, 8 up, 7 in 2023-09-23T18:44:04.286 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:04 smithi098 bash[79516]: cluster 2023-09-23T18:44:03.064456+0000 osd.6 (osd.6) 127 : cluster 3 failed to encode map e197 with expected crc 2023-09-23T18:44:04.287 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:04 smithi098 bash[76817]: cluster 2023-09-23T18:44:03.041738+0000 mon.a (mon.0) 1038 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:44:04.287 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:04 smithi098 bash[76817]: cluster 2023-09-23T18:44:03.047084+0000 osd.5 (osd.5) 94 : cluster 3 failed to encode map e197 with expected crc 2023-09-23T18:44:04.287 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:04 smithi098 bash[76817]: cluster 2023-09-23T18:44:03.049208+0000 osd.4 (osd.4) 157 : cluster 3 failed to encode map e197 with expected crc 2023-09-23T18:44:04.287 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:04 smithi098 bash[76817]: audit 2023-09-23T18:44:03.049416+0000 mon.a (mon.0) 1039 : audit 1 from='client.? 172.21.15.98:0/2731657990' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:44:04.287 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:04 smithi098 bash[76817]: cluster 2023-09-23T18:44:03.049494+0000 mon.a (mon.0) 1040 : cluster 0 osdmap e197: 8 total, 8 up, 7 in 2023-09-23T18:44:04.287 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:04 smithi098 bash[76817]: cluster 2023-09-23T18:44:03.064456+0000 osd.6 (osd.6) 127 : cluster 3 failed to encode map e197 with expected crc 2023-09-23T18:44:04.317 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:04.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:04.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:04.421 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:04.445 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:04.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:04.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:04.549 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:04.579 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:04.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:04.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:04.681 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:04.711 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:04.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:04.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:04.814 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:04.839 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:04.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:04.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:04.942 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:04.952 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T18:44:04.952 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:44:04.953 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T18:44:04.953 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T18:44:04.953 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T18:44:04.953 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T18:44:04.970 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:04.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:04.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:05.073 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:05.112 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:05.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:05.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:05.215 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:05.232 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:05.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:05.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:05.247 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:05 smithi098 bash[79516]: cluster 2023-09-23T18:44:03.978648+0000 mgr.y (mgr.26054) 775 : cluster 0 pgmap v788: 99 pgs: 1 peering, 1 unknown, 97 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 54 MiB/s wr, 864 op/s; 21 KiB/s, 0 objects/s recovering 2023-09-23T18:44:05.248 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:05 smithi098 bash[76817]: cluster 2023-09-23T18:44:03.978648+0000 mgr.y (mgr.26054) 775 : cluster 0 pgmap v788: 99 pgs: 1 peering, 1 unknown, 97 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 54 MiB/s wr, 864 op/s; 21 KiB/s, 0 objects/s recovering 2023-09-23T18:44:05.272 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 63 16 81313 81297 80.5095 53.25 0.0134728 0.0124154 2023-09-23T18:44:05.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:05 smithi173 bash[59422]: cluster 2023-09-23T18:44:03.978648+0000 mgr.y (mgr.26054) 775 : cluster 0 pgmap v788: 99 pgs: 1 peering, 1 unknown, 97 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 54 MiB/s wr, 864 op/s; 21 KiB/s, 0 objects/s recovering 2023-09-23T18:44:05.335 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:05.362 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:05.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:05.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:05.465 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:05.500 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:05.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:05.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:05.604 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:05.632 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:05.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:05.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:05.736 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:05.762 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:05.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:05.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:05.865 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:05.891 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:05.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:05.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:05.996 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:06.025 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:06.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:06.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:06.131 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:06.158 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:06.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:06.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:06.263 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:06.272 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 64 16 82503 82487 80.4137 74.375 0.00528385 0.0124211 2023-09-23T18:44:06.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:06 smithi173 bash[59422]: audit 2023-09-23T18:44:05.980228+0000 mon.a (mon.0) 1041 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T18:44:06.287 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:06.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:06.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:06.393 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:06.418 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:06.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:06.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:06.433 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:44:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:44:06] "GET /metrics HTTP/1.1" 200 36554 "" "Prometheus/2.33.4" 2023-09-23T18:44:06.434 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:06 smithi098 bash[79516]: audit 2023-09-23T18:44:05.980228+0000 mon.a (mon.0) 1041 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T18:44:06.434 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:06 smithi098 bash[76817]: audit 2023-09-23T18:44:05.980228+0000 mon.a (mon.0) 1041 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T18:44:06.522 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:06.554 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:06.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:06.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:06.658 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:06.684 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:06.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:06.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:06.787 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:06.808 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:06.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:06.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:06.911 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:06.938 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:06.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:06.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:07.042 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:07.076 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:07.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:07.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:07.180 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:07.206 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:07.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:07.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:07.273 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 65 16 82807 82791 79.4704 19 0.0038885 0.012393 2023-09-23T18:44:07.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:07 smithi173 bash[59422]: cluster 2023-09-23T18:44:05.979182+0000 mgr.y (mgr.26054) 776 : cluster 0 pgmap v789: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 50 MiB/s wr, 798 op/s; 17 KiB/s, 0 objects/s recovering 2023-09-23T18:44:07.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:07 smithi173 bash[59422]: audit 2023-09-23T18:44:06.091186+0000 mon.a (mon.0) 1042 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T18:44:07.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:07 smithi173 bash[59422]: cluster 2023-09-23T18:44:06.091432+0000 mon.a (mon.0) 1043 : cluster 0 osdmap e198: 8 total, 8 up, 7 in 2023-09-23T18:44:07.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:07 smithi173 bash[59422]: cluster 2023-09-23T18:44:06.095822+0000 osd.6 (osd.6) 128 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T18:44:07.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:07 smithi173 bash[59422]: cluster 2023-09-23T18:44:06.096193+0000 osd.7 (osd.7) 112 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T18:44:07.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:07 smithi173 bash[59422]: cluster 2023-09-23T18:44:06.099675+0000 osd.4 (osd.4) 158 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T18:44:07.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:07 smithi173 bash[59422]: cluster 2023-09-23T18:44:06.101651+0000 osd.6 (osd.6) 129 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T18:44:07.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:07 smithi173 bash[59422]: cluster 2023-09-23T18:44:06.120969+0000 osd.5 (osd.5) 95 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T18:44:07.310 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:07.341 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:07.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:07.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:07.370 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:44:07.385 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:07 smithi098 bash[76817]: cluster 2023-09-23T18:44:05.979182+0000 mgr.y (mgr.26054) 776 : cluster 0 pgmap v789: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 50 MiB/s wr, 798 op/s; 17 KiB/s, 0 objects/s recovering 2023-09-23T18:44:07.385 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:07 smithi098 bash[76817]: audit 2023-09-23T18:44:06.091186+0000 mon.a (mon.0) 1042 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T18:44:07.385 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:07 smithi098 bash[76817]: cluster 2023-09-23T18:44:06.091432+0000 mon.a (mon.0) 1043 : cluster 0 osdmap e198: 8 total, 8 up, 7 in 2023-09-23T18:44:07.385 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:07 smithi098 bash[76817]: cluster 2023-09-23T18:44:06.095822+0000 osd.6 (osd.6) 128 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T18:44:07.385 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:07 smithi098 bash[76817]: cluster 2023-09-23T18:44:06.096193+0000 osd.7 (osd.7) 112 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T18:44:07.386 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:07 smithi098 bash[76817]: cluster 2023-09-23T18:44:06.099675+0000 osd.4 (osd.4) 158 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T18:44:07.386 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:07 smithi098 bash[76817]: cluster 2023-09-23T18:44:06.101651+0000 osd.6 (osd.6) 129 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T18:44:07.386 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:07 smithi098 bash[76817]: cluster 2023-09-23T18:44:06.120969+0000 osd.5 (osd.5) 95 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T18:44:07.386 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:07 smithi098 bash[79516]: cluster 2023-09-23T18:44:05.979182+0000 mgr.y (mgr.26054) 776 : cluster 0 pgmap v789: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 50 MiB/s wr, 798 op/s; 17 KiB/s, 0 objects/s recovering 2023-09-23T18:44:07.386 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:07 smithi098 bash[79516]: audit 2023-09-23T18:44:06.091186+0000 mon.a (mon.0) 1042 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T18:44:07.386 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:07 smithi098 bash[79516]: cluster 2023-09-23T18:44:06.091432+0000 mon.a (mon.0) 1043 : cluster 0 osdmap e198: 8 total, 8 up, 7 in 2023-09-23T18:44:07.386 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:07 smithi098 bash[79516]: cluster 2023-09-23T18:44:06.095822+0000 osd.6 (osd.6) 128 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T18:44:07.386 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:07 smithi098 bash[79516]: cluster 2023-09-23T18:44:06.096193+0000 osd.7 (osd.7) 112 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T18:44:07.387 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:07 smithi098 bash[79516]: cluster 2023-09-23T18:44:06.099675+0000 osd.4 (osd.4) 158 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T18:44:07.387 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:07 smithi098 bash[79516]: cluster 2023-09-23T18:44:06.101651+0000 osd.6 (osd.6) 129 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T18:44:07.387 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:07 smithi098 bash[79516]: cluster 2023-09-23T18:44:06.120969+0000 osd.5 (osd.5) 95 : cluster 3 failed to encode map e198 with expected crc 2023-09-23T18:44:07.445 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:07.479 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:07.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:07.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:07.583 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:07.606 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:07.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:07.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:07.709 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:07.742 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:07.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:07.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:07.845 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:07.876 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:07.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:07.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:07.980 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:08.012 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:08.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:08.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:08.115 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:08.141 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:08.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:08.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:08.244 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:08.271 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:08.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:08.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:08.274 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 66 16 82807 82791 78.2682 0 - 0.012393 2023-09-23T18:44:08.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:08 smithi173 bash[59422]: cluster 2023-09-23T18:44:07.089572+0000 osd.7 (osd.7) 113 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T18:44:08.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:08 smithi173 bash[59422]: cluster 2023-09-23T18:44:07.089976+0000 mon.a (mon.0) 1044 : cluster 0 osdmap e199: 8 total, 8 up, 7 in 2023-09-23T18:44:08.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:08 smithi173 bash[59422]: cluster 2023-09-23T18:44:07.091977+0000 osd.6 (osd.6) 130 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T18:44:08.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:08 smithi173 bash[59422]: cluster 2023-09-23T18:44:07.094752+0000 osd.6 (osd.6) 131 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T18:44:08.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:08 smithi173 bash[59422]: cluster 2023-09-23T18:44:07.980014+0000 mgr.y (mgr.26054) 777 : cluster 0 pgmap v792: 99 pgs: 1 remapped+peering, 98 active+clean; 5.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 55 MiB/s wr, 887 op/s 2023-09-23T18:44:08.368 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:08 smithi098 bash[79516]: cluster 2023-09-23T18:44:07.089572+0000 osd.7 (osd.7) 113 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T18:44:08.369 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:08 smithi098 bash[79516]: cluster 2023-09-23T18:44:07.089976+0000 mon.a (mon.0) 1044 : cluster 0 osdmap e199: 8 total, 8 up, 7 in 2023-09-23T18:44:08.369 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:08 smithi098 bash[79516]: cluster 2023-09-23T18:44:07.091977+0000 osd.6 (osd.6) 130 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T18:44:08.369 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:08 smithi098 bash[79516]: cluster 2023-09-23T18:44:07.094752+0000 osd.6 (osd.6) 131 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T18:44:08.369 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:08 smithi098 bash[79516]: cluster 2023-09-23T18:44:07.980014+0000 mgr.y (mgr.26054) 777 : cluster 0 pgmap v792: 99 pgs: 1 remapped+peering, 98 active+clean; 5.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 55 MiB/s wr, 887 op/s 2023-09-23T18:44:08.369 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:08 smithi098 bash[76817]: cluster 2023-09-23T18:44:07.089572+0000 osd.7 (osd.7) 113 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T18:44:08.370 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:08 smithi098 bash[76817]: cluster 2023-09-23T18:44:07.089976+0000 mon.a (mon.0) 1044 : cluster 0 osdmap e199: 8 total, 8 up, 7 in 2023-09-23T18:44:08.370 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:08 smithi098 bash[76817]: cluster 2023-09-23T18:44:07.091977+0000 osd.6 (osd.6) 130 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T18:44:08.370 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:08 smithi098 bash[76817]: cluster 2023-09-23T18:44:07.094752+0000 osd.6 (osd.6) 131 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T18:44:08.370 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:08 smithi098 bash[76817]: cluster 2023-09-23T18:44:07.980014+0000 mgr.y (mgr.26054) 777 : cluster 0 pgmap v792: 99 pgs: 1 remapped+peering, 98 active+clean; 5.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 55 MiB/s wr, 887 op/s 2023-09-23T18:44:08.374 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:08.409 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:08.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:08.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:08.513 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:08.541 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:08.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:08.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:08.645 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:08.665 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:44:08.665 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [ 2023-09-23T18:44:08.666 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.666 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.45188.0:83037 7.e e200/187)", 2023-09-23T18:44:08.666 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:08.657271+0000", 2023-09-23T18:44:08.666 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.0054696800000000002, 2023-09-23T18:44:08.666 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0054961749999999998, 2023-09-23T18:44:08.666 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:44:08.666 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "queued for pg", 2023-09-23T18:44:08.666 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:44:08.666 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.666 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:44:08.666 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.657271+0000", 2023-09-23T18:44:08.667 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:44:08.667 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.667 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.667 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:44:08.667 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.657271+0000", 2023-09-23T18:44:08.667 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:44:08.667 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.669 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.669 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:44:08.669 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.657269+0000", 2023-09-23T18:44:08.669 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:44:08.669 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.669 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.670 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:44:08.670 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.657303+0000", 2023-09-23T18:44:08.670 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.349e-05 2023-09-23T18:44:08.670 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.670 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.670 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:44:08.670 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.657317+0000", 2023-09-23T18:44:08.670 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.4314000000000001e-05 2023-09-23T18:44:08.670 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.670 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.671 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:44:08.671 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.657326+0000", 2023-09-23T18:44:08.671 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.8629999999999997e-06 2023-09-23T18:44:08.671 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:44:08.671 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:44:08.671 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:44:08.671 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.671 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.671 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:83043 7.6 7:66354a76:::benchmark_data_smithi098_155798_object83042:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-09-23T18:44:08.671 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:08.661445+0000", 2023-09-23T18:44:08.671 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.0012949630000000001, 2023-09-23T18:44:08.672 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.001385863, 2023-09-23T18:44:08.672 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:44:08.672 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:44:08.672 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:44:08.672 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:44:08.672 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:44:08.672 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 83043 2023-09-23T18:44:08.672 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.672 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:44:08.673 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.673 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:44:08.673 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.661445+0000", 2023-09-23T18:44:08.673 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:44:08.673 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.673 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.673 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:44:08.673 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.661445+0000", 2023-09-23T18:44:08.673 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:44:08.673 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.673 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.674 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:44:08.674 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.661444+0000", 2023-09-23T18:44:08.674 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:44:08.674 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.674 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.674 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:44:08.674 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.661474+0000", 2023-09-23T18:44:08.674 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.0713000000000002e-05 2023-09-23T18:44:08.674 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.674 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.675 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:44:08.675 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.661491+0000", 2023-09-23T18:44:08.675 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.6232999999999999e-05 2023-09-23T18:44:08.675 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.675 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.675 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:44:08.675 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.661500+0000", 2023-09-23T18:44:08.675 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.7850000000000003e-06 2023-09-23T18:44:08.675 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.675 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.675 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:44:08.676 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.661561+0000", 2023-09-23T18:44:08.676 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.0922999999999999e-05 2023-09-23T18:44:08.676 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.676 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.676 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:44:08.676 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.661743+0000", 2023-09-23T18:44:08.676 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00018169799999999999 2023-09-23T18:44:08.676 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.676 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.676 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 1,7", 2023-09-23T18:44:08.676 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.661894+0000", 2023-09-23T18:44:08.677 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00015144400000000001 2023-09-23T18:44:08.677 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:44:08.677 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:44:08.677 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:44:08.677 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.677 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.677 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.45188.0:83014 7.d e200/187 7:b36c32fb:::benchmark_data_smithi098_155798_object83013:head v 200'5100, mlcod=200'5098)", 2023-09-23T18:44:08.677 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:08.632039+0000", 2023-09-23T18:44:08.677 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.030700924000000001, 2023-09-23T18:44:08.677 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.030872652, 2023-09-23T18:44:08.677 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:44:08.678 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T18:44:08.678 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:44:08.678 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.678 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:44:08.678 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.632039+0000", 2023-09-23T18:44:08.678 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:44:08.678 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.678 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.678 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:44:08.679 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.632039+0000", 2023-09-23T18:44:08.679 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:44:08.679 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.679 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.679 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:44:08.679 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.632038+0000", 2023-09-23T18:44:08.679 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:44:08.679 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.679 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.680 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:44:08.680 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.632092+0000", 2023-09-23T18:44:08.680 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.3905e-05 2023-09-23T18:44:08.680 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.680 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.680 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:44:08.680 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.632106+0000", 2023-09-23T18:44:08.680 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.4329e-05 2023-09-23T18:44:08.680 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.680 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.681 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:44:08.681 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.632128+0000", 2023-09-23T18:44:08.681 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.1263999999999998e-05 2023-09-23T18:44:08.681 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.681 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.681 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:44:08.681 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.634349+0000", 2023-09-23T18:44:08.681 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.002221607 2023-09-23T18:44:08.681 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.682 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.682 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:44:08.682 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.634371+0000", 2023-09-23T18:44:08.682 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.1869e-05 2023-09-23T18:44:08.682 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:44:08.682 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:44:08.682 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:44:08.682 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.682 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.682 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop_reply(client.45188.0:83030 7.6 e200/195)", 2023-09-23T18:44:08.683 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:08.661526+0000", 2023-09-23T18:44:08.683 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.0012144250000000001, 2023-09-23T18:44:08.683 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.001443656, 2023-09-23T18:44:08.683 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:44:08.683 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "queued for pg", 2023-09-23T18:44:08.683 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:44:08.683 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.683 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:44:08.683 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.661526+0000", 2023-09-23T18:44:08.683 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:44:08.683 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.683 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.683 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:44:08.683 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.661526+0000", 2023-09-23T18:44:08.684 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:44:08.684 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.684 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.684 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:44:08.684 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.661525+0000", 2023-09-23T18:44:08.684 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:44:08.684 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.684 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.684 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:44:08.684 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.661529+0000", 2023-09-23T18:44:08.684 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.9349999999999996e-06 2023-09-23T18:44:08.684 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.685 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.685 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:44:08.685 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.661538+0000", 2023-09-23T18:44:08.685 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.7360000000000007e-06 2023-09-23T18:44:08.685 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.685 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.685 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:44:08.685 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.661545+0000", 2023-09-23T18:44:08.685 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.5690000000000001e-06 2023-09-23T18:44:08.685 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:44:08.685 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:44:08.685 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:44:08.685 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.686 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.686 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.45188.0:83019 7.5 e200/195 7:a61a52c4:::benchmark_data_smithi098_155798_object83018:head v 200'4931, mlcod=200'4929)", 2023-09-23T18:44:08.686 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:08.633756+0000", 2023-09-23T18:44:08.686 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.028984143, 2023-09-23T18:44:08.686 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.029268974999999999, 2023-09-23T18:44:08.686 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:44:08.686 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:44:08.686 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:44:08.686 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.686 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:44:08.686 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.633756+0000", 2023-09-23T18:44:08.686 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:44:08.687 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.687 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.687 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:44:08.687 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.633756+0000", 2023-09-23T18:44:08.687 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:44:08.687 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.687 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.687 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:44:08.687 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.633755+0000", 2023-09-23T18:44:08.687 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:44:08.687 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.687 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.687 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:44:08.688 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.634049+0000", 2023-09-23T18:44:08.688 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00029406000000000001 2023-09-23T18:44:08.688 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.688 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.688 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:44:08.688 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.634073+0000", 2023-09-23T18:44:08.688 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.3640999999999999e-05 2023-09-23T18:44:08.688 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.688 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.688 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:44:08.688 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.634083+0000", 2023-09-23T18:44:08.688 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.029e-05 2023-09-23T18:44:08.688 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.689 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.689 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:44:08.689 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.635875+0000", 2023-09-23T18:44:08.689 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0017918420000000001 2023-09-23T18:44:08.689 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.689 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.689 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:44:08.689 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.635895+0000", 2023-09-23T18:44:08.689 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.0336999999999999e-05 2023-09-23T18:44:08.689 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.689 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.689 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:44:08.689 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.659921+0000", 2023-09-23T18:44:08.689 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.024025560000000001 2023-09-23T18:44:08.690 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:44:08.690 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:44:08.690 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:44:08.690 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.690 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.690 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.45188.0:83016 7.d e200/187 7:bbe4ce79:::benchmark_data_smithi098_155798_object83015:head v 200'5101, mlcod=200'5098)", 2023-09-23T18:44:08.690 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:08.635640+0000", 2023-09-23T18:44:08.690 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.027100085999999999, 2023-09-23T18:44:08.690 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.027460308999999999, 2023-09-23T18:44:08.690 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:44:08.690 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T18:44:08.690 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:44:08.691 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.691 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:44:08.691 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.635640+0000", 2023-09-23T18:44:08.691 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:44:08.691 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.691 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.691 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:44:08.691 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.635640+0000", 2023-09-23T18:44:08.691 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:44:08.691 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.691 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.691 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:44:08.691 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.635639+0000", 2023-09-23T18:44:08.692 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:44:08.692 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.692 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.692 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:44:08.692 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.635686+0000", 2023-09-23T18:44:08.692 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.6730000000000002e-05 2023-09-23T18:44:08.692 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.692 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.692 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:44:08.692 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.635704+0000", 2023-09-23T18:44:08.692 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.8097e-05 2023-09-23T18:44:08.692 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.692 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.692 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:44:08.693 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.635712+0000", 2023-09-23T18:44:08.693 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.1720000000000007e-06 2023-09-23T18:44:08.693 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.693 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.693 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:44:08.693 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.647320+0000", 2023-09-23T18:44:08.693 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.011607777999999999 2023-09-23T18:44:08.693 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.693 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.693 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:44:08.693 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.647343+0000", 2023-09-23T18:44:08.693 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.2857000000000002e-05 2023-09-23T18:44:08.694 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:44:08.694 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:44:08.694 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:44:08.694 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.694 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.694 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.45188.0:83020 7.d e200/187 7:b9e51f28:::benchmark_data_smithi098_155798_object83019:head v 200'5102, mlcod=200'5098)", 2023-09-23T18:44:08.694 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:08.636383+0000", 2023-09-23T18:44:08.694 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.026357225000000001, 2023-09-23T18:44:08.694 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.026783319, 2023-09-23T18:44:08.694 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:44:08.694 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T18:44:08.694 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:44:08.695 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.695 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:44:08.695 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.636383+0000", 2023-09-23T18:44:08.695 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:44:08.695 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.695 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.695 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:44:08.695 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.636383+0000", 2023-09-23T18:44:08.695 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:44:08.695 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.695 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.695 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:44:08.695 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.636382+0000", 2023-09-23T18:44:08.696 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:44:08.696 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.696 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.696 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:44:08.696 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.636430+0000", 2023-09-23T18:44:08.696 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.8056000000000002e-05 2023-09-23T18:44:08.696 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.696 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.696 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:44:08.696 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.636443+0000", 2023-09-23T18:44:08.696 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.2577e-05 2023-09-23T18:44:08.696 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.696 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.697 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:44:08.697 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.636450+0000", 2023-09-23T18:44:08.697 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.7240000000000003e-06 2023-09-23T18:44:08.697 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.697 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.697 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:44:08.697 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.654814+0000", 2023-09-23T18:44:08.697 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.018364577999999999 2023-09-23T18:44:08.697 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.697 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.697 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:44:08.697 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.654837+0000", 2023-09-23T18:44:08.697 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.2597000000000001e-05 2023-09-23T18:44:08.698 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:44:08.698 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:44:08.698 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:44:08.698 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.698 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.698 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.45188.0:83024 7.d e200/187 7:b0f7b810:::benchmark_data_smithi098_155798_object83023:head v 200'5103, mlcod=200'5098)", 2023-09-23T18:44:08.698 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:08.642096+0000", 2023-09-23T18:44:08.698 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.020644101000000002, 2023-09-23T18:44:08.698 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.021136551999999999, 2023-09-23T18:44:08.698 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:44:08.698 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T18:44:08.698 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:44:08.699 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.699 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:44:08.699 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.642096+0000", 2023-09-23T18:44:08.699 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:44:08.699 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.699 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.699 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:44:08.699 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.642096+0000", 2023-09-23T18:44:08.699 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:44:08.699 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.699 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.699 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:44:08.699 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.642095+0000", 2023-09-23T18:44:08.700 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:44:08.700 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.700 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.700 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:44:08.700 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.642148+0000", 2023-09-23T18:44:08.700 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.2463999999999998e-05 2023-09-23T18:44:08.700 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.700 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.700 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:44:08.700 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.642161+0000", 2023-09-23T18:44:08.700 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3831e-05 2023-09-23T18:44:08.700 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.700 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.701 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:44:08.701 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.642172+0000", 2023-09-23T18:44:08.701 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0213e-05 2023-09-23T18:44:08.701 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.701 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.701 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:44:08.701 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.657268+0000", 2023-09-23T18:44:08.701 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.015096416 2023-09-23T18:44:08.701 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.701 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.702 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:44:08.702 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.657301+0000", 2023-09-23T18:44:08.702 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.2920999999999999e-05 2023-09-23T18:44:08.702 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:44:08.702 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:44:08.702 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:44:08.702 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.702 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.702 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:83030 7.6 7:63411bfc:::benchmark_data_smithi098_155798_object83029:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-09-23T18:44:08.703 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:08.646179+0000", 2023-09-23T18:44:08.703 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.016561402999999999, 2023-09-23T18:44:08.703 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.017127880000000002, 2023-09-23T18:44:08.703 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:44:08.703 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:44:08.703 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:44:08.703 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:44:08.703 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:44:08.703 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 83030 2023-09-23T18:44:08.703 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.703 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:44:08.703 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.703 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:44:08.704 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.646179+0000", 2023-09-23T18:44:08.704 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:44:08.704 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.704 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.704 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:44:08.704 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.646179+0000", 2023-09-23T18:44:08.704 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:44:08.704 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.705 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.705 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:44:08.705 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.646177+0000", 2023-09-23T18:44:08.705 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T18:44:08.705 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.705 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.705 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:44:08.705 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.646230+0000", 2023-09-23T18:44:08.705 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.2497999999999998e-05 2023-09-23T18:44:08.705 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.705 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.705 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:44:08.705 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.646243+0000", 2023-09-23T18:44:08.706 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3657e-05 2023-09-23T18:44:08.706 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.706 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.706 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:44:08.706 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.646252+0000", 2023-09-23T18:44:08.706 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.4549999999999995e-06 2023-09-23T18:44:08.706 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.706 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.706 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:44:08.707 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.646347+0000", 2023-09-23T18:44:08.707 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.4837e-05 2023-09-23T18:44:08.707 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.707 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.707 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:44:08.707 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.646532+0000", 2023-09-23T18:44:08.707 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000185007 2023-09-23T18:44:08.707 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.707 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.707 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 1,7", 2023-09-23T18:44:08.707 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.646705+0000", 2023-09-23T18:44:08.707 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00017299899999999999 2023-09-23T18:44:08.708 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.708 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.708 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:44:08.708 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.654233+0000", 2023-09-23T18:44:08.708 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0075280199999999999 2023-09-23T18:44:08.708 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.708 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.708 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:44:08.708 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.654240+0000", 2023-09-23T18:44:08.708 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.2949999999999996e-06 2023-09-23T18:44:08.708 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:44:08.708 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:44:08.708 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:44:08.709 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.709 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.709 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:83033 7.4 7:243b66fc:::benchmark_data_smithi098_155798_object83032:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-09-23T18:44:08.709 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:08.649690+0000", 2023-09-23T18:44:08.709 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.013050533, 2023-09-23T18:44:08.709 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.013717729999999999, 2023-09-23T18:44:08.709 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:44:08.709 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:44:08.709 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:44:08.709 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:44:08.709 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:44:08.710 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 83033 2023-09-23T18:44:08.710 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.710 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:44:08.710 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.710 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:44:08.710 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.649690+0000", 2023-09-23T18:44:08.710 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:44:08.710 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.710 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.711 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:44:08.711 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.649690+0000", 2023-09-23T18:44:08.711 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:44:08.711 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.711 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.711 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:44:08.711 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.649688+0000", 2023-09-23T18:44:08.711 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:44:08.711 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.711 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.711 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:44:08.712 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.649754+0000", 2023-09-23T18:44:08.712 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.5809999999999995e-05 2023-09-23T18:44:08.712 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.712 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.712 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:44:08.712 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.649772+0000", 2023-09-23T18:44:08.712 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.7436e-05 2023-09-23T18:44:08.712 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.712 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.713 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:44:08.713 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.649781+0000", 2023-09-23T18:44:08.713 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.0450000000000006e-06 2023-09-23T18:44:08.713 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.713 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.713 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:44:08.713 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.650177+0000", 2023-09-23T18:44:08.713 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00039691100000000001 2023-09-23T18:44:08.713 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.713 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.714 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:44:08.714 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.650275+0000", 2023-09-23T18:44:08.714 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.7782999999999995e-05 2023-09-23T18:44:08.714 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.714 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.714 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:44:08.714 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.650448+0000", 2023-09-23T18:44:08.714 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000172855 2023-09-23T18:44:08.714 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.714 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.715 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:44:08.715 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.654260+0000", 2023-09-23T18:44:08.715 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0038123200000000001 2023-09-23T18:44:08.715 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.715 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.715 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:44:08.715 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.656793+0000", 2023-09-23T18:44:08.715 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.002532641 2023-09-23T18:44:08.715 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:44:08.715 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:44:08.716 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:44:08.716 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.716 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.716 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:83035 7.4 7:24bd52f2:::benchmark_data_smithi098_155798_object83034:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-09-23T18:44:08.716 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:08.650323+0000", 2023-09-23T18:44:08.716 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.012417638, 2023-09-23T18:44:08.716 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.013184859, 2023-09-23T18:44:08.716 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:44:08.716 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:44:08.717 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:44:08.717 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:44:08.717 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:44:08.717 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 83035 2023-09-23T18:44:08.717 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.717 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:44:08.717 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.717 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:44:08.717 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.650323+0000", 2023-09-23T18:44:08.717 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:44:08.717 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.718 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.718 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:44:08.718 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.650323+0000", 2023-09-23T18:44:08.718 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:44:08.718 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.718 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.718 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:44:08.718 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.650322+0000", 2023-09-23T18:44:08.718 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:44:08.718 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.718 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.718 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:44:08.718 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.650345+0000", 2023-09-23T18:44:08.719 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.3028e-05 2023-09-23T18:44:08.719 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.719 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.719 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:44:08.719 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.650355+0000", 2023-09-23T18:44:08.719 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.9380000000000004e-06 2023-09-23T18:44:08.719 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.719 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.719 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:44:08.719 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.650360+0000", 2023-09-23T18:44:08.719 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.6740000000000002e-06 2023-09-23T18:44:08.719 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.719 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.720 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:44:08.720 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.652355+0000", 2023-09-23T18:44:08.720 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0019943880000000002 2023-09-23T18:44:08.720 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.720 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.720 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:44:08.720 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.652491+0000", 2023-09-23T18:44:08.720 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00013624199999999999 2023-09-23T18:44:08.720 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.720 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.720 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:44:08.720 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.652623+0000", 2023-09-23T18:44:08.720 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000132442 2023-09-23T18:44:08.721 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.721 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.721 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:44:08.721 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.654402+0000", 2023-09-23T18:44:08.721 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0017788719999999999 2023-09-23T18:44:08.721 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.721 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.721 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:44:08.721 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.657025+0000", 2023-09-23T18:44:08.721 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.002622362 2023-09-23T18:44:08.721 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:44:08.721 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:44:08.721 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:44:08.722 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.722 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.722 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.45188.0:83036 7.e e200/187 7:78dab8e8:::benchmark_data_smithi098_155798_object83035:head v 200'5287, mlcod=200'5285)", 2023-09-23T18:44:08.722 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:08.651412+0000", 2023-09-23T18:44:08.722 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.011327719999999999, 2023-09-23T18:44:08.722 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.012178641, 2023-09-23T18:44:08.722 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:44:08.722 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T18:44:08.722 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:44:08.722 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.722 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:44:08.722 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.651412+0000", 2023-09-23T18:44:08.723 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:44:08.723 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.723 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.723 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:44:08.723 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.651412+0000", 2023-09-23T18:44:08.723 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:44:08.723 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.723 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.723 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:44:08.723 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.651411+0000", 2023-09-23T18:44:08.723 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:44:08.723 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.723 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.724 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:44:08.724 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.651447+0000", 2023-09-23T18:44:08.724 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.5179000000000003e-05 2023-09-23T18:44:08.724 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.724 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.724 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:44:08.724 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.651460+0000", 2023-09-23T18:44:08.724 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3645999999999999e-05 2023-09-23T18:44:08.724 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.724 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.724 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:44:08.724 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.651469+0000", 2023-09-23T18:44:08.725 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.5280000000000003e-06 2023-09-23T18:44:08.725 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.725 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.725 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:44:08.725 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.651672+0000", 2023-09-23T18:44:08.725 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000203544 2023-09-23T18:44:08.725 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:44:08.725 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:44:08.725 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:44:08.725 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:08.651691+0000", 2023-09-23T18:44:08.725 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.8635e-05 2023-09-23T18:44:08.725 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:44:08.725 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:44:08.726 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:44:08.726 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:44:08.726 INFO:teuthology.orchestra.run.smithi098.stdout: ], 2023-09-23T18:44:08.726 INFO:teuthology.orchestra.run.smithi098.stdout: "num_ops": 12 2023-09-23T18:44:08.726 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:44:08.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:08.727 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:08.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:08.828 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:08.852 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:08.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:08.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:08.935 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:44:08.956 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:09.012 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:09.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:09.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:09.116 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:09.154 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:09.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:09.157 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:09 smithi098 bash[76817]: cluster 2023-09-23T18:44:07.819956+0000 osd.4 (osd.4) 159 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T18:44:09.157 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:09 smithi098 bash[76817]: cluster 2023-09-23T18:44:07.820054+0000 osd.5 (osd.5) 96 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T18:44:09.157 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:09 smithi098 bash[76817]: cluster 2023-09-23T18:44:08.099621+0000 osd.6 (osd.6) 132 : cluster 3 failed to encode map e200 with expected crc 2023-09-23T18:44:09.157 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:09 smithi098 bash[76817]: cluster 2023-09-23T18:44:08.101604+0000 mon.a (mon.0) 1045 : cluster 0 osdmap e200: 8 total, 8 up, 7 in 2023-09-23T18:44:09.157 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:09 smithi098 bash[76817]: cluster 2023-09-23T18:44:08.274737+0000 osd.7 (osd.7) 114 : cluster 3 failed to encode map e200 with expected crc 2023-09-23T18:44:09.157 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:09 smithi098 bash[76817]: cluster 2023-09-23T18:44:08.277267+0000 osd.7 (osd.7) 115 : cluster 3 failed to encode map e200 with expected crc 2023-09-23T18:44:09.157 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:09 smithi098 bash[76817]: cluster 2023-09-23T18:44:08.614743+0000 osd.5 (osd.5) 97 : cluster 3 failed to encode map e200 with expected crc 2023-09-23T18:44:09.158 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:09 smithi098 bash[79516]: cluster 2023-09-23T18:44:07.819956+0000 osd.4 (osd.4) 159 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T18:44:09.158 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:09 smithi098 bash[79516]: cluster 2023-09-23T18:44:07.820054+0000 osd.5 (osd.5) 96 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T18:44:09.158 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:09 smithi098 bash[79516]: cluster 2023-09-23T18:44:08.099621+0000 osd.6 (osd.6) 132 : cluster 3 failed to encode map e200 with expected crc 2023-09-23T18:44:09.158 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:09 smithi098 bash[79516]: cluster 2023-09-23T18:44:08.101604+0000 mon.a (mon.0) 1045 : cluster 0 osdmap e200: 8 total, 8 up, 7 in 2023-09-23T18:44:09.158 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:09 smithi098 bash[79516]: cluster 2023-09-23T18:44:08.274737+0000 osd.7 (osd.7) 114 : cluster 3 failed to encode map e200 with expected crc 2023-09-23T18:44:09.158 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:09 smithi098 bash[79516]: cluster 2023-09-23T18:44:08.277267+0000 osd.7 (osd.7) 115 : cluster 3 failed to encode map e200 with expected crc 2023-09-23T18:44:09.158 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:09 smithi098 bash[79516]: cluster 2023-09-23T18:44:08.614743+0000 osd.5 (osd.5) 97 : cluster 3 failed to encode map e200 with expected crc 2023-09-23T18:44:09.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:09.260 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:09.273 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 67 16 83630 83614 77.8682 25.7188 0.0138699 0.0128365 2023-09-23T18:44:09.286 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:09.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:09.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:09.391 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:09.424 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:09.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:09.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:09.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:09 smithi173 bash[59422]: cluster 2023-09-23T18:44:07.819956+0000 osd.4 (osd.4) 159 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T18:44:09.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:09 smithi173 bash[59422]: cluster 2023-09-23T18:44:07.820054+0000 osd.5 (osd.5) 96 : cluster 3 failed to encode map e199 with expected crc 2023-09-23T18:44:09.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:09 smithi173 bash[59422]: cluster 2023-09-23T18:44:08.099621+0000 osd.6 (osd.6) 132 : cluster 3 failed to encode map e200 with expected crc 2023-09-23T18:44:09.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:09 smithi173 bash[59422]: cluster 2023-09-23T18:44:08.101604+0000 mon.a (mon.0) 1045 : cluster 0 osdmap e200: 8 total, 8 up, 7 in 2023-09-23T18:44:09.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:09 smithi173 bash[59422]: cluster 2023-09-23T18:44:08.274737+0000 osd.7 (osd.7) 114 : cluster 3 failed to encode map e200 with expected crc 2023-09-23T18:44:09.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:09 smithi173 bash[59422]: cluster 2023-09-23T18:44:08.277267+0000 osd.7 (osd.7) 115 : cluster 3 failed to encode map e200 with expected crc 2023-09-23T18:44:09.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:09 smithi173 bash[59422]: cluster 2023-09-23T18:44:08.614743+0000 osd.5 (osd.5) 97 : cluster 3 failed to encode map e200 with expected crc 2023-09-23T18:44:09.529 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:09.562 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:09.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:09.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:09.668 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:09.706 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:09.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:09.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:09.814 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:09.846 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:09.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:09.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:09.949 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:09.982 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:09.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:09.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:10.078 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T18:44:10.088 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:10.116 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:10.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:10.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:10.223 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:10.251 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:10.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:10.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:10.273 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 68 16 84599 84583 77.614 60.5625 0.0147891 0.0128774 2023-09-23T18:44:10.355 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:10.366 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:10 smithi173 bash[59422]: cluster 2023-09-23T18:44:08.380753+0000 osd.4 (osd.4) 160 : cluster 3 failed to encode map e200 with expected crc 2023-09-23T18:44:10.366 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:10 smithi173 bash[59422]: audit 2023-09-23T18:44:09.607806+0000 mon.a (mon.0) 1046 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:44:10.367 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:10 smithi173 bash[59422]: cluster 2023-09-23T18:44:09.980593+0000 mgr.y (mgr.26054) 778 : cluster 0 pgmap v794: 99 pgs: 1 remapped+peering, 98 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 53 MiB/s wr, 844 op/s 2023-09-23T18:44:10.382 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:10.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:10.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:10.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:10 smithi098 bash[79516]: cluster 2023-09-23T18:44:08.380753+0000 osd.4 (osd.4) 160 : cluster 3 failed to encode map e200 with expected crc 2023-09-23T18:44:10.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:10 smithi098 bash[79516]: audit 2023-09-23T18:44:09.607806+0000 mon.a (mon.0) 1046 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:44:10.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:10 smithi098 bash[79516]: cluster 2023-09-23T18:44:09.980593+0000 mgr.y (mgr.26054) 778 : cluster 0 pgmap v794: 99 pgs: 1 remapped+peering, 98 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 53 MiB/s wr, 844 op/s 2023-09-23T18:44:10.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:10 smithi098 bash[76817]: cluster 2023-09-23T18:44:08.380753+0000 osd.4 (osd.4) 160 : cluster 3 failed to encode map e200 with expected crc 2023-09-23T18:44:10.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:10 smithi098 bash[76817]: audit 2023-09-23T18:44:09.607806+0000 mon.a (mon.0) 1046 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:44:10.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:10 smithi098 bash[76817]: cluster 2023-09-23T18:44:09.980593+0000 mgr.y (mgr.26054) 778 : cluster 0 pgmap v794: 99 pgs: 1 remapped+peering, 98 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 53 MiB/s wr, 844 op/s 2023-09-23T18:44:10.486 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:10.515 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:10.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:10.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:10.558 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:44:10.621 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:10.647 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:10.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:10.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:10.750 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:10.781 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:10.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:10.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:10.886 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:10.912 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:10.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:10.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:11.015 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:11.041 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:11.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:11.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:11.145 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:11.171 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:11.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:11.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:11.273 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 69 16 85567 85551 77.3663 60.5 0.0115166 0.0129187 2023-09-23T18:44:11.274 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:11.303 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:11.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:11.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:11.407 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:11.416 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:11 smithi173 bash[59422]: audit 2023-09-23T18:44:10.163087+0000 mon.a (mon.0) 1047 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:44:11.432 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:11.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:11.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:11.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:11 smithi098 bash[76817]: audit 2023-09-23T18:44:10.163087+0000 mon.a (mon.0) 1047 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:44:11.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:11 smithi098 bash[79516]: audit 2023-09-23T18:44:10.163087+0000 mon.a (mon.0) 1047 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:44:11.536 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:11.564 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:11.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:11.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:11.668 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:11.695 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:11.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:11.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:11.799 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:11.829 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:11.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:11.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:11.934 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:11.959 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:11.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:11.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:12.061 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:12.086 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:12.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:12.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:12.172 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:12 smithi098 bash[76817]: cluster 2023-09-23T18:44:11.981175+0000 mgr.y (mgr.26054) 779 : cluster 0 pgmap v795: 99 pgs: 1 remapped+peering, 98 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 49 MiB/s wr, 776 op/s 2023-09-23T18:44:12.190 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:12.221 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:12.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:12.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:12.273 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 70 16 86548 86532 77.1372 61.3125 0.0106812 0.0129582 2023-09-23T18:44:12.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:12 smithi173 bash[59422]: cluster 2023-09-23T18:44:11.981175+0000 mgr.y (mgr.26054) 779 : cluster 0 pgmap v795: 99 pgs: 1 remapped+peering, 98 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 49 MiB/s wr, 776 op/s 2023-09-23T18:44:12.325 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:12.356 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:12.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:12.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:12.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:12 smithi098 bash[79516]: cluster 2023-09-23T18:44:11.981175+0000 mgr.y (mgr.26054) 779 : cluster 0 pgmap v795: 99 pgs: 1 remapped+peering, 98 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 49 MiB/s wr, 776 op/s 2023-09-23T18:44:12.462 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:12.494 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:12.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:12.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:12.598 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:44:12.599 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:12.645 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:12.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:12.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:12.749 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:12.775 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:12.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:12.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:12.878 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:12.912 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:12.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:12.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:13.016 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:13.043 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:13.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:13.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:13.146 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:13.175 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:13.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:13.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:13.184 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:13 smithi098 bash[76817]: audit 2023-09-23T18:44:12.985986+0000 mon.a (mon.0) 1048 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:44:13.273 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 71 16 87515 87499 76.9023 60.4375 0.0104575 0.0129981 2023-09-23T18:44:13.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:13 smithi173 bash[59422]: audit 2023-09-23T18:44:12.985986+0000 mon.a (mon.0) 1048 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:44:13.279 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:13.304 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:13.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:13.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:13.411 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:13.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:13 smithi098 bash[79516]: audit 2023-09-23T18:44:12.985986+0000 mon.a (mon.0) 1048 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:44:13.562 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:13.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:13.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:13.667 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:13.697 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:13.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:13.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:13.805 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:13.831 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:13.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:13.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:13.934 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:13.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:13.967 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:13.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:14.068 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:14.093 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:14.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:14.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:14.199 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:14.235 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:14.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:14.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:14.273 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 72 16 88509 88493 76.6973 62.125 0.0101101 0.0130328 2023-09-23T18:44:14.341 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:14.352 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:14 smithi173 bash[59422]: cluster 2023-09-23T18:44:13.981735+0000 mgr.y (mgr.26054) 780 : cluster 0 pgmap v796: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 50 MiB/s wr, 805 op/s; 4928/257562 objects misplaced (1.913%); 843 KiB/s, 13 objects/s recovering 2023-09-23T18:44:14.352 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:14 smithi173 bash[59422]: audit 2023-09-23T18:44:13.986298+0000 mon.a (mon.0) 1049 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T18:44:14.364 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:14.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:14.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:14.467 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:14.484 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:14 smithi098 bash[79516]: cluster 2023-09-23T18:44:13.981735+0000 mgr.y (mgr.26054) 780 : cluster 0 pgmap v796: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 50 MiB/s wr, 805 op/s; 4928/257562 objects misplaced (1.913%); 843 KiB/s, 13 objects/s recovering 2023-09-23T18:44:14.484 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:14 smithi098 bash[79516]: audit 2023-09-23T18:44:13.986298+0000 mon.a (mon.0) 1049 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T18:44:14.484 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:14 smithi098 bash[76817]: cluster 2023-09-23T18:44:13.981735+0000 mgr.y (mgr.26054) 780 : cluster 0 pgmap v796: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 50 MiB/s wr, 805 op/s; 4928/257562 objects misplaced (1.913%); 843 KiB/s, 13 objects/s recovering 2023-09-23T18:44:14.484 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:14 smithi098 bash[76817]: audit 2023-09-23T18:44:13.986298+0000 mon.a (mon.0) 1049 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T18:44:14.502 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:14.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:14.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:14.606 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:14.637 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:14.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:14.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:14.741 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:14.766 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:14.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:14.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:14.870 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:14.895 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:14.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:14.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:14.998 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:15.035 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:15.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:15.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:15.138 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:15.161 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:15.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:15.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:15.264 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:15.273 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 73 16 88837 88821 75.9285 20.5 0.00613725 0.0130261 2023-09-23T18:44:15.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:15 smithi173 bash[59422]: cluster 2023-09-23T18:44:14.209290+0000 osd.7 (osd.7) 116 : cluster 3 failed to encode map e201 with expected crc 2023-09-23T18:44:15.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:15 smithi173 bash[59422]: cluster 2023-09-23T18:44:14.211813+0000 osd.6 (osd.6) 133 : cluster 3 failed to encode map e201 with expected crc 2023-09-23T18:44:15.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:15 smithi173 bash[59422]: audit 2023-09-23T18:44:14.214886+0000 mon.a (mon.0) 1050 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T18:44:15.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:15 smithi173 bash[59422]: cluster 2023-09-23T18:44:14.215138+0000 mon.a (mon.0) 1051 : cluster 0 osdmap e201: 8 total, 8 up, 7 in 2023-09-23T18:44:15.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:15 smithi173 bash[59422]: cluster 2023-09-23T18:44:14.215694+0000 osd.6 (osd.6) 134 : cluster 3 failed to encode map e201 with expected crc 2023-09-23T18:44:15.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:15 smithi173 bash[59422]: cluster 2023-09-23T18:44:14.215757+0000 osd.4 (osd.4) 161 : cluster 3 failed to encode map e201 with expected crc 2023-09-23T18:44:15.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:15 smithi173 bash[59422]: cluster 2023-09-23T18:44:14.439260+0000 osd.5 (osd.5) 98 : cluster 3 failed to encode map e201 with expected crc 2023-09-23T18:44:15.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:15 smithi173 bash[59422]: cluster 2023-09-23T18:44:15.046994+0000 mon.a (mon.0) 1052 : cluster 0 osdmap e202: 8 total, 8 up, 7 in 2023-09-23T18:44:15.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:15 smithi173 bash[59422]: cluster 2023-09-23T18:44:15.051698+0000 osd.4 (osd.4) 162 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T18:44:15.287 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:15.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:15.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:15.392 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:15.415 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:15.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:15.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:15.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:15 smithi098 bash[76817]: cluster 2023-09-23T18:44:14.209290+0000 osd.7 (osd.7) 116 : cluster 3 failed to encode map e201 with expected crc 2023-09-23T18:44:15.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:15 smithi098 bash[76817]: cluster 2023-09-23T18:44:14.211813+0000 osd.6 (osd.6) 133 : cluster 3 failed to encode map e201 with expected crc 2023-09-23T18:44:15.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:15 smithi098 bash[76817]: audit 2023-09-23T18:44:14.214886+0000 mon.a (mon.0) 1050 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T18:44:15.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:15 smithi098 bash[76817]: cluster 2023-09-23T18:44:14.215138+0000 mon.a (mon.0) 1051 : cluster 0 osdmap e201: 8 total, 8 up, 7 in 2023-09-23T18:44:15.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:15 smithi098 bash[76817]: cluster 2023-09-23T18:44:14.215694+0000 osd.6 (osd.6) 134 : cluster 3 failed to encode map e201 with expected crc 2023-09-23T18:44:15.447 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:15 smithi098 bash[76817]: cluster 2023-09-23T18:44:14.215757+0000 osd.4 (osd.4) 161 : cluster 3 failed to encode map e201 with expected crc 2023-09-23T18:44:15.518 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:15.540 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:15.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:15.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:15.643 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:15.674 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:15.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:15.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:15.777 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:15.785 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:15 smithi098 bash[76817]: cluster 2023-09-23T18:44:14.439260+0000 osd.5 (osd.5) 98 : cluster 3 failed to encode map e201 with expected crc 2023-09-23T18:44:15.785 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:15 smithi098 bash[76817]: cluster 2023-09-23T18:44:15.046994+0000 mon.a (mon.0) 1052 : cluster 0 osdmap e202: 8 total, 8 up, 7 in 2023-09-23T18:44:15.785 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:15 smithi098 bash[76817]: cluster 2023-09-23T18:44:15.051698+0000 osd.4 (osd.4) 162 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T18:44:15.786 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:15 smithi098 bash[79516]: cluster 2023-09-23T18:44:14.209290+0000 osd.7 (osd.7) 116 : cluster 3 failed to encode map e201 with expected crc 2023-09-23T18:44:15.786 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:15 smithi098 bash[79516]: cluster 2023-09-23T18:44:14.211813+0000 osd.6 (osd.6) 133 : cluster 3 failed to encode map e201 with expected crc 2023-09-23T18:44:15.786 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:15 smithi098 bash[79516]: audit 2023-09-23T18:44:14.214886+0000 mon.a (mon.0) 1050 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T18:44:15.786 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:15 smithi098 bash[79516]: cluster 2023-09-23T18:44:14.215138+0000 mon.a (mon.0) 1051 : cluster 0 osdmap e201: 8 total, 8 up, 7 in 2023-09-23T18:44:15.786 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:15 smithi098 bash[79516]: cluster 2023-09-23T18:44:14.215694+0000 osd.6 (osd.6) 134 : cluster 3 failed to encode map e201 with expected crc 2023-09-23T18:44:15.786 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:15 smithi098 bash[79516]: cluster 2023-09-23T18:44:14.215757+0000 osd.4 (osd.4) 161 : cluster 3 failed to encode map e201 with expected crc 2023-09-23T18:44:15.786 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:15 smithi098 bash[79516]: cluster 2023-09-23T18:44:14.439260+0000 osd.5 (osd.5) 98 : cluster 3 failed to encode map e201 with expected crc 2023-09-23T18:44:15.786 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:15 smithi098 bash[79516]: cluster 2023-09-23T18:44:15.046994+0000 mon.a (mon.0) 1052 : cluster 0 osdmap e202: 8 total, 8 up, 7 in 2023-09-23T18:44:15.786 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:15 smithi098 bash[79516]: cluster 2023-09-23T18:44:15.051698+0000 osd.4 (osd.4) 162 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T18:44:15.799 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:15.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:15.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:15.902 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:15.935 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:15.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:15.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:16.038 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:16.068 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:16.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:16.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:16.172 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:16.198 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:16.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:16.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:16.273 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 74 16 88837 88821 74.9039 0 - 0.0130261 2023-09-23T18:44:16.302 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:16.328 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:16.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:16.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:16.431 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:16.439 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:16 smithi173 bash[59422]: cluster 2023-09-23T18:44:15.046271+0000 osd.6 (osd.6) 135 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T18:44:16.439 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:16 smithi173 bash[59422]: cluster 2023-09-23T18:44:15.048318+0000 osd.7 (osd.7) 117 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T18:44:16.439 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:16 smithi173 bash[59422]: cluster 2023-09-23T18:44:15.051689+0000 osd.7 (osd.7) 118 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T18:44:16.440 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:16 smithi173 bash[59422]: cluster 2023-09-23T18:44:15.052137+0000 osd.6 (osd.6) 136 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T18:44:16.440 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:16 smithi173 bash[59422]: cluster 2023-09-23T18:44:15.982485+0000 mgr.y (mgr.26054) 781 : cluster 0 pgmap v799: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 41 MiB/s wr, 651 op/s; 4928/262212 objects misplaced (1.879%); 844 KiB/s, 13 objects/s recovering 2023-09-23T18:44:16.440 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:16 smithi173 bash[59422]: audit 2023-09-23T18:44:15.983518+0000 mon.a (mon.0) 1053 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T18:44:16.440 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:16 smithi173 bash[59422]: audit 2023-09-23T18:44:16.059549+0000 mon.a (mon.0) 1054 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-23T18:44:16.440 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:16 smithi173 bash[59422]: cluster 2023-09-23T18:44:16.059688+0000 mon.a (mon.0) 1055 : cluster 0 osdmap e203: 8 total, 8 up, 7 in 2023-09-23T18:44:16.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:16 smithi098 bash[76817]: cluster 2023-09-23T18:44:15.046271+0000 osd.6 (osd.6) 135 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T18:44:16.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:16 smithi098 bash[76817]: cluster 2023-09-23T18:44:15.048318+0000 osd.7 (osd.7) 117 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T18:44:16.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:16 smithi098 bash[76817]: cluster 2023-09-23T18:44:15.051689+0000 osd.7 (osd.7) 118 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T18:44:16.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:16 smithi098 bash[76817]: cluster 2023-09-23T18:44:15.052137+0000 osd.6 (osd.6) 136 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T18:44:16.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:16 smithi098 bash[76817]: cluster 2023-09-23T18:44:15.982485+0000 mgr.y (mgr.26054) 781 : cluster 0 pgmap v799: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 41 MiB/s wr, 651 op/s; 4928/262212 objects misplaced (1.879%); 844 KiB/s, 13 objects/s recovering 2023-09-23T18:44:16.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:16 smithi098 bash[76817]: audit 2023-09-23T18:44:15.983518+0000 mon.a (mon.0) 1053 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T18:44:16.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:16 smithi098 bash[76817]: audit 2023-09-23T18:44:16.059549+0000 mon.a (mon.0) 1054 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-23T18:44:16.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:16 smithi098 bash[76817]: cluster 2023-09-23T18:44:16.059688+0000 mon.a (mon.0) 1055 : cluster 0 osdmap e203: 8 total, 8 up, 7 in 2023-09-23T18:44:16.441 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:44:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:44:16] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.33.4" 2023-09-23T18:44:16.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:16 smithi098 bash[79516]: cluster 2023-09-23T18:44:15.046271+0000 osd.6 (osd.6) 135 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T18:44:16.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:16 smithi098 bash[79516]: cluster 2023-09-23T18:44:15.048318+0000 osd.7 (osd.7) 117 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T18:44:16.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:16 smithi098 bash[79516]: cluster 2023-09-23T18:44:15.051689+0000 osd.7 (osd.7) 118 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T18:44:16.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:16 smithi098 bash[79516]: cluster 2023-09-23T18:44:15.052137+0000 osd.6 (osd.6) 136 : cluster 3 failed to encode map e202 with expected crc 2023-09-23T18:44:16.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:16 smithi098 bash[79516]: cluster 2023-09-23T18:44:15.982485+0000 mgr.y (mgr.26054) 781 : cluster 0 pgmap v799: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 41 MiB/s wr, 651 op/s; 4928/262212 objects misplaced (1.879%); 844 KiB/s, 13 objects/s recovering 2023-09-23T18:44:16.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:16 smithi098 bash[79516]: audit 2023-09-23T18:44:15.983518+0000 mon.a (mon.0) 1053 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T18:44:16.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:16 smithi098 bash[79516]: audit 2023-09-23T18:44:16.059549+0000 mon.a (mon.0) 1054 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-23T18:44:16.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:16 smithi098 bash[79516]: cluster 2023-09-23T18:44:16.059688+0000 mon.a (mon.0) 1055 : cluster 0 osdmap e203: 8 total, 8 up, 7 in 2023-09-23T18:44:16.459 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:16.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:16.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:16.562 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:16.590 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:16.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:16.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:16.695 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:16.724 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:16.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:16.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:16.827 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:16.852 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:16.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:16.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:16.955 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:16.968 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:44:16.980 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:16.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:16.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:17.083 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:17.107 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:17.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:17.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:17.210 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:17.239 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:17.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:17.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:17.274 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 75 16 89247 89231 74.2477 12.8125 0.0033197 0.0133568 2023-09-23T18:44:17.342 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:17.372 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:17.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:17.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:17.476 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:17.487 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:17 smithi173 bash[59422]: cluster 2023-09-23T18:44:16.055918+0000 osd.5 (osd.5) 99 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T18:44:17.487 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:17 smithi173 bash[59422]: cluster 2023-09-23T18:44:16.056037+0000 osd.6 (osd.6) 137 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T18:44:17.487 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:17 smithi173 bash[59422]: cluster 2023-09-23T18:44:16.060923+0000 osd.6 (osd.6) 138 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T18:44:17.487 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:17 smithi173 bash[59422]: cluster 2023-09-23T18:44:16.163500+0000 osd.4 (osd.4) 163 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T18:44:17.487 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:17 smithi173 bash[59422]: cluster 2023-09-23T18:44:16.289538+0000 osd.7 (osd.7) 119 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T18:44:17.487 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:17 smithi173 bash[59422]: cluster 2023-09-23T18:44:17.061992+0000 mon.a (mon.0) 1056 : cluster 0 osdmap e204: 8 total, 8 up, 7 in 2023-09-23T18:44:17.506 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:17.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:17.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:17.542 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:17 smithi098 bash[76817]: cluster 2023-09-23T18:44:16.055918+0000 osd.5 (osd.5) 99 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T18:44:17.543 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:17 smithi098 bash[76817]: cluster 2023-09-23T18:44:16.056037+0000 osd.6 (osd.6) 137 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T18:44:17.543 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:17 smithi098 bash[76817]: cluster 2023-09-23T18:44:16.060923+0000 osd.6 (osd.6) 138 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T18:44:17.543 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:17 smithi098 bash[76817]: cluster 2023-09-23T18:44:16.163500+0000 osd.4 (osd.4) 163 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T18:44:17.543 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:17 smithi098 bash[76817]: cluster 2023-09-23T18:44:16.289538+0000 osd.7 (osd.7) 119 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T18:44:17.543 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:17 smithi098 bash[76817]: cluster 2023-09-23T18:44:17.061992+0000 mon.a (mon.0) 1056 : cluster 0 osdmap e204: 8 total, 8 up, 7 in 2023-09-23T18:44:17.543 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:17 smithi098 bash[79516]: cluster 2023-09-23T18:44:16.055918+0000 osd.5 (osd.5) 99 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T18:44:17.543 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:17 smithi098 bash[79516]: cluster 2023-09-23T18:44:16.056037+0000 osd.6 (osd.6) 137 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T18:44:17.543 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:17 smithi098 bash[79516]: cluster 2023-09-23T18:44:16.060923+0000 osd.6 (osd.6) 138 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T18:44:17.543 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:17 smithi098 bash[79516]: cluster 2023-09-23T18:44:16.163500+0000 osd.4 (osd.4) 163 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T18:44:17.544 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:17 smithi098 bash[79516]: cluster 2023-09-23T18:44:16.289538+0000 osd.7 (osd.7) 119 : cluster 3 failed to encode map e203 with expected crc 2023-09-23T18:44:17.544 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:17 smithi098 bash[79516]: cluster 2023-09-23T18:44:17.061992+0000 mon.a (mon.0) 1056 : cluster 0 osdmap e204: 8 total, 8 up, 7 in 2023-09-23T18:44:17.610 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:17.632 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:17.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:17.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:17.735 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:17.766 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:17.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:17.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:17.869 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:17.896 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:17.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:17.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:17.981 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:44:17.999 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:18.022 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:18.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:18.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:18.125 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:18.153 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:18.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:18.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:18.256 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:18.274 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 76 16 89247 89231 73.2721 0 - 0.0133568 2023-09-23T18:44:18.284 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:18.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:18.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:18.388 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:18.418 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:18.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:18.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:18.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:18 smithi098 bash[76817]: cluster 2023-09-23T18:44:17.063620+0000 osd.6 (osd.6) 139 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T18:44:18.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:18 smithi098 bash[76817]: cluster 2023-09-23T18:44:17.175174+0000 osd.4 (osd.4) 164 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T18:44:18.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:18 smithi098 bash[76817]: cluster 2023-09-23T18:44:17.178158+0000 osd.4 (osd.4) 165 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T18:44:18.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:18 smithi098 bash[76817]: cluster 2023-09-23T18:44:17.180361+0000 osd.4 (osd.4) 166 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T18:44:18.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:18 smithi098 bash[76817]: cluster 2023-09-23T18:44:17.316169+0000 osd.5 (osd.5) 100 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T18:44:18.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:18 smithi098 bash[76817]: cluster 2023-09-23T18:44:17.316240+0000 osd.7 (osd.7) 120 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T18:44:18.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:18 smithi098 bash[76817]: cluster 2023-09-23T18:44:17.983236+0000 mgr.y (mgr.26054) 782 : cluster 0 pgmap v802: 99 pgs: 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 41 MiB/s wr, 655 op/s; 2787/265428 objects degraded (1.050%); 7189/265428 objects misplaced (2.708%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T18:44:18.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:18 smithi098 bash[76817]: cluster 2023-09-23T18:44:18.060297+0000 mon.a (mon.0) 1057 : cluster 0 osdmap e205: 8 total, 8 up, 7 in 2023-09-23T18:44:18.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:18 smithi098 bash[79516]: cluster 2023-09-23T18:44:17.063620+0000 osd.6 (osd.6) 139 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T18:44:18.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:18 smithi098 bash[79516]: cluster 2023-09-23T18:44:17.175174+0000 osd.4 (osd.4) 164 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T18:44:18.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:18 smithi098 bash[79516]: cluster 2023-09-23T18:44:17.178158+0000 osd.4 (osd.4) 165 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T18:44:18.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:18 smithi098 bash[79516]: cluster 2023-09-23T18:44:17.180361+0000 osd.4 (osd.4) 166 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T18:44:18.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:18 smithi098 bash[79516]: cluster 2023-09-23T18:44:17.316169+0000 osd.5 (osd.5) 100 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T18:44:18.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:18 smithi098 bash[79516]: cluster 2023-09-23T18:44:17.316240+0000 osd.7 (osd.7) 120 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T18:44:18.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:18 smithi098 bash[79516]: cluster 2023-09-23T18:44:17.983236+0000 mgr.y (mgr.26054) 782 : cluster 0 pgmap v802: 99 pgs: 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 41 MiB/s wr, 655 op/s; 2787/265428 objects degraded (1.050%); 7189/265428 objects misplaced (2.708%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T18:44:18.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:18 smithi098 bash[79516]: cluster 2023-09-23T18:44:18.060297+0000 mon.a (mon.0) 1057 : cluster 0 osdmap e205: 8 total, 8 up, 7 in 2023-09-23T18:44:18.522 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:18.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:18 smithi173 bash[59422]: cluster 2023-09-23T18:44:17.063620+0000 osd.6 (osd.6) 139 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T18:44:18.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:18 smithi173 bash[59422]: cluster 2023-09-23T18:44:17.175174+0000 osd.4 (osd.4) 164 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T18:44:18.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:18 smithi173 bash[59422]: cluster 2023-09-23T18:44:17.178158+0000 osd.4 (osd.4) 165 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T18:44:18.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:18 smithi173 bash[59422]: cluster 2023-09-23T18:44:17.180361+0000 osd.4 (osd.4) 166 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T18:44:18.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:18 smithi173 bash[59422]: cluster 2023-09-23T18:44:17.316169+0000 osd.5 (osd.5) 100 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T18:44:18.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:18 smithi173 bash[59422]: cluster 2023-09-23T18:44:17.316240+0000 osd.7 (osd.7) 120 : cluster 3 failed to encode map e204 with expected crc 2023-09-23T18:44:18.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:18 smithi173 bash[59422]: cluster 2023-09-23T18:44:17.983236+0000 mgr.y (mgr.26054) 782 : cluster 0 pgmap v802: 99 pgs: 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 41 MiB/s wr, 655 op/s; 2787/265428 objects degraded (1.050%); 7189/265428 objects misplaced (2.708%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T18:44:18.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:18 smithi173 bash[59422]: cluster 2023-09-23T18:44:18.060297+0000 mon.a (mon.0) 1057 : cluster 0 osdmap e205: 8 total, 8 up, 7 in 2023-09-23T18:44:18.545 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:18.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:18.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:18.648 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:18.670 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:18.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:18.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:18.773 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:18.985 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:18.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:18.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:19.091 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:19.134 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:19.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:19.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:19.219 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:19 smithi098 bash[76817]: cluster 2023-09-23T18:44:18.062598+0000 osd.6 (osd.6) 140 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T18:44:19.219 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:19 smithi098 bash[76817]: cluster 2023-09-23T18:44:18.073011+0000 osd.7 (osd.7) 121 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T18:44:19.246 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:19.270 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:19.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:19.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:19.275 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 77 16 89619 89603 72.6234 11.625 0.0104515 0.0137638 2023-09-23T18:44:19.375 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:19.399 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:19.400 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:19 smithi098 bash[76817]: cluster 2023-09-23T18:44:18.208026+0000 mon.a (mon.0) 1058 : cluster 3 Health check failed: Degraded data redundancy: 2787/265428 objects degraded (1.050%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:44:19.400 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:19 smithi098 bash[76817]: cluster 2023-09-23T18:44:18.308264+0000 osd.4 (osd.4) 167 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T18:44:19.400 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:19 smithi098 bash[76817]: cluster 2023-09-23T18:44:18.778612+0000 osd.5 (osd.5) 101 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T18:44:19.400 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:19 smithi098 bash[76817]: cluster 2023-09-23T18:44:18.780613+0000 osd.5 (osd.5) 102 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T18:44:19.400 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:19 smithi098 bash[76817]: cluster 2023-09-23T18:44:18.782034+0000 osd.5 (osd.5) 103 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T18:44:19.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:19.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:19.402 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:19 smithi098 bash[79516]: cluster 2023-09-23T18:44:18.062598+0000 osd.6 (osd.6) 140 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T18:44:19.402 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:19 smithi098 bash[79516]: cluster 2023-09-23T18:44:18.073011+0000 osd.7 (osd.7) 121 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T18:44:19.402 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:19 smithi098 bash[79516]: cluster 2023-09-23T18:44:18.208026+0000 mon.a (mon.0) 1058 : cluster 3 Health check failed: Degraded data redundancy: 2787/265428 objects degraded (1.050%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:44:19.402 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:19 smithi098 bash[79516]: cluster 2023-09-23T18:44:18.308264+0000 osd.4 (osd.4) 167 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T18:44:19.403 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:19 smithi098 bash[79516]: cluster 2023-09-23T18:44:18.778612+0000 osd.5 (osd.5) 101 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T18:44:19.403 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:19 smithi098 bash[79516]: cluster 2023-09-23T18:44:18.780613+0000 osd.5 (osd.5) 102 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T18:44:19.403 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:19 smithi098 bash[79516]: cluster 2023-09-23T18:44:18.782034+0000 osd.5 (osd.5) 103 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T18:44:19.503 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:19.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:19 smithi173 bash[59422]: cluster 2023-09-23T18:44:18.062598+0000 osd.6 (osd.6) 140 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T18:44:19.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:19 smithi173 bash[59422]: cluster 2023-09-23T18:44:18.073011+0000 osd.7 (osd.7) 121 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T18:44:19.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:19 smithi173 bash[59422]: cluster 2023-09-23T18:44:18.208026+0000 mon.a (mon.0) 1058 : cluster 3 Health check failed: Degraded data redundancy: 2787/265428 objects degraded (1.050%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:44:19.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:19 smithi173 bash[59422]: cluster 2023-09-23T18:44:18.308264+0000 osd.4 (osd.4) 167 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T18:44:19.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:19 smithi173 bash[59422]: cluster 2023-09-23T18:44:18.778612+0000 osd.5 (osd.5) 101 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T18:44:19.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:19 smithi173 bash[59422]: cluster 2023-09-23T18:44:18.780613+0000 osd.5 (osd.5) 102 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T18:44:19.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:19 smithi173 bash[59422]: cluster 2023-09-23T18:44:18.782034+0000 osd.5 (osd.5) 103 : cluster 3 failed to encode map e205 with expected crc 2023-09-23T18:44:19.532 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:19.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:19.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:19.635 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:19.662 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:19.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:19.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:19.768 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:19.798 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:19.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:19.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:19.904 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:19.928 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:19.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:19.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:20.031 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:20.059 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:20.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:20.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:20.163 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:20.192 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:20.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:20.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:20.274 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 78 16 90594 90578 72.4737 60.9375 0.00636695 0.0137915 2023-09-23T18:44:20.295 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:20.337 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:20.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:20.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:20.440 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:20.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:20 smithi098 bash[79516]: audit 2023-09-23T18:44:19.284336+0000 mon.a (mon.0) 1059 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:44:20.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:20 smithi098 bash[79516]: audit 2023-09-23T18:44:19.296935+0000 mon.a (mon.0) 1060 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:44:20.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:20 smithi098 bash[79516]: cluster 2023-09-23T18:44:19.984023+0000 mgr.y (mgr.26054) 783 : cluster 0 pgmap v804: 99 pgs: 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 33 MiB/s wr, 523 op/s; 2787/267285 objects degraded (1.043%); 7189/267285 objects misplaced (2.690%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T18:44:20.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:20 smithi098 bash[76817]: audit 2023-09-23T18:44:19.284336+0000 mon.a (mon.0) 1059 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:44:20.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:20 smithi098 bash[76817]: audit 2023-09-23T18:44:19.296935+0000 mon.a (mon.0) 1060 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:44:20.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:20 smithi098 bash[76817]: cluster 2023-09-23T18:44:19.984023+0000 mgr.y (mgr.26054) 783 : cluster 0 pgmap v804: 99 pgs: 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 33 MiB/s wr, 523 op/s; 2787/267285 objects degraded (1.043%); 7189/267285 objects misplaced (2.690%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T18:44:20.467 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:20.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:20.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:20.571 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:20.580 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:20 smithi173 bash[59422]: audit 2023-09-23T18:44:19.284336+0000 mon.a (mon.0) 1059 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:44:20.580 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:20 smithi173 bash[59422]: audit 2023-09-23T18:44:19.296935+0000 mon.a (mon.0) 1060 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:44:20.580 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:20 smithi173 bash[59422]: cluster 2023-09-23T18:44:19.984023+0000 mgr.y (mgr.26054) 783 : cluster 0 pgmap v804: 99 pgs: 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 33 MiB/s wr, 523 op/s; 2787/267285 objects degraded (1.043%); 7189/267285 objects misplaced (2.690%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T18:44:20.600 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:20.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:20.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:20.703 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:20.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:20.740 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:20.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:20.841 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:21.098 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:21.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:21.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:21.201 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:21.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:21.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:21.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:21.274 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 79 16 91575 91559 72.3325 61.3125 0.0111125 0.0138196 2023-09-23T18:44:21.333 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:21.365 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:21.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:21.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:21.474 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:21.501 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:21.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:21.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:21.604 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:21.629 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:21.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:21.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:21.735 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:21.761 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:21.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:21.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:21.867 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:21.892 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:21.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:21.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:21.999 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:22.022 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:22.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:22.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:22.126 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:22.170 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:22.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:22.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:22.274 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:2023-09-23T18:44:22.273642+0000 min lat: 0.00125843 max lat: 2.43785 avg lat: 0.0138468 2023-09-23T18:44:22.274 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:44:22.274 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 80 16 92547 92531 72.1878 60.75 0.0287831 0.0138468 2023-09-23T18:44:22.275 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:22.302 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:22.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:22.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:22.406 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:22.440 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:22.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:22.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:22.548 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:22.572 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:22.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:22.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:22.678 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:22.703 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:22.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:22.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:22.810 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:22.832 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:22.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:22.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:22.935 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:22.964 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:22.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:22.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:23.069 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:23.080 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:23 smithi173 bash[59422]: cluster 2023-09-23T18:44:21.984634+0000 mgr.y (mgr.26054) 784 : cluster 0 pgmap v805: 99 pgs: 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 29 MiB/s wr, 457 op/s; 2787/270444 objects degraded (1.031%); 7189/270444 objects misplaced (2.658%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T18:44:23.096 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:23.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:23.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:23.200 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:23.211 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:23 smithi098 bash[76817]: cluster 2023-09-23T18:44:21.984634+0000 mgr.y (mgr.26054) 784 : cluster 0 pgmap v805: 99 pgs: 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 29 MiB/s wr, 457 op/s; 2787/270444 objects degraded (1.031%); 7189/270444 objects misplaced (2.658%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T18:44:23.211 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:23 smithi098 bash[79516]: cluster 2023-09-23T18:44:21.984634+0000 mgr.y (mgr.26054) 784 : cluster 0 pgmap v805: 99 pgs: 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 29 MiB/s wr, 457 op/s; 2787/270444 objects degraded (1.031%); 7189/270444 objects misplaced (2.658%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T18:44:23.238 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:23.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:23.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:23.274 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 81 16 93389 93373 71.9466 52.625 0.0105456 0.0138934 2023-09-23T18:44:23.342 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:23.374 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:23.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:23.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:23.476 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:23.518 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:23.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:23.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:23.622 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:23.642 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:23.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:23.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:23.745 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:23.773 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:23.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:23.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:23.878 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:23.903 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:23.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:23.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:24.006 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:24.034 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:24.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:24.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:24.142 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:24.165 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:24.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:24.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:24.268 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:24.274 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 82 16 94168 94152 71.6632 48.6875 0.0415597 0.0139437 2023-09-23T18:44:24.277 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:24 smithi098 bash[79516]: cluster 2023-09-23T18:44:23.985308+0000 mgr.y (mgr.26054) 785 : cluster 0 pgmap v806: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 35 MiB/s wr, 564 op/s; 5581/275631 objects degraded (2.025%); 9563/275631 objects misplaced (3.469%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T18:44:24.277 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:24 smithi098 bash[79516]: cluster 2023-09-23T18:44:24.039390+0000 mon.a (mon.0) 1061 : cluster 3 Health check update: Degraded data redundancy: 5581/275631 objects degraded (2.025%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:44:24.277 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:24 smithi098 bash[76817]: cluster 2023-09-23T18:44:23.985308+0000 mgr.y (mgr.26054) 785 : cluster 0 pgmap v806: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 35 MiB/s wr, 564 op/s; 5581/275631 objects degraded (2.025%); 9563/275631 objects misplaced (3.469%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T18:44:24.278 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:24 smithi098 bash[76817]: cluster 2023-09-23T18:44:24.039390+0000 mon.a (mon.0) 1061 : cluster 3 Health check update: Degraded data redundancy: 5581/275631 objects degraded (2.025%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:44:24.295 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:24.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:24.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:24.400 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:24.409 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:24 smithi173 bash[59422]: cluster 2023-09-23T18:44:23.985308+0000 mgr.y (mgr.26054) 785 : cluster 0 pgmap v806: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 35 MiB/s wr, 564 op/s; 5581/275631 objects degraded (2.025%); 9563/275631 objects misplaced (3.469%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T18:44:24.409 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:24 smithi173 bash[59422]: cluster 2023-09-23T18:44:24.039390+0000 mon.a (mon.0) 1061 : cluster 3 Health check update: Degraded data redundancy: 5581/275631 objects degraded (2.025%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:44:24.423 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:24.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:24.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:24.526 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:24.548 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:24.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:24.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:24.650 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:24.676 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:24.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:24.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:24.782 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:24.813 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:24.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:24.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:24.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:24.943 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:24.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:24.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:25.049 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:25.077 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:25.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:25.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:25.184 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:25.298 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 83 16 95133 95117 71.5266 60.3125 0.0164109 0.013975 2023-09-23T18:44:25.339 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:25.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:25.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:25.446 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:25.470 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:25.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:25.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:25.573 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:25.613 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:25.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:25.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:25.717 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:25.726 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:25 smithi173 bash[59422]: audit 2023-09-23T18:44:24.620077+0000 mon.a (mon.0) 1062 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:44:25.726 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:25 smithi173 bash[59422]: audit 2023-09-23T18:44:24.622326+0000 mon.a (mon.0) 1063 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:44:25.746 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:25.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:25.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:25.849 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:25.858 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:25 smithi098 bash[79516]: audit 2023-09-23T18:44:24.620077+0000 mon.a (mon.0) 1062 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:44:25.858 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:25 smithi098 bash[79516]: audit 2023-09-23T18:44:24.622326+0000 mon.a (mon.0) 1063 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:44:25.858 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:25 smithi098 bash[76817]: audit 2023-09-23T18:44:24.620077+0000 mon.a (mon.0) 1062 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:44:25.858 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:25 smithi098 bash[76817]: audit 2023-09-23T18:44:24.622326+0000 mon.a (mon.0) 1063 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:44:25.874 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:25.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:25.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:25.979 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:26.007 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:26.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:26.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:26.110 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:26.139 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:26.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:26.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:26.190 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:44:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:44:26] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.33.4" 2023-09-23T18:44:26.242 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:26.271 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:26.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:26.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:26.276 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 84 16 95974 95958 71.2999 52.5625 0.0124205 0.0140191 2023-09-23T18:44:26.375 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:26.400 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:26.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:26.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:26.503 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:26.550 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:26.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:26.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:26.655 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:26.681 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:26.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:26.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:26.784 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:26.794 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:26 smithi098 bash[79516]: cluster 2023-09-23T18:44:25.985846+0000 mgr.y (mgr.26054) 786 : cluster 0 pgmap v807: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 40 MiB/s wr, 632 op/s; 5581/280881 objects degraded (1.987%); 9563/280881 objects misplaced (3.405%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:44:26.794 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:26 smithi098 bash[76817]: cluster 2023-09-23T18:44:25.985846+0000 mgr.y (mgr.26054) 786 : cluster 0 pgmap v807: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 40 MiB/s wr, 632 op/s; 5581/280881 objects degraded (1.987%); 9563/280881 objects misplaced (3.405%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:44:26.811 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:26.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:26.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:26.914 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:26.943 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:26.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:26.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:27.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:26 smithi173 bash[59422]: cluster 2023-09-23T18:44:25.985846+0000 mgr.y (mgr.26054) 786 : cluster 0 pgmap v807: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 40 MiB/s wr, 632 op/s; 5581/280881 objects degraded (1.987%); 9563/280881 objects misplaced (3.405%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:44:27.046 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:27.075 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:27.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:27.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:27.182 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:27.207 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:27.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:27.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:27.276 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 85 16 96896 96880 71.1392 57.625 0.0086513 0.0140514 2023-09-23T18:44:27.312 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:27.346 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:27.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:27.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:27.448 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:27.475 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:27.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:27.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:27.577 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:27.610 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:27.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:27.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:27.714 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:27.743 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:27.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:27.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:27.846 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:27.871 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:27.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:27.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:27.975 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:28.010 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:28.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:28.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:28.114 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:28.157 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:28.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:28.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:28.263 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:28.276 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 86 16 97786 97770 70.9589 55.625 0.00592752 0.0140868 2023-09-23T18:44:28.291 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:28.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:28.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:28.348 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:44:28.348 INFO:teuthology.orchestra.run.smithi098.stdout: "success": "" 2023-09-23T18:44:28.348 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:44:28.397 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:28.426 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:28.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:28.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:28.529 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:28.557 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:28.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:28.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:28.660 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:28.683 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:28.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:28.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:28.786 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:28.818 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:28.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:28.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:28.921 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:28.956 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:28.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:28.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:29.061 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:29.090 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:29.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:29.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:29.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:29 smithi098 bash[79516]: cluster 2023-09-23T18:44:27.986500+0000 mgr.y (mgr.26054) 787 : cluster 0 pgmap v808: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 42 MiB/s wr, 679 op/s; 5581/285831 objects degraded (1.953%); 9270/285831 objects misplaced (3.243%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T18:44:29.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:29 smithi098 bash[76817]: cluster 2023-09-23T18:44:27.986500+0000 mgr.y (mgr.26054) 787 : cluster 0 pgmap v808: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 42 MiB/s wr, 679 op/s; 5581/285831 objects degraded (1.953%); 9270/285831 objects misplaced (3.243%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T18:44:29.193 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:29.229 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:29.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:29.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:29.280 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 87 16 98613 98597 70.7354 51.6875 0.005175 0.0141312 2023-09-23T18:44:29.336 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:29.351 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:29 smithi173 bash[59422]: cluster 2023-09-23T18:44:27.986500+0000 mgr.y (mgr.26054) 787 : cluster 0 pgmap v808: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 42 MiB/s wr, 679 op/s; 5581/285831 objects degraded (1.953%); 9270/285831 objects misplaced (3.243%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T18:44:29.372 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:29.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:29.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:29.475 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:29.502 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:29.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:29.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:29.605 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:29.622 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:29.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:29.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:29.725 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:29.753 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:29.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:29.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:29.855 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:29.883 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:29.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:29.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:30.015 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:30.041 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:30.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:30.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:30.073 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:30 smithi098 bash[76817]: cluster 2023-09-23T18:44:30.046354+0000 mon.a (mon.0) 1064 : cluster 3 Health check update: Degraded data redundancy: 5581/285831 objects degraded (1.953%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:44:30.145 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:30.154 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:30 smithi098 bash[79516]: cluster 2023-09-23T18:44:30.046354+0000 mon.a (mon.0) 1064 : cluster 3 Health check update: Degraded data redundancy: 5581/285831 objects degraded (1.953%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:44:30.171 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:30.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:30.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:30.277 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:30.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:30 smithi173 bash[59422]: cluster 2023-09-23T18:44:30.046354+0000 mon.a (mon.0) 1064 : cluster 3 Health check update: Degraded data redundancy: 5581/285831 objects degraded (1.953%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:44:30.279 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 88 16 99374 99358 70.4724 47.5625 0.013491 0.0141845 2023-09-23T18:44:30.309 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:30.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:30.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:30.415 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:30.440 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:30.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:30.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:30.544 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:30.569 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:30.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:30.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:30.672 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:30.706 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:30.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:30.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:30.810 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:30.857 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:30.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:30.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:30.963 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:30.987 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:30.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:30.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:31.091 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:31.117 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:31.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:31.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:31.221 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:31.230 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:31 smithi173 bash[59422]: cluster 2023-09-23T18:44:29.987194+0000 mgr.y (mgr.26054) 788 : cluster 0 pgmap v809: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.9 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 44 MiB/s wr, 705 op/s; 5581/290673 objects degraded (1.920%); 9270/290673 objects misplaced (3.189%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T18:44:31.230 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:31 smithi173 bash[59422]: audit 2023-09-23T18:44:31.041579+0000 mon.a (mon.0) 1065 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:44:31.230 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:31 smithi173 bash[59422]: audit 2023-09-23T18:44:31.053113+0000 mon.a (mon.0) 1066 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:44:31.245 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:31.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:31.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:31.279 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 89 16 100133 100117 70.2138 47.4375 0.0209968 0.0142364 2023-09-23T18:44:31.348 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:31.375 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:31.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:31.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:31.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:31 smithi098 bash[79516]: cluster 2023-09-23T18:44:29.987194+0000 mgr.y (mgr.26054) 788 : cluster 0 pgmap v809: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.9 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 44 MiB/s wr, 705 op/s; 5581/290673 objects degraded (1.920%); 9270/290673 objects misplaced (3.189%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T18:44:31.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:31 smithi098 bash[79516]: audit 2023-09-23T18:44:31.041579+0000 mon.a (mon.0) 1065 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:44:31.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:31 smithi098 bash[79516]: audit 2023-09-23T18:44:31.053113+0000 mon.a (mon.0) 1066 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:44:31.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:31 smithi098 bash[76817]: cluster 2023-09-23T18:44:29.987194+0000 mgr.y (mgr.26054) 788 : cluster 0 pgmap v809: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.9 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 44 MiB/s wr, 705 op/s; 5581/290673 objects degraded (1.920%); 9270/290673 objects misplaced (3.189%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T18:44:31.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:31 smithi098 bash[76817]: audit 2023-09-23T18:44:31.041579+0000 mon.a (mon.0) 1065 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:44:31.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:31 smithi098 bash[76817]: audit 2023-09-23T18:44:31.053113+0000 mon.a (mon.0) 1066 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:44:31.479 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:31.505 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:31.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:31.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:31.610 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:31.637 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:31.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:31.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:31.740 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:31.766 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:31.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:31.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:31.871 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:31.901 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:31.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:31.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:32.005 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:32.030 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:32.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:32.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:32.133 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:32.156 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:32.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:32.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:32.259 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:32.279 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Total time run: 90.0125 2023-09-23T18:44:32.279 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Total writes made: 100942 2023-09-23T18:44:32.279 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Write size: 65536 2023-09-23T18:44:32.280 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Object size: 65536 2023-09-23T18:44:32.280 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Bandwidth (MB/sec): 70.0889 2023-09-23T18:44:32.280 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Stddev Bandwidth: 31.3594 2023-09-23T18:44:32.280 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Max bandwidth (MB/sec): 165.188 2023-09-23T18:44:32.280 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Min bandwidth (MB/sec): 0 2023-09-23T18:44:32.280 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Average IOPS: 1121 2023-09-23T18:44:32.280 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Stddev IOPS: 501.758 2023-09-23T18:44:32.280 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Max IOPS: 2643 2023-09-23T18:44:32.280 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Min IOPS: 0 2023-09-23T18:44:32.280 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Average Latency(s): 0.0142624 2023-09-23T18:44:32.281 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Stddev Latency(s): 0.0484946 2023-09-23T18:44:32.281 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Max latency(s): 2.43785 2023-09-23T18:44:32.281 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Min latency(s): 0.00125843 2023-09-23T18:44:32.287 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T18:44:32.291 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:32.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:32.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:32.397 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:32.427 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:32.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:32.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:32.530 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:32.560 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:32.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:32.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:32.666 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:32.690 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:32.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:32.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:32.796 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:32.819 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:32.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:32.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:32.923 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:32.950 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:32.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:32.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:33.053 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:33.089 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:33.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:33.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:33.193 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:33.201 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:33 smithi173 bash[59422]: cluster 2023-09-23T18:44:31.987783+0000 mgr.y (mgr.26054) 789 : cluster 0 pgmap v810: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 50 MiB/s wr, 797 op/s; 5581/295998 objects degraded (1.885%); 9270/295998 objects misplaced (3.132%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T18:44:33.201 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:33 smithi173 bash[59422]: audit 2023-09-23T18:44:32.283882+0000 mon.a (mon.0) 1067 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:44:33.201 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:33 smithi173 bash[59422]: audit 2023-09-23T18:44:32.284558+0000 mon.a (mon.0) 1068 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:44:33.201 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:33 smithi173 bash[59422]: audit 2023-09-23T18:44:32.290239+0000 mon.a (mon.0) 1069 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:44:33.217 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:33.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:33.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:33.320 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:33.343 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:33.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:33.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:33.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:33 smithi098 bash[76817]: cluster 2023-09-23T18:44:31.987783+0000 mgr.y (mgr.26054) 789 : cluster 0 pgmap v810: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 50 MiB/s wr, 797 op/s; 5581/295998 objects degraded (1.885%); 9270/295998 objects misplaced (3.132%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T18:44:33.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:33 smithi098 bash[76817]: audit 2023-09-23T18:44:32.283882+0000 mon.a (mon.0) 1067 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:44:33.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:33 smithi098 bash[76817]: audit 2023-09-23T18:44:32.284558+0000 mon.a (mon.0) 1068 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:44:33.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:33 smithi098 bash[76817]: audit 2023-09-23T18:44:32.290239+0000 mon.a (mon.0) 1069 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:44:33.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:33 smithi098 bash[79516]: cluster 2023-09-23T18:44:31.987783+0000 mgr.y (mgr.26054) 789 : cluster 0 pgmap v810: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 50 MiB/s wr, 797 op/s; 5581/295998 objects degraded (1.885%); 9270/295998 objects misplaced (3.132%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T18:44:33.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:33 smithi098 bash[79516]: audit 2023-09-23T18:44:32.283882+0000 mon.a (mon.0) 1067 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:44:33.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:33 smithi098 bash[79516]: audit 2023-09-23T18:44:32.284558+0000 mon.a (mon.0) 1068 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:44:33.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:33 smithi098 bash[79516]: audit 2023-09-23T18:44:32.290239+0000 mon.a (mon.0) 1069 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:44:33.446 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:33.473 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:33.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:33.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:33.577 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:33.607 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:33.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:33.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:33.710 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:33.734 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:33.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:33.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:33.839 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:33.869 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:33.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:33.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:33.972 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:33.998 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:34.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:34.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:34.101 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:34.130 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:34.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:34.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:34.235 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:34.272 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:34.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:34.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:34.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:34 smithi173 bash[59422]: cluster 2023-09-23T18:44:33.988403+0000 mgr.y (mgr.26054) 790 : cluster 0 pgmap v811: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 52 MiB/s wr, 859 op/s; 5581/299802 objects degraded (1.862%); 8941/299802 objects misplaced (2.982%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T18:44:34.378 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:34.386 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:34 smithi098 bash[79516]: cluster 2023-09-23T18:44:33.988403+0000 mgr.y (mgr.26054) 790 : cluster 0 pgmap v811: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 52 MiB/s wr, 859 op/s; 5581/299802 objects degraded (1.862%); 8941/299802 objects misplaced (2.982%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T18:44:34.386 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:34 smithi098 bash[76817]: cluster 2023-09-23T18:44:33.988403+0000 mgr.y (mgr.26054) 790 : cluster 0 pgmap v811: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 52 MiB/s wr, 859 op/s; 5581/299802 objects degraded (1.862%); 8941/299802 objects misplaced (2.982%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T18:44:34.402 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:34.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:34.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:34.506 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:34.531 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:34.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:34.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:34.634 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:34.663 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:34.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:34.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:34.767 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:34.791 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:34.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:34.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:34.895 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:34.923 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:34.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:34.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:35.027 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:35.079 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:35.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:35.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:35.185 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:35.194 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:35 smithi173 bash[59422]: cluster 2023-09-23T18:44:35.047423+0000 mon.a (mon.0) 1070 : cluster 3 Health check update: Degraded data redundancy: 5581/299802 objects degraded (1.862%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:44:35.212 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:35.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:35.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:35.316 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:35.324 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:35 smithi098 bash[76817]: cluster 2023-09-23T18:44:35.047423+0000 mon.a (mon.0) 1070 : cluster 3 Health check update: Degraded data redundancy: 5581/299802 objects degraded (1.862%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:44:35.325 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:35 smithi098 bash[79516]: cluster 2023-09-23T18:44:35.047423+0000 mon.a (mon.0) 1070 : cluster 3 Health check update: Degraded data redundancy: 5581/299802 objects degraded (1.862%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:44:35.338 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:35.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:35.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:35.443 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:35.464 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:35.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:35.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:35.568 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:35.592 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:35.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:35.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:35.695 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:35.723 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:35.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:35.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:35.826 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:35.871 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:35.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:35.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:35.977 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:36.057 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:36.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:36.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:36.086 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:44:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:44:36] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.33.4" 2023-09-23T18:44:36.163 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:36.190 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:36.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:36.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:36.295 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:36.329 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:36.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:36.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:36.432 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:36.439 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:36 smithi098 bash[79516]: cluster 2023-09-23T18:44:35.989029+0000 mgr.y (mgr.26054) 791 : cluster 0 pgmap v812: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 85 B/s rd, 46 MiB/s wr, 859 op/s; 5581/298230 objects degraded (1.871%); 8941/298230 objects misplaced (2.998%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T18:44:36.439 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:36 smithi098 bash[76817]: cluster 2023-09-23T18:44:35.989029+0000 mgr.y (mgr.26054) 791 : cluster 0 pgmap v812: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 85 B/s rd, 46 MiB/s wr, 859 op/s; 5581/298230 objects degraded (1.871%); 8941/298230 objects misplaced (2.998%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T18:44:36.453 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:36.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:36.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:36.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:36 smithi173 bash[59422]: cluster 2023-09-23T18:44:35.989029+0000 mgr.y (mgr.26054) 791 : cluster 0 pgmap v812: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 85 B/s rd, 46 MiB/s wr, 859 op/s; 5581/298230 objects degraded (1.871%); 8941/298230 objects misplaced (2.998%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T18:44:36.556 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:36.579 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:36.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:36.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:36.682 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:36.704 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:36.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:36.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:36.807 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:36.831 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:36.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:36.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:36.936 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:36.956 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:44:36.962 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:36.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:36.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:36.991 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:44:37.045 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:44:37.069 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:37.088 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:44:37.098 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:37.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:37.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:37.130 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:44:37.164 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:44:37.198 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:44:37.200 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:37.208 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:37 smithi098 bash[76817]: audit 2023-09-23T18:44:36.904161+0000 mon.a (mon.0) 1071 : audit 0 from='client.? 172.21.15.98:0/2645818763' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:44:37.209 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:37 smithi098 bash[79516]: audit 2023-09-23T18:44:36.904161+0000 mon.a (mon.0) 1071 : audit 0 from='client.? 172.21.15.98:0/2645818763' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:44:37.230 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:37.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:37.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:37.238 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:44:37.247 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:44:37.247 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:44:37.247 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:44:37.248 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:44:37.248 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:44:37.248 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:44:37.248 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:44:37.248 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:44:37.335 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:37.345 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:37 smithi173 bash[59422]: audit 2023-09-23T18:44:36.904161+0000 mon.a (mon.0) 1071 : audit 0 from='client.? 172.21.15.98:0/2645818763' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:44:37.359 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:37.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:37.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:37.465 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:37.489 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:37.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:37.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:37.591 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:37.621 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:37.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:37.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:37.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:37.753 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:37.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:37.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:37.859 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:37.887 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:37.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:37.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:37.992 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:38.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:38.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:38.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:38.120 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:38.145 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:38.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:38.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:38.161 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:38 smithi173 bash[59422]: cluster 2023-09-23T18:44:37.989603+0000 mgr.y (mgr.26054) 792 : cluster 0 pgmap v813: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 85 B/s rd, 38 MiB/s wr, 856 op/s; 5581/294093 objects degraded (1.898%); 8607/294093 objects misplaced (2.927%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T18:44:38.249 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:38.289 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:38.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:38.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:38.400 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:38.408 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:38 smithi098 bash[76817]: cluster 2023-09-23T18:44:37.989603+0000 mgr.y (mgr.26054) 792 : cluster 0 pgmap v813: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 85 B/s rd, 38 MiB/s wr, 856 op/s; 5581/294093 objects degraded (1.898%); 8607/294093 objects misplaced (2.927%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T18:44:38.408 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:38 smithi098 bash[79516]: cluster 2023-09-23T18:44:37.989603+0000 mgr.y (mgr.26054) 792 : cluster 0 pgmap v813: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 85 B/s rd, 38 MiB/s wr, 856 op/s; 5581/294093 objects degraded (1.898%); 8607/294093 objects misplaced (2.927%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T18:44:38.432 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:38.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:38.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:38.536 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:38.565 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:38.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:38.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:38.669 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:38.692 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:38.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:38.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:38.799 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:38.823 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:38.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:38.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:38.927 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:38.952 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:38.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:38.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:39.056 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:39.090 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:39.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:39.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:39.193 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:39.237 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:39.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:39.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:39.343 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:39.367 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:39.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:39.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:39.451 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T18:44:39.451 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:44:39.451 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-09-23T18:44:39.452 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:44:39.472 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:39.528 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:39.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:39.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:39.633 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:39.659 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:39.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:39.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:39.765 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:39.798 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:39.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:39.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:39.901 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:39.939 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:39.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:39 smithi098 bash[76817]: audit 2023-09-23T18:44:39.608239+0000 mon.a (mon.0) 1072 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:44:39.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:39 smithi098 bash[79516]: audit 2023-09-23T18:44:39.608239+0000 mon.a (mon.0) 1072 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:44:39.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:39.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:40.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:39 smithi173 bash[59422]: audit 2023-09-23T18:44:39.608239+0000 mon.a (mon.0) 1072 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:44:40.045 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:40.076 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:40.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:40.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:40.180 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:40.214 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:40.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:40.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:40.324 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:40.352 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:40.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:40.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:40.462 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:40.514 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:40.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:40.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:40.600 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:44:40.600 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:44:40.618 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:40.646 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:40.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:40.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:40.749 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:40.781 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:40.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:40.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:40.884 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:40.892 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:40 smithi098 bash[76817]: cluster 2023-09-23T18:44:39.992155+0000 mgr.y (mgr.26054) 793 : cluster 0 pgmap v814: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 85 B/s rd, 30 MiB/s wr, 870 op/s; 5581/288654 objects degraded (1.933%); 8607/288654 objects misplaced (2.982%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T18:44:40.892 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:40 smithi098 bash[76817]: cluster 2023-09-23T18:44:40.048537+0000 mon.a (mon.0) 1073 : cluster 3 Health check update: Degraded data redundancy: 5581/294093 objects degraded (1.898%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:44:40.893 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:40 smithi098 bash[79516]: cluster 2023-09-23T18:44:39.992155+0000 mgr.y (mgr.26054) 793 : cluster 0 pgmap v814: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 85 B/s rd, 30 MiB/s wr, 870 op/s; 5581/288654 objects degraded (1.933%); 8607/288654 objects misplaced (2.982%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T18:44:40.893 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:40 smithi098 bash[79516]: cluster 2023-09-23T18:44:40.048537+0000 mon.a (mon.0) 1073 : cluster 3 Health check update: Degraded data redundancy: 5581/294093 objects degraded (1.898%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:44:40.921 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:40.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:40.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:41.026 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:41.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:40 smithi173 bash[59422]: cluster 2023-09-23T18:44:39.992155+0000 mgr.y (mgr.26054) 793 : cluster 0 pgmap v814: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 85 B/s rd, 30 MiB/s wr, 870 op/s; 5581/288654 objects degraded (1.933%); 8607/288654 objects misplaced (2.982%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T18:44:41.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:40 smithi173 bash[59422]: cluster 2023-09-23T18:44:40.048537+0000 mon.a (mon.0) 1073 : cluster 3 Health check update: Degraded data redundancy: 5581/294093 objects degraded (1.898%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:44:41.054 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:41.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:41.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:41.158 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:41.200 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:41.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:41.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:41.308 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:41.336 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:41.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:41.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:41.440 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:41.477 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:41.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:41.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:41.581 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:41.611 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:41.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:41.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:41.692 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is unset 2023-09-23T18:44:41.715 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:41.743 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:41.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:41.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:41.755 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:41 smithi098 bash[79516]: audit 2023-09-23T18:44:40.971711+0000 mon.c (mon.2) 97 : audit 1 from='client.? 172.21.15.98:0/1746246476' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:44:41.755 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:41 smithi098 bash[79516]: audit 2023-09-23T18:44:40.973038+0000 mon.a (mon.0) 1074 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:44:41.756 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:41 smithi098 bash[76817]: audit 2023-09-23T18:44:40.971711+0000 mon.c (mon.2) 97 : audit 1 from='client.? 172.21.15.98:0/1746246476' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:44:41.756 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:41 smithi098 bash[76817]: audit 2023-09-23T18:44:40.973038+0000 mon.a (mon.0) 1074 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:44:41.845 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:41.878 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:41.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:41.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:41.982 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:42.010 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:42.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:42.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:42.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:41 smithi173 bash[59422]: audit 2023-09-23T18:44:40.971711+0000 mon.c (mon.2) 97 : audit 1 from='client.? 172.21.15.98:0/1746246476' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:44:42.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:41 smithi173 bash[59422]: audit 2023-09-23T18:44:40.973038+0000 mon.a (mon.0) 1074 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:44:42.114 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:42.137 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:42.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:42.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:42.242 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:42.244 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:44:42.267 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:42.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:42.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:42.372 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:42.394 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:42.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:42.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:42.502 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:42.530 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:42.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:42.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:42.633 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:42.658 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:42.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:42.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:42.761 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:42.791 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:42.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:42.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:42.895 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:42.922 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:42.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:42.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:42.966 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:42 smithi098 bash[76817]: cluster 2023-09-23T18:44:41.672206+0000 mon.a (mon.0) 1075 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:44:42.967 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:42 smithi098 bash[76817]: cluster 2023-09-23T18:44:41.679298+0000 osd.4 (osd.4) 168 : cluster 3 failed to encode map e206 with expected crc 2023-09-23T18:44:42.967 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:42 smithi098 bash[76817]: audit 2023-09-23T18:44:41.679577+0000 mon.a (mon.0) 1076 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:44:42.967 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:42 smithi098 bash[76817]: cluster 2023-09-23T18:44:41.679701+0000 mon.a (mon.0) 1077 : cluster 0 osdmap e206: 8 total, 8 up, 7 in 2023-09-23T18:44:42.967 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:42 smithi098 bash[76817]: cluster 2023-09-23T18:44:41.691122+0000 osd.6 (osd.6) 141 : cluster 3 failed to encode map e206 with expected crc 2023-09-23T18:44:42.967 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:42 smithi098 bash[76817]: cluster 2023-09-23T18:44:41.695082+0000 osd.7 (osd.7) 122 : cluster 3 failed to encode map e206 with expected crc 2023-09-23T18:44:42.967 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:42 smithi098 bash[76817]: cluster 2023-09-23T18:44:41.994418+0000 mgr.y (mgr.26054) 794 : cluster 0 pgmap v816: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 14 MiB/s wr, 899 op/s; 5581/282831 objects degraded (1.973%); 8607/282831 objects misplaced (3.043%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T18:44:42.967 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:42 smithi098 bash[79516]: cluster 2023-09-23T18:44:41.672206+0000 mon.a (mon.0) 1075 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:44:42.967 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:42 smithi098 bash[79516]: cluster 2023-09-23T18:44:41.679298+0000 osd.4 (osd.4) 168 : cluster 3 failed to encode map e206 with expected crc 2023-09-23T18:44:42.967 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:42 smithi098 bash[79516]: audit 2023-09-23T18:44:41.679577+0000 mon.a (mon.0) 1076 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:44:42.967 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:42 smithi098 bash[79516]: cluster 2023-09-23T18:44:41.679701+0000 mon.a (mon.0) 1077 : cluster 0 osdmap e206: 8 total, 8 up, 7 in 2023-09-23T18:44:42.968 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:42 smithi098 bash[79516]: cluster 2023-09-23T18:44:41.691122+0000 osd.6 (osd.6) 141 : cluster 3 failed to encode map e206 with expected crc 2023-09-23T18:44:42.968 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:42 smithi098 bash[79516]: cluster 2023-09-23T18:44:41.695082+0000 osd.7 (osd.7) 122 : cluster 3 failed to encode map e206 with expected crc 2023-09-23T18:44:42.968 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:42 smithi098 bash[79516]: cluster 2023-09-23T18:44:41.994418+0000 mgr.y (mgr.26054) 794 : cluster 0 pgmap v816: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 14 MiB/s wr, 899 op/s; 5581/282831 objects degraded (1.973%); 8607/282831 objects misplaced (3.043%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T18:44:43.025 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:43.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:42 smithi173 bash[59422]: cluster 2023-09-23T18:44:41.672206+0000 mon.a (mon.0) 1075 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:44:43.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:42 smithi173 bash[59422]: cluster 2023-09-23T18:44:41.679298+0000 osd.4 (osd.4) 168 : cluster 3 failed to encode map e206 with expected crc 2023-09-23T18:44:43.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:42 smithi173 bash[59422]: audit 2023-09-23T18:44:41.679577+0000 mon.a (mon.0) 1076 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:44:43.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:42 smithi173 bash[59422]: cluster 2023-09-23T18:44:41.679701+0000 mon.a (mon.0) 1077 : cluster 0 osdmap e206: 8 total, 8 up, 7 in 2023-09-23T18:44:43.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:42 smithi173 bash[59422]: cluster 2023-09-23T18:44:41.691122+0000 osd.6 (osd.6) 141 : cluster 3 failed to encode map e206 with expected crc 2023-09-23T18:44:43.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:42 smithi173 bash[59422]: cluster 2023-09-23T18:44:41.695082+0000 osd.7 (osd.7) 122 : cluster 3 failed to encode map e206 with expected crc 2023-09-23T18:44:43.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:42 smithi173 bash[59422]: cluster 2023-09-23T18:44:41.994418+0000 mgr.y (mgr.26054) 794 : cluster 0 pgmap v816: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 14 MiB/s wr, 899 op/s; 5581/282831 objects degraded (1.973%); 8607/282831 objects misplaced (3.043%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T18:44:43.050 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:43.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:43.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:43.154 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:43.179 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:43.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:43.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:43.225 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T18:44:43.281 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:43.305 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:43.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:43.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:43.413 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:43.438 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:43.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:43.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:43.543 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:43.572 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:43.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:43.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:43.675 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:43.702 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:43.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:43.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:43.807 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:43.816 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:43 smithi173 bash[59422]: cluster 2023-09-23T18:44:43.140285+0000 osd.0 (osd.0) 6 : cluster 0 7.4 scrub starts 2023-09-23T18:44:43.832 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:43.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:43.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:43.935 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:43.963 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:43.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:43.968 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:43 smithi098 bash[76817]: cluster 2023-09-23T18:44:43.140285+0000 osd.0 (osd.0) 6 : cluster 0 7.4 scrub starts 2023-09-23T18:44:43.968 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:43 smithi098 bash[79516]: cluster 2023-09-23T18:44:43.140285+0000 osd.0 (osd.0) 6 : cluster 0 7.4 scrub starts 2023-09-23T18:44:43.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:44.069 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:44.093 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:44.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:44.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:44.199 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:44.223 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:44.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:44.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:44.327 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:44.350 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:44.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:44.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:44.452 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:44.493 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:44.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:44.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:44.597 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:44.622 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:44.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:44.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:44.728 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:44.755 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:44.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:44.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:44.859 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:44.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:44 smithi173 bash[59422]: cluster 2023-09-23T18:44:43.591687+0000 osd.4 (osd.4) 169 : cluster 0 7.1 scrub starts 2023-09-23T18:44:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:44 smithi173 bash[59422]: cluster 2023-09-23T18:44:43.995138+0000 mgr.y (mgr.26054) 795 : cluster 0 pgmap v817: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 4.8 MiB/s wr, 914 op/s; 5581/276984 objects degraded (2.015%); 8227/276984 objects misplaced (2.970%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T18:44:44.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:44.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:44.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:44.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:44 smithi098 bash[79516]: cluster 2023-09-23T18:44:43.591687+0000 osd.4 (osd.4) 169 : cluster 0 7.1 scrub starts 2023-09-23T18:44:44.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:44 smithi098 bash[79516]: cluster 2023-09-23T18:44:43.995138+0000 mgr.y (mgr.26054) 795 : cluster 0 pgmap v817: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 4.8 MiB/s wr, 914 op/s; 5581/276984 objects degraded (2.015%); 8227/276984 objects misplaced (2.970%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T18:44:44.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:44 smithi098 bash[76817]: cluster 2023-09-23T18:44:43.591687+0000 osd.4 (osd.4) 169 : cluster 0 7.1 scrub starts 2023-09-23T18:44:44.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:44 smithi098 bash[76817]: cluster 2023-09-23T18:44:43.995138+0000 mgr.y (mgr.26054) 795 : cluster 0 pgmap v817: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 4.8 MiB/s wr, 914 op/s; 5581/276984 objects degraded (2.015%); 8227/276984 objects misplaced (2.970%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T18:44:44.988 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:45.019 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:45.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:45.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:45.123 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:45.169 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:45.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:45.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:45.274 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:45.301 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:45.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:45.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:45.405 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:45.438 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:45.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:45.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:45.541 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:45.570 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:45.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:45.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:45.673 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:45.697 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:45.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:45.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:45.804 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:45.817 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:45 smithi098 bash[79516]: cluster 2023-09-23T18:44:44.438056+0000 osd.3 (osd.3) 3 : cluster 0 7.3 scrub starts 2023-09-23T18:44:45.818 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:45 smithi098 bash[79516]: cluster 2023-09-23T18:44:45.049721+0000 mon.a (mon.0) 1078 : cluster 3 Health check update: Degraded data redundancy: 5581/276984 objects degraded (2.015%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:44:45.818 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:45 smithi098 bash[79516]: cluster 2023-09-23T18:44:45.154253+0000 osd.0 (osd.0) 7 : cluster 0 7.9 scrub starts 2023-09-23T18:44:45.818 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:45 smithi098 bash[76817]: cluster 2023-09-23T18:44:44.438056+0000 osd.3 (osd.3) 3 : cluster 0 7.3 scrub starts 2023-09-23T18:44:45.818 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:45 smithi098 bash[76817]: cluster 2023-09-23T18:44:45.049721+0000 mon.a (mon.0) 1078 : cluster 3 Health check update: Degraded data redundancy: 5581/276984 objects degraded (2.015%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:44:45.819 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:45 smithi098 bash[76817]: cluster 2023-09-23T18:44:45.154253+0000 osd.0 (osd.0) 7 : cluster 0 7.9 scrub starts 2023-09-23T18:44:45.844 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:45.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:45.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:45.948 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:45.975 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:45.976 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:45 smithi173 bash[59422]: cluster 2023-09-23T18:44:44.438056+0000 osd.3 (osd.3) 3 : cluster 0 7.3 scrub starts 2023-09-23T18:44:45.976 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:45 smithi173 bash[59422]: cluster 2023-09-23T18:44:45.049721+0000 mon.a (mon.0) 1078 : cluster 3 Health check update: Degraded data redundancy: 5581/276984 objects degraded (2.015%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:44:45.976 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:45 smithi173 bash[59422]: cluster 2023-09-23T18:44:45.154253+0000 osd.0 (osd.0) 7 : cluster 0 7.9 scrub starts 2023-09-23T18:44:45.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:45.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:46.078 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:46.111 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:46.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:46.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:46.214 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:46.236 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:46.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:46.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:46.252 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:44:46.339 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:46.349 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:44:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:44:46] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.33.4" 2023-09-23T18:44:46.374 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:46.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:46.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:46.477 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:46.511 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:46.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:46.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:46.614 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:46.670 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:46.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:46.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:46.768 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:44:46.774 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:46.814 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:46.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:46.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:46.923 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:46.940 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:46 smithi098 bash[79516]: cluster 2023-09-23T18:44:45.068358+0000 osd.6 (osd.6) 142 : cluster 0 7.7 scrub starts 2023-09-23T18:44:46.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:46 smithi098 bash[79516]: cluster 2023-09-23T18:44:45.419193+0000 osd.3 (osd.3) 4 : cluster 0 7.2 scrub starts 2023-09-23T18:44:46.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:46 smithi098 bash[79516]: cluster 2023-09-23T18:44:45.531359+0000 osd.4 (osd.4) 170 : cluster 0 7.1 scrub starts 2023-09-23T18:44:46.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:46 smithi098 bash[79516]: cluster 2023-09-23T18:44:45.995907+0000 mgr.y (mgr.26054) 796 : cluster 0 pgmap v818: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 1.0 MiB/s wr, 898 op/s; 5581/272262 objects degraded (2.050%); 8227/272262 objects misplaced (3.022%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T18:44:46.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:46 smithi098 bash[76817]: cluster 2023-09-23T18:44:45.068358+0000 osd.6 (osd.6) 142 : cluster 0 7.7 scrub starts 2023-09-23T18:44:46.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:46 smithi098 bash[76817]: cluster 2023-09-23T18:44:45.419193+0000 osd.3 (osd.3) 4 : cluster 0 7.2 scrub starts 2023-09-23T18:44:46.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:46 smithi098 bash[76817]: cluster 2023-09-23T18:44:45.531359+0000 osd.4 (osd.4) 170 : cluster 0 7.1 scrub starts 2023-09-23T18:44:46.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:46 smithi098 bash[76817]: cluster 2023-09-23T18:44:45.995907+0000 mgr.y (mgr.26054) 796 : cluster 0 pgmap v818: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 1.0 MiB/s wr, 898 op/s; 5581/272262 objects degraded (2.050%); 8227/272262 objects misplaced (3.022%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T18:44:46.947 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:46.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:46.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:47.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:46 smithi173 bash[59422]: cluster 2023-09-23T18:44:45.068358+0000 osd.6 (osd.6) 142 : cluster 0 7.7 scrub starts 2023-09-23T18:44:47.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:46 smithi173 bash[59422]: cluster 2023-09-23T18:44:45.419193+0000 osd.3 (osd.3) 4 : cluster 0 7.2 scrub starts 2023-09-23T18:44:47.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:46 smithi173 bash[59422]: cluster 2023-09-23T18:44:45.531359+0000 osd.4 (osd.4) 170 : cluster 0 7.1 scrub starts 2023-09-23T18:44:47.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:46 smithi173 bash[59422]: cluster 2023-09-23T18:44:45.995907+0000 mgr.y (mgr.26054) 796 : cluster 0 pgmap v818: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 1.0 MiB/s wr, 898 op/s; 5581/272262 objects degraded (2.050%); 8227/272262 objects misplaced (3.022%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T18:44:47.050 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:47.082 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:47.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:47.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:47.186 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:47.228 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:47.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:47.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:47.329 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:47.360 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:47.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:47.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:47.466 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:47.488 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:47.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:47.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:47.593 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:47.617 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:47.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:47.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:47.721 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:47.764 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:47.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:47.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:47.789 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:44:47.789 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:44:47.868 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:47.878 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:47 smithi098 bash[76817]: cluster 2023-09-23T18:44:46.103046+0000 osd.6 (osd.6) 143 : cluster 0 7.7 scrub starts 2023-09-23T18:44:47.878 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:47 smithi098 bash[76817]: cluster 2023-09-23T18:44:46.537060+0000 osd.4 (osd.4) 171 : cluster 0 7.1 scrub starts 2023-09-23T18:44:47.878 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:47 smithi098 bash[76817]: cluster 2023-09-23T18:44:47.150343+0000 osd.0 (osd.0) 8 : cluster 0 7.6 scrub starts 2023-09-23T18:44:47.878 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:47 smithi098 bash[79516]: cluster 2023-09-23T18:44:46.103046+0000 osd.6 (osd.6) 143 : cluster 0 7.7 scrub starts 2023-09-23T18:44:47.878 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:47 smithi098 bash[79516]: cluster 2023-09-23T18:44:46.537060+0000 osd.4 (osd.4) 171 : cluster 0 7.1 scrub starts 2023-09-23T18:44:47.879 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:47 smithi098 bash[79516]: cluster 2023-09-23T18:44:47.150343+0000 osd.0 (osd.0) 8 : cluster 0 7.6 scrub starts 2023-09-23T18:44:47.897 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:47.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:47.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:48.002 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:48.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:47 smithi173 bash[59422]: cluster 2023-09-23T18:44:46.103046+0000 osd.6 (osd.6) 143 : cluster 0 7.7 scrub starts 2023-09-23T18:44:48.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:47 smithi173 bash[59422]: cluster 2023-09-23T18:44:46.537060+0000 osd.4 (osd.4) 171 : cluster 0 7.1 scrub starts 2023-09-23T18:44:48.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:47 smithi173 bash[59422]: cluster 2023-09-23T18:44:47.150343+0000 osd.0 (osd.0) 8 : cluster 0 7.6 scrub starts 2023-09-23T18:44:48.045 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:48.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:48.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:48.149 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:48.179 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:48.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:48.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:48.284 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:48.312 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:48.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:48.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:48.417 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:48.447 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:48.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:48.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:48.552 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:48.585 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:48.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:48.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:48.689 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:48.711 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:48.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:48.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:48.815 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:48.849 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:48.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:48.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:48.952 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:48.970 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:48 smithi098 bash[79516]: cluster 2023-09-23T18:44:47.437741+0000 osd.3 (osd.3) 5 : cluster 0 7.13 scrub starts 2023-09-23T18:44:48.971 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:48 smithi098 bash[79516]: cluster 2023-09-23T18:44:47.996558+0000 mgr.y (mgr.26054) 797 : cluster 0 pgmap v819: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 892 op/s; 5581/267297 objects degraded (2.088%); 7847/267297 objects misplaced (2.936%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T18:44:48.971 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:48 smithi098 bash[79516]: cluster 2023-09-23T18:44:48.394454+0000 osd.3 (osd.3) 6 : cluster 0 7.3 scrub starts 2023-09-23T18:44:48.976 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:48 smithi098 bash[76817]: cluster 2023-09-23T18:44:47.437741+0000 osd.3 (osd.3) 5 : cluster 0 7.13 scrub starts 2023-09-23T18:44:48.977 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:48 smithi098 bash[76817]: cluster 2023-09-23T18:44:47.996558+0000 mgr.y (mgr.26054) 797 : cluster 0 pgmap v819: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 892 op/s; 5581/267297 objects degraded (2.088%); 7847/267297 objects misplaced (2.936%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T18:44:48.977 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:48 smithi098 bash[76817]: cluster 2023-09-23T18:44:48.394454+0000 osd.3 (osd.3) 6 : cluster 0 7.3 scrub starts 2023-09-23T18:44:48.977 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:48.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:48.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:49.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:48 smithi173 bash[59422]: cluster 2023-09-23T18:44:47.437741+0000 osd.3 (osd.3) 5 : cluster 0 7.13 scrub starts 2023-09-23T18:44:49.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:48 smithi173 bash[59422]: cluster 2023-09-23T18:44:47.996558+0000 mgr.y (mgr.26054) 797 : cluster 0 pgmap v819: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 892 op/s; 5581/267297 objects degraded (2.088%); 7847/267297 objects misplaced (2.936%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T18:44:49.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:48 smithi173 bash[59422]: cluster 2023-09-23T18:44:48.394454+0000 osd.3 (osd.3) 6 : cluster 0 7.3 scrub starts 2023-09-23T18:44:49.079 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:49.107 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:49.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:49.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:49.212 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:49.240 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:49.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:49.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:49.345 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:49.368 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:49.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:49.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:49.474 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:49.498 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:49.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:49.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:49.602 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:49.629 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:49.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:49.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:49.735 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:49.760 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:49.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:49.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:49.863 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:49.890 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:49.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:49.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:49.993 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:50.013 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:49 smithi098 bash[79516]: cluster 2023-09-23T18:44:49.051914+0000 osd.6 (osd.6) 144 : cluster 0 7.7 scrub starts 2023-09-23T18:44:50.014 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:49 smithi098 bash[76817]: cluster 2023-09-23T18:44:49.051914+0000 osd.6 (osd.6) 144 : cluster 0 7.7 scrub starts 2023-09-23T18:44:50.014 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:50.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:50.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:50.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:49 smithi173 bash[59422]: cluster 2023-09-23T18:44:49.051914+0000 osd.6 (osd.6) 144 : cluster 0 7.7 scrub starts 2023-09-23T18:44:50.118 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:50.150 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:50.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:50.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:50.253 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:50.286 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:50.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:50.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:50.390 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:50.417 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:50.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:50.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:50.520 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:50.550 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:50.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:50.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:50.653 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:50.696 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:50.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:50.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:50.802 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:50.832 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:50.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:50.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:50.841 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:44:50.940 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:50.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:50 smithi098 bash[76817]: cluster 2023-09-23T18:44:49.516508+0000 osd.4 (osd.4) 172 : cluster 0 7.1 scrub starts 2023-09-23T18:44:50.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:50 smithi098 bash[76817]: cluster 2023-09-23T18:44:49.997427+0000 mgr.y (mgr.26054) 798 : cluster 0 pgmap v820: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 871 op/s; 5581/262500 objects degraded (2.126%); 7847/262500 objects misplaced (2.989%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T18:44:50.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:50 smithi098 bash[76817]: cluster 2023-09-23T18:44:50.054367+0000 mon.a (mon.0) 1079 : cluster 3 Health check update: Degraded data redundancy: 5581/267297 objects degraded (2.088%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:44:50.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:50 smithi098 bash[76817]: cluster 2023-09-23T18:44:50.086219+0000 osd.6 (osd.6) 145 : cluster 0 7.7 scrub starts 2023-09-23T18:44:50.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:50 smithi098 bash[79516]: cluster 2023-09-23T18:44:49.516508+0000 osd.4 (osd.4) 172 : cluster 0 7.1 scrub starts 2023-09-23T18:44:50.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:50 smithi098 bash[79516]: cluster 2023-09-23T18:44:49.997427+0000 mgr.y (mgr.26054) 798 : cluster 0 pgmap v820: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 871 op/s; 5581/262500 objects degraded (2.126%); 7847/262500 objects misplaced (2.989%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T18:44:50.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:50 smithi098 bash[79516]: cluster 2023-09-23T18:44:50.054367+0000 mon.a (mon.0) 1079 : cluster 3 Health check update: Degraded data redundancy: 5581/267297 objects degraded (2.088%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:44:50.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:50 smithi098 bash[79516]: cluster 2023-09-23T18:44:50.086219+0000 osd.6 (osd.6) 145 : cluster 0 7.7 scrub starts 2023-09-23T18:44:50.968 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:50.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:50.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:51.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:50 smithi173 bash[59422]: cluster 2023-09-23T18:44:49.516508+0000 osd.4 (osd.4) 172 : cluster 0 7.1 scrub starts 2023-09-23T18:44:51.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:50 smithi173 bash[59422]: cluster 2023-09-23T18:44:49.997427+0000 mgr.y (mgr.26054) 798 : cluster 0 pgmap v820: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 871 op/s; 5581/262500 objects degraded (2.126%); 7847/262500 objects misplaced (2.989%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T18:44:51.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:50 smithi173 bash[59422]: cluster 2023-09-23T18:44:50.054367+0000 mon.a (mon.0) 1079 : cluster 3 Health check update: Degraded data redundancy: 5581/267297 objects degraded (2.088%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:44:51.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:50 smithi173 bash[59422]: cluster 2023-09-23T18:44:50.086219+0000 osd.6 (osd.6) 145 : cluster 0 7.7 scrub starts 2023-09-23T18:44:51.071 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:51.097 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:51.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:51.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:51.203 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:51.235 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:51.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:51.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:51.338 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:51.364 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:51.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:51.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:51.469 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:51.497 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:51.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:51.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:51.600 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:51.634 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:51.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:51.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:51.737 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:51.762 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:51.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:51.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:51.868 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:51.893 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:51.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:51.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:51.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:51 smithi098 bash[76817]: cluster 2023-09-23T18:44:50.472645+0000 osd.4 (osd.4) 173 : cluster 0 7.1 scrub starts 2023-09-23T18:44:51.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:51 smithi098 bash[76817]: cluster 2023-09-23T18:44:51.110329+0000 osd.6 (osd.6) 146 : cluster 0 7.7 scrub starts 2023-09-23T18:44:51.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:51 smithi098 bash[79516]: cluster 2023-09-23T18:44:50.472645+0000 osd.4 (osd.4) 173 : cluster 0 7.1 scrub starts 2023-09-23T18:44:51.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:51 smithi098 bash[79516]: cluster 2023-09-23T18:44:51.110329+0000 osd.6 (osd.6) 146 : cluster 0 7.7 scrub starts 2023-09-23T18:44:51.997 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:52.026 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:52.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:51 smithi173 bash[59422]: cluster 2023-09-23T18:44:50.472645+0000 osd.4 (osd.4) 173 : cluster 0 7.1 scrub starts 2023-09-23T18:44:52.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:51 smithi173 bash[59422]: cluster 2023-09-23T18:44:51.110329+0000 osd.6 (osd.6) 146 : cluster 0 7.7 scrub starts 2023-09-23T18:44:52.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:52.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:52.133 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:52.161 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:52.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:52.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:52.265 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:52.293 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:52.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:52.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:52.396 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:52.421 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:52.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:52.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:52.525 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:52.551 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:52.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:52.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:52.654 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:52.681 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:52.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:52.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:52.786 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:52.815 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:52.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:52.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:52.921 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:52.929 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:52 smithi098 bash[76817]: cluster 2023-09-23T18:44:51.504419+0000 osd.3 (osd.3) 7 : cluster 0 7.2 scrub starts 2023-09-23T18:44:52.929 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:52 smithi098 bash[76817]: cluster 2023-09-23T18:44:51.998078+0000 mgr.y (mgr.26054) 799 : cluster 0 pgmap v821: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 834 op/s; 5581/256989 objects degraded (2.172%); 7847/256989 objects misplaced (3.053%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:44:52.930 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:52 smithi098 bash[79516]: cluster 2023-09-23T18:44:51.504419+0000 osd.3 (osd.3) 7 : cluster 0 7.2 scrub starts 2023-09-23T18:44:52.930 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:52 smithi098 bash[79516]: cluster 2023-09-23T18:44:51.998078+0000 mgr.y (mgr.26054) 799 : cluster 0 pgmap v821: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 834 op/s; 5581/256989 objects degraded (2.172%); 7847/256989 objects misplaced (3.053%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:44:52.946 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:52.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:52.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:52 smithi173 bash[59422]: cluster 2023-09-23T18:44:51.504419+0000 osd.3 (osd.3) 7 : cluster 0 7.2 scrub starts 2023-09-23T18:44:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:52 smithi173 bash[59422]: cluster 2023-09-23T18:44:51.998078+0000 mgr.y (mgr.26054) 799 : cluster 0 pgmap v821: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 834 op/s; 5581/256989 objects degraded (2.172%); 7847/256989 objects misplaced (3.053%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:44:53.051 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:53.075 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:53.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:53.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:53.180 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:53.210 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:53.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:53.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:53.314 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:53.350 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:53.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:53.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:53.454 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:53.478 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:53.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:53.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:53.583 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:53.607 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:53.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:53.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:53.710 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:53.738 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:53.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:53.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:53.840 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:53.875 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:53.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:53.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:53.977 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:53.987 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:53 smithi173 bash[59422]: cluster 2023-09-23T18:44:52.499984+0000 osd.3 (osd.3) 8 : cluster 0 7.13 scrub starts 2023-09-23T18:44:53.987 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:53 smithi173 bash[59422]: cluster 2023-09-23T18:44:53.042776+0000 osd.0 (osd.0) 9 : cluster 0 7.16 scrub starts 2023-09-23T18:44:54.004 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:54.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:54.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:54.109 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:54.147 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:54.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:54.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:54.179 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:53 smithi098 bash[79516]: cluster 2023-09-23T18:44:52.499984+0000 osd.3 (osd.3) 8 : cluster 0 7.13 scrub starts 2023-09-23T18:44:54.179 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:53 smithi098 bash[79516]: cluster 2023-09-23T18:44:53.042776+0000 osd.0 (osd.0) 9 : cluster 0 7.16 scrub starts 2023-09-23T18:44:54.179 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:53 smithi098 bash[76817]: cluster 2023-09-23T18:44:52.499984+0000 osd.3 (osd.3) 8 : cluster 0 7.13 scrub starts 2023-09-23T18:44:54.179 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:53 smithi098 bash[76817]: cluster 2023-09-23T18:44:53.042776+0000 osd.0 (osd.0) 9 : cluster 0 7.16 scrub starts 2023-09-23T18:44:54.250 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:54.284 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:54.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:54.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:54.390 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:54.429 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:54.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:54.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:54.535 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:54.565 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:54.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:54.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:54.668 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:54.710 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:54.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:54.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:54.814 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:54.822 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:54 smithi173 bash[59422]: cluster 2023-09-23T18:44:53.495897+0000 osd.3 (osd.3) 9 : cluster 0 7.3 scrub starts 2023-09-23T18:44:54.822 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:54 smithi173 bash[59422]: cluster 2023-09-23T18:44:53.998716+0000 mgr.y (mgr.26054) 800 : cluster 0 pgmap v822: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 879 op/s; 5581/251163 objects degraded (2.222%); 7415/251163 objects misplaced (2.952%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T18:44:54.822 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:54 smithi173 bash[59422]: cluster 2023-09-23T18:44:54.085157+0000 osd.0 (osd.0) 10 : cluster 0 7.19 scrub starts 2023-09-23T18:44:54.822 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:54 smithi173 bash[59422]: audit 2023-09-23T18:44:54.608671+0000 mon.a (mon.0) 1080 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:44:54.838 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:54.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:54.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:54.941 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:54.971 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:54.973 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:54 smithi098 bash[79516]: cluster 2023-09-23T18:44:53.495897+0000 osd.3 (osd.3) 9 : cluster 0 7.3 scrub starts 2023-09-23T18:44:54.973 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:54 smithi098 bash[79516]: cluster 2023-09-23T18:44:53.998716+0000 mgr.y (mgr.26054) 800 : cluster 0 pgmap v822: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 879 op/s; 5581/251163 objects degraded (2.222%); 7415/251163 objects misplaced (2.952%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T18:44:54.973 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:54 smithi098 bash[79516]: cluster 2023-09-23T18:44:54.085157+0000 osd.0 (osd.0) 10 : cluster 0 7.19 scrub starts 2023-09-23T18:44:54.973 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:54 smithi098 bash[79516]: audit 2023-09-23T18:44:54.608671+0000 mon.a (mon.0) 1080 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:44:54.974 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:54 smithi098 bash[76817]: cluster 2023-09-23T18:44:53.495897+0000 osd.3 (osd.3) 9 : cluster 0 7.3 scrub starts 2023-09-23T18:44:54.974 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:54 smithi098 bash[76817]: cluster 2023-09-23T18:44:53.998716+0000 mgr.y (mgr.26054) 800 : cluster 0 pgmap v822: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 879 op/s; 5581/251163 objects degraded (2.222%); 7415/251163 objects misplaced (2.952%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T18:44:54.974 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:54 smithi098 bash[76817]: cluster 2023-09-23T18:44:54.085157+0000 osd.0 (osd.0) 10 : cluster 0 7.19 scrub starts 2023-09-23T18:44:54.974 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:54 smithi098 bash[76817]: audit 2023-09-23T18:44:54.608671+0000 mon.a (mon.0) 1080 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:44:54.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:54.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:55.076 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:55.099 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:55.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:55.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:55.203 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:55.234 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:55.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:55.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:55.339 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:55.365 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:55.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:55.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:55.468 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:55.498 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:55.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:55.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:55.601 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:55.627 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:55.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:55.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:55.732 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:55.754 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:55.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:55.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:55.857 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:55.866 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:55 smithi098 bash[79516]: cluster 2023-09-23T18:44:54.531782+0000 osd.3 (osd.3) 10 : cluster 0 7.2 scrub starts 2023-09-23T18:44:55.866 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:55 smithi098 bash[79516]: cluster 2023-09-23T18:44:55.055524+0000 mon.a (mon.0) 1081 : cluster 3 Health check update: Degraded data redundancy: 5581/251163 objects degraded (2.222%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:44:55.866 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:55 smithi098 bash[79516]: cluster 2023-09-23T18:44:55.128113+0000 osd.0 (osd.0) 11 : cluster 0 7.14 scrub starts 2023-09-23T18:44:55.867 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:55 smithi098 bash[76817]: cluster 2023-09-23T18:44:54.531782+0000 osd.3 (osd.3) 10 : cluster 0 7.2 scrub starts 2023-09-23T18:44:55.867 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:55 smithi098 bash[76817]: cluster 2023-09-23T18:44:55.055524+0000 mon.a (mon.0) 1081 : cluster 3 Health check update: Degraded data redundancy: 5581/251163 objects degraded (2.222%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:44:55.867 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:55 smithi098 bash[76817]: cluster 2023-09-23T18:44:55.128113+0000 osd.0 (osd.0) 11 : cluster 0 7.14 scrub starts 2023-09-23T18:44:55.891 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:44:55.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:55.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:44:55.995 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:56.024 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:56.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:56.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:56.127 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:56.153 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:56.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:56.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:56.164 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:44:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:44:56] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.33.4" 2023-09-23T18:44:56.257 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:56.270 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:55 smithi173 bash[59422]: cluster 2023-09-23T18:44:54.531782+0000 osd.3 (osd.3) 10 : cluster 0 7.2 scrub starts 2023-09-23T18:44:56.270 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:55 smithi173 bash[59422]: cluster 2023-09-23T18:44:55.055524+0000 mon.a (mon.0) 1081 : cluster 3 Health check update: Degraded data redundancy: 5581/251163 objects degraded (2.222%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:44:56.270 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:55 smithi173 bash[59422]: cluster 2023-09-23T18:44:55.128113+0000 osd.0 (osd.0) 11 : cluster 0 7.14 scrub starts 2023-09-23T18:44:56.292 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:56.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:56.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:56.396 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:56.423 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:56.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:56.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:56.528 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:56.553 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:56.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:56.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:56.655 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:56.682 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:56.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:56.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:56.788 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:56.814 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:56.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:56.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:56.917 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:56.947 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:56.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:56.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:57.050 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:57.059 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:56 smithi173 bash[59422]: cluster 2023-09-23T18:44:55.374618+0000 osd.4 (osd.4) 174 : cluster 0 7.1 scrub starts 2023-09-23T18:44:57.060 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:56 smithi173 bash[59422]: cluster 2023-09-23T18:44:55.999314+0000 mgr.y (mgr.26054) 801 : cluster 0 pgmap v823: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 852 op/s; 5581/246282 objects degraded (2.266%); 7415/246282 objects misplaced (3.011%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T18:44:57.078 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:57.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:57.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:57.181 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:57.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:56 smithi098 bash[76817]: cluster 2023-09-23T18:44:55.374618+0000 osd.4 (osd.4) 174 : cluster 0 7.1 scrub starts 2023-09-23T18:44:57.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:56 smithi098 bash[76817]: cluster 2023-09-23T18:44:55.999314+0000 mgr.y (mgr.26054) 801 : cluster 0 pgmap v823: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 852 op/s; 5581/246282 objects degraded (2.266%); 7415/246282 objects misplaced (3.011%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T18:44:57.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:56 smithi098 bash[79516]: cluster 2023-09-23T18:44:55.374618+0000 osd.4 (osd.4) 174 : cluster 0 7.1 scrub starts 2023-09-23T18:44:57.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:56 smithi098 bash[79516]: cluster 2023-09-23T18:44:55.999314+0000 mgr.y (mgr.26054) 801 : cluster 0 pgmap v823: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 852 op/s; 5581/246282 objects degraded (2.266%); 7415/246282 objects misplaced (3.011%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T18:44:57.204 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:57.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:57.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:57.308 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:57.332 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:57.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:57.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:57.439 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:57.469 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:57.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:57.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:57.573 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:57.606 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:57.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:57.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:57.713 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:57.737 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:57.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:57.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:57.840 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:57.848 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:57 smithi098 bash[76817]: cluster 2023-09-23T18:44:56.413138+0000 osd.4 (osd.4) 175 : cluster 0 7.1 scrub starts 2023-09-23T18:44:57.849 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:57 smithi098 bash[76817]: cluster 2023-09-23T18:44:57.157507+0000 osd.0 (osd.0) 12 : cluster 0 7.4 scrub starts 2023-09-23T18:44:57.849 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:57 smithi098 bash[79516]: cluster 2023-09-23T18:44:56.413138+0000 osd.4 (osd.4) 175 : cluster 0 7.1 scrub starts 2023-09-23T18:44:57.849 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:57 smithi098 bash[79516]: cluster 2023-09-23T18:44:57.157507+0000 osd.0 (osd.0) 12 : cluster 0 7.4 scrub starts 2023-09-23T18:44:57.879 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:57.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:57.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:57.986 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:58.013 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:58.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:58.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:58.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:57 smithi173 bash[59422]: cluster 2023-09-23T18:44:56.413138+0000 osd.4 (osd.4) 175 : cluster 0 7.1 scrub starts 2023-09-23T18:44:58.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:57 smithi173 bash[59422]: cluster 2023-09-23T18:44:57.157507+0000 osd.0 (osd.0) 12 : cluster 0 7.4 scrub starts 2023-09-23T18:44:58.117 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:58.147 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:58.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:58.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:58.250 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:58.277 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:44:58.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:58.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:44:58.380 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:58.405 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:58.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:58.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:58.508 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:58.536 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:58.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:58.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:58.639 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:58.668 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:58.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:58.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:58.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:58.797 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:58.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:58.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:44:58.803 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:58 smithi098 bash[76817]: cluster 2023-09-23T18:44:57.602469+0000 osd.3 (osd.3) 11 : cluster 0 7.13 scrub starts 2023-09-23T18:44:58.803 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:58 smithi098 bash[76817]: cluster 2023-09-23T18:44:57.999980+0000 mgr.y (mgr.26054) 802 : cluster 0 pgmap v824: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 863 op/s; 5581/241176 objects degraded (2.314%); 6977/241176 objects misplaced (2.893%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:44:58.804 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:58 smithi098 bash[76817]: cluster 2023-09-23T18:44:58.086210+0000 osd.6 (osd.6) 147 : cluster 0 7.7 scrub starts 2023-09-23T18:44:58.900 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:58.909 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:58 smithi173 bash[59422]: cluster 2023-09-23T18:44:57.602469+0000 osd.3 (osd.3) 11 : cluster 0 7.13 scrub starts 2023-09-23T18:44:58.909 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:58 smithi173 bash[59422]: cluster 2023-09-23T18:44:57.999980+0000 mgr.y (mgr.26054) 802 : cluster 0 pgmap v824: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 863 op/s; 5581/241176 objects degraded (2.314%); 6977/241176 objects misplaced (2.893%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:44:58.909 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:58 smithi173 bash[59422]: cluster 2023-09-23T18:44:58.086210+0000 osd.6 (osd.6) 147 : cluster 0 7.7 scrub starts 2023-09-23T18:44:58.925 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:44:58.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:58.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:44:59.028 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:59.060 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:59.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:59.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:59.164 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:59.174 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:58 smithi098 bash[79516]: cluster 2023-09-23T18:44:57.602469+0000 osd.3 (osd.3) 11 : cluster 0 7.13 scrub starts 2023-09-23T18:44:59.174 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:58 smithi098 bash[79516]: cluster 2023-09-23T18:44:57.999980+0000 mgr.y (mgr.26054) 802 : cluster 0 pgmap v824: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 863 op/s; 5581/241176 objects degraded (2.314%); 6977/241176 objects misplaced (2.893%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:44:59.174 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:58 smithi098 bash[79516]: cluster 2023-09-23T18:44:58.086210+0000 osd.6 (osd.6) 147 : cluster 0 7.7 scrub starts 2023-09-23T18:44:59.198 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:44:59.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:59.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:44:59.301 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:59.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:44:59.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:59.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:44:59.430 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:59.460 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:59.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:59.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:59.567 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:59.599 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:44:59.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:59.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:44:59.704 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:59.733 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:59.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:59.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:59.805 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:59 smithi098 bash[76817]: cluster 2023-09-23T18:44:59.064109+0000 osd.6 (osd.6) 148 : cluster 0 7.7 scrub starts 2023-09-23T18:44:59.805 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:44:59 smithi098 bash[76817]: cluster 2023-09-23T18:44:59.104768+0000 osd.0 (osd.0) 13 : cluster 0 7.9 scrub starts 2023-09-23T18:44:59.805 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:59 smithi098 bash[79516]: cluster 2023-09-23T18:44:59.064109+0000 osd.6 (osd.6) 148 : cluster 0 7.7 scrub starts 2023-09-23T18:44:59.805 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:44:59 smithi098 bash[79516]: cluster 2023-09-23T18:44:59.104768+0000 osd.0 (osd.0) 13 : cluster 0 7.9 scrub starts 2023-09-23T18:44:59.836 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:59.866 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:44:59.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:44:59.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:44:59.972 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:44:59.999 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:00.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:00.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:00.102 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:00.113 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:59 smithi173 bash[59422]: cluster 2023-09-23T18:44:59.064109+0000 osd.6 (osd.6) 148 : cluster 0 7.7 scrub starts 2023-09-23T18:45:00.113 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:44:59 smithi173 bash[59422]: cluster 2023-09-23T18:44:59.104768+0000 osd.0 (osd.0) 13 : cluster 0 7.9 scrub starts 2023-09-23T18:45:00.132 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:00.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:00.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:00.235 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:00.264 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:00.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:00.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:00.369 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:00.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:00.401 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:00.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:00.502 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:00.523 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:00.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:00.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:00.627 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:00.649 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:00.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:00.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:00.752 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:00.787 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:00.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:00.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:00.894 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:00.918 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:00.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:00.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:01.021 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:01.050 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:01.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:01.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:01.155 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:01.157 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:00 smithi098 bash[76817]: cluster 2023-09-23T18:44:59.526334+0000 osd.3 (osd.3) 12 : cluster 0 7.3 scrub starts 2023-09-23T18:45:01.157 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:00 smithi098 bash[76817]: cluster 2023-09-23T18:45:00.000611+0000 mgr.y (mgr.26054) 803 : cluster 0 pgmap v825: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 858 op/s; 5581/236367 objects degraded (2.361%); 6977/236367 objects misplaced (2.952%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:45:01.157 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:00 smithi098 bash[76817]: cluster 2023-09-23T18:45:00.056455+0000 mon.a (mon.0) 1082 : cluster 3 Health check update: Degraded data redundancy: 5581/241176 objects degraded (2.314%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:45:01.157 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:00 smithi098 bash[76817]: cluster 2023-09-23T18:45:00.108855+0000 osd.6 (osd.6) 149 : cluster 0 7.7 scrub starts 2023-09-23T18:45:01.158 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:00 smithi098 bash[76817]: cluster 2023-09-23T18:45:00.148744+0000 osd.0 (osd.0) 14 : cluster 0 7.6 scrub starts 2023-09-23T18:45:01.158 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:00 smithi098 bash[79516]: cluster 2023-09-23T18:44:59.526334+0000 osd.3 (osd.3) 12 : cluster 0 7.3 scrub starts 2023-09-23T18:45:01.158 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:00 smithi098 bash[79516]: cluster 2023-09-23T18:45:00.000611+0000 mgr.y (mgr.26054) 803 : cluster 0 pgmap v825: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 858 op/s; 5581/236367 objects degraded (2.361%); 6977/236367 objects misplaced (2.952%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:45:01.158 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:00 smithi098 bash[79516]: cluster 2023-09-23T18:45:00.056455+0000 mon.a (mon.0) 1082 : cluster 3 Health check update: Degraded data redundancy: 5581/241176 objects degraded (2.314%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:45:01.158 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:00 smithi098 bash[79516]: cluster 2023-09-23T18:45:00.108855+0000 osd.6 (osd.6) 149 : cluster 0 7.7 scrub starts 2023-09-23T18:45:01.158 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:00 smithi098 bash[79516]: cluster 2023-09-23T18:45:00.148744+0000 osd.0 (osd.0) 14 : cluster 0 7.6 scrub starts 2023-09-23T18:45:01.165 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:00 smithi173 bash[59422]: cluster 2023-09-23T18:44:59.526334+0000 osd.3 (osd.3) 12 : cluster 0 7.3 scrub starts 2023-09-23T18:45:01.165 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:00 smithi173 bash[59422]: cluster 2023-09-23T18:45:00.000611+0000 mgr.y (mgr.26054) 803 : cluster 0 pgmap v825: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 858 op/s; 5581/236367 objects degraded (2.361%); 6977/236367 objects misplaced (2.952%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:45:01.165 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:00 smithi173 bash[59422]: cluster 2023-09-23T18:45:00.056455+0000 mon.a (mon.0) 1082 : cluster 3 Health check update: Degraded data redundancy: 5581/241176 objects degraded (2.314%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:45:01.165 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:00 smithi173 bash[59422]: cluster 2023-09-23T18:45:00.108855+0000 osd.6 (osd.6) 149 : cluster 0 7.7 scrub starts 2023-09-23T18:45:01.165 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:00 smithi173 bash[59422]: cluster 2023-09-23T18:45:00.148744+0000 osd.0 (osd.0) 14 : cluster 0 7.6 scrub starts 2023-09-23T18:45:01.182 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:01.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:01.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:01.285 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:01.310 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:01.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:01.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:01.414 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:01.453 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:01.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:01.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:01.559 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:01.576 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:01.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:01.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:01.679 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:01.712 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:01.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:01.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:01.817 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:01.848 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:01.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:01.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:01.957 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:01.966 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:01 smithi173 bash[59422]: cluster 2023-09-23T18:45:00.447458+0000 osd.4 (osd.4) 176 : cluster 0 7.1 scrub starts 2023-09-23T18:45:01.982 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:01.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:01.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:02.087 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:02.111 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:02.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:02.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:02.132 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:01 smithi098 bash[76817]: cluster 2023-09-23T18:45:00.447458+0000 osd.4 (osd.4) 176 : cluster 0 7.1 scrub starts 2023-09-23T18:45:02.132 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:01 smithi098 bash[79516]: cluster 2023-09-23T18:45:00.447458+0000 osd.4 (osd.4) 176 : cluster 0 7.1 scrub starts 2023-09-23T18:45:02.218 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:02.245 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:02.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:02.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:02.347 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:02.374 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:02.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:02.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:02.477 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:02.500 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:02.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:02.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:02.605 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:02.632 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:02.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:02.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:02.740 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:02.773 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:02.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:02.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:02.877 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:02.902 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:02.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:02.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:03.007 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:03.007 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:02 smithi098 bash[79516]: cluster 2023-09-23T18:45:01.505992+0000 osd.3 (osd.3) 13 : cluster 0 7.2 scrub starts 2023-09-23T18:45:03.008 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:02 smithi098 bash[79516]: cluster 2023-09-23T18:45:02.001344+0000 mgr.y (mgr.26054) 804 : cluster 0 pgmap v826: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 871 op/s; 5581/231132 objects degraded (2.415%); 6977/231132 objects misplaced (3.019%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:45:03.008 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:02 smithi098 bash[79516]: cluster 2023-09-23T18:45:02.102168+0000 osd.6 (osd.6) 150 : cluster 0 7.7 scrub starts 2023-09-23T18:45:03.008 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:02 smithi098 bash[76817]: cluster 2023-09-23T18:45:01.505992+0000 osd.3 (osd.3) 13 : cluster 0 7.2 scrub starts 2023-09-23T18:45:03.008 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:02 smithi098 bash[76817]: cluster 2023-09-23T18:45:02.001344+0000 mgr.y (mgr.26054) 804 : cluster 0 pgmap v826: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 871 op/s; 5581/231132 objects degraded (2.415%); 6977/231132 objects misplaced (3.019%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:45:03.008 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:02 smithi098 bash[76817]: cluster 2023-09-23T18:45:02.102168+0000 osd.6 (osd.6) 150 : cluster 0 7.7 scrub starts 2023-09-23T18:45:03.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:02 smithi173 bash[59422]: cluster 2023-09-23T18:45:01.505992+0000 osd.3 (osd.3) 13 : cluster 0 7.2 scrub starts 2023-09-23T18:45:03.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:02 smithi173 bash[59422]: cluster 2023-09-23T18:45:02.001344+0000 mgr.y (mgr.26054) 804 : cluster 0 pgmap v826: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 871 op/s; 5581/231132 objects degraded (2.415%); 6977/231132 objects misplaced (3.019%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:45:03.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:02 smithi173 bash[59422]: cluster 2023-09-23T18:45:02.102168+0000 osd.6 (osd.6) 150 : cluster 0 7.7 scrub starts 2023-09-23T18:45:03.033 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:03.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:03.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:03.136 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:03.164 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:03.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:03.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:03.267 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:03.295 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:03.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:03.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:03.402 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:03.426 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:03.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:03.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:03.528 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:03.556 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:03.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:03.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:03.659 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:03.700 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:03.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:03.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:03.804 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:03.829 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:03.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:03.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:03.932 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:03.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:03 smithi098 bash[76817]: cluster 2023-09-23T18:45:03.224133+0000 osd.0 (osd.0) 15 : cluster 0 7.16 scrub starts 2023-09-23T18:45:03.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:03 smithi098 bash[79516]: cluster 2023-09-23T18:45:03.224133+0000 osd.0 (osd.0) 15 : cluster 0 7.16 scrub starts 2023-09-23T18:45:03.958 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:03.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:03.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:04.060 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:04.087 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:04.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:04.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:04.192 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:04.202 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:03 smithi173 bash[59422]: cluster 2023-09-23T18:45:03.224133+0000 osd.0 (osd.0) 15 : cluster 0 7.16 scrub starts 2023-09-23T18:45:04.217 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:04.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:04.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:04.320 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:04.352 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:04.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:04.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:04.455 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:04.484 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:04.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:04.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:04.587 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:04.622 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:04.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:04.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:04.725 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:04.757 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:04.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:04.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:04.860 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:04.884 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:04.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:04.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:04.987 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:05.013 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:05.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:05.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:05.117 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:05.124 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:04 smithi098 bash[79516]: cluster 2023-09-23T18:45:03.509413+0000 osd.3 (osd.3) 14 : cluster 0 7.13 scrub starts 2023-09-23T18:45:05.124 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:04 smithi098 bash[79516]: cluster 2023-09-23T18:45:04.001905+0000 mgr.y (mgr.26054) 805 : cluster 0 pgmap v827: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 862 op/s; 5581/225945 objects degraded (2.470%); 6497/225945 objects misplaced (2.875%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T18:45:05.124 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:04 smithi098 bash[76817]: cluster 2023-09-23T18:45:03.509413+0000 osd.3 (osd.3) 14 : cluster 0 7.13 scrub starts 2023-09-23T18:45:05.125 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:04 smithi098 bash[76817]: cluster 2023-09-23T18:45:04.001905+0000 mgr.y (mgr.26054) 805 : cluster 0 pgmap v827: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 862 op/s; 5581/225945 objects degraded (2.470%); 6497/225945 objects misplaced (2.875%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T18:45:05.143 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:05.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:05.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:05.247 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:05.256 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:04 smithi173 bash[59422]: cluster 2023-09-23T18:45:03.509413+0000 osd.3 (osd.3) 14 : cluster 0 7.13 scrub starts 2023-09-23T18:45:05.256 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:04 smithi173 bash[59422]: cluster 2023-09-23T18:45:04.001905+0000 mgr.y (mgr.26054) 805 : cluster 0 pgmap v827: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 862 op/s; 5581/225945 objects degraded (2.470%); 6497/225945 objects misplaced (2.875%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T18:45:05.273 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:05.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:05.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:05.377 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:05.415 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:05.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:05.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:05.520 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:05.547 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:05.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:05.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:05.650 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:05.681 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:05.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:05.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:05.786 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:05.814 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:05.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:05.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:05.916 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:05.940 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:05.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:05 smithi098 bash[79516]: cluster 2023-09-23T18:45:04.489755+0000 osd.3 (osd.3) 15 : cluster 0 7.3 scrub starts 2023-09-23T18:45:05.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:05 smithi098 bash[79516]: cluster 2023-09-23T18:45:05.057513+0000 mon.a (mon.0) 1083 : cluster 3 Health check update: Degraded data redundancy: 5581/225945 objects degraded (2.470%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:45:05.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:05 smithi098 bash[76817]: cluster 2023-09-23T18:45:04.489755+0000 osd.3 (osd.3) 15 : cluster 0 7.3 scrub starts 2023-09-23T18:45:05.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:05 smithi098 bash[76817]: cluster 2023-09-23T18:45:05.057513+0000 mon.a (mon.0) 1083 : cluster 3 Health check update: Degraded data redundancy: 5581/225945 objects degraded (2.470%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:45:05.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:05.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:06.044 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:06.052 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:05 smithi173 bash[59422]: cluster 2023-09-23T18:45:04.489755+0000 osd.3 (osd.3) 15 : cluster 0 7.3 scrub starts 2023-09-23T18:45:06.052 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:05 smithi173 bash[59422]: cluster 2023-09-23T18:45:05.057513+0000 mon.a (mon.0) 1083 : cluster 3 Health check update: Degraded data redundancy: 5581/225945 objects degraded (2.470%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:45:06.071 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:06.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:06.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:06.175 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:06.205 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:06.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:06.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:06.308 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:06.334 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:06.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:06.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:06.437 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:06.440 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:45:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:45:06] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.33.4" 2023-09-23T18:45:06.475 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:06.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:06.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:06.580 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:06.603 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:06.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:06.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:06.708 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:06.744 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:06.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:06.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:06.850 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:06.875 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:06.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:06.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:06.979 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:06.988 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:06 smithi098 bash[76817]: cluster 2023-09-23T18:45:05.361672+0000 osd.4 (osd.4) 177 : cluster 0 7.1 scrub starts 2023-09-23T18:45:06.988 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:06 smithi098 bash[76817]: cluster 2023-09-23T18:45:05.471610+0000 osd.3 (osd.3) 16 : cluster 0 7.2 scrub starts 2023-09-23T18:45:06.988 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:06 smithi098 bash[76817]: cluster 2023-09-23T18:45:06.002512+0000 mgr.y (mgr.26054) 806 : cluster 0 pgmap v828: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 818 op/s; 5581/221679 objects degraded (2.518%); 6497/221679 objects misplaced (2.931%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T18:45:06.988 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:06 smithi098 bash[76817]: cluster 2023-09-23T18:45:06.047430+0000 osd.6 (osd.6) 151 : cluster 0 7.7 scrub starts 2023-09-23T18:45:06.989 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:06 smithi098 bash[79516]: cluster 2023-09-23T18:45:05.361672+0000 osd.4 (osd.4) 177 : cluster 0 7.1 scrub starts 2023-09-23T18:45:06.989 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:06 smithi098 bash[79516]: cluster 2023-09-23T18:45:05.471610+0000 osd.3 (osd.3) 16 : cluster 0 7.2 scrub starts 2023-09-23T18:45:06.989 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:06 smithi098 bash[79516]: cluster 2023-09-23T18:45:06.002512+0000 mgr.y (mgr.26054) 806 : cluster 0 pgmap v828: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 818 op/s; 5581/221679 objects degraded (2.518%); 6497/221679 objects misplaced (2.931%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T18:45:06.989 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:06 smithi098 bash[79516]: cluster 2023-09-23T18:45:06.047430+0000 osd.6 (osd.6) 151 : cluster 0 7.7 scrub starts 2023-09-23T18:45:07.005 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:07.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:07.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:07.109 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:07.117 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:06 smithi173 bash[59422]: cluster 2023-09-23T18:45:05.361672+0000 osd.4 (osd.4) 177 : cluster 0 7.1 scrub starts 2023-09-23T18:45:07.117 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:06 smithi173 bash[59422]: cluster 2023-09-23T18:45:05.471610+0000 osd.3 (osd.3) 16 : cluster 0 7.2 scrub starts 2023-09-23T18:45:07.117 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:06 smithi173 bash[59422]: cluster 2023-09-23T18:45:06.002512+0000 mgr.y (mgr.26054) 806 : cluster 0 pgmap v828: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 818 op/s; 5581/221679 objects degraded (2.518%); 6497/221679 objects misplaced (2.931%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T18:45:07.117 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:06 smithi173 bash[59422]: cluster 2023-09-23T18:45:06.047430+0000 osd.6 (osd.6) 151 : cluster 0 7.7 scrub starts 2023-09-23T18:45:07.136 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:07.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:07.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:07.239 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:07.269 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:07.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:07.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:07.372 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:07.401 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:07.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:07.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:07.504 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:07.530 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:07.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:07.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:07.633 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:07.660 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:07.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:07.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:07.763 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:07.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:07.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:07.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:07.892 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:07.922 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:07.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:07.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:08.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:07 smithi173 bash[59422]: cluster 2023-09-23T18:45:07.051134+0000 osd.6 (osd.6) 152 : cluster 0 7.7 scrub starts 2023-09-23T18:45:08.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:07 smithi173 bash[59422]: cluster 2023-09-23T18:45:07.106246+0000 osd.0 (osd.0) 16 : cluster 0 7.19 scrub starts 2023-09-23T18:45:08.027 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:08.052 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:08.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:08.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:08.157 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:08.183 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:08.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:08.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:08.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:07 smithi098 bash[79516]: cluster 2023-09-23T18:45:07.051134+0000 osd.6 (osd.6) 152 : cluster 0 7.7 scrub starts 2023-09-23T18:45:08.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:07 smithi098 bash[79516]: cluster 2023-09-23T18:45:07.106246+0000 osd.0 (osd.0) 16 : cluster 0 7.19 scrub starts 2023-09-23T18:45:08.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:07 smithi098 bash[76817]: cluster 2023-09-23T18:45:07.051134+0000 osd.6 (osd.6) 152 : cluster 0 7.7 scrub starts 2023-09-23T18:45:08.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:07 smithi098 bash[76817]: cluster 2023-09-23T18:45:07.106246+0000 osd.0 (osd.0) 16 : cluster 0 7.19 scrub starts 2023-09-23T18:45:08.286 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:08.317 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:08.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:08.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:08.420 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:08.450 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:08.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:08.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:08.555 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:08.586 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:08.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:08.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:08.693 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:08.731 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:08.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:08.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:08.802 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:45:08.802 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:45:08.837 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:08.859 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:08.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:08.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:08.962 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:09.006 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:09.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:09.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:09.112 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:09.121 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:08 smithi098 bash[76817]: cluster 2023-09-23T18:45:08.003155+0000 mgr.y (mgr.26054) 807 : cluster 0 pgmap v829: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 812 op/s; 5581/217008 objects degraded (2.572%); 5907/217008 objects misplaced (2.722%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:45:09.121 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:08 smithi098 bash[76817]: cluster 2023-09-23T18:45:08.053701+0000 osd.6 (osd.6) 153 : cluster 0 7.7 scrub starts 2023-09-23T18:45:09.122 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:08 smithi098 bash[79516]: cluster 2023-09-23T18:45:08.003155+0000 mgr.y (mgr.26054) 807 : cluster 0 pgmap v829: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 812 op/s; 5581/217008 objects degraded (2.572%); 5907/217008 objects misplaced (2.722%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:45:09.122 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:08 smithi098 bash[79516]: cluster 2023-09-23T18:45:08.053701+0000 osd.6 (osd.6) 153 : cluster 0 7.7 scrub starts 2023-09-23T18:45:09.156 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:09.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:09.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:09.183 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:45:09.197 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-23T18:45:09.262 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:09.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:08 smithi173 bash[59422]: cluster 2023-09-23T18:45:08.003155+0000 mgr.y (mgr.26054) 807 : cluster 0 pgmap v829: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 812 op/s; 5581/217008 objects degraded (2.572%); 5907/217008 objects misplaced (2.722%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:45:09.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:08 smithi173 bash[59422]: cluster 2023-09-23T18:45:08.053701+0000 osd.6 (osd.6) 153 : cluster 0 7.7 scrub starts 2023-09-23T18:45:09.290 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:09.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:09.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:09.394 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:09.426 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:09.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:09.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:09.530 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:09.559 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:09.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:09.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:09.665 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:09.682 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:09.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:09.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:09.786 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:09.813 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:09.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:09.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:09.918 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:09.927 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:09 smithi098 bash[76817]: cluster 2023-09-23T18:45:09.076039+0000 osd.0 (osd.0) 17 : cluster 0 7.14 scrub starts 2023-09-23T18:45:09.927 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:09 smithi098 bash[76817]: audit 2023-09-23T18:45:09.609034+0000 mon.a (mon.0) 1084 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:45:09.927 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:09 smithi098 bash[79516]: cluster 2023-09-23T18:45:09.076039+0000 osd.0 (osd.0) 17 : cluster 0 7.14 scrub starts 2023-09-23T18:45:09.927 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:09 smithi098 bash[79516]: audit 2023-09-23T18:45:09.609034+0000 mon.a (mon.0) 1084 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:45:09.942 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:09.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:09.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:10.048 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:10.071 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:10.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:10.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:10.176 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:10.183 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:09 smithi173 bash[59422]: cluster 2023-09-23T18:45:09.076039+0000 osd.0 (osd.0) 17 : cluster 0 7.14 scrub starts 2023-09-23T18:45:10.183 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:09 smithi173 bash[59422]: audit 2023-09-23T18:45:09.609034+0000 mon.a (mon.0) 1084 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:45:10.200 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:10.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:10.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:10.304 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:10.331 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:10.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:10.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:10.437 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:10.466 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:10.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:10.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:10.568 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:10.603 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:10.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:10.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:10.708 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:10.732 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:10.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:10.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:10.839 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:10.864 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:10.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:10.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:10.968 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:10.979 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:10 smithi173 bash[59422]: audit 2023-09-23T18:45:09.166555+0000 mgr.y (mgr.26054) 808 : audit 0 from='client.54895 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:45:10.979 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:10 smithi173 bash[59422]: cluster 2023-09-23T18:45:09.442849+0000 osd.3 (osd.3) 17 : cluster 0 7.13 scrub starts 2023-09-23T18:45:10.979 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:10 smithi173 bash[59422]: cluster 2023-09-23T18:45:10.003623+0000 mgr.y (mgr.26054) 809 : cluster 0 pgmap v830: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 800 op/s; 5581/212346 objects degraded (2.628%); 5907/212346 objects misplaced (2.782%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T18:45:10.979 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:10 smithi173 bash[59422]: cluster 2023-09-23T18:45:10.058830+0000 mon.a (mon.0) 1085 : cluster 3 Health check update: Degraded data redundancy: 5581/217008 objects degraded (2.572%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:45:10.979 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:10 smithi173 bash[59422]: cluster 2023-09-23T18:45:10.112394+0000 osd.0 (osd.0) 18 : cluster 0 7.4 scrub starts 2023-09-23T18:45:10.979 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:10 smithi173 bash[59422]: cluster 2023-09-23T18:45:10.429727+0000 osd.3 (osd.3) 18 : cluster 0 7.3 scrub starts 2023-09-23T18:45:10.999 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:11.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:11.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:11.103 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:11.123 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:11.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:11.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:11.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:10 smithi098 bash[76817]: audit 2023-09-23T18:45:09.166555+0000 mgr.y (mgr.26054) 808 : audit 0 from='client.54895 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:45:11.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:10 smithi098 bash[76817]: cluster 2023-09-23T18:45:09.442849+0000 osd.3 (osd.3) 17 : cluster 0 7.13 scrub starts 2023-09-23T18:45:11.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:10 smithi098 bash[76817]: cluster 2023-09-23T18:45:10.003623+0000 mgr.y (mgr.26054) 809 : cluster 0 pgmap v830: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 800 op/s; 5581/212346 objects degraded (2.628%); 5907/212346 objects misplaced (2.782%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T18:45:11.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:10 smithi098 bash[76817]: cluster 2023-09-23T18:45:10.058830+0000 mon.a (mon.0) 1085 : cluster 3 Health check update: Degraded data redundancy: 5581/217008 objects degraded (2.572%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:45:11.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:10 smithi098 bash[76817]: cluster 2023-09-23T18:45:10.112394+0000 osd.0 (osd.0) 18 : cluster 0 7.4 scrub starts 2023-09-23T18:45:11.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:10 smithi098 bash[76817]: cluster 2023-09-23T18:45:10.429727+0000 osd.3 (osd.3) 18 : cluster 0 7.3 scrub starts 2023-09-23T18:45:11.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:10 smithi098 bash[79516]: audit 2023-09-23T18:45:09.166555+0000 mgr.y (mgr.26054) 808 : audit 0 from='client.54895 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:45:11.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:10 smithi098 bash[79516]: cluster 2023-09-23T18:45:09.442849+0000 osd.3 (osd.3) 17 : cluster 0 7.13 scrub starts 2023-09-23T18:45:11.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:10 smithi098 bash[79516]: cluster 2023-09-23T18:45:10.003623+0000 mgr.y (mgr.26054) 809 : cluster 0 pgmap v830: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 800 op/s; 5581/212346 objects degraded (2.628%); 5907/212346 objects misplaced (2.782%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T18:45:11.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:10 smithi098 bash[79516]: cluster 2023-09-23T18:45:10.058830+0000 mon.a (mon.0) 1085 : cluster 3 Health check update: Degraded data redundancy: 5581/217008 objects degraded (2.572%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:45:11.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:10 smithi098 bash[79516]: cluster 2023-09-23T18:45:10.112394+0000 osd.0 (osd.0) 18 : cluster 0 7.4 scrub starts 2023-09-23T18:45:11.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:10 smithi098 bash[79516]: cluster 2023-09-23T18:45:10.429727+0000 osd.3 (osd.3) 18 : cluster 0 7.3 scrub starts 2023-09-23T18:45:11.227 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:11.257 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:11.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:11.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:11.364 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:11.393 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:11.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:11.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:11.497 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:11.523 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:11.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:11.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:11.545 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:45:11.589 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:45:11.626 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:11.634 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:45:11.651 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:11.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:11.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:11.681 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:45:11.718 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:45:11.755 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:11.764 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:45:11.783 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:11.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:11.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:11.807 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:45:11.839 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:45:11.841 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:45:11.842 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:45:11.842 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:45:11.842 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:45:11.842 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:45:11.842 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:45:11.842 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:45:11.842 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:45:11.886 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:11.917 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:11.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:11.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:12.021 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:12.047 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:12.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:12.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:12.083 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is unset 2023-09-23T18:45:12.153 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:12.178 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:12.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:12.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:12.229 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:12 smithi098 bash[79516]: cluster 2023-09-23T18:45:11.065728+0000 osd.7 (osd.7) 123 : cluster 3 failed to encode map e207 with expected crc 2023-09-23T18:45:12.229 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:12 smithi098 bash[79516]: cluster 2023-09-23T18:45:11.066999+0000 mon.a (mon.0) 1086 : cluster 0 osdmap e207: 8 total, 8 up, 7 in 2023-09-23T18:45:12.229 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:12 smithi098 bash[79516]: cluster 2023-09-23T18:45:11.069300+0000 osd.4 (osd.4) 178 : cluster 3 failed to encode map e207 with expected crc 2023-09-23T18:45:12.229 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:12 smithi098 bash[79516]: cluster 2023-09-23T18:45:11.076933+0000 osd.6 (osd.6) 154 : cluster 3 failed to encode map e207 with expected crc 2023-09-23T18:45:12.229 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:12 smithi098 bash[79516]: cluster 2023-09-23T18:45:11.098734+0000 osd.0 (osd.0) 19 : cluster 0 7.9 scrub starts 2023-09-23T18:45:12.229 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:12 smithi098 bash[79516]: audit 2023-09-23T18:45:11.512274+0000 mon.a (mon.0) 1087 : audit 0 from='client.? 172.21.15.98:0/3743755350' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:45:12.229 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:12 smithi098 bash[79516]: audit 2023-09-23T18:45:11.874639+0000 mon.c (mon.2) 98 : audit 1 from='client.? 172.21.15.98:0/2630343417' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:45:12.229 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:12 smithi098 bash[79516]: audit 2023-09-23T18:45:11.875008+0000 mon.a (mon.0) 1088 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:45:12.284 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:12.318 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:12.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:12.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:12.422 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:12.430 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:12 smithi173 bash[59422]: cluster 2023-09-23T18:45:11.065728+0000 osd.7 (osd.7) 123 : cluster 3 failed to encode map e207 with expected crc 2023-09-23T18:45:12.430 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:12 smithi173 bash[59422]: cluster 2023-09-23T18:45:11.066999+0000 mon.a (mon.0) 1086 : cluster 0 osdmap e207: 8 total, 8 up, 7 in 2023-09-23T18:45:12.430 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:12 smithi173 bash[59422]: cluster 2023-09-23T18:45:11.069300+0000 osd.4 (osd.4) 178 : cluster 3 failed to encode map e207 with expected crc 2023-09-23T18:45:12.430 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:12 smithi173 bash[59422]: cluster 2023-09-23T18:45:11.076933+0000 osd.6 (osd.6) 154 : cluster 3 failed to encode map e207 with expected crc 2023-09-23T18:45:12.431 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:12 smithi173 bash[59422]: cluster 2023-09-23T18:45:11.098734+0000 osd.0 (osd.0) 19 : cluster 0 7.9 scrub starts 2023-09-23T18:45:12.431 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:12 smithi173 bash[59422]: audit 2023-09-23T18:45:11.512274+0000 mon.a (mon.0) 1087 : audit 0 from='client.? 172.21.15.98:0/3743755350' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:45:12.431 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:12 smithi173 bash[59422]: audit 2023-09-23T18:45:11.874639+0000 mon.c (mon.2) 98 : audit 1 from='client.? 172.21.15.98:0/2630343417' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:45:12.431 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:12 smithi173 bash[59422]: audit 2023-09-23T18:45:11.875008+0000 mon.a (mon.0) 1088 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:45:12.447 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:12.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:12.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:12.553 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:12.562 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:12 smithi098 bash[76817]: cluster 2023-09-23T18:45:11.065728+0000 osd.7 (osd.7) 123 : cluster 3 failed to encode map e207 with expected crc 2023-09-23T18:45:12.562 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:12 smithi098 bash[76817]: cluster 2023-09-23T18:45:11.066999+0000 mon.a (mon.0) 1086 : cluster 0 osdmap e207: 8 total, 8 up, 7 in 2023-09-23T18:45:12.562 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:12 smithi098 bash[76817]: cluster 2023-09-23T18:45:11.069300+0000 osd.4 (osd.4) 178 : cluster 3 failed to encode map e207 with expected crc 2023-09-23T18:45:12.562 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:12 smithi098 bash[76817]: cluster 2023-09-23T18:45:11.076933+0000 osd.6 (osd.6) 154 : cluster 3 failed to encode map e207 with expected crc 2023-09-23T18:45:12.562 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:12 smithi098 bash[76817]: cluster 2023-09-23T18:45:11.098734+0000 osd.0 (osd.0) 19 : cluster 0 7.9 scrub starts 2023-09-23T18:45:12.562 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:12 smithi098 bash[76817]: audit 2023-09-23T18:45:11.512274+0000 mon.a (mon.0) 1087 : audit 0 from='client.? 172.21.15.98:0/3743755350' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:45:12.562 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:12 smithi098 bash[76817]: audit 2023-09-23T18:45:11.874639+0000 mon.c (mon.2) 98 : audit 1 from='client.? 172.21.15.98:0/2630343417' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:45:12.562 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:12 smithi098 bash[76817]: audit 2023-09-23T18:45:11.875008+0000 mon.a (mon.0) 1088 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:45:12.581 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:12.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:12.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:12.687 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:12.718 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:12.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:12.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:12.820 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:12.849 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:12.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:12.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:12.955 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:12.983 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:12.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:12.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:13.089 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:13.122 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:13.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:13.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:13.225 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:13.236 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:13 smithi173 bash[59422]: cluster 2023-09-23T18:45:12.004261+0000 mgr.y (mgr.26054) 810 : cluster 0 pgmap v832: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 799 op/s; 5581/207129 objects degraded (2.694%); 5907/207129 objects misplaced (2.852%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T18:45:13.236 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:13 smithi173 bash[59422]: cluster 2023-09-23T18:45:12.064130+0000 mon.a (mon.0) 1089 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:45:13.236 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:13 smithi173 bash[59422]: cluster 2023-09-23T18:45:12.073094+0000 osd.5 (osd.5) 104 : cluster 3 failed to encode map e208 with expected crc 2023-09-23T18:45:13.236 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:13 smithi173 bash[59422]: cluster 2023-09-23T18:45:12.073843+0000 osd.4 (osd.4) 179 : cluster 3 failed to encode map e208 with expected crc 2023-09-23T18:45:13.236 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:13 smithi173 bash[59422]: audit 2023-09-23T18:45:12.075523+0000 mon.a (mon.0) 1090 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:45:13.236 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:13 smithi173 bash[59422]: cluster 2023-09-23T18:45:12.075718+0000 mon.a (mon.0) 1091 : cluster 0 osdmap e208: 8 total, 8 up, 7 in 2023-09-23T18:45:13.236 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:13 smithi173 bash[59422]: cluster 2023-09-23T18:45:12.080202+0000 osd.7 (osd.7) 124 : cluster 3 failed to encode map e208 with expected crc 2023-09-23T18:45:13.236 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:13 smithi173 bash[59422]: cluster 2023-09-23T18:45:12.087269+0000 osd.0 (osd.0) 20 : cluster 0 7.6 scrub starts 2023-09-23T18:45:13.236 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:13 smithi173 bash[59422]: cluster 2023-09-23T18:45:12.089966+0000 osd.6 (osd.6) 155 : cluster 3 failed to encode map e208 with expected crc 2023-09-23T18:45:13.251 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:13.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:13.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:13.356 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:13.375 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:13.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:13.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:13.386 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:13 smithi098 bash[79516]: cluster 2023-09-23T18:45:12.004261+0000 mgr.y (mgr.26054) 810 : cluster 0 pgmap v832: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 799 op/s; 5581/207129 objects degraded (2.694%); 5907/207129 objects misplaced (2.852%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T18:45:13.386 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:13 smithi098 bash[79516]: cluster 2023-09-23T18:45:12.064130+0000 mon.a (mon.0) 1089 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:45:13.386 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:13 smithi098 bash[79516]: cluster 2023-09-23T18:45:12.073094+0000 osd.5 (osd.5) 104 : cluster 3 failed to encode map e208 with expected crc 2023-09-23T18:45:13.386 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:13 smithi098 bash[79516]: cluster 2023-09-23T18:45:12.073843+0000 osd.4 (osd.4) 179 : cluster 3 failed to encode map e208 with expected crc 2023-09-23T18:45:13.386 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:13 smithi098 bash[79516]: audit 2023-09-23T18:45:12.075523+0000 mon.a (mon.0) 1090 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:45:13.386 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:13 smithi098 bash[79516]: cluster 2023-09-23T18:45:12.075718+0000 mon.a (mon.0) 1091 : cluster 0 osdmap e208: 8 total, 8 up, 7 in 2023-09-23T18:45:13.386 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:13 smithi098 bash[79516]: cluster 2023-09-23T18:45:12.080202+0000 osd.7 (osd.7) 124 : cluster 3 failed to encode map e208 with expected crc 2023-09-23T18:45:13.386 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:13 smithi098 bash[79516]: cluster 2023-09-23T18:45:12.087269+0000 osd.0 (osd.0) 20 : cluster 0 7.6 scrub starts 2023-09-23T18:45:13.386 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:13 smithi098 bash[79516]: cluster 2023-09-23T18:45:12.089966+0000 osd.6 (osd.6) 155 : cluster 3 failed to encode map e208 with expected crc 2023-09-23T18:45:13.387 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:13 smithi098 bash[76817]: cluster 2023-09-23T18:45:12.004261+0000 mgr.y (mgr.26054) 810 : cluster 0 pgmap v832: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 799 op/s; 5581/207129 objects degraded (2.694%); 5907/207129 objects misplaced (2.852%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T18:45:13.387 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:13 smithi098 bash[76817]: cluster 2023-09-23T18:45:12.064130+0000 mon.a (mon.0) 1089 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:45:13.387 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:13 smithi098 bash[76817]: cluster 2023-09-23T18:45:12.073094+0000 osd.5 (osd.5) 104 : cluster 3 failed to encode map e208 with expected crc 2023-09-23T18:45:13.387 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:13 smithi098 bash[76817]: cluster 2023-09-23T18:45:12.073843+0000 osd.4 (osd.4) 179 : cluster 3 failed to encode map e208 with expected crc 2023-09-23T18:45:13.387 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:13 smithi098 bash[76817]: audit 2023-09-23T18:45:12.075523+0000 mon.a (mon.0) 1090 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:45:13.387 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:13 smithi098 bash[76817]: cluster 2023-09-23T18:45:12.075718+0000 mon.a (mon.0) 1091 : cluster 0 osdmap e208: 8 total, 8 up, 7 in 2023-09-23T18:45:13.387 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:13 smithi098 bash[76817]: cluster 2023-09-23T18:45:12.080202+0000 osd.7 (osd.7) 124 : cluster 3 failed to encode map e208 with expected crc 2023-09-23T18:45:13.387 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:13 smithi098 bash[76817]: cluster 2023-09-23T18:45:12.087269+0000 osd.0 (osd.0) 20 : cluster 0 7.6 scrub starts 2023-09-23T18:45:13.387 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:13 smithi098 bash[76817]: cluster 2023-09-23T18:45:12.089966+0000 osd.6 (osd.6) 155 : cluster 3 failed to encode map e208 with expected crc 2023-09-23T18:45:13.478 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:13.501 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:13.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:13.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:13.604 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:13.656 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:13.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:13.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:13.762 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:13.788 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:13.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:13.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:13.892 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:13.897 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T18:45:13.918 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:13.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:13.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:14.021 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:14.044 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:14.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:14.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:14.148 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:14.177 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:14.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:14.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:14.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:14 smithi098 bash[76817]: cluster 2023-09-23T18:45:12.883833+0000 osd.2 (osd.2) 1 : cluster 0 7.8 scrub starts 2023-09-23T18:45:14.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:14 smithi098 bash[79516]: cluster 2023-09-23T18:45:12.883833+0000 osd.2 (osd.2) 1 : cluster 0 7.8 scrub starts 2023-09-23T18:45:14.282 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:14.292 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:14 smithi173 bash[59422]: cluster 2023-09-23T18:45:12.883833+0000 osd.2 (osd.2) 1 : cluster 0 7.8 scrub starts 2023-09-23T18:45:14.307 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:14.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:14.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:14.411 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:14.436 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:14.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:14.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:14.539 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:14.573 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:14.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:14.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:14.677 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:14.705 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:14.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:14.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:14.807 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:14.830 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:14.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:14.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:14.933 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:14.961 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:14.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:14.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:15.064 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:15.095 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:15.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:15.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:15.202 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:15.226 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:15.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:15.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:15.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:15 smithi173 bash[59422]: cluster 2023-09-23T18:45:14.004929+0000 mgr.y (mgr.26054) 811 : cluster 0 pgmap v834: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 3854/202782 objects degraded (1.901%); 7496/202782 objects misplaced (3.697%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T18:45:15.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:15 smithi173 bash[59422]: cluster 2023-09-23T18:45:14.135943+0000 osd.0 (osd.0) 21 : cluster 0 7.16 deep-scrub starts 2023-09-23T18:45:15.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:15 smithi173 bash[59422]: cluster 2023-09-23T18:45:15.059989+0000 mon.a (mon.0) 1092 : cluster 3 Health check update: Degraded data redundancy: 3854/202782 objects degraded (1.901%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:45:15.330 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:15 smithi098 bash[76817]: cluster 2023-09-23T18:45:14.004929+0000 mgr.y (mgr.26054) 811 : cluster 0 pgmap v834: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 3854/202782 objects degraded (1.901%); 7496/202782 objects misplaced (3.697%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T18:45:15.330 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:15 smithi098 bash[76817]: cluster 2023-09-23T18:45:14.135943+0000 osd.0 (osd.0) 21 : cluster 0 7.16 deep-scrub starts 2023-09-23T18:45:15.330 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:15 smithi098 bash[76817]: cluster 2023-09-23T18:45:15.059989+0000 mon.a (mon.0) 1092 : cluster 3 Health check update: Degraded data redundancy: 3854/202782 objects degraded (1.901%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:45:15.330 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:15 smithi098 bash[79516]: cluster 2023-09-23T18:45:14.004929+0000 mgr.y (mgr.26054) 811 : cluster 0 pgmap v834: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 3854/202782 objects degraded (1.901%); 7496/202782 objects misplaced (3.697%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T18:45:15.331 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:15 smithi098 bash[79516]: cluster 2023-09-23T18:45:14.135943+0000 osd.0 (osd.0) 21 : cluster 0 7.16 deep-scrub starts 2023-09-23T18:45:15.331 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:15 smithi098 bash[79516]: cluster 2023-09-23T18:45:15.059989+0000 mon.a (mon.0) 1092 : cluster 3 Health check update: Degraded data redundancy: 3854/202782 objects degraded (1.901%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:45:15.332 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":829,"stamp":"2023-09-23T18:45:08.002713+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4743501771,"num_objects":72336,"num_object_clones":0,"num_object_copies":217008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5581,"num_objects_misplaced":5907,"num_objects_unfound":0,"num_objects_dirty":72336,"num_whiteouts":0,"num_read":6246,"num_read_kb":13703,"num_write":130684,"num_write_kb":6475448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2453,"num_bytes_recovered":161940656,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":169343,"ondisk_log_size":169343,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":303,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17430732,"kb_used_data":14423764,"kb_used_omap":31,"kb_used_meta":3006432,"kb_avail":638785332,"statfs":{"total":671965249536,"available":654116179968,"internally_reserved":0,"allocated":14769934336,"data_stored":14764553467,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32507,"internal_metadata":3078586629},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-639500288,"num_objects":-9758,"num_object_clones":0,"num_object_copies":-29274,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1508,"num_objects_unfound":0,"num_objects_dirty":-9758,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9758,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":754,"num_bytes_recovered":49414144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003808"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":64,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.749009+0000","last_change":"2023-09-23T18:37:40.992912+0000","last_active":"2023-09-23T18:44:18.749009+0000","last_peered":"2023-09-23T18:44:18.749009+0000","last_clean":"2023-09-23T18:44:18.749009+0000","last_became_active":"2023-09-23T18:37:40.992453+0000","last_became_peered":"2023-09-23T18:37:40.992453+0000","last_unstale":"2023-09-23T18:44:18.749009+0000","last_undegraded":"2023-09-23T18:44:18.749009+0000","last_fullsized":"2023-09-23T18:44:18.749009+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:49.539251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":146,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.063826+0000","last_change":"2023-09-23T18:40:42.051229+0000","last_active":"2023-09-23T18:44:18.063826+0000","last_peered":"2023-09-23T18:44:18.063826+0000","last_clean":"2023-09-23T18:44:18.063826+0000","last_became_active":"2023-09-23T18:40:42.050622+0000","last_became_peered":"2023-09-23T18:40:42.050622+0000","last_unstale":"2023-09-23T18:44:18.063826+0000","last_undegraded":"2023-09-23T18:44:18.063826+0000","last_fullsized":"2023-09-23T18:44:18.063826+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:17.587802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":145,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.062588+0000","last_change":"2023-09-23T18:33:53.848655+0000","last_active":"2023-09-23T18:44:18.062588+0000","last_peered":"2023-09-23T18:44:18.062588+0000","last_clean":"2023-09-23T18:44:18.062588+0000","last_became_active":"2023-09-23T18:33:53.848279+0000","last_became_peered":"2023-09-23T18:33:53.848279+0000","last_unstale":"2023-09-23T18:44:18.062588+0000","last_undegraded":"2023-09-23T18:44:18.062588+0000","last_fullsized":"2023-09-23T18:44:18.062588+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:39.223612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":181,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:38.108355+0000","last_change":"2023-09-23T18:40:42.052899+0000","last_active":"2023-09-23T18:44:38.108355+0000","last_peered":"2023-09-23T18:44:38.108355+0000","last_clean":"2023-09-23T18:44:38.108355+0000","last_became_active":"2023-09-23T18:40:42.052569+0000","last_became_peered":"2023-09-23T18:40:42.052569+0000","last_unstale":"2023-09-23T18:44:38.108355+0000","last_undegraded":"2023-09-23T18:44:38.108355+0000","last_fullsized":"2023-09-23T18:44:38.108355+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:46.151073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":162,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.748405+0000","last_change":"2023-09-23T18:40:42.048469+0000","last_active":"2023-09-23T18:44:18.748405+0000","last_peered":"2023-09-23T18:44:18.748405+0000","last_clean":"2023-09-23T18:44:18.748405+0000","last_became_active":"2023-09-23T18:40:42.048139+0000","last_became_peered":"2023-09-23T18:40:42.048139+0000","last_unstale":"2023-09-23T18:44:18.748405+0000","last_undegraded":"2023-09-23T18:44:18.748405+0000","last_fullsized":"2023-09-23T18:44:18.748405+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:09:09.535543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":197,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.069885+0000","last_change":"2023-09-23T18:33:55.084557+0000","last_active":"2023-09-23T18:44:18.069885+0000","last_peered":"2023-09-23T18:44:18.069885+0000","last_clean":"2023-09-23T18:44:18.069885+0000","last_became_active":"2023-09-23T18:33:55.083987+0000","last_became_peered":"2023-09-23T18:33:55.083987+0000","last_unstale":"2023-09-23T18:44:18.069885+0000","last_undegraded":"2023-09-23T18:44:18.069885+0000","last_fullsized":"2023-09-23T18:44:18.069885+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.13","version":"206'6398","reported_seq":6439,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:45:06.369830+0000","last_change":"2023-09-23T18:45:03.542883+0000","last_active":"2023-09-23T18:45:06.369830+0000","last_peered":"2023-09-23T18:45:06.369830+0000","last_clean":"2023-09-23T18:45:06.369830+0000","last_became_active":"2023-09-23T18:43:59.486869+0000","last_became_peered":"2023-09-23T18:43:59.486869+0000","last_unstale":"2023-09-23T18:45:06.369830+0000","last_undegraded":"2023-09-23T18:45:06.369830+0000","last_fullsized":"2023-09-23T18:45:06.369830+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":195,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6398,"log_dups_size":0,"ondisk_log_size":6398,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143687680,"num_objects":2193,"num_object_clones":0,"num_object_copies":6579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4055,"num_write_kb":199904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":178,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.748402+0000","last_change":"2023-09-23T18:40:42.054577+0000","last_active":"2023-09-23T18:44:18.748402+0000","last_peered":"2023-09-23T18:44:18.748402+0000","last_clean":"2023-09-23T18:44:18.748402+0000","last_became_active":"2023-09-23T18:40:42.054170+0000","last_became_peered":"2023-09-23T18:40:42.054170+0000","last_unstale":"2023-09-23T18:44:18.748402+0000","last_undegraded":"2023-09-23T18:44:18.748402+0000","last_fullsized":"2023-09-23T18:44:18.748402+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:35.503748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":204,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:21.348282+0000","last_change":"2023-09-23T18:37:40.998526+0000","last_active":"2023-09-23T18:44:21.348282+0000","last_peered":"2023-09-23T18:44:21.348282+0000","last_clean":"2023-09-23T18:44:21.348282+0000","last_became_active":"2023-09-23T18:37:40.993211+0000","last_became_peered":"2023-09-23T18:37:40.993211+0000","last_unstale":"2023-09-23T18:44:21.348282+0000","last_undegraded":"2023-09-23T18:44:21.348282+0000","last_fullsized":"2023-09-23T18:44:21.348282+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:40.463496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.12","version":"206'6490","reported_seq":6531,"reported_epoch":206,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:45:07.836973+0000","last_change":"2023-09-23T18:44:18.828584+0000","last_active":"2023-09-23T18:45:07.836973+0000","last_peered":"2023-09-23T18:45:07.836973+0000","last_clean":"2023-09-23T18:44:14.786892+0000","last_became_active":"2023-09-23T18:44:18.822457+0000","last_became_peered":"2023-09-23T18:44:18.822457+0000","last_unstale":"2023-09-23T18:45:07.836973+0000","last_undegraded":"2023-09-23T18:44:18.138578+0000","last_fullsized":"2023-09-23T18:44:18.063005+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":195,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6490,"log_dups_size":0,"ondisk_log_size":6490,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144637952,"num_objects":2207,"num_object_clones":0,"num_object_copies":6621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2794,"num_objects_misplaced":2794,"num_objects_unfound":0,"num_objects_dirty":2207,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4147,"num_write_kb":203328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[1,2],"avail_no_missing":["1","2","3"],"object_location_counts":[{"shards":"1,2","objects":-587},{"shards":"1,2,3","objects":1824}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":204,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.077156+0000","last_change":"2023-09-23T18:40:42.047724+0000","last_active":"2023-09-23T18:44:18.077156+0000","last_peered":"2023-09-23T18:44:18.077156+0000","last_clean":"2023-09-23T18:44:18.077156+0000","last_became_active":"2023-09-23T18:40:42.047236+0000","last_became_peered":"2023-09-23T18:40:42.047236+0000","last_unstale":"2023-09-23T18:44:18.077156+0000","last_undegraded":"2023-09-23T18:44:18.077156+0000","last_fullsized":"2023-09-23T18:44:18.077156+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":170,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:21.348301+0000","last_change":"2023-09-23T18:28:03.298214+0000","last_active":"2023-09-23T18:44:21.348301+0000","last_peered":"2023-09-23T18:44:21.348301+0000","last_clean":"2023-09-23T18:44:21.348301+0000","last_became_active":"2023-09-23T18:28:03.297852+0000","last_became_peered":"2023-09-23T18:28:03.297852+0000","last_unstale":"2023-09-23T18:44:21.348301+0000","last_undegraded":"2023-09-23T18:44:21.348301+0000","last_fullsized":"2023-09-23T18:44:21.348301+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.146541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.11","version":"206'6559","reported_seq":6605,"reported_epoch":206,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:45:07.774730+0000","last_change":"2023-09-23T18:44:16.380800+0000","last_active":"2023-09-23T18:45:07.774730+0000","last_peered":"2023-09-23T18:45:07.774730+0000","last_clean":"2023-09-23T18:44:14.168767+0000","last_became_active":"2023-09-23T18:44:16.374974+0000","last_became_peered":"2023-09-23T18:44:16.374974+0000","last_unstale":"2023-09-23T18:45:07.774730+0000","last_undegraded":"2023-09-23T18:44:16.161569+0000","last_fullsized":"2023-09-23T18:44:16.057209+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":195,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6559,"log_dups_size":0,"ondisk_log_size":6559,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145391616,"num_objects":2219,"num_object_clones":0,"num_object_copies":6657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2787,"num_objects_misplaced":2787,"num_objects_unfound":0,"num_objects_dirty":2219,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4199,"num_write_kb":205344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[1,6],"avail_no_missing":["1","4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1797},{"shards":"1,6","objects":-568}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":175,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.076937+0000","last_change":"2023-09-23T18:28:03.296913+0000","last_active":"2023-09-23T18:44:18.076937+0000","last_peered":"2023-09-23T18:44:18.076937+0000","last_clean":"2023-09-23T18:44:18.076937+0000","last_became_active":"2023-09-23T18:28:03.296499+0000","last_became_peered":"2023-09-23T18:28:03.296499+0000","last_unstale":"2023-09-23T18:44:18.076937+0000","last_undegraded":"2023-09-23T18:44:18.076937+0000","last_fullsized":"2023-09-23T18:44:18.076937+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":141,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.062340+0000","last_change":"2023-09-23T18:33:53.850096+0000","last_active":"2023-09-23T18:44:18.062340+0000","last_peered":"2023-09-23T18:44:18.062340+0000","last_clean":"2023-09-23T18:44:18.062340+0000","last_became_active":"2023-09-23T18:33:53.849811+0000","last_became_peered":"2023-09-23T18:33:53.849811+0000","last_unstale":"2023-09-23T18:44:18.062340+0000","last_undegraded":"2023-09-23T18:44:18.062340+0000","last_fullsized":"2023-09-23T18:44:18.062340+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:44.938072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.10","version":"206'6514","reported_seq":13771,"reported_epoch":206,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T18:45:07.875769+0000","last_change":"2023-09-23T18:44:08.380017+0000","last_active":"2023-09-23T18:45:07.875769+0000","last_peered":"2023-09-23T18:45:07.875769+0000","last_clean":"2023-09-23T18:44:06.066977+0000","last_became_active":"2023-09-23T18:44:08.350837+0000","last_became_peered":"2023-09-23T18:44:08.350837+0000","last_unstale":"2023-09-23T18:45:07.875769+0000","last_undegraded":"2023-09-23T18:45:07.875769+0000","last_fullsized":"2023-09-23T18:44:08.099764+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":195,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6514,"log_dups_size":0,"ondisk_log_size":6514,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145555456,"num_objects":2221,"num_object_clones":0,"num_object_copies":6663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":326,"num_objects_unfound":0,"num_objects_dirty":2221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4145,"num_write_kb":203712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2405,"num_bytes_recovered":157614080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[1,2],"avail_no_missing":["1","3","6"],"object_location_counts":[{"shards":"1,2","objects":2058},{"shards":"1,3,6","objects":163}],"blocked_by":[],"up_primary":2,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":179,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.311885+0000","last_change":"2023-09-23T18:15:08.909339+0000","last_active":"2023-09-23T18:44:18.311885+0000","last_peered":"2023-09-23T18:44:18.311885+0000","last_clean":"2023-09-23T18:44:18.311885+0000","last_became_active":"2023-09-23T18:15:08.908912+0000","last_became_peered":"2023-09-23T18:15:08.908912+0000","last_unstale":"2023-09-23T18:44:18.311885+0000","last_undegraded":"2023-09-23T18:44:18.311885+0000","last_fullsized":"2023-09-23T18:44:18.311885+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":186,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.311371+0000","last_change":"2023-09-23T18:33:53.847489+0000","last_active":"2023-09-23T18:44:18.311371+0000","last_peered":"2023-09-23T18:44:18.311371+0000","last_clean":"2023-09-23T18:44:18.311371+0000","last_became_active":"2023-09-23T18:33:53.846935+0000","last_became_peered":"2023-09-23T18:33:53.846935+0000","last_unstale":"2023-09-23T18:44:18.311371+0000","last_undegraded":"2023-09-23T18:44:18.311371+0000","last_fullsized":"2023-09-23T18:44:18.311371+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"206'6400","reported_seq":6435,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:45:05.878703+0000","last_change":"2023-09-23T18:43:59.501690+0000","last_active":"2023-09-23T18:45:05.878703+0000","last_peered":"2023-09-23T18:45:05.878703+0000","last_clean":"2023-09-23T18:45:05.878703+0000","last_became_active":"2023-09-23T18:43:59.497605+0000","last_became_peered":"2023-09-23T18:43:59.497605+0000","last_unstale":"2023-09-23T18:45:05.878703+0000","last_undegraded":"2023-09-23T18:45:05.878703+0000","last_fullsized":"2023-09-23T18:45:05.878703+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":195,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6400,"log_dups_size":0,"ondisk_log_size":6400,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145817600,"num_objects":2225,"num_object_clones":0,"num_object_copies":6675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2225,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4041,"num_write_kb":200512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":174,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.076496+0000","last_change":"2023-09-23T18:27:33.585121+0000","last_active":"2023-09-23T18:44:18.076496+0000","last_peered":"2023-09-23T18:44:18.076496+0000","last_clean":"2023-09-23T18:44:18.076496+0000","last_became_active":"2023-09-23T18:27:33.584620+0000","last_became_peered":"2023-09-23T18:27:33.584620+0000","last_unstale":"2023-09-23T18:44:18.076496+0000","last_undegraded":"2023-09-23T18:44:18.076496+0000","last_fullsized":"2023-09-23T18:44:18.076496+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":195,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:26.351616+0000","last_change":"2023-09-23T18:33:53.852933+0000","last_active":"2023-09-23T18:44:26.351616+0000","last_peered":"2023-09-23T18:44:26.351616+0000","last_clean":"2023-09-23T18:44:26.351616+0000","last_became_active":"2023-09-23T18:33:53.852626+0000","last_became_peered":"2023-09-23T18:33:53.852626+0000","last_unstale":"2023-09-23T18:44:26.351616+0000","last_undegraded":"2023-09-23T18:44:26.351616+0000","last_fullsized":"2023-09-23T18:44:26.351616+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:00.174536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.16","version":"206'6458","reported_seq":6497,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:45:03.053236+0000","last_change":"2023-09-23T18:44:53.070852+0000","last_active":"2023-09-23T18:45:03.053236+0000","last_peered":"2023-09-23T18:45:03.053236+0000","last_clean":"2023-09-23T18:45:03.053236+0000","last_became_active":"2023-09-23T18:43:59.502108+0000","last_became_peered":"2023-09-23T18:43:59.502108+0000","last_unstale":"2023-09-23T18:45:03.053236+0000","last_undegraded":"2023-09-23T18:45:03.053236+0000","last_fullsized":"2023-09-23T18:45:03.053236+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":195,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6458,"log_dups_size":0,"ondisk_log_size":6458,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155942912,"num_objects":2380,"num_object_clones":0,"num_object_copies":7140,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2380,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4064,"num_write_kb":206176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":169,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.820732+0000","last_change":"2023-09-23T18:39:46.997790+0000","last_active":"2023-09-23T18:44:18.820732+0000","last_peered":"2023-09-23T18:44:18.820732+0000","last_clean":"2023-09-23T18:44:18.820732+0000","last_became_active":"2023-09-23T18:39:46.997089+0000","last_became_peered":"2023-09-23T18:39:46.997089+0000","last_unstale":"2023-09-23T18:44:18.820732+0000","last_undegraded":"2023-09-23T18:44:18.820732+0000","last_fullsized":"2023-09-23T18:44:18.820732+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:55.378220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":163,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.077113+0000","last_change":"2023-09-23T18:32:26.808237+0000","last_active":"2023-09-23T18:44:18.077113+0000","last_peered":"2023-09-23T18:44:18.077113+0000","last_clean":"2023-09-23T18:44:18.077113+0000","last_became_active":"2023-09-23T18:32:26.807843+0000","last_became_peered":"2023-09-23T18:32:26.807843+0000","last_unstale":"2023-09-23T18:44:18.077113+0000","last_undegraded":"2023-09-23T18:44:18.077113+0000","last_fullsized":"2023-09-23T18:44:18.077113+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.15","version":"206'6450","reported_seq":6486,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:45:05.812014+0000","last_change":"2023-09-23T18:43:59.480193+0000","last_active":"2023-09-23T18:45:05.812014+0000","last_peered":"2023-09-23T18:45:05.812014+0000","last_clean":"2023-09-23T18:45:05.812014+0000","last_became_active":"2023-09-23T18:43:59.477573+0000","last_became_peered":"2023-09-23T18:43:59.477573+0000","last_unstale":"2023-09-23T18:45:05.812014+0000","last_undegraded":"2023-09-23T18:45:05.812014+0000","last_fullsized":"2023-09-23T18:45:05.812014+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":195,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6450,"log_dups_size":0,"ondisk_log_size":6450,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148275200,"num_objects":2263,"num_object_clones":0,"num_object_copies":6789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4099,"num_write_kb":203552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":157,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.076338+0000","last_change":"2023-09-23T18:15:08.916658+0000","last_active":"2023-09-23T18:44:18.076338+0000","last_peered":"2023-09-23T18:44:18.076338+0000","last_clean":"2023-09-23T18:44:18.076338+0000","last_became_active":"2023-09-23T18:15:08.916301+0000","last_became_peered":"2023-09-23T18:15:08.916301+0000","last_unstale":"2023-09-23T18:44:18.076338+0000","last_undegraded":"2023-09-23T18:44:18.076338+0000","last_fullsized":"2023-09-23T18:44:18.076338+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":170,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.748433+0000","last_change":"2023-09-23T18:40:42.049370+0000","last_active":"2023-09-23T18:44:18.748433+0000","last_peered":"2023-09-23T18:44:18.748433+0000","last_clean":"2023-09-23T18:44:18.748433+0000","last_became_active":"2023-09-23T18:40:42.049003+0000","last_became_peered":"2023-09-23T18:40:42.049003+0000","last_unstale":"2023-09-23T18:44:18.748433+0000","last_undegraded":"2023-09-23T18:44:18.748433+0000","last_fullsized":"2023-09-23T18:44:18.748433+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:15.760457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"206'6324","reported_seq":6363,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:45:02.962326+0000","last_change":"2023-09-23T18:44:55.148046+0000","last_active":"2023-09-23T18:45:02.962326+0000","last_peered":"2023-09-23T18:45:02.962326+0000","last_clean":"2023-09-23T18:45:02.962326+0000","last_became_active":"2023-09-23T18:43:59.502491+0000","last_became_peered":"2023-09-23T18:43:59.502491+0000","last_unstale":"2023-09-23T18:45:02.962326+0000","last_undegraded":"2023-09-23T18:45:02.962326+0000","last_fullsized":"2023-09-23T18:45:02.962326+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":195,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6324,"log_dups_size":0,"ondisk_log_size":6324,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153550848,"num_objects":2343,"num_object_clones":0,"num_object_copies":7029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3977,"num_write_kb":202240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":218,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.068479+0000","last_change":"2023-09-23T18:40:42.073698+0000","last_active":"2023-09-23T18:44:18.068479+0000","last_peered":"2023-09-23T18:44:18.068479+0000","last_clean":"2023-09-23T18:44:18.068479+0000","last_became_active":"2023-09-23T18:40:42.059101+0000","last_became_peered":"2023-09-23T18:40:42.059101+0000","last_unstale":"2023-09-23T18:44:18.068479+0000","last_undegraded":"2023-09-23T18:44:18.068479+0000","last_fullsized":"2023-09-23T18:44:18.068479+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":149,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.064548+0000","last_change":"2023-09-23T18:40:42.058656+0000","last_active":"2023-09-23T18:44:18.064548+0000","last_peered":"2023-09-23T18:44:18.064548+0000","last_clean":"2023-09-23T18:44:18.064548+0000","last_became_active":"2023-09-23T18:40:42.057307+0000","last_became_peered":"2023-09-23T18:40:42.057307+0000","last_unstale":"2023-09-23T18:44:18.064548+0000","last_undegraded":"2023-09-23T18:44:18.064548+0000","last_fullsized":"2023-09-23T18:44:18.064548+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:08:45.979078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.b","version":"206'8169","reported_seq":8198,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:45:07.841319+0000","last_change":"2023-09-23T18:42:39.314324+0000","last_active":"2023-09-23T18:45:07.841319+0000","last_peered":"2023-09-23T18:45:07.841319+0000","last_clean":"2023-09-23T18:45:07.841319+0000","last_became_active":"2023-09-23T18:42:39.314006+0000","last_became_peered":"2023-09-23T18:42:39.314006+0000","last_unstale":"2023-09-23T18:45:07.841319+0000","last_undegraded":"2023-09-23T18:45:07.841319+0000","last_fullsized":"2023-09-23T18:45:07.841319+0000","mapping_epoch":187,"log_start":"192'3200","ondisk_log_start":"192'3200","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":4969,"log_dups_size":0,"ondisk_log_size":4969,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:52.663013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285540352,"num_objects":4357,"num_object_clones":0,"num_object_copies":13071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4357,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8169,"num_write_kb":400832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":186,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.077163+0000","last_change":"2023-09-23T18:37:41.034732+0000","last_active":"2023-09-23T18:44:18.077163+0000","last_peered":"2023-09-23T18:44:18.077163+0000","last_clean":"2023-09-23T18:44:18.077163+0000","last_became_active":"2023-09-23T18:37:40.985865+0000","last_became_peered":"2023-09-23T18:37:40.985865+0000","last_unstale":"2023-09-23T18:44:18.077163+0000","last_undegraded":"2023-09-23T18:44:18.077163+0000","last_fullsized":"2023-09-23T18:44:18.077163+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":104,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.076407+0000","last_change":"2023-09-23T18:32:26.806328+0000","last_active":"2023-09-23T18:44:18.076407+0000","last_peered":"2023-09-23T18:44:18.076407+0000","last_clean":"2023-09-23T18:44:18.076407+0000","last_became_active":"2023-09-23T18:32:26.803969+0000","last_became_peered":"2023-09-23T18:32:26.803969+0000","last_unstale":"2023-09-23T18:44:18.076407+0000","last_undegraded":"2023-09-23T18:44:18.076407+0000","last_fullsized":"2023-09-23T18:44:18.076407+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.a","version":"206'7990","reported_seq":8019,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:45:06.397716+0000","last_change":"2023-09-23T18:42:39.310080+0000","last_active":"2023-09-23T18:45:06.397716+0000","last_peered":"2023-09-23T18:45:06.397716+0000","last_clean":"2023-09-23T18:45:06.397716+0000","last_became_active":"2023-09-23T18:42:39.309414+0000","last_became_peered":"2023-09-23T18:42:39.309414+0000","last_unstale":"2023-09-23T18:45:06.397716+0000","last_undegraded":"2023-09-23T18:45:06.397716+0000","last_fullsized":"2023-09-23T18:45:06.397716+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":7990,"log_dups_size":0,"ondisk_log_size":7990,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:43.822106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289406976,"num_objects":4416,"num_object_clones":0,"num_object_copies":13248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7990,"num_write_kb":396992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":189,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.077143+0000","last_change":"2023-09-23T18:33:53.847817+0000","last_active":"2023-09-23T18:44:18.077143+0000","last_peered":"2023-09-23T18:44:18.077143+0000","last_clean":"2023-09-23T18:44:18.077143+0000","last_became_active":"2023-09-23T18:33:53.847342+0000","last_became_peered":"2023-09-23T18:33:53.847342+0000","last_unstale":"2023-09-23T18:44:18.077143+0000","last_undegraded":"2023-09-23T18:44:18.077143+0000","last_fullsized":"2023-09-23T18:44:18.077143+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":210,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.311246+0000","last_change":"2023-09-23T18:39:46.995444+0000","last_active":"2023-09-23T18:44:18.311246+0000","last_peered":"2023-09-23T18:44:18.311246+0000","last_clean":"2023-09-23T18:44:18.311246+0000","last_became_active":"2023-09-23T18:39:46.995033+0000","last_became_peered":"2023-09-23T18:39:46.995033+0000","last_unstale":"2023-09-23T18:44:18.311246+0000","last_undegraded":"2023-09-23T18:44:18.311246+0000","last_fullsized":"2023-09-23T18:44:18.311246+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"206'6282","reported_seq":6322,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:45:03.078190+0000","last_change":"2023-09-23T18:44:59.228023+0000","last_active":"2023-09-23T18:45:03.078190+0000","last_peered":"2023-09-23T18:45:03.078190+0000","last_clean":"2023-09-23T18:45:03.078190+0000","last_became_active":"2023-09-23T18:43:59.504968+0000","last_became_peered":"2023-09-23T18:43:59.504968+0000","last_unstale":"2023-09-23T18:45:03.078190+0000","last_undegraded":"2023-09-23T18:45:03.078190+0000","last_fullsized":"2023-09-23T18:45:03.078190+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6282,"log_dups_size":0,"ondisk_log_size":6282,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147259392,"num_objects":2247,"num_object_clones":0,"num_object_copies":6741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2247,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3949,"num_write_kb":198272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":171,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.076764+0000","last_change":"2023-09-23T18:32:26.854615+0000","last_active":"2023-09-23T18:44:18.076764+0000","last_peered":"2023-09-23T18:44:18.076764+0000","last_clean":"2023-09-23T18:44:18.076764+0000","last_became_active":"2023-09-23T18:32:26.806771+0000","last_became_peered":"2023-09-23T18:32:26.806771+0000","last_unstale":"2023-09-23T18:44:18.076764+0000","last_undegraded":"2023-09-23T18:44:18.076764+0000","last_fullsized":"2023-09-23T18:44:18.076764+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":180,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.068995+0000","last_change":"2023-09-23T18:27:33.589116+0000","last_active":"2023-09-23T18:44:18.068995+0000","last_peered":"2023-09-23T18:44:18.068995+0000","last_clean":"2023-09-23T18:44:18.068995+0000","last_became_active":"2023-09-23T18:27:33.588809+0000","last_became_peered":"2023-09-23T18:27:33.588809+0000","last_unstale":"2023-09-23T18:44:18.068995+0000","last_undegraded":"2023-09-23T18:44:18.068995+0000","last_fullsized":"2023-09-23T18:44:18.068995+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"206'6477","reported_seq":6516,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:45:04.162356+0000","last_change":"2023-09-23T18:43:59.464763+0000","last_active":"2023-09-23T18:45:04.162356+0000","last_peered":"2023-09-23T18:45:04.162356+0000","last_clean":"2023-09-23T18:45:04.162356+0000","last_became_active":"2023-09-23T18:43:59.458263+0000","last_became_peered":"2023-09-23T18:43:59.458263+0000","last_unstale":"2023-09-23T18:45:04.162356+0000","last_undegraded":"2023-09-23T18:45:04.162356+0000","last_fullsized":"2023-09-23T18:45:04.162356+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6477,"log_dups_size":0,"ondisk_log_size":6477,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153649152,"num_objects":2344,"num_object_clones":0,"num_object_copies":7032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4080,"num_write_kb":205600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":180,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.062100+0000","last_change":"2023-09-23T18:33:54.848776+0000","last_active":"2023-09-23T18:44:18.062100+0000","last_peered":"2023-09-23T18:44:18.062100+0000","last_clean":"2023-09-23T18:44:18.062100+0000","last_became_active":"2023-09-23T18:33:54.848380+0000","last_became_peered":"2023-09-23T18:33:54.848380+0000","last_unstale":"2023-09-23T18:44:18.062100+0000","last_undegraded":"2023-09-23T18:44:18.062100+0000","last_fullsized":"2023-09-23T18:44:18.062100+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:17:17.093993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":174,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:33.101055+0000","last_change":"2023-09-23T18:27:33.582033+0000","last_active":"2023-09-23T18:44:33.101055+0000","last_peered":"2023-09-23T18:44:33.101055+0000","last_clean":"2023-09-23T18:44:33.101055+0000","last_became_active":"2023-09-23T18:27:33.581667+0000","last_became_peered":"2023-09-23T18:27:33.581667+0000","last_unstale":"2023-09-23T18:44:33.101055+0000","last_undegraded":"2023-09-23T18:44:33.101055+0000","last_fullsized":"2023-09-23T18:44:33.101055+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:36.039922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"206'7972","reported_seq":8001,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:45:03.028654+0000","last_change":"2023-09-23T18:42:39.318187+0000","last_active":"2023-09-23T18:45:03.028654+0000","last_peered":"2023-09-23T18:45:03.028654+0000","last_clean":"2023-09-23T18:45:03.028654+0000","last_became_active":"2023-09-23T18:42:39.317693+0000","last_became_peered":"2023-09-23T18:42:39.317693+0000","last_unstale":"2023-09-23T18:45:03.028654+0000","last_undegraded":"2023-09-23T18:45:03.028654+0000","last_fullsized":"2023-09-23T18:45:03.028654+0000","mapping_epoch":187,"log_start":"192'1700","ondisk_log_start":"192'1700","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6272,"log_dups_size":0,"ondisk_log_size":6272,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:53.117569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306184192,"num_objects":4672,"num_object_clones":0,"num_object_copies":14016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7972,"num_write_kb":404608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":179,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.064176+0000","last_change":"2023-09-23T18:28:03.297628+0000","last_active":"2023-09-23T18:44:18.064176+0000","last_peered":"2023-09-23T18:44:18.064176+0000","last_clean":"2023-09-23T18:44:18.064176+0000","last_became_active":"2023-09-23T18:28:03.296694+0000","last_became_peered":"2023-09-23T18:28:03.296694+0000","last_unstale":"2023-09-23T18:44:18.064176+0000","last_undegraded":"2023-09-23T18:44:18.064176+0000","last_fullsized":"2023-09-23T18:44:18.064176+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:43.375652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":171,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.311677+0000","last_change":"2023-09-23T18:28:03.302326+0000","last_active":"2023-09-23T18:44:18.311677+0000","last_peered":"2023-09-23T18:44:18.311677+0000","last_clean":"2023-09-23T18:44:18.311677+0000","last_became_active":"2023-09-23T18:28:03.301970+0000","last_became_peered":"2023-09-23T18:28:03.301970+0000","last_unstale":"2023-09-23T18:44:18.311677+0000","last_undegraded":"2023-09-23T18:44:18.311677+0000","last_fullsized":"2023-09-23T18:44:18.311677+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"206'8145","reported_seq":8174,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:45:04.229414+0000","last_change":"2023-09-23T18:42:39.309814+0000","last_active":"2023-09-23T18:45:04.229414+0000","last_peered":"2023-09-23T18:45:04.229414+0000","last_clean":"2023-09-23T18:45:04.229414+0000","last_became_active":"2023-09-23T18:42:39.309499+0000","last_became_peered":"2023-09-23T18:42:39.309499+0000","last_unstale":"2023-09-23T18:45:04.229414+0000","last_undegraded":"2023-09-23T18:45:04.229414+0000","last_fullsized":"2023-09-23T18:45:04.229414+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":8145,"log_dups_size":0,"ondisk_log_size":8145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:26.106244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311230464,"num_objects":4749,"num_object_clones":0,"num_object_copies":14247,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4749,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8145,"num_write_kb":412608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":187,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.068868+0000","last_change":"2023-09-23T18:33:53.855371+0000","last_active":"2023-09-23T18:44:18.068868+0000","last_peered":"2023-09-23T18:44:18.068868+0000","last_clean":"2023-09-23T18:44:18.068868+0000","last_became_active":"2023-09-23T18:33:53.854996+0000","last_became_peered":"2023-09-23T18:33:53.854996+0000","last_unstale":"2023-09-23T18:44:18.068868+0000","last_undegraded":"2023-09-23T18:44:18.068868+0000","last_fullsized":"2023-09-23T18:44:18.068868+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":210,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.310734+0000","last_change":"2023-09-23T18:40:42.060184+0000","last_active":"2023-09-23T18:44:18.310734+0000","last_peered":"2023-09-23T18:44:18.310734+0000","last_clean":"2023-09-23T18:44:18.310734+0000","last_became_active":"2023-09-23T18:40:42.059725+0000","last_became_peered":"2023-09-23T18:40:42.059725+0000","last_unstale":"2023-09-23T18:44:18.310734+0000","last_undegraded":"2023-09-23T18:44:18.310734+0000","last_fullsized":"2023-09-23T18:44:18.310734+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"206'7969","reported_seq":7997,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:45:05.893716+0000","last_change":"2023-09-23T18:42:39.319875+0000","last_active":"2023-09-23T18:45:05.893716+0000","last_peered":"2023-09-23T18:45:05.893716+0000","last_clean":"2023-09-23T18:45:05.893716+0000","last_became_active":"2023-09-23T18:42:39.319468+0000","last_became_peered":"2023-09-23T18:42:39.319468+0000","last_unstale":"2023-09-23T18:45:05.893716+0000","last_undegraded":"2023-09-23T18:45:05.893716+0000","last_fullsized":"2023-09-23T18:45:05.893716+0000","mapping_epoch":187,"log_start":"192'1800","ondisk_log_start":"192'1800","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6169,"log_dups_size":0,"ondisk_log_size":6169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286916632,"num_objects":4379,"num_object_clones":0,"num_object_copies":13137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4379,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7969,"num_write_kb":395073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":172,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.310896+0000","last_change":"2023-09-23T18:27:33.585714+0000","last_active":"2023-09-23T18:44:18.310896+0000","last_peered":"2023-09-23T18:44:18.310896+0000","last_clean":"2023-09-23T18:44:18.310896+0000","last_became_active":"2023-09-23T18:27:33.584641+0000","last_became_peered":"2023-09-23T18:27:33.584641+0000","last_unstale":"2023-09-23T18:44:18.310896+0000","last_undegraded":"2023-09-23T18:44:18.310896+0000","last_fullsized":"2023-09-23T18:44:18.310896+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":186,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.312119+0000","last_change":"2023-09-23T18:33:53.851266+0000","last_active":"2023-09-23T18:44:18.312119+0000","last_peered":"2023-09-23T18:44:18.312119+0000","last_clean":"2023-09-23T18:44:18.312119+0000","last_became_active":"2023-09-23T18:33:53.850926+0000","last_became_peered":"2023-09-23T18:33:53.850926+0000","last_unstale":"2023-09-23T18:44:18.312119+0000","last_undegraded":"2023-09-23T18:44:18.312119+0000","last_fullsized":"2023-09-23T18:44:18.312119+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"206'8055","reported_seq":8082,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:45:05.896930+0000","last_change":"2023-09-23T18:42:39.318852+0000","last_active":"2023-09-23T18:45:05.896930+0000","last_peered":"2023-09-23T18:45:05.896930+0000","last_clean":"2023-09-23T18:45:05.896930+0000","last_became_active":"2023-09-23T18:42:39.318461+0000","last_became_peered":"2023-09-23T18:42:39.318461+0000","last_unstale":"2023-09-23T18:45:05.896930+0000","last_undegraded":"2023-09-23T18:45:05.896930+0000","last_fullsized":"2023-09-23T18:45:05.896930+0000","mapping_epoch":187,"log_start":"192'1900","ondisk_log_start":"192'1900","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6155,"log_dups_size":0,"ondisk_log_size":6155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289996800,"num_objects":4425,"num_object_clones":0,"num_object_copies":13275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8055,"num_write_kb":399360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":158,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.061797+0000","last_change":"2023-09-23T18:39:46.993941+0000","last_active":"2023-09-23T18:44:18.061797+0000","last_peered":"2023-09-23T18:44:18.061797+0000","last_clean":"2023-09-23T18:44:18.061797+0000","last_became_active":"2023-09-23T18:39:46.993625+0000","last_became_peered":"2023-09-23T18:39:46.993625+0000","last_unstale":"2023-09-23T18:44:18.061797+0000","last_undegraded":"2023-09-23T18:44:18.061797+0000","last_fullsized":"2023-09-23T18:44:18.061797+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:23.237229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":146,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:44:41.689545+0000","last_change":"2023-09-23T18:33:53.855430+0000","last_active":"2023-09-23T18:44:41.689545+0000","last_peered":"2023-09-23T18:44:41.689545+0000","last_clean":"2023-09-23T18:44:41.689545+0000","last_became_active":"2023-09-23T18:33:53.854935+0000","last_became_peered":"2023-09-23T18:33:53.854935+0000","last_unstale":"2023-09-23T18:44:41.689545+0000","last_undegraded":"2023-09-23T18:44:41.689545+0000","last_fullsized":"2023-09-23T18:44:41.689545+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:36.078411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"206'6420","reported_seq":6474,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:45:06.196040+0000","last_change":"2023-09-23T18:45:05.363486+0000","last_active":"2023-09-23T18:45:06.196040+0000","last_peered":"2023-09-23T18:45:06.196040+0000","last_clean":"2023-09-23T18:45:06.196040+0000","last_became_active":"2023-09-23T18:43:59.506884+0000","last_became_peered":"2023-09-23T18:43:59.506884+0000","last_unstale":"2023-09-23T18:45:06.196040+0000","last_undegraded":"2023-09-23T18:45:06.196040+0000","last_fullsized":"2023-09-23T18:45:06.196040+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6420,"log_dups_size":0,"ondisk_log_size":6420,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147816448,"num_objects":2255,"num_object_clones":0,"num_object_copies":6765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4059,"num_write_kb":202080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"77'130","reported_seq":672,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.063216+0000","last_change":"2023-09-23T18:33:53.850713+0000","last_active":"2023-09-23T18:44:18.063216+0000","last_peered":"2023-09-23T18:44:18.063216+0000","last_clean":"2023-09-23T18:44:18.063216+0000","last_became_active":"2023-09-23T18:33:53.850011+0000","last_became_peered":"2023-09-23T18:33:53.850011+0000","last_unstale":"2023-09-23T18:44:18.063216+0000","last_undegraded":"2023-09-23T18:44:18.063216+0000","last_fullsized":"2023-09-23T18:44:18.063216+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:58.943556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":772,"num_read_kb":1712,"num_write":126,"num_write_kb":2126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":127,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:44:41.697601+0000","last_change":"2023-09-23T18:40:42.058197+0000","last_active":"2023-09-23T18:44:41.697601+0000","last_peered":"2023-09-23T18:44:41.697601+0000","last_clean":"2023-09-23T18:44:41.697601+0000","last_became_active":"2023-09-23T18:40:42.057834+0000","last_became_peered":"2023-09-23T18:40:42.057834+0000","last_unstale":"2023-09-23T18:44:41.697601+0000","last_undegraded":"2023-09-23T18:44:41.697601+0000","last_fullsized":"2023-09-23T18:44:41.697601+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:28.133529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":120,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.062421+0000","last_change":"2023-09-23T18:33:54.843246+0000","last_active":"2023-09-23T18:44:18.062421+0000","last_peered":"2023-09-23T18:44:18.062421+0000","last_clean":"2023-09-23T18:44:18.062421+0000","last_became_active":"2023-09-23T18:33:54.842933+0000","last_became_peered":"2023-09-23T18:33:54.842933+0000","last_unstale":"2023-09-23T18:44:18.062421+0000","last_undegraded":"2023-09-23T18:44:18.062421+0000","last_fullsized":"2023-09-23T18:44:18.062421+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:43.716707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"206'6438","reported_seq":6492,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:45:05.878708+0000","last_change":"2023-09-23T18:45:02.102743+0000","last_active":"2023-09-23T18:45:05.878708+0000","last_peered":"2023-09-23T18:45:05.878708+0000","last_clean":"2023-09-23T18:45:05.878708+0000","last_became_active":"2023-09-23T18:43:59.473142+0000","last_became_peered":"2023-09-23T18:43:59.473142+0000","last_unstale":"2023-09-23T18:45:05.878708+0000","last_undegraded":"2023-09-23T18:45:05.878708+0000","last_fullsized":"2023-09-23T18:45:05.878708+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6438,"log_dups_size":0,"ondisk_log_size":6438,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148832256,"num_objects":2271,"num_object_clones":0,"num_object_copies":6813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4079,"num_write_kb":203200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"77'96","reported_seq":616,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:22.679909+0000","last_change":"2023-09-23T18:33:54.842857+0000","last_active":"2023-09-23T18:44:22.679909+0000","last_peered":"2023-09-23T18:44:22.679909+0000","last_clean":"2023-09-23T18:44:22.679909+0000","last_became_active":"2023-09-23T18:33:54.842134+0000","last_became_peered":"2023-09-23T18:33:54.842134+0000","last_unstale":"2023-09-23T18:44:22.679909+0000","last_undegraded":"2023-09-23T18:44:22.679909+0000","last_fullsized":"2023-09-23T18:44:22.679909+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":96,"log_dups_size":0,"ondisk_log_size":96,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:48.960158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":728,"num_read_kb":1234,"num_write":87,"num_write_kb":338,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":172,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.311487+0000","last_change":"2023-09-23T18:26:34.992535+0000","last_active":"2023-09-23T18:44:18.311487+0000","last_peered":"2023-09-23T18:44:18.311487+0000","last_clean":"2023-09-23T18:44:18.311487+0000","last_became_active":"2023-09-23T18:26:34.992177+0000","last_became_peered":"2023-09-23T18:26:34.992177+0000","last_unstale":"2023-09-23T18:44:18.311487+0000","last_undegraded":"2023-09-23T18:44:18.311487+0000","last_fullsized":"2023-09-23T18:44:18.311487+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":170,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.063460+0000","last_change":"2023-09-23T18:40:42.066006+0000","last_active":"2023-09-23T18:44:18.063460+0000","last_peered":"2023-09-23T18:44:18.063460+0000","last_clean":"2023-09-23T18:44:18.063460+0000","last_became_active":"2023-09-23T18:40:42.065431+0000","last_became_peered":"2023-09-23T18:40:42.065431+0000","last_unstale":"2023-09-23T18:44:18.063460+0000","last_undegraded":"2023-09-23T18:44:18.063460+0000","last_fullsized":"2023-09-23T18:44:18.063460+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:18.818629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"206'6372","reported_seq":6413,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:45:03.064594+0000","last_change":"2023-09-23T18:44:57.174542+0000","last_active":"2023-09-23T18:45:03.064594+0000","last_peered":"2023-09-23T18:45:03.064594+0000","last_clean":"2023-09-23T18:45:03.064594+0000","last_became_active":"2023-09-23T18:43:59.475213+0000","last_became_peered":"2023-09-23T18:43:59.475213+0000","last_unstale":"2023-09-23T18:45:03.064594+0000","last_undegraded":"2023-09-23T18:45:03.064594+0000","last_fullsized":"2023-09-23T18:45:03.064594+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6372,"log_dups_size":0,"ondisk_log_size":6372,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152240128,"num_objects":2323,"num_object_clones":0,"num_object_copies":6969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4025,"num_write_kb":203136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"77'112","reported_seq":557,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:44:41.697434+0000","last_change":"2023-09-23T18:40:45.106420+0000","last_active":"2023-09-23T18:44:41.697434+0000","last_peered":"2023-09-23T18:44:41.697434+0000","last_clean":"2023-09-23T18:44:41.697434+0000","last_became_active":"2023-09-23T18:40:45.104294+0000","last_became_peered":"2023-09-23T18:40:45.104294+0000","last_unstale":"2023-09-23T18:44:41.697434+0000","last_undegraded":"2023-09-23T18:44:41.697434+0000","last_fullsized":"2023-09-23T18:44:41.697434+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:45.419412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":774,"num_read_kb":596,"num_write":102,"num_write_kb":52,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":142,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.061553+0000","last_change":"2023-09-23T18:33:53.849084+0000","last_active":"2023-09-23T18:44:18.061553+0000","last_peered":"2023-09-23T18:44:18.061553+0000","last_clean":"2023-09-23T18:44:18.061553+0000","last_became_active":"2023-09-23T18:33:53.848757+0000","last_became_peered":"2023-09-23T18:33:53.848757+0000","last_unstale":"2023-09-23T18:44:18.061553+0000","last_undegraded":"2023-09-23T18:44:18.061553+0000","last_fullsized":"2023-09-23T18:44:18.061553+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:15.267742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":165,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:44:41.689757+0000","last_change":"2023-09-23T18:33:55.087304+0000","last_active":"2023-09-23T18:44:41.689757+0000","last_peered":"2023-09-23T18:44:41.689757+0000","last_clean":"2023-09-23T18:44:41.689757+0000","last_became_active":"2023-09-23T18:33:55.086832+0000","last_became_peered":"2023-09-23T18:33:55.086832+0000","last_unstale":"2023-09-23T18:44:41.689757+0000","last_undegraded":"2023-09-23T18:44:41.689757+0000","last_fullsized":"2023-09-23T18:44:41.689757+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:55.457664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"206'6506","reported_seq":6547,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:45:02.960638+0000","last_change":"2023-09-23T18:45:00.151781+0000","last_active":"2023-09-23T18:45:02.960638+0000","last_peered":"2023-09-23T18:45:02.960638+0000","last_clean":"2023-09-23T18:45:02.960638+0000","last_became_active":"2023-09-23T18:43:59.485259+0000","last_became_peered":"2023-09-23T18:43:59.485259+0000","last_unstale":"2023-09-23T18:45:02.960638+0000","last_undegraded":"2023-09-23T18:45:02.960638+0000","last_fullsized":"2023-09-23T18:45:02.960638+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6506,"log_dups_size":0,"ondisk_log_size":6506,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154501120,"num_objects":2357,"num_object_clones":0,"num_object_copies":7071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2357,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4111,"num_write_kb":207008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"77'147","reported_seq":597,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:44:41.697081+0000","last_change":"2023-09-23T18:32:29.845404+0000","last_active":"2023-09-23T18:44:41.697081+0000","last_peered":"2023-09-23T18:44:41.697081+0000","last_clean":"2023-09-23T18:44:41.697081+0000","last_became_active":"2023-09-23T18:32:29.844545+0000","last_became_peered":"2023-09-23T18:32:29.844545+0000","last_unstale":"2023-09-23T18:44:41.697081+0000","last_undegraded":"2023-09-23T18:44:41.697081+0000","last_fullsized":"2023-09-23T18:44:41.697081+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:44.834482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":624,"num_read_kb":1868,"num_write":143,"num_write_kb":5368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":408,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.076099+0000","last_change":"2023-09-23T18:26:34.999402+0000","last_active":"2023-09-23T18:44:18.076099+0000","last_peered":"2023-09-23T18:44:18.076099+0000","last_clean":"2023-09-23T18:44:18.076099+0000","last_became_active":"2023-09-23T18:26:34.998828+0000","last_became_peered":"2023-09-23T18:26:34.998828+0000","last_unstale":"2023-09-23T18:44:18.076099+0000","last_undegraded":"2023-09-23T18:44:18.076099+0000","last_fullsized":"2023-09-23T18:44:18.076099+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_clean_scrub_stamp":"2023-09-23T18:08:35.701374+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":121,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.069059+0000","last_change":"2023-09-23T18:40:42.058559+0000","last_active":"2023-09-23T18:44:18.069059+0000","last_peered":"2023-09-23T18:44:18.069059+0000","last_clean":"2023-09-23T18:44:18.069059+0000","last_became_active":"2023-09-23T18:40:42.058253+0000","last_became_peered":"2023-09-23T18:40:42.058253+0000","last_unstale":"2023-09-23T18:44:18.069059+0000","last_undegraded":"2023-09-23T18:44:18.069059+0000","last_fullsized":"2023-09-23T18:44:18.069059+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":156,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.069284+0000","last_change":"2023-09-23T18:15:08.921629+0000","last_active":"2023-09-23T18:44:18.069284+0000","last_peered":"2023-09-23T18:44:18.069284+0000","last_clean":"2023-09-23T18:44:18.069284+0000","last_became_active":"2023-09-23T18:15:08.921283+0000","last_became_peered":"2023-09-23T18:15:08.921283+0000","last_unstale":"2023-09-23T18:44:18.069284+0000","last_undegraded":"2023-09-23T18:44:18.069284+0000","last_fullsized":"2023-09-23T18:44:18.069284+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"206'6387","reported_seq":6424,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:45:05.918588+0000","last_change":"2023-09-23T18:43:59.463638+0000","last_active":"2023-09-23T18:45:05.918588+0000","last_peered":"2023-09-23T18:45:05.918588+0000","last_clean":"2023-09-23T18:45:05.918588+0000","last_became_active":"2023-09-23T18:43:59.460459+0000","last_became_peered":"2023-09-23T18:43:59.460459+0000","last_unstale":"2023-09-23T18:45:05.918588+0000","last_undegraded":"2023-09-23T18:45:05.918588+0000","last_fullsized":"2023-09-23T18:45:05.918588+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6387,"log_dups_size":0,"ondisk_log_size":6387,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147161088,"num_objects":2245,"num_object_clones":0,"num_object_copies":6735,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2245,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4035,"num_write_kb":200992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'120","reported_seq":667,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.076270+0000","last_change":"2023-09-23T18:33:53.852697+0000","last_active":"2023-09-23T18:44:18.076270+0000","last_peered":"2023-09-23T18:44:18.076270+0000","last_clean":"2023-09-23T18:44:18.076270+0000","last_became_active":"2023-09-23T18:33:53.852210+0000","last_became_peered":"2023-09-23T18:33:53.852210+0000","last_unstale":"2023-09-23T18:44:18.076270+0000","last_undegraded":"2023-09-23T18:44:18.076270+0000","last_fullsized":"2023-09-23T18:44:18.076270+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":120,"log_dups_size":0,"ondisk_log_size":120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":809,"num_read_kb":2749,"num_write":98,"num_write_kb":2087,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":170,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.749006+0000","last_change":"2023-09-23T18:39:46.997952+0000","last_active":"2023-09-23T18:44:18.749006+0000","last_peered":"2023-09-23T18:44:18.749006+0000","last_clean":"2023-09-23T18:44:18.749006+0000","last_became_active":"2023-09-23T18:39:46.997589+0000","last_became_peered":"2023-09-23T18:39:46.997589+0000","last_unstale":"2023-09-23T18:44:18.749006+0000","last_undegraded":"2023-09-23T18:44:18.749006+0000","last_fullsized":"2023-09-23T18:44:18.749006+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:09.887469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":201,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.749723+0000","last_change":"2023-09-23T18:39:46.996973+0000","last_active":"2023-09-23T18:44:18.749723+0000","last_peered":"2023-09-23T18:44:18.749723+0000","last_clean":"2023-09-23T18:44:18.749723+0000","last_became_active":"2023-09-23T18:39:46.996633+0000","last_became_peered":"2023-09-23T18:39:46.996633+0000","last_unstale":"2023-09-23T18:44:18.749723+0000","last_undegraded":"2023-09-23T18:44:18.749723+0000","last_fullsized":"2023-09-23T18:44:18.749723+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:08:05.607474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"206'6519","reported_seq":6558,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:45:07.812401+0000","last_change":"2023-09-23T18:43:59.488931+0000","last_active":"2023-09-23T18:45:07.812401+0000","last_peered":"2023-09-23T18:45:07.812401+0000","last_clean":"2023-09-23T18:45:07.812401+0000","last_became_active":"2023-09-23T18:43:59.476762+0000","last_became_peered":"2023-09-23T18:43:59.476762+0000","last_unstale":"2023-09-23T18:45:07.812401+0000","last_undegraded":"2023-09-23T18:45:07.812401+0000","last_fullsized":"2023-09-23T18:45:07.812401+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6519,"log_dups_size":0,"ondisk_log_size":6519,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145096704,"num_objects":2214,"num_object_clones":0,"num_object_copies":6642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4150,"num_write_kb":203648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'78","reported_seq":442,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.076814+0000","last_change":"2023-09-23T18:26:34.992345+0000","last_active":"2023-09-23T18:44:18.076814+0000","last_peered":"2023-09-23T18:44:18.076814+0000","last_clean":"2023-09-23T18:44:18.076814+0000","last_became_active":"2023-09-23T18:26:34.991774+0000","last_became_peered":"2023-09-23T18:26:34.991774+0000","last_unstale":"2023-09-23T18:44:18.076814+0000","last_undegraded":"2023-09-23T18:44:18.076814+0000","last_fullsized":"2023-09-23T18:44:18.076814+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":78,"log_dups_size":0,"ondisk_log_size":78,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":428,"num_read_kb":1125,"num_write":73,"num_write_kb":431,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":149,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.063291+0000","last_change":"2023-09-23T18:39:46.990343+0000","last_active":"2023-09-23T18:44:18.063291+0000","last_peered":"2023-09-23T18:44:18.063291+0000","last_clean":"2023-09-23T18:44:18.063291+0000","last_became_active":"2023-09-23T18:39:46.989904+0000","last_became_peered":"2023-09-23T18:39:46.989904+0000","last_unstale":"2023-09-23T18:44:18.063291+0000","last_undegraded":"2023-09-23T18:44:18.063291+0000","last_fullsized":"2023-09-23T18:44:18.063291+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:15.638308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":167,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.749389+0000","last_change":"2023-09-23T18:33:54.852309+0000","last_active":"2023-09-23T18:44:18.749389+0000","last_peered":"2023-09-23T18:44:18.749389+0000","last_clean":"2023-09-23T18:44:18.749389+0000","last_became_active":"2023-09-23T18:33:54.850255+0000","last_became_peered":"2023-09-23T18:33:54.850255+0000","last_unstale":"2023-09-23T18:44:18.749389+0000","last_undegraded":"2023-09-23T18:44:18.749389+0000","last_fullsized":"2023-09-23T18:44:18.749389+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:54.148992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"206'6405","reported_seq":6448,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:45:06.391942+0000","last_change":"2023-09-23T18:45:04.558483+0000","last_active":"2023-09-23T18:45:06.391942+0000","last_peered":"2023-09-23T18:45:06.391942+0000","last_clean":"2023-09-23T18:45:06.391942+0000","last_became_active":"2023-09-23T18:43:59.455678+0000","last_became_peered":"2023-09-23T18:43:59.455678+0000","last_unstale":"2023-09-23T18:45:06.391942+0000","last_undegraded":"2023-09-23T18:45:06.391942+0000","last_fullsized":"2023-09-23T18:45:06.391942+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6405,"log_dups_size":0,"ondisk_log_size":6405,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147685376,"num_objects":2253,"num_object_clones":0,"num_object_copies":6759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4061,"num_write_kb":202080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":179,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.063580+0000","last_change":"2023-09-23T18:28:03.298805+0000","last_active":"2023-09-23T18:44:18.063580+0000","last_peered":"2023-09-23T18:44:18.063580+0000","last_clean":"2023-09-23T18:44:18.063580+0000","last_became_active":"2023-09-23T18:28:03.298437+0000","last_became_peered":"2023-09-23T18:28:03.298437+0000","last_unstale":"2023-09-23T18:44:18.063580+0000","last_undegraded":"2023-09-23T18:44:18.063580+0000","last_fullsized":"2023-09-23T18:44:18.063580+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:27.831669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'181","reported_seq":680,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:44:41.688693+0000","last_change":"2023-09-23T18:33:53.854528+0000","last_active":"2023-09-23T18:44:41.688693+0000","last_peered":"2023-09-23T18:44:41.688693+0000","last_clean":"2023-09-23T18:44:41.688693+0000","last_became_active":"2023-09-23T18:33:53.851883+0000","last_became_peered":"2023-09-23T18:33:53.851883+0000","last_unstale":"2023-09-23T18:44:41.688693+0000","last_undegraded":"2023-09-23T18:44:41.688693+0000","last_fullsized":"2023-09-23T18:44:41.688693+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":181,"log_dups_size":0,"ondisk_log_size":181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:43.888844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":909,"num_read_kb":734,"num_write":148,"num_write_kb":75,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":114,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.076393+0000","last_change":"2023-09-23T18:39:46.992015+0000","last_active":"2023-09-23T18:44:18.076393+0000","last_peered":"2023-09-23T18:44:18.076393+0000","last_clean":"2023-09-23T18:44:18.076393+0000","last_became_active":"2023-09-23T18:39:46.991605+0000","last_became_peered":"2023-09-23T18:39:46.991605+0000","last_unstale":"2023-09-23T18:44:18.076393+0000","last_undegraded":"2023-09-23T18:44:18.076393+0000","last_fullsized":"2023-09-23T18:44:18.076393+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"206'6366","reported_seq":6413,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:45:06.378799+0000","last_change":"2023-09-23T18:45:05.476567+0000","last_active":"2023-09-23T18:45:06.378799+0000","last_peered":"2023-09-23T18:45:06.378799+0000","last_clean":"2023-09-23T18:45:06.378799+0000","last_became_active":"2023-09-23T18:43:59.507176+0000","last_became_peered":"2023-09-23T18:43:59.507176+0000","last_unstale":"2023-09-23T18:45:06.378799+0000","last_undegraded":"2023-09-23T18:45:06.378799+0000","last_fullsized":"2023-09-23T18:45:06.378799+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6366,"log_dups_size":0,"ondisk_log_size":6366,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145293312,"num_objects":2217,"num_object_clones":0,"num_object_copies":6651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4023,"num_write_kb":199680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"77'112","reported_seq":739,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.070022+0000","last_change":"2023-09-23T18:40:45.086639+0000","last_active":"2023-09-23T18:44:18.070022+0000","last_peered":"2023-09-23T18:44:18.070022+0000","last_clean":"2023-09-23T18:44:18.070022+0000","last_became_active":"2023-09-23T18:40:45.086119+0000","last_became_peered":"2023-09-23T18:40:45.086119+0000","last_unstale":"2023-09-23T18:44:18.070022+0000","last_undegraded":"2023-09-23T18:44:18.070022+0000","last_fullsized":"2023-09-23T18:44:18.070022+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":893,"num_read_kb":3172,"num_write":69,"num_write_kb":2237,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":180,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.749745+0000","last_change":"2023-09-23T18:33:53.851421+0000","last_active":"2023-09-23T18:44:18.749745+0000","last_peered":"2023-09-23T18:44:18.749745+0000","last_clean":"2023-09-23T18:44:18.749745+0000","last_became_active":"2023-09-23T18:33:53.851012+0000","last_became_peered":"2023-09-23T18:33:53.851012+0000","last_unstale":"2023-09-23T18:44:18.749745+0000","last_undegraded":"2023-09-23T18:44:18.749745+0000","last_fullsized":"2023-09-23T18:44:18.749745+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:03.971054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":165,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.311124+0000","last_change":"2023-09-23T18:32:26.805472+0000","last_active":"2023-09-23T18:44:18.311124+0000","last_peered":"2023-09-23T18:44:18.311124+0000","last_clean":"2023-09-23T18:44:18.311124+0000","last_became_active":"2023-09-23T18:32:26.805078+0000","last_became_peered":"2023-09-23T18:32:26.805078+0000","last_unstale":"2023-09-23T18:44:18.311124+0000","last_undegraded":"2023-09-23T18:44:18.311124+0000","last_fullsized":"2023-09-23T18:44:18.311124+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":149,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.749263+0000","last_change":"2023-09-23T18:26:34.994176+0000","last_active":"2023-09-23T18:44:18.749263+0000","last_peered":"2023-09-23T18:44:18.749263+0000","last_clean":"2023-09-23T18:44:18.749263+0000","last_became_active":"2023-09-23T18:26:34.993829+0000","last_became_peered":"2023-09-23T18:26:34.993829+0000","last_unstale":"2023-09-23T18:44:18.749263+0000","last_undegraded":"2023-09-23T18:44:18.749263+0000","last_fullsized":"2023-09-23T18:44:18.749263+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:27.111857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":186,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:44:41.689257+0000","last_change":"2023-09-23T18:40:42.053012+0000","last_active":"2023-09-23T18:44:41.689257+0000","last_peered":"2023-09-23T18:44:41.689257+0000","last_clean":"2023-09-23T18:44:41.689257+0000","last_became_active":"2023-09-23T18:40:42.052655+0000","last_became_peered":"2023-09-23T18:40:42.052655+0000","last_unstale":"2023-09-23T18:44:41.689257+0000","last_undegraded":"2023-09-23T18:44:41.689257+0000","last_fullsized":"2023-09-23T18:44:41.689257+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:33:51.009383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.18","version":"206'6409","reported_seq":6447,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:45:04.216214+0000","last_change":"2023-09-23T18:43:59.492253+0000","last_active":"2023-09-23T18:45:04.216214+0000","last_peered":"2023-09-23T18:45:04.216214+0000","last_clean":"2023-09-23T18:45:04.216214+0000","last_became_active":"2023-09-23T18:43:59.489977+0000","last_became_peered":"2023-09-23T18:43:59.489977+0000","last_unstale":"2023-09-23T18:45:04.216214+0000","last_undegraded":"2023-09-23T18:45:04.216214+0000","last_fullsized":"2023-09-23T18:45:04.216214+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":195,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6409,"log_dups_size":0,"ondisk_log_size":6409,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149323776,"num_objects":2279,"num_object_clones":0,"num_object_copies":6837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4013,"num_write_kb":201312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":129,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.311499+0000","last_change":"2023-09-23T18:33:54.847172+0000","last_active":"2023-09-23T18:44:18.311499+0000","last_peered":"2023-09-23T18:44:18.311499+0000","last_clean":"2023-09-23T18:44:18.311499+0000","last_became_active":"2023-09-23T18:33:54.846751+0000","last_became_peered":"2023-09-23T18:33:54.846751+0000","last_unstale":"2023-09-23T18:44:18.311499+0000","last_undegraded":"2023-09-23T18:44:18.311499+0000","last_fullsized":"2023-09-23T18:44:18.311499+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":201,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.311576+0000","last_change":"2023-09-23T18:33:53.841232+0000","last_active":"2023-09-23T18:44:18.311576+0000","last_peered":"2023-09-23T18:44:18.311576+0000","last_clean":"2023-09-23T18:44:18.311576+0000","last_became_active":"2023-09-23T18:33:53.840749+0000","last_became_peered":"2023-09-23T18:33:53.840749+0000","last_unstale":"2023-09-23T18:44:18.311576+0000","last_undegraded":"2023-09-23T18:44:18.311576+0000","last_fullsized":"2023-09-23T18:44:18.311576+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"206'6299","reported_seq":6337,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T18:45:03.212183+0000","last_change":"2023-09-23T18:44:54.142282+0000","last_active":"2023-09-23T18:45:03.212183+0000","last_peered":"2023-09-23T18:45:03.212183+0000","last_clean":"2023-09-23T18:45:03.212183+0000","last_became_active":"2023-09-23T18:43:59.508442+0000","last_became_peered":"2023-09-23T18:43:59.508442+0000","last_unstale":"2023-09-23T18:45:03.212183+0000","last_undegraded":"2023-09-23T18:45:03.212183+0000","last_fullsized":"2023-09-23T18:45:03.212183+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":195,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6299,"log_dups_size":0,"ondisk_log_size":6299,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147718144,"num_objects":2254,"num_object_clones":0,"num_object_copies":6762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3966,"num_write_kb":199040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":129,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.311497+0000","last_change":"2023-09-23T18:33:54.858971+0000","last_active":"2023-09-23T18:44:18.311497+0000","last_peered":"2023-09-23T18:44:18.311497+0000","last_clean":"2023-09-23T18:44:18.311497+0000","last_became_active":"2023-09-23T18:33:54.845921+0000","last_became_peered":"2023-09-23T18:33:54.845921+0000","last_unstale":"2023-09-23T18:44:18.311497+0000","last_undegraded":"2023-09-23T18:44:18.311497+0000","last_fullsized":"2023-09-23T18:44:18.311497+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":186,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.310900+0000","last_change":"2023-09-23T18:28:03.302261+0000","last_active":"2023-09-23T18:44:18.310900+0000","last_peered":"2023-09-23T18:44:18.310900+0000","last_clean":"2023-09-23T18:44:18.310900+0000","last_became_active":"2023-09-23T18:28:03.301911+0000","last_became_peered":"2023-09-23T18:28:03.301911+0000","last_unstale":"2023-09-23T18:44:18.310900+0000","last_undegraded":"2023-09-23T18:44:18.310900+0000","last_fullsized":"2023-09-23T18:44:18.310900+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":54,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.070117+0000","last_change":"2023-09-23T18:40:42.049209+0000","last_active":"2023-09-23T18:44:18.070117+0000","last_peered":"2023-09-23T18:44:18.070117+0000","last_clean":"2023-09-23T18:44:18.070117+0000","last_became_active":"2023-09-23T18:40:42.048654+0000","last_became_peered":"2023-09-23T18:40:42.048654+0000","last_unstale":"2023-09-23T18:44:18.070117+0000","last_undegraded":"2023-09-23T18:44:18.070117+0000","last_fullsized":"2023-09-23T18:44:18.070117+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":128,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.075154+0000","last_change":"2023-09-23T18:40:42.051196+0000","last_active":"2023-09-23T18:44:18.075154+0000","last_peered":"2023-09-23T18:44:18.075154+0000","last_clean":"2023-09-23T18:44:18.075154+0000","last_became_active":"2023-09-23T18:40:42.050613+0000","last_became_peered":"2023-09-23T18:40:42.050613+0000","last_unstale":"2023-09-23T18:44:18.075154+0000","last_undegraded":"2023-09-23T18:44:18.075154+0000","last_fullsized":"2023-09-23T18:44:18.075154+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:21.750773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":195,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.750106+0000","last_change":"2023-09-23T18:39:47.006553+0000","last_active":"2023-09-23T18:44:18.750106+0000","last_peered":"2023-09-23T18:44:18.750106+0000","last_clean":"2023-09-23T18:44:18.750106+0000","last_became_active":"2023-09-23T18:39:46.996464+0000","last_became_peered":"2023-09-23T18:39:46.996464+0000","last_unstale":"2023-09-23T18:44:18.750106+0000","last_undegraded":"2023-09-23T18:44:18.750106+0000","last_fullsized":"2023-09-23T18:44:18.750106+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:02.274848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":185,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T18:44:18.311229+0000","last_change":"2023-09-23T18:33:54.847156+0000","last_active":"2023-09-23T18:44:18.311229+0000","last_peered":"2023-09-23T18:44:18.311229+0000","last_clean":"2023-09-23T18:44:18.311229+0000","last_became_active":"2023-09-23T18:33:54.846765+0000","last_became_peered":"2023-09-23T18:33:54.846765+0000","last_unstale":"2023-09-23T18:44:18.311229+0000","last_undegraded":"2023-09-23T18:44:18.311229+0000","last_fullsized":"2023-09-23T18:44:18.311229+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":4738711576,"num_objects":72308,"num_object_clones":0,"num_object_copies":216924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5581,"num_objects_misplaced":5907,"num_objects_unfound":0,"num_objects_dirty":72308,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":129578,"num_write_kb":6460289,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2407,"num_bytes_recovered":157745152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14650978304,"data_stored":14650966088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":168173,"ondisk_log_size":168173,"up":78,"acting":75,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":1200,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26886,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5937,"num_read_kb":13190,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5621,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430607,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1921164,"kb_used_data":1611652,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91823988,"statfs":{"total":95995035648,"available":94027763712,"internally_reserved":0,"allocated":1650331648,"data_stored":1649751798,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:44:13 2023","interfaces":[{"interface":"back","average":{"1min":8.3620000000000001,"5min":4.1109999999999998,"15min":2.5419999999999998},"min":{"1min":0.32400000000000001,"5min":0.30199999999999999,"15min":0.252},"max":{"1min":97.415999999999997,"5min":97.415999999999997,"15min":97.415999999999997},"last":0.30099999999999999},{"interface":"front","average":{"1min":8.2579999999999991,"5min":4.032,"15min":2.5470000000000002},"min":{"1min":0.29299999999999998,"5min":0.28999999999999998,"15min":0.22900000000000001},"max":{"1min":96.581000000000003,"5min":96.581000000000003,"15min":96.581000000000003},"last":0.34200000000000003}]},{"osd":1,"last update":"Sat Sep 23 18:44:13 2023","interfaces":[{"interface":"back","average":{"1min":7.7220000000000004,"5min":3.2280000000000002,"15min":1.923},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":94.650999999999996,"5min":94.650999999999996,"15min":94.650999999999996},"last":4.1299999999999999},{"interface":"front","average":{"1min":8.8379999999999992,"5min":3.6800000000000002,"15min":2.1899999999999999},"min":{"1min":0.41899999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":94.843999999999994,"5min":94.843999999999994,"15min":94.843999999999994},"last":4.141}]},{"osd":2,"last update":"Sat Sep 23 18:45:05 2023","interfaces":[{"interface":"back","average":{"1min":2.157,"5min":1.407,"15min":1.617},"min":{"1min":0.373,"5min":0.34000000000000002,"15min":0.254},"max":{"1min":7.9880000000000004,"5min":8.7270000000000003,"15min":44.392000000000003},"last":6.8799999999999999},{"interface":"front","average":{"1min":1.7629999999999999,"5min":1.5600000000000001,"15min":1.633},"min":{"1min":0.40200000000000002,"5min":0.33600000000000002,"15min":0.29999999999999999},"max":{"1min":7.718,"5min":7.718,"15min":42.991999999999997},"last":2.492}]},{"osd":3,"last update":"Sat Sep 23 18:44:32 2023","interfaces":[{"interface":"back","average":{"1min":8.657,"5min":3.7559999999999998,"15min":2.1619999999999999},"min":{"1min":0.42199999999999999,"5min":0.36199999999999999,"15min":0.32800000000000001},"max":{"1min":94.572999999999993,"5min":94.572999999999993,"15min":94.572999999999993},"last":0.39200000000000002},{"interface":"front","average":{"1min":8.2159999999999993,"5min":4.2080000000000002,"15min":2.3359999999999999},"min":{"1min":0.35499999999999998,"5min":0.318,"15min":0.314},"max":{"1min":95.116,"5min":95.116,"15min":95.116},"last":0.63400000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:44:13 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":0.99099999999999999,"15min":1.044},"min":{"1min":0.34399999999999997,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":2.6709999999999998,"5min":5.1230000000000002,"15min":6.3529999999999998},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.016,"5min":1.1779999999999999,"15min":1.145},"min":{"1min":0.40500000000000003,"5min":0.38400000000000001,"15min":0.309},"max":{"1min":2.6850000000000001,"5min":7.6029999999999998,"15min":11.119},"last":0.61699999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:44:13 2023","interfaces":[{"interface":"back","average":{"1min":1.5369999999999999,"5min":1.351,"15min":1.145},"min":{"1min":0.42399999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.5329999999999995,"5min":8.7210000000000001,"15min":8.7210000000000001},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.069,"5min":1.02,"15min":1.0800000000000001},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.26800000000000002},"max":{"1min":3.4279999999999999,"5min":5.0129999999999999,"15min":7.46},"last":0.55200000000000005}]},{"osd":6,"last update":"Sat Sep 23 18:44:13 2023","interfaces":[{"interface":"back","average":{"1min":1.389,"5min":1.29,"15min":1.145},"min":{"1min":0.45700000000000002,"5min":0.41199999999999998,"15min":0.35299999999999998},"max":{"1min":5.915,"5min":5.915,"15min":5.915},"last":0.60399999999999998},{"interface":"front","average":{"1min":1.2290000000000001,"5min":1.306,"15min":1.1579999999999999},"min":{"1min":0.48599999999999999,"5min":0.41099999999999998,"15min":0.36499999999999999},"max":{"1min":3.3149999999999999,"5min":9.2850000000000001,"15min":9.2850000000000001},"last":0.65900000000000003}]}]},{"osd":6,"up_from":42,"seq":180388626839,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2085896,"kb_used_data":1784512,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91659256,"statfs":{"total":95995035648,"available":93859078144,"internally_reserved":0,"allocated":1827340288,"data_stored":1826764670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:44:08 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":1.1220000000000001,"15min":1.9750000000000001},"min":{"1min":0.33200000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.4980000000000002,"5min":6.758,"15min":182.655},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.98899999999999999,"5min":1.2230000000000001,"15min":2.0750000000000002},"min":{"1min":0.42499999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.5419999999999998,"5min":6.7930000000000001,"15min":182.58799999999999},"last":0.45300000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:44:08 2023","interfaces":[{"interface":"back","average":{"1min":5.9349999999999996,"5min":2.1680000000000001,"15min":1.3720000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":89.727000000000004,"5min":89.727000000000004,"15min":89.727000000000004},"last":2.5750000000000002},{"interface":"front","average":{"1min":5.2850000000000001,"5min":2.532,"15min":1.6180000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":89.584000000000003,"5min":89.584000000000003,"15min":89.584000000000003},"last":2.4260000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:44:08 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.7130000000000001,"15min":2.3220000000000001},"min":{"1min":0.40500000000000003,"5min":0.33500000000000002,"15min":0.28100000000000003},"max":{"1min":3.7970000000000002,"5min":10.99,"15min":122.242},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.085,"5min":1.8109999999999999,"15min":2.359},"min":{"1min":0.42499999999999999,"5min":0.34899999999999998,"15min":0.28799999999999998},"max":{"1min":3.2879999999999998,"5min":13.959,"15min":122.32899999999999},"last":2.3929999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:44:30 2023","interfaces":[{"interface":"back","average":{"1min":5.609,"5min":3.161,"15min":2.5249999999999999},"min":{"1min":0.433,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":89.245999999999995,"5min":89.897999999999996,"15min":123.52200000000001},"last":0.48699999999999999},{"interface":"front","average":{"1min":5.9320000000000004,"5min":3.6030000000000002,"15min":2.7250000000000001},"min":{"1min":0.40699999999999997,"5min":0.39800000000000002,"15min":0.36099999999999999},"max":{"1min":89.274000000000001,"5min":89.933999999999997,"15min":123.605},"last":2.4649999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":1.4470000000000001,"15min":1.133},"min":{"1min":0.437,"5min":0.32000000000000001,"15min":0.29999999999999999},"max":{"1min":3.6179999999999999,"5min":15.496,"15min":15.496},"last":0.44},{"interface":"front","average":{"1min":0.91200000000000003,"5min":1.306,"15min":1.0700000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.2930000000000001,"5min":14.875999999999999,"15min":14.875999999999999},"last":0.64300000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.153,"15min":1.026},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.32000000000000001},"max":{"1min":3.3090000000000002,"5min":7.4930000000000003,"15min":7.9240000000000004},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.85199999999999998,"5min":0.90400000000000003,"15min":0.91500000000000004},"min":{"1min":0.375,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.3,"5min":3.8220000000000001,"15min":4.5270000000000001},"last":0.73699999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:44:08 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":1.413,"15min":1.133},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":2.6110000000000002,"5min":7.9320000000000004,"15min":7.9320000000000004},"last":2.3580000000000001},{"interface":"front","average":{"1min":0.91000000000000003,"5min":1.6100000000000001,"15min":1.3959999999999999},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39100000000000001},"max":{"1min":2.6829999999999998,"5min":7.8209999999999997,"15min":9.1129999999999995},"last":0.56599999999999995}]}]},{"osd":1,"up_from":123,"seq":528280977544,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3189376,"kb_used_data":2806456,"kb_used_omap":7,"kb_used_meta":382776,"kb_avail":90555776,"statfs":{"total":95995035648,"available":92729114624,"internally_reserved":0,"allocated":2873810944,"data_stored":2872837583,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":391963406},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:44:10 2023","interfaces":[{"interface":"back","average":{"1min":4.9269999999999996,"5min":2.339,"15min":5.5789999999999997},"min":{"1min":0.375,"5min":0.186,"15min":0.186},"max":{"1min":53.530999999999999,"5min":53.530999999999999,"15min":173.41800000000001},"last":0.76700000000000002},{"interface":"front","average":{"1min":5.9909999999999997,"5min":2.742,"15min":5.4800000000000004},"min":{"1min":0.35999999999999999,"5min":0.249,"15min":0.17599999999999999},"max":{"1min":51.710999999999999,"5min":51.710999999999999,"15min":166.42599999999999},"last":1.671}]},{"osd":2,"last update":"Sat Sep 23 18:44:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8180000000000001,"5min":2.1640000000000001,"15min":2.952},"min":{"1min":0.224,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":6.3559999999999999,"5min":14.779,"15min":129.25899999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":3.0590000000000002,"5min":2.3820000000000001,"15min":2.9569999999999999},"min":{"1min":0.35599999999999998,"5min":0.33700000000000002,"15min":0.247},"max":{"1min":7.6790000000000003,"5min":16.032,"15min":129.35300000000001},"last":1.3360000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:44:10 2023","interfaces":[{"interface":"back","average":{"1min":2.7530000000000001,"5min":2.4100000000000001,"15min":2.0059999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.3129999999999997,"5min":16.123000000000001,"15min":55.014000000000003},"last":1.7390000000000001},{"interface":"front","average":{"1min":1.464,"5min":2.1629999999999998,"15min":1.9690000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.24399999999999999},"max":{"1min":5.681,"5min":15.717000000000001,"15min":55.841999999999999},"last":0.54000000000000004}]},{"osd":4,"last update":"Sat Sep 23 18:44:10 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":1.7470000000000001,"15min":1.4470000000000001},"min":{"1min":0.37,"5min":0.308,"15min":0.308},"max":{"1min":3.081,"5min":10.637,"15min":11.763},"last":0.56999999999999995},{"interface":"front","average":{"1min":1.171,"5min":1.444,"15min":1.4510000000000001},"min":{"1min":0.313,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.2229999999999999,"5min":8.2270000000000003,"15min":11.491},"last":0.92800000000000005}]},{"osd":5,"last update":"Sat Sep 23 18:44:10 2023","interfaces":[{"interface":"back","average":{"1min":2.5529999999999999,"5min":1.698,"15min":1.2749999999999999},"min":{"1min":0.41699999999999998,"5min":0.36699999999999999,"15min":0.35699999999999998},"max":{"1min":8.0299999999999994,"5min":9.2690000000000001,"15min":11.794},"last":1.583},{"interface":"front","average":{"1min":1.2629999999999999,"5min":1.6699999999999999,"15min":1.512},"min":{"1min":0.39300000000000002,"5min":0.371,"15min":0.371},"max":{"1min":3.157,"5min":10.692,"15min":11.733000000000001},"last":0.72699999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:44:10 2023","interfaces":[{"interface":"back","average":{"1min":1.488,"5min":1.8180000000000001,"15min":1.6479999999999999},"min":{"1min":0.34100000000000003,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.8419999999999996,"5min":11.041,"15min":11.776999999999999},"last":1.2929999999999999},{"interface":"front","average":{"1min":1.635,"5min":1.885,"15min":1.7270000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.4619999999999997,"5min":16.687000000000001,"15min":16.687000000000001},"last":1.014}]},{"osd":7,"last update":"Sat Sep 23 18:44:10 2023","interfaces":[{"interface":"back","average":{"1min":1.6759999999999999,"5min":2.0649999999999999,"15min":1.883},"min":{"1min":0.39300000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.8259999999999996,"5min":10.667,"15min":11.747999999999999},"last":1.179},{"interface":"front","average":{"1min":1.554,"5min":1.605,"15min":1.7070000000000001},"min":{"1min":0.40899999999999997,"5min":0.379,"15min":0.379},"max":{"1min":4.8890000000000002,"5min":8.4770000000000003,"15min":11.593},"last":1.141}]}]},{"osd":0,"up_from":88,"seq":377957122272,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2945416,"kb_used_data":2433856,"kb_used_omap":8,"kb_used_meta":511351,"kb_avail":90799736,"statfs":{"total":95995035648,"available":92978929664,"internally_reserved":0,"allocated":2492268544,"data_stored":2491345418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":523624318},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:44:18 2023","interfaces":[{"interface":"back","average":{"1min":2.4649999999999999,"5min":2.4100000000000001,"15min":1.972},"min":{"1min":0.191,"5min":0.191,"15min":0.17100000000000001},"max":{"1min":9.7609999999999992,"5min":38.545999999999999,"15min":38.545999999999999},"last":0.58499999999999996},{"interface":"front","average":{"1min":2.3530000000000002,"5min":2.508,"15min":2.2469999999999999},"min":{"1min":0.247,"5min":0.214,"15min":0.20100000000000001},"max":{"1min":7.1719999999999997,"5min":45.904000000000003,"15min":45.904000000000003},"last":2.488}]},{"osd":2,"last update":"Sat Sep 23 18:45:00 2023","interfaces":[{"interface":"back","average":{"1min":2.98,"5min":1.8260000000000001,"15min":2.3029999999999999},"min":{"1min":0.38600000000000001,"5min":0.24399999999999999,"15min":0.192},"max":{"1min":12.622999999999999,"5min":12.622999999999999,"15min":108.73999999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":2.403,"5min":1.381,"15min":2.0430000000000001},"min":{"1min":0.23899999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":12.297000000000001,"5min":12.297000000000001,"15min":108.136},"last":2.448}]},{"osd":3,"last update":"Sat Sep 23 18:44:26 2023","interfaces":[{"interface":"back","average":{"1min":1.7,"5min":3.7770000000000001,"15min":3.2869999999999999},"min":{"1min":0.32100000000000001,"5min":0.25900000000000001,"15min":0.216},"max":{"1min":9.7390000000000008,"5min":177.83600000000001,"15min":177.83600000000001},"last":0.47599999999999998},{"interface":"front","average":{"1min":2.177,"5min":3.835,"15min":3.3050000000000002},"min":{"1min":0.32700000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":11.731999999999999,"5min":181.71899999999999,"15min":181.71899999999999},"last":2.4089999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:44:18 2023","interfaces":[{"interface":"back","average":{"1min":1.4930000000000001,"5min":1.5229999999999999,"15min":1.351},"min":{"1min":0.31,"5min":0.31,"15min":0.28199999999999997},"max":{"1min":5.6680000000000001,"5min":9.9450000000000003,"15min":9.9450000000000003},"last":0.60799999999999998},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.365,"15min":1.4830000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.0190000000000001,"5min":6.6150000000000002,"15min":16.486999999999998},"last":0.51800000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:44:18 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.212,"15min":1.274},"min":{"1min":0.371,"5min":0.371,"15min":0.36499999999999999},"max":{"1min":4.0540000000000003,"5min":5.3209999999999997,"15min":10.606},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.016,"5min":1.1559999999999999,"15min":1.2509999999999999},"min":{"1min":0.34200000000000003,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.8900000000000001,"5min":8.2279999999999998,"15min":8.9250000000000007},"last":2.5430000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:44:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1719999999999999,"5min":1.5209999999999999,"15min":1.4330000000000001},"min":{"1min":0.376,"5min":0.376,"15min":0.35899999999999999},"max":{"1min":4.0720000000000001,"5min":11.302,"15min":11.302},"last":0.68300000000000005},{"interface":"front","average":{"1min":1.05,"5min":1.3480000000000001,"15min":1.341},"min":{"1min":0.372,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":2.9140000000000001,"5min":11.818,"15min":11.818},"last":2.5790000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:44:18 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":1.365,"15min":1.413},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39200000000000002},"max":{"1min":2.867,"5min":8.327,"15min":9.0600000000000005},"last":2.6110000000000002},{"interface":"front","average":{"1min":1.54,"5min":1.732,"15min":1.4359999999999999},"min":{"1min":0.442,"5min":0.42599999999999999,"15min":0.35799999999999998},"max":{"1min":5.6980000000000004,"5min":10.01,"15min":10.01},"last":0.63500000000000001}]}]},{"osd":2,"up_from":98,"seq":420906795220,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2690000,"kb_used_data":2190280,"kb_used_omap":11,"kb_used_meta":499636,"kb_avail":91055152,"statfs":{"total":95995035648,"available":93240475648,"internally_reserved":0,"allocated":2242846720,"data_stored":2241974218,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11949,"internal_metadata":511627603},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:44:05 2023","interfaces":[{"interface":"back","average":{"1min":1.2989999999999999,"5min":1.246,"15min":1.29},"min":{"1min":0.185,"5min":0.185,"15min":0.184},"max":{"1min":6.2169999999999996,"5min":21.498000000000001,"15min":21.498000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":2.2389999999999999,"5min":1.6870000000000001,"15min":1.4359999999999999},"min":{"1min":0.219,"5min":0.20300000000000001,"15min":0.17699999999999999},"max":{"1min":10.621,"5min":21.484999999999999,"15min":21.484999999999999},"last":0.496}]},{"osd":1,"last update":"Sat Sep 23 18:44:12 2023","interfaces":[{"interface":"back","average":{"1min":1.569,"5min":2.9220000000000002,"15min":3.2029999999999998},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":5.71,"5min":94.210999999999999,"15min":184.56299999999999},"last":1.177},{"interface":"front","average":{"1min":2.927,"5min":3.2759999999999998,"15min":3.3450000000000002},"min":{"1min":0.29699999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":11.638,"5min":90.451999999999998,"15min":183.922},"last":0.98299999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:44:37 2023","interfaces":[{"interface":"back","average":{"1min":1.478,"5min":4.7679999999999998,"15min":2.8410000000000002},"min":{"1min":0.24399999999999999,"5min":0.183,"15min":0.183},"max":{"1min":4.5499999999999998,"5min":161.773,"15min":161.773},"last":0.28000000000000003},{"interface":"front","average":{"1min":1.4379999999999999,"5min":4.4400000000000004,"15min":2.5150000000000001},"min":{"1min":0.29299999999999998,"5min":0.25,"15min":0.249},"max":{"1min":5.9489999999999998,"5min":158.03700000000001,"15min":158.03700000000001},"last":0.70199999999999996}]},{"osd":4,"last update":"Sat Sep 23 18:44:05 2023","interfaces":[{"interface":"back","average":{"1min":1.2529999999999999,"5min":1.645,"15min":1.7350000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.30099999999999999},"max":{"1min":5.2910000000000004,"5min":8.6690000000000005,"15min":71.587999999999994},"last":0.36799999999999999},{"interface":"front","average":{"1min":1.2310000000000001,"5min":1.1879999999999999,"15min":1.615},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33500000000000002},"max":{"1min":5.8079999999999998,"5min":8.5860000000000003,"15min":71.328999999999994},"last":0.54700000000000004}]},{"osd":5,"last update":"Sat Sep 23 18:44:05 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":1.0389999999999999,"15min":1.421},"min":{"1min":0.433,"5min":0.36799999999999999,"15min":0.35999999999999999},"max":{"1min":5.8490000000000002,"5min":5.8490000000000002,"15min":71.302000000000007},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.659,"5min":1.3859999999999999,"15min":1.6559999999999999},"min":{"1min":0.42599999999999999,"5min":0.35999999999999999,"15min":0.316},"max":{"1min":5.1369999999999996,"5min":7.9889999999999999,"15min":71.543999999999997},"last":0.46600000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:44:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.7749999999999999,"15min":1.7909999999999999},"min":{"1min":0.379,"5min":0.379,"15min":0.35199999999999998},"max":{"1min":2.9460000000000002,"5min":16.071000000000002,"15min":71.460999999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.2290000000000001,"5min":1.3069999999999999,"15min":1.5589999999999999},"min":{"1min":0.46200000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":5.899,"5min":8.6259999999999994,"15min":71.271000000000001},"last":0.56999999999999995}]},{"osd":7,"last update":"Sat Sep 23 18:44:05 2023","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":1.7030000000000001,"15min":1.8009999999999999},"min":{"1min":0.42099999999999999,"5min":0.38100000000000001,"15min":0.34799999999999998},"max":{"1min":3.2189999999999999,"5min":16.024000000000001,"15min":71.622},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.1539999999999999,"5min":1.3089999999999999,"15min":1.764},"min":{"1min":0.499,"5min":0.40100000000000002,"15min":0.39900000000000002},"max":{"1min":5.8739999999999997,"5min":8.8510000000000009,"15min":71.575000000000003},"last":0.59399999999999997}]}]},{"osd":3,"up_from":103,"seq":442381631695,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2241948,"kb_used_data":1829780,"kb_used_omap":4,"kb_used_meta":412155,"kb_avail":91503204,"statfs":{"total":95995035648,"available":93699280896,"internally_reserved":0,"allocated":1873694720,"data_stored":1872810048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4826,"internal_metadata":422047014},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:44:34 2023","interfaces":[{"interface":"back","average":{"1min":7.4900000000000002,"5min":2.7120000000000002,"15min":1.875},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.193},"max":{"1min":129.315,"5min":129.315,"15min":129.315},"last":0.39500000000000002},{"interface":"front","average":{"1min":7.9400000000000004,"5min":3.0019999999999998,"15min":1.804},"min":{"1min":0.219,"5min":0.219,"15min":0.20300000000000001},"max":{"1min":130.09200000000001,"5min":130.09200000000001,"15min":130.09200000000001},"last":1.8200000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:44:16 2023","interfaces":[{"interface":"back","average":{"1min":2.6800000000000002,"5min":1.6679999999999999,"15min":1.4059999999999999},"min":{"1min":0.27100000000000002,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":12.856999999999999,"5min":12.856999999999999,"15min":12.968999999999999},"last":3.2959999999999998},{"interface":"front","average":{"1min":2.351,"5min":1.6699999999999999,"15min":1.5609999999999999},"min":{"1min":0.23899999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":6.1509999999999998,"5min":11.087999999999999,"15min":11.74},"last":3.2650000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:44:34 2023","interfaces":[{"interface":"back","average":{"1min":1.7250000000000001,"5min":2.0569999999999999,"15min":2.7250000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.22700000000000001},"max":{"1min":4.9260000000000002,"5min":18.888000000000002,"15min":187.40000000000001},"last":2.2200000000000002},{"interface":"front","average":{"1min":2.3330000000000002,"5min":1.8540000000000001,"15min":2.2610000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.22500000000000001},"max":{"1min":12.898999999999999,"5min":12.898999999999999,"15min":187.71700000000001},"last":3.3300000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:44:34 2023","interfaces":[{"interface":"back","average":{"1min":1.6000000000000001,"5min":1.544,"15min":1.2210000000000001},"min":{"1min":0.34100000000000003,"5min":0.312,"15min":0.30599999999999999},"max":{"1min":9.9960000000000004,"5min":9.9960000000000004,"15min":9.9960000000000004},"last":3.3580000000000001},{"interface":"front","average":{"1min":1.355,"5min":1.772,"15min":1.3360000000000001},"min":{"1min":0.45200000000000001,"5min":0.36199999999999999,"15min":0.28399999999999997},"max":{"1min":7.2060000000000004,"5min":11.916,"15min":11.916},"last":0.51700000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:44:34 2023","interfaces":[{"interface":"back","average":{"1min":1.5,"5min":1.345,"15min":1.2070000000000001},"min":{"1min":0.48599999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":5.7939999999999996,"5min":7.8799999999999999,"15min":7.8799999999999999},"last":0.54500000000000004},{"interface":"front","average":{"1min":1.5740000000000001,"5min":1.782,"15min":1.3899999999999999},"min":{"1min":0.436,"5min":0.39100000000000001,"15min":0.36099999999999999},"max":{"1min":4.7750000000000004,"5min":10.18,"15min":10.18},"last":1.9590000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:44:34 2023","interfaces":[{"interface":"back","average":{"1min":1.5740000000000001,"5min":2.2829999999999999,"15min":1.6080000000000001},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.35599999999999998},"max":{"1min":4.6680000000000001,"5min":11.529,"15min":11.529},"last":2.1200000000000001},{"interface":"front","average":{"1min":1.7250000000000001,"5min":1.8140000000000001,"15min":1.349},"min":{"1min":0.375,"5min":0.375,"15min":0.33300000000000002},"max":{"1min":10.092000000000001,"5min":10.092000000000001,"15min":11.989000000000001},"last":3.8639999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:44:34 2023","interfaces":[{"interface":"back","average":{"1min":1.6319999999999999,"5min":1.8009999999999999,"15min":1.3839999999999999},"min":{"1min":0.40400000000000003,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":10.052,"5min":10.052,"15min":10.052},"last":3.3820000000000001},{"interface":"front","average":{"1min":1.4299999999999999,"5min":1.421,"15min":1.3260000000000001},"min":{"1min":0.439,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":5.7549999999999999,"5min":9.032,"15min":9.032},"last":0.48399999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019305,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2356932,"kb_used_data":1767228,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":91388220,"statfs":{"total":95995035648,"available":93581537280,"internally_reserved":0,"allocated":1809641472,"data_stored":1809069732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:45:02 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.284,"15min":1.782},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.253},"max":{"1min":5.8760000000000003,"5min":7.1139999999999999,"15min":82.623000000000005},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.4450000000000001,"5min":1.169,"15min":1.645},"min":{"1min":0.34399999999999997,"5min":0.313,"15min":0.23799999999999999},"max":{"1min":9.5039999999999996,"5min":9.5039999999999996,"15min":82.638000000000005},"last":1.3799999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:44:06 2023","interfaces":[{"interface":"back","average":{"1min":1.9490000000000001,"5min":1.714,"15min":1.571},"min":{"1min":0.48799999999999999,"5min":0.33000000000000002,"15min":0.29299999999999998},"max":{"1min":7.0060000000000002,"5min":9.9649999999999999,"15min":53.457000000000001},"last":1.2949999999999999},{"interface":"front","average":{"1min":2.1800000000000002,"5min":1.726,"15min":1.742},"min":{"1min":0.42899999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.0199999999999996,"5min":10.523,"15min":53.395000000000003},"last":1.278}]},{"osd":2,"last update":"Sat Sep 23 18:45:02 2023","interfaces":[{"interface":"back","average":{"1min":2.1190000000000002,"5min":1.4910000000000001,"15min":1.7090000000000001},"min":{"1min":0.40500000000000003,"5min":0.312,"15min":0.28799999999999998},"max":{"1min":6.2709999999999999,"5min":6.2709999999999999,"15min":53.249000000000002},"last":0.67100000000000004},{"interface":"front","average":{"1min":2.1520000000000001,"5min":1.7749999999999999,"15min":1.9530000000000001},"min":{"1min":0.45400000000000001,"5min":0.34499999999999997,"15min":0.309},"max":{"1min":6.2320000000000002,"5min":13.188000000000001,"15min":53.220999999999997},"last":0.54100000000000004}]},{"osd":3,"last update":"Sat Sep 23 18:44:40 2023","interfaces":[{"interface":"back","average":{"1min":8.2899999999999991,"5min":3.194,"15min":2.4660000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.32900000000000001},"max":{"1min":136.548,"5min":136.548,"15min":136.548},"last":0.40600000000000003},{"interface":"front","average":{"1min":8.3439999999999994,"5min":2.8799999999999999,"15min":2.427},"min":{"1min":0.38800000000000001,"5min":0.36099999999999999,"15min":0.34000000000000002},"max":{"1min":136.494,"5min":136.494,"15min":136.494},"last":0.502}]},{"osd":5,"last update":"Sat Sep 23 18:44:58 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":1.113,"15min":1.127},"min":{"1min":0.34000000000000002,"5min":0.307,"15min":0.29899999999999999},"max":{"1min":3.839,"5min":6.9249999999999998,"15min":7.819},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.67300000000000004,"5min":1.1060000000000001,"15min":1.101},"min":{"1min":0.39200000000000002,"5min":0.36099999999999999,"15min":0.28399999999999997},"max":{"1min":1.2,"5min":7.1440000000000001,"15min":7.1440000000000001},"last":0.83099999999999996}]},{"osd":6,"last update":"Sat Sep 23 18:44:22 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":1.2270000000000001,"15min":1.1839999999999999},"min":{"1min":0.36599999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.1699999999999999,"5min":6.9630000000000001,"15min":7.8419999999999996},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.873,"5min":1.141,"15min":1.0569999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26100000000000001},"max":{"1min":2.3610000000000002,"5min":6.8399999999999999,"15min":6.8399999999999999},"last":0.73399999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:45:02 2023","interfaces":[{"interface":"back","average":{"1min":0.91800000000000004,"5min":1.7669999999999999,"15min":1.514},"min":{"1min":0.40799999999999997,"5min":0.32500000000000001,"15min":0.311},"max":{"1min":2.6179999999999999,"5min":12.294,"15min":13.491},"last":0.80600000000000005},{"interface":"front","average":{"1min":1.294,"5min":1.968,"15min":1.6899999999999999},"min":{"1min":0.47599999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":3.9239999999999999,"5min":10.507,"15min":17.062999999999999},"last":0.65400000000000003}]}]},{"osd":5,"up_from":0,"seq":154618823073,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4224,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7725,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3429,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2463109120,"data_stored":2463105048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2870874112,"data_stored":2870870040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2231566336,"data_stored":2231566336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1849098240,"data_stored":1849098240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1792081920,"data_stored":1792081920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1808076800,"data_stored":1808072728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1636171776,"data_stored":1636171776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:45:15.339 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T18:45:15.339 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 41 2023-09-23T18:45:15.340 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:15.367 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:15.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:15.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:15.470 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:15.498 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:15.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:15.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:15.605 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:15.630 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:15.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:15.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:15.736 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:15.764 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:15.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:15.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:15.868 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:15.894 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:15.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:15.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:15.997 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:16.025 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:16.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:16.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:16.133 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:16.146 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:16 smithi173 bash[59422]: cluster 2023-09-23T18:45:14.366278+0000 osd.4 (osd.4) 180 : cluster 0 7.1 scrub starts 2023-09-23T18:45:16.147 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:16 smithi173 bash[59422]: cluster 2023-09-23T18:45:14.822586+0000 osd.2 (osd.2) 2 : cluster 0 7.18 scrub starts 2023-09-23T18:45:16.147 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:16 smithi173 bash[59422]: cluster 2023-09-23T18:45:15.096900+0000 osd.0 (osd.0) 22 : cluster 0 7.19 scrub starts 2023-09-23T18:45:16.168 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:16.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:16.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:16.274 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:16.312 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:16.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:16.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:16.419 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:16.428 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:16 smithi098 bash[76817]: cluster 2023-09-23T18:45:14.366278+0000 osd.4 (osd.4) 180 : cluster 0 7.1 scrub starts 2023-09-23T18:45:16.428 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:16 smithi098 bash[76817]: cluster 2023-09-23T18:45:14.822586+0000 osd.2 (osd.2) 2 : cluster 0 7.18 scrub starts 2023-09-23T18:45:16.428 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:16 smithi098 bash[76817]: cluster 2023-09-23T18:45:15.096900+0000 osd.0 (osd.0) 22 : cluster 0 7.19 scrub starts 2023-09-23T18:45:16.428 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:45:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:45:16] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.33.4" 2023-09-23T18:45:16.428 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:16 smithi098 bash[79516]: cluster 2023-09-23T18:45:14.366278+0000 osd.4 (osd.4) 180 : cluster 0 7.1 scrub starts 2023-09-23T18:45:16.429 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:16 smithi098 bash[79516]: cluster 2023-09-23T18:45:14.822586+0000 osd.2 (osd.2) 2 : cluster 0 7.18 scrub starts 2023-09-23T18:45:16.429 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:16 smithi098 bash[79516]: cluster 2023-09-23T18:45:15.096900+0000 osd.0 (osd.0) 22 : cluster 0 7.19 scrub starts 2023-09-23T18:45:16.447 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:16.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:16.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:16.552 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:16.582 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:16.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:16.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:16.686 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:16.735 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:16.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:16.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:16.839 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:16.865 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:16.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:16.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:16.969 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:16.999 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:17.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:17.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:17.102 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:17.114 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:17 smithi173 bash[59422]: cluster 2023-09-23T18:45:16.005595+0000 mgr.y (mgr.26054) 812 : cluster 0 pgmap v835: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 771 op/s; 3854/198498 objects degraded (1.942%); 7170/198498 objects misplaced (3.612%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T18:45:17.134 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:17.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:17.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:17.237 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:17.264 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:17.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:17.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:17.367 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:17.375 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:17 smithi098 bash[79516]: cluster 2023-09-23T18:45:16.005595+0000 mgr.y (mgr.26054) 812 : cluster 0 pgmap v835: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 771 op/s; 3854/198498 objects degraded (1.942%); 7170/198498 objects misplaced (3.612%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T18:45:17.375 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:17 smithi098 bash[76817]: cluster 2023-09-23T18:45:16.005595+0000 mgr.y (mgr.26054) 812 : cluster 0 pgmap v835: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 771 op/s; 3854/198498 objects degraded (1.942%); 7170/198498 objects misplaced (3.612%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T18:45:17.401 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:17.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:17.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:17.505 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:17.529 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:17.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:17.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:17.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:17.654 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:17.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:17.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:17.759 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:17.785 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:17.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:17.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:17.888 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:17.915 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:17.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:17.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:18.023 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:18.049 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:18.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:18.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:18.155 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:18.164 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:18 smithi098 bash[76817]: cluster 2023-09-23T18:45:17.135548+0000 osd.0 (osd.0) 23 : cluster 0 7.14 scrub starts 2023-09-23T18:45:18.164 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:18 smithi098 bash[76817]: cluster 2023-09-23T18:45:18.006835+0000 mgr.y (mgr.26054) 813 : cluster 0 pgmap v836: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 732 op/s; 3854/194766 objects degraded (1.979%); 6906/194766 objects misplaced (3.546%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T18:45:18.165 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:18 smithi098 bash[79516]: cluster 2023-09-23T18:45:17.135548+0000 osd.0 (osd.0) 23 : cluster 0 7.14 scrub starts 2023-09-23T18:45:18.165 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:18 smithi098 bash[79516]: cluster 2023-09-23T18:45:18.006835+0000 mgr.y (mgr.26054) 813 : cluster 0 pgmap v836: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 732 op/s; 3854/194766 objects degraded (1.979%); 6906/194766 objects misplaced (3.546%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T18:45:18.187 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:18.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:18.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:18.290 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:18.325 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:18.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:18.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:18.429 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:18.440 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:18 smithi173 bash[59422]: cluster 2023-09-23T18:45:17.135548+0000 osd.0 (osd.0) 23 : cluster 0 7.14 scrub starts 2023-09-23T18:45:18.440 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:18 smithi173 bash[59422]: cluster 2023-09-23T18:45:18.006835+0000 mgr.y (mgr.26054) 813 : cluster 0 pgmap v836: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 732 op/s; 3854/194766 objects degraded (1.979%); 6906/194766 objects misplaced (3.546%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T18:45:18.454 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:18.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:18.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:18.558 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:18.599 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:18.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:18.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:18.639 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:45:18.664 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:45:18.702 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:18.728 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:18.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:18.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:18.739 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:45:18.831 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:18.858 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:18.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:18.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:18.961 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:19.009 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:19.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:19.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:19.111 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:19.145 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:19.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:19.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:19.248 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:19.257 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:19 smithi098 bash[76817]: cluster 2023-09-23T18:45:18.447712+0000 osd.4 (osd.4) 181 : cluster 0 7.1 scrub starts 2023-09-23T18:45:19.258 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:19 smithi098 bash[79516]: cluster 2023-09-23T18:45:18.447712+0000 osd.4 (osd.4) 181 : cluster 0 7.1 scrub starts 2023-09-23T18:45:19.292 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:19.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:19.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:19.396 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:19.425 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:19.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:19.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:19.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:19 smithi173 bash[59422]: cluster 2023-09-23T18:45:18.447712+0000 osd.4 (osd.4) 181 : cluster 0 7.1 scrub starts 2023-09-23T18:45:19.529 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:19.553 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:19.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:19.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:19.656 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:19.694 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:19.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:19.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:19.758 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:45:19.804 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:19.837 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:19.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:19.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:19.940 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:19.977 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:19.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:19.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:20.080 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:20.103 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:20.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:20.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:20.207 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:20.237 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:20.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:20.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:20.341 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:20.373 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:20.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:20.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:20.476 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:20.485 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:20 smithi098 bash[76817]: cluster 2023-09-23T18:45:20.007668+0000 mgr.y (mgr.26054) 814 : cluster 0 pgmap v837: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 3854/190179 objects degraded (2.027%); 6906/190179 objects misplaced (3.631%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T18:45:20.485 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:20 smithi098 bash[76817]: cluster 2023-09-23T18:45:20.061110+0000 mon.a (mon.0) 1093 : cluster 3 Health check update: Degraded data redundancy: 3854/194766 objects degraded (1.979%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:45:20.486 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:20 smithi098 bash[79516]: cluster 2023-09-23T18:45:20.007668+0000 mgr.y (mgr.26054) 814 : cluster 0 pgmap v837: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 3854/190179 objects degraded (2.027%); 6906/190179 objects misplaced (3.631%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T18:45:20.486 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:20 smithi098 bash[79516]: cluster 2023-09-23T18:45:20.061110+0000 mon.a (mon.0) 1093 : cluster 3 Health check update: Degraded data redundancy: 3854/194766 objects degraded (1.979%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:45:20.499 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:20.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:20.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:20.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:20 smithi173 bash[59422]: cluster 2023-09-23T18:45:20.007668+0000 mgr.y (mgr.26054) 814 : cluster 0 pgmap v837: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 3854/190179 objects degraded (2.027%); 6906/190179 objects misplaced (3.631%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T18:45:20.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:20 smithi173 bash[59422]: cluster 2023-09-23T18:45:20.061110+0000 mon.a (mon.0) 1093 : cluster 3 Health check update: Degraded data redundancy: 3854/194766 objects degraded (1.979%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:45:20.603 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:20.628 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:20.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:20.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:20.731 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:20.757 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:20.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:20.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:20.863 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:20.896 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:20.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:20.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:21.001 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:21.023 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:21.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:21.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:21.126 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:21.152 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:21.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:21.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:21.257 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:21.266 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:21 smithi173 bash[59422]: cluster 2023-09-23T18:45:19.856667+0000 osd.2 (osd.2) 3 : cluster 0 7.10 scrub starts 2023-09-23T18:45:21.266 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:21 smithi173 bash[59422]: cluster 2023-09-23T18:45:20.131145+0000 osd.0 (osd.0) 24 : cluster 0 7.4 scrub starts 2023-09-23T18:45:21.282 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:21.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:21.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:21.387 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:21.396 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:21 smithi098 bash[76817]: cluster 2023-09-23T18:45:19.856667+0000 osd.2 (osd.2) 3 : cluster 0 7.10 scrub starts 2023-09-23T18:45:21.396 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:21 smithi098 bash[76817]: cluster 2023-09-23T18:45:20.131145+0000 osd.0 (osd.0) 24 : cluster 0 7.4 scrub starts 2023-09-23T18:45:21.396 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:21 smithi098 bash[79516]: cluster 2023-09-23T18:45:19.856667+0000 osd.2 (osd.2) 3 : cluster 0 7.10 scrub starts 2023-09-23T18:45:21.396 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:21 smithi098 bash[79516]: cluster 2023-09-23T18:45:20.131145+0000 osd.0 (osd.0) 24 : cluster 0 7.4 scrub starts 2023-09-23T18:45:21.419 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:21.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:21.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:21.523 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:21.555 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:21.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:21.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:21.658 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:21.689 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:21.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:21.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:21.792 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:21.823 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:21.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:21.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:21.930 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:21.956 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:21.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:21.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:22.059 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:22.084 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:22.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:22.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:22.189 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:22.223 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:22.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:22.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:22.330 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:22.341 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:22 smithi173 bash[59422]: cluster 2023-09-23T18:45:22.010257+0000 mgr.y (mgr.26054) 815 : cluster 0 pgmap v838: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 3854/185076 objects degraded (2.082%); 6906/185076 objects misplaced (3.731%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:45:22.359 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:22.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:22.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:22.465 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:22.473 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:22 smithi098 bash[79516]: cluster 2023-09-23T18:45:22.010257+0000 mgr.y (mgr.26054) 815 : cluster 0 pgmap v838: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 3854/185076 objects degraded (2.082%); 6906/185076 objects misplaced (3.731%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:45:22.474 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:22 smithi098 bash[76817]: cluster 2023-09-23T18:45:22.010257+0000 mgr.y (mgr.26054) 815 : cluster 0 pgmap v838: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 3854/185076 objects degraded (2.082%); 6906/185076 objects misplaced (3.731%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:45:22.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:22.491 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:22.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:22.592 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:22.621 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:22.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:22.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:22.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:22.754 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:22.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:22.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:22.803 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:45:22.807 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:45:22.860 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:22.890 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:22.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:22.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:22.994 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:23.037 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:23.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:23.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:23.143 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:23.172 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:23.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:23.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:23.276 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:23.301 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:23.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:23.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:23.406 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:23.434 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:23.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:23.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:23.536 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:23.586 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:23.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:23.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:23.690 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:23.721 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:23.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:23.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:23.823 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:23.846 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:23 smithi098 bash[76817]: cluster 2023-09-23T18:45:22.890912+0000 osd.2 (osd.2) 4 : cluster 0 7.8 scrub starts 2023-09-23T18:45:23.847 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:23 smithi098 bash[79516]: cluster 2023-09-23T18:45:22.890912+0000 osd.2 (osd.2) 4 : cluster 0 7.8 scrub starts 2023-09-23T18:45:23.859 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:23.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:23.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:23.962 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:23.990 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:23.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:23.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:24.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:23 smithi173 bash[59422]: cluster 2023-09-23T18:45:22.890912+0000 osd.2 (osd.2) 4 : cluster 0 7.8 scrub starts 2023-09-23T18:45:24.094 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:24.129 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:24.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:24.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:24.232 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:24.255 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:24.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:24.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:24.360 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:24.386 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:24.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:24.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:24.493 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:24.524 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:24.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:24.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:24.630 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:24.675 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:24.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:24.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:24.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:24 smithi173 bash[59422]: cluster 2023-09-23T18:45:23.403803+0000 osd.4 (osd.4) 182 : cluster 0 7.1 scrub starts 2023-09-23T18:45:24.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:24 smithi173 bash[59422]: cluster 2023-09-23T18:45:23.918958+0000 osd.2 (osd.2) 5 : cluster 0 7.18 scrub starts 2023-09-23T18:45:24.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:24 smithi173 bash[59422]: cluster 2023-09-23T18:45:24.011117+0000 mgr.y (mgr.26054) 816 : cluster 0 pgmap v839: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 749 op/s; 3854/180288 objects degraded (2.138%); 6624/180288 objects misplaced (3.674%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T18:45:24.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:24 smithi173 bash[59422]: audit 2023-09-23T18:45:24.613352+0000 mon.a (mon.0) 1094 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:45:24.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:24 smithi173 bash[59422]: audit 2023-09-23T18:45:24.616672+0000 mon.a (mon.0) 1095 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:45:24.780 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:24.810 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:24.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:24.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:24.913 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:24.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:24 smithi098 bash[76817]: cluster 2023-09-23T18:45:23.403803+0000 osd.4 (osd.4) 182 : cluster 0 7.1 scrub starts 2023-09-23T18:45:24.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:24 smithi098 bash[76817]: cluster 2023-09-23T18:45:23.918958+0000 osd.2 (osd.2) 5 : cluster 0 7.18 scrub starts 2023-09-23T18:45:24.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:24 smithi098 bash[76817]: cluster 2023-09-23T18:45:24.011117+0000 mgr.y (mgr.26054) 816 : cluster 0 pgmap v839: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 749 op/s; 3854/180288 objects degraded (2.138%); 6624/180288 objects misplaced (3.674%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T18:45:24.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:24 smithi098 bash[76817]: audit 2023-09-23T18:45:24.613352+0000 mon.a (mon.0) 1094 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:45:24.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:24 smithi098 bash[76817]: audit 2023-09-23T18:45:24.616672+0000 mon.a (mon.0) 1095 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:45:24.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:24 smithi098 bash[79516]: cluster 2023-09-23T18:45:23.403803+0000 osd.4 (osd.4) 182 : cluster 0 7.1 scrub starts 2023-09-23T18:45:24.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:24 smithi098 bash[79516]: cluster 2023-09-23T18:45:23.918958+0000 osd.2 (osd.2) 5 : cluster 0 7.18 scrub starts 2023-09-23T18:45:24.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:24 smithi098 bash[79516]: cluster 2023-09-23T18:45:24.011117+0000 mgr.y (mgr.26054) 816 : cluster 0 pgmap v839: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 749 op/s; 3854/180288 objects degraded (2.138%); 6624/180288 objects misplaced (3.674%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T18:45:24.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:24 smithi098 bash[79516]: audit 2023-09-23T18:45:24.613352+0000 mon.a (mon.0) 1094 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:45:24.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:24 smithi098 bash[79516]: audit 2023-09-23T18:45:24.616672+0000 mon.a (mon.0) 1095 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:45:24.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:24.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:24.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:25.048 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:25.077 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:25.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:25.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:25.180 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:25.212 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:25.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:25.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:25.319 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:25.341 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:25.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:25.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:25.456 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:25.481 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:25.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:25.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:25.584 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:25.608 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:25.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:25.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:25.711 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:25.735 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:25.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:25.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:25.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:25 smithi173 bash[59422]: cluster 2023-09-23T18:45:24.404257+0000 osd.4 (osd.4) 183 : cluster 0 7.1 scrub starts 2023-09-23T18:45:25.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:25 smithi173 bash[59422]: cluster 2023-09-23T18:45:24.908326+0000 osd.2 (osd.2) 6 : cluster 0 7.10 scrub starts 2023-09-23T18:45:25.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:25 smithi173 bash[59422]: cluster 2023-09-23T18:45:25.062188+0000 mon.a (mon.0) 1096 : cluster 3 Health check update: Degraded data redundancy: 3854/180288 objects degraded (2.138%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:45:25.846 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:25.879 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:25.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:25.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:25.986 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:26.010 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:26.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:26.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:26.086 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:25 smithi098 bash[79516]: cluster 2023-09-23T18:45:24.404257+0000 osd.4 (osd.4) 183 : cluster 0 7.1 scrub starts 2023-09-23T18:45:26.087 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:25 smithi098 bash[79516]: cluster 2023-09-23T18:45:24.908326+0000 osd.2 (osd.2) 6 : cluster 0 7.10 scrub starts 2023-09-23T18:45:26.087 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:25 smithi098 bash[79516]: cluster 2023-09-23T18:45:25.062188+0000 mon.a (mon.0) 1096 : cluster 3 Health check update: Degraded data redundancy: 3854/180288 objects degraded (2.138%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:45:26.087 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:45:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:45:26] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.33.4" 2023-09-23T18:45:26.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:25 smithi098 bash[76817]: cluster 2023-09-23T18:45:24.404257+0000 osd.4 (osd.4) 183 : cluster 0 7.1 scrub starts 2023-09-23T18:45:26.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:25 smithi098 bash[76817]: cluster 2023-09-23T18:45:24.908326+0000 osd.2 (osd.2) 6 : cluster 0 7.10 scrub starts 2023-09-23T18:45:26.088 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:25 smithi098 bash[76817]: cluster 2023-09-23T18:45:25.062188+0000 mon.a (mon.0) 1096 : cluster 3 Health check update: Degraded data redundancy: 3854/180288 objects degraded (2.138%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:45:26.113 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:26.140 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:26.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:26.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:26.246 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:26.276 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:26.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:26.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:26.382 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:26.411 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:26.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:26.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:26.514 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:26.541 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:26.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:26.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:26.646 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:26.676 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:26.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:26.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:26.781 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:26.810 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:26.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:26.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:26.913 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:26.945 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:26.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:26.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:26.975 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:26 smithi098 bash[76817]: cluster 2023-09-23T18:45:25.917024+0000 osd.2 (osd.2) 7 : cluster 0 7.8 scrub starts 2023-09-23T18:45:26.975 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:26 smithi098 bash[76817]: cluster 2023-09-23T18:45:26.011798+0000 mgr.y (mgr.26054) 817 : cluster 0 pgmap v840: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 750 op/s; 3854/175734 objects degraded (2.193%); 6624/175734 objects misplaced (3.769%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:45:27.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:26 smithi173 bash[59422]: cluster 2023-09-23T18:45:25.917024+0000 osd.2 (osd.2) 7 : cluster 0 7.8 scrub starts 2023-09-23T18:45:27.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:26 smithi173 bash[59422]: cluster 2023-09-23T18:45:26.011798+0000 mgr.y (mgr.26054) 817 : cluster 0 pgmap v840: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 750 op/s; 3854/175734 objects degraded (2.193%); 6624/175734 objects misplaced (3.769%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:45:27.052 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:27.080 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:27.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:27.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:27.184 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:27.218 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:27.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:27.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:27.262 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:26 smithi098 bash[79516]: cluster 2023-09-23T18:45:25.917024+0000 osd.2 (osd.2) 7 : cluster 0 7.8 scrub starts 2023-09-23T18:45:27.262 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:26 smithi098 bash[79516]: cluster 2023-09-23T18:45:26.011798+0000 mgr.y (mgr.26054) 817 : cluster 0 pgmap v840: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 750 op/s; 3854/175734 objects degraded (2.193%); 6624/175734 objects misplaced (3.769%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:45:27.325 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:27.353 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:27.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:27.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:27.460 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:27.486 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:27.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:27.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:27.589 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:27.621 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:27.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:27.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:27.725 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:27.749 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:27.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:27.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:27.852 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:27.889 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:27.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:27.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:27.994 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:28.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:28.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:28.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:28.119 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:28.157 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:28.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:28.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:28.262 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:28.296 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:28.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:28.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:28.401 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:28.444 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:28.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:28.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:28.550 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:28.577 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:28.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:28.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:28.680 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:28.707 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:28.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:28.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:28.812 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:28.919 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:28.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:28.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:29.022 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:29.058 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:29.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:29.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:29.161 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:29.189 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:29.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:29.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:29.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:29 smithi098 bash[76817]: cluster 2023-09-23T18:45:27.353764+0000 osd.4 (osd.4) 184 : cluster 0 7.1 scrub starts 2023-09-23T18:45:29.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:29 smithi098 bash[76817]: cluster 2023-09-23T18:45:27.950036+0000 osd.2 (osd.2) 8 : cluster 0 7.18 scrub starts 2023-09-23T18:45:29.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:29 smithi098 bash[76817]: cluster 2023-09-23T18:45:28.012468+0000 mgr.y (mgr.26054) 818 : cluster 0 pgmap v841: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 742 op/s; 3854/171762 objects degraded (2.244%); 6356/171762 objects misplaced (3.700%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T18:45:29.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:29 smithi098 bash[79516]: cluster 2023-09-23T18:45:27.353764+0000 osd.4 (osd.4) 184 : cluster 0 7.1 scrub starts 2023-09-23T18:45:29.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:29 smithi098 bash[79516]: cluster 2023-09-23T18:45:27.950036+0000 osd.2 (osd.2) 8 : cluster 0 7.18 scrub starts 2023-09-23T18:45:29.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:29 smithi098 bash[79516]: cluster 2023-09-23T18:45:28.012468+0000 mgr.y (mgr.26054) 818 : cluster 0 pgmap v841: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 742 op/s; 3854/171762 objects degraded (2.244%); 6356/171762 objects misplaced (3.700%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T18:45:29.292 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:29.317 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:29.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:29.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:29.421 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:29.445 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:29.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:29.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:29.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:29 smithi173 bash[59422]: cluster 2023-09-23T18:45:27.353764+0000 osd.4 (osd.4) 184 : cluster 0 7.1 scrub starts 2023-09-23T18:45:29.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:29 smithi173 bash[59422]: cluster 2023-09-23T18:45:27.950036+0000 osd.2 (osd.2) 8 : cluster 0 7.18 scrub starts 2023-09-23T18:45:29.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:29 smithi173 bash[59422]: cluster 2023-09-23T18:45:28.012468+0000 mgr.y (mgr.26054) 818 : cluster 0 pgmap v841: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 742 op/s; 3854/171762 objects degraded (2.244%); 6356/171762 objects misplaced (3.700%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T18:45:29.557 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:29.583 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:29.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:29.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:29.687 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:29.714 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:29.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:29.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:29.816 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:29.849 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:29.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:29.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:29.952 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:29.971 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:29.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:29.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:30.075 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:30.110 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:30.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:30.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:30.215 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:30.246 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:30.249 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:30 smithi098 bash[76817]: cluster 2023-09-23T18:45:30.063542+0000 mon.a (mon.0) 1097 : cluster 3 Health check update: Degraded data redundancy: 3854/171762 objects degraded (2.244%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:45:30.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:30.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:30.351 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:30.381 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:30.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:30.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:30.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:30 smithi173 bash[59422]: cluster 2023-09-23T18:45:30.063542+0000 mon.a (mon.0) 1097 : cluster 3 Health check update: Degraded data redundancy: 3854/171762 objects degraded (2.244%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:45:30.558 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:30.567 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:30 smithi098 bash[79516]: cluster 2023-09-23T18:45:30.063542+0000 mon.a (mon.0) 1097 : cluster 3 Health check update: Degraded data redundancy: 3854/171762 objects degraded (2.244%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:45:30.598 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:30.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:30.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:30.706 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:30.737 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:30.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:30.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:30.842 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:30.862 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:30.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:30.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:30.965 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:30.993 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:30.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:30.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:31.096 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:31.122 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:31.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:31.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:31.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:31 smithi098 bash[76817]: cluster 2023-09-23T18:45:30.013185+0000 mgr.y (mgr.26054) 819 : cluster 0 pgmap v842: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 769 op/s; 3854/167052 objects degraded (2.307%); 6356/167052 objects misplaced (3.805%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T18:45:31.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:31 smithi098 bash[79516]: cluster 2023-09-23T18:45:30.013185+0000 mgr.y (mgr.26054) 819 : cluster 0 pgmap v842: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 769 op/s; 3854/167052 objects degraded (2.307%); 6356/167052 objects misplaced (3.805%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T18:45:31.226 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:31.234 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:31 smithi173 bash[59422]: cluster 2023-09-23T18:45:30.013185+0000 mgr.y (mgr.26054) 819 : cluster 0 pgmap v842: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 769 op/s; 3854/167052 objects degraded (2.307%); 6356/167052 objects misplaced (3.805%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T18:45:31.247 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:31.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:31.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:31.354 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:31.383 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:31.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:31.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:31.486 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:31.511 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:31.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:31.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:31.619 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:31.643 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:31.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:31.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:31.746 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:31.779 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:31.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:31.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:31.883 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:31.911 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:31.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:31.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:32.015 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:32.042 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:32.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:32.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:32.151 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:32.175 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:32.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:32.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:32.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:32 smithi173 bash[59422]: cluster 2023-09-23T18:45:31.215965+0000 osd.0 (osd.0) 25 : cluster 0 7.9 scrub starts 2023-09-23T18:45:32.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:32 smithi173 bash[59422]: cluster 2023-09-23T18:45:32.013841+0000 mgr.y (mgr.26054) 820 : cluster 0 pgmap v843: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 776 op/s; 3854/162210 objects degraded (2.376%); 6356/162210 objects misplaced (3.918%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T18:45:32.281 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:32.305 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:32.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:32.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:32.408 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:32.417 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:32 smithi098 bash[76817]: cluster 2023-09-23T18:45:31.215965+0000 osd.0 (osd.0) 25 : cluster 0 7.9 scrub starts 2023-09-23T18:45:32.417 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:32 smithi098 bash[76817]: cluster 2023-09-23T18:45:32.013841+0000 mgr.y (mgr.26054) 820 : cluster 0 pgmap v843: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 776 op/s; 3854/162210 objects degraded (2.376%); 6356/162210 objects misplaced (3.918%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T18:45:32.418 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:32 smithi098 bash[79516]: cluster 2023-09-23T18:45:31.215965+0000 osd.0 (osd.0) 25 : cluster 0 7.9 scrub starts 2023-09-23T18:45:32.418 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:32 smithi098 bash[79516]: cluster 2023-09-23T18:45:32.013841+0000 mgr.y (mgr.26054) 820 : cluster 0 pgmap v843: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 776 op/s; 3854/162210 objects degraded (2.376%); 6356/162210 objects misplaced (3.918%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T18:45:32.441 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:32.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:32.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:32.548 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:32.575 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:32.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:32.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:32.679 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:32.709 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:32.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:32.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:32.811 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:32.831 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:32.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:32.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:32.933 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:32.965 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:32.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:32.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:33.068 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:33.095 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:33.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:33.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:33.198 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:33.209 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:33 smithi173 bash[59422]: audit 2023-09-23T18:45:32.706859+0000 mon.a (mon.0) 1098 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:45:33.221 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:33.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:33.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:33.329 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:33.343 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:33 smithi098 bash[76817]: audit 2023-09-23T18:45:32.706859+0000 mon.a (mon.0) 1098 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:45:33.343 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:33 smithi098 bash[79516]: audit 2023-09-23T18:45:32.706859+0000 mon.a (mon.0) 1098 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:45:33.362 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:33.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:33.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:33.465 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:33.487 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:33.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:33.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:33.593 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:33.626 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:33.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:33.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:33.730 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:33.765 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:33.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:33.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:33.868 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:33.899 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:33.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:33.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:34.003 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:34.030 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:34.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:34.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:34.134 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:34.163 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:34.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:34.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:34.268 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:34.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:34 smithi173 bash[59422]: cluster 2023-09-23T18:45:34.014500+0000 mgr.y (mgr.26054) 821 : cluster 0 pgmap v844: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 766 op/s; 3854/157461 objects degraded (2.448%); 6042/157461 objects misplaced (3.837%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T18:45:34.295 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:34.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:34.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:34.399 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:34.408 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:34 smithi098 bash[76817]: cluster 2023-09-23T18:45:34.014500+0000 mgr.y (mgr.26054) 821 : cluster 0 pgmap v844: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 766 op/s; 3854/157461 objects degraded (2.448%); 6042/157461 objects misplaced (3.837%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T18:45:34.408 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:34 smithi098 bash[79516]: cluster 2023-09-23T18:45:34.014500+0000 mgr.y (mgr.26054) 821 : cluster 0 pgmap v844: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 766 op/s; 3854/157461 objects degraded (2.448%); 6042/157461 objects misplaced (3.837%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T18:45:34.434 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:34.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:34.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:34.541 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:34.564 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:34.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:34.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:34.666 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:34.698 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:34.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:34.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:34.801 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:34.838 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:34.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:34.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:34.942 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:34.970 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:34.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:34.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:35.073 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:35.100 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:35.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:35.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:35.124 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:35 smithi098 bash[76817]: cluster 2023-09-23T18:45:35.064663+0000 mon.a (mon.0) 1099 : cluster 3 Health check update: Degraded data redundancy: 3854/157461 objects degraded (2.448%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:45:35.203 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:35.212 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:35 smithi173 bash[59422]: cluster 2023-09-23T18:45:35.064663+0000 mon.a (mon.0) 1099 : cluster 3 Health check update: Degraded data redundancy: 3854/157461 objects degraded (2.448%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:45:35.228 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:35.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:35.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:35.331 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:35.368 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:35.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:35.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:35.391 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:35 smithi098 bash[79516]: cluster 2023-09-23T18:45:35.064663+0000 mon.a (mon.0) 1099 : cluster 3 Health check update: Degraded data redundancy: 3854/157461 objects degraded (2.448%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:45:35.472 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:35.500 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:35.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:35.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:35.603 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:35.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:35.637 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:35.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:35.738 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:35.773 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:35.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:35.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:35.882 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:35.912 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:35.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:35.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:36.017 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:36.043 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:36.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:36.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:36.087 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:45:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:45:36] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.33.4" 2023-09-23T18:45:36.146 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:36.172 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:36.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:36.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:36.274 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:36.306 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:36.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:36.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:36.409 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:36.418 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:36 smithi173 bash[59422]: cluster 2023-09-23T18:45:36.015124+0000 mgr.y (mgr.26054) 822 : cluster 0 pgmap v845: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 3854/152907 objects degraded (2.520%); 6042/152907 objects misplaced (3.951%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T18:45:36.432 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:36.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:36.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:36.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:36 smithi098 bash[79516]: cluster 2023-09-23T18:45:36.015124+0000 mgr.y (mgr.26054) 822 : cluster 0 pgmap v845: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 3854/152907 objects degraded (2.520%); 6042/152907 objects misplaced (3.951%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T18:45:36.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:36 smithi098 bash[76817]: cluster 2023-09-23T18:45:36.015124+0000 mgr.y (mgr.26054) 822 : cluster 0 pgmap v845: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 3854/152907 objects degraded (2.520%); 6042/152907 objects misplaced (3.951%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T18:45:36.536 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:36.561 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:36.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:36.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:36.667 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:36.690 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:36.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:36.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:36.793 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:36.815 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:36.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:36.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:36.922 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:36.956 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:36.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:36.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:37.062 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:37.085 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:37.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:37.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:37.190 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:37.223 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:37.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:37.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:37.327 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:37.353 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:37.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:37.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:37.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:37 smithi098 bash[76817]: cluster 2023-09-23T18:45:35.249777+0000 osd.4 (osd.4) 185 : cluster 0 7.1 scrub starts 2023-09-23T18:45:37.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:37 smithi098 bash[79516]: cluster 2023-09-23T18:45:35.249777+0000 osd.4 (osd.4) 185 : cluster 0 7.1 scrub starts 2023-09-23T18:45:37.458 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:37.467 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:37 smithi173 bash[59422]: cluster 2023-09-23T18:45:35.249777+0000 osd.4 (osd.4) 185 : cluster 0 7.1 scrub starts 2023-09-23T18:45:37.482 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:37.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:37.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:37.585 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:37.617 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:37.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:37.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:37.722 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:37.749 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:37.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:37.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:37.853 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:37.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:37.891 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:37.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:37.992 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:38.024 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:38.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:38.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:38.129 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:38.161 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:38.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:38.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:38.266 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:38.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:38 smithi173 bash[59422]: cluster 2023-09-23T18:45:37.236080+0000 osd.0 (osd.0) 26 : cluster 0 7.6 scrub starts 2023-09-23T18:45:38.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:38 smithi173 bash[59422]: cluster 2023-09-23T18:45:38.015680+0000 mgr.y (mgr.26054) 823 : cluster 0 pgmap v846: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 739 op/s; 3854/149088 objects degraded (2.585%); 5718/149088 objects misplaced (3.835%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T18:45:38.299 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:38.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:38.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:38.404 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:38.405 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:38 smithi098 bash[79516]: cluster 2023-09-23T18:45:37.236080+0000 osd.0 (osd.0) 26 : cluster 0 7.6 scrub starts 2023-09-23T18:45:38.405 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:38 smithi098 bash[79516]: cluster 2023-09-23T18:45:38.015680+0000 mgr.y (mgr.26054) 823 : cluster 0 pgmap v846: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 739 op/s; 3854/149088 objects degraded (2.585%); 5718/149088 objects misplaced (3.835%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T18:45:38.405 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:38 smithi098 bash[76817]: cluster 2023-09-23T18:45:37.236080+0000 osd.0 (osd.0) 26 : cluster 0 7.6 scrub starts 2023-09-23T18:45:38.405 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:38 smithi098 bash[76817]: cluster 2023-09-23T18:45:38.015680+0000 mgr.y (mgr.26054) 823 : cluster 0 pgmap v846: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 739 op/s; 3854/149088 objects degraded (2.585%); 5718/149088 objects misplaced (3.835%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T18:45:38.447 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:38.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:38.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:38.550 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:38.566 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:38.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:38.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:38.669 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:38.696 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:38.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:38.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:38.799 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:38.846 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:38.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:38.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:38.853 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:45:38.853 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [], 2023-09-23T18:45:38.854 INFO:teuthology.orchestra.run.smithi098.stdout: "num_ops": 0 2023-09-23T18:45:38.854 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:45:38.954 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:38.973 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:38.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:38.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:39.086 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:39.108 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:39.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:39.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:39.212 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:39.241 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:39.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:39.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:39.346 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:39.355 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:39 smithi098 bash[76817]: cluster 2023-09-23T18:45:37.222702+0000 osd.4 (osd.4) 186 : cluster 0 7.1 scrub starts 2023-09-23T18:45:39.356 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:39 smithi098 bash[76817]: cluster 2023-09-23T18:45:38.265883+0000 osd.4 (osd.4) 187 : cluster 0 7.1 deep-scrub starts 2023-09-23T18:45:39.356 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:39 smithi098 bash[76817]: cluster 2023-09-23T18:45:38.268843+0000 osd.0 (osd.0) 27 : cluster 0 7.16 scrub starts 2023-09-23T18:45:39.356 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:39 smithi098 bash[76817]: audit 2023-09-23T18:45:38.969769+0000 mon.a (mon.0) 1100 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:45:39.356 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:39 smithi098 bash[76817]: audit 2023-09-23T18:45:38.989929+0000 mon.a (mon.0) 1101 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:45:39.356 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:39 smithi098 bash[79516]: cluster 2023-09-23T18:45:37.222702+0000 osd.4 (osd.4) 186 : cluster 0 7.1 scrub starts 2023-09-23T18:45:39.356 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:39 smithi098 bash[79516]: cluster 2023-09-23T18:45:38.265883+0000 osd.4 (osd.4) 187 : cluster 0 7.1 deep-scrub starts 2023-09-23T18:45:39.356 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:39 smithi098 bash[79516]: cluster 2023-09-23T18:45:38.268843+0000 osd.0 (osd.0) 27 : cluster 0 7.16 scrub starts 2023-09-23T18:45:39.356 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:39 smithi098 bash[79516]: audit 2023-09-23T18:45:38.969769+0000 mon.a (mon.0) 1100 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:45:39.356 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:39 smithi098 bash[79516]: audit 2023-09-23T18:45:38.989929+0000 mon.a (mon.0) 1101 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:45:39.371 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:39.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:39.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:39.476 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:39.488 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:39 smithi173 bash[59422]: cluster 2023-09-23T18:45:37.222702+0000 osd.4 (osd.4) 186 : cluster 0 7.1 scrub starts 2023-09-23T18:45:39.488 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:39 smithi173 bash[59422]: cluster 2023-09-23T18:45:38.265883+0000 osd.4 (osd.4) 187 : cluster 0 7.1 deep-scrub starts 2023-09-23T18:45:39.489 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:39 smithi173 bash[59422]: cluster 2023-09-23T18:45:38.268843+0000 osd.0 (osd.0) 27 : cluster 0 7.16 scrub starts 2023-09-23T18:45:39.489 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:39 smithi173 bash[59422]: audit 2023-09-23T18:45:38.969769+0000 mon.a (mon.0) 1100 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:45:39.489 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:39 smithi173 bash[59422]: audit 2023-09-23T18:45:38.989929+0000 mon.a (mon.0) 1101 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:45:39.506 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:39.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:39.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:39.612 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:39.650 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:39.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:39.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:39.755 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:39.786 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:39.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:39.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:39.890 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:39.917 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:39.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:39.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:40.021 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:40.055 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:40.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:40.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:40.159 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:40.160 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:40 smithi098 bash[76817]: cluster 2023-09-23T18:45:39.299482+0000 osd.0 (osd.0) 28 : cluster 0 7.19 deep-scrub starts 2023-09-23T18:45:40.160 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:40 smithi098 bash[76817]: audit 2023-09-23T18:45:39.609527+0000 mon.a (mon.0) 1102 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:45:40.160 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:40 smithi098 bash[76817]: cluster 2023-09-23T18:45:40.019823+0000 mgr.y (mgr.26054) 824 : cluster 0 pgmap v847: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 754 op/s; 3854/144585 objects degraded (2.666%); 5718/144585 objects misplaced (3.955%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T18:45:40.160 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:40 smithi098 bash[76817]: cluster 2023-09-23T18:45:40.065768+0000 mon.a (mon.0) 1103 : cluster 3 Health check update: Degraded data redundancy: 3854/149088 objects degraded (2.585%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:45:40.161 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:40 smithi098 bash[79516]: cluster 2023-09-23T18:45:39.299482+0000 osd.0 (osd.0) 28 : cluster 0 7.19 deep-scrub starts 2023-09-23T18:45:40.161 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:40 smithi098 bash[79516]: audit 2023-09-23T18:45:39.609527+0000 mon.a (mon.0) 1102 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:45:40.161 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:40 smithi098 bash[79516]: cluster 2023-09-23T18:45:40.019823+0000 mgr.y (mgr.26054) 824 : cluster 0 pgmap v847: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 754 op/s; 3854/144585 objects degraded (2.666%); 5718/144585 objects misplaced (3.955%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T18:45:40.161 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:40 smithi098 bash[79516]: cluster 2023-09-23T18:45:40.065768+0000 mon.a (mon.0) 1103 : cluster 3 Health check update: Degraded data redundancy: 3854/149088 objects degraded (2.585%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:45:40.185 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:40.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:40.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:40.288 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:40.319 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:40.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:40.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:40.422 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:40.461 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:40.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:40.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:40.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:40 smithi173 bash[59422]: cluster 2023-09-23T18:45:39.299482+0000 osd.0 (osd.0) 28 : cluster 0 7.19 deep-scrub starts 2023-09-23T18:45:40.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:40 smithi173 bash[59422]: audit 2023-09-23T18:45:39.609527+0000 mon.a (mon.0) 1102 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:45:40.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:40 smithi173 bash[59422]: cluster 2023-09-23T18:45:40.019823+0000 mgr.y (mgr.26054) 824 : cluster 0 pgmap v847: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 754 op/s; 3854/144585 objects degraded (2.666%); 5718/144585 objects misplaced (3.955%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T18:45:40.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:40 smithi173 bash[59422]: cluster 2023-09-23T18:45:40.065768+0000 mon.a (mon.0) 1103 : cluster 3 Health check update: Degraded data redundancy: 3854/149088 objects degraded (2.585%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:45:40.566 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:40.592 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:40.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:40.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:40.697 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:40.724 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:40.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:40.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:40.828 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:40.852 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:40.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:40.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:40.958 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:40.982 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:40.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:40.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:41.085 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:41.115 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:41.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:41.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:41.160 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:41 smithi098 bash[76817]: cluster 2023-09-23T18:45:39.985069+0000 osd.2 (osd.2) 9 : cluster 0 7.10 deep-scrub starts 2023-09-23T18:45:41.222 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:41.268 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:41.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:41.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:41.373 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:41.401 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:41.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:41.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:41.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:41 smithi098 bash[79516]: cluster 2023-09-23T18:45:39.985069+0000 osd.2 (osd.2) 9 : cluster 0 7.10 deep-scrub starts 2023-09-23T18:45:41.513 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:41.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:41 smithi173 bash[59422]: cluster 2023-09-23T18:45:39.985069+0000 osd.2 (osd.2) 9 : cluster 0 7.10 deep-scrub starts 2023-09-23T18:45:41.559 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:41.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:41.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:41.665 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:41.690 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:41.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:41.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:41.793 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:41.861 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:41.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:41.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:41.966 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:41.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:42.000 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:42.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:42.012 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-09-23T18:45:42.101 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:42.125 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:42.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:42.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:42.228 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:42.258 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:42.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:42.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:42.361 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:42.370 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:42 smithi098 bash[76817]: audit 2023-09-23T18:45:42.008033+0000 mon.a (mon.0) 1104 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T18:45:42.370 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:42 smithi098 bash[76817]: audit 2023-09-23T18:45:42.008182+0000 mon.b (mon.1) 17 : audit 1 from='client.? 172.21.15.98:0/3048955597' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T18:45:42.370 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:42 smithi098 bash[76817]: cluster 2023-09-23T18:45:42.020643+0000 mgr.y (mgr.26054) 825 : cluster 0 pgmap v848: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 754 op/s; 3854/139881 objects degraded (2.755%); 5718/139881 objects misplaced (4.088%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T18:45:42.370 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:42 smithi098 bash[79516]: audit 2023-09-23T18:45:42.008033+0000 mon.a (mon.0) 1104 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T18:45:42.371 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:42 smithi098 bash[79516]: audit 2023-09-23T18:45:42.008182+0000 mon.b (mon.1) 17 : audit 1 from='client.? 172.21.15.98:0/3048955597' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T18:45:42.371 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:42 smithi098 bash[79516]: cluster 2023-09-23T18:45:42.020643+0000 mgr.y (mgr.26054) 825 : cluster 0 pgmap v848: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 754 op/s; 3854/139881 objects degraded (2.755%); 5718/139881 objects misplaced (4.088%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T18:45:42.390 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:42.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:42.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:42.493 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:42.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:42 smithi173 bash[59422]: audit 2023-09-23T18:45:42.008033+0000 mon.a (mon.0) 1104 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T18:45:42.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:42 smithi173 bash[59422]: audit 2023-09-23T18:45:42.008182+0000 mon.b (mon.1) 17 : audit 1 from='client.? 172.21.15.98:0/3048955597' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T18:45:42.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:42 smithi173 bash[59422]: cluster 2023-09-23T18:45:42.020643+0000 mgr.y (mgr.26054) 825 : cluster 0 pgmap v848: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 754 op/s; 3854/139881 objects degraded (2.755%); 5718/139881 objects misplaced (4.088%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T18:45:42.533 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:42.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:42.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:42.638 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:42.668 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:42.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:42.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:42.772 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:42.800 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:42.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:42.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:42.904 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:42.930 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:42.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:42.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:43.033 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:43.064 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:43.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:43.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:43.169 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:43.215 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:43.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:43.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:43.293 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:43 smithi098 bash[79516]: cluster 2023-09-23T18:45:42.295524+0000 osd.0 (osd.0) 29 : cluster 0 7.14 scrub starts 2023-09-23T18:45:43.293 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:43 smithi098 bash[76817]: cluster 2023-09-23T18:45:42.295524+0000 osd.0 (osd.0) 29 : cluster 0 7.14 scrub starts 2023-09-23T18:45:43.321 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:43.342 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:43.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:43.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:43.448 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:43.457 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:43 smithi173 bash[59422]: cluster 2023-09-23T18:45:42.295524+0000 osd.0 (osd.0) 29 : cluster 0 7.14 scrub starts 2023-09-23T18:45:43.470 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:43.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:43.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:43.576 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:43.599 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:43.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:43.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:43.702 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:43.736 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:43.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:43.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:43.841 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:43.863 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:43.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:43.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:43.965 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:43.993 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:43.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:43.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:44.096 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:44.124 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:44.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:44.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:44.179 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:44 smithi098 bash[76817]: cluster 2023-09-23T18:45:42.952572+0000 osd.2 (osd.2) 10 : cluster 0 7.8 deep-scrub starts 2023-09-23T18:45:44.179 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:44 smithi098 bash[76817]: cluster 2023-09-23T18:45:43.261427+0000 osd.4 (osd.4) 188 : cluster 0 7.1 deep-scrub starts 2023-09-23T18:45:44.179 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:44 smithi098 bash[76817]: cluster 2023-09-23T18:45:43.300981+0000 osd.0 (osd.0) 30 : cluster 0 7.4 scrub starts 2023-09-23T18:45:44.180 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:44 smithi098 bash[76817]: cluster 2023-09-23T18:45:44.021329+0000 mgr.y (mgr.26054) 826 : cluster 0 pgmap v849: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 739 op/s; 3854/135555 objects degraded (2.843%); 5394/135555 objects misplaced (3.979%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T18:45:44.180 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:44 smithi098 bash[79516]: cluster 2023-09-23T18:45:42.952572+0000 osd.2 (osd.2) 10 : cluster 0 7.8 deep-scrub starts 2023-09-23T18:45:44.180 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:44 smithi098 bash[79516]: cluster 2023-09-23T18:45:43.261427+0000 osd.4 (osd.4) 188 : cluster 0 7.1 deep-scrub starts 2023-09-23T18:45:44.180 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:44 smithi098 bash[79516]: cluster 2023-09-23T18:45:43.300981+0000 osd.0 (osd.0) 30 : cluster 0 7.4 scrub starts 2023-09-23T18:45:44.180 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:44 smithi098 bash[79516]: cluster 2023-09-23T18:45:44.021329+0000 mgr.y (mgr.26054) 826 : cluster 0 pgmap v849: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 739 op/s; 3854/135555 objects degraded (2.843%); 5394/135555 objects misplaced (3.979%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T18:45:44.228 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:44.256 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:44.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:44.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:44.360 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:44.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:44 smithi173 bash[59422]: cluster 2023-09-23T18:45:42.952572+0000 osd.2 (osd.2) 10 : cluster 0 7.8 deep-scrub starts 2023-09-23T18:45:44.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:44 smithi173 bash[59422]: cluster 2023-09-23T18:45:43.261427+0000 osd.4 (osd.4) 188 : cluster 0 7.1 deep-scrub starts 2023-09-23T18:45:44.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:44 smithi173 bash[59422]: cluster 2023-09-23T18:45:43.300981+0000 osd.0 (osd.0) 30 : cluster 0 7.4 scrub starts 2023-09-23T18:45:44.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:44 smithi173 bash[59422]: cluster 2023-09-23T18:45:44.021329+0000 mgr.y (mgr.26054) 826 : cluster 0 pgmap v849: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 739 op/s; 3854/135555 objects degraded (2.843%); 5394/135555 objects misplaced (3.979%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T18:45:44.388 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:44.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:44.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:44.492 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:44.524 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:44.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:44.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:44.630 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:44.654 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:44.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:44.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:44.757 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:44.789 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:44.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:44.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:44.894 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:44.922 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:44.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:44.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:45.008 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T18:45:45.027 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:45.048 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:45.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:45.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:45.152 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:45.178 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:45.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:45.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:45.283 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:45.297 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:45 smithi098 bash[79516]: cluster 2023-09-23T18:45:44.241217+0000 osd.4 (osd.4) 189 : cluster 0 7.1 scrub starts 2023-09-23T18:45:45.298 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:45 smithi098 bash[79516]: cluster 2023-09-23T18:45:45.066810+0000 mon.a (mon.0) 1105 : cluster 3 Health check update: Degraded data redundancy: 3854/135555 objects degraded (2.843%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:45:45.299 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:45 smithi098 bash[76817]: cluster 2023-09-23T18:45:44.241217+0000 osd.4 (osd.4) 189 : cluster 0 7.1 scrub starts 2023-09-23T18:45:45.299 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:45 smithi098 bash[76817]: cluster 2023-09-23T18:45:45.066810+0000 mon.a (mon.0) 1105 : cluster 3 Health check update: Degraded data redundancy: 3854/135555 objects degraded (2.843%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:45:45.331 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:45.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:45.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:45.441 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:45.465 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:45.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:45.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:45.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:45 smithi173 bash[59422]: cluster 2023-09-23T18:45:44.241217+0000 osd.4 (osd.4) 189 : cluster 0 7.1 scrub starts 2023-09-23T18:45:45.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:45 smithi173 bash[59422]: cluster 2023-09-23T18:45:45.066810+0000 mon.a (mon.0) 1105 : cluster 3 Health check update: Degraded data redundancy: 3854/135555 objects degraded (2.843%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:45:45.568 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:45.585 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:45.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:45.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:45.688 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:45.723 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:45.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:45.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:45.832 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:45.861 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:45.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:45.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:45.964 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:45.992 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:45.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:45.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:46.096 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:46.121 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:46.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:46.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:46.224 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:46.253 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:46.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:46.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:46.356 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:46.379 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:46.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:46.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:46.392 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:46 smithi098 bash[76817]: cluster 2023-09-23T18:45:45.012688+0000 osd.2 (osd.2) 11 : cluster 0 7.18 deep-scrub starts 2023-09-23T18:45:46.392 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:46 smithi098 bash[76817]: cluster 2023-09-23T18:45:45.221940+0000 osd.4 (osd.4) 190 : cluster 0 7.1 scrub starts 2023-09-23T18:45:46.392 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:46 smithi098 bash[76817]: cluster 2023-09-23T18:45:45.381042+0000 osd.0 (osd.0) 31 : cluster 0 7.9 scrub starts 2023-09-23T18:45:46.392 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:46 smithi098 bash[76817]: cluster 2023-09-23T18:45:46.021895+0000 mgr.y (mgr.26054) 827 : cluster 0 pgmap v850: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 729 op/s; 3854/131187 objects degraded (2.938%); 5394/131187 objects misplaced (4.112%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T18:45:46.392 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:45:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:45:46] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.33.4" 2023-09-23T18:45:46.393 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:46 smithi098 bash[79516]: cluster 2023-09-23T18:45:45.012688+0000 osd.2 (osd.2) 11 : cluster 0 7.18 deep-scrub starts 2023-09-23T18:45:46.393 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:46 smithi098 bash[79516]: cluster 2023-09-23T18:45:45.221940+0000 osd.4 (osd.4) 190 : cluster 0 7.1 scrub starts 2023-09-23T18:45:46.393 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:46 smithi098 bash[79516]: cluster 2023-09-23T18:45:45.381042+0000 osd.0 (osd.0) 31 : cluster 0 7.9 scrub starts 2023-09-23T18:45:46.393 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:46 smithi098 bash[79516]: cluster 2023-09-23T18:45:46.021895+0000 mgr.y (mgr.26054) 827 : cluster 0 pgmap v850: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 729 op/s; 3854/131187 objects degraded (2.938%); 5394/131187 objects misplaced (4.112%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T18:45:46.481 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:46.509 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:46.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:46.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:46.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:46 smithi173 bash[59422]: cluster 2023-09-23T18:45:45.012688+0000 osd.2 (osd.2) 11 : cluster 0 7.18 deep-scrub starts 2023-09-23T18:45:46.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:46 smithi173 bash[59422]: cluster 2023-09-23T18:45:45.221940+0000 osd.4 (osd.4) 190 : cluster 0 7.1 scrub starts 2023-09-23T18:45:46.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:46 smithi173 bash[59422]: cluster 2023-09-23T18:45:45.381042+0000 osd.0 (osd.0) 31 : cluster 0 7.9 scrub starts 2023-09-23T18:45:46.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:46 smithi173 bash[59422]: cluster 2023-09-23T18:45:46.021895+0000 mgr.y (mgr.26054) 827 : cluster 0 pgmap v850: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 729 op/s; 3854/131187 objects degraded (2.938%); 5394/131187 objects misplaced (4.112%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T18:45:46.616 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:46.645 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:46.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:46.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:46.750 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:46.784 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:46.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:46.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:46.887 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:46.918 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:46.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:46.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:47.022 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:47.056 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:47.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:47.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:47.159 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:47.190 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:47.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:47.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:47.297 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:47.320 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:47.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:47.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:47.334 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-09-23T18:45:47.424 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:47.446 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:47.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:47.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:47.451 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:47 smithi098 bash[76817]: cluster 2023-09-23T18:45:46.406500+0000 osd.0 (osd.0) 32 : cluster 0 7.6 scrub starts 2023-09-23T18:45:47.451 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:47 smithi098 bash[79516]: cluster 2023-09-23T18:45:46.406500+0000 osd.0 (osd.0) 32 : cluster 0 7.6 scrub starts 2023-09-23T18:45:47.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:47 smithi173 bash[59422]: cluster 2023-09-23T18:45:46.406500+0000 osd.0 (osd.0) 32 : cluster 0 7.6 scrub starts 2023-09-23T18:45:47.549 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:47.595 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:47.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:47.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:47.702 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:47.728 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:47.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:47.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:47.831 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:47.850 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:47.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:47.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:47.953 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:47.988 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:47.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:47.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:48.092 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:48.120 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:48.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:48.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:48.224 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:48.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:48 smithi173 bash[59422]: cluster 2023-09-23T18:45:47.254649+0000 osd.4 (osd.4) 191 : cluster 0 7.1 scrub starts 2023-09-23T18:45:48.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:48 smithi173 bash[59422]: cluster 2023-09-23T18:45:48.022390+0000 mgr.y (mgr.26054) 828 : cluster 0 pgmap v851: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.6 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 656 op/s; 3854/129243 objects degraded (2.982%); 5394/129243 objects misplaced (4.174%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T18:45:48.251 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:48.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:48.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:48.354 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:48.378 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:48.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:48.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:48.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:48 smithi098 bash[76817]: cluster 2023-09-23T18:45:47.254649+0000 osd.4 (osd.4) 191 : cluster 0 7.1 scrub starts 2023-09-23T18:45:48.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:48 smithi098 bash[76817]: cluster 2023-09-23T18:45:48.022390+0000 mgr.y (mgr.26054) 828 : cluster 0 pgmap v851: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.6 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 656 op/s; 3854/129243 objects degraded (2.982%); 5394/129243 objects misplaced (4.174%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T18:45:48.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:48 smithi098 bash[79516]: cluster 2023-09-23T18:45:47.254649+0000 osd.4 (osd.4) 191 : cluster 0 7.1 scrub starts 2023-09-23T18:45:48.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:48 smithi098 bash[79516]: cluster 2023-09-23T18:45:48.022390+0000 mgr.y (mgr.26054) 828 : cluster 0 pgmap v851: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.6 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 656 op/s; 3854/129243 objects degraded (2.982%); 5394/129243 objects misplaced (4.174%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T18:45:48.482 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:48.520 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:48.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:48.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:48.625 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:48.654 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:48.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:48.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:48.760 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:48.785 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:48.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:48.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:48.891 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:48.924 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:48.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:48.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:49.033 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:49.058 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:49.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:49.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:49.161 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:49.189 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:49.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:49.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:49.292 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:49.333 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:49.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:49.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:49.443 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:49.470 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:49.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:49.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:49.573 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:49.599 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:49.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:49.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:49.705 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:49.729 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:49.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:49.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:49.834 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:49.858 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:49.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:49.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:49.900 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:45:49.957 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:45:49.964 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:49.989 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:49.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:49.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:49.995 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:45:50.047 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:45:50.091 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:45:50.094 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:50.107 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is set 2023-09-23T18:45:50.132 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:50.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:50.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:50.136 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:45:50.174 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:45:50.207 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:45:50.214 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:45:50.214 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:45:50.214 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:45:50.214 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:45:50.215 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:45:50.215 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:45:50.215 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:45:50.215 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:45:50.236 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:50.264 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:50.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:50.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:50.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:50 smithi173 bash[59422]: cluster 2023-09-23T18:45:48.235554+0000 osd.4 (osd.4) 192 : cluster 0 7.1 deep-scrub starts 2023-09-23T18:45:50.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:50 smithi173 bash[59422]: audit 2023-09-23T18:45:49.071275+0000 mon.a (mon.0) 1106 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:45:50.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:50 smithi173 bash[59422]: audit 2023-09-23T18:45:49.078582+0000 mon.a (mon.0) 1107 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:45:50.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:50 smithi173 bash[59422]: audit 2023-09-23T18:45:49.379036+0000 mon.a (mon.0) 1108 : audit 1 from='client.? 172.21.15.98:0/2577660226' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:45:50.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:50 smithi173 bash[59422]: audit 2023-09-23T18:45:49.717518+0000 mon.a (mon.0) 1109 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:45:50.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:50 smithi173 bash[59422]: audit 2023-09-23T18:45:49.719217+0000 mon.a (mon.0) 1110 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:45:50.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:50 smithi173 bash[59422]: audit 2023-09-23T18:45:49.727873+0000 mon.a (mon.0) 1111 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:45:50.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:50 smithi173 bash[59422]: audit 2023-09-23T18:45:49.812178+0000 mon.c (mon.2) 99 : audit 0 from='client.? 172.21.15.98:0/2137019245' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:45:50.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:50 smithi173 bash[59422]: cluster 2023-09-23T18:45:50.067797+0000 mon.a (mon.0) 1112 : cluster 3 Health check update: Degraded data redundancy: 3854/129243 objects degraded (2.982%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:45:50.359 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:50 smithi098 bash[76817]: cluster 2023-09-23T18:45:48.235554+0000 osd.4 (osd.4) 192 : cluster 0 7.1 deep-scrub starts 2023-09-23T18:45:50.359 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:50 smithi098 bash[76817]: audit 2023-09-23T18:45:49.071275+0000 mon.a (mon.0) 1106 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:45:50.359 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:50 smithi098 bash[76817]: audit 2023-09-23T18:45:49.078582+0000 mon.a (mon.0) 1107 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:45:50.359 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:50 smithi098 bash[76817]: audit 2023-09-23T18:45:49.379036+0000 mon.a (mon.0) 1108 : audit 1 from='client.? 172.21.15.98:0/2577660226' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:45:50.360 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:50 smithi098 bash[76817]: audit 2023-09-23T18:45:49.717518+0000 mon.a (mon.0) 1109 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:45:50.360 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:50 smithi098 bash[76817]: audit 2023-09-23T18:45:49.719217+0000 mon.a (mon.0) 1110 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:45:50.360 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:50 smithi098 bash[76817]: audit 2023-09-23T18:45:49.727873+0000 mon.a (mon.0) 1111 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:45:50.360 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:50 smithi098 bash[76817]: audit 2023-09-23T18:45:49.812178+0000 mon.c (mon.2) 99 : audit 0 from='client.? 172.21.15.98:0/2137019245' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:45:50.360 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:50 smithi098 bash[76817]: cluster 2023-09-23T18:45:50.067797+0000 mon.a (mon.0) 1112 : cluster 3 Health check update: Degraded data redundancy: 3854/129243 objects degraded (2.982%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:45:50.360 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:50 smithi098 bash[79516]: cluster 2023-09-23T18:45:48.235554+0000 osd.4 (osd.4) 192 : cluster 0 7.1 deep-scrub starts 2023-09-23T18:45:50.360 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:50 smithi098 bash[79516]: audit 2023-09-23T18:45:49.071275+0000 mon.a (mon.0) 1106 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:45:50.360 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:50 smithi098 bash[79516]: audit 2023-09-23T18:45:49.078582+0000 mon.a (mon.0) 1107 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:45:50.360 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:50 smithi098 bash[79516]: audit 2023-09-23T18:45:49.379036+0000 mon.a (mon.0) 1108 : audit 1 from='client.? 172.21.15.98:0/2577660226' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:45:50.361 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:50 smithi098 bash[79516]: audit 2023-09-23T18:45:49.717518+0000 mon.a (mon.0) 1109 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:45:50.361 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:50 smithi098 bash[79516]: audit 2023-09-23T18:45:49.719217+0000 mon.a (mon.0) 1110 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:45:50.361 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:50 smithi098 bash[79516]: audit 2023-09-23T18:45:49.727873+0000 mon.a (mon.0) 1111 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:45:50.361 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:50 smithi098 bash[79516]: audit 2023-09-23T18:45:49.812178+0000 mon.c (mon.2) 99 : audit 0 from='client.? 172.21.15.98:0/2137019245' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:45:50.361 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:50 smithi098 bash[79516]: cluster 2023-09-23T18:45:50.067797+0000 mon.a (mon.0) 1112 : cluster 3 Health check update: Degraded data redundancy: 3854/129243 objects degraded (2.982%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:45:50.373 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:50.409 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:50.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:50.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:50.512 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:50.539 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:50.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:50.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:50.642 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:50.668 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:50.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:50.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:50.772 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:50.798 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:50.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:50.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:50.902 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:50.943 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:50.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:50.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:51.047 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:51.080 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:51.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:51.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:51.086 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:51 smithi098 bash[79516]: cluster 2023-09-23T18:45:50.023059+0000 mgr.y (mgr.26054) 829 : cluster 0 pgmap v852: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 3854/123513 objects degraded (3.120%); 5098/123513 objects misplaced (4.128%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T18:45:51.086 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:51 smithi098 bash[79516]: cluster 2023-09-23T18:45:50.075373+0000 mon.a (mon.0) 1113 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:45:51.087 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:51 smithi098 bash[79516]: cluster 2023-09-23T18:45:50.091324+0000 osd.4 (osd.4) 193 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T18:45:51.087 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:51 smithi098 bash[79516]: audit 2023-09-23T18:45:50.094226+0000 mon.a (mon.0) 1114 : audit 1 from='client.? 172.21.15.98:0/2577660226' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:45:51.087 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:51 smithi098 bash[79516]: cluster 2023-09-23T18:45:50.094570+0000 mon.a (mon.0) 1115 : cluster 0 osdmap e209: 8 total, 8 up, 7 in 2023-09-23T18:45:51.087 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:51 smithi098 bash[79516]: cluster 2023-09-23T18:45:50.110289+0000 osd.6 (osd.6) 156 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T18:45:51.087 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:51 smithi098 bash[79516]: cluster 2023-09-23T18:45:50.110950+0000 osd.7 (osd.7) 125 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T18:45:51.087 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:51 smithi098 bash[79516]: cluster 2023-09-23T18:45:50.795677+0000 osd.5 (osd.5) 105 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T18:45:51.088 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:51 smithi098 bash[76817]: cluster 2023-09-23T18:45:50.023059+0000 mgr.y (mgr.26054) 829 : cluster 0 pgmap v852: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 3854/123513 objects degraded (3.120%); 5098/123513 objects misplaced (4.128%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T18:45:51.088 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:51 smithi098 bash[76817]: cluster 2023-09-23T18:45:50.075373+0000 mon.a (mon.0) 1113 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:45:51.088 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:51 smithi098 bash[76817]: cluster 2023-09-23T18:45:50.091324+0000 osd.4 (osd.4) 193 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T18:45:51.088 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:51 smithi098 bash[76817]: audit 2023-09-23T18:45:50.094226+0000 mon.a (mon.0) 1114 : audit 1 from='client.? 172.21.15.98:0/2577660226' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:45:51.088 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:51 smithi098 bash[76817]: cluster 2023-09-23T18:45:50.094570+0000 mon.a (mon.0) 1115 : cluster 0 osdmap e209: 8 total, 8 up, 7 in 2023-09-23T18:45:51.088 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:51 smithi098 bash[76817]: cluster 2023-09-23T18:45:50.110289+0000 osd.6 (osd.6) 156 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T18:45:51.088 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:51 smithi098 bash[76817]: cluster 2023-09-23T18:45:50.110950+0000 osd.7 (osd.7) 125 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T18:45:51.088 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:51 smithi098 bash[76817]: cluster 2023-09-23T18:45:50.795677+0000 osd.5 (osd.5) 105 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T18:45:51.183 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:51.218 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:51.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:51.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:51.324 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:51.336 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:51 smithi173 bash[59422]: cluster 2023-09-23T18:45:50.023059+0000 mgr.y (mgr.26054) 829 : cluster 0 pgmap v852: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 3854/123513 objects degraded (3.120%); 5098/123513 objects misplaced (4.128%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T18:45:51.336 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:51 smithi173 bash[59422]: cluster 2023-09-23T18:45:50.075373+0000 mon.a (mon.0) 1113 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:45:51.336 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:51 smithi173 bash[59422]: cluster 2023-09-23T18:45:50.091324+0000 osd.4 (osd.4) 193 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T18:45:51.336 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:51 smithi173 bash[59422]: audit 2023-09-23T18:45:50.094226+0000 mon.a (mon.0) 1114 : audit 1 from='client.? 172.21.15.98:0/2577660226' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:45:51.336 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:51 smithi173 bash[59422]: cluster 2023-09-23T18:45:50.094570+0000 mon.a (mon.0) 1115 : cluster 0 osdmap e209: 8 total, 8 up, 7 in 2023-09-23T18:45:51.337 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:51 smithi173 bash[59422]: cluster 2023-09-23T18:45:50.110289+0000 osd.6 (osd.6) 156 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T18:45:51.337 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:51 smithi173 bash[59422]: cluster 2023-09-23T18:45:50.110950+0000 osd.7 (osd.7) 125 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T18:45:51.337 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:51 smithi173 bash[59422]: cluster 2023-09-23T18:45:50.795677+0000 osd.5 (osd.5) 105 : cluster 3 failed to encode map e209 with expected crc 2023-09-23T18:45:51.351 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:51.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:51.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:51.454 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:51.487 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:51.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:51.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:51.594 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:51.621 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:51.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:51.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:51.724 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:51.748 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:51.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:51.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:51.851 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:51.878 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:51.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:51.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:51.987 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:52.011 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:52.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:52.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:52.116 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:52.142 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:52.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:52.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:52.246 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:52.280 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:52.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:52.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:52.335 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T18:45:52.335 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:45:52.336 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-09-23T18:45:52.336 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:45:52.384 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:52.417 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:52.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:52.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:52.529 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:52.559 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:52.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:52.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:52.664 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:52.696 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:52.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:52.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:52.804 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:52.833 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:52.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:52.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:52.936 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:52.959 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:52.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:52.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:53.062 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:53.087 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:53.088 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:53 smithi173 bash[59422]: cluster 2023-09-23T18:45:52.023744+0000 mgr.y (mgr.26054) 830 : cluster 0 pgmap v854: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 3854/119157 objects degraded (3.234%); 5098/119157 objects misplaced (4.278%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T18:45:53.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:53.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:53.195 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:53.243 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:53.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:53.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:53.349 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:53.379 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:53.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:53.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:53.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:53 smithi098 bash[76817]: cluster 2023-09-23T18:45:52.023744+0000 mgr.y (mgr.26054) 830 : cluster 0 pgmap v854: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 3854/119157 objects degraded (3.234%); 5098/119157 objects misplaced (4.278%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T18:45:53.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:53 smithi098 bash[79516]: cluster 2023-09-23T18:45:52.023744+0000 mgr.y (mgr.26054) 830 : cluster 0 pgmap v854: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 3854/119157 objects degraded (3.234%); 5098/119157 objects misplaced (4.278%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T18:45:53.482 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:53.510 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:53.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:53.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:53.613 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:53.649 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:53.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:53.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:53.754 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:53.789 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:53.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:53.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:53.897 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:53.923 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:53.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:53.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:54.027 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:54.055 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:54.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:54.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:54.158 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:54.180 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:54.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:54.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:54.285 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:54.322 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:54.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:54.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:54.426 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:54.459 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:54.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:54.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:54.565 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:54.592 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:54.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:54.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:54.694 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:54.724 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:54.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:54.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:54.818 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:45:54.828 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:54.839 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:45:54.908 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:54.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:54.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:55.014 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:55.041 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:55.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:55.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:55.097 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:55 smithi098 bash[76817]: cluster 2023-09-23T18:45:54.024420+0000 mgr.y (mgr.26054) 831 : cluster 0 pgmap v855: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 682 op/s; 3854/115086 objects degraded (3.349%); 4792/115086 objects misplaced (4.164%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T18:45:55.097 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:55 smithi098 bash[76817]: audit 2023-09-23T18:45:54.618872+0000 mon.a (mon.0) 1116 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:45:55.097 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:55 smithi098 bash[76817]: audit 2023-09-23T18:45:54.620276+0000 mon.a (mon.0) 1117 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:45:55.097 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:55 smithi098 bash[76817]: cluster 2023-09-23T18:45:55.069157+0000 mon.a (mon.0) 1118 : cluster 3 Health check update: Degraded data redundancy: 3854/115086 objects degraded (3.349%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:45:55.097 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:55 smithi098 bash[79516]: cluster 2023-09-23T18:45:54.024420+0000 mgr.y (mgr.26054) 831 : cluster 0 pgmap v855: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 682 op/s; 3854/115086 objects degraded (3.349%); 4792/115086 objects misplaced (4.164%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T18:45:55.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:55 smithi098 bash[79516]: audit 2023-09-23T18:45:54.618872+0000 mon.a (mon.0) 1116 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:45:55.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:55 smithi098 bash[79516]: audit 2023-09-23T18:45:54.620276+0000 mon.a (mon.0) 1117 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:45:55.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:55 smithi098 bash[79516]: cluster 2023-09-23T18:45:55.069157+0000 mon.a (mon.0) 1118 : cluster 3 Health check update: Degraded data redundancy: 3854/115086 objects degraded (3.349%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:45:55.147 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:55.189 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:55.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:55.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:55.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:55 smithi173 bash[59422]: cluster 2023-09-23T18:45:54.024420+0000 mgr.y (mgr.26054) 831 : cluster 0 pgmap v855: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 682 op/s; 3854/115086 objects degraded (3.349%); 4792/115086 objects misplaced (4.164%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T18:45:55.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:55 smithi173 bash[59422]: audit 2023-09-23T18:45:54.618872+0000 mon.a (mon.0) 1116 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:45:55.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:55 smithi173 bash[59422]: audit 2023-09-23T18:45:54.620276+0000 mon.a (mon.0) 1117 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:45:55.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:55 smithi173 bash[59422]: cluster 2023-09-23T18:45:55.069157+0000 mon.a (mon.0) 1118 : cluster 3 Health check update: Degraded data redundancy: 3854/115086 objects degraded (3.349%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:45:55.301 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:55.340 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:55.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:55.346 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:45:55.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:55.448 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:55.472 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:55.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:55.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:55.575 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:55.603 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:55.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:55.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:55.706 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:55.734 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:55.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:55.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:55.837 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:55.862 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:55.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:55.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:55.967 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:55.993 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:55.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:55.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:56.098 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:56.120 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:56.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:56.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:56.224 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:56.247 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:56.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:56.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:56.350 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:56.384 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:56.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:56.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:56.440 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:45:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:45:56] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.33.4" 2023-09-23T18:45:56.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:56.522 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:56.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:56.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:56.626 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:56.653 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:56.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:56.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:56.763 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:56.791 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:56.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:56.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:56.895 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:56.921 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:56.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:56.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:57.027 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:57.051 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:57.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:57.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:57.160 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:57.190 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:57.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:57.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:57.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:57 smithi173 bash[59422]: cluster 2023-09-23T18:45:56.025126+0000 mgr.y (mgr.26054) 832 : cluster 0 pgmap v856: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.3 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 3854/110469 objects degraded (3.489%); 4792/110469 objects misplaced (4.338%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T18:45:57.296 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:57.330 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:57.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:57.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:57.379 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:45:57.417 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:57 smithi098 bash[79516]: cluster 2023-09-23T18:45:56.025126+0000 mgr.y (mgr.26054) 832 : cluster 0 pgmap v856: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.3 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 3854/110469 objects degraded (3.489%); 4792/110469 objects misplaced (4.338%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T18:45:57.417 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:57 smithi098 bash[76817]: cluster 2023-09-23T18:45:56.025126+0000 mgr.y (mgr.26054) 832 : cluster 0 pgmap v856: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.3 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 3854/110469 objects degraded (3.489%); 4792/110469 objects misplaced (4.338%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T18:45:57.434 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:57.462 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:57.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:57.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:57.566 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:57.594 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:57.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:57.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:57.697 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:57.726 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:57.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:57.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:57.830 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:57.858 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:57.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:57.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:57.963 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:58.010 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:58.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:58.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:58.115 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:58.138 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:58.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:58.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:58.242 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:58.271 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:45:58.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:58.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:45:58.373 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:58.401 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:58.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:58.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:58.509 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:58.537 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:58.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:58.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:58.641 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:58.676 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:45:58.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:58.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:45:58.781 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:58.809 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:58.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:58.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:58.915 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:58.944 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:45:58.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:58.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:45:59.047 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:59.070 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:59.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:59.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:59.173 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:59.180 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:45:59 smithi098 bash[79516]: cluster 2023-09-23T18:45:58.025774+0000 mgr.y (mgr.26054) 833 : cluster 0 pgmap v857: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 691 op/s; 3854/108489 objects degraded (3.552%); 4792/108489 objects misplaced (4.417%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T18:45:59.181 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:45:59 smithi098 bash[76817]: cluster 2023-09-23T18:45:58.025774+0000 mgr.y (mgr.26054) 833 : cluster 0 pgmap v857: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 691 op/s; 3854/108489 objects degraded (3.552%); 4792/108489 objects misplaced (4.417%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T18:45:59.205 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:45:59.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:59.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:45:59.310 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:59.318 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:45:59 smithi173 bash[59422]: cluster 2023-09-23T18:45:58.025774+0000 mgr.y (mgr.26054) 833 : cluster 0 pgmap v857: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 691 op/s; 3854/108489 objects degraded (3.552%); 4792/108489 objects misplaced (4.417%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T18:45:59.340 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:59.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:59.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:59.417 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:45:59.417 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:45:59.448 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:59.491 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:45:59.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:59.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:45:59.600 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:59.624 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:45:59.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:59.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:45:59.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:59.764 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:45:59.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:59.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:45:59.867 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:59.893 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:45:59.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:45:59.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:45:59.996 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:00.025 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:00.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:00.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:00.130 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:00.161 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:00.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:00.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:00.265 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:00.273 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:00 smithi173 bash[59422]: cluster 2023-09-23T18:46:00.074834+0000 mon.a (mon.0) 1119 : cluster 3 Health check update: Degraded data redundancy: 3854/108489 objects degraded (3.552%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:46:00.289 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:00.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:00.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:00.392 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:00.413 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:00.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:00.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:00.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:00 smithi098 bash[76817]: cluster 2023-09-23T18:46:00.074834+0000 mon.a (mon.0) 1119 : cluster 3 Health check update: Degraded data redundancy: 3854/108489 objects degraded (3.552%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:46:00.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:00 smithi098 bash[79516]: cluster 2023-09-23T18:46:00.074834+0000 mon.a (mon.0) 1119 : cluster 3 Health check update: Degraded data redundancy: 3854/108489 objects degraded (3.552%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:46:00.516 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:00.542 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:00.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:00.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:00.647 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:00.673 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:00.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:00.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:00.777 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:00.803 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:00.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:00.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:00.906 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:00.931 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:00.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:00.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:01.034 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:01.063 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:01.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:01.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:01.167 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:01.190 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:01.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:01.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:01.293 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:01.302 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:01 smithi173 bash[59422]: cluster 2023-09-23T18:46:00.026500+0000 mgr.y (mgr.26054) 834 : cluster 0 pgmap v858: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.1 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 701 op/s; 3854/102456 objects degraded (3.762%); 4468/102456 objects misplaced (4.361%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T18:46:01.317 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:01.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:01.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:01.421 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:01.430 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:01 smithi098 bash[79516]: cluster 2023-09-23T18:46:00.026500+0000 mgr.y (mgr.26054) 834 : cluster 0 pgmap v858: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.1 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 701 op/s; 3854/102456 objects degraded (3.762%); 4468/102456 objects misplaced (4.361%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T18:46:01.430 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:01 smithi098 bash[76817]: cluster 2023-09-23T18:46:00.026500+0000 mgr.y (mgr.26054) 834 : cluster 0 pgmap v858: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.1 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 701 op/s; 3854/102456 objects degraded (3.762%); 4468/102456 objects misplaced (4.361%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T18:46:01.450 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:01.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:01.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:01.554 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:01.585 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:01.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:01.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:01.690 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:01.723 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:01.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:01.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:01.829 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:01.852 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:01.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:01.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:01.955 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:01.988 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:01.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:01.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:02.090 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:02.118 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:02.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:02.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:02.221 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:02.230 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:02 smithi173 bash[59422]: cluster 2023-09-23T18:46:02.027168+0000 mgr.y (mgr.26054) 835 : cluster 0 pgmap v859: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.0 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 714 op/s; 3854/97911 objects degraded (3.936%); 4468/97911 objects misplaced (4.563%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T18:46:02.245 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:02.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:02.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:02.350 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:02.374 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:02.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:02.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:02.476 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:02.507 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:02.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:02.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:02.539 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:02 smithi098 bash[76817]: cluster 2023-09-23T18:46:02.027168+0000 mgr.y (mgr.26054) 835 : cluster 0 pgmap v859: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.0 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 714 op/s; 3854/97911 objects degraded (3.936%); 4468/97911 objects misplaced (4.563%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T18:46:02.539 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:02 smithi098 bash[79516]: cluster 2023-09-23T18:46:02.027168+0000 mgr.y (mgr.26054) 835 : cluster 0 pgmap v859: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.0 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 714 op/s; 3854/97911 objects degraded (3.936%); 4468/97911 objects misplaced (4.563%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T18:46:02.615 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:02.636 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:02.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:02.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:02.741 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:02.770 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:02.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:02.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:02.874 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:02.909 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:02.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:02.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:03.015 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:03.038 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:03.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:03.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:03.141 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:03.167 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:03.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:03.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:03.270 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:03.295 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:03.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:03.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:03.398 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:03.423 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:03.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:03.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:03.527 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:03.554 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:03.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:03.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:03.658 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:03.689 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:03.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:03.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:03.796 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:03.821 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:03.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:03.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:03.925 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:03.966 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:03.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:03.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:04.069 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:04.096 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:04.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:04.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:04.200 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:04.232 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:04.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:04.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:04.338 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:04.378 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:04.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:04.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:04.484 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:04.513 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:04.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:04.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:04.619 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:04.649 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:04.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:04.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:04.754 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:04.781 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:04.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:04.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:04.884 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:04.919 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:04.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:04.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:05.028 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:05.056 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:05.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:05.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:05.161 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:05.169 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:05 smithi098 bash[76817]: cluster 2023-09-23T18:46:04.027815+0000 mgr.y (mgr.26054) 836 : cluster 0 pgmap v860: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.9 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 708 op/s; 3854/93630 objects degraded (4.116%); 4100/93630 objects misplaced (4.379%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:46:05.169 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:05 smithi098 bash[76817]: cluster 2023-09-23T18:46:05.075933+0000 mon.a (mon.0) 1120 : cluster 3 Health check update: Degraded data redundancy: 3854/93630 objects degraded (4.116%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:46:05.169 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:05 smithi098 bash[79516]: cluster 2023-09-23T18:46:04.027815+0000 mgr.y (mgr.26054) 836 : cluster 0 pgmap v860: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.9 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 708 op/s; 3854/93630 objects degraded (4.116%); 4100/93630 objects misplaced (4.379%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:46:05.169 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:05 smithi098 bash[79516]: cluster 2023-09-23T18:46:05.075933+0000 mon.a (mon.0) 1120 : cluster 3 Health check update: Degraded data redundancy: 3854/93630 objects degraded (4.116%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:46:05.191 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:05.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:05.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:05.297 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:05.321 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:05.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:05.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:05.425 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:05.434 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:05 smithi173 bash[59422]: cluster 2023-09-23T18:46:04.027815+0000 mgr.y (mgr.26054) 836 : cluster 0 pgmap v860: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.9 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 708 op/s; 3854/93630 objects degraded (4.116%); 4100/93630 objects misplaced (4.379%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:46:05.434 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:05 smithi173 bash[59422]: cluster 2023-09-23T18:46:05.075933+0000 mon.a (mon.0) 1120 : cluster 3 Health check update: Degraded data redundancy: 3854/93630 objects degraded (4.116%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:46:05.450 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:05.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:05.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:05.553 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:05.580 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:05.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:05.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:05.683 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:05.713 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:05.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:05.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:05.816 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:05.850 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:05.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:05.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:05.958 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:05.984 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:05.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:05.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:06.087 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:06.093 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:46:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:46:06] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.33.4" 2023-09-23T18:46:06.113 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:06.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:06.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:06.217 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:06.242 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:06.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:06.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:06.345 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:06.370 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:06.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:06.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:06.473 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:06.502 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:06.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:06.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:06.608 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:06.636 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:06.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:06.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:06.741 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:06.768 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:06.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:06.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:06.874 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:06.898 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:06.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:06.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:07.003 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:07.025 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:07.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:07.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:07.128 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:07.154 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:07.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:07.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:07.258 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:07.267 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:07 smithi173 bash[59422]: cluster 2023-09-23T18:46:06.028559+0000 mgr.y (mgr.26054) 837 : cluster 0 pgmap v861: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.8 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 3854/89196 objects degraded (4.321%); 4100/89196 objects misplaced (4.597%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T18:46:07.286 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:07.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:07.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:07.390 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:07.398 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:07 smithi098 bash[79516]: cluster 2023-09-23T18:46:06.028559+0000 mgr.y (mgr.26054) 837 : cluster 0 pgmap v861: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.8 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 3854/89196 objects degraded (4.321%); 4100/89196 objects misplaced (4.597%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T18:46:07.399 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:07 smithi098 bash[76817]: cluster 2023-09-23T18:46:06.028559+0000 mgr.y (mgr.26054) 837 : cluster 0 pgmap v861: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.8 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 3854/89196 objects degraded (4.321%); 4100/89196 objects misplaced (4.597%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T18:46:07.423 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:07.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:07.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:07.527 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:07.561 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:07.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:07.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:07.666 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:07.694 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:07.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:07.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:07.799 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:07.820 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:07.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:07.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:07.923 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:07.958 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:07.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:07.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:08.061 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:08.085 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:08.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:08.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:08.189 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:08.206 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:08.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:08.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:08.309 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:08.335 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:08.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:08.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:08.440 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:08.464 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:08.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:08.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:08.567 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:08.592 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:08.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:08.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:08.695 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:08.719 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:08.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:08.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:08.823 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:08.849 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:08.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:08.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:08.955 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:08.995 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:09.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:09.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:09.101 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:09.133 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:09.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:09.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:09.215 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:09 smithi098 bash[79516]: cluster 2023-09-23T18:46:08.029181+0000 mgr.y (mgr.26054) 838 : cluster 0 pgmap v862: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.8 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 3854/87189 objects degraded (4.420%); 4100/87189 objects misplaced (4.702%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T18:46:09.215 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:09 smithi098 bash[76817]: cluster 2023-09-23T18:46:08.029181+0000 mgr.y (mgr.26054) 838 : cluster 0 pgmap v862: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.8 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 3854/87189 objects degraded (4.420%); 4100/87189 objects misplaced (4.702%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T18:46:09.238 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:09.262 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:09.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:09.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:09.365 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:09.382 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:09.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:09.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:09.485 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:09.493 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:09 smithi173 bash[59422]: cluster 2023-09-23T18:46:08.029181+0000 mgr.y (mgr.26054) 838 : cluster 0 pgmap v862: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.8 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 3854/87189 objects degraded (4.420%); 4100/87189 objects misplaced (4.702%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T18:46:09.508 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:09.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:09.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:09.613 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:09.641 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:09.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:09.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:09.744 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:09.775 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:09.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:09.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:09.882 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:09.918 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:09.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:09.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:10.026 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:10.053 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:10.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:10.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:10.162 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:10.185 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:10.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:10.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:10.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:10 smithi098 bash[79516]: audit 2023-09-23T18:46:09.613629+0000 mon.a (mon.0) 1121 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:46:10.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:10 smithi098 bash[79516]: cluster 2023-09-23T18:46:10.029881+0000 mgr.y (mgr.26054) 839 : cluster 0 pgmap v863: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.7 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 752 op/s; 3775/81399 objects degraded (4.638%); 3775/81399 objects misplaced (4.638%); 2.4 MiB/s, 45 objects/s recovering 2023-09-23T18:46:10.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:10 smithi098 bash[79516]: cluster 2023-09-23T18:46:10.077051+0000 mon.a (mon.0) 1122 : cluster 3 Health check update: Degraded data redundancy: 3854/87189 objects degraded (4.420%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:46:10.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:10 smithi098 bash[76817]: audit 2023-09-23T18:46:09.613629+0000 mon.a (mon.0) 1121 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:46:10.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:10 smithi098 bash[76817]: cluster 2023-09-23T18:46:10.029881+0000 mgr.y (mgr.26054) 839 : cluster 0 pgmap v863: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.7 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 752 op/s; 3775/81399 objects degraded (4.638%); 3775/81399 objects misplaced (4.638%); 2.4 MiB/s, 45 objects/s recovering 2023-09-23T18:46:10.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:10 smithi098 bash[76817]: cluster 2023-09-23T18:46:10.077051+0000 mon.a (mon.0) 1122 : cluster 3 Health check update: Degraded data redundancy: 3854/87189 objects degraded (4.420%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:46:10.289 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:10.324 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:10.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:10.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:10.430 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:10.455 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:10.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:10.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:10.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:10 smithi173 bash[59422]: audit 2023-09-23T18:46:09.613629+0000 mon.a (mon.0) 1121 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:46:10.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:10 smithi173 bash[59422]: cluster 2023-09-23T18:46:10.029881+0000 mgr.y (mgr.26054) 839 : cluster 0 pgmap v863: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.7 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 752 op/s; 3775/81399 objects degraded (4.638%); 3775/81399 objects misplaced (4.638%); 2.4 MiB/s, 45 objects/s recovering 2023-09-23T18:46:10.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:10 smithi173 bash[59422]: cluster 2023-09-23T18:46:10.077051+0000 mon.a (mon.0) 1122 : cluster 3 Health check update: Degraded data redundancy: 3854/87189 objects degraded (4.420%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:46:10.563 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:10.591 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:10.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:10.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:10.696 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:10.719 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:10.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:10.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:10.822 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:10.855 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:10.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:10.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:10.958 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:10.982 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:10.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:10.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:11.085 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:11.116 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:11.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:11.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:11.226 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:11.254 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:11.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:11.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:11.358 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:11.389 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:11.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:11.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:11.492 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:11.517 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:11.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:11.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:11.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:11.645 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:11.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:11.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:11.749 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:11.780 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:11.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:11.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:11.885 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:11.910 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:11.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:11.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:12.017 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:12.044 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:12.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:12.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:12.148 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:12.182 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:12.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:12.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:12.286 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:12.313 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:12.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:12.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:12.417 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:12.435 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:12.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:12.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:12.537 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:12.563 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:12.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:12.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:12.667 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:12.688 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:12.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:12.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:12.794 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:12.825 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:12.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:12.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:12.933 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:12.962 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:12.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:12.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:13.068 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:13.090 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:13 smithi098 bash[76817]: cluster 2023-09-23T18:46:12.031018+0000 mgr.y (mgr.26054) 840 : cluster 0 pgmap v864: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.6 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 708 op/s; 3775/76941 objects degraded (4.906%); 3775/76941 objects misplaced (4.906%); 1.6 MiB/s, 32 objects/s recovering 2023-09-23T18:46:13.092 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:13.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:13.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:13.195 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:13.222 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:13.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:13.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:13.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:13 smithi173 bash[59422]: cluster 2023-09-23T18:46:12.031018+0000 mgr.y (mgr.26054) 840 : cluster 0 pgmap v864: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.6 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 708 op/s; 3775/76941 objects degraded (4.906%); 3775/76941 objects misplaced (4.906%); 1.6 MiB/s, 32 objects/s recovering 2023-09-23T18:46:13.326 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:13.350 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:13.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:13.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:13.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:13 smithi098 bash[79516]: cluster 2023-09-23T18:46:12.031018+0000 mgr.y (mgr.26054) 840 : cluster 0 pgmap v864: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.6 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 708 op/s; 3775/76941 objects degraded (4.906%); 3775/76941 objects misplaced (4.906%); 1.6 MiB/s, 32 objects/s recovering 2023-09-23T18:46:13.455 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:13.484 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:13.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:13.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:13.588 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:13.617 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:13.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:13.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:13.723 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:13.755 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:13.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:13.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:13.857 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:13.879 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:13.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:13.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:13.981 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:14.006 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:14.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:14.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:14.111 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:14.147 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:14.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:14.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:14.252 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:14.276 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:14.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:14.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:14.380 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:14.416 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:14.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:14.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:14.524 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:14.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:14.558 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:14.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:14.659 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:14.683 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:14.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:14.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:14.789 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:14.791 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:46:14.792 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:46:14.813 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:14.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:14.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:14.916 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:14.946 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:14.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:14.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:15.050 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:15.075 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:15.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:15.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:15.179 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:15.188 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:15 smithi173 bash[59422]: cluster 2023-09-23T18:46:14.031775+0000 mgr.y (mgr.26054) 841 : cluster 0 pgmap v865: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.5 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 3551/72615 objects degraded (4.890%); 3552/72615 objects misplaced (4.892%); 1.6 MiB/s, 50 objects/s recovering 2023-09-23T18:46:15.188 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:15 smithi173 bash[59422]: cluster 2023-09-23T18:46:15.078297+0000 mon.a (mon.0) 1123 : cluster 3 Health check update: Degraded data redundancy: 3551/72615 objects degraded (4.890%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:46:15.212 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:15.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:15.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:15.315 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:15.339 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:15.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:15.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:15.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:15 smithi098 bash[76817]: cluster 2023-09-23T18:46:14.031775+0000 mgr.y (mgr.26054) 841 : cluster 0 pgmap v865: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.5 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 3551/72615 objects degraded (4.890%); 3552/72615 objects misplaced (4.892%); 1.6 MiB/s, 50 objects/s recovering 2023-09-23T18:46:15.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:15 smithi098 bash[76817]: cluster 2023-09-23T18:46:15.078297+0000 mon.a (mon.0) 1123 : cluster 3 Health check update: Degraded data redundancy: 3551/72615 objects degraded (4.890%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:46:15.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:15 smithi098 bash[79516]: cluster 2023-09-23T18:46:14.031775+0000 mgr.y (mgr.26054) 841 : cluster 0 pgmap v865: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.5 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 3551/72615 objects degraded (4.890%); 3552/72615 objects misplaced (4.892%); 1.6 MiB/s, 50 objects/s recovering 2023-09-23T18:46:15.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:15 smithi098 bash[79516]: cluster 2023-09-23T18:46:15.078297+0000 mon.a (mon.0) 1123 : cluster 3 Health check update: Degraded data redundancy: 3551/72615 objects degraded (4.890%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:46:15.446 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:15.467 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:15.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:15.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:15.571 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:15.597 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:15.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:15.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:15.700 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:15.731 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:15.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:15.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:15.834 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:15.859 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:15.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:15.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:15.967 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:16.000 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:16.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:16.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:16.109 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:16.130 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:16.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:16.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:16.235 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:16.258 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:16.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:16.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:16.361 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:16.370 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:46:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:46:16] "GET /metrics HTTP/1.1" 200 36573 "" "Prometheus/2.33.4" 2023-09-23T18:46:16.383 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:16.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:16.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:16.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:16.514 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:16.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:16.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:16.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:16.645 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:16.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:16.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:16.749 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:16.780 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:16.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:16.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:16.883 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:16.912 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:16.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:16.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:17.015 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:17.040 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:17.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:17.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:17.144 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:17.180 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:17.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:17.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:17.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:17 smithi173 bash[59422]: cluster 2023-09-23T18:46:16.032369+0000 mgr.y (mgr.26054) 842 : cluster 0 pgmap v866: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.4 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 3551/68286 objects degraded (5.200%); 3552/68286 objects misplaced (5.202%); 656 KiB/s, 35 objects/s recovering 2023-09-23T18:46:17.285 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:17.310 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:17.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:17.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:17.416 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:17.425 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:17 smithi098 bash[76817]: cluster 2023-09-23T18:46:16.032369+0000 mgr.y (mgr.26054) 842 : cluster 0 pgmap v866: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.4 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 3551/68286 objects degraded (5.200%); 3552/68286 objects misplaced (5.202%); 656 KiB/s, 35 objects/s recovering 2023-09-23T18:46:17.425 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:17 smithi098 bash[79516]: cluster 2023-09-23T18:46:16.032369+0000 mgr.y (mgr.26054) 842 : cluster 0 pgmap v866: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.4 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 3551/68286 objects degraded (5.200%); 3552/68286 objects misplaced (5.202%); 656 KiB/s, 35 objects/s recovering 2023-09-23T18:46:17.457 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:17.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:17.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:17.562 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:17.587 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:17.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:17.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:17.691 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:17.713 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:17.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:17.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:17.816 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:17.841 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:17.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:17.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:17.944 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:17.973 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:17.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:17.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:18.076 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:18.101 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:18.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:18.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:18.204 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:18.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:18.241 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:18.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:18.342 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:18.371 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:18.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:18.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:18.474 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:18.506 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:18.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:18.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:18.610 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:18.637 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:18.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:18.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:18.740 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:18.759 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:18.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:18.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:18.862 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:18.887 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:18.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:18.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:18.921 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:46:18.932 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-23T18:46:18.990 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:19.022 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:19.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:19.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:19.128 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:19.148 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:19 smithi098 bash[76817]: cluster 2023-09-23T18:46:18.032937+0000 mgr.y (mgr.26054) 843 : cluster 0 pgmap v867: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.3 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 3551/66156 objects degraded (5.368%); 3552/66156 objects misplaced (5.369%); 656 KiB/s, 35 objects/s recovering 2023-09-23T18:46:19.152 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:19.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:19.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:19.256 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:19.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:19 smithi173 bash[59422]: cluster 2023-09-23T18:46:18.032937+0000 mgr.y (mgr.26054) 843 : cluster 0 pgmap v867: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.3 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 3551/66156 objects degraded (5.368%); 3552/66156 objects misplaced (5.369%); 656 KiB/s, 35 objects/s recovering 2023-09-23T18:46:19.280 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:19.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:19.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:19.384 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:19.407 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:19.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:19.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:19.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:19 smithi098 bash[79516]: cluster 2023-09-23T18:46:18.032937+0000 mgr.y (mgr.26054) 843 : cluster 0 pgmap v867: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.3 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 3551/66156 objects degraded (5.368%); 3552/66156 objects misplaced (5.369%); 656 KiB/s, 35 objects/s recovering 2023-09-23T18:46:19.510 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:19.532 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:19.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:19.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:19.636 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:19.666 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:19.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:19.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:19.770 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:19.794 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:19.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:19.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:19.898 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:19.932 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:19.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:19.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:20.037 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:20.066 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:20.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:20.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:20.146 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T18:46:20.170 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:20.195 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:20.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:20.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:20.298 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:20.300 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:20 smithi098 bash[79516]: audit 2023-09-23T18:46:18.897602+0000 mgr.y (mgr.26054) 844 : audit 0 from='client.35319 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:46:20.300 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:20 smithi098 bash[79516]: cluster 2023-09-23T18:46:20.079330+0000 mon.a (mon.0) 1124 : cluster 3 Health check update: Degraded data redundancy: 3551/66156 objects degraded (5.368%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:46:20.300 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:20 smithi098 bash[76817]: audit 2023-09-23T18:46:18.897602+0000 mgr.y (mgr.26054) 844 : audit 0 from='client.35319 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:46:20.301 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:20 smithi098 bash[76817]: cluster 2023-09-23T18:46:20.079330+0000 mon.a (mon.0) 1124 : cluster 3 Health check update: Degraded data redundancy: 3551/66156 objects degraded (5.368%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:46:20.321 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:20.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:20.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:20.424 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:20.456 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:20.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:20.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:20.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:20 smithi173 bash[59422]: audit 2023-09-23T18:46:18.897602+0000 mgr.y (mgr.26054) 844 : audit 0 from='client.35319 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:46:20.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:20 smithi173 bash[59422]: cluster 2023-09-23T18:46:20.079330+0000 mon.a (mon.0) 1124 : cluster 3 Health check update: Degraded data redundancy: 3551/66156 objects degraded (5.368%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:46:20.562 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:20.593 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:20.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:20.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:20.696 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:20.719 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:20.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:20.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:20.822 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:20.849 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:20.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:20.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:20.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:20.983 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:20.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:20.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:21.087 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:21.112 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:21.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:21.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:21.215 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:21.239 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:21.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:21.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:21.345 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:21.370 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:21.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:21.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:21.404 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:21 smithi098 bash[76817]: cluster 2023-09-23T18:46:20.033609+0000 mgr.y (mgr.26054) 845 : cluster 0 pgmap v868: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.2 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 752 op/s; 3318/60099 objects degraded (5.521%); 3318/60099 objects misplaced (5.521%); 656 KiB/s, 54 objects/s recovering 2023-09-23T18:46:21.405 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:21 smithi098 bash[79516]: cluster 2023-09-23T18:46:20.033609+0000 mgr.y (mgr.26054) 845 : cluster 0 pgmap v868: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.2 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 752 op/s; 3318/60099 objects degraded (5.521%); 3318/60099 objects misplaced (5.521%); 656 KiB/s, 54 objects/s recovering 2023-09-23T18:46:21.476 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:21.495 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:21.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:21.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:21.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:21 smithi173 bash[59422]: cluster 2023-09-23T18:46:20.033609+0000 mgr.y (mgr.26054) 845 : cluster 0 pgmap v868: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.2 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 752 op/s; 3318/60099 objects degraded (5.521%); 3318/60099 objects misplaced (5.521%); 656 KiB/s, 54 objects/s recovering 2023-09-23T18:46:21.598 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:21.623 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:21.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:21.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:21.727 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:21.763 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:21.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:21.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:21.869 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:21.887 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:21.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:21.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:21.989 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:22.030 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:22.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:22.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:22.136 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:22.164 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:22.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:22.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:22.267 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:22.300 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:22.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:22.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:22.347 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":867,"stamp":"2023-09-23T18:46:18.032550+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1448089547,"num_objects":22052,"num_object_clones":0,"num_object_copies":66156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3551,"num_objects_misplaced":3552,"num_objects_unfound":0,"num_objects_dirty":22052,"num_whiteouts":0,"num_read":6246,"num_read_kb":13703,"num_write":180968,"num_write_kb":6475448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4645,"num_bytes_recovered":285803696,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":200477,"ondisk_log_size":200477,"up":297,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":303,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":7981648,"kb_used_data":4800600,"kb_used_omap":31,"kb_used_meta":3180896,"kb_avail":648234416,"statfs":{"total":671965249536,"available":663792041984,"internally_reserved":0,"allocated":4915814400,"data_stored":4910400818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32507,"internal_metadata":3257237765},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-503316480,"num_objects":-7680,"num_object_clones":0,"num_object_copies":-23040,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-303,"num_objects_misplaced":-548,"num_objects_unfound":0,"num_objects_dirty":-7680,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7680,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":425,"num_bytes_recovered":8060928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004479"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":68,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:13.245124+0000","last_change":"2023-09-23T18:37:40.992912+0000","last_active":"2023-09-23T18:45:13.245124+0000","last_peered":"2023-09-23T18:45:13.245124+0000","last_clean":"2023-09-23T18:45:13.245124+0000","last_became_active":"2023-09-23T18:37:40.992453+0000","last_became_peered":"2023-09-23T18:37:40.992453+0000","last_unstale":"2023-09-23T18:45:13.245124+0000","last_undegraded":"2023-09-23T18:45:13.245124+0000","last_fullsized":"2023-09-23T18:45:13.245124+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:49.539251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":150,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:42.997946+0000","last_change":"2023-09-23T18:40:42.051229+0000","last_active":"2023-09-23T18:45:42.997946+0000","last_peered":"2023-09-23T18:45:42.997946+0000","last_clean":"2023-09-23T18:45:42.997946+0000","last_became_active":"2023-09-23T18:40:42.050622+0000","last_became_peered":"2023-09-23T18:40:42.050622+0000","last_unstale":"2023-09-23T18:45:42.997946+0000","last_undegraded":"2023-09-23T18:45:42.997946+0000","last_fullsized":"2023-09-23T18:45:42.997946+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:17.587802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":149,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:45:50.112153+0000","last_change":"2023-09-23T18:33:53.848655+0000","last_active":"2023-09-23T18:45:50.112153+0000","last_peered":"2023-09-23T18:45:50.112153+0000","last_clean":"2023-09-23T18:45:50.112153+0000","last_became_active":"2023-09-23T18:33:53.848279+0000","last_became_peered":"2023-09-23T18:33:53.848279+0000","last_unstale":"2023-09-23T18:45:50.112153+0000","last_undegraded":"2023-09-23T18:45:50.112153+0000","last_fullsized":"2023-09-23T18:45:50.112153+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:39.223612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":184,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.085388+0000","last_change":"2023-09-23T18:40:42.052899+0000","last_active":"2023-09-23T18:45:12.085388+0000","last_peered":"2023-09-23T18:45:12.085388+0000","last_clean":"2023-09-23T18:45:12.085388+0000","last_became_active":"2023-09-23T18:40:42.052569+0000","last_became_peered":"2023-09-23T18:40:42.052569+0000","last_unstale":"2023-09-23T18:45:12.085388+0000","last_undegraded":"2023-09-23T18:45:12.085388+0000","last_fullsized":"2023-09-23T18:45:12.085388+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:46.151073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":166,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:38.312835+0000","last_change":"2023-09-23T18:40:42.048469+0000","last_active":"2023-09-23T18:45:38.312835+0000","last_peered":"2023-09-23T18:45:38.312835+0000","last_clean":"2023-09-23T18:45:38.312835+0000","last_became_active":"2023-09-23T18:40:42.048139+0000","last_became_peered":"2023-09-23T18:40:42.048139+0000","last_unstale":"2023-09-23T18:45:38.312835+0000","last_undegraded":"2023-09-23T18:45:38.312835+0000","last_fullsized":"2023-09-23T18:45:38.312835+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:09:09.535543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":200,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.094131+0000","last_change":"2023-09-23T18:33:55.084557+0000","last_active":"2023-09-23T18:45:12.094131+0000","last_peered":"2023-09-23T18:45:12.094131+0000","last_clean":"2023-09-23T18:45:12.094131+0000","last_became_active":"2023-09-23T18:33:55.083987+0000","last_became_peered":"2023-09-23T18:33:55.083987+0000","last_unstale":"2023-09-23T18:45:12.094131+0000","last_undegraded":"2023-09-23T18:45:12.094131+0000","last_fullsized":"2023-09-23T18:45:12.094131+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.13","version":"209'7923","reported_seq":7968,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:46:16.566825+0000","last_change":"2023-09-23T18:45:09.503802+0000","last_active":"2023-09-23T18:46:16.566825+0000","last_peered":"2023-09-23T18:46:16.566825+0000","last_clean":"2023-09-23T18:46:16.566825+0000","last_became_active":"2023-09-23T18:43:59.486869+0000","last_became_peered":"2023-09-23T18:43:59.486869+0000","last_unstale":"2023-09-23T18:46:16.566825+0000","last_undegraded":"2023-09-23T18:46:16.566825+0000","last_fullsized":"2023-09-23T18:46:16.566825+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":195,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":7923,"log_dups_size":0,"ondisk_log_size":7923,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43745280,"num_objects":668,"num_object_clones":0,"num_object_copies":2004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":668,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5580,"num_write_kb":199904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":182,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:45:50.128828+0000","last_change":"2023-09-23T18:40:42.054577+0000","last_active":"2023-09-23T18:45:50.128828+0000","last_peered":"2023-09-23T18:45:50.128828+0000","last_clean":"2023-09-23T18:45:50.128828+0000","last_became_active":"2023-09-23T18:40:42.054170+0000","last_became_peered":"2023-09-23T18:40:42.054170+0000","last_unstale":"2023-09-23T18:45:50.128828+0000","last_undegraded":"2023-09-23T18:45:50.128828+0000","last_fullsized":"2023-09-23T18:45:50.128828+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:35.503748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":207,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.082729+0000","last_change":"2023-09-23T18:37:40.998526+0000","last_active":"2023-09-23T18:45:12.082729+0000","last_peered":"2023-09-23T18:45:12.082729+0000","last_clean":"2023-09-23T18:45:12.082729+0000","last_became_active":"2023-09-23T18:37:40.993211+0000","last_became_peered":"2023-09-23T18:37:40.993211+0000","last_unstale":"2023-09-23T18:45:12.082729+0000","last_undegraded":"2023-09-23T18:45:12.082729+0000","last_fullsized":"2023-09-23T18:45:12.082729+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:40.463496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.12","version":"209'7959","reported_seq":8003,"reported_epoch":209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:46:13.111173+0000","last_change":"2023-09-23T18:44:18.828584+0000","last_active":"2023-09-23T18:46:13.111173+0000","last_peered":"2023-09-23T18:46:13.111173+0000","last_clean":"2023-09-23T18:44:14.786892+0000","last_became_active":"2023-09-23T18:44:18.822457+0000","last_became_peered":"2023-09-23T18:44:18.822457+0000","last_unstale":"2023-09-23T18:46:13.111173+0000","last_undegraded":"2023-09-23T18:44:18.138578+0000","last_fullsized":"2023-09-23T18:44:18.063005+0000","mapping_epoch":204,"log_start":"192'1402","ondisk_log_start":"192'1402","created":195,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6557,"log_dups_size":0,"ondisk_log_size":6557,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48365568,"num_objects":738,"num_object_clones":0,"num_object_copies":2214,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2794,"num_objects_misplaced":2794,"num_objects_unfound":0,"num_objects_dirty":738,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5616,"num_write_kb":203328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[1,2],"avail_no_missing":["1","2","3"],"object_location_counts":[{"shards":"1,2","objects":-2056},{"shards":"1,2,3","objects":355}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":207,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.083954+0000","last_change":"2023-09-23T18:40:42.047724+0000","last_active":"2023-09-23T18:45:12.083954+0000","last_peered":"2023-09-23T18:45:12.083954+0000","last_clean":"2023-09-23T18:45:12.083954+0000","last_became_active":"2023-09-23T18:40:42.047236+0000","last_became_peered":"2023-09-23T18:40:42.047236+0000","last_unstale":"2023-09-23T18:45:12.083954+0000","last_undegraded":"2023-09-23T18:45:12.083954+0000","last_fullsized":"2023-09-23T18:45:12.083954+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":173,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.082774+0000","last_change":"2023-09-23T18:28:03.298214+0000","last_active":"2023-09-23T18:45:12.082774+0000","last_peered":"2023-09-23T18:45:12.082774+0000","last_clean":"2023-09-23T18:45:12.082774+0000","last_became_active":"2023-09-23T18:28:03.297852+0000","last_became_peered":"2023-09-23T18:28:03.297852+0000","last_unstale":"2023-09-23T18:45:12.082774+0000","last_undegraded":"2023-09-23T18:45:12.082774+0000","last_fullsized":"2023-09-23T18:45:12.082774+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.146541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.11","version":"209'8014","reported_seq":14152,"reported_epoch":209,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:46:13.142433+0000","last_change":"2023-09-23T18:45:10.697241+0000","last_active":"2023-09-23T18:46:13.142433+0000","last_peered":"2023-09-23T18:46:13.142433+0000","last_clean":"2023-09-23T18:44:14.168767+0000","last_became_active":"2023-09-23T18:44:16.374974+0000","last_became_peered":"2023-09-23T18:44:16.374974+0000","last_unstale":"2023-09-23T18:46:13.142433+0000","last_undegraded":"2023-09-23T18:44:16.161569+0000","last_fullsized":"2023-09-23T18:44:16.057209+0000","mapping_epoch":202,"log_start":"192'1432","ondisk_log_start":"192'1432","created":195,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6582,"log_dups_size":0,"ondisk_log_size":6582,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50036736,"num_objects":764,"num_object_clones":0,"num_object_copies":2292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":757,"num_objects_misplaced":758,"num_objects_unfound":0,"num_objects_dirty":764,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5654,"num_write_kb":205344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2030,"num_bytes_recovered":113246208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[1,6],"avail_no_missing":["1","4","6"],"object_location_counts":[{"shards":"1,6","objects":6},{"shards":"1,6,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":178,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.083691+0000","last_change":"2023-09-23T18:28:03.296913+0000","last_active":"2023-09-23T18:45:12.083691+0000","last_peered":"2023-09-23T18:45:12.083691+0000","last_clean":"2023-09-23T18:45:12.083691+0000","last_became_active":"2023-09-23T18:28:03.296499+0000","last_became_peered":"2023-09-23T18:28:03.296499+0000","last_unstale":"2023-09-23T18:45:12.083691+0000","last_undegraded":"2023-09-23T18:45:12.083691+0000","last_fullsized":"2023-09-23T18:45:12.083691+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":145,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:32.971236+0000","last_change":"2023-09-23T18:33:53.850096+0000","last_active":"2023-09-23T18:45:32.971236+0000","last_peered":"2023-09-23T18:45:32.971236+0000","last_clean":"2023-09-23T18:45:32.971236+0000","last_became_active":"2023-09-23T18:33:53.849811+0000","last_became_peered":"2023-09-23T18:33:53.849811+0000","last_unstale":"2023-09-23T18:45:32.971236+0000","last_undegraded":"2023-09-23T18:45:32.971236+0000","last_fullsized":"2023-09-23T18:45:32.971236+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:44.938072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.10","version":"209'8006","reported_seq":15766,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:46:14.294390+0000","last_change":"2023-09-23T18:45:40.017102+0000","last_active":"2023-09-23T18:46:14.294390+0000","last_peered":"2023-09-23T18:46:14.294390+0000","last_clean":"2023-09-23T18:46:14.294390+0000","last_became_active":"2023-09-23T18:45:12.085388+0000","last_became_peered":"2023-09-23T18:45:12.085388+0000","last_unstale":"2023-09-23T18:46:14.294390+0000","last_undegraded":"2023-09-23T18:46:14.294390+0000","last_fullsized":"2023-09-23T18:46:14.294390+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":195,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":8006,"log_dups_size":0,"ondisk_log_size":8006,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47775744,"num_objects":729,"num_object_clones":0,"num_object_copies":2187,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":729,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5637,"num_write_kb":203712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2568,"num_bytes_recovered":168296448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":182,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.079780+0000","last_change":"2023-09-23T18:15:08.909339+0000","last_active":"2023-09-23T18:45:12.079780+0000","last_peered":"2023-09-23T18:45:12.079780+0000","last_clean":"2023-09-23T18:45:12.079780+0000","last_became_active":"2023-09-23T18:15:08.908912+0000","last_became_peered":"2023-09-23T18:15:08.908912+0000","last_unstale":"2023-09-23T18:45:12.079780+0000","last_undegraded":"2023-09-23T18:45:12.079780+0000","last_fullsized":"2023-09-23T18:45:12.079780+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":189,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.079330+0000","last_change":"2023-09-23T18:33:53.847489+0000","last_active":"2023-09-23T18:45:12.079330+0000","last_peered":"2023-09-23T18:45:12.079330+0000","last_clean":"2023-09-23T18:45:12.079330+0000","last_became_active":"2023-09-23T18:33:53.846935+0000","last_became_peered":"2023-09-23T18:33:53.846935+0000","last_unstale":"2023-09-23T18:45:12.079330+0000","last_undegraded":"2023-09-23T18:45:12.079330+0000","last_fullsized":"2023-09-23T18:45:12.079330+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"209'7949","reported_seq":7987,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:46:15.827746+0000","last_change":"2023-09-23T18:43:59.501690+0000","last_active":"2023-09-23T18:46:15.827746+0000","last_peered":"2023-09-23T18:46:15.827746+0000","last_clean":"2023-09-23T18:46:15.827746+0000","last_became_active":"2023-09-23T18:43:59.497605+0000","last_became_peered":"2023-09-23T18:43:59.497605+0000","last_unstale":"2023-09-23T18:46:15.827746+0000","last_undegraded":"2023-09-23T18:46:15.827746+0000","last_fullsized":"2023-09-23T18:46:15.827746+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":195,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":7949,"log_dups_size":0,"ondisk_log_size":7949,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44302336,"num_objects":676,"num_object_clones":0,"num_object_copies":2028,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":676,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5590,"num_write_kb":200512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":177,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.083559+0000","last_change":"2023-09-23T18:27:33.585121+0000","last_active":"2023-09-23T18:45:12.083559+0000","last_peered":"2023-09-23T18:45:12.083559+0000","last_clean":"2023-09-23T18:45:12.083559+0000","last_became_active":"2023-09-23T18:27:33.584620+0000","last_became_peered":"2023-09-23T18:27:33.584620+0000","last_unstale":"2023-09-23T18:45:12.083559+0000","last_undegraded":"2023-09-23T18:45:12.083559+0000","last_fullsized":"2023-09-23T18:45:12.083559+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":198,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.082826+0000","last_change":"2023-09-23T18:33:53.852933+0000","last_active":"2023-09-23T18:45:12.082826+0000","last_peered":"2023-09-23T18:45:12.082826+0000","last_clean":"2023-09-23T18:45:12.082826+0000","last_became_active":"2023-09-23T18:33:53.852626+0000","last_became_peered":"2023-09-23T18:33:53.852626+0000","last_unstale":"2023-09-23T18:45:12.082826+0000","last_undegraded":"2023-09-23T18:45:12.082826+0000","last_fullsized":"2023-09-23T18:45:12.082826+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:00.174536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.16","version":"209'8063","reported_seq":8108,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:46:13.329067+0000","last_change":"2023-09-23T18:45:38.346598+0000","last_active":"2023-09-23T18:46:13.329067+0000","last_peered":"2023-09-23T18:46:13.329067+0000","last_clean":"2023-09-23T18:46:13.329067+0000","last_became_active":"2023-09-23T18:43:59.502108+0000","last_became_peered":"2023-09-23T18:43:59.502108+0000","last_unstale":"2023-09-23T18:46:13.329067+0000","last_undegraded":"2023-09-23T18:46:13.329067+0000","last_fullsized":"2023-09-23T18:46:13.329067+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":195,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":8063,"log_dups_size":0,"ondisk_log_size":8063,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50757632,"num_objects":775,"num_object_clones":0,"num_object_copies":2325,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":775,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5669,"num_write_kb":206176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":173,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:13.245145+0000","last_change":"2023-09-23T18:39:46.997790+0000","last_active":"2023-09-23T18:45:13.245145+0000","last_peered":"2023-09-23T18:45:13.245145+0000","last_clean":"2023-09-23T18:45:13.245145+0000","last_became_active":"2023-09-23T18:39:46.997089+0000","last_became_peered":"2023-09-23T18:39:46.997089+0000","last_unstale":"2023-09-23T18:45:13.245145+0000","last_undegraded":"2023-09-23T18:45:13.245145+0000","last_fullsized":"2023-09-23T18:45:13.245145+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:55.378220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":166,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.083942+0000","last_change":"2023-09-23T18:32:26.808237+0000","last_active":"2023-09-23T18:45:12.083942+0000","last_peered":"2023-09-23T18:45:12.083942+0000","last_clean":"2023-09-23T18:45:12.083942+0000","last_became_active":"2023-09-23T18:32:26.807843+0000","last_became_peered":"2023-09-23T18:32:26.807843+0000","last_unstale":"2023-09-23T18:45:12.083942+0000","last_undegraded":"2023-09-23T18:45:12.083942+0000","last_fullsized":"2023-09-23T18:45:12.083942+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.15","version":"209'8054","reported_seq":8093,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:46:15.888240+0000","last_change":"2023-09-23T18:43:59.480193+0000","last_active":"2023-09-23T18:46:15.888240+0000","last_peered":"2023-09-23T18:46:15.888240+0000","last_clean":"2023-09-23T18:46:15.888240+0000","last_became_active":"2023-09-23T18:43:59.477573+0000","last_became_peered":"2023-09-23T18:43:59.477573+0000","last_unstale":"2023-09-23T18:46:15.888240+0000","last_undegraded":"2023-09-23T18:46:15.888240+0000","last_fullsized":"2023-09-23T18:46:15.888240+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":195,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":8054,"log_dups_size":0,"ondisk_log_size":8054,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43155456,"num_objects":659,"num_object_clones":0,"num_object_copies":1977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":659,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5703,"num_write_kb":203552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":160,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.083326+0000","last_change":"2023-09-23T18:15:08.916658+0000","last_active":"2023-09-23T18:45:12.083326+0000","last_peered":"2023-09-23T18:45:12.083326+0000","last_clean":"2023-09-23T18:45:12.083326+0000","last_became_active":"2023-09-23T18:15:08.916301+0000","last_became_peered":"2023-09-23T18:15:08.916301+0000","last_unstale":"2023-09-23T18:45:12.083326+0000","last_undegraded":"2023-09-23T18:45:12.083326+0000","last_fullsized":"2023-09-23T18:45:12.083326+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":174,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:45:50.123093+0000","last_change":"2023-09-23T18:40:42.049370+0000","last_active":"2023-09-23T18:45:50.123093+0000","last_peered":"2023-09-23T18:45:50.123093+0000","last_clean":"2023-09-23T18:45:50.123093+0000","last_became_active":"2023-09-23T18:40:42.049003+0000","last_became_peered":"2023-09-23T18:40:42.049003+0000","last_unstale":"2023-09-23T18:45:50.123093+0000","last_undegraded":"2023-09-23T18:45:50.123093+0000","last_fullsized":"2023-09-23T18:45:50.123093+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:15.760457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"209'7920","reported_seq":7965,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:46:13.353365+0000","last_change":"2023-09-23T18:45:42.315826+0000","last_active":"2023-09-23T18:46:13.353365+0000","last_peered":"2023-09-23T18:46:13.353365+0000","last_clean":"2023-09-23T18:46:13.353365+0000","last_became_active":"2023-09-23T18:43:59.502491+0000","last_became_peered":"2023-09-23T18:43:59.502491+0000","last_unstale":"2023-09-23T18:46:13.353365+0000","last_undegraded":"2023-09-23T18:46:13.353365+0000","last_fullsized":"2023-09-23T18:46:13.353365+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":195,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":7920,"log_dups_size":0,"ondisk_log_size":7920,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48955392,"num_objects":747,"num_object_clones":0,"num_object_copies":2241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5573,"num_write_kb":202240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":221,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.092750+0000","last_change":"2023-09-23T18:40:42.073698+0000","last_active":"2023-09-23T18:45:12.092750+0000","last_peered":"2023-09-23T18:45:12.092750+0000","last_clean":"2023-09-23T18:45:12.092750+0000","last_became_active":"2023-09-23T18:40:42.059101+0000","last_became_peered":"2023-09-23T18:40:42.059101+0000","last_unstale":"2023-09-23T18:45:12.092750+0000","last_undegraded":"2023-09-23T18:45:12.092750+0000","last_fullsized":"2023-09-23T18:45:12.092750+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":153,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:32.971347+0000","last_change":"2023-09-23T18:40:42.058656+0000","last_active":"2023-09-23T18:45:32.971347+0000","last_peered":"2023-09-23T18:45:32.971347+0000","last_clean":"2023-09-23T18:45:32.971347+0000","last_became_active":"2023-09-23T18:40:42.057307+0000","last_became_peered":"2023-09-23T18:40:42.057307+0000","last_unstale":"2023-09-23T18:45:32.971347+0000","last_undegraded":"2023-09-23T18:45:32.971347+0000","last_fullsized":"2023-09-23T18:45:32.971347+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:08:45.979078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.b","version":"209'11094","reported_seq":11126,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:46:13.123418+0000","last_change":"2023-09-23T18:42:39.314324+0000","last_active":"2023-09-23T18:46:13.123418+0000","last_peered":"2023-09-23T18:46:13.123418+0000","last_clean":"2023-09-23T18:46:13.123418+0000","last_became_active":"2023-09-23T18:42:39.314006+0000","last_became_peered":"2023-09-23T18:42:39.314006+0000","last_unstale":"2023-09-23T18:46:13.123418+0000","last_undegraded":"2023-09-23T18:46:13.123418+0000","last_fullsized":"2023-09-23T18:46:13.123418+0000","mapping_epoch":187,"log_start":"205'6100","ondisk_log_start":"205'6100","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":4994,"log_dups_size":0,"ondisk_log_size":4994,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:52.663013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93847552,"num_objects":1432,"num_object_clones":0,"num_object_copies":4296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1432,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11094,"num_write_kb":400832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":189,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.084233+0000","last_change":"2023-09-23T18:37:41.034732+0000","last_active":"2023-09-23T18:45:12.084233+0000","last_peered":"2023-09-23T18:45:12.084233+0000","last_clean":"2023-09-23T18:45:12.084233+0000","last_became_active":"2023-09-23T18:37:40.985865+0000","last_became_peered":"2023-09-23T18:37:40.985865+0000","last_unstale":"2023-09-23T18:45:12.084233+0000","last_undegraded":"2023-09-23T18:45:12.084233+0000","last_fullsized":"2023-09-23T18:45:12.084233+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":107,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.084192+0000","last_change":"2023-09-23T18:32:26.806328+0000","last_active":"2023-09-23T18:45:12.084192+0000","last_peered":"2023-09-23T18:45:12.084192+0000","last_clean":"2023-09-23T18:45:12.084192+0000","last_became_active":"2023-09-23T18:32:26.803969+0000","last_became_peered":"2023-09-23T18:32:26.803969+0000","last_unstale":"2023-09-23T18:45:12.084192+0000","last_undegraded":"2023-09-23T18:45:12.084192+0000","last_fullsized":"2023-09-23T18:45:12.084192+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.a","version":"209'11149","reported_seq":11179,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:46:16.593569+0000","last_change":"2023-09-23T18:42:39.310080+0000","last_active":"2023-09-23T18:46:16.593569+0000","last_peered":"2023-09-23T18:46:16.593569+0000","last_clean":"2023-09-23T18:46:16.593569+0000","last_became_active":"2023-09-23T18:42:39.309414+0000","last_became_peered":"2023-09-23T18:42:39.309414+0000","last_unstale":"2023-09-23T18:46:16.593569+0000","last_undegraded":"2023-09-23T18:46:16.593569+0000","last_fullsized":"2023-09-23T18:46:16.593569+0000","mapping_epoch":187,"log_start":"192'1100","ondisk_log_start":"192'1100","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":10049,"log_dups_size":0,"ondisk_log_size":10049,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:43.822106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82378752,"num_objects":1257,"num_object_clones":0,"num_object_copies":3771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11149,"num_write_kb":396992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":192,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.083920+0000","last_change":"2023-09-23T18:33:53.847817+0000","last_active":"2023-09-23T18:45:12.083920+0000","last_peered":"2023-09-23T18:45:12.083920+0000","last_clean":"2023-09-23T18:45:12.083920+0000","last_became_active":"2023-09-23T18:33:53.847342+0000","last_became_peered":"2023-09-23T18:33:53.847342+0000","last_unstale":"2023-09-23T18:45:12.083920+0000","last_undegraded":"2023-09-23T18:45:12.083920+0000","last_fullsized":"2023-09-23T18:45:12.083920+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":213,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.079216+0000","last_change":"2023-09-23T18:39:46.995444+0000","last_active":"2023-09-23T18:45:12.079216+0000","last_peered":"2023-09-23T18:45:12.079216+0000","last_clean":"2023-09-23T18:45:12.079216+0000","last_became_active":"2023-09-23T18:39:46.995033+0000","last_became_peered":"2023-09-23T18:39:46.995033+0000","last_unstale":"2023-09-23T18:45:12.079216+0000","last_undegraded":"2023-09-23T18:45:12.079216+0000","last_fullsized":"2023-09-23T18:45:12.079216+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"209'7853","reported_seq":7899,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:46:13.314898+0000","last_change":"2023-09-23T18:45:45.718634+0000","last_active":"2023-09-23T18:46:13.314898+0000","last_peered":"2023-09-23T18:46:13.314898+0000","last_clean":"2023-09-23T18:46:13.314898+0000","last_became_active":"2023-09-23T18:43:59.504968+0000","last_became_peered":"2023-09-23T18:43:59.504968+0000","last_unstale":"2023-09-23T18:46:13.314898+0000","last_undegraded":"2023-09-23T18:46:13.314898+0000","last_fullsized":"2023-09-23T18:46:13.314898+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":7853,"log_dups_size":0,"ondisk_log_size":7853,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44302336,"num_objects":676,"num_object_clones":0,"num_object_copies":2028,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":676,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5520,"num_write_kb":198272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":174,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.083531+0000","last_change":"2023-09-23T18:32:26.854615+0000","last_active":"2023-09-23T18:45:12.083531+0000","last_peered":"2023-09-23T18:45:12.083531+0000","last_clean":"2023-09-23T18:45:12.083531+0000","last_became_active":"2023-09-23T18:32:26.806771+0000","last_became_peered":"2023-09-23T18:32:26.806771+0000","last_unstale":"2023-09-23T18:45:12.083531+0000","last_undegraded":"2023-09-23T18:45:12.083531+0000","last_fullsized":"2023-09-23T18:45:12.083531+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":183,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.093223+0000","last_change":"2023-09-23T18:27:33.589116+0000","last_active":"2023-09-23T18:45:12.093223+0000","last_peered":"2023-09-23T18:45:12.093223+0000","last_clean":"2023-09-23T18:45:12.093223+0000","last_became_active":"2023-09-23T18:27:33.588809+0000","last_became_peered":"2023-09-23T18:27:33.588809+0000","last_unstale":"2023-09-23T18:45:12.093223+0000","last_undegraded":"2023-09-23T18:45:12.093223+0000","last_fullsized":"2023-09-23T18:45:12.093223+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"209'8172","reported_seq":8218,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:46:14.176753+0000","last_change":"2023-09-23T18:45:43.111577+0000","last_active":"2023-09-23T18:46:14.176753+0000","last_peered":"2023-09-23T18:46:14.176753+0000","last_clean":"2023-09-23T18:46:14.176753+0000","last_became_active":"2023-09-23T18:43:59.458263+0000","last_became_peered":"2023-09-23T18:43:59.458263+0000","last_unstale":"2023-09-23T18:46:14.176753+0000","last_undegraded":"2023-09-23T18:46:14.176753+0000","last_fullsized":"2023-09-23T18:46:14.176753+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":8172,"log_dups_size":0,"ondisk_log_size":8172,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42565632,"num_objects":649,"num_object_clones":0,"num_object_copies":1947,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":649,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5775,"num_write_kb":205600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":184,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:45:50.118939+0000","last_change":"2023-09-23T18:33:54.848776+0000","last_active":"2023-09-23T18:45:50.118939+0000","last_peered":"2023-09-23T18:45:50.118939+0000","last_clean":"2023-09-23T18:45:50.118939+0000","last_became_active":"2023-09-23T18:33:54.848380+0000","last_became_peered":"2023-09-23T18:33:54.848380+0000","last_unstale":"2023-09-23T18:45:50.118939+0000","last_undegraded":"2023-09-23T18:45:50.118939+0000","last_fullsized":"2023-09-23T18:45:50.118939+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:17:17.093993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":177,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.084953+0000","last_change":"2023-09-23T18:27:33.582033+0000","last_active":"2023-09-23T18:45:12.084953+0000","last_peered":"2023-09-23T18:45:12.084953+0000","last_clean":"2023-09-23T18:45:12.084953+0000","last_became_active":"2023-09-23T18:27:33.581667+0000","last_became_peered":"2023-09-23T18:27:33.581667+0000","last_unstale":"2023-09-23T18:45:12.084953+0000","last_undegraded":"2023-09-23T18:45:12.084953+0000","last_fullsized":"2023-09-23T18:45:12.084953+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:36.039922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"209'11170","reported_seq":11201,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:46:13.386931+0000","last_change":"2023-09-23T18:42:39.318187+0000","last_active":"2023-09-23T18:46:13.386931+0000","last_peered":"2023-09-23T18:46:13.386931+0000","last_clean":"2023-09-23T18:46:13.386931+0000","last_became_active":"2023-09-23T18:42:39.317693+0000","last_became_peered":"2023-09-23T18:42:39.317693+0000","last_unstale":"2023-09-23T18:46:13.386931+0000","last_undegraded":"2023-09-23T18:46:13.386931+0000","last_fullsized":"2023-09-23T18:46:13.386931+0000","mapping_epoch":187,"log_start":"196'4900","ondisk_log_start":"196'4900","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6270,"log_dups_size":0,"ondisk_log_size":6270,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:53.117569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96600064,"num_objects":1474,"num_object_clones":0,"num_object_copies":4422,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1474,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11170,"num_write_kb":404608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":183,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:45:50.120321+0000","last_change":"2023-09-23T18:28:03.297628+0000","last_active":"2023-09-23T18:45:50.120321+0000","last_peered":"2023-09-23T18:45:50.120321+0000","last_clean":"2023-09-23T18:45:50.120321+0000","last_became_active":"2023-09-23T18:28:03.296694+0000","last_became_peered":"2023-09-23T18:28:03.296694+0000","last_unstale":"2023-09-23T18:45:50.120321+0000","last_undegraded":"2023-09-23T18:45:50.120321+0000","last_fullsized":"2023-09-23T18:45:50.120321+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:43.375652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":174,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.079599+0000","last_change":"2023-09-23T18:28:03.302326+0000","last_active":"2023-09-23T18:45:12.079599+0000","last_peered":"2023-09-23T18:45:12.079599+0000","last_clean":"2023-09-23T18:45:12.079599+0000","last_became_active":"2023-09-23T18:28:03.301970+0000","last_became_peered":"2023-09-23T18:28:03.301970+0000","last_unstale":"2023-09-23T18:45:12.079599+0000","last_undegraded":"2023-09-23T18:45:12.079599+0000","last_fullsized":"2023-09-23T18:45:12.079599+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"209'11457","reported_seq":11489,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:46:14.400412+0000","last_change":"2023-09-23T18:42:39.309814+0000","last_active":"2023-09-23T18:46:14.400412+0000","last_peered":"2023-09-23T18:46:14.400412+0000","last_clean":"2023-09-23T18:46:14.400412+0000","last_became_active":"2023-09-23T18:42:39.309499+0000","last_became_peered":"2023-09-23T18:42:39.309499+0000","last_unstale":"2023-09-23T18:46:14.400412+0000","last_undegraded":"2023-09-23T18:46:14.400412+0000","last_fullsized":"2023-09-23T18:46:14.400412+0000","mapping_epoch":187,"log_start":"192'1400","ondisk_log_start":"192'1400","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":10057,"log_dups_size":0,"ondisk_log_size":10057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:26.106244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94175232,"num_objects":1437,"num_object_clones":0,"num_object_copies":4311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11457,"num_write_kb":412608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":190,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.092967+0000","last_change":"2023-09-23T18:33:53.855371+0000","last_active":"2023-09-23T18:45:12.092967+0000","last_peered":"2023-09-23T18:45:12.092967+0000","last_clean":"2023-09-23T18:45:12.092967+0000","last_became_active":"2023-09-23T18:33:53.854996+0000","last_became_peered":"2023-09-23T18:33:53.854996+0000","last_unstale":"2023-09-23T18:45:12.092967+0000","last_undegraded":"2023-09-23T18:45:12.092967+0000","last_fullsized":"2023-09-23T18:45:12.092967+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":213,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.079474+0000","last_change":"2023-09-23T18:40:42.060184+0000","last_active":"2023-09-23T18:45:12.079474+0000","last_peered":"2023-09-23T18:45:12.079474+0000","last_clean":"2023-09-23T18:45:12.079474+0000","last_became_active":"2023-09-23T18:40:42.059725+0000","last_became_peered":"2023-09-23T18:40:42.059725+0000","last_unstale":"2023-09-23T18:45:12.079474+0000","last_undegraded":"2023-09-23T18:45:12.079474+0000","last_fullsized":"2023-09-23T18:45:12.079474+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"209'11078","reported_seq":11109,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:46:15.880534+0000","last_change":"2023-09-23T18:42:39.319875+0000","last_active":"2023-09-23T18:46:15.880534+0000","last_peered":"2023-09-23T18:46:15.880534+0000","last_clean":"2023-09-23T18:46:15.880534+0000","last_became_active":"2023-09-23T18:42:39.319468+0000","last_became_peered":"2023-09-23T18:42:39.319468+0000","last_unstale":"2023-09-23T18:46:15.880534+0000","last_undegraded":"2023-09-23T18:46:15.880534+0000","last_fullsized":"2023-09-23T18:46:15.880534+0000","mapping_epoch":187,"log_start":"197'4900","ondisk_log_start":"197'4900","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6178,"log_dups_size":0,"ondisk_log_size":6178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83165208,"num_objects":1270,"num_object_clones":0,"num_object_copies":3810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1270,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11078,"num_write_kb":395073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":175,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.078841+0000","last_change":"2023-09-23T18:27:33.585714+0000","last_active":"2023-09-23T18:45:12.078841+0000","last_peered":"2023-09-23T18:45:12.078841+0000","last_clean":"2023-09-23T18:45:12.078841+0000","last_became_active":"2023-09-23T18:27:33.584641+0000","last_became_peered":"2023-09-23T18:27:33.584641+0000","last_unstale":"2023-09-23T18:45:12.078841+0000","last_undegraded":"2023-09-23T18:45:12.078841+0000","last_fullsized":"2023-09-23T18:45:12.078841+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":189,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.079856+0000","last_change":"2023-09-23T18:33:53.851266+0000","last_active":"2023-09-23T18:45:12.079856+0000","last_peered":"2023-09-23T18:45:12.079856+0000","last_clean":"2023-09-23T18:45:12.079856+0000","last_became_active":"2023-09-23T18:33:53.850926+0000","last_became_peered":"2023-09-23T18:33:53.850926+0000","last_unstale":"2023-09-23T18:45:12.079856+0000","last_undegraded":"2023-09-23T18:45:12.079856+0000","last_fullsized":"2023-09-23T18:45:12.079856+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"209'11156","reported_seq":11186,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:46:15.936335+0000","last_change":"2023-09-23T18:42:39.318852+0000","last_active":"2023-09-23T18:46:15.936335+0000","last_peered":"2023-09-23T18:46:15.936335+0000","last_clean":"2023-09-23T18:46:15.936335+0000","last_became_active":"2023-09-23T18:42:39.318461+0000","last_became_peered":"2023-09-23T18:42:39.318461+0000","last_unstale":"2023-09-23T18:46:15.936335+0000","last_undegraded":"2023-09-23T18:46:15.936335+0000","last_fullsized":"2023-09-23T18:46:15.936335+0000","mapping_epoch":187,"log_start":"197'5000","ondisk_log_start":"197'5000","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6156,"log_dups_size":0,"ondisk_log_size":6156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86769664,"num_objects":1324,"num_object_clones":0,"num_object_copies":3972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11156,"num_write_kb":399360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":161,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.077761+0000","last_change":"2023-09-23T18:39:46.993941+0000","last_active":"2023-09-23T18:45:12.077761+0000","last_peered":"2023-09-23T18:45:12.077761+0000","last_clean":"2023-09-23T18:45:12.077761+0000","last_became_active":"2023-09-23T18:39:46.993625+0000","last_became_peered":"2023-09-23T18:39:46.993625+0000","last_unstale":"2023-09-23T18:45:12.077761+0000","last_undegraded":"2023-09-23T18:45:12.077761+0000","last_fullsized":"2023-09-23T18:45:12.077761+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:23.237229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":149,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.077759+0000","last_change":"2023-09-23T18:33:53.855430+0000","last_active":"2023-09-23T18:45:12.077759+0000","last_peered":"2023-09-23T18:45:12.077759+0000","last_clean":"2023-09-23T18:45:12.077759+0000","last_became_active":"2023-09-23T18:33:53.854935+0000","last_became_peered":"2023-09-23T18:33:53.854935+0000","last_unstale":"2023-09-23T18:45:12.077759+0000","last_undegraded":"2023-09-23T18:45:12.077759+0000","last_fullsized":"2023-09-23T18:45:12.077759+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:36.078411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"209'8072","reported_seq":8155,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:46:16.230888+0000","last_change":"2023-09-23T18:45:48.236107+0000","last_active":"2023-09-23T18:46:16.230888+0000","last_peered":"2023-09-23T18:46:16.230888+0000","last_clean":"2023-09-23T18:46:16.230888+0000","last_became_active":"2023-09-23T18:43:59.506884+0000","last_became_peered":"2023-09-23T18:43:59.506884+0000","last_unstale":"2023-09-23T18:46:16.230888+0000","last_undegraded":"2023-09-23T18:46:16.230888+0000","last_fullsized":"2023-09-23T18:46:16.230888+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":8072,"log_dups_size":0,"ondisk_log_size":8072,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39550976,"num_objects":603,"num_object_clones":0,"num_object_copies":1809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5711,"num_write_kb":202080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"77'130","reported_seq":675,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.077813+0000","last_change":"2023-09-23T18:33:53.850713+0000","last_active":"2023-09-23T18:45:12.077813+0000","last_peered":"2023-09-23T18:45:12.077813+0000","last_clean":"2023-09-23T18:45:12.077813+0000","last_became_active":"2023-09-23T18:33:53.850011+0000","last_became_peered":"2023-09-23T18:33:53.850011+0000","last_unstale":"2023-09-23T18:45:12.077813+0000","last_undegraded":"2023-09-23T18:45:12.077813+0000","last_fullsized":"2023-09-23T18:45:12.077813+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:58.943556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":772,"num_read_kb":1712,"num_write":126,"num_write_kb":2126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":130,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.081166+0000","last_change":"2023-09-23T18:40:42.058197+0000","last_active":"2023-09-23T18:45:12.081166+0000","last_peered":"2023-09-23T18:45:12.081166+0000","last_clean":"2023-09-23T18:45:12.081166+0000","last_became_active":"2023-09-23T18:40:42.057834+0000","last_became_peered":"2023-09-23T18:40:42.057834+0000","last_unstale":"2023-09-23T18:45:12.081166+0000","last_undegraded":"2023-09-23T18:45:12.081166+0000","last_fullsized":"2023-09-23T18:45:12.081166+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:28.133529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":123,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.078437+0000","last_change":"2023-09-23T18:33:54.843246+0000","last_active":"2023-09-23T18:45:12.078437+0000","last_peered":"2023-09-23T18:45:12.078437+0000","last_clean":"2023-09-23T18:45:12.078437+0000","last_became_active":"2023-09-23T18:33:54.842933+0000","last_became_peered":"2023-09-23T18:33:54.842933+0000","last_unstale":"2023-09-23T18:45:12.078437+0000","last_undegraded":"2023-09-23T18:45:12.078437+0000","last_fullsized":"2023-09-23T18:45:12.078437+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:43.716707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"209'8057","reported_seq":8120,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:46:15.880343+0000","last_change":"2023-09-23T18:45:08.064245+0000","last_active":"2023-09-23T18:46:15.880343+0000","last_peered":"2023-09-23T18:46:15.880343+0000","last_clean":"2023-09-23T18:46:15.880343+0000","last_became_active":"2023-09-23T18:43:59.473142+0000","last_became_peered":"2023-09-23T18:43:59.473142+0000","last_unstale":"2023-09-23T18:46:15.880343+0000","last_undegraded":"2023-09-23T18:46:15.880343+0000","last_fullsized":"2023-09-23T18:46:15.880343+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":8057,"log_dups_size":0,"ondisk_log_size":8057,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42729472,"num_objects":652,"num_object_clones":0,"num_object_copies":1956,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":652,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5698,"num_write_kb":203200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"77'96","reported_seq":619,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.078617+0000","last_change":"2023-09-23T18:33:54.842857+0000","last_active":"2023-09-23T18:45:12.078617+0000","last_peered":"2023-09-23T18:45:12.078617+0000","last_clean":"2023-09-23T18:45:12.078617+0000","last_became_active":"2023-09-23T18:33:54.842134+0000","last_became_peered":"2023-09-23T18:33:54.842134+0000","last_unstale":"2023-09-23T18:45:12.078617+0000","last_undegraded":"2023-09-23T18:45:12.078617+0000","last_fullsized":"2023-09-23T18:45:12.078617+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":96,"log_dups_size":0,"ondisk_log_size":96,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:48.960158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":728,"num_read_kb":1234,"num_write":87,"num_write_kb":338,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":175,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.079450+0000","last_change":"2023-09-23T18:26:34.992535+0000","last_active":"2023-09-23T18:45:12.079450+0000","last_peered":"2023-09-23T18:45:12.079450+0000","last_clean":"2023-09-23T18:45:12.079450+0000","last_became_active":"2023-09-23T18:26:34.992177+0000","last_became_peered":"2023-09-23T18:26:34.992177+0000","last_unstale":"2023-09-23T18:45:12.079450+0000","last_undegraded":"2023-09-23T18:45:12.079450+0000","last_fullsized":"2023-09-23T18:45:12.079450+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":173,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.077738+0000","last_change":"2023-09-23T18:40:42.066006+0000","last_active":"2023-09-23T18:45:12.077738+0000","last_peered":"2023-09-23T18:45:12.077738+0000","last_clean":"2023-09-23T18:45:12.077738+0000","last_became_active":"2023-09-23T18:40:42.065431+0000","last_became_peered":"2023-09-23T18:40:42.065431+0000","last_unstale":"2023-09-23T18:45:12.077738+0000","last_undegraded":"2023-09-23T18:45:12.077738+0000","last_fullsized":"2023-09-23T18:45:12.077738+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:18.818629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"209'7984","reported_seq":8031,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:46:13.329587+0000","last_change":"2023-09-23T18:45:43.333622+0000","last_active":"2023-09-23T18:46:13.329587+0000","last_peered":"2023-09-23T18:46:13.329587+0000","last_clean":"2023-09-23T18:46:13.329587+0000","last_became_active":"2023-09-23T18:43:59.475213+0000","last_became_peered":"2023-09-23T18:43:59.475213+0000","last_unstale":"2023-09-23T18:46:13.329587+0000","last_undegraded":"2023-09-23T18:46:13.329587+0000","last_fullsized":"2023-09-23T18:46:13.329587+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":7984,"log_dups_size":0,"ondisk_log_size":7984,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46596096,"num_objects":711,"num_object_clones":0,"num_object_copies":2133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":711,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5637,"num_write_kb":203136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"77'112","reported_seq":560,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.084696+0000","last_change":"2023-09-23T18:40:45.106420+0000","last_active":"2023-09-23T18:45:12.084696+0000","last_peered":"2023-09-23T18:45:12.084696+0000","last_clean":"2023-09-23T18:45:12.084696+0000","last_became_active":"2023-09-23T18:40:45.104294+0000","last_became_peered":"2023-09-23T18:40:45.104294+0000","last_unstale":"2023-09-23T18:45:12.084696+0000","last_undegraded":"2023-09-23T18:45:12.084696+0000","last_fullsized":"2023-09-23T18:45:12.084696+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:45.419412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":774,"num_read_kb":596,"num_write":102,"num_write_kb":52,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":146,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:45:50.095962+0000","last_change":"2023-09-23T18:33:53.849084+0000","last_active":"2023-09-23T18:45:50.095962+0000","last_peered":"2023-09-23T18:45:50.095962+0000","last_clean":"2023-09-23T18:45:50.095962+0000","last_became_active":"2023-09-23T18:33:53.848757+0000","last_became_peered":"2023-09-23T18:33:53.848757+0000","last_unstale":"2023-09-23T18:45:50.095962+0000","last_undegraded":"2023-09-23T18:45:50.095962+0000","last_fullsized":"2023-09-23T18:45:50.095962+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:15.267742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":168,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.078108+0000","last_change":"2023-09-23T18:33:55.087304+0000","last_active":"2023-09-23T18:45:12.078108+0000","last_peered":"2023-09-23T18:45:12.078108+0000","last_clean":"2023-09-23T18:45:12.078108+0000","last_became_active":"2023-09-23T18:33:55.086832+0000","last_became_peered":"2023-09-23T18:33:55.086832+0000","last_unstale":"2023-09-23T18:45:12.078108+0000","last_undegraded":"2023-09-23T18:45:12.078108+0000","last_fullsized":"2023-09-23T18:45:12.078108+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:55.457664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"209'8137","reported_seq":8184,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:46:13.386002+0000","last_change":"2023-09-23T18:45:46.495820+0000","last_active":"2023-09-23T18:46:13.386002+0000","last_peered":"2023-09-23T18:46:13.386002+0000","last_clean":"2023-09-23T18:46:13.386002+0000","last_became_active":"2023-09-23T18:43:59.485259+0000","last_became_peered":"2023-09-23T18:43:59.485259+0000","last_unstale":"2023-09-23T18:46:13.386002+0000","last_undegraded":"2023-09-23T18:46:13.386002+0000","last_fullsized":"2023-09-23T18:46:13.386002+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":8137,"log_dups_size":0,"ondisk_log_size":8137,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47611904,"num_objects":726,"num_object_clones":0,"num_object_copies":2178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":726,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5742,"num_write_kb":207008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"77'147","reported_seq":600,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.081173+0000","last_change":"2023-09-23T18:32:29.845404+0000","last_active":"2023-09-23T18:45:12.081173+0000","last_peered":"2023-09-23T18:45:12.081173+0000","last_clean":"2023-09-23T18:45:12.081173+0000","last_became_active":"2023-09-23T18:32:29.844545+0000","last_became_peered":"2023-09-23T18:32:29.844545+0000","last_unstale":"2023-09-23T18:45:12.081173+0000","last_undegraded":"2023-09-23T18:45:12.081173+0000","last_fullsized":"2023-09-23T18:45:12.081173+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:44.834482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":624,"num_read_kb":1868,"num_write":143,"num_write_kb":5368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":411,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.083799+0000","last_change":"2023-09-23T18:26:34.999402+0000","last_active":"2023-09-23T18:45:12.083799+0000","last_peered":"2023-09-23T18:45:12.083799+0000","last_clean":"2023-09-23T18:45:12.083799+0000","last_became_active":"2023-09-23T18:26:34.998828+0000","last_became_peered":"2023-09-23T18:26:34.998828+0000","last_unstale":"2023-09-23T18:45:12.083799+0000","last_undegraded":"2023-09-23T18:45:12.083799+0000","last_fullsized":"2023-09-23T18:45:12.083799+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_clean_scrub_stamp":"2023-09-23T18:08:35.701374+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":124,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.093361+0000","last_change":"2023-09-23T18:40:42.058559+0000","last_active":"2023-09-23T18:45:12.093361+0000","last_peered":"2023-09-23T18:45:12.093361+0000","last_clean":"2023-09-23T18:45:12.093361+0000","last_became_active":"2023-09-23T18:40:42.058253+0000","last_became_peered":"2023-09-23T18:40:42.058253+0000","last_unstale":"2023-09-23T18:45:12.093361+0000","last_undegraded":"2023-09-23T18:45:12.093361+0000","last_fullsized":"2023-09-23T18:45:12.093361+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":159,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.093931+0000","last_change":"2023-09-23T18:15:08.921629+0000","last_active":"2023-09-23T18:45:12.093931+0000","last_peered":"2023-09-23T18:45:12.093931+0000","last_clean":"2023-09-23T18:45:12.093931+0000","last_became_active":"2023-09-23T18:15:08.921283+0000","last_became_peered":"2023-09-23T18:15:08.921283+0000","last_unstale":"2023-09-23T18:45:12.093931+0000","last_undegraded":"2023-09-23T18:45:12.093931+0000","last_fullsized":"2023-09-23T18:45:12.093931+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"209'7956","reported_seq":7996,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:46:15.965145+0000","last_change":"2023-09-23T18:43:59.463638+0000","last_active":"2023-09-23T18:46:15.965145+0000","last_peered":"2023-09-23T18:46:15.965145+0000","last_clean":"2023-09-23T18:46:15.965145+0000","last_became_active":"2023-09-23T18:43:59.460459+0000","last_became_peered":"2023-09-23T18:43:59.460459+0000","last_unstale":"2023-09-23T18:46:15.965145+0000","last_undegraded":"2023-09-23T18:46:15.965145+0000","last_fullsized":"2023-09-23T18:46:15.965145+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":7956,"log_dups_size":0,"ondisk_log_size":7956,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44335104,"num_objects":676,"num_object_clones":0,"num_object_copies":2028,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":676,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5604,"num_write_kb":200992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'120","reported_seq":670,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.083645+0000","last_change":"2023-09-23T18:33:53.852697+0000","last_active":"2023-09-23T18:45:12.083645+0000","last_peered":"2023-09-23T18:45:12.083645+0000","last_clean":"2023-09-23T18:45:12.083645+0000","last_became_active":"2023-09-23T18:33:53.852210+0000","last_became_peered":"2023-09-23T18:33:53.852210+0000","last_unstale":"2023-09-23T18:45:12.083645+0000","last_undegraded":"2023-09-23T18:45:12.083645+0000","last_fullsized":"2023-09-23T18:45:12.083645+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":120,"log_dups_size":0,"ondisk_log_size":120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":809,"num_read_kb":2749,"num_write":98,"num_write_kb":2087,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":174,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:18.253746+0000","last_change":"2023-09-23T18:39:46.997952+0000","last_active":"2023-09-23T18:45:18.253746+0000","last_peered":"2023-09-23T18:45:18.253746+0000","last_clean":"2023-09-23T18:45:18.253746+0000","last_became_active":"2023-09-23T18:39:46.997589+0000","last_became_peered":"2023-09-23T18:39:46.997589+0000","last_unstale":"2023-09-23T18:45:18.253746+0000","last_undegraded":"2023-09-23T18:45:18.253746+0000","last_fullsized":"2023-09-23T18:45:18.253746+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:09.887469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":205,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:45:50.122760+0000","last_change":"2023-09-23T18:39:46.996973+0000","last_active":"2023-09-23T18:45:50.122760+0000","last_peered":"2023-09-23T18:45:50.122760+0000","last_clean":"2023-09-23T18:45:50.122760+0000","last_became_active":"2023-09-23T18:39:46.996633+0000","last_became_peered":"2023-09-23T18:39:46.996633+0000","last_unstale":"2023-09-23T18:45:50.122760+0000","last_undegraded":"2023-09-23T18:45:50.122760+0000","last_fullsized":"2023-09-23T18:45:50.122760+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:08:05.607474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"209'8028","reported_seq":8070,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:46:13.019565+0000","last_change":"2023-09-23T18:43:59.488931+0000","last_active":"2023-09-23T18:46:13.019565+0000","last_peered":"2023-09-23T18:46:13.019565+0000","last_clean":"2023-09-23T18:46:13.019565+0000","last_became_active":"2023-09-23T18:43:59.476762+0000","last_became_peered":"2023-09-23T18:43:59.476762+0000","last_unstale":"2023-09-23T18:46:13.019565+0000","last_undegraded":"2023-09-23T18:46:13.019565+0000","last_fullsized":"2023-09-23T18:46:13.019565+0000","mapping_epoch":187,"log_start":"192'1516","ondisk_log_start":"192'1516","created":187,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":6512,"log_dups_size":0,"ondisk_log_size":6512,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46202880,"num_objects":705,"num_object_clones":0,"num_object_copies":2115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5659,"num_write_kb":203648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'78","reported_seq":445,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.083583+0000","last_change":"2023-09-23T18:26:34.992345+0000","last_active":"2023-09-23T18:45:12.083583+0000","last_peered":"2023-09-23T18:45:12.083583+0000","last_clean":"2023-09-23T18:45:12.083583+0000","last_became_active":"2023-09-23T18:26:34.991774+0000","last_became_peered":"2023-09-23T18:26:34.991774+0000","last_unstale":"2023-09-23T18:45:12.083583+0000","last_undegraded":"2023-09-23T18:45:12.083583+0000","last_fullsized":"2023-09-23T18:45:12.083583+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":78,"log_dups_size":0,"ondisk_log_size":78,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":428,"num_read_kb":1125,"num_write":73,"num_write_kb":431,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":152,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.077890+0000","last_change":"2023-09-23T18:39:46.990343+0000","last_active":"2023-09-23T18:45:12.077890+0000","last_peered":"2023-09-23T18:45:12.077890+0000","last_clean":"2023-09-23T18:45:12.077890+0000","last_became_active":"2023-09-23T18:39:46.989904+0000","last_became_peered":"2023-09-23T18:39:46.989904+0000","last_unstale":"2023-09-23T18:45:12.077890+0000","last_undegraded":"2023-09-23T18:45:12.077890+0000","last_fullsized":"2023-09-23T18:45:12.077890+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:15.638308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":171,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:18.253806+0000","last_change":"2023-09-23T18:33:54.852309+0000","last_active":"2023-09-23T18:45:18.253806+0000","last_peered":"2023-09-23T18:45:18.253806+0000","last_clean":"2023-09-23T18:45:18.253806+0000","last_became_active":"2023-09-23T18:33:54.850255+0000","last_became_peered":"2023-09-23T18:33:54.850255+0000","last_unstale":"2023-09-23T18:45:18.253806+0000","last_undegraded":"2023-09-23T18:45:18.253806+0000","last_fullsized":"2023-09-23T18:45:18.253806+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:54.148992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"209'8046","reported_seq":8093,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:46:16.604346+0000","last_change":"2023-09-23T18:45:10.467026+0000","last_active":"2023-09-23T18:46:16.604346+0000","last_peered":"2023-09-23T18:46:16.604346+0000","last_clean":"2023-09-23T18:46:16.604346+0000","last_became_active":"2023-09-23T18:43:59.455678+0000","last_became_peered":"2023-09-23T18:43:59.455678+0000","last_unstale":"2023-09-23T18:46:16.604346+0000","last_undegraded":"2023-09-23T18:46:16.604346+0000","last_fullsized":"2023-09-23T18:46:16.604346+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":8046,"log_dups_size":0,"ondisk_log_size":8046,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40140800,"num_objects":612,"num_object_clones":0,"num_object_copies":1836,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":612,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5702,"num_write_kb":202080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":183,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:21.448869+0000","last_change":"2023-09-23T18:28:03.298805+0000","last_active":"2023-09-23T18:45:21.448869+0000","last_peered":"2023-09-23T18:45:21.448869+0000","last_clean":"2023-09-23T18:45:21.448869+0000","last_became_active":"2023-09-23T18:28:03.298437+0000","last_became_peered":"2023-09-23T18:28:03.298437+0000","last_unstale":"2023-09-23T18:45:21.448869+0000","last_undegraded":"2023-09-23T18:45:21.448869+0000","last_fullsized":"2023-09-23T18:45:21.448869+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:27.831669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'181","reported_seq":683,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.078855+0000","last_change":"2023-09-23T18:33:53.854528+0000","last_active":"2023-09-23T18:45:12.078855+0000","last_peered":"2023-09-23T18:45:12.078855+0000","last_clean":"2023-09-23T18:45:12.078855+0000","last_became_active":"2023-09-23T18:33:53.851883+0000","last_became_peered":"2023-09-23T18:33:53.851883+0000","last_unstale":"2023-09-23T18:45:12.078855+0000","last_undegraded":"2023-09-23T18:45:12.078855+0000","last_fullsized":"2023-09-23T18:45:12.078855+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":181,"log_dups_size":0,"ondisk_log_size":181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:43.888844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":909,"num_read_kb":734,"num_write":148,"num_write_kb":75,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":117,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.084100+0000","last_change":"2023-09-23T18:39:46.992015+0000","last_active":"2023-09-23T18:45:12.084100+0000","last_peered":"2023-09-23T18:45:12.084100+0000","last_clean":"2023-09-23T18:45:12.084100+0000","last_became_active":"2023-09-23T18:39:46.991605+0000","last_became_peered":"2023-09-23T18:39:46.991605+0000","last_unstale":"2023-09-23T18:45:12.084100+0000","last_undegraded":"2023-09-23T18:45:12.084100+0000","last_fullsized":"2023-09-23T18:45:12.084100+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"209'7939","reported_seq":7989,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:46:16.567322+0000","last_change":"2023-09-23T18:45:05.476567+0000","last_active":"2023-09-23T18:46:16.567322+0000","last_peered":"2023-09-23T18:46:16.567322+0000","last_clean":"2023-09-23T18:46:16.567322+0000","last_became_active":"2023-09-23T18:43:59.507176+0000","last_became_peered":"2023-09-23T18:43:59.507176+0000","last_unstale":"2023-09-23T18:46:16.567322+0000","last_undegraded":"2023-09-23T18:46:16.567322+0000","last_fullsized":"2023-09-23T18:46:16.567322+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":7939,"log_dups_size":0,"ondisk_log_size":7939,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42205184,"num_objects":644,"num_object_clones":0,"num_object_copies":1932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":644,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5596,"num_write_kb":199680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"77'112","reported_seq":742,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.094266+0000","last_change":"2023-09-23T18:40:45.086639+0000","last_active":"2023-09-23T18:45:12.094266+0000","last_peered":"2023-09-23T18:45:12.094266+0000","last_clean":"2023-09-23T18:45:12.094266+0000","last_became_active":"2023-09-23T18:40:45.086119+0000","last_became_peered":"2023-09-23T18:40:45.086119+0000","last_unstale":"2023-09-23T18:45:12.094266+0000","last_undegraded":"2023-09-23T18:45:12.094266+0000","last_fullsized":"2023-09-23T18:45:12.094266+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":893,"num_read_kb":3172,"num_write":69,"num_write_kb":2237,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":184,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:45:50.122837+0000","last_change":"2023-09-23T18:33:53.851421+0000","last_active":"2023-09-23T18:45:50.122837+0000","last_peered":"2023-09-23T18:45:50.122837+0000","last_clean":"2023-09-23T18:45:50.122837+0000","last_became_active":"2023-09-23T18:33:53.851012+0000","last_became_peered":"2023-09-23T18:33:53.851012+0000","last_unstale":"2023-09-23T18:45:50.122837+0000","last_undegraded":"2023-09-23T18:45:50.122837+0000","last_fullsized":"2023-09-23T18:45:50.122837+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:03.971054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":168,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.079085+0000","last_change":"2023-09-23T18:32:26.805472+0000","last_active":"2023-09-23T18:45:12.079085+0000","last_peered":"2023-09-23T18:45:12.079085+0000","last_clean":"2023-09-23T18:45:12.079085+0000","last_became_active":"2023-09-23T18:32:26.805078+0000","last_became_peered":"2023-09-23T18:32:26.805078+0000","last_unstale":"2023-09-23T18:45:12.079085+0000","last_undegraded":"2023-09-23T18:45:12.079085+0000","last_fullsized":"2023-09-23T18:45:12.079085+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":152,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.085122+0000","last_change":"2023-09-23T18:26:34.994176+0000","last_active":"2023-09-23T18:45:12.085122+0000","last_peered":"2023-09-23T18:45:12.085122+0000","last_clean":"2023-09-23T18:45:12.085122+0000","last_became_active":"2023-09-23T18:26:34.993829+0000","last_became_peered":"2023-09-23T18:26:34.993829+0000","last_unstale":"2023-09-23T18:45:12.085122+0000","last_undegraded":"2023-09-23T18:45:12.085122+0000","last_fullsized":"2023-09-23T18:45:12.085122+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:27.111857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":189,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.077961+0000","last_change":"2023-09-23T18:40:42.053012+0000","last_active":"2023-09-23T18:45:12.077961+0000","last_peered":"2023-09-23T18:45:12.077961+0000","last_clean":"2023-09-23T18:45:12.077961+0000","last_became_active":"2023-09-23T18:40:42.052655+0000","last_became_peered":"2023-09-23T18:40:42.052655+0000","last_unstale":"2023-09-23T18:45:12.077961+0000","last_undegraded":"2023-09-23T18:45:12.077961+0000","last_fullsized":"2023-09-23T18:45:12.077961+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:33:51.009383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.18","version":"209'7979","reported_seq":8024,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:46:14.197229+0000","last_change":"2023-09-23T18:45:45.170389+0000","last_active":"2023-09-23T18:46:14.197229+0000","last_peered":"2023-09-23T18:46:14.197229+0000","last_clean":"2023-09-23T18:46:14.197229+0000","last_became_active":"2023-09-23T18:43:59.489977+0000","last_became_peered":"2023-09-23T18:43:59.489977+0000","last_unstale":"2023-09-23T18:46:14.197229+0000","last_undegraded":"2023-09-23T18:46:14.197229+0000","last_fullsized":"2023-09-23T18:46:14.197229+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":195,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":7979,"log_dups_size":0,"ondisk_log_size":7979,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46432256,"num_objects":709,"num_object_clones":0,"num_object_copies":2127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":709,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5583,"num_write_kb":201312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":132,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.079444+0000","last_change":"2023-09-23T18:33:54.847172+0000","last_active":"2023-09-23T18:45:12.079444+0000","last_peered":"2023-09-23T18:45:12.079444+0000","last_clean":"2023-09-23T18:45:12.079444+0000","last_became_active":"2023-09-23T18:33:54.846751+0000","last_became_peered":"2023-09-23T18:33:54.846751+0000","last_unstale":"2023-09-23T18:45:12.079444+0000","last_undegraded":"2023-09-23T18:45:12.079444+0000","last_fullsized":"2023-09-23T18:45:12.079444+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":204,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.079591+0000","last_change":"2023-09-23T18:33:53.841232+0000","last_active":"2023-09-23T18:45:12.079591+0000","last_peered":"2023-09-23T18:45:12.079591+0000","last_clean":"2023-09-23T18:45:12.079591+0000","last_became_active":"2023-09-23T18:33:53.840749+0000","last_became_peered":"2023-09-23T18:33:53.840749+0000","last_unstale":"2023-09-23T18:45:12.079591+0000","last_undegraded":"2023-09-23T18:45:12.079591+0000","last_fullsized":"2023-09-23T18:45:12.079591+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"209'7842","reported_seq":7886,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T18:46:13.315011+0000","last_change":"2023-09-23T18:45:39.352024+0000","last_active":"2023-09-23T18:46:13.315011+0000","last_peered":"2023-09-23T18:46:13.315011+0000","last_clean":"2023-09-23T18:46:13.315011+0000","last_became_active":"2023-09-23T18:43:59.508442+0000","last_became_peered":"2023-09-23T18:43:59.508442+0000","last_unstale":"2023-09-23T18:46:13.315011+0000","last_undegraded":"2023-09-23T18:46:13.315011+0000","last_fullsized":"2023-09-23T18:46:13.315011+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":195,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:42:38.290195+0000","last_clean_scrub_stamp":"2023-09-23T18:42:38.290195+0000","objects_scrubbed":0,"log_size":7842,"log_dups_size":0,"ondisk_log_size":7842,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46596096,"num_objects":711,"num_object_clones":0,"num_object_copies":2133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":711,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5509,"num_write_kb":199040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":132,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.079651+0000","last_change":"2023-09-23T18:33:54.858971+0000","last_active":"2023-09-23T18:45:12.079651+0000","last_peered":"2023-09-23T18:45:12.079651+0000","last_clean":"2023-09-23T18:45:12.079651+0000","last_became_active":"2023-09-23T18:33:54.845921+0000","last_became_peered":"2023-09-23T18:33:54.845921+0000","last_unstale":"2023-09-23T18:45:12.079651+0000","last_undegraded":"2023-09-23T18:45:12.079651+0000","last_fullsized":"2023-09-23T18:45:12.079651+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":189,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.079084+0000","last_change":"2023-09-23T18:28:03.302261+0000","last_active":"2023-09-23T18:45:12.079084+0000","last_peered":"2023-09-23T18:45:12.079084+0000","last_clean":"2023-09-23T18:45:12.079084+0000","last_became_active":"2023-09-23T18:28:03.301911+0000","last_became_peered":"2023-09-23T18:28:03.301911+0000","last_unstale":"2023-09-23T18:45:12.079084+0000","last_undegraded":"2023-09-23T18:45:12.079084+0000","last_fullsized":"2023-09-23T18:45:12.079084+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":57,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.094105+0000","last_change":"2023-09-23T18:40:42.049209+0000","last_active":"2023-09-23T18:45:12.094105+0000","last_peered":"2023-09-23T18:45:12.094105+0000","last_clean":"2023-09-23T18:45:12.094105+0000","last_became_active":"2023-09-23T18:40:42.048654+0000","last_became_peered":"2023-09-23T18:40:42.048654+0000","last_unstale":"2023-09-23T18:45:12.094105+0000","last_undegraded":"2023-09-23T18:45:12.094105+0000","last_fullsized":"2023-09-23T18:45:12.094105+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":132,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:34.300548+0000","last_change":"2023-09-23T18:40:42.051196+0000","last_active":"2023-09-23T18:45:34.300548+0000","last_peered":"2023-09-23T18:45:34.300548+0000","last_clean":"2023-09-23T18:45:34.300548+0000","last_became_active":"2023-09-23T18:40:42.050613+0000","last_became_peered":"2023-09-23T18:40:42.050613+0000","last_unstale":"2023-09-23T18:45:34.300548+0000","last_undegraded":"2023-09-23T18:45:34.300548+0000","last_fullsized":"2023-09-23T18:45:34.300548+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:21.750773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":199,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:18.253923+0000","last_change":"2023-09-23T18:39:47.006553+0000","last_active":"2023-09-23T18:45:18.253923+0000","last_peered":"2023-09-23T18:45:18.253923+0000","last_clean":"2023-09-23T18:45:18.253923+0000","last_became_active":"2023-09-23T18:39:46.996464+0000","last_became_peered":"2023-09-23T18:39:46.996464+0000","last_unstale":"2023-09-23T18:45:18.253923+0000","last_undegraded":"2023-09-23T18:45:18.253923+0000","last_fullsized":"2023-09-23T18:45:18.253923+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:02.274848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":188,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T18:45:12.078793+0000","last_change":"2023-09-23T18:33:54.847156+0000","last_active":"2023-09-23T18:45:12.078793+0000","last_peered":"2023-09-23T18:45:12.078793+0000","last_clean":"2023-09-23T18:45:12.078793+0000","last_became_active":"2023-09-23T18:33:54.846765+0000","last_became_peered":"2023-09-23T18:33:54.846765+0000","last_unstale":"2023-09-23T18:45:12.078793+0000","last_undegraded":"2023-09-23T18:45:12.078793+0000","last_fullsized":"2023-09-23T18:45:12.078793+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":1443299352,"num_objects":22024,"num_object_clones":0,"num_object_copies":66072,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3551,"num_objects_misplaced":3552,"num_objects_unfound":0,"num_objects_dirty":22024,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":179862,"num_write_kb":6460289,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4599,"num_bytes_recovered":281608192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4817956864,"data_stored":4817944648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":199307,"ondisk_log_size":199307,"up":78,"acting":76,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":1200,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26886,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5937,"num_read_kb":13190,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5621,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430621,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":827684,"kb_used_data":530460,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92917468,"statfs":{"total":95995035648,"available":95147487232,"internally_reserved":0,"allocated":543191040,"data_stored":542610212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:45:14 2023","interfaces":[{"interface":"back","average":{"1min":2.4780000000000002,"5min":4.3090000000000002,"15min":2.6459999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.252},"max":{"1min":23.501999999999999,"5min":97.415999999999997,"15min":97.415999999999997},"last":0.42899999999999999},{"interface":"front","average":{"1min":2.8540000000000001,"5min":4.1790000000000003,"15min":2.6749999999999998},"min":{"1min":0.34100000000000003,"5min":0.28999999999999998,"15min":0.22900000000000001},"max":{"1min":28.573,"5min":96.581000000000003,"15min":96.581000000000003},"last":2.2149999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:45:14 2023","interfaces":[{"interface":"back","average":{"1min":1.764,"5min":2.8170000000000002,"15min":1.966},"min":{"1min":0.28000000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":9.5120000000000005,"5min":94.650999999999996,"15min":94.650999999999996},"last":3.5350000000000001},{"interface":"front","average":{"1min":1.831,"5min":3.1459999999999999,"15min":2.2189999999999999},"min":{"1min":0.39300000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":10.651999999999999,"5min":94.843999999999994,"15min":94.843999999999994},"last":3.5670000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:46:08 2023","interfaces":[{"interface":"back","average":{"1min":1.645,"5min":1.478,"15min":1.661},"min":{"1min":0.34300000000000003,"5min":0.34000000000000002,"15min":0.254},"max":{"1min":10.771000000000001,"5min":10.771000000000001,"15min":44.392000000000003},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.5329999999999999,"5min":1.4410000000000001,"15min":1.6699999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.29999999999999999},"max":{"1min":5.8810000000000002,"5min":7.718,"15min":42.991999999999997},"last":3.6819999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:45:33 2023","interfaces":[{"interface":"back","average":{"1min":7.2679999999999998,"5min":4.5800000000000001,"15min":2.5830000000000002},"min":{"1min":0.36499999999999999,"5min":0.36199999999999999,"15min":0.32800000000000001},"max":{"1min":136.345,"5min":136.345,"15min":136.345},"last":0.56100000000000005},{"interface":"front","average":{"1min":7.8300000000000001,"5min":4.7329999999999997,"15min":2.7970000000000002},"min":{"1min":0.35899999999999999,"5min":0.318,"15min":0.314},"max":{"1min":136.893,"5min":136.893,"15min":136.893},"last":2.4540000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:45:14 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":0.97899999999999998,"15min":1.0529999999999999},"min":{"1min":0.42499999999999999,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":4.0750000000000002,"5min":4.1120000000000001,"15min":6.3529999999999998},"last":1.8129999999999999},{"interface":"front","average":{"1min":1.0740000000000001,"5min":1.2110000000000001,"15min":1.1479999999999999},"min":{"1min":0.42899999999999999,"5min":0.38400000000000001,"15min":0.309},"max":{"1min":3.75,"5min":7.6029999999999998,"15min":11.119},"last":0.46800000000000003}]},{"osd":5,"last update":"Sat Sep 23 18:45:14 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":1.23,"15min":1.147},"min":{"1min":0.499,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.8370000000000002,"5min":8.7210000000000001,"15min":8.7210000000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.012,"15min":1.087},"min":{"1min":0.51000000000000001,"5min":0.41899999999999998,"15min":0.26800000000000002},"max":{"1min":4.1289999999999996,"5min":4.1289999999999996,"15min":7.46},"last":0.65300000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:45:14 2023","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":1.2869999999999999,"15min":1.1479999999999999},"min":{"1min":0.46899999999999997,"5min":0.41199999999999998,"15min":0.35299999999999998},"max":{"1min":4.1029999999999998,"5min":5.915,"15min":5.915},"last":0.622},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.341,"15min":1.159},"min":{"1min":0.46300000000000002,"5min":0.41099999999999998,"15min":0.36499999999999999},"max":{"1min":3.665,"5min":9.2850000000000001,"15min":9.2850000000000001},"last":0.71599999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626853,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":863456,"kb_used_data":549848,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":92881696,"statfs":{"total":95995035648,"available":95110856704,"internally_reserved":0,"allocated":563044352,"data_stored":562467756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:46:14 2023","interfaces":[{"interface":"back","average":{"1min":7.6230000000000002,"5min":2.3479999999999999,"15min":2.4279999999999999},"min":{"1min":0.33200000000000002,"5min":0.28100000000000003,"15min":0.26000000000000001},"max":{"1min":103.562,"5min":103.562,"15min":182.655},"last":3.3580000000000001},{"interface":"front","average":{"1min":7.6799999999999997,"5min":2.452,"15min":2.52},"min":{"1min":0.51000000000000001,"5min":0.36699999999999999,"15min":0.32800000000000001},"max":{"1min":103.643,"5min":103.643,"15min":182.58799999999999},"last":3.391}]},{"osd":1,"last update":"Sat Sep 23 18:46:14 2023","interfaces":[{"interface":"back","average":{"1min":7.6059999999999999,"5min":5.5579999999999998,"15min":2.5710000000000002},"min":{"1min":0.30299999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":103.887,"5min":284.07400000000001,"15min":284.07400000000001},"last":3.2440000000000002},{"interface":"front","average":{"1min":7.3760000000000003,"5min":5.5170000000000003,"15min":2.778},"min":{"1min":0.33900000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":103.852,"5min":281.19900000000001,"15min":281.19900000000001},"last":0.67000000000000004}]},{"osd":2,"last update":"Sat Sep 23 18:46:14 2023","interfaces":[{"interface":"back","average":{"1min":1.506,"5min":1.5169999999999999,"15min":2.4689999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.28100000000000003},"max":{"1min":5.4640000000000004,"5min":13.407999999999999,"15min":122.242},"last":0.76500000000000001},{"interface":"front","average":{"1min":1.456,"5min":1.536,"15min":2.4969999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.28799999999999998},"max":{"1min":5.4219999999999997,"5min":13.395,"15min":122.32899999999999},"last":0.69899999999999995}]},{"osd":3,"last update":"Sat Sep 23 18:45:33 2023","interfaces":[{"interface":"back","average":{"1min":1.325,"5min":2.0790000000000002,"15min":2.544},"min":{"1min":0.44400000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":5.7839999999999998,"5min":89.245999999999995,"15min":123.52200000000001},"last":3.3210000000000002},{"interface":"front","average":{"1min":1.6839999999999999,"5min":2.544,"15min":2.7749999999999999},"min":{"1min":0.40400000000000003,"5min":0.40100000000000002,"15min":0.36099999999999999},"max":{"1min":5.7539999999999996,"5min":89.274000000000001,"15min":123.605},"last":1.1950000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:45:33 2023","interfaces":[{"interface":"back","average":{"1min":1.089,"5min":1.476,"15min":1.1399999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":8.8330000000000002,"5min":15.496,"15min":15.496},"last":0.85199999999999998},{"interface":"front","average":{"1min":0.59199999999999997,"5min":1.296,"15min":1.0369999999999999},"min":{"1min":0.38100000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.88800000000000001,"5min":14.875999999999999,"15min":14.875999999999999},"last":3.2839999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:45:33 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":1.212,"15min":1.028},"min":{"1min":0.47699999999999998,"5min":0.38200000000000001,"15min":0.32000000000000001},"max":{"1min":8.8840000000000003,"5min":8.8840000000000003,"15min":8.8840000000000003},"last":0.82999999999999996},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.86699999999999999,"15min":0.88200000000000001},"min":{"1min":0.41499999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.351,"5min":3.8220000000000001,"15min":4.5270000000000001},"last":3.1850000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:46:14 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":1.113,"15min":1.1399999999999999},"min":{"1min":0.372,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.2810000000000001,"5min":7.6360000000000001,"15min":7.9320000000000004},"last":0.86699999999999999},{"interface":"front","average":{"1min":1.379,"5min":1.306,"15min":1.4119999999999999},"min":{"1min":0.40600000000000003,"5min":0.39900000000000002,"15min":0.39100000000000001},"max":{"1min":5.2800000000000002,"5min":8.9250000000000007,"15min":9.1129999999999995},"last":0.99299999999999999}]}]},{"osd":1,"up_from":123,"seq":528280977557,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1370020,"kb_used_data":950044,"kb_used_omap":7,"kb_used_meta":419960,"kb_avail":92375132,"statfs":{"total":95995035648,"available":94592135168,"internally_reserved":0,"allocated":972845056,"data_stored":971859633,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":430039822},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:45:13 2023","interfaces":[{"interface":"back","average":{"1min":3.1779999999999999,"5min":2.605,"15min":5.1790000000000003},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":8.2759999999999998,"5min":53.530999999999999,"15min":173.41800000000001},"last":5.1189999999999998},{"interface":"front","average":{"1min":3.6899999999999999,"5min":3.0070000000000001,"15min":5.1559999999999997},"min":{"1min":0.33700000000000002,"5min":0.249,"15min":0.17599999999999999},"max":{"1min":8.6020000000000003,"5min":51.710999999999999,"15min":166.42599999999999},"last":5.5910000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:45:13 2023","interfaces":[{"interface":"back","average":{"1min":3.4950000000000001,"5min":2.0329999999999999,"15min":3.0009999999999999},"min":{"1min":0.315,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":12.615,"5min":12.615,"15min":129.25899999999999},"last":0.20899999999999999},{"interface":"front","average":{"1min":3.0859999999999999,"5min":2.431,"15min":3.0089999999999999},"min":{"1min":0.56299999999999994,"5min":0.33700000000000002,"15min":0.247},"max":{"1min":14.602,"5min":16.032,"15min":129.35300000000001},"last":1.095}]},{"osd":3,"last update":"Sat Sep 23 18:45:13 2023","interfaces":[{"interface":"back","average":{"1min":3.274,"5min":2.621,"15min":2.1560000000000001},"min":{"1min":0.38200000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":14.744999999999999,"5min":16.123000000000001,"15min":55.014000000000003},"last":0.70199999999999996},{"interface":"front","average":{"1min":3.0910000000000002,"5min":2.1389999999999998,"15min":2.0950000000000002},"min":{"1min":0.318,"5min":0.246,"15min":0.24399999999999999},"max":{"1min":7.8170000000000002,"5min":15.717000000000001,"15min":55.841999999999999},"last":0.66700000000000004}]},{"osd":4,"last update":"Sat Sep 23 18:45:13 2023","interfaces":[{"interface":"back","average":{"1min":2.3490000000000002,"5min":1.77,"15min":1.5249999999999999},"min":{"1min":0.36399999999999999,"5min":0.308,"15min":0.308},"max":{"1min":7.4509999999999996,"5min":8.8670000000000009,"15min":11.763},"last":0.57199999999999995},{"interface":"front","average":{"1min":2.5150000000000001,"5min":1.579,"15min":1.518},"min":{"1min":0.38200000000000001,"5min":0.313,"15min":0.27700000000000002},"max":{"1min":7.7009999999999996,"5min":8.2270000000000003,"15min":11.491},"last":4.9809999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:45:13 2023","interfaces":[{"interface":"back","average":{"1min":2.4079999999999999,"5min":1.8540000000000001,"15min":1.3879999999999999},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.35699999999999998},"max":{"1min":8.7750000000000004,"5min":9.2690000000000001,"15min":11.794},"last":1.159},{"interface":"front","average":{"1min":2.3479999999999999,"5min":1.718,"15min":1.5720000000000001},"min":{"1min":0.433,"5min":0.371,"15min":0.371},"max":{"1min":7.4260000000000002,"5min":8.9309999999999992,"15min":11.733000000000001},"last":0.627}]},{"osd":6,"last update":"Sat Sep 23 18:45:13 2023","interfaces":[{"interface":"back","average":{"1min":2.9820000000000002,"5min":1.992,"15min":1.74},"min":{"1min":0.41299999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":12.574999999999999,"5min":12.574999999999999,"15min":12.574999999999999},"last":0.64800000000000002},{"interface":"front","average":{"1min":2.375,"5min":1.9410000000000001,"15min":1.766},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.5209999999999999,"5min":16.687000000000001,"15min":16.687000000000001},"last":5.0259999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:45:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6030000000000002,"5min":1.9850000000000001,"15min":1.9259999999999999},"min":{"1min":0.437,"5min":0.39300000000000002,"15min":0.33500000000000002},"max":{"1min":7.4770000000000003,"5min":8.9109999999999996,"15min":11.747999999999999},"last":0.60299999999999998},{"interface":"front","average":{"1min":2.3809999999999998,"5min":1.712,"15min":1.744},"min":{"1min":0.40999999999999998,"5min":0.40899999999999997,"15min":0.379},"max":{"1min":5.6100000000000003,"5min":8.4770000000000003,"15min":11.593},"last":5.0510000000000002}]}]},{"osd":0,"up_from":88,"seq":377957122286,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1311264,"kb_used_data":767896,"kb_used_omap":8,"kb_used_meta":543351,"kb_avail":92433888,"statfs":{"total":95995035648,"available":94652301312,"internally_reserved":0,"allocated":786325504,"data_stored":785401400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":556392318},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:45:23 2023","interfaces":[{"interface":"back","average":{"1min":2.7930000000000001,"5min":2.0790000000000002,"15min":2.0310000000000001},"min":{"1min":0.26500000000000001,"5min":0.191,"15min":0.17100000000000001},"max":{"1min":9.8919999999999995,"5min":16.355,"15min":38.545999999999999},"last":4.8719999999999999},{"interface":"front","average":{"1min":2.7719999999999998,"5min":2.1699999999999999,"15min":2.2650000000000001},"min":{"1min":0.246,"5min":0.23200000000000001,"15min":0.20100000000000001},"max":{"1min":12.08,"5min":21.535,"15min":45.904000000000003},"last":1.4670000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:46:00 2023","interfaces":[{"interface":"back","average":{"1min":2.8340000000000001,"5min":1.8819999999999999,"15min":2.4420000000000002},"min":{"1min":0.32600000000000001,"5min":0.24399999999999999,"15min":0.192},"max":{"1min":8.1059999999999999,"5min":12.622999999999999,"15min":108.73999999999999},"last":9.4489999999999998},{"interface":"front","average":{"1min":2.9119999999999999,"5min":1.631,"15min":2.1880000000000002},"min":{"1min":0.30499999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":5.7800000000000002,"5min":12.297000000000001,"15min":108.136},"last":4.2539999999999996}]},{"osd":3,"last update":"Sat Sep 23 18:45:27 2023","interfaces":[{"interface":"back","average":{"1min":2.9039999999999999,"5min":3.8969999999999998,"15min":3.4340000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.216},"max":{"1min":8.4689999999999994,"5min":177.83600000000001,"15min":177.83600000000001},"last":1.2909999999999999},{"interface":"front","average":{"1min":2.8359999999999999,"5min":3.9279999999999999,"15min":3.4470000000000001},"min":{"1min":0.41899999999999998,"5min":0.28199999999999997,"15min":0.24299999999999999},"max":{"1min":12.996,"5min":181.71899999999999,"15min":181.71899999999999},"last":4.2839999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:45:23 2023","interfaces":[{"interface":"back","average":{"1min":1.726,"5min":1.5349999999999999,"15min":1.4039999999999999},"min":{"1min":0.35999999999999999,"5min":0.31,"15min":0.28199999999999997},"max":{"1min":4.1289999999999996,"5min":9.9450000000000003,"15min":9.9450000000000003},"last":1.5620000000000001},{"interface":"front","average":{"1min":1.7170000000000001,"5min":1.337,"15min":1.5369999999999999},"min":{"1min":0.44600000000000001,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.1459999999999999,"5min":6.6150000000000002,"15min":16.486999999999998},"last":1.3480000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:45:23 2023","interfaces":[{"interface":"back","average":{"1min":1.758,"5min":1.272,"15min":1.3320000000000001},"min":{"1min":0.41699999999999998,"5min":0.371,"15min":0.36499999999999999},"max":{"1min":6.1130000000000004,"5min":6.1130000000000004,"15min":10.606},"last":1.371},{"interface":"front","average":{"1min":1.607,"5min":1.153,"15min":1.294},"min":{"1min":0.40400000000000003,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.843,"5min":7.843,"15min":8.9250000000000007},"last":4.3209999999999997}]},{"osd":6,"last update":"Sat Sep 23 18:45:23 2023","interfaces":[{"interface":"back","average":{"1min":1.9630000000000001,"5min":1.591,"15min":1.5009999999999999},"min":{"1min":0.379,"5min":0.376,"15min":0.35899999999999999},"max":{"1min":6.0640000000000001,"5min":11.302,"15min":11.302},"last":1.393},{"interface":"front","average":{"1min":1.786,"5min":1.4059999999999999,"15min":1.3959999999999999},"min":{"1min":0.42599999999999999,"5min":0.372,"15min":0.33200000000000002},"max":{"1min":7.8710000000000004,"5min":11.818,"15min":11.818},"last":4.3860000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:45:23 2023","interfaces":[{"interface":"back","average":{"1min":1.7210000000000001,"5min":1.4419999999999999,"15min":1.4630000000000001},"min":{"1min":0.44800000000000001,"5min":0.39700000000000002,"15min":0.39200000000000002},"max":{"1min":7.9550000000000001,"5min":8.327,"15min":9.0600000000000005},"last":4.3529999999999998},{"interface":"front","average":{"1min":1.7569999999999999,"5min":1.6479999999999999,"15min":1.4890000000000001},"min":{"1min":0.47599999999999998,"5min":0.442,"15min":0.35799999999999998},"max":{"1min":4.1790000000000003,"5min":10.01,"15min":10.01},"last":1.946}]}]},{"osd":2,"up_from":98,"seq":420906795234,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1220844,"kb_used_data":680932,"kb_used_omap":11,"kb_used_meta":539828,"kb_avail":92524308,"statfs":{"total":95995035648,"available":94744891392,"internally_reserved":0,"allocated":697274368,"data_stored":696397189,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11949,"internal_metadata":552784211},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:46:10 2023","interfaces":[{"interface":"back","average":{"1min":2.4089999999999998,"5min":2.077,"15min":1.7110000000000001},"min":{"1min":0.217,"5min":0.185,"15min":0.184},"max":{"1min":6.9580000000000002,"5min":48.015999999999998,"15min":48.015999999999998},"last":0.75600000000000001},{"interface":"front","average":{"1min":3.927,"5min":2.9060000000000001,"15min":2.0030000000000001},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.17699999999999999},"max":{"1min":9.9730000000000008,"5min":48.985999999999997,"15min":48.985999999999997},"last":4.9400000000000004}]},{"osd":1,"last update":"Sat Sep 23 18:45:17 2023","interfaces":[{"interface":"back","average":{"1min":3.04,"5min":2.8839999999999999,"15min":3.234},"min":{"1min":0.29699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":26.495000000000001,"5min":94.210999999999999,"15min":184.56299999999999},"last":0.64300000000000002},{"interface":"front","average":{"1min":3.73,"5min":3.4220000000000002,"15min":3.4319999999999999},"min":{"1min":0.28699999999999998,"5min":0.26100000000000001,"15min":0.23499999999999999},"max":{"1min":26.579999999999998,"5min":90.451999999999998,"15min":183.922},"last":4.8739999999999997}]},{"osd":3,"last update":"Sat Sep 23 18:45:38 2023","interfaces":[{"interface":"back","average":{"1min":1.9930000000000001,"5min":3.3319999999999999,"15min":2.9300000000000002},"min":{"1min":0.26800000000000002,"5min":0.183,"15min":0.183},"max":{"1min":6.2350000000000003,"5min":161.773,"15min":161.773},"last":0.38},{"interface":"front","average":{"1min":1.865,"5min":3.0920000000000001,"15min":2.5950000000000002},"min":{"1min":0.26500000000000001,"5min":0.25,"15min":0.249},"max":{"1min":7.1399999999999997,"5min":158.03700000000001,"15min":158.03700000000001},"last":0.28100000000000003}]},{"osd":4,"last update":"Sat Sep 23 18:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.093,"5min":1.23,"15min":1.7589999999999999},"min":{"1min":0.34000000000000002,"5min":0.32800000000000001,"15min":0.30099999999999999},"max":{"1min":3.6389999999999998,"5min":7.0190000000000001,"15min":71.587999999999994},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.702,"5min":1.2,"15min":1.6479999999999999},"min":{"1min":0.44700000000000001,"5min":0.33600000000000002,"15min":0.33500000000000002},"max":{"1min":6.0369999999999999,"5min":8.5860000000000003,"15min":71.328999999999994},"last":0.61399999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":1.1339999999999999,"15min":1.4610000000000001},"min":{"1min":0.46600000000000003,"5min":0.36799999999999999,"15min":0.35999999999999999},"max":{"1min":6.0039999999999996,"5min":6.0039999999999996,"15min":71.302000000000007},"last":0.72399999999999998},{"interface":"front","average":{"1min":2.7970000000000002,"5min":1.75,"15min":1.8120000000000001},"min":{"1min":0.39400000000000002,"5min":0.371,"15min":0.316},"max":{"1min":9.891,"5min":9.891,"15min":71.543999999999997},"last":4.9119999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.242,"5min":1.165,"15min":1.819},"min":{"1min":0.44500000000000001,"5min":0.374,"15min":0.35199999999999998},"max":{"1min":3.5779999999999998,"5min":4.8259999999999996,"15min":71.460999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.8149999999999999,"5min":1.353,"15min":1.6180000000000001},"min":{"1min":0.42599999999999999,"5min":0.39800000000000002,"15min":0.38100000000000001},"max":{"1min":5.9279999999999999,"5min":8.6259999999999994,"15min":71.271000000000001},"last":3.661}]},{"osd":7,"last update":"Sat Sep 23 18:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.506,"5min":1.25,"15min":1.845},"min":{"1min":0.40000000000000002,"5min":0.39100000000000001,"15min":0.34799999999999998},"max":{"1min":6.4699999999999998,"5min":6.4699999999999998,"15min":71.622},"last":0.629},{"interface":"front","average":{"1min":2.0310000000000001,"5min":1.266,"15min":1.8140000000000001},"min":{"1min":0.48399999999999999,"5min":0.40100000000000002,"15min":0.39900000000000002},"max":{"1min":5.9710000000000001,"5min":5.9710000000000001,"15min":71.575000000000003},"last":3.7050000000000001}]}]},{"osd":3,"up_from":103,"seq":442381631709,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1157696,"kb_used_data":684536,"kb_used_omap":4,"kb_used_meta":473147,"kb_avail":92587456,"statfs":{"total":95995035648,"available":94809554944,"internally_reserved":0,"allocated":700964864,"data_stored":700068130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4826,"internal_metadata":484502822},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:45:35 2023","interfaces":[{"interface":"back","average":{"1min":4.4269999999999996,"5min":3.0859999999999999,"15min":2.1309999999999998},"min":{"1min":0.24299999999999999,"5min":0.20799999999999999,"15min":0.193},"max":{"1min":42.695999999999998,"5min":129.315,"15min":129.315},"last":3.4609999999999999},{"interface":"front","average":{"1min":4.266,"5min":3.1619999999999999,"15min":2.0459999999999998},"min":{"1min":0.26400000000000001,"5min":0.219,"15min":0.20300000000000001},"max":{"1min":36.302999999999997,"5min":130.09200000000001,"15min":130.09200000000001},"last":10.491}]},{"osd":1,"last update":"Sat Sep 23 18:45:18 2023","interfaces":[{"interface":"back","average":{"1min":3.5329999999999999,"5min":1.8859999999999999,"15min":1.5589999999999999},"min":{"1min":0.42199999999999999,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":19.529,"5min":19.529,"15min":19.529},"last":8.0419999999999998},{"interface":"front","average":{"1min":3.5899999999999999,"5min":1.9299999999999999,"15min":1.708},"min":{"1min":0.29099999999999998,"5min":0.23899999999999999,"15min":0.23100000000000001},"max":{"1min":19.507000000000001,"5min":19.507000000000001,"15min":19.507000000000001},"last":8.4489999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:45:35 2023","interfaces":[{"interface":"back","average":{"1min":2.8250000000000002,"5min":1.8240000000000001,"15min":2.871},"min":{"1min":0.26400000000000001,"5min":0.248,"15min":0.22700000000000001},"max":{"1min":10.343,"5min":10.343,"15min":187.40000000000001},"last":3.6589999999999998},{"interface":"front","average":{"1min":3.3100000000000001,"5min":1.8260000000000001,"15min":2.4329999999999998},"min":{"1min":0.27400000000000002,"5min":0.251,"15min":0.22500000000000001},"max":{"1min":8.8130000000000006,"5min":12.898999999999999,"15min":187.71700000000001},"last":8.3940000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.6619999999999999,"5min":1.3999999999999999,"15min":1.272},"min":{"1min":0.36899999999999999,"5min":0.312,"15min":0.30599999999999999},"max":{"1min":5.2249999999999996,"5min":9.9960000000000004,"15min":9.9960000000000004},"last":7.9480000000000004},{"interface":"front","average":{"1min":1.71,"5min":1.599,"15min":1.387},"min":{"1min":0.51400000000000001,"5min":0.36199999999999999,"15min":0.28399999999999997},"max":{"1min":5.5780000000000003,"5min":11.916,"15min":11.916},"last":1.355}]},{"osd":5,"last update":"Sat Sep 23 18:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.821,"5min":1.3919999999999999,"15min":1.266},"min":{"1min":0.54500000000000004,"5min":0.39700000000000002,"15min":0.34100000000000003},"max":{"1min":5.5540000000000003,"5min":7.8799999999999999,"15min":7.8799999999999999},"last":1.333},{"interface":"front","average":{"1min":1.9910000000000001,"5min":1.611,"15min":1.46},"min":{"1min":0.46700000000000003,"5min":0.39100000000000001,"15min":0.36099999999999999},"max":{"1min":4.2969999999999997,"5min":7.8129999999999997,"15min":10.18},"last":3.7210000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:45:35 2023","interfaces":[{"interface":"back","average":{"1min":2.1309999999999998,"5min":1.8620000000000001,"15min":1.6879999999999999},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.35599999999999998},"max":{"1min":4.3319999999999999,"5min":8.6940000000000008,"15min":11.529},"last":3.6920000000000002},{"interface":"front","average":{"1min":1.8320000000000001,"5min":1.609,"15min":1.4059999999999999},"min":{"1min":0.43099999999999999,"5min":0.375,"15min":0.33300000000000002},"max":{"1min":5.2960000000000003,"5min":10.092000000000001,"15min":11.989000000000001},"last":8.0150000000000006}]},{"osd":7,"last update":"Sat Sep 23 18:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.784,"5min":1.5109999999999999,"15min":1.4370000000000001},"min":{"1min":0.40699999999999997,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":5.2670000000000003,"5min":10.052,"15min":10.052},"last":7.9889999999999999},{"interface":"front","average":{"1min":1.9670000000000001,"5min":1.381,"15min":1.3879999999999999},"min":{"1min":0.48399999999999999,"5min":0.39900000000000002,"15min":0.36599999999999999},"max":{"1min":5.6029999999999998,"5min":5.7549999999999999,"15min":9.032},"last":1.2989999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019319,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1230684,"kb_used_data":636884,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":92514468,"statfs":{"total":95995035648,"available":94734815232,"internally_reserved":0,"allocated":652169216,"data_stored":651596498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:46:05 2023","interfaces":[{"interface":"back","average":{"1min":1.4950000000000001,"5min":1.2210000000000001,"15min":1.8240000000000001},"min":{"1min":0.316,"5min":0.28199999999999997,"15min":0.253},"max":{"1min":9.2210000000000001,"5min":9.2210000000000001,"15min":82.623000000000005},"last":4.835},{"interface":"front","average":{"1min":1.867,"5min":1.2689999999999999,"15min":1.702},"min":{"1min":0.39500000000000002,"5min":0.32100000000000001,"15min":0.23799999999999999},"max":{"1min":9.2349999999999994,"5min":9.5039999999999996,"15min":82.638000000000005},"last":4.0990000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:46:13 2023","interfaces":[{"interface":"back","average":{"1min":8.6389999999999993,"5min":3.383,"15min":2.3109999999999999},"min":{"1min":0.36199999999999999,"5min":0.31,"15min":0.29299999999999998},"max":{"1min":73.537000000000006,"5min":73.537000000000006,"15min":73.537000000000006},"last":2.0529999999999999},{"interface":"front","average":{"1min":8.2479999999999993,"5min":3.3969999999999998,"15min":2.4140000000000001},"min":{"1min":0.35599999999999998,"5min":0.313,"15min":0.28399999999999997},"max":{"1min":73.575999999999993,"5min":73.575999999999993,"15min":73.575999999999993},"last":2.097}]},{"osd":2,"last update":"Sat Sep 23 18:46:05 2023","interfaces":[{"interface":"back","average":{"1min":6.8049999999999997,"5min":2.5379999999999998,"15min":2.097},"min":{"1min":0.45700000000000002,"5min":0.312,"15min":0.28799999999999998},"max":{"1min":92.471000000000004,"5min":92.471000000000004,"15min":92.471000000000004},"last":3.5470000000000002},{"interface":"front","average":{"1min":6.3769999999999998,"5min":2.5310000000000001,"15min":2.3130000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":92.387,"5min":92.387,"15min":92.387},"last":3.528}]},{"osd":3,"last update":"Sat Sep 23 18:45:41 2023","interfaces":[{"interface":"back","average":{"1min":1.506,"5min":2.9980000000000002,"15min":2.5089999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.32900000000000001},"max":{"1min":5.3890000000000002,"5min":136.548,"15min":136.548},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.4570000000000001,"5min":2.8849999999999998,"15min":2.46},"min":{"1min":0.377,"5min":0.36499999999999999,"15min":0.34000000000000002},"max":{"1min":4.2089999999999996,"5min":136.494,"15min":136.494},"last":2.972}]},{"osd":5,"last update":"Sat Sep 23 18:46:00 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":1.022,"15min":1.107},"min":{"1min":0.441,"5min":0.307,"15min":0.29899999999999999},"max":{"1min":2.7480000000000002,"5min":6.9249999999999998,"15min":7.819},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.95999999999999996,"15min":1.0780000000000001},"min":{"1min":0.38900000000000001,"5min":0.36099999999999999,"15min":0.28399999999999997},"max":{"1min":0.85999999999999999,"5min":7.1440000000000001,"15min":7.1440000000000001},"last":0.51100000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.3660000000000001,"5min":1.2829999999999999,"15min":1.2070000000000001},"min":{"1min":0.37,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.5229999999999997,"5min":6.9630000000000001,"15min":7.8419999999999996},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.86499999999999999,"5min":1.071,"15min":1.0449999999999999},"min":{"1min":0.35499999999999998,"5min":0.27700000000000002,"15min":0.26100000000000001},"max":{"1min":4.7320000000000002,"5min":6.8399999999999999,"15min":6.8399999999999999},"last":1.639}]},{"osd":7,"last update":"Sat Sep 23 18:46:05 2023","interfaces":[{"interface":"back","average":{"1min":1.546,"5min":1.3200000000000001,"15min":1.5449999999999999},"min":{"1min":0.41899999999999998,"5min":0.32500000000000001,"15min":0.311},"max":{"1min":8.1859999999999999,"5min":8.1859999999999999,"15min":13.491},"last":1.6850000000000001},{"interface":"front","average":{"1min":1.244,"5min":1.5940000000000001,"15min":1.6919999999999999},"min":{"1min":0.47299999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.1219999999999999,"5min":10.106,"15min":17.062999999999999},"last":0.747}]}]},{"osd":5,"up_from":0,"seq":154618823087,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4224,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7725,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3429,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":782700544,"data_stored":782696472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":946475008,"data_stored":946470936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":682557440,"data_stored":682557440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":691994624,"data_stored":691994624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":646447104,"data_stored":646447104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":541986816,"data_stored":541982744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":525795328,"data_stored":525795328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:46:22.355 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 41 2023-09-23T18:46:22.355 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:22 smithi098 bash[76817]: cluster 2023-09-23T18:46:22.034296+0000 mgr.y (mgr.26054) 846 : cluster 0 pgmap v869: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.1 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 713 op/s; 3318/55713 objects degraded (5.956%); 3318/55713 objects misplaced (5.956%); 0 B/s, 38 objects/s recovering 2023-09-23T18:46:22.355 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:22 smithi098 bash[79516]: cluster 2023-09-23T18:46:22.034296+0000 mgr.y (mgr.26054) 846 : cluster 0 pgmap v869: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.1 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 713 op/s; 3318/55713 objects degraded (5.956%); 3318/55713 objects misplaced (5.956%); 0 B/s, 38 objects/s recovering 2023-09-23T18:46:22.405 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:22.443 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:22.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:22.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:22.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:22 smithi173 bash[59422]: cluster 2023-09-23T18:46:22.034296+0000 mgr.y (mgr.26054) 846 : cluster 0 pgmap v869: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.1 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 713 op/s; 3318/55713 objects degraded (5.956%); 3318/55713 objects misplaced (5.956%); 0 B/s, 38 objects/s recovering 2023-09-23T18:46:22.551 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:22.576 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:22.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:22.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:22.684 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:22.738 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:22.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:22.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:22.843 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:22.871 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:22.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:22.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:22.927 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:46:22.969 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:46:22.974 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:23.016 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:23.021 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:46:23.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:23.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:23.066 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:46:23.119 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:46:23.125 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:23.148 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:23.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:23.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:23.154 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:46:23.195 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:46:23.231 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:46:23.240 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:46:23.240 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:46:23.240 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:46:23.240 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:46:23.240 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:46:23.240 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:46:23.240 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:46:23.240 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:46:23.251 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:23.274 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:23.275 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:23 smithi098 bash[76817]: audit 2023-09-23T18:46:22.857719+0000 mon.c (mon.2) 100 : audit 0 from='client.? 172.21.15.98:0/3437018175' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:46:23.275 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:23 smithi098 bash[79516]: audit 2023-09-23T18:46:22.857719+0000 mon.c (mon.2) 100 : audit 0 from='client.? 172.21.15.98:0/3437018175' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:46:23.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:23.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:23.377 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:23.411 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:23.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:23.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:23.517 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:23.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:23 smithi173 bash[59422]: audit 2023-09-23T18:46:22.857719+0000 mon.c (mon.2) 100 : audit 0 from='client.? 172.21.15.98:0/3437018175' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:46:23.545 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:23.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:23.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:23.651 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:23.678 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:23.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:23.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:23.781 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:23.821 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:23.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:23.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:23.926 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:23.956 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:23.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:23.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:24.059 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:24.096 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:24.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:24.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:24.169 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is set 2023-09-23T18:46:24.200 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:24.224 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:24.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:24.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:24.332 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:24.341 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:24 smithi098 bash[79516]: audit 2023-09-23T18:46:23.329465+0000 mon.a (mon.0) 1125 : audit 1 from='client.? 172.21.15.98:0/411813803' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:46:24.341 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:24 smithi098 bash[79516]: cluster 2023-09-23T18:46:24.035039+0000 mgr.y (mgr.26054) 847 : cluster 0 pgmap v870: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.1 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 3004/51789 objects degraded (5.800%); 3004/51789 objects misplaced (5.800%); 0 B/s, 64 objects/s recovering 2023-09-23T18:46:24.343 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:24 smithi098 bash[76817]: audit 2023-09-23T18:46:23.329465+0000 mon.a (mon.0) 1125 : audit 1 from='client.? 172.21.15.98:0/411813803' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:46:24.343 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:24 smithi098 bash[76817]: cluster 2023-09-23T18:46:24.035039+0000 mgr.y (mgr.26054) 847 : cluster 0 pgmap v870: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.1 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 3004/51789 objects degraded (5.800%); 3004/51789 objects misplaced (5.800%); 0 B/s, 64 objects/s recovering 2023-09-23T18:46:24.371 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:24.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:24.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:24.477 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:24.509 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:24.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:24.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:24.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:24 smithi173 bash[59422]: audit 2023-09-23T18:46:23.329465+0000 mon.a (mon.0) 1125 : audit 1 from='client.? 172.21.15.98:0/411813803' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:46:24.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:24 smithi173 bash[59422]: cluster 2023-09-23T18:46:24.035039+0000 mgr.y (mgr.26054) 847 : cluster 0 pgmap v870: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.1 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 3004/51789 objects degraded (5.800%); 3004/51789 objects misplaced (5.800%); 0 B/s, 64 objects/s recovering 2023-09-23T18:46:24.611 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:24.634 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:24.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:24.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:24.741 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:24.768 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:24.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:24.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:24.872 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:24.907 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:24.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:24.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:25.012 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:25.039 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:25.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:25.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:25.145 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:25 smithi098 bash[76817]: cluster 2023-09-23T18:46:24.128160+0000 mon.a (mon.0) 1126 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:46:25.145 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:25 smithi098 bash[76817]: audit 2023-09-23T18:46:24.152790+0000 mon.a (mon.0) 1127 : audit 1 from='client.? 172.21.15.98:0/411813803' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:46:25.145 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:25 smithi098 bash[76817]: cluster 2023-09-23T18:46:24.153308+0000 mon.a (mon.0) 1128 : cluster 0 osdmap e210: 8 total, 8 up, 7 in 2023-09-23T18:46:25.145 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:25 smithi098 bash[76817]: cluster 2023-09-23T18:46:24.162625+0000 osd.6 (osd.6) 157 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T18:46:25.145 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:25 smithi098 bash[76817]: cluster 2023-09-23T18:46:24.164034+0000 osd.6 (osd.6) 158 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T18:46:25.145 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:25 smithi098 bash[76817]: cluster 2023-09-23T18:46:24.165393+0000 osd.7 (osd.7) 126 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T18:46:25.146 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:25 smithi098 bash[76817]: audit 2023-09-23T18:46:24.612972+0000 mon.a (mon.0) 1129 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:46:25.146 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:25 smithi098 bash[76817]: cluster 2023-09-23T18:46:25.080374+0000 mon.a (mon.0) 1130 : cluster 3 Health check update: Degraded data redundancy: 3004/51789 objects degraded (5.800%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:46:25.146 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:25 smithi098 bash[79516]: cluster 2023-09-23T18:46:24.128160+0000 mon.a (mon.0) 1126 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:46:25.146 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:25 smithi098 bash[79516]: audit 2023-09-23T18:46:24.152790+0000 mon.a (mon.0) 1127 : audit 1 from='client.? 172.21.15.98:0/411813803' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:46:25.146 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:25 smithi098 bash[79516]: cluster 2023-09-23T18:46:24.153308+0000 mon.a (mon.0) 1128 : cluster 0 osdmap e210: 8 total, 8 up, 7 in 2023-09-23T18:46:25.146 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:25 smithi098 bash[79516]: cluster 2023-09-23T18:46:24.162625+0000 osd.6 (osd.6) 157 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T18:46:25.146 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:25 smithi098 bash[79516]: cluster 2023-09-23T18:46:24.164034+0000 osd.6 (osd.6) 158 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T18:46:25.146 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:25 smithi098 bash[79516]: cluster 2023-09-23T18:46:24.165393+0000 osd.7 (osd.7) 126 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T18:46:25.146 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:25 smithi098 bash[79516]: audit 2023-09-23T18:46:24.612972+0000 mon.a (mon.0) 1129 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:46:25.147 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:25 smithi098 bash[79516]: cluster 2023-09-23T18:46:25.080374+0000 mon.a (mon.0) 1130 : cluster 3 Health check update: Degraded data redundancy: 3004/51789 objects degraded (5.800%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:46:25.147 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:25.166 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:25.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:25.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:25.269 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:25.298 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:25.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:25.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:25.402 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:25.410 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:25 smithi173 bash[59422]: cluster 2023-09-23T18:46:24.128160+0000 mon.a (mon.0) 1126 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:46:25.410 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:25 smithi173 bash[59422]: audit 2023-09-23T18:46:24.152790+0000 mon.a (mon.0) 1127 : audit 1 from='client.? 172.21.15.98:0/411813803' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:46:25.411 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:25 smithi173 bash[59422]: cluster 2023-09-23T18:46:24.153308+0000 mon.a (mon.0) 1128 : cluster 0 osdmap e210: 8 total, 8 up, 7 in 2023-09-23T18:46:25.411 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:25 smithi173 bash[59422]: cluster 2023-09-23T18:46:24.162625+0000 osd.6 (osd.6) 157 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T18:46:25.411 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:25 smithi173 bash[59422]: cluster 2023-09-23T18:46:24.164034+0000 osd.6 (osd.6) 158 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T18:46:25.411 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:25 smithi173 bash[59422]: cluster 2023-09-23T18:46:24.165393+0000 osd.7 (osd.7) 126 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T18:46:25.411 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:25 smithi173 bash[59422]: audit 2023-09-23T18:46:24.612972+0000 mon.a (mon.0) 1129 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:46:25.411 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:25 smithi173 bash[59422]: cluster 2023-09-23T18:46:25.080374+0000 mon.a (mon.0) 1130 : cluster 3 Health check update: Degraded data redundancy: 3004/51789 objects degraded (5.800%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:46:25.429 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:25.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:25.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:25.535 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:25.559 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:25.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:25.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:25.662 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:25.685 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:25.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:25.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:25.791 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:25.819 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:25.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:25.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:25.923 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:25.950 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:25.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:25.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:26.056 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:26.094 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:26.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:26.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:26.201 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:26.210 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:26 smithi173 bash[59422]: cluster 2023-09-23T18:46:24.153781+0000 osd.4 (osd.4) 194 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T18:46:26.210 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:26 smithi173 bash[59422]: cluster 2023-09-23T18:46:26.035677+0000 mgr.y (mgr.26054) 848 : cluster 0 pgmap v872: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 994 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 3004/47577 objects degraded (6.314%); 3004/47577 objects misplaced (6.314%); 0 B/s, 54 objects/s recovering 2023-09-23T18:46:26.225 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:26.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:26.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:26.328 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:26.355 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:26.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:26.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:26.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:26 smithi098 bash[76817]: cluster 2023-09-23T18:46:24.153781+0000 osd.4 (osd.4) 194 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T18:46:26.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:26 smithi098 bash[76817]: cluster 2023-09-23T18:46:26.035677+0000 mgr.y (mgr.26054) 848 : cluster 0 pgmap v872: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 994 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 3004/47577 objects degraded (6.314%); 3004/47577 objects misplaced (6.314%); 0 B/s, 54 objects/s recovering 2023-09-23T18:46:26.441 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:46:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:46:26] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.33.4" 2023-09-23T18:46:26.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:26 smithi098 bash[79516]: cluster 2023-09-23T18:46:24.153781+0000 osd.4 (osd.4) 194 : cluster 3 failed to encode map e210 with expected crc 2023-09-23T18:46:26.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:26 smithi098 bash[79516]: cluster 2023-09-23T18:46:26.035677+0000 mgr.y (mgr.26054) 848 : cluster 0 pgmap v872: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 994 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 3004/47577 objects degraded (6.314%); 3004/47577 objects misplaced (6.314%); 0 B/s, 54 objects/s recovering 2023-09-23T18:46:26.459 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:26.489 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:26.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:26.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:26.592 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:26.629 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:26.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:26.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:26.737 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:26.769 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:26.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:26.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:26.872 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:26.896 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:26.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:26.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:26.999 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:27.024 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:27.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:27.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:27.128 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:27.169 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:27.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:27.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:27.274 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:27.307 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:27.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:27.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:27.396 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:46:27.411 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:27.435 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:27.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:27.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:27.537 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:27.565 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:27.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:27.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:27.638 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:46:27.669 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:27.700 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:27.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:27.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:27.806 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:27.831 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:27.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:27.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:27.937 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:27.962 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:27.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:27.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:28.070 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:28.096 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:28.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:28.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:28.150 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:46:28.202 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:28.227 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:28.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:28.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:28.333 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:28.357 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:28.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:28.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:28.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:28 smithi098 bash[76817]: cluster 2023-09-23T18:46:27.164890+0000 osd.4 (osd.4) 195 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T18:46:28.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:28 smithi098 bash[76817]: cluster 2023-09-23T18:46:27.165917+0000 mon.a (mon.0) 1131 : cluster 0 osdmap e211: 8 total, 8 up, 7 in 2023-09-23T18:46:28.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:28 smithi098 bash[76817]: cluster 2023-09-23T18:46:27.170922+0000 osd.7 (osd.7) 127 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T18:46:28.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:28 smithi098 bash[76817]: cluster 2023-09-23T18:46:27.172130+0000 osd.6 (osd.6) 159 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T18:46:28.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:28 smithi098 bash[76817]: cluster 2023-09-23T18:46:27.374382+0000 osd.5 (osd.5) 106 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T18:46:28.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:28 smithi098 bash[76817]: cluster 2023-09-23T18:46:28.036167+0000 mgr.y (mgr.26054) 849 : cluster 0 pgmap v874: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 954 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 3004/45681 objects degraded (6.576%); 3004/45681 objects misplaced (6.576%); 0 B/s, 39 objects/s recovering 2023-09-23T18:46:28.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:28 smithi098 bash[79516]: cluster 2023-09-23T18:46:27.164890+0000 osd.4 (osd.4) 195 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T18:46:28.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:28 smithi098 bash[79516]: cluster 2023-09-23T18:46:27.165917+0000 mon.a (mon.0) 1131 : cluster 0 osdmap e211: 8 total, 8 up, 7 in 2023-09-23T18:46:28.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:28 smithi098 bash[79516]: cluster 2023-09-23T18:46:27.170922+0000 osd.7 (osd.7) 127 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T18:46:28.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:28 smithi098 bash[79516]: cluster 2023-09-23T18:46:27.172130+0000 osd.6 (osd.6) 159 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T18:46:28.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:28 smithi098 bash[79516]: cluster 2023-09-23T18:46:27.374382+0000 osd.5 (osd.5) 106 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T18:46:28.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:28 smithi098 bash[79516]: cluster 2023-09-23T18:46:28.036167+0000 mgr.y (mgr.26054) 849 : cluster 0 pgmap v874: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 954 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 3004/45681 objects degraded (6.576%); 3004/45681 objects misplaced (6.576%); 0 B/s, 39 objects/s recovering 2023-09-23T18:46:28.463 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:28.472 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:28 smithi173 bash[59422]: cluster 2023-09-23T18:46:27.164890+0000 osd.4 (osd.4) 195 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T18:46:28.472 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:28 smithi173 bash[59422]: cluster 2023-09-23T18:46:27.165917+0000 mon.a (mon.0) 1131 : cluster 0 osdmap e211: 8 total, 8 up, 7 in 2023-09-23T18:46:28.472 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:28 smithi173 bash[59422]: cluster 2023-09-23T18:46:27.170922+0000 osd.7 (osd.7) 127 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T18:46:28.472 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:28 smithi173 bash[59422]: cluster 2023-09-23T18:46:27.172130+0000 osd.6 (osd.6) 159 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T18:46:28.472 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:28 smithi173 bash[59422]: cluster 2023-09-23T18:46:27.374382+0000 osd.5 (osd.5) 106 : cluster 3 failed to encode map e211 with expected crc 2023-09-23T18:46:28.472 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:28 smithi173 bash[59422]: cluster 2023-09-23T18:46:28.036167+0000 mgr.y (mgr.26054) 849 : cluster 0 pgmap v874: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 954 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 3004/45681 objects degraded (6.576%); 3004/45681 objects misplaced (6.576%); 0 B/s, 39 objects/s recovering 2023-09-23T18:46:28.490 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:28.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:28.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:28.596 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:28.624 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:28.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:28.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:28.730 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:28.768 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:28.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:28.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:28.876 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:28.908 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:28.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:28.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:29.014 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:29.043 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:29.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:29.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:29.152 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:29.189 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:29.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:29.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:29.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:29 smithi173 bash[59422]: cluster 2023-09-23T18:46:28.178821+0000 osd.6 (osd.6) 160 : cluster 3 failed to encode map e212 with expected crc 2023-09-23T18:46:29.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:29 smithi173 bash[59422]: cluster 2023-09-23T18:46:28.179438+0000 osd.7 (osd.7) 128 : cluster 3 failed to encode map e212 with expected crc 2023-09-23T18:46:29.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:29 smithi173 bash[59422]: cluster 2023-09-23T18:46:28.181098+0000 mon.a (mon.0) 1132 : cluster 0 osdmap e212: 8 total, 8 up, 7 in 2023-09-23T18:46:29.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:29 smithi173 bash[59422]: cluster 2023-09-23T18:46:28.182723+0000 osd.7 (osd.7) 129 : cluster 3 failed to encode map e212 with expected crc 2023-09-23T18:46:29.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:29 smithi173 bash[59422]: cluster 2023-09-23T18:46:28.199546+0000 osd.4 (osd.4) 196 : cluster 3 failed to encode map e212 with expected crc 2023-09-23T18:46:29.295 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:29.334 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:29.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:29.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:29.436 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:29 smithi098 bash[76817]: cluster 2023-09-23T18:46:28.178821+0000 osd.6 (osd.6) 160 : cluster 3 failed to encode map e212 with expected crc 2023-09-23T18:46:29.436 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:29 smithi098 bash[76817]: cluster 2023-09-23T18:46:28.179438+0000 osd.7 (osd.7) 128 : cluster 3 failed to encode map e212 with expected crc 2023-09-23T18:46:29.436 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:29 smithi098 bash[76817]: cluster 2023-09-23T18:46:28.181098+0000 mon.a (mon.0) 1132 : cluster 0 osdmap e212: 8 total, 8 up, 7 in 2023-09-23T18:46:29.436 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:29 smithi098 bash[76817]: cluster 2023-09-23T18:46:28.182723+0000 osd.7 (osd.7) 129 : cluster 3 failed to encode map e212 with expected crc 2023-09-23T18:46:29.436 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:29 smithi098 bash[76817]: cluster 2023-09-23T18:46:28.199546+0000 osd.4 (osd.4) 196 : cluster 3 failed to encode map e212 with expected crc 2023-09-23T18:46:29.437 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:29 smithi098 bash[79516]: cluster 2023-09-23T18:46:28.178821+0000 osd.6 (osd.6) 160 : cluster 3 failed to encode map e212 with expected crc 2023-09-23T18:46:29.437 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:29 smithi098 bash[79516]: cluster 2023-09-23T18:46:28.179438+0000 osd.7 (osd.7) 128 : cluster 3 failed to encode map e212 with expected crc 2023-09-23T18:46:29.437 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:29 smithi098 bash[79516]: cluster 2023-09-23T18:46:28.181098+0000 mon.a (mon.0) 1132 : cluster 0 osdmap e212: 8 total, 8 up, 7 in 2023-09-23T18:46:29.437 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:29 smithi098 bash[79516]: cluster 2023-09-23T18:46:28.182723+0000 osd.7 (osd.7) 129 : cluster 3 failed to encode map e212 with expected crc 2023-09-23T18:46:29.437 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:29 smithi098 bash[79516]: cluster 2023-09-23T18:46:28.199546+0000 osd.4 (osd.4) 196 : cluster 3 failed to encode map e212 with expected crc 2023-09-23T18:46:29.439 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:29.464 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:29.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:29.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:29.567 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:29.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:29.598 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:29.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:29.675 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:46:29.699 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:29.726 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:29.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:29.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:29.829 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:29.858 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:29.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:29.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:29.961 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:29.991 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:29.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:29.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:30.096 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:30.143 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:30.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:30.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:30.249 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:30.274 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:30.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:30.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:30.377 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:30.390 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:30 smithi098 bash[79516]: cluster 2023-09-23T18:46:30.037248+0000 mgr.y (mgr.26054) 850 : cluster 0 pgmap v876: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 97 active+clean; 836 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 2949/39990 objects degraded (7.374%); 2951/39990 objects misplaced (7.379%); 576 KiB/s, 8 objects/s recovering 2023-09-23T18:46:30.390 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:30 smithi098 bash[79516]: cluster 2023-09-23T18:46:30.082601+0000 mon.a (mon.0) 1133 : cluster 3 Health check update: Degraded data redundancy: 3004/45681 objects degraded (6.576%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:46:30.390 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:30 smithi098 bash[76817]: cluster 2023-09-23T18:46:30.037248+0000 mgr.y (mgr.26054) 850 : cluster 0 pgmap v876: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 97 active+clean; 836 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 2949/39990 objects degraded (7.374%); 2951/39990 objects misplaced (7.379%); 576 KiB/s, 8 objects/s recovering 2023-09-23T18:46:30.390 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:30 smithi098 bash[76817]: cluster 2023-09-23T18:46:30.082601+0000 mon.a (mon.0) 1133 : cluster 3 Health check update: Degraded data redundancy: 3004/45681 objects degraded (6.576%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:46:30.405 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:30.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:30.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:30.508 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:30.518 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:30 smithi173 bash[59422]: cluster 2023-09-23T18:46:30.037248+0000 mgr.y (mgr.26054) 850 : cluster 0 pgmap v876: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 97 active+clean; 836 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 2949/39990 objects degraded (7.374%); 2951/39990 objects misplaced (7.379%); 576 KiB/s, 8 objects/s recovering 2023-09-23T18:46:30.519 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:30 smithi173 bash[59422]: cluster 2023-09-23T18:46:30.082601+0000 mon.a (mon.0) 1133 : cluster 3 Health check update: Degraded data redundancy: 3004/45681 objects degraded (6.576%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:46:30.533 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:30.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:30.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:30.637 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:30.666 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:30.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:30.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:30.770 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:30.813 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:30.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:30.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:30.921 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:30.948 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:30.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:30.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:31.053 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:31.078 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:31.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:31.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:31.181 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:31.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:31.211 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:31.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:31.312 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:31.342 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:31.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:31.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:31.445 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:31.476 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:31.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:31.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:31.582 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:31.629 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:31.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:31.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:31.715 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:46:31.735 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:31.792 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:31.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:31.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:31.900 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:31.940 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:31.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:31.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:32.047 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:32.082 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:32.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:32.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:32.186 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:32.211 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:32.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:32.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:32.316 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:32.395 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:32.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:32.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:32.504 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:32.534 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:32.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:32.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:32.639 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:32.668 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:32.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:32.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:32.734 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:46:32.772 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:32.796 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:32.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:32.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:32.904 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:32.931 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:32.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:32.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:33.036 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:33.068 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:33.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:33.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:33.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:33 smithi173 bash[59422]: cluster 2023-09-23T18:46:32.038077+0000 mgr.y (mgr.26054) 851 : cluster 0 pgmap v877: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 736 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 701 op/s; 2739/35193 objects degraded (7.783%); 2741/35193 objects misplaced (7.788%); 438 KiB/s, 33 objects/s recovering 2023-09-23T18:46:33.173 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:33.201 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:33.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:33.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:33.303 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:33.330 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:33.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:33.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:33.435 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:33.469 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:33.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:33.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:33.492 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:33 smithi098 bash[76817]: cluster 2023-09-23T18:46:32.038077+0000 mgr.y (mgr.26054) 851 : cluster 0 pgmap v877: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 736 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 701 op/s; 2739/35193 objects degraded (7.783%); 2741/35193 objects misplaced (7.788%); 438 KiB/s, 33 objects/s recovering 2023-09-23T18:46:33.493 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:33 smithi098 bash[79516]: cluster 2023-09-23T18:46:32.038077+0000 mgr.y (mgr.26054) 851 : cluster 0 pgmap v877: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 736 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 701 op/s; 2739/35193 objects degraded (7.783%); 2741/35193 objects misplaced (7.788%); 438 KiB/s, 33 objects/s recovering 2023-09-23T18:46:33.576 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:33.601 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:33.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:33.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:33.704 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:33.736 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:33.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:33.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:33.841 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:33.869 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:33.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:33.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:33.971 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:33.999 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:34.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:34.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:34.103 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:34.120 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:34.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:34.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:34.223 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:34.258 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:34.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:34.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:34.364 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:34.399 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:34.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:34.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:34.505 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:34.529 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:34.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:34.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:34.637 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:34.671 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:34.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:34.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:34.775 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:34.811 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:34.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:34.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:34.918 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:34.946 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:34.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:34.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:35.049 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:35.077 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:35.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:35.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:35.181 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:35.217 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:35.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:35.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:35.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:35 smithi173 bash[59422]: cluster 2023-09-23T18:46:34.038837+0000 mgr.y (mgr.26054) 852 : cluster 0 pgmap v878: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 652 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 683 op/s; 2614/31158 objects degraded (8.389%); 2614/31158 objects misplaced (8.389%); 440 KiB/s, 48 objects/s recovering 2023-09-23T18:46:35.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:35 smithi173 bash[59422]: cluster 2023-09-23T18:46:35.083637+0000 mon.a (mon.0) 1134 : cluster 3 Health check update: Degraded data redundancy: 2614/31158 objects degraded (8.389%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:46:35.326 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:35.350 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:35.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:35.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:35.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:35 smithi098 bash[76817]: cluster 2023-09-23T18:46:34.038837+0000 mgr.y (mgr.26054) 852 : cluster 0 pgmap v878: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 652 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 683 op/s; 2614/31158 objects degraded (8.389%); 2614/31158 objects misplaced (8.389%); 440 KiB/s, 48 objects/s recovering 2023-09-23T18:46:35.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:35 smithi098 bash[76817]: cluster 2023-09-23T18:46:35.083637+0000 mon.a (mon.0) 1134 : cluster 3 Health check update: Degraded data redundancy: 2614/31158 objects degraded (8.389%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:46:35.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:35 smithi098 bash[79516]: cluster 2023-09-23T18:46:34.038837+0000 mgr.y (mgr.26054) 852 : cluster 0 pgmap v878: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 652 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 683 op/s; 2614/31158 objects degraded (8.389%); 2614/31158 objects misplaced (8.389%); 440 KiB/s, 48 objects/s recovering 2023-09-23T18:46:35.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:35 smithi098 bash[79516]: cluster 2023-09-23T18:46:35.083637+0000 mon.a (mon.0) 1134 : cluster 3 Health check update: Degraded data redundancy: 2614/31158 objects degraded (8.389%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:46:35.453 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:35.483 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:35.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:35.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:35.587 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:35.614 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:35.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:35.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:35.716 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:35.747 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:35.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:35.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:35.852 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:35.875 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:35.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:35.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:35.979 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:36.006 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:36.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:36.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:36.111 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:36.129 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:36.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:36.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:36.232 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:36.262 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:36.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:36.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:36.371 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:36.397 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:36.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:36.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:36.441 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:46:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:46:36] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.33.4" 2023-09-23T18:46:36.501 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:36.531 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:36.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:36.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:36.638 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:36.671 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:36.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:36.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:36.774 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:36.805 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:36.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:36.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:36.910 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:36.934 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:36.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:36.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:37.040 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:37.066 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:37.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:37.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:37.171 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:37.179 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:37 smithi173 bash[59422]: cluster 2023-09-23T18:46:36.039442+0000 mgr.y (mgr.26054) 853 : cluster 0 pgmap v879: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 557 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 717 op/s; 2614/26580 objects degraded (9.834%); 2614/26580 objects misplaced (9.834%); 397 KiB/s, 43 objects/s recovering 2023-09-23T18:46:37.203 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:37.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:37.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:37.306 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:37.329 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:37.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:37.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:37.397 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:37 smithi098 bash[76817]: cluster 2023-09-23T18:46:36.039442+0000 mgr.y (mgr.26054) 853 : cluster 0 pgmap v879: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 557 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 717 op/s; 2614/26580 objects degraded (9.834%); 2614/26580 objects misplaced (9.834%); 397 KiB/s, 43 objects/s recovering 2023-09-23T18:46:37.398 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:37 smithi098 bash[79516]: cluster 2023-09-23T18:46:36.039442+0000 mgr.y (mgr.26054) 853 : cluster 0 pgmap v879: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 557 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 717 op/s; 2614/26580 objects degraded (9.834%); 2614/26580 objects misplaced (9.834%); 397 KiB/s, 43 objects/s recovering 2023-09-23T18:46:37.432 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:37.470 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:37.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:37.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:37.573 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:37.602 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:37.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:37.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:37.706 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:37.740 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:37.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:37.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:37.844 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:37.869 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:37.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:37.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:37.973 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:37.998 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:37.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:38.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:38.101 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:38.123 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:38.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:38.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:38.230 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:38.259 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:38.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:38.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:38.362 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:38.389 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:38.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:38.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:38.492 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:38.510 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:38.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:38.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:38.612 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:38.636 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:38.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:38.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:38.740 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:38.768 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:38.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:38.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:38.874 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:38.908 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:38.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:38.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:39.012 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:39.042 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:39.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:39.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:39.146 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:39.155 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:39 smithi173 bash[59422]: cluster 2023-09-23T18:46:38.041060+0000 mgr.y (mgr.26054) 854 : cluster 0 pgmap v880: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 521 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 693 op/s; 2614/24855 objects degraded (10.517%); 2614/24855 objects misplaced (10.517%); 352 KiB/s, 38 objects/s recovering 2023-09-23T18:46:39.171 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:39.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:39.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:39.277 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:39.311 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:39.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:39.317 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:39 smithi098 bash[76817]: cluster 2023-09-23T18:46:38.041060+0000 mgr.y (mgr.26054) 854 : cluster 0 pgmap v880: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 521 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 693 op/s; 2614/24855 objects degraded (10.517%); 2614/24855 objects misplaced (10.517%); 352 KiB/s, 38 objects/s recovering 2023-09-23T18:46:39.317 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:39 smithi098 bash[79516]: cluster 2023-09-23T18:46:38.041060+0000 mgr.y (mgr.26054) 854 : cluster 0 pgmap v880: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 521 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 693 op/s; 2614/24855 objects degraded (10.517%); 2614/24855 objects misplaced (10.517%); 352 KiB/s, 38 objects/s recovering 2023-09-23T18:46:39.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:39.419 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:39.446 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:39.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:39.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:39.548 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:39.573 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:39.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:39.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:39.675 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:39.701 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:39.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:39.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:39.809 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:39.836 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:39.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:39.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:39.939 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:39.971 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:39.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:39.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:40.077 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:40.102 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:40.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:40.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:40.207 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:40.235 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:40.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:40.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:40.339 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:40.347 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:40 smithi098 bash[76817]: audit 2023-09-23T18:46:39.613403+0000 mon.a (mon.0) 1135 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:46:40.347 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:40 smithi098 bash[76817]: cluster 2023-09-23T18:46:40.084614+0000 mon.a (mon.0) 1136 : cluster 3 Health check update: Degraded data redundancy: 2614/24855 objects degraded (10.517%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:46:40.347 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:40 smithi098 bash[79516]: audit 2023-09-23T18:46:39.613403+0000 mon.a (mon.0) 1135 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:46:40.348 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:40 smithi098 bash[79516]: cluster 2023-09-23T18:46:40.084614+0000 mon.a (mon.0) 1136 : cluster 3 Health check update: Degraded data redundancy: 2614/24855 objects degraded (10.517%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:46:40.370 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:40.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:40.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:40.476 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:40.487 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:40 smithi173 bash[59422]: audit 2023-09-23T18:46:39.613403+0000 mon.a (mon.0) 1135 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:46:40.487 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:40 smithi173 bash[59422]: cluster 2023-09-23T18:46:40.084614+0000 mon.a (mon.0) 1136 : cluster 3 Health check update: Degraded data redundancy: 2614/24855 objects degraded (10.517%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:46:40.507 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:40.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:40.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:40.612 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:40.638 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:40.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:40.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:40.744 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:40.776 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:40.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:40.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:40.883 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:40.911 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:40.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:40.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:41.015 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:41.051 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:41.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:41.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:41.157 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:41.192 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:41.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:41.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:41.296 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:41.304 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:41 smithi098 bash[76817]: cluster 2023-09-23T18:46:40.041784+0000 mgr.y (mgr.26054) 855 : cluster 0 pgmap v881: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 409 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 2503/19476 objects degraded (12.852%); 2503/19476 objects misplaced (12.852%); 297 KiB/s, 42 objects/s recovering 2023-09-23T18:46:41.304 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:41 smithi098 bash[79516]: cluster 2023-09-23T18:46:40.041784+0000 mgr.y (mgr.26054) 855 : cluster 0 pgmap v881: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 409 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 2503/19476 objects degraded (12.852%); 2503/19476 objects misplaced (12.852%); 297 KiB/s, 42 objects/s recovering 2023-09-23T18:46:41.329 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:41.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:41.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:41.432 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:41.446 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:41 smithi173 bash[59422]: cluster 2023-09-23T18:46:40.041784+0000 mgr.y (mgr.26054) 855 : cluster 0 pgmap v881: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 409 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 2503/19476 objects degraded (12.852%); 2503/19476 objects misplaced (12.852%); 297 KiB/s, 42 objects/s recovering 2023-09-23T18:46:41.463 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:41.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:41.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:41.568 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:41.592 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:41.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:41.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:41.695 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:41.717 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:41.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:41.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:41.820 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:41.858 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:41.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:41.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:41.961 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:41.990 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:41.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:41.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:42.096 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:42.134 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:42.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:42.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:42.237 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:42.248 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:42 smithi173 bash[59422]: cluster 2023-09-23T18:46:42.042521+0000 mgr.y (mgr.26054) 856 : cluster 0 pgmap v882: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 304 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 2503/14442 objects degraded (17.331%); 2503/14442 objects misplaced (17.331%); 5.3 KiB/s, 37 objects/s recovering 2023-09-23T18:46:42.262 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:42.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:42.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:42.365 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:42.397 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:42.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:42.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:42.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:42 smithi098 bash[79516]: cluster 2023-09-23T18:46:42.042521+0000 mgr.y (mgr.26054) 856 : cluster 0 pgmap v882: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 304 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 2503/14442 objects degraded (17.331%); 2503/14442 objects misplaced (17.331%); 5.3 KiB/s, 37 objects/s recovering 2023-09-23T18:46:42.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:42 smithi098 bash[76817]: cluster 2023-09-23T18:46:42.042521+0000 mgr.y (mgr.26054) 856 : cluster 0 pgmap v882: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 304 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 2503/14442 objects degraded (17.331%); 2503/14442 objects misplaced (17.331%); 5.3 KiB/s, 37 objects/s recovering 2023-09-23T18:46:42.502 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:42.528 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:42.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:42.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:42.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:42.656 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:42.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:42.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:42.759 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:42.781 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:42.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:42.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:42.884 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:42.911 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:42.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:42.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:43.016 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:43.064 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:43.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:43.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:43.169 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:43.194 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:43.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:43.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:43.297 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:43.325 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:43.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:43.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:43.429 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:43.463 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:43.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:43.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:43.568 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:43.595 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:43.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:43.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:43.699 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:43.732 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:43.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:43.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:43.835 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:43.874 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:43.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:43.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:43.977 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:44.013 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:44.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:44.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:44.117 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:44.151 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:44.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:44.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:44.256 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:44.288 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:44.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:44.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:44.392 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:44.415 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:44.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:44.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:44.519 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:44.551 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:44.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:44.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:44.654 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:44.680 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:44.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:44.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:44.785 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:44.813 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:44.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:44.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:44.917 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:44.940 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:44.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:44.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:45.043 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:45.086 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:45.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:45.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:45.190 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:45.228 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:45.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:45.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:45.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:45 smithi173 bash[59422]: cluster 2023-09-23T18:46:44.043221+0000 mgr.y (mgr.26054) 857 : cluster 0 pgmap v883: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 219 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 2373/10359 objects degraded (22.908%); 2373/10359 objects misplaced (22.908%); 5.3 KiB/s, 30 objects/s recovering 2023-09-23T18:46:45.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:45 smithi173 bash[59422]: cluster 2023-09-23T18:46:45.085710+0000 mon.a (mon.0) 1137 : cluster 3 Health check update: Degraded data redundancy: 2373/10359 objects degraded (22.908%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:46:45.332 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:45.341 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:45 smithi098 bash[76817]: cluster 2023-09-23T18:46:44.043221+0000 mgr.y (mgr.26054) 857 : cluster 0 pgmap v883: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 219 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 2373/10359 objects degraded (22.908%); 2373/10359 objects misplaced (22.908%); 5.3 KiB/s, 30 objects/s recovering 2023-09-23T18:46:45.341 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:45 smithi098 bash[76817]: cluster 2023-09-23T18:46:45.085710+0000 mon.a (mon.0) 1137 : cluster 3 Health check update: Degraded data redundancy: 2373/10359 objects degraded (22.908%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:46:45.341 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:45 smithi098 bash[79516]: cluster 2023-09-23T18:46:44.043221+0000 mgr.y (mgr.26054) 857 : cluster 0 pgmap v883: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 219 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 2373/10359 objects degraded (22.908%); 2373/10359 objects misplaced (22.908%); 5.3 KiB/s, 30 objects/s recovering 2023-09-23T18:46:45.341 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:45 smithi098 bash[79516]: cluster 2023-09-23T18:46:45.085710+0000 mon.a (mon.0) 1137 : cluster 3 Health check update: Degraded data redundancy: 2373/10359 objects degraded (22.908%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:46:45.373 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:45.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:45.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:45.477 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:45.499 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:45.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:45.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:45.602 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:45.624 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:45.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:45.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:45.728 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:45.754 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:45.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:45.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:45.857 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:45.889 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:45.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:45.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:45.994 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:46.021 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:46.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:46.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:46.124 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:46.158 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:46.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:46.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:46.191 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:46:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:46:46] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.33.4" 2023-09-23T18:46:46.267 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:46.298 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:46.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:46.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:46.402 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:46.427 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:46.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:46.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:46.530 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:46.552 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:46.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:46.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:46.656 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:46.693 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:46.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:46.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:46.741 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Removed 100942 objects 2023-09-23T18:46:46.741 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Clean up completed and total clean up time :134.453 2023-09-23T18:46:46.798 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:46.828 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:46.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:46.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:46.932 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:46.963 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:46.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:46.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:47.067 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:47.087 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:47.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:47.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:47.190 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:47.223 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:47.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:47.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:47.327 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:47.338 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:47 smithi098 bash[79516]: cluster 2023-09-23T18:46:46.043847+0000 mgr.y (mgr.26054) 858 : cluster 0 pgmap v884: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 129 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 700 op/s; 2373/6063 objects degraded (39.139%); 2373/6063 objects misplaced (39.139%); 0 B/s, 20 objects/s recovering 2023-09-23T18:46:47.338 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:47 smithi098 bash[76817]: cluster 2023-09-23T18:46:46.043847+0000 mgr.y (mgr.26054) 858 : cluster 0 pgmap v884: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 129 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 700 op/s; 2373/6063 objects degraded (39.139%); 2373/6063 objects misplaced (39.139%); 0 B/s, 20 objects/s recovering 2023-09-23T18:46:47.351 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:47.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:47.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:47.454 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:47.476 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:47.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:47.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:47.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:47 smithi173 bash[59422]: cluster 2023-09-23T18:46:46.043847+0000 mgr.y (mgr.26054) 858 : cluster 0 pgmap v884: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 129 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 700 op/s; 2373/6063 objects degraded (39.139%); 2373/6063 objects misplaced (39.139%); 0 B/s, 20 objects/s recovering 2023-09-23T18:46:47.579 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:47.598 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:47.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:47.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:47.701 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:47.725 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:47.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:47.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:47.828 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:47.854 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:47.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:47.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:47.958 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:47.981 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:47.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:47.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:48.084 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:48.108 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:48.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:48.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:48.211 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:48.232 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:48.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:48.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:48.336 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:48.371 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:48.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:48.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:48.475 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:48.502 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:48.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:48.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:48.606 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:48.630 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:48.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:48.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:48.733 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:48.762 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:48.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:48.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:48.865 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:48.894 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:48.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:48.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:48.997 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:49.020 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:49.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:49.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:49.123 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:49.150 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:49.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:49.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:49.254 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:49.283 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:49.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:49.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:49.360 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:49 smithi098 bash[76817]: cluster 2023-09-23T18:46:48.044301+0000 mgr.y (mgr.26054) 859 : cluster 0 pgmap v885: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 91 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 2373/4260 objects degraded (55.704%); 2373/4260 objects misplaced (55.704%); 0 B/s, 20 objects/s recovering 2023-09-23T18:46:49.360 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:49 smithi098 bash[79516]: cluster 2023-09-23T18:46:48.044301+0000 mgr.y (mgr.26054) 859 : cluster 0 pgmap v885: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 91 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 2373/4260 objects degraded (55.704%); 2373/4260 objects misplaced (55.704%); 0 B/s, 20 objects/s recovering 2023-09-23T18:46:49.388 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:49.407 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:49.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:49.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:49.511 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:49.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:49 smithi173 bash[59422]: cluster 2023-09-23T18:46:48.044301+0000 mgr.y (mgr.26054) 859 : cluster 0 pgmap v885: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 91 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 2373/4260 objects degraded (55.704%); 2373/4260 objects misplaced (55.704%); 0 B/s, 20 objects/s recovering 2023-09-23T18:46:49.530 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:49.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:49.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:49.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:49.664 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:49.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:49.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:49.768 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:49.797 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:49.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:49.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:49.901 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:49.924 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:49.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:49.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:50.027 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:50.029 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:46:50.029 INFO:teuthology.orchestra.run.smithi098.stdout: "size": 20, 2023-09-23T18:46:50.029 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 600, 2023-09-23T18:46:50.030 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [ 2023-09-23T18:46:50.030 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.030 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:82304 7.10 7.22748870 (undecoded) ondisk+retry+write+known_if_redirected e198)", 2023-09-23T18:46:50.030 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:06.101814+0000", 2023-09-23T18:46:50.030 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 163.92348975900001, 2023-09-23T18:46:50.030 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.99571468500000004, 2023-09-23T18:46:50.030 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:46:50.030 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T18:46:50.030 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:46:50.030 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:46:50.030 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:46:50.030 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 82304 2023-09-23T18:46:50.030 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.031 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:46:50.031 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.031 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:46:50.031 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.101814+0000", 2023-09-23T18:46:50.031 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.031 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.031 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.031 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:46:50.031 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.101814+0000", 2023-09-23T18:46:50.031 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.031 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.031 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.032 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:46:50.032 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.101814+0000", 2023-09-23T18:46:50.032 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:46:50.032 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.032 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.032 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:46:50.032 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.101834+0000", 2023-09-23T18:46:50.032 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.0007999999999999e-05 2023-09-23T18:46:50.032 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.032 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.032 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:46:50.032 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.101843+0000", 2023-09-23T18:46:50.032 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.3230000000000004e-06 2023-09-23T18:46:50.033 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.033 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.033 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:46:50.033 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.101850+0000", 2023-09-23T18:46:50.033 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.6370000000000002e-06 2023-09-23T18:46:50.033 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.033 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.033 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.033 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.102021+0000", 2023-09-23T18:46:50.033 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00017134 2023-09-23T18:46:50.033 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.033 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.033 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:46:50.034 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.102028+0000", 2023-09-23T18:46:50.034 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.7320000000000001e-06 2023-09-23T18:46:50.034 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.034 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.034 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.034 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:07.097511+0000", 2023-09-23T18:46:50.034 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.99548390799999997 2023-09-23T18:46:50.034 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.034 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.034 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:46:50.034 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:07.097529+0000", 2023-09-23T18:46:50.034 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.7489000000000001e-05 2023-09-23T18:46:50.035 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.035 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:46:50.035 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.035 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.035 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.035 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:82308 7.10 7.f62fe0f0 (undecoded) ondisk+retry+write+known_if_redirected e198)", 2023-09-23T18:46:50.035 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:06.101855+0000", 2023-09-23T18:46:50.035 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 163.923449482, 2023-09-23T18:46:50.035 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.99593049099999997, 2023-09-23T18:46:50.035 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:46:50.035 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T18:46:50.035 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:46:50.036 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:46:50.036 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:46:50.036 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 82308 2023-09-23T18:46:50.036 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.036 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:46:50.036 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.036 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:46:50.036 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.101855+0000", 2023-09-23T18:46:50.036 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.036 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.036 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.036 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:46:50.036 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.101855+0000", 2023-09-23T18:46:50.037 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.037 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.037 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.037 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:46:50.037 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.101854+0000", 2023-09-23T18:46:50.037 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:46:50.037 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.037 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.037 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:46:50.037 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.101873+0000", 2023-09-23T18:46:50.037 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.8615e-05 2023-09-23T18:46:50.037 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.037 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.038 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:46:50.038 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.101879+0000", 2023-09-23T18:46:50.038 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.173e-06 2023-09-23T18:46:50.038 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.038 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.038 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:46:50.038 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.101884+0000", 2023-09-23T18:46:50.038 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.0690000000000001e-06 2023-09-23T18:46:50.038 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.038 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.038 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.038 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.102172+0000", 2023-09-23T18:46:50.038 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00028812400000000001 2023-09-23T18:46:50.039 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.039 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.039 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:46:50.039 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.102179+0000", 2023-09-23T18:46:50.039 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.4760000000000003e-06 2023-09-23T18:46:50.039 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.039 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.039 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.039 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:07.097778+0000", 2023-09-23T18:46:50.039 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.99559923100000003 2023-09-23T18:46:50.039 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.039 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.040 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:46:50.040 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:07.097785+0000", 2023-09-23T18:46:50.040 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.2409999999999998e-06 2023-09-23T18:46:50.040 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.040 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:46:50.040 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.040 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.040 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.040 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:82342 7.10 7.e4560530 (undecoded) ondisk+write+known_if_redirected e198)", 2023-09-23T18:46:50.040 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:06.125241+0000", 2023-09-23T18:46:50.040 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 163.90006287700001, 2023-09-23T18:46:50.040 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.97256867599999997, 2023-09-23T18:46:50.041 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:46:50.041 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T18:46:50.041 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:46:50.041 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:46:50.041 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:46:50.041 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 82342 2023-09-23T18:46:50.041 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.041 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:46:50.041 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.041 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:46:50.041 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.125241+0000", 2023-09-23T18:46:50.042 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.042 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.042 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.042 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:46:50.042 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.125241+0000", 2023-09-23T18:46:50.042 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.042 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.042 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.042 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:46:50.042 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.125240+0000", 2023-09-23T18:46:50.042 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:46:50.042 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.043 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.043 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:46:50.043 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.125262+0000", 2023-09-23T18:46:50.043 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.2790999999999999e-05 2023-09-23T18:46:50.043 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.043 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.043 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:46:50.043 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.125273+0000", 2023-09-23T18:46:50.043 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1090999999999999e-05 2023-09-23T18:46:50.043 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.043 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.043 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:46:50.044 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.125281+0000", 2023-09-23T18:46:50.044 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.4800000000000004e-06 2023-09-23T18:46:50.044 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.044 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.044 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.044 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.125330+0000", 2023-09-23T18:46:50.044 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.8599e-05 2023-09-23T18:46:50.044 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.044 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.044 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:46:50.044 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.125339+0000", 2023-09-23T18:46:50.044 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.0860000000000008e-06 2023-09-23T18:46:50.045 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.045 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.045 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.045 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:07.097803+0000", 2023-09-23T18:46:50.045 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.97246391899999995 2023-09-23T18:46:50.045 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.045 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.045 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:46:50.045 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:07.097810+0000", 2023-09-23T18:46:50.045 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.323e-06 2023-09-23T18:46:50.045 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.045 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:46:50.046 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.046 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.046 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.046 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:82397 7.10 7.bb9408f0 (undecoded) ondisk+write+known_if_redirected e198)", 2023-09-23T18:46:50.046 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:06.181348+0000", 2023-09-23T18:46:50.046 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 163.843956193, 2023-09-23T18:46:50.046 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.91650185900000003, 2023-09-23T18:46:50.046 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:46:50.046 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T18:46:50.046 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:46:50.046 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:46:50.047 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:46:50.047 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 82397 2023-09-23T18:46:50.047 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.047 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:46:50.047 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.047 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:46:50.047 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.181348+0000", 2023-09-23T18:46:50.047 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.047 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.047 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.047 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:46:50.047 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.181348+0000", 2023-09-23T18:46:50.048 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.048 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.048 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.048 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:46:50.048 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.181346+0000", 2023-09-23T18:46:50.048 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:46:50.048 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.048 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.048 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:46:50.048 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.181367+0000", 2023-09-23T18:46:50.048 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.0567000000000001e-05 2023-09-23T18:46:50.048 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.049 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.049 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:46:50.049 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.181378+0000", 2023-09-23T18:46:50.049 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0643e-05 2023-09-23T18:46:50.049 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.049 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.049 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:46:50.049 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.181386+0000", 2023-09-23T18:46:50.049 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.5599999999999994e-06 2023-09-23T18:46:50.049 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.049 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.049 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.050 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.181426+0000", 2023-09-23T18:46:50.050 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.9894000000000001e-05 2023-09-23T18:46:50.050 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.050 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.050 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:46:50.050 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.181436+0000", 2023-09-23T18:46:50.050 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.5049999999999996e-06 2023-09-23T18:46:50.050 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.050 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.050 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.050 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:07.097838+0000", 2023-09-23T18:46:50.050 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.91640275000000004 2023-09-23T18:46:50.050 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.051 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.051 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:46:50.051 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:07.097850+0000", 2023-09-23T18:46:50.051 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1301000000000001e-05 2023-09-23T18:46:50.051 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.051 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:46:50.051 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.051 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.051 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.051 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:82420 7.10 7.5066eb70 (undecoded) ondisk+write+known_if_redirected e198)", 2023-09-23T18:46:50.051 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:06.197206+0000", 2023-09-23T18:46:50.051 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 163.82809839699999, 2023-09-23T18:46:50.051 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.90066522400000004, 2023-09-23T18:46:50.052 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:46:50.052 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T18:46:50.052 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:46:50.052 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:46:50.052 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:46:50.052 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 82420 2023-09-23T18:46:50.052 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.052 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:46:50.052 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.052 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:46:50.052 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.197206+0000", 2023-09-23T18:46:50.052 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.052 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.053 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.053 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:46:50.053 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.197206+0000", 2023-09-23T18:46:50.053 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.053 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.053 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.053 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:46:50.053 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.197205+0000", 2023-09-23T18:46:50.053 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:46:50.053 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.053 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.053 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:46:50.054 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.197230+0000", 2023-09-23T18:46:50.054 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.5636e-05 2023-09-23T18:46:50.054 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.054 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.054 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:46:50.054 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.197242+0000", 2023-09-23T18:46:50.054 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1070000000000001e-05 2023-09-23T18:46:50.054 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.054 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.054 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:46:50.054 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.197249+0000", 2023-09-23T18:46:50.054 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.3119999999999996e-06 2023-09-23T18:46:50.055 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.055 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.055 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.055 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.197291+0000", 2023-09-23T18:46:50.055 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.1927e-05 2023-09-23T18:46:50.055 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.055 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.055 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:46:50.055 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.197296+0000", 2023-09-23T18:46:50.055 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.6210000000000001e-06 2023-09-23T18:46:50.055 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.055 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.056 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.056 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:07.097864+0000", 2023-09-23T18:46:50.056 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.90056744099999997 2023-09-23T18:46:50.056 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.056 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.056 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:46:50.056 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:07.097871+0000", 2023-09-23T18:46:50.056 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.0319999999999998e-06 2023-09-23T18:46:50.056 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.056 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:46:50.056 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.056 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.056 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.057 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:82423 7.10 7.bfa9c210 (undecoded) ondisk+write+known_if_redirected e198)", 2023-09-23T18:46:50.057 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:06.198041+0000", 2023-09-23T18:46:50.057 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 163.82726310499999, 2023-09-23T18:46:50.057 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.89987381, 2023-09-23T18:46:50.057 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:46:50.057 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T18:46:50.057 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:46:50.057 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:46:50.057 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:46:50.057 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 82423 2023-09-23T18:46:50.057 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.057 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:46:50.057 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.057 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:46:50.058 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.198041+0000", 2023-09-23T18:46:50.058 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.058 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.058 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.058 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:46:50.058 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.198041+0000", 2023-09-23T18:46:50.058 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.058 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.058 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.058 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:46:50.058 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.198040+0000", 2023-09-23T18:46:50.058 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:46:50.059 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.059 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.059 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:46:50.059 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.198062+0000", 2023-09-23T18:46:50.059 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.2475000000000001e-05 2023-09-23T18:46:50.059 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.059 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.059 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:46:50.059 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.198078+0000", 2023-09-23T18:46:50.059 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.5301000000000001e-05 2023-09-23T18:46:50.059 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.059 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.059 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:46:50.060 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.198086+0000", 2023-09-23T18:46:50.060 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.8609999999999998e-06 2023-09-23T18:46:50.060 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.060 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.060 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.060 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.201032+0000", 2023-09-23T18:46:50.060 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0029465649999999999 2023-09-23T18:46:50.060 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.060 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.060 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:46:50.060 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.201041+0000", 2023-09-23T18:46:50.060 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.5420000000000005e-06 2023-09-23T18:46:50.060 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.061 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.061 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.061 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:07.097903+0000", 2023-09-23T18:46:50.061 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.89686251699999997 2023-09-23T18:46:50.061 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.061 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.061 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:46:50.061 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:07.097915+0000", 2023-09-23T18:46:50.061 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1497e-05 2023-09-23T18:46:50.061 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.061 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:46:50.061 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.061 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.062 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.062 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:82455 7.10 7.80593c10 (undecoded) ondisk+write+known_if_redirected e198)", 2023-09-23T18:46:50.062 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:06.229373+0000", 2023-09-23T18:46:50.062 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 163.79593055000001, 2023-09-23T18:46:50.062 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.86860794500000005, 2023-09-23T18:46:50.062 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:46:50.062 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T18:46:50.062 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:46:50.062 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:46:50.062 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:46:50.062 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 82455 2023-09-23T18:46:50.062 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.063 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:46:50.063 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.063 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:46:50.063 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.229373+0000", 2023-09-23T18:46:50.063 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.063 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.063 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.063 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:46:50.063 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.229373+0000", 2023-09-23T18:46:50.063 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.063 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.063 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.063 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:46:50.064 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.229372+0000", 2023-09-23T18:46:50.064 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:46:50.064 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.064 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.064 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:46:50.064 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.229401+0000", 2023-09-23T18:46:50.064 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.9215999999999999e-05 2023-09-23T18:46:50.064 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.064 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.064 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:46:50.064 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.229418+0000", 2023-09-23T18:46:50.064 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.6463000000000001e-05 2023-09-23T18:46:50.065 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.065 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.065 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:46:50.065 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.229428+0000", 2023-09-23T18:46:50.065 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0088999999999999e-05 2023-09-23T18:46:50.065 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.065 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.065 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.065 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.229474+0000", 2023-09-23T18:46:50.065 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.5970000000000002e-05 2023-09-23T18:46:50.065 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.065 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.065 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:46:50.066 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.229480+0000", 2023-09-23T18:46:50.066 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.2859999999999996e-06 2023-09-23T18:46:50.066 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.066 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.066 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.066 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:07.097964+0000", 2023-09-23T18:46:50.066 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.86848375200000005 2023-09-23T18:46:50.066 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.066 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.066 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:46:50.066 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:07.097981+0000", 2023-09-23T18:46:50.067 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.7731999999999999e-05 2023-09-23T18:46:50.067 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.067 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:46:50.067 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.067 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.067 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.067 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:82479 7.10 7.47a33b90 (undecoded) ondisk+write+known_if_redirected e198)", 2023-09-23T18:46:50.067 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:06.250896+0000", 2023-09-23T18:46:50.067 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 163.77440836599999, 2023-09-23T18:46:50.067 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.84712643700000001, 2023-09-23T18:46:50.067 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:46:50.067 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T18:46:50.067 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:46:50.067 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:46:50.068 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:46:50.068 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 82479 2023-09-23T18:46:50.068 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.068 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:46:50.068 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.068 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:46:50.068 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.250896+0000", 2023-09-23T18:46:50.068 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.068 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.068 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.068 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:46:50.068 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.250896+0000", 2023-09-23T18:46:50.068 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.069 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.069 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.069 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:46:50.069 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.250894+0000", 2023-09-23T18:46:50.069 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:46:50.069 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.069 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.069 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:46:50.069 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.250915+0000", 2023-09-23T18:46:50.069 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.0767999999999998e-05 2023-09-23T18:46:50.069 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.069 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.070 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:46:50.070 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.250928+0000", 2023-09-23T18:46:50.070 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3108999999999999e-05 2023-09-23T18:46:50.070 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.070 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.070 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:46:50.070 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.250936+0000", 2023-09-23T18:46:50.070 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.4999999999999999e-06 2023-09-23T18:46:50.070 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.070 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.070 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.070 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.251017+0000", 2023-09-23T18:46:50.071 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.0801000000000001e-05 2023-09-23T18:46:50.071 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.071 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.071 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:46:50.071 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.251026+0000", 2023-09-23T18:46:50.071 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.6179999999999995e-06 2023-09-23T18:46:50.071 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.071 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.071 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.071 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:07.098012+0000", 2023-09-23T18:46:50.071 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.84698633899999998 2023-09-23T18:46:50.071 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.071 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.072 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:46:50.072 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:07.098022+0000", 2023-09-23T18:46:50.072 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.8749999999999995e-06 2023-09-23T18:46:50.072 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.072 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:46:50.072 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.072 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.072 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.072 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:82492 7.10 7.d23063b0 (undecoded) ondisk+write+known_if_redirected e198)", 2023-09-23T18:46:50.072 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:06.262366+0000", 2023-09-23T18:46:50.072 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 163.76293838000001, 2023-09-23T18:46:50.072 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.83571488999999999, 2023-09-23T18:46:50.073 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:46:50.073 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T18:46:50.073 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:46:50.073 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:46:50.073 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:46:50.073 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 82492 2023-09-23T18:46:50.073 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.073 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:46:50.073 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.073 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:46:50.073 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.262366+0000", 2023-09-23T18:46:50.073 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.074 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.074 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.074 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:46:50.074 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.262366+0000", 2023-09-23T18:46:50.074 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.074 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.074 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.074 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:46:50.074 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.262364+0000", 2023-09-23T18:46:50.074 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T18:46:50.074 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.074 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.075 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:46:50.075 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.262393+0000", 2023-09-23T18:46:50.075 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.8679000000000001e-05 2023-09-23T18:46:50.075 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.075 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.075 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:46:50.075 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.262412+0000", 2023-09-23T18:46:50.075 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.8991000000000002e-05 2023-09-23T18:46:50.075 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.075 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.075 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:46:50.075 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.262421+0000", 2023-09-23T18:46:50.075 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.9560000000000003e-06 2023-09-23T18:46:50.075 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.076 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.076 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.076 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.262474+0000", 2023-09-23T18:46:50.076 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.3128999999999999e-05 2023-09-23T18:46:50.076 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.076 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.076 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:46:50.076 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.262482+0000", 2023-09-23T18:46:50.076 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.7350000000000002e-06 2023-09-23T18:46:50.076 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.076 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.076 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.076 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:07.098065+0000", 2023-09-23T18:46:50.077 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.83558305499999996 2023-09-23T18:46:50.077 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.077 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.077 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:46:50.077 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:07.098081+0000", 2023-09-23T18:46:50.077 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.5130000000000001e-05 2023-09-23T18:46:50.077 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.077 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:46:50.077 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.077 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.077 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.077 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:82517 7.10 7.5dbc7fd0 (undecoded) ondisk+write+known_if_redirected e198)", 2023-09-23T18:46:50.078 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:06.294888+0000", 2023-09-23T18:46:50.078 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 163.73041584399999, 2023-09-23T18:46:50.078 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.80324006699999995, 2023-09-23T18:46:50.078 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:46:50.078 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T18:46:50.078 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:46:50.078 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:46:50.078 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:46:50.078 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 82517 2023-09-23T18:46:50.078 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.078 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:46:50.078 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.079 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:46:50.079 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.294888+0000", 2023-09-23T18:46:50.079 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.079 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.079 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.079 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:46:50.079 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.294888+0000", 2023-09-23T18:46:50.079 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.079 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.079 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.079 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:46:50.079 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.294886+0000", 2023-09-23T18:46:50.079 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T18:46:50.080 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.080 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.080 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:46:50.080 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.294911+0000", 2023-09-23T18:46:50.080 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.4887000000000001e-05 2023-09-23T18:46:50.080 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.080 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.080 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:46:50.080 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.294927+0000", 2023-09-23T18:46:50.080 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.5982999999999999e-05 2023-09-23T18:46:50.080 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.080 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.080 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:46:50.081 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.294935+0000", 2023-09-23T18:46:50.081 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.9219999999999997e-06 2023-09-23T18:46:50.081 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.081 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.081 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.081 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.294991+0000", 2023-09-23T18:46:50.081 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.5445000000000002e-05 2023-09-23T18:46:50.081 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.081 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.081 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:46:50.081 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.295002+0000", 2023-09-23T18:46:50.081 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1107999999999999e-05 2023-09-23T18:46:50.081 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.082 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.082 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.082 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:07.098118+0000", 2023-09-23T18:46:50.082 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.80311650199999995 2023-09-23T18:46:50.082 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.082 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.082 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:46:50.082 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:07.098128+0000", 2023-09-23T18:46:50.082 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.004e-05 2023-09-23T18:46:50.082 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.082 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:46:50.082 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.082 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.083 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.083 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:82533 7.10 7.79c102f0 (undecoded) ondisk+write+known_if_redirected e198)", 2023-09-23T18:46:50.083 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:06.311222+0000", 2023-09-23T18:46:50.083 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 163.71408174300001, 2023-09-23T18:46:50.083 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.78696533899999999, 2023-09-23T18:46:50.083 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:46:50.083 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T18:46:50.083 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:46:50.083 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:46:50.083 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:46:50.083 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 82533 2023-09-23T18:46:50.083 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.083 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:46:50.084 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.084 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:46:50.084 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.311222+0000", 2023-09-23T18:46:50.084 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.084 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.084 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.084 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:46:50.084 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.311222+0000", 2023-09-23T18:46:50.084 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.084 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.084 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.085 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:46:50.085 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.311220+0000", 2023-09-23T18:46:50.085 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T18:46:50.085 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.085 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.085 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:46:50.085 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.311250+0000", 2023-09-23T18:46:50.085 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.9777999999999998e-05 2023-09-23T18:46:50.085 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.085 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.085 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:46:50.085 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.311268+0000", 2023-09-23T18:46:50.085 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.8340000000000001e-05 2023-09-23T18:46:50.085 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.086 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.086 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:46:50.086 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.311276+0000", 2023-09-23T18:46:50.086 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.7959999999999996e-06 2023-09-23T18:46:50.086 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.086 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.086 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.086 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.311332+0000", 2023-09-23T18:46:50.086 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.6110999999999999e-05 2023-09-23T18:46:50.086 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.086 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.086 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:46:50.087 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:06.311344+0000", 2023-09-23T18:46:50.087 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1234e-05 2023-09-23T18:46:50.087 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.087 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.087 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.087 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:07.098171+0000", 2023-09-23T18:46:50.087 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.78682757999999997 2023-09-23T18:46:50.087 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.087 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.087 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:46:50.087 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:07.098188+0000", 2023-09-23T18:46:50.088 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.6436e-05 2023-09-23T18:46:50.088 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.088 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:46:50.088 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.088 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.088 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.088 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:134419 7.10 7:0b481da4:::benchmark_data_smithi098_155798_object33474:head [delete] snapc 0=[] ondisk+write+known_if_redirected e207)", 2023-09-23T18:46:50.088 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:45:11.681542+0000", 2023-09-23T18:46:50.088 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 98.343761977, 2023-09-23T18:46:50.088 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.46428311999999999, 2023-09-23T18:46:50.088 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:46:50.088 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:46:50.089 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:46:50.089 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:46:50.089 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:46:50.089 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 134419 2023-09-23T18:46:50.089 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.089 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:46:50.089 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.089 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:46:50.089 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:11.681542+0000", 2023-09-23T18:46:50.089 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.089 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.089 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.090 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:46:50.090 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:11.681542+0000", 2023-09-23T18:46:50.090 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.090 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.090 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.090 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:46:50.090 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:11.681541+0000", 2023-09-23T18:46:50.090 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:46:50.090 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.090 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.090 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:46:50.091 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:11.681546+0000", 2023-09-23T18:46:50.091 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.7740000000000001e-06 2023-09-23T18:46:50.091 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.091 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.091 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:46:50.091 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:11.681557+0000", 2023-09-23T18:46:50.091 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0963e-05 2023-09-23T18:46:50.091 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.091 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.091 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:46:50.091 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:11.681565+0000", 2023-09-23T18:46:50.091 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.6869999999999994e-06 2023-09-23T18:46:50.091 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.091 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.092 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.092 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:11.681631+0000", 2023-09-23T18:46:50.092 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.6210999999999993e-05 2023-09-23T18:46:50.092 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.092 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.092 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:46:50.092 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:11.681643+0000", 2023-09-23T18:46:50.092 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1337e-05 2023-09-23T18:46:50.092 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.092 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.092 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.092 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:12.129714+0000", 2023-09-23T18:46:50.092 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.44807098699999998 2023-09-23T18:46:50.093 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.093 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.093 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:46:50.093 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:12.130270+0000", 2023-09-23T18:46:50.093 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000556333 2023-09-23T18:46:50.093 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.093 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.093 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 1,7", 2023-09-23T18:46:50.093 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:12.130622+0000", 2023-09-23T18:46:50.093 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00035163000000000002 2023-09-23T18:46:50.093 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.094 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.094 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:46:50.094 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:12.133526+0000", 2023-09-23T18:46:50.094 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0029043570000000002 2023-09-23T18:46:50.094 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.094 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.094 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:46:50.094 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:12.145574+0000", 2023-09-23T18:46:50.094 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.012047877 2023-09-23T18:46:50.094 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.094 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.094 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:46:50.095 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:12.145757+0000", 2023-09-23T18:46:50.095 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00018337400000000001 2023-09-23T18:46:50.095 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.095 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.095 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:46:50.095 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:12.145794+0000", 2023-09-23T18:46:50.095 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.7219999999999999e-05 2023-09-23T18:46:50.095 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.095 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.095 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:46:50.095 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:12.145825+0000", 2023-09-23T18:46:50.095 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.0837000000000003e-05 2023-09-23T18:46:50.096 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.096 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:46:50.096 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.096 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.096 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.096 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:134492 7.10 7:0ad8a45e:::benchmark_data_smithi098_155798_object33547:head [delete] snapc 0=[] ondisk+write+known_if_redirected e207)", 2023-09-23T18:46:50.096 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:45:11.891492+0000", 2023-09-23T18:46:50.096 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 98.133811735999998, 2023-09-23T18:46:50.096 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.26046911499999997, 2023-09-23T18:46:50.096 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:46:50.096 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:46:50.096 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:46:50.097 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:46:50.097 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:46:50.097 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 134492 2023-09-23T18:46:50.097 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.097 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:46:50.097 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.097 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:46:50.097 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:11.891492+0000", 2023-09-23T18:46:50.097 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.097 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.097 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.097 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:46:50.097 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:11.891492+0000", 2023-09-23T18:46:50.098 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.098 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.098 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.098 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:46:50.098 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:11.891491+0000", 2023-09-23T18:46:50.098 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:46:50.098 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.098 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.098 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:46:50.098 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:11.891497+0000", 2023-09-23T18:46:50.098 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.4799999999999998e-06 2023-09-23T18:46:50.098 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.098 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.098 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:46:50.099 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:11.891509+0000", 2023-09-23T18:46:50.099 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.2043e-05 2023-09-23T18:46:50.099 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.099 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.099 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:46:50.099 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:11.891516+0000", 2023-09-23T18:46:50.099 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.6579999999999996e-06 2023-09-23T18:46:50.099 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.099 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.099 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.099 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:11.891562+0000", 2023-09-23T18:46:50.099 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.5595e-05 2023-09-23T18:46:50.099 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.099 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.100 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:46:50.100 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:11.891570+0000", 2023-09-23T18:46:50.100 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.3850000000000002e-06 2023-09-23T18:46:50.100 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.100 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.100 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.100 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:12.133683+0000", 2023-09-23T18:46:50.100 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.24211344700000001 2023-09-23T18:46:50.100 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.100 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.100 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:46:50.100 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:12.133790+0000", 2023-09-23T18:46:50.100 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000106909 2023-09-23T18:46:50.101 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.101 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.101 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 1,7", 2023-09-23T18:46:50.101 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:12.133904+0000", 2023-09-23T18:46:50.101 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00011388 2023-09-23T18:46:50.101 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.101 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.101 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:46:50.101 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:12.145720+0000", 2023-09-23T18:46:50.101 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.011815665 2023-09-23T18:46:50.101 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.101 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.101 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:46:50.101 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:12.145883+0000", 2023-09-23T18:46:50.102 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000163516 2023-09-23T18:46:50.102 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.102 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.102 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:46:50.102 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:12.151879+0000", 2023-09-23T18:46:50.102 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0059951989999999997 2023-09-23T18:46:50.102 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.102 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.102 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:46:50.102 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:12.151925+0000", 2023-09-23T18:46:50.102 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.6477999999999998e-05 2023-09-23T18:46:50.102 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.103 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.103 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:46:50.103 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:12.151961+0000", 2023-09-23T18:46:50.103 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.6341999999999999e-05 2023-09-23T18:46:50.103 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.103 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:46:50.103 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.103 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.103 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.103 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:145725 7.e 7:7e879141:::benchmark_data_smithi098_155798_object44780:head [delete] snapc 0=[] ondisk+write+known_if_redirected e208)", 2023-09-23T18:46:50.103 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:45:26.714954+0000", 2023-09-23T18:46:50.103 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 83.310350170000007, 2023-09-23T18:46:50.104 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.25973700799999999, 2023-09-23T18:46:50.104 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:46:50.104 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:46:50.104 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:46:50.104 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:46:50.104 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:46:50.104 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 145725 2023-09-23T18:46:50.104 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.104 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:46:50.104 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.104 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:46:50.104 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:26.714954+0000", 2023-09-23T18:46:50.104 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.104 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.105 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.105 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:46:50.105 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:26.714954+0000", 2023-09-23T18:46:50.105 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.105 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.105 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.105 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:46:50.105 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:26.714952+0000", 2023-09-23T18:46:50.105 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:46:50.105 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.105 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.105 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:46:50.105 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:26.714959+0000", 2023-09-23T18:46:50.106 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.6610000000000003e-06 2023-09-23T18:46:50.106 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.106 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.106 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:46:50.106 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:26.714969+0000", 2023-09-23T18:46:50.106 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0193e-05 2023-09-23T18:46:50.106 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.106 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.106 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:46:50.106 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:26.714981+0000", 2023-09-23T18:46:50.106 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.203e-05 2023-09-23T18:46:50.106 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.107 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.107 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.107 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:26.732112+0000", 2023-09-23T18:46:50.107 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.017130486 2023-09-23T18:46:50.107 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.107 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.107 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:46:50.107 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:26.732243+0000", 2023-09-23T18:46:50.107 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00013109600000000001 2023-09-23T18:46:50.107 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.107 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.107 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,3", 2023-09-23T18:46:50.107 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:26.732367+0000", 2023-09-23T18:46:50.108 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00012430500000000001 2023-09-23T18:46:50.108 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.108 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.108 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:46:50.108 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:26.756704+0000", 2023-09-23T18:46:50.108 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.024336880000000002 2023-09-23T18:46:50.108 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.108 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.108 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:46:50.108 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:26.974523+0000", 2023-09-23T18:46:50.108 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.21781929799999999 2023-09-23T18:46:50.108 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.108 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.109 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:46:50.109 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:26.974578+0000", 2023-09-23T18:46:50.109 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.5050000000000003e-05 2023-09-23T18:46:50.109 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.109 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.109 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:46:50.109 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:26.974631+0000", 2023-09-23T18:46:50.109 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.2679999999999997e-05 2023-09-23T18:46:50.109 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.109 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.109 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:46:50.109 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:26.974691+0000", 2023-09-23T18:46:50.109 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.9920999999999998e-05 2023-09-23T18:46:50.110 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.110 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:46:50.110 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.110 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.110 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.110 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:145747 7.e 7:733e6d95:::benchmark_data_smithi098_155798_object44802:head [delete] snapc 0=[] ondisk+write+known_if_redirected e208)", 2023-09-23T18:46:50.110 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:45:26.748960+0000", 2023-09-23T18:46:50.110 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 83.276343725999993, 2023-09-23T18:46:50.110 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.252090337, 2023-09-23T18:46:50.110 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:46:50.110 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:46:50.110 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:46:50.110 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:46:50.111 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:46:50.111 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 145747 2023-09-23T18:46:50.111 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.111 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:46:50.111 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.111 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:46:50.111 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:26.748960+0000", 2023-09-23T18:46:50.111 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.111 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.111 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.111 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:46:50.111 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:26.748960+0000", 2023-09-23T18:46:50.111 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.112 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.112 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.112 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:46:50.112 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:26.748959+0000", 2023-09-23T18:46:50.112 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:46:50.112 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.112 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.112 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:46:50.112 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:26.748965+0000", 2023-09-23T18:46:50.112 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.6509999999999998e-06 2023-09-23T18:46:50.112 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.112 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.112 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:46:50.113 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:26.748976+0000", 2023-09-23T18:46:50.113 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1664e-05 2023-09-23T18:46:50.113 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.113 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.113 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:46:50.113 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:26.748984+0000", 2023-09-23T18:46:50.113 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.2139999999999999e-06 2023-09-23T18:46:50.113 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.113 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.113 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.113 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:26.975314+0000", 2023-09-23T18:46:50.113 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.22632998300000001 2023-09-23T18:46:50.113 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.114 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.114 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:46:50.114 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:26.975434+0000", 2023-09-23T18:46:50.114 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000120772 2023-09-23T18:46:50.114 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.114 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.114 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,3", 2023-09-23T18:46:50.114 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:26.975554+0000", 2023-09-23T18:46:50.114 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000119251 2023-09-23T18:46:50.114 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.114 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.114 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:46:50.114 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:26.984852+0000", 2023-09-23T18:46:50.115 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0092986219999999994 2023-09-23T18:46:50.115 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.115 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.115 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:46:50.115 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:26.986470+0000", 2023-09-23T18:46:50.115 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0016174430000000001 2023-09-23T18:46:50.115 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.115 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.115 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:46:50.115 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:27.000962+0000", 2023-09-23T18:46:50.115 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.014492710000000001 2023-09-23T18:46:50.115 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.116 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.116 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:46:50.116 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:27.001013+0000", 2023-09-23T18:46:50.116 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.1177000000000001e-05 2023-09-23T18:46:50.116 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.116 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.116 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:46:50.116 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:27.001051+0000", 2023-09-23T18:46:50.116 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.7169999999999998e-05 2023-09-23T18:46:50.116 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.116 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:46:50.116 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.116 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.116 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.117 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:161068 7.e 7:7ce8d06f:::benchmark_data_smithi098_155798_object60123:head [delete] snapc 0=[] ondisk+write+known_if_redirected e208)", 2023-09-23T18:46:50.117 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:45:47.758329+0000", 2023-09-23T18:46:50.117 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 62.266974914999999, 2023-09-23T18:46:50.117 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.30433315399999999, 2023-09-23T18:46:50.117 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:46:50.117 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:46:50.117 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:46:50.117 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:46:50.117 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:46:50.117 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 161068 2023-09-23T18:46:50.117 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.117 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:46:50.117 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.118 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:46:50.118 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.758329+0000", 2023-09-23T18:46:50.118 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.118 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.118 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.118 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:46:50.118 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.758329+0000", 2023-09-23T18:46:50.118 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.118 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.118 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.118 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:46:50.118 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.758327+0000", 2023-09-23T18:46:50.118 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T18:46:50.119 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.119 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.119 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:46:50.119 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.758337+0000", 2023-09-23T18:46:50.119 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0291e-05 2023-09-23T18:46:50.119 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.119 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.119 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:46:50.119 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.758353+0000", 2023-09-23T18:46:50.119 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.5252000000000001e-05 2023-09-23T18:46:50.119 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.119 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.119 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:46:50.120 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.758362+0000", 2023-09-23T18:46:50.120 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.6020000000000006e-06 2023-09-23T18:46:50.120 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.120 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.120 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.120 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.758407+0000", 2023-09-23T18:46:50.120 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.5117999999999997e-05 2023-09-23T18:46:50.120 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.120 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.120 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:46:50.120 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.758498+0000", 2023-09-23T18:46:50.120 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.1044000000000004e-05 2023-09-23T18:46:50.120 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.121 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.121 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,3", 2023-09-23T18:46:50.121 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.758570+0000", 2023-09-23T18:46:50.121 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.1234000000000003e-05 2023-09-23T18:46:50.121 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.121 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.121 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:46:50.121 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.768944+0000", 2023-09-23T18:46:50.121 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.010373881 2023-09-23T18:46:50.121 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.121 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.121 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:46:50.121 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.781547+0000", 2023-09-23T18:46:50.122 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.012603237 2023-09-23T18:46:50.122 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.122 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.122 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:46:50.122 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:48.062552+0000", 2023-09-23T18:46:50.122 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.28100564300000003 2023-09-23T18:46:50.122 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.122 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.122 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:46:50.122 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:48.062626+0000", 2023-09-23T18:46:50.122 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.3398000000000004e-05 2023-09-23T18:46:50.122 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.122 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.123 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:46:50.123 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:48.062662+0000", 2023-09-23T18:46:50.123 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.6504000000000002e-05 2023-09-23T18:46:50.123 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.123 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:46:50.123 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.123 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.123 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.123 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:161070 7.e 7:70f51b5e:::benchmark_data_smithi098_155798_object60125:head [delete] snapc 0=[] ondisk+write+known_if_redirected e208)", 2023-09-23T18:46:50.123 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:45:47.762465+0000", 2023-09-23T18:46:50.123 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 62.262839204999999, 2023-09-23T18:46:50.123 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.30032821599999998, 2023-09-23T18:46:50.124 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:46:50.124 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:46:50.124 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:46:50.124 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:46:50.124 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:46:50.124 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 161070 2023-09-23T18:46:50.124 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.124 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:46:50.124 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.124 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:46:50.124 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.762465+0000", 2023-09-23T18:46:50.124 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.124 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.124 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.125 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:46:50.125 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.762465+0000", 2023-09-23T18:46:50.125 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.125 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.125 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.125 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:46:50.125 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.762463+0000", 2023-09-23T18:46:50.125 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:46:50.125 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.125 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.125 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:46:50.125 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.762469+0000", 2023-09-23T18:46:50.126 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.9769999999999997e-06 2023-09-23T18:46:50.126 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.126 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.126 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:46:50.126 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.762479+0000", 2023-09-23T18:46:50.126 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0305e-05 2023-09-23T18:46:50.126 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.126 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.126 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:46:50.126 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.762486+0000", 2023-09-23T18:46:50.126 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.9840000000000004e-06 2023-09-23T18:46:50.126 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.126 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.127 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.127 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.763793+0000", 2023-09-23T18:46:50.127 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.001306314 2023-09-23T18:46:50.127 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.127 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.127 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:46:50.127 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.763908+0000", 2023-09-23T18:46:50.127 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00011527 2023-09-23T18:46:50.127 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.127 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.127 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,3", 2023-09-23T18:46:50.127 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.764030+0000", 2023-09-23T18:46:50.127 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00012218600000000001 2023-09-23T18:46:50.127 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.128 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.128 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:46:50.128 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.769043+0000", 2023-09-23T18:46:50.128 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0050127319999999998 2023-09-23T18:46:50.128 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.128 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.128 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:46:50.128 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:48.060209+0000", 2023-09-23T18:46:50.128 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.29116564299999997 2023-09-23T18:46:50.128 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.128 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.128 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:46:50.128 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:48.062695+0000", 2023-09-23T18:46:50.129 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0024863509999999999 2023-09-23T18:46:50.129 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.129 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.129 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:46:50.129 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:48.062761+0000", 2023-09-23T18:46:50.129 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.6173999999999994e-05 2023-09-23T18:46:50.129 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.129 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.129 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:46:50.129 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:48.062793+0000", 2023-09-23T18:46:50.129 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.1921000000000002e-05 2023-09-23T18:46:50.129 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.129 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:46:50.130 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.130 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.130 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.130 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:161080 7.e 7:70d45034:::benchmark_data_smithi098_155798_object60135:head [delete] snapc 0=[] ondisk+write+known_if_redirected e208)", 2023-09-23T18:46:50.130 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:45:47.770480+0000", 2023-09-23T18:46:50.130 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 62.254824106000001, 2023-09-23T18:46:50.130 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.29244095399999998, 2023-09-23T18:46:50.130 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:46:50.130 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:46:50.130 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:46:50.130 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:46:50.130 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:46:50.131 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 161080 2023-09-23T18:46:50.131 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.131 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:46:50.131 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.131 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:46:50.131 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.770480+0000", 2023-09-23T18:46:50.131 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.131 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.131 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.131 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:46:50.131 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.770480+0000", 2023-09-23T18:46:50.131 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.131 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.131 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.132 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:46:50.132 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.770478+0000", 2023-09-23T18:46:50.132 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T18:46:50.132 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.132 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.132 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:46:50.132 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.770487+0000", 2023-09-23T18:46:50.132 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.7050000000000005e-06 2023-09-23T18:46:50.132 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.132 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.132 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:46:50.132 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.770500+0000", 2023-09-23T18:46:50.133 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.2703e-05 2023-09-23T18:46:50.133 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.133 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.133 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:46:50.133 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.770508+0000", 2023-09-23T18:46:50.133 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.7369999999999995e-06 2023-09-23T18:46:50.133 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.133 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.133 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.133 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.770555+0000", 2023-09-23T18:46:50.133 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.6239000000000002e-05 2023-09-23T18:46:50.133 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.133 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.134 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:46:50.134 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.770670+0000", 2023-09-23T18:46:50.134 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00011546 2023-09-23T18:46:50.134 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.134 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.134 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,3", 2023-09-23T18:46:50.134 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.770778+0000", 2023-09-23T18:46:50.134 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000107452 2023-09-23T18:46:50.134 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.134 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.134 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:46:50.134 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.778159+0000", 2023-09-23T18:46:50.135 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0073817739999999998 2023-09-23T18:46:50.135 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.135 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.135 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:46:50.135 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:48.062522+0000", 2023-09-23T18:46:50.135 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.28436292800000001 2023-09-23T18:46:50.135 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.135 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.135 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:46:50.135 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:48.062825+0000", 2023-09-23T18:46:50.135 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00030286799999999997 2023-09-23T18:46:50.136 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.136 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.136 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:46:50.136 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:48.062889+0000", 2023-09-23T18:46:50.136 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.4079999999999996e-05 2023-09-23T18:46:50.136 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.136 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.136 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:46:50.136 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:48.062921+0000", 2023-09-23T18:46:50.136 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.1708999999999997e-05 2023-09-23T18:46:50.136 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.136 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:46:50.136 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.137 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.137 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.137 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:161091 7.10 7:0fbc8b25:::benchmark_data_smithi098_155798_object60146:head [delete] snapc 0=[] ondisk+write+known_if_redirected e208)", 2023-09-23T18:46:50.137 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:45:47.790877+0000", 2023-09-23T18:46:50.137 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 62.234427285000002, 2023-09-23T18:46:50.137 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.25823926400000002, 2023-09-23T18:46:50.137 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:46:50.137 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:46:50.137 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:46:50.137 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:46:50.137 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:46:50.137 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 161091 2023-09-23T18:46:50.138 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.138 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:46:50.138 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.138 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:46:50.138 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.790877+0000", 2023-09-23T18:46:50.138 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.138 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.138 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.138 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:46:50.138 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.790877+0000", 2023-09-23T18:46:50.138 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.138 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.138 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.139 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:46:50.139 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.790875+0000", 2023-09-23T18:46:50.139 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:46:50.139 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.139 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.139 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:46:50.139 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.790882+0000", 2023-09-23T18:46:50.139 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.9489999999999999e-06 2023-09-23T18:46:50.139 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.139 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.139 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:46:50.139 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.790895+0000", 2023-09-23T18:46:50.139 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.2673e-05 2023-09-23T18:46:50.140 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.140 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.140 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:46:50.140 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.790903+0000", 2023-09-23T18:46:50.140 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.1359999999999997e-06 2023-09-23T18:46:50.140 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.140 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.140 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.140 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.790931+0000", 2023-09-23T18:46:50.140 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.7671999999999999e-05 2023-09-23T18:46:50.140 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.140 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.140 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:46:50.141 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.791013+0000", 2023-09-23T18:46:50.141 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.2384000000000003e-05 2023-09-23T18:46:50.141 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.141 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.141 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 1,7", 2023-09-23T18:46:50.141 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.791083+0000", 2023-09-23T18:46:50.141 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.9641999999999995e-05 2023-09-23T18:46:50.141 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.141 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.141 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:46:50.141 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.793014+0000", 2023-09-23T18:46:50.141 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0019313469999999999 2023-09-23T18:46:50.141 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.142 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.142 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:46:50.142 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.794213+0000", 2023-09-23T18:46:50.142 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.001198813 2023-09-23T18:46:50.142 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.142 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.142 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:46:50.142 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:48.049025+0000", 2023-09-23T18:46:50.142 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.25481231100000001 2023-09-23T18:46:50.142 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.142 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.142 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:46:50.142 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:48.049075+0000", 2023-09-23T18:46:50.143 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.9804e-05 2023-09-23T18:46:50.143 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.143 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.143 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:46:50.143 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:48.049116+0000", 2023-09-23T18:46:50.143 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.0899999999999998e-05 2023-09-23T18:46:50.143 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.143 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:46:50.143 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.143 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.143 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.143 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:194393 7.e 7:755f5512:::benchmark_data_smithi098_155798_object93448:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T18:46:50.143 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:46:36.026867+0000", 2023-09-23T18:46:50.144 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 13.998437291, 2023-09-23T18:46:50.144 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.26638529500000002, 2023-09-23T18:46:50.144 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:46:50.144 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:46:50.144 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:46:50.144 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:46:50.144 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:46:50.144 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 194393 2023-09-23T18:46:50.144 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.144 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:46:50.144 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.144 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:46:50.144 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:46:36.026867+0000", 2023-09-23T18:46:50.145 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.145 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.145 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.145 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:46:50.145 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:46:36.026867+0000", 2023-09-23T18:46:50.145 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:46:50.145 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.145 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.145 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:46:50.145 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:46:36.026866+0000", 2023-09-23T18:46:50.145 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:46:50.145 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.145 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.146 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:46:50.146 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:46:36.026872+0000", 2023-09-23T18:46:50.146 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.596e-06 2023-09-23T18:46:50.146 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.146 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.146 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:46:50.146 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:46:36.026884+0000", 2023-09-23T18:46:50.146 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1561e-05 2023-09-23T18:46:50.146 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.146 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.146 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:46:50.146 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:46:36.026891+0000", 2023-09-23T18:46:50.146 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.4970000000000004e-06 2023-09-23T18:46:50.147 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.147 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.147 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:46:50.147 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:46:36.037752+0000", 2023-09-23T18:46:50.147 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.010861107 2023-09-23T18:46:50.147 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.147 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.147 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:46:50.147 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:46:36.037838+0000", 2023-09-23T18:46:50.147 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.5729000000000005e-05 2023-09-23T18:46:50.147 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.147 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.148 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,3", 2023-09-23T18:46:50.148 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:46:36.037914+0000", 2023-09-23T18:46:50.148 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.5610000000000003e-05 2023-09-23T18:46:50.148 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.148 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.148 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:46:50.148 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:46:36.042629+0000", 2023-09-23T18:46:50.148 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0047148370000000004 2023-09-23T18:46:50.148 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.148 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.148 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:46:50.148 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:46:36.056804+0000", 2023-09-23T18:46:50.148 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.014175238 2023-09-23T18:46:50.148 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.149 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.149 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:46:50.149 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:46:36.292932+0000", 2023-09-23T18:46:50.149 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.23612791499999999 2023-09-23T18:46:50.149 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.149 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.149 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:46:50.149 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:46:36.293206+0000", 2023-09-23T18:46:50.149 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00027399100000000001 2023-09-23T18:46:50.149 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:46:50.149 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:46:50.149 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:46:50.149 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:46:36.293252+0000", 2023-09-23T18:46:50.150 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.634e-05 2023-09-23T18:46:50.150 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.150 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:46:50.150 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.150 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:46:50.150 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:46:50.150 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:46:50.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:50.152 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:50.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:50.253 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:50.276 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:50.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:50.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:50.330 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-09-23T18:46:50.380 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:50.393 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:50 smithi173 bash[59422]: audit 2023-09-23T18:46:49.809060+0000 mon.a (mon.0) 1138 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:46:50.393 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:50 smithi173 bash[59422]: cluster 2023-09-23T18:46:50.086531+0000 mon.a (mon.0) 1139 : cluster 3 Health check update: Degraded data redundancy: 2373/4260 objects degraded (55.704%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:46:50.411 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:50.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:50.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:50.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:50 smithi098 bash[76817]: audit 2023-09-23T18:46:49.809060+0000 mon.a (mon.0) 1138 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:46:50.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:50 smithi098 bash[76817]: cluster 2023-09-23T18:46:50.086531+0000 mon.a (mon.0) 1139 : cluster 3 Health check update: Degraded data redundancy: 2373/4260 objects degraded (55.704%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:46:50.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:50 smithi098 bash[79516]: audit 2023-09-23T18:46:49.809060+0000 mon.a (mon.0) 1138 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:46:50.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:50 smithi098 bash[79516]: cluster 2023-09-23T18:46:50.086531+0000 mon.a (mon.0) 1139 : cluster 3 Health check update: Degraded data redundancy: 2373/4260 objects degraded (55.704%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:46:50.515 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:50.544 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:50.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:50.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:50.647 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:50.677 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:50.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:50.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:50.781 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:50.809 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:50.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:50.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:50.913 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:50.939 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:50.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:50.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:51.042 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:51.073 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:51.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:51.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:51.177 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:51.206 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:51.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:51.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:51.310 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:51.323 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:51 smithi173 bash[59422]: cluster 2023-09-23T18:46:50.044918+0000 mgr.y (mgr.26054) 860 : cluster 0 pgmap v886: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 30 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 673 op/s; 2102/1296 objects degraded (162.191%); 2102/1296 objects misplaced (162.191%); 0 B/s, 42 objects/s recovering 2023-09-23T18:46:51.323 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:51 smithi173 bash[59422]: audit 2023-09-23T18:46:50.232507+0000 mon.a (mon.0) 1140 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:46:51.323 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:51 smithi173 bash[59422]: audit 2023-09-23T18:46:50.239192+0000 mon.a (mon.0) 1141 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:46:51.323 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:51 smithi173 bash[59422]: audit 2023-09-23T18:46:50.326791+0000 mon.a (mon.0) 1142 : audit 1 from='client.? 172.21.15.98:0/1664388161' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "41"}]: dispatch 2023-09-23T18:46:51.340 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:51.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:51.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:51.444 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:51.469 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:51 smithi098 bash[79516]: cluster 2023-09-23T18:46:50.044918+0000 mgr.y (mgr.26054) 860 : cluster 0 pgmap v886: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 30 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 673 op/s; 2102/1296 objects degraded (162.191%); 2102/1296 objects misplaced (162.191%); 0 B/s, 42 objects/s recovering 2023-09-23T18:46:51.469 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:51 smithi098 bash[79516]: audit 2023-09-23T18:46:50.232507+0000 mon.a (mon.0) 1140 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:46:51.469 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:51 smithi098 bash[79516]: audit 2023-09-23T18:46:50.239192+0000 mon.a (mon.0) 1141 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:46:51.469 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:51 smithi098 bash[79516]: audit 2023-09-23T18:46:50.326791+0000 mon.a (mon.0) 1142 : audit 1 from='client.? 172.21.15.98:0/1664388161' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "41"}]: dispatch 2023-09-23T18:46:51.469 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:51 smithi098 bash[76817]: cluster 2023-09-23T18:46:50.044918+0000 mgr.y (mgr.26054) 860 : cluster 0 pgmap v886: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 30 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 673 op/s; 2102/1296 objects degraded (162.191%); 2102/1296 objects misplaced (162.191%); 0 B/s, 42 objects/s recovering 2023-09-23T18:46:51.470 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:51 smithi098 bash[76817]: audit 2023-09-23T18:46:50.232507+0000 mon.a (mon.0) 1140 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:46:51.470 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:51 smithi098 bash[76817]: audit 2023-09-23T18:46:50.239192+0000 mon.a (mon.0) 1141 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:46:51.470 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:51 smithi098 bash[76817]: audit 2023-09-23T18:46:50.326791+0000 mon.a (mon.0) 1142 : audit 1 from='client.? 172.21.15.98:0/1664388161' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "41"}]: dispatch 2023-09-23T18:46:51.470 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:51.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:51.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:51.574 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:51.603 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:51.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:51.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:51.707 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:51.731 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:51.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:51.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:51.834 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:51.860 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:51.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:51.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:51.963 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:51.993 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:51.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:51.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:52.097 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:52.128 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:52.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:52.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:52.231 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:52.252 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:52.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:52.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:52.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:52 smithi173 bash[59422]: cluster 2023-09-23T18:46:52.045953+0000 mgr.y (mgr.26054) 861 : cluster 0 pgmap v887: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 21 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 538 op/s; 2102/900 objects degraded (233.556%); 2102/900 objects misplaced (233.556%); 0 B/s, 33 objects/s recovering 2023-09-23T18:46:52.356 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:52.385 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:52.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:52.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:52.464 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:52 smithi098 bash[79516]: cluster 2023-09-23T18:46:52.045953+0000 mgr.y (mgr.26054) 861 : cluster 0 pgmap v887: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 21 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 538 op/s; 2102/900 objects degraded (233.556%); 2102/900 objects misplaced (233.556%); 0 B/s, 33 objects/s recovering 2023-09-23T18:46:52.464 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:52 smithi098 bash[76817]: cluster 2023-09-23T18:46:52.045953+0000 mgr.y (mgr.26054) 861 : cluster 0 pgmap v887: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 21 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 538 op/s; 2102/900 objects degraded (233.556%); 2102/900 objects misplaced (233.556%); 0 B/s, 33 objects/s recovering 2023-09-23T18:46:52.489 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:52.518 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:52.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:52.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:52.621 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:52.652 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:52.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:52.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:52.756 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:52.787 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:52.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:52.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:52.890 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:52.912 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:52.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:52.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:53.015 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:53.044 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:53.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:53.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:53.147 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:53.176 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:53.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:53.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:53.280 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:53.308 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:53.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:53.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:53.412 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:53.445 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:53.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:53.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:53.548 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:53.577 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:53.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:53.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:53.681 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:53.708 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:53.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:53.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:53.812 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:53.835 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:53.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:53.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:53.938 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:53.966 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:53.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:53.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:54.070 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:54.093 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:54.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:54.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:54.197 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:54.225 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:54.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:54.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:54.329 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:54.353 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:54.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:54.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:54.456 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:54.480 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:54.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:54.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:54.583 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:54.606 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:54.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:54.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:54.710 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:54.729 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:54.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:54.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:54.832 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:54.860 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:54.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:54.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:54.963 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:54.988 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:54.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:54.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:55.092 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:55.116 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:55.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:55.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:55.219 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:55.231 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:55 smithi173 bash[59422]: cluster 2023-09-23T18:46:54.046570+0000 mgr.y (mgr.26054) 862 : cluster 0 pgmap v888: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 21 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 398 op/s; 1499/900 objects degraded (166.556%); 1499/900 objects misplaced (166.556%); 0 B/s, 83 objects/s recovering 2023-09-23T18:46:55.232 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:55 smithi173 bash[59422]: audit 2023-09-23T18:46:54.613532+0000 mon.a (mon.0) 1143 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:46:55.232 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:55 smithi173 bash[59422]: cluster 2023-09-23T18:46:55.087552+0000 mon.a (mon.0) 1144 : cluster 3 Health check update: Degraded data redundancy: 1499/900 objects degraded (166.556%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:46:55.247 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:55.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:55.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:55.351 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:55.361 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:55 smithi098 bash[79516]: cluster 2023-09-23T18:46:54.046570+0000 mgr.y (mgr.26054) 862 : cluster 0 pgmap v888: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 21 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 398 op/s; 1499/900 objects degraded (166.556%); 1499/900 objects misplaced (166.556%); 0 B/s, 83 objects/s recovering 2023-09-23T18:46:55.361 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:55 smithi098 bash[79516]: audit 2023-09-23T18:46:54.613532+0000 mon.a (mon.0) 1143 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:46:55.362 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:55 smithi098 bash[79516]: cluster 2023-09-23T18:46:55.087552+0000 mon.a (mon.0) 1144 : cluster 3 Health check update: Degraded data redundancy: 1499/900 objects degraded (166.556%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:46:55.362 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:55 smithi098 bash[76817]: cluster 2023-09-23T18:46:54.046570+0000 mgr.y (mgr.26054) 862 : cluster 0 pgmap v888: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 21 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 398 op/s; 1499/900 objects degraded (166.556%); 1499/900 objects misplaced (166.556%); 0 B/s, 83 objects/s recovering 2023-09-23T18:46:55.362 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:55 smithi098 bash[76817]: audit 2023-09-23T18:46:54.613532+0000 mon.a (mon.0) 1143 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:46:55.362 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:55 smithi098 bash[76817]: cluster 2023-09-23T18:46:55.087552+0000 mon.a (mon.0) 1144 : cluster 3 Health check update: Degraded data redundancy: 1499/900 objects degraded (166.556%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:46:55.375 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:55.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:55.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:55.412 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:46:55.435 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:46:55.458 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:46:55.479 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:55.480 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:46:55.500 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:55.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:55.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:55.504 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:46:55.530 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:46:55.557 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:46:55.585 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:46:55.587 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:46:55.587 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:46:55.587 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:46:55.587 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:46:55.587 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:46:55.587 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:46:55.587 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:46:55.587 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:46:55.603 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:55.626 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:55.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:55.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:55.730 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:55.757 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:55.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:55.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:55.860 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:55.912 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:55.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:55.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:56.017 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:56.048 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:56.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:56.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:56.151 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:56.161 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:56 smithi173 bash[59422]: audit 2023-09-23T18:46:55.379366+0000 mon.a (mon.0) 1145 : audit 0 from='client.? 172.21.15.98:0/1077066702' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:46:56.162 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:56 smithi173 bash[59422]: audit 2023-09-23T18:46:56.048559+0000 mon.a (mon.0) 1146 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:46:56.162 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:56 smithi173 bash[59422]: audit 2023-09-23T18:46:56.054817+0000 mon.a (mon.0) 1147 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:46:56.178 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:56.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:56.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:56.282 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:56.312 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:56.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:56.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:56.415 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:56.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:56 smithi098 bash[79516]: audit 2023-09-23T18:46:55.379366+0000 mon.a (mon.0) 1145 : audit 0 from='client.? 172.21.15.98:0/1077066702' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:46:56.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:56 smithi098 bash[79516]: audit 2023-09-23T18:46:56.048559+0000 mon.a (mon.0) 1146 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:46:56.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:56 smithi098 bash[79516]: audit 2023-09-23T18:46:56.054817+0000 mon.a (mon.0) 1147 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:46:56.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:56 smithi098 bash[76817]: audit 2023-09-23T18:46:55.379366+0000 mon.a (mon.0) 1145 : audit 0 from='client.? 172.21.15.98:0/1077066702' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:46:56.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:56 smithi098 bash[76817]: audit 2023-09-23T18:46:56.048559+0000 mon.a (mon.0) 1146 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:46:56.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:56 smithi098 bash[76817]: audit 2023-09-23T18:46:56.054817+0000 mon.a (mon.0) 1147 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:46:56.442 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:46:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:46:56] "GET /metrics HTTP/1.1" 200 36587 "" "Prometheus/2.33.4" 2023-09-23T18:46:56.447 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:56.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:56.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:56.550 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:56.578 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:56.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:56.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:56.681 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:56.713 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:46:56.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:56.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:46:56.817 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:56.847 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:56.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:56.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:56.951 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:56.980 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:56.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:56.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:57.083 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:57.105 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:57.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:57.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:57.212 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:57.234 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:57.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:57.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:57.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:57 smithi173 bash[59422]: cluster 2023-09-23T18:46:56.047169+0000 mgr.y (mgr.26054) 863 : cluster 0 pgmap v889: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 21 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 285 op/s; 1499/900 objects degraded (166.556%); 1499/900 objects misplaced (166.556%); 0 B/s, 72 objects/s recovering 2023-09-23T18:46:57.337 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:57.345 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:57 smithi098 bash[79516]: cluster 2023-09-23T18:46:56.047169+0000 mgr.y (mgr.26054) 863 : cluster 0 pgmap v889: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 21 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 285 op/s; 1499/900 objects degraded (166.556%); 1499/900 objects misplaced (166.556%); 0 B/s, 72 objects/s recovering 2023-09-23T18:46:57.345 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:57 smithi098 bash[76817]: cluster 2023-09-23T18:46:56.047169+0000 mgr.y (mgr.26054) 863 : cluster 0 pgmap v889: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 21 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 285 op/s; 1499/900 objects degraded (166.556%); 1499/900 objects misplaced (166.556%); 0 B/s, 72 objects/s recovering 2023-09-23T18:46:57.362 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:57.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:57.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:57.465 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:57.494 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:57.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:57.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:57.597 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:57.628 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:57.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:57.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:57.705 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T18:46:57.733 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:57.779 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:57.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:57.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:57.882 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:57.903 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:57.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:57.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:58.007 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:58.034 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:58.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:58.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:58.138 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:58.166 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:58.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:58.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:58.270 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:58.297 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:58.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:58.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:58.400 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:58.429 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:46:58.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:58.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:46:58.532 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:58.563 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:58.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:58.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:58.667 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:58.690 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:58.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:58.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:58.793 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:58.823 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:58.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:58.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:58.926 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:58.954 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:46:58.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:58.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:46:59.057 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:59.081 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:46:59.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:59.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:46:59.185 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:59.213 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:59.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:59.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:59.317 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:59.346 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:46:59.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:59.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:46:59.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:46:59 smithi098 bash[76817]: cluster 2023-09-23T18:46:58.048033+0000 mgr.y (mgr.26054) 864 : cluster 0 pgmap v890: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 21 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 162 op/s; 1499/900 objects degraded (166.556%); 1499/900 objects misplaced (166.556%); 0 B/s, 72 objects/s recovering 2023-09-23T18:46:59.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:46:59 smithi098 bash[79516]: cluster 2023-09-23T18:46:58.048033+0000 mgr.y (mgr.26054) 864 : cluster 0 pgmap v890: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 21 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 162 op/s; 1499/900 objects degraded (166.556%); 1499/900 objects misplaced (166.556%); 0 B/s, 72 objects/s recovering 2023-09-23T18:46:59.450 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:59.462 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:46:59 smithi173 bash[59422]: cluster 2023-09-23T18:46:58.048033+0000 mgr.y (mgr.26054) 864 : cluster 0 pgmap v890: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 21 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 162 op/s; 1499/900 objects degraded (166.556%); 1499/900 objects misplaced (166.556%); 0 B/s, 72 objects/s recovering 2023-09-23T18:46:59.476 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:59.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:59.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:59.580 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:59.605 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:46:59.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:59.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:46:59.618 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-09-23T18:46:59.618 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-09-23T18:46:59.618 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-09-23T18:46:59.708 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:59.752 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:46:59.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:59.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:46:59.855 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:59.888 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:46:59.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:46:59.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:46:59.992 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:00.019 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:00.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:00.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:00.126 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:00.155 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:00.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:00.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:00.258 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:00.285 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:00.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:00.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:00.388 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:00.420 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:00.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:00.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:00.523 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:00.554 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:00.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:00.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:00.657 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:00.686 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:00.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:00.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:00.789 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:00.818 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:00.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:00.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:00.921 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:00.954 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:00.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:00.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:01.058 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:01.089 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:01.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:01.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:01.127 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is unset 2023-09-23T18:47:01.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:01 smithi098 bash[79516]: cluster 2023-09-23T18:47:00.048810+0000 mgr.y (mgr.26054) 865 : cluster 0 pgmap v891: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 21 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 108 op/s; 755/900 objects degraded (83.889%); 755/900 objects misplaced (83.889%); 0 B/s, 134 objects/s recovering 2023-09-23T18:47:01.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:01 smithi098 bash[79516]: cluster 2023-09-23T18:47:00.108307+0000 mon.a (mon.0) 1148 : cluster 3 Health check update: Degraded data redundancy: 755/900 objects degraded (83.889%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:47:01.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:01 smithi098 bash[79516]: audit 2023-09-23T18:47:00.307436+0000 mon.a (mon.0) 1149 : audit 1 from='client.? 172.21.15.98:0/783595081' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:47:01.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:01 smithi098 bash[76817]: cluster 2023-09-23T18:47:00.048810+0000 mgr.y (mgr.26054) 865 : cluster 0 pgmap v891: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 21 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 108 op/s; 755/900 objects degraded (83.889%); 755/900 objects misplaced (83.889%); 0 B/s, 134 objects/s recovering 2023-09-23T18:47:01.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:01 smithi098 bash[76817]: cluster 2023-09-23T18:47:00.108307+0000 mon.a (mon.0) 1148 : cluster 3 Health check update: Degraded data redundancy: 755/900 objects degraded (83.889%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:47:01.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:01 smithi098 bash[76817]: audit 2023-09-23T18:47:00.307436+0000 mon.a (mon.0) 1149 : audit 1 from='client.? 172.21.15.98:0/783595081' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:47:01.192 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:01.218 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:01.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:01.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:01.322 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:01.334 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:01 smithi173 bash[59422]: cluster 2023-09-23T18:47:00.048810+0000 mgr.y (mgr.26054) 865 : cluster 0 pgmap v891: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 21 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 108 op/s; 755/900 objects degraded (83.889%); 755/900 objects misplaced (83.889%); 0 B/s, 134 objects/s recovering 2023-09-23T18:47:01.334 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:01 smithi173 bash[59422]: cluster 2023-09-23T18:47:00.108307+0000 mon.a (mon.0) 1148 : cluster 3 Health check update: Degraded data redundancy: 755/900 objects degraded (83.889%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:47:01.334 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:01 smithi173 bash[59422]: audit 2023-09-23T18:47:00.307436+0000 mon.a (mon.0) 1149 : audit 1 from='client.? 172.21.15.98:0/783595081' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:47:01.352 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:01.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:01.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:01.455 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:01.488 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:01.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:01.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:01.591 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:01.622 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:01.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:01.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:01.637 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:47:01.726 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:01.757 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:01.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:01.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:01.860 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:01.884 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:01.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:01.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:01.988 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:02.019 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:02.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:02.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:02.123 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:02.131 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:02 smithi098 bash[76817]: cluster 2023-09-23T18:47:01.111197+0000 mon.a (mon.0) 1150 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:47:02.131 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:02 smithi098 bash[76817]: cluster 2023-09-23T18:47:01.117632+0000 osd.5 (osd.5) 107 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T18:47:02.131 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:02 smithi098 bash[76817]: audit 2023-09-23T18:47:01.121443+0000 mon.a (mon.0) 1151 : audit 1 from='client.? 172.21.15.98:0/783595081' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:47:02.131 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:02 smithi098 bash[76817]: cluster 2023-09-23T18:47:01.121645+0000 mon.a (mon.0) 1152 : cluster 0 osdmap e213: 8 total, 8 up, 7 in 2023-09-23T18:47:02.131 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:02 smithi098 bash[76817]: cluster 2023-09-23T18:47:01.122842+0000 osd.6 (osd.6) 161 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T18:47:02.131 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:02 smithi098 bash[76817]: cluster 2023-09-23T18:47:01.305342+0000 osd.7 (osd.7) 130 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T18:47:02.131 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:02 smithi098 bash[76817]: cluster 2023-09-23T18:47:01.307850+0000 osd.7 (osd.7) 131 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T18:47:02.132 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:02 smithi098 bash[79516]: cluster 2023-09-23T18:47:01.111197+0000 mon.a (mon.0) 1150 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:47:02.132 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:02 smithi098 bash[79516]: cluster 2023-09-23T18:47:01.117632+0000 osd.5 (osd.5) 107 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T18:47:02.132 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:02 smithi098 bash[79516]: audit 2023-09-23T18:47:01.121443+0000 mon.a (mon.0) 1151 : audit 1 from='client.? 172.21.15.98:0/783595081' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:47:02.132 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:02 smithi098 bash[79516]: cluster 2023-09-23T18:47:01.121645+0000 mon.a (mon.0) 1152 : cluster 0 osdmap e213: 8 total, 8 up, 7 in 2023-09-23T18:47:02.132 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:02 smithi098 bash[79516]: cluster 2023-09-23T18:47:01.122842+0000 osd.6 (osd.6) 161 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T18:47:02.132 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:02 smithi098 bash[79516]: cluster 2023-09-23T18:47:01.305342+0000 osd.7 (osd.7) 130 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T18:47:02.132 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:02 smithi098 bash[79516]: cluster 2023-09-23T18:47:01.307850+0000 osd.7 (osd.7) 131 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T18:47:02.148 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:02.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:02.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:02.251 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:02.264 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:02 smithi173 bash[59422]: cluster 2023-09-23T18:47:01.111197+0000 mon.a (mon.0) 1150 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:47:02.264 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:02 smithi173 bash[59422]: cluster 2023-09-23T18:47:01.117632+0000 osd.5 (osd.5) 107 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T18:47:02.264 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:02 smithi173 bash[59422]: audit 2023-09-23T18:47:01.121443+0000 mon.a (mon.0) 1151 : audit 1 from='client.? 172.21.15.98:0/783595081' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:47:02.264 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:02 smithi173 bash[59422]: cluster 2023-09-23T18:47:01.121645+0000 mon.a (mon.0) 1152 : cluster 0 osdmap e213: 8 total, 8 up, 7 in 2023-09-23T18:47:02.264 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:02 smithi173 bash[59422]: cluster 2023-09-23T18:47:01.122842+0000 osd.6 (osd.6) 161 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T18:47:02.264 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:02 smithi173 bash[59422]: cluster 2023-09-23T18:47:01.305342+0000 osd.7 (osd.7) 130 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T18:47:02.264 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:02 smithi173 bash[59422]: cluster 2023-09-23T18:47:01.307850+0000 osd.7 (osd.7) 131 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T18:47:02.282 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:02.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:02.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:02.385 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:02.410 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:02.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:02.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:02.513 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:02.547 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:02.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:02.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:02.651 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:02.684 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:02.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:02.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:02.788 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:02.822 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:02.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:02.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:02.927 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:02.958 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:02.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:02.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:03.061 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:03.094 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:03.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:03.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:03.199 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:03.228 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:03.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:03.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:03.332 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:03.476 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:03 smithi098 bash[79516]: cluster 2023-09-23T18:47:01.705867+0000 osd.4 (osd.4) 197 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T18:47:03.476 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:03 smithi098 bash[79516]: cluster 2023-09-23T18:47:01.707385+0000 osd.4 (osd.4) 198 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T18:47:03.476 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:03 smithi098 bash[79516]: cluster 2023-09-23T18:47:02.049365+0000 mgr.y (mgr.26054) 866 : cluster 0 pgmap v893: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 21 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 755/900 objects degraded (83.889%); 755/900 objects misplaced (83.889%); 0 B/s, 134 objects/s recovering 2023-09-23T18:47:03.476 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:03 smithi098 bash[79516]: cluster 2023-09-23T18:47:02.129875+0000 osd.7 (osd.7) 132 : cluster 3 failed to encode map e214 with expected crc 2023-09-23T18:47:03.476 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:03 smithi098 bash[79516]: cluster 2023-09-23T18:47:02.129892+0000 osd.6 (osd.6) 162 : cluster 3 failed to encode map e214 with expected crc 2023-09-23T18:47:03.477 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:03 smithi098 bash[79516]: cluster 2023-09-23T18:47:02.132439+0000 mon.a (mon.0) 1153 : cluster 0 osdmap e214: 8 total, 8 up, 7 in 2023-09-23T18:47:03.477 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:03 smithi098 bash[79516]: cluster 2023-09-23T18:47:02.202728+0000 osd.6 (osd.6) 163 : cluster 0 7.7 scrub starts 2023-09-23T18:47:03.477 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:03 smithi098 bash[79516]: cluster 2023-09-23T18:47:02.216684+0000 osd.6 (osd.6) 164 : cluster 0 7.7 scrub ok 2023-09-23T18:47:03.477 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:03 smithi098 bash[79516]: cluster 2023-09-23T18:47:02.381808+0000 osd.1 (osd.1) 3 : cluster 0 7.0 scrub starts 2023-09-23T18:47:03.477 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:03 smithi098 bash[79516]: cluster 2023-09-23T18:47:02.398764+0000 osd.1 (osd.1) 4 : cluster 0 7.0 scrub ok 2023-09-23T18:47:03.477 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:03 smithi098 bash[79516]: cluster 2023-09-23T18:47:02.539482+0000 osd.7 (osd.7) 133 : cluster 0 7.5 scrub starts 2023-09-23T18:47:03.477 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:03 smithi098 bash[79516]: cluster 2023-09-23T18:47:02.559054+0000 osd.7 (osd.7) 134 : cluster 0 7.5 scrub ok 2023-09-23T18:47:03.477 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:03 smithi098 bash[76817]: cluster 2023-09-23T18:47:01.705867+0000 osd.4 (osd.4) 197 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T18:47:03.478 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:03 smithi098 bash[76817]: cluster 2023-09-23T18:47:01.707385+0000 osd.4 (osd.4) 198 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T18:47:03.478 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:03 smithi098 bash[76817]: cluster 2023-09-23T18:47:02.049365+0000 mgr.y (mgr.26054) 866 : cluster 0 pgmap v893: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 21 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 755/900 objects degraded (83.889%); 755/900 objects misplaced (83.889%); 0 B/s, 134 objects/s recovering 2023-09-23T18:47:03.478 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:03 smithi098 bash[76817]: cluster 2023-09-23T18:47:02.129875+0000 osd.7 (osd.7) 132 : cluster 3 failed to encode map e214 with expected crc 2023-09-23T18:47:03.478 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:03 smithi098 bash[76817]: cluster 2023-09-23T18:47:02.129892+0000 osd.6 (osd.6) 162 : cluster 3 failed to encode map e214 with expected crc 2023-09-23T18:47:03.478 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:03 smithi098 bash[76817]: cluster 2023-09-23T18:47:02.132439+0000 mon.a (mon.0) 1153 : cluster 0 osdmap e214: 8 total, 8 up, 7 in 2023-09-23T18:47:03.478 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:03 smithi098 bash[76817]: cluster 2023-09-23T18:47:02.202728+0000 osd.6 (osd.6) 163 : cluster 0 7.7 scrub starts 2023-09-23T18:47:03.478 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:03 smithi098 bash[76817]: cluster 2023-09-23T18:47:02.216684+0000 osd.6 (osd.6) 164 : cluster 0 7.7 scrub ok 2023-09-23T18:47:03.478 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:03 smithi098 bash[76817]: cluster 2023-09-23T18:47:02.381808+0000 osd.1 (osd.1) 3 : cluster 0 7.0 scrub starts 2023-09-23T18:47:03.478 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:03 smithi098 bash[76817]: cluster 2023-09-23T18:47:02.398764+0000 osd.1 (osd.1) 4 : cluster 0 7.0 scrub ok 2023-09-23T18:47:03.478 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:03 smithi098 bash[76817]: cluster 2023-09-23T18:47:02.539482+0000 osd.7 (osd.7) 133 : cluster 0 7.5 scrub starts 2023-09-23T18:47:03.479 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:03 smithi098 bash[76817]: cluster 2023-09-23T18:47:02.559054+0000 osd.7 (osd.7) 134 : cluster 0 7.5 scrub ok 2023-09-23T18:47:03.490 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:03 smithi173 bash[59422]: cluster 2023-09-23T18:47:01.705867+0000 osd.4 (osd.4) 197 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T18:47:03.491 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:03 smithi173 bash[59422]: cluster 2023-09-23T18:47:01.707385+0000 osd.4 (osd.4) 198 : cluster 3 failed to encode map e213 with expected crc 2023-09-23T18:47:03.491 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:03 smithi173 bash[59422]: cluster 2023-09-23T18:47:02.049365+0000 mgr.y (mgr.26054) 866 : cluster 0 pgmap v893: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 21 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 755/900 objects degraded (83.889%); 755/900 objects misplaced (83.889%); 0 B/s, 134 objects/s recovering 2023-09-23T18:47:03.491 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:03 smithi173 bash[59422]: cluster 2023-09-23T18:47:02.129875+0000 osd.7 (osd.7) 132 : cluster 3 failed to encode map e214 with expected crc 2023-09-23T18:47:03.491 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:03 smithi173 bash[59422]: cluster 2023-09-23T18:47:02.129892+0000 osd.6 (osd.6) 162 : cluster 3 failed to encode map e214 with expected crc 2023-09-23T18:47:03.491 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:03 smithi173 bash[59422]: cluster 2023-09-23T18:47:02.132439+0000 mon.a (mon.0) 1153 : cluster 0 osdmap e214: 8 total, 8 up, 7 in 2023-09-23T18:47:03.491 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:03 smithi173 bash[59422]: cluster 2023-09-23T18:47:02.202728+0000 osd.6 (osd.6) 163 : cluster 0 7.7 scrub starts 2023-09-23T18:47:03.491 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:03 smithi173 bash[59422]: cluster 2023-09-23T18:47:02.216684+0000 osd.6 (osd.6) 164 : cluster 0 7.7 scrub ok 2023-09-23T18:47:03.491 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:03 smithi173 bash[59422]: cluster 2023-09-23T18:47:02.381808+0000 osd.1 (osd.1) 3 : cluster 0 7.0 scrub starts 2023-09-23T18:47:03.491 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:03 smithi173 bash[59422]: cluster 2023-09-23T18:47:02.398764+0000 osd.1 (osd.1) 4 : cluster 0 7.0 scrub ok 2023-09-23T18:47:03.491 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:03 smithi173 bash[59422]: cluster 2023-09-23T18:47:02.539482+0000 osd.7 (osd.7) 133 : cluster 0 7.5 scrub starts 2023-09-23T18:47:03.492 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:03 smithi173 bash[59422]: cluster 2023-09-23T18:47:02.559054+0000 osd.7 (osd.7) 134 : cluster 0 7.5 scrub ok 2023-09-23T18:47:03.507 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:03.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:03.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:03.611 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:03.640 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:03.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:03.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:03.744 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:03.777 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:03.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:03.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:03.880 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:03.912 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:03.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:03.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:04.016 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:04.049 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:04.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:04.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:04.176 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:04.209 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:04.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:04.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:04.287 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:47:04.288 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:47:04.313 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:04.316 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:04 smithi098 bash[79516]: cluster 2023-09-23T18:47:03.131815+0000 osd.7 (osd.7) 135 : cluster 3 failed to encode map e215 with expected crc 2023-09-23T18:47:04.316 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:04 smithi098 bash[79516]: cluster 2023-09-23T18:47:03.132470+0000 osd.4 (osd.4) 199 : cluster 3 failed to encode map e215 with expected crc 2023-09-23T18:47:04.317 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:04 smithi098 bash[79516]: cluster 2023-09-23T18:47:03.135261+0000 osd.6 (osd.6) 165 : cluster 3 failed to encode map e215 with expected crc 2023-09-23T18:47:04.317 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:04 smithi098 bash[79516]: cluster 2023-09-23T18:47:03.135390+0000 mon.a (mon.0) 1154 : cluster 0 osdmap e215: 8 total, 8 up, 7 in 2023-09-23T18:47:04.317 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:04 smithi098 bash[79516]: cluster 2023-09-23T18:47:03.138349+0000 osd.6 (osd.6) 166 : cluster 3 failed to encode map e215 with expected crc 2023-09-23T18:47:04.317 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:04 smithi098 bash[79516]: cluster 2023-09-23T18:47:03.172275+0000 osd.6 (osd.6) 167 : cluster 0 7.17 scrub starts 2023-09-23T18:47:04.317 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:04 smithi098 bash[79516]: cluster 2023-09-23T18:47:03.185067+0000 osd.6 (osd.6) 168 : cluster 0 7.17 scrub ok 2023-09-23T18:47:04.317 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:04 smithi098 bash[79516]: cluster 2023-09-23T18:47:03.428541+0000 osd.3 (osd.3) 19 : cluster 0 7.3 scrub starts 2023-09-23T18:47:04.317 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:04 smithi098 bash[79516]: cluster 2023-09-23T18:47:03.443874+0000 osd.3 (osd.3) 20 : cluster 0 7.3 scrub ok 2023-09-23T18:47:04.317 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:04 smithi098 bash[79516]: cluster 2023-09-23T18:47:03.569333+0000 osd.7 (osd.7) 136 : cluster 0 7.15 scrub starts 2023-09-23T18:47:04.317 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:04 smithi098 bash[79516]: cluster 2023-09-23T18:47:03.587730+0000 osd.7 (osd.7) 137 : cluster 0 7.15 scrub ok 2023-09-23T18:47:04.317 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:04 smithi098 bash[79516]: cluster 2023-09-23T18:47:04.050148+0000 mgr.y (mgr.26054) 867 : cluster 0 pgmap v896: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 21 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 755/900 objects degraded (83.889%); 755/900 objects misplaced (83.889%); 0 B/s, 123 objects/s recovering 2023-09-23T18:47:04.318 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:04 smithi098 bash[76817]: cluster 2023-09-23T18:47:03.131815+0000 osd.7 (osd.7) 135 : cluster 3 failed to encode map e215 with expected crc 2023-09-23T18:47:04.318 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:04 smithi098 bash[76817]: cluster 2023-09-23T18:47:03.132470+0000 osd.4 (osd.4) 199 : cluster 3 failed to encode map e215 with expected crc 2023-09-23T18:47:04.318 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:04 smithi098 bash[76817]: cluster 2023-09-23T18:47:03.135261+0000 osd.6 (osd.6) 165 : cluster 3 failed to encode map e215 with expected crc 2023-09-23T18:47:04.318 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:04 smithi098 bash[76817]: cluster 2023-09-23T18:47:03.135390+0000 mon.a (mon.0) 1154 : cluster 0 osdmap e215: 8 total, 8 up, 7 in 2023-09-23T18:47:04.318 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:04 smithi098 bash[76817]: cluster 2023-09-23T18:47:03.138349+0000 osd.6 (osd.6) 166 : cluster 3 failed to encode map e215 with expected crc 2023-09-23T18:47:04.318 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:04 smithi098 bash[76817]: cluster 2023-09-23T18:47:03.172275+0000 osd.6 (osd.6) 167 : cluster 0 7.17 scrub starts 2023-09-23T18:47:04.318 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:04 smithi098 bash[76817]: cluster 2023-09-23T18:47:03.185067+0000 osd.6 (osd.6) 168 : cluster 0 7.17 scrub ok 2023-09-23T18:47:04.318 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:04 smithi098 bash[76817]: cluster 2023-09-23T18:47:03.428541+0000 osd.3 (osd.3) 19 : cluster 0 7.3 scrub starts 2023-09-23T18:47:04.318 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:04 smithi098 bash[76817]: cluster 2023-09-23T18:47:03.443874+0000 osd.3 (osd.3) 20 : cluster 0 7.3 scrub ok 2023-09-23T18:47:04.319 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:04 smithi098 bash[76817]: cluster 2023-09-23T18:47:03.569333+0000 osd.7 (osd.7) 136 : cluster 0 7.15 scrub starts 2023-09-23T18:47:04.319 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:04 smithi098 bash[76817]: cluster 2023-09-23T18:47:03.587730+0000 osd.7 (osd.7) 137 : cluster 0 7.15 scrub ok 2023-09-23T18:47:04.319 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:04 smithi098 bash[76817]: cluster 2023-09-23T18:47:04.050148+0000 mgr.y (mgr.26054) 867 : cluster 0 pgmap v896: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 21 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 755/900 objects degraded (83.889%); 755/900 objects misplaced (83.889%); 0 B/s, 123 objects/s recovering 2023-09-23T18:47:04.341 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:04.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:04.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:04.444 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:04.466 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:04.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:04.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:04.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:04 smithi173 bash[59422]: cluster 2023-09-23T18:47:03.131815+0000 osd.7 (osd.7) 135 : cluster 3 failed to encode map e215 with expected crc 2023-09-23T18:47:04.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:04 smithi173 bash[59422]: cluster 2023-09-23T18:47:03.132470+0000 osd.4 (osd.4) 199 : cluster 3 failed to encode map e215 with expected crc 2023-09-23T18:47:04.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:04 smithi173 bash[59422]: cluster 2023-09-23T18:47:03.135261+0000 osd.6 (osd.6) 165 : cluster 3 failed to encode map e215 with expected crc 2023-09-23T18:47:04.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:04 smithi173 bash[59422]: cluster 2023-09-23T18:47:03.135390+0000 mon.a (mon.0) 1154 : cluster 0 osdmap e215: 8 total, 8 up, 7 in 2023-09-23T18:47:04.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:04 smithi173 bash[59422]: cluster 2023-09-23T18:47:03.138349+0000 osd.6 (osd.6) 166 : cluster 3 failed to encode map e215 with expected crc 2023-09-23T18:47:04.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:04 smithi173 bash[59422]: cluster 2023-09-23T18:47:03.172275+0000 osd.6 (osd.6) 167 : cluster 0 7.17 scrub starts 2023-09-23T18:47:04.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:04 smithi173 bash[59422]: cluster 2023-09-23T18:47:03.185067+0000 osd.6 (osd.6) 168 : cluster 0 7.17 scrub ok 2023-09-23T18:47:04.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:04 smithi173 bash[59422]: cluster 2023-09-23T18:47:03.428541+0000 osd.3 (osd.3) 19 : cluster 0 7.3 scrub starts 2023-09-23T18:47:04.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:04 smithi173 bash[59422]: cluster 2023-09-23T18:47:03.443874+0000 osd.3 (osd.3) 20 : cluster 0 7.3 scrub ok 2023-09-23T18:47:04.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:04 smithi173 bash[59422]: cluster 2023-09-23T18:47:03.569333+0000 osd.7 (osd.7) 136 : cluster 0 7.15 scrub starts 2023-09-23T18:47:04.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:04 smithi173 bash[59422]: cluster 2023-09-23T18:47:03.587730+0000 osd.7 (osd.7) 137 : cluster 0 7.15 scrub ok 2023-09-23T18:47:04.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:04 smithi173 bash[59422]: cluster 2023-09-23T18:47:04.050148+0000 mgr.y (mgr.26054) 867 : cluster 0 pgmap v896: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 21 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 755/900 objects degraded (83.889%); 755/900 objects misplaced (83.889%); 0 B/s, 123 objects/s recovering 2023-09-23T18:47:04.570 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:04.589 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:04.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:04.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:04.619 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T18:47:04.619 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:47:04.620 INFO:tasks.thrashosds.thrasher:Adding osd 5 2023-09-23T18:47:04.620 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd in 5 2023-09-23T18:47:04.693 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:04.722 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:04.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:04.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:04.786 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:47:04.825 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:04.857 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:04.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:04.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:04.961 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:04.983 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:04.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:04.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:05.086 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:05.116 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:05.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:05.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:05.220 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:05.254 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:05.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:05.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:05.358 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:05.385 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:05.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:05.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:05.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:05.521 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:05.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:05.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:05.624 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:05.637 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:05 smithi098 bash[76817]: audit 2023-09-23T18:47:04.444487+0000 mon.a (mon.0) 1155 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:47:05.637 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:05 smithi098 bash[76817]: audit 2023-09-23T18:47:04.451710+0000 mon.a (mon.0) 1156 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:47:05.637 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:05 smithi098 bash[76817]: audit 2023-09-23T18:47:04.928512+0000 mon.a (mon.0) 1157 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:47:05.637 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:05 smithi098 bash[76817]: audit 2023-09-23T18:47:04.929559+0000 mon.a (mon.0) 1158 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:47:05.637 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:05 smithi098 bash[76817]: audit 2023-09-23T18:47:04.940441+0000 mon.a (mon.0) 1159 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:47:05.638 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:05 smithi098 bash[79516]: audit 2023-09-23T18:47:04.444487+0000 mon.a (mon.0) 1155 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:47:05.638 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:05 smithi098 bash[79516]: audit 2023-09-23T18:47:04.451710+0000 mon.a (mon.0) 1156 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:47:05.638 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:05 smithi098 bash[79516]: audit 2023-09-23T18:47:04.928512+0000 mon.a (mon.0) 1157 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:47:05.638 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:05 smithi098 bash[79516]: audit 2023-09-23T18:47:04.929559+0000 mon.a (mon.0) 1158 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:47:05.638 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:05 smithi098 bash[79516]: audit 2023-09-23T18:47:04.940441+0000 mon.a (mon.0) 1159 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:47:05.658 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:05.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:05.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:05.762 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:05.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:05 smithi173 bash[59422]: audit 2023-09-23T18:47:04.444487+0000 mon.a (mon.0) 1155 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:47:05.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:05 smithi173 bash[59422]: audit 2023-09-23T18:47:04.451710+0000 mon.a (mon.0) 1156 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:47:05.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:05 smithi173 bash[59422]: audit 2023-09-23T18:47:04.928512+0000 mon.a (mon.0) 1157 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:47:05.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:05 smithi173 bash[59422]: audit 2023-09-23T18:47:04.929559+0000 mon.a (mon.0) 1158 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:47:05.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:05 smithi173 bash[59422]: audit 2023-09-23T18:47:04.940441+0000 mon.a (mon.0) 1159 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:47:05.796 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:05.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:05.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:05.900 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:05.930 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:05.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:05.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:06.033 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:06.063 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:06.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:06.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:06.166 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:06.193 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:06.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:06.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:06.298 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:06.307 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:47:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:47:06] "GET /metrics HTTP/1.1" 200 36587 "" "Prometheus/2.33.4" 2023-09-23T18:47:06.311 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:47:06.323 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:06.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:06.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:06.427 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:06.462 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:06.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:06.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:06.566 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:06.597 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:06.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:06.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:06.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:06 smithi098 bash[76817]: cluster 2023-09-23T18:47:05.432475+0000 osd.2 (osd.2) 12 : cluster 0 7.8 scrub starts 2023-09-23T18:47:06.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:06 smithi098 bash[76817]: cluster 2023-09-23T18:47:05.455622+0000 osd.2 (osd.2) 13 : cluster 0 7.8 scrub ok 2023-09-23T18:47:06.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:06 smithi098 bash[76817]: cluster 2023-09-23T18:47:05.866092+0000 osd.4 (osd.4) 200 : cluster 0 7.1 scrub starts 2023-09-23T18:47:06.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:06 smithi098 bash[76817]: cluster 2023-09-23T18:47:05.879610+0000 osd.4 (osd.4) 201 : cluster 0 7.1 scrub ok 2023-09-23T18:47:06.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:06 smithi098 bash[76817]: cluster 2023-09-23T18:47:06.051074+0000 mgr.y (mgr.26054) 868 : cluster 0 pgmap v897: 99 pgs: 99 active+clean; 18 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 125 objects/s recovering 2023-09-23T18:47:06.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:06 smithi098 bash[76817]: audit 2023-09-23T18:47:06.052443+0000 mon.a (mon.0) 1160 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T18:47:06.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:06 smithi098 bash[79516]: cluster 2023-09-23T18:47:05.432475+0000 osd.2 (osd.2) 12 : cluster 0 7.8 scrub starts 2023-09-23T18:47:06.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:06 smithi098 bash[79516]: cluster 2023-09-23T18:47:05.455622+0000 osd.2 (osd.2) 13 : cluster 0 7.8 scrub ok 2023-09-23T18:47:06.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:06 smithi098 bash[79516]: cluster 2023-09-23T18:47:05.866092+0000 osd.4 (osd.4) 200 : cluster 0 7.1 scrub starts 2023-09-23T18:47:06.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:06 smithi098 bash[79516]: cluster 2023-09-23T18:47:05.879610+0000 osd.4 (osd.4) 201 : cluster 0 7.1 scrub ok 2023-09-23T18:47:06.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:06 smithi098 bash[79516]: cluster 2023-09-23T18:47:06.051074+0000 mgr.y (mgr.26054) 868 : cluster 0 pgmap v897: 99 pgs: 99 active+clean; 18 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 125 objects/s recovering 2023-09-23T18:47:06.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:06 smithi098 bash[79516]: audit 2023-09-23T18:47:06.052443+0000 mon.a (mon.0) 1160 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T18:47:06.701 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:06.731 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:06.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:06.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:06.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:06 smithi173 bash[59422]: cluster 2023-09-23T18:47:05.432475+0000 osd.2 (osd.2) 12 : cluster 0 7.8 scrub starts 2023-09-23T18:47:06.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:06 smithi173 bash[59422]: cluster 2023-09-23T18:47:05.455622+0000 osd.2 (osd.2) 13 : cluster 0 7.8 scrub ok 2023-09-23T18:47:06.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:06 smithi173 bash[59422]: cluster 2023-09-23T18:47:05.866092+0000 osd.4 (osd.4) 200 : cluster 0 7.1 scrub starts 2023-09-23T18:47:06.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:06 smithi173 bash[59422]: cluster 2023-09-23T18:47:05.879610+0000 osd.4 (osd.4) 201 : cluster 0 7.1 scrub ok 2023-09-23T18:47:06.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:06 smithi173 bash[59422]: cluster 2023-09-23T18:47:06.051074+0000 mgr.y (mgr.26054) 868 : cluster 0 pgmap v897: 99 pgs: 99 active+clean; 18 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 125 objects/s recovering 2023-09-23T18:47:06.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:06 smithi173 bash[59422]: audit 2023-09-23T18:47:06.052443+0000 mon.a (mon.0) 1160 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T18:47:06.834 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:06.864 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:06.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:06.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:06.968 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:07.003 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:07.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:07.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:07.107 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:07.128 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:07.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:07.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:07.231 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:07.266 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:07.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:07.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:07.370 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:07.401 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:07.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:07.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:07.506 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:07.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:07 smithi173 bash[59422]: cluster 2023-09-23T18:47:06.442156+0000 mon.a (mon.0) 1161 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 755/900 objects degraded (83.889%), 1 pg degraded, 1 pg undersized) 2023-09-23T18:47:07.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:07 smithi173 bash[59422]: cluster 2023-09-23T18:47:06.451429+0000 osd.2 (osd.2) 14 : cluster 0 7.18 scrub starts 2023-09-23T18:47:07.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:07 smithi173 bash[59422]: cluster 2023-09-23T18:47:06.453033+0000 osd.3 (osd.3) 21 : cluster 0 7.2 scrub starts 2023-09-23T18:47:07.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:07 smithi173 bash[59422]: cluster 2023-09-23T18:47:06.458622+0000 osd.4 (osd.4) 202 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T18:47:07.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:07 smithi173 bash[59422]: audit 2023-09-23T18:47:06.460624+0000 mon.a (mon.0) 1162 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-23T18:47:07.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:07 smithi173 bash[59422]: cluster 2023-09-23T18:47:06.460685+0000 mon.a (mon.0) 1163 : cluster 0 osdmap e216: 8 total, 8 up, 7 in 2023-09-23T18:47:07.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:07 smithi173 bash[59422]: cluster 2023-09-23T18:47:06.465374+0000 osd.7 (osd.7) 138 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T18:47:07.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:07 smithi173 bash[59422]: cluster 2023-09-23T18:47:06.479559+0000 osd.2 (osd.2) 15 : cluster 0 7.18 scrub ok 2023-09-23T18:47:07.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:07 smithi173 bash[59422]: cluster 2023-09-23T18:47:06.568936+0000 osd.6 (osd.6) 169 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T18:47:07.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:07 smithi173 bash[59422]: cluster 2023-09-23T18:47:06.572231+0000 osd.6 (osd.6) 170 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T18:47:07.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:07 smithi173 bash[59422]: cluster 2023-09-23T18:47:06.575046+0000 osd.6 (osd.6) 171 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T18:47:07.535 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:07.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:07.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:07.639 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:07.667 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:07.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:07.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:07.770 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:07.800 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:07.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:07.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:07.905 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:07.936 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:07.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:07.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:07.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:07 smithi098 bash[76817]: cluster 2023-09-23T18:47:06.442156+0000 mon.a (mon.0) 1161 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 755/900 objects degraded (83.889%), 1 pg degraded, 1 pg undersized) 2023-09-23T18:47:07.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:07 smithi098 bash[76817]: cluster 2023-09-23T18:47:06.451429+0000 osd.2 (osd.2) 14 : cluster 0 7.18 scrub starts 2023-09-23T18:47:07.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:07 smithi098 bash[76817]: cluster 2023-09-23T18:47:06.453033+0000 osd.3 (osd.3) 21 : cluster 0 7.2 scrub starts 2023-09-23T18:47:07.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:07 smithi098 bash[76817]: cluster 2023-09-23T18:47:06.458622+0000 osd.4 (osd.4) 202 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T18:47:07.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:07 smithi098 bash[76817]: audit 2023-09-23T18:47:06.460624+0000 mon.a (mon.0) 1162 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-23T18:47:07.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:07 smithi098 bash[76817]: cluster 2023-09-23T18:47:06.460685+0000 mon.a (mon.0) 1163 : cluster 0 osdmap e216: 8 total, 8 up, 7 in 2023-09-23T18:47:07.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:07 smithi098 bash[76817]: cluster 2023-09-23T18:47:06.465374+0000 osd.7 (osd.7) 138 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T18:47:07.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:07 smithi098 bash[76817]: cluster 2023-09-23T18:47:06.479559+0000 osd.2 (osd.2) 15 : cluster 0 7.18 scrub ok 2023-09-23T18:47:07.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:07 smithi098 bash[76817]: cluster 2023-09-23T18:47:06.568936+0000 osd.6 (osd.6) 169 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T18:47:07.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:07 smithi098 bash[76817]: cluster 2023-09-23T18:47:06.572231+0000 osd.6 (osd.6) 170 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T18:47:07.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:07 smithi098 bash[76817]: cluster 2023-09-23T18:47:06.575046+0000 osd.6 (osd.6) 171 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T18:47:07.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:07 smithi098 bash[79516]: cluster 2023-09-23T18:47:06.442156+0000 mon.a (mon.0) 1161 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 755/900 objects degraded (83.889%), 1 pg degraded, 1 pg undersized) 2023-09-23T18:47:07.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:07 smithi098 bash[79516]: cluster 2023-09-23T18:47:06.451429+0000 osd.2 (osd.2) 14 : cluster 0 7.18 scrub starts 2023-09-23T18:47:07.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:07 smithi098 bash[79516]: cluster 2023-09-23T18:47:06.453033+0000 osd.3 (osd.3) 21 : cluster 0 7.2 scrub starts 2023-09-23T18:47:07.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:07 smithi098 bash[79516]: cluster 2023-09-23T18:47:06.458622+0000 osd.4 (osd.4) 202 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T18:47:07.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:07 smithi098 bash[79516]: audit 2023-09-23T18:47:06.460624+0000 mon.a (mon.0) 1162 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-23T18:47:07.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:07 smithi098 bash[79516]: cluster 2023-09-23T18:47:06.460685+0000 mon.a (mon.0) 1163 : cluster 0 osdmap e216: 8 total, 8 up, 7 in 2023-09-23T18:47:07.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:07 smithi098 bash[79516]: cluster 2023-09-23T18:47:06.465374+0000 osd.7 (osd.7) 138 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T18:47:07.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:07 smithi098 bash[79516]: cluster 2023-09-23T18:47:06.479559+0000 osd.2 (osd.2) 15 : cluster 0 7.18 scrub ok 2023-09-23T18:47:07.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:07 smithi098 bash[79516]: cluster 2023-09-23T18:47:06.568936+0000 osd.6 (osd.6) 169 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T18:47:07.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:07 smithi098 bash[79516]: cluster 2023-09-23T18:47:06.572231+0000 osd.6 (osd.6) 170 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T18:47:07.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:07 smithi098 bash[79516]: cluster 2023-09-23T18:47:06.575046+0000 osd.6 (osd.6) 171 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T18:47:08.040 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:08.074 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:08.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:08.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:08.177 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:08.207 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:08.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:08.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:08.311 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:08.336 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:08.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:08.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:08.439 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:08.467 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:08.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:08.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:08.571 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:08.605 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:08.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:08.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:08.709 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:08.739 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:08.741 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:08 smithi173 bash[59422]: cluster 2023-09-23T18:47:07.096367+0000 osd.5 (osd.5) 108 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T18:47:08.741 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:08 smithi173 bash[59422]: cluster 2023-09-23T18:47:07.139423+0000 osd.6 (osd.6) 172 : cluster 0 7.12 scrub starts 2023-09-23T18:47:08.741 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:08 smithi173 bash[59422]: cluster 2023-09-23T18:47:07.158363+0000 osd.6 (osd.6) 173 : cluster 0 7.12 scrub ok 2023-09-23T18:47:08.741 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:08 smithi173 bash[59422]: cluster 2023-09-23T18:47:07.465840+0000 mon.a (mon.0) 1164 : cluster 0 osdmap e217: 8 total, 8 up, 7 in 2023-09-23T18:47:08.741 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:08 smithi173 bash[59422]: cluster 2023-09-23T18:47:07.467641+0000 osd.6 (osd.6) 174 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T18:47:08.741 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:08 smithi173 bash[59422]: cluster 2023-09-23T18:47:07.472045+0000 osd.7 (osd.7) 139 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T18:47:08.741 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:08 smithi173 bash[59422]: cluster 2023-09-23T18:47:07.472045+0000 osd.4 (osd.4) 203 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T18:47:08.741 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:08 smithi173 bash[59422]: cluster 2023-09-23T18:47:08.051628+0000 mgr.y (mgr.26054) 869 : cluster 0 pgmap v900: 99 pgs: 99 active+clean; 18 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 127 objects/s recovering 2023-09-23T18:47:08.741 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:08 smithi173 bash[59422]: audit 2023-09-23T18:47:08.052868+0000 mon.a (mon.0) 1165 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T18:47:08.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:08.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:08.843 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:08.871 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:08.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:08.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:08.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:08 smithi098 bash[76817]: cluster 2023-09-23T18:47:07.096367+0000 osd.5 (osd.5) 108 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T18:47:08.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:08 smithi098 bash[76817]: cluster 2023-09-23T18:47:07.139423+0000 osd.6 (osd.6) 172 : cluster 0 7.12 scrub starts 2023-09-23T18:47:08.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:08 smithi098 bash[76817]: cluster 2023-09-23T18:47:07.158363+0000 osd.6 (osd.6) 173 : cluster 0 7.12 scrub ok 2023-09-23T18:47:08.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:08 smithi098 bash[76817]: cluster 2023-09-23T18:47:07.465840+0000 mon.a (mon.0) 1164 : cluster 0 osdmap e217: 8 total, 8 up, 7 in 2023-09-23T18:47:08.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:08 smithi098 bash[76817]: cluster 2023-09-23T18:47:07.467641+0000 osd.6 (osd.6) 174 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T18:47:08.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:08 smithi098 bash[76817]: cluster 2023-09-23T18:47:07.472045+0000 osd.7 (osd.7) 139 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T18:47:08.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:08 smithi098 bash[76817]: cluster 2023-09-23T18:47:07.472045+0000 osd.4 (osd.4) 203 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T18:47:08.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:08 smithi098 bash[76817]: cluster 2023-09-23T18:47:08.051628+0000 mgr.y (mgr.26054) 869 : cluster 0 pgmap v900: 99 pgs: 99 active+clean; 18 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 127 objects/s recovering 2023-09-23T18:47:08.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:08 smithi098 bash[76817]: audit 2023-09-23T18:47:08.052868+0000 mon.a (mon.0) 1165 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T18:47:08.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:08 smithi098 bash[79516]: cluster 2023-09-23T18:47:07.096367+0000 osd.5 (osd.5) 108 : cluster 3 failed to encode map e216 with expected crc 2023-09-23T18:47:08.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:08 smithi098 bash[79516]: cluster 2023-09-23T18:47:07.139423+0000 osd.6 (osd.6) 172 : cluster 0 7.12 scrub starts 2023-09-23T18:47:08.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:08 smithi098 bash[79516]: cluster 2023-09-23T18:47:07.158363+0000 osd.6 (osd.6) 173 : cluster 0 7.12 scrub ok 2023-09-23T18:47:08.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:08 smithi098 bash[79516]: cluster 2023-09-23T18:47:07.465840+0000 mon.a (mon.0) 1164 : cluster 0 osdmap e217: 8 total, 8 up, 7 in 2023-09-23T18:47:08.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:08 smithi098 bash[79516]: cluster 2023-09-23T18:47:07.467641+0000 osd.6 (osd.6) 174 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T18:47:08.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:08 smithi098 bash[79516]: cluster 2023-09-23T18:47:07.472045+0000 osd.7 (osd.7) 139 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T18:47:08.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:08 smithi098 bash[79516]: cluster 2023-09-23T18:47:07.472045+0000 osd.4 (osd.4) 203 : cluster 3 failed to encode map e217 with expected crc 2023-09-23T18:47:08.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:08 smithi098 bash[79516]: cluster 2023-09-23T18:47:08.051628+0000 mgr.y (mgr.26054) 869 : cluster 0 pgmap v900: 99 pgs: 99 active+clean; 18 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 127 objects/s recovering 2023-09-23T18:47:08.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:08 smithi098 bash[79516]: audit 2023-09-23T18:47:08.052868+0000 mon.a (mon.0) 1165 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T18:47:08.974 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:09.001 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:09.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:09.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:09.105 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:09.126 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:09.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:09.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:09.229 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:09.259 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:09.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:09.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:09.362 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:09.385 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:09.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:09.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:09.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:09.517 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:09.521 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:09 smithi173 bash[59422]: cluster 2023-09-23T18:47:08.475813+0000 osd.6 (osd.6) 175 : cluster 3 failed to encode map e218 with expected crc 2023-09-23T18:47:09.521 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:09 smithi173 bash[59422]: audit 2023-09-23T18:47:08.477738+0000 mon.a (mon.0) 1166 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T18:47:09.521 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:09 smithi173 bash[59422]: cluster 2023-09-23T18:47:08.477803+0000 mon.a (mon.0) 1167 : cluster 0 osdmap e218: 8 total, 8 up, 7 in 2023-09-23T18:47:09.521 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:09 smithi173 bash[59422]: cluster 2023-09-23T18:47:08.482918+0000 osd.4 (osd.4) 204 : cluster 3 failed to encode map e218 with expected crc 2023-09-23T18:47:09.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:09.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:09.623 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:09.658 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:09.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:09.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:09.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:09 smithi098 bash[76817]: cluster 2023-09-23T18:47:08.475813+0000 osd.6 (osd.6) 175 : cluster 3 failed to encode map e218 with expected crc 2023-09-23T18:47:09.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:09 smithi098 bash[76817]: audit 2023-09-23T18:47:08.477738+0000 mon.a (mon.0) 1166 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T18:47:09.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:09 smithi098 bash[76817]: cluster 2023-09-23T18:47:08.477803+0000 mon.a (mon.0) 1167 : cluster 0 osdmap e218: 8 total, 8 up, 7 in 2023-09-23T18:47:09.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:09 smithi098 bash[76817]: cluster 2023-09-23T18:47:08.482918+0000 osd.4 (osd.4) 204 : cluster 3 failed to encode map e218 with expected crc 2023-09-23T18:47:09.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:09 smithi098 bash[79516]: cluster 2023-09-23T18:47:08.475813+0000 osd.6 (osd.6) 175 : cluster 3 failed to encode map e218 with expected crc 2023-09-23T18:47:09.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:09 smithi098 bash[79516]: audit 2023-09-23T18:47:08.477738+0000 mon.a (mon.0) 1166 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T18:47:09.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:09 smithi098 bash[79516]: cluster 2023-09-23T18:47:08.477803+0000 mon.a (mon.0) 1167 : cluster 0 osdmap e218: 8 total, 8 up, 7 in 2023-09-23T18:47:09.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:09 smithi098 bash[79516]: cluster 2023-09-23T18:47:08.482918+0000 osd.4 (osd.4) 204 : cluster 3 failed to encode map e218 with expected crc 2023-09-23T18:47:09.761 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:09.782 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:09.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:09.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:09.885 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:09.908 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:09.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:09.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:10.012 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:10.044 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:10.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:10.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:10.147 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:10.172 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:10.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:10.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:10.278 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:10.308 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:10.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:10.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:10.411 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:10.442 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:10.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:10.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:10.545 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:10.570 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:10.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:10.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:10.673 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:10.687 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:10 smithi173 bash[59422]: cluster 2023-09-23T18:47:08.648818+0000 osd.7 (osd.7) 140 : cluster 3 failed to encode map e218 with expected crc 2023-09-23T18:47:10.687 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:10 smithi173 bash[59422]: cluster 2023-09-23T18:47:08.984457+0000 osd.5 (osd.5) 109 : cluster 3 failed to encode map e218 with expected crc 2023-09-23T18:47:10.687 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:10 smithi173 bash[59422]: cluster 2023-09-23T18:47:09.482588+0000 mon.a (mon.0) 1168 : cluster 0 osdmap e219: 8 total, 8 up, 7 in 2023-09-23T18:47:10.687 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:10 smithi173 bash[59422]: cluster 2023-09-23T18:47:09.486127+0000 osd.4 (osd.4) 205 : cluster 3 failed to encode map e219 with expected crc 2023-09-23T18:47:10.687 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:10 smithi173 bash[59422]: cluster 2023-09-23T18:47:09.498503+0000 osd.7 (osd.7) 141 : cluster 3 failed to encode map e219 with expected crc 2023-09-23T18:47:10.687 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:10 smithi173 bash[59422]: audit 2023-09-23T18:47:09.618021+0000 mon.a (mon.0) 1169 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:47:10.687 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:10 smithi173 bash[59422]: audit 2023-09-23T18:47:09.619167+0000 mon.a (mon.0) 1170 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:47:10.687 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:10 smithi173 bash[59422]: cluster 2023-09-23T18:47:10.052313+0000 mgr.y (mgr.26054) 870 : cluster 0 pgmap v903: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 14 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 14 objects/s recovering 2023-09-23T18:47:10.687 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:10 smithi173 bash[59422]: audit 2023-09-23T18:47:10.053518+0000 mon.a (mon.0) 1171 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-09-23T18:47:10.706 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:10.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:10.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:10.809 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:10.820 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:10 smithi098 bash[76817]: cluster 2023-09-23T18:47:08.648818+0000 osd.7 (osd.7) 140 : cluster 3 failed to encode map e218 with expected crc 2023-09-23T18:47:10.820 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:10 smithi098 bash[76817]: cluster 2023-09-23T18:47:08.984457+0000 osd.5 (osd.5) 109 : cluster 3 failed to encode map e218 with expected crc 2023-09-23T18:47:10.821 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:10 smithi098 bash[76817]: cluster 2023-09-23T18:47:09.482588+0000 mon.a (mon.0) 1168 : cluster 0 osdmap e219: 8 total, 8 up, 7 in 2023-09-23T18:47:10.821 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:10 smithi098 bash[76817]: cluster 2023-09-23T18:47:09.486127+0000 osd.4 (osd.4) 205 : cluster 3 failed to encode map e219 with expected crc 2023-09-23T18:47:10.821 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:10 smithi098 bash[76817]: cluster 2023-09-23T18:47:09.498503+0000 osd.7 (osd.7) 141 : cluster 3 failed to encode map e219 with expected crc 2023-09-23T18:47:10.821 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:10 smithi098 bash[76817]: audit 2023-09-23T18:47:09.618021+0000 mon.a (mon.0) 1169 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:47:10.821 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:10 smithi098 bash[76817]: audit 2023-09-23T18:47:09.619167+0000 mon.a (mon.0) 1170 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:47:10.821 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:10 smithi098 bash[76817]: cluster 2023-09-23T18:47:10.052313+0000 mgr.y (mgr.26054) 870 : cluster 0 pgmap v903: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 14 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 14 objects/s recovering 2023-09-23T18:47:10.821 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:10 smithi098 bash[76817]: audit 2023-09-23T18:47:10.053518+0000 mon.a (mon.0) 1171 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-09-23T18:47:10.821 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:10 smithi098 bash[79516]: cluster 2023-09-23T18:47:08.648818+0000 osd.7 (osd.7) 140 : cluster 3 failed to encode map e218 with expected crc 2023-09-23T18:47:10.822 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:10 smithi098 bash[79516]: cluster 2023-09-23T18:47:08.984457+0000 osd.5 (osd.5) 109 : cluster 3 failed to encode map e218 with expected crc 2023-09-23T18:47:10.822 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:10 smithi098 bash[79516]: cluster 2023-09-23T18:47:09.482588+0000 mon.a (mon.0) 1168 : cluster 0 osdmap e219: 8 total, 8 up, 7 in 2023-09-23T18:47:10.822 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:10 smithi098 bash[79516]: cluster 2023-09-23T18:47:09.486127+0000 osd.4 (osd.4) 205 : cluster 3 failed to encode map e219 with expected crc 2023-09-23T18:47:10.822 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:10 smithi098 bash[79516]: cluster 2023-09-23T18:47:09.498503+0000 osd.7 (osd.7) 141 : cluster 3 failed to encode map e219 with expected crc 2023-09-23T18:47:10.822 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:10 smithi098 bash[79516]: audit 2023-09-23T18:47:09.618021+0000 mon.a (mon.0) 1169 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:47:10.822 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:10 smithi098 bash[79516]: audit 2023-09-23T18:47:09.619167+0000 mon.a (mon.0) 1170 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:47:10.822 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:10 smithi098 bash[79516]: cluster 2023-09-23T18:47:10.052313+0000 mgr.y (mgr.26054) 870 : cluster 0 pgmap v903: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 14 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 14 objects/s recovering 2023-09-23T18:47:10.822 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:10 smithi098 bash[79516]: audit 2023-09-23T18:47:10.053518+0000 mon.a (mon.0) 1171 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-09-23T18:47:10.831 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:10.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:10.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:10.934 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:10.961 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:10.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:10.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:11.065 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:11.091 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:11.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:11.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:11.194 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:11.213 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:11.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:11.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:11.317 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:11.348 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:11.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:11.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:11.452 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:11.482 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:11.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:11.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:11.585 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:11.598 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:11 smithi173 bash[59422]: cluster 2023-09-23T18:47:10.085592+0000 osd.6 (osd.6) 176 : cluster 3 failed to encode map e219 with expected crc 2023-09-23T18:47:11.598 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:11 smithi173 bash[59422]: cluster 2023-09-23T18:47:10.493769+0000 osd.5 (osd.5) 110 : cluster 3 failed to encode map e219 with expected crc 2023-09-23T18:47:11.598 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:11 smithi173 bash[59422]: cluster 2023-09-23T18:47:10.495445+0000 osd.0 (osd.0) 33 : cluster 0 7.14 starting backfill to osd.2 from (0'0,0'0] MAX to 212'8645 2023-09-23T18:47:11.599 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:11 smithi173 bash[59422]: audit 2023-09-23T18:47:10.495889+0000 mon.a (mon.0) 1172 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "22"}]': finished 2023-09-23T18:47:11.599 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:11 smithi173 bash[59422]: cluster 2023-09-23T18:47:10.496078+0000 mon.a (mon.0) 1173 : cluster 0 osdmap e220: 8 total, 8 up, 7 in 2023-09-23T18:47:11.599 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:11 smithi173 bash[59422]: cluster 2023-09-23T18:47:10.498945+0000 osd.7 (osd.7) 142 : cluster 3 failed to encode map e220 with expected crc 2023-09-23T18:47:11.599 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:11 smithi173 bash[59422]: cluster 2023-09-23T18:47:10.502440+0000 osd.7 (osd.7) 143 : cluster 3 failed to encode map e220 with expected crc 2023-09-23T18:47:11.599 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:11 smithi173 bash[59422]: cluster 2023-09-23T18:47:10.511194+0000 osd.0 (osd.0) 34 : cluster 0 7.14 starting backfill to osd.3 from (0'0,0'0] MAX to 212'8645 2023-09-23T18:47:11.599 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:11 smithi173 bash[59422]: cluster 2023-09-23T18:47:10.525285+0000 osd.4 (osd.4) 206 : cluster 3 failed to encode map e220 with expected crc 2023-09-23T18:47:11.618 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:11.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:11.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:11.721 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:11.742 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:11.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:11.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:11.845 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:11.878 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:11.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:11.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:11.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:11 smithi098 bash[79516]: cluster 2023-09-23T18:47:10.085592+0000 osd.6 (osd.6) 176 : cluster 3 failed to encode map e219 with expected crc 2023-09-23T18:47:11.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:11 smithi098 bash[79516]: cluster 2023-09-23T18:47:10.493769+0000 osd.5 (osd.5) 110 : cluster 3 failed to encode map e219 with expected crc 2023-09-23T18:47:11.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:11 smithi098 bash[79516]: cluster 2023-09-23T18:47:10.495445+0000 osd.0 (osd.0) 33 : cluster 0 7.14 starting backfill to osd.2 from (0'0,0'0] MAX to 212'8645 2023-09-23T18:47:11.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:11 smithi098 bash[79516]: audit 2023-09-23T18:47:10.495889+0000 mon.a (mon.0) 1172 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "22"}]': finished 2023-09-23T18:47:11.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:11 smithi098 bash[79516]: cluster 2023-09-23T18:47:10.496078+0000 mon.a (mon.0) 1173 : cluster 0 osdmap e220: 8 total, 8 up, 7 in 2023-09-23T18:47:11.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:11 smithi098 bash[79516]: cluster 2023-09-23T18:47:10.498945+0000 osd.7 (osd.7) 142 : cluster 3 failed to encode map e220 with expected crc 2023-09-23T18:47:11.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:11 smithi098 bash[79516]: cluster 2023-09-23T18:47:10.502440+0000 osd.7 (osd.7) 143 : cluster 3 failed to encode map e220 with expected crc 2023-09-23T18:47:11.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:11 smithi098 bash[79516]: cluster 2023-09-23T18:47:10.511194+0000 osd.0 (osd.0) 34 : cluster 0 7.14 starting backfill to osd.3 from (0'0,0'0] MAX to 212'8645 2023-09-23T18:47:11.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:11 smithi098 bash[79516]: cluster 2023-09-23T18:47:10.525285+0000 osd.4 (osd.4) 206 : cluster 3 failed to encode map e220 with expected crc 2023-09-23T18:47:11.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:11 smithi098 bash[76817]: cluster 2023-09-23T18:47:10.085592+0000 osd.6 (osd.6) 176 : cluster 3 failed to encode map e219 with expected crc 2023-09-23T18:47:11.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:11 smithi098 bash[76817]: cluster 2023-09-23T18:47:10.493769+0000 osd.5 (osd.5) 110 : cluster 3 failed to encode map e219 with expected crc 2023-09-23T18:47:11.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:11 smithi098 bash[76817]: cluster 2023-09-23T18:47:10.495445+0000 osd.0 (osd.0) 33 : cluster 0 7.14 starting backfill to osd.2 from (0'0,0'0] MAX to 212'8645 2023-09-23T18:47:11.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:11 smithi098 bash[76817]: audit 2023-09-23T18:47:10.495889+0000 mon.a (mon.0) 1172 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "22"}]': finished 2023-09-23T18:47:11.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:11 smithi098 bash[76817]: cluster 2023-09-23T18:47:10.496078+0000 mon.a (mon.0) 1173 : cluster 0 osdmap e220: 8 total, 8 up, 7 in 2023-09-23T18:47:11.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:11 smithi098 bash[76817]: cluster 2023-09-23T18:47:10.498945+0000 osd.7 (osd.7) 142 : cluster 3 failed to encode map e220 with expected crc 2023-09-23T18:47:11.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:11 smithi098 bash[76817]: cluster 2023-09-23T18:47:10.502440+0000 osd.7 (osd.7) 143 : cluster 3 failed to encode map e220 with expected crc 2023-09-23T18:47:11.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:11 smithi098 bash[76817]: cluster 2023-09-23T18:47:10.511194+0000 osd.0 (osd.0) 34 : cluster 0 7.14 starting backfill to osd.3 from (0'0,0'0] MAX to 212'8645 2023-09-23T18:47:11.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:11 smithi098 bash[76817]: cluster 2023-09-23T18:47:10.525285+0000 osd.4 (osd.4) 206 : cluster 3 failed to encode map e220 with expected crc 2023-09-23T18:47:11.982 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:12.015 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:12.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:12.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:12.118 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:12.153 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:12.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:12.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:12.257 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:12.283 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:12.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:12.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:12.387 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:12.411 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:12.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:12.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:12.514 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:12.540 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:12.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:12.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:12.643 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:12.668 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:12.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:12.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:12.775 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:12.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:12 smithi173 bash[59422]: cluster 2023-09-23T18:47:11.513295+0000 mon.a (mon.0) 1174 : cluster 0 osdmap e221: 8 total, 8 up, 7 in 2023-09-23T18:47:12.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:12 smithi173 bash[59422]: cluster 2023-09-23T18:47:11.514115+0000 osd.4 (osd.4) 207 : cluster 3 failed to encode map e221 with expected crc 2023-09-23T18:47:12.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:12 smithi173 bash[59422]: cluster 2023-09-23T18:47:11.514537+0000 osd.7 (osd.7) 144 : cluster 3 failed to encode map e221 with expected crc 2023-09-23T18:47:12.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:12 smithi173 bash[59422]: cluster 2023-09-23T18:47:11.516909+0000 osd.7 (osd.7) 145 : cluster 3 failed to encode map e221 with expected crc 2023-09-23T18:47:12.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:12 smithi173 bash[59422]: cluster 2023-09-23T18:47:12.053057+0000 mgr.y (mgr.26054) 871 : cluster 0 pgmap v906: 99 pgs: 2 peering, 1 active+recovering+undersized+remapped, 96 active+clean; 14 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:12.806 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:12.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:12.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:12.910 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:12.916 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:12 smithi098 bash[79516]: cluster 2023-09-23T18:47:11.513295+0000 mon.a (mon.0) 1174 : cluster 0 osdmap e221: 8 total, 8 up, 7 in 2023-09-23T18:47:12.916 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:12 smithi098 bash[79516]: cluster 2023-09-23T18:47:11.514115+0000 osd.4 (osd.4) 207 : cluster 3 failed to encode map e221 with expected crc 2023-09-23T18:47:12.916 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:12 smithi098 bash[79516]: cluster 2023-09-23T18:47:11.514537+0000 osd.7 (osd.7) 144 : cluster 3 failed to encode map e221 with expected crc 2023-09-23T18:47:12.917 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:12 smithi098 bash[79516]: cluster 2023-09-23T18:47:11.516909+0000 osd.7 (osd.7) 145 : cluster 3 failed to encode map e221 with expected crc 2023-09-23T18:47:12.917 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:12 smithi098 bash[79516]: cluster 2023-09-23T18:47:12.053057+0000 mgr.y (mgr.26054) 871 : cluster 0 pgmap v906: 99 pgs: 2 peering, 1 active+recovering+undersized+remapped, 96 active+clean; 14 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:12.917 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:12 smithi098 bash[76817]: cluster 2023-09-23T18:47:11.513295+0000 mon.a (mon.0) 1174 : cluster 0 osdmap e221: 8 total, 8 up, 7 in 2023-09-23T18:47:12.917 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:12 smithi098 bash[76817]: cluster 2023-09-23T18:47:11.514115+0000 osd.4 (osd.4) 207 : cluster 3 failed to encode map e221 with expected crc 2023-09-23T18:47:12.917 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:12 smithi098 bash[76817]: cluster 2023-09-23T18:47:11.514537+0000 osd.7 (osd.7) 144 : cluster 3 failed to encode map e221 with expected crc 2023-09-23T18:47:12.917 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:12 smithi098 bash[76817]: cluster 2023-09-23T18:47:11.516909+0000 osd.7 (osd.7) 145 : cluster 3 failed to encode map e221 with expected crc 2023-09-23T18:47:12.917 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:12 smithi098 bash[76817]: cluster 2023-09-23T18:47:12.053057+0000 mgr.y (mgr.26054) 871 : cluster 0 pgmap v906: 99 pgs: 2 peering, 1 active+recovering+undersized+remapped, 96 active+clean; 14 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:12.928 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:12.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:12.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:13.031 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:13.063 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:13.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:13.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:13.168 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:13.198 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:13.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:13.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:13.302 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:13.325 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:13.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:13.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:13.428 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:13.454 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:13.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:13.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:13.558 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:13.566 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:13 smithi098 bash[79516]: cluster 2023-09-23T18:47:12.526876+0000 mon.a (mon.0) 1175 : cluster 0 osdmap e222: 8 total, 8 up, 7 in 2023-09-23T18:47:13.566 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:13 smithi098 bash[79516]: cluster 2023-09-23T18:47:12.528605+0000 osd.7 (osd.7) 146 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T18:47:13.566 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:13 smithi098 bash[79516]: cluster 2023-09-23T18:47:12.532571+0000 osd.7 (osd.7) 147 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T18:47:13.567 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:13 smithi098 bash[79516]: cluster 2023-09-23T18:47:12.839880+0000 osd.4 (osd.4) 208 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T18:47:13.567 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:13 smithi098 bash[76817]: cluster 2023-09-23T18:47:12.526876+0000 mon.a (mon.0) 1175 : cluster 0 osdmap e222: 8 total, 8 up, 7 in 2023-09-23T18:47:13.567 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:13 smithi098 bash[76817]: cluster 2023-09-23T18:47:12.528605+0000 osd.7 (osd.7) 146 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T18:47:13.567 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:13 smithi098 bash[76817]: cluster 2023-09-23T18:47:12.532571+0000 osd.7 (osd.7) 147 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T18:47:13.567 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:13 smithi098 bash[76817]: cluster 2023-09-23T18:47:12.839880+0000 osd.4 (osd.4) 208 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T18:47:13.580 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:13.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:13.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:13.683 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:13.704 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:13.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:13.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:13.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:13 smithi173 bash[59422]: cluster 2023-09-23T18:47:12.526876+0000 mon.a (mon.0) 1175 : cluster 0 osdmap e222: 8 total, 8 up, 7 in 2023-09-23T18:47:13.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:13 smithi173 bash[59422]: cluster 2023-09-23T18:47:12.528605+0000 osd.7 (osd.7) 146 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T18:47:13.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:13 smithi173 bash[59422]: cluster 2023-09-23T18:47:12.532571+0000 osd.7 (osd.7) 147 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T18:47:13.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:13 smithi173 bash[59422]: cluster 2023-09-23T18:47:12.839880+0000 osd.4 (osd.4) 208 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T18:47:13.807 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:13.838 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:13.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:13.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:13.941 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:13.964 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:13.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:13.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:14.067 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:14.092 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:14.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:14.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:14.195 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:14.220 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:14.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:14.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:14.323 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:14.351 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:14.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:14.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:14.454 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:14.474 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:14.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:14.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:14.578 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:14.601 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:14.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:14.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:14.703 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:14.705 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:14 smithi098 bash[76817]: cluster 2023-09-23T18:47:12.897302+0000 osd.5 (osd.5) 111 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T18:47:14.705 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:14 smithi098 bash[76817]: cluster 2023-09-23T18:47:12.897333+0000 osd.6 (osd.6) 177 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T18:47:14.705 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:14 smithi098 bash[76817]: cluster 2023-09-23T18:47:14.053853+0000 mgr.y (mgr.26054) 872 : cluster 0 pgmap v908: 99 pgs: 2 peering, 1 active+recovering+undersized+remapped, 96 active+clean; 14 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:14.725 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:14.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:14.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:14.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:14 smithi173 bash[59422]: cluster 2023-09-23T18:47:12.897302+0000 osd.5 (osd.5) 111 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T18:47:14.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:14 smithi173 bash[59422]: cluster 2023-09-23T18:47:12.897333+0000 osd.6 (osd.6) 177 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T18:47:14.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:14 smithi173 bash[59422]: cluster 2023-09-23T18:47:14.053853+0000 mgr.y (mgr.26054) 872 : cluster 0 pgmap v908: 99 pgs: 2 peering, 1 active+recovering+undersized+remapped, 96 active+clean; 14 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:14.828 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:14.861 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:14.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:14.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:14.964 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:14.974 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:14 smithi098 bash[79516]: cluster 2023-09-23T18:47:12.897302+0000 osd.5 (osd.5) 111 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T18:47:14.974 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:14 smithi098 bash[79516]: cluster 2023-09-23T18:47:12.897333+0000 osd.6 (osd.6) 177 : cluster 3 failed to encode map e222 with expected crc 2023-09-23T18:47:14.975 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:14 smithi098 bash[79516]: cluster 2023-09-23T18:47:14.053853+0000 mgr.y (mgr.26054) 872 : cluster 0 pgmap v908: 99 pgs: 2 peering, 1 active+recovering+undersized+remapped, 96 active+clean; 14 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:14.988 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:14.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:14.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:15.092 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:15.120 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:15.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:15.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:15.223 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:15.246 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:15.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:15.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:15.350 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:15.370 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:15.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:15.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:15.473 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:15.493 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:15.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:15.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:15.596 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:15.621 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:15.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:15.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:15.724 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:15.754 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:15.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:15.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:15.858 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:15.883 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:15.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:15.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:15.987 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:16.010 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:16.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:16.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:16.113 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:16.138 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:16.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:16.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:16.242 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:16.260 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:16.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:16.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:16.351 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:47:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:47:16] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.33.4" 2023-09-23T18:47:16.352 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:16 smithi098 bash[76817]: cluster 2023-09-23T18:47:16.054529+0000 mgr.y (mgr.26054) 873 : cluster 0 pgmap v909: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 1 peering, 96 active+clean; 14 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:16.352 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:16 smithi098 bash[76817]: cluster 2023-09-23T18:47:16.094581+0000 mon.a (mon.0) 1176 : cluster 3 Health check failed: Degraded data redundancy: 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:47:16.364 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:16.387 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:16.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:16.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:16.490 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:16.518 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:16.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:16.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:16.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:16 smithi173 bash[59422]: cluster 2023-09-23T18:47:16.054529+0000 mgr.y (mgr.26054) 873 : cluster 0 pgmap v909: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 1 peering, 96 active+clean; 14 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:16.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:16 smithi173 bash[59422]: cluster 2023-09-23T18:47:16.094581+0000 mon.a (mon.0) 1176 : cluster 3 Health check failed: Degraded data redundancy: 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:47:16.621 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:16.628 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:16 smithi098 bash[79516]: cluster 2023-09-23T18:47:16.054529+0000 mgr.y (mgr.26054) 873 : cluster 0 pgmap v909: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 1 peering, 96 active+clean; 14 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:16.628 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:16 smithi098 bash[79516]: cluster 2023-09-23T18:47:16.094581+0000 mon.a (mon.0) 1176 : cluster 3 Health check failed: Degraded data redundancy: 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:47:16.641 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:16.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:16.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:16.744 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:16.770 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:16.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:16.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:16.873 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:16.893 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:16.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:16.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:16.996 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:17.022 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:17.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:17.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:17.125 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:17.152 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:17.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:17.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:17.255 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:17.279 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:17.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:17.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:17.382 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:17.408 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:17.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:17.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:17.511 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:17.538 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:17.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:17.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:17.641 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:17.663 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:17.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:17.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:17.766 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:17.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:17 smithi173 bash[59422]: cluster 2023-09-23T18:47:16.617119+0000 osd.3 (osd.3) 22 : cluster 0 7.14 scrub starts 2023-09-23T18:47:17.795 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:17.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:17.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:17.898 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:17.906 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:17 smithi098 bash[79516]: cluster 2023-09-23T18:47:16.617119+0000 osd.3 (osd.3) 22 : cluster 0 7.14 scrub starts 2023-09-23T18:47:17.906 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:17 smithi098 bash[76817]: cluster 2023-09-23T18:47:16.617119+0000 osd.3 (osd.3) 22 : cluster 0 7.14 scrub starts 2023-09-23T18:47:17.922 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:17.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:17.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:18.026 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:18.055 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:18.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:18.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:18.158 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:18.183 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:18.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:18.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:18.287 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:18.310 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:18.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:18.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:18.321 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:47:18.322 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:47:18.413 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:18.433 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:18.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:18.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:18.536 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:18.560 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:18.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:18.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:18.663 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:18.686 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:18.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:18.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:18.788 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:18.799 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:18 smithi098 bash[76817]: cluster 2023-09-23T18:47:17.653509+0000 osd.3 (osd.3) 23 : cluster 0 7.2 scrub starts 2023-09-23T18:47:18.799 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:18 smithi098 bash[76817]: cluster 2023-09-23T18:47:18.055163+0000 mgr.y (mgr.26054) 874 : cluster 0 pgmap v910: 99 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 96 active+clean; 14 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:18.800 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:18 smithi098 bash[76817]: audit 2023-09-23T18:47:18.056377+0000 mon.a (mon.0) 1177 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-09-23T18:47:18.800 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:18 smithi098 bash[79516]: cluster 2023-09-23T18:47:17.653509+0000 osd.3 (osd.3) 23 : cluster 0 7.2 scrub starts 2023-09-23T18:47:18.800 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:18 smithi098 bash[79516]: cluster 2023-09-23T18:47:18.055163+0000 mgr.y (mgr.26054) 874 : cluster 0 pgmap v910: 99 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 96 active+clean; 14 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:18.800 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:18 smithi098 bash[79516]: audit 2023-09-23T18:47:18.056377+0000 mon.a (mon.0) 1177 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-09-23T18:47:18.812 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:18.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:18.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:18.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:18.927 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:18 smithi173 bash[59422]: cluster 2023-09-23T18:47:17.653509+0000 osd.3 (osd.3) 23 : cluster 0 7.2 scrub starts 2023-09-23T18:47:18.927 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:18 smithi173 bash[59422]: cluster 2023-09-23T18:47:18.055163+0000 mgr.y (mgr.26054) 874 : cluster 0 pgmap v910: 99 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 96 active+clean; 14 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:18.927 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:18 smithi173 bash[59422]: audit 2023-09-23T18:47:18.056377+0000 mon.a (mon.0) 1177 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-09-23T18:47:18.943 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:18.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:18.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:19.046 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:19.070 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:19.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:19.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:19.174 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:19.198 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:19.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:19.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:19.302 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:19.330 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:19.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:19.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:19.434 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:19.464 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:19.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:19.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:19.567 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:19.592 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:19.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:19.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:19.696 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:19.703 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:19 smithi173 bash[59422]: cluster 2023-09-23T18:47:18.665190+0000 osd.4 (osd.4) 209 : cluster 3 failed to encode map e223 with expected crc 2023-09-23T18:47:19.703 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:19 smithi173 bash[59422]: audit 2023-09-23T18:47:18.666171+0000 mon.a (mon.0) 1178 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "23"}]': finished 2023-09-23T18:47:19.703 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:19 smithi173 bash[59422]: cluster 2023-09-23T18:47:18.666484+0000 mon.a (mon.0) 1179 : cluster 0 osdmap e223: 8 total, 8 up, 7 in 2023-09-23T18:47:19.703 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:19 smithi173 bash[59422]: cluster 2023-09-23T18:47:18.668498+0000 osd.4 (osd.4) 210 : cluster 3 failed to encode map e223 with expected crc 2023-09-23T18:47:19.704 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:19 smithi173 bash[59422]: cluster 2023-09-23T18:47:18.671589+0000 osd.7 (osd.7) 148 : cluster 3 failed to encode map e223 with expected crc 2023-09-23T18:47:19.717 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:19.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:19.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:19.821 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:19.829 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:19 smithi098 bash[79516]: cluster 2023-09-23T18:47:18.665190+0000 osd.4 (osd.4) 209 : cluster 3 failed to encode map e223 with expected crc 2023-09-23T18:47:19.829 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:19 smithi098 bash[79516]: audit 2023-09-23T18:47:18.666171+0000 mon.a (mon.0) 1178 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "23"}]': finished 2023-09-23T18:47:19.829 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:19 smithi098 bash[79516]: cluster 2023-09-23T18:47:18.666484+0000 mon.a (mon.0) 1179 : cluster 0 osdmap e223: 8 total, 8 up, 7 in 2023-09-23T18:47:19.829 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:19 smithi098 bash[79516]: cluster 2023-09-23T18:47:18.668498+0000 osd.4 (osd.4) 210 : cluster 3 failed to encode map e223 with expected crc 2023-09-23T18:47:19.830 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:19 smithi098 bash[79516]: cluster 2023-09-23T18:47:18.671589+0000 osd.7 (osd.7) 148 : cluster 3 failed to encode map e223 with expected crc 2023-09-23T18:47:19.830 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:19 smithi098 bash[76817]: cluster 2023-09-23T18:47:18.665190+0000 osd.4 (osd.4) 209 : cluster 3 failed to encode map e223 with expected crc 2023-09-23T18:47:19.830 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:19 smithi098 bash[76817]: audit 2023-09-23T18:47:18.666171+0000 mon.a (mon.0) 1178 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "23"}]': finished 2023-09-23T18:47:19.830 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:19 smithi098 bash[76817]: cluster 2023-09-23T18:47:18.666484+0000 mon.a (mon.0) 1179 : cluster 0 osdmap e223: 8 total, 8 up, 7 in 2023-09-23T18:47:19.830 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:19 smithi098 bash[76817]: cluster 2023-09-23T18:47:18.668498+0000 osd.4 (osd.4) 210 : cluster 3 failed to encode map e223 with expected crc 2023-09-23T18:47:19.830 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:19 smithi098 bash[76817]: cluster 2023-09-23T18:47:18.671589+0000 osd.7 (osd.7) 148 : cluster 3 failed to encode map e223 with expected crc 2023-09-23T18:47:19.844 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:19.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:19.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:19.947 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:19.970 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:19.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:19.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:20.074 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:20.100 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:20.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:20.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:20.204 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:20.225 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:20.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:20.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:20.328 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:20.352 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:20.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:20.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:20.455 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:20.479 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:20.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:20.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:20.582 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:20.604 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:20.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:20.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:20.707 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:20.728 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:20.731 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:20 smithi098 bash[79516]: cluster 2023-09-23T18:47:19.486039+0000 osd.5 (osd.5) 112 : cluster 3 failed to encode map e223 with expected crc 2023-09-23T18:47:20.731 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:20 smithi098 bash[79516]: cluster 2023-09-23T18:47:19.486243+0000 osd.6 (osd.6) 178 : cluster 3 failed to encode map e223 with expected crc 2023-09-23T18:47:20.731 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:20 smithi098 bash[79516]: cluster 2023-09-23T18:47:19.657552+0000 osd.3 (osd.3) 24 : cluster 0 7.14 scrub starts 2023-09-23T18:47:20.731 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:20 smithi098 bash[79516]: cluster 2023-09-23T18:47:19.679149+0000 osd.6 (osd.6) 179 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T18:47:20.731 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:20 smithi098 bash[79516]: cluster 2023-09-23T18:47:19.681133+0000 mon.a (mon.0) 1180 : cluster 0 osdmap e224: 8 total, 8 up, 7 in 2023-09-23T18:47:20.731 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:20 smithi098 bash[79516]: cluster 2023-09-23T18:47:19.682516+0000 osd.4 (osd.4) 211 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T18:47:20.731 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:20 smithi098 bash[79516]: cluster 2023-09-23T18:47:19.685196+0000 osd.7 (osd.7) 149 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T18:47:20.732 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:20 smithi098 bash[79516]: cluster 2023-09-23T18:47:19.685570+0000 osd.4 (osd.4) 212 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T18:47:20.732 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:20 smithi098 bash[79516]: cluster 2023-09-23T18:47:19.687182+0000 osd.7 (osd.7) 150 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T18:47:20.732 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:20 smithi098 bash[79516]: cluster 2023-09-23T18:47:20.055610+0000 mgr.y (mgr.26054) 875 : cluster 0 pgmap v913: 99 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 96 active+clean; 14 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 223 objects/s recovering 2023-09-23T18:47:20.732 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:20 smithi098 bash[79516]: audit 2023-09-23T18:47:20.057146+0000 mon.a (mon.0) 1181 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-09-23T18:47:20.732 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:20 smithi098 bash[76817]: cluster 2023-09-23T18:47:19.486039+0000 osd.5 (osd.5) 112 : cluster 3 failed to encode map e223 with expected crc 2023-09-23T18:47:20.732 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:20 smithi098 bash[76817]: cluster 2023-09-23T18:47:19.486243+0000 osd.6 (osd.6) 178 : cluster 3 failed to encode map e223 with expected crc 2023-09-23T18:47:20.732 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:20 smithi098 bash[76817]: cluster 2023-09-23T18:47:19.657552+0000 osd.3 (osd.3) 24 : cluster 0 7.14 scrub starts 2023-09-23T18:47:20.732 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:20 smithi098 bash[76817]: cluster 2023-09-23T18:47:19.679149+0000 osd.6 (osd.6) 179 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T18:47:20.732 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:20 smithi098 bash[76817]: cluster 2023-09-23T18:47:19.681133+0000 mon.a (mon.0) 1180 : cluster 0 osdmap e224: 8 total, 8 up, 7 in 2023-09-23T18:47:20.732 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:20 smithi098 bash[76817]: cluster 2023-09-23T18:47:19.682516+0000 osd.4 (osd.4) 211 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T18:47:20.733 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:20 smithi098 bash[76817]: cluster 2023-09-23T18:47:19.685196+0000 osd.7 (osd.7) 149 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T18:47:20.733 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:20 smithi098 bash[76817]: cluster 2023-09-23T18:47:19.685570+0000 osd.4 (osd.4) 212 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T18:47:20.733 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:20 smithi098 bash[76817]: cluster 2023-09-23T18:47:19.687182+0000 osd.7 (osd.7) 150 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T18:47:20.733 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:20 smithi098 bash[76817]: cluster 2023-09-23T18:47:20.055610+0000 mgr.y (mgr.26054) 875 : cluster 0 pgmap v913: 99 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 96 active+clean; 14 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 223 objects/s recovering 2023-09-23T18:47:20.733 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:20 smithi098 bash[76817]: audit 2023-09-23T18:47:20.057146+0000 mon.a (mon.0) 1181 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-09-23T18:47:20.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:20.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:20.835 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:20.844 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:20 smithi173 bash[59422]: cluster 2023-09-23T18:47:19.486039+0000 osd.5 (osd.5) 112 : cluster 3 failed to encode map e223 with expected crc 2023-09-23T18:47:20.844 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:20 smithi173 bash[59422]: cluster 2023-09-23T18:47:19.486243+0000 osd.6 (osd.6) 178 : cluster 3 failed to encode map e223 with expected crc 2023-09-23T18:47:20.844 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:20 smithi173 bash[59422]: cluster 2023-09-23T18:47:19.657552+0000 osd.3 (osd.3) 24 : cluster 0 7.14 scrub starts 2023-09-23T18:47:20.844 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:20 smithi173 bash[59422]: cluster 2023-09-23T18:47:19.679149+0000 osd.6 (osd.6) 179 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T18:47:20.844 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:20 smithi173 bash[59422]: cluster 2023-09-23T18:47:19.681133+0000 mon.a (mon.0) 1180 : cluster 0 osdmap e224: 8 total, 8 up, 7 in 2023-09-23T18:47:20.844 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:20 smithi173 bash[59422]: cluster 2023-09-23T18:47:19.682516+0000 osd.4 (osd.4) 211 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T18:47:20.845 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:20 smithi173 bash[59422]: cluster 2023-09-23T18:47:19.685196+0000 osd.7 (osd.7) 149 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T18:47:20.845 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:20 smithi173 bash[59422]: cluster 2023-09-23T18:47:19.685570+0000 osd.4 (osd.4) 212 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T18:47:20.845 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:20 smithi173 bash[59422]: cluster 2023-09-23T18:47:19.687182+0000 osd.7 (osd.7) 150 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T18:47:20.845 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:20 smithi173 bash[59422]: cluster 2023-09-23T18:47:20.055610+0000 mgr.y (mgr.26054) 875 : cluster 0 pgmap v913: 99 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+degraded+remapped, 96 active+clean; 14 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 223 objects/s recovering 2023-09-23T18:47:20.845 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:20 smithi173 bash[59422]: audit 2023-09-23T18:47:20.057146+0000 mon.a (mon.0) 1181 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-09-23T18:47:20.861 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:20.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:20.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:20.964 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:20.986 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:20.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:20.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:21.090 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:21.110 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:21.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:21.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:21.213 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:21.235 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:21.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:21.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:21.338 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:21.363 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:21.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:21.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:21.466 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:21.493 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:21.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:21.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:21.596 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:21.619 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:21.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:21.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:21.722 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:21.745 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:21.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:21.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:21.848 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:21.874 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:21.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:21.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:21.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:21 smithi098 bash[76817]: cluster 2023-09-23T18:47:20.417271+0000 osd.5 (osd.5) 113 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T18:47:21.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:21 smithi098 bash[76817]: audit 2023-09-23T18:47:20.690484+0000 mon.a (mon.0) 1182 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "24"}]': finished 2023-09-23T18:47:21.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:21 smithi098 bash[76817]: cluster 2023-09-23T18:47:20.690563+0000 mon.a (mon.0) 1183 : cluster 0 osdmap e225: 8 total, 8 up, 7 in 2023-09-23T18:47:21.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:21 smithi098 bash[76817]: cluster 2023-09-23T18:47:20.691019+0000 osd.7 (osd.7) 151 : cluster 3 failed to encode map e225 with expected crc 2023-09-23T18:47:21.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:21 smithi098 bash[76817]: cluster 2023-09-23T18:47:20.692489+0000 osd.0 (osd.0) 35 : cluster 0 7.16 starting backfill to osd.4 from (0'0,0'0] MAX to 212'8803 2023-09-23T18:47:21.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:21 smithi098 bash[76817]: cluster 2023-09-23T18:47:20.693300+0000 osd.7 (osd.7) 152 : cluster 3 failed to encode map e225 with expected crc 2023-09-23T18:47:21.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:21 smithi098 bash[76817]: cluster 2023-09-23T18:47:20.716185+0000 osd.5 (osd.5) 114 : cluster 3 failed to encode map e225 with expected crc 2023-09-23T18:47:21.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:21 smithi098 bash[76817]: cluster 2023-09-23T18:47:20.717246+0000 osd.5 (osd.5) 115 : cluster 3 failed to encode map e225 with expected crc 2023-09-23T18:47:21.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:21 smithi098 bash[76817]: cluster 2023-09-23T18:47:20.898571+0000 osd.4 (osd.4) 213 : cluster 3 failed to encode map e225 with expected crc 2023-09-23T18:47:21.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:21 smithi098 bash[79516]: cluster 2023-09-23T18:47:20.417271+0000 osd.5 (osd.5) 113 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T18:47:21.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:21 smithi098 bash[79516]: audit 2023-09-23T18:47:20.690484+0000 mon.a (mon.0) 1182 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "24"}]': finished 2023-09-23T18:47:21.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:21 smithi098 bash[79516]: cluster 2023-09-23T18:47:20.690563+0000 mon.a (mon.0) 1183 : cluster 0 osdmap e225: 8 total, 8 up, 7 in 2023-09-23T18:47:21.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:21 smithi098 bash[79516]: cluster 2023-09-23T18:47:20.691019+0000 osd.7 (osd.7) 151 : cluster 3 failed to encode map e225 with expected crc 2023-09-23T18:47:21.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:21 smithi098 bash[79516]: cluster 2023-09-23T18:47:20.692489+0000 osd.0 (osd.0) 35 : cluster 0 7.16 starting backfill to osd.4 from (0'0,0'0] MAX to 212'8803 2023-09-23T18:47:21.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:21 smithi098 bash[79516]: cluster 2023-09-23T18:47:20.693300+0000 osd.7 (osd.7) 152 : cluster 3 failed to encode map e225 with expected crc 2023-09-23T18:47:21.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:21 smithi098 bash[79516]: cluster 2023-09-23T18:47:20.716185+0000 osd.5 (osd.5) 114 : cluster 3 failed to encode map e225 with expected crc 2023-09-23T18:47:21.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:21 smithi098 bash[79516]: cluster 2023-09-23T18:47:20.717246+0000 osd.5 (osd.5) 115 : cluster 3 failed to encode map e225 with expected crc 2023-09-23T18:47:21.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:21 smithi098 bash[79516]: cluster 2023-09-23T18:47:20.898571+0000 osd.4 (osd.4) 213 : cluster 3 failed to encode map e225 with expected crc 2023-09-23T18:47:21.976 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:21.987 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:21 smithi173 bash[59422]: cluster 2023-09-23T18:47:20.417271+0000 osd.5 (osd.5) 113 : cluster 3 failed to encode map e224 with expected crc 2023-09-23T18:47:21.987 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:21 smithi173 bash[59422]: audit 2023-09-23T18:47:20.690484+0000 mon.a (mon.0) 1182 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "24"}]': finished 2023-09-23T18:47:21.987 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:21 smithi173 bash[59422]: cluster 2023-09-23T18:47:20.690563+0000 mon.a (mon.0) 1183 : cluster 0 osdmap e225: 8 total, 8 up, 7 in 2023-09-23T18:47:21.987 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:21 smithi173 bash[59422]: cluster 2023-09-23T18:47:20.691019+0000 osd.7 (osd.7) 151 : cluster 3 failed to encode map e225 with expected crc 2023-09-23T18:47:21.987 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:21 smithi173 bash[59422]: cluster 2023-09-23T18:47:20.692489+0000 osd.0 (osd.0) 35 : cluster 0 7.16 starting backfill to osd.4 from (0'0,0'0] MAX to 212'8803 2023-09-23T18:47:21.988 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:21 smithi173 bash[59422]: cluster 2023-09-23T18:47:20.693300+0000 osd.7 (osd.7) 152 : cluster 3 failed to encode map e225 with expected crc 2023-09-23T18:47:21.988 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:21 smithi173 bash[59422]: cluster 2023-09-23T18:47:20.716185+0000 osd.5 (osd.5) 114 : cluster 3 failed to encode map e225 with expected crc 2023-09-23T18:47:21.988 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:21 smithi173 bash[59422]: cluster 2023-09-23T18:47:20.717246+0000 osd.5 (osd.5) 115 : cluster 3 failed to encode map e225 with expected crc 2023-09-23T18:47:21.988 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:21 smithi173 bash[59422]: cluster 2023-09-23T18:47:20.898571+0000 osd.4 (osd.4) 213 : cluster 3 failed to encode map e225 with expected crc 2023-09-23T18:47:22.002 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:22.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:22.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:22.108 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:22.133 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:22.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:22.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:22.236 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:22.263 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:22.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:22.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:22.367 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:22.395 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:22.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:22.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:22.498 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:22.523 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:22.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:22.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:22.627 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:22.650 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:22.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:22.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:22.724 INFO:teuthology.orchestra.run.smithi098.stderr:pool 'unique_pool_2' removedpool 'unique_pool_2' removed 2023-09-23T18:47:22.754 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:22.763 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:22 smithi098 bash[79516]: cluster 2023-09-23T18:47:21.694111+0000 osd.7 (osd.7) 153 : cluster 3 failed to encode map e226 with expected crc 2023-09-23T18:47:22.763 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:22 smithi098 bash[79516]: cluster 2023-09-23T18:47:21.696379+0000 mon.a (mon.0) 1184 : cluster 0 osdmap e226: 8 total, 8 up, 7 in 2023-09-23T18:47:22.763 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:22 smithi098 bash[79516]: cluster 2023-09-23T18:47:21.696756+0000 osd.4 (osd.4) 214 : cluster 3 failed to encode map e226 with expected crc 2023-09-23T18:47:22.763 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:22 smithi098 bash[79516]: cluster 2023-09-23T18:47:21.697472+0000 osd.7 (osd.7) 154 : cluster 3 failed to encode map e226 with expected crc 2023-09-23T18:47:22.764 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:22 smithi098 bash[79516]: cluster 2023-09-23T18:47:21.702577+0000 osd.6 (osd.6) 180 : cluster 3 failed to encode map e226 with expected crc 2023-09-23T18:47:22.764 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:22 smithi098 bash[79516]: cluster 2023-09-23T18:47:22.056302+0000 mgr.y (mgr.26054) 876 : cluster 0 pgmap v916: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 97 active+clean; 14 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 254 objects/s recovering 2023-09-23T18:47:22.764 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:22 smithi098 bash[79516]: audit 2023-09-23T18:47:22.057587+0000 mon.a (mon.0) 1185 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-09-23T18:47:22.764 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:22 smithi098 bash[79516]: audit 2023-09-23T18:47:22.109474+0000 mon.c (mon.2) 101 : audit 1 from='client.? 172.21.15.98:0/3432070363' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:47:22.764 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:22 smithi098 bash[79516]: audit 2023-09-23T18:47:22.109873+0000 mon.a (mon.0) 1186 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:47:22.764 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:22 smithi098 bash[76817]: cluster 2023-09-23T18:47:21.694111+0000 osd.7 (osd.7) 153 : cluster 3 failed to encode map e226 with expected crc 2023-09-23T18:47:22.764 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:22 smithi098 bash[76817]: cluster 2023-09-23T18:47:21.696379+0000 mon.a (mon.0) 1184 : cluster 0 osdmap e226: 8 total, 8 up, 7 in 2023-09-23T18:47:22.764 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:22 smithi098 bash[76817]: cluster 2023-09-23T18:47:21.696756+0000 osd.4 (osd.4) 214 : cluster 3 failed to encode map e226 with expected crc 2023-09-23T18:47:22.764 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:22 smithi098 bash[76817]: cluster 2023-09-23T18:47:21.697472+0000 osd.7 (osd.7) 154 : cluster 3 failed to encode map e226 with expected crc 2023-09-23T18:47:22.765 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:22 smithi098 bash[76817]: cluster 2023-09-23T18:47:21.702577+0000 osd.6 (osd.6) 180 : cluster 3 failed to encode map e226 with expected crc 2023-09-23T18:47:22.765 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:22 smithi098 bash[76817]: cluster 2023-09-23T18:47:22.056302+0000 mgr.y (mgr.26054) 876 : cluster 0 pgmap v916: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 97 active+clean; 14 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 254 objects/s recovering 2023-09-23T18:47:22.765 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:22 smithi098 bash[76817]: audit 2023-09-23T18:47:22.057587+0000 mon.a (mon.0) 1185 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-09-23T18:47:22.765 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:22 smithi098 bash[76817]: audit 2023-09-23T18:47:22.109474+0000 mon.c (mon.2) 101 : audit 1 from='client.? 172.21.15.98:0/3432070363' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:47:22.765 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:22 smithi098 bash[76817]: audit 2023-09-23T18:47:22.109873+0000 mon.a (mon.0) 1186 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:47:22.796 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:22.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:22.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:22.902 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:22.932 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:22.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:22.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:23.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:22 smithi173 bash[59422]: cluster 2023-09-23T18:47:21.694111+0000 osd.7 (osd.7) 153 : cluster 3 failed to encode map e226 with expected crc 2023-09-23T18:47:23.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:22 smithi173 bash[59422]: cluster 2023-09-23T18:47:21.696379+0000 mon.a (mon.0) 1184 : cluster 0 osdmap e226: 8 total, 8 up, 7 in 2023-09-23T18:47:23.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:22 smithi173 bash[59422]: cluster 2023-09-23T18:47:21.696756+0000 osd.4 (osd.4) 214 : cluster 3 failed to encode map e226 with expected crc 2023-09-23T18:47:23.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:22 smithi173 bash[59422]: cluster 2023-09-23T18:47:21.697472+0000 osd.7 (osd.7) 154 : cluster 3 failed to encode map e226 with expected crc 2023-09-23T18:47:23.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:22 smithi173 bash[59422]: cluster 2023-09-23T18:47:21.702577+0000 osd.6 (osd.6) 180 : cluster 3 failed to encode map e226 with expected crc 2023-09-23T18:47:23.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:22 smithi173 bash[59422]: cluster 2023-09-23T18:47:22.056302+0000 mgr.y (mgr.26054) 876 : cluster 0 pgmap v916: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 97 active+clean; 14 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 254 objects/s recovering 2023-09-23T18:47:23.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:22 smithi173 bash[59422]: audit 2023-09-23T18:47:22.057587+0000 mon.a (mon.0) 1185 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-09-23T18:47:23.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:22 smithi173 bash[59422]: audit 2023-09-23T18:47:22.109474+0000 mon.c (mon.2) 101 : audit 1 from='client.? 172.21.15.98:0/3432070363' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:47:23.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:22 smithi173 bash[59422]: audit 2023-09-23T18:47:22.109873+0000 mon.a (mon.0) 1186 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:47:23.035 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:23.059 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:23.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:23.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:23.162 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:23.188 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:23.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:23.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:23.291 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:23.315 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:23.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:23.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:23.418 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:23.443 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:23.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:23.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:23.545 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:23.577 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:23.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:23.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:23.681 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:23.705 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:23.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:23.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:23.811 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:23.838 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:23.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:23.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:23.941 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:23.969 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:23.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:23.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:24.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:23 smithi173 bash[59422]: cluster 2023-09-23T18:47:22.702161+0000 osd.5 (osd.5) 116 : cluster 3 failed to encode map e227 with expected crc 2023-09-23T18:47:24.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:23 smithi173 bash[59422]: cluster 2023-09-23T18:47:22.702497+0000 osd.4 (osd.4) 215 : cluster 3 failed to encode map e227 with expected crc 2023-09-23T18:47:24.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:23 smithi173 bash[59422]: audit 2023-09-23T18:47:22.706781+0000 mon.a (mon.0) 1187 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "25"}]': finished 2023-09-23T18:47:24.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:23 smithi173 bash[59422]: audit 2023-09-23T18:47:22.707139+0000 mon.a (mon.0) 1188 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:47:24.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:23 smithi173 bash[59422]: cluster 2023-09-23T18:47:22.707218+0000 mon.a (mon.0) 1189 : cluster 0 osdmap e227: 8 total, 8 up, 7 in 2023-09-23T18:47:24.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:23 smithi173 bash[59422]: cluster 2023-09-23T18:47:22.747739+0000 osd.7 (osd.7) 155 : cluster 3 failed to encode map e227 with expected crc 2023-09-23T18:47:24.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:23 smithi173 bash[59422]: cluster 2023-09-23T18:47:22.773301+0000 osd.3 (osd.3) 25 : cluster 0 7.2 scrub starts 2023-09-23T18:47:24.072 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:24.104 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:24.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:24.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:24.180 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:23 smithi098 bash[79516]: cluster 2023-09-23T18:47:22.702161+0000 osd.5 (osd.5) 116 : cluster 3 failed to encode map e227 with expected crc 2023-09-23T18:47:24.180 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:23 smithi098 bash[79516]: cluster 2023-09-23T18:47:22.702497+0000 osd.4 (osd.4) 215 : cluster 3 failed to encode map e227 with expected crc 2023-09-23T18:47:24.180 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:23 smithi098 bash[79516]: audit 2023-09-23T18:47:22.706781+0000 mon.a (mon.0) 1187 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "25"}]': finished 2023-09-23T18:47:24.180 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:23 smithi098 bash[79516]: audit 2023-09-23T18:47:22.707139+0000 mon.a (mon.0) 1188 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:47:24.180 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:23 smithi098 bash[79516]: cluster 2023-09-23T18:47:22.707218+0000 mon.a (mon.0) 1189 : cluster 0 osdmap e227: 8 total, 8 up, 7 in 2023-09-23T18:47:24.180 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:23 smithi098 bash[79516]: cluster 2023-09-23T18:47:22.747739+0000 osd.7 (osd.7) 155 : cluster 3 failed to encode map e227 with expected crc 2023-09-23T18:47:24.181 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:23 smithi098 bash[79516]: cluster 2023-09-23T18:47:22.773301+0000 osd.3 (osd.3) 25 : cluster 0 7.2 scrub starts 2023-09-23T18:47:24.207 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:24.243 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:24.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:24.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:24.346 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:24.385 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:24.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:24.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:24.488 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:24.490 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:24 smithi098 bash[76817]: cluster 2023-09-23T18:47:22.702161+0000 osd.5 (osd.5) 116 : cluster 3 failed to encode map e227 with expected crc 2023-09-23T18:47:24.490 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:24 smithi098 bash[76817]: cluster 2023-09-23T18:47:22.702497+0000 osd.4 (osd.4) 215 : cluster 3 failed to encode map e227 with expected crc 2023-09-23T18:47:24.490 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:24 smithi098 bash[76817]: audit 2023-09-23T18:47:22.706781+0000 mon.a (mon.0) 1187 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "25"}]': finished 2023-09-23T18:47:24.490 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:24 smithi098 bash[76817]: audit 2023-09-23T18:47:22.707139+0000 mon.a (mon.0) 1188 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:47:24.490 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:24 smithi098 bash[76817]: cluster 2023-09-23T18:47:22.707218+0000 mon.a (mon.0) 1189 : cluster 0 osdmap e227: 8 total, 8 up, 7 in 2023-09-23T18:47:24.490 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:24 smithi098 bash[76817]: cluster 2023-09-23T18:47:22.747739+0000 osd.7 (osd.7) 155 : cluster 3 failed to encode map e227 with expected crc 2023-09-23T18:47:24.490 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:24 smithi098 bash[76817]: cluster 2023-09-23T18:47:22.773301+0000 osd.3 (osd.3) 25 : cluster 0 7.2 scrub starts 2023-09-23T18:47:24.516 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:24.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:24.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:24.619 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:24.643 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:24.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:24.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:24.747 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:24.754 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:24 smithi098 bash[76817]: cluster 2023-09-23T18:47:23.049742+0000 osd.6 (osd.6) 181 : cluster 3 failed to encode map e227 with expected crc 2023-09-23T18:47:24.754 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:24 smithi098 bash[76817]: cluster 2023-09-23T18:47:24.057105+0000 mgr.y (mgr.26054) 877 : cluster 0 pgmap v918: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:24.755 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:24 smithi098 bash[76817]: audit 2023-09-23T18:47:24.620378+0000 mon.a (mon.0) 1190 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:47:24.755 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:24 smithi098 bash[76817]: audit 2023-09-23T18:47:24.621710+0000 mon.a (mon.0) 1191 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:47:24.780 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:24.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:24.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:24.884 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:24.919 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:24.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:24.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:24.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:24 smithi098 bash[79516]: cluster 2023-09-23T18:47:23.049742+0000 osd.6 (osd.6) 181 : cluster 3 failed to encode map e227 with expected crc 2023-09-23T18:47:24.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:24 smithi098 bash[79516]: cluster 2023-09-23T18:47:24.057105+0000 mgr.y (mgr.26054) 877 : cluster 0 pgmap v918: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:24.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:24 smithi098 bash[79516]: audit 2023-09-23T18:47:24.620378+0000 mon.a (mon.0) 1190 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:47:24.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:24 smithi098 bash[79516]: audit 2023-09-23T18:47:24.621710+0000 mon.a (mon.0) 1191 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:47:25.022 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:25.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:24 smithi173 bash[59422]: cluster 2023-09-23T18:47:23.049742+0000 osd.6 (osd.6) 181 : cluster 3 failed to encode map e227 with expected crc 2023-09-23T18:47:25.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:24 smithi173 bash[59422]: cluster 2023-09-23T18:47:24.057105+0000 mgr.y (mgr.26054) 877 : cluster 0 pgmap v918: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:25.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:24 smithi173 bash[59422]: audit 2023-09-23T18:47:24.620378+0000 mon.a (mon.0) 1190 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:47:25.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:24 smithi173 bash[59422]: audit 2023-09-23T18:47:24.621710+0000 mon.a (mon.0) 1191 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:47:25.054 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:25.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:25.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:25.157 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:25.187 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:25.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:25.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:25.291 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:25.323 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:25.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:25.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:25.426 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:25.453 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:25.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:25.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:25.556 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:25.577 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:25.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:25.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:25.681 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:25.702 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:25.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:25.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:25.805 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:25.828 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:25.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:25.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:25.931 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:25.963 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:25.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:25.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:26.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:25 smithi173 bash[59422]: cluster 2023-09-23T18:47:24.700938+0000 mon.a (mon.0) 1192 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2 pgs degraded) 2023-09-23T18:47:26.066 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:26.074 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:25 smithi098 bash[79516]: cluster 2023-09-23T18:47:24.700938+0000 mon.a (mon.0) 1192 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2 pgs degraded) 2023-09-23T18:47:26.075 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:25 smithi098 bash[76817]: cluster 2023-09-23T18:47:24.700938+0000 mon.a (mon.0) 1192 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2 pgs degraded) 2023-09-23T18:47:26.088 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:26.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:26.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:26.192 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:26.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:26.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:26.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:26.330 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:26.343 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:47:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:47:26] "GET /metrics HTTP/1.1" 200 34211 "" "Prometheus/2.33.4" 2023-09-23T18:47:26.358 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:26.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:26.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:26.461 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:26.493 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:26.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:26.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:26.597 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:26.616 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:26.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:26.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:26.707 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:47:26.707 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:47:26.719 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:26.739 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:26.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:26.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:26.842 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:26.855 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:26 smithi098 bash[76817]: cluster 2023-09-23T18:47:26.057862+0000 mgr.y (mgr.26054) 878 : cluster 0 pgmap v919: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:26.856 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:26 smithi098 bash[79516]: cluster 2023-09-23T18:47:26.057862+0000 mgr.y (mgr.26054) 878 : cluster 0 pgmap v919: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:26.867 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:26.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:26.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:26.972 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:26.997 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:26.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:26.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:27.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:26 smithi173 bash[59422]: cluster 2023-09-23T18:47:26.057862+0000 mgr.y (mgr.26054) 878 : cluster 0 pgmap v919: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:27.099 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:27.128 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:27.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:27.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:27.232 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:27.261 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:27.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:27.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:27.364 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:27.401 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:27.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:27.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:27.505 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:27.542 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:27.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:27.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:27.557 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T18:47:27.558 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T18:47:27.558 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-09-23T18:47:27.558 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-09-23T18:47:27.645 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:27.668 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:27.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:27.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:27.771 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:27.804 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:27.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:27.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:27.907 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:27.939 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:27.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:27.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:28.044 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:28.067 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:28.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:28.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:28.171 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:28.203 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:28.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:28.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:28.308 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:28.342 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:28.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:28.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:28.447 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:28.470 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:28.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:28.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:28.525 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:47:28.546 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:47:28.565 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:47:28.573 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:28.585 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:47:28.592 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:28.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:28.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:28.610 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:47:28.631 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:47:28.654 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:47:28.675 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:47:28.676 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:47:28.677 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:47:28.677 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:47:28.677 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:47:28.677 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:47:28.677 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:47:28.677 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:47:28.677 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:47:28.696 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:28.714 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:28.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:28.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:28.818 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:28.854 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:28.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:28.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:28.957 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:28.981 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:28.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:28.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:29.085 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:29.116 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:29.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:29.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:29.219 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:29.255 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:29.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:29.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:29.287 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:29 smithi098 bash[79516]: cluster 2023-09-23T18:47:28.058383+0000 mgr.y (mgr.26054) 879 : cluster 0 pgmap v920: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:29.287 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:29 smithi098 bash[79516]: audit 2023-09-23T18:47:28.503481+0000 mon.a (mon.0) 1193 : audit 0 from='client.? 172.21.15.98:0/3516322459' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:47:29.287 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:29 smithi098 bash[76817]: cluster 2023-09-23T18:47:28.058383+0000 mgr.y (mgr.26054) 879 : cluster 0 pgmap v920: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:29.288 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:29 smithi098 bash[76817]: audit 2023-09-23T18:47:28.503481+0000 mon.a (mon.0) 1193 : audit 0 from='client.? 172.21.15.98:0/3516322459' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:47:29.360 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:29.386 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:29.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:29.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:29.489 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:29.502 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:29 smithi173 bash[59422]: cluster 2023-09-23T18:47:28.058383+0000 mgr.y (mgr.26054) 879 : cluster 0 pgmap v920: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:29.502 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:29 smithi173 bash[59422]: audit 2023-09-23T18:47:28.503481+0000 mon.a (mon.0) 1193 : audit 0 from='client.? 172.21.15.98:0/3516322459' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:47:29.516 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:29.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:29.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:29.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:29.652 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:29.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:29.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:29.756 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:29.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:29.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:29.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:29.890 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:29.917 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:29.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:29.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:30.021 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:30.046 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:30.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:30.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:30.149 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:30.178 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:30.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:30.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:30.281 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:30.307 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:30.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:30.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:30.410 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:30.439 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:30.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:30.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:30.542 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:30.567 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:30.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:30.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:30.671 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:30.705 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:30.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:30.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:30.809 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:30.831 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:30.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:30.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:30.934 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:30.962 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:30.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:30.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:31.065 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:31.090 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:31.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:31.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:31.193 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:31.223 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:31.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:31.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:31.326 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:31.337 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:31 smithi098 bash[79516]: cluster 2023-09-23T18:47:30.058883+0000 mgr.y (mgr.26054) 880 : cluster 0 pgmap v921: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:31.338 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:31 smithi098 bash[79516]: audit 2023-09-23T18:47:30.221866+0000 mon.a (mon.0) 1194 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:47:31.338 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:31 smithi098 bash[76817]: cluster 2023-09-23T18:47:30.058883+0000 mgr.y (mgr.26054) 880 : cluster 0 pgmap v921: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:31.338 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:31 smithi098 bash[76817]: audit 2023-09-23T18:47:30.221866+0000 mon.a (mon.0) 1194 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:47:31.351 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:31.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:31.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:31.455 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:31.477 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:31.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:31.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:31.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:31 smithi173 bash[59422]: cluster 2023-09-23T18:47:30.058883+0000 mgr.y (mgr.26054) 880 : cluster 0 pgmap v921: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:31.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:31 smithi173 bash[59422]: audit 2023-09-23T18:47:30.221866+0000 mon.a (mon.0) 1194 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:47:31.581 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:31.615 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:31.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:31.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:31.630 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T18:47:31.718 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:31.751 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:31.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:31.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:31.855 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:31.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:31.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:31.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:31.990 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:32.021 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:32.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:32.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:32.125 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:32.152 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:32.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:32.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:32.256 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:32.285 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:32.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:32.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:32.387 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:32.408 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:32.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:32.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:32.511 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:32.542 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:32.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:32.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:32.646 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:32.672 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:32.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:32.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:32.776 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:32.805 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:32.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:32.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:32.908 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:32.943 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:32.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:32.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:33.046 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:33.069 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:33.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:33.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:33.172 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:33.203 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:33.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:33.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:33.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:33 smithi173 bash[59422]: cluster 2023-09-23T18:47:32.059771+0000 mgr.y (mgr.26054) 881 : cluster 0 pgmap v922: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:33.307 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:33.340 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:33.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:33.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:33.445 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:33.477 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:33.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:33.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:33.581 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:33.615 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:33.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:33.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:33.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:33 smithi098 bash[79516]: cluster 2023-09-23T18:47:32.059771+0000 mgr.y (mgr.26054) 881 : cluster 0 pgmap v922: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:33.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:33 smithi098 bash[76817]: cluster 2023-09-23T18:47:32.059771+0000 mgr.y (mgr.26054) 881 : cluster 0 pgmap v922: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:33.718 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:33.753 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:33.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:33.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:33.857 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:33.891 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:33.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:33.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:33.995 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:34.021 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:34.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:34.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:34.125 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:34.152 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:34.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:34.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:34.256 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:34.268 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:34 smithi098 bash[79516]: cluster 2023-09-23T18:47:34.060646+0000 mgr.y (mgr.26054) 882 : cluster 0 pgmap v923: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:34.268 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:34 smithi098 bash[76817]: cluster 2023-09-23T18:47:34.060646+0000 mgr.y (mgr.26054) 882 : cluster 0 pgmap v923: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:34.283 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:34.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:34.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:34.387 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:34.418 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:34.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:34.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:34.521 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:34.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:34 smithi173 bash[59422]: cluster 2023-09-23T18:47:34.060646+0000 mgr.y (mgr.26054) 882 : cluster 0 pgmap v923: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:34.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:34.577 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:34.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:34.679 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:34.709 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:34.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:34.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:34.813 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:34.815 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:47:34.849 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:34.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:34.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:34.953 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:34.980 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:34.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:34.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:35.084 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:35.115 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:35.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:35.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:35.219 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:35.254 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:35.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:35.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:35.357 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:35.369 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:47:35.392 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:35.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:35.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:35.496 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:35.530 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:35.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:35.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:35.633 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:35.643 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:35 smithi098 bash[76817]: debug 2023-09-23T18:47:35.248+0000 7fc8a2a01700 -1 mon.a@0(leader).osd e227 definitely_dead 0 2023-09-23T18:47:35.661 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:35.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:35.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:35.764 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:35.797 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:35.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:35.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:35.901 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:35.934 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:35.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:35.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:36.038 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:36.069 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:36.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:36.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:36.128 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is unset 2023-09-23T18:47:36.130 INFO:teuthology.orchestra.run.smithi098.stderr:marked in osd.5. 2023-09-23T18:47:36.173 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:36.206 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:36.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:36.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:36.310 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:36.323 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:36 smithi173 bash[59422]: audit 2023-09-23T18:47:35.241110+0000 mon.a (mon.0) 1195 : audit 1 from='client.? 172.21.15.98:0/575294170' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:47:36.323 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:36 smithi173 bash[59422]: audit 2023-09-23T18:47:35.249631+0000 mon.c (mon.2) 102 : audit 0 from='client.? 172.21.15.98:0/543004928' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T18:47:36.323 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:36 smithi173 bash[59422]: audit 2023-09-23T18:47:35.250322+0000 mon.a (mon.0) 1196 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T18:47:36.342 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:36.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:36.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:36.441 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:47:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:47:36] "GET /metrics HTTP/1.1" 200 34211 "" "Prometheus/2.33.4" 2023-09-23T18:47:36.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:36 smithi098 bash[76817]: audit 2023-09-23T18:47:35.241110+0000 mon.a (mon.0) 1195 : audit 1 from='client.? 172.21.15.98:0/575294170' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:47:36.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:36 smithi098 bash[76817]: audit 2023-09-23T18:47:35.249631+0000 mon.c (mon.2) 102 : audit 0 from='client.? 172.21.15.98:0/543004928' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T18:47:36.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:36 smithi098 bash[76817]: audit 2023-09-23T18:47:35.250322+0000 mon.a (mon.0) 1196 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T18:47:36.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:36 smithi098 bash[79516]: audit 2023-09-23T18:47:35.241110+0000 mon.a (mon.0) 1195 : audit 1 from='client.? 172.21.15.98:0/575294170' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:47:36.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:36 smithi098 bash[79516]: audit 2023-09-23T18:47:35.249631+0000 mon.c (mon.2) 102 : audit 0 from='client.? 172.21.15.98:0/543004928' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T18:47:36.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:36 smithi098 bash[79516]: audit 2023-09-23T18:47:35.250322+0000 mon.a (mon.0) 1196 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T18:47:36.445 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:36.473 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:36.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:36.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:36.577 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:36.610 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:36.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:36.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:36.714 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:36.744 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:36.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:36.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:36.848 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:36.877 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:36.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:36.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:36.980 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:37.005 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:37.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:37.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:37.108 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:37.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:37 smithi173 bash[59422]: cluster 2023-09-23T18:47:36.061545+0000 mgr.y (mgr.26054) 883 : cluster 0 pgmap v924: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:37.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:37 smithi173 bash[59422]: cluster 2023-09-23T18:47:36.101508+0000 mon.a (mon.0) 1197 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:47:37.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:37 smithi173 bash[59422]: cluster 2023-09-23T18:47:36.101558+0000 mon.a (mon.0) 1198 : cluster 1 Cluster is now healthy 2023-09-23T18:47:37.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:37 smithi173 bash[59422]: cluster 2023-09-23T18:47:36.112806+0000 osd.5 (osd.5) 117 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T18:47:37.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:37 smithi173 bash[59422]: cluster 2023-09-23T18:47:36.113713+0000 osd.4 (osd.4) 216 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T18:47:37.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:37 smithi173 bash[59422]: cluster 2023-09-23T18:47:36.116228+0000 osd.5 (osd.5) 118 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T18:47:37.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:37 smithi173 bash[59422]: audit 2023-09-23T18:47:36.116443+0000 mon.a (mon.0) 1199 : audit 1 from='client.? 172.21.15.98:0/575294170' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:47:37.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:37 smithi173 bash[59422]: audit 2023-09-23T18:47:36.116643+0000 mon.a (mon.0) 1200 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T18:47:37.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:37 smithi173 bash[59422]: cluster 2023-09-23T18:47:36.116817+0000 mon.a (mon.0) 1201 : cluster 0 osdmap e228: 8 total, 8 up, 8 in 2023-09-23T18:47:37.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:37 smithi173 bash[59422]: cluster 2023-09-23T18:47:36.117813+0000 osd.4 (osd.4) 217 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T18:47:37.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:37 smithi173 bash[59422]: cluster 2023-09-23T18:47:36.118457+0000 osd.6 (osd.6) 182 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T18:47:37.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:37 smithi173 bash[59422]: cluster 2023-09-23T18:47:36.120212+0000 osd.7 (osd.7) 156 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T18:47:37.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:37 smithi173 bash[59422]: cluster 2023-09-23T18:47:36.122302+0000 osd.6 (osd.6) 183 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T18:47:37.143 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:37.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:37.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:37.246 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:37.276 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:37.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:37.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:37.379 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:37.401 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:37.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:37.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:37.437 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[79516]: cluster 2023-09-23T18:47:36.061545+0000 mgr.y (mgr.26054) 883 : cluster 0 pgmap v924: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:37.437 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[79516]: cluster 2023-09-23T18:47:36.101508+0000 mon.a (mon.0) 1197 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:47:37.438 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[79516]: cluster 2023-09-23T18:47:36.101558+0000 mon.a (mon.0) 1198 : cluster 1 Cluster is now healthy 2023-09-23T18:47:37.438 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[79516]: cluster 2023-09-23T18:47:36.112806+0000 osd.5 (osd.5) 117 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T18:47:37.438 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[79516]: cluster 2023-09-23T18:47:36.113713+0000 osd.4 (osd.4) 216 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T18:47:37.438 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[79516]: cluster 2023-09-23T18:47:36.116228+0000 osd.5 (osd.5) 118 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T18:47:37.438 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[79516]: audit 2023-09-23T18:47:36.116443+0000 mon.a (mon.0) 1199 : audit 1 from='client.? 172.21.15.98:0/575294170' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:47:37.438 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[79516]: audit 2023-09-23T18:47:36.116643+0000 mon.a (mon.0) 1200 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T18:47:37.438 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[79516]: cluster 2023-09-23T18:47:36.116817+0000 mon.a (mon.0) 1201 : cluster 0 osdmap e228: 8 total, 8 up, 8 in 2023-09-23T18:47:37.438 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[79516]: cluster 2023-09-23T18:47:36.117813+0000 osd.4 (osd.4) 217 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T18:47:37.438 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[79516]: cluster 2023-09-23T18:47:36.118457+0000 osd.6 (osd.6) 182 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T18:47:37.438 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[79516]: cluster 2023-09-23T18:47:36.120212+0000 osd.7 (osd.7) 156 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T18:47:37.438 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[79516]: cluster 2023-09-23T18:47:36.122302+0000 osd.6 (osd.6) 183 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T18:47:37.439 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[76817]: cluster 2023-09-23T18:47:36.061545+0000 mgr.y (mgr.26054) 883 : cluster 0 pgmap v924: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:37.439 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[76817]: cluster 2023-09-23T18:47:36.101508+0000 mon.a (mon.0) 1197 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:47:37.439 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[76817]: cluster 2023-09-23T18:47:36.101558+0000 mon.a (mon.0) 1198 : cluster 1 Cluster is now healthy 2023-09-23T18:47:37.439 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[76817]: cluster 2023-09-23T18:47:36.112806+0000 osd.5 (osd.5) 117 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T18:47:37.439 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[76817]: cluster 2023-09-23T18:47:36.113713+0000 osd.4 (osd.4) 216 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T18:47:37.439 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[76817]: cluster 2023-09-23T18:47:36.116228+0000 osd.5 (osd.5) 118 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T18:47:37.439 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[76817]: audit 2023-09-23T18:47:36.116443+0000 mon.a (mon.0) 1199 : audit 1 from='client.? 172.21.15.98:0/575294170' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:47:37.439 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[76817]: audit 2023-09-23T18:47:36.116643+0000 mon.a (mon.0) 1200 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T18:47:37.439 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[76817]: cluster 2023-09-23T18:47:36.116817+0000 mon.a (mon.0) 1201 : cluster 0 osdmap e228: 8 total, 8 up, 8 in 2023-09-23T18:47:37.439 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[76817]: cluster 2023-09-23T18:47:36.117813+0000 osd.4 (osd.4) 217 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T18:47:37.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[76817]: cluster 2023-09-23T18:47:36.118457+0000 osd.6 (osd.6) 182 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T18:47:37.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[76817]: cluster 2023-09-23T18:47:36.120212+0000 osd.7 (osd.7) 156 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T18:47:37.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:37 smithi098 bash[76817]: cluster 2023-09-23T18:47:36.122302+0000 osd.6 (osd.6) 183 : cluster 3 failed to encode map e228 with expected crc 2023-09-23T18:47:37.504 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:37.532 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:37.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:37.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:37.637 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:37.668 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:37.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:37.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:37.771 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:37.800 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:37.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:37.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:37.904 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:37.937 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:37.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:37.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:38.041 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:38.066 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:38.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:38.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:38.169 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:38.190 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:38.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:38.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:38.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:38 smithi173 bash[59422]: cluster 2023-09-23T18:47:37.116189+0000 osd.7 (osd.7) 157 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:38 smithi173 bash[59422]: cluster 2023-09-23T18:47:37.116536+0000 osd.4 (osd.4) 218 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:38 smithi173 bash[59422]: cluster 2023-09-23T18:47:37.117131+0000 mon.a (mon.0) 1202 : cluster 0 osdmap e229: 8 total, 8 up, 8 in 2023-09-23T18:47:38.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:38 smithi173 bash[59422]: cluster 2023-09-23T18:47:37.118678+0000 osd.5 (osd.5) 119 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:38 smithi173 bash[59422]: cluster 2023-09-23T18:47:37.119023+0000 osd.7 (osd.7) 158 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:38 smithi173 bash[59422]: cluster 2023-09-23T18:47:37.119758+0000 osd.6 (osd.6) 184 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:38 smithi173 bash[59422]: cluster 2023-09-23T18:47:37.121984+0000 osd.6 (osd.6) 185 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:38 smithi173 bash[59422]: cluster 2023-09-23T18:47:37.123096+0000 osd.5 (osd.5) 120 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:38 smithi173 bash[59422]: cluster 2023-09-23T18:47:37.123427+0000 osd.7 (osd.7) 159 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:38 smithi173 bash[59422]: cluster 2023-09-23T18:47:37.124451+0000 osd.6 (osd.6) 186 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:38 smithi173 bash[59422]: cluster 2023-09-23T18:47:37.126878+0000 osd.7 (osd.7) 160 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.293 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:38.322 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:38.324 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:38 smithi098 bash[79516]: cluster 2023-09-23T18:47:37.116189+0000 osd.7 (osd.7) 157 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.324 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:38 smithi098 bash[79516]: cluster 2023-09-23T18:47:37.116536+0000 osd.4 (osd.4) 218 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.324 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:38 smithi098 bash[79516]: cluster 2023-09-23T18:47:37.117131+0000 mon.a (mon.0) 1202 : cluster 0 osdmap e229: 8 total, 8 up, 8 in 2023-09-23T18:47:38.324 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:38 smithi098 bash[79516]: cluster 2023-09-23T18:47:37.118678+0000 osd.5 (osd.5) 119 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.324 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:38 smithi098 bash[79516]: cluster 2023-09-23T18:47:37.119023+0000 osd.7 (osd.7) 158 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.325 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:38 smithi098 bash[79516]: cluster 2023-09-23T18:47:37.119758+0000 osd.6 (osd.6) 184 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.325 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:38 smithi098 bash[79516]: cluster 2023-09-23T18:47:37.121984+0000 osd.6 (osd.6) 185 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.325 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:38 smithi098 bash[79516]: cluster 2023-09-23T18:47:37.123096+0000 osd.5 (osd.5) 120 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.325 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:38 smithi098 bash[79516]: cluster 2023-09-23T18:47:37.123427+0000 osd.7 (osd.7) 159 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.325 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:38 smithi098 bash[79516]: cluster 2023-09-23T18:47:37.124451+0000 osd.6 (osd.6) 186 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.325 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:38 smithi098 bash[79516]: cluster 2023-09-23T18:47:37.126878+0000 osd.7 (osd.7) 160 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.325 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:38 smithi098 bash[76817]: cluster 2023-09-23T18:47:37.116189+0000 osd.7 (osd.7) 157 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.325 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:38 smithi098 bash[76817]: cluster 2023-09-23T18:47:37.116536+0000 osd.4 (osd.4) 218 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.326 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:38 smithi098 bash[76817]: cluster 2023-09-23T18:47:37.117131+0000 mon.a (mon.0) 1202 : cluster 0 osdmap e229: 8 total, 8 up, 8 in 2023-09-23T18:47:38.326 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:38 smithi098 bash[76817]: cluster 2023-09-23T18:47:37.118678+0000 osd.5 (osd.5) 119 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.326 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:38 smithi098 bash[76817]: cluster 2023-09-23T18:47:37.119023+0000 osd.7 (osd.7) 158 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.326 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:38 smithi098 bash[76817]: cluster 2023-09-23T18:47:37.119758+0000 osd.6 (osd.6) 184 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.326 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:38 smithi098 bash[76817]: cluster 2023-09-23T18:47:37.121984+0000 osd.6 (osd.6) 185 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.326 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:38 smithi098 bash[76817]: cluster 2023-09-23T18:47:37.123096+0000 osd.5 (osd.5) 120 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.326 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:38 smithi098 bash[76817]: cluster 2023-09-23T18:47:37.123427+0000 osd.7 (osd.7) 159 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.326 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:38 smithi098 bash[76817]: cluster 2023-09-23T18:47:37.124451+0000 osd.6 (osd.6) 186 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.326 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:38 smithi098 bash[76817]: cluster 2023-09-23T18:47:37.126878+0000 osd.7 (osd.7) 160 : cluster 3 failed to encode map e229 with expected crc 2023-09-23T18:47:38.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:38.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:38.428 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:38.460 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:38.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:38.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:38.564 INFO:tasks.thrashosds.thrasher:Added osd 5 2023-09-23T18:47:38.564 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:38.592 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:38.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:38.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:38.696 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:38.728 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:38.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:38.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:38.832 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:38.864 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:38.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:38.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:38.968 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:38.998 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:39.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:39.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:39.103 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:39.139 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:39.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:39.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:39.244 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:39.270 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:39.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:39.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:39.317 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:47:39.374 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:39.397 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:39.400 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:39 smithi098 bash[76817]: cluster 2023-09-23T18:47:38.062068+0000 mgr.y (mgr.26054) 884 : cluster 0 pgmap v927: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:39.400 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:39 smithi098 bash[76817]: cluster 2023-09-23T18:47:38.130106+0000 osd.4 (osd.4) 219 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T18:47:39.400 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:39 smithi098 bash[76817]: cluster 2023-09-23T18:47:38.133516+0000 mon.a (mon.0) 1203 : cluster 0 osdmap e230: 8 total, 8 up, 8 in 2023-09-23T18:47:39.400 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:39 smithi098 bash[76817]: cluster 2023-09-23T18:47:38.133624+0000 osd.7 (osd.7) 161 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T18:47:39.400 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:39 smithi098 bash[76817]: cluster 2023-09-23T18:47:38.303244+0000 osd.6 (osd.6) 187 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T18:47:39.400 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:39 smithi098 bash[76817]: cluster 2023-09-23T18:47:38.303260+0000 osd.5 (osd.5) 121 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T18:47:39.401 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:39 smithi098 bash[79516]: cluster 2023-09-23T18:47:38.062068+0000 mgr.y (mgr.26054) 884 : cluster 0 pgmap v927: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:39.401 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:39 smithi098 bash[79516]: cluster 2023-09-23T18:47:38.130106+0000 osd.4 (osd.4) 219 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T18:47:39.401 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:39 smithi098 bash[79516]: cluster 2023-09-23T18:47:38.133516+0000 mon.a (mon.0) 1203 : cluster 0 osdmap e230: 8 total, 8 up, 8 in 2023-09-23T18:47:39.401 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:39 smithi098 bash[79516]: cluster 2023-09-23T18:47:38.133624+0000 osd.7 (osd.7) 161 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T18:47:39.401 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:39 smithi098 bash[79516]: cluster 2023-09-23T18:47:38.303244+0000 osd.6 (osd.6) 187 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T18:47:39.401 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:39 smithi098 bash[79516]: cluster 2023-09-23T18:47:38.303260+0000 osd.5 (osd.5) 121 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T18:47:39.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:39.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:39.503 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:39.514 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:39 smithi173 bash[59422]: cluster 2023-09-23T18:47:38.062068+0000 mgr.y (mgr.26054) 884 : cluster 0 pgmap v927: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:47:39.514 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:39 smithi173 bash[59422]: cluster 2023-09-23T18:47:38.130106+0000 osd.4 (osd.4) 219 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T18:47:39.515 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:39 smithi173 bash[59422]: cluster 2023-09-23T18:47:38.133516+0000 mon.a (mon.0) 1203 : cluster 0 osdmap e230: 8 total, 8 up, 8 in 2023-09-23T18:47:39.515 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:39 smithi173 bash[59422]: cluster 2023-09-23T18:47:38.133624+0000 osd.7 (osd.7) 161 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T18:47:39.515 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:39 smithi173 bash[59422]: cluster 2023-09-23T18:47:38.303244+0000 osd.6 (osd.6) 187 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T18:47:39.515 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:39 smithi173 bash[59422]: cluster 2023-09-23T18:47:38.303260+0000 osd.5 (osd.5) 121 : cluster 3 failed to encode map e230 with expected crc 2023-09-23T18:47:39.532 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:39.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:39.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:39.637 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:39.670 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:39.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:39.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:39.775 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:39.812 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:39.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:39.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:39.916 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:39.955 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:39.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:39.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:40.058 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:40.093 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:40.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:40.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:40.197 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:40.233 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:40.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:40.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:40.336 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:40.350 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:40 smithi173 bash[59422]: cluster 2023-09-23T18:47:39.129553+0000 osd.7 (osd.7) 162 : cluster 3 failed to encode map e231 with expected crc 2023-09-23T18:47:40.350 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:40 smithi173 bash[59422]: cluster 2023-09-23T18:47:39.129955+0000 osd.4 (osd.4) 220 : cluster 3 failed to encode map e231 with expected crc 2023-09-23T18:47:40.350 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:40 smithi173 bash[59422]: cluster 2023-09-23T18:47:39.132559+0000 mon.a (mon.0) 1204 : cluster 0 osdmap e231: 8 total, 8 up, 8 in 2023-09-23T18:47:40.351 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:40 smithi173 bash[59422]: cluster 2023-09-23T18:47:39.137421+0000 osd.5 (osd.5) 122 : cluster 3 failed to encode map e231 with expected crc 2023-09-23T18:47:40.351 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:40 smithi173 bash[59422]: audit 2023-09-23T18:47:39.619164+0000 mon.a (mon.0) 1205 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:47:40.351 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:40 smithi173 bash[59422]: audit 2023-09-23T18:47:39.620529+0000 mon.a (mon.0) 1206 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:47:40.351 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:40 smithi173 bash[59422]: cluster 2023-09-23T18:47:40.062648+0000 mgr.y (mgr.26054) 885 : cluster 0 pgmap v930: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 138 B/s, 0 objects/s recovering 2023-09-23T18:47:40.368 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:40.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:40.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:40.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:40 smithi098 bash[76817]: cluster 2023-09-23T18:47:39.129553+0000 osd.7 (osd.7) 162 : cluster 3 failed to encode map e231 with expected crc 2023-09-23T18:47:40.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:40 smithi098 bash[76817]: cluster 2023-09-23T18:47:39.129955+0000 osd.4 (osd.4) 220 : cluster 3 failed to encode map e231 with expected crc 2023-09-23T18:47:40.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:40 smithi098 bash[76817]: cluster 2023-09-23T18:47:39.132559+0000 mon.a (mon.0) 1204 : cluster 0 osdmap e231: 8 total, 8 up, 8 in 2023-09-23T18:47:40.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:40 smithi098 bash[76817]: cluster 2023-09-23T18:47:39.137421+0000 osd.5 (osd.5) 122 : cluster 3 failed to encode map e231 with expected crc 2023-09-23T18:47:40.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:40 smithi098 bash[76817]: audit 2023-09-23T18:47:39.619164+0000 mon.a (mon.0) 1205 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:47:40.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:40 smithi098 bash[76817]: audit 2023-09-23T18:47:39.620529+0000 mon.a (mon.0) 1206 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:47:40.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:40 smithi098 bash[76817]: cluster 2023-09-23T18:47:40.062648+0000 mgr.y (mgr.26054) 885 : cluster 0 pgmap v930: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 138 B/s, 0 objects/s recovering 2023-09-23T18:47:40.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:40 smithi098 bash[79516]: cluster 2023-09-23T18:47:39.129553+0000 osd.7 (osd.7) 162 : cluster 3 failed to encode map e231 with expected crc 2023-09-23T18:47:40.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:40 smithi098 bash[79516]: cluster 2023-09-23T18:47:39.129955+0000 osd.4 (osd.4) 220 : cluster 3 failed to encode map e231 with expected crc 2023-09-23T18:47:40.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:40 smithi098 bash[79516]: cluster 2023-09-23T18:47:39.132559+0000 mon.a (mon.0) 1204 : cluster 0 osdmap e231: 8 total, 8 up, 8 in 2023-09-23T18:47:40.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:40 smithi098 bash[79516]: cluster 2023-09-23T18:47:39.137421+0000 osd.5 (osd.5) 122 : cluster 3 failed to encode map e231 with expected crc 2023-09-23T18:47:40.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:40 smithi098 bash[79516]: audit 2023-09-23T18:47:39.619164+0000 mon.a (mon.0) 1205 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:47:40.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:40 smithi098 bash[79516]: audit 2023-09-23T18:47:39.620529+0000 mon.a (mon.0) 1206 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:47:40.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:40 smithi098 bash[79516]: cluster 2023-09-23T18:47:40.062648+0000 mgr.y (mgr.26054) 885 : cluster 0 pgmap v930: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 138 B/s, 0 objects/s recovering 2023-09-23T18:47:40.472 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:40.501 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:40.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:40.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:40.604 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:40.632 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:40.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:40.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:40.735 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:40.760 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:40.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:40.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:40.863 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:40.897 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:40.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:40.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:40.934 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:47:41.001 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:41.034 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:41.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:41.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:41.138 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:41.168 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:41.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:41.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:41.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:41 smithi098 bash[79516]: cluster 2023-09-23T18:47:40.138248+0000 mon.a (mon.0) 1207 : cluster 0 osdmap e232: 8 total, 8 up, 8 in 2023-09-23T18:47:41.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:41 smithi098 bash[79516]: cluster 2023-09-23T18:47:40.139986+0000 osd.5 (osd.5) 123 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T18:47:41.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:41 smithi098 bash[79516]: cluster 2023-09-23T18:47:40.141710+0000 osd.4 (osd.4) 221 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T18:47:41.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:41 smithi098 bash[79516]: cluster 2023-09-23T18:47:40.143402+0000 osd.5 (osd.5) 124 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T18:47:41.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:41 smithi098 bash[79516]: cluster 2023-09-23T18:47:40.276598+0000 osd.6 (osd.6) 188 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T18:47:41.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:41 smithi098 bash[79516]: cluster 2023-09-23T18:47:40.279194+0000 osd.6 (osd.6) 189 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T18:47:41.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:41 smithi098 bash[76817]: cluster 2023-09-23T18:47:40.138248+0000 mon.a (mon.0) 1207 : cluster 0 osdmap e232: 8 total, 8 up, 8 in 2023-09-23T18:47:41.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:41 smithi098 bash[76817]: cluster 2023-09-23T18:47:40.139986+0000 osd.5 (osd.5) 123 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T18:47:41.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:41 smithi098 bash[76817]: cluster 2023-09-23T18:47:40.141710+0000 osd.4 (osd.4) 221 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T18:47:41.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:41 smithi098 bash[76817]: cluster 2023-09-23T18:47:40.143402+0000 osd.5 (osd.5) 124 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T18:47:41.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:41 smithi098 bash[76817]: cluster 2023-09-23T18:47:40.276598+0000 osd.6 (osd.6) 188 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T18:47:41.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:41 smithi098 bash[76817]: cluster 2023-09-23T18:47:40.279194+0000 osd.6 (osd.6) 189 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T18:47:41.272 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:41.305 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:41.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:41.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:41.409 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:41.420 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:41 smithi173 bash[59422]: cluster 2023-09-23T18:47:40.138248+0000 mon.a (mon.0) 1207 : cluster 0 osdmap e232: 8 total, 8 up, 8 in 2023-09-23T18:47:41.421 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:41 smithi173 bash[59422]: cluster 2023-09-23T18:47:40.139986+0000 osd.5 (osd.5) 123 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T18:47:41.421 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:41 smithi173 bash[59422]: cluster 2023-09-23T18:47:40.141710+0000 osd.4 (osd.4) 221 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T18:47:41.421 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:41 smithi173 bash[59422]: cluster 2023-09-23T18:47:40.143402+0000 osd.5 (osd.5) 124 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T18:47:41.421 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:41 smithi173 bash[59422]: cluster 2023-09-23T18:47:40.276598+0000 osd.6 (osd.6) 188 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T18:47:41.421 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:41 smithi173 bash[59422]: cluster 2023-09-23T18:47:40.279194+0000 osd.6 (osd.6) 189 : cluster 3 failed to encode map e232 with expected crc 2023-09-23T18:47:41.427 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:47:41.441 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:41.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:41.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:41.545 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:41.572 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:41.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:41.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:41.675 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:41.711 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:41.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:41.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:41.814 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:41.845 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:41.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:41.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:41.948 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:41.980 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:41.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:41.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:42.084 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:42.107 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:42.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:42.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:42.210 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:42.221 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:42 smithi098 bash[79516]: cluster 2023-09-23T18:47:42.063561+0000 mgr.y (mgr.26054) 886 : cluster 0 pgmap v932: 73 pgs: 73 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 828 KiB/s, 5 objects/s recovering 2023-09-23T18:47:42.221 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:42 smithi098 bash[76817]: cluster 2023-09-23T18:47:42.063561+0000 mgr.y (mgr.26054) 886 : cluster 0 pgmap v932: 73 pgs: 73 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 828 KiB/s, 5 objects/s recovering 2023-09-23T18:47:42.234 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:42.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:42.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:42.338 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:42.365 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:42.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:42.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:42.468 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:42.498 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:42.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:42.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:42.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:42 smithi173 bash[59422]: cluster 2023-09-23T18:47:42.063561+0000 mgr.y (mgr.26054) 886 : cluster 0 pgmap v932: 73 pgs: 73 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 828 KiB/s, 5 objects/s recovering 2023-09-23T18:47:42.603 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:42.637 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:42.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:42.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:42.741 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:42.772 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:42.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:42.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:42.875 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:42.905 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:42.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:42.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:43.009 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:43.033 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:43.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:43.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:43.137 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:43.166 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:43.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:43.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:43.270 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:43.299 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:43.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:43.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:43.403 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:43.428 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:43.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:43.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:43.531 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:43.556 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:43.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:43.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:43.565 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T18:47:43.565 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:47:43.565 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T18:47:43.565 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 0.000000 2023-09-23T18:47:43.566 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 6 0 2023-09-23T18:47:43.659 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:43.682 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:43.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:43.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:43.785 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:43.817 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:43.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:43.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:43.921 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:43.949 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:43.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:43.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:44.054 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:44.085 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:44.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:44.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:44.189 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:44.217 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:44.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:44.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:44.321 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:44.354 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:44.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:44.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:44.458 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:44.488 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:44.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:44.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:44.591 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:44.623 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:44.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:44.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:44.726 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:44.758 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:44.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:44.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:44.861 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:44.892 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:44.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:44.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:44.996 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:45.028 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:45.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:45.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:45.132 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:45.156 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:45.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:45.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:45.259 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:45.271 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:45 smithi173 bash[59422]: cluster 2023-09-23T18:47:44.064089+0000 mgr.y (mgr.26054) 887 : cluster 0 pgmap v933: 73 pgs: 73 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 683 KiB/s, 4 objects/s recovering 2023-09-23T18:47:45.292 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:45.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:45.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:45.396 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:45.406 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:45 smithi098 bash[76817]: cluster 2023-09-23T18:47:44.064089+0000 mgr.y (mgr.26054) 887 : cluster 0 pgmap v933: 73 pgs: 73 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 683 KiB/s, 4 objects/s recovering 2023-09-23T18:47:45.407 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:45 smithi098 bash[79516]: cluster 2023-09-23T18:47:44.064089+0000 mgr.y (mgr.26054) 887 : cluster 0 pgmap v933: 73 pgs: 73 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 683 KiB/s, 4 objects/s recovering 2023-09-23T18:47:45.420 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:45.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:45.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:45.494 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:47:45.524 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:45.557 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:45.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:45.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:45.661 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:45.689 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:45.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:45.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:45.793 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:45.826 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:45.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:45.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:45.929 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:45.961 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:45.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:45.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:46.064 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:46.097 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:46.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:46.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:46.200 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:46.207 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:47:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:47:46] "GET /metrics HTTP/1.1" 200 34214 "" "Prometheus/2.33.4" 2023-09-23T18:47:46.226 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:46.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:46.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:46.329 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:46.360 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:46.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:46.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:46.463 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:46.495 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:46.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:46.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:46.599 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:46.621 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:46.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:46.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:46.724 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:46.755 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:46.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:46.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:46.858 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:46.885 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:46.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:46.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:46.988 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:47.021 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:47.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:47.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:47.125 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:47.155 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:47.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:47.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:47.258 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:47.269 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:47 smithi173 bash[59422]: cluster 2023-09-23T18:47:46.064799+0000 mgr.y (mgr.26054) 888 : cluster 0 pgmap v934: 73 pgs: 73 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 516 KiB/s, 3 objects/s recovering 2023-09-23T18:47:47.285 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:47.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:47.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:47.389 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:47.421 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:47.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:47.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:47.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:47 smithi098 bash[79516]: cluster 2023-09-23T18:47:46.064799+0000 mgr.y (mgr.26054) 888 : cluster 0 pgmap v934: 73 pgs: 73 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 516 KiB/s, 3 objects/s recovering 2023-09-23T18:47:47.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:47 smithi098 bash[76817]: cluster 2023-09-23T18:47:46.064799+0000 mgr.y (mgr.26054) 888 : cluster 0 pgmap v934: 73 pgs: 73 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 516 KiB/s, 3 objects/s recovering 2023-09-23T18:47:47.514 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:47:47.525 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:47.543 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:47.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:47.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:47.646 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:47.675 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:47.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:47.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:47.779 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:47.803 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:47.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:47.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:47.906 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:47.933 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:47.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:47.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:48.036 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:48.059 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:48.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:48.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:48.146 INFO:teuthology.orchestra.run.smithi098.stderr:pool 'unique_pool_3' created 2023-09-23T18:47:48.163 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:48.185 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:48.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:48.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:48.288 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:48.319 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:48.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:48.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:48.414 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:48 smithi098 bash[76817]: audit 2023-09-23T18:47:47.317119+0000 mon.c (mon.2) 103 : audit 1 from='client.? 172.21.15.98:0/3881182025' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T18:47:48.414 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:48 smithi098 bash[76817]: audit 2023-09-23T18:47:47.317624+0000 mon.a (mon.0) 1208 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T18:47:48.415 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:48 smithi098 bash[79516]: audit 2023-09-23T18:47:47.317119+0000 mon.c (mon.2) 103 : audit 1 from='client.? 172.21.15.98:0/3881182025' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T18:47:48.415 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:48 smithi098 bash[79516]: audit 2023-09-23T18:47:47.317624+0000 mon.a (mon.0) 1208 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T18:47:48.422 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:48.435 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:48 smithi173 bash[59422]: audit 2023-09-23T18:47:47.317119+0000 mon.c (mon.2) 103 : audit 1 from='client.? 172.21.15.98:0/3881182025' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T18:47:48.435 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:48 smithi173 bash[59422]: audit 2023-09-23T18:47:47.317624+0000 mon.a (mon.0) 1208 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T18:47:48.454 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:48.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:48.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:48.558 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:48.586 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:48.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:48.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:48.689 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:48.719 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:48.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:48.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:48.822 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:48.850 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:48.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:48.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:48.954 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:48.979 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:48.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:48.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:49.082 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:49.113 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:49.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:49.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:49.217 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:49.229 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:49 smithi098 bash[76817]: cluster 2023-09-23T18:47:48.065283+0000 mgr.y (mgr.26054) 889 : cluster 0 pgmap v935: 73 pgs: 73 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 458 KiB/s, 2 objects/s recovering 2023-09-23T18:47:49.229 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:49 smithi098 bash[76817]: cluster 2023-09-23T18:47:48.134047+0000 osd.5 (osd.5) 125 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T18:47:49.229 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:49 smithi098 bash[76817]: cluster 2023-09-23T18:47:48.134838+0000 osd.4 (osd.4) 222 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T18:47:49.230 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:49 smithi098 bash[76817]: audit 2023-09-23T18:47:48.136412+0000 mon.a (mon.0) 1209 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T18:47:49.230 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:49 smithi098 bash[76817]: cluster 2023-09-23T18:47:48.136505+0000 mon.a (mon.0) 1210 : cluster 0 osdmap e233: 8 total, 8 up, 8 in 2023-09-23T18:47:49.230 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:49 smithi098 bash[76817]: cluster 2023-09-23T18:47:48.136974+0000 osd.6 (osd.6) 190 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T18:47:49.230 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:49 smithi098 bash[76817]: cluster 2023-09-23T18:47:48.138050+0000 osd.5 (osd.5) 126 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T18:47:49.230 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:49 smithi098 bash[76817]: cluster 2023-09-23T18:47:48.139442+0000 osd.6 (osd.6) 191 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T18:47:49.230 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:49 smithi098 bash[76817]: cluster 2023-09-23T18:47:48.139697+0000 osd.7 (osd.7) 163 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T18:47:49.230 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:49 smithi098 bash[76817]: cluster 2023-09-23T18:47:48.140908+0000 osd.5 (osd.5) 127 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T18:47:49.230 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:49 smithi098 bash[79516]: cluster 2023-09-23T18:47:48.065283+0000 mgr.y (mgr.26054) 889 : cluster 0 pgmap v935: 73 pgs: 73 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 458 KiB/s, 2 objects/s recovering 2023-09-23T18:47:49.230 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:49 smithi098 bash[79516]: cluster 2023-09-23T18:47:48.134047+0000 osd.5 (osd.5) 125 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T18:47:49.230 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:49 smithi098 bash[79516]: cluster 2023-09-23T18:47:48.134838+0000 osd.4 (osd.4) 222 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T18:47:49.231 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:49 smithi098 bash[79516]: audit 2023-09-23T18:47:48.136412+0000 mon.a (mon.0) 1209 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T18:47:49.231 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:49 smithi098 bash[79516]: cluster 2023-09-23T18:47:48.136505+0000 mon.a (mon.0) 1210 : cluster 0 osdmap e233: 8 total, 8 up, 8 in 2023-09-23T18:47:49.231 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:49 smithi098 bash[79516]: cluster 2023-09-23T18:47:48.136974+0000 osd.6 (osd.6) 190 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T18:47:49.231 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:49 smithi098 bash[79516]: cluster 2023-09-23T18:47:48.138050+0000 osd.5 (osd.5) 126 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T18:47:49.231 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:49 smithi098 bash[79516]: cluster 2023-09-23T18:47:48.139442+0000 osd.6 (osd.6) 191 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T18:47:49.231 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:49 smithi098 bash[79516]: cluster 2023-09-23T18:47:48.139697+0000 osd.7 (osd.7) 163 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T18:47:49.231 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:49 smithi098 bash[79516]: cluster 2023-09-23T18:47:48.140908+0000 osd.5 (osd.5) 127 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T18:47:49.247 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:49.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:49.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:49.350 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:49.363 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:49 smithi173 bash[59422]: cluster 2023-09-23T18:47:48.065283+0000 mgr.y (mgr.26054) 889 : cluster 0 pgmap v935: 73 pgs: 73 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 458 KiB/s, 2 objects/s recovering 2023-09-23T18:47:49.363 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:49 smithi173 bash[59422]: cluster 2023-09-23T18:47:48.134047+0000 osd.5 (osd.5) 125 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T18:47:49.363 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:49 smithi173 bash[59422]: cluster 2023-09-23T18:47:48.134838+0000 osd.4 (osd.4) 222 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T18:47:49.363 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:49 smithi173 bash[59422]: audit 2023-09-23T18:47:48.136412+0000 mon.a (mon.0) 1209 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T18:47:49.364 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:49 smithi173 bash[59422]: cluster 2023-09-23T18:47:48.136505+0000 mon.a (mon.0) 1210 : cluster 0 osdmap e233: 8 total, 8 up, 8 in 2023-09-23T18:47:49.364 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:49 smithi173 bash[59422]: cluster 2023-09-23T18:47:48.136974+0000 osd.6 (osd.6) 190 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T18:47:49.364 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:49 smithi173 bash[59422]: cluster 2023-09-23T18:47:48.138050+0000 osd.5 (osd.5) 126 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T18:47:49.364 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:49 smithi173 bash[59422]: cluster 2023-09-23T18:47:48.139442+0000 osd.6 (osd.6) 191 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T18:47:49.364 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:49 smithi173 bash[59422]: cluster 2023-09-23T18:47:48.139697+0000 osd.7 (osd.7) 163 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T18:47:49.364 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:49 smithi173 bash[59422]: cluster 2023-09-23T18:47:48.140908+0000 osd.5 (osd.5) 127 : cluster 3 failed to encode map e233 with expected crc 2023-09-23T18:47:49.381 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:49.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:49.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:49.484 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:49.510 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:49.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:49.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:49.614 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:49.640 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:49.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:49.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:49.743 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:49.774 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:49.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:49.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:49.877 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:49.908 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:49.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:49.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:50.011 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:50.037 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:50.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:50.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:50.141 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:50.173 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:50.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:50.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:50.275 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:50.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:50 smithi173 bash[59422]: cluster 2023-09-23T18:47:49.137882+0000 osd.4 (osd.4) 223 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T18:47:50.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:50 smithi173 bash[59422]: cluster 2023-09-23T18:47:49.139012+0000 mon.a (mon.0) 1211 : cluster 0 osdmap e234: 8 total, 8 up, 8 in 2023-09-23T18:47:50.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:50 smithi173 bash[59422]: cluster 2023-09-23T18:47:49.141205+0000 osd.4 (osd.4) 224 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T18:47:50.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:50 smithi173 bash[59422]: cluster 2023-09-23T18:47:49.141353+0000 osd.6 (osd.6) 192 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T18:47:50.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:50 smithi173 bash[59422]: cluster 2023-09-23T18:47:49.142524+0000 osd.5 (osd.5) 128 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T18:47:50.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:50 smithi173 bash[59422]: cluster 2023-09-23T18:47:49.612412+0000 osd.7 (osd.7) 164 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T18:47:50.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:50 smithi173 bash[59422]: cluster 2023-09-23T18:47:50.065837+0000 mgr.y (mgr.26054) 890 : cluster 0 pgmap v938: 89 pgs: 7 creating+peering, 9 unknown, 73 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:47:50.294 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:50.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:50.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:50.397 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:50.425 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:50.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:50.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:50.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:50 smithi098 bash[79516]: cluster 2023-09-23T18:47:49.137882+0000 osd.4 (osd.4) 223 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T18:47:50.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:50 smithi098 bash[79516]: cluster 2023-09-23T18:47:49.139012+0000 mon.a (mon.0) 1211 : cluster 0 osdmap e234: 8 total, 8 up, 8 in 2023-09-23T18:47:50.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:50 smithi098 bash[79516]: cluster 2023-09-23T18:47:49.141205+0000 osd.4 (osd.4) 224 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T18:47:50.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:50 smithi098 bash[79516]: cluster 2023-09-23T18:47:49.141353+0000 osd.6 (osd.6) 192 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T18:47:50.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:50 smithi098 bash[79516]: cluster 2023-09-23T18:47:49.142524+0000 osd.5 (osd.5) 128 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T18:47:50.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:50 smithi098 bash[79516]: cluster 2023-09-23T18:47:49.612412+0000 osd.7 (osd.7) 164 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T18:47:50.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:50 smithi098 bash[79516]: cluster 2023-09-23T18:47:50.065837+0000 mgr.y (mgr.26054) 890 : cluster 0 pgmap v938: 89 pgs: 7 creating+peering, 9 unknown, 73 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:47:50.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:50 smithi098 bash[76817]: cluster 2023-09-23T18:47:49.137882+0000 osd.4 (osd.4) 223 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T18:47:50.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:50 smithi098 bash[76817]: cluster 2023-09-23T18:47:49.139012+0000 mon.a (mon.0) 1211 : cluster 0 osdmap e234: 8 total, 8 up, 8 in 2023-09-23T18:47:50.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:50 smithi098 bash[76817]: cluster 2023-09-23T18:47:49.141205+0000 osd.4 (osd.4) 224 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T18:47:50.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:50 smithi098 bash[76817]: cluster 2023-09-23T18:47:49.141353+0000 osd.6 (osd.6) 192 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T18:47:50.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:50 smithi098 bash[76817]: cluster 2023-09-23T18:47:49.142524+0000 osd.5 (osd.5) 128 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T18:47:50.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:50 smithi098 bash[76817]: cluster 2023-09-23T18:47:49.612412+0000 osd.7 (osd.7) 164 : cluster 3 failed to encode map e234 with expected crc 2023-09-23T18:47:50.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:50 smithi098 bash[76817]: cluster 2023-09-23T18:47:50.065837+0000 mgr.y (mgr.26054) 890 : cluster 0 pgmap v938: 89 pgs: 7 creating+peering, 9 unknown, 73 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:47:50.529 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:50.550 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:50.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:50.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:50.654 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:50.683 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:50.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:50.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:50.787 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:50.820 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:50.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:50.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:50.923 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:50.952 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:50.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:50.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:51.055 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:51.086 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:51.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:51.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:51.190 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:51.212 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:51.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:51.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:51.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:51 smithi173 bash[59422]: cluster 2023-09-23T18:47:50.152111+0000 osd.5 (osd.5) 129 : cluster 3 failed to encode map e235 with expected crc 2023-09-23T18:47:51.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:51 smithi173 bash[59422]: cluster 2023-09-23T18:47:50.152605+0000 osd.4 (osd.4) 225 : cluster 3 failed to encode map e235 with expected crc 2023-09-23T18:47:51.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:51 smithi173 bash[59422]: cluster 2023-09-23T18:47:50.154747+0000 mon.a (mon.0) 1212 : cluster 0 osdmap e235: 8 total, 8 up, 8 in 2023-09-23T18:47:51.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:51 smithi173 bash[59422]: audit 2023-09-23T18:47:50.238078+0000 mon.a (mon.0) 1213 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:47:51.315 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:51.349 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:51.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:51.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:51.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:51 smithi098 bash[76817]: cluster 2023-09-23T18:47:50.152111+0000 osd.5 (osd.5) 129 : cluster 3 failed to encode map e235 with expected crc 2023-09-23T18:47:51.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:51 smithi098 bash[76817]: cluster 2023-09-23T18:47:50.152605+0000 osd.4 (osd.4) 225 : cluster 3 failed to encode map e235 with expected crc 2023-09-23T18:47:51.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:51 smithi098 bash[76817]: cluster 2023-09-23T18:47:50.154747+0000 mon.a (mon.0) 1212 : cluster 0 osdmap e235: 8 total, 8 up, 8 in 2023-09-23T18:47:51.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:51 smithi098 bash[76817]: audit 2023-09-23T18:47:50.238078+0000 mon.a (mon.0) 1213 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:47:51.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:51 smithi098 bash[79516]: cluster 2023-09-23T18:47:50.152111+0000 osd.5 (osd.5) 129 : cluster 3 failed to encode map e235 with expected crc 2023-09-23T18:47:51.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:51 smithi098 bash[79516]: cluster 2023-09-23T18:47:50.152605+0000 osd.4 (osd.4) 225 : cluster 3 failed to encode map e235 with expected crc 2023-09-23T18:47:51.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:51 smithi098 bash[79516]: cluster 2023-09-23T18:47:50.154747+0000 mon.a (mon.0) 1212 : cluster 0 osdmap e235: 8 total, 8 up, 8 in 2023-09-23T18:47:51.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:51 smithi098 bash[79516]: audit 2023-09-23T18:47:50.238078+0000 mon.a (mon.0) 1213 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:47:51.454 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:51.488 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:51.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:51.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:51.592 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:51.614 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:51.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:51.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:51.702 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-09-23T18:47:51.718 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:51.766 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:51.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:51.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:51.869 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:51.902 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:51.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:51.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:52.006 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:52.034 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:52.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:52.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:52.137 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:52.167 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:52.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:52.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:52.271 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:52.293 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:52.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:52.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:52.396 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:52.409 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:52 smithi098 bash[79516]: cluster 2023-09-23T18:47:52.066510+0000 mgr.y (mgr.26054) 891 : cluster 0 pgmap v940: 89 pgs: 7 creating+peering, 82 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:47:52.409 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:52 smithi098 bash[76817]: cluster 2023-09-23T18:47:52.066510+0000 mgr.y (mgr.26054) 891 : cluster 0 pgmap v940: 89 pgs: 7 creating+peering, 82 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:47:52.423 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:52.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:52.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:52.527 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:52.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:52 smithi173 bash[59422]: cluster 2023-09-23T18:47:52.066510+0000 mgr.y (mgr.26054) 891 : cluster 0 pgmap v940: 89 pgs: 7 creating+peering, 82 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:47:52.548 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:52.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:52.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:52.651 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:52.679 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:52.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:52.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:52.783 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:52.817 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:52.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:52.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:52.921 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:52.948 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:52.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:52.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:53.051 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:53.080 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:53.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:53.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:53.183 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:53.216 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:53.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:53.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:53.224 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:47:53.224 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [], 2023-09-23T18:47:53.224 INFO:teuthology.orchestra.run.smithi098.stdout: "complaint_time": 30, 2023-09-23T18:47:53.224 INFO:teuthology.orchestra.run.smithi098.stdout: "num_blocked_ops": 0 2023-09-23T18:47:53.224 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:47:53.319 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:53.343 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:53.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:53.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:53.445 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:53.471 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:53.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:53.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:53.573 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:53.602 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:53.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:53.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:53.705 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:53.731 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:53.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:53.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:53.834 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:53.859 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:53.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:53.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:53.962 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:53.987 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:53.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:53.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:54.090 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:54.121 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:54.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:54.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:54.228 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:54.265 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:54.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:54.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:54.370 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:54.401 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:54.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:54.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:54.505 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:54.532 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:54.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:54.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:54.636 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:54.666 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:54.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:54.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:54.770 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:54.798 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:54.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:54.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:54.901 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:54.931 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:54.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:54.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:55.036 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:55.069 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:55.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:55.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:55.172 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:55.197 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:55.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:55.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:55.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:55 smithi173 bash[59422]: cluster 2023-09-23T18:47:54.067126+0000 mgr.y (mgr.26054) 892 : cluster 0 pgmap v941: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:47:55.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:55 smithi173 bash[59422]: audit 2023-09-23T18:47:54.614963+0000 mon.a (mon.0) 1214 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:47:55.301 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:55.334 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:55.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:55.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:55.438 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:55.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:55 smithi098 bash[76817]: cluster 2023-09-23T18:47:54.067126+0000 mgr.y (mgr.26054) 892 : cluster 0 pgmap v941: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:47:55.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:55 smithi098 bash[76817]: audit 2023-09-23T18:47:54.614963+0000 mon.a (mon.0) 1214 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:47:55.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:55 smithi098 bash[79516]: cluster 2023-09-23T18:47:54.067126+0000 mgr.y (mgr.26054) 892 : cluster 0 pgmap v941: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:47:55.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:55 smithi098 bash[79516]: audit 2023-09-23T18:47:54.614963+0000 mon.a (mon.0) 1214 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:47:55.470 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:55.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:55.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:55.577 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:55.609 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:55.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:55.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:55.713 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:55.738 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:55.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:55.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:55.842 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:55.873 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:55.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:55.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:55.976 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:56.009 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:56.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:56.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:56.113 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:56.143 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:56.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:56.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:56.246 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:56.277 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:56.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:56.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:56.381 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:56.388 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:47:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:47:56] "GET /metrics HTTP/1.1" 200 36494 "" "Prometheus/2.33.4" 2023-09-23T18:47:56.402 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:56.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:56.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:56.505 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:56.533 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:56.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:56.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:56.636 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:56.666 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:56.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:56.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:56.770 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:56.800 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:56.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:56.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:56.829 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:47:56.848 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:47:56.867 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:47:56.886 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:47:56.903 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:56.907 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:47:56.923 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:56.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:56.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:56.929 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:47:56.954 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:47:56.978 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:47:56.980 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:47:56.980 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:47:56.980 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:47:56.980 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:47:56.980 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:47:56.980 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:47:56.981 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:47:56.981 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:47:57.025 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:57.054 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:57.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:57.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:57.159 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:57.169 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:57 smithi098 bash[76817]: cluster 2023-09-23T18:47:56.067973+0000 mgr.y (mgr.26054) 893 : cluster 0 pgmap v942: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:47:57.169 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:57 smithi098 bash[76817]: audit 2023-09-23T18:47:56.807420+0000 mon.c (mon.2) 104 : audit 0 from='client.? 172.21.15.98:0/13120957' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:47:57.169 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:57 smithi098 bash[79516]: cluster 2023-09-23T18:47:56.067973+0000 mgr.y (mgr.26054) 893 : cluster 0 pgmap v942: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:47:57.169 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:57 smithi098 bash[79516]: audit 2023-09-23T18:47:56.807420+0000 mon.c (mon.2) 104 : audit 0 from='client.? 172.21.15.98:0/13120957' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:47:57.183 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:57.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:57.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:57.286 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:57.315 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:57.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:57.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:57.419 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:57.441 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:57.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:57.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:57.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:57 smithi173 bash[59422]: cluster 2023-09-23T18:47:56.067973+0000 mgr.y (mgr.26054) 893 : cluster 0 pgmap v942: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:47:57.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:57 smithi173 bash[59422]: audit 2023-09-23T18:47:56.807420+0000 mon.c (mon.2) 104 : audit 0 from='client.? 172.21.15.98:0/13120957' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:47:57.544 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:57.572 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:57.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:57.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:57.675 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:57.711 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:57.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:57.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:57.815 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:57.845 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:57.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:57.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:57.948 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:57.980 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:57.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:57.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:58.084 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:58.106 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:58.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:58.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:58.208 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:58.238 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:58.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:58.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:58.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:47:58 smithi173 bash[59422]: cluster 2023-09-23T18:47:58.068785+0000 mgr.y (mgr.26054) 894 : cluster 0 pgmap v943: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:47:58.341 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:58.349 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T18:47:58.371 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:58.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:58.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:58.455 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:47:58 smithi098 bash[76817]: cluster 2023-09-23T18:47:58.068785+0000 mgr.y (mgr.26054) 894 : cluster 0 pgmap v943: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:47:58.456 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:47:58 smithi098 bash[79516]: cluster 2023-09-23T18:47:58.068785+0000 mgr.y (mgr.26054) 894 : cluster 0 pgmap v943: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:47:58.474 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:58.497 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:47:58.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:58.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:47:58.600 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:58.636 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:58.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:58.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:58.740 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:58.772 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:58.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:58.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:58.876 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:58.904 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:47:58.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:58.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:47:59.008 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:59.040 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:59.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:59.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:47:59.143 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:59.175 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:47:59.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:59.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:47:59.279 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:59.310 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:47:59.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:59.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:47:59.414 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:59.446 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:47:59.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:59.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:47:59.549 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:59.578 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:59.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:59.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:59.682 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:59.713 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:47:59.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:59.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:47:59.816 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:59.844 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:47:59.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:59.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:47:59.947 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:59.980 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:47:59.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:47:59.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:00.084 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:00.117 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:00.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:00.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:00.221 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:00.250 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:00.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:00.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:00.354 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:00.383 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:00.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:00.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:00.487 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:00.517 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:00.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:00.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:00.621 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:00.652 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:00.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:00.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:00.755 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:00.787 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:00.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:00.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:00.891 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:00.923 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:00.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:00.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:01.027 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:01.060 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:01.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:01.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:01.164 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:01.176 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:01 smithi173 bash[59422]: cluster 2023-09-23T18:48:00.069401+0000 mgr.y (mgr.26054) 895 : cluster 0 pgmap v944: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:48:01.176 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:01 smithi173 bash[59422]: audit 2023-09-23T18:48:00.251487+0000 mon.a (mon.0) 1215 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:01.176 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:01 smithi173 bash[59422]: audit 2023-09-23T18:48:00.430630+0000 mon.c (mon.2) 105 : audit 1 from='client.? 172.21.15.98:0/223792126' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-09-23T18:48:01.176 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:01 smithi173 bash[59422]: audit 2023-09-23T18:48:00.431336+0000 mon.a (mon.0) 1216 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-09-23T18:48:01.176 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:01 smithi173 bash[59422]: audit 2023-09-23T18:48:00.507814+0000 mon.c (mon.2) 106 : audit 1 from='client.? 172.21.15.98:0/496124925' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:48:01.177 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:01 smithi173 bash[59422]: audit 2023-09-23T18:48:00.508362+0000 mon.a (mon.0) 1217 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:48:01.192 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:01.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:01.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:01.270 INFO:teuthology.orchestra.run.smithi098.stderr:set osd.6 primary-affinity to 0 (802) 2023-09-23T18:48:01.270 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is set 2023-09-23T18:48:01.296 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:01.323 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:01 smithi098 bash[79516]: cluster 2023-09-23T18:48:00.069401+0000 mgr.y (mgr.26054) 895 : cluster 0 pgmap v944: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:48:01.323 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:01 smithi098 bash[79516]: audit 2023-09-23T18:48:00.251487+0000 mon.a (mon.0) 1215 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:01.323 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:01 smithi098 bash[79516]: audit 2023-09-23T18:48:00.430630+0000 mon.c (mon.2) 105 : audit 1 from='client.? 172.21.15.98:0/223792126' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-09-23T18:48:01.323 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:01 smithi098 bash[79516]: audit 2023-09-23T18:48:00.431336+0000 mon.a (mon.0) 1216 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-09-23T18:48:01.323 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:01 smithi098 bash[79516]: audit 2023-09-23T18:48:00.507814+0000 mon.c (mon.2) 106 : audit 1 from='client.? 172.21.15.98:0/496124925' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:48:01.323 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:01 smithi098 bash[79516]: audit 2023-09-23T18:48:00.508362+0000 mon.a (mon.0) 1217 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:48:01.323 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:01 smithi098 bash[76817]: cluster 2023-09-23T18:48:00.069401+0000 mgr.y (mgr.26054) 895 : cluster 0 pgmap v944: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:48:01.324 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:01 smithi098 bash[76817]: audit 2023-09-23T18:48:00.251487+0000 mon.a (mon.0) 1215 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:01.324 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:01 smithi098 bash[76817]: audit 2023-09-23T18:48:00.430630+0000 mon.c (mon.2) 105 : audit 1 from='client.? 172.21.15.98:0/223792126' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-09-23T18:48:01.324 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:01 smithi098 bash[76817]: audit 2023-09-23T18:48:00.431336+0000 mon.a (mon.0) 1216 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-09-23T18:48:01.324 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:01 smithi098 bash[76817]: audit 2023-09-23T18:48:00.507814+0000 mon.c (mon.2) 106 : audit 1 from='client.? 172.21.15.98:0/496124925' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:48:01.324 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:01 smithi098 bash[76817]: audit 2023-09-23T18:48:00.508362+0000 mon.a (mon.0) 1217 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:48:01.331 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:01.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:01.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:01.435 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:01.469 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:01.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:01.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:01.573 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:01.598 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:48:01.606 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:01.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:01.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:01.710 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:01.735 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:01.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:01.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:01.838 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:01.868 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:01.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:01.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:01.971 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:01.996 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:01.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:01.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:02.099 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:02.126 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:02.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:02.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:02.230 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:02.240 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:02 smithi098 bash[79516]: cluster 2023-09-23T18:48:01.248713+0000 mon.a (mon.0) 1218 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:48:02.241 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:02 smithi098 bash[79516]: cluster 2023-09-23T18:48:01.257424+0000 osd.5 (osd.5) 130 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T18:48:02.241 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:02 smithi098 bash[79516]: cluster 2023-09-23T18:48:01.262030+0000 osd.5 (osd.5) 131 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T18:48:02.241 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:02 smithi098 bash[79516]: audit 2023-09-23T18:48:01.262095+0000 mon.a (mon.0) 1219 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2023-09-23T18:48:02.241 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:02 smithi098 bash[79516]: audit 2023-09-23T18:48:01.262217+0000 mon.a (mon.0) 1220 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:48:02.241 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:02 smithi098 bash[79516]: cluster 2023-09-23T18:48:01.262624+0000 mon.a (mon.0) 1221 : cluster 0 osdmap e236: 8 total, 8 up, 8 in 2023-09-23T18:48:02.241 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:02 smithi098 bash[79516]: cluster 2023-09-23T18:48:01.263384+0000 osd.7 (osd.7) 165 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T18:48:02.241 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:02 smithi098 bash[79516]: cluster 2023-09-23T18:48:01.263387+0000 osd.6 (osd.6) 193 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T18:48:02.241 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:02 smithi098 bash[79516]: cluster 2023-09-23T18:48:01.264638+0000 osd.6 (osd.6) 194 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T18:48:02.241 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:02 smithi098 bash[79516]: cluster 2023-09-23T18:48:01.267799+0000 osd.7 (osd.7) 166 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T18:48:02.242 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:02 smithi098 bash[76817]: cluster 2023-09-23T18:48:01.248713+0000 mon.a (mon.0) 1218 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:48:02.242 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:02 smithi098 bash[76817]: cluster 2023-09-23T18:48:01.257424+0000 osd.5 (osd.5) 130 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T18:48:02.242 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:02 smithi098 bash[76817]: cluster 2023-09-23T18:48:01.262030+0000 osd.5 (osd.5) 131 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T18:48:02.242 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:02 smithi098 bash[76817]: audit 2023-09-23T18:48:01.262095+0000 mon.a (mon.0) 1219 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2023-09-23T18:48:02.242 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:02 smithi098 bash[76817]: audit 2023-09-23T18:48:01.262217+0000 mon.a (mon.0) 1220 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:48:02.242 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:02 smithi098 bash[76817]: cluster 2023-09-23T18:48:01.262624+0000 mon.a (mon.0) 1221 : cluster 0 osdmap e236: 8 total, 8 up, 8 in 2023-09-23T18:48:02.242 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:02 smithi098 bash[76817]: cluster 2023-09-23T18:48:01.263384+0000 osd.7 (osd.7) 165 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T18:48:02.242 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:02 smithi098 bash[76817]: cluster 2023-09-23T18:48:01.263387+0000 osd.6 (osd.6) 193 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T18:48:02.242 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:02 smithi098 bash[76817]: cluster 2023-09-23T18:48:01.264638+0000 osd.6 (osd.6) 194 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T18:48:02.242 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:02 smithi098 bash[76817]: cluster 2023-09-23T18:48:01.267799+0000 osd.7 (osd.7) 166 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T18:48:02.261 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:02.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:02.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:02.365 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:02.402 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:02.404 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:02 smithi173 bash[59422]: cluster 2023-09-23T18:48:01.248713+0000 mon.a (mon.0) 1218 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:48:02.404 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:02 smithi173 bash[59422]: cluster 2023-09-23T18:48:01.257424+0000 osd.5 (osd.5) 130 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T18:48:02.404 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:02 smithi173 bash[59422]: cluster 2023-09-23T18:48:01.262030+0000 osd.5 (osd.5) 131 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T18:48:02.404 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:02 smithi173 bash[59422]: audit 2023-09-23T18:48:01.262095+0000 mon.a (mon.0) 1219 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2023-09-23T18:48:02.404 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:02 smithi173 bash[59422]: audit 2023-09-23T18:48:01.262217+0000 mon.a (mon.0) 1220 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:48:02.404 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:02 smithi173 bash[59422]: cluster 2023-09-23T18:48:01.262624+0000 mon.a (mon.0) 1221 : cluster 0 osdmap e236: 8 total, 8 up, 8 in 2023-09-23T18:48:02.404 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:02 smithi173 bash[59422]: cluster 2023-09-23T18:48:01.263384+0000 osd.7 (osd.7) 165 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T18:48:02.404 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:02 smithi173 bash[59422]: cluster 2023-09-23T18:48:01.263387+0000 osd.6 (osd.6) 193 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T18:48:02.404 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:02 smithi173 bash[59422]: cluster 2023-09-23T18:48:01.264638+0000 osd.6 (osd.6) 194 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T18:48:02.404 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:02 smithi173 bash[59422]: cluster 2023-09-23T18:48:01.267799+0000 osd.7 (osd.7) 166 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T18:48:02.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:02.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:02.506 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:02.540 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:02.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:02.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:02.644 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:02.678 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:02.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:02.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:02.783 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:02.810 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:02.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:02.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:02.913 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:02.945 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:02.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:02.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:03.049 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:03.077 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:03.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:03.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:03.181 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:03.212 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:03.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:03.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:03.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:03 smithi173 bash[59422]: cluster 2023-09-23T18:48:01.263728+0000 osd.4 (osd.4) 226 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T18:48:03.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:03 smithi173 bash[59422]: cluster 2023-09-23T18:48:02.070343+0000 mgr.y (mgr.26054) 896 : cluster 0 pgmap v946: 89 pgs: 3 peering, 86 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:48:03.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:03 smithi173 bash[59422]: cluster 2023-09-23T18:48:02.252818+0000 mon.a (mon.0) 1222 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T18:48:03.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:03 smithi173 bash[59422]: cluster 2023-09-23T18:48:02.264147+0000 osd.5 (osd.5) 132 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T18:48:03.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:03 smithi173 bash[59422]: cluster 2023-09-23T18:48:02.265053+0000 mon.a (mon.0) 1223 : cluster 0 osdmap e237: 8 total, 8 up, 8 in 2023-09-23T18:48:03.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:03 smithi173 bash[59422]: cluster 2023-09-23T18:48:02.266741+0000 osd.5 (osd.5) 133 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T18:48:03.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:03 smithi173 bash[59422]: cluster 2023-09-23T18:48:02.268773+0000 osd.6 (osd.6) 195 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T18:48:03.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:03 smithi173 bash[59422]: cluster 2023-09-23T18:48:02.268951+0000 osd.7 (osd.7) 167 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T18:48:03.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:03 smithi173 bash[59422]: cluster 2023-09-23T18:48:02.269906+0000 osd.5 (osd.5) 134 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T18:48:03.315 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:03.328 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:03 smithi098 bash[79516]: cluster 2023-09-23T18:48:01.263728+0000 osd.4 (osd.4) 226 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T18:48:03.329 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:03 smithi098 bash[79516]: cluster 2023-09-23T18:48:02.070343+0000 mgr.y (mgr.26054) 896 : cluster 0 pgmap v946: 89 pgs: 3 peering, 86 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:48:03.329 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:03 smithi098 bash[79516]: cluster 2023-09-23T18:48:02.252818+0000 mon.a (mon.0) 1222 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T18:48:03.329 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:03 smithi098 bash[79516]: cluster 2023-09-23T18:48:02.264147+0000 osd.5 (osd.5) 132 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T18:48:03.329 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:03 smithi098 bash[79516]: cluster 2023-09-23T18:48:02.265053+0000 mon.a (mon.0) 1223 : cluster 0 osdmap e237: 8 total, 8 up, 8 in 2023-09-23T18:48:03.329 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:03 smithi098 bash[79516]: cluster 2023-09-23T18:48:02.266741+0000 osd.5 (osd.5) 133 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T18:48:03.329 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:03 smithi098 bash[79516]: cluster 2023-09-23T18:48:02.268773+0000 osd.6 (osd.6) 195 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T18:48:03.329 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:03 smithi098 bash[79516]: cluster 2023-09-23T18:48:02.268951+0000 osd.7 (osd.7) 167 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T18:48:03.329 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:03 smithi098 bash[79516]: cluster 2023-09-23T18:48:02.269906+0000 osd.5 (osd.5) 134 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T18:48:03.329 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:03 smithi098 bash[76817]: cluster 2023-09-23T18:48:01.263728+0000 osd.4 (osd.4) 226 : cluster 3 failed to encode map e236 with expected crc 2023-09-23T18:48:03.329 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:03 smithi098 bash[76817]: cluster 2023-09-23T18:48:02.070343+0000 mgr.y (mgr.26054) 896 : cluster 0 pgmap v946: 89 pgs: 3 peering, 86 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:48:03.330 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:03 smithi098 bash[76817]: cluster 2023-09-23T18:48:02.252818+0000 mon.a (mon.0) 1222 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T18:48:03.330 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:03 smithi098 bash[76817]: cluster 2023-09-23T18:48:02.264147+0000 osd.5 (osd.5) 132 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T18:48:03.330 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:03 smithi098 bash[76817]: cluster 2023-09-23T18:48:02.265053+0000 mon.a (mon.0) 1223 : cluster 0 osdmap e237: 8 total, 8 up, 8 in 2023-09-23T18:48:03.330 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:03 smithi098 bash[76817]: cluster 2023-09-23T18:48:02.266741+0000 osd.5 (osd.5) 133 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T18:48:03.330 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:03 smithi098 bash[76817]: cluster 2023-09-23T18:48:02.268773+0000 osd.6 (osd.6) 195 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T18:48:03.330 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:03 smithi098 bash[76817]: cluster 2023-09-23T18:48:02.268951+0000 osd.7 (osd.7) 167 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T18:48:03.330 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:03 smithi098 bash[76817]: cluster 2023-09-23T18:48:02.269906+0000 osd.5 (osd.5) 134 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T18:48:03.351 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:03.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:03.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:03.455 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:03.488 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:03.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:03.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:03.558 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:48:03.592 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:03.619 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:03.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:03.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:03.722 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:03.758 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:03.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:03.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:03.862 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:03.889 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:03.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:03.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:03.992 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:04.023 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:04.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:04.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:04.127 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:04.154 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:04.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:04.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:04.258 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:04.271 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:04 smithi173 bash[59422]: cluster 2023-09-23T18:48:02.267256+0000 osd.4 (osd.4) 227 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T18:48:04.271 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:04 smithi173 bash[59422]: cluster 2023-09-23T18:48:02.270353+0000 osd.4 (osd.4) 228 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T18:48:04.271 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:04 smithi173 bash[59422]: cluster 2023-09-23T18:48:02.272588+0000 osd.4 (osd.4) 229 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T18:48:04.271 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:04 smithi173 bash[59422]: cluster 2023-09-23T18:48:04.070977+0000 mgr.y (mgr.26054) 897 : cluster 0 pgmap v948: 89 pgs: 3 peering, 86 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:48:04.291 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:04.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:04.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:04.395 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:04.426 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:04.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:04.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:04.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:04 smithi098 bash[79516]: cluster 2023-09-23T18:48:02.267256+0000 osd.4 (osd.4) 227 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T18:48:04.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:04 smithi098 bash[79516]: cluster 2023-09-23T18:48:02.270353+0000 osd.4 (osd.4) 228 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T18:48:04.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:04 smithi098 bash[79516]: cluster 2023-09-23T18:48:02.272588+0000 osd.4 (osd.4) 229 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T18:48:04.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:04 smithi098 bash[79516]: cluster 2023-09-23T18:48:04.070977+0000 mgr.y (mgr.26054) 897 : cluster 0 pgmap v948: 89 pgs: 3 peering, 86 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:48:04.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:04 smithi098 bash[76817]: cluster 2023-09-23T18:48:02.267256+0000 osd.4 (osd.4) 227 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T18:48:04.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:04 smithi098 bash[76817]: cluster 2023-09-23T18:48:02.270353+0000 osd.4 (osd.4) 228 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T18:48:04.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:04 smithi098 bash[76817]: cluster 2023-09-23T18:48:02.272588+0000 osd.4 (osd.4) 229 : cluster 3 failed to encode map e237 with expected crc 2023-09-23T18:48:04.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:04 smithi098 bash[76817]: cluster 2023-09-23T18:48:04.070977+0000 mgr.y (mgr.26054) 897 : cluster 0 pgmap v948: 89 pgs: 3 peering, 86 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:48:04.529 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:04.559 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:04.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:04.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:04.662 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:04.691 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:04.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:04.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:04.795 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:04.831 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:04.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:04.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:04.934 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:04.966 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:04.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:04.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:05.070 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:05.080 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:48:05.099 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:05.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:05.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:05.202 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:05.227 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:05.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:05.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:05.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:05 smithi173 bash[59422]: audit 2023-09-23T18:48:05.016777+0000 mon.a (mon.0) 1224 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:48:05.332 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:05.354 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:05.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:05.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:05.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:05 smithi098 bash[76817]: audit 2023-09-23T18:48:05.016777+0000 mon.a (mon.0) 1224 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:48:05.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:05 smithi098 bash[79516]: audit 2023-09-23T18:48:05.016777+0000 mon.a (mon.0) 1224 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:48:05.457 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:05.483 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:05.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:05.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:05.587 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:05.624 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:05.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:05.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:05.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:05.759 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:05.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:05.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:05.862 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:05.892 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:05.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:05.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:05.995 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:06.031 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:06.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:06.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:06.134 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:06.171 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:06.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:06.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:06.191 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:48:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:48:06] "GET /metrics HTTP/1.1" 200 36494 "" "Prometheus/2.33.4" 2023-09-23T18:48:06.275 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:06.306 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:06.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:06.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:06.409 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:06.438 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:06.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:06.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:06.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:06 smithi173 bash[59422]: audit 2023-09-23T18:48:05.393017+0000 mon.a (mon.0) 1225 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:06.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:06 smithi173 bash[59422]: audit 2023-09-23T18:48:05.399538+0000 mon.a (mon.0) 1226 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:06.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:06 smithi173 bash[59422]: cluster 2023-09-23T18:48:06.071865+0000 mgr.y (mgr.26054) 898 : cluster 0 pgmap v949: 89 pgs: 3 peering, 86 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:48:06.543 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:06.564 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:48:06.578 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:06.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:06.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:06.682 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:06.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:06 smithi098 bash[79516]: audit 2023-09-23T18:48:05.393017+0000 mon.a (mon.0) 1225 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:06.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:06 smithi098 bash[79516]: audit 2023-09-23T18:48:05.399538+0000 mon.a (mon.0) 1226 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:06.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:06 smithi098 bash[79516]: cluster 2023-09-23T18:48:06.071865+0000 mgr.y (mgr.26054) 898 : cluster 0 pgmap v949: 89 pgs: 3 peering, 86 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:48:06.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:06 smithi098 bash[76817]: audit 2023-09-23T18:48:05.393017+0000 mon.a (mon.0) 1225 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:06.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:06 smithi098 bash[76817]: audit 2023-09-23T18:48:05.399538+0000 mon.a (mon.0) 1226 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:06.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:06 smithi098 bash[76817]: cluster 2023-09-23T18:48:06.071865+0000 mgr.y (mgr.26054) 898 : cluster 0 pgmap v949: 89 pgs: 3 peering, 86 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:48:06.713 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:06.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:06.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:06.817 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:06.844 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:06.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:06.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:06.947 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:06.969 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:06.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:06.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:07.072 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:07.098 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:07.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:07.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:07.201 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:07.234 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:07.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:07.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:07.338 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:07.365 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:07.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:07.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:07.469 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:07.496 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:07.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:07.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:07.600 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:07.629 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:07.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:07.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:07.732 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:07.757 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:07.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:07.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:07.860 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:07.885 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:07.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:07.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:07.989 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:08.015 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:08.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:08.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:08.090 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T18:48:08.090 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:48:08.091 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T18:48:08.091 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T18:48:08.091 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T18:48:08.091 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T18:48:08.118 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:08.151 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:08.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:08.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:08.254 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:08.261 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:08 smithi098 bash[76817]: cluster 2023-09-23T18:48:08.072704+0000 mgr.y (mgr.26054) 899 : cluster 0 pgmap v950: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:48:08.261 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:08 smithi098 bash[76817]: cluster 2023-09-23T18:48:08.124126+0000 mon.a (mon.0) 1227 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T18:48:08.261 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:08 smithi098 bash[79516]: cluster 2023-09-23T18:48:08.072704+0000 mgr.y (mgr.26054) 899 : cluster 0 pgmap v950: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:48:08.261 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:08 smithi098 bash[79516]: cluster 2023-09-23T18:48:08.124126+0000 mon.a (mon.0) 1227 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T18:48:08.274 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:08.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:08.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:08.377 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:08.391 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:08 smithi173 bash[59422]: cluster 2023-09-23T18:48:08.072704+0000 mgr.y (mgr.26054) 899 : cluster 0 pgmap v950: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:48:08.391 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:08 smithi173 bash[59422]: cluster 2023-09-23T18:48:08.124126+0000 mon.a (mon.0) 1227 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T18:48:08.409 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:08.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:08.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:08.512 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:08.544 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:08.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:08.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:08.640 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:48:08.648 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:08.680 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:08.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:08.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:08.784 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:08.802 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:08.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:08.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:08.905 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:08.926 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:08.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:08.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:09.029 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:09.057 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:09.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:09.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:09.161 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:09.188 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:09.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:09.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:09.202 INFO:teuthology.orchestra.run.smithi098.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-09-23T18:48:09.227 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:09 smithi098 bash[76817]: audit 2023-09-23T18:48:08.843423+0000 mon.a (mon.0) 1228 : audit 1 from='client.? 172.21.15.98:0/3572346781' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:48:09.227 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:09 smithi098 bash[79516]: audit 2023-09-23T18:48:08.843423+0000 mon.a (mon.0) 1228 : audit 1 from='client.? 172.21.15.98:0/3572346781' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:48:09.292 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:09.321 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:09.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:09.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:09.425 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:09.438 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:09 smithi173 bash[59422]: audit 2023-09-23T18:48:08.843423+0000 mon.a (mon.0) 1228 : audit 1 from='client.? 172.21.15.98:0/3572346781' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:48:09.460 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:09.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:09.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:09.564 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:09.593 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:09.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:09.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:09.697 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:09.733 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:09.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:09.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:09.836 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:09.870 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:09.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:09.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:09.973 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:09.994 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:09.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:09.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:10.097 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:10.124 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:10.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:10.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:10.226 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:10 smithi098 bash[76817]: audit 2023-09-23T18:48:09.196107+0000 mon.a (mon.0) 1229 : audit 1 from='client.? 172.21.15.98:0/3572346781' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:48:10.226 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:10 smithi098 bash[76817]: cluster 2023-09-23T18:48:09.196223+0000 mon.a (mon.0) 1230 : cluster 0 osdmap e238: 8 total, 8 up, 8 in 2023-09-23T18:48:10.226 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:10 smithi098 bash[76817]: cluster 2023-09-23T18:48:09.197585+0000 osd.4 (osd.4) 230 : cluster 3 failed to encode map e238 with expected crc 2023-09-23T18:48:10.226 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:10 smithi098 bash[76817]: cluster 2023-09-23T18:48:09.284230+0000 osd.6 (osd.6) 196 : cluster 3 failed to encode map e238 with expected crc 2023-09-23T18:48:10.226 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:10 smithi098 bash[76817]: cluster 2023-09-23T18:48:09.287183+0000 osd.6 (osd.6) 197 : cluster 3 failed to encode map e238 with expected crc 2023-09-23T18:48:10.226 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:10 smithi098 bash[76817]: audit 2023-09-23T18:48:09.624863+0000 mon.a (mon.0) 1231 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:10.226 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:10 smithi098 bash[76817]: audit 2023-09-23T18:48:09.626007+0000 mon.a (mon.0) 1232 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:48:10.226 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:10 smithi098 bash[76817]: cluster 2023-09-23T18:48:10.073443+0000 mgr.y (mgr.26054) 900 : cluster 0 pgmap v952: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:48:10.230 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:10.253 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:10.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:10.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:10.356 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:10.386 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:10.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:10.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:10.501 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:10.513 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:10 smithi173 bash[59422]: audit 2023-09-23T18:48:09.196107+0000 mon.a (mon.0) 1229 : audit 1 from='client.? 172.21.15.98:0/3572346781' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:48:10.513 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:10 smithi173 bash[59422]: cluster 2023-09-23T18:48:09.196223+0000 mon.a (mon.0) 1230 : cluster 0 osdmap e238: 8 total, 8 up, 8 in 2023-09-23T18:48:10.513 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:10 smithi173 bash[59422]: cluster 2023-09-23T18:48:09.197585+0000 osd.4 (osd.4) 230 : cluster 3 failed to encode map e238 with expected crc 2023-09-23T18:48:10.513 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:10 smithi173 bash[59422]: cluster 2023-09-23T18:48:09.284230+0000 osd.6 (osd.6) 196 : cluster 3 failed to encode map e238 with expected crc 2023-09-23T18:48:10.513 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:10 smithi173 bash[59422]: cluster 2023-09-23T18:48:09.287183+0000 osd.6 (osd.6) 197 : cluster 3 failed to encode map e238 with expected crc 2023-09-23T18:48:10.513 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:10 smithi173 bash[59422]: audit 2023-09-23T18:48:09.624863+0000 mon.a (mon.0) 1231 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:10.513 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:10 smithi173 bash[59422]: audit 2023-09-23T18:48:09.626007+0000 mon.a (mon.0) 1232 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:48:10.513 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:10 smithi173 bash[59422]: cluster 2023-09-23T18:48:10.073443+0000 mgr.y (mgr.26054) 900 : cluster 0 pgmap v952: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:48:10.532 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:10.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:10.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:10.635 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:10.649 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:10 smithi098 bash[79516]: audit 2023-09-23T18:48:09.196107+0000 mon.a (mon.0) 1229 : audit 1 from='client.? 172.21.15.98:0/3572346781' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:48:10.649 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:10 smithi098 bash[79516]: cluster 2023-09-23T18:48:09.196223+0000 mon.a (mon.0) 1230 : cluster 0 osdmap e238: 8 total, 8 up, 8 in 2023-09-23T18:48:10.649 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:10 smithi098 bash[79516]: cluster 2023-09-23T18:48:09.197585+0000 osd.4 (osd.4) 230 : cluster 3 failed to encode map e238 with expected crc 2023-09-23T18:48:10.649 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:10 smithi098 bash[79516]: cluster 2023-09-23T18:48:09.284230+0000 osd.6 (osd.6) 196 : cluster 3 failed to encode map e238 with expected crc 2023-09-23T18:48:10.649 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:10 smithi098 bash[79516]: cluster 2023-09-23T18:48:09.287183+0000 osd.6 (osd.6) 197 : cluster 3 failed to encode map e238 with expected crc 2023-09-23T18:48:10.649 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:10 smithi098 bash[79516]: audit 2023-09-23T18:48:09.624863+0000 mon.a (mon.0) 1231 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:10.649 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:10 smithi098 bash[79516]: audit 2023-09-23T18:48:09.626007+0000 mon.a (mon.0) 1232 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:48:10.649 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:10 smithi098 bash[79516]: cluster 2023-09-23T18:48:10.073443+0000 mgr.y (mgr.26054) 900 : cluster 0 pgmap v952: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:48:10.668 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:10.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:10.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:10.770 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:10.795 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:10.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:10.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:10.899 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:10.924 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:10.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:10.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:11.027 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:11.054 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:11.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:11.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:11.158 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:11.186 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:11.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:11.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:11.290 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:11.331 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:11.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:11.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:11.436 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:11.486 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:11.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:11.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:11.590 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:11.612 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:11.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:11.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:11.715 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:11.727 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:11 smithi173 bash[59422]: cluster 2023-09-23T18:48:10.435151+0000 osd.7 (osd.7) 168 : cluster 3 failed to encode map e238 with expected crc 2023-09-23T18:48:11.748 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:11.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:11.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:11.851 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:11.863 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:11 smithi098 bash[76817]: cluster 2023-09-23T18:48:10.435151+0000 osd.7 (osd.7) 168 : cluster 3 failed to encode map e238 with expected crc 2023-09-23T18:48:11.863 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:11 smithi098 bash[79516]: cluster 2023-09-23T18:48:10.435151+0000 osd.7 (osd.7) 168 : cluster 3 failed to encode map e238 with expected crc 2023-09-23T18:48:11.880 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:11.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:11.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:11.984 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:12.011 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:12.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:12.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:12.114 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:12.146 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:12.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:12.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:12.249 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:12.271 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:12.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:12.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:12.374 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:12.404 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:12.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:12.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:12.460 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:48:12.508 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:12.541 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:12.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:12.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:12.646 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:12.679 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:12.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:12.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:12.783 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:12.815 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:12.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:12.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:12.918 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:12.926 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:12 smithi098 bash[79516]: audit 2023-09-23T18:48:11.635690+0000 mon.a (mon.0) 1233 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:12.926 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:12 smithi098 bash[79516]: audit 2023-09-23T18:48:11.642481+0000 mon.a (mon.0) 1234 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:12.926 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:12 smithi098 bash[79516]: cluster 2023-09-23T18:48:12.074072+0000 mgr.y (mgr.26054) 901 : cluster 0 pgmap v953: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:48:12.927 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:12 smithi098 bash[76817]: audit 2023-09-23T18:48:11.635690+0000 mon.a (mon.0) 1233 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:12.927 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:12 smithi098 bash[76817]: audit 2023-09-23T18:48:11.642481+0000 mon.a (mon.0) 1234 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:12.927 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:12 smithi098 bash[76817]: cluster 2023-09-23T18:48:12.074072+0000 mgr.y (mgr.26054) 901 : cluster 0 pgmap v953: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:48:12.940 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:12.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:12.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:13.017 DEBUG:teuthology.orchestra.run.smithi098:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-09-23T18:48:13.019 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T18:48:13.019 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T18:48:13.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:12 smithi173 bash[59422]: audit 2023-09-23T18:48:11.635690+0000 mon.a (mon.0) 1233 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:13.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:12 smithi173 bash[59422]: audit 2023-09-23T18:48:11.642481+0000 mon.a (mon.0) 1234 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:13.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:12 smithi173 bash[59422]: cluster 2023-09-23T18:48:12.074072+0000 mgr.y (mgr.26054) 901 : cluster 0 pgmap v953: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:48:13.044 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:13.064 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:13.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:13.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:13.167 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:13.201 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:13.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:13.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:13.305 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:13.338 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:13.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:13.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:13.442 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:13.475 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:13.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:13.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:13.564 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:hints = 1 2023-09-23T18:48:13.565 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T18:48:13.565 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Object prefix: benchmark_data_smithi098_176514 2023-09-23T18:48:13.565 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:48:13.565 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 0 0 0 0 0 0 - 0 2023-09-23T18:48:13.578 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:13.611 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:13.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:13.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:13.715 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:13.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:13.745 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:13.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:13.846 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:13.876 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:13.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:13.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:13.980 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:14.009 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:14.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:14.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:14.114 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:14.144 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:14.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:14.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:14.248 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:14.270 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:14.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:14.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:14.375 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:14.397 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:14.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:14.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:14.501 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:14.520 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:14 smithi173 bash[59422]: audit 2023-09-23T18:48:13.517631+0000 mon.a (mon.0) 1235 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:14.520 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:14 smithi173 bash[59422]: audit 2023-09-23T18:48:13.522867+0000 mon.a (mon.0) 1236 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:14.520 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:14 smithi173 bash[59422]: cluster 2023-09-23T18:48:14.074545+0000 mgr.y (mgr.26054) 902 : cluster 0 pgmap v954: 89 pgs: 89 active+clean; 6.8 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 230 KiB/s wr, 3 op/s 2023-09-23T18:48:14.520 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:14 smithi173 bash[59422]: audit 2023-09-23T18:48:14.091345+0000 mon.a (mon.0) 1237 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:48:14.520 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:14 smithi173 bash[59422]: audit 2023-09-23T18:48:14.092104+0000 mon.a (mon.0) 1238 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:48:14.520 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:14 smithi173 bash[59422]: audit 2023-09-23T18:48:14.097163+0000 mon.a (mon.0) 1239 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:14.526 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:14.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:14.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:14.565 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 1 16 2373 2357 147.305 147.312 0.00452569 0.00676016 2023-09-23T18:48:14.631 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:14.657 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:14.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:14.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:14.760 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:14.792 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:14.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:14.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:14.898 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:14.908 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:14 smithi098 bash[79516]: audit 2023-09-23T18:48:13.517631+0000 mon.a (mon.0) 1235 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:14.908 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:14 smithi098 bash[79516]: audit 2023-09-23T18:48:13.522867+0000 mon.a (mon.0) 1236 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:14.908 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:14 smithi098 bash[79516]: cluster 2023-09-23T18:48:14.074545+0000 mgr.y (mgr.26054) 902 : cluster 0 pgmap v954: 89 pgs: 89 active+clean; 6.8 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 230 KiB/s wr, 3 op/s 2023-09-23T18:48:14.908 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:14 smithi098 bash[79516]: audit 2023-09-23T18:48:14.091345+0000 mon.a (mon.0) 1237 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:48:14.908 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:14 smithi098 bash[79516]: audit 2023-09-23T18:48:14.092104+0000 mon.a (mon.0) 1238 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:48:14.909 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:14 smithi098 bash[79516]: audit 2023-09-23T18:48:14.097163+0000 mon.a (mon.0) 1239 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:14.909 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:14 smithi098 bash[76817]: audit 2023-09-23T18:48:13.517631+0000 mon.a (mon.0) 1235 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:14.909 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:14 smithi098 bash[76817]: audit 2023-09-23T18:48:13.522867+0000 mon.a (mon.0) 1236 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:14.909 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:14 smithi098 bash[76817]: cluster 2023-09-23T18:48:14.074545+0000 mgr.y (mgr.26054) 902 : cluster 0 pgmap v954: 89 pgs: 89 active+clean; 6.8 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 230 KiB/s wr, 3 op/s 2023-09-23T18:48:14.909 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:14 smithi098 bash[76817]: audit 2023-09-23T18:48:14.091345+0000 mon.a (mon.0) 1237 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:48:14.909 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:14 smithi098 bash[76817]: audit 2023-09-23T18:48:14.092104+0000 mon.a (mon.0) 1238 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:48:14.909 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:14 smithi098 bash[76817]: audit 2023-09-23T18:48:14.097163+0000 mon.a (mon.0) 1239 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:14.928 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:14.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:14.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:15.033 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:15.059 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:15.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:15.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:15.163 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:15.189 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:15.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:15.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:15.292 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:15.319 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:15.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:15.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:15.422 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:15.448 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:15.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:15.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:15.551 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:15.565 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 2 16 5149 5133 160.395 173.5 0.00737978 0.00622065 2023-09-23T18:48:15.577 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:15.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:15.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:15.680 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:15.705 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:15.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:15.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:15.808 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:15.836 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:15.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:15.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:15.940 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:15.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:15.974 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:15.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:16.075 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:16.107 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:16.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:16.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:16.212 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:16.241 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:16.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:16.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:16.347 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:16.356 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:16 smithi098 bash[76817]: audit 2023-09-23T18:48:15.267532+0000 mon.a (mon.0) 1240 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:16.356 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:16 smithi098 bash[76817]: cluster 2023-09-23T18:48:16.075070+0000 mgr.y (mgr.26054) 903 : cluster 0 pgmap v955: 89 pgs: 89 active+clean; 85 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 8.0 MiB/s wr, 128 op/s 2023-09-23T18:48:16.357 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:16 smithi098 bash[79516]: audit 2023-09-23T18:48:15.267532+0000 mon.a (mon.0) 1240 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:16.357 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:16 smithi098 bash[79516]: cluster 2023-09-23T18:48:16.075070+0000 mgr.y (mgr.26054) 903 : cluster 0 pgmap v955: 89 pgs: 89 active+clean; 85 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 8.0 MiB/s wr, 128 op/s 2023-09-23T18:48:16.357 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:48:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:48:16] "GET /metrics HTTP/1.1" 200 36502 "" "Prometheus/2.33.4" 2023-09-23T18:48:16.373 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:16.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:16.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:16.381 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:48:16.477 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:16.503 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:16.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:16.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:16.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:16 smithi173 bash[59422]: audit 2023-09-23T18:48:15.267532+0000 mon.a (mon.0) 1240 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:48:16.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:16 smithi173 bash[59422]: cluster 2023-09-23T18:48:16.075070+0000 mgr.y (mgr.26054) 903 : cluster 0 pgmap v955: 89 pgs: 89 active+clean; 85 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 8.0 MiB/s wr, 128 op/s 2023-09-23T18:48:16.565 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 3 16 7911 7895 164.466 172.625 0.00229053 0.00606821 2023-09-23T18:48:16.607 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:16.636 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:16.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:16.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:16.741 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:16.767 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:16.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:16.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:16.870 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:16.901 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:16.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:16.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:17.005 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:17.029 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:17.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:17.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:17.135 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:17.152 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:17.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:17.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:17.255 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:17.281 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:17.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:17.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:17.386 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:17.409 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:17.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:17.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:17.513 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:17.542 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:17.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:17.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:17.565 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 4 16 10067 10051 157.034 134.75 0.00270247 0.00636288 2023-09-23T18:48:17.646 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:17.673 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:17.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:17.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:17.777 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:17.805 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:17.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:17.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:17.908 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:17.935 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:17.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:17.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:18.013 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:48:18.013 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:48:18.038 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:18.069 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:18.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:18.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:18.173 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:18.202 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:18.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:18.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:18.305 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:18.331 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:18.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:18.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:18.434 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:18.461 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:18.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:18.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:18.566 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 5 16 12178 12162 152.013 131.938 0.0040597 0.00657102 2023-09-23T18:48:18.568 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:18.595 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:18.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:18.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:18.699 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:18.721 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:18.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:18.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:18.824 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:18.846 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:18.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:18.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:18.948 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:18.982 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:18.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:18.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:19.090 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:19.118 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:19.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:19.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:19.133 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:19 smithi173 bash[59422]: cluster 2023-09-23T18:48:18.075586+0000 mgr.y (mgr.26054) 904 : cluster 0 pgmap v956: 89 pgs: 89 active+clean; 237 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 23 MiB/s wr, 370 op/s 2023-09-23T18:48:19.222 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:19.249 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:19.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:19.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:19.353 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:19.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:19.390 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:19.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:19.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:19 smithi098 bash[79516]: cluster 2023-09-23T18:48:18.075586+0000 mgr.y (mgr.26054) 904 : cluster 0 pgmap v956: 89 pgs: 89 active+clean; 237 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 23 MiB/s wr, 370 op/s 2023-09-23T18:48:19.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:19 smithi098 bash[76817]: cluster 2023-09-23T18:48:18.075586+0000 mgr.y (mgr.26054) 904 : cluster 0 pgmap v956: 89 pgs: 89 active+clean; 237 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 23 MiB/s wr, 370 op/s 2023-09-23T18:48:19.492 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:19.519 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:19.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:19.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:19.565 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 6 16 14465 14449 150.498 142.938 0.00663932 0.00663813 2023-09-23T18:48:19.624 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:19.657 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:19.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:19.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:19.761 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:19.797 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:19.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:19.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:19.900 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:19.932 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:19.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:19.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:20.035 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:20.060 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:20.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:20.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:20.165 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:20.196 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:20.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:20.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:20.300 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:20.321 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:20.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:20.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:20.428 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:20.452 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:20.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:20.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:20.556 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:20.565 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 7 16 16847 16831 150.264 148.875 0.00354714 0.00664676 2023-09-23T18:48:20.579 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:20.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:20.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:20.685 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:20.714 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:20.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:20.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:20.817 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:20.843 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:20.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:20.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:20.948 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:20.981 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:20.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:20.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:21.084 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:21.117 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:21.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:21.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:21.128 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:21 smithi173 bash[59422]: cluster 2023-09-23T18:48:20.076239+0000 mgr.y (mgr.26054) 905 : cluster 0 pgmap v957: 89 pgs: 89 active+clean; 559 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2023-09-23T18:48:21.220 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:21.259 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:21.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:21.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:21.365 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:21.405 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:21.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:21.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:21.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:21 smithi098 bash[76817]: cluster 2023-09-23T18:48:20.076239+0000 mgr.y (mgr.26054) 905 : cluster 0 pgmap v957: 89 pgs: 89 active+clean; 559 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2023-09-23T18:48:21.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:21 smithi098 bash[79516]: cluster 2023-09-23T18:48:20.076239+0000 mgr.y (mgr.26054) 905 : cluster 0 pgmap v957: 89 pgs: 89 active+clean; 559 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2023-09-23T18:48:21.512 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:21.541 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:21.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:21.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:21.566 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 8 16 19019 19003 148.448 135.75 0.00655452 0.0067308 2023-09-23T18:48:21.648 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:21.679 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:21.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:21.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:21.787 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:21.814 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:21.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:21.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:21.918 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:21.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:21.949 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:21.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:22.049 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:22.073 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:22.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:22.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:22.179 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:22.204 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:22.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:22.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:22.306 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:22.336 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:22.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:22.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:22.440 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:22.470 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:22.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:22.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:22.566 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 9 16 20907 20891 145.064 118 0.00469588 0.00688564 2023-09-23T18:48:22.573 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:22.602 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:22.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:22.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:22.706 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:22.734 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:22.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:22.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:22.839 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:22.870 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:22.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:22.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:22.976 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:23.009 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:23.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:23.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:23.112 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:23.142 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:23.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:23.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:23.248 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:23.270 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:23.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:23.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:23.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:23 smithi173 bash[59422]: cluster 2023-09-23T18:48:22.076782+0000 mgr.y (mgr.26054) 906 : cluster 0 pgmap v958: 89 pgs: 89 active+clean; 957 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-09-23T18:48:23.376 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:23.384 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:23 smithi098 bash[76817]: cluster 2023-09-23T18:48:22.076782+0000 mgr.y (mgr.26054) 906 : cluster 0 pgmap v958: 89 pgs: 89 active+clean; 957 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-09-23T18:48:23.384 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:23 smithi098 bash[79516]: cluster 2023-09-23T18:48:22.076782+0000 mgr.y (mgr.26054) 906 : cluster 0 pgmap v958: 89 pgs: 89 active+clean; 957 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-09-23T18:48:23.407 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:23.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:23.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:23.511 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:23.537 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:23.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:23.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:23.566 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 10 16 23002 22986 143.65 130.938 0.00517569 0.00695451 2023-09-23T18:48:23.616 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T18:48:23.642 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:23.694 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:23.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:23.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:23.797 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:23.820 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:23.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:23.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:23.925 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:23.953 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:23.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:23.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:24.056 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:24.085 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:24.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:24.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:24.189 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:24.196 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:24 smithi173 bash[59422]: cluster 2023-09-23T18:48:24.077337+0000 mgr.y (mgr.26054) 907 : cluster 0 pgmap v959: 89 pgs: 89 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2023-09-23T18:48:24.212 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:24.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:24.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:24.315 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:24.325 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:24 smithi098 bash[76817]: cluster 2023-09-23T18:48:24.077337+0000 mgr.y (mgr.26054) 907 : cluster 0 pgmap v959: 89 pgs: 89 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2023-09-23T18:48:24.325 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:24 smithi098 bash[79516]: cluster 2023-09-23T18:48:24.077337+0000 mgr.y (mgr.26054) 907 : cluster 0 pgmap v959: 89 pgs: 89 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2023-09-23T18:48:24.342 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:24.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:24.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:24.445 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:24.472 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:24.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:24.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:24.566 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 11 16 24796 24780 140.783 112.125 0.00480832 0.00709703 2023-09-23T18:48:24.576 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:24.606 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:24.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:24.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:24.709 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:24.730 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:24.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:24.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:24.833 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:24.859 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:24.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:24.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:24.963 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:24.995 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:24.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:24.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:25.100 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:25.128 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:25.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:25.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:25.231 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:25.251 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:25.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:25.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:25.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:25 smithi173 bash[59422]: audit 2023-09-23T18:48:24.616063+0000 mon.a (mon.0) 1241 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:48:25.355 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:25.380 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:25.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:25.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:25.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:25 smithi098 bash[79516]: audit 2023-09-23T18:48:24.616063+0000 mon.a (mon.0) 1241 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:48:25.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:25 smithi098 bash[76817]: audit 2023-09-23T18:48:24.616063+0000 mon.a (mon.0) 1241 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:48:25.485 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:25.514 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:25.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:25.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:25.566 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 12 16 26203 26187 136.379 87.9375 0.00250132 0.00729036 2023-09-23T18:48:25.617 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:25.641 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:25.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:25.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:25.743 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:25.769 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:25.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:25.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:25.872 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:25.895 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:25.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:25.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:26.000 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:26.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:26.036 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:26.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:26.137 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:26.162 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:26.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:26.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:26.269 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:26.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:26 smithi173 bash[59422]: cluster 2023-09-23T18:48:26.077852+0000 mgr.y (mgr.26054) 908 : cluster 0 pgmap v960: 89 pgs: 89 active+clean; 1.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-09-23T18:48:26.293 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:26.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:26.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:26.333 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:48:26.372 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:48:26.398 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:26.405 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:48:26.406 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:26 smithi098 bash[79516]: cluster 2023-09-23T18:48:26.077852+0000 mgr.y (mgr.26054) 908 : cluster 0 pgmap v960: 89 pgs: 89 active+clean; 1.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-09-23T18:48:26.407 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:26 smithi098 bash[76817]: cluster 2023-09-23T18:48:26.077852+0000 mgr.y (mgr.26054) 908 : cluster 0 pgmap v960: 89 pgs: 89 active+clean; 1.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-09-23T18:48:26.407 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:48:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:48:26] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.33.4" 2023-09-23T18:48:26.423 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:26.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:26.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:26.434 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:48:26.462 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:48:26.491 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:48:26.525 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:48:26.526 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:26.554 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:48:26.555 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:26.555 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:48:26.555 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:48:26.555 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:48:26.556 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:48:26.556 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:48:26.556 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:48:26.556 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:48:26.556 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:48:26.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:26.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:26.566 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 13 16 27654 27638 132.863 90.6875 0.0129703 0.00752 2023-09-23T18:48:26.659 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:26.691 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:26.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:26.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:26.795 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:26.831 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:26.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:26.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:26.934 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:26.959 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:26.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:26.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:27.063 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:27.094 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:27.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:27.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:27.199 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:27.230 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:27.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:27.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:27.336 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:27.363 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:27.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:27.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:27.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:27 smithi098 bash[76817]: audit 2023-09-23T18:48:26.289663+0000 mon.c (mon.2) 107 : audit 0 from='client.? 172.21.15.98:0/1363173815' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:48:27.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:27 smithi098 bash[79516]: audit 2023-09-23T18:48:26.289663+0000 mon.c (mon.2) 107 : audit 0 from='client.? 172.21.15.98:0/1363173815' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:48:27.467 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:27.476 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:27 smithi173 bash[59422]: audit 2023-09-23T18:48:26.289663+0000 mon.c (mon.2) 107 : audit 0 from='client.? 172.21.15.98:0/1363173815' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:48:27.505 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:27.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:27.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:27.566 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 14 16 29361 29345 130.993 106.688 0.0117612 0.00762756 2023-09-23T18:48:27.610 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:27.638 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:27.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:27.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:27.749 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:27.771 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:27.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:27.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:27.874 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:27.902 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:27.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:27.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:28.005 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:28.034 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:28.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:28.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:28.137 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:28.167 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:28.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:28.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:28.271 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:28.279 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:28 smithi098 bash[76817]: cluster 2023-09-23T18:48:28.078428+0000 mgr.y (mgr.26054) 909 : cluster 0 pgmap v961: 89 pgs: 89 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-09-23T18:48:28.279 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:28 smithi098 bash[79516]: cluster 2023-09-23T18:48:28.078428+0000 mgr.y (mgr.26054) 909 : cluster 0 pgmap v961: 89 pgs: 89 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-09-23T18:48:28.300 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:28.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:28.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:28.404 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:28.429 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:28.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:28.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:28.444 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:48:28.482 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:28 smithi173 bash[59422]: cluster 2023-09-23T18:48:28.078428+0000 mgr.y (mgr.26054) 909 : cluster 0 pgmap v961: 89 pgs: 89 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-09-23T18:48:28.534 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:28.566 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 15 16 31118 31102 129.58 109.812 0.0103405 0.00771233 2023-09-23T18:48:28.578 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:28.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:28.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:28.683 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:28.709 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:28.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:28.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:28.813 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:28.849 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:28.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:28.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:28.952 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:28.980 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:28.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:28.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:29.086 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:29.112 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:29.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:29.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:29.216 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:29.237 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:29.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:29.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:29.340 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:29.374 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:29.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:29.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:29.479 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:29.508 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:29.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:29.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:29.567 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 16 16 32479 32463 126.797 85.0625 0.00841605 0.00788032 2023-09-23T18:48:29.611 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:29.637 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:29.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:29.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:29.741 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:29.762 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:29.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:29.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:29.865 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:29.896 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:29.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:29.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:30.000 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:30.020 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:30.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:30.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:30.123 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:30.155 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:30.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:30.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:30.261 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:30.287 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:30.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:30.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:30.392 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:30.419 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:30.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:30.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:30.524 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:30.547 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:30.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:30.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:30.567 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 17 16 33812 33796 124.239 83.3125 0.00421838 0.00802205 2023-09-23T18:48:30.650 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:30.679 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:30.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:30.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:30.784 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:30.811 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:30.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:30.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:30.915 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:30.934 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:30.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:30.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:31.036 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:31.061 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:31.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:31.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:31.158 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is set 2023-09-23T18:48:31.165 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:31.174 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:31 smithi098 bash[79516]: cluster 2023-09-23T18:48:30.079474+0000 mgr.y (mgr.26054) 910 : cluster 0 pgmap v962: 89 pgs: 89 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-09-23T18:48:31.174 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:31 smithi098 bash[79516]: audit 2023-09-23T18:48:30.417597+0000 mon.c (mon.2) 108 : audit 1 from='client.? 172.21.15.98:0/1353668637' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:48:31.175 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:31 smithi098 bash[79516]: audit 2023-09-23T18:48:30.417996+0000 mon.a (mon.0) 1242 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:48:31.175 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:31 smithi098 bash[76817]: cluster 2023-09-23T18:48:30.079474+0000 mgr.y (mgr.26054) 910 : cluster 0 pgmap v962: 89 pgs: 89 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-09-23T18:48:31.175 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:31 smithi098 bash[76817]: audit 2023-09-23T18:48:30.417597+0000 mon.c (mon.2) 108 : audit 1 from='client.? 172.21.15.98:0/1353668637' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:48:31.175 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:31 smithi098 bash[76817]: audit 2023-09-23T18:48:30.417996+0000 mon.a (mon.0) 1242 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:48:31.196 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:31.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:31.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:31.300 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:31.310 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:31 smithi173 bash[59422]: cluster 2023-09-23T18:48:30.079474+0000 mgr.y (mgr.26054) 910 : cluster 0 pgmap v962: 89 pgs: 89 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-09-23T18:48:31.310 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:31 smithi173 bash[59422]: audit 2023-09-23T18:48:30.417597+0000 mon.c (mon.2) 108 : audit 1 from='client.? 172.21.15.98:0/1353668637' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:48:31.310 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:31 smithi173 bash[59422]: audit 2023-09-23T18:48:30.417996+0000 mon.a (mon.0) 1242 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:48:31.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:31.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:31.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:31.432 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:31.462 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:31.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:31.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:31.567 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 18 16 35434 35418 122.968 101.375 0.00755749 0.00812604 2023-09-23T18:48:31.567 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:31.603 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:31.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:31.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:31.713 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:31.746 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:31.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:31.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:31.849 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:31.883 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:31.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:31.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:31.985 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:32.015 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:32.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:32.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:32.120 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:32.153 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:32.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:32.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:32.256 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:32.286 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:32.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:32.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:32.391 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:32.402 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:32 smithi098 bash[79516]: cluster 2023-09-23T18:48:31.135521+0000 mon.a (mon.0) 1243 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:48:32.403 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:32 smithi098 bash[79516]: audit 2023-09-23T18:48:31.142008+0000 mon.a (mon.0) 1244 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:48:32.403 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:32 smithi098 bash[79516]: cluster 2023-09-23T18:48:31.142165+0000 mon.a (mon.0) 1245 : cluster 0 osdmap e239: 8 total, 8 up, 8 in 2023-09-23T18:48:32.403 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:32 smithi098 bash[79516]: cluster 2023-09-23T18:48:31.144667+0000 osd.4 (osd.4) 231 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T18:48:32.403 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:32 smithi098 bash[79516]: cluster 2023-09-23T18:48:31.148106+0000 osd.5 (osd.5) 135 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T18:48:32.403 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:32 smithi098 bash[79516]: cluster 2023-09-23T18:48:31.149193+0000 osd.4 (osd.4) 232 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T18:48:32.403 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:32 smithi098 bash[79516]: cluster 2023-09-23T18:48:31.157162+0000 osd.6 (osd.6) 198 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T18:48:32.404 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:32 smithi098 bash[76817]: cluster 2023-09-23T18:48:31.135521+0000 mon.a (mon.0) 1243 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:48:32.404 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:32 smithi098 bash[76817]: audit 2023-09-23T18:48:31.142008+0000 mon.a (mon.0) 1244 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:48:32.404 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:32 smithi098 bash[76817]: cluster 2023-09-23T18:48:31.142165+0000 mon.a (mon.0) 1245 : cluster 0 osdmap e239: 8 total, 8 up, 8 in 2023-09-23T18:48:32.404 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:32 smithi098 bash[76817]: cluster 2023-09-23T18:48:31.144667+0000 osd.4 (osd.4) 231 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T18:48:32.404 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:32 smithi098 bash[76817]: cluster 2023-09-23T18:48:31.148106+0000 osd.5 (osd.5) 135 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T18:48:32.404 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:32 smithi098 bash[76817]: cluster 2023-09-23T18:48:31.149193+0000 osd.4 (osd.4) 232 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T18:48:32.404 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:32 smithi098 bash[76817]: cluster 2023-09-23T18:48:31.157162+0000 osd.6 (osd.6) 198 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T18:48:32.421 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:32.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:32.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:32.499 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:48:32.526 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:32.564 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:32 smithi173 bash[59422]: cluster 2023-09-23T18:48:31.135521+0000 mon.a (mon.0) 1243 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:48:32.565 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:32 smithi173 bash[59422]: audit 2023-09-23T18:48:31.142008+0000 mon.a (mon.0) 1244 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:48:32.565 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:32 smithi173 bash[59422]: cluster 2023-09-23T18:48:31.142165+0000 mon.a (mon.0) 1245 : cluster 0 osdmap e239: 8 total, 8 up, 8 in 2023-09-23T18:48:32.565 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:32 smithi173 bash[59422]: cluster 2023-09-23T18:48:31.144667+0000 osd.4 (osd.4) 231 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T18:48:32.565 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:32 smithi173 bash[59422]: cluster 2023-09-23T18:48:31.148106+0000 osd.5 (osd.5) 135 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T18:48:32.565 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:32 smithi173 bash[59422]: cluster 2023-09-23T18:48:31.149193+0000 osd.4 (osd.4) 232 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T18:48:32.565 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:32 smithi173 bash[59422]: cluster 2023-09-23T18:48:31.157162+0000 osd.6 (osd.6) 198 : cluster 3 failed to encode map e239 with expected crc 2023-09-23T18:48:32.567 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 19 16 36759 36743 120.854 82.8125 0.0521575 0.00825601 2023-09-23T18:48:32.577 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:32.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:32.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:32.682 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:32.708 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:32.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:32.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:32.811 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:32.839 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:32.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:32.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:32.943 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:32.962 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:32.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:32.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:33.065 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:33.083 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:33.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:33.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:33.185 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:33.215 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:33.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:33.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:33.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:33 smithi173 bash[59422]: cluster 2023-09-23T18:48:32.080041+0000 mgr.y (mgr.26054) 911 : cluster 0 pgmap v964: 89 pgs: 89 active+clean; 2.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T18:48:33.321 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:33.366 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:33.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:33.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:33.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:33 smithi098 bash[76817]: cluster 2023-09-23T18:48:32.080041+0000 mgr.y (mgr.26054) 911 : cluster 0 pgmap v964: 89 pgs: 89 active+clean; 2.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T18:48:33.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:33 smithi098 bash[79516]: cluster 2023-09-23T18:48:32.080041+0000 mgr.y (mgr.26054) 911 : cluster 0 pgmap v964: 89 pgs: 89 active+clean; 2.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T18:48:33.468 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:33.505 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:33.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:33.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:33.567 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:2023-09-23T18:48:33.565810+0000 min lat: 0.00122919 max lat: 0.250456 avg lat: 0.00837141 2023-09-23T18:48:33.567 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:48:33.567 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 20 16 38221 38205 119.38 91.375 0.00848352 0.00837141 2023-09-23T18:48:33.610 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:33.645 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:33.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:33.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:33.749 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:33.773 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:33.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:33.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:33.877 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:33.921 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:33.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:33.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:33.931 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:48:33.931 INFO:teuthology.orchestra.run.smithi098.stdout: "success": "" 2023-09-23T18:48:33.932 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:48:34.031 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:34.093 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:34.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:34.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:34.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:34 smithi098 bash[79516]: cluster 2023-09-23T18:48:34.080567+0000 mgr.y (mgr.26054) 912 : cluster 0 pgmap v965: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-09-23T18:48:34.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:34 smithi098 bash[76817]: cluster 2023-09-23T18:48:34.080567+0000 mgr.y (mgr.26054) 912 : cluster 0 pgmap v965: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-09-23T18:48:34.199 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:34.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:34.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:34.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:34.332 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:34.360 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:34.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:34.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:34.463 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:34.473 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:34 smithi173 bash[59422]: cluster 2023-09-23T18:48:34.080567+0000 mgr.y (mgr.26054) 912 : cluster 0 pgmap v965: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-09-23T18:48:34.489 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:34.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:34.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:34.567 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 21 16 39728 39712 118.179 94.1875 0.00958809 0.00845569 2023-09-23T18:48:34.593 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:34.620 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:34.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:34.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:34.723 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:34.748 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:34.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:34.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:34.851 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:34.877 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:34.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:34.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:34.982 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:35.019 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:35.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:35.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:35.101 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:48:35.101 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-09-23T18:48:35.101 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.101 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35412.0:40451 8.5 8:a5a73b12:::benchmark_data_smithi098_176514_object40450:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e239)", 2023-09-23T18:48:35.101 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:48:35.093698+0000", 2023-09-23T18:48:35.101 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.0057534539999999999, 2023-09-23T18:48:35.102 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0057888640000000003, 2023-09-23T18:48:35.102 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:48:35.102 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:48:35.102 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:48:35.102 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35412", 2023-09-23T18:48:35.102 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:48:35.102 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 40451 2023-09-23T18:48:35.102 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.102 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:48:35.102 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.103 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:48:35.103 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.093698+0000", 2023-09-23T18:48:35.103 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:48:35.103 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.103 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.103 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:48:35.103 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.093698+0000", 2023-09-23T18:48:35.103 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999752 2023-09-23T18:48:35.103 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.103 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.104 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:48:35.104 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.093674+0000", 2023-09-23T18:48:35.104 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000101357 2023-09-23T18:48:35.104 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.104 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.104 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:48:35.104 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.093775+0000", 2023-09-23T18:48:35.104 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.1109999999999999e-05 2023-09-23T18:48:35.104 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.104 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.104 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:48:35.105 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.093796+0000", 2023-09-23T18:48:35.105 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.4970000000000001e-06 2023-09-23T18:48:35.105 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.105 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.105 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:48:35.105 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.093805+0000", 2023-09-23T18:48:35.105 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.3279999999999998e-05 2023-09-23T18:48:35.105 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.105 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.105 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:48:35.105 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.093858+0000", 2023-09-23T18:48:35.106 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000140362 2023-09-23T18:48:35.106 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.106 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.106 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:48:35.106 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.093998+0000", 2023-09-23T18:48:35.106 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00017458799999999999 2023-09-23T18:48:35.106 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.106 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.106 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 6,7", 2023-09-23T18:48:35.107 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.094173+0000", 2023-09-23T18:48:35.107 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001920894 2023-09-23T18:48:35.107 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.107 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.107 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:48:35.107 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.096094+0000", 2023-09-23T18:48:35.107 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002395458 2023-09-23T18:48:35.107 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:48:35.107 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:48:35.107 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:48:35.107 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.107 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.108 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35412.0:40454 8.2 8:47bfae7b:::benchmark_data_smithi098_176514_object40453:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e239)", 2023-09-23T18:48:35.108 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:48:35.094324+0000", 2023-09-23T18:48:35.108 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.0051276469999999999, 2023-09-23T18:48:35.108 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0052590659999999997, 2023-09-23T18:48:35.108 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:48:35.108 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:48:35.108 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:48:35.108 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35412", 2023-09-23T18:48:35.108 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:48:35.108 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 40454 2023-09-23T18:48:35.109 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.109 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:48:35.109 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.109 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:48:35.109 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.094324+0000", 2023-09-23T18:48:35.109 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:48:35.109 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.109 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.109 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:48:35.109 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.094324+0000", 2023-09-23T18:48:35.109 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:48:35.110 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.110 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.110 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:48:35.110 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.094323+0000", 2023-09-23T18:48:35.110 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.2308000000000003e-05 2023-09-23T18:48:35.110 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.111 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.111 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:48:35.111 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.094355+0000", 2023-09-23T18:48:35.111 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1973e-05 2023-09-23T18:48:35.111 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.111 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.111 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:48:35.111 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.094367+0000", 2023-09-23T18:48:35.111 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.34e-06 2023-09-23T18:48:35.111 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.111 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.111 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:48:35.112 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.094374+0000", 2023-09-23T18:48:35.112 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.94e-05 2023-09-23T18:48:35.112 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.112 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.112 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:48:35.112 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.094434+0000", 2023-09-23T18:48:35.112 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014168500000000001 2023-09-23T18:48:35.112 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.112 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.112 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:48:35.112 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.094576+0000", 2023-09-23T18:48:35.112 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000181991 2023-09-23T18:48:35.113 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.113 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.113 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:48:35.113 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.094758+0000", 2023-09-23T18:48:35.113 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002337287 2023-09-23T18:48:35.113 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.113 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.113 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:48:35.113 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.097095+0000", 2023-09-23T18:48:35.113 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002770752 2023-09-23T18:48:35.113 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:48:35.114 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:48:35.114 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:48:35.114 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.114 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.114 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop_reply(client.35412.0:40454 8.2 e239/233 ondisk, result = 0)", 2023-09-23T18:48:35.114 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:48:35.097513+0000", 2023-09-23T18:48:35.114 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.0019387429999999999, 2023-09-23T18:48:35.114 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0021573510000000001, 2023-09-23T18:48:35.114 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:48:35.114 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "started", 2023-09-23T18:48:35.114 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:48:35.115 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.115 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:48:35.115 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.097513+0000", 2023-09-23T18:48:35.115 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:48:35.115 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.115 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.115 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:48:35.115 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.097513+0000", 2023-09-23T18:48:35.115 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:48:35.115 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.115 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.115 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:48:35.116 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.097511+0000", 2023-09-23T18:48:35.116 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.0809999999999999e-06 2023-09-23T18:48:35.116 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.116 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.116 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:48:35.116 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.097518+0000", 2023-09-23T18:48:35.116 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3011e-05 2023-09-23T18:48:35.116 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.116 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.116 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:48:35.116 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.097531+0000", 2023-09-23T18:48:35.116 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.1389999999999998e-06 2023-09-23T18:48:35.117 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.117 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.117 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:48:35.117 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.097538+0000", 2023-09-23T18:48:35.117 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0020678559999999999 2023-09-23T18:48:35.117 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.117 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.117 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:48:35.117 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.099606+0000", 2023-09-23T18:48:35.117 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.4320000000000005e-06 2023-09-23T18:48:35.117 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.117 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.118 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:48:35.118 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.099615+0000", 2023-09-23T18:48:35.118 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0021019480000000002 2023-09-23T18:48:35.118 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:48:35.118 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:48:35.118 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:48:35.118 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.118 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.118 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35412.0:40445 8.4 e239/233 8:29fd0a21:::benchmark_data_smithi098_176514_object40444:head v 239'2497, mlcod=239'2493)", 2023-09-23T18:48:35.118 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:48:35.097981+0000", 2023-09-23T18:48:35.118 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.001470655, 2023-09-23T18:48:35.119 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00175165, 2023-09-23T18:48:35.119 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:48:35.119 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "started", 2023-09-23T18:48:35.119 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:48:35.119 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.119 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:48:35.119 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.097981+0000", 2023-09-23T18:48:35.119 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:48:35.119 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.119 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.119 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:48:35.119 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.097981+0000", 2023-09-23T18:48:35.119 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:48:35.120 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.120 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.120 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:48:35.120 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.097980+0000", 2023-09-23T18:48:35.120 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.0199999999999999e-05 2023-09-23T18:48:35.120 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.120 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.120 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:48:35.120 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.098010+0000", 2023-09-23T18:48:35.120 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2669e-05 2023-09-23T18:48:35.120 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.120 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.121 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:48:35.121 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.098023+0000", 2023-09-23T18:48:35.121 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.7059999999999998e-06 2023-09-23T18:48:35.121 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.121 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.121 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:48:35.121 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.098029+0000", 2023-09-23T18:48:35.121 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6521999999999999e-05 2023-09-23T18:48:35.121 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.121 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.121 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:48:35.122 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.098086+0000", 2023-09-23T18:48:35.122 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.5024999999999998e-05 2023-09-23T18:48:35.122 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.122 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.122 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:48:35.122 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.098111+0000", 2023-09-23T18:48:35.122 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012966400000000001 2023-09-23T18:48:35.122 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:48:35.122 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:48:35.122 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:48:35.122 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.122 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.123 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35412.0:40460 8.b 8:dd55ae92:::benchmark_data_smithi098_176514_object40459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e239)", 2023-09-23T18:48:35.123 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:48:35.098064+0000", 2023-09-23T18:48:35.123 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.0013874670000000001, 2023-09-23T18:48:35.123 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001733821, 2023-09-23T18:48:35.123 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:48:35.123 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:48:35.123 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:48:35.123 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35412", 2023-09-23T18:48:35.123 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:48:35.123 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 40460 2023-09-23T18:48:35.123 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.123 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:48:35.124 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.124 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:48:35.124 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.098064+0000", 2023-09-23T18:48:35.124 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:48:35.124 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.124 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.124 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:48:35.124 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.098064+0000", 2023-09-23T18:48:35.124 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:48:35.124 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.124 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.124 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:48:35.125 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.098063+0000", 2023-09-23T18:48:35.125 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.5732000000000003e-05 2023-09-23T18:48:35.125 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.125 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.125 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:48:35.125 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.098098+0000", 2023-09-23T18:48:35.125 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2747e-05 2023-09-23T18:48:35.125 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.125 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.125 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:48:35.125 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.098111+0000", 2023-09-23T18:48:35.125 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.6510000000000004e-06 2023-09-23T18:48:35.126 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.126 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.126 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:48:35.126 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.098118+0000", 2023-09-23T18:48:35.126 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1582999999999996e-05 2023-09-23T18:48:35.126 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.126 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.126 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:48:35.126 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.098179+0000", 2023-09-23T18:48:35.126 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015111599999999999 2023-09-23T18:48:35.126 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.127 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.127 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:48:35.127 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.098330+0000", 2023-09-23T18:48:35.127 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00017477599999999999 2023-09-23T18:48:35.127 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:48:35.127 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:48:35.127 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,7", 2023-09-23T18:48:35.127 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:48:35.098505+0000", 2023-09-23T18:48:35.127 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00044097 2023-09-23T18:48:35.127 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:48:35.127 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:48:35.127 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:48:35.128 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:48:35.128 INFO:teuthology.orchestra.run.smithi173.stdout: ], 2023-09-23T18:48:35.128 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 5 2023-09-23T18:48:35.128 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:48:35.128 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:35.151 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:35.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:35.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:35.254 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:35.278 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:35.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:35.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:35.382 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:35.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:35.411 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:35.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:35.512 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:35.543 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:35.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:35.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:35.567 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 22 16 41124 41108 116.773 87.25 0.0206837 0.00855739 2023-09-23T18:48:35.649 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:35.678 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:35.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:35.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:35.777 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:48:35.783 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:35.817 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:35.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:35.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:35.919 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:35.961 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:35.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:35.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:36.066 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:36.087 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:48:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:48:36] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.33.4" 2023-09-23T18:48:36.099 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:36.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:36.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:36.209 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:36.241 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:36.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:36.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:36.307 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T18:48:36.344 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:36.387 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:36.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:36.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:36.489 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:36.516 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:36.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:36.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:36.567 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 23 16 42539 42523 115.541 88.4375 0.0253822 0.00864866 2023-09-23T18:48:36.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:36.647 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:36.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:36.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:36.752 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:36.779 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:36.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:36.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:36.882 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:36.917 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:36.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:36.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:37.020 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:37.047 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:37.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:37.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:37.152 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:37.178 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:37.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:37.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:37.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:37 smithi173 bash[59422]: cluster 2023-09-23T18:48:36.081131+0000 mgr.y (mgr.26054) 913 : cluster 0 pgmap v966: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-09-23T18:48:37.286 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:37.295 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:37 smithi098 bash[79516]: cluster 2023-09-23T18:48:36.081131+0000 mgr.y (mgr.26054) 913 : cluster 0 pgmap v966: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-09-23T18:48:37.295 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:37 smithi098 bash[76817]: cluster 2023-09-23T18:48:36.081131+0000 mgr.y (mgr.26054) 913 : cluster 0 pgmap v966: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-09-23T18:48:37.309 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:37.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:37.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:37.412 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:37.439 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:37.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:37.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:37.542 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:37.567 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 24 16 43998 43982 114.526 91.1875 0.0100711 0.00872666 2023-09-23T18:48:37.575 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:37.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:37.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:37.678 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:37.703 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:37.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:37.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:37.806 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:37.828 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:37.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:37.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:37.934 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:37.961 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:37.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:37.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:38.023 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:48:38.066 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:38.096 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:38.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:38.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:38.146 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:38 smithi098 bash[76817]: cluster 2023-09-23T18:48:38.081671+0000 mgr.y (mgr.26054) 914 : cluster 0 pgmap v967: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-23T18:48:38.199 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:38.227 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:38.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:38.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:38.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:38 smithi173 bash[59422]: cluster 2023-09-23T18:48:38.081671+0000 mgr.y (mgr.26054) 914 : cluster 0 pgmap v967: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-23T18:48:38.330 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:38.356 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:38.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:38.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:38.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:38 smithi098 bash[79516]: cluster 2023-09-23T18:48:38.081671+0000 mgr.y (mgr.26054) 914 : cluster 0 pgmap v967: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-23T18:48:38.461 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:38.490 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:38.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:38.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:38.568 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 25 16 45438 45422 113.544 90 0.00786466 0.00880124 2023-09-23T18:48:38.592 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:38.617 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:38.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:38.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:38.721 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:38.744 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:38.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:38.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:38.849 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:38.871 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:38.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:38.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:38.974 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:39.006 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:39.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:39.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:39.109 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:39.135 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:39.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:39.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:39.238 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:39.264 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:39.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:39.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:39.370 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:39.398 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:39.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:39.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:39.505 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:39.542 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:39.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:39.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:39.568 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 26 16 46489 46473 111.703 65.6875 0.0112192 0.00894624 2023-09-23T18:48:39.645 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:39.680 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:39.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:39.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:39.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:39 smithi173 bash[59422]: audit 2023-09-23T18:48:39.616311+0000 mon.a (mon.0) 1246 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:48:39.785 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:39.811 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:39.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:39.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:39.914 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:39.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:39 smithi098 bash[79516]: audit 2023-09-23T18:48:39.616311+0000 mon.a (mon.0) 1246 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:48:39.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:39 smithi098 bash[76817]: audit 2023-09-23T18:48:39.616311+0000 mon.a (mon.0) 1246 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:48:39.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:39.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:39.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:40.049 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:40.058 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:48:40.091 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:40.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:40.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:40.195 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:40.220 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:40.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:40.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:40.323 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:40.346 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:40.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:40.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:40.451 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:40.477 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:40.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:40.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:40.517 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T18:48:40.517 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:48:40.517 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T18:48:40.517 INFO:tasks.thrashosds.thrasher:Setting osd 7 primary_affinity to 1.000000 2023-09-23T18:48:40.517 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-09-23T18:48:40.568 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 27 16 47885 47869 110.797 87.25 0.00508418 0.00901954 2023-09-23T18:48:40.581 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:40.609 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:40.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:40.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:40.713 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:40.739 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:40.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:40.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:40.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:40 smithi173 bash[59422]: audit 2023-09-23T18:48:40.019114+0000 mon.a (mon.0) 1247 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-09-23T18:48:40.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:40 smithi173 bash[59422]: cluster 2023-09-23T18:48:40.082357+0000 mgr.y (mgr.26054) 915 : cluster 0 pgmap v968: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T18:48:40.843 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:40.868 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:40.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:40.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:40.971 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:40 smithi098 bash[79516]: audit 2023-09-23T18:48:40.019114+0000 mon.a (mon.0) 1247 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-09-23T18:48:40.971 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:40 smithi098 bash[79516]: cluster 2023-09-23T18:48:40.082357+0000 mgr.y (mgr.26054) 915 : cluster 0 pgmap v968: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T18:48:40.971 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:40 smithi098 bash[76817]: audit 2023-09-23T18:48:40.019114+0000 mon.a (mon.0) 1247 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-09-23T18:48:40.971 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:40 smithi098 bash[76817]: cluster 2023-09-23T18:48:40.082357+0000 mgr.y (mgr.26054) 915 : cluster 0 pgmap v968: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T18:48:40.971 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:41.008 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:41.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:41.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:41.113 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:41.138 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:41.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:41.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:41.169 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:48:41.243 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:41.266 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:41.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:41.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:41.369 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:41.403 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:41.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:41.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:41.506 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:41.533 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:41.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:41.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:41.568 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 28 16 49079 49063 109.505 74.625 0.00252118 0.00911865 2023-09-23T18:48:41.636 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:41.671 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:41.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:41.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:41.775 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:41.797 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:41.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:41.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:41.900 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:41.939 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:41.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:41.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:42.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:41 smithi173 bash[59422]: audit 2023-09-23T18:48:40.687195+0000 mon.a (mon.0) 1248 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-09-23T18:48:42.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:41 smithi173 bash[59422]: cluster 2023-09-23T18:48:40.687354+0000 mon.a (mon.0) 1249 : cluster 0 osdmap e240: 8 total, 8 up, 8 in 2023-09-23T18:48:42.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:41 smithi173 bash[59422]: cluster 2023-09-23T18:48:40.687689+0000 osd.5 (osd.5) 136 : cluster 3 failed to encode map e240 with expected crc 2023-09-23T18:48:42.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:41 smithi173 bash[59422]: cluster 2023-09-23T18:48:40.690160+0000 osd.6 (osd.6) 199 : cluster 3 failed to encode map e240 with expected crc 2023-09-23T18:48:42.045 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:42.078 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:42.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:42.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:42.184 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:42.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:41 smithi098 bash[79516]: audit 2023-09-23T18:48:40.687195+0000 mon.a (mon.0) 1248 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-09-23T18:48:42.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:41 smithi098 bash[79516]: cluster 2023-09-23T18:48:40.687354+0000 mon.a (mon.0) 1249 : cluster 0 osdmap e240: 8 total, 8 up, 8 in 2023-09-23T18:48:42.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:41 smithi098 bash[79516]: cluster 2023-09-23T18:48:40.687689+0000 osd.5 (osd.5) 136 : cluster 3 failed to encode map e240 with expected crc 2023-09-23T18:48:42.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:41 smithi098 bash[79516]: cluster 2023-09-23T18:48:40.690160+0000 osd.6 (osd.6) 199 : cluster 3 failed to encode map e240 with expected crc 2023-09-23T18:48:42.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:41 smithi098 bash[76817]: audit 2023-09-23T18:48:40.687195+0000 mon.a (mon.0) 1248 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-09-23T18:48:42.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:41 smithi098 bash[76817]: cluster 2023-09-23T18:48:40.687354+0000 mon.a (mon.0) 1249 : cluster 0 osdmap e240: 8 total, 8 up, 8 in 2023-09-23T18:48:42.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:41 smithi098 bash[76817]: cluster 2023-09-23T18:48:40.687689+0000 osd.5 (osd.5) 136 : cluster 3 failed to encode map e240 with expected crc 2023-09-23T18:48:42.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:41 smithi098 bash[76817]: cluster 2023-09-23T18:48:40.690160+0000 osd.6 (osd.6) 199 : cluster 3 failed to encode map e240 with expected crc 2023-09-23T18:48:42.219 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:42.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:42.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:42.325 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:42.354 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:42.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:42.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:42.460 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:42.484 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:42.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:42.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:42.568 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 29 16 50141 50125 108.018 66.375 0.00368636 0.00921329 2023-09-23T18:48:42.587 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:42.615 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:42.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:42.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:42.718 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:42.728 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:42 smithi098 bash[79516]: cluster 2023-09-23T18:48:41.690558+0000 osd.7 (osd.7) 169 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T18:48:42.729 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:42 smithi098 bash[79516]: cluster 2023-09-23T18:48:41.692553+0000 osd.6 (osd.6) 200 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T18:48:42.729 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:42 smithi098 bash[79516]: cluster 2023-09-23T18:48:41.695473+0000 mon.a (mon.0) 1250 : cluster 0 osdmap e241: 8 total, 8 up, 8 in 2023-09-23T18:48:42.729 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:42 smithi098 bash[79516]: cluster 2023-09-23T18:48:41.697217+0000 osd.5 (osd.5) 137 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T18:48:42.729 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:42 smithi098 bash[79516]: cluster 2023-09-23T18:48:41.697787+0000 osd.4 (osd.4) 233 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T18:48:42.729 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:42 smithi098 bash[79516]: cluster 2023-09-23T18:48:41.699595+0000 osd.4 (osd.4) 234 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T18:48:42.729 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:42 smithi098 bash[79516]: cluster 2023-09-23T18:48:41.700097+0000 osd.5 (osd.5) 138 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T18:48:42.729 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:42 smithi098 bash[79516]: cluster 2023-09-23T18:48:41.701357+0000 osd.4 (osd.4) 235 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T18:48:42.729 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:42 smithi098 bash[79516]: cluster 2023-09-23T18:48:42.082816+0000 mgr.y (mgr.26054) 916 : cluster 0 pgmap v971: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-09-23T18:48:42.729 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:42 smithi098 bash[76817]: cluster 2023-09-23T18:48:41.690558+0000 osd.7 (osd.7) 169 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T18:48:42.729 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:42 smithi098 bash[76817]: cluster 2023-09-23T18:48:41.692553+0000 osd.6 (osd.6) 200 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T18:48:42.729 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:42 smithi098 bash[76817]: cluster 2023-09-23T18:48:41.695473+0000 mon.a (mon.0) 1250 : cluster 0 osdmap e241: 8 total, 8 up, 8 in 2023-09-23T18:48:42.730 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:42 smithi098 bash[76817]: cluster 2023-09-23T18:48:41.697217+0000 osd.5 (osd.5) 137 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T18:48:42.730 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:42 smithi098 bash[76817]: cluster 2023-09-23T18:48:41.697787+0000 osd.4 (osd.4) 233 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T18:48:42.730 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:42 smithi098 bash[76817]: cluster 2023-09-23T18:48:41.699595+0000 osd.4 (osd.4) 234 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T18:48:42.730 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:42 smithi098 bash[76817]: cluster 2023-09-23T18:48:41.700097+0000 osd.5 (osd.5) 138 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T18:48:42.730 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:42 smithi098 bash[76817]: cluster 2023-09-23T18:48:41.701357+0000 osd.4 (osd.4) 235 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T18:48:42.730 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:42 smithi098 bash[76817]: cluster 2023-09-23T18:48:42.082816+0000 mgr.y (mgr.26054) 916 : cluster 0 pgmap v971: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-09-23T18:48:42.743 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:42.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:42.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:42.847 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:42.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:42.879 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:42.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:42.980 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:42.989 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:42 smithi173 bash[59422]: cluster 2023-09-23T18:48:41.690558+0000 osd.7 (osd.7) 169 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T18:48:42.990 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:42 smithi173 bash[59422]: cluster 2023-09-23T18:48:41.692553+0000 osd.6 (osd.6) 200 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T18:48:42.990 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:42 smithi173 bash[59422]: cluster 2023-09-23T18:48:41.695473+0000 mon.a (mon.0) 1250 : cluster 0 osdmap e241: 8 total, 8 up, 8 in 2023-09-23T18:48:42.990 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:42 smithi173 bash[59422]: cluster 2023-09-23T18:48:41.697217+0000 osd.5 (osd.5) 137 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T18:48:42.990 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:42 smithi173 bash[59422]: cluster 2023-09-23T18:48:41.697787+0000 osd.4 (osd.4) 233 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T18:48:42.990 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:42 smithi173 bash[59422]: cluster 2023-09-23T18:48:41.699595+0000 osd.4 (osd.4) 234 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T18:48:42.990 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:42 smithi173 bash[59422]: cluster 2023-09-23T18:48:41.700097+0000 osd.5 (osd.5) 138 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T18:48:42.990 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:42 smithi173 bash[59422]: cluster 2023-09-23T18:48:41.701357+0000 osd.4 (osd.4) 235 : cluster 3 failed to encode map e241 with expected crc 2023-09-23T18:48:42.990 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:42 smithi173 bash[59422]: cluster 2023-09-23T18:48:42.082816+0000 mgr.y (mgr.26054) 916 : cluster 0 pgmap v971: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-09-23T18:48:43.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:43.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:43.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:43.123 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:43.159 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:43.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:43.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:43.262 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:43.298 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:43.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:43.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:43.403 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:43.429 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:43.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:43.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:43.535 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:43.566 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:43.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:43.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:43.568 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 30 16 51138 51122 106.494 62.3125 0.0120827 0.00938363 2023-09-23T18:48:43.669 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:43.701 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:43.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:43.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:43.808 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:43.835 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:43.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:43.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:43.938 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:43.974 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:43.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:43.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:44.078 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:44.110 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:44.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:44.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:44.215 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:44.231 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:44 smithi098 bash[79516]: cluster 2023-09-23T18:48:44.083433+0000 mgr.y (mgr.26054) 917 : cluster 0 pgmap v972: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-09-23T18:48:44.231 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:44 smithi098 bash[76817]: cluster 2023-09-23T18:48:44.083433+0000 mgr.y (mgr.26054) 917 : cluster 0 pgmap v972: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-09-23T18:48:44.246 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:44.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:44.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:44.350 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:44.374 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:44.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:44.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:44.482 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:44.485 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:44 smithi173 bash[59422]: cluster 2023-09-23T18:48:44.083433+0000 mgr.y (mgr.26054) 917 : cluster 0 pgmap v972: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-09-23T18:48:44.524 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:44.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:44.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:44.568 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 31 16 52431 52415 105.665 80.8125 0.0083841 0.00945814 2023-09-23T18:48:44.629 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:44.661 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:44.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:44.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:44.764 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:44.796 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:44.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:44.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:44.899 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:44.922 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:44.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:44.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:45.026 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:45.050 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:45.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:45.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:45.153 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:45.181 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:45.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:45.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:45.288 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:45.314 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:45.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:45.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:45.417 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:45.443 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:45.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:45.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:45.552 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:45.568 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 32 16 53730 53714 104.9 81.1875 0.0196905 0.0095268 2023-09-23T18:48:45.583 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:45.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:45.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:45.689 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:45.713 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:45.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:45.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:45.820 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:45.852 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:45.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:45.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:45.955 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:45.984 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:45.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:45.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:46.088 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:46.118 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:46.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:46.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:46.221 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:46.249 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:46.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:46.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:46.351 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:46.385 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:46.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:46.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:46.442 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:48:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:48:46] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.33.4" 2023-09-23T18:48:46.488 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:46.518 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:46.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:46.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:46.568 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 33 16 54866 54850 103.873 71 0.00838258 0.00962144 2023-09-23T18:48:46.622 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:46.650 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:46.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:46.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:46.753 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:46.786 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:46.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:46.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:46.892 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:46.917 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:46.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:46.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:47.021 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:47.055 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:47.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:47.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:47.145 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:48:47.145 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-09-23T18:48:47.145 INFO:teuthology.orchestra.run.smithi173.stdout: "complaint_time": 30, 2023-09-23T18:48:47.145 INFO:teuthology.orchestra.run.smithi173.stdout: "num_blocked_ops": 0 2023-09-23T18:48:47.146 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:48:47.161 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:47.170 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:47 smithi173 bash[59422]: cluster 2023-09-23T18:48:46.083915+0000 mgr.y (mgr.26054) 918 : cluster 0 pgmap v973: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-09-23T18:48:47.174 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:47 smithi098 bash[76817]: cluster 2023-09-23T18:48:46.083915+0000 mgr.y (mgr.26054) 918 : cluster 0 pgmap v973: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-09-23T18:48:47.188 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:47.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:47.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:47.297 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:47.324 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:47.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:47.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:47.428 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:47.452 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:47.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:47.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:47.488 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:47 smithi098 bash[79516]: cluster 2023-09-23T18:48:46.083915+0000 mgr.y (mgr.26054) 918 : cluster 0 pgmap v973: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-09-23T18:48:47.555 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:47.568 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 34 16 55886 55870 102.692 63.75 0.0281271 0.00973245 2023-09-23T18:48:47.583 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:47.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:47.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:47.687 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:47.710 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:47.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:47.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:47.814 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:47.840 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:47.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:47.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:47.944 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:47.967 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:47.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:47.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:48.071 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:48.098 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:48.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:48.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:48.201 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:48.236 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:48.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:48.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:48.281 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T18:48:48.347 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:48.374 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:48.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:48.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:48.478 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:48.512 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:48.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:48.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:48.568 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 35 16 56906 56890 101.58 63.75 0.00604304 0.00983801 2023-09-23T18:48:48.617 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:48.642 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:48.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:48.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:48.745 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:48.774 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:48.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:48.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:48.878 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:48.908 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:48.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:48.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:49.012 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:49.036 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:49.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:49.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:49.060 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:48:49.090 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:48:49.125 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:48:49.139 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:49.140 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:49 smithi098 bash[76817]: cluster 2023-09-23T18:48:48.084500+0000 mgr.y (mgr.26054) 919 : cluster 0 pgmap v974: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T18:48:49.140 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:49 smithi098 bash[76817]: audit 2023-09-23T18:48:49.028005+0000 mon.a (mon.0) 1251 : audit 0 from='client.? 172.21.15.98:0/3590378070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:48:49.165 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:49.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:49.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:49.185 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:48:49.215 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:48:49.258 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:48:49.268 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:49.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:49 smithi173 bash[59422]: cluster 2023-09-23T18:48:48.084500+0000 mgr.y (mgr.26054) 919 : cluster 0 pgmap v974: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T18:48:49.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:49 smithi173 bash[59422]: audit 2023-09-23T18:48:49.028005+0000 mon.a (mon.0) 1251 : audit 0 from='client.? 172.21.15.98:0/3590378070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:48:49.286 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:48:49.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:49.298 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:49.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:49.323 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:48:49.328 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:48:49.328 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:48:49.328 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:48:49.329 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:48:49.329 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:48:49.329 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:48:49.329 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:48:49.329 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:48:49.399 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:49.426 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:49.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:49.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:49.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:49 smithi098 bash[79516]: cluster 2023-09-23T18:48:48.084500+0000 mgr.y (mgr.26054) 919 : cluster 0 pgmap v974: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T18:48:49.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:49 smithi098 bash[79516]: audit 2023-09-23T18:48:49.028005+0000 mon.a (mon.0) 1251 : audit 0 from='client.? 172.21.15.98:0/3590378070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:48:49.530 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:49.569 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 36 16 58039 58023 100.725 70.8125 0.0047835 0.00992156 2023-09-23T18:48:49.579 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:49.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:49.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:49.684 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:49.708 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:49.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:49.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:49.816 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:49.852 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:49.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:49.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:49.956 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:49.990 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:49.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:49.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:50.095 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:50.135 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:50.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:50.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:50.240 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:50.278 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:50.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:50.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:50.383 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:50.418 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:50.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:50.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:50.524 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:50.551 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:50.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:50.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:50.569 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 37 16 59235 59219 100.023 74.75 0.00835804 0.0099922 2023-09-23T18:48:50.654 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:50.684 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:50.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:50.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:50.787 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:50.813 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:50.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:50.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:50.916 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:50.960 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:50.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:50.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:51.065 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:51.103 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:51.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:51.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:51.142 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:51 smithi173 bash[59422]: cluster 2023-09-23T18:48:50.084984+0000 mgr.y (mgr.26054) 920 : cluster 0 pgmap v975: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T18:48:51.142 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:51 smithi173 bash[59422]: audit 2023-09-23T18:48:51.111141+0000 mon.b (mon.1) 18 : audit 1 from='client.? 172.21.15.98:0/2863114133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:48:51.142 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:51 smithi173 bash[59422]: audit 2023-09-23T18:48:51.112016+0000 mon.a (mon.0) 1252 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:48:51.160 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is unset 2023-09-23T18:48:51.207 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:51.232 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:51.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:51.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:51.337 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:51.346 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:51 smithi098 bash[79516]: cluster 2023-09-23T18:48:50.084984+0000 mgr.y (mgr.26054) 920 : cluster 0 pgmap v975: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T18:48:51.346 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:51 smithi098 bash[79516]: audit 2023-09-23T18:48:51.111141+0000 mon.b (mon.1) 18 : audit 1 from='client.? 172.21.15.98:0/2863114133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:48:51.346 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:51 smithi098 bash[79516]: audit 2023-09-23T18:48:51.112016+0000 mon.a (mon.0) 1252 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:48:51.347 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:51 smithi098 bash[76817]: cluster 2023-09-23T18:48:50.084984+0000 mgr.y (mgr.26054) 920 : cluster 0 pgmap v975: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T18:48:51.347 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:51 smithi098 bash[76817]: audit 2023-09-23T18:48:51.111141+0000 mon.b (mon.1) 18 : audit 1 from='client.? 172.21.15.98:0/2863114133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:48:51.347 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:51 smithi098 bash[76817]: audit 2023-09-23T18:48:51.112016+0000 mon.a (mon.0) 1252 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:48:51.363 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:51.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:51.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:51.467 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:51.501 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:51.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:51.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:51.569 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 38 15 60402 60387 99.3113 73 0.014188 0.0100642 2023-09-23T18:48:51.607 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:51.641 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:51.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:51.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:51.745 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:51.770 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:51.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:51.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:51.876 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:51.903 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:51.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:51.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:52.007 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:52.032 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:52.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:52.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:52.140 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:52.143 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:52 smithi098 bash[79516]: cluster 2023-09-23T18:48:51.141074+0000 mon.a (mon.0) 1253 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:48:52.143 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:52 smithi098 bash[79516]: audit 2023-09-23T18:48:51.147475+0000 mon.a (mon.0) 1254 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:48:52.143 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:52 smithi098 bash[79516]: cluster 2023-09-23T18:48:51.147560+0000 mon.a (mon.0) 1255 : cluster 0 osdmap e242: 8 total, 8 up, 8 in 2023-09-23T18:48:52.143 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:52 smithi098 bash[79516]: cluster 2023-09-23T18:48:51.151392+0000 osd.6 (osd.6) 201 : cluster 3 failed to encode map e242 with expected crc 2023-09-23T18:48:52.143 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:52 smithi098 bash[79516]: cluster 2023-09-23T18:48:51.152774+0000 osd.5 (osd.5) 139 : cluster 3 failed to encode map e242 with expected crc 2023-09-23T18:48:52.143 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:52 smithi098 bash[79516]: cluster 2023-09-23T18:48:51.163798+0000 osd.7 (osd.7) 170 : cluster 3 failed to encode map e242 with expected crc 2023-09-23T18:48:52.144 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:52 smithi098 bash[79516]: cluster 2023-09-23T18:48:52.085661+0000 mgr.y (mgr.26054) 921 : cluster 0 pgmap v977: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T18:48:52.174 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:52.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:52.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:52.278 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:52.307 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:52.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:52.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:52.413 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:52.424 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:52 smithi173 bash[59422]: cluster 2023-09-23T18:48:51.141074+0000 mon.a (mon.0) 1253 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:48:52.424 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:52 smithi173 bash[59422]: audit 2023-09-23T18:48:51.147475+0000 mon.a (mon.0) 1254 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:48:52.424 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:52 smithi173 bash[59422]: cluster 2023-09-23T18:48:51.147560+0000 mon.a (mon.0) 1255 : cluster 0 osdmap e242: 8 total, 8 up, 8 in 2023-09-23T18:48:52.424 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:52 smithi173 bash[59422]: cluster 2023-09-23T18:48:51.151392+0000 osd.6 (osd.6) 201 : cluster 3 failed to encode map e242 with expected crc 2023-09-23T18:48:52.425 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:52 smithi173 bash[59422]: cluster 2023-09-23T18:48:51.152774+0000 osd.5 (osd.5) 139 : cluster 3 failed to encode map e242 with expected crc 2023-09-23T18:48:52.425 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:52 smithi173 bash[59422]: cluster 2023-09-23T18:48:51.163798+0000 osd.7 (osd.7) 170 : cluster 3 failed to encode map e242 with expected crc 2023-09-23T18:48:52.425 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:52 smithi173 bash[59422]: cluster 2023-09-23T18:48:52.085661+0000 mgr.y (mgr.26054) 921 : cluster 0 pgmap v977: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T18:48:52.440 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:52.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:52 smithi098 bash[76817]: cluster 2023-09-23T18:48:51.141074+0000 mon.a (mon.0) 1253 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:48:52.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:52 smithi098 bash[76817]: audit 2023-09-23T18:48:51.147475+0000 mon.a (mon.0) 1254 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:48:52.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:52 smithi098 bash[76817]: cluster 2023-09-23T18:48:51.147560+0000 mon.a (mon.0) 1255 : cluster 0 osdmap e242: 8 total, 8 up, 8 in 2023-09-23T18:48:52.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:52 smithi098 bash[76817]: cluster 2023-09-23T18:48:51.151392+0000 osd.6 (osd.6) 201 : cluster 3 failed to encode map e242 with expected crc 2023-09-23T18:48:52.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:52 smithi098 bash[76817]: cluster 2023-09-23T18:48:51.152774+0000 osd.5 (osd.5) 139 : cluster 3 failed to encode map e242 with expected crc 2023-09-23T18:48:52.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:52 smithi098 bash[76817]: cluster 2023-09-23T18:48:51.163798+0000 osd.7 (osd.7) 170 : cluster 3 failed to encode map e242 with expected crc 2023-09-23T18:48:52.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:52 smithi098 bash[76817]: cluster 2023-09-23T18:48:52.085661+0000 mgr.y (mgr.26054) 921 : cluster 0 pgmap v977: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T18:48:52.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:52.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:52.544 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:52.569 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 39 16 61601 61585 98.6846 74.875 0.0350305 0.0101269 2023-09-23T18:48:52.576 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:52.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:52.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:52.680 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:52.709 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:52.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:52.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:52.812 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:52.839 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:52.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:52.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:52.945 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:52.977 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:52.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:52.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:53.049 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:48:53.080 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:53.109 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:48:53.110 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:53.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:53.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:53.144 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:48:53.213 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:53.248 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:53.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:53.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:53.354 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:53.388 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:53.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:53.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:53.492 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:53.515 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:53.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:53.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:53.569 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:2023-09-23T18:48:53.567743+0000 min lat: 0.00122919 max lat: 0.407963 avg lat: 0.0101979 2023-09-23T18:48:53.569 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:48:53.569 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 40 16 62736 62720 97.9907 70.9375 0.00751757 0.0101979 2023-09-23T18:48:53.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:53.652 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:53.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:53.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:53.756 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:53.781 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:53.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:53.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:53.884 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:53.907 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:53.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:53.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:54.011 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:54.042 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:54.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:54.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:54.146 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:54.178 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:54.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:54.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:54.192 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:54 smithi173 bash[59422]: cluster 2023-09-23T18:48:54.086141+0000 mgr.y (mgr.26054) 922 : cluster 0 pgmap v978: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T18:48:54.282 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:54.313 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:54.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:54.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:54.421 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:54.430 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:54 smithi098 bash[79516]: cluster 2023-09-23T18:48:54.086141+0000 mgr.y (mgr.26054) 922 : cluster 0 pgmap v978: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T18:48:54.431 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:54 smithi098 bash[76817]: cluster 2023-09-23T18:48:54.086141+0000 mgr.y (mgr.26054) 922 : cluster 0 pgmap v978: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T18:48:54.447 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:54.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:54.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:54.551 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:54.569 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 41 16 63893 63877 97.3643 72.3125 0.00957591 0.0102648 2023-09-23T18:48:54.578 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:54.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:54.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:54.632 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:48:54.683 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:54.717 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:54.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:54.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:54.821 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:54.856 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:54.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:54.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:54.959 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:54.985 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:54.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:54.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:55.090 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:55.119 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:55.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:55.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:55.222 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:55.230 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:55 smithi098 bash[76817]: audit 2023-09-23T18:48:54.616777+0000 mon.a (mon.0) 1256 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:48:55.231 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:55 smithi098 bash[79516]: audit 2023-09-23T18:48:54.616777+0000 mon.a (mon.0) 1256 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:48:55.251 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:55.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:55.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:55.355 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:55.364 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:55 smithi173 bash[59422]: audit 2023-09-23T18:48:54.616777+0000 mon.a (mon.0) 1256 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:48:55.390 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:55.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:55.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:55.495 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:55.520 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:55.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:55.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:55.569 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 42 16 64996 64980 96.6873 68.9375 0.0205857 0.0103376 2023-09-23T18:48:55.627 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:55.660 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:55.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:55.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:55.764 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:55.787 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:55.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:55.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:55.890 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:55.927 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:55.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:55.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:56.029 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:56.052 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:56.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:56.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:56.156 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:56.204 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:56.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:56.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:56.310 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:56.317 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:56 smithi173 bash[59422]: cluster 2023-09-23T18:48:56.086722+0000 mgr.y (mgr.26054) 923 : cluster 0 pgmap v979: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T18:48:56.330 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:56.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:56.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:56.434 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:56.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:56 smithi098 bash[79516]: cluster 2023-09-23T18:48:56.086722+0000 mgr.y (mgr.26054) 923 : cluster 0 pgmap v979: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T18:48:56.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:56 smithi098 bash[76817]: cluster 2023-09-23T18:48:56.086722+0000 mgr.y (mgr.26054) 923 : cluster 0 pgmap v979: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T18:48:56.442 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:48:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:48:56] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.33.4" 2023-09-23T18:48:56.460 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:56.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:56.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:56.563 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:56.569 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 43 16 66024 66008 95.9328 64.25 0.00722396 0.0104183 2023-09-23T18:48:56.589 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:56.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:56.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:56.692 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:56.722 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:56.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:56.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:56.825 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:56.861 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:56.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:56.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:56.967 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:56.992 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:56.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:56.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:57.095 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:57.129 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:57.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:57.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:57.233 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:57.268 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:57.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:57.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:57.363 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:48:57.378 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:57.420 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:57.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:57.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:57.528 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:57.563 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:57.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:57.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:57.569 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 44 16 67133 67117 95.3276 69.3125 0.00854287 0.0104831 2023-09-23T18:48:57.669 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:57.699 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:57.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:57.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:57.805 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:57.827 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:48:57.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:57.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:48:57.931 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:57.957 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:57.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:57.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:58.068 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:58.105 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:48:58.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:58.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:48:58.195 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:48:58 smithi173 bash[59422]: cluster 2023-09-23T18:48:58.087319+0000 mgr.y (mgr.26054) 924 : cluster 0 pgmap v980: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T18:48:58.209 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:58.239 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:58.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:58.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:58.337 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:48:58 smithi098 bash[79516]: cluster 2023-09-23T18:48:58.087319+0000 mgr.y (mgr.26054) 924 : cluster 0 pgmap v980: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T18:48:58.338 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:48:58 smithi098 bash[76817]: cluster 2023-09-23T18:48:58.087319+0000 mgr.y (mgr.26054) 924 : cluster 0 pgmap v980: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T18:48:58.343 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:58.371 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:58.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:58.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:58.478 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:58.502 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:58.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:58.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:58.569 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 45 16 68248 68232 94.7576 69.6875 0.00983823 0.010547 2023-09-23T18:48:58.607 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:58.753 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:58.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:58.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:58.800 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T18:48:58.800 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:48:58.856 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:58.879 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:48:58.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:58.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:48:58.982 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:59.000 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:59.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:59.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:59.103 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:59.138 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:48:59.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:59.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:48:59.198 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:48:59.245 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:59.276 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:48:59.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:59.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:48:59.381 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:59.413 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:59.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:59.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:59.517 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:59.543 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:48:59.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:59.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:48:59.570 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 46 16 69199 69183 93.9897 59.4375 0.00973072 0.0106341 2023-09-23T18:48:59.647 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:59.677 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:48:59.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:59.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:48:59.782 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:59.811 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:59.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:59.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:48:59.902 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T18:48:59.917 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:59.989 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:48:59.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:48:59.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:00.097 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:00.185 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:00.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:00.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:00.289 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:00.315 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:00.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:00.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:00.422 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:00.449 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:00.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:00.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:00.553 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:00.569 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 47 16 70255 70239 93.394 66 0.0112626 0.0107019 2023-09-23T18:49:00.584 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:00.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:00.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:00.689 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:00.714 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:00.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:00.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:00.823 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:00.850 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:00.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:00.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:00.958 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:00.989 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:00.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:00.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:01.094 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:01.115 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:01.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:01.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:01.218 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:01.226 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:01 smithi098 bash[79516]: cluster 2023-09-23T18:49:00.087763+0000 mgr.y (mgr.26054) 925 : cluster 0 pgmap v981: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T18:49:01.226 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:01 smithi098 bash[76817]: cluster 2023-09-23T18:49:00.087763+0000 mgr.y (mgr.26054) 925 : cluster 0 pgmap v981: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T18:49:01.258 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:01.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:01.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:01.362 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:01.401 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:01.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:01.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:01.505 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:01.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:01 smithi173 bash[59422]: cluster 2023-09-23T18:49:00.087763+0000 mgr.y (mgr.26054) 925 : cluster 0 pgmap v981: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T18:49:01.550 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:01.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:01.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:01.570 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 48 16 71294 71278 92.8011 64.9375 0.0134081 0.0107699 2023-09-23T18:49:01.655 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:01.682 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:01.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:01.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:01.787 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:01.829 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:01.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:01.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:01.932 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:01.962 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:01.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:01.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:02.067 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:02.099 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:02.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:02.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:02.203 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:02.242 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:02.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:02.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:02.344 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:02.383 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:02.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:02.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:02.488 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:02.515 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:02.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:02.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:02.570 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 49 16 72326 72310 92.2234 64.5 0.0159562 0.0108378 2023-09-23T18:49:02.619 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:02.645 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:02.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:02.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:02.750 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:02.774 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:02.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:02.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:02.877 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:02.904 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:02.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:02.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:03.009 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:03.033 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:03.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:03.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:03.138 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:03.154 INFO:teuthology.orchestra.run.smithi098.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-09-23T18:49:03.162 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:03.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:03.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:03.268 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:03.277 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:03 smithi098 bash[76817]: cluster 2023-09-23T18:49:02.088381+0000 mgr.y (mgr.26054) 926 : cluster 0 pgmap v982: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T18:49:03.277 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:03 smithi098 bash[76817]: audit 2023-09-23T18:49:02.278825+0000 mon.a (mon.0) 1257 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T18:49:03.278 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:03 smithi098 bash[76817]: audit 2023-09-23T18:49:02.279020+0000 mon.b (mon.1) 19 : audit 1 from='client.? 172.21.15.98:0/3030946333' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T18:49:03.278 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:03 smithi098 bash[79516]: cluster 2023-09-23T18:49:02.088381+0000 mgr.y (mgr.26054) 926 : cluster 0 pgmap v982: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T18:49:03.278 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:03 smithi098 bash[79516]: audit 2023-09-23T18:49:02.278825+0000 mon.a (mon.0) 1257 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T18:49:03.278 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:03 smithi098 bash[79516]: audit 2023-09-23T18:49:02.279020+0000 mon.b (mon.1) 19 : audit 1 from='client.? 172.21.15.98:0/3030946333' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T18:49:03.298 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:03.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:03.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:03.402 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:03.429 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:03.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:03.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:03 smithi173 bash[59422]: cluster 2023-09-23T18:49:02.088381+0000 mgr.y (mgr.26054) 926 : cluster 0 pgmap v982: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T18:49:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:03 smithi173 bash[59422]: audit 2023-09-23T18:49:02.278825+0000 mon.a (mon.0) 1257 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T18:49:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:03 smithi173 bash[59422]: audit 2023-09-23T18:49:02.279020+0000 mon.b (mon.1) 19 : audit 1 from='client.? 172.21.15.98:0/3030946333' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T18:49:03.532 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:03.554 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:03.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:03.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:03.570 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 50 16 73383 73367 91.7 66.0625 0.00637548 0.0108996 2023-09-23T18:49:03.658 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:03.717 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:03.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:03.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:03.820 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:03.850 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:03.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:03.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:03.956 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:03.987 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:03.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:03.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:04.092 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:04.137 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:04.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:04.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:04.241 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:04.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:04 smithi173 bash[59422]: audit 2023-09-23T18:49:03.142899+0000 mon.a (mon.0) 1258 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T18:49:04.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:04 smithi173 bash[59422]: cluster 2023-09-23T18:49:03.143059+0000 mon.a (mon.0) 1259 : cluster 0 osdmap e243: 8 total, 8 up, 8 in 2023-09-23T18:49:04.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:04 smithi173 bash[59422]: cluster 2023-09-23T18:49:03.145085+0000 osd.4 (osd.4) 236 : cluster 3 failed to encode map e243 with expected crc 2023-09-23T18:49:04.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:04 smithi173 bash[59422]: cluster 2023-09-23T18:49:03.145744+0000 osd.6 (osd.6) 202 : cluster 3 failed to encode map e243 with expected crc 2023-09-23T18:49:04.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:04 smithi173 bash[59422]: cluster 2023-09-23T18:49:03.153213+0000 osd.5 (osd.5) 140 : cluster 3 failed to encode map e243 with expected crc 2023-09-23T18:49:04.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:04 smithi173 bash[59422]: cluster 2023-09-23T18:49:04.089345+0000 mgr.y (mgr.26054) 927 : cluster 0 pgmap v984: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T18:49:04.297 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:04.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:04.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:04.400 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:04.434 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:04.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:04.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:04.486 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:04 smithi098 bash[79516]: audit 2023-09-23T18:49:03.142899+0000 mon.a (mon.0) 1258 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T18:49:04.486 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:04 smithi098 bash[79516]: cluster 2023-09-23T18:49:03.143059+0000 mon.a (mon.0) 1259 : cluster 0 osdmap e243: 8 total, 8 up, 8 in 2023-09-23T18:49:04.486 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:04 smithi098 bash[79516]: cluster 2023-09-23T18:49:03.145085+0000 osd.4 (osd.4) 236 : cluster 3 failed to encode map e243 with expected crc 2023-09-23T18:49:04.486 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:04 smithi098 bash[79516]: cluster 2023-09-23T18:49:03.145744+0000 osd.6 (osd.6) 202 : cluster 3 failed to encode map e243 with expected crc 2023-09-23T18:49:04.486 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:04 smithi098 bash[79516]: cluster 2023-09-23T18:49:03.153213+0000 osd.5 (osd.5) 140 : cluster 3 failed to encode map e243 with expected crc 2023-09-23T18:49:04.486 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:04 smithi098 bash[79516]: cluster 2023-09-23T18:49:04.089345+0000 mgr.y (mgr.26054) 927 : cluster 0 pgmap v984: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T18:49:04.487 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:04 smithi098 bash[76817]: audit 2023-09-23T18:49:03.142899+0000 mon.a (mon.0) 1258 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T18:49:04.487 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:04 smithi098 bash[76817]: cluster 2023-09-23T18:49:03.143059+0000 mon.a (mon.0) 1259 : cluster 0 osdmap e243: 8 total, 8 up, 8 in 2023-09-23T18:49:04.487 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:04 smithi098 bash[76817]: cluster 2023-09-23T18:49:03.145085+0000 osd.4 (osd.4) 236 : cluster 3 failed to encode map e243 with expected crc 2023-09-23T18:49:04.487 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:04 smithi098 bash[76817]: cluster 2023-09-23T18:49:03.145744+0000 osd.6 (osd.6) 202 : cluster 3 failed to encode map e243 with expected crc 2023-09-23T18:49:04.487 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:04 smithi098 bash[76817]: cluster 2023-09-23T18:49:03.153213+0000 osd.5 (osd.5) 140 : cluster 3 failed to encode map e243 with expected crc 2023-09-23T18:49:04.487 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:04 smithi098 bash[76817]: cluster 2023-09-23T18:49:04.089345+0000 mgr.y (mgr.26054) 927 : cluster 0 pgmap v984: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T18:49:04.539 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:04.570 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 51 16 74349 74333 91.0857 60.375 0.0109415 0.0109727 2023-09-23T18:49:04.579 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:04.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:04.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:04.683 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:04.714 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:04.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:04.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:04.789 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:49:04.818 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:04.852 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:04.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:04.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:04.957 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:04.995 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:04.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:04.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:05.099 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:05.132 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:05.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:05.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:05.238 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:05.266 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:05.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:05.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:05.370 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:05.410 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:05.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:05.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:05.513 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:05.544 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:05.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:05.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:05.570 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 52 16 75404 75388 90.6019 65.9375 0.0100166 0.011031 2023-09-23T18:49:05.649 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:05.690 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:05.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:05.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:05.795 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:05.825 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:05.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:05.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:05.932 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:05.966 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:05.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:05.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:06.069 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:06.086 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:49:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:49:06] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.33.4" 2023-09-23T18:49:06.217 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:06.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:06.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:06.322 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:06.351 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:06.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:06.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:06.454 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:06.475 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:06.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:06.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:06.570 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 53 16 76476 76460 90.1565 67 0.00434983 0.0110857 2023-09-23T18:49:06.578 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:06.604 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:06.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:06.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:06.708 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:06.747 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:06.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:06.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:06.851 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:06.985 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:06.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:06.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:07.088 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:07.115 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:07.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:07.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:07.220 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:07.245 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:07.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:07.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:07.264 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:07 smithi173 bash[59422]: cluster 2023-09-23T18:49:06.089982+0000 mgr.y (mgr.26054) 928 : cluster 0 pgmap v985: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T18:49:07.348 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:07.387 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:07.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:07.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:07.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:07 smithi098 bash[79516]: cluster 2023-09-23T18:49:06.089982+0000 mgr.y (mgr.26054) 928 : cluster 0 pgmap v985: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T18:49:07.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:07 smithi098 bash[76817]: cluster 2023-09-23T18:49:06.089982+0000 mgr.y (mgr.26054) 928 : cluster 0 pgmap v985: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T18:49:07.493 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:07.522 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:07.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:07.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:07.570 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 54 16 77506 77490 89.679 64.375 0.0044873 0.0111452 2023-09-23T18:49:07.629 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:07.677 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:07.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:07.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:07.780 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:07.806 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:07.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:07.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:07.913 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:07.943 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:07.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:07.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:08.048 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:08.077 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:08.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:08.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:08.180 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:08.211 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:08.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:08.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:08.315 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:08.343 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:08.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:08.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:08.447 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:08.473 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:08.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:08.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:08.570 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 55 16 78518 78502 89.1983 63.25 0.0195696 0.0112055 2023-09-23T18:49:08.576 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:08.604 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:08.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:08.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:08.710 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:08.734 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:08.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:08.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:08.838 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:08.864 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:08.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:08.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:08.969 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:09.004 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:09.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:09.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:09.108 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:09.139 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:09.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:09.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:09.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:09 smithi098 bash[76817]: cluster 2023-09-23T18:49:08.090584+0000 mgr.y (mgr.26054) 929 : cluster 0 pgmap v986: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-09-23T18:49:09.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:09 smithi098 bash[79516]: cluster 2023-09-23T18:49:08.090584+0000 mgr.y (mgr.26054) 929 : cluster 0 pgmap v986: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-09-23T18:49:09.245 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:09.271 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:09.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:09.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:09.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:09 smithi173 bash[59422]: cluster 2023-09-23T18:49:08.090584+0000 mgr.y (mgr.26054) 929 : cluster 0 pgmap v986: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-09-23T18:49:09.375 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:09.570 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 56 16 79488 79472 88.688 60.625 0.0240624 0.0112697 2023-09-23T18:49:09.583 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:09.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:09.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:09.644 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:49:09.678 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:49:09.686 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:09.710 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:49:09.714 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:09.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:09.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:09.756 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:49:09.810 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:49:09.818 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:09.841 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:09.841 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:49:09.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:09.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:09.877 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:49:09.923 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:49:09.930 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:49:09.930 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:49:09.930 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:49:09.930 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:49:09.931 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:49:09.931 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:49:09.931 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:49:09.931 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:49:09.944 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:09.973 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:09.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:09.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:10.078 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:10.109 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:10.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:10.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:10.154 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:10 smithi173 bash[59422]: audit 2023-09-23T18:49:09.594713+0000 mon.b (mon.1) 20 : audit 0 from='client.? 172.21.15.98:0/4270165515' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:49:10.154 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:10 smithi173 bash[59422]: audit 2023-09-23T18:49:09.621200+0000 mon.a (mon.0) 1260 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:49:10.170 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:49:10.170 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-09-23T18:49:10.170 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.170 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35412.0:79987 8.9 8:9ce9428f:::benchmark_data_smithi098_176514_object79986:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e243)", 2023-09-23T18:49:10.170 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:49:10.162944+0000", 2023-09-23T18:49:10.170 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.003493711, 2023-09-23T18:49:10.170 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0035282450000000002, 2023-09-23T18:49:10.170 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:49:10.171 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:49:10.171 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:49:10.171 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35412", 2023-09-23T18:49:10.171 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:49:10.171 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 79987 2023-09-23T18:49:10.171 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.171 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:49:10.171 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.171 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:49:10.171 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.162944+0000", 2023-09-23T18:49:10.171 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:49:10.171 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.172 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.172 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:49:10.172 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.162944+0000", 2023-09-23T18:49:10.172 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:49:10.172 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.172 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.172 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:49:10.172 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.162943+0000", 2023-09-23T18:49:10.172 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012792000000000001 2023-09-23T18:49:10.172 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.172 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.173 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:49:10.173 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.163070+0000", 2023-09-23T18:49:10.173 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1935e-05 2023-09-23T18:49:10.173 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.173 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.173 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:49:10.173 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.163082+0000", 2023-09-23T18:49:10.173 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.5889999999999999e-06 2023-09-23T18:49:10.173 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.173 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.173 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:49:10.173 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.163089+0000", 2023-09-23T18:49:10.174 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.9759999999999998e-05 2023-09-23T18:49:10.175 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.175 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.175 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:49:10.175 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.163139+0000", 2023-09-23T18:49:10.175 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012299000000000001 2023-09-23T18:49:10.175 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.175 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.175 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:49:10.175 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.163262+0000", 2023-09-23T18:49:10.175 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014594599999999999 2023-09-23T18:49:10.175 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.176 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.176 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,6", 2023-09-23T18:49:10.176 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.163408+0000", 2023-09-23T18:49:10.176 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000463349 2023-09-23T18:49:10.176 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:49:10.176 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:49:10.176 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:49:10.176 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.176 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.176 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35412.0:79980 8.2 e243/233 8:41bab8dd:::benchmark_data_smithi098_176514_object79979:head v 243'4966, mlcod=243'4963)", 2023-09-23T18:49:10.176 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:49:10.165119+0000", 2023-09-23T18:49:10.176 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.0013192449999999999, 2023-09-23T18:49:10.177 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0014400159999999999, 2023-09-23T18:49:10.177 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:49:10.177 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "started", 2023-09-23T18:49:10.177 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:49:10.177 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.177 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:49:10.177 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.165119+0000", 2023-09-23T18:49:10.177 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:49:10.177 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.177 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.177 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:49:10.177 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.165119+0000", 2023-09-23T18:49:10.177 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:49:10.178 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.178 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.178 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:49:10.178 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.165118+0000", 2023-09-23T18:49:10.178 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.6611999999999999e-05 2023-09-23T18:49:10.178 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.178 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.178 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:49:10.178 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.165144+0000", 2023-09-23T18:49:10.178 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7762999999999999e-05 2023-09-23T18:49:10.178 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.178 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.179 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:49:10.179 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.165162+0000", 2023-09-23T18:49:10.179 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.7950000000000002e-06 2023-09-23T18:49:10.179 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.179 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.179 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:49:10.179 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.165169+0000", 2023-09-23T18:49:10.179 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.2328000000000004e-05 2023-09-23T18:49:10.179 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.179 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.179 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:49:10.180 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.165231+0000", 2023-09-23T18:49:10.180 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8386e-05 2023-09-23T18:49:10.180 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.180 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.180 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:49:10.180 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.165250+0000", 2023-09-23T18:49:10.180 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000130884 2023-09-23T18:49:10.180 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:49:10.180 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:49:10.180 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:49:10.180 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.180 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.181 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35412.0:79973 8.8 8:15893c45:::benchmark_data_smithi098_176514_object79972:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e243)", 2023-09-23T18:49:10.181 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:49:10.148299+0000", 2023-09-23T18:49:10.181 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.018139130999999999, 2023-09-23T18:49:10.181 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.018329708, 2023-09-23T18:49:10.181 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:49:10.181 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:49:10.181 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:49:10.181 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35412", 2023-09-23T18:49:10.181 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:49:10.181 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 79973 2023-09-23T18:49:10.181 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.181 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:49:10.182 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.182 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:49:10.182 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.148299+0000", 2023-09-23T18:49:10.182 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:49:10.182 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.182 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.182 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:49:10.182 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.148299+0000", 2023-09-23T18:49:10.182 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:49:10.182 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.182 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.182 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:49:10.183 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.148297+0000", 2023-09-23T18:49:10.183 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4493999999999997e-05 2023-09-23T18:49:10.183 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.183 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.183 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:49:10.183 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.148352+0000", 2023-09-23T18:49:10.183 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1362000000000001e-05 2023-09-23T18:49:10.183 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.183 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.183 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:49:10.183 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.148363+0000", 2023-09-23T18:49:10.183 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.1010000000000003e-06 2023-09-23T18:49:10.184 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.184 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.184 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:49:10.184 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.148370+0000", 2023-09-23T18:49:10.184 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1548000000000002e-05 2023-09-23T18:49:10.184 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.184 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.184 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:49:10.184 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.148422+0000", 2023-09-23T18:49:10.184 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00028122100000000001 2023-09-23T18:49:10.184 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.185 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.185 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:49:10.185 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.148703+0000", 2023-09-23T18:49:10.185 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018254499999999999 2023-09-23T18:49:10.185 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.185 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.185 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 3,7", 2023-09-23T18:49:10.185 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.148885+0000", 2023-09-23T18:49:10.185 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0078720349999999994 2023-09-23T18:49:10.185 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.185 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.186 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:49:10.186 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.156757+0000", 2023-09-23T18:49:10.186 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0084584480000000004 2023-09-23T18:49:10.186 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:49:10.186 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:49:10.186 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:49:10.186 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.186 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.186 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35412.0:79975 8.4 8:2367c566:::benchmark_data_smithi098_176514_object79974:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e243)", 2023-09-23T18:49:10.186 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:49:10.148607+0000", 2023-09-23T18:49:10.186 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.017830764999999998, 2023-09-23T18:49:10.187 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.018106697000000001, 2023-09-23T18:49:10.187 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:49:10.187 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:49:10.187 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:49:10.187 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35412", 2023-09-23T18:49:10.187 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:49:10.187 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 79975 2023-09-23T18:49:10.187 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.187 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:49:10.187 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.187 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:49:10.187 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.148607+0000", 2023-09-23T18:49:10.188 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:49:10.188 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.188 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.188 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:49:10.188 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.148607+0000", 2023-09-23T18:49:10.188 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:49:10.188 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.188 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.188 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:49:10.188 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.148606+0000", 2023-09-23T18:49:10.188 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5078999999999999e-05 2023-09-23T18:49:10.188 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.188 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.189 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:49:10.189 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.148661+0000", 2023-09-23T18:49:10.189 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1483e-05 2023-09-23T18:49:10.189 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.189 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.189 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:49:10.189 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.148673+0000", 2023-09-23T18:49:10.189 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.4189999999999999e-06 2023-09-23T18:49:10.189 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.189 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.189 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:49:10.189 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.148679+0000", 2023-09-23T18:49:10.189 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.2208e-05 2023-09-23T18:49:10.190 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.190 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.190 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:49:10.190 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.148722+0000", 2023-09-23T18:49:10.190 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000164265 2023-09-23T18:49:10.190 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.190 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.190 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:49:10.190 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.148886+0000", 2023-09-23T18:49:10.190 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00019898 2023-09-23T18:49:10.190 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.190 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.191 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:49:10.191 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.149085+0000", 2023-09-23T18:49:10.191 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.011683025999999999 2023-09-23T18:49:10.191 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.191 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.191 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:49:10.191 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.160768+0000", 2023-09-23T18:49:10.191 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0039149359999999999 2023-09-23T18:49:10.191 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.191 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.191 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:49:10.191 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.164683+0000", 2023-09-23T18:49:10.191 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.016075553999999999 2023-09-23T18:49:10.192 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:49:10.192 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:49:10.192 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:49:10.192 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.192 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.192 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35412.0:79982 8.8 8:163d740f:::benchmark_data_smithi098_176514_object79981:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e243)", 2023-09-23T18:49:10.192 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:49:10.154774+0000", 2023-09-23T18:49:10.192 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.011663876, 2023-09-23T18:49:10.192 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.012031257, 2023-09-23T18:49:10.192 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:49:10.193 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:49:10.193 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:49:10.193 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35412", 2023-09-23T18:49:10.193 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:49:10.193 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 79982 2023-09-23T18:49:10.193 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.193 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:49:10.193 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.193 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:49:10.193 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.154774+0000", 2023-09-23T18:49:10.193 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:49:10.193 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.194 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.194 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:49:10.194 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.154774+0000", 2023-09-23T18:49:10.194 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:49:10.194 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.194 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.194 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:49:10.194 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.154772+0000", 2023-09-23T18:49:10.194 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.4473e-05 2023-09-23T18:49:10.194 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.194 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.194 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:49:10.195 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.154807+0000", 2023-09-23T18:49:10.195 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6339000000000001e-05 2023-09-23T18:49:10.195 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.195 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.195 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:49:10.195 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.154823+0000", 2023-09-23T18:49:10.195 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1369999999999999e-06 2023-09-23T18:49:10.195 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.195 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.195 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:49:10.195 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.154829+0000", 2023-09-23T18:49:10.195 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001959667 2023-09-23T18:49:10.195 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.196 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.196 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:49:10.196 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.156789+0000", 2023-09-23T18:49:10.196 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000158851 2023-09-23T18:49:10.196 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.196 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.196 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:49:10.196 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.156948+0000", 2023-09-23T18:49:10.196 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00019753699999999999 2023-09-23T18:49:10.196 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.196 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.196 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 3,7", 2023-09-23T18:49:10.196 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.157145+0000", 2023-09-23T18:49:10.197 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0023712719999999998 2023-09-23T18:49:10.197 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:49:10.197 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:49:10.197 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:49:10.197 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.197 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.197 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35412.0:79985 8.8 8:1f8ff6cf:::benchmark_data_smithi098_176514_object79984:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e243)", 2023-09-23T18:49:10.197 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:49:10.158920+0000", 2023-09-23T18:49:10.197 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.0075185080000000001, 2023-09-23T18:49:10.197 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0079647390000000002, 2023-09-23T18:49:10.197 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:49:10.197 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:49:10.197 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:49:10.198 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35412", 2023-09-23T18:49:10.198 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:49:10.198 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 79985 2023-09-23T18:49:10.198 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.198 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:49:10.198 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.198 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:49:10.198 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.158920+0000", 2023-09-23T18:49:10.198 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:49:10.198 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.198 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.198 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:49:10.199 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.158920+0000", 2023-09-23T18:49:10.199 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:49:10.199 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.199 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.199 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:49:10.199 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.158918+0000", 2023-09-23T18:49:10.199 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.9446000000000001e-05 2023-09-23T18:49:10.199 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.199 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.199 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:49:10.199 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.158957+0000", 2023-09-23T18:49:10.199 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8060999999999999e-05 2023-09-23T18:49:10.199 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.200 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.200 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:49:10.200 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.158975+0000", 2023-09-23T18:49:10.200 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.1300000000000003e-06 2023-09-23T18:49:10.200 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.200 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.200 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:49:10.200 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.158982+0000", 2023-09-23T18:49:10.200 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001511004 2023-09-23T18:49:10.200 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.200 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.200 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:49:10.201 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.160493+0000", 2023-09-23T18:49:10.201 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012871599999999999 2023-09-23T18:49:10.201 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.201 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.201 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:49:10.201 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.160622+0000", 2023-09-23T18:49:10.201 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015896399999999999 2023-09-23T18:49:10.201 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.201 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.201 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 3,7", 2023-09-23T18:49:10.201 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.160781+0000", 2023-09-23T18:49:10.201 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0018616640000000001 2023-09-23T18:49:10.201 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:49:10.202 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:49:10.202 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:49:10.202 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.202 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.202 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35412.0:79979 8.2 e243/233 8:4d818471:::benchmark_data_smithi098_176514_object79978:head v 243'4965, mlcod=243'4963)", 2023-09-23T18:49:10.202 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:49:10.161431+0000", 2023-09-23T18:49:10.202 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.0050074990000000003, 2023-09-23T18:49:10.202 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0055319239999999997, 2023-09-23T18:49:10.202 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:49:10.202 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "started", 2023-09-23T18:49:10.202 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:49:10.202 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.203 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:49:10.203 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.161431+0000", 2023-09-23T18:49:10.203 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:49:10.203 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.203 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.203 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:49:10.203 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.161431+0000", 2023-09-23T18:49:10.203 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:49:10.203 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.203 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.203 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:49:10.203 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.161430+0000", 2023-09-23T18:49:10.203 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.1125999999999999e-05 2023-09-23T18:49:10.203 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.204 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.204 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:49:10.204 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.161471+0000", 2023-09-23T18:49:10.204 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1712000000000001e-05 2023-09-23T18:49:10.204 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.204 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.204 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:49:10.204 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.161483+0000", 2023-09-23T18:49:10.204 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1340000000000001e-06 2023-09-23T18:49:10.204 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.204 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.204 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:49:10.204 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.161489+0000", 2023-09-23T18:49:10.205 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00033427700000000001 2023-09-23T18:49:10.205 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.205 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.205 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:49:10.205 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.161823+0000", 2023-09-23T18:49:10.205 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.0480999999999999e-05 2023-09-23T18:49:10.205 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:49:10.205 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:49:10.205 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:49:10.205 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:49:10.161843+0000", 2023-09-23T18:49:10.205 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00041288499999999999 2023-09-23T18:49:10.205 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:49:10.205 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:49:10.206 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:49:10.206 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:49:10.206 INFO:teuthology.orchestra.run.smithi173.stdout: ], 2023-09-23T18:49:10.206 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 7 2023-09-23T18:49:10.206 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:49:10.206 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:10 smithi098 bash[79516]: audit 2023-09-23T18:49:09.594713+0000 mon.b (mon.1) 20 : audit 0 from='client.? 172.21.15.98:0/4270165515' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:49:10.207 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:10 smithi098 bash[79516]: audit 2023-09-23T18:49:09.621200+0000 mon.a (mon.0) 1260 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:49:10.207 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:10 smithi098 bash[76817]: audit 2023-09-23T18:49:09.594713+0000 mon.b (mon.1) 20 : audit 0 from='client.? 172.21.15.98:0/4270165515' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:49:10.207 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:10 smithi098 bash[76817]: audit 2023-09-23T18:49:09.621200+0000 mon.a (mon.0) 1260 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:49:10.212 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:10.271 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:10.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:10.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:10.377 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:10.404 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:10.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:10.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:10.509 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:10.538 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:10.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:10.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:10.571 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 57 16 80379 80363 88.109 55.6875 0.0166821 0.011344 2023-09-23T18:49:10.645 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:10.674 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:10.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:10.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:10.777 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:10.809 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:10.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:10.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:10.915 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:10.944 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:10.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:10.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:11.047 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:11.074 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:11.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:11.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:11.155 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:11 smithi173 bash[59422]: cluster 2023-09-23T18:49:10.091216+0000 mgr.y (mgr.26054) 930 : cluster 0 pgmap v987: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T18:49:11.155 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:11 smithi173 bash[59422]: audit 2023-09-23T18:49:10.271830+0000 mon.a (mon.0) 1261 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:49:11.155 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:11 smithi173 bash[59422]: audit 2023-09-23T18:49:10.272300+0000 mon.b (mon.1) 21 : audit 1 from='client.? 172.21.15.98:0/468179966' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:49:11.177 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is unset 2023-09-23T18:49:11.179 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:11.188 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:11 smithi098 bash[76817]: cluster 2023-09-23T18:49:10.091216+0000 mgr.y (mgr.26054) 930 : cluster 0 pgmap v987: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T18:49:11.188 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:11 smithi098 bash[76817]: audit 2023-09-23T18:49:10.271830+0000 mon.a (mon.0) 1261 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:49:11.188 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:11 smithi098 bash[76817]: audit 2023-09-23T18:49:10.272300+0000 mon.b (mon.1) 21 : audit 1 from='client.? 172.21.15.98:0/468179966' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:49:11.189 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:11 smithi098 bash[79516]: cluster 2023-09-23T18:49:10.091216+0000 mgr.y (mgr.26054) 930 : cluster 0 pgmap v987: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T18:49:11.189 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:11 smithi098 bash[79516]: audit 2023-09-23T18:49:10.271830+0000 mon.a (mon.0) 1261 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:49:11.189 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:11 smithi098 bash[79516]: audit 2023-09-23T18:49:10.272300+0000 mon.b (mon.1) 21 : audit 1 from='client.? 172.21.15.98:0/468179966' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:49:11.216 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:11.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:11.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:11.320 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:11.346 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:11.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:11.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:11.449 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:11.475 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:11.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:11.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:11.566 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T18:49:11.571 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 58 16 81383 81367 87.6717 62.75 0.0157941 0.0114008 2023-09-23T18:49:11.578 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:11.643 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:11.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:11.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:11.751 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:11.784 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:11.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:11.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:11.890 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:11.916 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:11.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:11.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:12.025 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:12.081 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:12.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:12.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:12.185 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:12.232 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:12.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:12.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:12.309 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T18:49:12.309 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:49:12.309 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T18:49:12.310 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-09-23T18:49:12.336 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:12.380 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:12.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:12.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:12.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:12 smithi098 bash[76817]: cluster 2023-09-23T18:49:11.151585+0000 mon.a (mon.0) 1262 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:49:12.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:12 smithi098 bash[76817]: cluster 2023-09-23T18:49:11.151627+0000 mon.a (mon.0) 1263 : cluster 1 Cluster is now healthy 2023-09-23T18:49:12.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:12 smithi098 bash[76817]: audit 2023-09-23T18:49:11.165137+0000 mon.a (mon.0) 1264 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:49:12.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:12 smithi098 bash[76817]: cluster 2023-09-23T18:49:11.165234+0000 mon.a (mon.0) 1265 : cluster 0 osdmap e244: 8 total, 8 up, 8 in 2023-09-23T18:49:12.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:12 smithi098 bash[76817]: cluster 2023-09-23T18:49:11.171548+0000 osd.5 (osd.5) 141 : cluster 3 failed to encode map e244 with expected crc 2023-09-23T18:49:12.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:12 smithi098 bash[76817]: cluster 2023-09-23T18:49:11.177570+0000 osd.4 (osd.4) 237 : cluster 3 failed to encode map e244 with expected crc 2023-09-23T18:49:12.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:12 smithi098 bash[76817]: cluster 2023-09-23T18:49:11.179518+0000 osd.6 (osd.6) 203 : cluster 3 failed to encode map e244 with expected crc 2023-09-23T18:49:12.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:12 smithi098 bash[76817]: cluster 2023-09-23T18:49:12.091767+0000 mgr.y (mgr.26054) 931 : cluster 0 pgmap v989: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T18:49:12.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:12 smithi098 bash[79516]: cluster 2023-09-23T18:49:11.151585+0000 mon.a (mon.0) 1262 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:49:12.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:12 smithi098 bash[79516]: cluster 2023-09-23T18:49:11.151627+0000 mon.a (mon.0) 1263 : cluster 1 Cluster is now healthy 2023-09-23T18:49:12.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:12 smithi098 bash[79516]: audit 2023-09-23T18:49:11.165137+0000 mon.a (mon.0) 1264 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:49:12.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:12 smithi098 bash[79516]: cluster 2023-09-23T18:49:11.165234+0000 mon.a (mon.0) 1265 : cluster 0 osdmap e244: 8 total, 8 up, 8 in 2023-09-23T18:49:12.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:12 smithi098 bash[79516]: cluster 2023-09-23T18:49:11.171548+0000 osd.5 (osd.5) 141 : cluster 3 failed to encode map e244 with expected crc 2023-09-23T18:49:12.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:12 smithi098 bash[79516]: cluster 2023-09-23T18:49:11.177570+0000 osd.4 (osd.4) 237 : cluster 3 failed to encode map e244 with expected crc 2023-09-23T18:49:12.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:12 smithi098 bash[79516]: cluster 2023-09-23T18:49:11.179518+0000 osd.6 (osd.6) 203 : cluster 3 failed to encode map e244 with expected crc 2023-09-23T18:49:12.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:12 smithi098 bash[79516]: cluster 2023-09-23T18:49:12.091767+0000 mgr.y (mgr.26054) 931 : cluster 0 pgmap v989: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T18:49:12.489 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:12.513 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:12.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:12.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:12.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:12 smithi173 bash[59422]: cluster 2023-09-23T18:49:11.151585+0000 mon.a (mon.0) 1262 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:49:12.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:12 smithi173 bash[59422]: cluster 2023-09-23T18:49:11.151627+0000 mon.a (mon.0) 1263 : cluster 1 Cluster is now healthy 2023-09-23T18:49:12.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:12 smithi173 bash[59422]: audit 2023-09-23T18:49:11.165137+0000 mon.a (mon.0) 1264 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:49:12.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:12 smithi173 bash[59422]: cluster 2023-09-23T18:49:11.165234+0000 mon.a (mon.0) 1265 : cluster 0 osdmap e244: 8 total, 8 up, 8 in 2023-09-23T18:49:12.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:12 smithi173 bash[59422]: cluster 2023-09-23T18:49:11.171548+0000 osd.5 (osd.5) 141 : cluster 3 failed to encode map e244 with expected crc 2023-09-23T18:49:12.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:12 smithi173 bash[59422]: cluster 2023-09-23T18:49:11.177570+0000 osd.4 (osd.4) 237 : cluster 3 failed to encode map e244 with expected crc 2023-09-23T18:49:12.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:12 smithi173 bash[59422]: cluster 2023-09-23T18:49:11.179518+0000 osd.6 (osd.6) 203 : cluster 3 failed to encode map e244 with expected crc 2023-09-23T18:49:12.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:12 smithi173 bash[59422]: cluster 2023-09-23T18:49:12.091767+0000 mgr.y (mgr.26054) 931 : cluster 0 pgmap v989: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T18:49:12.571 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 59 16 82297 82281 87.1538 57.125 0.0162784 0.0114682 2023-09-23T18:49:12.616 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:12.650 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:12.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:12.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:12.756 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:12.781 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:12.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:12.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:12.885 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:12.914 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:12.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:12.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:13.019 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:13.046 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:13.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:13.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:13.151 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:13.182 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:13.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:13.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:13.287 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:13.317 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:13.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:13.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:13.421 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:13.450 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:13.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:13.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:13.554 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:13.572 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:2023-09-23T18:49:13.569928+0000 min lat: 0.00122919 max lat: 0.407963 avg lat: 0.0115253 2023-09-23T18:49:13.572 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:49:13.572 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 60 16 83281 83265 86.7257 61.5 0.0125212 0.0115253 2023-09-23T18:49:13.592 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:13.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:13.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:13.697 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:13.729 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:13.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:13.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:13.747 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:49:13.832 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:13.861 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:13.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:13.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:13.965 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:14.008 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:14.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:14.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:14.112 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:14.149 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:14.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:14.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:14.255 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:14.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:14 smithi173 bash[59422]: cluster 2023-09-23T18:49:14.092232+0000 mgr.y (mgr.26054) 932 : cluster 0 pgmap v990: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T18:49:14.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:14 smithi173 bash[59422]: audit 2023-09-23T18:49:14.157659+0000 mon.a (mon.0) 1266 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:49:14.293 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:14.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:14.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:14.399 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:14.408 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:14 smithi098 bash[76817]: cluster 2023-09-23T18:49:14.092232+0000 mgr.y (mgr.26054) 932 : cluster 0 pgmap v990: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T18:49:14.408 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:14 smithi098 bash[76817]: audit 2023-09-23T18:49:14.157659+0000 mon.a (mon.0) 1266 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:49:14.409 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:14 smithi098 bash[79516]: cluster 2023-09-23T18:49:14.092232+0000 mgr.y (mgr.26054) 932 : cluster 0 pgmap v990: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T18:49:14.410 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:14 smithi098 bash[79516]: audit 2023-09-23T18:49:14.157659+0000 mon.a (mon.0) 1266 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:49:14.444 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:14.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:14.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:14.552 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:14.571 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 61 16 84181 84165 86.226 56.25 0.0195578 0.0115905 2023-09-23T18:49:14.588 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:14.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:14.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:14.691 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:14.705 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:49:14.718 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:14.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:14.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:14.823 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:14.848 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:14.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:14.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:14.954 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:14.984 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:14.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:14.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:15.090 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:15.122 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:15.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:15.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:15.227 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:15.256 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:15.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:15.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:15.361 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:15.388 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:15.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:15.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:15.493 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:15.522 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:15.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:15.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:15.571 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 62 16 85121 85105 85.7828 58.75 0.0125169 0.0116515 2023-09-23T18:49:15.626 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:15.655 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:15.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:15.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:15.759 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:15.786 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:15.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:15.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:15.893 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:15.925 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:15.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:15.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:16.030 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:16.060 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:16.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:16.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:16.165 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:16.175 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:49:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:49:16] "GET /metrics HTTP/1.1" 200 36566 "" "Prometheus/2.33.4" 2023-09-23T18:49:16.201 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:16.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:16.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:16.306 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:16.332 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:16.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:16.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:16.435 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:16.470 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:16.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:16.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:16.545 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:49:16.572 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 63 16 86058 86042 85.3506 58.5625 0.00701526 0.0117111 2023-09-23T18:49:16.577 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:16.611 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:16.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:16.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:16.721 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:16.763 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:16.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:16.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:16.868 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:16.900 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:16.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:16.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:17.006 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:17.032 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:17.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:17.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:17.138 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:17.162 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:17.162 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:49:17.166 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:17 smithi173 bash[59422]: cluster 2023-09-23T18:49:16.093019+0000 mgr.y (mgr.26054) 933 : cluster 0 pgmap v991: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 996 op/s 2023-09-23T18:49:17.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:17.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:17.268 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:17.278 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:17 smithi098 bash[79516]: cluster 2023-09-23T18:49:16.093019+0000 mgr.y (mgr.26054) 933 : cluster 0 pgmap v991: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 996 op/s 2023-09-23T18:49:17.279 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:17 smithi098 bash[76817]: cluster 2023-09-23T18:49:16.093019+0000 mgr.y (mgr.26054) 933 : cluster 0 pgmap v991: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 996 op/s 2023-09-23T18:49:17.300 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:17.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:17.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:17.403 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:17.433 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:17.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:17.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:17.537 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:17.565 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:17.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:17.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:17.571 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 64 16 86990 86974 84.9271 58.25 0.0332118 0.0117689 2023-09-23T18:49:17.669 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:17.691 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:17.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:17.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:17.794 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:17.837 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:17.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:17.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:17.943 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:17.978 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:17.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:17.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:18.083 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:18.109 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:18.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:18.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:18.175 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:49:18.217 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:18.264 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:18.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:18.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:18.367 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:18.434 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:18.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:18.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:18.538 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:18.572 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:18.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:18.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:18.575 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 65 16 87801 87785 84.3952 50.6875 0.0153237 0.0118434 2023-09-23T18:49:18.676 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:18.702 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:18.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:18.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:18.807 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:18.832 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:18.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:18.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:18.938 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:18.994 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:18.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:18.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:19.097 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:19.129 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:19.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:19.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:19.198 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:49:19.232 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:19.267 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:19.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:19.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:19.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:19 smithi173 bash[59422]: cluster 2023-09-23T18:49:18.093599+0000 mgr.y (mgr.26054) 934 : cluster 0 pgmap v992: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 968 op/s 2023-09-23T18:49:19.371 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:19.407 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:19.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:19.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:19.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:19 smithi098 bash[76817]: cluster 2023-09-23T18:49:18.093599+0000 mgr.y (mgr.26054) 934 : cluster 0 pgmap v992: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 968 op/s 2023-09-23T18:49:19.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:19 smithi098 bash[79516]: cluster 2023-09-23T18:49:18.093599+0000 mgr.y (mgr.26054) 934 : cluster 0 pgmap v992: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 968 op/s 2023-09-23T18:49:19.509 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:19.543 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:19.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:19.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:19.576 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 66 16 88599 88583 83.8721 49.875 0.0242413 0.0119164 2023-09-23T18:49:19.648 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:19.679 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:19.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:19.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:19.783 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:19.822 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:19.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:19.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:19.926 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:19.971 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:19.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:19.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:20.074 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:20.101 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:20.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:20.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:20.207 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:20.238 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:20.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:20.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:20.343 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:20.393 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:20.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:20.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:20.505 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:20.528 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:20.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:20.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:20.576 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 67 16 89463 89447 83.4263 54 0.00953859 0.0119804 2023-09-23T18:49:20.633 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:20.667 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:20.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:20.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:20.767 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:20 smithi098 bash[76817]: audit 2023-09-23T18:49:19.765352+0000 mon.a (mon.0) 1267 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:49:20.767 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:20 smithi098 bash[76817]: audit 2023-09-23T18:49:19.775501+0000 mon.a (mon.0) 1268 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:49:20.767 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:20 smithi098 bash[76817]: cluster 2023-09-23T18:49:20.094135+0000 mgr.y (mgr.26054) 935 : cluster 0 pgmap v993: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2023-09-23T18:49:20.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:20.811 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:20.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:20.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:20.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:20.931 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:20 smithi173 bash[59422]: audit 2023-09-23T18:49:19.765352+0000 mon.a (mon.0) 1267 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:49:20.932 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:20 smithi173 bash[59422]: audit 2023-09-23T18:49:19.775501+0000 mon.a (mon.0) 1268 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:49:20.932 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:20 smithi173 bash[59422]: cluster 2023-09-23T18:49:20.094135+0000 mgr.y (mgr.26054) 935 : cluster 0 pgmap v993: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2023-09-23T18:49:20.953 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:20.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:20.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:21.057 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:21.082 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:20 smithi098 bash[79516]: audit 2023-09-23T18:49:19.765352+0000 mon.a (mon.0) 1267 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:49:21.082 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:20 smithi098 bash[79516]: audit 2023-09-23T18:49:19.775501+0000 mon.a (mon.0) 1268 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:49:21.082 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:20 smithi098 bash[79516]: cluster 2023-09-23T18:49:20.094135+0000 mgr.y (mgr.26054) 935 : cluster 0 pgmap v993: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2023-09-23T18:49:21.087 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:21.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:21.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:21.192 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:21.215 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:21.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:21.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:21.320 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:21.360 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:21.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:21.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:21.464 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:21.489 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:21.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:21.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:21.576 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 68 16 90282 90266 82.9521 51.1875 0.00980858 0.0120491 2023-09-23T18:49:21.592 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:21.618 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:21.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:21.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:21.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:21.758 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:21.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:21.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:21.863 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:21.894 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:21.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:21.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:21.999 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:22.037 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:22.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:22.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:22.143 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:22.170 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:22.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:22.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:22.273 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:22.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:22 smithi173 bash[59422]: audit 2023-09-23T18:49:21.085566+0000 mon.a (mon.0) 1269 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:49:22.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:22 smithi173 bash[59422]: audit 2023-09-23T18:49:21.096694+0000 mon.a (mon.0) 1270 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:49:22.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:22 smithi173 bash[59422]: audit 2023-09-23T18:49:21.666751+0000 mon.a (mon.0) 1271 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:49:22.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:22 smithi173 bash[59422]: audit 2023-09-23T18:49:21.667570+0000 mon.a (mon.0) 1272 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:49:22.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:22 smithi173 bash[59422]: audit 2023-09-23T18:49:21.672444+0000 mon.a (mon.0) 1273 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:49:22.302 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:22.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:22.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:22.406 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:22.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:22 smithi098 bash[79516]: audit 2023-09-23T18:49:21.085566+0000 mon.a (mon.0) 1269 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:49:22.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:22 smithi098 bash[79516]: audit 2023-09-23T18:49:21.096694+0000 mon.a (mon.0) 1270 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:49:22.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:22 smithi098 bash[79516]: audit 2023-09-23T18:49:21.666751+0000 mon.a (mon.0) 1271 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:49:22.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:22 smithi098 bash[79516]: audit 2023-09-23T18:49:21.667570+0000 mon.a (mon.0) 1272 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:49:22.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:22 smithi098 bash[79516]: audit 2023-09-23T18:49:21.672444+0000 mon.a (mon.0) 1273 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:49:22.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:22 smithi098 bash[76817]: audit 2023-09-23T18:49:21.085566+0000 mon.a (mon.0) 1269 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:49:22.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:22 smithi098 bash[76817]: audit 2023-09-23T18:49:21.096694+0000 mon.a (mon.0) 1270 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:49:22.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:22 smithi098 bash[76817]: audit 2023-09-23T18:49:21.666751+0000 mon.a (mon.0) 1271 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:49:22.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:22 smithi098 bash[76817]: audit 2023-09-23T18:49:21.667570+0000 mon.a (mon.0) 1272 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:49:22.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:22 smithi098 bash[76817]: audit 2023-09-23T18:49:21.672444+0000 mon.a (mon.0) 1273 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:49:22.444 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:22.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:22.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:22.553 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:22.576 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 69 16 91073 91057 82.4657 49.4375 0.028126 0.0121202 2023-09-23T18:49:22.578 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:22.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:22.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:22.682 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:22.732 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:49:22.732 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-09-23T18:49:22.733 INFO:teuthology.orchestra.run.smithi173.stdout: "complaint_time": 30, 2023-09-23T18:49:22.733 INFO:teuthology.orchestra.run.smithi173.stdout: "num_blocked_ops": 0 2023-09-23T18:49:22.733 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:49:22.740 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:22.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:22.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:22.844 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:22.880 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:22.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:22.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:22.982 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:23.011 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:23.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:23.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:23.115 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:23.144 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:23.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:23.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:23.249 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:23.273 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:23.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:23.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:23.378 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:23.387 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:23 smithi173 bash[59422]: cluster 2023-09-23T18:49:22.094711+0000 mgr.y (mgr.26054) 936 : cluster 0 pgmap v994: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T18:49:23.404 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:23.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:23.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:23.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:23 smithi098 bash[79516]: cluster 2023-09-23T18:49:22.094711+0000 mgr.y (mgr.26054) 936 : cluster 0 pgmap v994: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T18:49:23.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:23 smithi098 bash[76817]: cluster 2023-09-23T18:49:22.094711+0000 mgr.y (mgr.26054) 936 : cluster 0 pgmap v994: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T18:49:23.507 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:23.536 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:23.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:23.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:23.577 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 70 16 91950 91934 82.0706 54.8125 0.0143523 0.0121794 2023-09-23T18:49:23.641 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:23.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:23.674 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:23.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:23.775 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:23.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:23.805 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:23.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:23.906 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:23.933 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:23.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:23.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:24.033 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T18:49:24.040 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:24.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:24.138 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:24.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:24.239 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:24.284 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:24.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:24.293 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:24 smithi173 bash[59422]: cluster 2023-09-23T18:49:24.095238+0000 mgr.y (mgr.26054) 937 : cluster 0 pgmap v995: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 816 op/s 2023-09-23T18:49:24.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:24.394 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:24.436 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:24.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:24.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:24.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:24 smithi098 bash[76817]: cluster 2023-09-23T18:49:24.095238+0000 mgr.y (mgr.26054) 937 : cluster 0 pgmap v995: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 816 op/s 2023-09-23T18:49:24.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:24 smithi098 bash[79516]: cluster 2023-09-23T18:49:24.095238+0000 mgr.y (mgr.26054) 937 : cluster 0 pgmap v995: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 816 op/s 2023-09-23T18:49:24.542 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:24.572 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:24.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:24.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:24.577 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 71 16 92805 92789 81.6672 53.4375 0.0291712 0.0122379 2023-09-23T18:49:24.675 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:24.699 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:24.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:24.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:24.803 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:24.829 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:24.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:24.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:24.936 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:24.963 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:24.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:24.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:25.066 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:25.092 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:25.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:25.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:25.195 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:25.230 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:25.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:25.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:25.335 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:25.369 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:25.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:25.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:25.473 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:25.503 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:25.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:25.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:25.577 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 72 16 93713 93697 81.321 56.75 0.0191813 0.01229 2023-09-23T18:49:25.606 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:25.625 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:25.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:25.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:25.728 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:25.763 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:25.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:25.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:25.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:25 smithi173 bash[59422]: audit 2023-09-23T18:49:24.625241+0000 mon.a (mon.0) 1274 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:49:25.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:25 smithi173 bash[59422]: audit 2023-09-23T18:49:24.630579+0000 mon.a (mon.0) 1275 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:49:25.870 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:25.901 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:25.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:25.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:25.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:25 smithi098 bash[79516]: audit 2023-09-23T18:49:24.625241+0000 mon.a (mon.0) 1274 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:49:25.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:25 smithi098 bash[79516]: audit 2023-09-23T18:49:24.630579+0000 mon.a (mon.0) 1275 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:49:25.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:25 smithi098 bash[76817]: audit 2023-09-23T18:49:24.625241+0000 mon.a (mon.0) 1274 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:49:25.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:25 smithi098 bash[76817]: audit 2023-09-23T18:49:24.630579+0000 mon.a (mon.0) 1275 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:49:26.004 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:26.036 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:26.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:26.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:26.141 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:26.170 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:26.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:26.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:26.275 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:26.307 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:26.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:26.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:26.411 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:26.432 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:49:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:49:26] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.33.4" 2023-09-23T18:49:26.434 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:26.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:26.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:26.536 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:26.566 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:26.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:26.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:26.577 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 73 16 94554 94538 80.927 52.5625 0.0107337 0.0123502 2023-09-23T18:49:26.673 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:26.691 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:26 smithi098 bash[79516]: cluster 2023-09-23T18:49:26.095810+0000 mgr.y (mgr.26054) 938 : cluster 0 pgmap v996: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s 2023-09-23T18:49:26.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:26 smithi098 bash[76817]: cluster 2023-09-23T18:49:26.095810+0000 mgr.y (mgr.26054) 938 : cluster 0 pgmap v996: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s 2023-09-23T18:49:26.698 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:26.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:26.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:26.801 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:26.811 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:26 smithi173 bash[59422]: cluster 2023-09-23T18:49:26.095810+0000 mgr.y (mgr.26054) 938 : cluster 0 pgmap v996: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s 2023-09-23T18:49:26.840 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:26.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:26.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:26.950 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:26.981 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:26.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:26.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:27.087 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:27.115 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:27.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:27.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:27.221 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:27.251 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:27.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:27.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:27.354 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:27.390 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:27.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:27.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:27.495 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:27.520 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:27.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:27.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:27.577 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 74 16 95438 95422 80.58 55.25 0.0151606 0.0124044 2023-09-23T18:49:27.624 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:27.654 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:27.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:27.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:27.763 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:27.793 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:27.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:27.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:27.899 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:27.930 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:27.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:27.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:28.036 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:28.069 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:28.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:28.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:28.172 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:28.200 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:28.202 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:28 smithi173 bash[59422]: cluster 2023-09-23T18:49:28.096385+0000 mgr.y (mgr.26054) 939 : cluster 0 pgmap v997: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s 2023-09-23T18:49:28.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:28.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:28.305 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:28.336 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:28.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:28.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:28.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:28 smithi098 bash[79516]: cluster 2023-09-23T18:49:28.096385+0000 mgr.y (mgr.26054) 939 : cluster 0 pgmap v997: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s 2023-09-23T18:49:28.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:28 smithi098 bash[76817]: cluster 2023-09-23T18:49:28.096385+0000 mgr.y (mgr.26054) 939 : cluster 0 pgmap v997: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s 2023-09-23T18:49:28.445 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:28.473 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:28.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:28.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:28.577 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:28.578 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 75 16 96316 96300 80.2372 54.875 0.0137399 0.0124577 2023-09-23T18:49:28.613 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:28.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:28.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:28.717 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:28.744 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:28.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:28.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:28.849 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:28.855 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:49:28.899 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:28.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:28.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:29.008 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:29.040 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:29.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:29.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:29.147 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:29.176 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:29.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:29.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:29.280 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:29.311 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:29.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:29.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:29.416 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:29.448 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:29.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:29.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:29.552 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:29.578 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 76 16 97184 97168 79.8953 54.25 0.00711704 0.0125114 2023-09-23T18:49:29.578 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:29.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:29.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:29.684 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:29.714 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:29.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:29.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:29.821 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:29.845 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:29.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:29.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:29.948 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:29.990 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:29.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:29.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:30.081 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:49:30.098 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:30.127 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:30.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:30.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:30.174 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:49:30.220 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:49:30.230 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:30.240 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:30 smithi098 bash[76817]: audit 2023-09-23T18:49:30.028568+0000 mon.a (mon.0) 1276 : audit 0 from='client.? 172.21.15.98:0/932567939' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:49:30.241 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:30 smithi098 bash[79516]: audit 2023-09-23T18:49:30.028568+0000 mon.a (mon.0) 1276 : audit 0 from='client.? 172.21.15.98:0/932567939' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:49:30.264 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:30.266 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:49:30.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:30.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:30.309 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:49:30.349 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:49:30.368 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:30.412 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:49:30.412 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:30.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:30.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:30.461 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:49:30.471 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:49:30.471 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:49:30.471 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:49:30.471 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:49:30.471 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:49:30.471 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:49:30.471 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:49:30.471 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:49:30.516 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:30.524 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:30 smithi173 bash[59422]: audit 2023-09-23T18:49:30.028568+0000 mon.a (mon.0) 1276 : audit 0 from='client.? 172.21.15.98:0/932567939' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:49:30.541 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:30.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:30.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:30.577 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 77 16 97967 97951 79.4932 48.9375 0.0558783 0.0125736 2023-09-23T18:49:30.645 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:30.676 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:30.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:30.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:30.780 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:30.811 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:30.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:30.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:30.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:30.946 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:30.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:30.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:31.050 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:31.074 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:31.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:31.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:31.156 INFO:teuthology.orchestra.run.smithi098.stdout:moved 30 / 267 (11.236%) 2023-09-23T18:49:31.156 INFO:teuthology.orchestra.run.smithi098.stdout:avg 33.375 2023-09-23T18:49:31.156 INFO:teuthology.orchestra.run.smithi098.stdout:stddev 8.99913 -> 6.20358 (expected baseline 5.40399) 2023-09-23T18:49:31.156 INFO:teuthology.orchestra.run.smithi098.stdout:min osd.2 with 18 -> 31 pgs (0.539326 -> 0.928839 * mean) 2023-09-23T18:49:31.156 INFO:teuthology.orchestra.run.smithi098.stdout:max osd.1 with 43 -> 42 pgs (1.28839 -> 1.25843 * mean) 2023-09-23T18:49:31.156 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:49:31.156 INFO:teuthology.orchestra.run.smithi098.stdout:oload 110 2023-09-23T18:49:31.156 INFO:teuthology.orchestra.run.smithi098.stdout:max_change 3 2023-09-23T18:49:31.156 INFO:teuthology.orchestra.run.smithi098.stdout:max_change_osds 4 2023-09-23T18:49:31.157 INFO:teuthology.orchestra.run.smithi098.stdout:average_utilization 0.0300 2023-09-23T18:49:31.157 INFO:teuthology.orchestra.run.smithi098.stdout:overload_utilization 0.0330 2023-09-23T18:49:31.157 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2 weight 0.4944 -> 0.9987 2023-09-23T18:49:31.157 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5 weight 1.0000 -> 0.6936 2023-09-23T18:49:31.157 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3 weight 0.5346 -> 0.6315 2023-09-23T18:49:31.157 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6 weight 1.0000 -> 0.8751 2023-09-23T18:49:31.167 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is set 2023-09-23T18:49:31.178 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:31.203 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:31.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:31.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:31.307 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:31.315 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:31 smithi098 bash[76817]: cluster 2023-09-23T18:49:30.096893+0000 mgr.y (mgr.26054) 940 : cluster 0 pgmap v998: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s 2023-09-23T18:49:31.315 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:31 smithi098 bash[76817]: audit 2023-09-23T18:49:30.658043+0000 mon.a (mon.0) 1277 : audit 1 from='client.? 172.21.15.98:0/987334869' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:49:31.315 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:31 smithi098 bash[76817]: audit 2023-09-23T18:49:30.939707+0000 mon.a (mon.0) 1278 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'65449','3':'41387','5':'45459','6':'57353'}"}]: dispatch 2023-09-23T18:49:31.316 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:31 smithi098 bash[79516]: cluster 2023-09-23T18:49:30.096893+0000 mgr.y (mgr.26054) 940 : cluster 0 pgmap v998: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s 2023-09-23T18:49:31.316 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:31 smithi098 bash[79516]: audit 2023-09-23T18:49:30.658043+0000 mon.a (mon.0) 1277 : audit 1 from='client.? 172.21.15.98:0/987334869' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:49:31.316 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:31 smithi098 bash[79516]: audit 2023-09-23T18:49:30.939707+0000 mon.a (mon.0) 1278 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'65449','3':'41387','5':'45459','6':'57353'}"}]: dispatch 2023-09-23T18:49:31.329 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:31.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:31.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:31.433 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:31.446 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:31 smithi173 bash[59422]: cluster 2023-09-23T18:49:30.096893+0000 mgr.y (mgr.26054) 940 : cluster 0 pgmap v998: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s 2023-09-23T18:49:31.447 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:31 smithi173 bash[59422]: audit 2023-09-23T18:49:30.658043+0000 mon.a (mon.0) 1277 : audit 1 from='client.? 172.21.15.98:0/987334869' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:49:31.447 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:31 smithi173 bash[59422]: audit 2023-09-23T18:49:30.939707+0000 mon.a (mon.0) 1278 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'65449','3':'41387','5':'45459','6':'57353'}"}]: dispatch 2023-09-23T18:49:31.458 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:31.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:31.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:31.562 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:31.577 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 78 16 98462 98446 78.8707 30.9375 0.0114886 0.0126077 2023-09-23T18:49:31.593 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:31.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:31.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:31.696 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:31.729 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:31.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:31.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:31.832 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:31.864 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:31.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:31.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:31.968 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:31.993 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:31.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:31.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:32.096 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:32.123 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:32.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:32.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:32.226 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:32.254 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:32.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:32.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:32.360 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:32.389 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:32.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:32.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:32.481 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:32 smithi098 bash[76817]: audit 2023-09-23T18:49:30.938759+0000 mgr.y (mgr.26054) 941 : audit 0 from='client.35463 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:49:32.481 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:32 smithi098 bash[76817]: cluster 2023-09-23T18:49:31.138045+0000 mon.a (mon.0) 1279 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:49:32.481 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:32 smithi098 bash[76817]: audit 2023-09-23T18:49:31.152684+0000 mon.a (mon.0) 1280 : audit 1 from='client.? 172.21.15.98:0/987334869' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:49:32.482 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:32 smithi098 bash[76817]: audit 2023-09-23T18:49:31.152801+0000 mon.a (mon.0) 1281 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'65449','3':'41387','5':'45459','6':'57353'}"}]': finished 2023-09-23T18:49:32.482 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:32 smithi098 bash[76817]: cluster 2023-09-23T18:49:31.152876+0000 mon.a (mon.0) 1282 : cluster 0 osdmap e245: 8 total, 8 up, 8 in 2023-09-23T18:49:32.482 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:32 smithi098 bash[76817]: cluster 2023-09-23T18:49:31.153396+0000 osd.4 (osd.4) 238 : cluster 3 failed to encode map e245 with expected crc 2023-09-23T18:49:32.482 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:32 smithi098 bash[76817]: cluster 2023-09-23T18:49:31.154405+0000 osd.7 (osd.7) 171 : cluster 3 failed to encode map e245 with expected crc 2023-09-23T18:49:32.482 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:32 smithi098 bash[76817]: cluster 2023-09-23T18:49:31.165657+0000 osd.5 (osd.5) 142 : cluster 3 failed to encode map e245 with expected crc 2023-09-23T18:49:32.482 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:32 smithi098 bash[79516]: audit 2023-09-23T18:49:30.938759+0000 mgr.y (mgr.26054) 941 : audit 0 from='client.35463 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:49:32.482 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:32 smithi098 bash[79516]: cluster 2023-09-23T18:49:31.138045+0000 mon.a (mon.0) 1279 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:49:32.482 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:32 smithi098 bash[79516]: audit 2023-09-23T18:49:31.152684+0000 mon.a (mon.0) 1280 : audit 1 from='client.? 172.21.15.98:0/987334869' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:49:32.482 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:32 smithi098 bash[79516]: audit 2023-09-23T18:49:31.152801+0000 mon.a (mon.0) 1281 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'65449','3':'41387','5':'45459','6':'57353'}"}]': finished 2023-09-23T18:49:32.483 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:32 smithi098 bash[79516]: cluster 2023-09-23T18:49:31.152876+0000 mon.a (mon.0) 1282 : cluster 0 osdmap e245: 8 total, 8 up, 8 in 2023-09-23T18:49:32.483 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:32 smithi098 bash[79516]: cluster 2023-09-23T18:49:31.153396+0000 osd.4 (osd.4) 238 : cluster 3 failed to encode map e245 with expected crc 2023-09-23T18:49:32.483 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:32 smithi098 bash[79516]: cluster 2023-09-23T18:49:31.154405+0000 osd.7 (osd.7) 171 : cluster 3 failed to encode map e245 with expected crc 2023-09-23T18:49:32.483 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:32 smithi098 bash[79516]: cluster 2023-09-23T18:49:31.165657+0000 osd.5 (osd.5) 142 : cluster 3 failed to encode map e245 with expected crc 2023-09-23T18:49:32.492 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:32.503 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:32 smithi173 bash[59422]: audit 2023-09-23T18:49:30.938759+0000 mgr.y (mgr.26054) 941 : audit 0 from='client.35463 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:49:32.504 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:32 smithi173 bash[59422]: cluster 2023-09-23T18:49:31.138045+0000 mon.a (mon.0) 1279 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:49:32.504 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:32 smithi173 bash[59422]: audit 2023-09-23T18:49:31.152684+0000 mon.a (mon.0) 1280 : audit 1 from='client.? 172.21.15.98:0/987334869' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:49:32.504 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:32 smithi173 bash[59422]: audit 2023-09-23T18:49:31.152801+0000 mon.a (mon.0) 1281 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'65449','3':'41387','5':'45459','6':'57353'}"}]': finished 2023-09-23T18:49:32.504 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:32 smithi173 bash[59422]: cluster 2023-09-23T18:49:31.152876+0000 mon.a (mon.0) 1282 : cluster 0 osdmap e245: 8 total, 8 up, 8 in 2023-09-23T18:49:32.504 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:32 smithi173 bash[59422]: cluster 2023-09-23T18:49:31.153396+0000 osd.4 (osd.4) 238 : cluster 3 failed to encode map e245 with expected crc 2023-09-23T18:49:32.504 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:32 smithi173 bash[59422]: cluster 2023-09-23T18:49:31.154405+0000 osd.7 (osd.7) 171 : cluster 3 failed to encode map e245 with expected crc 2023-09-23T18:49:32.505 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:32 smithi173 bash[59422]: cluster 2023-09-23T18:49:31.165657+0000 osd.5 (osd.5) 142 : cluster 3 failed to encode map e245 with expected crc 2023-09-23T18:49:32.527 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:32.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:32.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:32.578 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 79 16 98462 98446 77.8723 0 - 0.0126077 2023-09-23T18:49:32.632 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:32.660 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:32.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:32.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:32.765 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:32.794 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:32.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:32.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:32.897 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:32.928 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:32.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:32.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:33.032 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:33.062 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:33.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:33.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:33.165 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:33.174 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:33 smithi098 bash[76817]: cluster 2023-09-23T18:49:32.097918+0000 mgr.y (mgr.26054) 942 : cluster 0 pgmap v1000: 89 pgs: 13 peering, 76 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 828 op/s 2023-09-23T18:49:33.175 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:33 smithi098 bash[76817]: cluster 2023-09-23T18:49:32.148668+0000 mon.a (mon.0) 1283 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:49:33.175 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:33 smithi098 bash[76817]: cluster 2023-09-23T18:49:32.157293+0000 mon.a (mon.0) 1284 : cluster 0 osdmap e246: 8 total, 8 up, 8 in 2023-09-23T18:49:33.175 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:33 smithi098 bash[76817]: cluster 2023-09-23T18:49:32.157727+0000 osd.7 (osd.7) 172 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T18:49:33.175 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:33 smithi098 bash[76817]: cluster 2023-09-23T18:49:32.157760+0000 osd.6 (osd.6) 204 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T18:49:33.175 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:33 smithi098 bash[76817]: cluster 2023-09-23T18:49:32.158681+0000 osd.5 (osd.5) 143 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T18:49:33.175 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:33 smithi098 bash[76817]: cluster 2023-09-23T18:49:32.159973+0000 osd.4 (osd.4) 239 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T18:49:33.175 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:33 smithi098 bash[76817]: cluster 2023-09-23T18:49:32.160141+0000 osd.5 (osd.5) 144 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T18:49:33.175 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:33 smithi098 bash[76817]: cluster 2023-09-23T18:49:32.161280+0000 osd.7 (osd.7) 173 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T18:49:33.175 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:33 smithi098 bash[76817]: cluster 2023-09-23T18:49:32.162469+0000 osd.4 (osd.4) 240 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T18:49:33.176 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:33 smithi098 bash[79516]: cluster 2023-09-23T18:49:32.097918+0000 mgr.y (mgr.26054) 942 : cluster 0 pgmap v1000: 89 pgs: 13 peering, 76 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 828 op/s 2023-09-23T18:49:33.176 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:33 smithi098 bash[79516]: cluster 2023-09-23T18:49:32.148668+0000 mon.a (mon.0) 1283 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:49:33.176 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:33 smithi098 bash[79516]: cluster 2023-09-23T18:49:32.157293+0000 mon.a (mon.0) 1284 : cluster 0 osdmap e246: 8 total, 8 up, 8 in 2023-09-23T18:49:33.176 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:33 smithi098 bash[79516]: cluster 2023-09-23T18:49:32.157727+0000 osd.7 (osd.7) 172 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T18:49:33.176 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:33 smithi098 bash[79516]: cluster 2023-09-23T18:49:32.157760+0000 osd.6 (osd.6) 204 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T18:49:33.176 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:33 smithi098 bash[79516]: cluster 2023-09-23T18:49:32.158681+0000 osd.5 (osd.5) 143 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T18:49:33.176 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:33 smithi098 bash[79516]: cluster 2023-09-23T18:49:32.159973+0000 osd.4 (osd.4) 239 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T18:49:33.176 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:33 smithi098 bash[79516]: cluster 2023-09-23T18:49:32.160141+0000 osd.5 (osd.5) 144 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T18:49:33.176 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:33 smithi098 bash[79516]: cluster 2023-09-23T18:49:32.161280+0000 osd.7 (osd.7) 173 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T18:49:33.177 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:33 smithi098 bash[79516]: cluster 2023-09-23T18:49:32.162469+0000 osd.4 (osd.4) 240 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T18:49:33.193 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:33.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:33.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:33.306 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:33.350 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:33 smithi173 bash[59422]: cluster 2023-09-23T18:49:32.097918+0000 mgr.y (mgr.26054) 942 : cluster 0 pgmap v1000: 89 pgs: 13 peering, 76 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 828 op/s 2023-09-23T18:49:33.350 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:33 smithi173 bash[59422]: cluster 2023-09-23T18:49:32.148668+0000 mon.a (mon.0) 1283 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:49:33.351 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:33 smithi173 bash[59422]: cluster 2023-09-23T18:49:32.157293+0000 mon.a (mon.0) 1284 : cluster 0 osdmap e246: 8 total, 8 up, 8 in 2023-09-23T18:49:33.351 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:33 smithi173 bash[59422]: cluster 2023-09-23T18:49:32.157727+0000 osd.7 (osd.7) 172 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T18:49:33.351 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:33 smithi173 bash[59422]: cluster 2023-09-23T18:49:32.157760+0000 osd.6 (osd.6) 204 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T18:49:33.351 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:33 smithi173 bash[59422]: cluster 2023-09-23T18:49:32.158681+0000 osd.5 (osd.5) 143 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T18:49:33.351 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:33 smithi173 bash[59422]: cluster 2023-09-23T18:49:32.159973+0000 osd.4 (osd.4) 239 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T18:49:33.351 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:33 smithi173 bash[59422]: cluster 2023-09-23T18:49:32.160141+0000 osd.5 (osd.5) 144 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T18:49:33.352 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:33 smithi173 bash[59422]: cluster 2023-09-23T18:49:32.161280+0000 osd.7 (osd.7) 173 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T18:49:33.352 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:33 smithi173 bash[59422]: cluster 2023-09-23T18:49:32.162469+0000 osd.4 (osd.4) 240 : cluster 3 failed to encode map e246 with expected crc 2023-09-23T18:49:33.364 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:33.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:33.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:33.467 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:33.518 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:33.518 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T18:49:33.519 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 0.05 2023-09-23T18:49:33.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:33.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:33.578 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:2023-09-23T18:49:33.576468+0000 min lat: 0.00122919 max lat: 2.31786 avg lat: 0.0127241 2023-09-23T18:49:33.578 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:49:33.578 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 80 16 98473 98457 76.9075 0.34375 0.00602706 0.0127241 2023-09-23T18:49:33.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:33.760 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:33.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:33.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:33.862 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:33.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:33.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:33.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:33.989 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:34.018 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:34.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:34.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:34.120 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:34.143 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:34.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:34.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:34.200 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:49:34.252 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:34 smithi173 bash[59422]: cluster 2023-09-23T18:49:33.156820+0000 osd.5 (osd.5) 145 : cluster 3 failed to encode map e247 with expected crc 2023-09-23T18:49:34.252 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:34 smithi173 bash[59422]: cluster 2023-09-23T18:49:33.157848+0000 mon.a (mon.0) 1285 : cluster 0 osdmap e247: 8 total, 8 up, 8 in 2023-09-23T18:49:34.252 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:34 smithi173 bash[59422]: cluster 2023-09-23T18:49:33.159854+0000 osd.5 (osd.5) 146 : cluster 3 failed to encode map e247 with expected crc 2023-09-23T18:49:34.252 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:34 smithi173 bash[59422]: cluster 2023-09-23T18:49:33.162095+0000 osd.4 (osd.4) 241 : cluster 3 failed to encode map e247 with expected crc 2023-09-23T18:49:34.252 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:34 smithi173 bash[59422]: cluster 2023-09-23T18:49:33.164502+0000 osd.6 (osd.6) 205 : cluster 3 failed to encode map e247 with expected crc 2023-09-23T18:49:34.253 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:34 smithi173 bash[59422]: cluster 2023-09-23T18:49:34.098647+0000 mgr.y (mgr.26054) 943 : cluster 0 pgmap v1003: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 peering, 73 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s; 12261/295227 objects degraded (4.153%); 92 B/s, 0 objects/s recovering 2023-09-23T18:49:34.253 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:34.291 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:34.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:34.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:34.395 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:34.420 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:34.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:34.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:34.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:34 smithi098 bash[76817]: cluster 2023-09-23T18:49:33.156820+0000 osd.5 (osd.5) 145 : cluster 3 failed to encode map e247 with expected crc 2023-09-23T18:49:34.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:34 smithi098 bash[76817]: cluster 2023-09-23T18:49:33.157848+0000 mon.a (mon.0) 1285 : cluster 0 osdmap e247: 8 total, 8 up, 8 in 2023-09-23T18:49:34.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:34 smithi098 bash[76817]: cluster 2023-09-23T18:49:33.159854+0000 osd.5 (osd.5) 146 : cluster 3 failed to encode map e247 with expected crc 2023-09-23T18:49:34.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:34 smithi098 bash[76817]: cluster 2023-09-23T18:49:33.162095+0000 osd.4 (osd.4) 241 : cluster 3 failed to encode map e247 with expected crc 2023-09-23T18:49:34.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:34 smithi098 bash[76817]: cluster 2023-09-23T18:49:33.164502+0000 osd.6 (osd.6) 205 : cluster 3 failed to encode map e247 with expected crc 2023-09-23T18:49:34.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:34 smithi098 bash[76817]: cluster 2023-09-23T18:49:34.098647+0000 mgr.y (mgr.26054) 943 : cluster 0 pgmap v1003: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 peering, 73 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s; 12261/295227 objects degraded (4.153%); 92 B/s, 0 objects/s recovering 2023-09-23T18:49:34.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:34 smithi098 bash[79516]: cluster 2023-09-23T18:49:33.156820+0000 osd.5 (osd.5) 145 : cluster 3 failed to encode map e247 with expected crc 2023-09-23T18:49:34.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:34 smithi098 bash[79516]: cluster 2023-09-23T18:49:33.157848+0000 mon.a (mon.0) 1285 : cluster 0 osdmap e247: 8 total, 8 up, 8 in 2023-09-23T18:49:34.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:34 smithi098 bash[79516]: cluster 2023-09-23T18:49:33.159854+0000 osd.5 (osd.5) 146 : cluster 3 failed to encode map e247 with expected crc 2023-09-23T18:49:34.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:34 smithi098 bash[79516]: cluster 2023-09-23T18:49:33.162095+0000 osd.4 (osd.4) 241 : cluster 3 failed to encode map e247 with expected crc 2023-09-23T18:49:34.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:34 smithi098 bash[79516]: cluster 2023-09-23T18:49:33.164502+0000 osd.6 (osd.6) 205 : cluster 3 failed to encode map e247 with expected crc 2023-09-23T18:49:34.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:34 smithi098 bash[79516]: cluster 2023-09-23T18:49:34.098647+0000 mgr.y (mgr.26054) 943 : cluster 0 pgmap v1003: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 peering, 73 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s; 12261/295227 objects degraded (4.153%); 92 B/s, 0 objects/s recovering 2023-09-23T18:49:34.523 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:34.554 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:34.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:34.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:34.562 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T18:49:34.562 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:49:34.578 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 81 16 98838 98822 76.2397 22.8125 0.0238197 0.013108 2023-09-23T18:49:34.658 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:34.694 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:34.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:34.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:34.800 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:34.852 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:34.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:34.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:34.957 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:34.980 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:34.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:34.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:35.084 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:35.126 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:35.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:35.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:35.231 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:35.255 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:35.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:35.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:35.359 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:35.381 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:35.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:35.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:35.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:35 smithi098 bash[76817]: cluster 2023-09-23T18:49:34.154878+0000 mon.a (mon.0) 1286 : cluster 3 Health check failed: Degraded data redundancy: 12261/295227 objects degraded (4.153%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T18:49:35.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:35 smithi098 bash[76817]: cluster 2023-09-23T18:49:34.168618+0000 osd.5 (osd.5) 147 : cluster 3 failed to encode map e248 with expected crc 2023-09-23T18:49:35.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:35 smithi098 bash[76817]: cluster 2023-09-23T18:49:34.170531+0000 osd.6 (osd.6) 206 : cluster 3 failed to encode map e248 with expected crc 2023-09-23T18:49:35.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:35 smithi098 bash[76817]: cluster 2023-09-23T18:49:34.173662+0000 osd.7 (osd.7) 174 : cluster 3 failed to encode map e248 with expected crc 2023-09-23T18:49:35.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:35 smithi098 bash[76817]: cluster 2023-09-23T18:49:34.177462+0000 mon.a (mon.0) 1287 : cluster 0 osdmap e248: 8 total, 8 up, 8 in 2023-09-23T18:49:35.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:35 smithi098 bash[79516]: cluster 2023-09-23T18:49:34.154878+0000 mon.a (mon.0) 1286 : cluster 3 Health check failed: Degraded data redundancy: 12261/295227 objects degraded (4.153%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T18:49:35.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:35 smithi098 bash[79516]: cluster 2023-09-23T18:49:34.168618+0000 osd.5 (osd.5) 147 : cluster 3 failed to encode map e248 with expected crc 2023-09-23T18:49:35.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:35 smithi098 bash[79516]: cluster 2023-09-23T18:49:34.170531+0000 osd.6 (osd.6) 206 : cluster 3 failed to encode map e248 with expected crc 2023-09-23T18:49:35.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:35 smithi098 bash[79516]: cluster 2023-09-23T18:49:34.173662+0000 osd.7 (osd.7) 174 : cluster 3 failed to encode map e248 with expected crc 2023-09-23T18:49:35.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:35 smithi098 bash[79516]: cluster 2023-09-23T18:49:34.177462+0000 mon.a (mon.0) 1287 : cluster 0 osdmap e248: 8 total, 8 up, 8 in 2023-09-23T18:49:35.484 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:35.486 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:49:35.512 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:35.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:35.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:35.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:35 smithi173 bash[59422]: cluster 2023-09-23T18:49:34.154878+0000 mon.a (mon.0) 1286 : cluster 3 Health check failed: Degraded data redundancy: 12261/295227 objects degraded (4.153%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T18:49:35.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:35 smithi173 bash[59422]: cluster 2023-09-23T18:49:34.168618+0000 osd.5 (osd.5) 147 : cluster 3 failed to encode map e248 with expected crc 2023-09-23T18:49:35.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:35 smithi173 bash[59422]: cluster 2023-09-23T18:49:34.170531+0000 osd.6 (osd.6) 206 : cluster 3 failed to encode map e248 with expected crc 2023-09-23T18:49:35.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:35 smithi173 bash[59422]: cluster 2023-09-23T18:49:34.173662+0000 osd.7 (osd.7) 174 : cluster 3 failed to encode map e248 with expected crc 2023-09-23T18:49:35.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:35 smithi173 bash[59422]: cluster 2023-09-23T18:49:34.177462+0000 mon.a (mon.0) 1287 : cluster 0 osdmap e248: 8 total, 8 up, 8 in 2023-09-23T18:49:35.578 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 82 15 99348 99333 75.6994 31.9375 0.0348445 0.0132023 2023-09-23T18:49:35.614 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T18:49:35.615 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:35.638 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:35.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:35.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:35.744 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:35.787 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:35.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:35.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:35.891 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:35.921 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:35.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:35.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:36.028 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:36.058 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:36.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:36.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:36.087 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:49:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:49:36] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.33.4" 2023-09-23T18:49:36.160 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:36.186 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:36.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:36.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:36.296 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:36.322 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:36.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:36.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:36.429 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:36.448 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:36 smithi098 bash[76817]: cluster 2023-09-23T18:49:35.180191+0000 osd.6 (osd.6) 207 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T18:49:36.448 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:36 smithi098 bash[76817]: cluster 2023-09-23T18:49:35.180906+0000 mon.a (mon.0) 1288 : cluster 0 osdmap e249: 8 total, 8 up, 8 in 2023-09-23T18:49:36.448 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:36 smithi098 bash[76817]: cluster 2023-09-23T18:49:35.189397+0000 osd.6 (osd.6) 208 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T18:49:36.448 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:36 smithi098 bash[76817]: cluster 2023-09-23T18:49:35.200367+0000 osd.4 (osd.4) 242 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T18:49:36.448 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:36 smithi098 bash[76817]: cluster 2023-09-23T18:49:35.202582+0000 osd.4 (osd.4) 243 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T18:49:36.448 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:36 smithi098 bash[76817]: cluster 2023-09-23T18:49:35.203574+0000 osd.5 (osd.5) 148 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T18:49:36.448 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:36 smithi098 bash[76817]: cluster 2023-09-23T18:49:35.203656+0000 osd.4 (osd.4) 244 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T18:49:36.448 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:36 smithi098 bash[76817]: cluster 2023-09-23T18:49:36.099357+0000 mgr.y (mgr.26054) 944 : cluster 0 pgmap v1006: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 14 peering, 71 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 10 MiB/s wr, 161 op/s; 24493/295914 objects degraded (8.277%); 155 B/s, 3 objects/s recovering 2023-09-23T18:49:36.449 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:36 smithi098 bash[79516]: cluster 2023-09-23T18:49:35.180191+0000 osd.6 (osd.6) 207 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T18:49:36.449 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:36 smithi098 bash[79516]: cluster 2023-09-23T18:49:35.180906+0000 mon.a (mon.0) 1288 : cluster 0 osdmap e249: 8 total, 8 up, 8 in 2023-09-23T18:49:36.449 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:36 smithi098 bash[79516]: cluster 2023-09-23T18:49:35.189397+0000 osd.6 (osd.6) 208 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T18:49:36.449 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:36 smithi098 bash[79516]: cluster 2023-09-23T18:49:35.200367+0000 osd.4 (osd.4) 242 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T18:49:36.449 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:36 smithi098 bash[79516]: cluster 2023-09-23T18:49:35.202582+0000 osd.4 (osd.4) 243 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T18:49:36.449 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:36 smithi098 bash[79516]: cluster 2023-09-23T18:49:35.203574+0000 osd.5 (osd.5) 148 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T18:49:36.449 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:36 smithi098 bash[79516]: cluster 2023-09-23T18:49:35.203656+0000 osd.4 (osd.4) 244 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T18:49:36.449 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:36 smithi098 bash[79516]: cluster 2023-09-23T18:49:36.099357+0000 mgr.y (mgr.26054) 944 : cluster 0 pgmap v1006: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 14 peering, 71 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 10 MiB/s wr, 161 op/s; 24493/295914 objects degraded (8.277%); 155 B/s, 3 objects/s recovering 2023-09-23T18:49:36.475 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:36.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:36.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:36.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:36 smithi173 bash[59422]: cluster 2023-09-23T18:49:35.180191+0000 osd.6 (osd.6) 207 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T18:49:36.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:36 smithi173 bash[59422]: cluster 2023-09-23T18:49:35.180906+0000 mon.a (mon.0) 1288 : cluster 0 osdmap e249: 8 total, 8 up, 8 in 2023-09-23T18:49:36.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:36 smithi173 bash[59422]: cluster 2023-09-23T18:49:35.189397+0000 osd.6 (osd.6) 208 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T18:49:36.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:36 smithi173 bash[59422]: cluster 2023-09-23T18:49:35.200367+0000 osd.4 (osd.4) 242 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T18:49:36.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:36 smithi173 bash[59422]: cluster 2023-09-23T18:49:35.202582+0000 osd.4 (osd.4) 243 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T18:49:36.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:36 smithi173 bash[59422]: cluster 2023-09-23T18:49:35.203574+0000 osd.5 (osd.5) 148 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T18:49:36.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:36 smithi173 bash[59422]: cluster 2023-09-23T18:49:35.203656+0000 osd.4 (osd.4) 244 : cluster 3 failed to encode map e249 with expected crc 2023-09-23T18:49:36.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:36 smithi173 bash[59422]: cluster 2023-09-23T18:49:36.099357+0000 mgr.y (mgr.26054) 944 : cluster 0 pgmap v1006: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 14 peering, 71 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 10 MiB/s wr, 161 op/s; 24493/295914 objects degraded (8.277%); 155 B/s, 3 objects/s recovering 2023-09-23T18:49:36.578 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 83 16 99843 99827 75.1593 30.875 0.0395928 0.013298 2023-09-23T18:49:36.581 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:36.608 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:36.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:36.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:36.711 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:36.738 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:36.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:36.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:36.841 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:36.864 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:36.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:36.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:36.972 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:36.997 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:36.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:36.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:37.100 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:37.134 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:37.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:37.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:37.237 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:37.272 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:37.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:37.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:37.375 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:37.397 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:37.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:37.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:37.503 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:37.529 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:37.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:37.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:37.578 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 84 16 100398 100382 74.6775 34.6875 0.0378019 0.0133851 2023-09-23T18:49:37.633 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:37.677 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:37.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:37.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:37.781 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:37.825 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:37.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:37.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:37.928 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:37.951 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:37.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:37.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:38.056 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:38.089 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:38.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:38.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:38.193 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:38.222 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:38.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:38.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:38.327 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:38.354 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:38.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:38.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:38.431 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:49:38.435 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:49:38.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:38 smithi098 bash[76817]: cluster 2023-09-23T18:49:38.100122+0000 mgr.y (mgr.26054) 945 : cluster 0 pgmap v1007: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 peering, 79 active+clean; 6.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 15 MiB/s wr, 246 op/s; 49359/298368 objects degraded (16.543%); 6101/298368 objects misplaced (2.045%); 463 KiB/s, 9 objects/s recovering 2023-09-23T18:49:38.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:38 smithi098 bash[76817]: cluster 2023-09-23T18:49:38.151548+0000 mon.a (mon.0) 1289 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:49:38.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:38 smithi098 bash[79516]: cluster 2023-09-23T18:49:38.100122+0000 mgr.y (mgr.26054) 945 : cluster 0 pgmap v1007: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 peering, 79 active+clean; 6.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 15 MiB/s wr, 246 op/s; 49359/298368 objects degraded (16.543%); 6101/298368 objects misplaced (2.045%); 463 KiB/s, 9 objects/s recovering 2023-09-23T18:49:38.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:38 smithi098 bash[79516]: cluster 2023-09-23T18:49:38.151548+0000 mon.a (mon.0) 1289 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:49:38.460 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:38.467 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:38 smithi173 bash[59422]: cluster 2023-09-23T18:49:38.100122+0000 mgr.y (mgr.26054) 945 : cluster 0 pgmap v1007: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 peering, 79 active+clean; 6.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 15 MiB/s wr, 246 op/s; 49359/298368 objects degraded (16.543%); 6101/298368 objects misplaced (2.045%); 463 KiB/s, 9 objects/s recovering 2023-09-23T18:49:38.468 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:38 smithi173 bash[59422]: cluster 2023-09-23T18:49:38.151548+0000 mon.a (mon.0) 1289 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:49:38.483 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:38.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:38.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:38.580 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 85 16 100928 100912 74.1878 33.125 0.0181817 0.0134723 2023-09-23T18:49:38.589 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:38.637 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:38.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:38.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:38.745 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:38.775 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:38.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:38.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:38.879 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:38.932 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:38.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:38.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:39.038 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:39.070 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:39.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:39.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:39.181 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:39.207 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:39.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:39.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:39.313 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:39.337 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:39.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:39.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:39.441 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:39.471 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:49:39.475 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:39.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:39.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:39.579 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 86 16 101438 101422 73.6958 31.875 0.015765 0.0135633 2023-09-23T18:49:39.584 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:39.610 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:39.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:39.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:39.713 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:39.740 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:39.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:39.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:39.847 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:39.875 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:39.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:39.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:39.981 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:40.011 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:40.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:40.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:40.117 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:40.145 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:40.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:40.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:40.249 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:40.290 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:40.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:40.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:40.395 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:40.396 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:49:40.425 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:40.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:40.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:40.533 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:40.560 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:40.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:40.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:40.579 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 87 16 101991 101975 73.246 34.5625 0.0224196 0.0136444 2023-09-23T18:49:40.642 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:40 smithi098 bash[76817]: audit 2023-09-23T18:49:39.631052+0000 mon.a (mon.0) 1290 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:49:40.643 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:40 smithi098 bash[76817]: audit 2023-09-23T18:49:39.636413+0000 mon.a (mon.0) 1291 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:49:40.643 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:40 smithi098 bash[76817]: cluster 2023-09-23T18:49:40.100883+0000 mgr.y (mgr.26054) 946 : cluster 0 pgmap v1008: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 19 MiB/s wr, 305 op/s; 49359/300345 objects degraded (16.434%); 6101/300345 objects misplaced (2.031%); 396 KiB/s, 8 objects/s recovering 2023-09-23T18:49:40.643 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:40 smithi098 bash[76817]: cluster 2023-09-23T18:49:40.137797+0000 mon.a (mon.0) 1292 : cluster 3 Health check update: Degraded data redundancy: 49359/298368 objects degraded (16.543%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:49:40.667 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:40.675 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:40 smithi173 bash[59422]: audit 2023-09-23T18:49:39.631052+0000 mon.a (mon.0) 1290 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:49:40.675 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:40 smithi173 bash[59422]: audit 2023-09-23T18:49:39.636413+0000 mon.a (mon.0) 1291 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:49:40.675 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:40 smithi173 bash[59422]: cluster 2023-09-23T18:49:40.100883+0000 mgr.y (mgr.26054) 946 : cluster 0 pgmap v1008: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 19 MiB/s wr, 305 op/s; 49359/300345 objects degraded (16.434%); 6101/300345 objects misplaced (2.031%); 396 KiB/s, 8 objects/s recovering 2023-09-23T18:49:40.676 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:40 smithi173 bash[59422]: cluster 2023-09-23T18:49:40.137797+0000 mon.a (mon.0) 1292 : cluster 3 Health check update: Degraded data redundancy: 49359/298368 objects degraded (16.543%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:49:40.689 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:40.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:40.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:40.796 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:40.819 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:40.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:40.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:40.927 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:40.951 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:40.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:40.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:41.027 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:40 smithi098 bash[79516]: audit 2023-09-23T18:49:39.631052+0000 mon.a (mon.0) 1290 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:49:41.027 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:40 smithi098 bash[79516]: audit 2023-09-23T18:49:39.636413+0000 mon.a (mon.0) 1291 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:49:41.027 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:40 smithi098 bash[79516]: cluster 2023-09-23T18:49:40.100883+0000 mgr.y (mgr.26054) 946 : cluster 0 pgmap v1008: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 19 MiB/s wr, 305 op/s; 49359/300345 objects degraded (16.434%); 6101/300345 objects misplaced (2.031%); 396 KiB/s, 8 objects/s recovering 2023-09-23T18:49:41.027 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:40 smithi098 bash[79516]: cluster 2023-09-23T18:49:40.137797+0000 mon.a (mon.0) 1292 : cluster 3 Health check update: Degraded data redundancy: 49359/298368 objects degraded (16.543%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:49:41.054 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:41.087 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:41.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:41.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:41.194 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:41.220 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:41.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:41.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:41.323 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:41.344 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:41.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:41.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:41.448 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:41.485 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:41.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:41.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:41.580 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 88 16 102513 102497 72.7844 32.625 0.00560642 0.0137317 2023-09-23T18:49:41.590 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:41.617 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:41.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:41.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:41.724 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:41.747 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:41.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:41.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:41.850 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:41.870 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:41.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:41.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:41.974 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:41.996 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:42.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:42.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:42.101 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:42.132 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:42.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:42.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:42.238 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:42.265 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:42.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:42.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:42.369 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:42.393 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:42.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:42.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:42.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:42 smithi098 bash[76817]: cluster 2023-09-23T18:49:42.101716+0000 mgr.y (mgr.26054) 947 : cluster 0 pgmap v1009: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 29 MiB/s wr, 456 op/s; 49359/306195 objects degraded (16.120%); 6015/306195 objects misplaced (1.964%); 1.0 MiB/s, 17 objects/s recovering 2023-09-23T18:49:42.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:42 smithi098 bash[79516]: cluster 2023-09-23T18:49:42.101716+0000 mgr.y (mgr.26054) 947 : cluster 0 pgmap v1009: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 29 MiB/s wr, 456 op/s; 49359/306195 objects degraded (16.120%); 6015/306195 objects misplaced (1.964%); 1.0 MiB/s, 17 objects/s recovering 2023-09-23T18:49:42.479 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:42 smithi173 bash[59422]: cluster 2023-09-23T18:49:42.101716+0000 mgr.y (mgr.26054) 947 : cluster 0 pgmap v1009: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 29 MiB/s wr, 456 op/s; 49359/306195 objects degraded (16.120%); 6015/306195 objects misplaced (1.964%); 1.0 MiB/s, 17 objects/s recovering 2023-09-23T18:49:42.497 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:42.523 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:42.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:42.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:42.580 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 89 16 102973 102957 72.2896 28.75 0.0487968 0.0138272 2023-09-23T18:49:42.628 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:42.660 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:42.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:42.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:42.766 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:42.808 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:42.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:42.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:42.912 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:42.936 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:42.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:42.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:43.044 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:43.069 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:43.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:43.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:43.172 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:43.209 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:43.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:43.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:43.315 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:43.343 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:43.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:43.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:43.452 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:43.491 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:43.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:43.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:43.580 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 90 5 103475 103470 71.8426 32.0625 0.078113 0.0139132 2023-09-23T18:49:43.598 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:43.629 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:43.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:43.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:43.732 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:43.766 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:43.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:43.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:43.869 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:43.896 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:43.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:43.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:44.000 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:44.033 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:44.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:44.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:44.135 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:44.164 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:44.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:44.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:44.267 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:44.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:44 smithi173 bash[59422]: cluster 2023-09-23T18:49:44.102465+0000 mgr.y (mgr.26054) 948 : cluster 0 pgmap v1010: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 27 MiB/s wr, 430 op/s; 49359/308064 objects degraded (16.022%); 6015/308064 objects misplaced (1.953%); 831 KiB/s, 13 objects/s recovering 2023-09-23T18:49:44.290 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:44.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:44.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:44.393 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:44.415 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:44.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:44.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:44.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:44 smithi098 bash[79516]: cluster 2023-09-23T18:49:44.102465+0000 mgr.y (mgr.26054) 948 : cluster 0 pgmap v1010: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 27 MiB/s wr, 430 op/s; 49359/308064 objects degraded (16.022%); 6015/308064 objects misplaced (1.953%); 831 KiB/s, 13 objects/s recovering 2023-09-23T18:49:44.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:44 smithi098 bash[76817]: cluster 2023-09-23T18:49:44.102465+0000 mgr.y (mgr.26054) 948 : cluster 0 pgmap v1010: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 27 MiB/s wr, 430 op/s; 49359/308064 objects degraded (16.022%); 6015/308064 objects misplaced (1.953%); 831 KiB/s, 13 objects/s recovering 2023-09-23T18:49:44.518 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:44.548 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:44.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:44.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:44.580 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Total time run: 90.0284 2023-09-23T18:49:44.580 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Total writes made: 103475 2023-09-23T18:49:44.580 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Write size: 65536 2023-09-23T18:49:44.580 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Object size: 65536 2023-09-23T18:49:44.580 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Bandwidth (MB/sec): 71.835 2023-09-23T18:49:44.580 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Stddev Bandwidth: 33.3556 2023-09-23T18:49:44.581 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Max bandwidth (MB/sec): 173.5 2023-09-23T18:49:44.581 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Min bandwidth (MB/sec): 0 2023-09-23T18:49:44.581 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Average IOPS: 1149 2023-09-23T18:49:44.581 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Stddev IOPS: 533.702 2023-09-23T18:49:44.581 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Max IOPS: 2776 2023-09-23T18:49:44.581 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Min IOPS: 0 2023-09-23T18:49:44.581 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Average Latency(s): 0.0139141 2023-09-23T18:49:44.581 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Stddev Latency(s): 0.0339846 2023-09-23T18:49:44.581 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Max latency(s): 2.69167 2023-09-23T18:49:44.581 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Min latency(s): 0.00122919 2023-09-23T18:49:44.598 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T18:49:44.651 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:44.700 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:44.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:44.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:44.803 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:44.845 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:44.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:44.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:44.951 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:44.957 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T18:49:44.957 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:49:44.979 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:44.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:44.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:45.085 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:45.135 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:45.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:45.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:45.240 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:45.249 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:45 smithi173 bash[59422]: cluster 2023-09-23T18:49:45.139573+0000 mon.a (mon.0) 1293 : cluster 3 Health check update: Degraded data redundancy: 49359/308064 objects degraded (16.022%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:49:45.265 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:45.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:45.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:45.344 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:45 smithi098 bash[79516]: cluster 2023-09-23T18:49:45.139573+0000 mon.a (mon.0) 1293 : cluster 3 Health check update: Degraded data redundancy: 49359/308064 objects degraded (16.022%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:49:45.344 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:45 smithi098 bash[76817]: cluster 2023-09-23T18:49:45.139573+0000 mon.a (mon.0) 1293 : cluster 3 Health check update: Degraded data redundancy: 49359/308064 objects degraded (16.022%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:49:45.369 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:45.399 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:45.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:45.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:45.616 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:45.652 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:45.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:45.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:45.760 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:45.783 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:45.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:45.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:45.871 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T18:49:45.888 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:45.947 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:45.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:45.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:46.051 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:46.116 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:46.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:46.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:46.226 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:46.269 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:46.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:46.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:46.276 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:49:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:49:46] "GET /metrics HTTP/1.1" 200 36617 "" "Prometheus/2.33.4" 2023-09-23T18:49:46.377 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:46.412 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:46.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:46.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:46.517 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:46.542 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:46.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:46.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:46.646 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:46.653 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:46 smithi098 bash[76817]: audit 2023-09-23T18:49:45.302951+0000 mon.a (mon.0) 1294 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:49:46.653 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:46 smithi098 bash[76817]: cluster 2023-09-23T18:49:46.103116+0000 mgr.y (mgr.26054) 949 : cluster 0 pgmap v1011: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 93 B/s rd, 26 MiB/s wr, 426 op/s; 49359/308571 objects degraded (15.996%); 6015/308571 objects misplaced (1.949%); 756 KiB/s, 11 objects/s recovering 2023-09-23T18:49:46.654 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:46 smithi098 bash[79516]: audit 2023-09-23T18:49:45.302951+0000 mon.a (mon.0) 1294 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:49:46.654 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:46 smithi098 bash[79516]: cluster 2023-09-23T18:49:46.103116+0000 mgr.y (mgr.26054) 949 : cluster 0 pgmap v1011: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 93 B/s rd, 26 MiB/s wr, 426 op/s; 49359/308571 objects degraded (15.996%); 6015/308571 objects misplaced (1.949%); 756 KiB/s, 11 objects/s recovering 2023-09-23T18:49:46.667 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:46.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:46.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:46.770 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:46.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:46 smithi173 bash[59422]: audit 2023-09-23T18:49:45.302951+0000 mon.a (mon.0) 1294 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:49:46.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:46 smithi173 bash[59422]: cluster 2023-09-23T18:49:46.103116+0000 mgr.y (mgr.26054) 949 : cluster 0 pgmap v1011: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 93 B/s rd, 26 MiB/s wr, 426 op/s; 49359/308571 objects degraded (15.996%); 6015/308571 objects misplaced (1.949%); 756 KiB/s, 11 objects/s recovering 2023-09-23T18:49:46.812 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:46.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:46.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:46.919 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:46.950 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:46.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:46.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:47.053 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:47.084 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:47.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:47.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:47.187 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:47.215 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:47.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:47.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:47.323 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:47.363 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:47.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:47.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:47.470 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:47.496 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:47.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:47.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:47.600 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:47.632 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:47.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:47.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:47.737 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:47.777 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:47.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:47.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:47.883 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:47.916 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:47.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:47.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:48.020 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:48.047 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:48.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:48.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:48.154 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:48.184 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:48.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:48.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:48.297 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:48.305 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:48 smithi098 bash[79516]: cluster 2023-09-23T18:49:48.103814+0000 mgr.y (mgr.26054) 950 : cluster 0 pgmap v1012: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 25 MiB/s wr, 463 op/s; 49359/308403 objects degraded (16.005%); 5891/308403 objects misplaced (1.910%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T18:49:48.306 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:48 smithi098 bash[76817]: cluster 2023-09-23T18:49:48.103814+0000 mgr.y (mgr.26054) 950 : cluster 0 pgmap v1012: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 25 MiB/s wr, 463 op/s; 49359/308403 objects degraded (16.005%); 5891/308403 objects misplaced (1.910%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T18:49:48.326 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:48.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:48.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:48.432 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:48.485 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:48.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:48.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:48.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:48 smithi173 bash[59422]: cluster 2023-09-23T18:49:48.103814+0000 mgr.y (mgr.26054) 950 : cluster 0 pgmap v1012: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 25 MiB/s wr, 463 op/s; 49359/308403 objects degraded (16.005%); 5891/308403 objects misplaced (1.910%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T18:49:48.590 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:48.616 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:48.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:48.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:48.719 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:48.754 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:48.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:48.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:48.859 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:48.891 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:48.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:48.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:48.995 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:49.024 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:49.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:49.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:49.127 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:49.156 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:49.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:49.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:49.258 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:49.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:49.303 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:49.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:49.404 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:49.444 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:49.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:49.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:49.549 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:49.585 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:49.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:49.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:49.692 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:49.725 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:49.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:49.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:49.828 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:49.853 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:49.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:49.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:49.959 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:49.991 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:49.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:49.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:50.095 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:50.129 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:50.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:50.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:50.201 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:50 smithi173 bash[59422]: cluster 2023-09-23T18:49:50.104486+0000 mgr.y (mgr.26054) 951 : cluster 0 pgmap v1013: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 438 op/s; 49359/306861 objects degraded (16.085%); 5891/306861 objects misplaced (1.920%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T18:49:50.201 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:50 smithi173 bash[59422]: cluster 2023-09-23T18:49:50.144610+0000 mon.a (mon.0) 1295 : cluster 3 Health check update: Degraded data redundancy: 49359/308403 objects degraded (16.005%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:49:50.236 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:50.282 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:50.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:50.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:50.389 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:50.415 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:50.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:50.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:50.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:50 smithi098 bash[76817]: cluster 2023-09-23T18:49:50.104486+0000 mgr.y (mgr.26054) 951 : cluster 0 pgmap v1013: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 438 op/s; 49359/306861 objects degraded (16.085%); 5891/306861 objects misplaced (1.920%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T18:49:50.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:50 smithi098 bash[76817]: cluster 2023-09-23T18:49:50.144610+0000 mon.a (mon.0) 1295 : cluster 3 Health check update: Degraded data redundancy: 49359/308403 objects degraded (16.005%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:49:50.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:50 smithi098 bash[79516]: cluster 2023-09-23T18:49:50.104486+0000 mgr.y (mgr.26054) 951 : cluster 0 pgmap v1013: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 438 op/s; 49359/306861 objects degraded (16.085%); 5891/306861 objects misplaced (1.920%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T18:49:50.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:50 smithi098 bash[79516]: cluster 2023-09-23T18:49:50.144610+0000 mon.a (mon.0) 1295 : cluster 3 Health check update: Degraded data redundancy: 49359/308403 objects degraded (16.005%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:49:50.518 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:50.547 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:50.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:50.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:50.652 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:50.675 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:49:50.681 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:50.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:50.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:50.787 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:50.817 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:50.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:50.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:50.921 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:50.972 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:50.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:50.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:51.079 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:51.111 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:51.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:51.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:51.214 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:51.233 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:51.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:51.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:51.336 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:51.370 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:51.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:51.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:51.474 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:51.495 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:51.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:51.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:51.600 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:51.631 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:51.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:51.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:51.738 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:51.780 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:51.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:51.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:51.886 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:51.909 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:51.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:51.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:52.013 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:52.044 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:52.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:52.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:52.147 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:52.180 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:52.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:52.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:52.283 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:52.306 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:52.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:52.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:52.409 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:52.436 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:52.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:52.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:52.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:52 smithi098 bash[76817]: cluster 2023-09-23T18:49:52.105347+0000 mgr.y (mgr.26054) 952 : cluster 0 pgmap v1014: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 545 op/s; 49359/301029 objects degraded (16.397%); 5799/301029 objects misplaced (1.926%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T18:49:52.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:52 smithi098 bash[79516]: cluster 2023-09-23T18:49:52.105347+0000 mgr.y (mgr.26054) 952 : cluster 0 pgmap v1014: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 545 op/s; 49359/301029 objects degraded (16.397%); 5799/301029 objects misplaced (1.926%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T18:49:52.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:52 smithi173 bash[59422]: cluster 2023-09-23T18:49:52.105347+0000 mgr.y (mgr.26054) 952 : cluster 0 pgmap v1014: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 545 op/s; 49359/301029 objects degraded (16.397%); 5799/301029 objects misplaced (1.926%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T18:49:52.539 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:52.573 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:52.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:52.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:52.623 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:49:52.675 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:49:52.677 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:52.705 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:52.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:52.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:52.724 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:49:52.776 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:49:52.811 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:52.814 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:49:52.835 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:52.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:52.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:52.860 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:49:52.912 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:49:52.940 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:52.943 INFO:teuthology.orchestra.run.smithi098.stdout:moved 5 / 267 (1.87266%) 2023-09-23T18:49:52.943 INFO:teuthology.orchestra.run.smithi098.stdout:avg 33.375 2023-09-23T18:49:52.943 INFO:teuthology.orchestra.run.smithi098.stdout:stddev 6.20358 -> 5.82961 (expected baseline 5.40399) 2023-09-23T18:49:52.943 INFO:teuthology.orchestra.run.smithi098.stdout:min osd.5 with 21 -> 21 pgs (0.629214 -> 0.629214 * mean) 2023-09-23T18:49:52.944 INFO:teuthology.orchestra.run.smithi098.stdout:max osd.1 with 42 -> 41 pgs (1.25843 -> 1.22846 * mean) 2023-09-23T18:49:52.944 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:49:52.944 INFO:teuthology.orchestra.run.smithi098.stdout:oload 1000 2023-09-23T18:49:52.944 INFO:teuthology.orchestra.run.smithi098.stdout:max_change 0.05 2023-09-23T18:49:52.944 INFO:teuthology.orchestra.run.smithi098.stdout:max_change_osds 4 2023-09-23T18:49:52.944 INFO:teuthology.orchestra.run.smithi098.stdout:average_utilization 0.0322 2023-09-23T18:49:52.944 INFO:teuthology.orchestra.run.smithi098.stdout:overload_utilization 0.3218 2023-09-23T18:49:52.944 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2 weight 0.9987 -> 1.0000 2023-09-23T18:49:52.944 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3 weight 0.6315 -> 0.6815 2023-09-23T18:49:52.945 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7 weight 0.8539 -> 0.9039 2023-09-23T18:49:52.951 INFO:teuthology.orchestra.run.smithi098.stderr:no change 2023-09-23T18:49:52.959 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:49:52.963 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:52.968 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:49:52.968 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:49:52.969 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:49:52.969 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:49:52.969 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:49:52.969 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:49:52.969 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:49:52.969 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:49:52.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:52.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:53.072 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:53.096 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:53.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:53.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:53.200 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:53.212 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:49:52 smithi098 bash[73085]: debug 2023-09-23T18:49:52.938+0000 7f90a56ad700 -1 mgr.server reply reply (3) No such process no change 2023-09-23T18:49:53.212 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:53 smithi098 bash[76817]: audit 2023-09-23T18:49:52.445247+0000 mon.a (mon.0) 1296 : audit 1 from='client.? 172.21.15.98:0/3833784593' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:49:53.212 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:53 smithi098 bash[76817]: audit 2023-09-23T18:49:52.570712+0000 mon.c (mon.2) 109 : audit 0 from='client.? 172.21.15.98:0/3174318230' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:49:53.212 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:53 smithi098 bash[76817]: audit 2023-09-23T18:49:52.940234+0000 mgr.y (mgr.26054) 953 : audit 0 from='client.45416 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:49:53.225 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:53.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:53.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:53.241 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is set 2023-09-23T18:49:53.329 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:53.364 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:53.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:53.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:53.469 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:53.478 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:53 smithi098 bash[79516]: audit 2023-09-23T18:49:52.445247+0000 mon.a (mon.0) 1296 : audit 1 from='client.? 172.21.15.98:0/3833784593' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:49:53.478 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:53 smithi098 bash[79516]: audit 2023-09-23T18:49:52.570712+0000 mon.c (mon.2) 109 : audit 0 from='client.? 172.21.15.98:0/3174318230' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:49:53.478 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:53 smithi098 bash[79516]: audit 2023-09-23T18:49:52.940234+0000 mgr.y (mgr.26054) 953 : audit 0 from='client.45416 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:49:53.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:53 smithi173 bash[59422]: audit 2023-09-23T18:49:52.445247+0000 mon.a (mon.0) 1296 : audit 1 from='client.? 172.21.15.98:0/3833784593' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:49:53.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:53 smithi173 bash[59422]: audit 2023-09-23T18:49:52.570712+0000 mon.c (mon.2) 109 : audit 0 from='client.? 172.21.15.98:0/3174318230' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:49:53.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:53 smithi173 bash[59422]: audit 2023-09-23T18:49:52.940234+0000 mgr.y (mgr.26054) 953 : audit 0 from='client.45416 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:49:53.530 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:53.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:53.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:53.635 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:53.674 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:53.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:53.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:53.779 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:53.809 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:53.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:53.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:53.917 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:53.960 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:53.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:53.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:54.064 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:54.089 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:54.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:54.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:54.191 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:54.219 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:54.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:54.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:54.322 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:54.351 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:54.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:54.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:54.454 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:54.493 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:54.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:54.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:54.498 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:54 smithi098 bash[76817]: cluster 2023-09-23T18:49:53.205921+0000 mon.a (mon.0) 1297 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:49:54.498 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:54 smithi098 bash[76817]: cluster 2023-09-23T18:49:53.227098+0000 osd.7 (osd.7) 175 : cluster 3 failed to encode map e250 with expected crc 2023-09-23T18:49:54.498 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:54 smithi098 bash[76817]: audit 2023-09-23T18:49:53.231371+0000 mon.a (mon.0) 1298 : audit 1 from='client.? 172.21.15.98:0/3833784593' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:49:54.499 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:54 smithi098 bash[76817]: cluster 2023-09-23T18:49:53.231701+0000 mon.a (mon.0) 1299 : cluster 0 osdmap e250: 8 total, 8 up, 8 in 2023-09-23T18:49:54.499 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:54 smithi098 bash[76817]: cluster 2023-09-23T18:49:53.235246+0000 osd.6 (osd.6) 209 : cluster 3 failed to encode map e250 with expected crc 2023-09-23T18:49:54.499 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:54 smithi098 bash[76817]: cluster 2023-09-23T18:49:54.106052+0000 mgr.y (mgr.26054) 954 : cluster 0 pgmap v1016: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 5.1 MiB/s wr, 459 op/s; 49359/299169 objects degraded (16.499%); 5799/299169 objects misplaced (1.938%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T18:49:54.499 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:54 smithi098 bash[79516]: cluster 2023-09-23T18:49:53.205921+0000 mon.a (mon.0) 1297 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:49:54.499 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:54 smithi098 bash[79516]: cluster 2023-09-23T18:49:53.227098+0000 osd.7 (osd.7) 175 : cluster 3 failed to encode map e250 with expected crc 2023-09-23T18:49:54.499 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:54 smithi098 bash[79516]: audit 2023-09-23T18:49:53.231371+0000 mon.a (mon.0) 1298 : audit 1 from='client.? 172.21.15.98:0/3833784593' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:49:54.499 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:54 smithi098 bash[79516]: cluster 2023-09-23T18:49:53.231701+0000 mon.a (mon.0) 1299 : cluster 0 osdmap e250: 8 total, 8 up, 8 in 2023-09-23T18:49:54.499 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:54 smithi098 bash[79516]: cluster 2023-09-23T18:49:53.235246+0000 osd.6 (osd.6) 209 : cluster 3 failed to encode map e250 with expected crc 2023-09-23T18:49:54.499 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:54 smithi098 bash[79516]: cluster 2023-09-23T18:49:54.106052+0000 mgr.y (mgr.26054) 954 : cluster 0 pgmap v1016: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 5.1 MiB/s wr, 459 op/s; 49359/299169 objects degraded (16.499%); 5799/299169 objects misplaced (1.938%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T18:49:54.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:54 smithi173 bash[59422]: cluster 2023-09-23T18:49:53.205921+0000 mon.a (mon.0) 1297 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:49:54.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:54 smithi173 bash[59422]: cluster 2023-09-23T18:49:53.227098+0000 osd.7 (osd.7) 175 : cluster 3 failed to encode map e250 with expected crc 2023-09-23T18:49:54.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:54 smithi173 bash[59422]: audit 2023-09-23T18:49:53.231371+0000 mon.a (mon.0) 1298 : audit 1 from='client.? 172.21.15.98:0/3833784593' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:49:54.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:54 smithi173 bash[59422]: cluster 2023-09-23T18:49:53.231701+0000 mon.a (mon.0) 1299 : cluster 0 osdmap e250: 8 total, 8 up, 8 in 2023-09-23T18:49:54.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:54 smithi173 bash[59422]: cluster 2023-09-23T18:49:53.235246+0000 osd.6 (osd.6) 209 : cluster 3 failed to encode map e250 with expected crc 2023-09-23T18:49:54.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:54 smithi173 bash[59422]: cluster 2023-09-23T18:49:54.106052+0000 mgr.y (mgr.26054) 954 : cluster 0 pgmap v1016: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 5.1 MiB/s wr, 459 op/s; 49359/299169 objects degraded (16.499%); 5799/299169 objects misplaced (1.938%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T18:49:54.598 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:54.625 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:54.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:54.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:54.728 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:54.750 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:54.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:54.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:54.853 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:54.885 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:54.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:54.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:54.988 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:55.011 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:55.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:55.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:55.114 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:55.162 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:55.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:55.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:55.268 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:55.294 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:55.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:55.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:55.397 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:55.434 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:55.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:55.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:55.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:55 smithi098 bash[76817]: cluster 2023-09-23T18:49:53.229624+0000 osd.4 (osd.4) 245 : cluster 3 failed to encode map e250 with expected crc 2023-09-23T18:49:55.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:55 smithi098 bash[76817]: audit 2023-09-23T18:49:54.621826+0000 mon.a (mon.0) 1300 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:49:55.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:55 smithi098 bash[76817]: cluster 2023-09-23T18:49:55.146541+0000 mon.a (mon.0) 1301 : cluster 3 Health check update: Degraded data redundancy: 49359/299169 objects degraded (16.499%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:49:55.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:55 smithi098 bash[79516]: cluster 2023-09-23T18:49:53.229624+0000 osd.4 (osd.4) 245 : cluster 3 failed to encode map e250 with expected crc 2023-09-23T18:49:55.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:55 smithi098 bash[79516]: audit 2023-09-23T18:49:54.621826+0000 mon.a (mon.0) 1300 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:49:55.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:55 smithi098 bash[79516]: cluster 2023-09-23T18:49:55.146541+0000 mon.a (mon.0) 1301 : cluster 3 Health check update: Degraded data redundancy: 49359/299169 objects degraded (16.499%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:49:55.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:55 smithi173 bash[59422]: cluster 2023-09-23T18:49:53.229624+0000 osd.4 (osd.4) 245 : cluster 3 failed to encode map e250 with expected crc 2023-09-23T18:49:55.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:55 smithi173 bash[59422]: audit 2023-09-23T18:49:54.621826+0000 mon.a (mon.0) 1300 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:49:55.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:55 smithi173 bash[59422]: cluster 2023-09-23T18:49:55.146541+0000 mon.a (mon.0) 1301 : cluster 3 Health check update: Degraded data redundancy: 49359/299169 objects degraded (16.499%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:49:55.536 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:55.547 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:49:55.547 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-09-23T18:49:55.547 INFO:teuthology.orchestra.run.smithi173.stdout: "complaint_time": 30, 2023-09-23T18:49:55.547 INFO:teuthology.orchestra.run.smithi173.stdout: "num_blocked_ops": 0 2023-09-23T18:49:55.547 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:49:55.567 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:55.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:55.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:55.671 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:55.703 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:55.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:55.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:55.808 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:55.842 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:55.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:55.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:55.949 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:55.983 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:55.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:55.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:56.051 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T18:49:56.051 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-09-23T18:49:56.088 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:56.114 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:56.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:56.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:56.217 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:56.258 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:56.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:56.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:56.362 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:56.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:56 smithi173 bash[59422]: cluster 2023-09-23T18:49:56.106895+0000 mgr.y (mgr.26054) 955 : cluster 0 pgmap v1017: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 3.4 MiB/s wr, 468 op/s; 49359/297750 objects degraded (16.577%); 5799/297750 objects misplaced (1.948%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T18:49:56.397 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:56.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:56.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:56.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:56 smithi098 bash[76817]: cluster 2023-09-23T18:49:56.106895+0000 mgr.y (mgr.26054) 955 : cluster 0 pgmap v1017: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 3.4 MiB/s wr, 468 op/s; 49359/297750 objects degraded (16.577%); 5799/297750 objects misplaced (1.948%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T18:49:56.442 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:49:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:49:56] "GET /metrics HTTP/1.1" 200 36619 "" "Prometheus/2.33.4" 2023-09-23T18:49:56.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:56 smithi098 bash[79516]: cluster 2023-09-23T18:49:56.106895+0000 mgr.y (mgr.26054) 955 : cluster 0 pgmap v1017: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 3.4 MiB/s wr, 468 op/s; 49359/297750 objects degraded (16.577%); 5799/297750 objects misplaced (1.948%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T18:49:56.478 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T18:49:56.500 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:56.537 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:56.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:56.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:56.646 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:56.683 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:56.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:56.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:56.789 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:56.814 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:49:56.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:56.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:49:56.920 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:56.948 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:56.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:56.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:57.053 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:57.094 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:57.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:57.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:57.200 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:57.230 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:57.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:57.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:57.336 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:57.367 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:57.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:57.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:57.472 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:57.501 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:57.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:57.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:57.608 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:57.633 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:57.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:57.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:57.741 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:57.772 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:57.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:57.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:57.875 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:57.901 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:57.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:57.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:58.004 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:58.031 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:58.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:58.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:58.084 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:49:58.109 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:49:58.138 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:58.166 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:58.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:58.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:58.270 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:58.299 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:58.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:58.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:58.406 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:58.418 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:49:58 smithi098 bash[79516]: cluster 2023-09-23T18:49:58.107481+0000 mgr.y (mgr.26054) 956 : cluster 0 pgmap v1018: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 49359/293427 objects degraded (16.822%); 5716/293427 objects misplaced (1.948%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T18:49:58.419 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:49:58 smithi098 bash[76817]: cluster 2023-09-23T18:49:58.107481+0000 mgr.y (mgr.26054) 956 : cluster 0 pgmap v1018: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 49359/293427 objects degraded (16.822%); 5716/293427 objects misplaced (1.948%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T18:49:58.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:58.447 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:58.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:58.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:49:58 smithi173 bash[59422]: cluster 2023-09-23T18:49:58.107481+0000 mgr.y (mgr.26054) 956 : cluster 0 pgmap v1018: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 49359/293427 objects degraded (16.822%); 5716/293427 objects misplaced (1.948%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T18:49:58.547 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:58.597 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:58.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:58.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:58.700 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:58.732 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:58.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:58.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:58.838 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:58.867 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:58.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:58.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:58.974 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:59.009 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:49:59.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:59.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:49:59.111 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:59.142 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:49:59.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:59.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:49:59.247 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:59.281 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:59.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:59.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:49:59.387 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:59.429 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:49:59.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:59.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:49:59.533 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:59.556 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:49:59.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:59.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:49:59.659 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:59.690 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:49:59.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:59.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:49:59.793 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:59.826 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:49:59.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:59.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:49:59.932 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:59.975 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:49:59.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:49:59.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:00.079 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:00.104 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:00.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:00.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:00.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:00 smithi098 bash[76817]: cluster 2023-09-23T18:50:00.000116+0000 mon.a (mon.0) 1302 : cluster 3 Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 49359/293427 objects degraded (16.822%), 8 pgs degraded 2023-09-23T18:50:00.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:00 smithi098 bash[76817]: cluster 2023-09-23T18:50:00.000158+0000 mon.a (mon.0) 1303 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T18:50:00.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:00 smithi098 bash[76817]: cluster 2023-09-23T18:50:00.000177+0000 mon.a (mon.0) 1304 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 49359/293427 objects degraded (16.822%), 8 pgs degraded 2023-09-23T18:50:00.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:00 smithi098 bash[76817]: cluster 2023-09-23T18:50:00.000196+0000 mon.a (mon.0) 1305 : cluster 3 pg 8.1 is active+recovery_wait+undersized+degraded+remapped, acting [1,0] 2023-09-23T18:50:00.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:00 smithi098 bash[76817]: cluster 2023-09-23T18:50:00.000211+0000 mon.a (mon.0) 1306 : cluster 3 pg 8.4 is active+recovery_wait+undersized+degraded+remapped, acting [5,4] 2023-09-23T18:50:00.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:00 smithi098 bash[76817]: cluster 2023-09-23T18:50:00.000229+0000 mon.a (mon.0) 1307 : cluster 3 pg 8.9 is active+recovery_wait+undersized+degraded+remapped, acting [0,6] 2023-09-23T18:50:00.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:00 smithi098 bash[76817]: cluster 2023-09-23T18:50:00.000243+0000 mon.a (mon.0) 1308 : cluster 3 pg 8.a is active+recovery_wait+undersized+degraded+remapped, acting [1,2] 2023-09-23T18:50:00.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:00 smithi098 bash[76817]: cluster 2023-09-23T18:50:00.000259+0000 mon.a (mon.0) 1309 : cluster 3 pg 8.b is active+recovery_wait+undersized+degraded+remapped, acting [4,1] 2023-09-23T18:50:00.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:00 smithi098 bash[76817]: cluster 2023-09-23T18:50:00.000276+0000 mon.a (mon.0) 1310 : cluster 3 pg 8.c is active+recovery_wait+undersized+degraded+remapped, acting [0,3] 2023-09-23T18:50:00.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:00 smithi098 bash[76817]: cluster 2023-09-23T18:50:00.000293+0000 mon.a (mon.0) 1311 : cluster 3 pg 8.d is active+recovery_wait+undersized+degraded+remapped, acting [5,7] 2023-09-23T18:50:00.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:00 smithi098 bash[76817]: cluster 2023-09-23T18:50:00.000310+0000 mon.a (mon.0) 1312 : cluster 3 pg 8.f is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2023-09-23T18:50:00.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:00 smithi098 bash[79516]: cluster 2023-09-23T18:50:00.000116+0000 mon.a (mon.0) 1302 : cluster 3 Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 49359/293427 objects degraded (16.822%), 8 pgs degraded 2023-09-23T18:50:00.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:00 smithi098 bash[79516]: cluster 2023-09-23T18:50:00.000158+0000 mon.a (mon.0) 1303 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T18:50:00.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:00 smithi098 bash[79516]: cluster 2023-09-23T18:50:00.000177+0000 mon.a (mon.0) 1304 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 49359/293427 objects degraded (16.822%), 8 pgs degraded 2023-09-23T18:50:00.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:00 smithi098 bash[79516]: cluster 2023-09-23T18:50:00.000196+0000 mon.a (mon.0) 1305 : cluster 3 pg 8.1 is active+recovery_wait+undersized+degraded+remapped, acting [1,0] 2023-09-23T18:50:00.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:00 smithi098 bash[79516]: cluster 2023-09-23T18:50:00.000211+0000 mon.a (mon.0) 1306 : cluster 3 pg 8.4 is active+recovery_wait+undersized+degraded+remapped, acting [5,4] 2023-09-23T18:50:00.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:00 smithi098 bash[79516]: cluster 2023-09-23T18:50:00.000229+0000 mon.a (mon.0) 1307 : cluster 3 pg 8.9 is active+recovery_wait+undersized+degraded+remapped, acting [0,6] 2023-09-23T18:50:00.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:00 smithi098 bash[79516]: cluster 2023-09-23T18:50:00.000243+0000 mon.a (mon.0) 1308 : cluster 3 pg 8.a is active+recovery_wait+undersized+degraded+remapped, acting [1,2] 2023-09-23T18:50:00.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:00 smithi098 bash[79516]: cluster 2023-09-23T18:50:00.000259+0000 mon.a (mon.0) 1309 : cluster 3 pg 8.b is active+recovery_wait+undersized+degraded+remapped, acting [4,1] 2023-09-23T18:50:00.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:00 smithi098 bash[79516]: cluster 2023-09-23T18:50:00.000276+0000 mon.a (mon.0) 1310 : cluster 3 pg 8.c is active+recovery_wait+undersized+degraded+remapped, acting [0,3] 2023-09-23T18:50:00.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:00 smithi098 bash[79516]: cluster 2023-09-23T18:50:00.000293+0000 mon.a (mon.0) 1311 : cluster 3 pg 8.d is active+recovery_wait+undersized+degraded+remapped, acting [5,7] 2023-09-23T18:50:00.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:00 smithi098 bash[79516]: cluster 2023-09-23T18:50:00.000310+0000 mon.a (mon.0) 1312 : cluster 3 pg 8.f is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2023-09-23T18:50:00.209 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:00.242 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:00.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:00.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:00.346 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:00.354 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:00 smithi173 bash[59422]: cluster 2023-09-23T18:50:00.000116+0000 mon.a (mon.0) 1302 : cluster 3 Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 49359/293427 objects degraded (16.822%), 8 pgs degraded 2023-09-23T18:50:00.355 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:00 smithi173 bash[59422]: cluster 2023-09-23T18:50:00.000158+0000 mon.a (mon.0) 1303 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T18:50:00.355 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:00 smithi173 bash[59422]: cluster 2023-09-23T18:50:00.000177+0000 mon.a (mon.0) 1304 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 49359/293427 objects degraded (16.822%), 8 pgs degraded 2023-09-23T18:50:00.355 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:00 smithi173 bash[59422]: cluster 2023-09-23T18:50:00.000196+0000 mon.a (mon.0) 1305 : cluster 3 pg 8.1 is active+recovery_wait+undersized+degraded+remapped, acting [1,0] 2023-09-23T18:50:00.355 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:00 smithi173 bash[59422]: cluster 2023-09-23T18:50:00.000211+0000 mon.a (mon.0) 1306 : cluster 3 pg 8.4 is active+recovery_wait+undersized+degraded+remapped, acting [5,4] 2023-09-23T18:50:00.355 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:00 smithi173 bash[59422]: cluster 2023-09-23T18:50:00.000229+0000 mon.a (mon.0) 1307 : cluster 3 pg 8.9 is active+recovery_wait+undersized+degraded+remapped, acting [0,6] 2023-09-23T18:50:00.355 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:00 smithi173 bash[59422]: cluster 2023-09-23T18:50:00.000243+0000 mon.a (mon.0) 1308 : cluster 3 pg 8.a is active+recovery_wait+undersized+degraded+remapped, acting [1,2] 2023-09-23T18:50:00.355 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:00 smithi173 bash[59422]: cluster 2023-09-23T18:50:00.000259+0000 mon.a (mon.0) 1309 : cluster 3 pg 8.b is active+recovery_wait+undersized+degraded+remapped, acting [4,1] 2023-09-23T18:50:00.355 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:00 smithi173 bash[59422]: cluster 2023-09-23T18:50:00.000276+0000 mon.a (mon.0) 1310 : cluster 3 pg 8.c is active+recovery_wait+undersized+degraded+remapped, acting [0,3] 2023-09-23T18:50:00.355 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:00 smithi173 bash[59422]: cluster 2023-09-23T18:50:00.000293+0000 mon.a (mon.0) 1311 : cluster 3 pg 8.d is active+recovery_wait+undersized+degraded+remapped, acting [5,7] 2023-09-23T18:50:00.355 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:00 smithi173 bash[59422]: cluster 2023-09-23T18:50:00.000310+0000 mon.a (mon.0) 1312 : cluster 3 pg 8.f is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2023-09-23T18:50:00.381 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:00.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:00.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:00.486 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:00.514 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:00.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:00.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:00.619 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:00.648 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:00.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:00.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:00.751 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:00.774 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:00.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:00.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:00.881 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:00.908 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:50:00.913 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:00.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:00.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:01.023 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:01.047 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:01.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:01.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:01.155 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:01.182 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:01.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:01.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:01.285 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:50:01.287 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:01.296 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:01 smithi098 bash[76817]: cluster 2023-09-23T18:50:00.108104+0000 mgr.y (mgr.26054) 957 : cluster 0 pgmap v1019: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 49359/291432 objects degraded (16.937%); 5716/291432 objects misplaced (1.961%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T18:50:01.296 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:01 smithi098 bash[76817]: cluster 2023-09-23T18:50:00.147969+0000 mon.a (mon.0) 1313 : cluster 3 Health check update: Degraded data redundancy: 49359/293427 objects degraded (16.822%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:50:01.296 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:01 smithi098 bash[79516]: cluster 2023-09-23T18:50:00.108104+0000 mgr.y (mgr.26054) 957 : cluster 0 pgmap v1019: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 49359/291432 objects degraded (16.937%); 5716/291432 objects misplaced (1.961%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T18:50:01.296 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:01 smithi098 bash[79516]: cluster 2023-09-23T18:50:00.147969+0000 mon.a (mon.0) 1313 : cluster 3 Health check update: Degraded data redundancy: 49359/293427 objects degraded (16.822%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:50:01.309 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:01 smithi173 bash[59422]: cluster 2023-09-23T18:50:00.108104+0000 mgr.y (mgr.26054) 957 : cluster 0 pgmap v1019: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 49359/291432 objects degraded (16.937%); 5716/291432 objects misplaced (1.961%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T18:50:01.309 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:01 smithi173 bash[59422]: cluster 2023-09-23T18:50:00.147969+0000 mon.a (mon.0) 1313 : cluster 3 Health check update: Degraded data redundancy: 49359/293427 objects degraded (16.822%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:50:01.320 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:01.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:01.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:01.423 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:01.470 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:01.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:01.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:01.574 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:01.595 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:01.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:01.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:01.699 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:01.740 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:01.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:01.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:01.846 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:01.870 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:01.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:01.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:01.977 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:02.006 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:02.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:02.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:02.108 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:02.143 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:02.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:02.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:02.213 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:02 smithi173 bash[59422]: cluster 2023-09-23T18:50:02.108916+0000 mgr.y (mgr.26054) 958 : cluster 0 pgmap v1020: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 49388/285339 objects degraded (17.309%); 5597/285339 objects misplaced (1.962%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T18:50:02.251 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:02.281 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:02.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:02.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:02.384 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:02 smithi098 bash[76817]: cluster 2023-09-23T18:50:02.108916+0000 mgr.y (mgr.26054) 958 : cluster 0 pgmap v1020: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 49388/285339 objects degraded (17.309%); 5597/285339 objects misplaced (1.962%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T18:50:02.384 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:02 smithi098 bash[79516]: cluster 2023-09-23T18:50:02.108916+0000 mgr.y (mgr.26054) 958 : cluster 0 pgmap v1020: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 49388/285339 objects degraded (17.309%); 5597/285339 objects misplaced (1.962%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T18:50:02.387 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:02.412 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:02.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:02.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:02.519 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:02.542 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:02.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:02.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:02.645 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:02.672 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:02.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:02.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:02.775 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:02.815 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:02.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:02.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:02.923 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:02.947 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:02.948 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:50:02.948 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:50:02.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:02.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:03.053 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:03.078 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:03.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:03.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:03.183 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:03.209 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:03.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:03.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:03.313 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:03.356 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:03.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:03.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:03.463 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:03.494 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:03.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:03.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:03.598 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:03.633 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:03.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:03.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:03.737 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:03.771 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:03.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:03.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:03.874 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:03.922 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:03.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:03.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:04.028 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:04.061 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:04.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:04.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:04.168 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:04.190 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:04.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:04.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:04.294 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:04.328 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:04.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:04.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:04.434 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:04.470 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:04.473 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:04 smithi098 bash[76817]: cluster 2023-09-23T18:50:04.109540+0000 mgr.y (mgr.26054) 959 : cluster 0 pgmap v1021: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 49388/283236 objects degraded (17.437%); 5597/283236 objects misplaced (1.976%); 1017 KiB/s, 15 objects/s recovering 2023-09-23T18:50:04.473 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:04 smithi098 bash[79516]: cluster 2023-09-23T18:50:04.109540+0000 mgr.y (mgr.26054) 959 : cluster 0 pgmap v1021: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 49388/283236 objects degraded (17.437%); 5597/283236 objects misplaced (1.976%); 1017 KiB/s, 15 objects/s recovering 2023-09-23T18:50:04.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:04.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:04.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:04 smithi173 bash[59422]: cluster 2023-09-23T18:50:04.109540+0000 mgr.y (mgr.26054) 959 : cluster 0 pgmap v1021: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 49388/283236 objects degraded (17.437%); 5597/283236 objects misplaced (1.976%); 1017 KiB/s, 15 objects/s recovering 2023-09-23T18:50:04.575 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:04.603 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:04.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:04.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:04.705 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:04.737 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:04.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:04.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:04.839 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:04.861 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:04.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:04.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:04.966 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:05.011 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:05.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:05.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:05.117 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:05.145 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:05.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:05.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:05.248 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:05.277 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:05.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:05.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:05.382 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:05.390 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:05 smithi173 bash[59422]: cluster 2023-09-23T18:50:05.151587+0000 mon.a (mon.0) 1314 : cluster 3 Health check update: Degraded data redundancy: 49388/283236 objects degraded (17.437%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:50:05.407 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:05.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:05.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:05.511 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:05.534 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:05.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:05.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:05.632 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:50:05.633 INFO:teuthology.orchestra.run.smithi173.stdout: "size": 20, 2023-09-23T18:50:05.633 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 600, 2023-09-23T18:50:05.633 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-09-23T18:50:05.633 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.633 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44954.0:170216 6.c 6:37c18518:::benchmark_data_smithi098_131983_object65547:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-09-23T18:50:05.633 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:40:41.048233+0000", 2023-09-23T18:50:05.634 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 564.57982673699996, 2023-09-23T18:50:05.634 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2877881769999999, 2023-09-23T18:50:05.634 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:50:05.634 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:50:05.634 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:50:05.635 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44954", 2023-09-23T18:50:05.635 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/343998332", 2023-09-23T18:50:05.635 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 170216 2023-09-23T18:50:05.635 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.635 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:50:05.635 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.635 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:50:05.636 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:41.048233+0000", 2023-09-23T18:50:05.636 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:50:05.636 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.636 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.636 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:50:05.637 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:41.048233+0000", 2023-09-23T18:50:05.637 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-09-23T18:50:05.637 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.637 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.637 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:50:05.637 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:41.048232+0000", 2023-09-23T18:50:05.637 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.4159999999999997e-06 2023-09-23T18:50:05.637 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.637 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.637 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:50:05.638 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:41.048237+0000", 2023-09-23T18:50:05.638 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.2530000000000004e-06 2023-09-23T18:50:05.638 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.638 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.638 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:50:05.638 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:41.048245+0000", 2023-09-23T18:50:05.638 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.3810000000000001e-06 2023-09-23T18:50:05.638 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.638 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.639 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:50:05.639 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:41.048249+0000", 2023-09-23T18:50:05.639 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000211614 2023-09-23T18:50:05.639 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.639 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.639 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.639 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:41.048461+0000", 2023-09-23T18:50:05.639 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2663000000000001e-05 2023-09-23T18:50:05.639 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.639 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.639 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:50:05.640 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:41.048474+0000", 2023-09-23T18:50:05.640 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.000380482 2023-09-23T18:50:05.640 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.640 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.640 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.640 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:42.048854+0000", 2023-09-23T18:50:05.640 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2265e-05 2023-09-23T18:50:05.640 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.640 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.640 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:50:05.640 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:42.048866+0000", 2023-09-23T18:50:05.641 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.240822418 2023-09-23T18:50:05.641 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.641 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.641 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.641 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.289689+0000", 2023-09-23T18:50:05.641 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0035047059999999998 2023-09-23T18:50:05.641 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.641 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.641 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:50:05.641 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.293193+0000", 2023-09-23T18:50:05.642 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00076717099999999998 2023-09-23T18:50:05.642 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.642 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.642 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T18:50:05.642 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.293961+0000", 2023-09-23T18:50:05.642 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.039782645999999998 2023-09-23T18:50:05.642 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.642 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.642 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:50:05.643 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.333743+0000", 2023-09-23T18:50:05.643 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00024898200000000002 2023-09-23T18:50:05.643 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.643 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.643 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:50:05.643 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.333992+0000", 2023-09-23T18:50:05.643 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0017978969999999999 2023-09-23T18:50:05.643 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.643 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.643 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:50:05.643 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.335790+0000", 2023-09-23T18:50:05.644 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.6393999999999999e-05 2023-09-23T18:50:05.644 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.644 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.644 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:50:05.644 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.335886+0000", 2023-09-23T18:50:05.644 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.3510000000000001e-05 2023-09-23T18:50:05.644 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.644 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.644 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:50:05.644 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.335940+0000", 2023-09-23T18:50:05.644 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.1091e-05 2023-09-23T18:50:05.645 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.645 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.645 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:50:05.645 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.336021+0000", 2023-09-23T18:50:05.645 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2877881769999999 2023-09-23T18:50:05.645 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.645 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:50:05.646 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.646 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.646 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.646 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44954.0:170222 6.c 6:3173f1e7:::benchmark_data_smithi098_131983_object65553:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-09-23T18:50:05.646 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:40:41.070001+0000", 2023-09-23T18:50:05.647 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 564.55805910399999, 2023-09-23T18:50:05.647 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2664350400000002, 2023-09-23T18:50:05.647 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:50:05.647 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:50:05.647 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:50:05.647 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44954", 2023-09-23T18:50:05.647 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/343998332", 2023-09-23T18:50:05.647 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 170222 2023-09-23T18:50:05.647 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.648 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:50:05.648 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.648 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:50:05.648 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:41.070001+0000", 2023-09-23T18:50:05.648 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:50:05.648 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.648 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.648 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:50:05.648 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:41.070001+0000", 2023-09-23T18:50:05.648 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:50:05.649 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.649 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.649 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:50:05.649 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:41.069999+0000", 2023-09-23T18:50:05.649 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.0169999999999999e-06 2023-09-23T18:50:05.649 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.649 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.649 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:50:05.649 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:41.070006+0000", 2023-09-23T18:50:05.649 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6218999999999998e-05 2023-09-23T18:50:05.650 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.650 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.650 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:50:05.650 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:41.070022+0000", 2023-09-23T18:50:05.650 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0645999999999999e-05 2023-09-23T18:50:05.650 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.650 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.650 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:50:05.650 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:41.070033+0000", 2023-09-23T18:50:05.651 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.4635999999999996e-05 2023-09-23T18:50:05.651 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.651 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.651 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.651 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:41.070097+0000", 2023-09-23T18:50:05.651 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1041e-05 2023-09-23T18:50:05.651 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.651 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.651 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:50:05.652 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:41.070108+0000", 2023-09-23T18:50:05.652 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.97878113099999997 2023-09-23T18:50:05.652 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.652 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.652 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.652 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:42.048889+0000", 2023-09-23T18:50:05.652 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.7600000000000008e-06 2023-09-23T18:50:05.652 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.652 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.652 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:50:05.653 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:42.048898+0000", 2023-09-23T18:50:05.653 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.276906519 2023-09-23T18:50:05.653 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.653 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.653 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.653 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.325805+0000", 2023-09-23T18:50:05.653 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00020134100000000001 2023-09-23T18:50:05.653 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.654 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.654 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:50:05.654 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.326006+0000", 2023-09-23T18:50:05.654 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018119300000000001 2023-09-23T18:50:05.654 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.654 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.654 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T18:50:05.654 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.326187+0000", 2023-09-23T18:50:05.654 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0075671590000000004 2023-09-23T18:50:05.655 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.655 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.655 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:50:05.655 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.333754+0000", 2023-09-23T18:50:05.655 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0023170589999999998 2023-09-23T18:50:05.655 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.655 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.655 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:50:05.655 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.336071+0000", 2023-09-23T18:50:05.656 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.9578000000000004e-05 2023-09-23T18:50:05.656 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.656 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.656 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:50:05.656 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.336141+0000", 2023-09-23T18:50:05.656 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000147804 2023-09-23T18:50:05.656 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.656 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.656 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:50:05.657 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.336289+0000", 2023-09-23T18:50:05.657 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4252000000000002e-05 2023-09-23T18:50:05.657 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.657 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.657 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:50:05.657 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.336343+0000", 2023-09-23T18:50:05.657 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.2440000000000003e-05 2023-09-23T18:50:05.657 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.657 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.657 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:50:05.658 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.336436+0000", 2023-09-23T18:50:05.658 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2664350400000002 2023-09-23T18:50:05.658 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.658 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:50:05.658 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.658 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.658 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.658 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44954.0:170227 6.c 6:382f2840:::benchmark_data_smithi098_131983_object65558:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-09-23T18:50:05.658 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:40:41.080833+0000", 2023-09-23T18:50:05.658 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 564.54722662200004, 2023-09-23T18:50:05.659 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2557285220000001, 2023-09-23T18:50:05.659 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:50:05.659 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:50:05.659 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:50:05.659 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44954", 2023-09-23T18:50:05.659 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/343998332", 2023-09-23T18:50:05.659 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 170227 2023-09-23T18:50:05.659 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.659 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:50:05.659 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.660 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:50:05.660 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:41.080833+0000", 2023-09-23T18:50:05.660 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:50:05.660 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.660 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.660 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:50:05.660 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:41.080833+0000", 2023-09-23T18:50:05.660 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:50:05.660 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.660 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.661 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:50:05.661 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:41.080831+0000", 2023-09-23T18:50:05.661 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4222999999999999e-05 2023-09-23T18:50:05.661 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.661 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.661 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:50:05.661 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:41.080846+0000", 2023-09-23T18:50:05.661 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9326000000000001e-05 2023-09-23T18:50:05.661 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.661 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.662 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:50:05.662 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:41.080865+0000", 2023-09-23T18:50:05.662 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.2099999999999993e-06 2023-09-23T18:50:05.662 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.662 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.662 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:50:05.662 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:41.080873+0000", 2023-09-23T18:50:05.662 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.8350000000000002e-05 2023-09-23T18:50:05.663 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.663 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.663 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.663 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:41.080931+0000", 2023-09-23T18:50:05.663 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2465e-05 2023-09-23T18:50:05.663 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.663 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.663 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:50:05.663 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:41.080944+0000", 2023-09-23T18:50:05.663 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.96797599599999995 2023-09-23T18:50:05.664 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.664 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.664 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.664 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:42.048920+0000", 2023-09-23T18:50:05.664 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.106e-06 2023-09-23T18:50:05.664 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.665 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.665 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:50:05.665 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:42.048928+0000", 2023-09-23T18:50:05.665 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2797435269999999 2023-09-23T18:50:05.665 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.666 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.666 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.666 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.328672+0000", 2023-09-23T18:50:05.666 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.6417999999999998e-05 2023-09-23T18:50:05.666 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.666 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.666 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:50:05.666 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.328758+0000", 2023-09-23T18:50:05.666 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013553999999999999 2023-09-23T18:50:05.666 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.666 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.667 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T18:50:05.667 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.328893+0000", 2023-09-23T18:50:05.667 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0049417009999999997 2023-09-23T18:50:05.667 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.667 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.667 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:50:05.667 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.333835+0000", 2023-09-23T18:50:05.667 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0023686240000000002 2023-09-23T18:50:05.667 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.667 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.667 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:50:05.667 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.336204+0000", 2023-09-23T18:50:05.668 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.3544000000000002e-05 2023-09-23T18:50:05.668 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.668 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.668 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:50:05.668 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.336247+0000", 2023-09-23T18:50:05.668 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000222301 2023-09-23T18:50:05.668 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.668 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.668 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:50:05.668 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.336470+0000", 2023-09-23T18:50:05.668 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.6760000000000001e-05 2023-09-23T18:50:05.668 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.668 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.669 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:50:05.669 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.336496+0000", 2023-09-23T18:50:05.669 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.5110999999999994e-05 2023-09-23T18:50:05.669 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.669 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.669 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:50:05.669 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:40:43.336562+0000", 2023-09-23T18:50:05.669 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2557285220000001 2023-09-23T18:50:05.669 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.669 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:50:05.669 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.669 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.670 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.670 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.44954.0:192482 6.c 6:3d8ddd2d:::benchmark_data_smithi098_131983_object87813:head [delete] snapc 0=[] ondisk+write+known_if_redirected e178)", 2023-09-23T18:50:05.670 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:41:17.165737+0000", 2023-09-23T18:50:05.670 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 528.46232268799997, 2023-09-23T18:50:05.670 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.30528654999999999, 2023-09-23T18:50:05.670 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:50:05.670 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:50:05.670 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:50:05.670 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.44954", 2023-09-23T18:50:05.670 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/343998332", 2023-09-23T18:50:05.670 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 192482 2023-09-23T18:50:05.670 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.671 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:50:05.671 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.671 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:50:05.671 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:41:17.165737+0000", 2023-09-23T18:50:05.671 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:50:05.671 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.671 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.671 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:50:05.671 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:41:17.165737+0000", 2023-09-23T18:50:05.671 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:50:05.671 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.671 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.671 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:50:05.672 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:41:17.165736+0000", 2023-09-23T18:50:05.672 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0858e-05 2023-09-23T18:50:05.672 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.672 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.672 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:50:05.672 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:41:17.165747+0000", 2023-09-23T18:50:05.672 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4481000000000001e-05 2023-09-23T18:50:05.672 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.672 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.672 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:50:05.673 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:41:17.165761+0000", 2023-09-23T18:50:05.673 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.5780000000000004e-06 2023-09-23T18:50:05.673 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.673 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.673 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:50:05.673 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:41:17.165768+0000", 2023-09-23T18:50:05.674 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.003509454 2023-09-23T18:50:05.674 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.674 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.674 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.674 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:41:17.169277+0000", 2023-09-23T18:50:05.674 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012852600000000001 2023-09-23T18:50:05.674 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.674 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.674 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:50:05.674 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:41:17.169406+0000", 2023-09-23T18:50:05.674 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000170839 2023-09-23T18:50:05.675 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.675 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.675 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T18:50:05.675 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:41:17.169577+0000", 2023-09-23T18:50:05.675 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0043646240000000001 2023-09-23T18:50:05.675 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.675 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.675 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:50:05.675 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:41:17.173941+0000", 2023-09-23T18:50:05.675 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0010119630000000001 2023-09-23T18:50:05.675 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.675 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.675 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:50:05.676 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:41:17.174953+0000", 2023-09-23T18:50:05.676 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.007125536 2023-09-23T18:50:05.676 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.676 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.676 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:50:05.676 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:41:17.182079+0000", 2023-09-23T18:50:05.676 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.28851627699999999 2023-09-23T18:50:05.676 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.676 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.676 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:50:05.676 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:41:17.470595+0000", 2023-09-23T18:50:05.676 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000172905 2023-09-23T18:50:05.677 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.677 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.677 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:50:05.677 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:41:17.470768+0000", 2023-09-23T18:50:05.677 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00025541700000000002 2023-09-23T18:50:05.677 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.677 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.677 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:50:05.677 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:41:17.471023+0000", 2023-09-23T18:50:05.677 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.30528654999999999 2023-09-23T18:50:05.677 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.677 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:50:05.678 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.678 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.678 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.678 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45188.0:80514 7.d 7:b8ede8b5:::benchmark_data_smithi098_155798_object80513:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e197)", 2023-09-23T18:50:05.678 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:44:04.323229+0000", 2023-09-23T18:50:05.678 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 361.30483073800002, 2023-09-23T18:50:05.678 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.247874017, 2023-09-23T18:50:05.678 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:50:05.678 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:50:05.678 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:50:05.678 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45188", 2023-09-23T18:50:05.678 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:50:05.679 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 80514 2023-09-23T18:50:05.679 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.679 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:50:05.679 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.679 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:50:05.679 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:04.323229+0000", 2023-09-23T18:50:05.679 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:50:05.679 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.679 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.679 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:50:05.679 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:04.323229+0000", 2023-09-23T18:50:05.679 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:50:05.679 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.680 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.680 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:50:05.680 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:04.323228+0000", 2023-09-23T18:50:05.680 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.9762999999999999e-05 2023-09-23T18:50:05.680 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.680 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.680 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:50:05.680 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:04.323268+0000", 2023-09-23T18:50:05.680 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.5364e-05 2023-09-23T18:50:05.680 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.680 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.680 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:50:05.680 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:04.323293+0000", 2023-09-23T18:50:05.681 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.0000000000000004e-06 2023-09-23T18:50:05.681 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.681 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.681 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:50:05.681 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:04.323298+0000", 2023-09-23T18:50:05.681 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.1829000000000002e-05 2023-09-23T18:50:05.681 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.681 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.681 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.681 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:04.323340+0000", 2023-09-23T18:50:05.681 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000137047 2023-09-23T18:50:05.681 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.682 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.682 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:50:05.682 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:04.323477+0000", 2023-09-23T18:50:05.682 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018196299999999999 2023-09-23T18:50:05.682 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.682 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.682 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,1", 2023-09-23T18:50:05.682 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:04.323659+0000", 2023-09-23T18:50:05.682 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0029863149999999998 2023-09-23T18:50:05.682 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.682 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.682 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:50:05.683 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:04.326645+0000", 2023-09-23T18:50:05.683 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00099791599999999995 2023-09-23T18:50:05.683 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.683 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.683 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:50:05.683 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:04.327643+0000", 2023-09-23T18:50:05.683 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.24334156400000001 2023-09-23T18:50:05.683 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.683 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.683 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:50:05.683 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:04.570985+0000", 2023-09-23T18:50:05.683 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.7916999999999994e-05 2023-09-23T18:50:05.684 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.684 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.684 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:50:05.684 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:04.571053+0000", 2023-09-23T18:50:05.684 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.0093999999999998e-05 2023-09-23T18:50:05.684 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.684 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.684 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:50:05.684 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:04.571103+0000", 2023-09-23T18:50:05.684 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.247874017 2023-09-23T18:50:05.684 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.684 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:50:05.685 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.685 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.685 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.685 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45188.0:88846 7.12 7.a2512212 (undecoded) ondisk+write+known_if_redirected e203)", 2023-09-23T18:50:05.685 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:44:16.408821+0000", 2023-09-23T18:50:05.685 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 349.21923897900001, 2023-09-23T18:50:05.685 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.65844427800000005, 2023-09-23T18:50:05.685 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:50:05.685 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T18:50:05.685 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:50:05.685 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45188", 2023-09-23T18:50:05.685 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:50:05.685 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 88846 2023-09-23T18:50:05.686 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.686 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:50:05.686 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.686 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:50:05.686 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.408821+0000", 2023-09-23T18:50:05.686 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:50:05.686 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.686 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.686 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:50:05.686 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.408821+0000", 2023-09-23T18:50:05.686 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:50:05.687 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.687 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.687 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:50:05.687 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.408819+0000", 2023-09-23T18:50:05.687 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6378000000000002e-05 2023-09-23T18:50:05.687 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.687 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.687 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:50:05.687 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.408865+0000", 2023-09-23T18:50:05.687 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4486e-05 2023-09-23T18:50:05.687 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.687 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.687 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:50:05.688 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.408880+0000", 2023-09-23T18:50:05.688 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1330000000000004e-06 2023-09-23T18:50:05.688 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.688 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.688 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:50:05.688 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.408886+0000", 2023-09-23T18:50:05.688 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.7057999999999994e-05 2023-09-23T18:50:05.688 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.688 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.688 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.688 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.408953+0000", 2023-09-23T18:50:05.688 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2869e-05 2023-09-23T18:50:05.688 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.689 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.689 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:50:05.689 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.408966+0000", 2023-09-23T18:50:05.689 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.65828120999999995 2023-09-23T18:50:05.689 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.689 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.689 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.689 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:17.067247+0000", 2023-09-23T18:50:05.689 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8134999999999998e-05 2023-09-23T18:50:05.689 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.689 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.689 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:50:05.690 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:17.067265+0000", 2023-09-23T18:50:05.690 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.65844427800000005 2023-09-23T18:50:05.690 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.690 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:50:05.690 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.690 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.690 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.690 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45188.0:88873 7.12 7.92aa91d2 (undecoded) ondisk+write+known_if_redirected e203)", 2023-09-23T18:50:05.690 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:44:16.447364+0000", 2023-09-23T18:50:05.690 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 349.18069611599998, 2023-09-23T18:50:05.690 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.61998016499999997, 2023-09-23T18:50:05.690 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:50:05.690 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T18:50:05.691 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:50:05.691 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45188", 2023-09-23T18:50:05.691 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:50:05.691 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 88873 2023-09-23T18:50:05.691 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.691 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:50:05.691 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.691 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:50:05.691 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.447364+0000", 2023-09-23T18:50:05.691 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:50:05.691 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.691 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.691 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:50:05.692 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.447364+0000", 2023-09-23T18:50:05.692 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999943 2023-09-23T18:50:05.692 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.692 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.692 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:50:05.692 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.447358+0000", 2023-09-23T18:50:05.692 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.5668999999999999e-05 2023-09-23T18:50:05.692 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.692 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.692 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:50:05.692 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.447403+0000", 2023-09-23T18:50:05.692 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8379000000000001e-05 2023-09-23T18:50:05.693 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.693 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.693 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:50:05.693 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.447422+0000", 2023-09-23T18:50:05.693 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.1609999999999992e-06 2023-09-23T18:50:05.693 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.693 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.693 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:50:05.693 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.447431+0000", 2023-09-23T18:50:05.693 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.9915999999999999e-05 2023-09-23T18:50:05.693 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.693 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.693 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.694 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.447511+0000", 2023-09-23T18:50:05.694 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.2630000000000003e-06 2023-09-23T18:50:05.694 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.694 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.694 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:50:05.694 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.447519+0000", 2023-09-23T18:50:05.694 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.61980976799999998 2023-09-23T18:50:05.694 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.694 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.694 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.694 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:17.067329+0000", 2023-09-23T18:50:05.694 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4727e-05 2023-09-23T18:50:05.695 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.695 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.695 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:50:05.695 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:17.067344+0000", 2023-09-23T18:50:05.695 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.61998016499999997 2023-09-23T18:50:05.695 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.695 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:50:05.695 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.695 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.695 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.695 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45188.0:88883 7.12 7.dd2e2b12 (undecoded) ondisk+write+known_if_redirected e203)", 2023-09-23T18:50:05.695 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:44:16.459238+0000", 2023-09-23T18:50:05.695 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 349.16882204799998, 2023-09-23T18:50:05.696 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.60814314999999997, 2023-09-23T18:50:05.696 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:50:05.696 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T18:50:05.696 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:50:05.696 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45188", 2023-09-23T18:50:05.696 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:50:05.696 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 88883 2023-09-23T18:50:05.696 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.696 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:50:05.696 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.696 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:50:05.696 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.459238+0000", 2023-09-23T18:50:05.697 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:50:05.697 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.697 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.697 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:50:05.697 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.459238+0000", 2023-09-23T18:50:05.697 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:50:05.697 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.697 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.697 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:50:05.697 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.459236+0000", 2023-09-23T18:50:05.697 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.2135999999999997e-05 2023-09-23T18:50:05.697 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.697 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.698 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:50:05.698 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.459288+0000", 2023-09-23T18:50:05.698 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4219e-05 2023-09-23T18:50:05.698 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.698 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.698 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:50:05.698 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.459312+0000", 2023-09-23T18:50:05.698 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2267e-05 2023-09-23T18:50:05.698 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.698 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.698 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:50:05.698 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.459324+0000", 2023-09-23T18:50:05.698 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.4166999999999997e-05 2023-09-23T18:50:05.699 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.699 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.699 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.699 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.459388+0000", 2023-09-23T18:50:05.699 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.8419999999999999e-06 2023-09-23T18:50:05.699 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.699 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.699 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:50:05.699 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.459395+0000", 2023-09-23T18:50:05.699 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.60797717799999995 2023-09-23T18:50:05.699 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.699 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.700 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.700 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:17.067372+0000", 2023-09-23T18:50:05.700 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.4090000000000003e-06 2023-09-23T18:50:05.700 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.700 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.700 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:50:05.700 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:17.067381+0000", 2023-09-23T18:50:05.700 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.60814314999999997 2023-09-23T18:50:05.700 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.700 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:50:05.700 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.700 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.700 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.701 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45188.0:88917 7.12 7.a393932 (undecoded) ondisk+write+known_if_redirected e203)", 2023-09-23T18:50:05.701 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:44:16.499173+0000", 2023-09-23T18:50:05.701 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 349.12888629700001, 2023-09-23T18:50:05.701 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.56824535200000004, 2023-09-23T18:50:05.701 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:50:05.701 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T18:50:05.701 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:50:05.701 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45188", 2023-09-23T18:50:05.701 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:50:05.701 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 88917 2023-09-23T18:50:05.701 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.701 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:50:05.701 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.702 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:50:05.702 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.499173+0000", 2023-09-23T18:50:05.702 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:50:05.702 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.702 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.702 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:50:05.702 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.499173+0000", 2023-09-23T18:50:05.702 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:50:05.702 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.702 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.702 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:50:05.702 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.499172+0000", 2023-09-23T18:50:05.703 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.4804999999999999e-05 2023-09-23T18:50:05.703 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.703 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.703 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:50:05.703 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.499257+0000", 2023-09-23T18:50:05.703 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.0996e-05 2023-09-23T18:50:05.703 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.703 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.703 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:50:05.703 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.499278+0000", 2023-09-23T18:50:05.703 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.3819999999999993e-06 2023-09-23T18:50:05.703 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.704 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.704 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:50:05.704 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.499287+0000", 2023-09-23T18:50:05.704 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.4744000000000001e-05 2023-09-23T18:50:05.704 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.704 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.704 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.704 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.499362+0000", 2023-09-23T18:50:05.704 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.0909999999999999e-06 2023-09-23T18:50:05.704 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.704 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.704 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:50:05.704 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.499371+0000", 2023-09-23T18:50:05.705 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.56803980700000001 2023-09-23T18:50:05.705 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.705 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.705 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.705 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:17.067411+0000", 2023-09-23T18:50:05.705 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.0390000000000005e-06 2023-09-23T18:50:05.705 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.705 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.705 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:50:05.705 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:17.067419+0000", 2023-09-23T18:50:05.705 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.56824535200000004 2023-09-23T18:50:05.705 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.706 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:50:05.706 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.706 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.706 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.706 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45188.0:88995 7.12 7.f9c73072 (undecoded) ondisk+write+known_if_redirected e203)", 2023-09-23T18:50:05.706 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:44:16.590471+0000", 2023-09-23T18:50:05.706 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 349.03758868300002, 2023-09-23T18:50:05.706 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.47700915799999999, 2023-09-23T18:50:05.706 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:50:05.706 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T18:50:05.706 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:50:05.706 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45188", 2023-09-23T18:50:05.706 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:50:05.707 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 88995 2023-09-23T18:50:05.707 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.707 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:50:05.707 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.707 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:50:05.707 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.590471+0000", 2023-09-23T18:50:05.707 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:50:05.707 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.707 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.707 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:50:05.707 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.590471+0000", 2023-09-23T18:50:05.707 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:50:05.707 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.708 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.708 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:50:05.708 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.590469+0000", 2023-09-23T18:50:05.708 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.2138000000000005e-05 2023-09-23T18:50:05.708 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.708 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.708 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:50:05.708 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.590551+0000", 2023-09-23T18:50:05.708 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4532000000000001e-05 2023-09-23T18:50:05.708 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.708 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.708 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:50:05.709 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.590566+0000", 2023-09-23T18:50:05.709 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.1219999999999997e-06 2023-09-23T18:50:05.709 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.709 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.709 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:50:05.709 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.590573+0000", 2023-09-23T18:50:05.709 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.3973000000000005e-05 2023-09-23T18:50:05.709 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.709 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.709 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.709 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.590647+0000", 2023-09-23T18:50:05.709 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2034e-05 2023-09-23T18:50:05.709 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.710 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.710 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:50:05.710 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.590659+0000", 2023-09-23T18:50:05.710 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.476810285 2023-09-23T18:50:05.710 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.710 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.710 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.710 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:17.067469+0000", 2023-09-23T18:50:05.710 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0835999999999999e-05 2023-09-23T18:50:05.710 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.710 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.710 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:50:05.710 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:17.067480+0000", 2023-09-23T18:50:05.711 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.47700915799999999 2023-09-23T18:50:05.711 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.711 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:50:05.711 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.711 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.711 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.711 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45188.0:88996 7.12 7.89b19a32 (undecoded) ondisk+write+known_if_redirected e203)", 2023-09-23T18:50:05.711 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:44:16.591695+0000", 2023-09-23T18:50:05.711 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 349.03636416099999, 2023-09-23T18:50:05.711 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.47582676200000001, 2023-09-23T18:50:05.711 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:50:05.711 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T18:50:05.712 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:50:05.712 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45188", 2023-09-23T18:50:05.712 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:50:05.712 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 88996 2023-09-23T18:50:05.712 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.712 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:50:05.712 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.712 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:50:05.712 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.591695+0000", 2023-09-23T18:50:05.712 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:50:05.712 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.712 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.712 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:50:05.713 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.591695+0000", 2023-09-23T18:50:05.713 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:50:05.713 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.713 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.713 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:50:05.713 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.591694+0000", 2023-09-23T18:50:05.713 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.2924999999999996e-05 2023-09-23T18:50:05.713 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.713 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.713 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:50:05.713 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.591777+0000", 2023-09-23T18:50:05.713 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.6044e-05 2023-09-23T18:50:05.714 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.714 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.714 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:50:05.714 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.591803+0000", 2023-09-23T18:50:05.714 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.6599999999999995e-06 2023-09-23T18:50:05.714 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.714 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.714 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:50:05.714 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.591811+0000", 2023-09-23T18:50:05.714 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.4690000000000006e-05 2023-09-23T18:50:05.714 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.714 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.715 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.715 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.591876+0000", 2023-09-23T18:50:05.715 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.9629999999999998e-06 2023-09-23T18:50:05.715 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.715 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.715 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:50:05.715 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.591884+0000", 2023-09-23T18:50:05.715 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.475628575 2023-09-23T18:50:05.715 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.715 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.715 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.715 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:17.067512+0000", 2023-09-23T18:50:05.715 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0147e-05 2023-09-23T18:50:05.716 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.716 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.716 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:50:05.716 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:17.067522+0000", 2023-09-23T18:50:05.716 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.47582676200000001 2023-09-23T18:50:05.716 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.716 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:50:05.716 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.716 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.716 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.716 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45188.0:89042 7.12 7.7dbe50b2 (undecoded) ondisk+write+known_if_redirected e203)", 2023-09-23T18:50:05.716 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:44:16.640405+0000", 2023-09-23T18:50:05.717 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 348.98765446099998, 2023-09-23T18:50:05.717 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.427154215, 2023-09-23T18:50:05.717 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:50:05.717 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T18:50:05.717 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:50:05.717 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45188", 2023-09-23T18:50:05.717 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:50:05.717 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 89042 2023-09-23T18:50:05.717 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.717 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:50:05.717 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.717 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:50:05.718 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.640405+0000", 2023-09-23T18:50:05.718 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:50:05.718 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.718 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.718 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:50:05.718 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.640405+0000", 2023-09-23T18:50:05.718 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:50:05.718 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.718 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.718 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:50:05.718 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.640403+0000", 2023-09-23T18:50:05.718 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.3573999999999999e-05 2023-09-23T18:50:05.718 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.719 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.719 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:50:05.719 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.640457+0000", 2023-09-23T18:50:05.719 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6603999999999999e-05 2023-09-23T18:50:05.719 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.719 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.719 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:50:05.719 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.640473+0000", 2023-09-23T18:50:05.719 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.7919999999999993e-06 2023-09-23T18:50:05.719 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.719 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.719 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:50:05.720 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.640481+0000", 2023-09-23T18:50:05.720 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00094353000000000004 2023-09-23T18:50:05.720 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.720 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.720 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.720 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.641425+0000", 2023-09-23T18:50:05.720 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.6120999999999999e-05 2023-09-23T18:50:05.720 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.720 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.720 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:50:05.720 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.641451+0000", 2023-09-23T18:50:05.720 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.42610115199999998 2023-09-23T18:50:05.720 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.721 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.721 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.721 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:17.067552+0000", 2023-09-23T18:50:05.721 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.4329999999999998e-06 2023-09-23T18:50:05.721 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.721 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.721 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:50:05.721 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:17.067559+0000", 2023-09-23T18:50:05.721 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.427154215 2023-09-23T18:50:05.721 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.721 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:50:05.721 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.721 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.722 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.722 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45188.0:89048 7.12 7.dd891a52 (undecoded) ondisk+write+known_if_redirected e203)", 2023-09-23T18:50:05.722 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:44:16.646895+0000", 2023-09-23T18:50:05.722 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 348.98116508499999, 2023-09-23T18:50:05.722 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.420697302, 2023-09-23T18:50:05.722 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:50:05.722 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T18:50:05.722 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:50:05.722 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45188", 2023-09-23T18:50:05.722 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:50:05.722 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 89048 2023-09-23T18:50:05.722 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.723 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:50:05.723 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.723 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:50:05.723 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.646895+0000", 2023-09-23T18:50:05.723 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:50:05.723 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.723 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.723 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:50:05.723 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.646895+0000", 2023-09-23T18:50:05.723 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:50:05.723 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.723 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.724 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:50:05.724 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.646893+0000", 2023-09-23T18:50:05.724 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.7313999999999999e-05 2023-09-23T18:50:05.724 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.724 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.724 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:50:05.724 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.646970+0000", 2023-09-23T18:50:05.724 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2981e-05 2023-09-23T18:50:05.724 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.724 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.724 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:50:05.724 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.646983+0000", 2023-09-23T18:50:05.724 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.3999999999999997e-06 2023-09-23T18:50:05.725 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.725 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.725 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:50:05.725 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.646990+0000", 2023-09-23T18:50:05.725 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.3283999999999999e-05 2023-09-23T18:50:05.725 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.725 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.725 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.725 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.647053+0000", 2023-09-23T18:50:05.725 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0881e-05 2023-09-23T18:50:05.725 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.725 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.726 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:50:05.726 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.647064+0000", 2023-09-23T18:50:05.726 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.42052035500000001 2023-09-23T18:50:05.726 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.726 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.726 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.726 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:17.067584+0000", 2023-09-23T18:50:05.726 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.7570000000000004e-06 2023-09-23T18:50:05.726 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.726 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.726 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:50:05.726 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:17.067592+0000", 2023-09-23T18:50:05.727 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.420697302 2023-09-23T18:50:05.727 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.727 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:50:05.727 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.727 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.727 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.727 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45188.0:89063 7.12 7.f3c2aef2 (undecoded) ondisk+write+known_if_redirected e203)", 2023-09-23T18:50:05.727 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:44:16.667818+0000", 2023-09-23T18:50:05.727 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 348.96024167899998, 2023-09-23T18:50:05.727 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.39981200099999997, 2023-09-23T18:50:05.727 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:50:05.727 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T18:50:05.727 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:50:05.728 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45188", 2023-09-23T18:50:05.728 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:50:05.728 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 89063 2023-09-23T18:50:05.728 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.728 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:50:05.728 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.728 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:50:05.728 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.667818+0000", 2023-09-23T18:50:05.728 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:50:05.728 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.728 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.728 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:50:05.728 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.667818+0000", 2023-09-23T18:50:05.729 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:50:05.729 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.729 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.729 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:50:05.729 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.667816+0000", 2023-09-23T18:50:05.729 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.6995999999999993e-05 2023-09-23T18:50:05.729 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.729 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.729 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:50:05.729 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.667883+0000", 2023-09-23T18:50:05.729 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4148000000000001e-05 2023-09-23T18:50:05.729 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.729 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.730 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:50:05.730 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.667897+0000", 2023-09-23T18:50:05.730 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.1840000000000005e-06 2023-09-23T18:50:05.730 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.730 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.730 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:50:05.730 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.667906+0000", 2023-09-23T18:50:05.730 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5495000000000003e-05 2023-09-23T18:50:05.730 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.730 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.730 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.730 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.667962+0000", 2023-09-23T18:50:05.731 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.5490000000000003e-06 2023-09-23T18:50:05.731 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.731 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.731 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:50:05.731 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.667968+0000", 2023-09-23T18:50:05.731 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.39965336499999998 2023-09-23T18:50:05.731 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.731 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.731 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.731 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:17.067622+0000", 2023-09-23T18:50:05.731 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.1040000000000007e-06 2023-09-23T18:50:05.731 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.731 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.732 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:50:05.732 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:17.067630+0000", 2023-09-23T18:50:05.732 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.39981200099999997 2023-09-23T18:50:05.732 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.732 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:50:05.732 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.732 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.732 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.732 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45188.0:89096 7.12 7.24767ed2 (undecoded) ondisk+write+known_if_redirected e203)", 2023-09-23T18:50:05.732 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:44:16.706269+0000", 2023-09-23T18:50:05.732 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 348.92179062299999, 2023-09-23T18:50:05.732 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.36139157700000002, 2023-09-23T18:50:05.732 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:50:05.733 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T18:50:05.733 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:50:05.733 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45188", 2023-09-23T18:50:05.733 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:50:05.733 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 89096 2023-09-23T18:50:05.733 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.733 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:50:05.733 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.733 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:50:05.733 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.706269+0000", 2023-09-23T18:50:05.733 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:50:05.733 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.733 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.734 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:50:05.734 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.706269+0000", 2023-09-23T18:50:05.734 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:50:05.734 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.734 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.734 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:50:05.734 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.706267+0000", 2023-09-23T18:50:05.734 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.0422999999999998e-05 2023-09-23T18:50:05.734 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.734 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.734 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:50:05.734 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.706358+0000", 2023-09-23T18:50:05.734 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4126e-05 2023-09-23T18:50:05.735 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.735 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.735 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:50:05.735 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.706372+0000", 2023-09-23T18:50:05.735 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.4500000000000001e-06 2023-09-23T18:50:05.735 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.735 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.735 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:50:05.735 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.706378+0000", 2023-09-23T18:50:05.735 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7958000000000003e-05 2023-09-23T18:50:05.735 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.735 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.736 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.736 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.706436+0000", 2023-09-23T18:50:05.736 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.6450000000000005e-06 2023-09-23T18:50:05.736 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.736 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.736 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:50:05.736 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.706444+0000", 2023-09-23T18:50:05.736 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.36120601800000002 2023-09-23T18:50:05.736 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.736 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.736 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.736 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:17.067650+0000", 2023-09-23T18:50:05.736 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0573e-05 2023-09-23T18:50:05.737 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.737 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.737 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:50:05.737 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:17.067661+0000", 2023-09-23T18:50:05.737 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.36139157700000002 2023-09-23T18:50:05.737 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.737 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:50:05.737 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.737 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.737 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.737 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45188.0:89108 7.12 7.b55d9032 (undecoded) ondisk+write+known_if_redirected e203)", 2023-09-23T18:50:05.737 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:44:16.723016+0000", 2023-09-23T18:50:05.737 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 348.90504320000002, 2023-09-23T18:50:05.738 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.34467711600000001, 2023-09-23T18:50:05.738 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:50:05.738 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T18:50:05.738 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:50:05.738 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45188", 2023-09-23T18:50:05.738 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:50:05.738 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 89108 2023-09-23T18:50:05.738 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.738 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:50:05.738 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.738 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:50:05.738 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.723016+0000", 2023-09-23T18:50:05.739 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:50:05.739 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.739 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.739 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:50:05.739 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.723016+0000", 2023-09-23T18:50:05.739 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999857 2023-09-23T18:50:05.739 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.739 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.739 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:50:05.739 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.723002+0000", 2023-09-23T18:50:05.739 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.185e-05 2023-09-23T18:50:05.739 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.739 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.740 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:50:05.740 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.723054+0000", 2023-09-23T18:50:05.740 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6558999999999998e-05 2023-09-23T18:50:05.740 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.740 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.740 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:50:05.740 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.723071+0000", 2023-09-23T18:50:05.740 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0319999999999999e-05 2023-09-23T18:50:05.740 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.740 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.740 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:50:05.740 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.723081+0000", 2023-09-23T18:50:05.740 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.6970000000000004e-05 2023-09-23T18:50:05.741 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.741 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.741 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.741 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.723148+0000", 2023-09-23T18:50:05.741 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2081e-05 2023-09-23T18:50:05.741 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.741 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.741 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:50:05.741 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.723160+0000", 2023-09-23T18:50:05.741 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.34452258099999999 2023-09-23T18:50:05.741 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.741 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.742 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.742 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:17.067682+0000", 2023-09-23T18:50:05.742 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1069e-05 2023-09-23T18:50:05.742 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.742 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.742 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:50:05.742 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:17.067694+0000", 2023-09-23T18:50:05.742 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.34467711600000001 2023-09-23T18:50:05.742 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.742 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:50:05.742 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.742 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.742 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.743 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45188.0:89151 7.12 7.8aff9612 (undecoded) ondisk+write+known_if_redirected e203)", 2023-09-23T18:50:05.743 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:44:16.786263+0000", 2023-09-23T18:50:05.743 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 348.84179621200002, 2023-09-23T18:50:05.743 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.28146290299999999, 2023-09-23T18:50:05.743 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:50:05.743 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T18:50:05.743 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:50:05.743 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45188", 2023-09-23T18:50:05.743 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:50:05.743 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 89151 2023-09-23T18:50:05.743 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.743 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:50:05.744 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.744 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:50:05.744 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.786263+0000", 2023-09-23T18:50:05.744 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:50:05.744 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.744 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.744 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:50:05.744 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.786263+0000", 2023-09-23T18:50:05.744 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-09-23T18:50:05.744 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.744 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.744 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:50:05.744 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.786261+0000", 2023-09-23T18:50:05.745 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.5999000000000003e-05 2023-09-23T18:50:05.745 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.745 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.745 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:50:05.745 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.786327+0000", 2023-09-23T18:50:05.745 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.0140999999999999e-05 2023-09-23T18:50:05.745 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.745 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.745 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:50:05.745 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.786347+0000", 2023-09-23T18:50:05.745 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.2259999999999994e-06 2023-09-23T18:50:05.745 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.745 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.746 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:50:05.746 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.786356+0000", 2023-09-23T18:50:05.746 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7182999999999998e-05 2023-09-23T18:50:05.746 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.746 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.746 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.746 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.786414+0000", 2023-09-23T18:50:05.746 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.2259999999999999e-06 2023-09-23T18:50:05.746 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.746 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.746 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:50:05.746 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:16.786421+0000", 2023-09-23T18:50:05.747 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.28129405800000001 2023-09-23T18:50:05.747 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.747 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.747 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.747 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:17.067715+0000", 2023-09-23T18:50:05.747 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1378e-05 2023-09-23T18:50:05.747 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.747 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.747 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:50:05.747 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:44:17.067726+0000", 2023-09-23T18:50:05.747 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.28146290299999999 2023-09-23T18:50:05.747 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.747 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:50:05.748 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.748 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.748 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.748 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45188.0:159503 7.d 7:bdd0cf76:::benchmark_data_smithi098_155798_object58558:head [delete] snapc 0=[] ondisk+write+known_if_redirected e208)", 2023-09-23T18:50:05.748 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:45:45.458514+0000", 2023-09-23T18:50:05.748 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 260.169545153, 2023-09-23T18:50:05.748 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.25318658900000002, 2023-09-23T18:50:05.748 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:50:05.748 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:50:05.748 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:50:05.748 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45188", 2023-09-23T18:50:05.748 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:50:05.748 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 159503 2023-09-23T18:50:05.749 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.749 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:50:05.749 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.749 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:50:05.749 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:45.458514+0000", 2023-09-23T18:50:05.749 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:50:05.749 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.749 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.749 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:50:05.749 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:45.458514+0000", 2023-09-23T18:50:05.749 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:50:05.749 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.749 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.750 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:50:05.750 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:45.458513+0000", 2023-09-23T18:50:05.750 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.2320000000000004e-06 2023-09-23T18:50:05.750 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.750 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.750 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:50:05.750 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:45.458520+0000", 2023-09-23T18:50:05.750 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1788e-05 2023-09-23T18:50:05.750 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.750 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.750 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:50:05.750 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:45.458532+0000", 2023-09-23T18:50:05.751 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.2529999999999998e-06 2023-09-23T18:50:05.751 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.751 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.751 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:50:05.751 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:45.458539+0000", 2023-09-23T18:50:05.751 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6616999999999997e-05 2023-09-23T18:50:05.751 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.751 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.751 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.751 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:45.458596+0000", 2023-09-23T18:50:05.751 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000103646 2023-09-23T18:50:05.751 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.751 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.752 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:50:05.752 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:45.458700+0000", 2023-09-23T18:50:05.752 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000155625 2023-09-23T18:50:05.752 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.752 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.752 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,1", 2023-09-23T18:50:05.752 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:45.458855+0000", 2023-09-23T18:50:05.752 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.003528167 2023-09-23T18:50:05.752 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.752 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.752 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:50:05.752 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:45.462384+0000", 2023-09-23T18:50:05.752 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00082096800000000004 2023-09-23T18:50:05.753 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.753 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.753 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:50:05.753 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:45.463205+0000", 2023-09-23T18:50:05.753 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.24837115900000001 2023-09-23T18:50:05.753 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.753 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.753 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:50:05.753 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:45.711576+0000", 2023-09-23T18:50:05.753 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.7455999999999999e-05 2023-09-23T18:50:05.753 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.753 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.753 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:50:05.754 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:45.711643+0000", 2023-09-23T18:50:05.754 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7938e-05 2023-09-23T18:50:05.754 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.754 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.754 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:50:05.754 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:45.711701+0000", 2023-09-23T18:50:05.754 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.25318658900000002 2023-09-23T18:50:05.754 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.754 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:50:05.754 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.754 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.754 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.755 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45188.0:161083 7.17 7:e8d86609:::benchmark_data_smithi098_155798_object60138:head [delete] snapc 0=[] ondisk+write+known_if_redirected e208)", 2023-09-23T18:50:05.755 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:45:47.780549+0000", 2023-09-23T18:50:05.755 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 257.84751109699999, 2023-09-23T18:50:05.755 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.270521925, 2023-09-23T18:50:05.755 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:50:05.755 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:50:05.755 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:50:05.755 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45188", 2023-09-23T18:50:05.755 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:50:05.755 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 161083 2023-09-23T18:50:05.755 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.755 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:50:05.755 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.756 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:50:05.756 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:47.780549+0000", 2023-09-23T18:50:05.756 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:50:05.756 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.756 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.756 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:50:05.756 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:47.780549+0000", 2023-09-23T18:50:05.756 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:50:05.756 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.756 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.756 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:50:05.756 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:47.780547+0000", 2023-09-23T18:50:05.757 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.3009999999999997e-06 2023-09-23T18:50:05.757 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.757 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.757 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:50:05.757 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:47.780553+0000", 2023-09-23T18:50:05.757 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0957e-05 2023-09-23T18:50:05.757 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.757 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.757 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:50:05.757 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:47.780564+0000", 2023-09-23T18:50:05.757 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.3680000000000001e-06 2023-09-23T18:50:05.757 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.757 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.758 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:50:05.758 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:47.780569+0000", 2023-09-23T18:50:05.758 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.8368999999999998e-05 2023-09-23T18:50:05.758 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.758 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.758 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.758 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:47.780607+0000", 2023-09-23T18:50:05.758 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.6866999999999993e-05 2023-09-23T18:50:05.758 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.758 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.758 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:50:05.758 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:47.780694+0000", 2023-09-23T18:50:05.759 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00040783900000000001 2023-09-23T18:50:05.759 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.759 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.759 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,1", 2023-09-23T18:50:05.759 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:47.781102+0000", 2023-09-23T18:50:05.759 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0019621640000000002 2023-09-23T18:50:05.759 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.759 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.759 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:50:05.759 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:47.783064+0000", 2023-09-23T18:50:05.759 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0037017869999999998 2023-09-23T18:50:05.761 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.761 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.761 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:50:05.761 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:47.786766+0000", 2023-09-23T18:50:05.761 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.264092942 2023-09-23T18:50:05.761 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.761 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.761 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:50:05.761 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:48.050859+0000", 2023-09-23T18:50:05.762 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000113201 2023-09-23T18:50:05.762 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.762 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.762 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:50:05.762 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:48.050972+0000", 2023-09-23T18:50:05.762 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.8231000000000002e-05 2023-09-23T18:50:05.762 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.762 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.762 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:50:05.762 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:48.051070+0000", 2023-09-23T18:50:05.762 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.270521925 2023-09-23T18:50:05.762 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.763 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:50:05.763 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.763 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.763 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.763 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45188.0:161088 7.d 7:bcb6ea84:::benchmark_data_smithi098_155798_object60143:head [delete] snapc 0=[] ondisk+write+known_if_redirected e208)", 2023-09-23T18:50:05.763 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:45:47.783161+0000", 2023-09-23T18:50:05.763 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 257.84489902799999, 2023-09-23T18:50:05.763 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.27999077, 2023-09-23T18:50:05.763 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:50:05.763 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:50:05.763 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:50:05.763 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45188", 2023-09-23T18:50:05.763 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:50:05.764 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 161088 2023-09-23T18:50:05.764 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.764 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:50:05.764 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.764 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:50:05.764 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:47.783161+0000", 2023-09-23T18:50:05.764 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:50:05.764 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.764 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.764 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:50:05.764 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:47.783161+0000", 2023-09-23T18:50:05.764 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:50:05.765 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.765 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.765 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:50:05.765 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:47.783160+0000", 2023-09-23T18:50:05.765 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6300000000000003e-06 2023-09-23T18:50:05.765 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.765 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.765 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:50:05.765 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:47.783165+0000", 2023-09-23T18:50:05.765 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0495e-05 2023-09-23T18:50:05.765 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.765 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.765 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:50:05.766 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:47.783176+0000", 2023-09-23T18:50:05.766 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.2229999999999998e-06 2023-09-23T18:50:05.766 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.766 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.766 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:50:05.766 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:47.783181+0000", 2023-09-23T18:50:05.766 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.9904000000000001e-05 2023-09-23T18:50:05.766 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.766 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.766 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:50:05.766 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:47.783241+0000", 2023-09-23T18:50:05.766 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000168021 2023-09-23T18:50:05.767 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.767 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.767 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:50:05.767 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:47.783409+0000", 2023-09-23T18:50:05.767 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0001897 2023-09-23T18:50:05.767 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.767 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.767 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,1", 2023-09-23T18:50:05.767 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:47.783599+0000", 2023-09-23T18:50:05.767 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0034202490000000002 2023-09-23T18:50:05.767 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.767 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.767 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:50:05.768 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:47.787019+0000", 2023-09-23T18:50:05.768 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.26432649499999999 2023-09-23T18:50:05.768 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.768 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.768 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:50:05.768 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:48.051345+0000", 2023-09-23T18:50:05.768 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.011660752999999999 2023-09-23T18:50:05.768 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.768 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.768 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:50:05.768 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:48.063006+0000", 2023-09-23T18:50:05.768 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.3916000000000001e-05 2023-09-23T18:50:05.769 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.769 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.769 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:50:05.769 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:48.063090+0000", 2023-09-23T18:50:05.769 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1334999999999995e-05 2023-09-23T18:50:05.769 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:50:05.769 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:50:05.769 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:50:05.769 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:45:48.063151+0000", 2023-09-23T18:50:05.769 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.27999077 2023-09-23T18:50:05.769 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.769 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:50:05.769 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.770 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:50:05.770 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:50:05.770 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:50:05.770 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:05.788 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:05.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:05.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:05.820 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:05 smithi098 bash[79516]: cluster 2023-09-23T18:50:05.151587+0000 mon.a (mon.0) 1314 : cluster 3 Health check update: Degraded data redundancy: 49388/283236 objects degraded (17.437%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:50:05.820 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:05 smithi098 bash[76817]: cluster 2023-09-23T18:50:05.151587+0000 mon.a (mon.0) 1314 : cluster 3 Health check update: Degraded data redundancy: 49388/283236 objects degraded (17.437%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:50:05.891 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:05.922 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:05.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:05.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:06.028 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:06.050 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:06.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:06.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:06.153 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:06.189 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:06.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:06.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:06.296 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:06.324 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:06.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:06.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:06.430 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:06.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:06 smithi098 bash[76817]: cluster 2023-09-23T18:50:06.110181+0000 mgr.y (mgr.26054) 960 : cluster 0 pgmap v1022: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 49388/281652 objects degraded (17.535%); 5597/281652 objects misplaced (1.987%); 922 KiB/s, 14 objects/s recovering 2023-09-23T18:50:06.440 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:50:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:50:06] "GET /metrics HTTP/1.1" 200 36619 "" "Prometheus/2.33.4" 2023-09-23T18:50:06.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:06 smithi098 bash[79516]: cluster 2023-09-23T18:50:06.110181+0000 mgr.y (mgr.26054) 960 : cluster 0 pgmap v1022: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 49388/281652 objects degraded (17.535%); 5597/281652 objects misplaced (1.987%); 922 KiB/s, 14 objects/s recovering 2023-09-23T18:50:06.486 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:06.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:06.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:06.590 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:06.597 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:06 smithi173 bash[59422]: cluster 2023-09-23T18:50:06.110181+0000 mgr.y (mgr.26054) 960 : cluster 0 pgmap v1022: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 49388/281652 objects degraded (17.535%); 5597/281652 objects misplaced (1.987%); 922 KiB/s, 14 objects/s recovering 2023-09-23T18:50:06.615 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:06.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:06.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:06.725 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:06.760 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T18:50:06.761 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:06.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:06.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:06.865 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:06.894 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:06.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:06.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:07.007 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:07.037 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:07.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:07.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:07.141 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:07.172 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:07.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:07.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:07.275 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:07.315 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:07.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:07.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:07.422 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:07.445 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:07.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:07.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:07.551 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:07.576 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:07.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:07.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:07.681 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:07.713 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:07.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:07.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:07.821 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:07.847 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:07.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:07.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:07.950 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:07.974 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:07.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:07.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:08.081 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:08.112 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:08.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:08.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:08.217 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:08.229 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:08 smithi098 bash[79516]: cluster 2023-09-23T18:50:08.110942+0000 mgr.y (mgr.26054) 961 : cluster 0 pgmap v1023: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 49466/277446 objects degraded (17.829%); 5439/277446 objects misplaced (1.960%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T18:50:08.247 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:08.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:08.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:08.350 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:08.374 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:08.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:08.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:08.477 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:08.487 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:08 smithi173 bash[59422]: cluster 2023-09-23T18:50:08.110942+0000 mgr.y (mgr.26054) 961 : cluster 0 pgmap v1023: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 49466/277446 objects degraded (17.829%); 5439/277446 objects misplaced (1.960%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T18:50:08.500 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:08.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:08.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:08.603 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:08.613 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:08 smithi098 bash[76817]: cluster 2023-09-23T18:50:08.110942+0000 mgr.y (mgr.26054) 961 : cluster 0 pgmap v1023: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 49466/277446 objects degraded (17.829%); 5439/277446 objects misplaced (1.960%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T18:50:08.653 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:08.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:08.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:08.760 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:08.790 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:08.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:08.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:08.895 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:08.925 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:08.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:08.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:09.030 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:09.059 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:09.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:09.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:09.166 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:09.195 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:09.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:09.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:09.298 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:09.325 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:09.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:09.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:09.428 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:09.469 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:09.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:09.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:09.572 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:09.607 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:09.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:09.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:09.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:09 smithi098 bash[79516]: audit 2023-09-23T18:50:09.622140+0000 mon.a (mon.0) 1315 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:50:09.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:09 smithi098 bash[76817]: audit 2023-09-23T18:50:09.622140+0000 mon.a (mon.0) 1315 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:50:09.710 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:09.741 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:09.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:09.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:09.845 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:09.886 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:09.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:09.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:09.990 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:10.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:09 smithi173 bash[59422]: audit 2023-09-23T18:50:09.622140+0000 mon.a (mon.0) 1315 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:50:10.028 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:10.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:10.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:10.138 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:10.156 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:10.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:10.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:10.259 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:10.281 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:10.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:10.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:10.384 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:10.416 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:10.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:10.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:10.521 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:10.558 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:10.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:10.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:10.663 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:10.705 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:10.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:10.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:10.813 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:10.821 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:10 smithi173 bash[59422]: cluster 2023-09-23T18:50:10.111795+0000 mgr.y (mgr.26054) 962 : cluster 0 pgmap v1024: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 49466/275562 objects degraded (17.951%); 5439/275562 objects misplaced (1.974%); 906 KiB/s, 14 objects/s recovering 2023-09-23T18:50:10.821 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:10 smithi173 bash[59422]: cluster 2023-09-23T18:50:10.152601+0000 mon.a (mon.0) 1316 : cluster 3 Health check update: Degraded data redundancy: 49466/277446 objects degraded (17.829%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:50:10.837 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:10.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:10.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:10.941 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:10.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:10 smithi098 bash[76817]: cluster 2023-09-23T18:50:10.111795+0000 mgr.y (mgr.26054) 962 : cluster 0 pgmap v1024: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 49466/275562 objects degraded (17.951%); 5439/275562 objects misplaced (1.974%); 906 KiB/s, 14 objects/s recovering 2023-09-23T18:50:10.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:10 smithi098 bash[76817]: cluster 2023-09-23T18:50:10.152601+0000 mon.a (mon.0) 1316 : cluster 3 Health check update: Degraded data redundancy: 49466/277446 objects degraded (17.829%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:50:10.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:10 smithi098 bash[79516]: cluster 2023-09-23T18:50:10.111795+0000 mgr.y (mgr.26054) 962 : cluster 0 pgmap v1024: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 49466/275562 objects degraded (17.951%); 5439/275562 objects misplaced (1.974%); 906 KiB/s, 14 objects/s recovering 2023-09-23T18:50:10.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:10 smithi098 bash[79516]: cluster 2023-09-23T18:50:10.152601+0000 mon.a (mon.0) 1316 : cluster 3 Health check update: Degraded data redundancy: 49466/277446 objects degraded (17.829%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:50:10.970 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:10.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:10.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:11.076 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:11.103 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:11.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:11.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:11.208 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:11.246 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:11.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:11.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:11.351 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:11.376 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:11.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:11.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:11.484 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:11.513 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:11.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:11.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:11.600 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:50:11.616 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:11.666 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:11.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:11.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:11.770 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:11.792 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:11.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:11.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:11.898 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:11.931 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:11.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:11.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:12.038 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:12.072 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:12.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:12.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:12.179 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:12.206 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:12.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:12.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:12.315 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:12.323 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:12 smithi098 bash[76817]: cluster 2023-09-23T18:50:12.112637+0000 mgr.y (mgr.26054) 963 : cluster 0 pgmap v1025: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 49529/270084 objects degraded (18.338%); 5272/270084 objects misplaced (1.952%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T18:50:12.323 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:12 smithi098 bash[79516]: cluster 2023-09-23T18:50:12.112637+0000 mgr.y (mgr.26054) 963 : cluster 0 pgmap v1025: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 49529/270084 objects degraded (18.338%); 5272/270084 objects misplaced (1.952%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T18:50:12.341 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:12.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:12.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:12.446 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:12.468 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:12.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:12.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:12.530 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:12 smithi173 bash[59422]: cluster 2023-09-23T18:50:12.112637+0000 mgr.y (mgr.26054) 963 : cluster 0 pgmap v1025: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 49529/270084 objects degraded (18.338%); 5272/270084 objects misplaced (1.952%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T18:50:12.573 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:12.600 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:12.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:12.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:12.705 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:12.749 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:12.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:12.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:12.860 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:12.880 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:12.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:12.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:12.985 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:13.019 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:13.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:13.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:13.125 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:13.155 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:13.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:13.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:13.265 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:13.292 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:13.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:13.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:13.402 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:13.426 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:13.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:13.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:13.529 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:13.562 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:13.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:13.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:13.666 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:13.704 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:13.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:13.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:13.809 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:13.841 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:13.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:13.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:13.945 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:13.971 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:13.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:13.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:14.078 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:14.147 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:14.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:14.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:14.253 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:14.283 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:14.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:14.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:14.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:14 smithi173 bash[59422]: cluster 2023-09-23T18:50:14.113276+0000 mgr.y (mgr.26054) 964 : cluster 0 pgmap v1026: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 49529/268290 objects degraded (18.461%); 5272/268290 objects misplaced (1.965%); 981 KiB/s, 15 objects/s recovering 2023-09-23T18:50:14.387 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:14.414 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:14.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:14.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:14.518 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:14.526 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:14 smithi098 bash[76817]: cluster 2023-09-23T18:50:14.113276+0000 mgr.y (mgr.26054) 964 : cluster 0 pgmap v1026: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 49529/268290 objects degraded (18.461%); 5272/268290 objects misplaced (1.965%); 981 KiB/s, 15 objects/s recovering 2023-09-23T18:50:14.527 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:14 smithi098 bash[79516]: cluster 2023-09-23T18:50:14.113276+0000 mgr.y (mgr.26054) 964 : cluster 0 pgmap v1026: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 49529/268290 objects degraded (18.461%); 5272/268290 objects misplaced (1.965%); 981 KiB/s, 15 objects/s recovering 2023-09-23T18:50:14.555 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:14.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:14.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:14.659 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:14.695 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:14.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:14.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:14.803 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:14.840 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:14.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:14.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:14.947 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:14.971 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:14.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:14.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:15.076 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:15.099 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:15.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:15.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:15.202 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:15.244 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:15.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:15.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:15.350 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:15.372 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:15.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:15.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:15.478 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:15.499 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:15.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:15.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:15.603 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:15.630 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:15.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:15.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:15.734 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:15.775 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:15.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:15.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:15.881 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:15.902 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:15.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:15.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:16.006 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:16.026 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:16.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:16.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:16.129 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:16.154 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:16.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:16.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:16.257 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:16.266 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:16 smithi098 bash[79516]: cluster 2023-09-23T18:50:15.153925+0000 mon.a (mon.0) 1317 : cluster 3 Health check update: Degraded data redundancy: 49529/268290 objects degraded (18.461%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:50:16.266 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:50:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:50:16] "GET /metrics HTTP/1.1" 200 36619 "" "Prometheus/2.33.4" 2023-09-23T18:50:16.266 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:16 smithi098 bash[76817]: cluster 2023-09-23T18:50:15.153925+0000 mon.a (mon.0) 1317 : cluster 3 Health check update: Degraded data redundancy: 49529/268290 objects degraded (18.461%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:50:16.307 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:16.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:16.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:16.409 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:16.417 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:16 smithi173 bash[59422]: cluster 2023-09-23T18:50:15.153925+0000 mon.a (mon.0) 1317 : cluster 3 Health check update: Degraded data redundancy: 49529/268290 objects degraded (18.461%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:50:16.431 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:16.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:16.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:16.534 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:16.565 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:16.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:16.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:16.675 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:16.711 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:16.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:16.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:16.814 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:16.848 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:16.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:16.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:16.956 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:16.994 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:16.996 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:50:16.996 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-09-23T18:50:16.996 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 0 2023-09-23T18:50:16.996 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:50:16.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:16.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:17.099 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:17.129 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:17.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:17.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:17.232 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:17.256 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:17.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:17.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:17.359 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:17.368 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:17 smithi098 bash[76817]: cluster 2023-09-23T18:50:16.113919+0000 mgr.y (mgr.26054) 965 : cluster 0 pgmap v1027: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 456 op/s; 49529/266793 objects degraded (18.565%); 5272/266793 objects misplaced (1.976%); 981 KiB/s, 15 objects/s recovering 2023-09-23T18:50:17.369 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:17 smithi098 bash[79516]: cluster 2023-09-23T18:50:16.113919+0000 mgr.y (mgr.26054) 965 : cluster 0 pgmap v1027: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 456 op/s; 49529/266793 objects degraded (18.565%); 5272/266793 objects misplaced (1.976%); 981 KiB/s, 15 objects/s recovering 2023-09-23T18:50:17.396 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:17.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:17.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:17.499 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:17.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:17 smithi173 bash[59422]: cluster 2023-09-23T18:50:16.113919+0000 mgr.y (mgr.26054) 965 : cluster 0 pgmap v1027: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 456 op/s; 49529/266793 objects degraded (18.565%); 5272/266793 objects misplaced (1.976%); 981 KiB/s, 15 objects/s recovering 2023-09-23T18:50:17.536 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:17.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:17.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:17.647 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:17.677 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:17.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:17.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:17.783 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:17.812 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:17.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:17.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:17.917 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:17.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:17.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:17.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:17.984 INFO:teuthology.orchestra.run.smithi098.stdout:moved 11 / 267 (4.11985%) 2023-09-23T18:50:17.984 INFO:teuthology.orchestra.run.smithi098.stdout:avg 33.375 2023-09-23T18:50:17.985 INFO:teuthology.orchestra.run.smithi098.stdout:stddev 6.20358 -> 7.38135 (expected baseline 5.40399) 2023-09-23T18:50:17.985 INFO:teuthology.orchestra.run.smithi098.stdout:min osd.5 with 21 -> 16 pgs (0.629214 -> 0.479401 * mean) 2023-09-23T18:50:17.985 INFO:teuthology.orchestra.run.smithi098.stdout:max osd.1 with 42 -> 42 pgs (1.25843 -> 1.25843 * mean) 2023-09-23T18:50:17.985 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:50:17.985 INFO:teuthology.orchestra.run.smithi098.stdout:oload 110 2023-09-23T18:50:17.985 INFO:teuthology.orchestra.run.smithi098.stdout:max_change 1 2023-09-23T18:50:17.985 INFO:teuthology.orchestra.run.smithi098.stdout:max_change_osds 4 2023-09-23T18:50:17.985 INFO:teuthology.orchestra.run.smithi098.stdout:average_utilization 0.0298 2023-09-23T18:50:17.985 INFO:teuthology.orchestra.run.smithi098.stdout:overload_utilization 0.0328 2023-09-23T18:50:17.985 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2 weight 0.9987 -> 1.0000 2023-09-23T18:50:17.986 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5 weight 0.6936 -> 0.4803 2023-09-23T18:50:17.986 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3 weight 0.6315 -> 0.7531 2023-09-23T18:50:17.986 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7 weight 0.8539 -> 0.9936 2023-09-23T18:50:17.993 INFO:teuthology.orchestra.run.smithi098.stderr:no change 2023-09-23T18:50:18.038 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:50:18.050 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:18.074 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:18.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:18.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:18.098 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:50:18.142 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T18:50:18.156 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:50:18.178 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:18.207 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:18.208 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:50:18.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:18.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:18.247 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:50:18.291 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:50:18.310 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:18.331 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:50:18.333 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:18.334 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:18 smithi173 bash[59422]: audit 2023-09-23T18:50:17.968651+0000 mon.c (mon.2) 110 : audit 0 from='client.? 172.21.15.98:0/2182368813' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:50:18.335 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:18 smithi173 bash[59422]: audit 2023-09-23T18:50:17.975516+0000 mgr.y (mgr.26054) 966 : audit 0 from='client.35481 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:50:18.335 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:18 smithi173 bash[59422]: cluster 2023-09-23T18:50:18.114664+0000 mgr.y (mgr.26054) 967 : cluster 0 pgmap v1028: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 49588/262542 objects degraded (18.888%); 5113/262542 objects misplaced (1.947%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T18:50:18.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:18.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:18.371 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:50:18.384 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:50:18.385 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:50:18.385 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:50:18.385 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:50:18.385 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:50:18.385 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:50:18.385 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:50:18.385 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:50:18.393 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:18 smithi098 bash[76817]: audit 2023-09-23T18:50:17.968651+0000 mon.c (mon.2) 110 : audit 0 from='client.? 172.21.15.98:0/2182368813' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:50:18.393 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:18 smithi098 bash[76817]: audit 2023-09-23T18:50:17.975516+0000 mgr.y (mgr.26054) 966 : audit 0 from='client.35481 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:50:18.393 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:18 smithi098 bash[76817]: cluster 2023-09-23T18:50:18.114664+0000 mgr.y (mgr.26054) 967 : cluster 0 pgmap v1028: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 49588/262542 objects degraded (18.888%); 5113/262542 objects misplaced (1.947%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T18:50:18.394 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:50:17 smithi098 bash[73085]: debug 2023-09-23T18:50:17.973+0000 7f90a56ad700 -1 mgr.server reply reply (4) Interrupted system call no change 2023-09-23T18:50:18.394 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:18 smithi098 bash[79516]: audit 2023-09-23T18:50:17.968651+0000 mon.c (mon.2) 110 : audit 0 from='client.? 172.21.15.98:0/2182368813' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:50:18.395 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:18 smithi098 bash[79516]: audit 2023-09-23T18:50:17.975516+0000 mgr.y (mgr.26054) 966 : audit 0 from='client.35481 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:50:18.395 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:18 smithi098 bash[79516]: cluster 2023-09-23T18:50:18.114664+0000 mgr.y (mgr.26054) 967 : cluster 0 pgmap v1028: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 49588/262542 objects degraded (18.888%); 5113/262542 objects misplaced (1.947%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T18:50:18.436 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:18.471 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:18.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:18.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:18.576 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:18.600 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:18.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:18.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:18.703 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:18.742 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:18.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:18.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:18.846 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:18.869 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:18.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:18.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:18.974 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:19.019 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:19.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:19.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:19.124 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:19.150 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:19.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:19.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:19.208 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is unset 2023-09-23T18:50:19.257 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:19.284 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:19.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:19.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:19.391 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:19.417 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:19.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:19.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:19.523 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:19.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:19 smithi173 bash[59422]: audit 2023-09-23T18:50:19.084916+0000 mon.c (mon.2) 111 : audit 1 from='client.? 172.21.15.98:0/207202164' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:50:19.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:19 smithi173 bash[59422]: audit 2023-09-23T18:50:19.085759+0000 mon.a (mon.0) 1318 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:50:19.543 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:19 smithi098 bash[79516]: audit 2023-09-23T18:50:19.084916+0000 mon.c (mon.2) 111 : audit 1 from='client.? 172.21.15.98:0/207202164' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:50:19.543 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:19 smithi098 bash[79516]: audit 2023-09-23T18:50:19.085759+0000 mon.a (mon.0) 1318 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:50:19.543 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:19 smithi098 bash[76817]: audit 2023-09-23T18:50:19.084916+0000 mon.c (mon.2) 111 : audit 1 from='client.? 172.21.15.98:0/207202164' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:50:19.543 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:19 smithi098 bash[76817]: audit 2023-09-23T18:50:19.085759+0000 mon.a (mon.0) 1318 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:50:19.551 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:19.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:19.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:19.657 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:19.694 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:19.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:19.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:19.891 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:19.918 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:19.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:19.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:20.022 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:20.065 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:20.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:20.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:20.174 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:20.200 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:20.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:20.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:20.306 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:20.330 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:20.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:20.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:20.435 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:20.447 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:20 smithi173 bash[59422]: cluster 2023-09-23T18:50:19.180825+0000 mon.a (mon.0) 1319 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:50:20.447 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:20 smithi173 bash[59422]: audit 2023-09-23T18:50:19.193013+0000 mon.a (mon.0) 1320 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:50:20.447 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:20 smithi173 bash[59422]: cluster 2023-09-23T18:50:19.193221+0000 mon.a (mon.0) 1321 : cluster 0 osdmap e251: 8 total, 8 up, 8 in 2023-09-23T18:50:20.447 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:20 smithi173 bash[59422]: cluster 2023-09-23T18:50:19.207462+0000 osd.5 (osd.5) 149 : cluster 3 failed to encode map e251 with expected crc 2023-09-23T18:50:20.447 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:20 smithi173 bash[59422]: cluster 2023-09-23T18:50:19.208853+0000 osd.4 (osd.4) 246 : cluster 3 failed to encode map e251 with expected crc 2023-09-23T18:50:20.447 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:20 smithi173 bash[59422]: cluster 2023-09-23T18:50:19.209781+0000 osd.5 (osd.5) 150 : cluster 3 failed to encode map e251 with expected crc 2023-09-23T18:50:20.447 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:20 smithi173 bash[59422]: cluster 2023-09-23T18:50:19.212765+0000 osd.6 (osd.6) 210 : cluster 3 failed to encode map e251 with expected crc 2023-09-23T18:50:20.447 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:20 smithi173 bash[59422]: cluster 2023-09-23T18:50:20.115223+0000 mgr.y (mgr.26054) 968 : cluster 0 pgmap v1030: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 49588/260772 objects degraded (19.016%); 5113/260772 objects misplaced (1.961%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T18:50:20.448 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:20 smithi173 bash[59422]: cluster 2023-09-23T18:50:20.155008+0000 mon.a (mon.0) 1322 : cluster 3 Health check update: Degraded data redundancy: 49588/262542 objects degraded (18.888%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:50:20.463 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:20.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:20.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:20.549 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:20 smithi098 bash[76817]: cluster 2023-09-23T18:50:19.180825+0000 mon.a (mon.0) 1319 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:50:20.549 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:20 smithi098 bash[76817]: audit 2023-09-23T18:50:19.193013+0000 mon.a (mon.0) 1320 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:50:20.549 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:20 smithi098 bash[76817]: cluster 2023-09-23T18:50:19.193221+0000 mon.a (mon.0) 1321 : cluster 0 osdmap e251: 8 total, 8 up, 8 in 2023-09-23T18:50:20.550 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:20 smithi098 bash[76817]: cluster 2023-09-23T18:50:19.207462+0000 osd.5 (osd.5) 149 : cluster 3 failed to encode map e251 with expected crc 2023-09-23T18:50:20.550 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:20 smithi098 bash[76817]: cluster 2023-09-23T18:50:19.208853+0000 osd.4 (osd.4) 246 : cluster 3 failed to encode map e251 with expected crc 2023-09-23T18:50:20.550 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:20 smithi098 bash[76817]: cluster 2023-09-23T18:50:19.209781+0000 osd.5 (osd.5) 150 : cluster 3 failed to encode map e251 with expected crc 2023-09-23T18:50:20.550 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:20 smithi098 bash[76817]: cluster 2023-09-23T18:50:19.212765+0000 osd.6 (osd.6) 210 : cluster 3 failed to encode map e251 with expected crc 2023-09-23T18:50:20.550 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:20 smithi098 bash[76817]: cluster 2023-09-23T18:50:20.115223+0000 mgr.y (mgr.26054) 968 : cluster 0 pgmap v1030: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 49588/260772 objects degraded (19.016%); 5113/260772 objects misplaced (1.961%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T18:50:20.550 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:20 smithi098 bash[76817]: cluster 2023-09-23T18:50:20.155008+0000 mon.a (mon.0) 1322 : cluster 3 Health check update: Degraded data redundancy: 49588/262542 objects degraded (18.888%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:50:20.550 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:20 smithi098 bash[79516]: cluster 2023-09-23T18:50:19.180825+0000 mon.a (mon.0) 1319 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:50:20.550 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:20 smithi098 bash[79516]: audit 2023-09-23T18:50:19.193013+0000 mon.a (mon.0) 1320 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:50:20.551 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:20 smithi098 bash[79516]: cluster 2023-09-23T18:50:19.193221+0000 mon.a (mon.0) 1321 : cluster 0 osdmap e251: 8 total, 8 up, 8 in 2023-09-23T18:50:20.551 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:20 smithi098 bash[79516]: cluster 2023-09-23T18:50:19.207462+0000 osd.5 (osd.5) 149 : cluster 3 failed to encode map e251 with expected crc 2023-09-23T18:50:20.551 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:20 smithi098 bash[79516]: cluster 2023-09-23T18:50:19.208853+0000 osd.4 (osd.4) 246 : cluster 3 failed to encode map e251 with expected crc 2023-09-23T18:50:20.551 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:20 smithi098 bash[79516]: cluster 2023-09-23T18:50:19.209781+0000 osd.5 (osd.5) 150 : cluster 3 failed to encode map e251 with expected crc 2023-09-23T18:50:20.551 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:20 smithi098 bash[79516]: cluster 2023-09-23T18:50:19.212765+0000 osd.6 (osd.6) 210 : cluster 3 failed to encode map e251 with expected crc 2023-09-23T18:50:20.551 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:20 smithi098 bash[79516]: cluster 2023-09-23T18:50:20.115223+0000 mgr.y (mgr.26054) 968 : cluster 0 pgmap v1030: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 49588/260772 objects degraded (19.016%); 5113/260772 objects misplaced (1.961%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T18:50:20.551 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:20 smithi098 bash[79516]: cluster 2023-09-23T18:50:20.155008+0000 mon.a (mon.0) 1322 : cluster 3 Health check update: Degraded data redundancy: 49588/262542 objects degraded (18.888%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:50:20.567 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:20.595 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:20.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:20.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:20.702 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:20.735 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:20.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:20.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:20.838 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:20.855 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:20.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:20.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:20.958 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:20.975 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:20.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:20.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:21.077 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:21.121 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:21.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:21.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:21.227 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:21.264 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:21.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:21.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:21.368 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:21.396 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:21.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:21.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:21.499 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:21.525 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:21.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:21.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:21.631 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:21.660 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:21.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:21.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:21.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:21.801 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:21.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:21.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:21.889 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T18:50:21.890 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-09-23T18:50:21.907 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:21.931 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:21.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:21.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:22.034 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:22.043 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:21 smithi173 bash[59422]: audit 2023-09-23T18:50:21.896247+0000 mon.a (mon.0) 1323 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:50:22.061 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:22.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:22.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:22.164 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:22.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:21 smithi098 bash[76817]: audit 2023-09-23T18:50:21.896247+0000 mon.a (mon.0) 1323 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:50:22.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:21 smithi098 bash[79516]: audit 2023-09-23T18:50:21.896247+0000 mon.a (mon.0) 1323 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:50:22.202 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:22.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:22.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:22.309 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:22.344 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:22.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:22.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:22.447 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:22.495 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:22.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:22.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:22.598 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:22.623 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:22.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:22.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:22.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:22.751 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:22.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:22.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:22.859 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:22.899 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:22.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:22.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:22.942 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:50:23.004 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:23.025 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:23.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:23.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:23.128 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:23.153 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:23.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:23.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:23.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:22 smithi098 bash[76817]: cluster 2023-09-23T18:50:22.116091+0000 mgr.y (mgr.26054) 969 : cluster 0 pgmap v1031: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 49633/255432 objects degraded (19.431%); 4978/255432 objects misplaced (1.949%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:50:23.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:22 smithi098 bash[79516]: cluster 2023-09-23T18:50:22.116091+0000 mgr.y (mgr.26054) 969 : cluster 0 pgmap v1031: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 49633/255432 objects degraded (19.431%); 4978/255432 objects misplaced (1.949%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:50:23.258 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:23.265 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:22 smithi173 bash[59422]: cluster 2023-09-23T18:50:22.116091+0000 mgr.y (mgr.26054) 969 : cluster 0 pgmap v1031: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 49633/255432 objects degraded (19.431%); 4978/255432 objects misplaced (1.949%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:50:23.281 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:23.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:23.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:23.384 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:23.405 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:23.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:23.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:23.508 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:23.536 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:23.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:23.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:23.641 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:23.668 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:23.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:23.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:23.773 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:23.798 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:23.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:23.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:23.901 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:23.923 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:23.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:23.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:24.027 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:24.059 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:24.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:24.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:24.165 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:24.373 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:24.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:24.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:24.438 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:50:24.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:24 smithi098 bash[76817]: cluster 2023-09-23T18:50:24.116759+0000 mgr.y (mgr.26054) 970 : cluster 0 pgmap v1032: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 49633/253569 objects degraded (19.574%); 4978/253569 objects misplaced (1.963%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:50:24.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:24 smithi098 bash[79516]: cluster 2023-09-23T18:50:24.116759+0000 mgr.y (mgr.26054) 970 : cluster 0 pgmap v1032: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 49633/253569 objects degraded (19.574%); 4978/253569 objects misplaced (1.963%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:50:24.478 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:24.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:24 smithi173 bash[59422]: cluster 2023-09-23T18:50:24.116759+0000 mgr.y (mgr.26054) 970 : cluster 0 pgmap v1032: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 49633/253569 objects degraded (19.574%); 4978/253569 objects misplaced (1.963%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:50:24.530 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:24.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:24.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:24.635 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:24.647 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:50:24.657 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:24.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:24.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:24.760 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:24.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:24.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:24.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:24.892 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:24.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:24.929 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:24.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:25.030 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:25.069 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:25.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:25.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:25.173 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:25.200 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:25.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:25.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:25.309 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:25.336 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:25.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:25.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:25.440 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:25.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:25 smithi098 bash[76817]: audit 2023-09-23T18:50:24.622577+0000 mon.a (mon.0) 1324 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:50:25.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:25 smithi098 bash[76817]: cluster 2023-09-23T18:50:25.156266+0000 mon.a (mon.0) 1325 : cluster 3 Health check update: Degraded data redundancy: 49633/253569 objects degraded (19.574%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:50:25.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:25 smithi098 bash[79516]: audit 2023-09-23T18:50:24.622577+0000 mon.a (mon.0) 1324 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:50:25.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:25 smithi098 bash[79516]: cluster 2023-09-23T18:50:25.156266+0000 mon.a (mon.0) 1325 : cluster 3 Health check update: Degraded data redundancy: 49633/253569 objects degraded (19.574%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:50:25.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:25.500 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:25.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:25.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:25 smithi173 bash[59422]: audit 2023-09-23T18:50:24.622577+0000 mon.a (mon.0) 1324 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:50:25.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:25 smithi173 bash[59422]: cluster 2023-09-23T18:50:25.156266+0000 mon.a (mon.0) 1325 : cluster 3 Health check update: Degraded data redundancy: 49633/253569 objects degraded (19.574%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:50:25.601 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:25.639 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:25.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:25.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:25.745 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:25.777 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:25.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:25.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:25.881 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:25.909 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:25.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:25.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:26.013 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:26.041 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:26.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:26.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:26.146 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:26.173 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:26.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:26.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:26.277 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:26.311 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:26.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:26.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:26.415 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:26.423 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:26 smithi173 bash[59422]: cluster 2023-09-23T18:50:26.117376+0000 mgr.y (mgr.26054) 971 : cluster 0 pgmap v1033: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 484 op/s; 49633/252267 objects degraded (19.675%); 4978/252267 objects misplaced (1.973%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:50:26.435 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:26.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:26.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:26.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:26 smithi098 bash[76817]: cluster 2023-09-23T18:50:26.117376+0000 mgr.y (mgr.26054) 971 : cluster 0 pgmap v1033: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 484 op/s; 49633/252267 objects degraded (19.675%); 4978/252267 objects misplaced (1.973%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:50:26.442 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:50:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:50:26] "GET /metrics HTTP/1.1" 200 36622 "" "Prometheus/2.33.4" 2023-09-23T18:50:26.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:26 smithi098 bash[79516]: cluster 2023-09-23T18:50:26.117376+0000 mgr.y (mgr.26054) 971 : cluster 0 pgmap v1033: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 484 op/s; 49633/252267 objects degraded (19.675%); 4978/252267 objects misplaced (1.973%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:50:26.541 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:26.579 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:26.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:26.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:26.688 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:26.721 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:26.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:26.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:26.824 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:26.851 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:26.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:26.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:26.894 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:50:26.954 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:26.999 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:27.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:27.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:27.103 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:27.140 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:27.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:27.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:27.248 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:27.288 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:27.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:27.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:27.395 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:27.474 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:27.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:27.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:27.581 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:27.618 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:27.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:27.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:27.731 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:27.776 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:27.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:27.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:27.883 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:27.914 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:27.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:27.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:27.951 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T18:50:27.951 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:50:28.022 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:28.051 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:28.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:28.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:28.153 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:28.239 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:28.242 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:28 smithi173 bash[59422]: cluster 2023-09-23T18:50:28.118089+0000 mgr.y (mgr.26054) 972 : cluster 0 pgmap v1034: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 49720/247971 objects degraded (20.051%); 4806/247971 objects misplaced (1.938%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T18:50:28.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:28.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:28.343 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:28.379 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:28.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:28.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:28.508 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:28.538 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:28.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:28.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:28.645 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:28.654 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:28 smithi098 bash[76817]: cluster 2023-09-23T18:50:28.118089+0000 mgr.y (mgr.26054) 972 : cluster 0 pgmap v1034: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 49720/247971 objects degraded (20.051%); 4806/247971 objects misplaced (1.938%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T18:50:28.654 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:28 smithi098 bash[79516]: cluster 2023-09-23T18:50:28.118089+0000 mgr.y (mgr.26054) 972 : cluster 0 pgmap v1034: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 49720/247971 objects degraded (20.051%); 4806/247971 objects misplaced (1.938%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T18:50:28.672 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:28.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:28.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:28.775 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:28.802 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:28.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:28.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:28.828 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T18:50:28.905 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:28.929 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:28.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:28.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:28.938 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:50:28.938 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:50:29.036 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:29.064 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:29.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:29.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:29.173 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:29.221 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:29.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:29.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:29.331 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:29.360 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:29.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:29.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:29.463 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:29.497 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:29.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:29.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:29.603 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:29.640 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:29.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:29.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:29.746 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:29.786 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:29.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:29.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:29.892 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:29.922 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:29.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:29.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:29.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:29 smithi098 bash[76817]: audit 2023-09-23T18:50:28.789597+0000 mon.a (mon.0) 1326 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:50:29.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:29 smithi098 bash[76817]: audit 2023-09-23T18:50:28.801883+0000 mon.a (mon.0) 1327 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:50:29.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:29 smithi098 bash[76817]: audit 2023-09-23T18:50:29.194215+0000 mon.a (mon.0) 1328 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:50:29.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:29 smithi098 bash[76817]: audit 2023-09-23T18:50:29.204773+0000 mon.a (mon.0) 1329 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:50:29.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:29 smithi098 bash[79516]: audit 2023-09-23T18:50:28.789597+0000 mon.a (mon.0) 1326 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:50:29.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:29 smithi098 bash[79516]: audit 2023-09-23T18:50:28.801883+0000 mon.a (mon.0) 1327 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:50:29.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:29 smithi098 bash[79516]: audit 2023-09-23T18:50:29.194215+0000 mon.a (mon.0) 1328 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:50:29.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:29 smithi098 bash[79516]: audit 2023-09-23T18:50:29.204773+0000 mon.a (mon.0) 1329 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:50:30.026 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:30.072 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:30.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:30.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:30.175 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:30.183 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:29 smithi173 bash[59422]: audit 2023-09-23T18:50:28.789597+0000 mon.a (mon.0) 1326 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:50:30.183 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:29 smithi173 bash[59422]: audit 2023-09-23T18:50:28.801883+0000 mon.a (mon.0) 1327 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:50:30.183 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:29 smithi173 bash[59422]: audit 2023-09-23T18:50:29.194215+0000 mon.a (mon.0) 1328 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:50:30.184 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:29 smithi173 bash[59422]: audit 2023-09-23T18:50:29.204773+0000 mon.a (mon.0) 1329 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:50:30.206 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:30.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:30.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:30.311 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:30.333 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:30.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:30.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:30.440 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:30.480 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:30.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:30.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:30.584 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:30.616 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:30.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:30.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:30.718 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:30.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:30.746 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:30.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:30.847 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:30.876 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:30.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:30.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:30.982 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:31.012 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:31.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:31.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:31.119 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:31.157 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:31.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:31.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:31.261 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:31.286 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:31.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:31.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:31.389 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:31.413 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:31.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:31.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:31.420 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:31 smithi173 bash[59422]: cluster 2023-09-23T18:50:30.118796+0000 mgr.y (mgr.26054) 973 : cluster 0 pgmap v1035: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 450 op/s; 49720/246021 objects degraded (20.210%); 4806/246021 objects misplaced (1.953%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T18:50:31.420 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:31 smithi173 bash[59422]: cluster 2023-09-23T18:50:30.157401+0000 mon.a (mon.0) 1330 : cluster 3 Health check update: Degraded data redundancy: 49720/247971 objects degraded (20.051%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:50:31.420 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:31 smithi173 bash[59422]: audit 2023-09-23T18:50:30.697099+0000 mon.a (mon.0) 1331 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:50:31.420 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:31 smithi173 bash[59422]: audit 2023-09-23T18:50:30.698544+0000 mon.a (mon.0) 1332 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:50:31.421 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:31 smithi173 bash[59422]: audit 2023-09-23T18:50:30.711820+0000 mon.a (mon.0) 1333 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:50:31.520 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:31.528 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:31 smithi098 bash[76817]: cluster 2023-09-23T18:50:30.118796+0000 mgr.y (mgr.26054) 973 : cluster 0 pgmap v1035: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 450 op/s; 49720/246021 objects degraded (20.210%); 4806/246021 objects misplaced (1.953%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T18:50:31.528 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:31 smithi098 bash[76817]: cluster 2023-09-23T18:50:30.157401+0000 mon.a (mon.0) 1330 : cluster 3 Health check update: Degraded data redundancy: 49720/247971 objects degraded (20.051%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:50:31.529 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:31 smithi098 bash[76817]: audit 2023-09-23T18:50:30.697099+0000 mon.a (mon.0) 1331 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:50:31.529 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:31 smithi098 bash[76817]: audit 2023-09-23T18:50:30.698544+0000 mon.a (mon.0) 1332 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:50:31.529 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:31 smithi098 bash[76817]: audit 2023-09-23T18:50:30.711820+0000 mon.a (mon.0) 1333 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:50:31.529 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:31 smithi098 bash[79516]: cluster 2023-09-23T18:50:30.118796+0000 mgr.y (mgr.26054) 973 : cluster 0 pgmap v1035: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 450 op/s; 49720/246021 objects degraded (20.210%); 4806/246021 objects misplaced (1.953%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T18:50:31.529 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:31 smithi098 bash[79516]: cluster 2023-09-23T18:50:30.157401+0000 mon.a (mon.0) 1330 : cluster 3 Health check update: Degraded data redundancy: 49720/247971 objects degraded (20.051%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:50:31.529 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:31 smithi098 bash[79516]: audit 2023-09-23T18:50:30.697099+0000 mon.a (mon.0) 1331 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:50:31.529 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:31 smithi098 bash[79516]: audit 2023-09-23T18:50:30.698544+0000 mon.a (mon.0) 1332 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:50:31.529 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:31 smithi098 bash[79516]: audit 2023-09-23T18:50:30.711820+0000 mon.a (mon.0) 1333 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:50:31.559 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:31.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:31.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:31.668 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:31.700 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:31.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:31.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:31.808 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:31.825 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:31.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:31.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:31.927 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:31.955 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:31.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:31.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:32.059 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:32.093 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:32.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:32.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:32.197 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:32.236 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:32.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:32.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:32.339 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:32.368 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:32.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:32.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:32.472 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:32.508 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:32.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:32.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:32.612 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:32.647 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:32.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:32.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:32.758 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:32.784 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:32.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:32.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:32.895 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:32.923 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:32.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:32.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:33.027 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:33.052 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:33.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:33.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:33.164 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:33.166 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:33 smithi098 bash[76817]: cluster 2023-09-23T18:50:32.119611+0000 mgr.y (mgr.26054) 974 : cluster 0 pgmap v1036: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 49720/241311 objects degraded (20.604%); 4806/241311 objects misplaced (1.992%); 933 KiB/s, 14 objects/s recovering 2023-09-23T18:50:33.200 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:33.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:33.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:33.302 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:33.311 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:33 smithi173 bash[59422]: cluster 2023-09-23T18:50:32.119611+0000 mgr.y (mgr.26054) 974 : cluster 0 pgmap v1036: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 49720/241311 objects degraded (20.604%); 4806/241311 objects misplaced (1.992%); 933 KiB/s, 14 objects/s recovering 2023-09-23T18:50:33.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:33.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:33.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:33.431 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:33.438 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:33 smithi098 bash[79516]: cluster 2023-09-23T18:50:32.119611+0000 mgr.y (mgr.26054) 974 : cluster 0 pgmap v1036: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 49720/241311 objects degraded (20.604%); 4806/241311 objects misplaced (1.992%); 933 KiB/s, 14 objects/s recovering 2023-09-23T18:50:33.479 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:33.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:33.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:33.583 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:33.634 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:33.637 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:50:33.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:33.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:33.740 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:33.767 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:33.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:33.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:33.870 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:33.906 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:33.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:33.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:34.010 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:34.032 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:34.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:34.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:34.137 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:34.176 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:34.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:34.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:34.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:34 smithi173 bash[59422]: cluster 2023-09-23T18:50:34.120615+0000 mgr.y (mgr.26054) 975 : cluster 0 pgmap v1037: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 49751/238731 objects degraded (20.840%); 4671/238731 objects misplaced (1.957%); 1008 KiB/s, 15 objects/s recovering 2023-09-23T18:50:34.281 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:34.289 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:34 smithi098 bash[79516]: cluster 2023-09-23T18:50:34.120615+0000 mgr.y (mgr.26054) 975 : cluster 0 pgmap v1037: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 49751/238731 objects degraded (20.840%); 4671/238731 objects misplaced (1.957%); 1008 KiB/s, 15 objects/s recovering 2023-09-23T18:50:34.290 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:34 smithi098 bash[76817]: cluster 2023-09-23T18:50:34.120615+0000 mgr.y (mgr.26054) 975 : cluster 0 pgmap v1037: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 49751/238731 objects degraded (20.840%); 4671/238731 objects misplaced (1.957%); 1008 KiB/s, 15 objects/s recovering 2023-09-23T18:50:34.312 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:34.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:34.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:34.416 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:34.439 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:34.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:34.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:34.543 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:34.569 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:34.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:34.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:34.672 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:34.703 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:34.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:34.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:34.811 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:34.841 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:34.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:34.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:34.944 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:34.967 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:34.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:34.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:35.070 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:35.093 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:35.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:35.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:35.198 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:35.221 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:35.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:35.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:35.328 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:35.357 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:35.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:35.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:35.461 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:35.477 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:35 smithi098 bash[79516]: cluster 2023-09-23T18:50:35.158630+0000 mon.a (mon.0) 1334 : cluster 3 Health check update: Degraded data redundancy: 49751/238731 objects degraded (20.840%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:50:35.478 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:35 smithi098 bash[76817]: cluster 2023-09-23T18:50:35.158630+0000 mon.a (mon.0) 1334 : cluster 3 Health check update: Degraded data redundancy: 49751/238731 objects degraded (20.840%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:50:35.505 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:35.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:35.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:35.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:35 smithi173 bash[59422]: cluster 2023-09-23T18:50:35.158630+0000 mon.a (mon.0) 1334 : cluster 3 Health check update: Degraded data redundancy: 49751/238731 objects degraded (20.840%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:50:35.611 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:35.638 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:35.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:35.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:35.741 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:35.773 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:35.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:35.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:35.877 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:35.904 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:35.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:35.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:36.008 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:36.036 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:36.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:36.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:36.145 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:36.177 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:36.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:36.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:36.227 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:50:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:50:36] "GET /metrics HTTP/1.1" 200 36622 "" "Prometheus/2.33.4" 2023-09-23T18:50:36.284 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:36.321 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:36.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:36.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:36.425 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:36.433 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:36 smithi173 bash[59422]: cluster 2023-09-23T18:50:36.121288+0000 mgr.y (mgr.26054) 976 : cluster 0 pgmap v1038: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 448 op/s; 49751/237432 objects degraded (20.954%); 4671/237432 objects misplaced (1.967%); 1008 KiB/s, 15 objects/s recovering 2023-09-23T18:50:36.448 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:36.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:36.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:36.553 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:36.574 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:36.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:36.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:36.680 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:36.690 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:36 smithi098 bash[76817]: cluster 2023-09-23T18:50:36.121288+0000 mgr.y (mgr.26054) 976 : cluster 0 pgmap v1038: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 448 op/s; 49751/237432 objects degraded (20.954%); 4671/237432 objects misplaced (1.967%); 1008 KiB/s, 15 objects/s recovering 2023-09-23T18:50:36.690 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:36 smithi098 bash[79516]: cluster 2023-09-23T18:50:36.121288+0000 mgr.y (mgr.26054) 976 : cluster 0 pgmap v1038: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 448 op/s; 49751/237432 objects degraded (20.954%); 4671/237432 objects misplaced (1.967%); 1008 KiB/s, 15 objects/s recovering 2023-09-23T18:50:36.714 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:36.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:36.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:36.817 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:36.844 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:36.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:36.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:36.948 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:36.977 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:36.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:36.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:37.088 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:37.109 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:37.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:37.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:37.211 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:37.237 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:37.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:37.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:37.341 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:37.394 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:37.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:37.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:37.500 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:37.527 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:37.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:37.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:37.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:37.657 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:37.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:37.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:37.761 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:37.796 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:37.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:37.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:37.905 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:37.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:37.930 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:37.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:38.031 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:38.059 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:38.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:38.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:38.164 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:38.190 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:38.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:38.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:38.294 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:38.321 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:38.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:38.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:38.424 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:38.467 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:38.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:38.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:38.473 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:38 smithi098 bash[76817]: cluster 2023-09-23T18:50:38.122024+0000 mgr.y (mgr.26054) 977 : cluster 0 pgmap v1039: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 49815/233136 objects degraded (21.367%); 4516/233136 objects misplaced (1.937%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T18:50:38.474 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:38 smithi098 bash[79516]: cluster 2023-09-23T18:50:38.122024+0000 mgr.y (mgr.26054) 977 : cluster 0 pgmap v1039: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 49815/233136 objects degraded (21.367%); 4516/233136 objects misplaced (1.937%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T18:50:38.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:38 smithi173 bash[59422]: cluster 2023-09-23T18:50:38.122024+0000 mgr.y (mgr.26054) 977 : cluster 0 pgmap v1039: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 49815/233136 objects degraded (21.367%); 4516/233136 objects misplaced (1.937%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T18:50:38.574 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:38.603 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:38.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:38.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:38.627 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T18:50:38.627 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:50:38.710 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:38.737 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:38.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:38.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:38.840 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:38.863 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:38.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:38.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:38.969 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:39.006 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:39.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:39.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:39.110 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:39.141 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:39.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:39.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:39.245 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:39.289 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:39.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:39.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:39.394 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:39.418 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:39.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:39.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:39.523 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:39.557 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:39.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:39.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:39.663 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:39.672 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T18:50:39.691 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:39.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:39.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:39.794 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:39.818 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:39.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:39.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:39.921 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:39.930 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:39 smithi098 bash[76817]: audit 2023-09-23T18:50:39.626632+0000 mon.a (mon.0) 1335 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:50:39.930 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:39 smithi098 bash[79516]: audit 2023-09-23T18:50:39.626632+0000 mon.a (mon.0) 1335 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:50:39.960 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:39.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:39.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:40.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:39 smithi173 bash[59422]: audit 2023-09-23T18:50:39.626632+0000 mon.a (mon.0) 1335 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:50:40.066 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:40.100 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:40.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:40.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:40.209 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:40.245 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:40.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:40.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:40.353 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:40.378 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:40.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:40.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:40.484 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:40.511 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:40.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:40.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:40.622 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:40.646 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:40.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:40.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:40.752 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:40.775 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:40.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:40.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:40.881 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:40.890 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:40 smithi098 bash[76817]: cluster 2023-09-23T18:50:40.122712+0000 mgr.y (mgr.26054) 978 : cluster 0 pgmap v1040: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 467 op/s; 49815/231141 objects degraded (21.552%); 4516/231141 objects misplaced (1.954%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T18:50:40.890 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:40 smithi098 bash[76817]: cluster 2023-09-23T18:50:40.159675+0000 mon.a (mon.0) 1336 : cluster 3 Health check update: Degraded data redundancy: 49815/233136 objects degraded (21.367%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:50:40.890 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:40 smithi098 bash[79516]: cluster 2023-09-23T18:50:40.122712+0000 mgr.y (mgr.26054) 978 : cluster 0 pgmap v1040: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 467 op/s; 49815/231141 objects degraded (21.552%); 4516/231141 objects misplaced (1.954%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T18:50:40.890 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:40 smithi098 bash[79516]: cluster 2023-09-23T18:50:40.159675+0000 mon.a (mon.0) 1336 : cluster 3 Health check update: Degraded data redundancy: 49815/233136 objects degraded (21.367%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:50:40.911 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:40.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:40.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:41.018 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:41.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:40 smithi173 bash[59422]: cluster 2023-09-23T18:50:40.122712+0000 mgr.y (mgr.26054) 978 : cluster 0 pgmap v1040: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 467 op/s; 49815/231141 objects degraded (21.552%); 4516/231141 objects misplaced (1.954%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T18:50:41.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:40 smithi173 bash[59422]: cluster 2023-09-23T18:50:40.159675+0000 mon.a (mon.0) 1336 : cluster 3 Health check update: Degraded data redundancy: 49815/233136 objects degraded (21.367%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:50:41.041 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:41.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:41.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:41.145 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:41.176 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:41.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:41.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:41.279 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:41.309 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:41.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:41.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:41.412 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:41.442 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:41.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:41.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:41.517 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:50:41.547 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:41.561 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:50:41.585 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:41.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:41.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:41.606 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:50:41.661 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:50:41.691 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:41.708 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:50:41.758 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:41.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:41.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:41.780 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:50:41.832 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:50:41.864 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:41.875 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:50:41.883 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:50:41.883 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:50:41.883 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:50:41.883 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:50:41.883 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:50:41.883 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:50:41.883 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:50:41.883 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:50:41.891 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:41.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:41.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:41.922 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:41 smithi098 bash[76817]: audit 2023-09-23T18:50:41.434192+0000 mon.a (mon.0) 1337 : audit 0 from='client.? 172.21.15.98:0/1533116072' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:50:41.923 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:41 smithi098 bash[79516]: audit 2023-09-23T18:50:41.434192+0000 mon.a (mon.0) 1337 : audit 0 from='client.? 172.21.15.98:0/1533116072' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:50:41.994 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:42.004 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:41 smithi173 bash[59422]: audit 2023-09-23T18:50:41.434192+0000 mon.a (mon.0) 1337 : audit 0 from='client.? 172.21.15.98:0/1533116072' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:50:42.019 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:42.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:42.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:42.127 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:42.154 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:42.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:42.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:42.259 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:42.291 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:42.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:42.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:42.394 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:42.417 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:42.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:42.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:42.521 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:42.550 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:42.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:42.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:42.656 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:42.689 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:42.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:42.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:42.714 INFO:teuthology.orchestra.run.smithi098.stdout:moved 4 / 267 (1.49813%) 2023-09-23T18:50:42.714 INFO:teuthology.orchestra.run.smithi098.stdout:avg 33.375 2023-09-23T18:50:42.714 INFO:teuthology.orchestra.run.smithi098.stdout:stddev 6.20358 -> 6.6132 (expected baseline 5.40399) 2023-09-23T18:50:42.714 INFO:teuthology.orchestra.run.smithi098.stdout:min osd.5 with 21 -> 19 pgs (0.629214 -> 0.569288 * mean) 2023-09-23T18:50:42.714 INFO:teuthology.orchestra.run.smithi098.stdout:max osd.1 with 42 -> 41 pgs (1.25843 -> 1.22846 * mean) 2023-09-23T18:50:42.715 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:50:42.715 INFO:teuthology.orchestra.run.smithi098.stdout:oload 110 2023-09-23T18:50:42.715 INFO:teuthology.orchestra.run.smithi098.stdout:max_change 0.05 2023-09-23T18:50:42.715 INFO:teuthology.orchestra.run.smithi098.stdout:max_change_osds 4 2023-09-23T18:50:42.715 INFO:teuthology.orchestra.run.smithi098.stdout:average_utilization 0.0275 2023-09-23T18:50:42.715 INFO:teuthology.orchestra.run.smithi098.stdout:overload_utilization 0.0303 2023-09-23T18:50:42.715 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5 weight 0.6936 -> 0.6437 2023-09-23T18:50:42.715 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2 weight 0.9987 -> 1.0000 2023-09-23T18:50:42.715 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3 weight 0.6315 -> 0.6815 2023-09-23T18:50:42.715 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6 weight 0.8751 -> 0.8251 2023-09-23T18:50:42.725 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is unset 2023-09-23T18:50:42.792 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:42.818 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:42.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:42.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:42.845 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:42 smithi098 bash[79516]: audit 2023-09-23T18:50:41.844121+0000 mgr.y (mgr.26054) 979 : audit 0 from='client.35493 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:50:42.845 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:42 smithi098 bash[79516]: audit 2023-09-23T18:50:41.845242+0000 mon.a (mon.0) 1338 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'65536','3':'44663','5':'42183','6':'54077'}"}]: dispatch 2023-09-23T18:50:42.846 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:42 smithi098 bash[79516]: audit 2023-09-23T18:50:41.849429+0000 mon.c (mon.2) 112 : audit 1 from='client.? 172.21.15.98:0/1647782820' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:50:42.846 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:42 smithi098 bash[79516]: audit 2023-09-23T18:50:41.851054+0000 mon.a (mon.0) 1339 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:50:42.846 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:42 smithi098 bash[79516]: cluster 2023-09-23T18:50:42.123483+0000 mgr.y (mgr.26054) 980 : cluster 0 pgmap v1041: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 49815/226428 objects degraded (22.000%); 4516/226428 objects misplaced (1.994%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T18:50:42.846 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:42 smithi098 bash[76817]: audit 2023-09-23T18:50:41.844121+0000 mgr.y (mgr.26054) 979 : audit 0 from='client.35493 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:50:42.846 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:42 smithi098 bash[76817]: audit 2023-09-23T18:50:41.845242+0000 mon.a (mon.0) 1338 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'65536','3':'44663','5':'42183','6':'54077'}"}]: dispatch 2023-09-23T18:50:42.846 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:42 smithi098 bash[76817]: audit 2023-09-23T18:50:41.849429+0000 mon.c (mon.2) 112 : audit 1 from='client.? 172.21.15.98:0/1647782820' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:50:42.846 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:42 smithi098 bash[76817]: audit 2023-09-23T18:50:41.851054+0000 mon.a (mon.0) 1339 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:50:42.846 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:42 smithi098 bash[76817]: cluster 2023-09-23T18:50:42.123483+0000 mgr.y (mgr.26054) 980 : cluster 0 pgmap v1041: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 49815/226428 objects degraded (22.000%); 4516/226428 objects misplaced (1.994%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T18:50:42.925 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:42.956 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:42.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:42.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:43.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:42 smithi173 bash[59422]: audit 2023-09-23T18:50:41.844121+0000 mgr.y (mgr.26054) 979 : audit 0 from='client.35493 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:50:43.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:42 smithi173 bash[59422]: audit 2023-09-23T18:50:41.845242+0000 mon.a (mon.0) 1338 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'65536','3':'44663','5':'42183','6':'54077'}"}]: dispatch 2023-09-23T18:50:43.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:42 smithi173 bash[59422]: audit 2023-09-23T18:50:41.849429+0000 mon.c (mon.2) 112 : audit 1 from='client.? 172.21.15.98:0/1647782820' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:50:43.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:42 smithi173 bash[59422]: audit 2023-09-23T18:50:41.851054+0000 mon.a (mon.0) 1339 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:50:43.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:42 smithi173 bash[59422]: cluster 2023-09-23T18:50:42.123483+0000 mgr.y (mgr.26054) 980 : cluster 0 pgmap v1041: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 49815/226428 objects degraded (22.000%); 4516/226428 objects misplaced (1.994%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T18:50:43.061 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:43.087 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:43.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:43.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:43.193 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:43.220 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:43.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:43.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:43.323 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:43.356 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:43.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:43.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:43.459 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:43.488 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:43.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:43.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:43.591 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:43.621 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:43.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:43.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:43.725 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:43.742 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:43.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:43.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:43.845 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:43.870 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:43.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:43.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:43.973 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:43.984 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:43 smithi173 bash[59422]: cluster 2023-09-23T18:50:42.695156+0000 mon.a (mon.0) 1340 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:50:43.984 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:43 smithi173 bash[59422]: cluster 2023-09-23T18:50:42.705693+0000 osd.5 (osd.5) 151 : cluster 3 failed to encode map e252 with expected crc 2023-09-23T18:50:43.984 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:43 smithi173 bash[59422]: cluster 2023-09-23T18:50:42.705779+0000 osd.6 (osd.6) 211 : cluster 3 failed to encode map e252 with expected crc 2023-09-23T18:50:43.984 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:43 smithi173 bash[59422]: audit 2023-09-23T18:50:42.711028+0000 mon.a (mon.0) 1341 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'65536','3':'44663','5':'42183','6':'54077'}"}]': finished 2023-09-23T18:50:43.984 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:43 smithi173 bash[59422]: audit 2023-09-23T18:50:42.711182+0000 mon.a (mon.0) 1342 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:50:43.984 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:43 smithi173 bash[59422]: cluster 2023-09-23T18:50:42.711413+0000 mon.a (mon.0) 1343 : cluster 0 osdmap e252: 8 total, 8 up, 8 in 2023-09-23T18:50:43.984 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:43 smithi173 bash[59422]: cluster 2023-09-23T18:50:42.719794+0000 osd.4 (osd.4) 247 : cluster 3 failed to encode map e252 with expected crc 2023-09-23T18:50:43.984 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:43 smithi173 bash[59422]: cluster 2023-09-23T18:50:42.721944+0000 osd.7 (osd.7) 176 : cluster 3 failed to encode map e252 with expected crc 2023-09-23T18:50:44.002 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:44.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:44.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:44.106 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:44.134 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:44.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:44.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:44.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:43 smithi098 bash[79516]: cluster 2023-09-23T18:50:42.695156+0000 mon.a (mon.0) 1340 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:50:44.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:43 smithi098 bash[79516]: cluster 2023-09-23T18:50:42.705693+0000 osd.5 (osd.5) 151 : cluster 3 failed to encode map e252 with expected crc 2023-09-23T18:50:44.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:43 smithi098 bash[79516]: cluster 2023-09-23T18:50:42.705779+0000 osd.6 (osd.6) 211 : cluster 3 failed to encode map e252 with expected crc 2023-09-23T18:50:44.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:43 smithi098 bash[79516]: audit 2023-09-23T18:50:42.711028+0000 mon.a (mon.0) 1341 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'65536','3':'44663','5':'42183','6':'54077'}"}]': finished 2023-09-23T18:50:44.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:43 smithi098 bash[79516]: audit 2023-09-23T18:50:42.711182+0000 mon.a (mon.0) 1342 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:50:44.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:43 smithi098 bash[79516]: cluster 2023-09-23T18:50:42.711413+0000 mon.a (mon.0) 1343 : cluster 0 osdmap e252: 8 total, 8 up, 8 in 2023-09-23T18:50:44.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:43 smithi098 bash[79516]: cluster 2023-09-23T18:50:42.719794+0000 osd.4 (osd.4) 247 : cluster 3 failed to encode map e252 with expected crc 2023-09-23T18:50:44.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:43 smithi098 bash[79516]: cluster 2023-09-23T18:50:42.721944+0000 osd.7 (osd.7) 176 : cluster 3 failed to encode map e252 with expected crc 2023-09-23T18:50:44.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:43 smithi098 bash[76817]: cluster 2023-09-23T18:50:42.695156+0000 mon.a (mon.0) 1340 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:50:44.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:43 smithi098 bash[76817]: cluster 2023-09-23T18:50:42.705693+0000 osd.5 (osd.5) 151 : cluster 3 failed to encode map e252 with expected crc 2023-09-23T18:50:44.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:43 smithi098 bash[76817]: cluster 2023-09-23T18:50:42.705779+0000 osd.6 (osd.6) 211 : cluster 3 failed to encode map e252 with expected crc 2023-09-23T18:50:44.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:43 smithi098 bash[76817]: audit 2023-09-23T18:50:42.711028+0000 mon.a (mon.0) 1341 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'65536','3':'44663','5':'42183','6':'54077'}"}]': finished 2023-09-23T18:50:44.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:43 smithi098 bash[76817]: audit 2023-09-23T18:50:42.711182+0000 mon.a (mon.0) 1342 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:50:44.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:43 smithi098 bash[76817]: cluster 2023-09-23T18:50:42.711413+0000 mon.a (mon.0) 1343 : cluster 0 osdmap e252: 8 total, 8 up, 8 in 2023-09-23T18:50:44.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:43 smithi098 bash[76817]: cluster 2023-09-23T18:50:42.719794+0000 osd.4 (osd.4) 247 : cluster 3 failed to encode map e252 with expected crc 2023-09-23T18:50:44.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:43 smithi098 bash[76817]: cluster 2023-09-23T18:50:42.721944+0000 osd.7 (osd.7) 176 : cluster 3 failed to encode map e252 with expected crc 2023-09-23T18:50:44.238 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:44.266 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:44.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:44.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:44.369 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:44.394 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:44.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:44.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:44.497 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:44.521 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:44.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:44.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:44.624 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:44.654 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:44.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:44.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:44.758 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:44.769 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:44 smithi098 bash[76817]: cluster 2023-09-23T18:50:43.711290+0000 mon.a (mon.0) 1344 : cluster 0 osdmap e253: 8 total, 8 up, 8 in 2023-09-23T18:50:44.769 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:44 smithi098 bash[76817]: cluster 2023-09-23T18:50:43.713315+0000 osd.6 (osd.6) 212 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T18:50:44.770 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:44 smithi098 bash[76817]: cluster 2023-09-23T18:50:43.716380+0000 osd.5 (osd.5) 152 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T18:50:44.770 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:44 smithi098 bash[76817]: cluster 2023-09-23T18:50:43.717070+0000 osd.7 (osd.7) 177 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T18:50:44.770 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:44 smithi098 bash[76817]: cluster 2023-09-23T18:50:44.057940+0000 osd.4 (osd.4) 248 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T18:50:44.770 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:44 smithi098 bash[76817]: cluster 2023-09-23T18:50:44.063871+0000 osd.4 (osd.4) 249 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T18:50:44.770 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:44 smithi098 bash[76817]: cluster 2023-09-23T18:50:44.124073+0000 mgr.y (mgr.26054) 981 : cluster 0 pgmap v1044: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 49861/223926 objects degraded (22.267%); 4370/223926 objects misplaced (1.952%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T18:50:44.770 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:44 smithi098 bash[76817]: cluster 2023-09-23T18:50:44.710969+0000 mon.a (mon.0) 1345 : cluster 0 osdmap e254: 8 total, 8 up, 8 in 2023-09-23T18:50:44.770 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:44 smithi098 bash[79516]: cluster 2023-09-23T18:50:43.711290+0000 mon.a (mon.0) 1344 : cluster 0 osdmap e253: 8 total, 8 up, 8 in 2023-09-23T18:50:44.770 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:44 smithi098 bash[79516]: cluster 2023-09-23T18:50:43.713315+0000 osd.6 (osd.6) 212 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T18:50:44.770 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:44 smithi098 bash[79516]: cluster 2023-09-23T18:50:43.716380+0000 osd.5 (osd.5) 152 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T18:50:44.771 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:44 smithi098 bash[79516]: cluster 2023-09-23T18:50:43.717070+0000 osd.7 (osd.7) 177 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T18:50:44.771 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:44 smithi098 bash[79516]: cluster 2023-09-23T18:50:44.057940+0000 osd.4 (osd.4) 248 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T18:50:44.771 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:44 smithi098 bash[79516]: cluster 2023-09-23T18:50:44.063871+0000 osd.4 (osd.4) 249 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T18:50:44.771 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:44 smithi098 bash[79516]: cluster 2023-09-23T18:50:44.124073+0000 mgr.y (mgr.26054) 981 : cluster 0 pgmap v1044: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 49861/223926 objects degraded (22.267%); 4370/223926 objects misplaced (1.952%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T18:50:44.771 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:44 smithi098 bash[79516]: cluster 2023-09-23T18:50:44.710969+0000 mon.a (mon.0) 1345 : cluster 0 osdmap e254: 8 total, 8 up, 8 in 2023-09-23T18:50:44.782 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:44.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:44.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:44.884 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:44.896 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:44 smithi173 bash[59422]: cluster 2023-09-23T18:50:43.711290+0000 mon.a (mon.0) 1344 : cluster 0 osdmap e253: 8 total, 8 up, 8 in 2023-09-23T18:50:44.896 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:44 smithi173 bash[59422]: cluster 2023-09-23T18:50:43.713315+0000 osd.6 (osd.6) 212 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T18:50:44.896 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:44 smithi173 bash[59422]: cluster 2023-09-23T18:50:43.716380+0000 osd.5 (osd.5) 152 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T18:50:44.896 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:44 smithi173 bash[59422]: cluster 2023-09-23T18:50:43.717070+0000 osd.7 (osd.7) 177 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T18:50:44.896 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:44 smithi173 bash[59422]: cluster 2023-09-23T18:50:44.057940+0000 osd.4 (osd.4) 248 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T18:50:44.896 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:44 smithi173 bash[59422]: cluster 2023-09-23T18:50:44.063871+0000 osd.4 (osd.4) 249 : cluster 3 failed to encode map e253 with expected crc 2023-09-23T18:50:44.896 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:44 smithi173 bash[59422]: cluster 2023-09-23T18:50:44.124073+0000 mgr.y (mgr.26054) 981 : cluster 0 pgmap v1044: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 49861/223926 objects degraded (22.267%); 4370/223926 objects misplaced (1.952%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T18:50:44.897 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:44 smithi173 bash[59422]: cluster 2023-09-23T18:50:44.710969+0000 mon.a (mon.0) 1345 : cluster 0 osdmap e254: 8 total, 8 up, 8 in 2023-09-23T18:50:44.914 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:44.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:44.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:45.018 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:45.046 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:45.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:45.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:45.149 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:45.188 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:45.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:45.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:45.295 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:45.319 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T18:50:45.320 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 0.05 2023-09-23T18:50:45.323 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:50:45.359 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:45.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:45.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:45.464 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:45.488 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:45.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:45.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:45.591 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:45.625 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:45.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:45.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:45.730 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:45.756 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:45.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:45.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:45.861 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:45.888 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:45.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:45.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:45.991 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:46.000 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:45 smithi173 bash[59422]: cluster 2023-09-23T18:50:44.710337+0000 osd.5 (osd.5) 153 : cluster 3 failed to encode map e254 with expected crc 2023-09-23T18:50:46.000 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:45 smithi173 bash[59422]: cluster 2023-09-23T18:50:44.974898+0000 osd.4 (osd.4) 250 : cluster 3 failed to encode map e254 with expected crc 2023-09-23T18:50:46.000 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:45 smithi173 bash[59422]: cluster 2023-09-23T18:50:45.160988+0000 mon.a (mon.0) 1346 : cluster 3 Health check update: Degraded data redundancy: 49861/223926 objects degraded (22.267%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:50:46.017 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:46.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:46.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:46.087 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:45 smithi098 bash[79516]: cluster 2023-09-23T18:50:44.710337+0000 osd.5 (osd.5) 153 : cluster 3 failed to encode map e254 with expected crc 2023-09-23T18:50:46.087 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:45 smithi098 bash[79516]: cluster 2023-09-23T18:50:44.974898+0000 osd.4 (osd.4) 250 : cluster 3 failed to encode map e254 with expected crc 2023-09-23T18:50:46.087 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:45 smithi098 bash[79516]: cluster 2023-09-23T18:50:45.160988+0000 mon.a (mon.0) 1346 : cluster 3 Health check update: Degraded data redundancy: 49861/223926 objects degraded (22.267%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:50:46.087 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:50:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:50:46] "GET /metrics HTTP/1.1" 200 36620 "" "Prometheus/2.33.4" 2023-09-23T18:50:46.088 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:45 smithi098 bash[76817]: cluster 2023-09-23T18:50:44.710337+0000 osd.5 (osd.5) 153 : cluster 3 failed to encode map e254 with expected crc 2023-09-23T18:50:46.088 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:45 smithi098 bash[76817]: cluster 2023-09-23T18:50:44.974898+0000 osd.4 (osd.4) 250 : cluster 3 failed to encode map e254 with expected crc 2023-09-23T18:50:46.088 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:45 smithi098 bash[76817]: cluster 2023-09-23T18:50:45.160988+0000 mon.a (mon.0) 1346 : cluster 3 Health check update: Degraded data redundancy: 49861/223926 objects degraded (22.267%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:50:46.119 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:46.146 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:46.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:46.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:46.250 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:46.276 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:46.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:46.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:46.381 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:46.419 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:46.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:46.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:46.524 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:46.549 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:46.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:46.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:46.654 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:46.679 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:46.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:46.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:46.781 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:46.793 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:46 smithi173 bash[59422]: cluster 2023-09-23T18:50:44.975033+0000 osd.6 (osd.6) 213 : cluster 3 failed to encode map e254 with expected crc 2023-09-23T18:50:46.793 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:46 smithi173 bash[59422]: cluster 2023-09-23T18:50:46.124842+0000 mgr.y (mgr.26054) 982 : cluster 0 pgmap v1046: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 427 op/s; 49869/223440 objects degraded (22.319%); 4370/223440 objects misplaced (1.956%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T18:50:46.814 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:46.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:46.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:46.917 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:46.950 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:46.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:46.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:47.061 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:47.070 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:46 smithi098 bash[79516]: cluster 2023-09-23T18:50:44.975033+0000 osd.6 (osd.6) 213 : cluster 3 failed to encode map e254 with expected crc 2023-09-23T18:50:47.070 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:46 smithi098 bash[79516]: cluster 2023-09-23T18:50:46.124842+0000 mgr.y (mgr.26054) 982 : cluster 0 pgmap v1046: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 427 op/s; 49869/223440 objects degraded (22.319%); 4370/223440 objects misplaced (1.956%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T18:50:47.070 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:46 smithi098 bash[76817]: cluster 2023-09-23T18:50:44.975033+0000 osd.6 (osd.6) 213 : cluster 3 failed to encode map e254 with expected crc 2023-09-23T18:50:47.070 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:46 smithi098 bash[76817]: cluster 2023-09-23T18:50:46.124842+0000 mgr.y (mgr.26054) 982 : cluster 0 pgmap v1046: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 427 op/s; 49869/223440 objects degraded (22.319%); 4370/223440 objects misplaced (1.956%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T18:50:47.103 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:47.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:47.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:47.212 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:47.251 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:47.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:47.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:47.324 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:50:47.354 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:47.392 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:47.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:47.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:47.495 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:47.523 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:47.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:47.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:47.628 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:47.663 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:47.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:47.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:47.768 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:47.803 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:47.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:47.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:47.826 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:50:47.907 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:47.933 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:47.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:47.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:48.036 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:48.065 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:48.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:48.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:48.167 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:48.192 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:48.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:48.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:48.296 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:48.340 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:48.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:48.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:48.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:48 smithi098 bash[76817]: cluster 2023-09-23T18:50:48.125589+0000 mgr.y (mgr.26054) 983 : cluster 0 pgmap v1047: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 326 op/s; 56274/220545 objects degraded (25.516%); 4207/220545 objects misplaced (1.908%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T18:50:48.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:48 smithi098 bash[79516]: cluster 2023-09-23T18:50:48.125589+0000 mgr.y (mgr.26054) 983 : cluster 0 pgmap v1047: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 326 op/s; 56274/220545 objects degraded (25.516%); 4207/220545 objects misplaced (1.908%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T18:50:48.446 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:48.473 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:48.475 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:48 smithi173 bash[59422]: cluster 2023-09-23T18:50:48.125589+0000 mgr.y (mgr.26054) 983 : cluster 0 pgmap v1047: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 326 op/s; 56274/220545 objects degraded (25.516%); 4207/220545 objects misplaced (1.908%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T18:50:48.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:48.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:48.576 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:48.598 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:48.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:48.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:48.704 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:48.740 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:48.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:48.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:48.849 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:48.875 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:48.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:48.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:48.979 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:49.009 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:49.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:49.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:49.112 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:49.146 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:49.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:49.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:49.251 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:49.304 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:49.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:49.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:49.411 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:49.436 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:49.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:49.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:49.540 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:49.575 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:49.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:49.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:49.679 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:49.710 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:49.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:49.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:49.815 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:49.839 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:49.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:49.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:49.866 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:50:49.876 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:50:49.948 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:49.988 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:49.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:49.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:50.092 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:50.125 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:50.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:50.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:50.218 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:50 smithi173 bash[59422]: cluster 2023-09-23T18:50:50.130918+0000 mgr.y (mgr.26054) 984 : cluster 0 pgmap v1048: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 288 op/s; 56274/219087 objects degraded (25.686%); 4207/219087 objects misplaced (1.920%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T18:50:50.218 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:50 smithi173 bash[59422]: cluster 2023-09-23T18:50:50.162150+0000 mon.a (mon.0) 1347 : cluster 3 Health check update: Degraded data redundancy: 56274/220545 objects degraded (25.516%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:50:50.229 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:50.259 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:50.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:50.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:50.366 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:50.403 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:50.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:50.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:50.508 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:50.534 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:50.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:50.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:50.636 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:50.645 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:50 smithi098 bash[76817]: cluster 2023-09-23T18:50:50.130918+0000 mgr.y (mgr.26054) 984 : cluster 0 pgmap v1048: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 288 op/s; 56274/219087 objects degraded (25.686%); 4207/219087 objects misplaced (1.920%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T18:50:50.645 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:50 smithi098 bash[76817]: cluster 2023-09-23T18:50:50.162150+0000 mon.a (mon.0) 1347 : cluster 3 Health check update: Degraded data redundancy: 56274/220545 objects degraded (25.516%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:50:50.646 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:50 smithi098 bash[79516]: cluster 2023-09-23T18:50:50.130918+0000 mgr.y (mgr.26054) 984 : cluster 0 pgmap v1048: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 288 op/s; 56274/219087 objects degraded (25.686%); 4207/219087 objects misplaced (1.920%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T18:50:50.646 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:50 smithi098 bash[79516]: cluster 2023-09-23T18:50:50.162150+0000 mon.a (mon.0) 1347 : cluster 3 Health check update: Degraded data redundancy: 56274/220545 objects degraded (25.516%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:50:50.673 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:50.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:50.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:50.779 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:50.818 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:50.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:50.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:50.922 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:50.963 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:50.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:50.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:51.068 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:51.106 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:51.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:51.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:51.211 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:51.247 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:51.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:51.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:51.351 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:51.385 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:51.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:51.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:51.491 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:51.512 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:51.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:51.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:51.617 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:51.643 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:51.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:51.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:51.748 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:51.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:51.791 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:51.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:51.892 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:51.914 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:50:51.922 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:51.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:51.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:52.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:52.047 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:52.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:52.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:52.149 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:52.181 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:52.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:52.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:52.287 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:52.296 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:52 smithi098 bash[76817]: cluster 2023-09-23T18:50:52.131657+0000 mgr.y (mgr.26054) 985 : cluster 0 pgmap v1049: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 415 op/s; 56274/215013 objects degraded (26.172%); 4207/215013 objects misplaced (1.957%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:50:52.296 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:52 smithi098 bash[79516]: cluster 2023-09-23T18:50:52.131657+0000 mgr.y (mgr.26054) 985 : cluster 0 pgmap v1049: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 415 op/s; 56274/215013 objects degraded (26.172%); 4207/215013 objects misplaced (1.957%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:50:52.322 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:52.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:52.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:52.426 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:52.450 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:52.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:52.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:52.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:52 smithi173 bash[59422]: cluster 2023-09-23T18:50:52.131657+0000 mgr.y (mgr.26054) 985 : cluster 0 pgmap v1049: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 415 op/s; 56274/215013 objects degraded (26.172%); 4207/215013 objects misplaced (1.957%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:50:52.563 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:52.590 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:52.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:52.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:52.693 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:52.740 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:52.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:52.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:52.845 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:52.871 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:52.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:52.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:52.976 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:53.009 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:53.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:53.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:53.113 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:53.155 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:53.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:53.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:53.261 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:53.287 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:53.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:53.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:53.393 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:53.435 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:53.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:53.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:53.540 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:53.566 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:53.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:53.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:53.672 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:53.707 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:53.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:53.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:53.811 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:53.837 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:53.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:53.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:53.948 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:53.984 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:53.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:53.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:54.088 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:54.119 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:54.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:54.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:54.223 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:54 smithi098 bash[79516]: cluster 2023-09-23T18:50:54.132396+0000 mgr.y (mgr.26054) 986 : cluster 0 pgmap v1050: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 56279/211899 objects degraded (26.559%); 4113/211899 objects misplaced (1.941%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T18:50:54.224 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:54 smithi098 bash[76817]: cluster 2023-09-23T18:50:54.132396+0000 mgr.y (mgr.26054) 986 : cluster 0 pgmap v1050: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 56279/211899 objects degraded (26.559%); 4113/211899 objects misplaced (1.941%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T18:50:54.228 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:54.268 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:54.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:54.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:54.372 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:54.387 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:54 smithi173 bash[59422]: cluster 2023-09-23T18:50:54.132396+0000 mgr.y (mgr.26054) 986 : cluster 0 pgmap v1050: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 56279/211899 objects degraded (26.559%); 4113/211899 objects misplaced (1.941%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T18:50:54.400 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:54.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:54.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:54.505 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:54.532 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:54.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:54.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:54.636 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:54.681 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:54.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:54.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:54.788 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:54.810 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:54.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:54.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:54.913 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:54.945 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:54.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:54.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:55.047 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:55.082 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:55.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:55.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:55.189 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:55.217 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:55.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:55.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:55.320 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:55.346 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:55.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:55.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:55.449 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:55.482 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:55.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:55.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:55.586 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:55.633 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:55.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:55.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:55.736 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:55.743 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:55 smithi173 bash[59422]: audit 2023-09-23T18:50:54.627550+0000 mon.a (mon.0) 1348 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:50:55.744 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:55 smithi173 bash[59422]: audit 2023-09-23T18:50:54.632319+0000 mon.a (mon.0) 1349 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:50:55.744 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:55 smithi173 bash[59422]: cluster 2023-09-23T18:50:55.163181+0000 mon.a (mon.0) 1350 : cluster 3 Health check update: Degraded data redundancy: 56279/211899 objects degraded (26.559%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:50:55.772 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:55.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:55.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:55.874 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:55.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:55.903 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:55.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:55.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:55 smithi098 bash[76817]: audit 2023-09-23T18:50:54.627550+0000 mon.a (mon.0) 1348 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:50:55.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:55 smithi098 bash[76817]: audit 2023-09-23T18:50:54.632319+0000 mon.a (mon.0) 1349 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:50:55.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:55 smithi098 bash[76817]: cluster 2023-09-23T18:50:55.163181+0000 mon.a (mon.0) 1350 : cluster 3 Health check update: Degraded data redundancy: 56279/211899 objects degraded (26.559%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:50:55.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:55 smithi098 bash[79516]: audit 2023-09-23T18:50:54.627550+0000 mon.a (mon.0) 1348 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:50:55.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:55 smithi098 bash[79516]: audit 2023-09-23T18:50:54.632319+0000 mon.a (mon.0) 1349 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:50:55.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:55 smithi098 bash[79516]: cluster 2023-09-23T18:50:55.163181+0000 mon.a (mon.0) 1350 : cluster 3 Health check update: Degraded data redundancy: 56279/211899 objects degraded (26.559%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:50:56.003 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:56.047 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:56.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:56.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:56.150 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:56.188 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:56.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:56.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:56.293 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:56.302 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:50:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:50:56] "GET /metrics HTTP/1.1" 200 36607 "" "Prometheus/2.33.4" 2023-09-23T18:50:56.318 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:56.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:56.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:56.423 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:56.448 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:56.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:56.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:56.556 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:56.585 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:56.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:56.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:56.637 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:56 smithi098 bash[76817]: cluster 2023-09-23T18:50:56.135137+0000 mgr.y (mgr.26054) 987 : cluster 0 pgmap v1051: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 377 op/s; 56279/210969 objects degraded (26.676%); 4113/210969 objects misplaced (1.950%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T18:50:56.687 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:56.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:56 smithi098 bash[79516]: cluster 2023-09-23T18:50:56.135137+0000 mgr.y (mgr.26054) 987 : cluster 0 pgmap v1051: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 377 op/s; 56279/210969 objects degraded (26.676%); 4113/210969 objects misplaced (1.950%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T18:50:56.697 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:56 smithi173 bash[59422]: cluster 2023-09-23T18:50:56.135137+0000 mgr.y (mgr.26054) 987 : cluster 0 pgmap v1051: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 377 op/s; 56279/210969 objects degraded (26.676%); 4113/210969 objects misplaced (1.950%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T18:50:56.721 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:56.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:56.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:56.825 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:56.862 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:56.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:56.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:56.970 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:57.005 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:57.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:57.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:57.111 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:57.136 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:57.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:57.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:57.239 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:57.265 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:50:57.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:57.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:50:57.369 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:57.393 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:50:57.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:57.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:50:57.500 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:57.552 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:57.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:57.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:57.658 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:57.684 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:57.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:57.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:57.789 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:57.819 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:57.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:57.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:57.924 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:57.958 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:57.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:57.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:58.063 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:58.087 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:58.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:58.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:58.192 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:58.215 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:58.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:58.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:58.318 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:58.355 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:58.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:58.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:58.461 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:58.487 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:58.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:58.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:58.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:50:58 smithi173 bash[59422]: cluster 2023-09-23T18:50:58.135862+0000 mgr.y (mgr.26054) 988 : cluster 0 pgmap v1052: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 470 op/s; 56325/206475 objects degraded (27.279%); 3977/206475 objects misplaced (1.926%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T18:50:58.534 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:50:58 smithi098 bash[76817]: cluster 2023-09-23T18:50:58.135862+0000 mgr.y (mgr.26054) 988 : cluster 0 pgmap v1052: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 470 op/s; 56325/206475 objects degraded (27.279%); 3977/206475 objects misplaced (1.926%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T18:50:58.534 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:50:58 smithi098 bash[79516]: cluster 2023-09-23T18:50:58.135862+0000 mgr.y (mgr.26054) 988 : cluster 0 pgmap v1052: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 470 op/s; 56325/206475 objects degraded (27.279%); 3977/206475 objects misplaced (1.926%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T18:50:58.591 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:58.622 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:50:58.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:58.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:50:58.729 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:58.771 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:58.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:58.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:58.875 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:58.929 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:50:58.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:58.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:50:59.035 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:59.057 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:59.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:59.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:59.160 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:59.189 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:59.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:59.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:59.294 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:59.337 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:59.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:59.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:59.441 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:59.473 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:59.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:59.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:50:59.579 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:59.603 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:50:59.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:59.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:50:59.706 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:59.733 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:50:59.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:59.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:50:59.836 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:59.862 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:50:59.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:59.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:50:59.970 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:59.993 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:50:59.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:50:59.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:00.097 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:00.134 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:00.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:00.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:00.241 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:00.251 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:00 smithi173 bash[59422]: cluster 2023-09-23T18:51:00.136563+0000 mgr.y (mgr.26054) 989 : cluster 0 pgmap v1053: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 439 op/s; 56325/204702 objects degraded (27.516%); 3977/204702 objects misplaced (1.943%); 954 KiB/s, 14 objects/s recovering 2023-09-23T18:51:00.251 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:00 smithi173 bash[59422]: cluster 2023-09-23T18:51:00.164325+0000 mon.a (mon.0) 1351 : cluster 3 Health check update: Degraded data redundancy: 56325/206475 objects degraded (27.279%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:00.265 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:00.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:00.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:00.371 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:00.386 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:00 smithi098 bash[76817]: cluster 2023-09-23T18:51:00.136563+0000 mgr.y (mgr.26054) 989 : cluster 0 pgmap v1053: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 439 op/s; 56325/204702 objects degraded (27.516%); 3977/204702 objects misplaced (1.943%); 954 KiB/s, 14 objects/s recovering 2023-09-23T18:51:00.387 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:00 smithi098 bash[76817]: cluster 2023-09-23T18:51:00.164325+0000 mon.a (mon.0) 1351 : cluster 3 Health check update: Degraded data redundancy: 56325/206475 objects degraded (27.279%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:00.388 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:00 smithi098 bash[79516]: cluster 2023-09-23T18:51:00.136563+0000 mgr.y (mgr.26054) 989 : cluster 0 pgmap v1053: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 439 op/s; 56325/204702 objects degraded (27.516%); 3977/204702 objects misplaced (1.943%); 954 KiB/s, 14 objects/s recovering 2023-09-23T18:51:00.388 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:00 smithi098 bash[79516]: cluster 2023-09-23T18:51:00.164325+0000 mon.a (mon.0) 1351 : cluster 3 Health check update: Degraded data redundancy: 56325/206475 objects degraded (27.279%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:00.400 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:00.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:00.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:00.510 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:00.540 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:00.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:00.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:00.644 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:00.675 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:00.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:00.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:00.780 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:00.807 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:00.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:00.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:00.911 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:00.937 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:00.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:00.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:01.040 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:01.070 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:01.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:01.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:01.173 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:01.199 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:01.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:01.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:01.302 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:01.332 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:01.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:01.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:01.440 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:01.463 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:01.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:01.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:01.567 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:01.604 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:01.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:01.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:01.706 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:01.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:01.754 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:01.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:01.855 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:01.888 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:01.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:01.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:02.001 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:02.039 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:02.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:02.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:02.143 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:02.163 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:02.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:02.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:02.266 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:02.306 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:02.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:02.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:02.411 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:02.436 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:02.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:02.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:02.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:02 smithi173 bash[59422]: cluster 2023-09-23T18:51:02.137337+0000 mgr.y (mgr.26054) 990 : cluster 0 pgmap v1054: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 56325/200646 objects degraded (28.072%); 3977/200646 objects misplaced (1.982%); 954 KiB/s, 14 objects/s recovering 2023-09-23T18:51:02.539 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:02.573 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:02.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:02.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:02.589 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:02 smithi098 bash[76817]: cluster 2023-09-23T18:51:02.137337+0000 mgr.y (mgr.26054) 990 : cluster 0 pgmap v1054: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 56325/200646 objects degraded (28.072%); 3977/200646 objects misplaced (1.982%); 954 KiB/s, 14 objects/s recovering 2023-09-23T18:51:02.589 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:02 smithi098 bash[79516]: cluster 2023-09-23T18:51:02.137337+0000 mgr.y (mgr.26054) 990 : cluster 0 pgmap v1054: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 56325/200646 objects degraded (28.072%); 3977/200646 objects misplaced (1.982%); 954 KiB/s, 14 objects/s recovering 2023-09-23T18:51:02.676 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:02.710 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:02.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:02.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:02.816 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:02.844 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:02.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:02.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:02.953 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:02.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:02.978 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:02.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:03.079 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:03.107 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:03.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:03.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:03.213 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:03.244 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:03.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:03.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:03.350 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:03.386 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:03.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:03.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:03.491 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:03.516 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:03.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:03.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:03.619 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:03.663 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:03.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:03.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:03.770 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:03.798 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:03.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:03.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:03.906 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:03.930 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:03.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:03.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:04.037 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:04.063 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:04.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:04.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:04.170 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:04.193 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:04.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:04.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:04.297 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:04.319 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:04.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:04.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:04.423 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:04.433 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:04 smithi173 bash[59422]: cluster 2023-09-23T18:51:04.138091+0000 mgr.y (mgr.26054) 991 : cluster 0 pgmap v1055: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 56414/197493 objects degraded (28.565%); 3807/197493 objects misplaced (1.928%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T18:51:04.453 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:04.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:04.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:04.557 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:04.581 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:04.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:04.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:04.688 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:04.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:04 smithi098 bash[79516]: cluster 2023-09-23T18:51:04.138091+0000 mgr.y (mgr.26054) 991 : cluster 0 pgmap v1055: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 56414/197493 objects degraded (28.565%); 3807/197493 objects misplaced (1.928%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T18:51:04.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:04 smithi098 bash[76817]: cluster 2023-09-23T18:51:04.138091+0000 mgr.y (mgr.26054) 991 : cluster 0 pgmap v1055: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 56414/197493 objects degraded (28.565%); 3807/197493 objects misplaced (1.928%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T18:51:04.723 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:04.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:04.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:04.829 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:04.851 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:04.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:04.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:04.954 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:04.992 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:04.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:04.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:05.099 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:05.129 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:05.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:05.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:05.226 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:05 smithi098 bash[79516]: cluster 2023-09-23T18:51:05.165495+0000 mon.a (mon.0) 1352 : cluster 3 Health check update: Degraded data redundancy: 56414/197493 objects degraded (28.565%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:05.227 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:05 smithi098 bash[76817]: cluster 2023-09-23T18:51:05.165495+0000 mon.a (mon.0) 1352 : cluster 3 Health check update: Degraded data redundancy: 56414/197493 objects degraded (28.565%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:05.233 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:05.256 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:05.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:05.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:05.364 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:05.374 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:05 smithi173 bash[59422]: cluster 2023-09-23T18:51:05.165495+0000 mon.a (mon.0) 1352 : cluster 3 Health check update: Degraded data redundancy: 56414/197493 objects degraded (28.565%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:05.392 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:05.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:05.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:05.498 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:05.532 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:05.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:05.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:05.634 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:05.667 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:05.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:05.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:05.774 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:05.814 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:05.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:05.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:05.922 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:05.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:05.975 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:05.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:06.076 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:06.086 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:51:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:51:06] "GET /metrics HTTP/1.1" 200 36607 "" "Prometheus/2.33.4" 2023-09-23T18:51:06.102 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:06.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:06.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:06.205 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:06.232 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:06.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:06.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:06.335 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:06.344 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:06 smithi173 bash[59422]: cluster 2023-09-23T18:51:06.138874+0000 mgr.y (mgr.26054) 992 : cluster 0 pgmap v1056: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 422 op/s; 56414/196668 objects degraded (28.685%); 3807/196668 objects misplaced (1.936%); 912 KiB/s, 14 objects/s recovering 2023-09-23T18:51:06.362 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:06.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:06.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:06.368 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:06 smithi098 bash[79516]: cluster 2023-09-23T18:51:06.138874+0000 mgr.y (mgr.26054) 992 : cluster 0 pgmap v1056: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 422 op/s; 56414/196668 objects degraded (28.685%); 3807/196668 objects misplaced (1.936%); 912 KiB/s, 14 objects/s recovering 2023-09-23T18:51:06.368 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:06 smithi098 bash[76817]: cluster 2023-09-23T18:51:06.138874+0000 mgr.y (mgr.26054) 992 : cluster 0 pgmap v1056: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 422 op/s; 56414/196668 objects degraded (28.685%); 3807/196668 objects misplaced (1.936%); 912 KiB/s, 14 objects/s recovering 2023-09-23T18:51:06.465 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:06.504 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:06.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:06.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:06.610 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:06.642 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:06.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:06.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:06.745 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:06.776 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:06.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:06.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:06.884 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:06.890 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:51:06.890 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [ 2023-09-23T18:51:06.890 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.890 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.35412.0:143352 8.f e254/246 8:fedfaabd:::benchmark_data_smithi098_176514_object39874:head v 254'9000, mlcod=0'0)", 2023-09-23T18:51:06.890 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:51:06.868654+0000", 2023-09-23T18:51:06.890 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.016941583, 2023-09-23T18:51:06.890 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.016964491000000002, 2023-09-23T18:51:06.890 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:06.890 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T18:51:06.891 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:06.891 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.891 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:06.891 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.868654+0000", 2023-09-23T18:51:06.891 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:06.891 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.891 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.891 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:06.891 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.868654+0000", 2023-09-23T18:51:06.891 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:06.891 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.891 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.892 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:06.892 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.868653+0000", 2023-09-23T18:51:06.892 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:51:06.892 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.892 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.892 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:06.892 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.868660+0000", 2023-09-23T18:51:06.892 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.7780000000000002e-06 2023-09-23T18:51:06.892 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.892 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.892 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:06.892 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.868673+0000", 2023-09-23T18:51:06.893 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3210000000000001e-05 2023-09-23T18:51:06.893 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.893 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.893 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:06.893 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.868679+0000", 2023-09-23T18:51:06.893 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.6370000000000002e-06 2023-09-23T18:51:06.893 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.893 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.893 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:06.893 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.868809+0000", 2023-09-23T18:51:06.893 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00012993100000000001 2023-09-23T18:51:06.893 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.893 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.894 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:51:06.894 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.868859+0000", 2023-09-23T18:51:06.894 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.9341000000000001e-05 2023-09-23T18:51:06.894 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:06.894 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:06.894 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:06.894 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.894 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.894 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:143356 8.7 8:e786d912:::benchmark_data_smithi098_176514_object39878:head [delete] snapc 0=[] ondisk+write+known_if_redirected e254)", 2023-09-23T18:51:06.894 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:51:06.871373+0000", 2023-09-23T18:51:06.894 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.014221851000000001, 2023-09-23T18:51:06.894 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.014290146, 2023-09-23T18:51:06.895 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:06.895 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:51:06.895 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:51:06.895 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:51:06.895 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:51:06.895 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 143356 2023-09-23T18:51:06.895 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.895 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:06.895 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.895 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:06.895 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.871373+0000", 2023-09-23T18:51:06.895 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:06.896 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.896 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.896 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:06.896 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.871373+0000", 2023-09-23T18:51:06.896 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:06.896 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.896 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.896 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:06.896 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.871372+0000", 2023-09-23T18:51:06.896 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:51:06.896 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.896 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.897 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:06.897 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.871379+0000", 2023-09-23T18:51:06.897 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.7129999999999999e-06 2023-09-23T18:51:06.897 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.897 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.897 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:06.897 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.871389+0000", 2023-09-23T18:51:06.897 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0767999999999999e-05 2023-09-23T18:51:06.897 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.897 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.897 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:06.897 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.871396+0000", 2023-09-23T18:51:06.898 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.0099999999999998e-06 2023-09-23T18:51:06.898 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.898 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.898 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:06.898 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.878559+0000", 2023-09-23T18:51:06.898 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0071623499999999996 2023-09-23T18:51:06.898 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.898 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.898 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:51:06.898 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.881295+0000", 2023-09-23T18:51:06.898 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0027363380000000001 2023-09-23T18:51:06.899 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.899 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.899 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 3,6", 2023-09-23T18:51:06.899 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.881688+0000", 2023-09-23T18:51:06.899 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00039258899999999999 2023-09-23T18:51:06.899 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:06.899 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:06.899 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:06.899 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.899 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.899 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.35412.0:143358 8.f e254/246)", 2023-09-23T18:51:06.899 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:51:06.882956+0000", 2023-09-23T18:51:06.899 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.0026388169999999999, 2023-09-23T18:51:06.900 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0027513559999999999, 2023-09-23T18:51:06.900 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:06.900 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "queued for pg", 2023-09-23T18:51:06.900 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:06.900 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.900 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:06.900 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.882956+0000", 2023-09-23T18:51:06.900 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:06.900 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.900 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.900 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:06.900 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.882956+0000", 2023-09-23T18:51:06.901 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:06.901 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.901 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.901 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:06.901 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.882955+0000", 2023-09-23T18:51:06.901 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:51:06.901 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.901 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.901 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:06.901 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.882963+0000", 2023-09-23T18:51:06.901 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.3989999999999998e-06 2023-09-23T18:51:06.901 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.901 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.902 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:06.902 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.882978+0000", 2023-09-23T18:51:06.902 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.4942e-05 2023-09-23T18:51:06.902 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.902 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.902 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:06.902 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.882986+0000", 2023-09-23T18:51:06.902 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.225e-06 2023-09-23T18:51:06.902 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:06.902 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:06.902 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:06.902 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.903 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.903 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop_reply(client.35412.0:143347 8.1 e254/246 ondisk, result = 0)", 2023-09-23T18:51:06.903 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:51:06.885354+0000", 2023-09-23T18:51:06.903 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.00024075600000000001, 2023-09-23T18:51:06.903 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00038088199999999998, 2023-09-23T18:51:06.903 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:06.903 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T18:51:06.903 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:06.903 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.903 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:06.903 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.885354+0000", 2023-09-23T18:51:06.903 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:06.903 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.904 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.904 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:06.904 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.885354+0000", 2023-09-23T18:51:06.904 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:06.904 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.904 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.904 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:06.904 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.885353+0000", 2023-09-23T18:51:06.904 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:51:06.904 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.904 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.904 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:06.905 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.885360+0000", 2023-09-23T18:51:06.905 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.3810000000000004e-06 2023-09-23T18:51:06.905 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.905 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.905 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:06.905 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.885377+0000", 2023-09-23T18:51:06.905 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.7351999999999999e-05 2023-09-23T18:51:06.905 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.905 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.905 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:06.905 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.885385+0000", 2023-09-23T18:51:06.905 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.2600000000000005e-06 2023-09-23T18:51:06.905 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.906 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.906 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:06.906 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.885550+0000", 2023-09-23T18:51:06.906 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00016469600000000001 2023-09-23T18:51:06.906 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.906 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.906 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:51:06.906 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.885586+0000", 2023-09-23T18:51:06.906 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.5936000000000003e-05 2023-09-23T18:51:06.906 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:06.906 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:06.906 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:06.907 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.907 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.907 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:143345 8.7 8:ee89298d:::benchmark_data_smithi098_176514_object39867:head [delete] snapc 0=[] ondisk+write+known_if_redirected e254)", 2023-09-23T18:51:06.907 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:51:06.847728+0000", 2023-09-23T18:51:06.907 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.037867576, 2023-09-23T18:51:06.907 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.038044758999999997, 2023-09-23T18:51:06.907 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:06.907 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:51:06.907 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:51:06.907 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:51:06.907 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:51:06.907 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 143345 2023-09-23T18:51:06.908 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.908 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:06.908 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.908 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:06.908 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.847728+0000", 2023-09-23T18:51:06.908 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:06.908 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.908 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.908 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:06.908 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.847728+0000", 2023-09-23T18:51:06.908 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:06.908 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.908 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.909 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:06.909 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.847726+0000", 2023-09-23T18:51:06.909 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T18:51:06.909 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.909 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.909 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:06.909 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.847735+0000", 2023-09-23T18:51:06.909 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.9509999999999995e-06 2023-09-23T18:51:06.909 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.909 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.909 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:06.909 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.847752+0000", 2023-09-23T18:51:06.910 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.7683999999999999e-05 2023-09-23T18:51:06.910 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.910 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.910 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:06.910 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.847761+0000", 2023-09-23T18:51:06.910 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.867e-06 2023-09-23T18:51:06.910 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.910 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.910 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:06.910 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.847903+0000", 2023-09-23T18:51:06.910 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00014196200000000001 2023-09-23T18:51:06.910 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.911 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.911 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:51:06.911 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.848239+0000", 2023-09-23T18:51:06.911 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00033548900000000001 2023-09-23T18:51:06.911 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.911 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.911 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 3,6", 2023-09-23T18:51:06.911 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.848619+0000", 2023-09-23T18:51:06.911 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00038073099999999998 2023-09-23T18:51:06.911 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.911 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.911 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:06.911 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.867898+0000", 2023-09-23T18:51:06.912 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.019278765 2023-09-23T18:51:06.912 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.912 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.912 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:06.912 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.868017+0000", 2023-09-23T18:51:06.912 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000118906 2023-09-23T18:51:06.912 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:06.912 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:06.912 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:06.912 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.912 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.912 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:143347 8.1 8:87741c0b:::benchmark_data_smithi098_176514_object39869:head [delete] snapc 0=[] ondisk+write+known_if_redirected e254)", 2023-09-23T18:51:06.913 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:51:06.852466+0000", 2023-09-23T18:51:06.913 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.033129178000000002, 2023-09-23T18:51:06.913 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.033355174000000001, 2023-09-23T18:51:06.913 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:06.913 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:51:06.913 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:51:06.913 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:51:06.913 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:51:06.913 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 143347 2023-09-23T18:51:06.913 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.913 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:06.914 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.914 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:06.914 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.852466+0000", 2023-09-23T18:51:06.914 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:06.914 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.914 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.914 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:06.914 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.852466+0000", 2023-09-23T18:51:06.914 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:06.914 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.914 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.914 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:06.915 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.852464+0000", 2023-09-23T18:51:06.915 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T18:51:06.915 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.915 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.915 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:06.915 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.852472+0000", 2023-09-23T18:51:06.915 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.1000000000000004e-06 2023-09-23T18:51:06.915 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.915 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.915 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:06.915 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.852485+0000", 2023-09-23T18:51:06.915 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3121e-05 2023-09-23T18:51:06.916 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.916 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.916 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:06.916 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.852493+0000", 2023-09-23T18:51:06.916 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.4229999999999999e-06 2023-09-23T18:51:06.916 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.916 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.916 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:06.916 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.852546+0000", 2023-09-23T18:51:06.916 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.304e-05 2023-09-23T18:51:06.916 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.916 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.917 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:51:06.917 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.852724+0000", 2023-09-23T18:51:06.917 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00017832500000000001 2023-09-23T18:51:06.917 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.917 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.917 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:51:06.917 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.853090+0000", 2023-09-23T18:51:06.917 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00036593500000000001 2023-09-23T18:51:06.917 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.917 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.917 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:06.917 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.867133+0000", 2023-09-23T18:51:06.917 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.014042689000000001 2023-09-23T18:51:06.918 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.918 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.918 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:51:06.918 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.870973+0000", 2023-09-23T18:51:06.918 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0038401469999999999 2023-09-23T18:51:06.918 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:06.918 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:06.918 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:06.918 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:06.885611+0000", 2023-09-23T18:51:06.918 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.014637542999999999 2023-09-23T18:51:06.918 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:06.918 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:06.919 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:06.919 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:06.919 INFO:teuthology.orchestra.run.smithi098.stdout: ], 2023-09-23T18:51:06.919 INFO:teuthology.orchestra.run.smithi098.stdout: "num_ops": 6 2023-09-23T18:51:06.919 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:51:06.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:06.920 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:06.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:07.021 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:07.094 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:07.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:07.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:07.197 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:07.225 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:07.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:07.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:07.328 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:07.351 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:07.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:07.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:07.460 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:07.488 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:07.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:07.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:07.596 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:07.618 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:07.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:07.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:07.721 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:07.754 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:07.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:07.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:07.858 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:07.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:07.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:07.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:07.989 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:08.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:08.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:08.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:08.119 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:08.144 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:08.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:08.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:08.226 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:08 smithi173 bash[59422]: cluster 2023-09-23T18:51:08.139656+0000 mgr.y (mgr.26054) 993 : cluster 0 pgmap v1057: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 56468/192165 objects degraded (29.385%); 3662/192165 objects misplaced (1.906%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T18:51:08.251 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:08.276 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:08.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:08.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:08.380 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:08.404 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:08.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:08.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:08.483 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:08 smithi098 bash[79516]: cluster 2023-09-23T18:51:08.139656+0000 mgr.y (mgr.26054) 993 : cluster 0 pgmap v1057: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 56468/192165 objects degraded (29.385%); 3662/192165 objects misplaced (1.906%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T18:51:08.485 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:08 smithi098 bash[76817]: cluster 2023-09-23T18:51:08.139656+0000 mgr.y (mgr.26054) 993 : cluster 0 pgmap v1057: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 56468/192165 objects degraded (29.385%); 3662/192165 objects misplaced (1.906%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T18:51:08.509 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:08.536 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:08.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:08.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:08.639 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:08.682 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:08.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:08.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:08.690 INFO:teuthology.orchestra.run.smithi098.stdout:moved 2 / 267 (0.749064%) 2023-09-23T18:51:08.691 INFO:teuthology.orchestra.run.smithi098.stdout:avg 33.375 2023-09-23T18:51:08.691 INFO:teuthology.orchestra.run.smithi098.stdout:stddev 6.6132 -> 6.55625 (expected baseline 5.40399) 2023-09-23T18:51:08.691 INFO:teuthology.orchestra.run.smithi098.stdout:min osd.5 with 19 -> 19 pgs (0.569288 -> 0.569288 * mean) 2023-09-23T18:51:08.691 INFO:teuthology.orchestra.run.smithi098.stdout:max osd.1 with 41 -> 41 pgs (1.22846 -> 1.22846 * mean) 2023-09-23T18:51:08.691 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:51:08.691 INFO:teuthology.orchestra.run.smithi098.stdout:oload 1000 2023-09-23T18:51:08.691 INFO:teuthology.orchestra.run.smithi098.stdout:max_change 0.05 2023-09-23T18:51:08.691 INFO:teuthology.orchestra.run.smithi098.stdout:max_change_osds 4 2023-09-23T18:51:08.691 INFO:teuthology.orchestra.run.smithi098.stdout:average_utilization 0.0249 2023-09-23T18:51:08.691 INFO:teuthology.orchestra.run.smithi098.stdout:overload_utilization 0.2491 2023-09-23T18:51:08.691 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3 weight 0.6815 -> 0.7315 2023-09-23T18:51:08.692 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7 weight 0.8539 -> 0.9039 2023-09-23T18:51:08.699 INFO:teuthology.orchestra.run.smithi098.stderr:no change 2023-09-23T18:51:08.787 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:08.821 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:08.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:08.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:08.924 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:08.942 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:51:08 smithi098 bash[73085]: debug 2023-09-23T18:51:08.681+0000 7f90a56ad700 -1 mgr.server reply reply (2) No such file or directory no change 2023-09-23T18:51:08.947 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:08.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:08.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:09.050 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:09.089 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:09.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:09.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:09.193 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:09.216 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:09.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:09.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:09.321 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:09.328 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:09 smithi173 bash[59422]: audit 2023-09-23T18:51:08.682674+0000 mgr.y (mgr.26054) 994 : audit 0 from='client.45446 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:51:09.343 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:09.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:09.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:09.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:09 smithi098 bash[76817]: audit 2023-09-23T18:51:08.682674+0000 mgr.y (mgr.26054) 994 : audit 0 from='client.45446 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:51:09.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:09 smithi098 bash[79516]: audit 2023-09-23T18:51:08.682674+0000 mgr.y (mgr.26054) 994 : audit 0 from='client.45446 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:51:09.449 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:09.478 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:09.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:09.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:09.586 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:09.613 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:09.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:09.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:09.717 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:09.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:09.749 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:09.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:09.850 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:09.874 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:09.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:09.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:09.977 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:10.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:10.041 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:10.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:10.142 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:10.178 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:10.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:10.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:10.286 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:10.316 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:10.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:10.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:10.419 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:10.431 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:10 smithi173 bash[59422]: audit 2023-09-23T18:51:09.623738+0000 mon.a (mon.0) 1353 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:51:10.431 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:10 smithi173 bash[59422]: cluster 2023-09-23T18:51:10.140227+0000 mgr.y (mgr.26054) 995 : cluster 0 pgmap v1058: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 56468/190224 objects degraded (29.685%); 3662/190224 objects misplaced (1.925%); 917 KiB/s, 14 objects/s recovering 2023-09-23T18:51:10.432 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:10 smithi173 bash[59422]: cluster 2023-09-23T18:51:10.166808+0000 mon.a (mon.0) 1354 : cluster 3 Health check update: Degraded data redundancy: 56468/192165 objects degraded (29.385%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:10.450 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:10.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:10.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:10.488 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:10 smithi098 bash[76817]: audit 2023-09-23T18:51:09.623738+0000 mon.a (mon.0) 1353 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:51:10.488 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:10 smithi098 bash[76817]: cluster 2023-09-23T18:51:10.140227+0000 mgr.y (mgr.26054) 995 : cluster 0 pgmap v1058: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 56468/190224 objects degraded (29.685%); 3662/190224 objects misplaced (1.925%); 917 KiB/s, 14 objects/s recovering 2023-09-23T18:51:10.488 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:10 smithi098 bash[76817]: cluster 2023-09-23T18:51:10.166808+0000 mon.a (mon.0) 1354 : cluster 3 Health check update: Degraded data redundancy: 56468/192165 objects degraded (29.385%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:10.489 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:10 smithi098 bash[79516]: audit 2023-09-23T18:51:09.623738+0000 mon.a (mon.0) 1353 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:51:10.489 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:10 smithi098 bash[79516]: cluster 2023-09-23T18:51:10.140227+0000 mgr.y (mgr.26054) 995 : cluster 0 pgmap v1058: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 56468/190224 objects degraded (29.685%); 3662/190224 objects misplaced (1.925%); 917 KiB/s, 14 objects/s recovering 2023-09-23T18:51:10.489 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:10 smithi098 bash[79516]: cluster 2023-09-23T18:51:10.166808+0000 mon.a (mon.0) 1354 : cluster 3 Health check update: Degraded data redundancy: 56468/192165 objects degraded (29.385%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:10.554 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:10.578 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:10.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:10.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:10.681 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:10.714 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:10.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:10.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:10.819 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:10.872 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:10.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:10.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:10.975 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:10.998 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:11.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:11.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:11.104 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:11.131 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:11.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:11.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:11.236 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:11.263 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:11.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:11.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:11.369 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:11.401 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:11.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:11.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:11.507 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:11.557 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:11.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:11.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:11.663 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:11.692 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:11.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:11.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:11.796 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:11.823 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:11.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:11.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:11.927 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:11.951 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:11.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:11.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:12.054 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:12.075 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:12.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:12.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:12.113 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T18:51:12.181 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:12.224 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:12.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:12.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:12.330 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:12.363 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:12 smithi098 bash[76817]: cluster 2023-09-23T18:51:12.141070+0000 mgr.y (mgr.26054) 996 : cluster 0 pgmap v1059: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 56468/186342 objects degraded (30.303%); 3662/186342 objects misplaced (1.965%); 917 KiB/s, 14 objects/s recovering 2023-09-23T18:51:12.364 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:12 smithi098 bash[79516]: cluster 2023-09-23T18:51:12.141070+0000 mgr.y (mgr.26054) 996 : cluster 0 pgmap v1059: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 56468/186342 objects degraded (30.303%); 3662/186342 objects misplaced (1.965%); 917 KiB/s, 14 objects/s recovering 2023-09-23T18:51:12.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:12.401 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:12.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:12.502 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:12.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:12 smithi173 bash[59422]: cluster 2023-09-23T18:51:12.141070+0000 mgr.y (mgr.26054) 996 : cluster 0 pgmap v1059: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 56468/186342 objects degraded (30.303%); 3662/186342 objects misplaced (1.965%); 917 KiB/s, 14 objects/s recovering 2023-09-23T18:51:12.541 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:12.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:12.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:12.647 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:12.672 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:12.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:12.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:12.775 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:12.804 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:12.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:12.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:12.908 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:12.935 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:12.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:12.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:13.045 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:13.083 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:51:13.088 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:13.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:13.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:13.127 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:51:13.192 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:51:13.198 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:13.239 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:13.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:13.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:13.265 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:51:13.304 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:51:13.335 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:51:13.346 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:13.374 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:51:13.387 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:13.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:13.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:13.421 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:51:13.424 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:51:13.424 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:51:13.424 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:51:13.424 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:51:13.424 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:51:13.424 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:51:13.425 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:51:13.425 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:51:13.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:13 smithi098 bash[76817]: audit 2023-09-23T18:51:12.994668+0000 mon.c (mon.2) 113 : audit 0 from='client.? 172.21.15.98:0/3039539887' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:51:13.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:13 smithi098 bash[79516]: audit 2023-09-23T18:51:12.994668+0000 mon.c (mon.2) 113 : audit 0 from='client.? 172.21.15.98:0/3039539887' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:51:13.490 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:13.499 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:13 smithi173 bash[59422]: audit 2023-09-23T18:51:12.994668+0000 mon.c (mon.2) 113 : audit 0 from='client.? 172.21.15.98:0/3039539887' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:51:13.520 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:13.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:13.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:13.624 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:13.670 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:13.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:13.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:13.778 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:13.819 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:13.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:13.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:13.930 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:13.961 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:13.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:13.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:14.074 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:14.108 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:14.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:14.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:14.214 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:14.253 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:14.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:14.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:14.265 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is set 2023-09-23T18:51:14.359 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:14.391 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:14.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:14.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:14.455 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:14 smithi098 bash[79516]: audit 2023-09-23T18:51:13.438115+0000 mon.c (mon.2) 114 : audit 1 from='client.? 172.21.15.98:0/1618468949' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:51:14.455 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:14 smithi098 bash[79516]: audit 2023-09-23T18:51:13.438740+0000 mon.a (mon.0) 1355 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:51:14.455 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:14 smithi098 bash[79516]: cluster 2023-09-23T18:51:14.141792+0000 mgr.y (mgr.26054) 997 : cluster 0 pgmap v1060: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 56511/183381 objects degraded (30.816%); 3530/183381 objects misplaced (1.925%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T18:51:14.455 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:14 smithi098 bash[76817]: audit 2023-09-23T18:51:13.438115+0000 mon.c (mon.2) 114 : audit 1 from='client.? 172.21.15.98:0/1618468949' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:51:14.455 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:14 smithi098 bash[76817]: audit 2023-09-23T18:51:13.438740+0000 mon.a (mon.0) 1355 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:51:14.456 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:14 smithi098 bash[76817]: cluster 2023-09-23T18:51:14.141792+0000 mgr.y (mgr.26054) 997 : cluster 0 pgmap v1060: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 56511/183381 objects degraded (30.816%); 3530/183381 objects misplaced (1.925%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T18:51:14.494 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:14.503 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:14 smithi173 bash[59422]: audit 2023-09-23T18:51:13.438115+0000 mon.c (mon.2) 114 : audit 1 from='client.? 172.21.15.98:0/1618468949' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:51:14.503 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:14 smithi173 bash[59422]: audit 2023-09-23T18:51:13.438740+0000 mon.a (mon.0) 1355 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:51:14.503 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:14 smithi173 bash[59422]: cluster 2023-09-23T18:51:14.141792+0000 mgr.y (mgr.26054) 997 : cluster 0 pgmap v1060: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 56511/183381 objects degraded (30.816%); 3530/183381 objects misplaced (1.925%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T18:51:14.517 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:14.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:14.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:14.621 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:14.649 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:14.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:14.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:14.752 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:14.784 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:14.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:14.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:14.891 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:14.916 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:14.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:14.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:15.019 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:15.064 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:15.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:15.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:15.170 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:15.201 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:15.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:15.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:15.307 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:15.316 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:15 smithi098 bash[76817]: cluster 2023-09-23T18:51:14.231619+0000 mon.a (mon.0) 1356 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:51:15.316 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:15 smithi098 bash[76817]: cluster 2023-09-23T18:51:14.249524+0000 osd.5 (osd.5) 154 : cluster 3 failed to encode map e255 with expected crc 2023-09-23T18:51:15.316 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:15 smithi098 bash[76817]: cluster 2023-09-23T18:51:14.253776+0000 osd.4 (osd.4) 251 : cluster 3 failed to encode map e255 with expected crc 2023-09-23T18:51:15.316 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:15 smithi098 bash[76817]: audit 2023-09-23T18:51:14.255717+0000 mon.a (mon.0) 1357 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:51:15.316 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:15 smithi098 bash[76817]: cluster 2023-09-23T18:51:14.255999+0000 mon.a (mon.0) 1358 : cluster 0 osdmap e255: 8 total, 8 up, 8 in 2023-09-23T18:51:15.316 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:15 smithi098 bash[76817]: cluster 2023-09-23T18:51:15.167969+0000 mon.a (mon.0) 1359 : cluster 3 Health check update: Degraded data redundancy: 56511/183381 objects degraded (30.816%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:15.317 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:15 smithi098 bash[79516]: cluster 2023-09-23T18:51:14.231619+0000 mon.a (mon.0) 1356 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:51:15.317 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:15 smithi098 bash[79516]: cluster 2023-09-23T18:51:14.249524+0000 osd.5 (osd.5) 154 : cluster 3 failed to encode map e255 with expected crc 2023-09-23T18:51:15.317 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:15 smithi098 bash[79516]: cluster 2023-09-23T18:51:14.253776+0000 osd.4 (osd.4) 251 : cluster 3 failed to encode map e255 with expected crc 2023-09-23T18:51:15.317 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:15 smithi098 bash[79516]: audit 2023-09-23T18:51:14.255717+0000 mon.a (mon.0) 1357 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:51:15.317 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:15 smithi098 bash[79516]: cluster 2023-09-23T18:51:14.255999+0000 mon.a (mon.0) 1358 : cluster 0 osdmap e255: 8 total, 8 up, 8 in 2023-09-23T18:51:15.317 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:15 smithi098 bash[79516]: cluster 2023-09-23T18:51:15.167969+0000 mon.a (mon.0) 1359 : cluster 3 Health check update: Degraded data redundancy: 56511/183381 objects degraded (30.816%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:15.348 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:15.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:15.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:15.452 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:15.480 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:15.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:15.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:15.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:15 smithi173 bash[59422]: cluster 2023-09-23T18:51:14.231619+0000 mon.a (mon.0) 1356 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:51:15.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:15 smithi173 bash[59422]: cluster 2023-09-23T18:51:14.249524+0000 osd.5 (osd.5) 154 : cluster 3 failed to encode map e255 with expected crc 2023-09-23T18:51:15.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:15 smithi173 bash[59422]: cluster 2023-09-23T18:51:14.253776+0000 osd.4 (osd.4) 251 : cluster 3 failed to encode map e255 with expected crc 2023-09-23T18:51:15.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:15 smithi173 bash[59422]: audit 2023-09-23T18:51:14.255717+0000 mon.a (mon.0) 1357 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:51:15.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:15 smithi173 bash[59422]: cluster 2023-09-23T18:51:14.255999+0000 mon.a (mon.0) 1358 : cluster 0 osdmap e255: 8 total, 8 up, 8 in 2023-09-23T18:51:15.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:15 smithi173 bash[59422]: cluster 2023-09-23T18:51:15.167969+0000 mon.a (mon.0) 1359 : cluster 3 Health check update: Degraded data redundancy: 56511/183381 objects degraded (30.816%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:15.585 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:15.636 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:15.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:15.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:15.742 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:15.770 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:15.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:15.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:15.880 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:15.908 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:15.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:15.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:16.011 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:16.054 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:16.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:16.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:16.159 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:16.185 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:16.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:16.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:16.277 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:16 smithi098 bash[76817]: cluster 2023-09-23T18:51:16.142635+0000 mgr.y (mgr.26054) 998 : cluster 0 pgmap v1062: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 56511/182496 objects degraded (30.966%); 3530/182496 objects misplaced (1.934%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T18:51:16.278 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:51:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:51:16] "GET /metrics HTTP/1.1" 200 36613 "" "Prometheus/2.33.4" 2023-09-23T18:51:16.278 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:16 smithi098 bash[79516]: cluster 2023-09-23T18:51:16.142635+0000 mgr.y (mgr.26054) 998 : cluster 0 pgmap v1062: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 56511/182496 objects degraded (30.966%); 3530/182496 objects misplaced (1.934%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T18:51:16.292 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:16.319 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:16.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:16.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:16.422 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:16.447 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:16.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:16.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:16.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:16 smithi173 bash[59422]: cluster 2023-09-23T18:51:16.142635+0000 mgr.y (mgr.26054) 998 : cluster 0 pgmap v1062: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 56511/182496 objects degraded (30.966%); 3530/182496 objects misplaced (1.934%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T18:51:16.552 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:16.582 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:16.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:16.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:16.685 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:16.711 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:16.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:16.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:16.819 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:16.876 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:16.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:16.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:16.981 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:17.031 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:17.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:17.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:17.136 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:17.157 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:17.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:17.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:17.171 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:51:17.260 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:17.281 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:17.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:17.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:17.365 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T18:51:17.365 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:51:17.366 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [0, 1, 2, 3, 4, 6, 7, 5] 2023-09-23T18:51:17.366 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd out 5 2023-09-23T18:51:17.385 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:17.461 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:17.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:17.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:17.568 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:17.615 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:17.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:17.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:17.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:17.749 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:17.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:17.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:17.853 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:17.888 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:17.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:17.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:17.993 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:18.015 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:18.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:18.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:18.119 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:18.144 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:18.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:18.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:18.249 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:18.275 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:18.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:18 smithi173 bash[59422]: cluster 2023-09-23T18:51:18.143332+0000 mgr.y (mgr.26054) 999 : cluster 0 pgmap v1063: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 56546/178263 objects degraded (31.721%); 3397/178263 objects misplaced (1.906%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:51:18.280 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:51:18.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:18.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:18.382 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:18.406 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:18.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:18.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:18.509 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:18.536 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:18.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:18.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:18.639 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:18.651 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:18 smithi098 bash[76817]: cluster 2023-09-23T18:51:18.143332+0000 mgr.y (mgr.26054) 999 : cluster 0 pgmap v1063: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 56546/178263 objects degraded (31.721%); 3397/178263 objects misplaced (1.906%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:51:18.651 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:18 smithi098 bash[79516]: cluster 2023-09-23T18:51:18.143332+0000 mgr.y (mgr.26054) 999 : cluster 0 pgmap v1063: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 56546/178263 objects degraded (31.721%); 3397/178263 objects misplaced (1.906%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:51:18.669 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:18.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:18.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:18.771 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:18.789 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:51:18.794 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:18.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:18.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:18.898 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:18.958 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:18.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:18.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:19.066 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:19.095 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:19.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:19.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:19.198 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:19.225 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:19.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:19.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:19.330 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:19.359 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:19.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:19.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:19.464 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:19.491 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:19.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:19.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:19.594 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:19.621 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:19.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:19.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:19.725 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:19.766 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:19.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:19.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:19.871 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:19.893 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:19.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:19.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:20.000 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:20.032 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:20.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:20.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:20.135 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:20.176 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:20.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:20.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:20.224 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:20 smithi173 bash[59422]: cluster 2023-09-23T18:51:20.144099+0000 mgr.y (mgr.26054) 1000 : cluster 0 pgmap v1064: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 458 op/s; 56546/176469 objects degraded (32.043%); 3397/176469 objects misplaced (1.925%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:51:20.225 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:20 smithi173 bash[59422]: cluster 2023-09-23T18:51:20.169047+0000 mon.a (mon.0) 1360 : cluster 3 Health check update: Degraded data redundancy: 56546/178263 objects degraded (31.721%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:20.279 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:20.312 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:20.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:20.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:20.415 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:20.424 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:20 smithi098 bash[79516]: cluster 2023-09-23T18:51:20.144099+0000 mgr.y (mgr.26054) 1000 : cluster 0 pgmap v1064: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 458 op/s; 56546/176469 objects degraded (32.043%); 3397/176469 objects misplaced (1.925%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:51:20.424 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:20 smithi098 bash[79516]: cluster 2023-09-23T18:51:20.169047+0000 mon.a (mon.0) 1360 : cluster 3 Health check update: Degraded data redundancy: 56546/178263 objects degraded (31.721%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:20.424 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:20 smithi098 bash[76817]: cluster 2023-09-23T18:51:20.144099+0000 mgr.y (mgr.26054) 1000 : cluster 0 pgmap v1064: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 458 op/s; 56546/176469 objects degraded (32.043%); 3397/176469 objects misplaced (1.925%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:51:20.424 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:20 smithi098 bash[76817]: cluster 2023-09-23T18:51:20.169047+0000 mon.a (mon.0) 1360 : cluster 3 Health check update: Degraded data redundancy: 56546/178263 objects degraded (31.721%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:20.447 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:20.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:20.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:20.551 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:20.576 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:20.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:20.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:20.679 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:20.711 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:20.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:20.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:20.817 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:20.851 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:20.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:20.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:20.957 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:20.986 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:20.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:20.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:21.095 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:21.125 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:21.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:21.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:21.231 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:21.256 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:21.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:21.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:21.360 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:21.391 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:21.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:21.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:21.494 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:21.518 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:21.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:21.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:21.624 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:21.659 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:21.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:21.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:21.764 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:21.794 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:21.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:21.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:21.859 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:51:21.861 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:51:21.897 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:21.924 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:21.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:21.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:22.031 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:22.055 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:22.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:22.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:22.158 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:22.182 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:22.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:22.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:22.287 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:22.312 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:22.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:22.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:22.417 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:22.426 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:22 smithi173 bash[59422]: cluster 2023-09-23T18:51:22.144830+0000 mgr.y (mgr.26054) 1001 : cluster 0 pgmap v1065: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 401 op/s; 56546/174294 objects degraded (32.443%); 3397/174294 objects misplaced (1.949%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:51:22.440 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:22.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:22 smithi098 bash[79516]: cluster 2023-09-23T18:51:22.144830+0000 mgr.y (mgr.26054) 1001 : cluster 0 pgmap v1065: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 401 op/s; 56546/174294 objects degraded (32.443%); 3397/174294 objects misplaced (1.949%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:51:22.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:22 smithi098 bash[76817]: cluster 2023-09-23T18:51:22.144830+0000 mgr.y (mgr.26054) 1001 : cluster 0 pgmap v1065: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 401 op/s; 56546/174294 objects degraded (32.443%); 3397/174294 objects misplaced (1.949%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:51:22.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:22.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:22.548 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:22.594 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:22.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:22.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:22.702 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:22.740 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:22.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:22.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:22.850 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:22.893 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:22.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:22.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:22.998 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:23.024 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:23.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:23.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:23.127 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:23.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:23.154 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:23.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:23.255 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:23.286 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:23.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:23.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:23.392 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:23.423 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:23.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:23.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:23.533 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:23.559 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:23.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:23.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:23.662 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:23.689 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:23.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:23.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:23.792 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:23.820 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:23.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:23.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:23.924 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:23.950 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:23.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:23.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:24.053 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:24.087 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:24.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:24.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:24.192 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:24.217 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:24.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:24.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:24.321 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:24.330 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:24 smithi098 bash[76817]: cluster 2023-09-23T18:51:24.145665+0000 mgr.y (mgr.26054) 1002 : cluster 0 pgmap v1066: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 467 op/s; 56621/169353 objects degraded (33.434%); 3232/169353 objects misplaced (1.908%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:51:24.331 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:24 smithi098 bash[79516]: cluster 2023-09-23T18:51:24.145665+0000 mgr.y (mgr.26054) 1002 : cluster 0 pgmap v1066: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 467 op/s; 56621/169353 objects degraded (33.434%); 3232/169353 objects misplaced (1.908%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:51:24.356 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:24.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:24.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:24.459 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:24.469 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:24 smithi173 bash[59422]: cluster 2023-09-23T18:51:24.145665+0000 mgr.y (mgr.26054) 1002 : cluster 0 pgmap v1066: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 467 op/s; 56621/169353 objects degraded (33.434%); 3232/169353 objects misplaced (1.908%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:51:24.482 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:24.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:24.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:24.585 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:24.625 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:24.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:24.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:24.730 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:24.756 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:24.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:24.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:24.859 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:24.881 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:24.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:24.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:24.983 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:25.010 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:25.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:25.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:25.077 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:51:25.119 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:25.162 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:25.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:25.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:25.270 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:25.309 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:25.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:25.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:25.414 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:25.437 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:25.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:25.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:25.540 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:25.571 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:25.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:25.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:25.675 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:25.685 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:25 smithi173 bash[59422]: audit 2023-09-23T18:51:24.629190+0000 mon.a (mon.0) 1361 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:51:25.685 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:25 smithi173 bash[59422]: audit 2023-09-23T18:51:24.632980+0000 mon.a (mon.0) 1362 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:51:25.685 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:25 smithi173 bash[59422]: cluster 2023-09-23T18:51:25.169995+0000 mon.a (mon.0) 1363 : cluster 3 Health check update: Degraded data redundancy: 56621/169353 objects degraded (33.434%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:25.702 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:25.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:25.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:25.805 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:25.830 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:25.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:25.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:25.937 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:25.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:25 smithi098 bash[76817]: audit 2023-09-23T18:51:24.629190+0000 mon.a (mon.0) 1361 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:51:25.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:25 smithi098 bash[76817]: audit 2023-09-23T18:51:24.632980+0000 mon.a (mon.0) 1362 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:51:25.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:25 smithi098 bash[76817]: cluster 2023-09-23T18:51:25.169995+0000 mon.a (mon.0) 1363 : cluster 3 Health check update: Degraded data redundancy: 56621/169353 objects degraded (33.434%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:25.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:25 smithi098 bash[79516]: audit 2023-09-23T18:51:24.629190+0000 mon.a (mon.0) 1361 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:51:25.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:25 smithi098 bash[79516]: audit 2023-09-23T18:51:24.632980+0000 mon.a (mon.0) 1362 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:51:25.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:25 smithi098 bash[79516]: cluster 2023-09-23T18:51:25.169995+0000 mon.a (mon.0) 1363 : cluster 3 Health check update: Degraded data redundancy: 56621/169353 objects degraded (33.434%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:25.968 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:25.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:25.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:26.072 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:26.107 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:26.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:26.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:26.213 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:26.225 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:51:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:51:26] "GET /metrics HTTP/1.1" 200 36609 "" "Prometheus/2.33.4" 2023-09-23T18:51:26.269 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:26.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:26.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:26.376 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:26.398 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:26.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:26.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:26.501 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:26.526 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:26.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:26.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:26.630 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:26.659 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:26.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:26.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:26.764 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:26.809 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:26.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:26.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:26.913 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:26.921 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:26 smithi173 bash[59422]: cluster 2023-09-23T18:51:26.146262+0000 mgr.y (mgr.26054) 1003 : cluster 0 pgmap v1067: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 418 op/s; 56621/168444 objects degraded (33.614%); 3232/168444 objects misplaced (1.919%); 1011 KiB/s, 15 objects/s recovering 2023-09-23T18:51:26.936 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:26.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:26.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:26.949 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:26 smithi098 bash[76817]: cluster 2023-09-23T18:51:26.146262+0000 mgr.y (mgr.26054) 1003 : cluster 0 pgmap v1067: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 418 op/s; 56621/168444 objects degraded (33.614%); 3232/168444 objects misplaced (1.919%); 1011 KiB/s, 15 objects/s recovering 2023-09-23T18:51:26.949 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:26 smithi098 bash[79516]: cluster 2023-09-23T18:51:26.146262+0000 mgr.y (mgr.26054) 1003 : cluster 0 pgmap v1067: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 418 op/s; 56621/168444 objects degraded (33.614%); 3232/168444 objects misplaced (1.919%); 1011 KiB/s, 15 objects/s recovering 2023-09-23T18:51:27.038 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:27.067 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:27.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:27.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:27.172 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:27.197 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:27.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:27.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:27.305 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:27.332 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:27.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:27.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:27.437 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:27.468 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:27.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:27.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:27.573 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:27.598 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:27.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:27.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:27.701 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:27.725 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:27.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:27.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:27.829 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:27.856 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:27.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:27.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:27.960 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:27.984 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:27.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:27.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:28.092 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:28.126 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:28.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:28.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:28.229 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:28.254 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:28.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:28.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:28.360 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:28.392 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:28.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:28.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:28.499 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:28.507 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:28 smithi098 bash[76817]: cluster 2023-09-23T18:51:28.147044+0000 mgr.y (mgr.26054) 1004 : cluster 0 pgmap v1068: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 56669/163968 objects degraded (34.561%); 3077/163968 objects misplaced (1.877%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T18:51:28.507 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:28 smithi098 bash[79516]: cluster 2023-09-23T18:51:28.147044+0000 mgr.y (mgr.26054) 1004 : cluster 0 pgmap v1068: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 56669/163968 objects degraded (34.561%); 3077/163968 objects misplaced (1.877%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T18:51:28.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:28 smithi173 bash[59422]: cluster 2023-09-23T18:51:28.147044+0000 mgr.y (mgr.26054) 1004 : cluster 0 pgmap v1068: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 56669/163968 objects degraded (34.561%); 3077/163968 objects misplaced (1.877%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T18:51:28.536 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:28.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:28.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:28.638 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:28.681 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:28.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:28.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:28.790 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:28.813 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:28.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:28.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:28.920 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:28.950 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:28.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:28.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:29.056 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:29.091 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:29.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:29.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:29.198 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:29.229 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:29.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:29.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:29.337 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:29.372 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:29.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:29.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:29.475 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:29.517 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:29.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:29.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:29.621 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:29.653 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:29.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:29.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:29.761 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:29.780 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:51:29.780 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:51:29.785 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:29.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:29.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:29.888 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:29.932 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:29.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:29.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:30.037 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:30.061 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:30.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:30.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:30.231 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:30.268 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:30.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:30.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:30.377 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:30.387 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:30 smithi173 bash[59422]: cluster 2023-09-23T18:51:30.147694+0000 mgr.y (mgr.26054) 1005 : cluster 0 pgmap v1069: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 449 op/s; 56669/162069 objects degraded (34.966%); 3077/162069 objects misplaced (1.899%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T18:51:30.387 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:30 smithi173 bash[59422]: cluster 2023-09-23T18:51:30.171148+0000 mon.a (mon.0) 1364 : cluster 3 Health check update: Degraded data redundancy: 56669/163968 objects degraded (34.561%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:30.400 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:30.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:30.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:30.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:30 smithi098 bash[76817]: cluster 2023-09-23T18:51:30.147694+0000 mgr.y (mgr.26054) 1005 : cluster 0 pgmap v1069: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 449 op/s; 56669/162069 objects degraded (34.966%); 3077/162069 objects misplaced (1.899%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T18:51:30.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:30 smithi098 bash[76817]: cluster 2023-09-23T18:51:30.171148+0000 mon.a (mon.0) 1364 : cluster 3 Health check update: Degraded data redundancy: 56669/163968 objects degraded (34.561%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:30.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:30 smithi098 bash[79516]: cluster 2023-09-23T18:51:30.147694+0000 mgr.y (mgr.26054) 1005 : cluster 0 pgmap v1069: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 449 op/s; 56669/162069 objects degraded (34.966%); 3077/162069 objects misplaced (1.899%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T18:51:30.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:30 smithi098 bash[79516]: cluster 2023-09-23T18:51:30.171148+0000 mon.a (mon.0) 1364 : cluster 3 Health check update: Degraded data redundancy: 56669/163968 objects degraded (34.561%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:30.503 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:30.529 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:30.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:30.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:30.635 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:30.658 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:30.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:30.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:30.761 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:30.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:30.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:30.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:30.889 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:30.927 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:30.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:30.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:31.032 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:31.078 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:31.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:31.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:31.183 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:31.212 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:31.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:31.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:31.316 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:31.352 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:31.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:31.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:31.456 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:31.470 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:31 smithi173 bash[59422]: audit 2023-09-23T18:51:30.803007+0000 mon.a (mon.0) 1365 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:51:31.487 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:31.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:31.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:31.590 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:31.600 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:31 smithi098 bash[79516]: audit 2023-09-23T18:51:30.803007+0000 mon.a (mon.0) 1365 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:51:31.601 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:31 smithi098 bash[76817]: audit 2023-09-23T18:51:30.803007+0000 mon.a (mon.0) 1365 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:51:31.628 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:31.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:31.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:31.731 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:31.763 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:31.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:31.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:31.869 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:31.891 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:31.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:31.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:31.993 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:32.019 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:32.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:32.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:32.123 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:32.149 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:32.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:32.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:32.252 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:32.258 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:32 smithi098 bash[76817]: cluster 2023-09-23T18:51:32.148414+0000 mgr.y (mgr.26054) 1006 : cluster 0 pgmap v1070: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 56669/159771 objects degraded (35.469%); 3077/159771 objects misplaced (1.926%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T18:51:32.258 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:32 smithi098 bash[79516]: cluster 2023-09-23T18:51:32.148414+0000 mgr.y (mgr.26054) 1006 : cluster 0 pgmap v1070: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 56669/159771 objects degraded (35.469%); 3077/159771 objects misplaced (1.926%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T18:51:32.273 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:32.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:32.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:32.376 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:32.384 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:32 smithi173 bash[59422]: cluster 2023-09-23T18:51:32.148414+0000 mgr.y (mgr.26054) 1006 : cluster 0 pgmap v1070: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 56669/159771 objects degraded (35.469%); 3077/159771 objects misplaced (1.926%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T18:51:32.396 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:32.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:32.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:32.500 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:32.530 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:32.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:32.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:32.635 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:32.672 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:32.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:32.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:32.776 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:32.806 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:32.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:32.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:32.912 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:32.943 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:32.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:32.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:33.048 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:33.074 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:33.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:33.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:33.180 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:33.212 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:33.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:33.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:33.318 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:33.356 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:33.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:33.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:33.462 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:33.505 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:33.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:33.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:33.609 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:33.646 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:33.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:33.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:33.751 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:33.792 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:33.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:33.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:33.899 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:33.928 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:33.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:33.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:34.035 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:34.059 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:34.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:34.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:34.164 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:34.191 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:34.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:34.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:34.265 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T18:51:34.295 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:34.303 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:34 smithi173 bash[59422]: cluster 2023-09-23T18:51:34.149272+0000 mgr.y (mgr.26054) 1007 : cluster 0 pgmap v1071: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 56730/155247 objects degraded (36.542%); 2920/155247 objects misplaced (1.881%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T18:51:34.316 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:34.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:34.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:34.419 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:34.453 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:34.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:34.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:34.556 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:34.567 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:34 smithi098 bash[76817]: cluster 2023-09-23T18:51:34.149272+0000 mgr.y (mgr.26054) 1007 : cluster 0 pgmap v1071: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 56730/155247 objects degraded (36.542%); 2920/155247 objects misplaced (1.881%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T18:51:34.568 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:34 smithi098 bash[79516]: cluster 2023-09-23T18:51:34.149272+0000 mgr.y (mgr.26054) 1007 : cluster 0 pgmap v1071: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 56730/155247 objects degraded (36.542%); 2920/155247 objects misplaced (1.881%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T18:51:34.589 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:34.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:34.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:34.697 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:34.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:34.729 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:34.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:34.830 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:34.856 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:34.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:34.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:34.959 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:34.994 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:35.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:35.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:35.101 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:35.144 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:35.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:35.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:35.242 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:35 smithi173 bash[59422]: cluster 2023-09-23T18:51:35.173464+0000 mon.a (mon.0) 1366 : cluster 3 Health check update: Degraded data redundancy: 56730/155247 objects degraded (36.542%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:35.244 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:35 smithi098 bash[79516]: cluster 2023-09-23T18:51:35.173464+0000 mon.a (mon.0) 1366 : cluster 3 Health check update: Degraded data redundancy: 56730/155247 objects degraded (36.542%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:35.251 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:35.281 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:35.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:35.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:35.387 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:35.412 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:35.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:35.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:35.523 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:35.545 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:35.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:35.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:35.648 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:35.657 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:35 smithi098 bash[76817]: cluster 2023-09-23T18:51:35.173464+0000 mon.a (mon.0) 1366 : cluster 3 Health check update: Degraded data redundancy: 56730/155247 objects degraded (36.542%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:35.684 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:35.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:35.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:35.792 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:35.835 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:35.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:35.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:35.941 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:35.970 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:35.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:35.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:36.075 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:36.112 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:36.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:36.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:36.218 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:36.227 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:51:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:51:36] "GET /metrics HTTP/1.1" 200 36609 "" "Prometheus/2.33.4" 2023-09-23T18:51:36.242 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:36.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:36.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:36.347 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:36.356 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:36 smithi173 bash[59422]: cluster 2023-09-23T18:51:36.149892+0000 mgr.y (mgr.26054) 1008 : cluster 0 pgmap v1072: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 416 op/s; 56730/154362 objects degraded (36.751%); 2920/154362 objects misplaced (1.892%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T18:51:36.373 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:36.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:36.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:36.477 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:36.486 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:36 smithi098 bash[79516]: cluster 2023-09-23T18:51:36.149892+0000 mgr.y (mgr.26054) 1008 : cluster 0 pgmap v1072: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 416 op/s; 56730/154362 objects degraded (36.751%); 2920/154362 objects misplaced (1.892%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T18:51:36.487 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:36 smithi098 bash[76817]: cluster 2023-09-23T18:51:36.149892+0000 mgr.y (mgr.26054) 1008 : cluster 0 pgmap v1072: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 416 op/s; 56730/154362 objects degraded (36.751%); 2920/154362 objects misplaced (1.892%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T18:51:36.508 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:36.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:36.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:36.619 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:36.650 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:36.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:36.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:36.753 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:36.795 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:36.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:36.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:36.899 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:36.964 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:36.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:36.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:37.071 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:37.102 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:37.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:37.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:37.206 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:37.246 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:37.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:37.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:37.355 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:37.387 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:37.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:37.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:37.498 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:37.532 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:37.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:37.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:37.636 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:37.672 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:37.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:37.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:37.778 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:37.840 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:37.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:37.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:37.948 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:37.979 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:37.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:37.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:38.083 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:38.112 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:38.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:38.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:38.216 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:38.241 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:38.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:38.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:38.344 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:38.377 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:38.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:38.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:38.484 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:38.495 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:38 smithi173 bash[59422]: audit 2023-09-23T18:51:37.358713+0000 mon.a (mon.0) 1367 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:51:38.496 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:38 smithi173 bash[59422]: audit 2023-09-23T18:51:37.369177+0000 mon.a (mon.0) 1368 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:51:38.496 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:38 smithi173 bash[59422]: cluster 2023-09-23T18:51:38.150660+0000 mgr.y (mgr.26054) 1009 : cluster 0 pgmap v1073: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 56790/149766 objects degraded (37.919%); 2766/149766 objects misplaced (1.847%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T18:51:38.514 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:38.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:38.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:38.620 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:38.628 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:38 smithi098 bash[76817]: audit 2023-09-23T18:51:37.358713+0000 mon.a (mon.0) 1367 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:51:38.628 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:38 smithi098 bash[76817]: audit 2023-09-23T18:51:37.369177+0000 mon.a (mon.0) 1368 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:51:38.628 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:38 smithi098 bash[76817]: cluster 2023-09-23T18:51:38.150660+0000 mgr.y (mgr.26054) 1009 : cluster 0 pgmap v1073: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 56790/149766 objects degraded (37.919%); 2766/149766 objects misplaced (1.847%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T18:51:38.628 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:38 smithi098 bash[79516]: audit 2023-09-23T18:51:37.358713+0000 mon.a (mon.0) 1367 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:51:38.629 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:38 smithi098 bash[79516]: audit 2023-09-23T18:51:37.369177+0000 mon.a (mon.0) 1368 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:51:38.629 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:38 smithi098 bash[79516]: cluster 2023-09-23T18:51:38.150660+0000 mgr.y (mgr.26054) 1009 : cluster 0 pgmap v1073: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 56790/149766 objects degraded (37.919%); 2766/149766 objects misplaced (1.847%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T18:51:38.653 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:38.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:38.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:38.759 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:38.817 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:38.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:38.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:38.921 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:38.956 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:38.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:38.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:39.059 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:39.098 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:39.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:39.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:39.155 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:51:39.204 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:39.247 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:39.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:39.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:39.351 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:39.388 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:39.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:39.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:39.494 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:39.547 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:39.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:39.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:39.656 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:39.688 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:39.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:39.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:39.793 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:39.822 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:39.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:39.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:39.928 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:39.952 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:39.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:39.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:40.057 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:40.081 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:40.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:40.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:40.187 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:40.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:40.214 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:40.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:40.303 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:51:40.315 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:40.347 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:40.348 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:51:40.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:40.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:40.351 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:40 smithi098 bash[76817]: audit 2023-09-23T18:51:39.347268+0000 mon.a (mon.0) 1369 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:51:40.351 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:40 smithi098 bash[76817]: audit 2023-09-23T18:51:39.364610+0000 mon.a (mon.0) 1370 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:51:40.390 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:51:40.444 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:51:40.450 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:40.472 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:40.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:40.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:40.490 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:51:40.535 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:51:40.569 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:51:40.576 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:40.585 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:40 smithi173 bash[59422]: audit 2023-09-23T18:51:39.347268+0000 mon.a (mon.0) 1369 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:51:40.586 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:40 smithi173 bash[59422]: audit 2023-09-23T18:51:39.364610+0000 mon.a (mon.0) 1370 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:51:40.586 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:40 smithi173 bash[59422]: audit 2023-09-23T18:51:39.624458+0000 mon.a (mon.0) 1371 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:51:40.586 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:40 smithi173 bash[59422]: audit 2023-09-23T18:51:40.043003+0000 mon.a (mon.0) 1372 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:51:40.586 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:40 smithi173 bash[59422]: audit 2023-09-23T18:51:40.044219+0000 mon.a (mon.0) 1373 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:51:40.586 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:40 smithi173 bash[59422]: audit 2023-09-23T18:51:40.051549+0000 mon.a (mon.0) 1374 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:51:40.586 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:40 smithi173 bash[59422]: cluster 2023-09-23T18:51:40.151325+0000 mgr.y (mgr.26054) 1010 : cluster 0 pgmap v1074: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 441 op/s; 56790/148071 objects degraded (38.353%); 2766/148071 objects misplaced (1.868%); 1013 KiB/s, 15 objects/s recovering 2023-09-23T18:51:40.586 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:40 smithi173 bash[59422]: cluster 2023-09-23T18:51:40.174990+0000 mon.a (mon.0) 1375 : cluster 3 Health check update: Degraded data redundancy: 56790/149766 objects degraded (37.919%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:40.586 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:40 smithi173 bash[59422]: audit 2023-09-23T18:51:40.220206+0000 mon.a (mon.0) 1376 : audit 0 from='client.? 172.21.15.98:0/2642477604' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:51:40.601 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:40.603 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:51:40.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:40.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:40.606 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:51:40.606 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:51:40.606 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:51:40.606 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:51:40.606 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:51:40.606 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:51:40.606 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:51:40.607 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:51:40.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:40 smithi098 bash[79516]: audit 2023-09-23T18:51:39.347268+0000 mon.a (mon.0) 1369 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:51:40.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:40 smithi098 bash[79516]: audit 2023-09-23T18:51:39.364610+0000 mon.a (mon.0) 1370 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:51:40.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:40 smithi098 bash[79516]: audit 2023-09-23T18:51:39.624458+0000 mon.a (mon.0) 1371 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:51:40.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:40 smithi098 bash[79516]: audit 2023-09-23T18:51:40.043003+0000 mon.a (mon.0) 1372 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:51:40.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:40 smithi098 bash[79516]: audit 2023-09-23T18:51:40.044219+0000 mon.a (mon.0) 1373 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:51:40.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:40 smithi098 bash[79516]: audit 2023-09-23T18:51:40.051549+0000 mon.a (mon.0) 1374 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:51:40.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:40 smithi098 bash[79516]: cluster 2023-09-23T18:51:40.151325+0000 mgr.y (mgr.26054) 1010 : cluster 0 pgmap v1074: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 441 op/s; 56790/148071 objects degraded (38.353%); 2766/148071 objects misplaced (1.868%); 1013 KiB/s, 15 objects/s recovering 2023-09-23T18:51:40.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:40 smithi098 bash[79516]: cluster 2023-09-23T18:51:40.174990+0000 mon.a (mon.0) 1375 : cluster 3 Health check update: Degraded data redundancy: 56790/149766 objects degraded (37.919%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:40.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:40 smithi098 bash[79516]: audit 2023-09-23T18:51:40.220206+0000 mon.a (mon.0) 1376 : audit 0 from='client.? 172.21.15.98:0/2642477604' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:51:40.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:40 smithi098 bash[76817]: audit 2023-09-23T18:51:39.624458+0000 mon.a (mon.0) 1371 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:51:40.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:40 smithi098 bash[76817]: audit 2023-09-23T18:51:40.043003+0000 mon.a (mon.0) 1372 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:51:40.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:40 smithi098 bash[76817]: audit 2023-09-23T18:51:40.044219+0000 mon.a (mon.0) 1373 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:51:40.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:40 smithi098 bash[76817]: audit 2023-09-23T18:51:40.051549+0000 mon.a (mon.0) 1374 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:51:40.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:40 smithi098 bash[76817]: cluster 2023-09-23T18:51:40.151325+0000 mgr.y (mgr.26054) 1010 : cluster 0 pgmap v1074: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 441 op/s; 56790/148071 objects degraded (38.353%); 2766/148071 objects misplaced (1.868%); 1013 KiB/s, 15 objects/s recovering 2023-09-23T18:51:40.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:40 smithi098 bash[76817]: cluster 2023-09-23T18:51:40.174990+0000 mon.a (mon.0) 1375 : cluster 3 Health check update: Degraded data redundancy: 56790/149766 objects degraded (37.919%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:40.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:40 smithi098 bash[76817]: audit 2023-09-23T18:51:40.220206+0000 mon.a (mon.0) 1376 : audit 0 from='client.? 172.21.15.98:0/2642477604' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:51:40.704 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:40.732 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:40.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:40.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:40.840 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:40.880 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:40.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:40.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:40.984 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:41.012 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:41.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:41.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:41.116 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:41.155 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:41.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:41.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:41.262 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:41.288 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:41.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:41.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:41.392 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:41.430 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:41.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:41.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:41.537 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:41.564 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:41.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:41.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:41.568 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:41 smithi098 bash[76817]: debug 2023-09-23T18:51:41.560+0000 7fc8a2a01700 -1 mon.a@0(leader).osd e255 definitely_dead 0 2023-09-23T18:51:41.669 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:41.704 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:41.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:41.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:41.807 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:41.833 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:41.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:41.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:41.940 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:41.962 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:41.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:41.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:42.075 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:42.082 INFO:teuthology.orchestra.run.smithi098.stderr:marked out osd.5. 2023-09-23T18:51:42.102 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:42.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:42.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:42.126 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:42 smithi098 bash[76817]: audit 2023-09-23T18:51:41.564914+0000 mon.a (mon.0) 1377 : audit 1 from='client.? 172.21.15.98:0/1014783429' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T18:51:42.126 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:42 smithi098 bash[76817]: cluster 2023-09-23T18:51:41.565056+0000 mon.a (mon.0) 1378 : cluster 1 Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T18:51:42.126 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:42 smithi098 bash[79516]: audit 2023-09-23T18:51:41.564914+0000 mon.a (mon.0) 1377 : audit 1 from='client.? 172.21.15.98:0/1014783429' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T18:51:42.126 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:42 smithi098 bash[79516]: cluster 2023-09-23T18:51:41.565056+0000 mon.a (mon.0) 1378 : cluster 1 Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T18:51:42.207 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:42.214 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:42 smithi173 bash[59422]: audit 2023-09-23T18:51:41.564914+0000 mon.a (mon.0) 1377 : audit 1 from='client.? 172.21.15.98:0/1014783429' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T18:51:42.214 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:42 smithi173 bash[59422]: cluster 2023-09-23T18:51:41.565056+0000 mon.a (mon.0) 1378 : cluster 1 Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T18:51:42.228 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:42.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:42.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:42.331 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:42.360 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:42.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:42.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:42.463 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:42.494 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:42.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:42.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:42.598 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:42.625 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:42.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:42.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:42.728 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:42.757 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:42.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:42.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:42.861 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:42.894 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:42.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:42.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:42.998 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:43.018 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:43.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:43.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:43.085 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is set 2023-09-23T18:51:43.122 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:43.145 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:43.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:43.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:43.247 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:43.259 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:43 smithi098 bash[76817]: cluster 2023-09-23T18:51:42.069923+0000 osd.4 (osd.4) 252 : cluster 3 failed to encode map e256 with expected crc 2023-09-23T18:51:43.259 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:43 smithi098 bash[76817]: audit 2023-09-23T18:51:42.071402+0000 mon.a (mon.0) 1379 : audit 1 from='client.? 172.21.15.98:0/1014783429' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T18:51:43.259 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:43 smithi098 bash[76817]: cluster 2023-09-23T18:51:42.071491+0000 mon.a (mon.0) 1380 : cluster 0 osdmap e256: 8 total, 8 up, 7 in 2023-09-23T18:51:43.259 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:43 smithi098 bash[76817]: cluster 2023-09-23T18:51:42.079356+0000 osd.5 (osd.5) 155 : cluster 3 failed to encode map e256 with expected crc 2023-09-23T18:51:43.259 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:43 smithi098 bash[76817]: cluster 2023-09-23T18:51:42.080029+0000 osd.6 (osd.6) 214 : cluster 3 failed to encode map e256 with expected crc 2023-09-23T18:51:43.259 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:43 smithi098 bash[76817]: cluster 2023-09-23T18:51:42.151875+0000 mgr.y (mgr.26054) 1011 : cluster 0 pgmap v1076: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 446 op/s; 56790/146364 objects degraded (38.801%); 2766/146364 objects misplaced (1.890%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:51:43.260 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:43 smithi098 bash[76817]: audit 2023-09-23T18:51:42.258703+0000 mon.a (mon.0) 1381 : audit 1 from='client.? 172.21.15.98:0/3121049660' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:51:43.260 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:43 smithi098 bash[79516]: cluster 2023-09-23T18:51:42.069923+0000 osd.4 (osd.4) 252 : cluster 3 failed to encode map e256 with expected crc 2023-09-23T18:51:43.260 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:43 smithi098 bash[79516]: audit 2023-09-23T18:51:42.071402+0000 mon.a (mon.0) 1379 : audit 1 from='client.? 172.21.15.98:0/1014783429' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T18:51:43.260 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:43 smithi098 bash[79516]: cluster 2023-09-23T18:51:42.071491+0000 mon.a (mon.0) 1380 : cluster 0 osdmap e256: 8 total, 8 up, 7 in 2023-09-23T18:51:43.260 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:43 smithi098 bash[79516]: cluster 2023-09-23T18:51:42.079356+0000 osd.5 (osd.5) 155 : cluster 3 failed to encode map e256 with expected crc 2023-09-23T18:51:43.260 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:43 smithi098 bash[79516]: cluster 2023-09-23T18:51:42.080029+0000 osd.6 (osd.6) 214 : cluster 3 failed to encode map e256 with expected crc 2023-09-23T18:51:43.260 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:43 smithi098 bash[79516]: cluster 2023-09-23T18:51:42.151875+0000 mgr.y (mgr.26054) 1011 : cluster 0 pgmap v1076: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 446 op/s; 56790/146364 objects degraded (38.801%); 2766/146364 objects misplaced (1.890%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:51:43.260 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:43 smithi098 bash[79516]: audit 2023-09-23T18:51:42.258703+0000 mon.a (mon.0) 1381 : audit 1 from='client.? 172.21.15.98:0/3121049660' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:51:43.270 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:43.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:43.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:43.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:43 smithi173 bash[59422]: cluster 2023-09-23T18:51:42.069923+0000 osd.4 (osd.4) 252 : cluster 3 failed to encode map e256 with expected crc 2023-09-23T18:51:43.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:43 smithi173 bash[59422]: audit 2023-09-23T18:51:42.071402+0000 mon.a (mon.0) 1379 : audit 1 from='client.? 172.21.15.98:0/1014783429' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T18:51:43.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:43 smithi173 bash[59422]: cluster 2023-09-23T18:51:42.071491+0000 mon.a (mon.0) 1380 : cluster 0 osdmap e256: 8 total, 8 up, 7 in 2023-09-23T18:51:43.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:43 smithi173 bash[59422]: cluster 2023-09-23T18:51:42.079356+0000 osd.5 (osd.5) 155 : cluster 3 failed to encode map e256 with expected crc 2023-09-23T18:51:43.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:43 smithi173 bash[59422]: cluster 2023-09-23T18:51:42.080029+0000 osd.6 (osd.6) 214 : cluster 3 failed to encode map e256 with expected crc 2023-09-23T18:51:43.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:43 smithi173 bash[59422]: cluster 2023-09-23T18:51:42.151875+0000 mgr.y (mgr.26054) 1011 : cluster 0 pgmap v1076: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 446 op/s; 56790/146364 objects degraded (38.801%); 2766/146364 objects misplaced (1.890%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T18:51:43.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:43 smithi173 bash[59422]: audit 2023-09-23T18:51:42.258703+0000 mon.a (mon.0) 1381 : audit 1 from='client.? 172.21.15.98:0/3121049660' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:51:43.373 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:43.393 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:43.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:43.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:43.496 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:43.521 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:43.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:43.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:43.625 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:43.653 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:43.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:43.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:43.757 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:43.789 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:43.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:43.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:43.892 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:43.912 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:43.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:43.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:44.016 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:44.048 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:44.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:44.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:44.152 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:44.184 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:44.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:44.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:44.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:44 smithi098 bash[79516]: cluster 2023-09-23T18:51:43.063151+0000 mon.a (mon.0) 1382 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:51:44.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:44 smithi098 bash[79516]: cluster 2023-09-23T18:51:43.069545+0000 osd.4 (osd.4) 253 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T18:51:44.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:44 smithi098 bash[79516]: cluster 2023-09-23T18:51:43.073773+0000 osd.5 (osd.5) 156 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T18:51:44.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:44 smithi098 bash[79516]: cluster 2023-09-23T18:51:43.073786+0000 osd.6 (osd.6) 215 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T18:51:44.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:44 smithi098 bash[79516]: cluster 2023-09-23T18:51:43.076207+0000 osd.5 (osd.5) 157 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T18:51:44.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:44 smithi098 bash[79516]: audit 2023-09-23T18:51:43.076716+0000 mon.a (mon.0) 1383 : audit 1 from='client.? 172.21.15.98:0/3121049660' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:51:44.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:44 smithi098 bash[79516]: cluster 2023-09-23T18:51:43.076931+0000 mon.a (mon.0) 1384 : cluster 0 osdmap e257: 8 total, 8 up, 7 in 2023-09-23T18:51:44.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:44 smithi098 bash[79516]: cluster 2023-09-23T18:51:43.080509+0000 osd.5 (osd.5) 158 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T18:51:44.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:44 smithi098 bash[79516]: cluster 2023-09-23T18:51:44.072319+0000 mon.a (mon.0) 1385 : cluster 0 osdmap e258: 8 total, 8 up, 7 in 2023-09-23T18:51:44.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:44 smithi098 bash[76817]: cluster 2023-09-23T18:51:43.063151+0000 mon.a (mon.0) 1382 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:51:44.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:44 smithi098 bash[76817]: cluster 2023-09-23T18:51:43.069545+0000 osd.4 (osd.4) 253 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T18:51:44.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:44 smithi098 bash[76817]: cluster 2023-09-23T18:51:43.073773+0000 osd.5 (osd.5) 156 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T18:51:44.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:44 smithi098 bash[76817]: cluster 2023-09-23T18:51:43.073786+0000 osd.6 (osd.6) 215 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T18:51:44.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:44 smithi098 bash[76817]: cluster 2023-09-23T18:51:43.076207+0000 osd.5 (osd.5) 157 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T18:51:44.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:44 smithi098 bash[76817]: audit 2023-09-23T18:51:43.076716+0000 mon.a (mon.0) 1383 : audit 1 from='client.? 172.21.15.98:0/3121049660' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:51:44.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:44 smithi098 bash[76817]: cluster 2023-09-23T18:51:43.076931+0000 mon.a (mon.0) 1384 : cluster 0 osdmap e257: 8 total, 8 up, 7 in 2023-09-23T18:51:44.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:44 smithi098 bash[76817]: cluster 2023-09-23T18:51:43.080509+0000 osd.5 (osd.5) 158 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T18:51:44.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:44 smithi098 bash[76817]: cluster 2023-09-23T18:51:44.072319+0000 mon.a (mon.0) 1385 : cluster 0 osdmap e258: 8 total, 8 up, 7 in 2023-09-23T18:51:44.288 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:44.298 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:44 smithi173 bash[59422]: cluster 2023-09-23T18:51:43.063151+0000 mon.a (mon.0) 1382 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:51:44.298 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:44 smithi173 bash[59422]: cluster 2023-09-23T18:51:43.069545+0000 osd.4 (osd.4) 253 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T18:51:44.298 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:44 smithi173 bash[59422]: cluster 2023-09-23T18:51:43.073773+0000 osd.5 (osd.5) 156 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T18:51:44.298 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:44 smithi173 bash[59422]: cluster 2023-09-23T18:51:43.073786+0000 osd.6 (osd.6) 215 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T18:51:44.298 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:44 smithi173 bash[59422]: cluster 2023-09-23T18:51:43.076207+0000 osd.5 (osd.5) 157 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T18:51:44.298 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:44 smithi173 bash[59422]: audit 2023-09-23T18:51:43.076716+0000 mon.a (mon.0) 1383 : audit 1 from='client.? 172.21.15.98:0/3121049660' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:51:44.299 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:44 smithi173 bash[59422]: cluster 2023-09-23T18:51:43.076931+0000 mon.a (mon.0) 1384 : cluster 0 osdmap e257: 8 total, 8 up, 7 in 2023-09-23T18:51:44.299 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:44 smithi173 bash[59422]: cluster 2023-09-23T18:51:43.080509+0000 osd.5 (osd.5) 158 : cluster 3 failed to encode map e257 with expected crc 2023-09-23T18:51:44.299 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:44 smithi173 bash[59422]: cluster 2023-09-23T18:51:44.072319+0000 mon.a (mon.0) 1385 : cluster 0 osdmap e258: 8 total, 8 up, 7 in 2023-09-23T18:51:44.326 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:44.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:44.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:44.432 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:44.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:44.467 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:44.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:44.567 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:44.598 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:44.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:44.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:44.705 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:44.736 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:44.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:44.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:44.840 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:44.864 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:44.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:44.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:44.968 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:44.995 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:45.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:45.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:45.101 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:45.121 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:45.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:45.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:45.223 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:45.234 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:45 smithi173 bash[59422]: cluster 2023-09-23T18:51:44.072981+0000 osd.7 (osd.7) 178 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T18:51:45.234 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:45 smithi173 bash[59422]: cluster 2023-09-23T18:51:44.073059+0000 osd.4 (osd.4) 254 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T18:51:45.234 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:45 smithi173 bash[59422]: cluster 2023-09-23T18:51:44.073210+0000 osd.1 (osd.1) 5 : cluster 0 8.3 starting backfill to osd.2 from (0'0,0'0] MAX to 255'9927 2023-09-23T18:51:45.234 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:45 smithi173 bash[59422]: cluster 2023-09-23T18:51:44.073479+0000 osd.5 (osd.5) 159 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T18:51:45.234 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:45 smithi173 bash[59422]: cluster 2023-09-23T18:51:44.077889+0000 osd.4 (osd.4) 255 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T18:51:45.234 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:45 smithi173 bash[59422]: cluster 2023-09-23T18:51:44.077992+0000 osd.7 (osd.7) 179 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T18:51:45.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:45 smithi173 bash[59422]: cluster 2023-09-23T18:51:44.080679+0000 osd.4 (osd.4) 256 : cluster 0 8.2 starting backfill to osd.6 from (0'0,0'0] MAX to 255'9939 2023-09-23T18:51:45.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:45 smithi173 bash[59422]: cluster 2023-09-23T18:51:44.097339+0000 osd.6 (osd.6) 216 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T18:51:45.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:45 smithi173 bash[59422]: cluster 2023-09-23T18:51:44.152672+0000 mgr.y (mgr.26054) 1012 : cluster 0 pgmap v1079: 89 pgs: 6 peering, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 392 op/s; 54168/142707 objects degraded (37.957%); 5217/142707 objects misplaced (3.656%); 1.8 MiB/s, 27 objects/s recovering 2023-09-23T18:51:45.248 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:45.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:45.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:45.351 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:45.377 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:45.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:45.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:45.447 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:45 smithi098 bash[76817]: cluster 2023-09-23T18:51:44.072981+0000 osd.7 (osd.7) 178 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T18:51:45.448 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:45 smithi098 bash[76817]: cluster 2023-09-23T18:51:44.073059+0000 osd.4 (osd.4) 254 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T18:51:45.448 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:45 smithi098 bash[76817]: cluster 2023-09-23T18:51:44.073210+0000 osd.1 (osd.1) 5 : cluster 0 8.3 starting backfill to osd.2 from (0'0,0'0] MAX to 255'9927 2023-09-23T18:51:45.448 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:45 smithi098 bash[76817]: cluster 2023-09-23T18:51:44.073479+0000 osd.5 (osd.5) 159 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T18:51:45.448 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:45 smithi098 bash[76817]: cluster 2023-09-23T18:51:44.077889+0000 osd.4 (osd.4) 255 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T18:51:45.448 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:45 smithi098 bash[76817]: cluster 2023-09-23T18:51:44.077992+0000 osd.7 (osd.7) 179 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T18:51:45.448 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:45 smithi098 bash[76817]: cluster 2023-09-23T18:51:44.080679+0000 osd.4 (osd.4) 256 : cluster 0 8.2 starting backfill to osd.6 from (0'0,0'0] MAX to 255'9939 2023-09-23T18:51:45.448 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:45 smithi098 bash[76817]: cluster 2023-09-23T18:51:44.097339+0000 osd.6 (osd.6) 216 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T18:51:45.448 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:45 smithi098 bash[76817]: cluster 2023-09-23T18:51:44.152672+0000 mgr.y (mgr.26054) 1012 : cluster 0 pgmap v1079: 89 pgs: 6 peering, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 392 op/s; 54168/142707 objects degraded (37.957%); 5217/142707 objects misplaced (3.656%); 1.8 MiB/s, 27 objects/s recovering 2023-09-23T18:51:45.449 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:45 smithi098 bash[79516]: cluster 2023-09-23T18:51:44.072981+0000 osd.7 (osd.7) 178 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T18:51:45.449 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:45 smithi098 bash[79516]: cluster 2023-09-23T18:51:44.073059+0000 osd.4 (osd.4) 254 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T18:51:45.449 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:45 smithi098 bash[79516]: cluster 2023-09-23T18:51:44.073210+0000 osd.1 (osd.1) 5 : cluster 0 8.3 starting backfill to osd.2 from (0'0,0'0] MAX to 255'9927 2023-09-23T18:51:45.449 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:45 smithi098 bash[79516]: cluster 2023-09-23T18:51:44.073479+0000 osd.5 (osd.5) 159 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T18:51:45.449 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:45 smithi098 bash[79516]: cluster 2023-09-23T18:51:44.077889+0000 osd.4 (osd.4) 255 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T18:51:45.449 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:45 smithi098 bash[79516]: cluster 2023-09-23T18:51:44.077992+0000 osd.7 (osd.7) 179 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T18:51:45.449 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:45 smithi098 bash[79516]: cluster 2023-09-23T18:51:44.080679+0000 osd.4 (osd.4) 256 : cluster 0 8.2 starting backfill to osd.6 from (0'0,0'0] MAX to 255'9939 2023-09-23T18:51:45.450 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:45 smithi098 bash[79516]: cluster 2023-09-23T18:51:44.097339+0000 osd.6 (osd.6) 216 : cluster 3 failed to encode map e258 with expected crc 2023-09-23T18:51:45.450 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:45 smithi098 bash[79516]: cluster 2023-09-23T18:51:44.152672+0000 mgr.y (mgr.26054) 1012 : cluster 0 pgmap v1079: 89 pgs: 6 peering, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 392 op/s; 54168/142707 objects degraded (37.957%); 5217/142707 objects misplaced (3.656%); 1.8 MiB/s, 27 objects/s recovering 2023-09-23T18:51:45.480 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:45.507 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:45.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:45.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:45.610 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:45.641 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:45.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:45.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:45.743 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:45.762 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:45.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:45.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:45.864 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:45.898 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:45.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:45.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:46.002 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:46.029 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:46.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:46.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:46.132 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:46.159 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:46.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:46.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:46.262 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:46.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:46 smithi173 bash[59422]: cluster 2023-09-23T18:51:45.087249+0000 mon.a (mon.0) 1386 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T18:51:46.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:46 smithi173 bash[59422]: cluster 2023-09-23T18:51:45.178762+0000 mon.a (mon.0) 1387 : cluster 3 Health check update: Degraded data redundancy: 54168/142707 objects degraded (37.957%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:46.284 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:46.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:46.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:46.342 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:46 smithi098 bash[76817]: cluster 2023-09-23T18:51:45.087249+0000 mon.a (mon.0) 1386 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T18:51:46.342 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:46 smithi098 bash[76817]: cluster 2023-09-23T18:51:45.178762+0000 mon.a (mon.0) 1387 : cluster 3 Health check update: Degraded data redundancy: 54168/142707 objects degraded (37.957%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:46.342 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:51:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:51:46] "GET /metrics HTTP/1.1" 200 36606 "" "Prometheus/2.33.4" 2023-09-23T18:51:46.342 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:46 smithi098 bash[79516]: cluster 2023-09-23T18:51:45.087249+0000 mon.a (mon.0) 1386 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T18:51:46.342 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:46 smithi098 bash[79516]: cluster 2023-09-23T18:51:45.178762+0000 mon.a (mon.0) 1387 : cluster 3 Health check update: Degraded data redundancy: 54168/142707 objects degraded (37.957%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:46.387 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:46.440 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:46.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:46.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:46.544 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:46.569 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:46.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:46.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:46.673 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:46.701 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:46.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:46.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:46.805 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:46.829 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:46.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:46.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:46.932 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:46.955 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:46.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:46.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:47.059 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:47.093 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:47.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:47.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:47.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:47 smithi098 bash[79516]: cluster 2023-09-23T18:51:46.153842+0000 mgr.y (mgr.26054) 1013 : cluster 0 pgmap v1080: 89 pgs: 6 peering, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 308 op/s; 60549/142518 objects degraded (42.485%); 5217/142518 objects misplaced (3.661%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T18:51:47.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:47 smithi098 bash[76817]: cluster 2023-09-23T18:51:46.153842+0000 mgr.y (mgr.26054) 1013 : cluster 0 pgmap v1080: 89 pgs: 6 peering, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 308 op/s; 60549/142518 objects degraded (42.485%); 5217/142518 objects misplaced (3.661%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T18:51:47.197 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:47.226 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:47.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:47.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:47.330 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:47.354 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:47.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:47.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:47.458 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:47.465 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:47 smithi173 bash[59422]: cluster 2023-09-23T18:51:46.153842+0000 mgr.y (mgr.26054) 1013 : cluster 0 pgmap v1080: 89 pgs: 6 peering, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 308 op/s; 60549/142518 objects degraded (42.485%); 5217/142518 objects misplaced (3.661%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T18:51:47.481 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:47.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:47.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:47.584 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:47.610 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:47.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:47.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:47.717 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:47.749 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:47.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:47.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:47.853 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:47.878 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:47.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:47.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:47.981 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:48.010 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:48.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:48.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:48.115 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:48.143 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:48.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:48.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:48.247 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:48.271 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:48.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:48.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:48.274 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:51:48.365 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:51:48.374 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:48.443 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:48.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:48.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:48.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:48 smithi173 bash[59422]: cluster 2023-09-23T18:51:48.155119+0000 mgr.y (mgr.26054) 1014 : cluster 0 pgmap v1081: 89 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 219 op/s; 63134/142356 objects degraded (44.349%); 5513/142356 objects misplaced (3.873%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T18:51:48.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:48 smithi173 bash[59422]: cluster 2023-09-23T18:51:48.206546+0000 mon.a (mon.0) 1388 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2023-09-23T18:51:48.547 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:48.573 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:48.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:48.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:48.676 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:48.687 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:48 smithi098 bash[76817]: cluster 2023-09-23T18:51:48.155119+0000 mgr.y (mgr.26054) 1014 : cluster 0 pgmap v1081: 89 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 219 op/s; 63134/142356 objects degraded (44.349%); 5513/142356 objects misplaced (3.873%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T18:51:48.688 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:48 smithi098 bash[76817]: cluster 2023-09-23T18:51:48.206546+0000 mon.a (mon.0) 1388 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2023-09-23T18:51:48.688 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:48 smithi098 bash[79516]: cluster 2023-09-23T18:51:48.155119+0000 mgr.y (mgr.26054) 1014 : cluster 0 pgmap v1081: 89 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 219 op/s; 63134/142356 objects degraded (44.349%); 5513/142356 objects misplaced (3.873%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T18:51:48.688 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:48 smithi098 bash[79516]: cluster 2023-09-23T18:51:48.206546+0000 mon.a (mon.0) 1388 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2023-09-23T18:51:48.707 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:48.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:48.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:48.810 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:48.836 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:48.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:48.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:48.940 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:48.969 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:48.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:48.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:49.072 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:49.102 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:49.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:49.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:49.206 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:49.230 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:49.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:49.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:49.334 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:49.337 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:51:49.337 INFO:teuthology.orchestra.run.smithi098.stdout: "size": 20, 2023-09-23T18:51:49.337 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 600, 2023-09-23T18:51:49.337 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [ 2023-09-23T18:51:49.337 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.337 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:88883 7.12 7:48d474bb:::benchmark_data_smithi098_155798_object88882:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e204)", 2023-09-23T18:51:49.337 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:17.067257+0000", 2023-09-23T18:51:49.337 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 452.26075953399999, 2023-09-23T18:51:49.337 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.78994038, 2023-09-23T18:51:49.338 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:49.338 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:51:49.338 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:51:49.338 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:51:49.338 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:51:49.338 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 88883 2023-09-23T18:51:49.338 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.338 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:49.338 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.338 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:49.338 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067257+0000", 2023-09-23T18:51:49.339 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.339 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.339 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.339 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:49.339 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067257+0000", 2023-09-23T18:51:49.339 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.339 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.339 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.339 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:49.339 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067257+0000", 2023-09-23T18:51:49.339 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:51:49.339 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.339 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.340 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:49.340 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067291+0000", 2023-09-23T18:51:49.340 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.4356000000000001e-05 2023-09-23T18:51:49.340 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.340 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.340 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:49.340 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067299+0000", 2023-09-23T18:51:49.340 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.8839999999999994e-06 2023-09-23T18:51:49.340 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.340 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.340 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:49.340 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067304+0000", 2023-09-23T18:51:49.341 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.5339999999999999e-06 2023-09-23T18:51:49.341 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.341 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.341 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.341 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067430+0000", 2023-09-23T18:51:49.341 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00012510399999999999 2023-09-23T18:51:49.341 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.341 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.341 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.341 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067436+0000", 2023-09-23T18:51:49.341 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.922e-06 2023-09-23T18:51:49.341 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.342 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.342 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.342 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.830824+0000", 2023-09-23T18:51:49.342 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.7633876989999999 2023-09-23T18:51:49.342 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.342 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.342 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:51:49.342 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.831102+0000", 2023-09-23T18:51:49.342 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00027776199999999997 2023-09-23T18:51:49.342 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.342 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.343 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,2,6", 2023-09-23T18:51:49.343 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.831348+0000", 2023-09-23T18:51:49.343 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00024587799999999999 2023-09-23T18:51:49.343 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.343 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.343 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:51:49.343 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.835050+0000", 2023-09-23T18:51:49.343 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0037017410000000001 2023-09-23T18:51:49.343 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.343 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.343 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.343 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.836675+0000", 2023-09-23T18:51:49.344 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.001625339 2023-09-23T18:51:49.344 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.344 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.344 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.344 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.847432+0000", 2023-09-23T18:51:49.344 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.010756794 2023-09-23T18:51:49.344 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.344 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.344 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.344 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.854510+0000", 2023-09-23T18:51:49.344 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0070784910000000001 2023-09-23T18:51:49.345 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.345 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.345 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:51:49.345 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.854566+0000", 2023-09-23T18:51:49.345 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.5396000000000003e-05 2023-09-23T18:51:49.345 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.345 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.345 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:51:49.345 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.857198+0000", 2023-09-23T18:51:49.345 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0026320829999999999 2023-09-23T18:51:49.345 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.345 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:49.346 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.346 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.346 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.346 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:88917 7.12 7:4c9c9c50:::benchmark_data_smithi098_155798_object88916:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e204)", 2023-09-23T18:51:49.346 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:17.067318+0000", 2023-09-23T18:51:49.346 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 452.26069829900001, 2023-09-23T18:51:49.346 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.7915008320000001, 2023-09-23T18:51:49.346 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:49.346 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:51:49.347 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:51:49.347 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:51:49.347 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:51:49.347 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 88917 2023-09-23T18:51:49.347 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.347 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:49.347 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.347 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:49.347 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067318+0000", 2023-09-23T18:51:49.347 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.347 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.347 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.348 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:49.348 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067318+0000", 2023-09-23T18:51:49.348 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.348 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.348 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.348 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:49.348 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067318+0000", 2023-09-23T18:51:49.348 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:51:49.348 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.348 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.348 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:49.348 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067355+0000", 2023-09-23T18:51:49.349 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.6637999999999997e-05 2023-09-23T18:51:49.349 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.349 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.349 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:49.349 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067364+0000", 2023-09-23T18:51:49.349 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.8910000000000007e-06 2023-09-23T18:51:49.349 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.349 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.349 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:49.349 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067371+0000", 2023-09-23T18:51:49.349 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.4320000000000004e-06 2023-09-23T18:51:49.349 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.349 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.350 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.350 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067476+0000", 2023-09-23T18:51:49.350 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000105209 2023-09-23T18:51:49.350 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.350 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.350 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.350 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067486+0000", 2023-09-23T18:51:49.350 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0436e-05 2023-09-23T18:51:49.350 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.350 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.350 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.350 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.832442+0000", 2023-09-23T18:51:49.351 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.7649554219999999 2023-09-23T18:51:49.351 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.351 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.351 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:51:49.351 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.832559+0000", 2023-09-23T18:51:49.351 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000116798 2023-09-23T18:51:49.351 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.351 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.351 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,2,6", 2023-09-23T18:51:49.351 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.832738+0000", 2023-09-23T18:51:49.351 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00017931300000000001 2023-09-23T18:51:49.351 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.352 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.352 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:51:49.352 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.836688+0000", 2023-09-23T18:51:49.352 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0039494949999999999 2023-09-23T18:51:49.352 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.352 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.352 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.352 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.836723+0000", 2023-09-23T18:51:49.352 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.5788999999999999e-05 2023-09-23T18:51:49.352 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.352 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.352 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.352 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.857354+0000", 2023-09-23T18:51:49.353 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.020630239000000002 2023-09-23T18:51:49.353 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.353 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.353 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.353 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.857528+0000", 2023-09-23T18:51:49.353 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00017434800000000001 2023-09-23T18:51:49.353 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.353 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.353 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:51:49.353 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.857600+0000", 2023-09-23T18:51:49.353 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.1851e-05 2023-09-23T18:51:49.353 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.354 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.354 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:51:49.354 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.858819+0000", 2023-09-23T18:51:49.354 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0012195750000000001 2023-09-23T18:51:49.354 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.354 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:49.354 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.354 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.354 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.354 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:88995 7.12 7:4e0ce39f:::benchmark_data_smithi098_155798_object88994:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e204)", 2023-09-23T18:51:49.354 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:17.067377+0000", 2023-09-23T18:51:49.354 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 452.26063981200002, 2023-09-23T18:51:49.355 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.7924068630000001, 2023-09-23T18:51:49.355 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:49.355 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:51:49.355 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:51:49.355 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:51:49.355 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:51:49.355 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 88995 2023-09-23T18:51:49.355 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.355 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:49.355 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.355 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:49.355 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067377+0000", 2023-09-23T18:51:49.355 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.356 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.356 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.356 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:49.356 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067377+0000", 2023-09-23T18:51:49.356 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.356 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.356 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.356 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:49.356 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067376+0000", 2023-09-23T18:51:49.356 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:51:49.356 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.356 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.356 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:49.357 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067403+0000", 2023-09-23T18:51:49.357 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.6882000000000001e-05 2023-09-23T18:51:49.357 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.357 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.357 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:49.357 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067408+0000", 2023-09-23T18:51:49.357 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.1730000000000003e-06 2023-09-23T18:51:49.357 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.357 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.357 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:49.357 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067412+0000", 2023-09-23T18:51:49.358 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.6969999999999999e-06 2023-09-23T18:51:49.358 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.358 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.358 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.358 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067510+0000", 2023-09-23T18:51:49.358 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.8091000000000007e-05 2023-09-23T18:51:49.358 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.358 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.358 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.358 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067517+0000", 2023-09-23T18:51:49.358 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.798e-06 2023-09-23T18:51:49.358 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.359 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.359 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.359 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.833787+0000", 2023-09-23T18:51:49.359 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.7662698400000001 2023-09-23T18:51:49.359 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.359 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.359 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:51:49.359 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.833900+0000", 2023-09-23T18:51:49.359 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000113557 2023-09-23T18:51:49.359 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.359 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.359 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,2,6", 2023-09-23T18:51:49.359 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.834096+0000", 2023-09-23T18:51:49.360 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00019535200000000001 2023-09-23T18:51:49.360 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.360 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.360 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.360 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.836778+0000", 2023-09-23T18:51:49.360 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00268213 2023-09-23T18:51:49.360 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.360 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.360 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:51:49.360 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.836788+0000", 2023-09-23T18:51:49.360 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0088999999999999e-05 2023-09-23T18:51:49.360 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.360 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.361 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.361 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.857393+0000", 2023-09-23T18:51:49.361 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.020604560000000001 2023-09-23T18:51:49.361 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.361 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.361 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.361 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.858896+0000", 2023-09-23T18:51:49.361 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.001503321 2023-09-23T18:51:49.361 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.361 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.361 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:51:49.361 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.858979+0000", 2023-09-23T18:51:49.362 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.3078000000000007e-05 2023-09-23T18:51:49.362 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.362 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.362 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:51:49.362 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.859784+0000", 2023-09-23T18:51:49.362 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00080483799999999999 2023-09-23T18:51:49.362 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.362 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:49.362 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.362 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.362 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.362 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:88996 7.12 7:4c598d91:::benchmark_data_smithi098_155798_object88995:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e204)", 2023-09-23T18:51:49.363 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:17.067437+0000", 2023-09-23T18:51:49.363 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 452.260579474, 2023-09-23T18:51:49.363 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.804874629, 2023-09-23T18:51:49.363 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:49.363 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:51:49.363 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:51:49.363 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:51:49.363 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:51:49.363 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 88996 2023-09-23T18:51:49.363 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.363 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:49.363 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.363 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:49.364 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067437+0000", 2023-09-23T18:51:49.364 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.364 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.364 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.364 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:49.364 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067437+0000", 2023-09-23T18:51:49.364 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.364 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.364 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.364 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:49.364 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067437+0000", 2023-09-23T18:51:49.364 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:51:49.364 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.365 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.365 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:49.365 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067491+0000", 2023-09-23T18:51:49.365 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.4487000000000002e-05 2023-09-23T18:51:49.365 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.365 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.365 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:49.365 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067501+0000", 2023-09-23T18:51:49.365 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.3030000000000005e-06 2023-09-23T18:51:49.365 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.365 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.365 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:49.365 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067506+0000", 2023-09-23T18:51:49.366 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.9840000000000001e-06 2023-09-23T18:51:49.366 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.366 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.366 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.366 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067543+0000", 2023-09-23T18:51:49.366 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.7391999999999997e-05 2023-09-23T18:51:49.366 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.366 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.366 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.366 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067554+0000", 2023-09-23T18:51:49.367 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1099e-05 2023-09-23T18:51:49.367 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.367 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.367 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.367 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.835098+0000", 2023-09-23T18:51:49.367 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.7675440500000001 2023-09-23T18:51:49.367 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.367 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.367 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:51:49.367 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.835267+0000", 2023-09-23T18:51:49.367 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00016878800000000001 2023-09-23T18:51:49.367 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.368 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.368 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,2,6", 2023-09-23T18:51:49.368 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.835521+0000", 2023-09-23T18:51:49.368 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000254317 2023-09-23T18:51:49.368 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.368 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.368 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:51:49.368 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.840048+0000", 2023-09-23T18:51:49.368 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0045265000000000001 2023-09-23T18:51:49.368 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.368 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.368 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.368 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.840093+0000", 2023-09-23T18:51:49.369 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.4824000000000003e-05 2023-09-23T18:51:49.369 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.369 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.369 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.369 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.864393+0000", 2023-09-23T18:51:49.369 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.024300381999999999 2023-09-23T18:51:49.369 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.369 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.369 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.369 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.870765+0000", 2023-09-23T18:51:49.369 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0063719909999999996 2023-09-23T18:51:49.369 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.370 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.370 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:51:49.370 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.870955+0000", 2023-09-23T18:51:49.370 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00019026199999999999 2023-09-23T18:51:49.370 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.370 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.370 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:51:49.370 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.872312+0000", 2023-09-23T18:51:49.370 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0013566800000000001 2023-09-23T18:51:49.370 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.370 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:49.370 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.371 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.371 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.371 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:89042 7.12 7:4d0a7dbe:::benchmark_data_smithi098_155798_object89041:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e204)", 2023-09-23T18:51:49.371 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:17.067512+0000", 2023-09-23T18:51:49.371 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 452.260504384, 2023-09-23T18:51:49.371 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.8092041409999999, 2023-09-23T18:51:49.371 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:49.371 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:51:49.371 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:51:49.371 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:51:49.371 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:51:49.371 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 89042 2023-09-23T18:51:49.371 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.372 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:49.372 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.372 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:49.372 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067512+0000", 2023-09-23T18:51:49.372 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.372 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.372 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.372 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:49.372 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067512+0000", 2023-09-23T18:51:49.372 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.372 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.372 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.373 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:49.373 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067512+0000", 2023-09-23T18:51:49.373 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:51:49.373 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.373 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.373 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:49.373 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067603+0000", 2023-09-23T18:51:49.373 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.1670999999999997e-05 2023-09-23T18:51:49.373 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.373 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.373 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:49.373 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067615+0000", 2023-09-23T18:51:49.374 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1525000000000001e-05 2023-09-23T18:51:49.374 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.374 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.374 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:49.374 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067622+0000", 2023-09-23T18:51:49.374 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.153e-06 2023-09-23T18:51:49.374 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.374 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.374 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.374 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067660+0000", 2023-09-23T18:51:49.374 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.8260999999999998e-05 2023-09-23T18:51:49.374 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.375 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.375 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.375 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067667+0000", 2023-09-23T18:51:49.375 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.9249999999999998e-06 2023-09-23T18:51:49.375 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.375 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.375 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.375 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.836823+0000", 2023-09-23T18:51:49.375 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.769156065 2023-09-23T18:51:49.375 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.375 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.375 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:51:49.376 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.836975+0000", 2023-09-23T18:51:49.376 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000152075 2023-09-23T18:51:49.376 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.376 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.376 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,2,6", 2023-09-23T18:51:49.376 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.837241+0000", 2023-09-23T18:51:49.376 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00026524200000000001 2023-09-23T18:51:49.376 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.376 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.376 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:51:49.376 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.840252+0000", 2023-09-23T18:51:49.376 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.003010988 2023-09-23T18:51:49.377 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.377 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.377 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.377 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.840289+0000", 2023-09-23T18:51:49.377 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.7172999999999998e-05 2023-09-23T18:51:49.377 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.377 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.377 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.377 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.864439+0000", 2023-09-23T18:51:49.377 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.024149968000000001 2023-09-23T18:51:49.377 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.378 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.378 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.378 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.872393+0000", 2023-09-23T18:51:49.378 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0079541209999999998 2023-09-23T18:51:49.378 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.378 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.378 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:51:49.378 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.872637+0000", 2023-09-23T18:51:49.378 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00024411999999999999 2023-09-23T18:51:49.378 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.378 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.378 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:51:49.379 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.876717+0000", 2023-09-23T18:51:49.379 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0040794519999999999 2023-09-23T18:51:49.379 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.379 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:49.379 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.379 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.379 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.379 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:89048 7.12 7:4a5891bb:::benchmark_data_smithi098_155798_object89047:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e204)", 2023-09-23T18:51:49.379 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:17.067630+0000", 2023-09-23T18:51:49.379 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 452.26038642600003, 2023-09-23T18:51:49.379 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.8157532240000001, 2023-09-23T18:51:49.379 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:49.380 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:51:49.380 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:51:49.380 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:51:49.380 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:51:49.380 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 89048 2023-09-23T18:51:49.380 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.380 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:49.380 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.380 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:49.380 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067630+0000", 2023-09-23T18:51:49.380 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.380 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.380 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.381 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:49.381 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067630+0000", 2023-09-23T18:51:49.381 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.381 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.381 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.381 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:49.381 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067630+0000", 2023-09-23T18:51:49.381 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:51:49.381 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.381 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.381 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:49.381 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067673+0000", 2023-09-23T18:51:49.382 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.3337000000000002e-05 2023-09-23T18:51:49.382 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.382 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.382 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:49.382 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067679+0000", 2023-09-23T18:51:49.382 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.6509999999999998e-06 2023-09-23T18:51:49.382 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.382 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.382 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:49.382 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067683+0000", 2023-09-23T18:51:49.382 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.1239999999999998e-06 2023-09-23T18:51:49.382 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.382 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.383 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.383 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067715+0000", 2023-09-23T18:51:49.383 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.1752999999999999e-05 2023-09-23T18:51:49.383 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.383 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.383 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.383 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067721+0000", 2023-09-23T18:51:49.383 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.3180000000000003e-06 2023-09-23T18:51:49.383 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.383 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.383 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.383 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.838418+0000", 2023-09-23T18:51:49.384 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.7706973700000002 2023-09-23T18:51:49.384 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.384 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.384 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:51:49.384 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.838585+0000", 2023-09-23T18:51:49.384 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00016633899999999999 2023-09-23T18:51:49.384 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.384 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.384 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,2,6", 2023-09-23T18:51:49.384 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.838852+0000", 2023-09-23T18:51:49.384 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00026699100000000001 2023-09-23T18:51:49.384 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.385 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.385 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.385 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.840338+0000", 2023-09-23T18:51:49.385 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0014868780000000001 2023-09-23T18:51:49.385 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.385 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.385 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:51:49.385 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.840348+0000", 2023-09-23T18:51:49.385 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.9569999999999997e-06 2023-09-23T18:51:49.385 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.385 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.385 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.386 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.868441+0000", 2023-09-23T18:51:49.386 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.028092310999999998 2023-09-23T18:51:49.386 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.386 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.386 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.386 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.881827+0000", 2023-09-23T18:51:49.386 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.013386376 2023-09-23T18:51:49.386 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.386 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.386 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:51:49.386 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.881900+0000", 2023-09-23T18:51:49.386 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.2614000000000006e-05 2023-09-23T18:51:49.387 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.387 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.387 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:51:49.387 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.883384+0000", 2023-09-23T18:51:49.387 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.001483875 2023-09-23T18:51:49.387 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.387 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:49.387 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.387 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.387 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.387 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:89063 7.12 7:4f7543cf:::benchmark_data_smithi098_155798_object89062:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e204)", 2023-09-23T18:51:49.387 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:17.067691+0000", 2023-09-23T18:51:49.388 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 452.260325868, 2023-09-23T18:51:49.388 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.819025036, 2023-09-23T18:51:49.388 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:49.388 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:51:49.388 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:51:49.388 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:51:49.388 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:51:49.388 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 89063 2023-09-23T18:51:49.388 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.388 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:49.388 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.388 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:49.388 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067691+0000", 2023-09-23T18:51:49.389 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.389 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.389 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.389 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:49.389 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067691+0000", 2023-09-23T18:51:49.389 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.389 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.389 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.389 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:49.389 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067690+0000", 2023-09-23T18:51:49.389 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:51:49.389 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.389 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.390 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:49.390 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067717+0000", 2023-09-23T18:51:49.390 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.6599000000000001e-05 2023-09-23T18:51:49.390 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.390 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.390 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:49.390 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067723+0000", 2023-09-23T18:51:49.390 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.5809999999999996e-06 2023-09-23T18:51:49.390 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.390 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.390 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:49.391 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067726+0000", 2023-09-23T18:51:49.391 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.8550000000000004e-06 2023-09-23T18:51:49.391 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.391 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.391 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.391 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067764+0000", 2023-09-23T18:51:49.391 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.7149e-05 2023-09-23T18:51:49.391 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.391 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.391 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.391 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067774+0000", 2023-09-23T18:51:49.391 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0276e-05 2023-09-23T18:51:49.392 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.392 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.392 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.392 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.840375+0000", 2023-09-23T18:51:49.392 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.7726012280000001 2023-09-23T18:51:49.392 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.392 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.392 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:51:49.392 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.840488+0000", 2023-09-23T18:51:49.392 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000112519 2023-09-23T18:51:49.392 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.392 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.392 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,2,6", 2023-09-23T18:51:49.393 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.840688+0000", 2023-09-23T18:51:49.393 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00020051699999999999 2023-09-23T18:51:49.393 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.393 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.393 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:51:49.393 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.844282+0000", 2023-09-23T18:51:49.393 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0035937780000000002 2023-09-23T18:51:49.393 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.393 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.393 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.393 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.852987+0000", 2023-09-23T18:51:49.393 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0087051869999999996 2023-09-23T18:51:49.394 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.394 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.394 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.394 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.876782+0000", 2023-09-23T18:51:49.394 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.02379496 2023-09-23T18:51:49.394 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.394 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.394 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.394 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.883447+0000", 2023-09-23T18:51:49.394 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0066650909999999997 2023-09-23T18:51:49.394 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.395 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.395 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:51:49.395 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.883640+0000", 2023-09-23T18:51:49.395 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000192552 2023-09-23T18:51:49.395 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.395 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.395 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:51:49.395 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.886716+0000", 2023-09-23T18:51:49.395 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0030762699999999999 2023-09-23T18:51:49.395 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.395 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:49.395 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.396 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.396 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.396 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:89096 7.12 7:4b7e6e24:::benchmark_data_smithi098_155798_object89095:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e204)", 2023-09-23T18:51:49.396 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:17.067732+0000", 2023-09-23T18:51:49.396 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 452.26028460200001, 2023-09-23T18:51:49.396 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.8204347279999999, 2023-09-23T18:51:49.396 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:49.396 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:51:49.396 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:51:49.396 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:51:49.396 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:51:49.396 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 89096 2023-09-23T18:51:49.397 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.397 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:49.397 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.397 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:49.397 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067732+0000", 2023-09-23T18:51:49.397 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.397 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.397 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.397 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:49.397 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067732+0000", 2023-09-23T18:51:49.397 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.397 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.397 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.398 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:49.398 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067732+0000", 2023-09-23T18:51:49.398 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:51:49.398 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.398 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.398 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:49.398 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067756+0000", 2023-09-23T18:51:49.398 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.3890999999999999e-05 2023-09-23T18:51:49.398 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.398 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.398 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:49.398 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067763+0000", 2023-09-23T18:51:49.399 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.0260000000000002e-06 2023-09-23T18:51:49.399 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.399 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.399 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:49.399 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067769+0000", 2023-09-23T18:51:49.399 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.0050000000000001e-06 2023-09-23T18:51:49.399 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.399 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.399 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.399 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067807+0000", 2023-09-23T18:51:49.399 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.8751000000000002e-05 2023-09-23T18:51:49.400 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.400 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.400 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.400 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067815+0000", 2023-09-23T18:51:49.400 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.2629999999999997e-06 2023-09-23T18:51:49.400 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.400 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.400 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.400 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.841659+0000", 2023-09-23T18:51:49.400 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.7738441950000001 2023-09-23T18:51:49.400 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.400 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.400 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:51:49.401 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.841766+0000", 2023-09-23T18:51:49.401 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000106808 2023-09-23T18:51:49.401 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.401 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.401 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,2,6", 2023-09-23T18:51:49.401 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.841930+0000", 2023-09-23T18:51:49.401 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00016433299999999999 2023-09-23T18:51:49.401 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.401 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.401 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:51:49.401 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.845624+0000", 2023-09-23T18:51:49.401 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0036938230000000002 2023-09-23T18:51:49.402 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.402 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.402 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.402 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.853038+0000", 2023-09-23T18:51:49.402 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0074141390000000001 2023-09-23T18:51:49.402 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.402 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.402 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.402 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.876823+0000", 2023-09-23T18:51:49.402 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.023785524999999998 2023-09-23T18:51:49.402 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.402 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.402 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.403 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.886776+0000", 2023-09-23T18:51:49.403 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0099523770000000001 2023-09-23T18:51:49.403 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.403 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.403 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:51:49.403 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.886845+0000", 2023-09-23T18:51:49.403 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.8682000000000004e-05 2023-09-23T18:51:49.403 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.403 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.403 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:51:49.403 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.888167+0000", 2023-09-23T18:51:49.403 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.001322387 2023-09-23T18:51:49.403 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.404 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:49.404 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.404 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.404 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.404 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:89108 7.12 7:4c09baad:::benchmark_data_smithi098_155798_object89107:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e204)", 2023-09-23T18:51:49.404 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:17.067777+0000", 2023-09-23T18:51:49.404 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 452.26023974600002, 2023-09-23T18:51:49.404 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.8220463360000001, 2023-09-23T18:51:49.404 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:49.404 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:51:49.404 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:51:49.404 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:51:49.405 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:51:49.405 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 89108 2023-09-23T18:51:49.405 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.405 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:49.405 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.405 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:49.405 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067777+0000", 2023-09-23T18:51:49.405 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.405 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.405 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.405 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:49.405 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067777+0000", 2023-09-23T18:51:49.405 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.406 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.406 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.406 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:49.406 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067777+0000", 2023-09-23T18:51:49.406 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:51:49.406 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.406 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.406 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:49.406 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067799+0000", 2023-09-23T18:51:49.406 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.2374999999999998e-05 2023-09-23T18:51:49.406 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.406 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.406 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:49.407 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067806+0000", 2023-09-23T18:51:49.407 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.1010000000000003e-06 2023-09-23T18:51:49.407 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.407 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.407 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:49.407 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067812+0000", 2023-09-23T18:51:49.407 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.8760000000000002e-06 2023-09-23T18:51:49.407 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.407 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.407 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.407 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067852+0000", 2023-09-23T18:51:49.407 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.9681000000000002e-05 2023-09-23T18:51:49.408 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.408 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.408 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.408 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067860+0000", 2023-09-23T18:51:49.408 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.8510000000000005e-06 2023-09-23T18:51:49.408 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.408 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.408 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.408 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.842954+0000", 2023-09-23T18:51:49.408 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.7750934730000001 2023-09-23T18:51:49.408 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.408 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.409 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:51:49.409 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.843069+0000", 2023-09-23T18:51:49.409 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00011529 2023-09-23T18:51:49.409 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.409 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.409 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,2,6", 2023-09-23T18:51:49.409 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.843247+0000", 2023-09-23T18:51:49.409 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00017771900000000001 2023-09-23T18:51:49.409 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.409 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.409 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:51:49.409 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.847384+0000", 2023-09-23T18:51:49.409 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0041367490000000003 2023-09-23T18:51:49.410 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.410 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.410 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.410 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.853075+0000", 2023-09-23T18:51:49.410 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0056918280000000003 2023-09-23T18:51:49.410 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.410 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.410 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.410 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.876854+0000", 2023-09-23T18:51:49.410 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.023778560000000001 2023-09-23T18:51:49.410 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.410 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.411 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.411 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.888316+0000", 2023-09-23T18:51:49.411 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.011462393 2023-09-23T18:51:49.411 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.411 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.411 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:51:49.411 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.888379+0000", 2023-09-23T18:51:49.411 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.3037e-05 2023-09-23T18:51:49.411 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.411 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.411 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:51:49.411 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.889823+0000", 2023-09-23T18:51:49.411 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.001443883 2023-09-23T18:51:49.412 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.412 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:49.412 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.412 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.412 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.412 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:89151 7.12 7:4869ff51:::benchmark_data_smithi098_155798_object89150:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e204)", 2023-09-23T18:51:49.412 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:17.067819+0000", 2023-09-23T18:51:49.412 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 452.26019793799998, 2023-09-23T18:51:49.412 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.823156191, 2023-09-23T18:51:49.412 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:49.412 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:51:49.412 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:51:49.413 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:51:49.413 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:51:49.413 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 89151 2023-09-23T18:51:49.413 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.413 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:49.413 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.413 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:49.413 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067819+0000", 2023-09-23T18:51:49.413 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.413 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.413 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.413 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:49.413 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067819+0000", 2023-09-23T18:51:49.414 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.414 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.414 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.414 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:49.414 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067818+0000", 2023-09-23T18:51:49.414 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:51:49.414 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.414 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.414 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:49.414 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067851+0000", 2023-09-23T18:51:49.414 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.2849999999999999e-05 2023-09-23T18:51:49.414 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.415 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.415 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:49.415 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067859+0000", 2023-09-23T18:51:49.415 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.0350000000000001e-06 2023-09-23T18:51:49.415 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.415 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.415 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:49.415 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067865+0000", 2023-09-23T18:51:49.415 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.5759999999999996e-06 2023-09-23T18:51:49.415 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.415 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.415 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.415 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067896+0000", 2023-09-23T18:51:49.416 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.1306999999999997e-05 2023-09-23T18:51:49.416 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.416 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.416 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.416 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067903+0000", 2023-09-23T18:51:49.416 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.8519999999999998e-06 2023-09-23T18:51:49.416 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.416 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.416 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.416 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.844313+0000", 2023-09-23T18:51:49.416 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.776410343 2023-09-23T18:51:49.416 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.416 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.417 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:51:49.417 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.844472+0000", 2023-09-23T18:51:49.417 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00015917399999999999 2023-09-23T18:51:49.417 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.417 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.417 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,2,6", 2023-09-23T18:51:49.417 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.844655+0000", 2023-09-23T18:51:49.417 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00018273900000000001 2023-09-23T18:51:49.417 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.417 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.417 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:51:49.417 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.847441+0000", 2023-09-23T18:51:49.417 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0027861079999999998 2023-09-23T18:51:49.418 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.418 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.418 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.418 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.857269+0000", 2023-09-23T18:51:49.418 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0098273990000000006 2023-09-23T18:51:49.418 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.418 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.418 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.418 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.876883+0000", 2023-09-23T18:51:49.418 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.019614571000000001 2023-09-23T18:51:49.418 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.418 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.419 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.419 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.889889+0000", 2023-09-23T18:51:49.419 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.013006208 2023-09-23T18:51:49.419 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.419 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.419 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:51:49.419 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.889950+0000", 2023-09-23T18:51:49.419 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.0532000000000003e-05 2023-09-23T18:51:49.419 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.419 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.419 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:51:49.419 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.890975+0000", 2023-09-23T18:51:49.420 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0010250369999999999 2023-09-23T18:51:49.420 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.420 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:49.420 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.420 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.420 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.420 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:89180 7.12 7:493661e7:::benchmark_data_smithi098_155798_object89179:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e204)", 2023-09-23T18:51:49.420 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:17.067872+0000", 2023-09-23T18:51:49.420 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 452.26014458399999, 2023-09-23T18:51:49.420 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.8241869770000001, 2023-09-23T18:51:49.420 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:49.420 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:51:49.421 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:51:49.421 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:51:49.421 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:51:49.421 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 89180 2023-09-23T18:51:49.421 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.421 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:49.421 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.421 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:49.421 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067872+0000", 2023-09-23T18:51:49.421 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.421 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.421 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.422 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:49.422 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067872+0000", 2023-09-23T18:51:49.422 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.422 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.422 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.422 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:49.422 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067872+0000", 2023-09-23T18:51:49.422 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:51:49.422 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.422 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.422 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:49.422 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067899+0000", 2023-09-23T18:51:49.422 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.745e-05 2023-09-23T18:51:49.423 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.423 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.423 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:49.423 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067904+0000", 2023-09-23T18:51:49.423 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.6450000000000004e-06 2023-09-23T18:51:49.423 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.423 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.423 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:49.423 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067907+0000", 2023-09-23T18:51:49.423 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.5329999999999999e-06 2023-09-23T18:51:49.423 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.423 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.424 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.424 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067937+0000", 2023-09-23T18:51:49.424 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.0097e-05 2023-09-23T18:51:49.424 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.424 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.424 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.424 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067944+0000", 2023-09-23T18:51:49.424 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.3620000000000002e-06 2023-09-23T18:51:49.424 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.424 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.424 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.424 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.847486+0000", 2023-09-23T18:51:49.424 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.779541834 2023-09-23T18:51:49.425 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.425 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.425 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:51:49.425 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.847710+0000", 2023-09-23T18:51:49.425 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00022469300000000001 2023-09-23T18:51:49.425 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.425 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.425 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,2,6", 2023-09-23T18:51:49.425 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.848090+0000", 2023-09-23T18:51:49.425 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00037930499999999997 2023-09-23T18:51:49.425 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.425 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.425 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:51:49.426 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.852939+0000", 2023-09-23T18:51:49.426 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0048494380000000002 2023-09-23T18:51:49.426 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.426 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.426 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.426 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.857312+0000", 2023-09-23T18:51:49.426 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0043725279999999997 2023-09-23T18:51:49.426 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.426 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.426 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.426 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.880645+0000", 2023-09-23T18:51:49.426 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.023333139999999999 2023-09-23T18:51:49.426 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.427 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.427 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.427 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.891037+0000", 2023-09-23T18:51:49.427 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.010392235 2023-09-23T18:51:49.427 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.427 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.427 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:51:49.427 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.891096+0000", 2023-09-23T18:51:49.427 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.9314999999999997e-05 2023-09-23T18:51:49.427 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.427 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.427 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:51:49.427 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.892059+0000", 2023-09-23T18:51:49.428 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00096289200000000002 2023-09-23T18:51:49.428 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.428 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:49.428 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.428 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.428 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.428 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:89222 7.12 7:4bedbc26:::benchmark_data_smithi098_155798_object89221:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e204)", 2023-09-23T18:51:49.428 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:17.067915+0000", 2023-09-23T18:51:49.428 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 452.26010186399998, 2023-09-23T18:51:49.428 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.8282308810000001, 2023-09-23T18:51:49.428 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:49.428 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:51:49.429 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:51:49.429 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:51:49.429 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:51:49.429 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 89222 2023-09-23T18:51:49.429 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.429 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:49.429 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.429 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:49.429 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067915+0000", 2023-09-23T18:51:49.429 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.429 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.429 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.430 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:49.430 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067915+0000", 2023-09-23T18:51:49.430 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.430 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.430 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.430 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:49.430 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067914+0000", 2023-09-23T18:51:49.430 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:51:49.430 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.430 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.430 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:49.430 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067941+0000", 2023-09-23T18:51:49.430 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.6361999999999999e-05 2023-09-23T18:51:49.431 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.431 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.431 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:49.431 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067954+0000", 2023-09-23T18:51:49.431 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3407000000000001e-05 2023-09-23T18:51:49.431 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.431 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.431 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:49.431 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067958+0000", 2023-09-23T18:51:49.431 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.8959999999999996e-06 2023-09-23T18:51:49.431 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.431 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.431 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.432 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067992+0000", 2023-09-23T18:51:49.432 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.4072000000000002e-05 2023-09-23T18:51:49.432 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.432 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.432 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.432 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.068000+0000", 2023-09-23T18:51:49.432 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.4390000000000003e-06 2023-09-23T18:51:49.432 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.432 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.432 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.432 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.849548+0000", 2023-09-23T18:51:49.432 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.781548691 2023-09-23T18:51:49.432 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.433 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.433 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:51:49.433 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.849721+0000", 2023-09-23T18:51:49.433 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00017322 2023-09-23T18:51:49.433 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.433 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.433 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,2,6", 2023-09-23T18:51:49.433 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.850016+0000", 2023-09-23T18:51:49.433 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000294538 2023-09-23T18:51:49.433 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.433 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.433 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:51:49.434 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.852997+0000", 2023-09-23T18:51:49.434 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0029812329999999998 2023-09-23T18:51:49.434 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.434 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.434 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.434 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.857443+0000", 2023-09-23T18:51:49.434 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0044461259999999999 2023-09-23T18:51:49.434 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.434 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.434 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.434 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.892114+0000", 2023-09-23T18:51:49.434 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.034670339000000001 2023-09-23T18:51:49.434 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.435 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.435 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.435 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.894627+0000", 2023-09-23T18:51:49.435 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0025135280000000001 2023-09-23T18:51:49.435 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.435 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.435 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:51:49.435 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.894688+0000", 2023-09-23T18:51:49.435 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.0380000000000001e-05 2023-09-23T18:51:49.435 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.435 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.435 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:51:49.436 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.896146+0000", 2023-09-23T18:51:49.436 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0014581559999999999 2023-09-23T18:51:49.436 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.436 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:49.436 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.436 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.436 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.436 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:89225 7.12 7:48911597:::benchmark_data_smithi098_155798_object89224:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e204)", 2023-09-23T18:51:49.436 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:17.067964+0000", 2023-09-23T18:51:49.436 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 452.26005245900001, 2023-09-23T18:51:49.436 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.8295856160000001, 2023-09-23T18:51:49.437 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:49.437 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:51:49.437 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:51:49.437 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:51:49.437 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:51:49.437 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 89225 2023-09-23T18:51:49.437 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.437 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:49.437 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.437 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:49.437 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067964+0000", 2023-09-23T18:51:49.437 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.438 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.438 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.438 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:49.438 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067964+0000", 2023-09-23T18:51:49.438 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.438 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.438 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.438 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:49.438 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067964+0000", 2023-09-23T18:51:49.438 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:51:49.438 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.438 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.439 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:49.439 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067988+0000", 2023-09-23T18:51:49.439 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.4324999999999998e-05 2023-09-23T18:51:49.439 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.439 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.439 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:49.439 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.067995+0000", 2023-09-23T18:51:49.439 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.8560000000000001e-06 2023-09-23T18:51:49.439 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.439 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.439 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:49.439 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.068000+0000", 2023-09-23T18:51:49.439 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.3669999999999996e-06 2023-09-23T18:51:49.440 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.440 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.440 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.440 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.068046+0000", 2023-09-23T18:51:49.440 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.6143999999999997e-05 2023-09-23T18:51:49.440 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.440 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.440 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.440 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.068053+0000", 2023-09-23T18:51:49.440 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.9639999999999997e-06 2023-09-23T18:51:49.440 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.440 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.441 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.441 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.851189+0000", 2023-09-23T18:51:49.441 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.7831354589999999 2023-09-23T18:51:49.441 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.441 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.441 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:51:49.441 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.851340+0000", 2023-09-23T18:51:49.441 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00015081 2023-09-23T18:51:49.441 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.441 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.441 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,2,6", 2023-09-23T18:51:49.441 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.851621+0000", 2023-09-23T18:51:49.442 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00028185299999999999 2023-09-23T18:51:49.442 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.442 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.442 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:51:49.442 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.857283+0000", 2023-09-23T18:51:49.442 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0056612989999999998 2023-09-23T18:51:49.442 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.442 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.442 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.442 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.857478+0000", 2023-09-23T18:51:49.442 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00019512199999999999 2023-09-23T18:51:49.442 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.443 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.443 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.443 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.893581+0000", 2023-09-23T18:51:49.443 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.036103108000000002 2023-09-23T18:51:49.443 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.443 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.443 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.443 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.896201+0000", 2023-09-23T18:51:49.443 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0026203860000000002 2023-09-23T18:51:49.443 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.443 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.443 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:51:49.443 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.896257+0000", 2023-09-23T18:51:49.444 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.5568000000000001e-05 2023-09-23T18:51:49.444 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.444 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.444 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:51:49.444 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.897550+0000", 2023-09-23T18:51:49.444 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0012930050000000001 2023-09-23T18:51:49.444 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.444 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:49.444 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.444 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.444 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.444 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:89247 7.12 7:4cee6b26:::benchmark_data_smithi098_155798_object89246:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e204)", 2023-09-23T18:51:49.444 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:44:17.068015+0000", 2023-09-23T18:51:49.445 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 452.26000213700001, 2023-09-23T18:51:49.445 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.836348112, 2023-09-23T18:51:49.445 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:49.445 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:51:49.445 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:51:49.445 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:51:49.445 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:51:49.445 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 89247 2023-09-23T18:51:49.445 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.445 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:49.445 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.445 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:49.446 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.068015+0000", 2023-09-23T18:51:49.446 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.446 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.446 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.446 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:49.446 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.068015+0000", 2023-09-23T18:51:49.446 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.446 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.446 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.446 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:49.446 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.068014+0000", 2023-09-23T18:51:49.446 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:51:49.447 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.447 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.447 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:49.447 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.068043+0000", 2023-09-23T18:51:49.447 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.8603999999999999e-05 2023-09-23T18:51:49.447 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.447 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.447 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:49.447 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.068050+0000", 2023-09-23T18:51:49.447 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.2799999999999998e-06 2023-09-23T18:51:49.447 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.447 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.447 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:49.448 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.068057+0000", 2023-09-23T18:51:49.448 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.8800000000000002e-06 2023-09-23T18:51:49.448 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.448 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.448 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.448 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.068087+0000", 2023-09-23T18:51:49.448 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.0595000000000001e-05 2023-09-23T18:51:49.448 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.448 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.448 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.448 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:17.068094+0000", 2023-09-23T18:51:49.448 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.3339999999999998e-06 2023-09-23T18:51:49.448 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.449 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.449 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.449 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.859819+0000", 2023-09-23T18:51:49.449 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.791725668 2023-09-23T18:51:49.449 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.449 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.449 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:51:49.449 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.859983+0000", 2023-09-23T18:51:49.449 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00016395900000000001 2023-09-23T18:51:49.449 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.449 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.449 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,2,6", 2023-09-23T18:51:49.450 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.862808+0000", 2023-09-23T18:51:49.450 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.002825195 2023-09-23T18:51:49.450 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.450 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.450 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:51:49.450 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.864580+0000", 2023-09-23T18:51:49.450 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0017710740000000001 2023-09-23T18:51:49.450 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.450 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.450 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.450 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.870433+0000", 2023-09-23T18:51:49.450 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0058530309999999999 2023-09-23T18:51:49.451 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.451 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.451 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.451 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.893649+0000", 2023-09-23T18:51:49.451 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.023216442 2023-09-23T18:51:49.451 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.451 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.451 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.451 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.902442+0000", 2023-09-23T18:51:49.451 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0087931869999999992 2023-09-23T18:51:49.451 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.451 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.452 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:51:49.452 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.902499+0000", 2023-09-23T18:51:49.452 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.6634e-05 2023-09-23T18:51:49.452 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.452 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.452 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:51:49.452 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:44:18.904363+0000", 2023-09-23T18:51:49.452 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.001863948 2023-09-23T18:51:49.452 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.452 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:49.452 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.452 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.453 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.453 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:159642 8.3 8:cadce5e6:::benchmark_data_smithi098_176514_object56164:head [delete] snapc 0=[] ondisk+write+known_if_redirected e256)", 2023-09-23T18:51:49.453 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:51:42.134999+0000", 2023-09-23T18:51:49.453 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 7.1930173980000003, 2023-09-23T18:51:49.453 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.1302095539999999, 2023-09-23T18:51:49.453 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:49.453 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:51:49.453 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:51:49.453 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:51:49.453 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:51:49.453 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 159642 2023-09-23T18:51:49.453 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.453 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:49.454 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.454 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:49.454 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.134999+0000", 2023-09-23T18:51:49.454 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.454 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.454 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.454 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:49.454 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.134999+0000", 2023-09-23T18:51:49.454 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.454 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.454 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.454 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:49.455 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.134999+0000", 2023-09-23T18:51:49.455 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:51:49.455 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.455 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.455 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:49.455 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.135004+0000", 2023-09-23T18:51:49.455 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.4480000000000002e-06 2023-09-23T18:51:49.455 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.455 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.455 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:49.455 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.135013+0000", 2023-09-23T18:51:49.455 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.2669999999999995e-06 2023-09-23T18:51:49.455 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.456 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.456 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:49.456 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.135020+0000", 2023-09-23T18:51:49.456 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.3040000000000001e-06 2023-09-23T18:51:49.456 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.456 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.456 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.456 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.137121+0000", 2023-09-23T18:51:49.456 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0021015669999999999 2023-09-23T18:51:49.456 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.456 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.456 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.456 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.137136+0000", 2023-09-23T18:51:49.457 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.4681000000000001e-05 2023-09-23T18:51:49.457 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.457 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.457 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.457 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:43.072508+0000", 2023-09-23T18:51:49.457 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.935372494 2023-09-23T18:51:49.457 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.457 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.457 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.457 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:43.072529+0000", 2023-09-23T18:51:49.457 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.0373e-05 2023-09-23T18:51:49.457 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.457 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.458 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.458 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.242889+0000", 2023-09-23T18:51:49.458 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1703600240000001 2023-09-23T18:51:49.458 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.458 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.458 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:51:49.458 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.243057+0000", 2023-09-23T18:51:49.458 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00016831499999999999 2023-09-23T18:51:49.458 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.458 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.458 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T18:51:49.458 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.243755+0000", 2023-09-23T18:51:49.458 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00069763699999999998 2023-09-23T18:51:49.459 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.459 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.459 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.459 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.256620+0000", 2023-09-23T18:51:49.459 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.012865501 2023-09-23T18:51:49.459 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.459 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.459 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.459 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.256700+0000", 2023-09-23T18:51:49.459 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.9764000000000004e-05 2023-09-23T18:51:49.459 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.459 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.460 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.460 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.256773+0000", 2023-09-23T18:51:49.460 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.3165999999999997e-05 2023-09-23T18:51:49.460 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.460 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.460 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:51:49.460 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.265001+0000", 2023-09-23T18:51:49.460 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0082283240000000004 2023-09-23T18:51:49.460 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.460 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.460 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:51:49.460 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.265060+0000", 2023-09-23T18:51:49.461 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.8501000000000001e-05 2023-09-23T18:51:49.461 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.461 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.461 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:51:49.461 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.265209+0000", 2023-09-23T18:51:49.461 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00014898900000000001 2023-09-23T18:51:49.461 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.461 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:49.461 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.461 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.461 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.461 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:159661 8.3 8:c78bae83:::benchmark_data_smithi098_176514_object56183:head [delete] snapc 0=[] ondisk+write+known_if_redirected e256)", 2023-09-23T18:51:49.461 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:51:42.177720+0000", 2023-09-23T18:51:49.462 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 7.1502963800000003, 2023-09-23T18:51:49.462 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.0918106490000001, 2023-09-23T18:51:49.462 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:49.462 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:51:49.462 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:51:49.462 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:51:49.462 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:51:49.462 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 159661 2023-09-23T18:51:49.462 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.462 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:49.462 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.462 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:49.463 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.177720+0000", 2023-09-23T18:51:49.463 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.463 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.463 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.463 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:49.463 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.177720+0000", 2023-09-23T18:51:49.463 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.463 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.463 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.463 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:49.463 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.177719+0000", 2023-09-23T18:51:49.464 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T18:51:49.464 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.464 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.464 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:49.464 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.177728+0000", 2023-09-23T18:51:49.464 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.4339999999999998e-06 2023-09-23T18:51:49.464 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.464 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.464 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:49.464 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.177747+0000", 2023-09-23T18:51:49.464 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.9156000000000001e-05 2023-09-23T18:51:49.464 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.465 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.465 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:49.465 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.177754+0000", 2023-09-23T18:51:49.465 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.255e-06 2023-09-23T18:51:49.465 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.465 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.465 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.465 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.177822+0000", 2023-09-23T18:51:49.465 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.7792000000000004e-05 2023-09-23T18:51:49.465 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.465 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.465 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.465 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.177831+0000", 2023-09-23T18:51:49.466 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.9190000000000005e-06 2023-09-23T18:51:49.466 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.466 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.466 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.466 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:43.072559+0000", 2023-09-23T18:51:49.466 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.89472804500000003 2023-09-23T18:51:49.466 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.466 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.466 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.466 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:43.072569+0000", 2023-09-23T18:51:49.466 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.764e-06 2023-09-23T18:51:49.467 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.467 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.467 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.467 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.256826+0000", 2023-09-23T18:51:49.467 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1842568199999999 2023-09-23T18:51:49.467 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.467 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.467 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:51:49.467 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.256984+0000", 2023-09-23T18:51:49.467 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00015851799999999999 2023-09-23T18:51:49.467 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.467 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.467 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T18:51:49.468 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.257128+0000", 2023-09-23T18:51:49.468 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00014333799999999999 2023-09-23T18:51:49.468 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.468 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.468 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.468 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.264925+0000", 2023-09-23T18:51:49.468 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0077970160000000004 2023-09-23T18:51:49.468 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.468 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.468 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:51:49.468 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.265223+0000", 2023-09-23T18:51:49.469 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00029835899999999999 2023-09-23T18:51:49.469 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.469 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.469 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.469 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.269250+0000", 2023-09-23T18:51:49.469 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0040271020000000003 2023-09-23T18:51:49.469 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.469 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.469 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.469 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.269349+0000", 2023-09-23T18:51:49.469 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.8581999999999999e-05 2023-09-23T18:51:49.469 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.469 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.470 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:51:49.470 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.269404+0000", 2023-09-23T18:51:49.470 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.5686999999999997e-05 2023-09-23T18:51:49.470 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.470 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.470 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:51:49.470 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.269531+0000", 2023-09-23T18:51:49.470 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000126606 2023-09-23T18:51:49.470 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.470 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:49.470 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.470 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.470 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.471 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:159666 8.3 8:c3cbf68a:::benchmark_data_smithi098_176514_object56188:head [delete] snapc 0=[] ondisk+write+known_if_redirected e256)", 2023-09-23T18:51:49.471 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:51:42.189753+0000", 2023-09-23T18:51:49.471 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 7.1382642260000004, 2023-09-23T18:51:49.471 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.088349258, 2023-09-23T18:51:49.471 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:49.471 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:51:49.471 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:51:49.471 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:51:49.471 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:51:49.471 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 159666 2023-09-23T18:51:49.471 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.472 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:49.472 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.472 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:49.472 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.189753+0000", 2023-09-23T18:51:49.472 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.472 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.472 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.472 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:49.472 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.189753+0000", 2023-09-23T18:51:49.472 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.472 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.472 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.473 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:49.473 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.189751+0000", 2023-09-23T18:51:49.473 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T18:51:49.473 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.473 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.473 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:49.473 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.189760+0000", 2023-09-23T18:51:49.473 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.8550000000000008e-06 2023-09-23T18:51:49.473 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.473 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.473 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:49.473 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.189780+0000", 2023-09-23T18:51:49.473 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.9823e-05 2023-09-23T18:51:49.474 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.474 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.474 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:49.474 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.189790+0000", 2023-09-23T18:51:49.474 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0129000000000001e-05 2023-09-23T18:51:49.474 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.474 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.474 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.474 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.189864+0000", 2023-09-23T18:51:49.474 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.4560000000000004e-05 2023-09-23T18:51:49.474 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.474 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.475 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.475 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.189878+0000", 2023-09-23T18:51:49.475 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.4034e-05 2023-09-23T18:51:49.475 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.475 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.475 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.475 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:43.072593+0000", 2023-09-23T18:51:49.475 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.88271504899999997 2023-09-23T18:51:49.475 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.475 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.475 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.475 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:43.072602+0000", 2023-09-23T18:51:49.476 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.8999999999999995e-06 2023-09-23T18:51:49.476 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.476 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.476 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.476 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.260516+0000", 2023-09-23T18:51:49.476 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1879133719999999 2023-09-23T18:51:49.476 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.476 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.476 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:51:49.476 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.260684+0000", 2023-09-23T18:51:49.476 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00016815999999999999 2023-09-23T18:51:49.476 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.477 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.477 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T18:51:49.477 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.260868+0000", 2023-09-23T18:51:49.477 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00018460899999999999 2023-09-23T18:51:49.477 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.477 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.477 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.477 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.264987+0000", 2023-09-23T18:51:49.477 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0041186219999999997 2023-09-23T18:51:49.477 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.477 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.477 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:51:49.477 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.269265+0000", 2023-09-23T18:51:49.478 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0042779640000000004 2023-09-23T18:51:49.478 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.478 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.478 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.478 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.269307+0000", 2023-09-23T18:51:49.478 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.2024999999999998e-05 2023-09-23T18:51:49.478 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.478 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.478 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.478 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.277880+0000", 2023-09-23T18:51:49.478 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0085734130000000002 2023-09-23T18:51:49.478 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.479 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.479 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:51:49.479 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.277948+0000", 2023-09-23T18:51:49.479 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.7912999999999998e-05 2023-09-23T18:51:49.479 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.479 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.479 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:51:49.479 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.278102+0000", 2023-09-23T18:51:49.479 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00015351199999999999 2023-09-23T18:51:49.479 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.479 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:49.479 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.479 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.480 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.480 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:159669 8.3 8:cc8e7570:::benchmark_data_smithi098_176514_object56191:head [delete] snapc 0=[] ondisk+write+known_if_redirected e256)", 2023-09-23T18:51:49.480 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:51:42.199325+0000", 2023-09-23T18:51:49.480 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 7.1286921789999997, 2023-09-23T18:51:49.480 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.0791055709999999, 2023-09-23T18:51:49.480 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:49.480 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:51:49.480 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:51:49.480 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:51:49.480 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:51:49.480 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 159669 2023-09-23T18:51:49.480 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.480 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:49.481 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.481 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:49.481 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.199325+0000", 2023-09-23T18:51:49.481 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.481 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.481 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.481 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:49.481 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.199325+0000", 2023-09-23T18:51:49.481 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.481 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.481 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.481 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:49.481 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.199323+0000", 2023-09-23T18:51:49.482 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T18:51:49.482 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.482 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.482 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:49.482 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.199332+0000", 2023-09-23T18:51:49.482 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.166e-06 2023-09-23T18:51:49.482 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.482 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.482 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:49.482 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.199355+0000", 2023-09-23T18:51:49.482 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.2663999999999998e-05 2023-09-23T18:51:49.482 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.483 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.483 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:49.483 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.199364+0000", 2023-09-23T18:51:49.483 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.8300000000000008e-06 2023-09-23T18:51:49.483 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.483 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.483 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.483 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.199435+0000", 2023-09-23T18:51:49.483 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.0331000000000001e-05 2023-09-23T18:51:49.483 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.483 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.483 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.483 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.199449+0000", 2023-09-23T18:51:49.484 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.4329e-05 2023-09-23T18:51:49.484 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.484 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.484 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.484 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:43.072631+0000", 2023-09-23T18:51:49.484 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.87318185500000001 2023-09-23T18:51:49.484 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.484 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.484 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.484 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:43.072641+0000", 2023-09-23T18:51:49.484 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.8519999999999999e-06 2023-09-23T18:51:49.484 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.484 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.485 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.485 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.265274+0000", 2023-09-23T18:51:49.485 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.192633346 2023-09-23T18:51:49.485 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.485 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.485 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:51:49.485 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.265445+0000", 2023-09-23T18:51:49.485 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000170812 2023-09-23T18:51:49.485 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.485 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.485 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T18:51:49.485 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.265611+0000", 2023-09-23T18:51:49.486 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000166529 2023-09-23T18:51:49.486 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.486 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.486 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.486 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.269591+0000", 2023-09-23T18:51:49.486 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0039792860000000003 2023-09-23T18:51:49.486 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.486 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.486 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:51:49.486 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.273717+0000", 2023-09-23T18:51:49.486 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0041265879999999996 2023-09-23T18:51:49.486 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.486 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.487 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.487 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.277786+0000", 2023-09-23T18:51:49.487 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0040685549999999997 2023-09-23T18:51:49.487 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.487 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.487 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.487 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.278176+0000", 2023-09-23T18:51:49.487 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00039034600000000002 2023-09-23T18:51:49.487 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.487 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.487 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:51:49.487 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.278299+0000", 2023-09-23T18:51:49.488 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000123189 2023-09-23T18:51:49.488 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.488 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.488 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:51:49.488 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.278430+0000", 2023-09-23T18:51:49.488 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000130739 2023-09-23T18:51:49.488 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.488 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:49.488 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.488 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.488 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.488 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:159670 8.3 8:c9836ad6:::benchmark_data_smithi098_176514_object56192:head [delete] snapc 0=[] ondisk+write+known_if_redirected e256)", 2023-09-23T18:51:49.489 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:51:42.199460+0000", 2023-09-23T18:51:49.489 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 7.128556433, 2023-09-23T18:51:49.489 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.0926747140000002, 2023-09-23T18:51:49.489 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:49.489 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:51:49.489 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:51:49.489 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:51:49.489 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:51:49.489 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 159670 2023-09-23T18:51:49.489 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.489 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:49.490 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.490 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:49.490 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.199460+0000", 2023-09-23T18:51:49.490 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.490 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.490 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.490 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:49.490 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.199460+0000", 2023-09-23T18:51:49.490 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.490 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.490 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.491 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:49.491 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.199459+0000", 2023-09-23T18:51:49.491 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:51:49.491 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.491 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.491 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:49.491 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.199464+0000", 2023-09-23T18:51:49.491 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.989e-06 2023-09-23T18:51:49.491 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.491 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.491 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:49.491 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.199476+0000", 2023-09-23T18:51:49.491 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1420000000000001e-05 2023-09-23T18:51:49.492 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.492 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.492 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:49.492 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.199482+0000", 2023-09-23T18:51:49.492 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.981e-06 2023-09-23T18:51:49.492 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.492 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.492 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.492 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.199525+0000", 2023-09-23T18:51:49.492 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.3229000000000002e-05 2023-09-23T18:51:49.492 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.492 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.493 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.493 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.199532+0000", 2023-09-23T18:51:49.493 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.0310000000000001e-06 2023-09-23T18:51:49.493 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.493 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.493 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.493 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:43.072667+0000", 2023-09-23T18:51:49.493 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.87313442799999996 2023-09-23T18:51:49.493 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.493 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.493 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.493 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:43.072676+0000", 2023-09-23T18:51:49.493 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.4870000000000008e-06 2023-09-23T18:51:49.493 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.494 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.494 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.494 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.269631+0000", 2023-09-23T18:51:49.494 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.196954801 2023-09-23T18:51:49.494 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.494 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.494 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:51:49.494 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.269772+0000", 2023-09-23T18:51:49.494 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00014142899999999999 2023-09-23T18:51:49.494 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.494 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.494 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T18:51:49.495 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.269925+0000", 2023-09-23T18:51:49.495 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00015308100000000001 2023-09-23T18:51:49.495 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.495 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.495 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.495 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.277833+0000", 2023-09-23T18:51:49.495 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0079080950000000004 2023-09-23T18:51:49.495 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.495 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.495 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.495 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.278482+0000", 2023-09-23T18:51:49.495 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00064835999999999995 2023-09-23T18:51:49.496 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.496 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.496 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:51:49.496 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.282851+0000", 2023-09-23T18:51:49.496 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0043689660000000002 2023-09-23T18:51:49.496 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.496 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.496 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.496 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.291922+0000", 2023-09-23T18:51:49.496 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0090711509999999995 2023-09-23T18:51:49.496 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.496 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.496 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:51:49.497 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.291986+0000", 2023-09-23T18:51:49.497 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.4321999999999998e-05 2023-09-23T18:51:49.497 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.497 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.497 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:51:49.497 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.292135+0000", 2023-09-23T18:51:49.497 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000148818 2023-09-23T18:51:49.497 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.497 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:49.497 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.497 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.497 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.497 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:159673 8.3 8:c7ee463d:::benchmark_data_smithi098_176514_object56195:head [delete] snapc 0=[] ondisk+write+known_if_redirected e256)", 2023-09-23T18:51:49.498 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:51:42.216666+0000", 2023-09-23T18:51:49.498 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 7.1113511389999999, 2023-09-23T18:51:49.498 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.0757003389999999, 2023-09-23T18:51:49.498 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:51:49.498 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:51:49.498 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:51:49.498 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:51:49.498 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:51:49.498 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 159673 2023-09-23T18:51:49.498 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.498 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:51:49.498 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.499 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:51:49.499 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.216666+0000", 2023-09-23T18:51:49.499 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.499 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.499 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.499 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:51:49.499 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.216666+0000", 2023-09-23T18:51:49.499 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:51:49.499 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.499 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.499 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:51:49.499 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.216664+0000", 2023-09-23T18:51:49.500 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T18:51:49.500 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.500 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.500 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:51:49.500 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.216674+0000", 2023-09-23T18:51:49.500 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0091999999999999e-05 2023-09-23T18:51:49.500 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.500 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.500 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:51:49.500 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.216694+0000", 2023-09-23T18:51:49.500 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.0630000000000001e-05 2023-09-23T18:51:49.500 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.501 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.501 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:51:49.501 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.216706+0000", 2023-09-23T18:51:49.501 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1534e-05 2023-09-23T18:51:49.501 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.501 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.501 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.501 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.216793+0000", 2023-09-23T18:51:49.501 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.6802e-05 2023-09-23T18:51:49.501 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.501 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.501 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.502 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:42.216811+0000", 2023-09-23T18:51:49.502 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.8558000000000001e-05 2023-09-23T18:51:49.502 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.502 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.502 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.502 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:43.072704+0000", 2023-09-23T18:51:49.502 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.85589219500000002 2023-09-23T18:51:49.502 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.502 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.502 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:51:49.502 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:43.072714+0000", 2023-09-23T18:51:49.503 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.9939999999999995e-06 2023-09-23T18:51:49.503 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.503 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.503 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:51:49.503 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.273752+0000", 2023-09-23T18:51:49.503 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.2010384080000001 2023-09-23T18:51:49.503 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.503 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.503 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:51:49.503 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.273912+0000", 2023-09-23T18:51:49.503 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00015973300000000001 2023-09-23T18:51:49.504 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.504 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.504 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T18:51:49.504 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.274083+0000", 2023-09-23T18:51:49.504 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00017084399999999999 2023-09-23T18:51:49.504 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.504 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.504 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.504 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.278981+0000", 2023-09-23T18:51:49.504 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0048986309999999996 2023-09-23T18:51:49.504 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.504 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.505 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:51:49.505 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.282862+0000", 2023-09-23T18:51:49.505 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0038812320000000001 2023-09-23T18:51:49.505 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.505 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.505 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.505 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.284165+0000", 2023-09-23T18:51:49.505 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0013029980000000001 2023-09-23T18:51:49.505 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.505 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.505 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:51:49.505 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.292203+0000", 2023-09-23T18:51:49.505 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0080373500000000004 2023-09-23T18:51:49.506 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.506 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.506 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:51:49.506 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.292250+0000", 2023-09-23T18:51:49.506 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.689e-05 2023-09-23T18:51:49.506 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:51:49.506 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:51:49.506 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:51:49.506 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.292366+0000", 2023-09-23T18:51:49.506 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000116366 2023-09-23T18:51:49.506 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.506 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:49.507 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.507 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:51:49.507 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:51:49.507 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:51:49.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:49.509 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:49.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:49.609 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:49.636 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:49.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:49.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:49.740 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:49.763 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:49.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:49.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:49.867 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:49.892 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:49.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:49.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:49.995 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:50.026 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:50.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:50.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:50.128 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:50.151 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:50.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:50.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:50.255 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:50.262 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:50 smithi173 bash[59422]: cluster 2023-09-23T18:51:50.156197+0000 mgr.y (mgr.26054) 1015 : cluster 0 pgmap v1082: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 169 op/s; 63134/142305 objects degraded (44.365%); 5513/142305 objects misplaced (3.874%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T18:51:50.262 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:50 smithi173 bash[59422]: cluster 2023-09-23T18:51:50.180268+0000 mon.a (mon.0) 1389 : cluster 3 Health check update: Degraded data redundancy: 63134/142356 objects degraded (44.349%), 13 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:50.278 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:50.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:50.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:50.381 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:50.408 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:50.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:50.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:50.485 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T18:51:50.497 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:50 smithi098 bash[79516]: cluster 2023-09-23T18:51:50.156197+0000 mgr.y (mgr.26054) 1015 : cluster 0 pgmap v1082: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 169 op/s; 63134/142305 objects degraded (44.365%); 5513/142305 objects misplaced (3.874%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T18:51:50.497 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:50 smithi098 bash[79516]: cluster 2023-09-23T18:51:50.180268+0000 mon.a (mon.0) 1389 : cluster 3 Health check update: Degraded data redundancy: 63134/142356 objects degraded (44.349%), 13 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:50.498 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:50 smithi098 bash[76817]: cluster 2023-09-23T18:51:50.156197+0000 mgr.y (mgr.26054) 1015 : cluster 0 pgmap v1082: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 169 op/s; 63134/142305 objects degraded (44.365%); 5513/142305 objects misplaced (3.874%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T18:51:50.498 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:50 smithi098 bash[76817]: cluster 2023-09-23T18:51:50.180268+0000 mon.a (mon.0) 1389 : cluster 3 Health check update: Degraded data redundancy: 63134/142356 objects degraded (44.349%), 13 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:50.511 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:50.538 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:50.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:50.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:50.641 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:50.669 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:50.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:50.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:50.772 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:50.802 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:50.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:50.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:50.905 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:50.927 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:50.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:50.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:51.035 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:51.061 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:51.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:51.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:51.164 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:51.190 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:51.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:51.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:51.297 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:51.324 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:51.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:51.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:51.344 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T18:51:51.344 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:51:51.345 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T18:51:51.345 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.652757 2023-09-23T18:51:51.345 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 0.6527572438099544 2023-09-23T18:51:51.427 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:51.453 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:51.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:51.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:51.556 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:51.585 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:51.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:51.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:51.689 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:51.717 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:51.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:51.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:51.821 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:51.846 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:51.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:51.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:51.948 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:51.985 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:51.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:51.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:52.088 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:52.119 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:52.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:52.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:52.223 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:52.254 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:52.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:52.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:52.357 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:52.387 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:52.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:52.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:52.492 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:52.503 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:52 smithi173 bash[59422]: cluster 2023-09-23T18:51:52.157059+0000 mgr.y (mgr.26054) 1016 : cluster 0 pgmap v1083: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 53 op/s; 63134/142305 objects degraded (44.365%); 5513/142305 objects misplaced (3.874%); 507 KiB/s, 8 objects/s recovering 2023-09-23T18:51:52.524 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:52.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:52.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:52.628 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:52.650 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:52.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:52.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:52.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:52 smithi098 bash[79516]: cluster 2023-09-23T18:51:52.157059+0000 mgr.y (mgr.26054) 1016 : cluster 0 pgmap v1083: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 53 op/s; 63134/142305 objects degraded (44.365%); 5513/142305 objects misplaced (3.874%); 507 KiB/s, 8 objects/s recovering 2023-09-23T18:51:52.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:52 smithi098 bash[76817]: cluster 2023-09-23T18:51:52.157059+0000 mgr.y (mgr.26054) 1016 : cluster 0 pgmap v1083: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 53 op/s; 63134/142305 objects degraded (44.365%); 5513/142305 objects misplaced (3.874%); 507 KiB/s, 8 objects/s recovering 2023-09-23T18:51:52.757 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:52.787 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:52.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:52.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:52.891 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:52.918 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:52.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:52.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:53.021 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:53.048 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:53.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:53.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:53.151 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:53.184 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:53.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:53.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:53.287 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:53.315 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:53.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:53.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:53.417 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:53.444 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:53.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:53.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:53.547 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:53.571 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:53.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:53.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:53.675 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:53.705 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:53.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:53.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:53.809 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:53.843 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:53.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:53.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:53.947 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:53.973 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:53.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:53.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:54.014 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:51:54.014 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:51:54.077 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:54.105 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:54.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:54.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:54.208 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:54.238 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:54.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:54.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:54.341 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:54.376 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:54.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:54.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:54.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:54 smithi098 bash[76817]: cluster 2023-09-23T18:51:54.158386+0000 mgr.y (mgr.26054) 1017 : cluster 0 pgmap v1084: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 47 op/s; 63134/142305 objects degraded (44.365%); 5513/142305 objects misplaced (3.874%); 457 KiB/s, 7 objects/s recovering 2023-09-23T18:51:54.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:54 smithi098 bash[79516]: cluster 2023-09-23T18:51:54.158386+0000 mgr.y (mgr.26054) 1017 : cluster 0 pgmap v1084: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 47 op/s; 63134/142305 objects degraded (44.365%); 5513/142305 objects misplaced (3.874%); 457 KiB/s, 7 objects/s recovering 2023-09-23T18:51:54.479 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:54.507 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:54.509 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:54 smithi173 bash[59422]: cluster 2023-09-23T18:51:54.158386+0000 mgr.y (mgr.26054) 1017 : cluster 0 pgmap v1084: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 47 op/s; 63134/142305 objects degraded (44.365%); 5513/142305 objects misplaced (3.874%); 457 KiB/s, 7 objects/s recovering 2023-09-23T18:51:54.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:54.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:54.610 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:54.639 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:54.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:54.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:54.741 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:54.774 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:54.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:54.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:54.878 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:54.910 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:54.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:54.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:55.014 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:55.026 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:51:55.042 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:55.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:55.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:55.145 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:55.171 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:55.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:55.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:55.274 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:55.301 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:55.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:55.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:55.344 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:55 smithi098 bash[79516]: audit 2023-09-23T18:51:54.625528+0000 mon.a (mon.0) 1390 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:51:55.344 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:55 smithi098 bash[79516]: cluster 2023-09-23T18:51:55.181489+0000 mon.a (mon.0) 1391 : cluster 3 Health check update: Degraded data redundancy: 63134/142305 objects degraded (44.365%), 13 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:55.345 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:55 smithi098 bash[76817]: audit 2023-09-23T18:51:54.625528+0000 mon.a (mon.0) 1390 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:51:55.345 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:55 smithi098 bash[76817]: cluster 2023-09-23T18:51:55.181489+0000 mon.a (mon.0) 1391 : cluster 3 Health check update: Degraded data redundancy: 63134/142305 objects degraded (44.365%), 13 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:55.404 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:55.413 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:55 smithi173 bash[59422]: audit 2023-09-23T18:51:54.625528+0000 mon.a (mon.0) 1390 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:51:55.413 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:55 smithi173 bash[59422]: cluster 2023-09-23T18:51:55.181489+0000 mon.a (mon.0) 1391 : cluster 3 Health check update: Degraded data redundancy: 63134/142305 objects degraded (44.365%), 13 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:51:55.428 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:55.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:55.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:55.530 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:55.554 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:55.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:55.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:55.658 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:55.684 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:55.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:55.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:55.787 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:55.818 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:55.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:55.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:55.921 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:55.955 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:55.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:55.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:56.039 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:51:56.059 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:56.081 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:56.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:56.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:56.184 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:56.210 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:56.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:56.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:56.313 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:56.324 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:56 smithi173 bash[59422]: cluster 2023-09-23T18:51:56.159093+0000 mgr.y (mgr.26054) 1018 : cluster 0 pgmap v1085: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 11 op/s; 63134/142305 objects degraded (44.365%); 5513/142305 objects misplaced (3.874%); 133 KiB/s, 2 objects/s recovering 2023-09-23T18:51:56.341 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:56.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:56.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:56.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:56 smithi098 bash[79516]: cluster 2023-09-23T18:51:56.159093+0000 mgr.y (mgr.26054) 1018 : cluster 0 pgmap v1085: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 11 op/s; 63134/142305 objects degraded (44.365%); 5513/142305 objects misplaced (3.874%); 133 KiB/s, 2 objects/s recovering 2023-09-23T18:51:56.442 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:51:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:51:56] "GET /metrics HTTP/1.1" 200 36612 "" "Prometheus/2.33.4" 2023-09-23T18:51:56.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:56 smithi098 bash[76817]: cluster 2023-09-23T18:51:56.159093+0000 mgr.y (mgr.26054) 1018 : cluster 0 pgmap v1085: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 11 op/s; 63134/142305 objects degraded (44.365%); 5513/142305 objects misplaced (3.874%); 133 KiB/s, 2 objects/s recovering 2023-09-23T18:51:56.444 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:56.469 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:56.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:56.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:56.573 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:56.594 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:56.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:56.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:56.697 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:56.719 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:56.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:56.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:56.823 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:56.850 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:56.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:56.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:56.954 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:56.975 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:56.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:56.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:57.078 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:57.106 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:51:57.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:57.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:51:57.209 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:57.236 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:57.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:57.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:57.340 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:57.364 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:57.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:57.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:57.467 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:57.491 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:57.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:57.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:57.593 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:57.618 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:57.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:57.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:57.721 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:57.746 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:51:57.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:57.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:51:57.851 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:57.878 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:57.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:57.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:57.980 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:58.005 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:58.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:58.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:58.108 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:58.134 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:58.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:58.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:58.238 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:58.260 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:58.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:58.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:58.364 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:58.372 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:51:58 smithi098 bash[79516]: cluster 2023-09-23T18:51:58.160499+0000 mgr.y (mgr.26054) 1019 : cluster 0 pgmap v1086: 89 pgs: 1 activating+undersized+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 23 op/s; 64300/141663 objects degraded (45.389%); 10078/141663 objects misplaced (7.114%); 5.4 MiB/s, 87 objects/s recovering 2023-09-23T18:51:58.372 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:51:58 smithi098 bash[76817]: cluster 2023-09-23T18:51:58.160499+0000 mgr.y (mgr.26054) 1019 : cluster 0 pgmap v1086: 89 pgs: 1 activating+undersized+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 23 op/s; 64300/141663 objects degraded (45.389%); 10078/141663 objects misplaced (7.114%); 5.4 MiB/s, 87 objects/s recovering 2023-09-23T18:51:58.384 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:58.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:58.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:58.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:58.497 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:51:58 smithi173 bash[59422]: cluster 2023-09-23T18:51:58.160499+0000 mgr.y (mgr.26054) 1019 : cluster 0 pgmap v1086: 89 pgs: 1 activating+undersized+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 23 op/s; 64300/141663 objects degraded (45.389%); 10078/141663 objects misplaced (7.114%); 5.4 MiB/s, 87 objects/s recovering 2023-09-23T18:51:58.512 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:58.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:58.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:58.615 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:58.642 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:51:58.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:58.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:51:58.745 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:58.769 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:51:58.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:58.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:51:58.872 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:58.898 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:58.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:58.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:59.003 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:59.029 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:59.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:59.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:59.132 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:59.154 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:59.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:59.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:59.258 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:59.284 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:59.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:59.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:51:59.387 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:59.413 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:59.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:59.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:59.517 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:59.540 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:51:59.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:59.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:51:59.643 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:59.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:59.667 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:51:59.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:51:59.768 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:59.793 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:51:59.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:59.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:51:59.896 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:59.923 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:51:59.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:51:59.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:00.029 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:00.058 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:00.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:00.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:00.161 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:00.183 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:00.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:00.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:00.288 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:00.324 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:00.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:00.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:00.432 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:00.441 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:00 smithi173 bash[59422]: cluster 2023-09-23T18:52:00.161164+0000 mgr.y (mgr.26054) 1020 : cluster 0 pgmap v1087: 89 pgs: 1 activating+undersized+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 19 op/s; 64300/141663 objects degraded (45.389%); 10078/141663 objects misplaced (7.114%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T18:52:00.441 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:00 smithi173 bash[59422]: cluster 2023-09-23T18:52:00.182954+0000 mon.a (mon.0) 1392 : cluster 3 Health check update: Degraded data redundancy: 64300/141663 objects degraded (45.389%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:00.455 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:00.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:00.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:00.562 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:00.587 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:00.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:00.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:00.691 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:00.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:00 smithi098 bash[79516]: cluster 2023-09-23T18:52:00.161164+0000 mgr.y (mgr.26054) 1020 : cluster 0 pgmap v1087: 89 pgs: 1 activating+undersized+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 19 op/s; 64300/141663 objects degraded (45.389%); 10078/141663 objects misplaced (7.114%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T18:52:00.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:00 smithi098 bash[79516]: cluster 2023-09-23T18:52:00.182954+0000 mon.a (mon.0) 1392 : cluster 3 Health check update: Degraded data redundancy: 64300/141663 objects degraded (45.389%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:00.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:00 smithi098 bash[76817]: cluster 2023-09-23T18:52:00.161164+0000 mgr.y (mgr.26054) 1020 : cluster 0 pgmap v1087: 89 pgs: 1 activating+undersized+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 19 op/s; 64300/141663 objects degraded (45.389%); 10078/141663 objects misplaced (7.114%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T18:52:00.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:00 smithi098 bash[76817]: cluster 2023-09-23T18:52:00.182954+0000 mon.a (mon.0) 1392 : cluster 3 Health check update: Degraded data redundancy: 64300/141663 objects degraded (45.389%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:00.721 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:00.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:00.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:00.824 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:00.849 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:00.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:00.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:00.954 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:00.981 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:00.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:00.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:01.085 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:01.195 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:01.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:01.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:01.298 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:01.369 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:01.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:01.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:01.472 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:01.496 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:01.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:01.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:01.604 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:01.640 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:01.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:01.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:01.748 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:01.781 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:01.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:01.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:01.888 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:01.910 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:01.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:01.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:02.013 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:02.038 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:02.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:02.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:02.146 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:02.178 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:02.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:02.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:02.283 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:02.311 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:02.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:02.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:02.419 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:02.450 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:02.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:02.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:02.527 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:02 smithi098 bash[76817]: cluster 2023-09-23T18:52:02.161812+0000 mgr.y (mgr.26054) 1021 : cluster 0 pgmap v1088: 89 pgs: 1 activating+undersized+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 18 op/s; 64274/141624 objects degraded (45.384%); 10078/141624 objects misplaced (7.116%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T18:52:02.528 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:02 smithi098 bash[79516]: cluster 2023-09-23T18:52:02.161812+0000 mgr.y (mgr.26054) 1021 : cluster 0 pgmap v1088: 89 pgs: 1 activating+undersized+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 18 op/s; 64274/141624 objects degraded (45.384%); 10078/141624 objects misplaced (7.116%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T18:52:02.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:02 smithi173 bash[59422]: cluster 2023-09-23T18:52:02.161812+0000 mgr.y (mgr.26054) 1021 : cluster 0 pgmap v1088: 89 pgs: 1 activating+undersized+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 18 op/s; 64274/141624 objects degraded (45.384%); 10078/141624 objects misplaced (7.116%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T18:52:02.554 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:02.575 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:02.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:02.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:02.678 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:02.700 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:02.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:02.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:02.805 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:02.829 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:02.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:02.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:02.933 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:02.971 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:02.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:02.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:03.075 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:03.121 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:03.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:03.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:03.225 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:03.279 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:03.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:03.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:03.389 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:03.419 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:03.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:03.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:03.527 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:03.555 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:03.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:03.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:03.663 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:03.686 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:03.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:03.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:03.794 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:03.821 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:03.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:03.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:03.931 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:03.955 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:03.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:03.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:04.061 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:04.102 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:04.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:04.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:04.208 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:04.231 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:04.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:04.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:04.336 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:04.372 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:04.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:04.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:04.379 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:04 smithi098 bash[76817]: cluster 2023-09-23T18:52:04.162710+0000 mgr.y (mgr.26054) 1022 : cluster 0 pgmap v1089: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 130 op/s; 56108/137610 objects degraded (40.773%); 10794/137610 objects misplaced (7.844%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T18:52:04.379 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:04 smithi098 bash[79516]: cluster 2023-09-23T18:52:04.162710+0000 mgr.y (mgr.26054) 1022 : cluster 0 pgmap v1089: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 130 op/s; 56108/137610 objects degraded (40.773%); 10794/137610 objects misplaced (7.844%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T18:52:04.478 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:04.507 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:04.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:04.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:04.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:04 smithi173 bash[59422]: cluster 2023-09-23T18:52:04.162710+0000 mgr.y (mgr.26054) 1022 : cluster 0 pgmap v1089: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 130 op/s; 56108/137610 objects degraded (40.773%); 10794/137610 objects misplaced (7.844%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T18:52:04.610 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:04.643 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:04.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:04.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:04.747 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:04.773 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:04.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:04.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:04.876 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:04.900 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:04.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:04.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:05.008 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:05.034 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:05.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:05.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:05.137 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:05.160 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:05.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:05.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:05.265 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:05.274 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:05 smithi173 bash[59422]: cluster 2023-09-23T18:52:05.184106+0000 mon.a (mon.0) 1393 : cluster 3 Health check update: Degraded data redundancy: 56108/137610 objects degraded (40.773%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:05.289 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:05.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:05.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:05.392 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:05.423 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:05.424 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:05 smithi098 bash[79516]: cluster 2023-09-23T18:52:05.184106+0000 mon.a (mon.0) 1393 : cluster 3 Health check update: Degraded data redundancy: 56108/137610 objects degraded (40.773%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:05.425 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:05 smithi098 bash[76817]: cluster 2023-09-23T18:52:05.184106+0000 mon.a (mon.0) 1393 : cluster 3 Health check update: Degraded data redundancy: 56108/137610 objects degraded (40.773%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:05.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:05.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:05.527 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:05.558 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:05.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:05.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:05.661 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:05.687 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:05.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:05.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:05.790 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:05.817 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:05.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:05.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:05.920 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:05.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:05.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:05.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:06.050 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:06.077 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:06.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:06.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:06.182 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:06.216 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:06.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:06.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:06.264 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:06 smithi173 bash[59422]: cluster 2023-09-23T18:52:06.163411+0000 mgr.y (mgr.26054) 1023 : cluster 0 pgmap v1090: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 141 op/s; 55850/137223 objects degraded (40.700%); 10794/137223 objects misplaced (7.866%); 7.7 MiB/s, 122 objects/s recovering 2023-09-23T18:52:06.315 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:52:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:52:06] "GET /metrics HTTP/1.1" 200 36612 "" "Prometheus/2.33.4" 2023-09-23T18:52:06.318 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:06.350 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:06.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:06.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:06.455 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:06.487 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:06.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:06.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:06.592 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:06.602 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:06 smithi098 bash[79516]: cluster 2023-09-23T18:52:06.163411+0000 mgr.y (mgr.26054) 1023 : cluster 0 pgmap v1090: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 141 op/s; 55850/137223 objects degraded (40.700%); 10794/137223 objects misplaced (7.866%); 7.7 MiB/s, 122 objects/s recovering 2023-09-23T18:52:06.603 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:06 smithi098 bash[76817]: cluster 2023-09-23T18:52:06.163411+0000 mgr.y (mgr.26054) 1023 : cluster 0 pgmap v1090: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 141 op/s; 55850/137223 objects degraded (40.700%); 10794/137223 objects misplaced (7.866%); 7.7 MiB/s, 122 objects/s recovering 2023-09-23T18:52:06.622 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:06.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:06.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:06.728 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:06.746 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:06.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:06.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:06.849 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:06.870 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:06.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:06.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:06.975 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:07.002 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:07.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:07.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:07.109 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:07.161 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:07.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:07.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:07.268 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:07.303 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:07.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:07.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:07.407 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:07.447 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:07.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:07.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:07.553 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:07.580 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:07.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:07.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:07.683 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:07.713 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:07.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:07.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:07.818 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:07.848 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:07.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:07.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:07.954 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:07.977 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:07.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:07.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:08.080 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:08.105 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:08.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:08.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:08.210 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:08.246 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:08.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:08.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:08.258 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:08 smithi098 bash[76817]: cluster 2023-09-23T18:52:08.164243+0000 mgr.y (mgr.26054) 1024 : cluster 0 pgmap v1091: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 270 op/s; 55850/132567 objects degraded (42.130%); 10379/132567 objects misplaced (7.829%); 8.5 MiB/s, 135 objects/s recovering 2023-09-23T18:52:08.353 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:08.380 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:08.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:08.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:08.485 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:08.516 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:08.519 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:08 smithi173 bash[59422]: cluster 2023-09-23T18:52:08.164243+0000 mgr.y (mgr.26054) 1024 : cluster 0 pgmap v1091: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 270 op/s; 55850/132567 objects degraded (42.130%); 10379/132567 objects misplaced (7.829%); 8.5 MiB/s, 135 objects/s recovering 2023-09-23T18:52:08.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:08.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:08.621 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:08.643 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:08.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:08.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:08.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:08 smithi098 bash[79516]: cluster 2023-09-23T18:52:08.164243+0000 mgr.y (mgr.26054) 1024 : cluster 0 pgmap v1091: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 270 op/s; 55850/132567 objects degraded (42.130%); 10379/132567 objects misplaced (7.829%); 8.5 MiB/s, 135 objects/s recovering 2023-09-23T18:52:08.746 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:08.773 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:08.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:08.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:08.877 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:08.901 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:08.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:08.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:09.005 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:09.033 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:09.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:09.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:09.139 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:09.169 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:09.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:09.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:09.275 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:09.307 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:09.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:09.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:09.412 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:09.442 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:09.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:09.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:09.545 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:09.550 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:52:09.581 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:09.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:09.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:09.603 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:52:09.655 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:52:09.688 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:09.696 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:09 smithi173 bash[59422]: audit 2023-09-23T18:52:09.504001+0000 mon.c (mon.2) 115 : audit 0 from='client.? 172.21.15.98:0/1091299462' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:52:09.698 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:52:09.711 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:09.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:09.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:09.747 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:52:09.807 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:52:09.818 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:09.840 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:52:09.844 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:09.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:09.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:09.856 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:09 smithi098 bash[79516]: audit 2023-09-23T18:52:09.504001+0000 mon.c (mon.2) 115 : audit 0 from='client.? 172.21.15.98:0/1091299462' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:52:09.856 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:09 smithi098 bash[76817]: audit 2023-09-23T18:52:09.504001+0000 mon.c (mon.2) 115 : audit 0 from='client.? 172.21.15.98:0/1091299462' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:52:09.893 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:52:09.897 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:52:09.897 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:52:09.897 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:52:09.897 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:52:09.897 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:52:09.897 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:52:09.898 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:52:09.898 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:52:09.948 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:09.988 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:09.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:09.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:10.091 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:10.117 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:52:10.117 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:52:10.125 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:10.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:10.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:10.228 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:10.255 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:10.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:10.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:10.358 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:10.390 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:10.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:10.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:10.496 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:10.528 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:10.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:10.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:10.584 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is unset 2023-09-23T18:52:10.632 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:10.678 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:10.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:10.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:10.781 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:10.791 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:10 smithi098 bash[79516]: audit 2023-09-23T18:52:09.625476+0000 mon.a (mon.0) 1394 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:52:10.791 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:10 smithi098 bash[79516]: audit 2023-09-23T18:52:10.163687+0000 mon.c (mon.2) 116 : audit 1 from='client.? 172.21.15.98:0/963053692' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:52:10.791 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:10 smithi098 bash[79516]: audit 2023-09-23T18:52:10.164767+0000 mon.a (mon.0) 1395 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:52:10.791 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:10 smithi098 bash[79516]: cluster 2023-09-23T18:52:10.164940+0000 mgr.y (mgr.26054) 1025 : cluster 0 pgmap v1092: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 297 op/s; 55899/130950 objects degraded (42.687%); 10257/130950 objects misplaced (7.833%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T18:52:10.792 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:10 smithi098 bash[79516]: cluster 2023-09-23T18:52:10.185319+0000 mon.a (mon.0) 1396 : cluster 3 Health check update: Degraded data redundancy: 55850/132567 objects degraded (42.130%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:10.792 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:10 smithi098 bash[76817]: audit 2023-09-23T18:52:09.625476+0000 mon.a (mon.0) 1394 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:52:10.792 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:10 smithi098 bash[76817]: audit 2023-09-23T18:52:10.163687+0000 mon.c (mon.2) 116 : audit 1 from='client.? 172.21.15.98:0/963053692' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:52:10.792 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:10 smithi098 bash[76817]: audit 2023-09-23T18:52:10.164767+0000 mon.a (mon.0) 1395 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:52:10.792 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:10 smithi098 bash[76817]: cluster 2023-09-23T18:52:10.164940+0000 mgr.y (mgr.26054) 1025 : cluster 0 pgmap v1092: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 297 op/s; 55899/130950 objects degraded (42.687%); 10257/130950 objects misplaced (7.833%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T18:52:10.792 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:10 smithi098 bash[76817]: cluster 2023-09-23T18:52:10.185319+0000 mon.a (mon.0) 1396 : cluster 3 Health check update: Degraded data redundancy: 55850/132567 objects degraded (42.130%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:10.813 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:10.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:10.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:10.917 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:10.925 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:10 smithi173 bash[59422]: audit 2023-09-23T18:52:09.625476+0000 mon.a (mon.0) 1394 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:52:10.926 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:10 smithi173 bash[59422]: audit 2023-09-23T18:52:10.163687+0000 mon.c (mon.2) 116 : audit 1 from='client.? 172.21.15.98:0/963053692' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:52:10.926 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:10 smithi173 bash[59422]: audit 2023-09-23T18:52:10.164767+0000 mon.a (mon.0) 1395 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:52:10.926 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:10 smithi173 bash[59422]: cluster 2023-09-23T18:52:10.164940+0000 mgr.y (mgr.26054) 1025 : cluster 0 pgmap v1092: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 297 op/s; 55899/130950 objects degraded (42.687%); 10257/130950 objects misplaced (7.833%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T18:52:10.926 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:10 smithi173 bash[59422]: cluster 2023-09-23T18:52:10.185319+0000 mon.a (mon.0) 1396 : cluster 3 Health check update: Degraded data redundancy: 55850/132567 objects degraded (42.130%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:10.945 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:10.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:10.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:11.049 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:11.072 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:11.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:11.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:11.175 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:11.203 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:11.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:11.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:11.306 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:11.335 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:11.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:11.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:11.438 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:11.472 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:11.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:11.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:11.575 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:11.595 INFO:teuthology.orchestra.run.smithi098.stderr:set osd.0 primary-affinity to 0.652757 (8427792) 2023-09-23T18:52:11.633 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:11.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:11.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:11.739 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:11.748 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:11 smithi173 bash[59422]: cluster 2023-09-23T18:52:10.556467+0000 mon.a (mon.0) 1397 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:52:11.748 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:11 smithi173 bash[59422]: cluster 2023-09-23T18:52:10.568277+0000 osd.7 (osd.7) 180 : cluster 3 failed to encode map e259 with expected crc 2023-09-23T18:52:11.748 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:11 smithi173 bash[59422]: audit 2023-09-23T18:52:10.572787+0000 mon.a (mon.0) 1398 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:52:11.748 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:11 smithi173 bash[59422]: cluster 2023-09-23T18:52:10.573139+0000 mon.a (mon.0) 1399 : cluster 0 osdmap e259: 8 total, 8 up, 7 in 2023-09-23T18:52:11.748 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:11 smithi173 bash[59422]: cluster 2023-09-23T18:52:10.574820+0000 osd.5 (osd.5) 160 : cluster 3 failed to encode map e259 with expected crc 2023-09-23T18:52:11.748 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:11 smithi173 bash[59422]: cluster 2023-09-23T18:52:10.574822+0000 osd.6 (osd.6) 217 : cluster 3 failed to encode map e259 with expected crc 2023-09-23T18:52:11.748 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:11 smithi173 bash[59422]: cluster 2023-09-23T18:52:10.584314+0000 osd.4 (osd.4) 257 : cluster 3 failed to encode map e259 with expected crc 2023-09-23T18:52:11.748 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:11 smithi173 bash[59422]: audit 2023-09-23T18:52:10.624947+0000 mon.c (mon.2) 117 : audit 1 from='client.? 172.21.15.98:0/2485816178' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6527572438099544}]: dispatch 2023-09-23T18:52:11.748 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:11 smithi173 bash[59422]: audit 2023-09-23T18:52:10.625599+0000 mon.a (mon.0) 1400 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6527572438099544}]: dispatch 2023-09-23T18:52:11.769 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:11.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:11.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:11.873 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:11.882 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:11 smithi098 bash[76817]: cluster 2023-09-23T18:52:10.556467+0000 mon.a (mon.0) 1397 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:52:11.883 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:11 smithi098 bash[76817]: cluster 2023-09-23T18:52:10.568277+0000 osd.7 (osd.7) 180 : cluster 3 failed to encode map e259 with expected crc 2023-09-23T18:52:11.883 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:11 smithi098 bash[76817]: audit 2023-09-23T18:52:10.572787+0000 mon.a (mon.0) 1398 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:52:11.883 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:11 smithi098 bash[76817]: cluster 2023-09-23T18:52:10.573139+0000 mon.a (mon.0) 1399 : cluster 0 osdmap e259: 8 total, 8 up, 7 in 2023-09-23T18:52:11.883 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:11 smithi098 bash[76817]: cluster 2023-09-23T18:52:10.574820+0000 osd.5 (osd.5) 160 : cluster 3 failed to encode map e259 with expected crc 2023-09-23T18:52:11.883 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:11 smithi098 bash[76817]: cluster 2023-09-23T18:52:10.574822+0000 osd.6 (osd.6) 217 : cluster 3 failed to encode map e259 with expected crc 2023-09-23T18:52:11.883 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:11 smithi098 bash[76817]: cluster 2023-09-23T18:52:10.584314+0000 osd.4 (osd.4) 257 : cluster 3 failed to encode map e259 with expected crc 2023-09-23T18:52:11.883 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:11 smithi098 bash[76817]: audit 2023-09-23T18:52:10.624947+0000 mon.c (mon.2) 117 : audit 1 from='client.? 172.21.15.98:0/2485816178' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6527572438099544}]: dispatch 2023-09-23T18:52:11.883 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:11 smithi098 bash[76817]: audit 2023-09-23T18:52:10.625599+0000 mon.a (mon.0) 1400 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6527572438099544}]: dispatch 2023-09-23T18:52:11.884 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:11 smithi098 bash[79516]: cluster 2023-09-23T18:52:10.556467+0000 mon.a (mon.0) 1397 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:52:11.884 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:11 smithi098 bash[79516]: cluster 2023-09-23T18:52:10.568277+0000 osd.7 (osd.7) 180 : cluster 3 failed to encode map e259 with expected crc 2023-09-23T18:52:11.884 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:11 smithi098 bash[79516]: audit 2023-09-23T18:52:10.572787+0000 mon.a (mon.0) 1398 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:52:11.884 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:11 smithi098 bash[79516]: cluster 2023-09-23T18:52:10.573139+0000 mon.a (mon.0) 1399 : cluster 0 osdmap e259: 8 total, 8 up, 7 in 2023-09-23T18:52:11.884 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:11 smithi098 bash[79516]: cluster 2023-09-23T18:52:10.574820+0000 osd.5 (osd.5) 160 : cluster 3 failed to encode map e259 with expected crc 2023-09-23T18:52:11.884 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:11 smithi098 bash[79516]: cluster 2023-09-23T18:52:10.574822+0000 osd.6 (osd.6) 217 : cluster 3 failed to encode map e259 with expected crc 2023-09-23T18:52:11.884 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:11 smithi098 bash[79516]: cluster 2023-09-23T18:52:10.584314+0000 osd.4 (osd.4) 257 : cluster 3 failed to encode map e259 with expected crc 2023-09-23T18:52:11.884 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:11 smithi098 bash[79516]: audit 2023-09-23T18:52:10.624947+0000 mon.c (mon.2) 117 : audit 1 from='client.? 172.21.15.98:0/2485816178' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6527572438099544}]: dispatch 2023-09-23T18:52:11.884 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:11 smithi098 bash[79516]: audit 2023-09-23T18:52:10.625599+0000 mon.a (mon.0) 1400 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6527572438099544}]: dispatch 2023-09-23T18:52:11.897 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:11.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:11.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:12.004 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:12.046 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:12.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:12.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:12.151 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:12.176 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:12.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:12.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:12.278 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:12.319 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:12.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:12.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:12.424 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:12.468 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:12.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:12.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:12.575 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:12.626 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:12.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:12.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:12.731 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:12.764 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:12.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:12.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:12.869 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:12.879 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:12 smithi173 bash[59422]: audit 2023-09-23T18:52:11.578261+0000 mon.a (mon.0) 1401 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6527572438099544}]': finished 2023-09-23T18:52:12.879 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:12 smithi173 bash[59422]: cluster 2023-09-23T18:52:11.578308+0000 osd.7 (osd.7) 181 : cluster 3 failed to encode map e260 with expected crc 2023-09-23T18:52:12.880 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:12 smithi173 bash[59422]: cluster 2023-09-23T18:52:11.578521+0000 mon.a (mon.0) 1402 : cluster 0 osdmap e260: 8 total, 8 up, 7 in 2023-09-23T18:52:12.880 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:12 smithi173 bash[59422]: cluster 2023-09-23T18:52:11.585061+0000 osd.6 (osd.6) 218 : cluster 3 failed to encode map e260 with expected crc 2023-09-23T18:52:12.880 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:12 smithi173 bash[59422]: cluster 2023-09-23T18:52:11.595501+0000 osd.4 (osd.4) 258 : cluster 3 failed to encode map e260 with expected crc 2023-09-23T18:52:12.880 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:12 smithi173 bash[59422]: cluster 2023-09-23T18:52:11.603359+0000 osd.6 (osd.6) 219 : cluster 3 failed to encode map e260 with expected crc 2023-09-23T18:52:12.880 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:12 smithi173 bash[59422]: cluster 2023-09-23T18:52:11.606166+0000 osd.5 (osd.5) 161 : cluster 3 failed to encode map e260 with expected crc 2023-09-23T18:52:12.880 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:12 smithi173 bash[59422]: cluster 2023-09-23T18:52:12.165555+0000 mgr.y (mgr.26054) 1026 : cluster 0 pgmap v1095: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 294 op/s; 55631/130551 objects degraded (42.612%); 10257/130551 objects misplaced (7.857%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T18:52:12.880 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:12 smithi173 bash[59422]: cluster 2023-09-23T18:52:12.574122+0000 mon.a (mon.0) 1403 : cluster 0 osdmap e261: 8 total, 8 up, 7 in 2023-09-23T18:52:12.892 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:12.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:12.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:12.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:12 smithi098 bash[76817]: audit 2023-09-23T18:52:11.578261+0000 mon.a (mon.0) 1401 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6527572438099544}]': finished 2023-09-23T18:52:12.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:12 smithi098 bash[76817]: cluster 2023-09-23T18:52:11.578308+0000 osd.7 (osd.7) 181 : cluster 3 failed to encode map e260 with expected crc 2023-09-23T18:52:12.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:12 smithi098 bash[76817]: cluster 2023-09-23T18:52:11.578521+0000 mon.a (mon.0) 1402 : cluster 0 osdmap e260: 8 total, 8 up, 7 in 2023-09-23T18:52:12.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:12 smithi098 bash[76817]: cluster 2023-09-23T18:52:11.585061+0000 osd.6 (osd.6) 218 : cluster 3 failed to encode map e260 with expected crc 2023-09-23T18:52:12.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:12 smithi098 bash[76817]: cluster 2023-09-23T18:52:11.595501+0000 osd.4 (osd.4) 258 : cluster 3 failed to encode map e260 with expected crc 2023-09-23T18:52:12.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:12 smithi098 bash[76817]: cluster 2023-09-23T18:52:11.603359+0000 osd.6 (osd.6) 219 : cluster 3 failed to encode map e260 with expected crc 2023-09-23T18:52:12.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:12 smithi098 bash[76817]: cluster 2023-09-23T18:52:11.606166+0000 osd.5 (osd.5) 161 : cluster 3 failed to encode map e260 with expected crc 2023-09-23T18:52:12.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:12 smithi098 bash[76817]: cluster 2023-09-23T18:52:12.165555+0000 mgr.y (mgr.26054) 1026 : cluster 0 pgmap v1095: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 294 op/s; 55631/130551 objects degraded (42.612%); 10257/130551 objects misplaced (7.857%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T18:52:12.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:12 smithi098 bash[76817]: cluster 2023-09-23T18:52:12.574122+0000 mon.a (mon.0) 1403 : cluster 0 osdmap e261: 8 total, 8 up, 7 in 2023-09-23T18:52:12.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:12 smithi098 bash[79516]: audit 2023-09-23T18:52:11.578261+0000 mon.a (mon.0) 1401 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6527572438099544}]': finished 2023-09-23T18:52:12.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:12 smithi098 bash[79516]: cluster 2023-09-23T18:52:11.578308+0000 osd.7 (osd.7) 181 : cluster 3 failed to encode map e260 with expected crc 2023-09-23T18:52:12.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:12 smithi098 bash[79516]: cluster 2023-09-23T18:52:11.578521+0000 mon.a (mon.0) 1402 : cluster 0 osdmap e260: 8 total, 8 up, 7 in 2023-09-23T18:52:12.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:12 smithi098 bash[79516]: cluster 2023-09-23T18:52:11.585061+0000 osd.6 (osd.6) 218 : cluster 3 failed to encode map e260 with expected crc 2023-09-23T18:52:12.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:12 smithi098 bash[79516]: cluster 2023-09-23T18:52:11.595501+0000 osd.4 (osd.4) 258 : cluster 3 failed to encode map e260 with expected crc 2023-09-23T18:52:12.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:12 smithi098 bash[79516]: cluster 2023-09-23T18:52:11.603359+0000 osd.6 (osd.6) 219 : cluster 3 failed to encode map e260 with expected crc 2023-09-23T18:52:12.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:12 smithi098 bash[79516]: cluster 2023-09-23T18:52:11.606166+0000 osd.5 (osd.5) 161 : cluster 3 failed to encode map e260 with expected crc 2023-09-23T18:52:12.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:12 smithi098 bash[79516]: cluster 2023-09-23T18:52:12.165555+0000 mgr.y (mgr.26054) 1026 : cluster 0 pgmap v1095: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 294 op/s; 55631/130551 objects degraded (42.612%); 10257/130551 objects misplaced (7.857%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T18:52:12.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:12 smithi098 bash[79516]: cluster 2023-09-23T18:52:12.574122+0000 mon.a (mon.0) 1403 : cluster 0 osdmap e261: 8 total, 8 up, 7 in 2023-09-23T18:52:12.997 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:13.032 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:13.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:13.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:13.137 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:13.159 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:13.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:13.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:13.261 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:13.286 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:13.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:13.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:13.389 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:13.407 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:13.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:13.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:13.516 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:13.548 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:13.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:13.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:13.650 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:13.685 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:13.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:13.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:13.790 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:13.822 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:13.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:13.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:13.927 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:13.936 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:13 smithi173 bash[59422]: cluster 2023-09-23T18:52:12.571651+0000 osd.4 (osd.4) 259 : cluster 3 failed to encode map e261 with expected crc 2023-09-23T18:52:13.936 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:13 smithi173 bash[59422]: cluster 2023-09-23T18:52:12.575997+0000 osd.7 (osd.7) 182 : cluster 3 failed to encode map e261 with expected crc 2023-09-23T18:52:13.936 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:13 smithi173 bash[59422]: cluster 2023-09-23T18:52:12.578554+0000 osd.6 (osd.6) 220 : cluster 3 failed to encode map e261 with expected crc 2023-09-23T18:52:13.937 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:13 smithi173 bash[59422]: cluster 2023-09-23T18:52:12.605818+0000 osd.5 (osd.5) 162 : cluster 3 failed to encode map e261 with expected crc 2023-09-23T18:52:13.955 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:13.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:13.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:14.001 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:13 smithi098 bash[76817]: cluster 2023-09-23T18:52:12.571651+0000 osd.4 (osd.4) 259 : cluster 3 failed to encode map e261 with expected crc 2023-09-23T18:52:14.002 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:13 smithi098 bash[76817]: cluster 2023-09-23T18:52:12.575997+0000 osd.7 (osd.7) 182 : cluster 3 failed to encode map e261 with expected crc 2023-09-23T18:52:14.002 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:13 smithi098 bash[76817]: cluster 2023-09-23T18:52:12.578554+0000 osd.6 (osd.6) 220 : cluster 3 failed to encode map e261 with expected crc 2023-09-23T18:52:14.002 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:13 smithi098 bash[76817]: cluster 2023-09-23T18:52:12.605818+0000 osd.5 (osd.5) 162 : cluster 3 failed to encode map e261 with expected crc 2023-09-23T18:52:14.002 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:13 smithi098 bash[79516]: cluster 2023-09-23T18:52:12.571651+0000 osd.4 (osd.4) 259 : cluster 3 failed to encode map e261 with expected crc 2023-09-23T18:52:14.002 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:13 smithi098 bash[79516]: cluster 2023-09-23T18:52:12.575997+0000 osd.7 (osd.7) 182 : cluster 3 failed to encode map e261 with expected crc 2023-09-23T18:52:14.002 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:13 smithi098 bash[79516]: cluster 2023-09-23T18:52:12.578554+0000 osd.6 (osd.6) 220 : cluster 3 failed to encode map e261 with expected crc 2023-09-23T18:52:14.002 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:13 smithi098 bash[79516]: cluster 2023-09-23T18:52:12.605818+0000 osd.5 (osd.5) 162 : cluster 3 failed to encode map e261 with expected crc 2023-09-23T18:52:14.058 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:14.087 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:14.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:14.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:14.190 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:14.215 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:14.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:14.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:14.318 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:14.350 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:14.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:14.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:14.455 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:14.480 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:14.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:14.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:14.583 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:14.608 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:14.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:14.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:14.625 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:14 smithi098 bash[76817]: cluster 2023-09-23T18:52:14.166329+0000 mgr.y (mgr.26054) 1027 : cluster 0 pgmap v1097: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 349 op/s; 55631/126273 objects degraded (44.056%); 9894/126273 objects misplaced (7.835%); 3.4 MiB/s, 55 objects/s recovering 2023-09-23T18:52:14.625 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:14 smithi098 bash[79516]: cluster 2023-09-23T18:52:14.166329+0000 mgr.y (mgr.26054) 1027 : cluster 0 pgmap v1097: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 349 op/s; 55631/126273 objects degraded (44.056%); 9894/126273 objects misplaced (7.835%); 3.4 MiB/s, 55 objects/s recovering 2023-09-23T18:52:14.713 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:14.748 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:14.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:14.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:14.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:14 smithi173 bash[59422]: cluster 2023-09-23T18:52:14.166329+0000 mgr.y (mgr.26054) 1027 : cluster 0 pgmap v1097: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 349 op/s; 55631/126273 objects degraded (44.056%); 9894/126273 objects misplaced (7.835%); 3.4 MiB/s, 55 objects/s recovering 2023-09-23T18:52:14.851 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:14.887 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:14.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:14.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:14.992 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:15.022 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:15.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:15.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:15.130 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:15.159 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:15.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:15.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:15.265 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:15.302 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:15.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:15.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:15.332 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T18:52:15.417 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:15.439 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:15.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:15.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:15.542 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:15.598 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:15.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:15.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:15.702 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:15.738 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:15.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:15.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:15.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:15 smithi173 bash[59422]: cluster 2023-09-23T18:52:15.187758+0000 mon.a (mon.0) 1404 : cluster 3 Health check update: Degraded data redundancy: 55631/126273 objects degraded (44.056%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:15.845 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:15.854 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:15 smithi098 bash[76817]: cluster 2023-09-23T18:52:15.187758+0000 mon.a (mon.0) 1404 : cluster 3 Health check update: Degraded data redundancy: 55631/126273 objects degraded (44.056%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:15.854 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:15 smithi098 bash[79516]: cluster 2023-09-23T18:52:15.187758+0000 mon.a (mon.0) 1404 : cluster 3 Health check update: Degraded data redundancy: 55631/126273 objects degraded (44.056%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:15.869 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:15.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:15.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:15.979 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:16.018 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:16.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:16.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:16.121 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:16.131 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:52:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:52:16] "GET /metrics HTTP/1.1" 200 36618 "" "Prometheus/2.33.4" 2023-09-23T18:52:16.146 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:16.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:16.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:16.252 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:16.298 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:16.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:16.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:16.403 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:16.430 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:16.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:16.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:16.470 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:52:16.537 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:16.580 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:16.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:16.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:16.688 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:16.730 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:16.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:16.737 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:16 smithi098 bash[76817]: cluster 2023-09-23T18:52:16.167103+0000 mgr.y (mgr.26054) 1028 : cluster 0 pgmap v1098: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 365 op/s; 55451/124368 objects degraded (44.586%); 9752/124368 objects misplaced (7.841%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T18:52:16.737 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:16 smithi098 bash[79516]: cluster 2023-09-23T18:52:16.167103+0000 mgr.y (mgr.26054) 1028 : cluster 0 pgmap v1098: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 365 op/s; 55451/124368 objects degraded (44.586%); 9752/124368 objects misplaced (7.841%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T18:52:16.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:16.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:16 smithi173 bash[59422]: cluster 2023-09-23T18:52:16.167103+0000 mgr.y (mgr.26054) 1028 : cluster 0 pgmap v1098: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 365 op/s; 55451/124368 objects degraded (44.586%); 9752/124368 objects misplaced (7.841%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T18:52:16.838 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:16.864 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:16.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:16.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:16.966 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:17.013 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:17.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:17.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:17.117 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:17.152 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:17.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:17.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:17.258 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:17.305 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:17.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:17.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:17.334 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:52:17.411 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:17.440 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:17.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:17.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:17.542 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:17.572 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:17.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:17.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:17.676 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:17.712 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:17.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:17.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:17.816 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:17.843 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:17.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:17.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:17.948 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:17.975 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:17.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:17.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:18.081 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:18.148 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:18.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:18.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:18.252 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:18.291 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:18.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:18.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:18.399 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:18.408 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:18 smithi098 bash[79516]: cluster 2023-09-23T18:52:18.167736+0000 mgr.y (mgr.26054) 1029 : cluster 0 pgmap v1099: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 463 op/s; 55451/119979 objects degraded (46.217%); 9373/119979 objects misplaced (7.812%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T18:52:18.409 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:18 smithi098 bash[76817]: cluster 2023-09-23T18:52:18.167736+0000 mgr.y (mgr.26054) 1029 : cluster 0 pgmap v1099: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 463 op/s; 55451/119979 objects degraded (46.217%); 9373/119979 objects misplaced (7.812%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T18:52:18.440 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:18.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:18.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:18.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:18 smithi173 bash[59422]: cluster 2023-09-23T18:52:18.167736+0000 mgr.y (mgr.26054) 1029 : cluster 0 pgmap v1099: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 463 op/s; 55451/119979 objects degraded (46.217%); 9373/119979 objects misplaced (7.812%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T18:52:18.548 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:18.580 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:18.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:18.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:18.685 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:18.721 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:18.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:18.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:18.828 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:18.868 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:18.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:18.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:18.975 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:19.003 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:19.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:19.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:19.107 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:19.137 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:19.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:19.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:19.244 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:19.268 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:19.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:19.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:19.371 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:19.403 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:19.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:19.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:19.508 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:19.538 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:19.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:19.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:19.642 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:19.670 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:19.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:19.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:19.777 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:19.806 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:19.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:19.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:19.910 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:19.938 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:19.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:19.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:20.045 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:20.082 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:20.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:20.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:20.185 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:20.213 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:20.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:20.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:20.272 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:52:20.318 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:20.355 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:20.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:20.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:20.361 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T18:52:20.361 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:52:20.361 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-09-23T18:52:20.362 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T18:52:20.362 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T18:52:20.362 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-09-23T18:52:20.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:20 smithi173 bash[59422]: cluster 2023-09-23T18:52:20.168365+0000 mgr.y (mgr.26054) 1030 : cluster 0 pgmap v1100: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 55512/118296 objects degraded (46.926%); 9237/118296 objects misplaced (7.808%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T18:52:20.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:20 smithi173 bash[59422]: cluster 2023-09-23T18:52:20.189639+0000 mon.a (mon.0) 1405 : cluster 3 Health check update: Degraded data redundancy: 55451/119979 objects degraded (46.217%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:20.461 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:20.493 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:20.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:20.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:20.596 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:20.630 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:20.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:20.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:20.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:20 smithi098 bash[76817]: cluster 2023-09-23T18:52:20.168365+0000 mgr.y (mgr.26054) 1030 : cluster 0 pgmap v1100: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 55512/118296 objects degraded (46.926%); 9237/118296 objects misplaced (7.808%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T18:52:20.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:20 smithi098 bash[76817]: cluster 2023-09-23T18:52:20.189639+0000 mon.a (mon.0) 1405 : cluster 3 Health check update: Degraded data redundancy: 55451/119979 objects degraded (46.217%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:20.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:20 smithi098 bash[79516]: cluster 2023-09-23T18:52:20.168365+0000 mgr.y (mgr.26054) 1030 : cluster 0 pgmap v1100: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 55512/118296 objects degraded (46.926%); 9237/118296 objects misplaced (7.808%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T18:52:20.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:20 smithi098 bash[79516]: cluster 2023-09-23T18:52:20.189639+0000 mon.a (mon.0) 1405 : cluster 3 Health check update: Degraded data redundancy: 55451/119979 objects degraded (46.217%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:20.733 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:20.761 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:20.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:20.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:20.866 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:20.890 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:20.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:20.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:21.000 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:21.031 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:21.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:21.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:21.134 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:21.169 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:21.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:21.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:21.273 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:21.289 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:52:21.295 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:52:21.295 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:21.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:21.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:21.400 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:21.437 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:21.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:21.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:21.543 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:21.578 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:21.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:21.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:21.683 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:21.729 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:21.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:21.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:21.834 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:21.872 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:21.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:21.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:21.975 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:21.999 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:22.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:22.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:22.103 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:22.134 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:22.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:22.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:22.237 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:22.270 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:22.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:22.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:22.373 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:22.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:22 smithi173 bash[59422]: cluster 2023-09-23T18:52:22.169067+0000 mgr.y (mgr.26054) 1031 : cluster 0 pgmap v1101: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 421 op/s; 55252/117903 objects degraded (46.862%); 9237/117903 objects misplaced (7.834%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T18:52:22.400 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:22.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:22.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:22.503 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:22.513 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:22 smithi098 bash[76817]: cluster 2023-09-23T18:52:22.169067+0000 mgr.y (mgr.26054) 1031 : cluster 0 pgmap v1101: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 421 op/s; 55252/117903 objects degraded (46.862%); 9237/117903 objects misplaced (7.834%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T18:52:22.513 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:22 smithi098 bash[79516]: cluster 2023-09-23T18:52:22.169067+0000 mgr.y (mgr.26054) 1031 : cluster 0 pgmap v1101: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 421 op/s; 55252/117903 objects degraded (46.862%); 9237/117903 objects misplaced (7.834%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T18:52:22.533 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:22.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:22.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:22.636 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:22.660 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:22.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:22.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:22.765 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:22.797 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:22.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:22.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:22.900 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:22.925 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:22.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:22.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:23.028 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:23.060 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:23.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:23.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:23.164 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:23.213 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:23.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:23.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:23.319 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:23.346 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:23.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:23.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:23.453 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:23.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:23.484 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:23.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:23.585 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:23.614 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:23.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:23.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:23.720 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:23.767 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:23.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:23.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:23.873 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:23.906 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:23.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:23.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:24.012 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:24.042 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:24.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:24.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:24.145 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:24.171 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:24.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:24.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:24.275 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:24.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:24 smithi173 bash[59422]: cluster 2023-09-23T18:52:24.169921+0000 mgr.y (mgr.26054) 1032 : cluster 0 pgmap v1102: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 55252/113418 objects degraded (48.715%); 8834/113418 objects misplaced (7.789%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T18:52:24.307 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:24.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:24.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:24.413 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:24.444 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:24.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:24.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:24.549 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:24.584 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:24.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:24.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:24.688 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:24.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:24 smithi098 bash[76817]: cluster 2023-09-23T18:52:24.169921+0000 mgr.y (mgr.26054) 1032 : cluster 0 pgmap v1102: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 55252/113418 objects degraded (48.715%); 8834/113418 objects misplaced (7.789%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T18:52:24.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:24 smithi098 bash[79516]: cluster 2023-09-23T18:52:24.169921+0000 mgr.y (mgr.26054) 1032 : cluster 0 pgmap v1102: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 55252/113418 objects degraded (48.715%); 8834/113418 objects misplaced (7.789%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T18:52:24.719 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:24.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:24.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:24.827 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:24.858 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:24.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:24.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:24.965 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:24.993 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:24.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:24.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:25.095 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:25.124 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:25.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:25.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:25.229 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:25.243 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:52:25.256 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:25.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:25.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:25.361 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:25.369 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:25 smithi098 bash[79516]: audit 2023-09-23T18:52:24.625890+0000 mon.a (mon.0) 1406 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:52:25.370 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:25 smithi098 bash[79516]: cluster 2023-09-23T18:52:25.190746+0000 mon.a (mon.0) 1407 : cluster 3 Health check update: Degraded data redundancy: 55252/113418 objects degraded (48.715%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:25.370 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:25 smithi098 bash[76817]: audit 2023-09-23T18:52:24.625890+0000 mon.a (mon.0) 1406 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:52:25.370 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:25 smithi098 bash[76817]: cluster 2023-09-23T18:52:25.190746+0000 mon.a (mon.0) 1407 : cluster 3 Health check update: Degraded data redundancy: 55252/113418 objects degraded (48.715%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:25.393 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:25.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:25.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:25.498 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:25.505 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:25 smithi173 bash[59422]: audit 2023-09-23T18:52:24.625890+0000 mon.a (mon.0) 1406 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:52:25.505 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:25 smithi173 bash[59422]: cluster 2023-09-23T18:52:25.190746+0000 mon.a (mon.0) 1407 : cluster 3 Health check update: Degraded data redundancy: 55252/113418 objects degraded (48.715%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:25.521 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:25.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:25.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:25.626 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:25.654 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:25.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:25.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:25.762 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:25.809 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:25.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:25.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:25.914 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:25.938 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:25.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:25.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:26.045 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:26.074 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:26.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:26.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:26.180 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:26.211 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:26.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:26.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:26.315 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:26.338 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:26 smithi098 bash[79516]: cluster 2023-09-23T18:52:26.170630+0000 mgr.y (mgr.26054) 1033 : cluster 0 pgmap v1103: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 416 op/s; 55044/111270 objects degraded (49.469%); 8683/111270 objects misplaced (7.804%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T18:52:26.339 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:26 smithi098 bash[76817]: cluster 2023-09-23T18:52:26.170630+0000 mgr.y (mgr.26054) 1033 : cluster 0 pgmap v1103: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 416 op/s; 55044/111270 objects degraded (49.469%); 8683/111270 objects misplaced (7.804%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T18:52:26.339 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:52:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:52:26] "GET /metrics HTTP/1.1" 200 36615 "" "Prometheus/2.33.4" 2023-09-23T18:52:26.340 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:26.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:26.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:26.445 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:26.484 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:26.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:26.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:26.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:26 smithi173 bash[59422]: cluster 2023-09-23T18:52:26.170630+0000 mgr.y (mgr.26054) 1033 : cluster 0 pgmap v1103: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 416 op/s; 55044/111270 objects degraded (49.469%); 8683/111270 objects misplaced (7.804%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T18:52:26.587 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:26.616 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:26.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:26.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:26.719 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:26.742 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:26.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:26.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:26.844 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:26.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:26.877 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:26.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:26.978 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:27.007 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:27.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:27.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:27.110 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:27.130 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:27.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:27.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:27.234 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:27.258 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:27.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:27.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:27.365 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:27.394 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:27.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:27.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:27.497 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:27.519 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:27.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:27.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:27.625 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:27.649 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:27.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:27.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:27.756 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:27.775 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:27.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:27.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:27.882 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:27.907 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:27.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:27.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:28.010 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:28.038 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:28.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:28.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:28.143 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:28.178 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:28.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:28.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:28.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:28 smithi173 bash[59422]: cluster 2023-09-23T18:52:28.172148+0000 mgr.y (mgr.26054) 1034 : cluster 0 pgmap v1104: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 55044/106530 objects degraded (51.670%); 8264/106530 objects misplaced (7.757%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:52:28.281 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:28.305 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:28.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:28.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:28.408 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:28.430 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:28.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:28.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:28.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:28 smithi098 bash[76817]: cluster 2023-09-23T18:52:28.172148+0000 mgr.y (mgr.26054) 1034 : cluster 0 pgmap v1104: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 55044/106530 objects degraded (51.670%); 8264/106530 objects misplaced (7.757%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:52:28.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:28 smithi098 bash[79516]: cluster 2023-09-23T18:52:28.172148+0000 mgr.y (mgr.26054) 1034 : cluster 0 pgmap v1104: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 55044/106530 objects degraded (51.670%); 8264/106530 objects misplaced (7.757%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:52:28.534 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:28.558 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:28.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:28.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:28.663 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:28.707 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:28.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:28.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:28.814 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:28.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:28.845 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:28.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:28.946 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:28.969 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:28.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:28.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:29.072 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:29.110 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:29.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:29.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:29.221 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:29.247 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:29.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:29.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:29.350 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:29.376 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:29.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:29.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:29.483 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:29.510 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:29.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:29.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:29.614 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:29.657 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:29.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:29.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:29.766 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:29.800 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:29.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:29.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:29.906 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:29.929 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:29.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:29.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:30.037 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:30.068 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:30.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:30.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:30.173 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:30.204 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:30.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:30.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:30.308 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:30.336 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:30.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:30.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:30.440 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:30.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:30 smithi098 bash[76817]: cluster 2023-09-23T18:52:30.172814+0000 mgr.y (mgr.26054) 1035 : cluster 0 pgmap v1105: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 417 op/s; 55097/104925 objects degraded (52.511%); 8132/104925 objects misplaced (7.750%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T18:52:30.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:30 smithi098 bash[76817]: cluster 2023-09-23T18:52:30.191781+0000 mon.a (mon.0) 1408 : cluster 3 Health check update: Degraded data redundancy: 55044/106530 objects degraded (51.670%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:30.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:30 smithi098 bash[79516]: cluster 2023-09-23T18:52:30.172814+0000 mgr.y (mgr.26054) 1035 : cluster 0 pgmap v1105: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 417 op/s; 55097/104925 objects degraded (52.511%); 8132/104925 objects misplaced (7.750%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T18:52:30.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:30 smithi098 bash[79516]: cluster 2023-09-23T18:52:30.191781+0000 mon.a (mon.0) 1408 : cluster 3 Health check update: Degraded data redundancy: 55044/106530 objects degraded (51.670%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:30.449 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:30 smithi173 bash[59422]: cluster 2023-09-23T18:52:30.172814+0000 mgr.y (mgr.26054) 1035 : cluster 0 pgmap v1105: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 417 op/s; 55097/104925 objects degraded (52.511%); 8132/104925 objects misplaced (7.750%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T18:52:30.449 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:30 smithi173 bash[59422]: cluster 2023-09-23T18:52:30.191781+0000 mon.a (mon.0) 1408 : cluster 3 Health check update: Degraded data redundancy: 55044/106530 objects degraded (51.670%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:30.466 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:30.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:30.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:30.572 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:30.610 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:30.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:30.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:30.712 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:30.754 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:30.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:30.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:30.860 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:30.892 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:30.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:30.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:30.998 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:31.019 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:31.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:31.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:31.122 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:31.150 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:31.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:31.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:31.253 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:31.282 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:31.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:31.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:31.387 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:31.417 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:31.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:31.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:31.520 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:31.554 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:31.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:31.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:31.657 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:31.693 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:31.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:31.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:31.797 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:31.931 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:31.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:31.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:32.035 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:32.062 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:32.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:32.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:32.169 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:32.201 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:32.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:32.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:32.301 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:32 smithi098 bash[79516]: cluster 2023-09-23T18:52:32.173553+0000 mgr.y (mgr.26054) 1036 : cluster 0 pgmap v1106: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 383 op/s; 54807/104490 objects degraded (52.452%); 8132/104490 objects misplaced (7.783%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T18:52:32.301 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:32 smithi098 bash[76817]: cluster 2023-09-23T18:52:32.173553+0000 mgr.y (mgr.26054) 1036 : cluster 0 pgmap v1106: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 383 op/s; 54807/104490 objects degraded (52.452%); 8132/104490 objects misplaced (7.783%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T18:52:32.305 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:32.333 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:32.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:32.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:32.437 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:32.448 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:32 smithi173 bash[59422]: cluster 2023-09-23T18:52:32.173553+0000 mgr.y (mgr.26054) 1036 : cluster 0 pgmap v1106: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 383 op/s; 54807/104490 objects degraded (52.452%); 8132/104490 objects misplaced (7.783%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T18:52:32.468 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:32.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:32.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:32.570 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:32.603 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:32.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:32.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:32.711 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:32.740 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:32.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:32.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:32.846 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:32.878 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:32.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:32.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:32.984 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:33.005 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:33.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:33.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:33.113 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:33.146 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:33.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:33.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:33.250 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:33.284 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:33.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:33.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:33.387 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:33.420 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:33.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:33.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:33.525 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:33.558 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:33.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:33.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:33.662 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:33.687 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:33.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:33.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:33.737 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:52:33.737 INFO:teuthology.orchestra.run.smithi173.stdout: "success": "" 2023-09-23T18:52:33.737 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:52:33.792 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:33.830 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:33.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:33.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:33.938 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:33.970 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:33.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:33.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:34.075 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:34.098 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:34.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:34.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:34.203 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:34.234 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:34.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:34.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:34.336 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:34.388 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:34.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:34.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:34.489 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:34 smithi098 bash[76817]: cluster 2023-09-23T18:52:34.174630+0000 mgr.y (mgr.26054) 1037 : cluster 0 pgmap v1107: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 54807/100062 objects degraded (54.773%); 7753/100062 objects misplaced (7.748%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T18:52:34.490 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:34 smithi098 bash[79516]: cluster 2023-09-23T18:52:34.174630+0000 mgr.y (mgr.26054) 1037 : cluster 0 pgmap v1107: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 54807/100062 objects degraded (54.773%); 7753/100062 objects misplaced (7.748%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T18:52:34.492 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:34.499 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:34 smithi173 bash[59422]: cluster 2023-09-23T18:52:34.174630+0000 mgr.y (mgr.26054) 1037 : cluster 0 pgmap v1107: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 54807/100062 objects degraded (54.773%); 7753/100062 objects misplaced (7.748%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T18:52:34.510 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:52:34.514 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:34.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:34.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:34.572 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:52:34.619 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:34.627 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:52:34.646 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:34.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:34.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:34.681 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:52:34.729 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:52:34.729 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:52:34.729 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [], 2023-09-23T18:52:34.730 INFO:teuthology.orchestra.run.smithi098.stdout: "complaint_time": 30, 2023-09-23T18:52:34.730 INFO:teuthology.orchestra.run.smithi098.stdout: "num_blocked_ops": 0 2023-09-23T18:52:34.730 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:52:34.753 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:34.779 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:52:34.780 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:34.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:34.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:34.822 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:52:34.856 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:52:34.869 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:52:34.870 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:52:34.870 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:52:34.870 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:52:34.870 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:52:34.870 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:52:34.870 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:52:34.870 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:52:34.885 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:34.925 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:34.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:34.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:35.028 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:35.061 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:35.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:35.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:35.168 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:35.207 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:35.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:35.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:35.263 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:35 smithi098 bash[76817]: audit 2023-09-23T18:52:34.465063+0000 mon.c (mon.2) 118 : audit 0 from='client.? 172.21.15.98:0/302458012' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:52:35.263 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:35 smithi098 bash[76817]: audit 2023-09-23T18:52:35.156408+0000 mon.c (mon.2) 119 : audit 1 from='client.? 172.21.15.98:0/3895175448' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:52:35.263 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:35 smithi098 bash[76817]: audit 2023-09-23T18:52:35.156966+0000 mon.a (mon.0) 1409 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:52:35.264 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:35 smithi098 bash[76817]: cluster 2023-09-23T18:52:35.193952+0000 mon.a (mon.0) 1410 : cluster 3 Health check update: Degraded data redundancy: 54807/100062 objects degraded (54.773%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:35.289 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is unset 2023-09-23T18:52:35.311 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:35.332 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:35 smithi098 bash[79516]: audit 2023-09-23T18:52:34.465063+0000 mon.c (mon.2) 118 : audit 0 from='client.? 172.21.15.98:0/302458012' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:52:35.332 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:35 smithi098 bash[79516]: audit 2023-09-23T18:52:35.156408+0000 mon.c (mon.2) 119 : audit 1 from='client.? 172.21.15.98:0/3895175448' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:52:35.332 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:35 smithi098 bash[79516]: audit 2023-09-23T18:52:35.156966+0000 mon.a (mon.0) 1409 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:52:35.332 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:35 smithi098 bash[79516]: cluster 2023-09-23T18:52:35.193952+0000 mon.a (mon.0) 1410 : cluster 3 Health check update: Degraded data redundancy: 54807/100062 objects degraded (54.773%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:35.342 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:35.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:35.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:35.445 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:35.454 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:35 smithi173 bash[59422]: audit 2023-09-23T18:52:34.465063+0000 mon.c (mon.2) 118 : audit 0 from='client.? 172.21.15.98:0/302458012' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:52:35.454 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:35 smithi173 bash[59422]: audit 2023-09-23T18:52:35.156408+0000 mon.c (mon.2) 119 : audit 1 from='client.? 172.21.15.98:0/3895175448' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:52:35.454 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:35 smithi173 bash[59422]: audit 2023-09-23T18:52:35.156966+0000 mon.a (mon.0) 1409 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:52:35.454 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:35 smithi173 bash[59422]: cluster 2023-09-23T18:52:35.193952+0000 mon.a (mon.0) 1410 : cluster 3 Health check update: Degraded data redundancy: 54807/100062 objects degraded (54.773%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:35.469 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:35.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:35.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:35.577 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:35.610 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:35.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:35.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:35.714 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:35.744 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:35.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:35.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:35.848 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:35.874 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:35.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:35.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:35.977 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:36.008 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:36.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:36.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:36.113 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:36.133 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:36.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:36.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:36.238 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:36.262 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:36.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:36.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:36.264 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:36 smithi098 bash[76817]: cluster 2023-09-23T18:52:35.255168+0000 mon.a (mon.0) 1411 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:52:36.265 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:36 smithi098 bash[76817]: cluster 2023-09-23T18:52:35.273318+0000 osd.5 (osd.5) 163 : cluster 3 failed to encode map e262 with expected crc 2023-09-23T18:52:36.265 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:52:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:52:36] "GET /metrics HTTP/1.1" 200 36615 "" "Prometheus/2.33.4" 2023-09-23T18:52:36.364 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:36.397 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:36.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:36.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:36.505 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:36.514 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:36 smithi173 bash[59422]: cluster 2023-09-23T18:52:35.255168+0000 mon.a (mon.0) 1411 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:52:36.514 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:36 smithi173 bash[59422]: cluster 2023-09-23T18:52:35.273318+0000 osd.5 (osd.5) 163 : cluster 3 failed to encode map e262 with expected crc 2023-09-23T18:52:36.514 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:36 smithi173 bash[59422]: audit 2023-09-23T18:52:35.274224+0000 mon.a (mon.0) 1412 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:52:36.514 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:36 smithi173 bash[59422]: cluster 2023-09-23T18:52:35.274417+0000 mon.a (mon.0) 1413 : cluster 0 osdmap e262: 8 total, 8 up, 7 in 2023-09-23T18:52:36.514 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:36 smithi173 bash[59422]: cluster 2023-09-23T18:52:35.282100+0000 osd.4 (osd.4) 260 : cluster 3 failed to encode map e262 with expected crc 2023-09-23T18:52:36.514 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:36 smithi173 bash[59422]: cluster 2023-09-23T18:52:36.175342+0000 mgr.y (mgr.26054) 1038 : cluster 0 pgmap v1109: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 440 op/s; 54647/98040 objects degraded (55.739%); 7591/98040 objects misplaced (7.743%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T18:52:36.516 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:36 smithi098 bash[76817]: audit 2023-09-23T18:52:35.274224+0000 mon.a (mon.0) 1412 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:52:36.517 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:36 smithi098 bash[76817]: cluster 2023-09-23T18:52:35.274417+0000 mon.a (mon.0) 1413 : cluster 0 osdmap e262: 8 total, 8 up, 7 in 2023-09-23T18:52:36.517 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:36 smithi098 bash[76817]: cluster 2023-09-23T18:52:35.282100+0000 osd.4 (osd.4) 260 : cluster 3 failed to encode map e262 with expected crc 2023-09-23T18:52:36.517 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:36 smithi098 bash[76817]: cluster 2023-09-23T18:52:36.175342+0000 mgr.y (mgr.26054) 1038 : cluster 0 pgmap v1109: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 440 op/s; 54647/98040 objects degraded (55.739%); 7591/98040 objects misplaced (7.743%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T18:52:36.517 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:36 smithi098 bash[79516]: cluster 2023-09-23T18:52:35.255168+0000 mon.a (mon.0) 1411 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:52:36.517 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:36 smithi098 bash[79516]: cluster 2023-09-23T18:52:35.273318+0000 osd.5 (osd.5) 163 : cluster 3 failed to encode map e262 with expected crc 2023-09-23T18:52:36.517 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:36 smithi098 bash[79516]: audit 2023-09-23T18:52:35.274224+0000 mon.a (mon.0) 1412 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:52:36.517 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:36 smithi098 bash[79516]: cluster 2023-09-23T18:52:35.274417+0000 mon.a (mon.0) 1413 : cluster 0 osdmap e262: 8 total, 8 up, 7 in 2023-09-23T18:52:36.517 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:36 smithi098 bash[79516]: cluster 2023-09-23T18:52:35.282100+0000 osd.4 (osd.4) 260 : cluster 3 failed to encode map e262 with expected crc 2023-09-23T18:52:36.518 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:36 smithi098 bash[79516]: cluster 2023-09-23T18:52:36.175342+0000 mgr.y (mgr.26054) 1038 : cluster 0 pgmap v1109: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 2.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 440 op/s; 54647/98040 objects degraded (55.739%); 7591/98040 objects misplaced (7.743%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T18:52:36.527 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:36.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:36.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:36.631 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:36.660 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:36.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:36.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:36.769 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:36.798 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:36.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:36.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:36.901 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:36.927 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:36.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:36.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:37.031 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:37.061 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:37.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:37.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:37.165 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:37.189 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:37.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:37.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:37.291 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:37.331 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:37.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:37.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:37.434 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:37.466 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:37.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:37.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:37.572 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:37.596 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:37.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:37.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:37.702 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:37.730 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:37.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:37.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:37.833 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:37.869 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:37.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:37.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:37.893 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T18:52:37.986 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:38.010 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:38.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:38.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:38.113 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:38.146 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:38.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:38.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:38.252 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:38.283 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:38.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:38.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:38.387 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:38.412 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:38.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:38.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:38.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:38 smithi098 bash[79516]: cluster 2023-09-23T18:52:38.176193+0000 mgr.y (mgr.26054) 1039 : cluster 0 pgmap v1110: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 432 op/s; 54647/93561 objects degraded (58.408%); 7194/93561 objects misplaced (7.689%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:52:38.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:38 smithi098 bash[76817]: cluster 2023-09-23T18:52:38.176193+0000 mgr.y (mgr.26054) 1039 : cluster 0 pgmap v1110: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 432 op/s; 54647/93561 objects degraded (58.408%); 7194/93561 objects misplaced (7.689%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:52:38.515 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:38.523 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:38 smithi173 bash[59422]: cluster 2023-09-23T18:52:38.176193+0000 mgr.y (mgr.26054) 1039 : cluster 0 pgmap v1110: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 432 op/s; 54647/93561 objects degraded (58.408%); 7194/93561 objects misplaced (7.689%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:52:38.544 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:38.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:38.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:38.647 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:38.683 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:38.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:38.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:38.788 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:38.812 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:38.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:38.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:38.917 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:38.944 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:38.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:38.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:39.050 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:39.085 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:39.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:39.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:39.195 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:39.219 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:39.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:39.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:39.322 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:39.351 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:52:39.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:39.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:52:39.454 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:39.483 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:39.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:39.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:39.589 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:39.618 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:39.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:39.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:39.722 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:39.763 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:39.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:39.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:39.867 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:39.907 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:39.909 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:52:39.915 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:52:39.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:39.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:40.018 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:40.044 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:40.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:40.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:40.147 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:40.173 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:40.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:40.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:40.179 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T18:52:40.179 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:52:40.180 INFO:tasks.thrashosds.thrasher:Killing osd 4, live_osds are [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T18:52:40.180 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-09-23T18:52:40.180 DEBUG:teuthology.orchestra.run.smithi173:> sudo systemctl stop ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.4 2023-09-23T18:52:40.277 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:40.278 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 18:52:40 smithi173 systemd[1]: Stopping Ceph osd.4 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T18:52:40.332 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:40.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:40.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:40.439 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:40.480 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:40.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:40.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:40.586 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:40.640 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:40.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:40.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:40.746 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:40.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:40 smithi173 bash[59422]: audit 2023-09-23T18:52:39.632060+0000 mon.a (mon.0) 1414 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:52:40.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:40 smithi173 bash[59422]: audit 2023-09-23T18:52:39.633637+0000 mon.a (mon.0) 1415 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:52:40.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:40 smithi173 bash[59422]: audit 2023-09-23T18:52:40.141254+0000 mon.a (mon.0) 1416 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:52:40.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:40 smithi173 bash[59422]: cluster 2023-09-23T18:52:40.176875+0000 mgr.y (mgr.26054) 1040 : cluster 0 pgmap v1111: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 433 op/s; 54704/91926 objects degraded (59.509%); 7061/91926 objects misplaced (7.681%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:52:40.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:40 smithi173 bash[59422]: cluster 2023-09-23T18:52:40.195230+0000 mon.a (mon.0) 1417 : cluster 3 Health check update: Degraded data redundancy: 54647/93561 objects degraded (58.408%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:40.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:40 smithi173 bash[59422]: cluster 2023-09-23T18:52:40.439887+0000 mon.a (mon.0) 1418 : cluster 1 osd.4 marked itself down and dead 2023-09-23T18:52:40.779 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 18:52:40 smithi173 bash[31296]: debug 2023-09-23T18:52:40.436+0000 7f8ace555700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T18:52:40.779 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 18:52:40 smithi173 bash[31296]: debug 2023-09-23T18:52:40.436+0000 7f8ace555700 -1 osd.4 262 *** Got signal Terminated *** 2023-09-23T18:52:40.779 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 18:52:40 smithi173 bash[31296]: debug 2023-09-23T18:52:40.436+0000 7f8ace555700 -1 osd.4 262 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T18:52:40.801 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:40.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:40.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:40.904 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:40.931 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:40.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:40.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:40.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:40 smithi098 bash[76817]: audit 2023-09-23T18:52:39.632060+0000 mon.a (mon.0) 1414 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:52:40.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:40 smithi098 bash[76817]: audit 2023-09-23T18:52:39.633637+0000 mon.a (mon.0) 1415 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:52:40.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:40 smithi098 bash[76817]: audit 2023-09-23T18:52:40.141254+0000 mon.a (mon.0) 1416 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:52:40.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:40 smithi098 bash[76817]: cluster 2023-09-23T18:52:40.176875+0000 mgr.y (mgr.26054) 1040 : cluster 0 pgmap v1111: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 433 op/s; 54704/91926 objects degraded (59.509%); 7061/91926 objects misplaced (7.681%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:52:40.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:40 smithi098 bash[76817]: cluster 2023-09-23T18:52:40.195230+0000 mon.a (mon.0) 1417 : cluster 3 Health check update: Degraded data redundancy: 54647/93561 objects degraded (58.408%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:40.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:40 smithi098 bash[76817]: cluster 2023-09-23T18:52:40.439887+0000 mon.a (mon.0) 1418 : cluster 1 osd.4 marked itself down and dead 2023-09-23T18:52:40.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:40 smithi098 bash[79516]: audit 2023-09-23T18:52:39.632060+0000 mon.a (mon.0) 1414 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:52:40.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:40 smithi098 bash[79516]: audit 2023-09-23T18:52:39.633637+0000 mon.a (mon.0) 1415 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:52:40.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:40 smithi098 bash[79516]: audit 2023-09-23T18:52:40.141254+0000 mon.a (mon.0) 1416 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:52:40.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:40 smithi098 bash[79516]: cluster 2023-09-23T18:52:40.176875+0000 mgr.y (mgr.26054) 1040 : cluster 0 pgmap v1111: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 433 op/s; 54704/91926 objects degraded (59.509%); 7061/91926 objects misplaced (7.681%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:52:40.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:40 smithi098 bash[79516]: cluster 2023-09-23T18:52:40.195230+0000 mon.a (mon.0) 1417 : cluster 3 Health check update: Degraded data redundancy: 54647/93561 objects degraded (58.408%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:40.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:40 smithi098 bash[79516]: cluster 2023-09-23T18:52:40.439887+0000 mon.a (mon.0) 1418 : cluster 1 osd.4 marked itself down and dead 2023-09-23T18:52:41.037 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:41.078 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:41.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:41.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:41.184 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:41.214 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:41.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:41.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:41.317 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:41.344 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:41.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:41.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:41.446 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:41.475 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:41.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:41.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:41.578 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:41.604 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:41.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:41.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:41.707 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:41.732 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:41.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:41.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:41.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:41 smithi173 bash[59422]: cluster 2023-09-23T18:52:41.200576+0000 mon.a (mon.0) 1419 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:52:41.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:41 smithi173 bash[59422]: cluster 2023-09-23T18:52:41.211100+0000 mon.a (mon.0) 1420 : cluster 0 osdmap e263: 8 total, 7 up, 7 in 2023-09-23T18:52:41.835 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:41.860 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:41.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:41.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:41.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:41 smithi098 bash[79516]: cluster 2023-09-23T18:52:41.200576+0000 mon.a (mon.0) 1419 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:52:41.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:41 smithi098 bash[79516]: cluster 2023-09-23T18:52:41.211100+0000 mon.a (mon.0) 1420 : cluster 0 osdmap e263: 8 total, 7 up, 7 in 2023-09-23T18:52:41.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:41 smithi098 bash[76817]: cluster 2023-09-23T18:52:41.200576+0000 mon.a (mon.0) 1419 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:52:41.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:41 smithi098 bash[76817]: cluster 2023-09-23T18:52:41.211100+0000 mon.a (mon.0) 1420 : cluster 0 osdmap e263: 8 total, 7 up, 7 in 2023-09-23T18:52:41.963 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:41.995 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:42.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:42.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:42.123 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:42.148 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:42.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:42.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:42.252 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:42.282 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:42.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:42.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:42.385 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:42.409 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:42.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:42.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:42.514 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:42.542 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:42.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:42.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:42.646 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:42.656 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:42 smithi098 bash[76817]: cluster 2023-09-23T18:52:41.214315+0000 osd.6 (osd.6) 221 : cluster 3 failed to encode map e263 with expected crc 2023-09-23T18:52:42.656 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:42 smithi098 bash[76817]: cluster 2023-09-23T18:52:41.215143+0000 osd.7 (osd.7) 183 : cluster 3 failed to encode map e263 with expected crc 2023-09-23T18:52:42.656 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:42 smithi098 bash[76817]: cluster 2023-09-23T18:52:41.235219+0000 osd.5 (osd.5) 164 : cluster 3 failed to encode map e263 with expected crc 2023-09-23T18:52:42.656 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:42 smithi098 bash[76817]: cluster 2023-09-23T18:52:42.177503+0000 mgr.y (mgr.26054) 1041 : cluster 0 pgmap v1113: 89 pgs: 1 stale+active+recovering+undersized+remapped, 1 stale+active+recovery_wait+undersized+degraded+remapped, 1 stale+active+remapped+backfill_wait, 17 stale+active+clean, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 57 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 356 op/s; 54426/91512 objects degraded (59.474%); 7061/91512 objects misplaced (7.716%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T18:52:42.656 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:42 smithi098 bash[76817]: cluster 2023-09-23T18:52:42.211597+0000 osd.0 (osd.0) 36 : cluster 1 8.2 continuing backfill to osd.6 from (242'4063,262'11059] 8:4205b97c:::benchmark_data_smithi098_176514_object70291:head to 262'11059 2023-09-23T18:52:42.656 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:42 smithi098 bash[76817]: cluster 2023-09-23T18:52:42.218183+0000 mon.a (mon.0) 1421 : cluster 0 osdmap e264: 8 total, 7 up, 7 in 2023-09-23T18:52:42.674 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:42.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:42.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:42.778 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:42.791 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:52:42.807 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:42.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:42.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:42.910 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:42.923 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:42 smithi173 bash[59422]: cluster 2023-09-23T18:52:41.214315+0000 osd.6 (osd.6) 221 : cluster 3 failed to encode map e263 with expected crc 2023-09-23T18:52:42.923 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:42 smithi173 bash[59422]: cluster 2023-09-23T18:52:41.215143+0000 osd.7 (osd.7) 183 : cluster 3 failed to encode map e263 with expected crc 2023-09-23T18:52:42.923 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:42 smithi173 bash[59422]: cluster 2023-09-23T18:52:41.235219+0000 osd.5 (osd.5) 164 : cluster 3 failed to encode map e263 with expected crc 2023-09-23T18:52:42.923 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:42 smithi173 bash[59422]: cluster 2023-09-23T18:52:42.177503+0000 mgr.y (mgr.26054) 1041 : cluster 0 pgmap v1113: 89 pgs: 1 stale+active+recovering+undersized+remapped, 1 stale+active+recovery_wait+undersized+degraded+remapped, 1 stale+active+remapped+backfill_wait, 17 stale+active+clean, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 57 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 356 op/s; 54426/91512 objects degraded (59.474%); 7061/91512 objects misplaced (7.716%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T18:52:42.923 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:42 smithi173 bash[59422]: cluster 2023-09-23T18:52:42.211597+0000 osd.0 (osd.0) 36 : cluster 1 8.2 continuing backfill to osd.6 from (242'4063,262'11059] 8:4205b97c:::benchmark_data_smithi098_176514_object70291:head to 262'11059 2023-09-23T18:52:42.923 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:42 smithi173 bash[59422]: cluster 2023-09-23T18:52:42.218183+0000 mon.a (mon.0) 1421 : cluster 0 osdmap e264: 8 total, 7 up, 7 in 2023-09-23T18:52:42.941 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:42.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:42 smithi098 bash[79516]: cluster 2023-09-23T18:52:41.214315+0000 osd.6 (osd.6) 221 : cluster 3 failed to encode map e263 with expected crc 2023-09-23T18:52:42.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:42 smithi098 bash[79516]: cluster 2023-09-23T18:52:41.215143+0000 osd.7 (osd.7) 183 : cluster 3 failed to encode map e263 with expected crc 2023-09-23T18:52:42.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:42 smithi098 bash[79516]: cluster 2023-09-23T18:52:41.235219+0000 osd.5 (osd.5) 164 : cluster 3 failed to encode map e263 with expected crc 2023-09-23T18:52:42.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:42 smithi098 bash[79516]: cluster 2023-09-23T18:52:42.177503+0000 mgr.y (mgr.26054) 1041 : cluster 0 pgmap v1113: 89 pgs: 1 stale+active+recovering+undersized+remapped, 1 stale+active+recovery_wait+undersized+degraded+remapped, 1 stale+active+remapped+backfill_wait, 17 stale+active+clean, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 57 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 356 op/s; 54426/91512 objects degraded (59.474%); 7061/91512 objects misplaced (7.716%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T18:52:42.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:42 smithi098 bash[79516]: cluster 2023-09-23T18:52:42.211597+0000 osd.0 (osd.0) 36 : cluster 1 8.2 continuing backfill to osd.6 from (242'4063,262'11059] 8:4205b97c:::benchmark_data_smithi098_176514_object70291:head to 262'11059 2023-09-23T18:52:42.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:42 smithi098 bash[79516]: cluster 2023-09-23T18:52:42.218183+0000 mon.a (mon.0) 1421 : cluster 0 osdmap e264: 8 total, 7 up, 7 in 2023-09-23T18:52:42.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:42.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:43.044 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:43.073 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:43.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:43.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:43.176 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:43.208 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:43.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:43.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:43.312 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:43.339 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:43.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:43.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:43.442 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:43.472 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:43.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:43.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:43.578 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:43.604 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:43.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:43.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:43.707 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:43.716 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:43 smithi098 bash[76817]: cluster 2023-09-23T18:52:42.211900+0000 osd.6 (osd.6) 222 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.716 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:43 smithi098 bash[76817]: cluster 2023-09-23T18:52:42.215463+0000 osd.5 (osd.5) 165 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.716 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:43 smithi098 bash[76817]: cluster 2023-09-23T18:52:42.215923+0000 osd.6 (osd.6) 223 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.716 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:43 smithi098 bash[76817]: cluster 2023-09-23T18:52:42.216043+0000 osd.7 (osd.7) 184 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.716 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:43 smithi098 bash[76817]: cluster 2023-09-23T18:52:42.217217+0000 osd.7 (osd.7) 185 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.716 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:43 smithi098 bash[76817]: cluster 2023-09-23T18:52:42.217485+0000 osd.6 (osd.6) 224 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.716 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:43 smithi098 bash[76817]: cluster 2023-09-23T18:52:42.218024+0000 osd.5 (osd.5) 166 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.716 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:43 smithi098 bash[76817]: cluster 2023-09-23T18:52:42.218352+0000 osd.7 (osd.7) 186 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.717 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:43 smithi098 bash[76817]: cluster 2023-09-23T18:52:42.220190+0000 osd.7 (osd.7) 187 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.717 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:43 smithi098 bash[76817]: cluster 2023-09-23T18:52:42.222573+0000 osd.5 (osd.5) 167 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.717 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:43 smithi098 bash[76817]: cluster 2023-09-23T18:52:43.210275+0000 mon.a (mon.0) 1422 : cluster 0 osdmap e265: 8 total, 7 up, 7 in 2023-09-23T18:52:43.717 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:43 smithi098 bash[79516]: cluster 2023-09-23T18:52:42.211900+0000 osd.6 (osd.6) 222 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.717 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:43 smithi098 bash[79516]: cluster 2023-09-23T18:52:42.215463+0000 osd.5 (osd.5) 165 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.717 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:43 smithi098 bash[79516]: cluster 2023-09-23T18:52:42.215923+0000 osd.6 (osd.6) 223 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.717 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:43 smithi098 bash[79516]: cluster 2023-09-23T18:52:42.216043+0000 osd.7 (osd.7) 184 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.717 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:43 smithi098 bash[79516]: cluster 2023-09-23T18:52:42.217217+0000 osd.7 (osd.7) 185 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.717 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:43 smithi098 bash[79516]: cluster 2023-09-23T18:52:42.217485+0000 osd.6 (osd.6) 224 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:43 smithi098 bash[79516]: cluster 2023-09-23T18:52:42.218024+0000 osd.5 (osd.5) 166 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:43 smithi098 bash[79516]: cluster 2023-09-23T18:52:42.218352+0000 osd.7 (osd.7) 186 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:43 smithi098 bash[79516]: cluster 2023-09-23T18:52:42.220190+0000 osd.7 (osd.7) 187 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:43 smithi098 bash[79516]: cluster 2023-09-23T18:52:42.222573+0000 osd.5 (osd.5) 167 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:43 smithi098 bash[79516]: cluster 2023-09-23T18:52:43.210275+0000 mon.a (mon.0) 1422 : cluster 0 osdmap e265: 8 total, 7 up, 7 in 2023-09-23T18:52:43.736 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:43.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:43.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:43.812 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:52:43.839 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:43.872 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:43.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:43.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:43.922 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:43 smithi173 bash[59422]: cluster 2023-09-23T18:52:42.211900+0000 osd.6 (osd.6) 222 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.923 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:43 smithi173 bash[59422]: cluster 2023-09-23T18:52:42.215463+0000 osd.5 (osd.5) 165 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.923 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:43 smithi173 bash[59422]: cluster 2023-09-23T18:52:42.215923+0000 osd.6 (osd.6) 223 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.923 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:43 smithi173 bash[59422]: cluster 2023-09-23T18:52:42.216043+0000 osd.7 (osd.7) 184 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.923 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:43 smithi173 bash[59422]: cluster 2023-09-23T18:52:42.217217+0000 osd.7 (osd.7) 185 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.923 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:43 smithi173 bash[59422]: cluster 2023-09-23T18:52:42.217485+0000 osd.6 (osd.6) 224 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.923 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:43 smithi173 bash[59422]: cluster 2023-09-23T18:52:42.218024+0000 osd.5 (osd.5) 166 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.923 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:43 smithi173 bash[59422]: cluster 2023-09-23T18:52:42.218352+0000 osd.7 (osd.7) 186 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.923 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:43 smithi173 bash[59422]: cluster 2023-09-23T18:52:42.220190+0000 osd.7 (osd.7) 187 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.923 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:43 smithi173 bash[59422]: cluster 2023-09-23T18:52:42.222573+0000 osd.5 (osd.5) 167 : cluster 3 failed to encode map e264 with expected crc 2023-09-23T18:52:43.924 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:43 smithi173 bash[59422]: cluster 2023-09-23T18:52:43.210275+0000 mon.a (mon.0) 1422 : cluster 0 osdmap e265: 8 total, 7 up, 7 in 2023-09-23T18:52:43.978 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:43.999 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:44.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:44.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:44.102 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:44.128 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:44.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:44.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:44.174 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 18:52:43 smithi173 bash[122356]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-4 2023-09-23T18:52:44.174 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 18:52:44 smithi173 bash[122566]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-4 2023-09-23T18:52:44.232 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:44.265 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:44.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:44.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:44.371 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:44.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:44.405 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:44.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:44.506 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:44.532 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:44.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:44.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:44.639 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:44.672 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:44.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:44.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:44.775 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:44.784 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:44 smithi098 bash[79516]: cluster 2023-09-23T18:52:43.506369+0000 osd.5 (osd.5) 168 : cluster 3 failed to encode map e265 with expected crc 2023-09-23T18:52:44.784 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:44 smithi098 bash[79516]: cluster 2023-09-23T18:52:44.178030+0000 mgr.y (mgr.26054) 1042 : cluster 0 pgmap v1116: 89 pgs: 2 remapped+peering, 15 active+undersized, 1 active+recovery_wait+degraded, 1 stale+active+recovering+undersized+remapped, 1 stale+active+remapped+backfill_wait, 7 stale+active+clean, 1 active+remapped+backfill_wait, 3 active+undersized+degraded, 3 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 49 active+clean; 1.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 243 op/s; 43802/89175 objects degraded (49.119%); 8389/89175 objects misplaced (9.407%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T18:52:44.785 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:44 smithi098 bash[76817]: cluster 2023-09-23T18:52:43.506369+0000 osd.5 (osd.5) 168 : cluster 3 failed to encode map e265 with expected crc 2023-09-23T18:52:44.785 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:44 smithi098 bash[76817]: cluster 2023-09-23T18:52:44.178030+0000 mgr.y (mgr.26054) 1042 : cluster 0 pgmap v1116: 89 pgs: 2 remapped+peering, 15 active+undersized, 1 active+recovery_wait+degraded, 1 stale+active+recovering+undersized+remapped, 1 stale+active+remapped+backfill_wait, 7 stale+active+clean, 1 active+remapped+backfill_wait, 3 active+undersized+degraded, 3 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 49 active+clean; 1.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 243 op/s; 43802/89175 objects degraded (49.119%); 8389/89175 objects misplaced (9.407%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T18:52:44.806 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:44.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:44.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:44.830 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:52:44.910 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:44.932 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:44.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:44.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:45.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:44 smithi173 bash[59422]: cluster 2023-09-23T18:52:43.506369+0000 osd.5 (osd.5) 168 : cluster 3 failed to encode map e265 with expected crc 2023-09-23T18:52:45.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:44 smithi173 bash[59422]: cluster 2023-09-23T18:52:44.178030+0000 mgr.y (mgr.26054) 1042 : cluster 0 pgmap v1116: 89 pgs: 2 remapped+peering, 15 active+undersized, 1 active+recovery_wait+degraded, 1 stale+active+recovering+undersized+remapped, 1 stale+active+remapped+backfill_wait, 7 stale+active+clean, 1 active+remapped+backfill_wait, 3 active+undersized+degraded, 3 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 49 active+clean; 1.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 243 op/s; 43802/89175 objects degraded (49.119%); 8389/89175 objects misplaced (9.407%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T18:52:45.035 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:45.059 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:45.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:45.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:45.162 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:45.183 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:45.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:45.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:45.286 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:45.316 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:45.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:45.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:45.419 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:45.448 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:45.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:45.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:45.551 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:45.599 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:45.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:45.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:45.702 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:45.728 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:45.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:45.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:45.831 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:45.842 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:45 smithi098 bash[76817]: cluster 2023-09-23T18:52:43.766416+0000 osd.7 (osd.7) 188 : cluster 3 failed to encode map e265 with expected crc 2023-09-23T18:52:45.842 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:45 smithi098 bash[76817]: cluster 2023-09-23T18:52:43.800483+0000 osd.6 (osd.6) 225 : cluster 3 failed to encode map e265 with expected crc 2023-09-23T18:52:45.842 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:45 smithi098 bash[76817]: cluster 2023-09-23T18:52:45.196492+0000 mon.a (mon.0) 1423 : cluster 3 Health check update: Degraded data redundancy: 43802/89175 objects degraded (49.119%), 13 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:45.842 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:45 smithi098 bash[79516]: cluster 2023-09-23T18:52:43.766416+0000 osd.7 (osd.7) 188 : cluster 3 failed to encode map e265 with expected crc 2023-09-23T18:52:45.843 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:45 smithi098 bash[79516]: cluster 2023-09-23T18:52:43.800483+0000 osd.6 (osd.6) 225 : cluster 3 failed to encode map e265 with expected crc 2023-09-23T18:52:45.843 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:45 smithi098 bash[79516]: cluster 2023-09-23T18:52:45.196492+0000 mon.a (mon.0) 1423 : cluster 3 Health check update: Degraded data redundancy: 43802/89175 objects degraded (49.119%), 13 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:45.857 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:45.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:45.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:45.962 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:45.983 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:45.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:45.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:46.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:45 smithi173 bash[59422]: cluster 2023-09-23T18:52:43.766416+0000 osd.7 (osd.7) 188 : cluster 3 failed to encode map e265 with expected crc 2023-09-23T18:52:46.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:45 smithi173 bash[59422]: cluster 2023-09-23T18:52:43.800483+0000 osd.6 (osd.6) 225 : cluster 3 failed to encode map e265 with expected crc 2023-09-23T18:52:46.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:45 smithi173 bash[59422]: cluster 2023-09-23T18:52:45.196492+0000 mon.a (mon.0) 1423 : cluster 3 Health check update: Degraded data redundancy: 43802/89175 objects degraded (49.119%), 13 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:46.087 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:46.095 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:52:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:52:46] "GET /metrics HTTP/1.1" 200 36612 "" "Prometheus/2.33.4" 2023-09-23T18:52:46.109 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:46.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:46.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:46.212 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:46.238 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:46.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:46.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:46.345 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:46.365 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:46.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:46.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:46.468 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:46.491 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:46.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:46.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:46.594 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:46.627 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:46.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:46.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:46.684 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:46 smithi098 bash[76817]: cluster 2023-09-23T18:52:46.179069+0000 mgr.y (mgr.26054) 1043 : cluster 0 pgmap v1117: 89 pgs: 2 remapped+peering, 1 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 1 active+recovery_wait+degraded, 1 stale+active+recovering+undersized+remapped, 2 stale+active+clean, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 37 active+clean; 1.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 232 op/s; 46862/87747 objects degraded (53.406%); 6935/87747 objects misplaced (7.903%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:52:46.685 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:46 smithi098 bash[76817]: audit 2023-09-23T18:52:46.543543+0000 mon.a (mon.0) 1424 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:52:46.685 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:46 smithi098 bash[76817]: audit 2023-09-23T18:52:46.548852+0000 mon.a (mon.0) 1425 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:52:46.730 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:46.755 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:46.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:46.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:46.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:46 smithi173 bash[59422]: cluster 2023-09-23T18:52:46.179069+0000 mgr.y (mgr.26054) 1043 : cluster 0 pgmap v1117: 89 pgs: 2 remapped+peering, 1 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 1 active+recovery_wait+degraded, 1 stale+active+recovering+undersized+remapped, 2 stale+active+clean, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 37 active+clean; 1.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 232 op/s; 46862/87747 objects degraded (53.406%); 6935/87747 objects misplaced (7.903%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:52:46.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:46 smithi173 bash[59422]: audit 2023-09-23T18:52:46.543543+0000 mon.a (mon.0) 1424 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:52:46.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:46 smithi173 bash[59422]: audit 2023-09-23T18:52:46.548852+0000 mon.a (mon.0) 1425 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:52:46.859 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:46.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:46.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:46.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:46.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:46 smithi098 bash[79516]: cluster 2023-09-23T18:52:46.179069+0000 mgr.y (mgr.26054) 1043 : cluster 0 pgmap v1117: 89 pgs: 2 remapped+peering, 1 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 1 active+recovery_wait+degraded, 1 stale+active+recovering+undersized+remapped, 2 stale+active+clean, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 37 active+clean; 1.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 232 op/s; 46862/87747 objects degraded (53.406%); 6935/87747 objects misplaced (7.903%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:52:46.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:46 smithi098 bash[79516]: audit 2023-09-23T18:52:46.543543+0000 mon.a (mon.0) 1424 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:52:46.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:46 smithi098 bash[79516]: audit 2023-09-23T18:52:46.548852+0000 mon.a (mon.0) 1425 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:52:46.989 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:47.022 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:47.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:47.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:47.090 DEBUG:teuthology.orchestra.run.smithi173:> sudo pkill -f 'journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.4.service' 2023-09-23T18:52:47.125 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T18:52:47.125 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-09-23T18:52:47.126 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:47.195 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:47.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:47.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:47.301 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:47.333 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:47.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:47.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:47.437 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:47.500 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:47.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:47.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:47.603 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:47.627 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:47.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:47.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:47.730 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:47.755 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:47.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:47.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:47.857 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:47.880 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:47.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:47.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:47.983 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:48.007 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:48.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:48.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:48.111 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:48.130 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:48.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:48.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:48.240 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:48.263 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:48.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:48.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:48.366 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:48.404 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:48.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:48.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:48.508 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:48.514 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:48 smithi098 bash[79516]: cluster 2023-09-23T18:52:48.179803+0000 mgr.y (mgr.26054) 1044 : cluster 0 pgmap v1118: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 1 stale+active+recovering+undersized+remapped, 2 stale+active+clean, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 37 active+clean; 1.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 199 op/s; 51795/87345 objects degraded (59.299%); 6939/87345 objects misplaced (7.944%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:52:48.515 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:48 smithi098 bash[76817]: cluster 2023-09-23T18:52:48.179803+0000 mgr.y (mgr.26054) 1044 : cluster 0 pgmap v1118: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 1 stale+active+recovering+undersized+remapped, 2 stale+active+clean, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 37 active+clean; 1.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 199 op/s; 51795/87345 objects degraded (59.299%); 6939/87345 objects misplaced (7.944%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:52:48.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:48 smithi173 bash[59422]: cluster 2023-09-23T18:52:48.179803+0000 mgr.y (mgr.26054) 1044 : cluster 0 pgmap v1118: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 1 stale+active+recovering+undersized+remapped, 2 stale+active+clean, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 37 active+clean; 1.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 199 op/s; 51795/87345 objects degraded (59.299%); 6939/87345 objects misplaced (7.944%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:52:48.530 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:48.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:48.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:48.636 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:48.656 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:48.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:48.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:48.759 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:48.781 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:48.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:48.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:48.884 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:48.904 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:48.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:48.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:49.007 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:49.037 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:49.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:49.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:49.140 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:49.166 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:49.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:49.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:49.269 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:49.290 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:49.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:49.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:49.392 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:49.420 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:49.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:49.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:49.523 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:49.548 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:49.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:49.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:49.652 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:49.678 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:49.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:49.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:49.784 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:49.807 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:49.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:49.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:49.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:49.942 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:49.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:49.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:50.046 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:50.068 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:50.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:50.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:50.173 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:50.203 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:50.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:50.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:50.309 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:50.317 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:50 smithi098 bash[76817]: audit 2023-09-23T18:52:49.254738+0000 mon.a (mon.0) 1426 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:52:50.317 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:50 smithi098 bash[76817]: audit 2023-09-23T18:52:49.263176+0000 mon.a (mon.0) 1427 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:52:50.318 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:50 smithi098 bash[76817]: audit 2023-09-23T18:52:49.730874+0000 mon.a (mon.0) 1428 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:52:50.318 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:50 smithi098 bash[76817]: audit 2023-09-23T18:52:49.731709+0000 mon.a (mon.0) 1429 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:52:50.318 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:50 smithi098 bash[76817]: audit 2023-09-23T18:52:49.739820+0000 mon.a (mon.0) 1430 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:52:50.318 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:50 smithi098 bash[76817]: cluster 2023-09-23T18:52:50.180512+0000 mgr.y (mgr.26054) 1045 : cluster 0 pgmap v1119: 89 pgs: 1 activating+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 37 active+clean; 1.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 190 op/s; 58574/86928 objects degraded (67.382%); 8855/86928 objects misplaced (10.187%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T18:52:50.318 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:50 smithi098 bash[76817]: cluster 2023-09-23T18:52:50.197593+0000 mon.a (mon.0) 1431 : cluster 3 Health check update: Degraded data redundancy: 51795/87345 objects degraded (59.299%), 23 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:50.318 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:50 smithi098 bash[76817]: cluster 2023-09-23T18:52:50.203048+0000 mon.a (mon.0) 1432 : cluster 0 osdmap e266: 8 total, 7 up, 7 in 2023-09-23T18:52:50.318 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:50 smithi098 bash[79516]: audit 2023-09-23T18:52:49.254738+0000 mon.a (mon.0) 1426 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:52:50.318 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:50 smithi098 bash[79516]: audit 2023-09-23T18:52:49.263176+0000 mon.a (mon.0) 1427 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:52:50.318 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:50 smithi098 bash[79516]: audit 2023-09-23T18:52:49.730874+0000 mon.a (mon.0) 1428 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:52:50.318 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:50 smithi098 bash[79516]: audit 2023-09-23T18:52:49.731709+0000 mon.a (mon.0) 1429 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:52:50.319 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:50 smithi098 bash[79516]: audit 2023-09-23T18:52:49.739820+0000 mon.a (mon.0) 1430 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:52:50.319 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:50 smithi098 bash[79516]: cluster 2023-09-23T18:52:50.180512+0000 mgr.y (mgr.26054) 1045 : cluster 0 pgmap v1119: 89 pgs: 1 activating+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 37 active+clean; 1.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 190 op/s; 58574/86928 objects degraded (67.382%); 8855/86928 objects misplaced (10.187%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T18:52:50.319 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:50 smithi098 bash[79516]: cluster 2023-09-23T18:52:50.197593+0000 mon.a (mon.0) 1431 : cluster 3 Health check update: Degraded data redundancy: 51795/87345 objects degraded (59.299%), 23 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:50.319 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:50 smithi098 bash[79516]: cluster 2023-09-23T18:52:50.203048+0000 mon.a (mon.0) 1432 : cluster 0 osdmap e266: 8 total, 7 up, 7 in 2023-09-23T18:52:50.335 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:50.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:50.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:50.440 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:50.468 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:50.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:50.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:50.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:50 smithi173 bash[59422]: audit 2023-09-23T18:52:49.254738+0000 mon.a (mon.0) 1426 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:52:50.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:50 smithi173 bash[59422]: audit 2023-09-23T18:52:49.263176+0000 mon.a (mon.0) 1427 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:52:50.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:50 smithi173 bash[59422]: audit 2023-09-23T18:52:49.730874+0000 mon.a (mon.0) 1428 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:52:50.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:50 smithi173 bash[59422]: audit 2023-09-23T18:52:49.731709+0000 mon.a (mon.0) 1429 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:52:50.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:50 smithi173 bash[59422]: audit 2023-09-23T18:52:49.739820+0000 mon.a (mon.0) 1430 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:52:50.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:50 smithi173 bash[59422]: cluster 2023-09-23T18:52:50.180512+0000 mgr.y (mgr.26054) 1045 : cluster 0 pgmap v1119: 89 pgs: 1 activating+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 37 active+clean; 1.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 190 op/s; 58574/86928 objects degraded (67.382%); 8855/86928 objects misplaced (10.187%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T18:52:50.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:50 smithi173 bash[59422]: cluster 2023-09-23T18:52:50.197593+0000 mon.a (mon.0) 1431 : cluster 3 Health check update: Degraded data redundancy: 51795/87345 objects degraded (59.299%), 23 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:50.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:50 smithi173 bash[59422]: cluster 2023-09-23T18:52:50.203048+0000 mon.a (mon.0) 1432 : cluster 0 osdmap e266: 8 total, 7 up, 7 in 2023-09-23T18:52:50.572 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:50.599 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:50.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:50.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:50.705 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:50.729 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:50.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:50.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:50.832 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:50.866 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:50.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:50.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:50.970 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:50.995 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:50.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:50.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:51.099 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:51.142 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:51.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:51.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:51.247 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:51.261 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:51 smithi098 bash[76817]: cluster 2023-09-23T18:52:50.202495+0000 osd.7 (osd.7) 189 : cluster 3 failed to encode map e266 with expected crc 2023-09-23T18:52:51.261 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:51 smithi098 bash[76817]: cluster 2023-09-23T18:52:50.203769+0000 osd.6 (osd.6) 226 : cluster 3 failed to encode map e266 with expected crc 2023-09-23T18:52:51.261 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:51 smithi098 bash[76817]: cluster 2023-09-23T18:52:50.223051+0000 osd.5 (osd.5) 169 : cluster 3 failed to encode map e266 with expected crc 2023-09-23T18:52:51.261 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:51 smithi098 bash[76817]: cluster 2023-09-23T18:52:50.225507+0000 osd.5 (osd.5) 170 : cluster 3 failed to encode map e266 with expected crc 2023-09-23T18:52:51.261 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:51 smithi098 bash[76817]: cluster 2023-09-23T18:52:51.205865+0000 mon.a (mon.0) 1433 : cluster 0 osdmap e267: 8 total, 7 up, 7 in 2023-09-23T18:52:51.272 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:51.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:51.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:51.376 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:51.401 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:51.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:51.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:51.504 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:51.513 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:51 smithi173 bash[59422]: cluster 2023-09-23T18:52:50.202495+0000 osd.7 (osd.7) 189 : cluster 3 failed to encode map e266 with expected crc 2023-09-23T18:52:51.513 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:51 smithi173 bash[59422]: cluster 2023-09-23T18:52:50.203769+0000 osd.6 (osd.6) 226 : cluster 3 failed to encode map e266 with expected crc 2023-09-23T18:52:51.513 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:51 smithi173 bash[59422]: cluster 2023-09-23T18:52:50.223051+0000 osd.5 (osd.5) 169 : cluster 3 failed to encode map e266 with expected crc 2023-09-23T18:52:51.513 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:51 smithi173 bash[59422]: cluster 2023-09-23T18:52:50.225507+0000 osd.5 (osd.5) 170 : cluster 3 failed to encode map e266 with expected crc 2023-09-23T18:52:51.513 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:51 smithi173 bash[59422]: cluster 2023-09-23T18:52:51.205865+0000 mon.a (mon.0) 1433 : cluster 0 osdmap e267: 8 total, 7 up, 7 in 2023-09-23T18:52:51.531 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:51.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:51.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:51.634 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:51.659 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:51.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:51.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:51.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:51 smithi098 bash[79516]: cluster 2023-09-23T18:52:50.202495+0000 osd.7 (osd.7) 189 : cluster 3 failed to encode map e266 with expected crc 2023-09-23T18:52:51.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:51 smithi098 bash[79516]: cluster 2023-09-23T18:52:50.203769+0000 osd.6 (osd.6) 226 : cluster 3 failed to encode map e266 with expected crc 2023-09-23T18:52:51.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:51 smithi098 bash[79516]: cluster 2023-09-23T18:52:50.223051+0000 osd.5 (osd.5) 169 : cluster 3 failed to encode map e266 with expected crc 2023-09-23T18:52:51.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:51 smithi098 bash[79516]: cluster 2023-09-23T18:52:50.225507+0000 osd.5 (osd.5) 170 : cluster 3 failed to encode map e266 with expected crc 2023-09-23T18:52:51.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:51 smithi098 bash[79516]: cluster 2023-09-23T18:52:51.205865+0000 mon.a (mon.0) 1433 : cluster 0 osdmap e267: 8 total, 7 up, 7 in 2023-09-23T18:52:51.762 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:51.789 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:51.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:51.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:51.893 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:51.921 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:51.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:51.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:52.024 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:52.046 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:52.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:52.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:52.125 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [4] live_osds: [0, 2, 3, 5, 6, 7, 1] 2023-09-23T18:52:52.125 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:52:52.125 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-09-23T18:52:52.126 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:52:52.148 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:52.199 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:52.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:52.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:52.303 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:52.330 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:52.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:52.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:52.433 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:52.460 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:52.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:52.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:52.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:52 smithi173 bash[59422]: cluster 2023-09-23T18:52:51.204528+0000 osd.5 (osd.5) 171 : cluster 3 failed to encode map e267 with expected crc 2023-09-23T18:52:52.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:52 smithi173 bash[59422]: cluster 2023-09-23T18:52:51.219154+0000 osd.6 (osd.6) 227 : cluster 3 failed to encode map e267 with expected crc 2023-09-23T18:52:52.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:52 smithi173 bash[59422]: cluster 2023-09-23T18:52:51.228657+0000 osd.7 (osd.7) 190 : cluster 3 failed to encode map e267 with expected crc 2023-09-23T18:52:52.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:52 smithi173 bash[59422]: cluster 2023-09-23T18:52:51.230643+0000 osd.7 (osd.7) 191 : cluster 3 failed to encode map e267 with expected crc 2023-09-23T18:52:52.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:52 smithi173 bash[59422]: cluster 2023-09-23T18:52:52.181235+0000 mgr.y (mgr.26054) 1046 : cluster 0 pgmap v1122: 89 pgs: 1 activating+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 37 active+clean; 1.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 94 op/s; 58558/86898 objects degraded (67.387%); 8855/86898 objects misplaced (10.190%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T18:52:52.568 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:52.601 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:52.606 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:52 smithi098 bash[76817]: cluster 2023-09-23T18:52:51.204528+0000 osd.5 (osd.5) 171 : cluster 3 failed to encode map e267 with expected crc 2023-09-23T18:52:52.606 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:52 smithi098 bash[76817]: cluster 2023-09-23T18:52:51.219154+0000 osd.6 (osd.6) 227 : cluster 3 failed to encode map e267 with expected crc 2023-09-23T18:52:52.606 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:52 smithi098 bash[76817]: cluster 2023-09-23T18:52:51.228657+0000 osd.7 (osd.7) 190 : cluster 3 failed to encode map e267 with expected crc 2023-09-23T18:52:52.606 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:52 smithi098 bash[76817]: cluster 2023-09-23T18:52:51.230643+0000 osd.7 (osd.7) 191 : cluster 3 failed to encode map e267 with expected crc 2023-09-23T18:52:52.606 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:52 smithi098 bash[76817]: cluster 2023-09-23T18:52:52.181235+0000 mgr.y (mgr.26054) 1046 : cluster 0 pgmap v1122: 89 pgs: 1 activating+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 37 active+clean; 1.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 94 op/s; 58558/86898 objects degraded (67.387%); 8855/86898 objects misplaced (10.190%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T18:52:52.607 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:52 smithi098 bash[79516]: cluster 2023-09-23T18:52:51.204528+0000 osd.5 (osd.5) 171 : cluster 3 failed to encode map e267 with expected crc 2023-09-23T18:52:52.607 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:52 smithi098 bash[79516]: cluster 2023-09-23T18:52:51.219154+0000 osd.6 (osd.6) 227 : cluster 3 failed to encode map e267 with expected crc 2023-09-23T18:52:52.607 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:52 smithi098 bash[79516]: cluster 2023-09-23T18:52:51.228657+0000 osd.7 (osd.7) 190 : cluster 3 failed to encode map e267 with expected crc 2023-09-23T18:52:52.607 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:52 smithi098 bash[79516]: cluster 2023-09-23T18:52:51.230643+0000 osd.7 (osd.7) 191 : cluster 3 failed to encode map e267 with expected crc 2023-09-23T18:52:52.607 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:52 smithi098 bash[79516]: cluster 2023-09-23T18:52:52.181235+0000 mgr.y (mgr.26054) 1046 : cluster 0 pgmap v1122: 89 pgs: 1 activating+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 37 active+clean; 1.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 94 op/s; 58558/86898 objects degraded (67.387%); 8855/86898 objects misplaced (10.190%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T18:52:52.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:52.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:52.709 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:52.733 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:52.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:52.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:52.836 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:52.862 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:52.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:52.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:52.965 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:52.994 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:52.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:52.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:53.100 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:53.131 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:53.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:53.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:53.234 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:53.264 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:53.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:53.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:53.368 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:53.401 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:53.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:53.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:53.504 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:53.535 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:53.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:53.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:53.639 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:53.659 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:53.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:53.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:53.762 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:53.786 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:53.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:53.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:53.889 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:53.915 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:53.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:53.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:54.018 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:54.054 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:54.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:54.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:54.159 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:54.182 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:54.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:54.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:54.285 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:54.311 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:54.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:54.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:54.415 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:54.423 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:54 smithi098 bash[79516]: cluster 2023-09-23T18:52:54.181922+0000 mgr.y (mgr.26054) 1047 : cluster 0 pgmap v1123: 89 pgs: 1 active+recovery_wait+degraded, 1 activating+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 37 active+clean; 1.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 67 op/s; 58482/86130 objects degraded (67.900%); 8804/86130 objects misplaced (10.222%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T18:52:54.423 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:54 smithi098 bash[76817]: cluster 2023-09-23T18:52:54.181922+0000 mgr.y (mgr.26054) 1047 : cluster 0 pgmap v1123: 89 pgs: 1 active+recovery_wait+degraded, 1 activating+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 37 active+clean; 1.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 67 op/s; 58482/86130 objects degraded (67.900%); 8804/86130 objects misplaced (10.222%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T18:52:54.445 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:54.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:54.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:54.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:54 smithi173 bash[59422]: cluster 2023-09-23T18:52:54.181922+0000 mgr.y (mgr.26054) 1047 : cluster 0 pgmap v1123: 89 pgs: 1 active+recovery_wait+degraded, 1 activating+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 37 active+clean; 1.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 67 op/s; 58482/86130 objects degraded (67.900%); 8804/86130 objects misplaced (10.222%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T18:52:54.548 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:54.575 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:54.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:54.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:54.678 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:54.706 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:54.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:54.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:54.809 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:54.836 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:54.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:54.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:54.940 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:54.973 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:54.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:54.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:55.077 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:55.104 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:55.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:55.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:55.207 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:55.232 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:55.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:55.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:55.339 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:55.363 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:55.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:55.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:55.467 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:55.497 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:55.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:55.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:55.600 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:55.631 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:55.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:55.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:55.736 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:55.772 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:55.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:55.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:55.875 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:55.884 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:55 smithi098 bash[79516]: audit 2023-09-23T18:52:54.629349+0000 mon.a (mon.0) 1434 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:52:55.884 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:55 smithi098 bash[79516]: audit 2023-09-23T18:52:54.630462+0000 mon.a (mon.0) 1435 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:52:55.884 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:55 smithi098 bash[79516]: cluster 2023-09-23T18:52:55.199278+0000 mon.a (mon.0) 1436 : cluster 3 Health check update: Degraded data redundancy: 58482/86130 objects degraded (67.900%), 24 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:55.884 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:55 smithi098 bash[76817]: audit 2023-09-23T18:52:54.629349+0000 mon.a (mon.0) 1434 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:52:55.885 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:55 smithi098 bash[76817]: audit 2023-09-23T18:52:54.630462+0000 mon.a (mon.0) 1435 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:52:55.885 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:55 smithi098 bash[76817]: cluster 2023-09-23T18:52:55.199278+0000 mon.a (mon.0) 1436 : cluster 3 Health check update: Degraded data redundancy: 58482/86130 objects degraded (67.900%), 24 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:55.899 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:55.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:55.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:56.007 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:56.017 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:55 smithi173 bash[59422]: audit 2023-09-23T18:52:54.629349+0000 mon.a (mon.0) 1434 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:52:56.017 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:55 smithi173 bash[59422]: audit 2023-09-23T18:52:54.630462+0000 mon.a (mon.0) 1435 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:52:56.017 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:55 smithi173 bash[59422]: cluster 2023-09-23T18:52:55.199278+0000 mon.a (mon.0) 1436 : cluster 3 Health check update: Degraded data redundancy: 58482/86130 objects degraded (67.900%), 24 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:52:56.034 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:56.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:56.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:56.137 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:56.146 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:52:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:52:56] "GET /metrics HTTP/1.1" 200 36617 "" "Prometheus/2.33.4" 2023-09-23T18:52:56.160 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:56.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:56.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:56.268 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:56.300 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:56.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:56.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:56.403 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:56.430 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:56.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:56.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:56.535 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:56.561 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:56.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:56.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:56.638 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:56 smithi098 bash[76817]: cluster 2023-09-23T18:52:56.182887+0000 mgr.y (mgr.26054) 1048 : cluster 0 pgmap v1124: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 106 op/s; 53075/84792 objects degraded (62.594%); 6680/84792 objects misplaced (7.878%); 5.5 MiB/s, 92 objects/s recovering 2023-09-23T18:52:56.638 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:56 smithi098 bash[79516]: cluster 2023-09-23T18:52:56.182887+0000 mgr.y (mgr.26054) 1048 : cluster 0 pgmap v1124: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 106 op/s; 53075/84792 objects degraded (62.594%); 6680/84792 objects misplaced (7.878%); 5.5 MiB/s, 92 objects/s recovering 2023-09-23T18:52:56.665 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:56.687 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:56.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:56.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:56.792 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:56.800 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:56 smithi173 bash[59422]: cluster 2023-09-23T18:52:56.182887+0000 mgr.y (mgr.26054) 1048 : cluster 0 pgmap v1124: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 106 op/s; 53075/84792 objects degraded (62.594%); 6680/84792 objects misplaced (7.878%); 5.5 MiB/s, 92 objects/s recovering 2023-09-23T18:52:56.817 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:56.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:56.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:56.920 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:56.943 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:56.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:56.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:57.047 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:57.072 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:57.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:57.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:57.184 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:57.210 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:57.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:57.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:57.313 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:57.343 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:52:57.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:57.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:52:57.446 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:57.481 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:57.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:57.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:57.586 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:57.623 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:57.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:57.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:57.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:57.758 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:57.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:57.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:57.861 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:57.891 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:57.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:57.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:57.995 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:58.021 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:58.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:58.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:58.124 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:58.157 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:58.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:58.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:58.261 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:58.263 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:52:58 smithi098 bash[76817]: cluster 2023-09-23T18:52:58.183630+0000 mgr.y (mgr.26054) 1049 : cluster 0 pgmap v1125: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 149 op/s; 52957/83328 objects degraded (63.552%); 6570/83328 objects misplaced (7.885%); 3.3 MiB/s, 75 objects/s recovering 2023-09-23T18:52:58.288 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:58.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:58.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:58.392 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:58.404 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:52:58 smithi173 bash[59422]: cluster 2023-09-23T18:52:58.183630+0000 mgr.y (mgr.26054) 1049 : cluster 0 pgmap v1125: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 149 op/s; 52957/83328 objects degraded (63.552%); 6570/83328 objects misplaced (7.885%); 3.3 MiB/s, 75 objects/s recovering 2023-09-23T18:52:58.423 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:58.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:58.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:58.529 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:58.554 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:52:58.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:58.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:52:58.657 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:58.686 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:52:58.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:58.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:52:58.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:52:58 smithi098 bash[79516]: cluster 2023-09-23T18:52:58.183630+0000 mgr.y (mgr.26054) 1049 : cluster 0 pgmap v1125: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 149 op/s; 52957/83328 objects degraded (63.552%); 6570/83328 objects misplaced (7.885%); 3.3 MiB/s, 75 objects/s recovering 2023-09-23T18:52:58.789 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:58.822 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:52:58.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:58.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:52:58.928 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:58.958 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:58.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:58.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:59.061 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:59.169 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:59.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:59.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:59.274 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:59.306 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:52:59.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:59.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:52:59.409 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:59.435 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:59.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:59.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:59.539 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:59.571 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:52:59.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:59.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:52:59.678 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:59.875 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:52:59.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:52:59.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:52:59.978 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:00.260 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:00.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:00.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:00.364 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:00.392 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:00.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:00.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:00.498 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:00.521 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:00.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:00.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:00.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:00 smithi173 bash[59422]: cluster 2023-09-23T18:53:00.184469+0000 mgr.y (mgr.26054) 1050 : cluster 0 pgmap v1126: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 158 op/s; 52718/82194 objects degraded (64.139%); 6449/82194 objects misplaced (7.846%); 3.1 MiB/s, 66 objects/s recovering 2023-09-23T18:53:00.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:00 smithi173 bash[59422]: cluster 2023-09-23T18:53:00.200163+0000 mon.a (mon.0) 1437 : cluster 3 Health check update: Degraded data redundancy: 52957/83328 objects degraded (63.552%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:00.553 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:00 smithi098 bash[79516]: cluster 2023-09-23T18:53:00.184469+0000 mgr.y (mgr.26054) 1050 : cluster 0 pgmap v1126: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 158 op/s; 52718/82194 objects degraded (64.139%); 6449/82194 objects misplaced (7.846%); 3.1 MiB/s, 66 objects/s recovering 2023-09-23T18:53:00.554 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:00 smithi098 bash[79516]: cluster 2023-09-23T18:53:00.200163+0000 mon.a (mon.0) 1437 : cluster 3 Health check update: Degraded data redundancy: 52957/83328 objects degraded (63.552%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:00.554 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:00 smithi098 bash[76817]: cluster 2023-09-23T18:53:00.184469+0000 mgr.y (mgr.26054) 1050 : cluster 0 pgmap v1126: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 158 op/s; 52718/82194 objects degraded (64.139%); 6449/82194 objects misplaced (7.846%); 3.1 MiB/s, 66 objects/s recovering 2023-09-23T18:53:00.554 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:00 smithi098 bash[76817]: cluster 2023-09-23T18:53:00.200163+0000 mon.a (mon.0) 1437 : cluster 3 Health check update: Degraded data redundancy: 52957/83328 objects degraded (63.552%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:00.625 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:00.647 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:00.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:00.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:00.753 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:00.985 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:00.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:00.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:01.094 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:01.121 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:01.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:01.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:01.225 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:01.252 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:01.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:01.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:01.360 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:01.392 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:01.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:01.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:01.496 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:01.525 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:01.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:01.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:01.629 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:01.657 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:01.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:01.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:01.762 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:01.791 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:01.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:01.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:01.894 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:02.001 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:02.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:02.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:02.105 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:02.133 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:02.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:02.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:02.237 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:02.263 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:02.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:02.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:02.370 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:02.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:02 smithi173 bash[59422]: cluster 2023-09-23T18:53:02.185117+0000 mgr.y (mgr.26054) 1051 : cluster 0 pgmap v1127: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 151 op/s; 52583/81921 objects degraded (64.187%); 6449/81921 objects misplaced (7.872%); 3.0 MiB/s, 63 objects/s recovering 2023-09-23T18:53:02.398 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:02.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:02.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:02.501 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:02.529 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:02.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:02.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:02.632 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:02.642 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:02 smithi098 bash[76817]: cluster 2023-09-23T18:53:02.185117+0000 mgr.y (mgr.26054) 1051 : cluster 0 pgmap v1127: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 151 op/s; 52583/81921 objects degraded (64.187%); 6449/81921 objects misplaced (7.872%); 3.0 MiB/s, 63 objects/s recovering 2023-09-23T18:53:02.642 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:02 smithi098 bash[79516]: cluster 2023-09-23T18:53:02.185117+0000 mgr.y (mgr.26054) 1051 : cluster 0 pgmap v1127: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 151 op/s; 52583/81921 objects degraded (64.187%); 6449/81921 objects misplaced (7.872%); 3.0 MiB/s, 63 objects/s recovering 2023-09-23T18:53:02.655 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:02.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:02.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:02.704 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:53:02.758 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:02.787 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:02.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:02.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:02.891 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:02.920 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:02.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:02.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:03.025 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:03.051 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:03.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:03.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:03.155 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:03.186 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:03.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:03.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:03.290 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:03.315 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:03.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:03.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:03.422 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:03.446 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:03.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:03.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:03.549 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:03.577 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:03.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:03.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:03.680 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:04.421 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:04.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:04.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:04.525 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:04.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:04 smithi173 bash[59422]: cluster 2023-09-23T18:53:04.185901+0000 mgr.y (mgr.26054) 1052 : cluster 0 pgmap v1128: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 178 op/s; 52469/80478 objects degraded (65.197%); 6336/80478 objects misplaced (7.873%); 3.0 MiB/s, 77 objects/s recovering 2023-09-23T18:53:04.551 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:04.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:04.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:04.654 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:04.681 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:04.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:04.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:04.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:04 smithi098 bash[76817]: cluster 2023-09-23T18:53:04.185901+0000 mgr.y (mgr.26054) 1052 : cluster 0 pgmap v1128: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 178 op/s; 52469/80478 objects degraded (65.197%); 6336/80478 objects misplaced (7.873%); 3.0 MiB/s, 77 objects/s recovering 2023-09-23T18:53:04.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:04 smithi098 bash[79516]: cluster 2023-09-23T18:53:04.185901+0000 mgr.y (mgr.26054) 1052 : cluster 0 pgmap v1128: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 178 op/s; 52469/80478 objects degraded (65.197%); 6336/80478 objects misplaced (7.873%); 3.0 MiB/s, 77 objects/s recovering 2023-09-23T18:53:04.784 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:04.812 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:04.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:04.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:04.920 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:04.945 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:04.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:04.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:05.048 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:05.073 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:05.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:05.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:05.182 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:05.211 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:05.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:05.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:05.318 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:05.347 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:05.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:05.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:05.368 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:05 smithi098 bash[76817]: cluster 2023-09-23T18:53:05.201275+0000 mon.a (mon.0) 1438 : cluster 3 Health check update: Degraded data redundancy: 52469/80478 objects degraded (65.197%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:05.368 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:05 smithi098 bash[79516]: cluster 2023-09-23T18:53:05.201275+0000 mon.a (mon.0) 1438 : cluster 3 Health check update: Degraded data redundancy: 52469/80478 objects degraded (65.197%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:05.453 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:05.512 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:05.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:05.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:05.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:05 smithi173 bash[59422]: cluster 2023-09-23T18:53:05.201275+0000 mon.a (mon.0) 1438 : cluster 3 Health check update: Degraded data redundancy: 52469/80478 objects degraded (65.197%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:05.616 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:05.638 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:05.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:05.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:05.740 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:05.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:05.776 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:05.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:05.877 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:05.903 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:05.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:05.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:06.006 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:06.028 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:06.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:06.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:06.133 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:06.155 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:06.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:06.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:06.259 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:06.285 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:06.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:06.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:06.389 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:06.396 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:06 smithi098 bash[76817]: cluster 2023-09-23T18:53:06.186799+0000 mgr.y (mgr.26054) 1053 : cluster 0 pgmap v1129: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.6 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 201 op/s; 52025/78885 objects degraded (65.950%); 6205/78885 objects misplaced (7.866%); 2.6 MiB/s, 70 objects/s recovering 2023-09-23T18:53:06.397 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:53:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:53:06] "GET /metrics HTTP/1.1" 200 36617 "" "Prometheus/2.33.4" 2023-09-23T18:53:06.397 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:06 smithi098 bash[79516]: cluster 2023-09-23T18:53:06.186799+0000 mgr.y (mgr.26054) 1053 : cluster 0 pgmap v1129: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.6 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 201 op/s; 52025/78885 objects degraded (65.950%); 6205/78885 objects misplaced (7.866%); 2.6 MiB/s, 70 objects/s recovering 2023-09-23T18:53:06.411 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:06.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:06.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:06.514 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:06.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:06 smithi173 bash[59422]: cluster 2023-09-23T18:53:06.186799+0000 mgr.y (mgr.26054) 1053 : cluster 0 pgmap v1129: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.6 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 201 op/s; 52025/78885 objects degraded (65.950%); 6205/78885 objects misplaced (7.866%); 2.6 MiB/s, 70 objects/s recovering 2023-09-23T18:53:06.536 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:06.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:06.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:06.639 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:06.677 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:06.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:06.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:06.782 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:06.809 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:06.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:06.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:06.918 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:06.947 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:06.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:06.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:07.051 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:07.075 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:07.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:07.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:07.178 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:07.203 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:07.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:07.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:07.312 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:07.341 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:07.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:07.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:07.444 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:07.471 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:07.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:07.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:07.576 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:07.601 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:07.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:07.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:07.704 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:07.746 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:07.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:07.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:07.852 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:07.881 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:07.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:07.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:07.985 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:08.020 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:08.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:08.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:08.124 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:08.147 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:08.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:08.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:08.251 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:08.276 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:08.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:08.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:08.379 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:08.388 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:08 smithi098 bash[76817]: cluster 2023-09-23T18:53:08.187664+0000 mgr.y (mgr.26054) 1054 : cluster 0 pgmap v1130: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.6 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 209 op/s; 51906/77232 objects degraded (67.208%); 6092/77232 objects misplaced (7.888%); 2.2 MiB/s, 77 objects/s recovering 2023-09-23T18:53:08.388 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:08 smithi098 bash[79516]: cluster 2023-09-23T18:53:08.187664+0000 mgr.y (mgr.26054) 1054 : cluster 0 pgmap v1130: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.6 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 209 op/s; 51906/77232 objects degraded (67.208%); 6092/77232 objects misplaced (7.888%); 2.2 MiB/s, 77 objects/s recovering 2023-09-23T18:53:08.404 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:08.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:08.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:08.509 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:08.518 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:08 smithi173 bash[59422]: cluster 2023-09-23T18:53:08.187664+0000 mgr.y (mgr.26054) 1054 : cluster 0 pgmap v1130: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.6 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 209 op/s; 51906/77232 objects degraded (67.208%); 6092/77232 objects misplaced (7.888%); 2.2 MiB/s, 77 objects/s recovering 2023-09-23T18:53:08.535 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:08.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:08.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:08.638 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:08.666 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:08.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:08.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:08.769 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:08.799 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:08.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:08.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:08.903 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:08.932 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:08.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:08.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:09.038 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:09.068 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:09.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:09.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:09.175 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:09.202 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:09.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:09.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:09.307 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:09.329 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:09.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:09.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:09.433 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:09.455 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:09.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:09.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:09.559 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:09.589 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:09.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:09.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:09.693 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:09.721 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:09.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:09.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:09.824 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:09.857 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:09.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:09.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:09.960 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:09.961 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:09 smithi098 bash[76817]: audit 2023-09-23T18:53:09.626898+0000 mon.a (mon.0) 1439 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:53:09.962 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:09 smithi098 bash[79516]: audit 2023-09-23T18:53:09.626898+0000 mon.a (mon.0) 1439 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:53:09.989 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:09.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:09.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:10.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:09 smithi173 bash[59422]: audit 2023-09-23T18:53:09.626898+0000 mon.a (mon.0) 1439 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:53:10.093 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:10.125 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:10.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:10.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:10.229 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:10.251 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:10.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:10.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:10.354 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:10.381 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:10.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:10.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:10.484 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:10.508 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:10.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:10.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:10.614 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:10.644 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:10.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:10.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:10.749 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:10.756 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:10 smithi098 bash[79516]: cluster 2023-09-23T18:53:10.188400+0000 mgr.y (mgr.26054) 1055 : cluster 0 pgmap v1131: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 203 op/s; 51644/76011 objects degraded (67.943%); 5958/76011 objects misplaced (7.838%); 2.1 MiB/s, 64 objects/s recovering 2023-09-23T18:53:10.756 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:10 smithi098 bash[79516]: cluster 2023-09-23T18:53:10.202459+0000 mon.a (mon.0) 1440 : cluster 3 Health check update: Degraded data redundancy: 51906/77232 objects degraded (67.208%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:10.757 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:10 smithi098 bash[76817]: cluster 2023-09-23T18:53:10.188400+0000 mgr.y (mgr.26054) 1055 : cluster 0 pgmap v1131: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 203 op/s; 51644/76011 objects degraded (67.943%); 5958/76011 objects misplaced (7.838%); 2.1 MiB/s, 64 objects/s recovering 2023-09-23T18:53:10.757 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:10 smithi098 bash[76817]: cluster 2023-09-23T18:53:10.202459+0000 mon.a (mon.0) 1440 : cluster 3 Health check update: Degraded data redundancy: 51906/77232 objects degraded (67.208%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:10.771 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:10.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:10.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:10.875 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:10.898 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:10.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:10.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:11.005 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:11.015 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:10 smithi173 bash[59422]: cluster 2023-09-23T18:53:10.188400+0000 mgr.y (mgr.26054) 1055 : cluster 0 pgmap v1131: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 203 op/s; 51644/76011 objects degraded (67.943%); 5958/76011 objects misplaced (7.838%); 2.1 MiB/s, 64 objects/s recovering 2023-09-23T18:53:11.015 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:10 smithi173 bash[59422]: cluster 2023-09-23T18:53:10.202459+0000 mon.a (mon.0) 1440 : cluster 3 Health check update: Degraded data redundancy: 51906/77232 objects degraded (67.208%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:11.032 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:11.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:11.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:11.135 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:11.157 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:11.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:11.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:11.260 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:11.298 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:11.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:11.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:11.401 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:11.435 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:11.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:11.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:11.539 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:11.570 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:11.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:11.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:11.674 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:11.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:11.706 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:11.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:11.807 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:11.831 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:11.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:11.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:11.935 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:11.964 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:11.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:11.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:12.067 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:12.094 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:12.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:12.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:12.198 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:12.224 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:12.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:12.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:12.328 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:12.339 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:12 smithi173 bash[59422]: cluster 2023-09-23T18:53:12.188977+0000 mgr.y (mgr.26054) 1056 : cluster 0 pgmap v1132: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.5 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 179 op/s; 51500/75726 objects degraded (68.008%); 5958/75726 objects misplaced (7.868%); 2.1 MiB/s, 63 objects/s recovering 2023-09-23T18:53:12.356 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:12.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:12.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:12.460 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:12.490 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:12.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:12.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:12.498 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:53:12.498 INFO:teuthology.orchestra.run.smithi098.stdout: "size": 20, 2023-09-23T18:53:12.498 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 600, 2023-09-23T18:53:12.498 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [ 2023-09-23T18:53:12.498 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.498 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.45188.0:161068 7.e e208/187 7:7ce8d06f:::benchmark_data_smithi098_155798_object60123:head v 208'10232, mlcod=208'10229)", 2023-09-23T18:53:12.498 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:45:47.758645+0000", 2023-09-23T18:53:12.498 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 444.72859206499999, 2023-09-23T18:53:12.498 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.30230962099999997, 2023-09-23T18:53:12.498 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:53:12.499 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:53:12.499 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:53:12.499 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.499 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:53:12.499 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.758645+0000", 2023-09-23T18:53:12.499 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.499 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.499 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.499 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:53:12.499 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.758645+0000", 2023-09-23T18:53:12.499 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.499 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.500 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.500 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:53:12.500 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.758644+0000", 2023-09-23T18:53:12.500 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:53:12.500 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.500 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.500 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:53:12.500 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.758652+0000", 2023-09-23T18:53:12.500 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.3050000000000004e-06 2023-09-23T18:53:12.500 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.500 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.500 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:53:12.500 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.758662+0000", 2023-09-23T18:53:12.501 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0438999999999999e-05 2023-09-23T18:53:12.501 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.501 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.501 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:53:12.501 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.758671+0000", 2023-09-23T18:53:12.501 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.1200000000000008e-06 2023-09-23T18:53:12.501 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.501 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.501 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.501 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.758730+0000", 2023-09-23T18:53:12.501 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.8254999999999997e-05 2023-09-23T18:53:12.501 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.502 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.502 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:53:12.502 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.758746+0000", 2023-09-23T18:53:12.502 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.6124e-05 2023-09-23T18:53:12.502 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.502 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.502 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:53:12.502 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:48.060916+0000", 2023-09-23T18:53:12.502 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.30217071699999998 2023-09-23T18:53:12.502 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.502 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.502 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:53:12.502 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:48.060955+0000", 2023-09-23T18:53:12.503 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.8682e-05 2023-09-23T18:53:12.503 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.503 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:53:12.503 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.503 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.503 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.503 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45188.0:161072 7.16 7:68b07035:::benchmark_data_smithi098_155798_object60127:head [delete] snapc 0=[] ondisk+write+known_if_redirected e208)", 2023-09-23T18:53:12.503 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:45:47.759401+0000", 2023-09-23T18:53:12.503 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 444.72783701399999, 2023-09-23T18:53:12.503 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.30165135599999998, 2023-09-23T18:53:12.503 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:53:12.503 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:53:12.504 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:53:12.504 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45188", 2023-09-23T18:53:12.504 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/2179845095", 2023-09-23T18:53:12.504 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 161072 2023-09-23T18:53:12.504 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.504 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:53:12.504 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.504 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:53:12.504 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.759401+0000", 2023-09-23T18:53:12.504 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.504 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.505 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.505 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:53:12.505 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.759401+0000", 2023-09-23T18:53:12.505 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.505 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.505 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.505 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:53:12.505 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.759399+0000", 2023-09-23T18:53:12.505 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:53:12.505 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.505 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.505 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:53:12.506 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.759404+0000", 2023-09-23T18:53:12.506 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.6380000000000003e-06 2023-09-23T18:53:12.506 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.506 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.506 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:53:12.506 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.759412+0000", 2023-09-23T18:53:12.506 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.4810000000000006e-06 2023-09-23T18:53:12.506 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.506 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.506 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:53:12.506 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.759418+0000", 2023-09-23T18:53:12.506 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.9279999999999998e-06 2023-09-23T18:53:12.507 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.507 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.507 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.507 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.766900+0000", 2023-09-23T18:53:12.507 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0074818819999999996 2023-09-23T18:53:12.507 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.507 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.507 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:53:12.507 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.769194+0000", 2023-09-23T18:53:12.507 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0022943799999999999 2023-09-23T18:53:12.507 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.507 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.507 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 1,7", 2023-09-23T18:53:12.508 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.769603+0000", 2023-09-23T18:53:12.508 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000408293 2023-09-23T18:53:12.508 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.508 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.508 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.508 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:47.776691+0000", 2023-09-23T18:53:12.508 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0070883630000000003 2023-09-23T18:53:12.508 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.508 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.508 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.508 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:48.047811+0000", 2023-09-23T18:53:12.508 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.27112022400000002 2023-09-23T18:53:12.509 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.509 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.509 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:53:12.509 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:48.060969+0000", 2023-09-23T18:53:12.509 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.013158077000000001 2023-09-23T18:53:12.509 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.509 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.509 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:53:12.509 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:48.061009+0000", 2023-09-23T18:53:12.509 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.9617999999999999e-05 2023-09-23T18:53:12.509 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.509 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.510 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:53:12.510 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:45:48.061052+0000", 2023-09-23T18:53:12.510 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.2802000000000001e-05 2023-09-23T18:53:12.510 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.510 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:53:12.510 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.510 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.510 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.510 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:144860 8.9 8:9da0a931:::benchmark_data_smithi098_176514_object41382:head [delete] snapc 0=[] ondisk+write+known_if_redirected e254)", 2023-09-23T18:53:12.510 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:51:10.054752+0000", 2023-09-23T18:53:12.510 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 122.43248584200001, 2023-09-23T18:53:12.510 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.18732516799999999, 2023-09-23T18:53:12.511 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:53:12.511 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:53:12.511 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:53:12.511 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:53:12.511 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:53:12.511 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 144860 2023-09-23T18:53:12.511 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.511 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:53:12.511 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.511 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:53:12.511 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:10.054752+0000", 2023-09-23T18:53:12.511 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.511 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.512 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.512 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:53:12.512 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:10.054752+0000", 2023-09-23T18:53:12.512 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.512 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.512 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.512 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:53:12.512 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:10.054750+0000", 2023-09-23T18:53:12.512 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T18:53:12.512 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.512 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.512 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:53:12.513 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:10.054762+0000", 2023-09-23T18:53:12.513 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.2115e-05 2023-09-23T18:53:12.513 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.513 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.513 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:53:12.513 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:10.163958+0000", 2023-09-23T18:53:12.513 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.109196326 2023-09-23T18:53:12.513 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.513 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.513 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:53:12.513 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:10.163967+0000", 2023-09-23T18:53:12.513 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.8929999999999994e-06 2023-09-23T18:53:12.514 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.514 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.514 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.514 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:10.169259+0000", 2023-09-23T18:53:12.514 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0052918610000000001 2023-09-23T18:53:12.514 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.514 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.514 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:53:12.514 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:10.169683+0000", 2023-09-23T18:53:12.514 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00042397000000000002 2023-09-23T18:53:12.514 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.514 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.515 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 3,6", 2023-09-23T18:53:12.515 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:10.171679+0000", 2023-09-23T18:53:12.515 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0019961570000000001 2023-09-23T18:53:12.515 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.515 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.515 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.515 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:10.183633+0000", 2023-09-23T18:53:12.515 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.01195359 2023-09-23T18:53:12.515 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.515 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.515 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:53:12.515 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:10.202888+0000", 2023-09-23T18:53:12.516 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.019255441000000002 2023-09-23T18:53:12.516 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.516 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.516 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.516 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:10.237888+0000", 2023-09-23T18:53:12.516 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.035000112 2023-09-23T18:53:12.516 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.516 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.516 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:53:12.516 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:10.237964+0000", 2023-09-23T18:53:12.516 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.5696000000000002e-05 2023-09-23T18:53:12.516 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.516 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.517 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:53:12.517 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:10.242077+0000", 2023-09-23T18:53:12.517 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0041130560000000004 2023-09-23T18:53:12.517 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.517 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:53:12.517 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.517 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.517 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.517 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:159691 8.c 8:3aece8e5:::benchmark_data_smithi098_176514_object56213:head [delete] snapc 0=[] ondisk+write+known_if_redirected e258)", 2023-09-23T18:53:12.517 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:51:44.329037+0000", 2023-09-23T18:53:12.517 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 88.158200090999998, 2023-09-23T18:53:12.517 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 15.469587658, 2023-09-23T18:53:12.518 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:53:12.518 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:53:12.518 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:53:12.518 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:53:12.518 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:53:12.518 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 159691 2023-09-23T18:53:12.518 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.518 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:53:12.518 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.518 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:53:12.518 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.329037+0000", 2023-09-23T18:53:12.518 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.518 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.519 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.519 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:53:12.519 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.329037+0000", 2023-09-23T18:53:12.519 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.519 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.519 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.519 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:53:12.519 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.329036+0000", 2023-09-23T18:53:12.519 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T18:53:12.519 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.519 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.519 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:53:12.519 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.329046+0000", 2023-09-23T18:53:12.519 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0434999999999999e-05 2023-09-23T18:53:12.521 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.521 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.521 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:53:12.521 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.329065+0000", 2023-09-23T18:53:12.521 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.8723999999999999e-05 2023-09-23T18:53:12.521 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.521 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.522 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:53:12.522 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.329074+0000", 2023-09-23T18:53:12.522 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.594e-06 2023-09-23T18:53:12.522 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.522 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.522 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.522 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.329141+0000", 2023-09-23T18:53:12.522 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.6613999999999996e-05 2023-09-23T18:53:12.522 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.522 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.522 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:53:12.522 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.329289+0000", 2023-09-23T18:53:12.522 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000148274 2023-09-23T18:53:12.523 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.523 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.523 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:53:12.523 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.330196+0000", 2023-09-23T18:53:12.523 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00090710599999999997 2023-09-23T18:53:12.523 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.523 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.523 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:53:12.523 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.339326+0000", 2023-09-23T18:53:12.523 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0091293759999999998 2023-09-23T18:53:12.523 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.523 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.524 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.524 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.342457+0000", 2023-09-23T18:53:12.524 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0031313420000000001 2023-09-23T18:53:12.524 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.524 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.524 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.524 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:59.794038+0000", 2023-09-23T18:53:12.524 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 15.451581349 2023-09-23T18:53:12.524 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.524 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.524 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:53:12.524 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:59.794093+0000", 2023-09-23T18:53:12.524 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.4722999999999998e-05 2023-09-23T18:53:12.525 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.525 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.525 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:53:12.525 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:59.798625+0000", 2023-09-23T18:53:12.525 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0045319339999999996 2023-09-23T18:53:12.525 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.525 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:53:12.525 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.525 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.525 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.525 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:159727 8.c 8:368486c8:::benchmark_data_smithi098_176514_object56249:head [delete] snapc 0=[] ondisk+write+known_if_redirected e258)", 2023-09-23T18:53:12.525 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:51:44.554582+0000", 2023-09-23T18:53:12.525 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 87.932655569999994, 2023-09-23T18:53:12.526 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 15.293908524000001, 2023-09-23T18:53:12.526 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:53:12.526 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:53:12.526 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:53:12.526 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:53:12.526 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:53:12.526 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 159727 2023-09-23T18:53:12.526 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.526 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:53:12.526 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.526 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:53:12.526 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.554582+0000", 2023-09-23T18:53:12.527 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.527 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.527 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.527 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:53:12.527 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.554582+0000", 2023-09-23T18:53:12.527 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.527 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.527 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.527 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:53:12.527 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.554581+0000", 2023-09-23T18:53:12.527 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:53:12.527 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.527 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.528 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:53:12.528 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.554587+0000", 2023-09-23T18:53:12.528 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.438e-06 2023-09-23T18:53:12.528 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.528 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.528 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:53:12.528 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.554600+0000", 2023-09-23T18:53:12.528 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.2707999999999999e-05 2023-09-23T18:53:12.528 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.528 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.528 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:53:12.528 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.554607+0000", 2023-09-23T18:53:12.529 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.6350000000000006e-06 2023-09-23T18:53:12.529 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.529 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.529 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.529 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.554669+0000", 2023-09-23T18:53:12.529 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.1190000000000002e-05 2023-09-23T18:53:12.529 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.529 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.529 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:53:12.529 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.554801+0000", 2023-09-23T18:53:12.529 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00013243200000000001 2023-09-23T18:53:12.529 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.529 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.530 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:53:12.530 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.554988+0000", 2023-09-23T18:53:12.530 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000186794 2023-09-23T18:53:12.530 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.530 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.530 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:53:12.530 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.559667+0000", 2023-09-23T18:53:12.530 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0046791879999999999 2023-09-23T18:53:12.530 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.530 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.530 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.530 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.559824+0000", 2023-09-23T18:53:12.531 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00015708399999999999 2023-09-23T18:53:12.531 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.531 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.531 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.531 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:59.844964+0000", 2023-09-23T18:53:12.531 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 15.28513937 2023-09-23T18:53:12.531 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.531 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.531 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:53:12.531 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:59.845027+0000", 2023-09-23T18:53:12.531 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.3177999999999997e-05 2023-09-23T18:53:12.531 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.532 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.532 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:53:12.532 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:59.848490+0000", 2023-09-23T18:53:12.532 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.003463767 2023-09-23T18:53:12.532 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.532 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:53:12.532 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.532 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.532 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.532 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:159738 8.c 8:38f807f9:::benchmark_data_smithi098_176514_object56260:head [delete] snapc 0=[] ondisk+write+known_if_redirected e258)", 2023-09-23T18:53:12.532 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:51:44.593296+0000", 2023-09-23T18:53:12.532 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 87.893941605999999, 2023-09-23T18:53:12.533 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 15.258760556, 2023-09-23T18:53:12.533 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:53:12.533 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:53:12.533 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:53:12.533 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:53:12.533 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:53:12.533 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 159738 2023-09-23T18:53:12.533 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.533 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:53:12.533 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.533 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:53:12.533 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.593296+0000", 2023-09-23T18:53:12.533 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.534 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.534 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.534 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:53:12.534 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.593296+0000", 2023-09-23T18:53:12.534 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.534 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.534 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.534 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:53:12.534 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.593294+0000", 2023-09-23T18:53:12.534 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:53:12.534 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.534 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.535 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:53:12.535 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.593301+0000", 2023-09-23T18:53:12.535 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.798e-06 2023-09-23T18:53:12.535 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.535 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.535 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:53:12.535 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.593325+0000", 2023-09-23T18:53:12.535 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.4219e-05 2023-09-23T18:53:12.535 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.535 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.535 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:53:12.535 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.593334+0000", 2023-09-23T18:53:12.536 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.0709999999999995e-06 2023-09-23T18:53:12.536 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.536 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.536 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.536 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.593394+0000", 2023-09-23T18:53:12.536 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.0630000000000001e-05 2023-09-23T18:53:12.536 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.536 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.536 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:53:12.536 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.593513+0000", 2023-09-23T18:53:12.536 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00011895700000000001 2023-09-23T18:53:12.536 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.537 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.537 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:53:12.537 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.593692+0000", 2023-09-23T18:53:12.537 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00017911700000000001 2023-09-23T18:53:12.537 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.537 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.537 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:53:12.537 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.598536+0000", 2023-09-23T18:53:12.537 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0048433649999999996 2023-09-23T18:53:12.537 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.537 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.537 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.537 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.598740+0000", 2023-09-23T18:53:12.538 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000204274 2023-09-23T18:53:12.538 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.538 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.538 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.538 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:59.848557+0000", 2023-09-23T18:53:12.538 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 15.249816690999999 2023-09-23T18:53:12.538 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.538 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.538 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:53:12.538 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:59.848599+0000", 2023-09-23T18:53:12.538 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.2153000000000001e-05 2023-09-23T18:53:12.538 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.539 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.539 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:53:12.539 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:59.852056+0000", 2023-09-23T18:53:12.539 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0034577779999999999 2023-09-23T18:53:12.539 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.539 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:53:12.539 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.539 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.539 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.539 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:159754 8.c 8:39453603:::benchmark_data_smithi098_176514_object56276:head [delete] snapc 0=[] ondisk+write+known_if_redirected e258)", 2023-09-23T18:53:12.539 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:51:44.635306+0000", 2023-09-23T18:53:12.539 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 87.851931339000004, 2023-09-23T18:53:12.540 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 15.242445987, 2023-09-23T18:53:12.540 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:53:12.540 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:53:12.540 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:53:12.540 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:53:12.540 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:53:12.540 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 159754 2023-09-23T18:53:12.540 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.540 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:53:12.540 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.540 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:53:12.540 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.635306+0000", 2023-09-23T18:53:12.540 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.541 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.541 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.541 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:53:12.541 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.635306+0000", 2023-09-23T18:53:12.541 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.541 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.541 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.541 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:53:12.541 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.635305+0000", 2023-09-23T18:53:12.541 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:53:12.541 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.541 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.541 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:53:12.542 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.635312+0000", 2023-09-23T18:53:12.542 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.4730000000000003e-06 2023-09-23T18:53:12.542 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.542 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.542 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:53:12.542 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.635329+0000", 2023-09-23T18:53:12.542 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.7095000000000001e-05 2023-09-23T18:53:12.542 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.542 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.542 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:53:12.542 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.635338+0000", 2023-09-23T18:53:12.542 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.183e-06 2023-09-23T18:53:12.542 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.543 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.543 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.543 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.635412+0000", 2023-09-23T18:53:12.543 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.3232000000000006e-05 2023-09-23T18:53:12.543 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.543 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.543 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:53:12.543 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.635570+0000", 2023-09-23T18:53:12.543 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000158157 2023-09-23T18:53:12.543 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.543 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.544 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:53:12.544 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.635807+0000", 2023-09-23T18:53:12.544 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00023666200000000001 2023-09-23T18:53:12.544 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.544 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.544 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:53:12.544 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.640536+0000", 2023-09-23T18:53:12.544 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0047296029999999998 2023-09-23T18:53:12.544 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.544 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.544 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.544 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.640935+0000", 2023-09-23T18:53:12.544 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00039884099999999999 2023-09-23T18:53:12.545 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.545 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.545 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.545 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:59.873550+0000", 2023-09-23T18:53:12.545 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 15.232615204 2023-09-23T18:53:12.545 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.545 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.545 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:53:12.545 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:59.873614+0000", 2023-09-23T18:53:12.545 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.3738999999999994e-05 2023-09-23T18:53:12.545 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.545 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.545 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:53:12.546 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:59.877752+0000", 2023-09-23T18:53:12.546 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0041382789999999999 2023-09-23T18:53:12.546 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.546 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:53:12.546 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.546 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.546 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.546 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:159768 8.c 8:3a729375:::benchmark_data_smithi098_176514_object56290:head [delete] snapc 0=[] ondisk+write+known_if_redirected e258)", 2023-09-23T18:53:12.546 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:51:44.672517+0000", 2023-09-23T18:53:12.546 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 87.814720604000001, 2023-09-23T18:53:12.546 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 15.208726524999999, 2023-09-23T18:53:12.547 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:53:12.547 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:53:12.547 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:53:12.547 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:53:12.547 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:53:12.547 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 159768 2023-09-23T18:53:12.547 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.547 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:53:12.547 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.547 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:53:12.547 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.672517+0000", 2023-09-23T18:53:12.547 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.547 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.548 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.548 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:53:12.548 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.672517+0000", 2023-09-23T18:53:12.548 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.548 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.548 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.548 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:53:12.548 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.672515+0000", 2023-09-23T18:53:12.548 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:53:12.548 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.548 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.548 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:53:12.548 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.672525+0000", 2023-09-23T18:53:12.549 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.5580000000000006e-06 2023-09-23T18:53:12.549 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.549 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.549 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:53:12.549 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.672546+0000", 2023-09-23T18:53:12.549 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.0775999999999999e-05 2023-09-23T18:53:12.549 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.549 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.549 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:53:12.549 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.672562+0000", 2023-09-23T18:53:12.549 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.6283999999999999e-05 2023-09-23T18:53:12.549 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.550 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.550 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.550 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.672628+0000", 2023-09-23T18:53:12.550 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.6198000000000006e-05 2023-09-23T18:53:12.550 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.550 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.550 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:53:12.550 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.672754+0000", 2023-09-23T18:53:12.550 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00012568900000000001 2023-09-23T18:53:12.550 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.550 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.550 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:53:12.551 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.672926+0000", 2023-09-23T18:53:12.551 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000171762 2023-09-23T18:53:12.551 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.551 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.551 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:53:12.551 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.676338+0000", 2023-09-23T18:53:12.551 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0034124519999999998 2023-09-23T18:53:12.551 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.551 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.551 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.551 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:44.676627+0000", 2023-09-23T18:53:12.551 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00028873700000000001 2023-09-23T18:53:12.551 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.552 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.552 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.552 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:59.877821+0000", 2023-09-23T18:53:12.552 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 15.201194285 2023-09-23T18:53:12.552 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.552 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.552 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:53:12.552 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:59.877883+0000", 2023-09-23T18:53:12.552 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.2095999999999996e-05 2023-09-23T18:53:12.552 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.552 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.552 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:53:12.553 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:51:59.881243+0000", 2023-09-23T18:53:12.553 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0033602620000000001 2023-09-23T18:53:12.553 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.553 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:53:12.553 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.553 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.553 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.553 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:162229 8.c 8:3ea2a2f3:::benchmark_data_smithi098_176514_object58751:head [delete] snapc 0=[] ondisk+write+known_if_redirected e258)", 2023-09-23T18:53:12.553 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:52:05.360650+0000", 2023-09-23T18:53:12.553 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 67.126587190999999, 2023-09-23T18:53:12.553 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.28854172300000003, 2023-09-23T18:53:12.553 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:53:12.554 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:53:12.554 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:53:12.554 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:53:12.554 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:53:12.554 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 162229 2023-09-23T18:53:12.554 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.554 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:53:12.554 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.554 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:53:12.554 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:05.360650+0000", 2023-09-23T18:53:12.554 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.554 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.555 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.555 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:53:12.555 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:05.360650+0000", 2023-09-23T18:53:12.555 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.555 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.555 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.555 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:53:12.555 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:05.360648+0000", 2023-09-23T18:53:12.555 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T18:53:12.555 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.555 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.555 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:53:12.556 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:05.360659+0000", 2023-09-23T18:53:12.556 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0139000000000001e-05 2023-09-23T18:53:12.556 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.556 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.556 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:53:12.556 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:05.360679+0000", 2023-09-23T18:53:12.556 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.0293999999999999e-05 2023-09-23T18:53:12.556 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.556 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.556 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:53:12.556 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:05.360690+0000", 2023-09-23T18:53:12.556 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0815000000000001e-05 2023-09-23T18:53:12.557 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.557 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.557 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.557 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:05.364645+0000", 2023-09-23T18:53:12.557 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0039551339999999999 2023-09-23T18:53:12.557 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.557 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.557 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for rw locks", 2023-09-23T18:53:12.557 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:05.365005+0000", 2023-09-23T18:53:12.557 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00036013199999999999 2023-09-23T18:53:12.557 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.557 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.558 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.558 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:05.601100+0000", 2023-09-23T18:53:12.558 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.236095473 2023-09-23T18:53:12.558 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.558 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.558 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:53:12.558 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:05.601181+0000", 2023-09-23T18:53:12.558 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.0542000000000002e-05 2023-09-23T18:53:12.558 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.558 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.558 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:53:12.558 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:05.601284+0000", 2023-09-23T18:53:12.559 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000103263 2023-09-23T18:53:12.559 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.559 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.559 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.559 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:05.620164+0000", 2023-09-23T18:53:12.559 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.018879371999999998 2023-09-23T18:53:12.559 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.559 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.559 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:53:12.559 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:05.624231+0000", 2023-09-23T18:53:12.559 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0040675520000000003 2023-09-23T18:53:12.559 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.560 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.560 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.560 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:05.645336+0000", 2023-09-23T18:53:12.560 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.021104708999999999 2023-09-23T18:53:12.560 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.560 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.560 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:53:12.560 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:05.645389+0000", 2023-09-23T18:53:12.560 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.3273999999999998e-05 2023-09-23T18:53:12.560 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.560 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.560 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:53:12.560 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:05.649192+0000", 2023-09-23T18:53:12.561 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.003802897 2023-09-23T18:53:12.561 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.561 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:53:12.561 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.561 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.561 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.561 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:162795 8.c 8:3e4f45d1:::benchmark_data_smithi098_176514_object59317:head [delete] snapc 0=[] ondisk+write+known_if_redirected e258)", 2023-09-23T18:53:12.561 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:52:06.624191+0000", 2023-09-23T18:53:12.561 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 65.863046603000001, 2023-09-23T18:53:12.561 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.16670504999999999, 2023-09-23T18:53:12.561 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:53:12.561 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:53:12.562 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:53:12.562 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:53:12.562 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:53:12.562 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 162795 2023-09-23T18:53:12.562 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.562 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:53:12.562 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.562 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:53:12.562 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:06.624191+0000", 2023-09-23T18:53:12.562 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.562 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.562 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.563 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:53:12.563 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:06.624191+0000", 2023-09-23T18:53:12.563 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.563 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.563 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.563 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:53:12.563 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:06.624190+0000", 2023-09-23T18:53:12.563 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:53:12.563 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.563 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.563 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:53:12.563 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:06.624194+0000", 2023-09-23T18:53:12.563 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.1599999999999998e-06 2023-09-23T18:53:12.564 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.564 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.564 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:53:12.564 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:06.624200+0000", 2023-09-23T18:53:12.564 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.3110000000000002e-06 2023-09-23T18:53:12.564 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.564 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.564 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:53:12.564 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:06.624205+0000", 2023-09-23T18:53:12.564 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.8999999999999997e-06 2023-09-23T18:53:12.564 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.564 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.564 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.565 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:06.624254+0000", 2023-09-23T18:53:12.565 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.8916999999999999e-05 2023-09-23T18:53:12.565 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.565 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.565 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for rw locks", 2023-09-23T18:53:12.565 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:06.624345+0000", 2023-09-23T18:53:12.565 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.1638000000000006e-05 2023-09-23T18:53:12.565 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.565 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.565 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.565 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:06.653588+0000", 2023-09-23T18:53:12.565 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.029242535 2023-09-23T18:53:12.566 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.566 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.566 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:53:12.566 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:06.653700+0000", 2023-09-23T18:53:12.566 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000112521 2023-09-23T18:53:12.566 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.566 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.566 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:53:12.566 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:06.653922+0000", 2023-09-23T18:53:12.566 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000221437 2023-09-23T18:53:12.566 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.566 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.567 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.567 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:06.780201+0000", 2023-09-23T18:53:12.567 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.126279383 2023-09-23T18:53:12.567 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.567 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.567 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.567 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:06.786667+0000", 2023-09-23T18:53:12.567 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0064661129999999999 2023-09-23T18:53:12.567 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.567 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.567 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:53:12.567 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:06.786686+0000", 2023-09-23T18:53:12.567 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.9009999999999999e-05 2023-09-23T18:53:12.568 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.568 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.568 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:53:12.568 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:06.786790+0000", 2023-09-23T18:53:12.568 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000104077 2023-09-23T18:53:12.568 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.568 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.568 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:53:12.568 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:06.790896+0000", 2023-09-23T18:53:12.568 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0041054799999999999 2023-09-23T18:53:12.568 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.568 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:53:12.569 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.569 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.569 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.569 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:177213 8.c 8:31cf0416:::benchmark_data_smithi098_176514_object73735:head [delete] snapc 0=[] ondisk+write+known_if_redirected e262)", 2023-09-23T18:53:12.569 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:52:40.001721+0000", 2023-09-23T18:53:12.569 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 32.485516560000001, 2023-09-23T18:53:12.569 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.18284790000000001, 2023-09-23T18:53:12.569 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:53:12.569 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:53:12.569 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:53:12.569 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:53:12.569 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:53:12.570 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 177213 2023-09-23T18:53:12.570 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.570 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:53:12.570 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.570 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:53:12.570 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:40.001721+0000", 2023-09-23T18:53:12.570 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.570 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.570 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.570 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:53:12.570 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:40.001721+0000", 2023-09-23T18:53:12.570 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.571 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.571 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.571 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:53:12.571 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:40.001719+0000", 2023-09-23T18:53:12.571 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:53:12.571 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.571 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.571 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:53:12.571 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:40.001727+0000", 2023-09-23T18:53:12.571 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.1470000000000004e-06 2023-09-23T18:53:12.571 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.571 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.571 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:53:12.571 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:40.001741+0000", 2023-09-23T18:53:12.572 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.4408e-05 2023-09-23T18:53:12.572 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.572 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.572 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:53:12.572 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:40.001748+0000", 2023-09-23T18:53:12.572 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.25e-06 2023-09-23T18:53:12.572 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.572 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.572 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.572 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:40.024349+0000", 2023-09-23T18:53:12.572 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.022600896999999998 2023-09-23T18:53:12.572 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.572 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.573 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for rw locks", 2023-09-23T18:53:12.573 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:40.024474+0000", 2023-09-23T18:53:12.573 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0001247 2023-09-23T18:53:12.573 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.573 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.573 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.573 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:40.123352+0000", 2023-09-23T18:53:12.573 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.098877796000000004 2023-09-23T18:53:12.573 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.573 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.573 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:53:12.574 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:40.123595+0000", 2023-09-23T18:53:12.574 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00024344300000000001 2023-09-23T18:53:12.574 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.574 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.574 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:53:12.574 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:40.128467+0000", 2023-09-23T18:53:12.574 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0048717229999999997 2023-09-23T18:53:12.574 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.574 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.574 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.574 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:40.153027+0000", 2023-09-23T18:53:12.574 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.024560595000000001 2023-09-23T18:53:12.574 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.575 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.575 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.575 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:40.153074+0000", 2023-09-23T18:53:12.575 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.6980000000000001e-05 2023-09-23T18:53:12.575 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.575 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.575 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:53:12.575 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:40.180375+0000", 2023-09-23T18:53:12.575 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.027300313 2023-09-23T18:53:12.575 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.575 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.575 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:53:12.575 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:40.180427+0000", 2023-09-23T18:53:12.576 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.1918e-05 2023-09-23T18:53:12.576 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.576 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.576 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:53:12.576 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:40.184569+0000", 2023-09-23T18:53:12.576 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0041422500000000001 2023-09-23T18:53:12.576 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.576 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:53:12.576 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.576 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.576 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.576 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:177712 8.c 8:3f365382:::benchmark_data_smithi098_176514_object74234:head [delete] snapc 0=[] ondisk+write+known_if_redirected e262)", 2023-09-23T18:53:12.576 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:52:41.214058+0000", 2023-09-23T18:53:12.577 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 31.273179764999998, 2023-09-23T18:53:12.577 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.6088531530000001, 2023-09-23T18:53:12.577 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:53:12.577 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T18:53:12.577 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:53:12.577 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:53:12.577 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:53:12.577 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 177712 2023-09-23T18:53:12.577 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.577 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:53:12.577 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.577 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:53:12.578 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.214058+0000", 2023-09-23T18:53:12.578 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.578 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.578 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.578 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:53:12.578 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.214058+0000", 2023-09-23T18:53:12.578 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.578 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.578 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.578 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:53:12.578 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.214056+0000", 2023-09-23T18:53:12.578 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:53:12.579 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.579 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.579 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:53:12.579 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.214062+0000", 2023-09-23T18:53:12.579 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.4229999999999996e-06 2023-09-23T18:53:12.579 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.579 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.579 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:53:12.579 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.214073+0000", 2023-09-23T18:53:12.579 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0750999999999999e-05 2023-09-23T18:53:12.579 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.579 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.579 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:53:12.580 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.214080+0000", 2023-09-23T18:53:12.580 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.6190000000000002e-06 2023-09-23T18:53:12.580 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.580 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.580 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.580 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.221251+0000", 2023-09-23T18:53:12.580 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0071708529999999996 2023-09-23T18:53:12.580 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.580 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.580 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:53:12.580 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.221516+0000", 2023-09-23T18:53:12.580 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00026511200000000002 2023-09-23T18:53:12.581 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.581 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.581 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:53:12.581 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.221948+0000", 2023-09-23T18:53:12.581 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00043208899999999998 2023-09-23T18:53:12.581 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.581 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.581 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.581 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.263100+0000", 2023-09-23T18:53:12.581 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.041151803000000001 2023-09-23T18:53:12.581 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.581 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.581 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:53:12.582 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.263111+0000", 2023-09-23T18:53:12.582 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0550999999999999e-05 2023-09-23T18:53:12.582 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.582 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.582 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.582 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.214962+0000", 2023-09-23T18:53:12.582 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.95185135399999998 2023-09-23T18:53:12.582 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.582 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.582 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:53:12.582 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.214975+0000", 2023-09-23T18:53:12.582 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.2741999999999999e-05 2023-09-23T18:53:12.583 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.583 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.583 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.583 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.774526+0000", 2023-09-23T18:53:12.583 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.5595514069999998 2023-09-23T18:53:12.583 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.583 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.583 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for missing object", 2023-09-23T18:53:12.583 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.774691+0000", 2023-09-23T18:53:12.583 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00016537400000000001 2023-09-23T18:53:12.583 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.583 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.584 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.584 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.822764+0000", 2023-09-23T18:53:12.584 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.048072230000000001 2023-09-23T18:53:12.584 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.584 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.584 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:53:12.584 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.822911+0000", 2023-09-23T18:53:12.584 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00014724200000000001 2023-09-23T18:53:12.584 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.584 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:53:12.584 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.584 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.585 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.585 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:177718 8.c 8:3bd7a2a7:::benchmark_data_smithi098_176514_object74240:head [delete] snapc 0=[] ondisk+write+known_if_redirected e262)", 2023-09-23T18:53:12.585 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:52:41.246632+0000", 2023-09-23T18:53:12.585 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 31.240605427999999, 2023-09-23T18:53:12.585 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.5773552309999999, 2023-09-23T18:53:12.585 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:53:12.585 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:53:12.585 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:53:12.585 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:53:12.585 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:53:12.585 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 177718 2023-09-23T18:53:12.585 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.585 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:53:12.586 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.586 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:53:12.586 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.246632+0000", 2023-09-23T18:53:12.586 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.586 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.586 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.586 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:53:12.586 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.246632+0000", 2023-09-23T18:53:12.586 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.586 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.586 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.587 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:53:12.587 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.246632+0000", 2023-09-23T18:53:12.587 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T18:53:12.587 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.587 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.587 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:53:12.587 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.246634+0000", 2023-09-23T18:53:12.587 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.785e-06 2023-09-23T18:53:12.587 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.587 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.587 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:53:12.587 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.246640+0000", 2023-09-23T18:53:12.587 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.6540000000000004e-06 2023-09-23T18:53:12.587 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.588 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.588 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:53:12.588 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.246645+0000", 2023-09-23T18:53:12.588 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.8999999999999997e-06 2023-09-23T18:53:12.588 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.588 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.588 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.588 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.263476+0000", 2023-09-23T18:53:12.588 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.016830903000000001 2023-09-23T18:53:12.588 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.588 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.588 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:53:12.589 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.263485+0000", 2023-09-23T18:53:12.589 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.9420000000000001e-06 2023-09-23T18:53:12.589 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.589 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.589 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.589 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.215003+0000", 2023-09-23T18:53:12.589 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.95151825700000003 2023-09-23T18:53:12.589 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.589 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.589 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:53:12.589 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.215010+0000", 2023-09-23T18:53:12.589 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.1930000000000004e-06 2023-09-23T18:53:12.590 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.590 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.590 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.590 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.775508+0000", 2023-09-23T18:53:12.590 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.560497998 2023-09-23T18:53:12.590 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.590 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.590 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:53:12.590 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.775636+0000", 2023-09-23T18:53:12.590 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00012799799999999999 2023-09-23T18:53:12.590 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.590 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.591 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 3,5", 2023-09-23T18:53:12.591 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.775769+0000", 2023-09-23T18:53:12.591 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0001326 2023-09-23T18:53:12.591 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.591 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.591 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:53:12.591 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.793848+0000", 2023-09-23T18:53:12.591 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.018078707999999999 2023-09-23T18:53:12.591 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.591 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.591 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.591 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.822969+0000", 2023-09-23T18:53:12.592 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.029121131000000001 2023-09-23T18:53:12.592 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.592 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.592 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.592 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.823229+0000", 2023-09-23T18:53:12.592 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00026025499999999998 2023-09-23T18:53:12.592 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.592 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.592 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:53:12.592 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.823290+0000", 2023-09-23T18:53:12.592 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.0936000000000001e-05 2023-09-23T18:53:12.592 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.592 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.593 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:53:12.593 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.823987+0000", 2023-09-23T18:53:12.593 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00069737399999999995 2023-09-23T18:53:12.593 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.593 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:53:12.593 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.593 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.593 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.593 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:177728 8.e 8:7caaa159:::benchmark_data_smithi098_176514_object74250:head [delete] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T18:53:12.593 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:52:41.272787+0000", 2023-09-23T18:53:12.593 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 31.214450495000001, 2023-09-23T18:53:12.593 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0440459440000001, 2023-09-23T18:53:12.594 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:53:12.594 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:53:12.594 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:53:12.594 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:53:12.594 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:53:12.594 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 177728 2023-09-23T18:53:12.594 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.594 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:53:12.594 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.594 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:53:12.594 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.272787+0000", 2023-09-23T18:53:12.594 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.594 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.595 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.595 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:53:12.595 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.272787+0000", 2023-09-23T18:53:12.595 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.595 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.595 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.595 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:53:12.595 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.272786+0000", 2023-09-23T18:53:12.595 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T18:53:12.595 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.595 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.595 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:53:12.596 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.272792+0000", 2023-09-23T18:53:12.596 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.2469999999999999e-06 2023-09-23T18:53:12.596 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.596 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.596 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:53:12.596 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.272808+0000", 2023-09-23T18:53:12.596 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.6781000000000001e-05 2023-09-23T18:53:12.596 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.596 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.596 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:53:12.596 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.272816+0000", 2023-09-23T18:53:12.596 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.6170000000000001e-06 2023-09-23T18:53:12.596 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.597 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.597 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.597 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.272856+0000", 2023-09-23T18:53:12.597 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.9598999999999998e-05 2023-09-23T18:53:12.597 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.597 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.597 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:53:12.597 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.272869+0000", 2023-09-23T18:53:12.597 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3390000000000001e-05 2023-09-23T18:53:12.597 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.597 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.597 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.598 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.302470+0000", 2023-09-23T18:53:12.598 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.029600654 2023-09-23T18:53:12.598 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.598 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.598 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:53:12.598 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.302615+0000", 2023-09-23T18:53:12.598 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00014523100000000001 2023-09-23T18:53:12.598 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.598 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.598 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 2", 2023-09-23T18:53:12.598 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.302752+0000", 2023-09-23T18:53:12.598 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00013697500000000001 2023-09-23T18:53:12.598 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.599 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.599 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.599 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.312348+0000", 2023-09-23T18:53:12.599 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0095965359999999993 2023-09-23T18:53:12.599 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.599 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.599 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:53:12.599 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.316685+0000", 2023-09-23T18:53:12.599 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0043369380000000003 2023-09-23T18:53:12.599 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.599 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.599 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:53:12.600 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.316745+0000", 2023-09-23T18:53:12.600 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.931e-05 2023-09-23T18:53:12.600 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.600 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.600 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:53:12.600 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.316833+0000", 2023-09-23T18:53:12.600 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.8399999999999994e-05 2023-09-23T18:53:12.600 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.600 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:53:12.600 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.600 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.600 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.601 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:177731 8.c 8:3660b231:::benchmark_data_smithi098_176514_object74253:head [delete] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T18:53:12.601 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:52:41.277257+0000", 2023-09-23T18:53:12.601 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 31.209980503000001, 2023-09-23T18:53:12.601 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.547396682, 2023-09-23T18:53:12.601 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:53:12.601 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:53:12.601 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:53:12.601 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:53:12.601 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:53:12.601 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 177731 2023-09-23T18:53:12.601 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.601 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:53:12.602 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.602 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:53:12.602 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.277257+0000", 2023-09-23T18:53:12.602 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.602 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.602 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.602 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:53:12.602 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.277257+0000", 2023-09-23T18:53:12.602 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.602 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.602 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.602 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:53:12.602 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.277255+0000", 2023-09-23T18:53:12.603 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999976 2023-09-23T18:53:12.603 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.603 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.603 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:53:12.603 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.277264+0000", 2023-09-23T18:53:12.603 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.2299999999999997e-06 2023-09-23T18:53:12.603 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.603 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.603 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:53:12.603 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.277280+0000", 2023-09-23T18:53:12.603 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.5749000000000001e-05 2023-09-23T18:53:12.603 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.604 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.604 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:53:12.604 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.277288+0000", 2023-09-23T18:53:12.604 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.5450000000000003e-06 2023-09-23T18:53:12.604 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.604 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.604 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.604 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.277342+0000", 2023-09-23T18:53:12.604 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.3202000000000003e-05 2023-09-23T18:53:12.604 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.604 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.604 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:53:12.605 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.277352+0000", 2023-09-23T18:53:12.605 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.049e-05 2023-09-23T18:53:12.605 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.605 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.605 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.605 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.215035+0000", 2023-09-23T18:53:12.605 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.93768318900000003 2023-09-23T18:53:12.605 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.605 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.605 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:53:12.605 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.215042+0000", 2023-09-23T18:53:12.605 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.063e-06 2023-09-23T18:53:12.606 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.606 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.606 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.606 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.780284+0000", 2023-09-23T18:53:12.606 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.565241935 2023-09-23T18:53:12.606 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.606 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.606 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:53:12.606 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.780452+0000", 2023-09-23T18:53:12.606 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00016806500000000001 2023-09-23T18:53:12.606 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.606 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.606 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 3,5", 2023-09-23T18:53:12.607 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.780606+0000", 2023-09-23T18:53:12.607 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00015362899999999999 2023-09-23T18:53:12.607 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.607 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.607 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:53:12.607 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.793857+0000", 2023-09-23T18:53:12.607 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.013251028 2023-09-23T18:53:12.607 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.607 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.607 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.607 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.823025+0000", 2023-09-23T18:53:12.607 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.029168058 2023-09-23T18:53:12.607 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.608 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.608 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.608 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.824049+0000", 2023-09-23T18:53:12.608 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.001023625 2023-09-23T18:53:12.608 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.608 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.608 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:53:12.608 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.824104+0000", 2023-09-23T18:53:12.608 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.5092e-05 2023-09-23T18:53:12.608 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.608 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.608 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:53:12.608 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.824654+0000", 2023-09-23T18:53:12.608 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00055001200000000001 2023-09-23T18:53:12.609 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.609 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:53:12.609 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.609 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.609 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.609 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:177733 8.c 8:38e94f97:::benchmark_data_smithi098_176514_object74255:head [delete] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T18:53:12.609 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:52:41.284941+0000", 2023-09-23T18:53:12.609 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 31.202296035, 2023-09-23T18:53:12.609 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.5402894329999999, 2023-09-23T18:53:12.609 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:53:12.609 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:53:12.609 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:53:12.609 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:53:12.610 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:53:12.610 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 177733 2023-09-23T18:53:12.610 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.610 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:53:12.610 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.610 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:53:12.610 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.284941+0000", 2023-09-23T18:53:12.610 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.610 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.610 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.610 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:53:12.610 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.284941+0000", 2023-09-23T18:53:12.610 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.611 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.611 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.611 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:53:12.611 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.284939+0000", 2023-09-23T18:53:12.611 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T18:53:12.611 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.611 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.611 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:53:12.611 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.284948+0000", 2023-09-23T18:53:12.611 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.3790000000000006e-06 2023-09-23T18:53:12.611 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.611 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.611 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:53:12.612 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.284967+0000", 2023-09-23T18:53:12.612 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.9187000000000001e-05 2023-09-23T18:53:12.612 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.612 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.612 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:53:12.612 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.284978+0000", 2023-09-23T18:53:12.612 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1316e-05 2023-09-23T18:53:12.612 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.612 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.612 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.612 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.285046+0000", 2023-09-23T18:53:12.612 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.7244999999999997e-05 2023-09-23T18:53:12.613 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.613 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.613 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:53:12.613 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.285058+0000", 2023-09-23T18:53:12.613 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.2115e-05 2023-09-23T18:53:12.613 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.613 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.613 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.613 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.215066+0000", 2023-09-23T18:53:12.613 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.93000865700000002 2023-09-23T18:53:12.613 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.613 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.614 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:53:12.614 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.215073+0000", 2023-09-23T18:53:12.614 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.5760000000000002e-06 2023-09-23T18:53:12.614 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.614 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.614 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.614 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.785093+0000", 2023-09-23T18:53:12.614 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.5700198749999998 2023-09-23T18:53:12.614 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.614 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.614 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:53:12.614 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.785271+0000", 2023-09-23T18:53:12.615 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00017769199999999999 2023-09-23T18:53:12.615 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.615 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.615 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 3,5", 2023-09-23T18:53:12.615 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.785372+0000", 2023-09-23T18:53:12.615 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000101728 2023-09-23T18:53:12.615 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.615 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.615 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:53:12.615 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.793963+0000", 2023-09-23T18:53:12.615 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0085905679999999998 2023-09-23T18:53:12.615 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.616 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.616 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.616 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.823076+0000", 2023-09-23T18:53:12.616 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0291135 2023-09-23T18:53:12.616 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.616 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.616 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.616 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.824742+0000", 2023-09-23T18:53:12.616 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0016652010000000001 2023-09-23T18:53:12.616 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.616 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.616 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:53:12.616 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.824797+0000", 2023-09-23T18:53:12.617 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.5121000000000003e-05 2023-09-23T18:53:12.617 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.617 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.617 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:53:12.617 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.825231+0000", 2023-09-23T18:53:12.617 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00043427100000000002 2023-09-23T18:53:12.617 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.617 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:53:12.617 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.617 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.617 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.617 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:177737 8.c 8:3c8a8733:::benchmark_data_smithi098_176514_object74259:head [delete] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T18:53:12.617 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:52:41.300423+0000", 2023-09-23T18:53:12.617 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 31.186814208000001, 2023-09-23T18:53:12.618 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.5396166099999999, 2023-09-23T18:53:12.618 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:53:12.618 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:53:12.618 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:53:12.618 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:53:12.618 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:53:12.618 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 177737 2023-09-23T18:53:12.618 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.618 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:53:12.618 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.618 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:53:12.618 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.300423+0000", 2023-09-23T18:53:12.618 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.619 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.619 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.619 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:53:12.619 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.300423+0000", 2023-09-23T18:53:12.619 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.619 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.619 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.619 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:53:12.619 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.300422+0000", 2023-09-23T18:53:12.619 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:53:12.619 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.619 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.619 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:53:12.620 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.300429+0000", 2023-09-23T18:53:12.620 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.0380000000000002e-06 2023-09-23T18:53:12.620 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.620 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.620 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:53:12.620 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.300442+0000", 2023-09-23T18:53:12.620 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3199e-05 2023-09-23T18:53:12.620 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.620 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.620 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:53:12.620 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.300448+0000", 2023-09-23T18:53:12.620 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.387e-06 2023-09-23T18:53:12.620 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.620 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.621 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.621 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.300500+0000", 2023-09-23T18:53:12.621 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.1603e-05 2023-09-23T18:53:12.621 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.621 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.621 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:53:12.621 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.300519+0000", 2023-09-23T18:53:12.621 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.8672999999999999e-05 2023-09-23T18:53:12.621 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.621 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.621 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.621 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.215097+0000", 2023-09-23T18:53:12.621 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.91457812199999999 2023-09-23T18:53:12.622 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.622 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.622 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:53:12.622 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.215103+0000", 2023-09-23T18:53:12.622 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.2110000000000003e-06 2023-09-23T18:53:12.622 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.622 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.622 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.622 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.788886+0000", 2023-09-23T18:53:12.622 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.5737828739999999 2023-09-23T18:53:12.622 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.623 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.623 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:53:12.623 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.789053+0000", 2023-09-23T18:53:12.623 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000167023 2023-09-23T18:53:12.623 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.623 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.623 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 3,5", 2023-09-23T18:53:12.623 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.789386+0000", 2023-09-23T18:53:12.623 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00033287699999999998 2023-09-23T18:53:12.623 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.623 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.623 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:53:12.623 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.799231+0000", 2023-09-23T18:53:12.624 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0098455330000000001 2023-09-23T18:53:12.624 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.624 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.624 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.624 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.823126+0000", 2023-09-23T18:53:12.624 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.023895085999999999 2023-09-23T18:53:12.624 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.624 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.624 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.624 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.839172+0000", 2023-09-23T18:53:12.624 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.016045954000000001 2023-09-23T18:53:12.624 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.624 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.625 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:53:12.625 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.839232+0000", 2023-09-23T18:53:12.625 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.0007000000000003e-05 2023-09-23T18:53:12.625 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.625 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.625 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:53:12.625 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.840040+0000", 2023-09-23T18:53:12.625 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00080755200000000001 2023-09-23T18:53:12.625 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.625 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:53:12.625 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.625 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.626 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.626 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:177738 8.c 8:359aa3ee:::benchmark_data_smithi098_176514_object74260:head [delete] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-09-23T18:53:12.626 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:52:41.302446+0000", 2023-09-23T18:53:12.626 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 31.184791423, 2023-09-23T18:53:12.626 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.5477939909999998, 2023-09-23T18:53:12.626 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:53:12.626 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:53:12.626 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:53:12.626 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:53:12.626 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:53:12.626 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 177738 2023-09-23T18:53:12.626 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.626 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:53:12.627 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.627 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:53:12.627 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.302446+0000", 2023-09-23T18:53:12.627 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.627 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.627 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.627 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:53:12.627 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.302446+0000", 2023-09-23T18:53:12.627 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.627 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.627 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.627 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:53:12.628 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.302445+0000", 2023-09-23T18:53:12.628 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:53:12.628 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.628 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.628 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:53:12.628 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.302451+0000", 2023-09-23T18:53:12.628 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.6889999999999998e-06 2023-09-23T18:53:12.628 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.628 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.628 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:53:12.628 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.302463+0000", 2023-09-23T18:53:12.628 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1796e-05 2023-09-23T18:53:12.628 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.629 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.629 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:53:12.629 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.302470+0000", 2023-09-23T18:53:12.629 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.2609999999999999e-06 2023-09-23T18:53:12.629 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.629 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.629 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.629 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.302526+0000", 2023-09-23T18:53:12.629 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.6369000000000002e-05 2023-09-23T18:53:12.629 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.629 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.629 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:53:12.629 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:41.302536+0000", 2023-09-23T18:53:12.630 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.7929999999999992e-06 2023-09-23T18:53:12.630 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.630 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.630 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.630 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.215298+0000", 2023-09-23T18:53:12.630 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.91276279699999996 2023-09-23T18:53:12.630 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.630 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.630 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:53:12.630 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.215308+0000", 2023-09-23T18:53:12.630 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.3780000000000007e-06 2023-09-23T18:53:12.630 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.631 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.631 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.631 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.793994+0000", 2023-09-23T18:53:12.631 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.5786865649999999 2023-09-23T18:53:12.631 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.631 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.631 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:53:12.631 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.794160+0000", 2023-09-23T18:53:12.631 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00016512499999999999 2023-09-23T18:53:12.631 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.631 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.631 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 3,5", 2023-09-23T18:53:12.632 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.794347+0000", 2023-09-23T18:53:12.632 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00018754000000000001 2023-09-23T18:53:12.632 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.632 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.632 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:53:12.632 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.799423+0000", 2023-09-23T18:53:12.632 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0050762100000000003 2023-09-23T18:53:12.632 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.632 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.632 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.632 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.823180+0000", 2023-09-23T18:53:12.632 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.023756643000000001 2023-09-23T18:53:12.632 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.633 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.633 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.633 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.849558+0000", 2023-09-23T18:53:12.633 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.026378308999999999 2023-09-23T18:53:12.633 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.633 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.633 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:53:12.633 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.849599+0000", 2023-09-23T18:53:12.633 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.0813999999999999e-05 2023-09-23T18:53:12.633 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.633 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.633 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:53:12.633 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.850240+0000", 2023-09-23T18:53:12.634 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00064103999999999995 2023-09-23T18:53:12.634 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.634 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:53:12.634 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.634 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.634 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.634 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:177771 8.c 8:3b27a463:::benchmark_data_smithi098_176514_object74293:head [delete] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-09-23T18:53:12.634 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:52:42.450345+0000", 2023-09-23T18:53:12.634 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 30.036892344999998, 2023-09-23T18:53:12.634 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.400284546, 2023-09-23T18:53:12.634 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:53:12.634 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:53:12.635 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:53:12.635 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:53:12.635 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:53:12.635 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 177771 2023-09-23T18:53:12.635 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.635 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:53:12.635 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.635 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:53:12.635 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.450345+0000", 2023-09-23T18:53:12.635 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.635 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.635 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.636 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:53:12.636 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.450345+0000", 2023-09-23T18:53:12.636 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.636 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.636 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.636 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:53:12.636 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.450344+0000", 2023-09-23T18:53:12.636 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:53:12.636 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.636 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.636 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:53:12.636 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.450352+0000", 2023-09-23T18:53:12.637 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.2640000000000008e-06 2023-09-23T18:53:12.637 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.637 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.637 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:53:12.637 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.450366+0000", 2023-09-23T18:53:12.637 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.4234e-05 2023-09-23T18:53:12.637 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.637 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.637 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:53:12.637 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.450375+0000", 2023-09-23T18:53:12.637 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.5369999999999997e-06 2023-09-23T18:53:12.638 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.638 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.638 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.638 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.450441+0000", 2023-09-23T18:53:12.638 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.6571999999999999e-05 2023-09-23T18:53:12.638 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.638 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.638 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T18:53:12.638 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:42.450457+0000", 2023-09-23T18:53:12.638 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.5849999999999999e-05 2023-09-23T18:53:12.638 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.638 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.639 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.639 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.801985+0000", 2023-09-23T18:53:12.639 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.351527393 2023-09-23T18:53:12.639 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.639 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.639 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:53:12.639 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.802172+0000", 2023-09-23T18:53:12.639 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00018719099999999999 2023-09-23T18:53:12.639 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.639 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.639 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 3,5", 2023-09-23T18:53:12.639 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.802369+0000", 2023-09-23T18:53:12.640 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000197302 2023-09-23T18:53:12.640 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.640 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.640 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:53:12.640 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.810346+0000", 2023-09-23T18:53:12.640 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0079772130000000004 2023-09-23T18:53:12.640 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.640 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.640 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.640 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.841247+0000", 2023-09-23T18:53:12.640 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.030901076999999999 2023-09-23T18:53:12.640 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.640 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.641 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.641 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.850268+0000", 2023-09-23T18:53:12.641 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0090204480000000004 2023-09-23T18:53:12.641 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.641 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.641 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:53:12.641 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.850317+0000", 2023-09-23T18:53:12.641 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.8656999999999998e-05 2023-09-23T18:53:12.641 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.641 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.641 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:53:12.641 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:52:43.850630+0000", 2023-09-23T18:53:12.641 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000313181 2023-09-23T18:53:12.641 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.642 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:53:12.642 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.642 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.642 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.642 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.35412.0:181950 8.c 8:3a5a3399:::benchmark_data_smithi098_176514_object78472:head [delete] snapc 0=[] ondisk+write+known_if_redirected e267)", 2023-09-23T18:53:12.642 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:53:09.798806+0000", 2023-09-23T18:53:12.642 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 2.6884316950000002, 2023-09-23T18:53:12.642 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.173092789, 2023-09-23T18:53:12.642 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T18:53:12.642 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:53:12.642 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T18:53:12.642 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.35412", 2023-09-23T18:53:12.642 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3176780747", 2023-09-23T18:53:12.643 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 181950 2023-09-23T18:53:12.643 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.643 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T18:53:12.643 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.643 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T18:53:12.643 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:53:09.798806+0000", 2023-09-23T18:53:12.643 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.643 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.643 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.643 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T18:53:12.643 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:53:09.798806+0000", 2023-09-23T18:53:12.643 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T18:53:12.644 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.644 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.644 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T18:53:12.644 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:53:09.798804+0000", 2023-09-23T18:53:12.644 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T18:53:12.644 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.644 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.644 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T18:53:12.644 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:53:09.798810+0000", 2023-09-23T18:53:12.644 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.5840000000000002e-06 2023-09-23T18:53:12.644 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.644 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.644 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T18:53:12.644 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:53:09.798820+0000", 2023-09-23T18:53:12.645 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0316000000000001e-05 2023-09-23T18:53:12.645 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.645 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.645 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T18:53:12.645 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:53:09.798828+0000", 2023-09-23T18:53:12.645 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.2400000000000001e-06 2023-09-23T18:53:12.645 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.645 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.645 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T18:53:12.645 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:53:09.798869+0000", 2023-09-23T18:53:12.645 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.1857999999999998e-05 2023-09-23T18:53:12.645 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.645 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.646 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T18:53:12.646 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:53:09.959805+0000", 2023-09-23T18:53:12.646 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.16093552999999999 2023-09-23T18:53:12.646 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.646 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.646 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 3,5", 2023-09-23T18:53:12.646 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:53:09.959995+0000", 2023-09-23T18:53:12.646 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00018999899999999999 2023-09-23T18:53:12.646 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.646 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.646 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.646 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:53:09.965778+0000", 2023-09-23T18:53:12.647 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0057831519999999997 2023-09-23T18:53:12.647 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.647 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.647 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T18:53:12.647 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:53:09.965961+0000", 2023-09-23T18:53:12.647 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00018320399999999999 2023-09-23T18:53:12.647 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.647 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.647 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:53:12.647 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:53:09.971340+0000", 2023-09-23T18:53:12.647 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0053786679999999996 2023-09-23T18:53:12.647 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.648 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.648 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T18:53:12.648 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:53:09.971396+0000", 2023-09-23T18:53:12.648 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.6431999999999998e-05 2023-09-23T18:53:12.648 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T18:53:12.648 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T18:53:12.648 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T18:53:12.648 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:53:09.971899+0000", 2023-09-23T18:53:12.648 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00050221399999999996 2023-09-23T18:53:12.648 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.648 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:53:12.648 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.648 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T18:53:12.649 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T18:53:12.649 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:53:12.649 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:12.650 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:12 smithi098 bash[76817]: cluster 2023-09-23T18:53:12.188977+0000 mgr.y (mgr.26054) 1056 : cluster 0 pgmap v1132: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.5 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 179 op/s; 51500/75726 objects degraded (68.008%); 5958/75726 objects misplaced (7.868%); 2.1 MiB/s, 63 objects/s recovering 2023-09-23T18:53:12.650 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:12 smithi098 bash[79516]: cluster 2023-09-23T18:53:12.188977+0000 mgr.y (mgr.26054) 1056 : cluster 0 pgmap v1132: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.5 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 179 op/s; 51500/75726 objects degraded (68.008%); 5958/75726 objects misplaced (7.868%); 2.1 MiB/s, 63 objects/s recovering 2023-09-23T18:53:12.672 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:12.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:12.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:12.776 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:12.832 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:12.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:12.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:12.936 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:12.964 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:12.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:12.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:13.066 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:13.092 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:13.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:13.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:13.196 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:13.222 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:13.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:13.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:13.289 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is set 2023-09-23T18:53:13.327 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:13.355 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:13.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:13.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:13.458 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:13.477 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:13.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:13.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:13.508 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:13 smithi098 bash[79516]: audit 2023-09-23T18:53:13.251266+0000 mon.a (mon.0) 1441 : audit 1 from='client.? 172.21.15.98:0/232256248' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:53:13.508 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:13 smithi098 bash[76817]: audit 2023-09-23T18:53:13.251266+0000 mon.a (mon.0) 1441 : audit 1 from='client.? 172.21.15.98:0/232256248' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:53:13.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:13 smithi173 bash[59422]: audit 2023-09-23T18:53:13.251266+0000 mon.a (mon.0) 1441 : audit 1 from='client.? 172.21.15.98:0/232256248' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:53:13.580 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:13.614 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:13.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:13.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:13.717 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:13.740 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:13.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:13.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:13.844 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:13.862 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:13.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:13.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:13.965 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:13.992 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:13.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:13.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:14.096 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:14.126 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:14.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:14.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:14.230 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:14.257 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:14.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:14.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:14.362 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:14.386 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:14.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:14.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:14.493 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:14.502 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:14 smithi098 bash[76817]: cluster 2023-09-23T18:53:13.261664+0000 mon.a (mon.0) 1442 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:53:14.502 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:14 smithi098 bash[76817]: cluster 2023-09-23T18:53:13.271215+0000 osd.5 (osd.5) 172 : cluster 3 failed to encode map e268 with expected crc 2023-09-23T18:53:14.502 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:14 smithi098 bash[76817]: audit 2023-09-23T18:53:13.271976+0000 mon.a (mon.0) 1443 : audit 1 from='client.? 172.21.15.98:0/232256248' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:53:14.503 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:14 smithi098 bash[76817]: cluster 2023-09-23T18:53:13.272046+0000 mon.a (mon.0) 1444 : cluster 0 osdmap e268: 8 total, 7 up, 7 in 2023-09-23T18:53:14.503 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:14 smithi098 bash[76817]: cluster 2023-09-23T18:53:13.273904+0000 osd.6 (osd.6) 228 : cluster 3 failed to encode map e268 with expected crc 2023-09-23T18:53:14.503 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:14 smithi098 bash[76817]: cluster 2023-09-23T18:53:13.286855+0000 osd.7 (osd.7) 192 : cluster 3 failed to encode map e268 with expected crc 2023-09-23T18:53:14.503 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:14 smithi098 bash[76817]: cluster 2023-09-23T18:53:14.189470+0000 mgr.y (mgr.26054) 1057 : cluster 0 pgmap v1134: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.5 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 205 op/s; 51379/74301 objects degraded (69.150%); 5838/74301 objects misplaced (7.857%); 2.3 MiB/s, 72 objects/s recovering 2023-09-23T18:53:14.503 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:14 smithi098 bash[79516]: cluster 2023-09-23T18:53:13.261664+0000 mon.a (mon.0) 1442 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:53:14.503 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:14 smithi098 bash[79516]: cluster 2023-09-23T18:53:13.271215+0000 osd.5 (osd.5) 172 : cluster 3 failed to encode map e268 with expected crc 2023-09-23T18:53:14.503 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:14 smithi098 bash[79516]: audit 2023-09-23T18:53:13.271976+0000 mon.a (mon.0) 1443 : audit 1 from='client.? 172.21.15.98:0/232256248' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:53:14.503 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:14 smithi098 bash[79516]: cluster 2023-09-23T18:53:13.272046+0000 mon.a (mon.0) 1444 : cluster 0 osdmap e268: 8 total, 7 up, 7 in 2023-09-23T18:53:14.503 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:14 smithi098 bash[79516]: cluster 2023-09-23T18:53:13.273904+0000 osd.6 (osd.6) 228 : cluster 3 failed to encode map e268 with expected crc 2023-09-23T18:53:14.503 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:14 smithi098 bash[79516]: cluster 2023-09-23T18:53:13.286855+0000 osd.7 (osd.7) 192 : cluster 3 failed to encode map e268 with expected crc 2023-09-23T18:53:14.504 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:14 smithi098 bash[79516]: cluster 2023-09-23T18:53:14.189470+0000 mgr.y (mgr.26054) 1057 : cluster 0 pgmap v1134: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.5 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 205 op/s; 51379/74301 objects degraded (69.150%); 5838/74301 objects misplaced (7.857%); 2.3 MiB/s, 72 objects/s recovering 2023-09-23T18:53:14.519 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:14.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:14.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:14.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:14 smithi173 bash[59422]: cluster 2023-09-23T18:53:13.261664+0000 mon.a (mon.0) 1442 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:53:14.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:14 smithi173 bash[59422]: cluster 2023-09-23T18:53:13.271215+0000 osd.5 (osd.5) 172 : cluster 3 failed to encode map e268 with expected crc 2023-09-23T18:53:14.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:14 smithi173 bash[59422]: audit 2023-09-23T18:53:13.271976+0000 mon.a (mon.0) 1443 : audit 1 from='client.? 172.21.15.98:0/232256248' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:53:14.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:14 smithi173 bash[59422]: cluster 2023-09-23T18:53:13.272046+0000 mon.a (mon.0) 1444 : cluster 0 osdmap e268: 8 total, 7 up, 7 in 2023-09-23T18:53:14.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:14 smithi173 bash[59422]: cluster 2023-09-23T18:53:13.273904+0000 osd.6 (osd.6) 228 : cluster 3 failed to encode map e268 with expected crc 2023-09-23T18:53:14.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:14 smithi173 bash[59422]: cluster 2023-09-23T18:53:13.286855+0000 osd.7 (osd.7) 192 : cluster 3 failed to encode map e268 with expected crc 2023-09-23T18:53:14.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:14 smithi173 bash[59422]: cluster 2023-09-23T18:53:14.189470+0000 mgr.y (mgr.26054) 1057 : cluster 0 pgmap v1134: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.5 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 205 op/s; 51379/74301 objects degraded (69.150%); 5838/74301 objects misplaced (7.857%); 2.3 MiB/s, 72 objects/s recovering 2023-09-23T18:53:14.623 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:14.659 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:14.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:14.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:14.732 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:53:14.762 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:14.783 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:14.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:14.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:14.887 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:14.901 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:53:14.925 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:14.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:14.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:14.947 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:53:14.987 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:53:15.024 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:53:15.034 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:15.066 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:53:15.066 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:15.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:15.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:15.095 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:53:15.095 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:53:15.095 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:53:15.096 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:53:15.096 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:53:15.096 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-09-23T18:53:15.096 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:53:15.096 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:53:15.096 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:53:15.168 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:15.193 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:15.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:15.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:15.296 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:15.331 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:15.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:15.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:15.434 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:15.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:15 smithi098 bash[79516]: audit 2023-09-23T18:53:14.699495+0000 mon.b (mon.1) 22 : audit 0 from='client.? 172.21.15.98:0/2909635904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:53:15.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:15 smithi098 bash[79516]: cluster 2023-09-23T18:53:15.203346+0000 mon.a (mon.0) 1445 : cluster 3 Health check update: Degraded data redundancy: 51379/74301 objects degraded (69.150%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:15.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:15 smithi098 bash[76817]: audit 2023-09-23T18:53:14.699495+0000 mon.b (mon.1) 22 : audit 0 from='client.? 172.21.15.98:0/2909635904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:53:15.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:15 smithi098 bash[76817]: cluster 2023-09-23T18:53:15.203346+0000 mon.a (mon.0) 1445 : cluster 3 Health check update: Degraded data redundancy: 51379/74301 objects degraded (69.150%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:15.464 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:15.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:15.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:15.567 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:15.579 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:15 smithi173 bash[59422]: audit 2023-09-23T18:53:14.699495+0000 mon.b (mon.1) 22 : audit 0 from='client.? 172.21.15.98:0/2909635904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:53:15.579 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:15 smithi173 bash[59422]: cluster 2023-09-23T18:53:15.203346+0000 mon.a (mon.0) 1445 : cluster 3 Health check update: Degraded data redundancy: 51379/74301 objects degraded (69.150%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:15.600 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:15.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:15.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:15.703 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:15.738 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:15.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:15.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:15.842 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:15.880 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:15.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:15.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:15.986 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:16.006 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:16.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:16.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:16.109 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:16.139 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:16.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:16.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:16.243 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:16.274 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:16.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:16.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:16.285 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:53:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:53:16] "GET /metrics HTTP/1.1" 200 36616 "" "Prometheus/2.33.4" 2023-09-23T18:53:16.377 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:16.403 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:16.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:16.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:16.511 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:16.518 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:16 smithi098 bash[76817]: cluster 2023-09-23T18:53:16.190275+0000 mgr.y (mgr.26054) 1058 : cluster 0 pgmap v1135: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.5 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 198 op/s; 50997/72933 objects degraded (69.923%); 5713/72933 objects misplaced (7.833%); 2.2 MiB/s, 72 objects/s recovering 2023-09-23T18:53:16.518 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:16 smithi098 bash[79516]: cluster 2023-09-23T18:53:16.190275+0000 mgr.y (mgr.26054) 1058 : cluster 0 pgmap v1135: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.5 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 198 op/s; 50997/72933 objects degraded (69.923%); 5713/72933 objects misplaced (7.833%); 2.2 MiB/s, 72 objects/s recovering 2023-09-23T18:53:16.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:16 smithi173 bash[59422]: cluster 2023-09-23T18:53:16.190275+0000 mgr.y (mgr.26054) 1058 : cluster 0 pgmap v1135: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.5 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 198 op/s; 50997/72933 objects degraded (69.923%); 5713/72933 objects misplaced (7.833%); 2.2 MiB/s, 72 objects/s recovering 2023-09-23T18:53:16.533 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:16.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:16.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:16.637 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:16.664 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:16.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:16.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:16.767 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:16.798 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:16.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:16.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:16.904 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:16.926 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:16.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:16.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:17.029 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:17.058 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:17.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:17.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:17.165 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:17.192 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:17.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:17.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:17.295 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:17.320 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:17.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:17.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:17.426 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:17.456 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:17.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:17.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:17.560 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:17.587 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:17.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:17.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:17.691 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:17.716 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:17.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:17.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:17.820 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:17.850 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:17.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:17.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:17.953 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:17.981 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:17.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:17.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:18.085 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:18.113 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:18.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:18.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:18.219 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:18.248 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:18.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:18.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:18.352 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:18.380 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:18.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:18.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:18.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:18 smithi098 bash[76817]: cluster 2023-09-23T18:53:18.191181+0000 mgr.y (mgr.26054) 1059 : cluster 0 pgmap v1136: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.5 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 189 op/s; 50890/71550 objects degraded (71.125%); 5595/71550 objects misplaced (7.820%); 2.2 MiB/s, 73 objects/s recovering 2023-09-23T18:53:18.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:18 smithi098 bash[79516]: cluster 2023-09-23T18:53:18.191181+0000 mgr.y (mgr.26054) 1059 : cluster 0 pgmap v1136: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.5 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 189 op/s; 50890/71550 objects degraded (71.125%); 5595/71550 objects misplaced (7.820%); 2.2 MiB/s, 73 objects/s recovering 2023-09-23T18:53:18.484 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:18.513 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:18.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:18.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:18.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:18 smithi173 bash[59422]: cluster 2023-09-23T18:53:18.191181+0000 mgr.y (mgr.26054) 1059 : cluster 0 pgmap v1136: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.5 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 189 op/s; 50890/71550 objects degraded (71.125%); 5595/71550 objects misplaced (7.820%); 2.2 MiB/s, 73 objects/s recovering 2023-09-23T18:53:18.616 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:18.640 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:18.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:18.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:18.743 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:18.772 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:18.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:18.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:18.875 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:18.902 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:18.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:18.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:19.006 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:19.030 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:19.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:19.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:19.133 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:19.164 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:19.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:19.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:19.267 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:19.293 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:19.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:19.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:19.396 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:19.424 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:19.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:19.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:19.528 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:19.562 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:19.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:19.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:19.602 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T18:53:19.668 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:19.698 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:19.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:19.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:19.801 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:19.831 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:19.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:19.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:19.934 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:19.961 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:19.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:19.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:20.065 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:20.093 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:20.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:20.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:20.196 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:20.220 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:20.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:20.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:20.328 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:20.356 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:20.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:20.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:20.459 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:20.469 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:20 smithi098 bash[79516]: cluster 2023-09-23T18:53:20.191918+0000 mgr.y (mgr.26054) 1060 : cluster 0 pgmap v1137: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.4 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 183 op/s; 50663/70497 objects degraded (71.865%); 5459/70497 objects misplaced (7.744%); 2.2 MiB/s, 74 objects/s recovering 2023-09-23T18:53:20.469 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:20 smithi098 bash[79516]: cluster 2023-09-23T18:53:20.204491+0000 mon.a (mon.0) 1446 : cluster 3 Health check update: Degraded data redundancy: 50890/71550 objects degraded (71.125%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:20.469 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:20 smithi098 bash[76817]: cluster 2023-09-23T18:53:20.191918+0000 mgr.y (mgr.26054) 1060 : cluster 0 pgmap v1137: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.4 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 183 op/s; 50663/70497 objects degraded (71.865%); 5459/70497 objects misplaced (7.744%); 2.2 MiB/s, 74 objects/s recovering 2023-09-23T18:53:20.469 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:20 smithi098 bash[76817]: cluster 2023-09-23T18:53:20.204491+0000 mon.a (mon.0) 1446 : cluster 3 Health check update: Degraded data redundancy: 50890/71550 objects degraded (71.125%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:20.490 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:20.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:20.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:20.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:20 smithi173 bash[59422]: cluster 2023-09-23T18:53:20.191918+0000 mgr.y (mgr.26054) 1060 : cluster 0 pgmap v1137: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.4 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 183 op/s; 50663/70497 objects degraded (71.865%); 5459/70497 objects misplaced (7.744%); 2.2 MiB/s, 74 objects/s recovering 2023-09-23T18:53:20.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:20 smithi173 bash[59422]: cluster 2023-09-23T18:53:20.204491+0000 mon.a (mon.0) 1446 : cluster 3 Health check update: Degraded data redundancy: 50890/71550 objects degraded (71.125%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:20.599 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:20.629 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:20.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:20.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:20.733 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:20.758 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:20.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:20.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:20.869 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:20.896 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:20.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:20.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:20.999 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:21.024 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:21.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:21.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:21.127 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:21.155 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:21.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:21.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:21.261 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:21.289 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:21.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:21.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:21.392 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:21.424 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:21.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:21.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:21.528 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:21.554 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:21.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:21.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:21.658 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:21.683 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:21.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:21.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:21.790 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:21.827 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:53:21.828 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:53:21.829 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:21.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:21.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:21.933 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:21.971 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:21.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:21.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:22.076 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:22.104 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:22.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:22.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:22.207 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:22.231 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:22.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:22.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:22.336 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:22.370 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:22.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:22.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:22.474 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:22.509 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:22.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:22.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:22.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:22 smithi173 bash[59422]: cluster 2023-09-23T18:53:22.192539+0000 mgr.y (mgr.26054) 1061 : cluster 0 pgmap v1138: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.4 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 184 op/s; 50519/70197 objects degraded (71.967%); 5459/70197 objects misplaced (7.777%); 2.2 MiB/s, 73 objects/s recovering 2023-09-23T18:53:22.614 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:22.644 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:22.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:22.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:22.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:22 smithi098 bash[76817]: cluster 2023-09-23T18:53:22.192539+0000 mgr.y (mgr.26054) 1061 : cluster 0 pgmap v1138: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.4 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 184 op/s; 50519/70197 objects degraded (71.967%); 5459/70197 objects misplaced (7.777%); 2.2 MiB/s, 73 objects/s recovering 2023-09-23T18:53:22.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:22 smithi098 bash[79516]: cluster 2023-09-23T18:53:22.192539+0000 mgr.y (mgr.26054) 1061 : cluster 0 pgmap v1138: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.4 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 184 op/s; 50519/70197 objects degraded (71.967%); 5459/70197 objects misplaced (7.777%); 2.2 MiB/s, 73 objects/s recovering 2023-09-23T18:53:22.750 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:22.777 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:22.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:22.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:22.882 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:22.917 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:22.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:22.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:23.020 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:23.044 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:23.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:23.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:23.148 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:23.175 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:23.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:23.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:23.278 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:23.310 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:23.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:23.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:23.414 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:23.445 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:23.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:23.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:23.548 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:23.575 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:23.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:23.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:23.680 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:23.706 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:23.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:23.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:23.800 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:53:23.809 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:23.829 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:53:23.887 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:23.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:23.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:23.994 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:24.022 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:24.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:24.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:24.125 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:24.155 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:24.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:24.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:24.259 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:24.293 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:24.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:24.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:24.402 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:24.410 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:24 smithi098 bash[76817]: cluster 2023-09-23T18:53:24.193262+0000 mgr.y (mgr.26054) 1062 : cluster 0 pgmap v1139: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.4 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 168 op/s; 50392/68784 objects degraded (73.261%); 5343/68784 objects misplaced (7.768%); 2.1 MiB/s, 71 objects/s recovering 2023-09-23T18:53:24.411 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:24 smithi098 bash[79516]: cluster 2023-09-23T18:53:24.193262+0000 mgr.y (mgr.26054) 1062 : cluster 0 pgmap v1139: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.4 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 168 op/s; 50392/68784 objects degraded (73.261%); 5343/68784 objects misplaced (7.768%); 2.1 MiB/s, 71 objects/s recovering 2023-09-23T18:53:24.434 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:24.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:24.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:24.498 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:53:24.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:24 smithi173 bash[59422]: cluster 2023-09-23T18:53:24.193262+0000 mgr.y (mgr.26054) 1062 : cluster 0 pgmap v1139: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.4 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 168 op/s; 50392/68784 objects degraded (73.261%); 5343/68784 objects misplaced (7.768%); 2.1 MiB/s, 71 objects/s recovering 2023-09-23T18:53:24.543 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:24.572 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:24.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:24.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:24.676 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:24.706 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:24.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:24.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:24.816 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:24.845 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:24.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:24.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:24.949 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:24.976 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:24.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:24.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:25.079 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:25.107 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:25.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:25.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:25.210 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:25.238 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:25.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:25.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:25.346 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:25.372 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:25.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:25.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:25.476 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:25.505 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:25.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:25.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:25.612 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:25.634 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:25.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:25.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:25.738 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:25.744 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:25 smithi098 bash[79516]: audit 2023-09-23T18:53:24.631156+0000 mon.a (mon.0) 1447 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:53:25.745 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:25 smithi098 bash[79516]: audit 2023-09-23T18:53:24.639454+0000 mon.a (mon.0) 1448 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:53:25.745 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:25 smithi098 bash[79516]: cluster 2023-09-23T18:53:25.205466+0000 mon.a (mon.0) 1449 : cluster 3 Health check update: Degraded data redundancy: 50392/68784 objects degraded (73.261%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:25.745 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:25 smithi098 bash[76817]: audit 2023-09-23T18:53:24.631156+0000 mon.a (mon.0) 1447 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:53:25.745 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:25 smithi098 bash[76817]: audit 2023-09-23T18:53:24.639454+0000 mon.a (mon.0) 1448 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:53:25.745 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:25 smithi098 bash[76817]: cluster 2023-09-23T18:53:25.205466+0000 mon.a (mon.0) 1449 : cluster 3 Health check update: Degraded data redundancy: 50392/68784 objects degraded (73.261%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:25.758 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:25.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:25.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:25.862 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:25.891 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:25.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:25.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:26.001 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:26.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:25 smithi173 bash[59422]: audit 2023-09-23T18:53:24.631156+0000 mon.a (mon.0) 1447 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:53:26.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:25 smithi173 bash[59422]: audit 2023-09-23T18:53:24.639454+0000 mon.a (mon.0) 1448 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:53:26.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:25 smithi173 bash[59422]: cluster 2023-09-23T18:53:25.205466+0000 mon.a (mon.0) 1449 : cluster 3 Health check update: Degraded data redundancy: 50392/68784 objects degraded (73.261%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:26.154 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:26.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:26.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:26.258 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:26.283 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:26.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:26.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:26.386 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:26.416 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:26.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:26.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:26.442 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:53:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:53:26] "GET /metrics HTTP/1.1" 200 36614 "" "Prometheus/2.33.4" 2023-09-23T18:53:26.520 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:26.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:26.550 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:26.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:26.557 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T18:53:26.566 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-23T18:53:26.652 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:26.677 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:26.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:26.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:26.781 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:26.789 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:26 smithi098 bash[76817]: cluster 2023-09-23T18:53:26.194494+0000 mgr.y (mgr.26054) 1063 : cluster 0 pgmap v1140: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.4 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 193 op/s; 49981/67320 objects degraded (74.244%); 5209/67320 objects misplaced (7.738%); 2.6 MiB/s, 75 objects/s recovering 2023-09-23T18:53:26.789 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:26 smithi098 bash[79516]: cluster 2023-09-23T18:53:26.194494+0000 mgr.y (mgr.26054) 1063 : cluster 0 pgmap v1140: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.4 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 193 op/s; 49981/67320 objects degraded (74.244%); 5209/67320 objects misplaced (7.738%); 2.6 MiB/s, 75 objects/s recovering 2023-09-23T18:53:26.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:26.815 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:26.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:26.917 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:26.926 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:26 smithi173 bash[59422]: cluster 2023-09-23T18:53:26.194494+0000 mgr.y (mgr.26054) 1063 : cluster 0 pgmap v1140: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.4 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 193 op/s; 49981/67320 objects degraded (74.244%); 5209/67320 objects misplaced (7.738%); 2.6 MiB/s, 75 objects/s recovering 2023-09-23T18:53:26.942 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:26.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:26.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:27.046 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:27.074 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:27.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:27.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:27.180 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:27.204 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:27.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:27.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:27.309 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:27.335 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:27.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:27.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:27.441 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:27.465 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:27.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:27.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:27.568 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:27.591 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:27.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:27.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:27.694 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:27.725 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:27.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:27.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:27.828 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:27.837 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:27 smithi098 bash[76817]: audit 2023-09-23T18:53:26.529751+0000 mgr.y (mgr.26054) 1064 : audit 0 from='client.45521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:53:27.837 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:27 smithi098 bash[79516]: audit 2023-09-23T18:53:26.529751+0000 mgr.y (mgr.26054) 1064 : audit 0 from='client.45521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:53:27.853 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:27.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:27.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:27.960 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:27.973 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:27 smithi173 bash[59422]: audit 2023-09-23T18:53:26.529751+0000 mgr.y (mgr.26054) 1064 : audit 0 from='client.45521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:53:27.988 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:27.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:27.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:28.094 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:28.123 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:28.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:28.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:28.226 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:28.261 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:28.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:28.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:28.370 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:28.393 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:28.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:28.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:28.499 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:28.525 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:28.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:28.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:28.628 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:28.655 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:28.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:28.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:28.759 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:28.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:28 smithi173 bash[59422]: cluster 2023-09-23T18:53:28.195378+0000 mgr.y (mgr.26054) 1065 : cluster 0 pgmap v1141: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.3 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 199 op/s; 49866/65730 objects degraded (75.865%); 5075/65730 objects misplaced (7.721%); 2.2 MiB/s, 89 objects/s recovering 2023-09-23T18:53:28.811 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:28.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:28.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:28.916 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:28.925 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:28 smithi098 bash[76817]: cluster 2023-09-23T18:53:28.195378+0000 mgr.y (mgr.26054) 1065 : cluster 0 pgmap v1141: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.3 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 199 op/s; 49866/65730 objects degraded (75.865%); 5075/65730 objects misplaced (7.721%); 2.2 MiB/s, 89 objects/s recovering 2023-09-23T18:53:28.925 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:28 smithi098 bash[79516]: cluster 2023-09-23T18:53:28.195378+0000 mgr.y (mgr.26054) 1065 : cluster 0 pgmap v1141: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.3 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 199 op/s; 49866/65730 objects degraded (75.865%); 5075/65730 objects misplaced (7.721%); 2.2 MiB/s, 89 objects/s recovering 2023-09-23T18:53:28.947 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:28.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:28.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:29.051 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:29.074 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:29.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:29.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:29.179 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:29.203 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:29.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:29.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:29.309 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:29.337 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:29.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:29.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:29.442 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:29.469 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:29.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:29.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:29.577 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:29.605 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:29.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:29.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:29.709 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:29.734 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:29.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:29.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:29.837 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:29.866 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:29.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:29.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:29.969 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:29.998 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:30.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:30.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:30.101 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:30.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:30.133 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:30.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:30.234 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:30.262 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:30.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:30.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:30.366 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:30.373 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:30 smithi098 bash[79516]: cluster 2023-09-23T18:53:30.196130+0000 mgr.y (mgr.26054) 1066 : cluster 0 pgmap v1142: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.3 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 196 op/s; 49588/64461 objects degraded (76.927%); 4924/64461 objects misplaced (7.639%); 2.3 MiB/s, 73 objects/s recovering 2023-09-23T18:53:30.373 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:30 smithi098 bash[79516]: cluster 2023-09-23T18:53:30.206544+0000 mon.a (mon.0) 1450 : cluster 3 Health check update: Degraded data redundancy: 49866/65730 objects degraded (75.865%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:30.374 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:30 smithi098 bash[76817]: cluster 2023-09-23T18:53:30.196130+0000 mgr.y (mgr.26054) 1066 : cluster 0 pgmap v1142: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.3 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 196 op/s; 49588/64461 objects degraded (76.927%); 4924/64461 objects misplaced (7.639%); 2.3 MiB/s, 73 objects/s recovering 2023-09-23T18:53:30.374 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:30 smithi098 bash[76817]: cluster 2023-09-23T18:53:30.206544+0000 mon.a (mon.0) 1450 : cluster 3 Health check update: Degraded data redundancy: 49866/65730 objects degraded (75.865%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:30.392 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:30.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:30.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:30.497 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:30.511 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:30 smithi173 bash[59422]: cluster 2023-09-23T18:53:30.196130+0000 mgr.y (mgr.26054) 1066 : cluster 0 pgmap v1142: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.3 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 196 op/s; 49588/64461 objects degraded (76.927%); 4924/64461 objects misplaced (7.639%); 2.3 MiB/s, 73 objects/s recovering 2023-09-23T18:53:30.511 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:30 smithi173 bash[59422]: cluster 2023-09-23T18:53:30.206544+0000 mon.a (mon.0) 1450 : cluster 3 Health check update: Degraded data redundancy: 49866/65730 objects degraded (75.865%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:30.512 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":1140,"stamp":"2023-09-23T18:53:26.193477+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1473517515,"num_objects":22440,"num_object_clones":0,"num_object_copies":67320,"num_objects_missing_on_primary":4539,"num_objects_missing":4539,"num_objects_degraded":49981,"num_objects_misplaced":5209,"num_objects_unfound":0,"num_objects_dirty":22440,"num_whiteouts":0,"num_read":6246,"num_read_kb":13703,"num_write":185646,"num_write_kb":6637560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9959,"num_bytes_recovered":582290362,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121588,"ondisk_log_size":121588,"up":225,"acting":222,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":9843908,"kb_used_data":4836300,"kb_used_omap":31,"kb_used_meta":5007456,"kb_avail":646372156,"statfs":{"total":671965249536,"available":661885087744,"internally_reserved":0,"allocated":4952371200,"data_stored":4945790097,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32507,"internal_metadata":5127635205},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-152502272,"num_objects":-2327,"num_object_clones":0,"num_object_copies":-6981,"num_objects_missing_on_primary":-213,"num_objects_missing":-213,"num_objects_degraded":-1398,"num_objects_misplaced":-629,"num_objects_unfound":0,"num_objects_dirty":-2327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2327,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":908,"num_bytes_recovered":32112640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004340"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":130,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.206063+0000","last_change":"2023-09-23T18:37:40.992912+0000","last_active":"2023-09-23T18:52:51.206063+0000","last_peered":"2023-09-23T18:52:51.206063+0000","last_clean":"2023-09-23T18:52:51.206063+0000","last_became_active":"2023-09-23T18:37:40.992453+0000","last_became_peered":"2023-09-23T18:37:40.992453+0000","last_unstale":"2023-09-23T18:52:51.206063+0000","last_undegraded":"2023-09-23T18:52:51.206063+0000","last_fullsized":"2023-09-23T18:52:51.206063+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:49.539251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":40,"reported_epoch":267,"state":"active+undersized","last_fresh":"2023-09-23T18:52:51.222016+0000","last_change":"2023-09-23T18:52:42.216770+0000","last_active":"2023-09-23T18:52:51.222016+0000","last_peered":"2023-09-23T18:52:51.222016+0000","last_clean":"2023-09-23T18:52:35.288500+0000","last_became_active":"2023-09-23T18:52:42.216770+0000","last_became_peered":"2023-09-23T18:52:42.216770+0000","last_unstale":"2023-09-23T18:52:51.222016+0000","last_undegraded":"2023-09-23T18:52:51.222016+0000","last_fullsized":"2023-09-23T18:52:42.209092+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:51.467498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":217,"reported_epoch":268,"state":"active+undersized","last_fresh":"2023-09-23T18:53:13.274723+0000","last_change":"2023-09-23T18:52:42.232134+0000","last_active":"2023-09-23T18:53:13.274723+0000","last_peered":"2023-09-23T18:53:13.274723+0000","last_clean":"2023-09-23T18:52:35.278178+0000","last_became_active":"2023-09-23T18:52:42.232134+0000","last_became_peered":"2023-09-23T18:52:42.232134+0000","last_unstale":"2023-09-23T18:53:13.274723+0000","last_undegraded":"2023-09-23T18:53:13.274723+0000","last_fullsized":"2023-09-23T18:52:42.207957+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:32.825733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":271,"reported_epoch":267,"state":"active+undersized","last_fresh":"2023-09-23T18:52:51.204980+0000","last_change":"2023-09-23T18:52:42.311717+0000","last_active":"2023-09-23T18:52:51.204980+0000","last_peered":"2023-09-23T18:52:51.204980+0000","last_clean":"2023-09-23T18:52:39.326812+0000","last_became_active":"2023-09-23T18:52:42.311717+0000","last_became_peered":"2023-09-23T18:52:42.311717+0000","last_unstale":"2023-09-23T18:52:51.204980+0000","last_undegraded":"2023-09-23T18:52:51.204980+0000","last_fullsized":"2023-09-23T18:52:42.211780+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:55.150724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":34,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.221313+0000","last_change":"2023-09-23T18:49:32.171708+0000","last_active":"2023-09-23T18:52:51.221313+0000","last_peered":"2023-09-23T18:52:51.221313+0000","last_clean":"2023-09-23T18:52:51.221313+0000","last_became_active":"2023-09-23T18:49:32.170937+0000","last_became_peered":"2023-09-23T18:49:32.170937+0000","last_unstale":"2023-09-23T18:52:51.221313+0000","last_undegraded":"2023-09-23T18:52:51.221313+0000","last_fullsized":"2023-09-23T18:52:51.221313+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:33:12.132319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":246,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.208071+0000","last_change":"2023-09-23T18:51:43.095478+0000","last_active":"2023-09-23T18:52:51.208071+0000","last_peered":"2023-09-23T18:52:51.208071+0000","last_clean":"2023-09-23T18:52:51.208071+0000","last_became_active":"2023-09-23T18:51:43.095247+0000","last_became_peered":"2023-09-23T18:51:43.095247+0000","last_unstale":"2023-09-23T18:52:51.208071+0000","last_undegraded":"2023-09-23T18:52:51.208071+0000","last_fullsized":"2023-09-23T18:52:51.208071+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:24.911633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":269,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.221456+0000","last_change":"2023-09-23T18:52:12.601437+0000","last_active":"2023-09-23T18:52:51.221456+0000","last_peered":"2023-09-23T18:52:51.221456+0000","last_clean":"2023-09-23T18:52:51.221456+0000","last_became_active":"2023-09-23T18:52:12.600910+0000","last_became_peered":"2023-09-23T18:52:12.600910+0000","last_unstale":"2023-09-23T18:52:51.221456+0000","last_undegraded":"2023-09-23T18:52:51.221456+0000","last_fullsized":"2023-09-23T18:52:51.221456+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:06.985657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":269,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.223766+0000","last_change":"2023-09-23T18:37:40.998526+0000","last_active":"2023-09-23T18:52:51.223766+0000","last_peered":"2023-09-23T18:52:51.223766+0000","last_clean":"2023-09-23T18:52:51.223766+0000","last_became_active":"2023-09-23T18:37:40.993211+0000","last_became_peered":"2023-09-23T18:37:40.993211+0000","last_unstale":"2023-09-23T18:52:51.223766+0000","last_undegraded":"2023-09-23T18:52:51.223766+0000","last_fullsized":"2023-09-23T18:52:51.223766+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:40.463496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":258,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.234614+0000","last_change":"2023-09-23T18:49:32.183841+0000","last_active":"2023-09-23T18:52:51.234614+0000","last_peered":"2023-09-23T18:52:51.234614+0000","last_clean":"2023-09-23T18:52:51.234614+0000","last_became_active":"2023-09-23T18:49:32.183073+0000","last_became_peered":"2023-09-23T18:49:32.183073+0000","last_unstale":"2023-09-23T18:52:51.234614+0000","last_undegraded":"2023-09-23T18:52:51.234614+0000","last_fullsized":"2023-09-23T18:52:51.234614+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":235,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.223432+0000","last_change":"2023-09-23T18:28:03.298214+0000","last_active":"2023-09-23T18:52:51.223432+0000","last_peered":"2023-09-23T18:52:51.223432+0000","last_clean":"2023-09-23T18:52:51.223432+0000","last_became_active":"2023-09-23T18:28:03.297852+0000","last_became_peered":"2023-09-23T18:28:03.297852+0000","last_unstale":"2023-09-23T18:52:51.223432+0000","last_undegraded":"2023-09-23T18:52:51.223432+0000","last_fullsized":"2023-09-23T18:52:51.223432+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.146541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":243,"reported_epoch":267,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:52:51.234565+0000","last_change":"2023-09-23T18:52:42.279649+0000","last_active":"2023-09-23T18:52:51.234565+0000","last_peered":"2023-09-23T18:52:51.234565+0000","last_clean":"2023-09-23T18:52:35.300253+0000","last_became_active":"2023-09-23T18:52:42.279649+0000","last_became_peered":"2023-09-23T18:52:42.279649+0000","last_unstale":"2023-09-23T18:52:51.234565+0000","last_undegraded":"2023-09-23T18:52:42.224974+0000","last_fullsized":"2023-09-23T18:52:42.224780+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":207,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.212083+0000","last_change":"2023-09-23T18:33:53.850096+0000","last_active":"2023-09-23T18:52:51.212083+0000","last_peered":"2023-09-23T18:52:51.212083+0000","last_clean":"2023-09-23T18:52:51.212083+0000","last_became_active":"2023-09-23T18:33:53.849811+0000","last_became_peered":"2023-09-23T18:33:53.849811+0000","last_unstale":"2023-09-23T18:52:51.212083+0000","last_undegraded":"2023-09-23T18:52:51.212083+0000","last_fullsized":"2023-09-23T18:52:51.212083+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:44.938072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":69,"reported_epoch":267,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:52:51.234236+0000","last_change":"2023-09-23T18:52:42.228565+0000","last_active":"2023-09-23T18:52:51.234236+0000","last_peered":"2023-09-23T18:52:51.234236+0000","last_clean":"2023-09-23T18:23:09.660586+0000","last_became_active":"2023-09-23T18:52:42.228565+0000","last_became_peered":"2023-09-23T18:52:42.228565+0000","last_unstale":"2023-09-23T18:52:51.234236+0000","last_undegraded":"2023-09-23T18:52:42.223179+0000","last_fullsized":"2023-09-23T18:52:42.223046+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":4}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":108,"reported_epoch":267,"state":"active+undersized","last_fresh":"2023-09-23T18:52:51.224252+0000","last_change":"2023-09-23T18:52:42.224216+0000","last_active":"2023-09-23T18:52:51.224252+0000","last_peered":"2023-09-23T18:52:51.224252+0000","last_clean":"2023-09-23T18:28:03.293385+0000","last_became_active":"2023-09-23T18:52:42.224216+0000","last_became_peered":"2023-09-23T18:52:42.224216+0000","last_unstale":"2023-09-23T18:52:51.224252+0000","last_undegraded":"2023-09-23T18:52:51.224252+0000","last_fullsized":"2023-09-23T18:52:42.212901+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:34:48.203769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":242,"reported_epoch":267,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:52:51.234179+0000","last_change":"2023-09-23T18:52:42.230355+0000","last_active":"2023-09-23T18:52:51.234179+0000","last_peered":"2023-09-23T18:52:51.234179+0000","last_clean":"2023-09-23T18:52:35.293912+0000","last_became_active":"2023-09-23T18:52:42.230355+0000","last_became_peered":"2023-09-23T18:52:42.230355+0000","last_unstale":"2023-09-23T18:52:51.234179+0000","last_undegraded":"2023-09-23T18:52:42.222732+0000","last_fullsized":"2023-09-23T18:52:42.222576+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":260,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.222625+0000","last_change":"2023-09-23T18:33:53.852933+0000","last_active":"2023-09-23T18:52:51.222625+0000","last_peered":"2023-09-23T18:52:51.222625+0000","last_clean":"2023-09-23T18:52:51.222625+0000","last_became_active":"2023-09-23T18:33:53.852626+0000","last_became_peered":"2023-09-23T18:33:53.852626+0000","last_unstale":"2023-09-23T18:52:51.222625+0000","last_undegraded":"2023-09-23T18:52:51.222625+0000","last_fullsized":"2023-09-23T18:52:51.222625+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:00.174536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":133,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.233838+0000","last_change":"2023-09-23T18:52:12.628462+0000","last_active":"2023-09-23T18:52:51.233838+0000","last_peered":"2023-09-23T18:52:51.233838+0000","last_clean":"2023-09-23T18:52:51.233838+0000","last_became_active":"2023-09-23T18:52:12.628129+0000","last_became_peered":"2023-09-23T18:52:12.628129+0000","last_unstale":"2023-09-23T18:52:51.233838+0000","last_undegraded":"2023-09-23T18:52:51.233838+0000","last_fullsized":"2023-09-23T18:52:51.233838+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":217,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.234894+0000","last_change":"2023-09-23T18:49:32.172245+0000","last_active":"2023-09-23T18:52:51.234894+0000","last_peered":"2023-09-23T18:52:51.234894+0000","last_clean":"2023-09-23T18:52:51.234894+0000","last_became_active":"2023-09-23T18:49:32.171718+0000","last_became_peered":"2023-09-23T18:49:32.171718+0000","last_unstale":"2023-09-23T18:52:51.234894+0000","last_undegraded":"2023-09-23T18:52:51.234894+0000","last_fullsized":"2023-09-23T18:52:51.234894+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":225,"reported_epoch":267,"state":"active+undersized","last_fresh":"2023-09-23T18:52:51.233809+0000","last_change":"2023-09-23T18:52:42.228737+0000","last_active":"2023-09-23T18:52:51.233809+0000","last_peered":"2023-09-23T18:52:51.233809+0000","last_clean":"2023-09-23T18:52:35.293640+0000","last_became_active":"2023-09-23T18:52:42.228737+0000","last_became_peered":"2023-09-23T18:52:42.228737+0000","last_unstale":"2023-09-23T18:52:51.233809+0000","last_undegraded":"2023-09-23T18:52:51.233809+0000","last_fullsized":"2023-09-23T18:52:42.222819+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":144,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.222412+0000","last_change":"2023-09-23T18:52:12.600644+0000","last_active":"2023-09-23T18:52:51.222412+0000","last_peered":"2023-09-23T18:52:51.222412+0000","last_clean":"2023-09-23T18:52:51.222412+0000","last_became_active":"2023-09-23T18:52:12.600263+0000","last_became_peered":"2023-09-23T18:52:12.600263+0000","last_unstale":"2023-09-23T18:52:51.222412+0000","last_undegraded":"2023-09-23T18:52:51.222412+0000","last_fullsized":"2023-09-23T18:52:51.222412+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:33:25.181447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":75,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.204362+0000","last_change":"2023-09-23T18:49:32.185489+0000","last_active":"2023-09-23T18:52:51.204362+0000","last_peered":"2023-09-23T18:52:51.204362+0000","last_clean":"2023-09-23T18:52:51.204362+0000","last_became_active":"2023-09-23T18:49:32.165294+0000","last_became_peered":"2023-09-23T18:49:32.165294+0000","last_unstale":"2023-09-23T18:52:51.204362+0000","last_undegraded":"2023-09-23T18:52:51.204362+0000","last_fullsized":"2023-09-23T18:52:51.204362+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.340334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":215,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.208221+0000","last_change":"2023-09-23T18:40:42.058656+0000","last_active":"2023-09-23T18:52:51.208221+0000","last_peered":"2023-09-23T18:52:51.208221+0000","last_clean":"2023-09-23T18:52:51.208221+0000","last_became_active":"2023-09-23T18:40:42.057307+0000","last_became_peered":"2023-09-23T18:40:42.057307+0000","last_unstale":"2023-09-23T18:52:51.208221+0000","last_undegraded":"2023-09-23T18:52:51.208221+0000","last_fullsized":"2023-09-23T18:52:51.208221+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:08:45.979078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.4","version":"268'11631","reported_seq":14984,"reported_epoch":268,"state":"active+recovering+degraded+remapped","last_fresh":"2023-09-23T18:53:24.762656+0000","last_change":"2023-09-23T18:52:50.087048+0000","last_active":"2023-09-23T18:53:24.762656+0000","last_peered":"2023-09-23T18:53:24.762656+0000","last_clean":"2023-09-23T18:49:31.150310+0000","last_became_active":"2023-09-23T18:52:49.517633+0000","last_became_peered":"2023-09-23T18:52:49.517633+0000","last_unstale":"2023-09-23T18:53:24.762656+0000","last_undegraded":"2023-09-23T18:52:49.467132+0000","last_fullsized":"2023-09-23T18:53:24.762656+0000","mapping_epoch":264,"log_start":"242'4120","ondisk_log_start":"242'4120","created":233,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_clean_scrub_stamp":"2023-09-23T18:47:48.124451+0000","objects_scrubbed":0,"log_size":7511,"log_dups_size":0,"ondisk_log_size":7511,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:04.103036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88801280,"num_objects":1355,"num_object_clones":0,"num_object_copies":4065,"num_objects_missing_on_primary":4539,"num_objects_missing":4539,"num_objects_degraded":1355,"num_objects_misplaced":1343,"num_objects_unfound":0,"num_objects_dirty":1355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11631,"num_write_kb":415552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1637,"num_bytes_recovered":107151360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[{"shards":"2,5","objects":306},{"shards":"2,5,6","objects":12},{"shards":"5,6","objects":1037}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":248,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.234676+0000","last_change":"2023-09-23T18:37:41.034732+0000","last_active":"2023-09-23T18:52:51.234676+0000","last_peered":"2023-09-23T18:52:51.234676+0000","last_clean":"2023-09-23T18:52:51.234676+0000","last_became_active":"2023-09-23T18:37:40.985865+0000","last_became_peered":"2023-09-23T18:37:40.985865+0000","last_unstale":"2023-09-23T18:52:51.234676+0000","last_undegraded":"2023-09-23T18:52:51.234676+0000","last_fullsized":"2023-09-23T18:52:51.234676+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":164,"reported_epoch":267,"state":"active+undersized","last_fresh":"2023-09-23T18:52:51.233957+0000","last_change":"2023-09-23T18:52:42.228583+0000","last_active":"2023-09-23T18:52:51.233957+0000","last_peered":"2023-09-23T18:52:51.233957+0000","last_clean":"2023-09-23T18:52:35.293491+0000","last_became_active":"2023-09-23T18:52:42.228583+0000","last_became_peered":"2023-09-23T18:52:42.228583+0000","last_unstale":"2023-09-23T18:52:51.233957+0000","last_undegraded":"2023-09-23T18:52:51.233957+0000","last_fullsized":"2023-09-23T18:52:42.223129+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"268'11404","reported_seq":11452,"reported_epoch":268,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:53:25.378434+0000","last_change":"2023-09-23T18:52:42.228742+0000","last_active":"2023-09-23T18:53:25.378434+0000","last_peered":"2023-09-23T18:53:25.378434+0000","last_clean":"2023-09-23T18:52:41.199666+0000","last_became_active":"2023-09-23T18:52:42.228742+0000","last_became_peered":"2023-09-23T18:52:42.228742+0000","last_unstale":"2023-09-23T18:53:25.378434+0000","last_undegraded":"2023-09-23T18:52:42.223060+0000","last_fullsized":"2023-09-23T18:52:42.222936+0000","mapping_epoch":263,"log_start":"242'4002","ondisk_log_start":"242'4002","created":233,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_clean_scrub_stamp":"2023-09-23T18:47:48.124451+0000","objects_scrubbed":0,"log_size":7402,"log_dups_size":0,"ondisk_log_size":7402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90177536,"num_objects":1376,"num_object_clones":0,"num_object_copies":4128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1376,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1376,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11404,"num_write_kb":408960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":1376}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":257,"reported_epoch":267,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:52:51.234412+0000","last_change":"2023-09-23T18:52:42.232449+0000","last_active":"2023-09-23T18:52:51.234412+0000","last_peered":"2023-09-23T18:52:51.234412+0000","last_clean":"2023-09-23T18:52:35.294276+0000","last_became_active":"2023-09-23T18:52:42.232449+0000","last_became_peered":"2023-09-23T18:52:42.232449+0000","last_unstale":"2023-09-23T18:52:51.234412+0000","last_undegraded":"2023-09-23T18:52:42.224469+0000","last_fullsized":"2023-09-23T18:52:42.224186+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[{"shards":"1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":170,"reported_epoch":267,"state":"active+undersized","last_fresh":"2023-09-23T18:52:51.206614+0000","last_change":"2023-09-23T18:52:42.226514+0000","last_active":"2023-09-23T18:52:51.206614+0000","last_peered":"2023-09-23T18:52:51.206614+0000","last_clean":"2023-09-23T18:39:19.103247+0000","last_became_active":"2023-09-23T18:52:42.226514+0000","last_became_peered":"2023-09-23T18:52:42.226514+0000","last_unstale":"2023-09-23T18:52:51.206614+0000","last_undegraded":"2023-09-23T18:52:51.206614+0000","last_fullsized":"2023-09-23T18:52:42.208357+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:55.909289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"268'11751","reported_seq":11797,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:53:24.267857+0000","last_change":"2023-09-23T18:47:49.621608+0000","last_active":"2023-09-23T18:53:24.267857+0000","last_peered":"2023-09-23T18:53:24.267857+0000","last_clean":"2023-09-23T18:53:24.267857+0000","last_became_active":"2023-09-23T18:47:49.620658+0000","last_became_peered":"2023-09-23T18:47:49.620658+0000","last_unstale":"2023-09-23T18:53:24.267857+0000","last_undegraded":"2023-09-23T18:53:24.267857+0000","last_fullsized":"2023-09-23T18:53:24.267857+0000","mapping_epoch":233,"log_start":"242'4050","ondisk_log_start":"242'4050","created":233,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_clean_scrub_stamp":"2023-09-23T18:47:48.124451+0000","objects_scrubbed":0,"log_size":7701,"log_dups_size":0,"ondisk_log_size":7701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:24.043184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93388800,"num_objects":1425,"num_object_clones":0,"num_object_copies":4275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11751,"num_write_kb":421632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":259,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.234200+0000","last_change":"2023-09-23T18:49:32.230776+0000","last_active":"2023-09-23T18:52:51.234200+0000","last_peered":"2023-09-23T18:52:51.234200+0000","last_clean":"2023-09-23T18:52:51.234200+0000","last_became_active":"2023-09-23T18:49:32.185564+0000","last_became_peered":"2023-09-23T18:49:32.185564+0000","last_unstale":"2023-09-23T18:52:51.234200+0000","last_undegraded":"2023-09-23T18:52:51.234200+0000","last_fullsized":"2023-09-23T18:52:51.234200+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":110,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.209363+0000","last_change":"2023-09-23T18:48:02.281253+0000","last_active":"2023-09-23T18:52:51.209363+0000","last_peered":"2023-09-23T18:52:51.209363+0000","last_clean":"2023-09-23T18:52:51.209363+0000","last_became_active":"2023-09-23T18:48:02.280890+0000","last_became_peered":"2023-09-23T18:48:02.280890+0000","last_unstale":"2023-09-23T18:52:51.209363+0000","last_undegraded":"2023-09-23T18:52:51.209363+0000","last_fullsized":"2023-09-23T18:52:51.209363+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:17.182894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.7","version":"268'11365","reported_seq":11411,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:53:22.396101+0000","last_change":"2023-09-23T18:47:49.157315+0000","last_active":"2023-09-23T18:53:22.396101+0000","last_peered":"2023-09-23T18:53:22.396101+0000","last_clean":"2023-09-23T18:53:22.396101+0000","last_became_active":"2023-09-23T18:47:49.156977+0000","last_became_peered":"2023-09-23T18:47:49.156977+0000","last_unstale":"2023-09-23T18:53:22.396101+0000","last_undegraded":"2023-09-23T18:53:22.396101+0000","last_fullsized":"2023-09-23T18:53:22.396101+0000","mapping_epoch":233,"log_start":"243'4975","ondisk_log_start":"243'4975","created":233,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_clean_scrub_stamp":"2023-09-23T18:47:48.124451+0000","objects_scrubbed":0,"log_size":6390,"log_dups_size":0,"ondisk_log_size":6390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:32.587999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93126656,"num_objects":1421,"num_object_clones":0,"num_object_copies":4263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11365,"num_write_kb":409152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":224,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.222847+0000","last_change":"2023-09-23T18:51:43.090742+0000","last_active":"2023-09-23T18:52:51.222847+0000","last_peered":"2023-09-23T18:52:51.222847+0000","last_clean":"2023-09-23T18:52:51.222847+0000","last_became_active":"2023-09-23T18:51:43.090409+0000","last_became_peered":"2023-09-23T18:51:43.090409+0000","last_unstale":"2023-09-23T18:52:51.222847+0000","last_undegraded":"2023-09-23T18:52:51.222847+0000","last_fullsized":"2023-09-23T18:52:51.222847+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:25.046714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":245,"reported_epoch":267,"state":"active+undersized","last_fresh":"2023-09-23T18:52:51.204282+0000","last_change":"2023-09-23T18:52:42.218903+0000","last_active":"2023-09-23T18:52:51.204282+0000","last_peered":"2023-09-23T18:52:51.204282+0000","last_clean":"2023-09-23T18:52:35.313355+0000","last_became_active":"2023-09-23T18:52:42.218903+0000","last_became_peered":"2023-09-23T18:52:42.218903+0000","last_unstale":"2023-09-23T18:52:51.204282+0000","last_undegraded":"2023-09-23T18:52:51.204282+0000","last_fullsized":"2023-09-23T18:52:42.212092+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:27:07.445633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"268'11540","reported_seq":16598,"reported_epoch":268,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T18:53:22.784706+0000","last_change":"2023-09-23T18:52:43.749315+0000","last_active":"2023-09-23T18:53:22.784706+0000","last_peered":"2023-09-23T18:53:22.784706+0000","last_clean":"2023-09-23T18:49:31.126813+0000","last_became_active":"2023-09-23T18:52:43.701300+0000","last_became_peered":"2023-09-23T18:52:43.701300+0000","last_unstale":"2023-09-23T18:53:22.784706+0000","last_undegraded":"2023-09-23T18:52:43.241011+0000","last_fullsized":"2023-09-23T18:53:22.784706+0000","mapping_epoch":264,"log_start":"239'2900","ondisk_log_start":"239'2900","created":233,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_clean_scrub_stamp":"2023-09-23T18:47:48.124451+0000","objects_scrubbed":0,"log_size":8640,"log_dups_size":0,"ondisk_log_size":8640,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:49.044224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91881472,"num_objects":1402,"num_object_clones":0,"num_object_copies":4206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1402,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11540,"num_write_kb":414144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2497,"num_bytes_recovered":163643392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1,5],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":327},{"shards":"1,3,5","objects":1075}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":251,"reported_epoch":267,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:52:51.221656+0000","last_change":"2023-09-23T18:52:42.228011+0000","last_active":"2023-09-23T18:52:51.221656+0000","last_peered":"2023-09-23T18:52:51.221656+0000","last_clean":"2023-09-23T18:52:35.288267+0000","last_became_active":"2023-09-23T18:52:42.228011+0000","last_became_peered":"2023-09-23T18:52:42.228011+0000","last_unstale":"2023-09-23T18:52:51.221656+0000","last_undegraded":"2023-09-23T18:52:42.216361+0000","last_fullsized":"2023-09-23T18:52:42.216241+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:46.353701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":73,"reported_epoch":267,"state":"active+undersized","last_fresh":"2023-09-23T18:52:51.222719+0000","last_change":"2023-09-23T18:52:42.241857+0000","last_active":"2023-09-23T18:52:51.222719+0000","last_peered":"2023-09-23T18:52:51.222719+0000","last_clean":"2023-09-23T18:27:33.584159+0000","last_became_active":"2023-09-23T18:52:42.241857+0000","last_became_peered":"2023-09-23T18:52:42.241857+0000","last_unstale":"2023-09-23T18:52:51.222719+0000","last_undegraded":"2023-09-23T18:52:51.222719+0000","last_fullsized":"2023-09-23T18:52:42.223096+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:47.301648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"268'11951","reported_seq":12002,"reported_epoch":268,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:53:22.426230+0000","last_change":"2023-09-23T18:49:33.764253+0000","last_active":"2023-09-23T18:53:22.426230+0000","last_peered":"2023-09-23T18:53:22.426230+0000","last_clean":"2023-09-23T18:49:31.133494+0000","last_became_active":"2023-09-23T18:49:33.743548+0000","last_became_peered":"2023-09-23T18:49:33.743548+0000","last_unstale":"2023-09-23T18:53:22.426230+0000","last_undegraded":"2023-09-23T18:49:33.224651+0000","last_fullsized":"2023-09-23T18:49:33.170193+0000","mapping_epoch":246,"log_start":"244'5486","ondisk_log_start":"244'5486","created":233,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_clean_scrub_stamp":"2023-09-23T18:47:48.124451+0000","objects_scrubbed":0,"log_size":6465,"log_dups_size":0,"ondisk_log_size":6465,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:37.695780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95485952,"num_objects":1457,"num_object_clones":0,"num_object_copies":4371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6396,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11951,"num_write_kb":429056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0],"avail_no_missing":["1","0","5"],"object_location_counts":[{"shards":"0,1","objects":-4939},{"shards":"0,1,5","objects":1149}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":112,"reported_epoch":267,"state":"active+undersized","last_fresh":"2023-09-23T18:52:51.212295+0000","last_change":"2023-09-23T18:52:42.257473+0000","last_active":"2023-09-23T18:52:51.212295+0000","last_peered":"2023-09-23T18:52:51.212295+0000","last_clean":"2023-09-23T18:28:03.294831+0000","last_became_active":"2023-09-23T18:52:42.257473+0000","last_became_peered":"2023-09-23T18:52:42.257473+0000","last_unstale":"2023-09-23T18:52:51.212295+0000","last_undegraded":"2023-09-23T18:52:51.212295+0000","last_fullsized":"2023-09-23T18:52:42.216838+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:43.866696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":263,"reported_epoch":267,"state":"active+undersized","last_fresh":"2023-09-23T18:52:51.209409+0000","last_change":"2023-09-23T18:52:42.215546+0000","last_active":"2023-09-23T18:52:51.209409+0000","last_peered":"2023-09-23T18:52:51.209409+0000","last_clean":"2023-09-23T18:49:11.184769+0000","last_became_active":"2023-09-23T18:52:42.215546+0000","last_became_peered":"2023-09-23T18:52:42.215546+0000","last_unstale":"2023-09-23T18:52:51.209409+0000","last_undegraded":"2023-09-23T18:52:51.209409+0000","last_fullsized":"2023-09-23T18:52:42.212441+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:15.862591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.2","version":"268'11495","reported_seq":12284,"reported_epoch":268,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:53:24.328708+0000","last_change":"2023-09-23T18:52:50.048880+0000","last_active":"2023-09-23T18:53:24.328708+0000","last_peered":"2023-09-23T18:53:24.328708+0000","last_clean":"2023-09-23T18:51:42.049077+0000","last_became_active":"2023-09-23T18:52:42.229036+0000","last_became_peered":"2023-09-23T18:52:42.229036+0000","last_unstale":"2023-09-23T18:53:24.328708+0000","last_undegraded":"2023-09-23T18:52:42.211646+0000","last_fullsized":"2023-09-23T18:52:42.211460+0000","mapping_epoch":263,"log_start":"242'4063","ondisk_log_start":"242'4063","created":233,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_clean_scrub_stamp":"2023-09-23T18:47:48.124451+0000","objects_scrubbed":0,"log_size":7432,"log_dups_size":0,"ondisk_log_size":7432,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:18.412723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91422720,"num_objects":1395,"num_object_clones":0,"num_object_copies":4185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1236,"num_objects_misplaced":1395,"num_objects_unfound":0,"num_objects_dirty":1395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11495,"num_write_kb":412480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":362,"num_bytes_recovered":23724032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":1395}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":70,"reported_epoch":267,"state":"active+undersized","last_fresh":"2023-09-23T18:52:51.209967+0000","last_change":"2023-09-23T18:52:42.226857+0000","last_active":"2023-09-23T18:52:51.209967+0000","last_peered":"2023-09-23T18:52:51.209967+0000","last_clean":"2023-09-23T18:27:04.247054+0000","last_became_active":"2023-09-23T18:52:42.226857+0000","last_became_peered":"2023-09-23T18:52:42.226857+0000","last_unstale":"2023-09-23T18:52:51.209967+0000","last_undegraded":"2023-09-23T18:52:51.209967+0000","last_fullsized":"2023-09-23T18:52:42.213127+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:16:34.775129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":108,"reported_epoch":267,"state":"active+undersized","last_fresh":"2023-09-23T18:52:51.207318+0000","last_change":"2023-09-23T18:52:42.226205+0000","last_active":"2023-09-23T18:52:51.207318+0000","last_peered":"2023-09-23T18:52:51.207318+0000","last_clean":"2023-09-23T18:28:03.292763+0000","last_became_active":"2023-09-23T18:52:42.226205+0000","last_became_peered":"2023-09-23T18:52:42.226205+0000","last_unstale":"2023-09-23T18:52:51.207318+0000","last_undegraded":"2023-09-23T18:52:51.207318+0000","last_fullsized":"2023-09-23T18:52:42.215984+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:46.266274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"268'11447","reported_seq":11502,"reported_epoch":268,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:53:22.494565+0000","last_change":"2023-09-23T18:51:44.256381+0000","last_active":"2023-09-23T18:53:22.494565+0000","last_peered":"2023-09-23T18:53:22.494565+0000","last_clean":"2023-09-23T18:51:42.071938+0000","last_became_active":"2023-09-23T18:51:44.206691+0000","last_became_peered":"2023-09-23T18:51:44.206691+0000","last_unstale":"2023-09-23T18:53:22.494565+0000","last_undegraded":"2023-09-23T18:53:22.494565+0000","last_fullsized":"2023-09-23T18:53:22.494565+0000","mapping_epoch":257,"log_start":"244'5045","ondisk_log_start":"244'5045","created":233,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_clean_scrub_stamp":"2023-09-23T18:47:48.124451+0000","objects_scrubbed":0,"log_size":6402,"log_dups_size":0,"ondisk_log_size":6402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:27.087714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90898432,"num_objects":1387,"num_object_clones":0,"num_object_copies":4161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1387,"num_objects_unfound":0,"num_objects_dirty":1387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11447,"num_write_kb":410688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,5],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":1387}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":230,"reported_epoch":267,"state":"active+undersized","last_fresh":"2023-09-23T18:52:51.207090+0000","last_change":"2023-09-23T18:52:42.229904+0000","last_active":"2023-09-23T18:52:51.207090+0000","last_peered":"2023-09-23T18:52:51.207090+0000","last_clean":"2023-09-23T18:52:35.278437+0000","last_became_active":"2023-09-23T18:52:42.229904+0000","last_became_peered":"2023-09-23T18:52:42.229904+0000","last_unstale":"2023-09-23T18:52:51.207090+0000","last_undegraded":"2023-09-23T18:52:51.207090+0000","last_fullsized":"2023-09-23T18:52:42.207924+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:02.974422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":217,"reported_epoch":267,"state":"active+undersized","last_fresh":"2023-09-23T18:52:51.207953+0000","last_change":"2023-09-23T18:52:42.214476+0000","last_active":"2023-09-23T18:52:51.207953+0000","last_peered":"2023-09-23T18:52:51.207953+0000","last_clean":"2023-09-23T18:52:35.284612+0000","last_became_active":"2023-09-23T18:52:42.214476+0000","last_became_peered":"2023-09-23T18:52:42.214476+0000","last_unstale":"2023-09-23T18:52:51.207953+0000","last_undegraded":"2023-09-23T18:52:51.207953+0000","last_fullsized":"2023-09-23T18:52:42.207818+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:30:54.422547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"268'11582","reported_seq":11634,"reported_epoch":268,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:53:24.166536+0000","last_change":"2023-09-23T18:52:42.219511+0000","last_active":"2023-09-23T18:53:24.166536+0000","last_peered":"2023-09-23T18:53:24.166536+0000","last_clean":"2023-09-23T18:52:41.118062+0000","last_became_active":"2023-09-23T18:52:42.219511+0000","last_became_peered":"2023-09-23T18:52:42.219511+0000","last_unstale":"2023-09-23T18:53:24.166536+0000","last_undegraded":"2023-09-23T18:52:42.211026+0000","last_fullsized":"2023-09-23T18:52:42.210899+0000","mapping_epoch":263,"log_start":"242'3832","ondisk_log_start":"242'3832","created":233,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_clean_scrub_stamp":"2023-09-23T18:47:48.124451+0000","objects_scrubbed":0,"log_size":7750,"log_dups_size":0,"ondisk_log_size":7750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:07.872875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95158272,"num_objects":1452,"num_object_clones":0,"num_object_copies":4356,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1452,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1452,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11582,"num_write_kb":417088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":1452}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"77'130","reported_seq":738,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.206757+0000","last_change":"2023-09-23T18:33:53.850713+0000","last_active":"2023-09-23T18:52:51.206757+0000","last_peered":"2023-09-23T18:52:51.206757+0000","last_clean":"2023-09-23T18:52:51.206757+0000","last_became_active":"2023-09-23T18:33:53.850011+0000","last_became_peered":"2023-09-23T18:33:53.850011+0000","last_unstale":"2023-09-23T18:52:51.206757+0000","last_undegraded":"2023-09-23T18:52:51.206757+0000","last_fullsized":"2023-09-23T18:52:51.206757+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:58.943556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":772,"num_read_kb":1712,"num_write":126,"num_write_kb":2126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":50,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.221863+0000","last_change":"2023-09-23T18:51:43.092309+0000","last_active":"2023-09-23T18:52:51.221863+0000","last_peered":"2023-09-23T18:52:51.221863+0000","last_clean":"2023-09-23T18:52:51.221863+0000","last_became_active":"2023-09-23T18:51:43.092000+0000","last_became_peered":"2023-09-23T18:51:43.092000+0000","last_unstale":"2023-09-23T18:52:51.221863+0000","last_undegraded":"2023-09-23T18:52:51.221863+0000","last_fullsized":"2023-09-23T18:52:51.221863+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:29.319431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":210,"reported_epoch":267,"state":"active+undersized","last_fresh":"2023-09-23T18:52:51.212167+0000","last_change":"2023-09-23T18:52:42.234004+0000","last_active":"2023-09-23T18:52:51.212167+0000","last_peered":"2023-09-23T18:52:51.212167+0000","last_clean":"2023-09-23T18:52:37.503953+0000","last_became_active":"2023-09-23T18:52:42.234004+0000","last_became_peered":"2023-09-23T18:52:42.234004+0000","last_unstale":"2023-09-23T18:52:51.212167+0000","last_undegraded":"2023-09-23T18:52:51.212167+0000","last_fullsized":"2023-09-23T18:52:42.216455+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:31.438036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"268'11368","reported_seq":11421,"reported_epoch":268,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:53:22.899473+0000","last_change":"2023-09-23T18:50:45.089630+0000","last_active":"2023-09-23T18:53:22.899473+0000","last_peered":"2023-09-23T18:53:22.899473+0000","last_clean":"2023-09-23T18:50:42.683972+0000","last_became_active":"2023-09-23T18:50:45.070163+0000","last_became_peered":"2023-09-23T18:50:45.070163+0000","last_unstale":"2023-09-23T18:53:22.899473+0000","last_undegraded":"2023-09-23T18:50:44.840509+0000","last_fullsized":"2023-09-23T18:50:44.804010+0000","mapping_epoch":253,"log_start":"239'2726","ondisk_log_start":"239'2726","created":233,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_clean_scrub_stamp":"2023-09-23T18:47:48.124451+0000","objects_scrubbed":0,"log_size":8642,"log_dups_size":0,"ondisk_log_size":8642,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:45.859554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93585408,"num_objects":1428,"num_object_clones":0,"num_object_copies":4284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6397,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11368,"num_write_kb":409472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":1428},{"shards":"3,7","objects":-3180}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"77'96","reported_seq":658,"reported_epoch":267,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:52:51.212546+0000","last_change":"2023-09-23T18:51:43.095730+0000","last_active":"2023-09-23T18:52:51.212546+0000","last_peered":"2023-09-23T18:52:51.212546+0000","last_clean":"2023-09-23T18:47:22.707953+0000","last_became_active":"2023-09-23T18:51:43.095277+0000","last_became_peered":"2023-09-23T18:51:43.095277+0000","last_unstale":"2023-09-23T18:52:51.212546+0000","last_undegraded":"2023-09-23T18:51:43.074521+0000","last_fullsized":"2023-09-23T18:52:51.212546+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":96,"log_dups_size":0,"ondisk_log_size":96,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:26:54.057934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":728,"num_read_kb":1234,"num_write":87,"num_write_kb":338,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":60,"reported_epoch":267,"state":"active+undersized","last_fresh":"2023-09-23T18:52:51.204601+0000","last_change":"2023-09-23T18:52:42.221592+0000","last_active":"2023-09-23T18:52:51.204601+0000","last_peered":"2023-09-23T18:52:51.204601+0000","last_clean":"2023-09-23T18:21:39.442892+0000","last_became_active":"2023-09-23T18:52:42.221592+0000","last_became_peered":"2023-09-23T18:52:42.221592+0000","last_unstale":"2023-09-23T18:52:51.204601+0000","last_undegraded":"2023-09-23T18:52:51.204601+0000","last_fullsized":"2023-09-23T18:52:42.213001+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:18.364655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":["0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":254,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.206427+0000","last_change":"2023-09-23T18:50:43.719949+0000","last_active":"2023-09-23T18:52:51.206427+0000","last_peered":"2023-09-23T18:52:51.206427+0000","last_clean":"2023-09-23T18:52:51.206427+0000","last_became_active":"2023-09-23T18:50:43.719666+0000","last_became_peered":"2023-09-23T18:50:43.719666+0000","last_unstale":"2023-09-23T18:52:51.206427+0000","last_undegraded":"2023-09-23T18:52:51.206427+0000","last_fullsized":"2023-09-23T18:52:51.206427+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:56.379780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"268'11693","reported_seq":18164,"reported_epoch":268,"state":"active+recovering+degraded+remapped","last_fresh":"2023-09-23T18:53:22.631576+0000","last_change":"2023-09-23T18:52:44.072223+0000","last_active":"2023-09-23T18:53:22.631576+0000","last_peered":"2023-09-23T18:53:22.631576+0000","last_clean":"2023-09-23T18:49:31.134852+0000","last_became_active":"2023-09-23T18:52:43.898155+0000","last_became_peered":"2023-09-23T18:52:43.898155+0000","last_unstale":"2023-09-23T18:53:22.631576+0000","last_undegraded":"2023-09-23T18:52:43.413364+0000","last_fullsized":"2023-09-23T18:53:22.631576+0000","mapping_epoch":264,"log_start":"244'5220","ondisk_log_start":"244'5220","created":233,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_clean_scrub_stamp":"2023-09-23T18:47:48.124451+0000","objects_scrubbed":0,"log_size":6473,"log_dups_size":0,"ondisk_log_size":6473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:41.590208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93388800,"num_objects":1425,"num_object_clones":0,"num_object_copies":4275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1425,"num_objects_misplaced":1084,"num_objects_unfound":0,"num_objects_dirty":1425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11693,"num_write_kb":419776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2652,"num_bytes_recovered":101253120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2,7],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1,2,7","objects":341},{"shards":"1,7","objects":1084}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'112","reported_seq":666,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.228384+0000","last_change":"2023-09-23T18:52:51.227977+0000","last_active":"2023-09-23T18:52:51.228384+0000","last_peered":"2023-09-23T18:52:51.228384+0000","last_clean":"2023-09-23T18:52:51.228384+0000","last_became_active":"2023-09-23T18:52:51.227007+0000","last_became_peered":"2023-09-23T18:52:51.227007+0000","last_unstale":"2023-09-23T18:52:51.228384+0000","last_undegraded":"2023-09-23T18:52:51.228384+0000","last_fullsized":"2023-09-23T18:52:51.228384+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:37.308459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":774,"num_read_kb":596,"num_write":102,"num_write_kb":52,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":208,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.206729+0000","last_change":"2023-09-23T18:33:53.849084+0000","last_active":"2023-09-23T18:52:51.206729+0000","last_peered":"2023-09-23T18:52:51.206729+0000","last_clean":"2023-09-23T18:52:51.206729+0000","last_became_active":"2023-09-23T18:33:53.848757+0000","last_became_peered":"2023-09-23T18:33:53.848757+0000","last_unstale":"2023-09-23T18:52:51.206729+0000","last_undegraded":"2023-09-23T18:52:51.206729+0000","last_fullsized":"2023-09-23T18:52:51.206729+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:15.267742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":249,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:53:13.274608+0000","last_change":"2023-09-23T18:51:43.095954+0000","last_active":"2023-09-23T18:53:13.274608+0000","last_peered":"2023-09-23T18:53:13.274608+0000","last_clean":"2023-09-23T18:53:13.274608+0000","last_became_active":"2023-09-23T18:51:43.095609+0000","last_became_peered":"2023-09-23T18:51:43.095609+0000","last_unstale":"2023-09-23T18:53:13.274608+0000","last_undegraded":"2023-09-23T18:53:13.274608+0000","last_fullsized":"2023-09-23T18:53:13.274608+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:40:03.370292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"268'11439","reported_seq":11490,"reported_epoch":268,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:53:24.264055+0000","last_change":"2023-09-23T18:49:33.610847+0000","last_active":"2023-09-23T18:53:24.264055+0000","last_peered":"2023-09-23T18:53:24.264055+0000","last_clean":"2023-09-23T18:49:31.118770+0000","last_became_active":"2023-09-23T18:49:33.583808+0000","last_became_peered":"2023-09-23T18:49:33.583808+0000","last_unstale":"2023-09-23T18:53:24.264055+0000","last_undegraded":"2023-09-23T18:49:33.226799+0000","last_fullsized":"2023-09-23T18:49:33.161662+0000","mapping_epoch":246,"log_start":"241'3720","ondisk_log_start":"241'3720","created":233,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_clean_scrub_stamp":"2023-09-23T18:47:48.124451+0000","objects_scrubbed":0,"log_size":7719,"log_dups_size":0,"ondisk_log_size":7719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:47.340251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90374144,"num_objects":1379,"num_object_clones":0,"num_object_copies":4137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6100,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1379,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11439,"num_write_kb":410176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":1070},{"shards":"0,6","objects":-4721}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"77'147","reported_seq":670,"reported_epoch":267,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:52:51.210118+0000","last_change":"2023-09-23T18:51:44.082793+0000","last_active":"2023-09-23T18:52:51.210118+0000","last_peered":"2023-09-23T18:52:51.210118+0000","last_clean":"2023-09-23T18:47:22.770560+0000","last_became_active":"2023-09-23T18:51:44.082084+0000","last_became_peered":"2023-09-23T18:51:44.082084+0000","last_unstale":"2023-09-23T18:52:51.210118+0000","last_undegraded":"2023-09-23T18:51:44.071084+0000","last_fullsized":"2023-09-23T18:51:44.070465+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:02.082206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":624,"num_read_kb":1868,"num_write":143,"num_write_kb":5368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":470,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.233635+0000","last_change":"2023-09-23T18:26:34.999402+0000","last_active":"2023-09-23T18:52:51.233635+0000","last_peered":"2023-09-23T18:52:51.233635+0000","last_clean":"2023-09-23T18:52:51.233635+0000","last_became_active":"2023-09-23T18:26:34.998828+0000","last_became_peered":"2023-09-23T18:26:34.998828+0000","last_unstale":"2023-09-23T18:52:51.233635+0000","last_undegraded":"2023-09-23T18:52:51.233635+0000","last_fullsized":"2023-09-23T18:52:51.233635+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_clean_scrub_stamp":"2023-09-23T18:08:35.701374+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":33,"reported_epoch":267,"state":"active+undersized","last_fresh":"2023-09-23T18:52:53.029757+0000","last_change":"2023-09-23T18:52:42.240757+0000","last_active":"2023-09-23T18:52:53.029757+0000","last_peered":"2023-09-23T18:52:53.029757+0000","last_clean":"2023-09-23T18:52:35.289166+0000","last_became_active":"2023-09-23T18:52:42.240757+0000","last_became_peered":"2023-09-23T18:52:42.240757+0000","last_unstale":"2023-09-23T18:52:53.029757+0000","last_undegraded":"2023-09-23T18:52:53.029757+0000","last_fullsized":"2023-09-23T18:52:42.223021+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:29.628112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":16,"reported_epoch":267,"state":"active+undersized","last_fresh":"2023-09-23T18:52:51.234748+0000","last_change":"2023-09-23T18:52:42.230595+0000","last_active":"2023-09-23T18:52:51.234748+0000","last_peered":"2023-09-23T18:52:51.234748+0000","last_clean":"2023-09-23T18:15:08.873352+0000","last_became_active":"2023-09-23T18:52:42.230595+0000","last_became_peered":"2023-09-23T18:52:42.230595+0000","last_unstale":"2023-09-23T18:52:51.234748+0000","last_undegraded":"2023-09-23T18:52:51.234748+0000","last_fullsized":"2023-09-23T18:52:42.225458+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"268'11358","reported_seq":11672,"reported_epoch":268,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:53:22.371947+0000","last_change":"2023-09-23T18:52:43.815336+0000","last_active":"2023-09-23T18:53:22.371947+0000","last_peered":"2023-09-23T18:53:22.371947+0000","last_clean":"2023-09-23T18:49:31.161738+0000","last_became_active":"2023-09-23T18:49:33.601578+0000","last_became_peered":"2023-09-23T18:49:33.601578+0000","last_unstale":"2023-09-23T18:53:22.371947+0000","last_undegraded":"2023-09-23T18:49:33.254936+0000","last_fullsized":"2023-09-23T18:49:33.167476+0000","mapping_epoch":246,"log_start":"243'4967","ondisk_log_start":"243'4967","created":233,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_clean_scrub_stamp":"2023-09-23T18:47:48.124451+0000","objects_scrubbed":0,"log_size":6391,"log_dups_size":0,"ondisk_log_size":6391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:11.523212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91881472,"num_objects":1402,"num_object_clones":0,"num_object_copies":4206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5950,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11358,"num_write_kb":408320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":129,"num_bytes_recovered":8454144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2],"avail_no_missing":["1","2","5"],"object_location_counts":[{"shards":"1,2","objects":-4548},{"shards":"1,2,5","objects":1101}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"77'120","reported_seq":729,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.233616+0000","last_change":"2023-09-23T18:33:53.852697+0000","last_active":"2023-09-23T18:52:51.233616+0000","last_peered":"2023-09-23T18:52:51.233616+0000","last_clean":"2023-09-23T18:52:51.233616+0000","last_became_active":"2023-09-23T18:33:53.852210+0000","last_became_peered":"2023-09-23T18:33:53.852210+0000","last_unstale":"2023-09-23T18:52:51.233616+0000","last_undegraded":"2023-09-23T18:52:51.233616+0000","last_fullsized":"2023-09-23T18:52:51.233616+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":120,"log_dups_size":0,"ondisk_log_size":120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":809,"num_read_kb":2749,"num_write":98,"num_write_kb":2087,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":260,"reported_epoch":267,"state":"active+undersized","last_fresh":"2023-09-23T18:52:51.207259+0000","last_change":"2023-09-23T18:52:42.227755+0000","last_active":"2023-09-23T18:52:51.207259+0000","last_peered":"2023-09-23T18:52:51.207259+0000","last_clean":"2023-09-23T18:52:35.314523+0000","last_became_active":"2023-09-23T18:52:42.227755+0000","last_became_peered":"2023-09-23T18:52:42.227755+0000","last_unstale":"2023-09-23T18:52:51.207259+0000","last_undegraded":"2023-09-23T18:52:51.207259+0000","last_fullsized":"2023-09-23T18:52:42.211528+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:23:59.338321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":294,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.204705+0000","last_change":"2023-09-23T18:51:43.087834+0000","last_active":"2023-09-23T18:52:51.204705+0000","last_peered":"2023-09-23T18:52:51.204705+0000","last_clean":"2023-09-23T18:52:51.204705+0000","last_became_active":"2023-09-23T18:51:43.087309+0000","last_became_peered":"2023-09-23T18:51:43.087309+0000","last_unstale":"2023-09-23T18:52:51.204705+0000","last_undegraded":"2023-09-23T18:52:51.204705+0000","last_fullsized":"2023-09-23T18:52:51.204705+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:53.913425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"268'11577","reported_seq":11877,"reported_epoch":268,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:53:22.940294+0000","last_change":"2023-09-23T18:52:43.994109+0000","last_active":"2023-09-23T18:53:22.940294+0000","last_peered":"2023-09-23T18:53:22.940294+0000","last_clean":"2023-09-23T18:49:31.137054+0000","last_became_active":"2023-09-23T18:49:33.721628+0000","last_became_peered":"2023-09-23T18:49:33.721628+0000","last_unstale":"2023-09-23T18:53:22.940294+0000","last_undegraded":"2023-09-23T18:49:33.215740+0000","last_fullsized":"2023-09-23T18:49:33.167390+0000","mapping_epoch":246,"log_start":"239'2929","ondisk_log_start":"239'2929","created":233,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_clean_scrub_stamp":"2023-09-23T18:47:48.124451+0000","objects_scrubbed":0,"log_size":8648,"log_dups_size":0,"ondisk_log_size":8648,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:00.195637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89849856,"num_objects":1371,"num_object_clones":0,"num_object_copies":4113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6051,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1371,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11577,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":122,"num_bytes_recovered":7995392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-4680}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"77'78","reported_seq":510,"reported_epoch":267,"state":"active+undersized","last_fresh":"2023-09-23T18:52:51.234353+0000","last_change":"2023-09-23T18:52:42.231648+0000","last_active":"2023-09-23T18:52:51.234353+0000","last_peered":"2023-09-23T18:52:51.234353+0000","last_clean":"2023-09-23T18:52:35.294557+0000","last_became_active":"2023-09-23T18:52:42.231648+0000","last_became_peered":"2023-09-23T18:52:42.231648+0000","last_unstale":"2023-09-23T18:52:51.234353+0000","last_undegraded":"2023-09-23T18:52:51.234353+0000","last_fullsized":"2023-09-23T18:52:42.222509+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":78,"log_dups_size":0,"ondisk_log_size":78,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":428,"num_read_kb":1125,"num_write":73,"num_write_kb":431,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":233,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.206562+0000","last_change":"2023-09-23T18:49:32.172810+0000","last_active":"2023-09-23T18:52:51.206562+0000","last_peered":"2023-09-23T18:52:51.206562+0000","last_clean":"2023-09-23T18:52:51.206562+0000","last_became_active":"2023-09-23T18:49:32.172207+0000","last_became_peered":"2023-09-23T18:49:32.172207+0000","last_unstale":"2023-09-23T18:52:51.206562+0000","last_undegraded":"2023-09-23T18:52:51.206562+0000","last_fullsized":"2023-09-23T18:52:51.206562+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:08:40.365756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":250,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.233870+0000","last_change":"2023-09-23T18:52:12.598045+0000","last_active":"2023-09-23T18:52:51.233870+0000","last_peered":"2023-09-23T18:52:51.233870+0000","last_clean":"2023-09-23T18:52:51.233870+0000","last_became_active":"2023-09-23T18:52:12.597683+0000","last_became_peered":"2023-09-23T18:52:12.597683+0000","last_unstale":"2023-09-23T18:52:51.233870+0000","last_undegraded":"2023-09-23T18:52:51.233870+0000","last_fullsized":"2023-09-23T18:52:51.233870+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.c","version":"268'11510","reported_seq":13548,"reported_epoch":268,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T18:53:24.013626+0000","last_change":"2023-09-23T18:52:50.029997+0000","last_active":"2023-09-23T18:53:24.013626+0000","last_peered":"2023-09-23T18:53:24.013626+0000","last_clean":"2023-09-23T18:49:31.136647+0000","last_became_active":"2023-09-23T18:52:43.727996+0000","last_became_peered":"2023-09-23T18:52:43.727996+0000","last_unstale":"2023-09-23T18:53:24.013626+0000","last_undegraded":"2023-09-23T18:52:43.367780+0000","last_fullsized":"2023-09-23T18:53:24.013626+0000","mapping_epoch":264,"log_start":"242'3756","ondisk_log_start":"242'3756","created":233,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_clean_scrub_stamp":"2023-09-23T18:47:48.124451+0000","objects_scrubbed":0,"log_size":7754,"log_dups_size":0,"ondisk_log_size":7754,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:34.632007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91750400,"num_objects":1400,"num_object_clones":0,"num_object_copies":4200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1400,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11510,"num_write_kb":413120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":986,"num_bytes_recovered":62849024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3,5],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3,5","objects":1400}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":245,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.227182+0000","last_change":"2023-09-23T18:28:03.298805+0000","last_active":"2023-09-23T18:52:51.227182+0000","last_peered":"2023-09-23T18:52:51.227182+0000","last_clean":"2023-09-23T18:52:51.227182+0000","last_became_active":"2023-09-23T18:28:03.298437+0000","last_became_peered":"2023-09-23T18:28:03.298437+0000","last_unstale":"2023-09-23T18:52:51.227182+0000","last_undegraded":"2023-09-23T18:52:51.227182+0000","last_fullsized":"2023-09-23T18:52:51.227182+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:27.831669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'181","reported_seq":752,"reported_epoch":268,"state":"active+undersized","last_fresh":"2023-09-23T18:53:13.274391+0000","last_change":"2023-09-23T18:52:42.226581+0000","last_active":"2023-09-23T18:53:13.274391+0000","last_peered":"2023-09-23T18:53:13.274391+0000","last_clean":"2023-09-23T18:52:35.278710+0000","last_became_active":"2023-09-23T18:52:42.226581+0000","last_became_peered":"2023-09-23T18:52:42.226581+0000","last_unstale":"2023-09-23T18:53:13.274391+0000","last_undegraded":"2023-09-23T18:53:13.274391+0000","last_fullsized":"2023-09-23T18:52:42.216292+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":181,"log_dups_size":0,"ondisk_log_size":181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:12.871102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":909,"num_read_kb":734,"num_write":148,"num_write_kb":75,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":176,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.233898+0000","last_change":"2023-09-23T18:39:46.992015+0000","last_active":"2023-09-23T18:52:51.233898+0000","last_peered":"2023-09-23T18:52:51.233898+0000","last_clean":"2023-09-23T18:52:51.233898+0000","last_became_active":"2023-09-23T18:39:46.991605+0000","last_became_peered":"2023-09-23T18:39:46.991605+0000","last_unstale":"2023-09-23T18:52:51.233898+0000","last_undegraded":"2023-09-23T18:52:51.233898+0000","last_fullsized":"2023-09-23T18:52:51.233898+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"268'11429","reported_seq":16016,"reported_epoch":268,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:53:25.384969+0000","last_change":"2023-09-23T18:51:44.498746+0000","last_active":"2023-09-23T18:53:25.384969+0000","last_peered":"2023-09-23T18:53:25.384969+0000","last_clean":"2023-09-23T18:49:31.154298+0000","last_became_active":"2023-09-23T18:51:44.457915+0000","last_became_peered":"2023-09-23T18:51:44.457915+0000","last_unstale":"2023-09-23T18:53:25.384969+0000","last_undegraded":"2023-09-23T18:51:44.300624+0000","last_fullsized":"2023-09-23T18:51:44.082992+0000","mapping_epoch":257,"log_start":"239'2331","ondisk_log_start":"239'2331","created":233,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:47:48.124451+0000","last_clean_scrub_stamp":"2023-09-23T18:47:48.124451+0000","objects_scrubbed":0,"log_size":9098,"log_dups_size":0,"ondisk_log_size":9098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87556120,"num_objects":1337,"num_object_clones":0,"num_object_copies":4011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9408,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1337,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11429,"num_write_kb":408449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1510,"num_bytes_recovered":98828288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2],"avail_no_missing":["7","5","6"],"object_location_counts":[{"shards":"2,5,7","objects":338},{"shards":"5,6,7","objects":999}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"77'112","reported_seq":595,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.210475+0000","last_change":"2023-09-23T18:48:02.275916+0000","last_active":"2023-09-23T18:52:51.210475+0000","last_peered":"2023-09-23T18:52:51.210475+0000","last_clean":"2023-09-23T18:52:51.210475+0000","last_became_active":"2023-09-23T18:48:02.275336+0000","last_became_peered":"2023-09-23T18:48:02.275336+0000","last_unstale":"2023-09-23T18:52:51.210475+0000","last_undegraded":"2023-09-23T18:52:51.210475+0000","last_fullsized":"2023-09-23T18:52:51.210475+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:37.263244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":893,"num_read_kb":3172,"num_write":69,"num_write_kb":2237,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":252,"reported_epoch":267,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:52:51.204840+0000","last_change":"2023-09-23T18:52:42.311695+0000","last_active":"2023-09-23T18:52:51.204840+0000","last_peered":"2023-09-23T18:52:51.204840+0000","last_clean":"2023-09-23T18:52:35.316014+0000","last_became_active":"2023-09-23T18:52:42.311695+0000","last_became_peered":"2023-09-23T18:52:42.311695+0000","last_unstale":"2023-09-23T18:52:51.204840+0000","last_undegraded":"2023-09-23T18:52:42.211548+0000","last_fullsized":"2023-09-23T18:52:42.211475+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:20.976228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":238,"reported_epoch":267,"state":"active+undersized","last_fresh":"2023-09-23T18:52:51.207041+0000","last_change":"2023-09-23T18:52:42.227191+0000","last_active":"2023-09-23T18:52:51.207041+0000","last_peered":"2023-09-23T18:52:51.207041+0000","last_clean":"2023-09-23T18:51:14.259249+0000","last_became_active":"2023-09-23T18:52:42.227191+0000","last_became_peered":"2023-09-23T18:52:42.227191+0000","last_unstale":"2023-09-23T18:52:51.207041+0000","last_undegraded":"2023-09-23T18:52:51.207041+0000","last_fullsized":"2023-09-23T18:52:42.210658+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:08.930546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":["0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":220,"reported_epoch":267,"state":"active+undersized","last_fresh":"2023-09-23T18:52:51.204788+0000","last_change":"2023-09-23T18:52:42.228333+0000","last_active":"2023-09-23T18:52:51.204788+0000","last_peered":"2023-09-23T18:52:51.204788+0000","last_clean":"2023-09-23T18:52:35.313521+0000","last_became_active":"2023-09-23T18:52:42.228333+0000","last_became_peered":"2023-09-23T18:52:42.228333+0000","last_unstale":"2023-09-23T18:52:51.204788+0000","last_undegraded":"2023-09-23T18:52:51.204788+0000","last_fullsized":"2023-09-23T18:52:42.212580+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:52.889391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":34,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.236061+0000","last_change":"2023-09-23T18:49:32.165166+0000","last_active":"2023-09-23T18:52:51.236061+0000","last_peered":"2023-09-23T18:52:51.236061+0000","last_clean":"2023-09-23T18:52:51.236061+0000","last_became_active":"2023-09-23T18:49:32.164777+0000","last_became_peered":"2023-09-23T18:49:32.164777+0000","last_unstale":"2023-09-23T18:52:51.236061+0000","last_undegraded":"2023-09-23T18:52:51.236061+0000","last_fullsized":"2023-09-23T18:52:51.236061+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:11.493077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":24,"reported_epoch":267,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:52:51.207040+0000","last_change":"2023-09-23T18:52:42.225412+0000","last_active":"2023-09-23T18:52:51.207040+0000","last_peered":"2023-09-23T18:52:51.207040+0000","last_clean":"2023-09-23T18:15:12.312222+0000","last_became_active":"2023-09-23T18:52:42.225412+0000","last_became_peered":"2023-09-23T18:52:42.225412+0000","last_unstale":"2023-09-23T18:52:51.207040+0000","last_undegraded":"2023-09-23T18:52:42.208523+0000","last_fullsized":"2023-09-23T18:52:42.208300+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:06.007099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":123,"reported_epoch":267,"state":"active+undersized","last_fresh":"2023-09-23T18:52:51.221414+0000","last_change":"2023-09-23T18:52:42.254681+0000","last_active":"2023-09-23T18:52:51.221414+0000","last_peered":"2023-09-23T18:52:51.221414+0000","last_clean":"2023-09-23T18:28:03.301070+0000","last_became_active":"2023-09-23T18:52:42.254681+0000","last_became_peered":"2023-09-23T18:52:42.254681+0000","last_unstale":"2023-09-23T18:52:51.221414+0000","last_undegraded":"2023-09-23T18:52:51.221414+0000","last_fullsized":"2023-09-23T18:52:42.216343+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:44:04.582227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":165,"reported_epoch":267,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:52:51.211232+0000","last_change":"2023-09-23T18:52:42.224702+0000","last_active":"2023-09-23T18:52:51.211232+0000","last_peered":"2023-09-23T18:52:51.211232+0000","last_clean":"2023-09-23T18:47:22.710511+0000","last_became_active":"2023-09-23T18:52:42.224702+0000","last_became_peered":"2023-09-23T18:52:42.224702+0000","last_unstale":"2023-09-23T18:52:51.211232+0000","last_undegraded":"2023-09-23T18:52:42.208597+0000","last_fullsized":"2023-09-23T18:52:42.207514+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:37.009837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":88,"reported_epoch":267,"state":"active+undersized","last_fresh":"2023-09-23T18:52:51.222823+0000","last_change":"2023-09-23T18:52:42.225901+0000","last_active":"2023-09-23T18:52:51.222823+0000","last_peered":"2023-09-23T18:52:51.222823+0000","last_clean":"2023-09-23T18:27:33.583428+0000","last_became_active":"2023-09-23T18:52:42.225901+0000","last_became_peered":"2023-09-23T18:52:42.225901+0000","last_unstale":"2023-09-23T18:52:51.222823+0000","last_undegraded":"2023-09-23T18:52:51.222823+0000","last_fullsized":"2023-09-23T18:52:42.218501+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:42:36.831691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":34,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.222513+0000","last_change":"2023-09-23T18:49:32.165623+0000","last_active":"2023-09-23T18:52:51.222513+0000","last_peered":"2023-09-23T18:52:51.222513+0000","last_clean":"2023-09-23T18:52:51.222513+0000","last_became_active":"2023-09-23T18:49:32.165174+0000","last_became_peered":"2023-09-23T18:49:32.165174+0000","last_unstale":"2023-09-23T18:52:51.222513+0000","last_undegraded":"2023-09-23T18:52:51.222513+0000","last_fullsized":"2023-09-23T18:52:51.222513+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:49.061095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":34,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.223366+0000","last_change":"2023-09-23T18:49:32.170049+0000","last_active":"2023-09-23T18:52:51.223366+0000","last_peered":"2023-09-23T18:52:51.223366+0000","last_clean":"2023-09-23T18:52:51.223366+0000","last_became_active":"2023-09-23T18:49:32.169725+0000","last_became_peered":"2023-09-23T18:49:32.169725+0000","last_unstale":"2023-09-23T18:52:51.223366+0000","last_undegraded":"2023-09-23T18:52:51.223366+0000","last_fullsized":"2023-09-23T18:52:51.223366+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:49.209921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":306,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:52:51.206862+0000","last_change":"2023-09-23T18:51:43.107148+0000","last_active":"2023-09-23T18:52:51.206862+0000","last_peered":"2023-09-23T18:52:51.206862+0000","last_clean":"2023-09-23T18:52:51.206862+0000","last_became_active":"2023-09-23T18:51:43.089248+0000","last_became_peered":"2023-09-23T18:51:43.089248+0000","last_unstale":"2023-09-23T18:52:51.206862+0000","last_undegraded":"2023-09-23T18:52:51.206862+0000","last_fullsized":"2023-09-23T18:52:51.206862+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:21.898095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":108,"reported_epoch":267,"state":"active+undersized","last_fresh":"2023-09-23T18:52:51.206931+0000","last_change":"2023-09-23T18:52:42.225289+0000","last_active":"2023-09-23T18:52:51.206931+0000","last_peered":"2023-09-23T18:52:51.206931+0000","last_clean":"2023-09-23T18:33:51.826014+0000","last_became_active":"2023-09-23T18:52:42.225289+0000","last_became_peered":"2023-09-23T18:52:42.225289+0000","last_unstale":"2023-09-23T18:52:51.206931+0000","last_undegraded":"2023-09-23T18:52:51.206931+0000","last_fullsized":"2023-09-23T18:52:42.207481+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:18.322662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":80,"acting":80,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":3002,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26886,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":79,"acting":79,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":1468727320,"num_objects":22412,"num_object_clones":0,"num_object_copies":67236,"num_objects_missing_on_primary":4539,"num_objects_missing":4539,"num_objects_degraded":49948,"num_objects_misplaced":5209,"num_objects_unfound":0,"num_objects_dirty":22412,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":184540,"num_write_kb":6622401,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9896,"num_bytes_recovered":573898752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7719759872,"data_stored":7719747656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120418,"ondisk_log_size":120418,"up":41,"acting":39,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":20,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5937,"num_read_kb":13190,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5621,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":22,"acting":21,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430707,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1574900,"kb_used_data":675756,"kb_used_omap":0,"kb_used_meta":899136,"kb_avail":92170252,"statfs":{"total":95995035648,"available":94382338048,"internally_reserved":0,"allocated":691974144,"data_stored":691321915,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920715264},"hb_peers":[0,1,2,3,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:52:25 2023","interfaces":[{"interface":"back","average":{"1min":1.641,"5min":1.4119999999999999,"15min":2.3340000000000001},"min":{"1min":0.31,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.0609999999999999,"5min":13.186999999999999,"15min":97.415999999999997},"last":0.48199999999999998},{"interface":"front","average":{"1min":2.665,"5min":1.5229999999999999,"15min":2.448},"min":{"1min":0.36499999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.5599999999999996,"5min":12.951000000000001,"15min":96.581000000000003},"last":0.372}]},{"osd":1,"last update":"Sat Sep 23 18:52:25 2023","interfaces":[{"interface":"back","average":{"1min":2.452,"5min":1.8899999999999999,"15min":2.2999999999999998},"min":{"1min":0.32100000000000001,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":10.401999999999999,"5min":16.056000000000001,"15min":94.650999999999996},"last":0.38900000000000001},{"interface":"front","average":{"1min":3.0800000000000001,"5min":2.02,"15min":2.5649999999999999},"min":{"1min":0.41599999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":11.292,"5min":16.085999999999999,"15min":94.843999999999994},"last":0.44900000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:53:22 2023","interfaces":[{"interface":"back","average":{"1min":2.2160000000000002,"5min":1.8720000000000001,"15min":1.7330000000000001},"min":{"1min":0.35399999999999998,"5min":0.32900000000000001,"15min":0.311},"max":{"1min":11.698,"5min":12.35,"15min":44.392000000000003},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.768,"5min":1.488,"15min":1.645},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.0890000000000004,"5min":7.4050000000000002,"15min":42.991999999999997},"last":0.53100000000000003}]},{"osd":3,"last update":"Sat Sep 23 18:52:42 2023","interfaces":[{"interface":"back","average":{"1min":1.331,"5min":2.9199999999999999,"15min":3.327},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":6.1470000000000002,"5min":118.07599999999999,"15min":136.345},"last":0.56899999999999995},{"interface":"front","average":{"1min":1.915,"5min":2.7130000000000001,"15min":3.4550000000000001},"min":{"1min":0.36599999999999999,"5min":0.32600000000000001,"15min":0.314},"max":{"1min":6.5119999999999996,"5min":118.69,"15min":136.893},"last":0.61899999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:52:25 2023","interfaces":[{"interface":"back","average":{"1min":1.4450000000000001,"5min":1.1319999999999999,"15min":1.016},"min":{"1min":0.39400000000000002,"5min":0.32500000000000001,"15min":0.29199999999999998},"max":{"1min":6.4809999999999999,"5min":6.4809999999999999,"15min":6.4809999999999999},"last":1.3480000000000001},{"interface":"front","average":{"1min":1.48,"5min":1.73,"15min":1.2849999999999999},"min":{"1min":0.40999999999999998,"5min":0.38900000000000001,"15min":0.35099999999999998},"max":{"1min":6.1100000000000003,"5min":13.010999999999999,"15min":13.010999999999999},"last":0.61299999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:52:25 2023","interfaces":[{"interface":"back","average":{"1min":1.5660000000000001,"5min":1.206,"15min":1.1839999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.29799999999999999},"max":{"1min":6.5780000000000003,"5min":7.0209999999999999,"15min":8.7210000000000001},"last":0.97399999999999998},{"interface":"front","average":{"1min":1.4870000000000001,"5min":1.4670000000000001,"15min":1.1499999999999999},"min":{"1min":0.47699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":6.5549999999999997,"5min":8.6519999999999992,"15min":8.6519999999999992},"last":0.58699999999999997}]},{"osd":6,"last update":"Sat Sep 23 18:52:25 2023","interfaces":[{"interface":"back","average":{"1min":1.6439999999999999,"5min":1.329,"15min":1.232},"min":{"1min":0.503,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":6.4500000000000002,"5min":6.4500000000000002,"15min":6.4500000000000002},"last":0.747},{"interface":"front","average":{"1min":1.165,"5min":1.2210000000000001,"15min":1.2010000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.36199999999999999},"max":{"1min":7.6029999999999998,"5min":7.6029999999999998,"15min":9.2850000000000001},"last":0.68700000000000006}]}]},{"osd":6,"up_from":42,"seq":180388626939,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1733800,"kb_used_data":1135904,"kb_used_omap":0,"kb_used_meta":597888,"kb_avail":92011352,"statfs":{"total":95995035648,"available":94219624448,"internally_reserved":0,"allocated":1163165696,"data_stored":1162525843,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612237312},"hb_peers":[0,1,2,3,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":1.181,"15min":2.0779999999999998},"min":{"1min":0.309,"5min":0.309,"15min":0.26000000000000001},"max":{"1min":5.2880000000000003,"5min":5.6859999999999999,"15min":103.562},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.427,"5min":1.3120000000000001,"15min":2.1880000000000002},"min":{"1min":0.39600000000000002,"5min":0.38400000000000001,"15min":0.32800000000000001},"max":{"1min":5.3300000000000001,"5min":5.7389999999999999,"15min":103.643},"last":0.52400000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5249999999999999,"5min":1.4370000000000001,"15min":3.3980000000000001},"min":{"1min":0.31,"5min":0.27600000000000002,"15min":0.25800000000000001},"max":{"1min":5.5039999999999996,"5min":6.8099999999999996,"15min":284.07400000000001},"last":0.624},{"interface":"front","average":{"1min":1.7989999999999999,"5min":1.712,"15min":3.585},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.0050000000000008,"5min":9.0050000000000008,"15min":281.19900000000001},"last":0.96499999999999997}]},{"osd":2,"last update":"Sat Sep 23 18:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":1.3660000000000001,"15min":2.306},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.314},"max":{"1min":3.9990000000000001,"5min":7.1799999999999997,"15min":122.242},"last":0.59699999999999998},{"interface":"front","average":{"1min":1.4419999999999999,"5min":1.4179999999999999,"15min":2.3460000000000001},"min":{"1min":0.433,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.9860000000000002,"5min":8.8580000000000005,"15min":122.32899999999999},"last":0.71299999999999997}]},{"osd":3,"last update":"Sat Sep 23 18:52:42 2023","interfaces":[{"interface":"back","average":{"1min":1.9259999999999999,"5min":1.679,"15min":3.1440000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.27500000000000002},"max":{"1min":11.720000000000001,"5min":11.720000000000001,"15min":123.52200000000001},"last":0.73099999999999998},{"interface":"front","average":{"1min":1.948,"5min":1.883,"15min":3.3460000000000001},"min":{"1min":0.49099999999999999,"5min":0.373,"15min":0.373},"max":{"1min":11.702,"5min":11.702,"15min":123.605},"last":0.67800000000000005}]},{"osd":4,"last update":"Sat Sep 23 18:51:40 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":1.5229999999999999,"15min":1.306},"min":{"1min":0.36099999999999999,"5min":0.34000000000000002,"15min":0.29999999999999999},"max":{"1min":5.5460000000000003,"5min":23.635999999999999,"15min":23.635999999999999},"last":3.4969999999999999},{"interface":"front","average":{"1min":1.121,"5min":1.091,"15min":1.117},"min":{"1min":0.48099999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.7000000000000002,"5min":5.2469999999999999,"15min":14.875999999999999},"last":0.56899999999999995}]},{"osd":5,"last update":"Sat Sep 23 18:52:42 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":4.2130000000000001,"15min":2.1230000000000002},"min":{"1min":0.38500000000000001,"5min":0.371,"15min":0.371},"max":{"1min":3.5409999999999999,"5min":336.26299999999998,"15min":336.26299999999998},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.70099999999999996,"5min":4.298,"15min":2.0409999999999999},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.34200000000000003},"max":{"1min":2.3199999999999998,"5min":336.07799999999997,"15min":336.07799999999997},"last":0.81699999999999995}]},{"osd":7,"last update":"Sat Sep 23 18:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":1.196,"15min":1.1950000000000001},"min":{"1min":0.45600000000000002,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":4.2460000000000004,"5min":7.0700000000000003,"15min":7.9320000000000004},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.93000000000000005,"5min":1.1200000000000001,"15min":1.3260000000000001},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.39900000000000002},"max":{"1min":3.6139999999999999,"5min":6.6580000000000004,"15min":8.9250000000000007},"last":0.75800000000000001}]}]},{"osd":1,"up_from":123,"seq":528280977643,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1225496,"kb_used_data":636816,"kb_used_omap":7,"kb_used_meta":588664,"kb_avail":92519656,"statfs":{"total":95995035648,"available":94740127744,"internally_reserved":0,"allocated":652099584,"data_stored":650840088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":602792718},"hb_peers":[0,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:52:26 2023","interfaces":[{"interface":"back","average":{"1min":3.1890000000000001,"5min":2.1960000000000002,"15min":2.298},"min":{"1min":0.21299999999999999,"5min":0.20499999999999999,"15min":0.17499999999999999},"max":{"1min":11.1,"5min":13.286,"15min":53.530999999999999},"last":0.34300000000000003},{"interface":"front","average":{"1min":3.2589999999999999,"5min":2.27,"15min":2.552},"min":{"1min":0.27700000000000002,"5min":0.222,"15min":0.17599999999999999},"max":{"1min":11.025,"5min":11.318,"15min":51.710999999999999},"last":0.65900000000000003}]},{"osd":2,"last update":"Sat Sep 23 18:52:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3559999999999999,"5min":3.3769999999999998,"15min":3.1720000000000002},"min":{"1min":0.28499999999999998,"5min":0.23300000000000001,"15min":0.19900000000000001},"max":{"1min":5.7930000000000001,"5min":180.578,"15min":180.578},"last":0.46400000000000002},{"interface":"front","average":{"1min":2.5350000000000001,"5min":3.5640000000000001,"15min":3.3780000000000001},"min":{"1min":0.308,"5min":0.28199999999999997,"15min":0.247},"max":{"1min":9.1649999999999991,"5min":180.12299999999999,"15min":180.12299999999999},"last":0.872}]},{"osd":3,"last update":"Sat Sep 23 18:52:26 2023","interfaces":[{"interface":"back","average":{"1min":2.653,"5min":2.0179999999999998,"15min":2.6509999999999998},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":9.1199999999999992,"5min":11.92,"15min":55.014000000000003},"last":0.50600000000000001},{"interface":"front","average":{"1min":2.246,"5min":1.885,"15min":2.4089999999999998},"min":{"1min":0.32300000000000001,"5min":0.30099999999999999,"15min":0.246},"max":{"1min":5.8289999999999997,"5min":8.8160000000000007,"15min":55.841999999999999},"last":0.41899999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:52:26 2023","interfaces":[{"interface":"back","average":{"1min":1.956,"5min":2.4500000000000002,"15min":1.966},"min":{"1min":0.39000000000000001,"5min":0.371,"15min":0.308},"max":{"1min":5.8780000000000001,"5min":56.871000000000002,"15min":56.871000000000002},"last":6.5590000000000002},{"interface":"front","average":{"1min":2.5070000000000001,"5min":4.1920000000000002,"15min":2.4710000000000001},"min":{"1min":0.43099999999999999,"5min":0.315,"15min":0.27700000000000002},"max":{"1min":8.0380000000000003,"5min":179.82499999999999,"15min":179.82499999999999},"last":4.0129999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:52:26 2023","interfaces":[{"interface":"back","average":{"1min":2,"5min":2.4260000000000002,"15min":2.0430000000000001},"min":{"1min":0.378,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":9.1440000000000001,"5min":54.488999999999997,"15min":54.488999999999997},"last":0.92600000000000005},{"interface":"front","average":{"1min":2.012,"5min":4.3230000000000004,"15min":2.5609999999999999},"min":{"1min":0.436,"5min":0.42099999999999999,"15min":0.371},"max":{"1min":5.8529999999999998,"5min":180.52099999999999,"15min":180.52099999999999},"last":0.53300000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:52:26 2023","interfaces":[{"interface":"back","average":{"1min":2.0499999999999998,"5min":1.833,"15min":1.839},"min":{"1min":0.47499999999999998,"5min":0.38500000000000001,"15min":0.33900000000000002},"max":{"1min":5.9009999999999998,"5min":8.1159999999999997,"15min":12.574999999999999},"last":0.59699999999999998},{"interface":"front","average":{"1min":2.3130000000000002,"5min":1.909,"15min":1.885},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.32900000000000001},"max":{"1min":7.9580000000000002,"5min":8.0069999999999997,"15min":16.687000000000001},"last":0.64700000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:52:26 2023","interfaces":[{"interface":"back","average":{"1min":2.141,"5min":1.9670000000000001,"15min":1.964},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.33500000000000002},"max":{"1min":8.0099999999999998,"5min":8.0800000000000001,"15min":11.747999999999999},"last":0.623},{"interface":"front","average":{"1min":2.3679999999999999,"5min":1.845,"15min":1.7909999999999999},"min":{"1min":0.40200000000000002,"5min":0.373,"15min":0.373},"max":{"1min":8.0099999999999998,"5min":8.0099999999999998,"15min":11.593},"last":0.61199999999999999}]}]},{"osd":0,"up_from":88,"seq":377957122372,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1269264,"kb_used_data":550984,"kb_used_omap":8,"kb_used_meta":718263,"kb_avail":92475888,"statfs":{"total":95995035648,"available":94695309312,"internally_reserved":0,"allocated":564207616,"data_stored":563071022,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":735502206},"hb_peers":[1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:52:35 2023","interfaces":[{"interface":"back","average":{"1min":1.6819999999999999,"5min":3.8140000000000001,"15min":2.8010000000000002},"min":{"1min":0.33700000000000002,"5min":0.23899999999999999,"15min":0.191},"max":{"1min":6.3789999999999996,"5min":123.81,"15min":123.81},"last":4.0880000000000001},{"interface":"front","average":{"1min":2.508,"5min":3.8029999999999999,"15min":2.7989999999999999},"min":{"1min":0.27800000000000002,"5min":0.22600000000000001,"15min":0.20999999999999999},"max":{"1min":11.974,"5min":121.92400000000001,"15min":121.92400000000001},"last":4.1319999999999997}]},{"osd":2,"last update":"Sat Sep 23 18:53:19 2023","interfaces":[{"interface":"back","average":{"1min":1.143,"5min":2.1749999999999998,"15min":2.0449999999999999},"min":{"1min":0.28899999999999998,"5min":0.224,"15min":0.224},"max":{"1min":4.5510000000000002,"5min":11.808999999999999,"15min":12.622999999999999},"last":1.232},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.8400000000000001,"15min":1.5860000000000001},"min":{"1min":0.27200000000000002,"5min":0.25700000000000001,"15min":0.19400000000000001},"max":{"1min":4.516,"5min":11.894,"15min":12.297000000000001},"last":1.1120000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:52:42 2023","interfaces":[{"interface":"back","average":{"1min":5.1310000000000002,"5min":2.9169999999999998,"15min":3.919},"min":{"1min":0.40200000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":42.735999999999997,"5min":42.735999999999997,"15min":177.83600000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":4.7030000000000003,"5min":2.7599999999999998,"15min":3.8279999999999998},"min":{"1min":0.33600000000000002,"5min":0.29599999999999999,"15min":0.24299999999999999},"max":{"1min":43,"5min":43,"15min":181.71899999999999},"last":0.32300000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:52:35 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":1.669,"15min":1.5720000000000001},"min":{"1min":0.36399999999999999,"5min":0.32400000000000001,"15min":0.29499999999999998},"max":{"1min":2.8239999999999998,"5min":8.7569999999999997,"15min":9.9450000000000003},"last":2.9860000000000002},{"interface":"front","average":{"1min":2.9079999999999999,"5min":1.8149999999999999,"15min":1.6060000000000001},"min":{"1min":0.41499999999999998,"5min":0.372,"15min":0.33000000000000002},"max":{"1min":9.1219999999999999,"5min":9.4239999999999995,"15min":10.561999999999999},"last":3.9140000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:52:35 2023","interfaces":[{"interface":"back","average":{"1min":3.0649999999999999,"5min":2.0049999999999999,"15min":1.5940000000000001},"min":{"1min":0.44900000000000001,"5min":0.40600000000000003,"15min":0.36499999999999999},"max":{"1min":9.0960000000000001,"5min":9.5879999999999992,"15min":10.606},"last":0.48399999999999999},{"interface":"front","average":{"1min":1.619,"5min":1.605,"15min":1.339},"min":{"1min":0.41799999999999998,"5min":0.34999999999999998,"15min":0.32900000000000001},"max":{"1min":8.6720000000000006,"5min":8.6720000000000006,"15min":8.6720000000000006},"last":1.0329999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:52:35 2023","interfaces":[{"interface":"back","average":{"1min":3.0510000000000002,"5min":1.97,"15min":1.7430000000000001},"min":{"1min":0.50900000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":9.0570000000000004,"5min":9.4760000000000009,"15min":11.302},"last":0.53000000000000003},{"interface":"front","average":{"1min":1.6950000000000001,"5min":1.742,"15min":1.498},"min":{"1min":0.42999999999999999,"5min":0.41499999999999998,"15min":0.33200000000000002},"max":{"1min":8.6300000000000008,"5min":8.6300000000000008,"15min":11.818},"last":1.0800000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:52:35 2023","interfaces":[{"interface":"back","average":{"1min":1.6120000000000001,"5min":1.5960000000000001,"15min":1.4790000000000001},"min":{"1min":0.48999999999999999,"5min":0.372,"15min":0.372},"max":{"1min":8.7040000000000006,"5min":8.7040000000000006,"15min":10.191000000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.5629999999999999,"15min":1.601},"min":{"1min":0.48699999999999999,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":2.8410000000000002,"5min":7.649,"15min":10.01},"last":0.621}]}]},{"osd":2,"up_from":98,"seq":420906795319,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":982544,"kb_used_data":291080,"kb_used_omap":11,"kb_used_meta":691380,"kb_avail":92762608,"statfs":{"total":95995035648,"available":94988910592,"internally_reserved":0,"allocated":298065920,"data_stored":296927894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11949,"internal_metadata":707973459},"hb_peers":[0,1,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:53:24 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":3.5,"15min":2.3239999999999998},"min":{"1min":0.30099999999999999,"5min":0.20200000000000001,"15min":0.184},"max":{"1min":6.9669999999999996,"5min":166.46100000000001,"15min":166.46100000000001},"last":0.30099999999999999},{"interface":"front","average":{"1min":1.736,"5min":3.2749999999999999,"15min":2.508},"min":{"1min":0.25800000000000001,"5min":0.23899999999999999,"15min":0.18099999999999999},"max":{"1min":10.585000000000001,"5min":162.911,"15min":162.911},"last":0.32900000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:52:28 2023","interfaces":[{"interface":"back","average":{"1min":1.623,"5min":3.7839999999999998,"15min":3.895},"min":{"1min":0.35299999999999998,"5min":0.216,"15min":0.20699999999999999},"max":{"1min":5.2450000000000001,"5min":156.35400000000001,"15min":184.56299999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.95,"5min":3.4649999999999999,"15min":3.9769999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.223},"max":{"1min":9.0259999999999998,"5min":156.376,"15min":183.922},"last":0.57099999999999995}]},{"osd":3,"last update":"Sat Sep 23 18:52:57 2023","interfaces":[{"interface":"back","average":{"1min":2.0819999999999999,"5min":3.258,"15min":3.5350000000000001},"min":{"1min":0.28000000000000003,"5min":0.24299999999999999,"15min":0.183},"max":{"1min":6.2300000000000004,"5min":157.58099999999999,"15min":161.773},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.895,"5min":3.827,"15min":3.4820000000000002},"min":{"1min":0.32000000000000001,"5min":0.25700000000000001,"15min":0.25},"max":{"1min":5.3090000000000002,"5min":159.34,"15min":159.34},"last":0.38500000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:52:21 2023","interfaces":[{"interface":"back","average":{"1min":1.655,"5min":1.492,"15min":1.667},"min":{"1min":0.35199999999999998,"5min":0.34000000000000002,"15min":0.30099999999999999},"max":{"1min":4.9640000000000004,"5min":5.9859999999999998,"15min":71.587999999999994},"last":4.8239999999999998},{"interface":"front","average":{"1min":1.5089999999999999,"5min":1.7829999999999999,"15min":1.633},"min":{"1min":0.39700000000000002,"5min":0.33700000000000002,"15min":0.33600000000000002},"max":{"1min":5.202,"5min":16.123000000000001,"15min":71.328999999999994},"last":0.61799999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:53:24 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":4.6109999999999998,"15min":2.5219999999999998},"min":{"1min":0.55700000000000005,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.4859999999999998,"5min":296.86200000000002,"15min":296.86200000000002},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.96,"5min":4.7160000000000002,"15min":2.766},"min":{"1min":0.40799999999999997,"5min":0.36299999999999999,"15min":0.35999999999999999},"max":{"1min":10.563000000000001,"5min":296.69499999999999,"15min":296.69499999999999},"last":0.70199999999999996}]},{"osd":6,"last update":"Sat Sep 23 18:53:24 2023","interfaces":[{"interface":"back","average":{"1min":1.417,"5min":1.5069999999999999,"15min":1.7589999999999999},"min":{"1min":0.39300000000000002,"5min":0.375,"15min":0.35199999999999998},"max":{"1min":11.021000000000001,"5min":11.021000000000001,"15min":71.460999999999999},"last":0.68000000000000005},{"interface":"front","average":{"1min":1.3440000000000001,"5min":1.8999999999999999,"15min":1.772},"min":{"1min":0.44400000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.5599999999999996,"5min":16.247,"15min":71.271000000000001},"last":0.78200000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:53:24 2023","interfaces":[{"interface":"back","average":{"1min":1.456,"5min":1.522,"15min":1.7450000000000001},"min":{"1min":0.42299999999999999,"5min":0.374,"15min":0.374},"max":{"1min":10.971,"5min":10.971,"15min":71.622},"last":0.72199999999999998},{"interface":"front","average":{"1min":1.145,"5min":1.7829999999999999,"15min":1.7310000000000001},"min":{"1min":0.52700000000000002,"5min":0.41999999999999998,"15min":0.39900000000000002},"max":{"1min":4.46,"5min":16.178999999999998,"15min":71.575000000000003},"last":0.81000000000000005}]}]},{"osd":3,"up_from":103,"seq":442381631794,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1111252,"kb_used_data":493708,"kb_used_omap":4,"kb_used_meta":617531,"kb_avail":92633900,"statfs":{"total":95995035648,"available":94857113600,"internally_reserved":0,"allocated":505556992,"data_stored":504414323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4826,"internal_metadata":632352038},"hb_peers":[0,1,2,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:52:51 2023","interfaces":[{"interface":"back","average":{"1min":5.1600000000000001,"5min":2.2280000000000002,"15min":2.7599999999999998},"min":{"1min":0.34899999999999998,"5min":0.20399999999999999,"15min":0.193},"max":{"1min":36.802999999999997,"5min":36.802999999999997,"15min":129.315},"last":0.252},{"interface":"front","average":{"1min":4.4909999999999997,"5min":2.3199999999999998,"15min":2.8359999999999999},"min":{"1min":0.251,"5min":0.247,"15min":0.19700000000000001},"max":{"1min":35.578000000000003,"5min":35.578000000000003,"15min":130.09200000000001},"last":2.036}]},{"osd":1,"last update":"Sat Sep 23 18:52:32 2023","interfaces":[{"interface":"back","average":{"1min":3.9369999999999998,"5min":2.395,"15min":2.5779999999999998},"min":{"1min":0.34999999999999998,"5min":0.249,"15min":0.20399999999999999},"max":{"1min":36.133000000000003,"5min":36.133000000000003,"15min":157.67400000000001},"last":2.0089999999999999},{"interface":"front","average":{"1min":4.8129999999999997,"5min":2.4780000000000002,"15min":2.6680000000000001},"min":{"1min":0.39600000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":36.176000000000002,"5min":36.176000000000002,"15min":157.84700000000001},"last":2.0219999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:52:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8879999999999999,"5min":2.4009999999999998,"15min":2.5},"min":{"1min":0.47399999999999998,"5min":0.22,"15min":0.22},"max":{"1min":10.49,"5min":14.212999999999999,"15min":23.518000000000001},"last":2.097},{"interface":"front","average":{"1min":1.964,"5min":1.7390000000000001,"15min":1.976},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.22500000000000001},"max":{"1min":6.3680000000000003,"5min":11.289999999999999,"15min":24.266999999999999},"last":0.27400000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:51:50 2023","interfaces":[{"interface":"back","average":{"1min":1.4399999999999999,"5min":1.403,"15min":1.4450000000000001},"min":{"1min":0.38300000000000001,"5min":0.32700000000000001,"15min":0.30599999999999999},"max":{"1min":5.1539999999999999,"5min":11.361000000000001,"15min":11.361000000000001},"last":1.645},{"interface":"front","average":{"1min":2.371,"5min":1.4079999999999999,"15min":1.5609999999999999},"min":{"1min":0.41499999999999998,"5min":0.34799999999999998,"15min":0.33900000000000002},"max":{"1min":8.0860000000000003,"5min":8.4930000000000003,"15min":11.916},"last":0.314}]},{"osd":5,"last update":"Sat Sep 23 18:52:51 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":1.5149999999999999,"15min":1.452},"min":{"1min":0.372,"5min":0.372,"15min":0.34100000000000003},"max":{"1min":6.5270000000000001,"5min":8.5169999999999995,"15min":8.5169999999999995},"last":0.372},{"interface":"front","average":{"1min":2.0449999999999999,"5min":1.796,"15min":1.8049999999999999},"min":{"1min":0.35999999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":10.436,"5min":11.052,"15min":11.052},"last":2.1339999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:52:51 2023","interfaces":[{"interface":"back","average":{"1min":2.0659999999999998,"5min":1.9750000000000001,"15min":2.0819999999999999},"min":{"1min":0.38500000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":10.465,"5min":19.242999999999999,"15min":19.242999999999999},"last":2.1789999999999998},{"interface":"front","average":{"1min":1.6830000000000001,"5min":1.669,"15min":1.722},"min":{"1min":0.45800000000000002,"5min":0.41399999999999998,"15min":0.33300000000000002},"max":{"1min":6.4249999999999998,"5min":11.335000000000001,"15min":11.989000000000001},"last":0.41699999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:52:51 2023","interfaces":[{"interface":"back","average":{"1min":2.0470000000000002,"5min":1.7210000000000001,"15min":1.702},"min":{"1min":0.48899999999999999,"5min":0.39100000000000001,"15min":0.35399999999999998},"max":{"1min":6.46,"5min":11.382999999999999,"15min":11.382999999999999},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.3520000000000001,"5min":1.601,"15min":1.5740000000000001},"min":{"1min":0.39900000000000002,"5min":0.38900000000000001,"15min":0.36599999999999999},"max":{"1min":6.5590000000000002,"5min":12.170999999999999,"15min":12.170999999999999},"last":0.40000000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019393,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1946652,"kb_used_data":1052052,"kb_used_omap":0,"kb_used_meta":894592,"kb_avail":91798500,"statfs":{"total":95995035648,"available":94001664000,"internally_reserved":0,"allocated":1077301248,"data_stored":1076689012,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916062208},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:52:11 2023","interfaces":[{"interface":"back","average":{"1min":2.71,"5min":2.4769999999999999,"15min":1.748},"min":{"1min":0.44500000000000001,"5min":0.248,"15min":0.248},"max":{"1min":7.3780000000000001,"5min":68.971000000000004,"15min":68.971000000000004},"last":1.659},{"interface":"front","average":{"1min":2.379,"5min":2.2509999999999999,"15min":1.671},"min":{"1min":0.29799999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":6.3639999999999999,"5min":69.200000000000003,"15min":69.200000000000003},"last":1.643}]},{"osd":1,"last update":"Sat Sep 23 18:52:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6399999999999999,"5min":3.069,"15min":3.1139999999999999},"min":{"1min":0.41599999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.9740000000000002,"5min":63.616,"15min":73.537000000000006},"last":0.33000000000000002},{"interface":"front","average":{"1min":2.3759999999999999,"5min":3.4489999999999998,"15min":3.23},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.28399999999999997},"max":{"1min":8.4580000000000002,"5min":63.648000000000003,"15min":73.575999999999993},"last":0.38400000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:52:11 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":1.4670000000000001,"15min":2.4769999999999999},"min":{"1min":0.34000000000000002,"5min":0.31900000000000001,"15min":0.312},"max":{"1min":5.0700000000000003,"5min":8.2650000000000006,"15min":111.34699999999999},"last":0.55800000000000005},{"interface":"front","average":{"1min":2.169,"5min":1.853,"15min":2.6949999999999998},"min":{"1min":0.40300000000000002,"5min":0.32700000000000001,"15min":0.309},"max":{"1min":7.8319999999999999,"5min":10.317,"15min":111.422},"last":0.59899999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:51:53 2023","interfaces":[{"interface":"back","average":{"1min":2.5299999999999998,"5min":2.6829999999999998,"15min":2.9489999999999998},"min":{"1min":0.41099999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":15.252000000000001,"5min":64.177000000000007,"15min":136.548},"last":4.6890000000000001},{"interface":"front","average":{"1min":2.0790000000000002,"5min":2.7519999999999998,"15min":2.8239999999999998},"min":{"1min":0.39200000000000002,"5min":0.36399999999999999,"15min":0.36099999999999999},"max":{"1min":12.465999999999999,"5min":64.313999999999993,"15min":136.494},"last":6.0170000000000003}]},{"osd":5,"last update":"Sat Sep 23 18:52:05 2023","interfaces":[{"interface":"back","average":{"1min":1.351,"5min":1.24,"15min":1.0429999999999999},"min":{"1min":0.375,"5min":0.313,"15min":0.307},"max":{"1min":4.7160000000000002,"5min":5.1859999999999999,"15min":6.9249999999999998},"last":4.7249999999999996},{"interface":"front","average":{"1min":1.6910000000000001,"5min":1.5569999999999999,"15min":1.1679999999999999},"min":{"1min":0.40500000000000003,"5min":0.36399999999999999,"15min":0.33500000000000002},"max":{"1min":7.4169999999999998,"5min":8.0470000000000006,"15min":8.0470000000000006},"last":2.198}]},{"osd":6,"last update":"Sat Sep 23 18:51:38 2023","interfaces":[{"interface":"back","average":{"1min":1.389,"5min":1.3240000000000001,"15min":1.1970000000000001},"min":{"1min":0.38500000000000001,"5min":0.36099999999999999,"15min":0.28100000000000003},"max":{"1min":4.7469999999999999,"5min":11.503,"15min":11.503},"last":6.194},{"interface":"front","average":{"1min":1.2210000000000001,"5min":1.2170000000000001,"15min":1.095},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.27700000000000002},"max":{"1min":4.8899999999999997,"5min":7.4100000000000001,"15min":7.4100000000000001},"last":0.378}]},{"osd":7,"last update":"Sat Sep 23 18:52:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5149999999999999,"5min":1.302,"15min":1.4470000000000001},"min":{"1min":0.436,"5min":0.39100000000000001,"15min":0.32500000000000001},"max":{"1min":4.7770000000000001,"5min":7.6890000000000001,"15min":12.294},"last":0.35599999999999998},{"interface":"front","average":{"1min":1.512,"5min":1.3700000000000001,"15min":1.5349999999999999},"min":{"1min":0.48499999999999999,"5min":0.41899999999999998,"15min":0.32500000000000001},"max":{"1min":4.734,"5min":5.3899999999999997,"15min":10.507},"last":4.7640000000000002}]}]},{"osd":5,"up_from":0,"seq":154618823173,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4224,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7725,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3429,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":556269568,"data_stored":556269568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":647168000,"data_stored":647168000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":289869824,"data_stored":289865752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":500760576,"data_stored":500760576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1074528256,"data_stored":1074528256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2813923328,"data_stored":2813919256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1155399680,"data_stored":1155399680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":681840640,"data_stored":681836568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:53:30.518 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 41 2023-09-23T18:53:30.529 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:30.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:30.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:30.633 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:30.654 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:30.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:30.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:30.757 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:30.790 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:30.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:30.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:30.896 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:30.927 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:30.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:30.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:30.978 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:53:30.981 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:53:31.031 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:31.057 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:31.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:31.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:31.161 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:31.196 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:31.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:31.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:31.301 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:31.328 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:31.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:31.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:31.434 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:31.458 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:31.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:31.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:31.561 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:31.595 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:31.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:31.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:31.699 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:31.732 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:31.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:31.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:31.836 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:31.868 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:31.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:31.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:31.971 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:31.997 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:32.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:32.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:32.103 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:32.130 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:32.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:32.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:32.234 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:32.263 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:32.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:32.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:32.368 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:32.391 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:32.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:32.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:32.495 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:32.504 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:32 smithi098 bash[79516]: cluster 2023-09-23T18:53:32.196840+0000 mgr.y (mgr.26054) 1067 : cluster 0 pgmap v1143: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.3 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 180 op/s; 49362/64008 objects degraded (77.118%); 4924/64008 objects misplaced (7.693%); 2.3 MiB/s, 73 objects/s recovering 2023-09-23T18:53:32.505 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:32 smithi098 bash[76817]: cluster 2023-09-23T18:53:32.196840+0000 mgr.y (mgr.26054) 1067 : cluster 0 pgmap v1143: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.3 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 180 op/s; 49362/64008 objects degraded (77.118%); 4924/64008 objects misplaced (7.693%); 2.3 MiB/s, 73 objects/s recovering 2023-09-23T18:53:32.518 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:32.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:32.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:32.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:32 smithi173 bash[59422]: cluster 2023-09-23T18:53:32.196840+0000 mgr.y (mgr.26054) 1067 : cluster 0 pgmap v1143: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.3 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 180 op/s; 49362/64008 objects degraded (77.118%); 4924/64008 objects misplaced (7.693%); 2.3 MiB/s, 73 objects/s recovering 2023-09-23T18:53:32.623 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:32.649 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:32.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:32.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:32.752 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:32.784 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:32.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:32.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:32.888 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:32.920 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:32.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:32.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:33.025 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:33.056 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:33.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:33.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:33.163 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:33.192 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:33.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:33.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:33.317 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:33.342 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:33.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:33.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:33.446 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:33.484 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:33.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:33.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:33.588 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:33.611 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:33.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:33.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:33.717 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:33.748 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:33.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:33.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:33.853 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:33.881 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:33.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:33.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:33.985 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:34.015 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:34.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:34.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:34.118 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:34.144 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:34.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:34.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:34.250 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:34.280 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:34.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:34.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:34.384 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:34.392 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:34 smithi098 bash[79516]: cluster 2023-09-23T18:53:34.197695+0000 mgr.y (mgr.26054) 1068 : cluster 0 pgmap v1144: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.3 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 222 op/s; 49202/62187 objects degraded (79.119%); 4766/62187 objects misplaced (7.664%); 2.6 MiB/s, 95 objects/s recovering 2023-09-23T18:53:34.392 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:34 smithi098 bash[76817]: cluster 2023-09-23T18:53:34.197695+0000 mgr.y (mgr.26054) 1068 : cluster 0 pgmap v1144: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.3 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 222 op/s; 49202/62187 objects degraded (79.119%); 4766/62187 objects misplaced (7.664%); 2.6 MiB/s, 95 objects/s recovering 2023-09-23T18:53:34.412 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:34.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:34.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:34.515 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:34.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:34 smithi173 bash[59422]: cluster 2023-09-23T18:53:34.197695+0000 mgr.y (mgr.26054) 1068 : cluster 0 pgmap v1144: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.3 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 222 op/s; 49202/62187 objects degraded (79.119%); 4766/62187 objects misplaced (7.664%); 2.6 MiB/s, 95 objects/s recovering 2023-09-23T18:53:34.546 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:34.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:34.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:34.655 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:34.679 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:34.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:34.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:34.782 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:34.814 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:34.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:34.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:34.917 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:34.947 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:34.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:34.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:35.050 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:35.081 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:35.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:35.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:35.184 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:35.218 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:35.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:35.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:35.320 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:35.349 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:35.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:35.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:35.456 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:35.482 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:35.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:35.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:35.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:35 smithi173 bash[59422]: cluster 2023-09-23T18:53:35.207407+0000 mon.a (mon.0) 1451 : cluster 3 Health check update: Degraded data redundancy: 49202/62187 objects degraded (79.119%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:35.585 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:35.594 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:35 smithi098 bash[79516]: cluster 2023-09-23T18:53:35.207407+0000 mon.a (mon.0) 1451 : cluster 3 Health check update: Degraded data redundancy: 49202/62187 objects degraded (79.119%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:35.594 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:35 smithi098 bash[76817]: cluster 2023-09-23T18:53:35.207407+0000 mon.a (mon.0) 1451 : cluster 3 Health check update: Degraded data redundancy: 49202/62187 objects degraded (79.119%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:35.615 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:35.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:35.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:35.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:35.746 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:35.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:35.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:35.850 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:35.880 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:35.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:35.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:35.984 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:36.013 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:36.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:36.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:36.123 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:36.157 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:36.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:36.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:36.264 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:36.292 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:36.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:36.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:36.396 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:36.423 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:36.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:36.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:36.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:36 smithi098 bash[76817]: cluster 2023-09-23T18:53:36.198561+0000 mgr.y (mgr.26054) 1069 : cluster 0 pgmap v1145: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.2 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 232 op/s; 48685/60417 objects degraded (80.582%); 4619/60417 objects misplaced (7.645%); 3.0 MiB/s, 83 objects/s recovering 2023-09-23T18:53:36.443 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:53:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:53:36] "GET /metrics HTTP/1.1" 200 36614 "" "Prometheus/2.33.4" 2023-09-23T18:53:36.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:36 smithi098 bash[79516]: cluster 2023-09-23T18:53:36.198561+0000 mgr.y (mgr.26054) 1069 : cluster 0 pgmap v1145: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.2 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 232 op/s; 48685/60417 objects degraded (80.582%); 4619/60417 objects misplaced (7.645%); 3.0 MiB/s, 83 objects/s recovering 2023-09-23T18:53:36.527 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:36.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:36 smithi173 bash[59422]: cluster 2023-09-23T18:53:36.198561+0000 mgr.y (mgr.26054) 1069 : cluster 0 pgmap v1145: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.2 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 232 op/s; 48685/60417 objects degraded (80.582%); 4619/60417 objects misplaced (7.645%); 3.0 MiB/s, 83 objects/s recovering 2023-09-23T18:53:36.552 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:36.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:36.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:36.655 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:36.685 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:36.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:36.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:36.794 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:36.820 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:36.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:36.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:36.924 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:36.958 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:36.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:36.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:37.061 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:37.090 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:37.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:37.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:37.194 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:37.222 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:37.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:37.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:37.325 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:37.352 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:37.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:37.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:37.456 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:37.482 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:37.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:37.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:37.585 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:37.617 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:37.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:37.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:37.721 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:37.747 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:37.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:37.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:37.852 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:37.883 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:37.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:37.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:37.988 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:38.014 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:38.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:38.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:38.117 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:38.142 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:38.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:38.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:38.246 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:38.274 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:38.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:38.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:38.328 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:53:38.379 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:38.414 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:38.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:38.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:38.518 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:38.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:38 smithi173 bash[59422]: cluster 2023-09-23T18:53:38.199377+0000 mgr.y (mgr.26054) 1070 : cluster 0 pgmap v1146: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.2 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 241 op/s; 48549/58614 objects degraded (82.828%); 4483/58614 objects misplaced (7.648%); 2.6 MiB/s, 97 objects/s recovering 2023-09-23T18:53:38.549 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:38.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:38.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:38.652 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:38.661 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:38 smithi098 bash[76817]: cluster 2023-09-23T18:53:38.199377+0000 mgr.y (mgr.26054) 1070 : cluster 0 pgmap v1146: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.2 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 241 op/s; 48549/58614 objects degraded (82.828%); 4483/58614 objects misplaced (7.648%); 2.6 MiB/s, 97 objects/s recovering 2023-09-23T18:53:38.662 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:38 smithi098 bash[79516]: cluster 2023-09-23T18:53:38.199377+0000 mgr.y (mgr.26054) 1070 : cluster 0 pgmap v1146: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.2 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 241 op/s; 48549/58614 objects degraded (82.828%); 4483/58614 objects misplaced (7.648%); 2.6 MiB/s, 97 objects/s recovering 2023-09-23T18:53:38.677 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:38.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:38.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:38.781 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:38.807 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:38.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:38.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:38.910 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:38.933 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:38.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:38.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:39.039 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:39.065 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:39.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:39.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:39.168 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:39.195 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:39.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:39.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:39.298 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:39.330 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:39.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:39.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:39.440 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:39.460 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:39.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:39.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:39.567 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:39.596 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:39.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:39.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:39.700 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:39.728 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:39.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:39.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:39.831 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:39.840 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:39 smithi098 bash[79516]: audit 2023-09-23T18:53:39.627057+0000 mon.a (mon.0) 1452 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:53:39.840 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:39 smithi098 bash[76817]: audit 2023-09-23T18:53:39.627057+0000 mon.a (mon.0) 1452 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:53:39.862 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:39.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:39.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:39.965 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:39.985 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:39.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:39.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:40.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:39 smithi173 bash[59422]: audit 2023-09-23T18:53:39.627057+0000 mon.a (mon.0) 1452 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:53:40.088 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:40.120 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:40.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:40.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:40.224 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:40.254 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:40.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:40.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:40.357 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:40.399 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:40.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:40.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:40.504 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:40.528 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:40.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:40.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:40.631 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:40.656 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:40.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:40.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:40.761 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:40.788 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:40.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:40.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:40.892 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:40.927 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:40.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:40.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:41.023 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:40 smithi098 bash[79516]: cluster 2023-09-23T18:53:40.200161+0000 mgr.y (mgr.26054) 1071 : cluster 0 pgmap v1147: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.2 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 233 op/s; 48275/57321 objects degraded (84.219%); 4335/57321 objects misplaced (7.563%); 2.7 MiB/s, 80 objects/s recovering 2023-09-23T18:53:41.023 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:40 smithi098 bash[79516]: cluster 2023-09-23T18:53:40.208476+0000 mon.a (mon.0) 1453 : cluster 3 Health check update: Degraded data redundancy: 48549/58614 objects degraded (82.828%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:41.023 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:40 smithi098 bash[76817]: cluster 2023-09-23T18:53:40.200161+0000 mgr.y (mgr.26054) 1071 : cluster 0 pgmap v1147: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.2 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 233 op/s; 48275/57321 objects degraded (84.219%); 4335/57321 objects misplaced (7.563%); 2.7 MiB/s, 80 objects/s recovering 2023-09-23T18:53:41.023 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:40 smithi098 bash[76817]: cluster 2023-09-23T18:53:40.208476+0000 mon.a (mon.0) 1453 : cluster 3 Health check update: Degraded data redundancy: 48549/58614 objects degraded (82.828%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:41.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:40 smithi173 bash[59422]: cluster 2023-09-23T18:53:40.200161+0000 mgr.y (mgr.26054) 1071 : cluster 0 pgmap v1147: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.2 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 233 op/s; 48275/57321 objects degraded (84.219%); 4335/57321 objects misplaced (7.563%); 2.7 MiB/s, 80 objects/s recovering 2023-09-23T18:53:41.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:40 smithi173 bash[59422]: cluster 2023-09-23T18:53:40.208476+0000 mon.a (mon.0) 1453 : cluster 3 Health check update: Degraded data redundancy: 48549/58614 objects degraded (82.828%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:41.030 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:41.057 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:41.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:41.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:41.163 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:41.189 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:41.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:41.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:41.294 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:41.323 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:41.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:41.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:41.427 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:41.457 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:41.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:41.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:41.560 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:41.582 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:41.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:41.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:41.687 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:41.714 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:41.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:41.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:41.819 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:41.845 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:41.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:41.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:41.949 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:41.970 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:41.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:41.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:42.075 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:42.104 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:42.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:42.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:42.210 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:42.232 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:42.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:42.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:42.335 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:42.355 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:42.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:42.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:42.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:42 smithi098 bash[79516]: cluster 2023-09-23T18:53:42.200824+0000 mgr.y (mgr.26054) 1072 : cluster 0 pgmap v1148: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.2 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 210 op/s; 48051/56877 objects degraded (84.482%); 4335/56877 objects misplaced (7.622%); 2.7 MiB/s, 80 objects/s recovering 2023-09-23T18:53:42.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:42 smithi098 bash[76817]: cluster 2023-09-23T18:53:42.200824+0000 mgr.y (mgr.26054) 1072 : cluster 0 pgmap v1148: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.2 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 210 op/s; 48051/56877 objects degraded (84.482%); 4335/56877 objects misplaced (7.622%); 2.7 MiB/s, 80 objects/s recovering 2023-09-23T18:53:42.458 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:42.484 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:42.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:42.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:42.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:42 smithi173 bash[59422]: cluster 2023-09-23T18:53:42.200824+0000 mgr.y (mgr.26054) 1072 : cluster 0 pgmap v1148: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.2 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 210 op/s; 48051/56877 objects degraded (84.482%); 4335/56877 objects misplaced (7.622%); 2.7 MiB/s, 80 objects/s recovering 2023-09-23T18:53:42.587 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:42.616 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:42.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:42.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:42.720 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:42.747 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:42.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:42.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:42.852 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:42.881 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:42.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:42.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:42.985 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:43.013 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:43.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:43.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:43.117 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:43.137 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:43.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:43.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:43.244 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:43.271 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:43.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:43.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:43.378 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:43.407 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:43.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:43.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:43.510 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:43.535 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:43.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:43.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:43.639 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:43.663 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:43.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:43.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:43.767 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:43.791 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:43.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:43.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:43.897 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:43.921 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:43.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:43.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:44.025 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:44.048 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:44.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:44.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:44.155 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:44.189 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:44.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:44.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:44.288 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:44 smithi098 bash[79516]: cluster 2023-09-23T18:53:44.201674+0000 mgr.y (mgr.26054) 1073 : cluster 0 pgmap v1149: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.1 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 245 op/s; 47896/55161 objects degraded (86.829%); 4188/55161 objects misplaced (7.592%); 2.8 MiB/s, 99 objects/s recovering 2023-09-23T18:53:44.288 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:44 smithi098 bash[76817]: cluster 2023-09-23T18:53:44.201674+0000 mgr.y (mgr.26054) 1073 : cluster 0 pgmap v1149: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.1 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 245 op/s; 47896/55161 objects degraded (86.829%); 4188/55161 objects misplaced (7.592%); 2.8 MiB/s, 99 objects/s recovering 2023-09-23T18:53:44.298 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:44.325 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:44.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:44.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:44.429 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:44.460 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:44.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:44.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:44.474 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:53:44.475 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:53:44.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:44 smithi173 bash[59422]: cluster 2023-09-23T18:53:44.201674+0000 mgr.y (mgr.26054) 1073 : cluster 0 pgmap v1149: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.1 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 245 op/s; 47896/55161 objects degraded (86.829%); 4188/55161 objects misplaced (7.592%); 2.8 MiB/s, 99 objects/s recovering 2023-09-23T18:53:44.563 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:44.590 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:44.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:44.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:44.693 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:44.730 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:44.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:44.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:44.837 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:44.862 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:44.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:44.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:44.967 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:44.998 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:45.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:45.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:45.111 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:45.141 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:45.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:45.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:45.247 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:45.272 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:45.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:45.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:45.381 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:45.391 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:45 smithi098 bash[76817]: cluster 2023-09-23T18:53:45.209404+0000 mon.a (mon.0) 1454 : cluster 3 Health check update: Degraded data redundancy: 47896/55161 objects degraded (86.829%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:45.391 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:45 smithi098 bash[79516]: cluster 2023-09-23T18:53:45.209404+0000 mon.a (mon.0) 1454 : cluster 3 Health check update: Degraded data redundancy: 47896/55161 objects degraded (86.829%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:45.406 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:45.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:45.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:45.509 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:45.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:45 smithi173 bash[59422]: cluster 2023-09-23T18:53:45.209404+0000 mon.a (mon.0) 1454 : cluster 3 Health check update: Degraded data redundancy: 47896/55161 objects degraded (86.829%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:45.540 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:45.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:45.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:45.645 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:45.673 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:45.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:45.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:45.777 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:45.801 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:45.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:45.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:45.908 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:45.943 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:45.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:45.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:46.051 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:46.082 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:46.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:46.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:46.185 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:46.214 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:46.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:46.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:46.318 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:46.328 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:46 smithi173 bash[59422]: cluster 2023-09-23T18:53:46.202747+0000 mgr.y (mgr.26054) 1074 : cluster 0 pgmap v1150: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.1 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 240 op/s; 47438/53514 objects degraded (88.646%); 4043/53514 objects misplaced (7.555%); 3.0 MiB/s, 85 objects/s recovering 2023-09-23T18:53:46.342 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:46.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:46.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:46.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:46 smithi098 bash[76817]: cluster 2023-09-23T18:53:46.202747+0000 mgr.y (mgr.26054) 1074 : cluster 0 pgmap v1150: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.1 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 240 op/s; 47438/53514 objects degraded (88.646%); 4043/53514 objects misplaced (7.555%); 3.0 MiB/s, 85 objects/s recovering 2023-09-23T18:53:46.443 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:53:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:53:46] "GET /metrics HTTP/1.1" 200 36612 "" "Prometheus/2.33.4" 2023-09-23T18:53:46.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:46 smithi098 bash[79516]: cluster 2023-09-23T18:53:46.202747+0000 mgr.y (mgr.26054) 1074 : cluster 0 pgmap v1150: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.1 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 240 op/s; 47438/53514 objects degraded (88.646%); 4043/53514 objects misplaced (7.555%); 3.0 MiB/s, 85 objects/s recovering 2023-09-23T18:53:46.447 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:46.472 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:46.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:46.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:46.575 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:46.600 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:46.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:46.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:46.708 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:46.738 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:46.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:46.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:46.842 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:46.866 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:46.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:46.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:46.969 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:46.997 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:47.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:47.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:47.101 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:47.118 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:47.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:47.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:47.221 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:47.246 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:47.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:47.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:47.351 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:47.377 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:47.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:47.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:47.481 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:47.506 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:47.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:47.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:47.610 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:47.636 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:47.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:47.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:47.739 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:47.769 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:47.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:47.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:47.872 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:47.908 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:47.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:47.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:48.016 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:48.039 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:48.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:48.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:48.142 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:48.174 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:48.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:48.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:48.278 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:48.308 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:48.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:48.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:48.410 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:48.418 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:48 smithi098 bash[79516]: cluster 2023-09-23T18:53:48.203835+0000 mgr.y (mgr.26054) 1075 : cluster 0 pgmap v1151: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.1 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 242 op/s; 47295/51675 objects degraded (91.524%); 3720/51675 objects misplaced (7.199%); 3.5 MiB/s, 93 objects/s recovering 2023-09-23T18:53:48.418 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:48 smithi098 bash[76817]: cluster 2023-09-23T18:53:48.203835+0000 mgr.y (mgr.26054) 1075 : cluster 0 pgmap v1151: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.1 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 242 op/s; 47295/51675 objects degraded (91.524%); 3720/51675 objects misplaced (7.199%); 3.5 MiB/s, 93 objects/s recovering 2023-09-23T18:53:48.433 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:48.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:48.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:48.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:48 smithi173 bash[59422]: cluster 2023-09-23T18:53:48.203835+0000 mgr.y (mgr.26054) 1075 : cluster 0 pgmap v1151: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.1 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 242 op/s; 47295/51675 objects degraded (91.524%); 3720/51675 objects misplaced (7.199%); 3.5 MiB/s, 93 objects/s recovering 2023-09-23T18:53:48.540 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:48.574 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:48.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:48.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:48.678 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:48.707 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:48.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:48.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:48.810 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:48.838 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:48.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:48.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:48.942 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:48.966 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:48.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:48.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:49.074 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:49.102 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:49.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:49.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:49.210 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:49.236 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:49.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:49.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:49.340 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:49.367 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:49.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:49.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:49.471 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:49.494 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:49.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:49.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:49.597 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:49.625 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:49.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:49.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:49.731 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:49.755 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:49.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:49.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:49.859 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:49.884 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:49.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:49.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:49.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:49 smithi098 bash[79516]: audit 2023-09-23T18:53:49.800316+0000 mon.a (mon.0) 1455 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:53:49.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:49 smithi098 bash[76817]: audit 2023-09-23T18:53:49.800316+0000 mon.a (mon.0) 1455 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:53:49.990 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:49.999 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:49 smithi173 bash[59422]: audit 2023-09-23T18:53:49.800316+0000 mon.a (mon.0) 1455 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:53:50.014 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:50.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:50.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:50.120 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:50.150 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:50.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:50.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:50.260 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:50.280 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:50.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:50.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:50.383 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:50.412 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:50.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:50.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:50.515 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:50.542 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:50.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:50.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:50.646 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:50.672 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:50.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:50.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:50.776 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:50.804 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:50.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:50.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:50.907 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:50.934 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:50.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:50.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:51.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:50 smithi173 bash[59422]: cluster 2023-09-23T18:53:50.204564+0000 mgr.y (mgr.26054) 1076 : cluster 0 pgmap v1152: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.0 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 230 op/s; 46984/50298 objects degraded (93.411%); 3578/50298 objects misplaced (7.114%); 3.6 MiB/s, 76 objects/s recovering 2023-09-23T18:53:51.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:50 smithi173 bash[59422]: cluster 2023-09-23T18:53:50.210668+0000 mon.a (mon.0) 1456 : cluster 3 Health check update: Degraded data redundancy: 47295/51675 objects degraded (91.524%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:51.040 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:51.061 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:51.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:51.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:51.164 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:51.173 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:50 smithi098 bash[76817]: cluster 2023-09-23T18:53:50.204564+0000 mgr.y (mgr.26054) 1076 : cluster 0 pgmap v1152: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.0 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 230 op/s; 46984/50298 objects degraded (93.411%); 3578/50298 objects misplaced (7.114%); 3.6 MiB/s, 76 objects/s recovering 2023-09-23T18:53:51.173 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:50 smithi098 bash[76817]: cluster 2023-09-23T18:53:50.210668+0000 mon.a (mon.0) 1456 : cluster 3 Health check update: Degraded data redundancy: 47295/51675 objects degraded (91.524%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:51.173 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:50 smithi098 bash[79516]: cluster 2023-09-23T18:53:50.204564+0000 mgr.y (mgr.26054) 1076 : cluster 0 pgmap v1152: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.0 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 230 op/s; 46984/50298 objects degraded (93.411%); 3578/50298 objects misplaced (7.114%); 3.6 MiB/s, 76 objects/s recovering 2023-09-23T18:53:51.173 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:50 smithi098 bash[79516]: cluster 2023-09-23T18:53:50.210668+0000 mon.a (mon.0) 1456 : cluster 3 Health check update: Degraded data redundancy: 47295/51675 objects degraded (91.524%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:51.220 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:51.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:51.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:51.323 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:51.349 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:51.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:51.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:51.455 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:51.481 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:51.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:51.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:51.586 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:51.609 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:51.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:51.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:51.712 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:51.741 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:51.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:51.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:51.844 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:51.870 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:51.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:51.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:51.974 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:52.011 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:52.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:52.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:52.116 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:52.141 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:52.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:52.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:52.245 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:52.274 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:52.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:52.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:52.378 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:52.387 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:52 smithi173 bash[59422]: cluster 2023-09-23T18:53:52.205535+0000 mgr.y (mgr.26054) 1077 : cluster 0 pgmap v1153: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.0 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 207 op/s; 46764/49848 objects degraded (93.813%); 3578/49848 objects misplaced (7.178%); 3.6 MiB/s, 75 objects/s recovering 2023-09-23T18:53:52.402 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:52.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:52.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:52.505 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:52.515 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:52 smithi098 bash[79516]: cluster 2023-09-23T18:53:52.205535+0000 mgr.y (mgr.26054) 1077 : cluster 0 pgmap v1153: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.0 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 207 op/s; 46764/49848 objects degraded (93.813%); 3578/49848 objects misplaced (7.178%); 3.6 MiB/s, 75 objects/s recovering 2023-09-23T18:53:52.516 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:52 smithi098 bash[76817]: cluster 2023-09-23T18:53:52.205535+0000 mgr.y (mgr.26054) 1077 : cluster 0 pgmap v1153: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 1.0 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 207 op/s; 46764/49848 objects degraded (93.813%); 3578/49848 objects misplaced (7.178%); 3.6 MiB/s, 75 objects/s recovering 2023-09-23T18:53:52.536 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:52.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:52.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:52.638 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:52.661 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:52.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:52.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:52.768 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:52.799 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:52.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:52.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:52.904 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:52.926 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:52.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:52.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:53.030 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:53.054 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:53.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:53.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:53.160 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:53.179 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:53.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:53.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:53.281 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:53.305 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:53.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:53.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:53.415 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:53.443 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:53.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:53.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:53.550 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:53.575 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:53.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:53.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:53.678 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:53.706 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:53.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:53.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:53.811 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:53.837 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:53.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:53.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:53.942 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:53.982 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:53.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:53.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:54.090 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:54.123 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:54.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:54.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:54.234 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:54.257 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:54.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:54.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:54.360 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:54.389 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:54.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:54.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:54.496 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:54.505 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:54 smithi173 bash[59422]: cluster 2023-09-23T18:53:54.206680+0000 mgr.y (mgr.26054) 1078 : cluster 0 pgmap v1154: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 998 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 252 op/s; 46601/47793 objects degraded (97.506%); 3269/47793 objects misplaced (6.840%); 4.4 MiB/s, 88 objects/s recovering 2023-09-23T18:53:54.524 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:54.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:54.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:54.631 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:54.655 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T18:53:54.655 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:54 smithi098 bash[79516]: cluster 2023-09-23T18:53:54.206680+0000 mgr.y (mgr.26054) 1078 : cluster 0 pgmap v1154: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 998 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 252 op/s; 46601/47793 objects degraded (97.506%); 3269/47793 objects misplaced (6.840%); 4.4 MiB/s, 88 objects/s recovering 2023-09-23T18:53:54.656 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:54 smithi098 bash[76817]: cluster 2023-09-23T18:53:54.206680+0000 mgr.y (mgr.26054) 1078 : cluster 0 pgmap v1154: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 998 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 252 op/s; 46601/47793 objects degraded (97.506%); 3269/47793 objects misplaced (6.840%); 4.4 MiB/s, 88 objects/s recovering 2023-09-23T18:53:54.657 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:54.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:54.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:54.760 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:54.785 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:54.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:54.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:54.888 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:54.914 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:54.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:54.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:55.017 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:55.046 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:55.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:55.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:55.149 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:55.181 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:55.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:55.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:55.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:55 smithi173 bash[59422]: audit 2023-09-23T18:53:54.627616+0000 mon.a (mon.0) 1457 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:53:55.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:55 smithi173 bash[59422]: cluster 2023-09-23T18:53:55.211583+0000 mon.a (mon.0) 1458 : cluster 3 Health check update: Degraded data redundancy: 46601/47793 objects degraded (97.506%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:55.287 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:55.311 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:55.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:55.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:55.413 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:55.435 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:55.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:55.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:55.541 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:55.553 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:55 smithi098 bash[76817]: audit 2023-09-23T18:53:54.627616+0000 mon.a (mon.0) 1457 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:53:55.553 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:55 smithi098 bash[76817]: cluster 2023-09-23T18:53:55.211583+0000 mon.a (mon.0) 1458 : cluster 3 Health check update: Degraded data redundancy: 46601/47793 objects degraded (97.506%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:55.553 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:55 smithi098 bash[79516]: audit 2023-09-23T18:53:54.627616+0000 mon.a (mon.0) 1457 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:53:55.553 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:55 smithi098 bash[79516]: cluster 2023-09-23T18:53:55.211583+0000 mon.a (mon.0) 1458 : cluster 3 Health check update: Degraded data redundancy: 46601/47793 objects degraded (97.506%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:53:55.566 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:55.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:55.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:55.669 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:55.699 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:55.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:55.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:55.802 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:55.828 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:55.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:55.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:55.931 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:55.955 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:55.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:55.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:56.061 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:56.084 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:56.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:56.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:56.190 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:56.192 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:53:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:53:56] "GET /metrics HTTP/1.1" 200 36614 "" "Prometheus/2.33.4" 2023-09-23T18:53:56.217 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:56.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:56.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:56.320 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:56.344 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:56.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:56.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:56.447 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:56.475 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:56.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:56.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:56.577 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:56.600 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:56.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:56.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:56.704 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:56.731 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:56.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:56.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:56.835 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:56.861 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:53:56.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:56.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:53:56.965 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:56.990 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:56.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:56.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:57.097 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:57.121 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:57.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:57.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:57.224 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:57.237 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:57 smithi173 bash[59422]: audit 2023-09-23T18:53:56.051860+0000 mon.a (mon.0) 1459 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:53:57.237 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:57 smithi173 bash[59422]: audit 2023-09-23T18:53:56.063172+0000 mon.a (mon.0) 1460 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:53:57.237 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:57 smithi173 bash[59422]: cluster 2023-09-23T18:53:56.208104+0000 mgr.y (mgr.26054) 1079 : cluster 0 pgmap v1155: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 952 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 266 op/s; 45973/45561 objects degraded (100.904%); 3090/45561 objects misplaced (6.782%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T18:53:57.254 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:57.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:57.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:57.357 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:57.380 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:57.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:57.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:57.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:57 smithi098 bash[76817]: audit 2023-09-23T18:53:56.051860+0000 mon.a (mon.0) 1459 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:53:57.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:57 smithi098 bash[76817]: audit 2023-09-23T18:53:56.063172+0000 mon.a (mon.0) 1460 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:53:57.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:57 smithi098 bash[76817]: cluster 2023-09-23T18:53:56.208104+0000 mgr.y (mgr.26054) 1079 : cluster 0 pgmap v1155: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 952 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 266 op/s; 45973/45561 objects degraded (100.904%); 3090/45561 objects misplaced (6.782%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T18:53:57.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:57 smithi098 bash[79516]: audit 2023-09-23T18:53:56.051860+0000 mon.a (mon.0) 1459 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:53:57.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:57 smithi098 bash[79516]: audit 2023-09-23T18:53:56.063172+0000 mon.a (mon.0) 1460 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:53:57.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:57 smithi098 bash[79516]: cluster 2023-09-23T18:53:56.208104+0000 mgr.y (mgr.26054) 1079 : cluster 0 pgmap v1155: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 952 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 266 op/s; 45973/45561 objects degraded (100.904%); 3090/45561 objects misplaced (6.782%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T18:53:57.483 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:57.507 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:57.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:57.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:57.612 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:57.640 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:57.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:57.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:57.745 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:57.773 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:57.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:57.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:57.877 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:57.914 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:57.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:57.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:58.018 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:58.039 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:53:58.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:58.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:53:58.142 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:58.173 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:58.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:58.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:58.277 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:53:58 smithi098 bash[76817]: cluster 2023-09-23T18:53:58.209176+0000 mgr.y (mgr.26054) 1080 : cluster 0 pgmap v1156: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 904 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 285 op/s; 45779/43239 objects degraded (105.874%); 2782/43239 objects misplaced (6.434%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T18:53:58.278 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:53:58 smithi098 bash[79516]: cluster 2023-09-23T18:53:58.209176+0000 mgr.y (mgr.26054) 1080 : cluster 0 pgmap v1156: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 904 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 285 op/s; 45779/43239 objects degraded (105.874%); 2782/43239 objects misplaced (6.434%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T18:53:58.282 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:58.306 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:58.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:58.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:58.410 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:58.434 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:58.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:58.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:58.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:53:58 smithi173 bash[59422]: cluster 2023-09-23T18:53:58.209176+0000 mgr.y (mgr.26054) 1080 : cluster 0 pgmap v1156: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 904 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 285 op/s; 45779/43239 objects degraded (105.874%); 2782/43239 objects misplaced (6.434%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T18:53:58.537 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:58.566 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:53:58.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:58.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:53:58.668 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:58.700 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:58.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:58.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:58.804 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:58.837 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:58.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:58.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:58.940 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:58.982 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:58.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:58.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:59.086 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:59.125 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:59.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:59.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:59.232 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:59.273 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:53:59.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:59.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:53:59.380 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:59.407 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:53:59.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:59.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:53:59.488 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:53:59.510 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:59.550 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:59.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:59.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:59.660 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:59.695 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:53:59.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:59.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:53:59.802 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:59.842 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:59.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:59.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:53:59.951 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:59.985 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:53:59.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:53:59.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:00.089 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:00.118 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:00.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:00.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:00.221 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:00.251 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:00.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:00.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:00.355 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:00.390 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:00.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:00.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:00.498 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:00.524 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:00.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:00.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:00.627 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:00.656 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:00.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:00.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:00.709 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:54:00.741 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:54:00.760 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:00.780 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:54:00.785 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:00.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:00.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:00.813 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:54:00.845 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:54:00.880 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:54:00.893 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:00.913 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:54:00.913 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:54:00.913 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:54:00.913 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:54:00.913 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:54:00.913 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-09-23T18:54:00.913 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:54:00.914 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:54:00.914 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:54:00.919 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:00.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:00.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:01.023 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:01.031 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:00 smithi098 bash[79516]: audit 2023-09-23T18:53:59.795029+0000 mon.a (mon.0) 1461 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:54:01.031 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:00 smithi098 bash[79516]: audit 2023-09-23T18:53:59.805905+0000 mon.a (mon.0) 1462 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:54:01.032 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:00 smithi098 bash[79516]: cluster 2023-09-23T18:54:00.210170+0000 mgr.y (mgr.26054) 1081 : cluster 0 pgmap v1157: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 869 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 280 op/s; 45431/41583 objects degraded (109.254%); 2605/41583 objects misplaced (6.265%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T18:54:01.032 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:00 smithi098 bash[79516]: cluster 2023-09-23T18:54:00.214694+0000 mon.a (mon.0) 1463 : cluster 3 Health check update: Degraded data redundancy: 45779/43239 objects degraded (105.874%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:01.032 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:00 smithi098 bash[79516]: audit 2023-09-23T18:54:00.445271+0000 mon.a (mon.0) 1464 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:54:01.032 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:00 smithi098 bash[79516]: audit 2023-09-23T18:54:00.447253+0000 mon.a (mon.0) 1465 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:54:01.032 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:00 smithi098 bash[79516]: audit 2023-09-23T18:54:00.460478+0000 mon.a (mon.0) 1466 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:54:01.032 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:00 smithi098 bash[79516]: audit 2023-09-23T18:54:00.673275+0000 mon.a (mon.0) 1467 : audit 0 from='client.? 172.21.15.98:0/1246178595' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:54:01.032 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:00 smithi098 bash[76817]: audit 2023-09-23T18:53:59.795029+0000 mon.a (mon.0) 1461 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:54:01.032 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:00 smithi098 bash[76817]: audit 2023-09-23T18:53:59.805905+0000 mon.a (mon.0) 1462 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:54:01.032 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:00 smithi098 bash[76817]: cluster 2023-09-23T18:54:00.210170+0000 mgr.y (mgr.26054) 1081 : cluster 0 pgmap v1157: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 869 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 280 op/s; 45431/41583 objects degraded (109.254%); 2605/41583 objects misplaced (6.265%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T18:54:01.033 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:00 smithi098 bash[76817]: cluster 2023-09-23T18:54:00.214694+0000 mon.a (mon.0) 1463 : cluster 3 Health check update: Degraded data redundancy: 45779/43239 objects degraded (105.874%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:01.033 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:00 smithi098 bash[76817]: audit 2023-09-23T18:54:00.445271+0000 mon.a (mon.0) 1464 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:54:01.033 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:00 smithi098 bash[76817]: audit 2023-09-23T18:54:00.447253+0000 mon.a (mon.0) 1465 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:54:01.033 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:00 smithi098 bash[76817]: audit 2023-09-23T18:54:00.460478+0000 mon.a (mon.0) 1466 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:54:01.033 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:00 smithi098 bash[76817]: audit 2023-09-23T18:54:00.673275+0000 mon.a (mon.0) 1467 : audit 0 from='client.? 172.21.15.98:0/1246178595' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:54:01.051 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:01.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:01.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:01.158 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:01.184 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:01.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:01.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:01.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:00 smithi173 bash[59422]: audit 2023-09-23T18:53:59.795029+0000 mon.a (mon.0) 1461 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:54:01.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:00 smithi173 bash[59422]: audit 2023-09-23T18:53:59.805905+0000 mon.a (mon.0) 1462 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:54:01.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:00 smithi173 bash[59422]: cluster 2023-09-23T18:54:00.210170+0000 mgr.y (mgr.26054) 1081 : cluster 0 pgmap v1157: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 869 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 280 op/s; 45431/41583 objects degraded (109.254%); 2605/41583 objects misplaced (6.265%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T18:54:01.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:00 smithi173 bash[59422]: cluster 2023-09-23T18:54:00.214694+0000 mon.a (mon.0) 1463 : cluster 3 Health check update: Degraded data redundancy: 45779/43239 objects degraded (105.874%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:01.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:00 smithi173 bash[59422]: audit 2023-09-23T18:54:00.445271+0000 mon.a (mon.0) 1464 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:54:01.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:00 smithi173 bash[59422]: audit 2023-09-23T18:54:00.447253+0000 mon.a (mon.0) 1465 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:54:01.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:00 smithi173 bash[59422]: audit 2023-09-23T18:54:00.460478+0000 mon.a (mon.0) 1466 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:54:01.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:00 smithi173 bash[59422]: audit 2023-09-23T18:54:00.673275+0000 mon.a (mon.0) 1467 : audit 0 from='client.? 172.21.15.98:0/1246178595' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:54:01.288 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:01.323 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:01.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:01.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:01.428 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:01.456 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:01.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:01.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:01.559 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:01.581 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:01.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:01.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:01.684 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:01.710 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:01.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:01.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:01.815 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:01.838 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:01.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:01.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:01.941 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:01.966 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:01.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:01.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:02.073 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:02.101 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:02.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:02.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:02.205 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:02.226 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:02.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:02.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:02.308 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:02 smithi098 bash[79516]: cluster 2023-09-23T18:54:02.211162+0000 mgr.y (mgr.26054) 1082 : cluster 0 pgmap v1158: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 857 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 257 op/s; 45148/41013 objects degraded (110.082%); 2605/41013 objects misplaced (6.352%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T18:54:02.308 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:02 smithi098 bash[76817]: cluster 2023-09-23T18:54:02.211162+0000 mgr.y (mgr.26054) 1082 : cluster 0 pgmap v1158: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 857 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 257 op/s; 45148/41013 objects degraded (110.082%); 2605/41013 objects misplaced (6.352%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T18:54:02.330 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:02.356 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:02.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:02.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:02.460 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:02.507 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:02.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:02.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:02.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:02 smithi173 bash[59422]: cluster 2023-09-23T18:54:02.211162+0000 mgr.y (mgr.26054) 1082 : cluster 0 pgmap v1158: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 857 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 257 op/s; 45148/41013 objects degraded (110.082%); 2605/41013 objects misplaced (6.352%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T18:54:02.611 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:02.642 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:02.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:02.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:02.754 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:02.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:02.789 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:02.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:02.890 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:02.922 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:02.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:02.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:03.028 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:03.055 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:03.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:03.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:03.161 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:03.182 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:03.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:03.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:03.272 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-09-23T18:54:03.285 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:03.298 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is set 2023-09-23T18:54:03.325 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:03.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:03.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:03.432 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:03.455 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:03.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:03.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:03 smithi173 bash[59422]: audit 2023-09-23T18:54:02.510949+0000 mon.c (mon.2) 120 : audit 1 from='client.? 172.21.15.98:0/3939180503' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:54:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:03 smithi173 bash[59422]: audit 2023-09-23T18:54:02.512335+0000 mon.a (mon.0) 1468 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:54:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:03 smithi173 bash[59422]: audit 2023-09-23T18:54:03.267193+0000 mon.a (mon.0) 1469 : audit 1 from='client.? 172.21.15.98:0/2255887997' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "41"}]: dispatch 2023-09-23T18:54:03.560 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:03.593 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:03 smithi098 bash[76817]: audit 2023-09-23T18:54:02.510949+0000 mon.c (mon.2) 120 : audit 1 from='client.? 172.21.15.98:0/3939180503' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:54:03.593 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:03 smithi098 bash[76817]: audit 2023-09-23T18:54:02.512335+0000 mon.a (mon.0) 1468 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:54:03.593 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:03 smithi098 bash[76817]: audit 2023-09-23T18:54:03.267193+0000 mon.a (mon.0) 1469 : audit 1 from='client.? 172.21.15.98:0/2255887997' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "41"}]: dispatch 2023-09-23T18:54:03.594 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:03 smithi098 bash[79516]: audit 2023-09-23T18:54:02.510949+0000 mon.c (mon.2) 120 : audit 1 from='client.? 172.21.15.98:0/3939180503' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:54:03.594 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:03 smithi098 bash[79516]: audit 2023-09-23T18:54:02.512335+0000 mon.a (mon.0) 1468 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:54:03.594 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:03 smithi098 bash[79516]: audit 2023-09-23T18:54:03.267193+0000 mon.a (mon.0) 1469 : audit 1 from='client.? 172.21.15.98:0/2255887997' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "41"}]: dispatch 2023-09-23T18:54:03.599 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:03.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:03.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:03.703 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:03.731 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:03.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:03.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:03.834 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:03.863 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:03.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:03.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:03.971 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:04.000 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:04.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:04.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:04.105 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:04.132 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:04.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:04.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:04.239 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:04.265 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:04.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:04.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:04.369 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:04.392 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:04.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:04.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:04.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:04 smithi098 bash[76817]: cluster 2023-09-23T18:54:03.274508+0000 mon.a (mon.0) 1470 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:54:04.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:04 smithi098 bash[76817]: audit 2023-09-23T18:54:03.290581+0000 mon.a (mon.0) 1471 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:54:04.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:04 smithi098 bash[76817]: cluster 2023-09-23T18:54:03.290654+0000 mon.a (mon.0) 1472 : cluster 0 osdmap e269: 8 total, 7 up, 7 in 2023-09-23T18:54:04.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:04 smithi098 bash[76817]: cluster 2023-09-23T18:54:03.300088+0000 osd.7 (osd.7) 193 : cluster 3 failed to encode map e269 with expected crc 2023-09-23T18:54:04.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:04 smithi098 bash[76817]: cluster 2023-09-23T18:54:03.315658+0000 osd.5 (osd.5) 173 : cluster 3 failed to encode map e269 with expected crc 2023-09-23T18:54:04.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:04 smithi098 bash[76817]: cluster 2023-09-23T18:54:03.317802+0000 osd.5 (osd.5) 174 : cluster 3 failed to encode map e269 with expected crc 2023-09-23T18:54:04.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:04 smithi098 bash[76817]: cluster 2023-09-23T18:54:03.338220+0000 osd.6 (osd.6) 229 : cluster 3 failed to encode map e269 with expected crc 2023-09-23T18:54:04.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:04 smithi098 bash[76817]: cluster 2023-09-23T18:54:04.212064+0000 mgr.y (mgr.26054) 1083 : cluster 0 pgmap v1160: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 812 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 297 op/s; 44958/38865 objects degraded (115.677%); 2394/38865 objects misplaced (6.160%); 4.2 MiB/s, 75 objects/s recovering 2023-09-23T18:54:04.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:04 smithi098 bash[79516]: cluster 2023-09-23T18:54:03.274508+0000 mon.a (mon.0) 1470 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:54:04.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:04 smithi098 bash[79516]: audit 2023-09-23T18:54:03.290581+0000 mon.a (mon.0) 1471 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:54:04.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:04 smithi098 bash[79516]: cluster 2023-09-23T18:54:03.290654+0000 mon.a (mon.0) 1472 : cluster 0 osdmap e269: 8 total, 7 up, 7 in 2023-09-23T18:54:04.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:04 smithi098 bash[79516]: cluster 2023-09-23T18:54:03.300088+0000 osd.7 (osd.7) 193 : cluster 3 failed to encode map e269 with expected crc 2023-09-23T18:54:04.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:04 smithi098 bash[79516]: cluster 2023-09-23T18:54:03.315658+0000 osd.5 (osd.5) 173 : cluster 3 failed to encode map e269 with expected crc 2023-09-23T18:54:04.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:04 smithi098 bash[79516]: cluster 2023-09-23T18:54:03.317802+0000 osd.5 (osd.5) 174 : cluster 3 failed to encode map e269 with expected crc 2023-09-23T18:54:04.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:04 smithi098 bash[79516]: cluster 2023-09-23T18:54:03.338220+0000 osd.6 (osd.6) 229 : cluster 3 failed to encode map e269 with expected crc 2023-09-23T18:54:04.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:04 smithi098 bash[79516]: cluster 2023-09-23T18:54:04.212064+0000 mgr.y (mgr.26054) 1083 : cluster 0 pgmap v1160: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 812 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 297 op/s; 44958/38865 objects degraded (115.677%); 2394/38865 objects misplaced (6.160%); 4.2 MiB/s, 75 objects/s recovering 2023-09-23T18:54:04.495 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:04.519 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:04.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:04.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:04.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:04 smithi173 bash[59422]: cluster 2023-09-23T18:54:03.274508+0000 mon.a (mon.0) 1470 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:54:04.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:04 smithi173 bash[59422]: audit 2023-09-23T18:54:03.290581+0000 mon.a (mon.0) 1471 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:54:04.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:04 smithi173 bash[59422]: cluster 2023-09-23T18:54:03.290654+0000 mon.a (mon.0) 1472 : cluster 0 osdmap e269: 8 total, 7 up, 7 in 2023-09-23T18:54:04.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:04 smithi173 bash[59422]: cluster 2023-09-23T18:54:03.300088+0000 osd.7 (osd.7) 193 : cluster 3 failed to encode map e269 with expected crc 2023-09-23T18:54:04.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:04 smithi173 bash[59422]: cluster 2023-09-23T18:54:03.315658+0000 osd.5 (osd.5) 173 : cluster 3 failed to encode map e269 with expected crc 2023-09-23T18:54:04.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:04 smithi173 bash[59422]: cluster 2023-09-23T18:54:03.317802+0000 osd.5 (osd.5) 174 : cluster 3 failed to encode map e269 with expected crc 2023-09-23T18:54:04.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:04 smithi173 bash[59422]: cluster 2023-09-23T18:54:03.338220+0000 osd.6 (osd.6) 229 : cluster 3 failed to encode map e269 with expected crc 2023-09-23T18:54:04.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:04 smithi173 bash[59422]: cluster 2023-09-23T18:54:04.212064+0000 mgr.y (mgr.26054) 1083 : cluster 0 pgmap v1160: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 812 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 297 op/s; 44958/38865 objects degraded (115.677%); 2394/38865 objects misplaced (6.160%); 4.2 MiB/s, 75 objects/s recovering 2023-09-23T18:54:04.623 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:04.647 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:04.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:04.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:04.750 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:04.782 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:04.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:04.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:04.890 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:04.912 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:04.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:04.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:05.015 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:05.045 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:05.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:05.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:05.151 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:05.175 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:05.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:05.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:05.279 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:05.303 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:05.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:05.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:05.406 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:05.433 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:05.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:05.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:05.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:05 smithi098 bash[76817]: cluster 2023-09-23T18:54:05.215721+0000 mon.a (mon.0) 1473 : cluster 3 Health check update: Degraded data redundancy: 44958/38865 objects degraded (115.677%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:05.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:05 smithi098 bash[79516]: cluster 2023-09-23T18:54:05.215721+0000 mon.a (mon.0) 1473 : cluster 3 Health check update: Degraded data redundancy: 44958/38865 objects degraded (115.677%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:05.537 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:05.567 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:05.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:05.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:05.671 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:05.701 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:05.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:05.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:05.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:05 smithi173 bash[59422]: cluster 2023-09-23T18:54:05.215721+0000 mon.a (mon.0) 1473 : cluster 3 Health check update: Degraded data redundancy: 44958/38865 objects degraded (115.677%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:05.807 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:05.830 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:05.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:05.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:05.933 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:05.959 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:05.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:05.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:06.064 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:06.100 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:06.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:06.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:06.202 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:06.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:06.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:06.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:06.333 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:06.342 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:06 smithi098 bash[79516]: cluster 2023-09-23T18:54:06.213401+0000 mgr.y (mgr.26054) 1084 : cluster 0 pgmap v1161: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 762 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 304 op/s; 44263/36429 objects degraded (121.505%); 2197/36429 objects misplaced (6.031%); 4.3 MiB/s, 77 objects/s recovering 2023-09-23T18:54:06.343 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:54:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:54:06] "GET /metrics HTTP/1.1" 200 36614 "" "Prometheus/2.33.4" 2023-09-23T18:54:06.343 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:06 smithi098 bash[76817]: cluster 2023-09-23T18:54:06.213401+0000 mgr.y (mgr.26054) 1084 : cluster 0 pgmap v1161: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 762 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 304 op/s; 44263/36429 objects degraded (121.505%); 2197/36429 objects misplaced (6.031%); 4.3 MiB/s, 77 objects/s recovering 2023-09-23T18:54:06.360 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:06.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:06.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:06.464 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:06.474 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:06 smithi173 bash[59422]: cluster 2023-09-23T18:54:06.213401+0000 mgr.y (mgr.26054) 1084 : cluster 0 pgmap v1161: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 762 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 304 op/s; 44263/36429 objects degraded (121.505%); 2197/36429 objects misplaced (6.031%); 4.3 MiB/s, 77 objects/s recovering 2023-09-23T18:54:06.491 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:06.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:06.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:06.594 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:06.622 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:06.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:06.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:06.725 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:06.766 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:06.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:06.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:06.872 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:06.901 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:06.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:06.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:07.005 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:07.039 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:07.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:07.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:07.145 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:07.171 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:07.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:07.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:07.275 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:07.313 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:07.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:07.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:07.418 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:07.442 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:07.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:07.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:07.545 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:07.578 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:07.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:07.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:07.684 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:07.718 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:07.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:07.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:07.823 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:07.851 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:07.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:07.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:07.954 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:07.995 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:07.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:07.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:08.099 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:08.131 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:08.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:08.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:08.236 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:08.260 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:08.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:08.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:08.329 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:08 smithi098 bash[76817]: cluster 2023-09-23T18:54:08.214524+0000 mgr.y (mgr.26054) 1085 : cluster 0 pgmap v1162: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 706 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 316 op/s; 44032/33741 objects degraded (130.500%); 2080/33741 objects misplaced (6.165%); 3.0 MiB/s, 77 objects/s recovering 2023-09-23T18:54:08.330 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:08 smithi098 bash[79516]: cluster 2023-09-23T18:54:08.214524+0000 mgr.y (mgr.26054) 1085 : cluster 0 pgmap v1162: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 706 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 316 op/s; 44032/33741 objects degraded (130.500%); 2080/33741 objects misplaced (6.165%); 3.0 MiB/s, 77 objects/s recovering 2023-09-23T18:54:08.365 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:08.393 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:08.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:08.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:08.497 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:08.532 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:08.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:08.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:08.636 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:08.645 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:08 smithi173 bash[59422]: cluster 2023-09-23T18:54:08.214524+0000 mgr.y (mgr.26054) 1085 : cluster 0 pgmap v1162: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 706 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 316 op/s; 44032/33741 objects degraded (130.500%); 2080/33741 objects misplaced (6.165%); 3.0 MiB/s, 77 objects/s recovering 2023-09-23T18:54:08.663 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:08.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:08.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:08.748 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:54:08.749 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:54:08.766 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:08.795 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-09-23T18:54:08.796 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:08.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:08.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:08.899 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:08.927 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:08.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:08.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:09.030 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:09.061 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:09.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:09.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:09.164 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:09.195 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:09.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:09.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:09.299 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:09.332 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:09.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:09.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:09.437 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:09.460 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:09.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:09.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:09.568 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:09.601 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:09.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:09.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:09.705 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:09.725 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:09.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:09.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:09.828 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:09.837 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:09 smithi173 bash[59422]: audit 2023-09-23T18:54:09.627896+0000 mon.a (mon.0) 1474 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:54:09.862 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:09.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:09.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:09.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:09 smithi098 bash[76817]: audit 2023-09-23T18:54:09.627896+0000 mon.a (mon.0) 1474 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:54:09.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:09 smithi098 bash[79516]: audit 2023-09-23T18:54:09.627896+0000 mon.a (mon.0) 1474 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:54:09.966 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:10.001 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:10.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:10.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:10.108 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:10.158 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:10.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:10.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:10.262 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:10.286 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:10.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:10.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:10.391 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:10.413 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:10.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:10.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:10.516 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:10.551 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:10.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:10.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:10.654 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:10.683 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:10.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:10.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:10.750 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:54:10.788 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:10.792 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:54:10.805 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:10 smithi098 bash[76817]: cluster 2023-09-23T18:54:10.215680+0000 mgr.y (mgr.26054) 1086 : cluster 0 pgmap v1163: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 674 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 312 op/s; 43732/32214 objects degraded (135.755%); 1965/32214 objects misplaced (6.100%); 2.5 MiB/s, 68 objects/s recovering 2023-09-23T18:54:10.805 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:10 smithi098 bash[76817]: cluster 2023-09-23T18:54:10.216971+0000 mon.a (mon.0) 1475 : cluster 3 Health check update: Degraded data redundancy: 44032/33741 objects degraded (130.500%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:10.805 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:10 smithi098 bash[79516]: cluster 2023-09-23T18:54:10.215680+0000 mgr.y (mgr.26054) 1086 : cluster 0 pgmap v1163: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 674 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 312 op/s; 43732/32214 objects degraded (135.755%); 1965/32214 objects misplaced (6.100%); 2.5 MiB/s, 68 objects/s recovering 2023-09-23T18:54:10.805 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:10 smithi098 bash[79516]: cluster 2023-09-23T18:54:10.216971+0000 mon.a (mon.0) 1475 : cluster 3 Health check update: Degraded data redundancy: 44032/33741 objects degraded (130.500%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:10.820 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:10.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:10.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:10.923 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:10.975 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:10.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:10.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:11.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:10 smithi173 bash[59422]: cluster 2023-09-23T18:54:10.215680+0000 mgr.y (mgr.26054) 1086 : cluster 0 pgmap v1163: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 674 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 312 op/s; 43732/32214 objects degraded (135.755%); 1965/32214 objects misplaced (6.100%); 2.5 MiB/s, 68 objects/s recovering 2023-09-23T18:54:11.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:10 smithi173 bash[59422]: cluster 2023-09-23T18:54:10.216971+0000 mon.a (mon.0) 1475 : cluster 3 Health check update: Degraded data redundancy: 44032/33741 objects degraded (130.500%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:11.079 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:11.108 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:11.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:11.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:11.211 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:11.235 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:11.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:11.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:11.339 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:11.377 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:11.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:11.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:11.480 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:11.505 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:11.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:11.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:11.609 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:11.639 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:11.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:11.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:11.741 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:11.775 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:11.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:11.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:11.881 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:11.903 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:11.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:11.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:12.011 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:12.038 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:12.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:12.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:12.144 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:12.167 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:12.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:12.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:12.270 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:12.281 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:12 smithi098 bash[79516]: cluster 2023-09-23T18:54:12.216493+0000 mgr.y (mgr.26054) 1087 : cluster 0 pgmap v1164: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 655 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 323 op/s; 43335/31311 objects degraded (138.402%); 1869/31311 objects misplaced (5.969%); 3.1 MiB/s, 78 objects/s recovering 2023-09-23T18:54:12.282 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:12 smithi098 bash[76817]: cluster 2023-09-23T18:54:12.216493+0000 mgr.y (mgr.26054) 1087 : cluster 0 pgmap v1164: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 655 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 323 op/s; 43335/31311 objects degraded (138.402%); 1869/31311 objects misplaced (5.969%); 3.1 MiB/s, 78 objects/s recovering 2023-09-23T18:54:12.294 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:12.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:12.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:12.398 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:12.423 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:12.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:12.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:12.526 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:12.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:12 smithi173 bash[59422]: cluster 2023-09-23T18:54:12.216493+0000 mgr.y (mgr.26054) 1087 : cluster 0 pgmap v1164: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 655 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 323 op/s; 43335/31311 objects degraded (138.402%); 1869/31311 objects misplaced (5.969%); 3.1 MiB/s, 78 objects/s recovering 2023-09-23T18:54:12.547 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:12.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:12.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:12.650 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:12.678 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:12.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:12.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:12.705 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:54:12.705 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [], 2023-09-23T18:54:12.705 INFO:teuthology.orchestra.run.smithi098.stdout: "complaint_time": 30, 2023-09-23T18:54:12.705 INFO:teuthology.orchestra.run.smithi098.stdout: "num_blocked_ops": 0 2023-09-23T18:54:12.705 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:54:12.784 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:12.812 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:12.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:12.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:12.919 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:12.943 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:12.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:12.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:13.050 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:13.084 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:13.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:13.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:13.187 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:13.211 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:13.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:13.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:13.316 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:13.351 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:13.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:13.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:13.458 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:13.489 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:13.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:13.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:13.595 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:13.619 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:13.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:13.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:13.724 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:13.750 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:13.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:13.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:13.797 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [4] live_osds: [0, 2, 3, 5, 6, 7, 1] 2023-09-23T18:54:13.797 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:54:13.797 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-09-23T18:54:13.797 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T18:54:13.797 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T18:54:13.798 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-09-23T18:54:13.853 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:13.881 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:13.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:13.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:13.984 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:14.028 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:14.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:14.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:14.138 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:14.170 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:14.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:14.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:14.273 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:14.307 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:14.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:14.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:14.362 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T18:54:14.412 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:14.447 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:14.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:14.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:14.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:14 smithi173 bash[59422]: cluster 2023-09-23T18:54:14.217404+0000 mgr.y (mgr.26054) 1088 : cluster 0 pgmap v1165: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 607 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 301 op/s; 43157/28983 objects degraded (148.905%); 1787/28983 objects misplaced (6.166%); 2.2 MiB/s, 75 objects/s recovering 2023-09-23T18:54:14.557 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:14.566 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:14 smithi098 bash[76817]: cluster 2023-09-23T18:54:14.217404+0000 mgr.y (mgr.26054) 1088 : cluster 0 pgmap v1165: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 607 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 301 op/s; 43157/28983 objects degraded (148.905%); 1787/28983 objects misplaced (6.166%); 2.2 MiB/s, 75 objects/s recovering 2023-09-23T18:54:14.566 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:14 smithi098 bash[79516]: cluster 2023-09-23T18:54:14.217404+0000 mgr.y (mgr.26054) 1088 : cluster 0 pgmap v1165: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 607 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 301 op/s; 43157/28983 objects degraded (148.905%); 1787/28983 objects misplaced (6.166%); 2.2 MiB/s, 75 objects/s recovering 2023-09-23T18:54:14.593 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:14.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:14.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:14.703 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:14.727 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:14.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:14.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:14.832 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:14.859 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:14.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:14.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:14.966 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:14.994 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:14.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:14.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:15.100 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:15.137 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:15.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:15.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:15.244 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:15.272 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:15.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:15.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:15.375 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:15.383 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:15 smithi098 bash[79516]: cluster 2023-09-23T18:54:15.218313+0000 mon.a (mon.0) 1476 : cluster 3 Health check update: Degraded data redundancy: 43157/28983 objects degraded (148.905%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:15.384 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:15 smithi098 bash[76817]: cluster 2023-09-23T18:54:15.218313+0000 mon.a (mon.0) 1476 : cluster 3 Health check update: Degraded data redundancy: 43157/28983 objects degraded (148.905%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:15.413 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:15.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:15.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:15.520 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:15.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:15 smithi173 bash[59422]: cluster 2023-09-23T18:54:15.218313+0000 mon.a (mon.0) 1476 : cluster 3 Health check update: Degraded data redundancy: 43157/28983 objects degraded (148.905%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:15.549 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:15.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:15.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:15.654 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:15.679 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:15.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:15.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:15.783 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:15.810 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:15.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:15.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:15.913 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:15.940 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:15.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:15.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:16.044 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:16.080 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:16.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:16.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:16.184 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:16.212 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:16.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:16.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:16.315 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:16.344 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:16.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:16.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:16.442 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:54:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:54:16] "GET /metrics HTTP/1.1" 200 36628 "" "Prometheus/2.33.4" 2023-09-23T18:54:16.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:16 smithi098 bash[79516]: cluster 2023-09-23T18:54:16.218384+0000 mgr.y (mgr.26054) 1089 : cluster 0 pgmap v1166: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 555 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 342 op/s; 42450/26517 objects degraded (160.086%); 1594/26517 objects misplaced (6.011%); 3.0 MiB/s, 85 objects/s recovering 2023-09-23T18:54:16.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:16 smithi098 bash[76817]: cluster 2023-09-23T18:54:16.218384+0000 mgr.y (mgr.26054) 1089 : cluster 0 pgmap v1166: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 555 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 342 op/s; 42450/26517 objects degraded (160.086%); 1594/26517 objects misplaced (6.011%); 3.0 MiB/s, 85 objects/s recovering 2023-09-23T18:54:16.449 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:16.485 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:16.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:16.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:16.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:16 smithi173 bash[59422]: cluster 2023-09-23T18:54:16.218384+0000 mgr.y (mgr.26054) 1089 : cluster 0 pgmap v1166: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 555 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 342 op/s; 42450/26517 objects degraded (160.086%); 1594/26517 objects misplaced (6.011%); 3.0 MiB/s, 85 objects/s recovering 2023-09-23T18:54:16.592 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:16.626 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:16.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:16.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:16.731 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:16.759 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:16.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:16.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:16.862 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:16.884 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:16.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:16.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:16.891 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:54:16.894 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:54:16.987 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:17.013 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:17.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:17.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:17.118 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:17.152 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:17.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:17.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:17.257 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:17.280 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:17.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:17.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:17.383 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:17.429 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:17.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:17.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:17.536 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:17.571 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:17.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:17.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:17.677 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:17.701 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:17.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:17.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:17.812 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:17.840 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:17.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:17.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:17.944 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:17.984 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:17.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:17.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:18.088 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:18.110 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:18.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:18.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:18.213 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:18.245 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:18.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:18.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:18.350 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:18.360 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:18 smithi173 bash[59422]: cluster 2023-09-23T18:54:18.220088+0000 mgr.y (mgr.26054) 1090 : cluster 0 pgmap v1167: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 506 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 341 op/s; 42255/24132 objects degraded (175.099%); 1500/24132 objects misplaced (6.216%); 2.0 MiB/s, 89 objects/s recovering 2023-09-23T18:54:18.376 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:18.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:18.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:18.486 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:18.495 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:18 smithi098 bash[79516]: cluster 2023-09-23T18:54:18.220088+0000 mgr.y (mgr.26054) 1090 : cluster 0 pgmap v1167: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 506 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 341 op/s; 42255/24132 objects degraded (175.099%); 1500/24132 objects misplaced (6.216%); 2.0 MiB/s, 89 objects/s recovering 2023-09-23T18:54:18.495 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:18 smithi098 bash[76817]: cluster 2023-09-23T18:54:18.220088+0000 mgr.y (mgr.26054) 1090 : cluster 0 pgmap v1167: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 506 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 341 op/s; 42255/24132 objects degraded (175.099%); 1500/24132 objects misplaced (6.216%); 2.0 MiB/s, 89 objects/s recovering 2023-09-23T18:54:18.513 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:18.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:18.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:18.619 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:18.642 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:18.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:18.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:18.745 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:18.781 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:18.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:18.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:18.887 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:18.913 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:18.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:18.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:18.927 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:54:18.927 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:54:19.017 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:19.056 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:19.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:19.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:19.159 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:19.187 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:19.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:19.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:19.291 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:19.323 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:19.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:19.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:19.428 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:19.462 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:19.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:19.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:19.566 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:19.590 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:19.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:19.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:19.694 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:19.724 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:19.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:19.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:19.829 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:19.855 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:19.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:19.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:19.962 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:19.991 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:19.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:19.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:20.094 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:20.125 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:20.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:20.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:20.228 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:20.257 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:20.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:20.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:20.364 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:20.387 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:20.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:20.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:20.490 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:20.499 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:20 smithi098 bash[76817]: cluster 2023-09-23T18:54:20.221364+0000 mgr.y (mgr.26054) 1091 : cluster 0 pgmap v1168: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 472 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 311 op/s; 41954/22521 objects degraded (186.288%); 1391/22521 objects misplaced (6.176%); 2.0 MiB/s, 71 objects/s recovering 2023-09-23T18:54:20.499 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:20 smithi098 bash[76817]: cluster 2023-09-23T18:54:20.221935+0000 mon.a (mon.0) 1477 : cluster 3 Health check update: Degraded data redundancy: 42255/24132 objects degraded (175.099%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:20.499 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:20 smithi098 bash[79516]: cluster 2023-09-23T18:54:20.221364+0000 mgr.y (mgr.26054) 1091 : cluster 0 pgmap v1168: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 472 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 311 op/s; 41954/22521 objects degraded (186.288%); 1391/22521 objects misplaced (6.176%); 2.0 MiB/s, 71 objects/s recovering 2023-09-23T18:54:20.499 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:20 smithi098 bash[79516]: cluster 2023-09-23T18:54:20.221935+0000 mon.a (mon.0) 1477 : cluster 3 Health check update: Degraded data redundancy: 42255/24132 objects degraded (175.099%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:20.517 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:20.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:20.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:20.622 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:20.629 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:20 smithi173 bash[59422]: cluster 2023-09-23T18:54:20.221364+0000 mgr.y (mgr.26054) 1091 : cluster 0 pgmap v1168: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 472 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 311 op/s; 41954/22521 objects degraded (186.288%); 1391/22521 objects misplaced (6.176%); 2.0 MiB/s, 71 objects/s recovering 2023-09-23T18:54:20.629 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:20 smithi173 bash[59422]: cluster 2023-09-23T18:54:20.221935+0000 mon.a (mon.0) 1477 : cluster 3 Health check update: Degraded data redundancy: 42255/24132 objects degraded (175.099%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:20.645 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:20.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:20.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:20.753 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:20.774 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:20.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:20.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:20.882 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:20.924 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:20.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:20.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:21.030 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:21.056 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:21.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:21.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:21.160 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:21.185 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:21.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:21.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:21.289 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:21.316 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:21.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:21.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:21.420 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:21.446 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:21.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:21.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:21.553 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:21.581 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:21.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:21.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:21.687 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:21.708 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:21.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:21.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:21.812 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:21.845 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:21.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:21.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:21.952 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:21.985 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:21.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:21.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:22.094 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:22.121 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:22.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:22.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:22.228 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:22.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:22.263 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:22.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:22.364 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:22.395 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:22.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:22.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:22.501 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:22.542 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:22.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:22.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:22.557 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:22 smithi098 bash[76817]: cluster 2023-09-23T18:54:22.222626+0000 mgr.y (mgr.26054) 1092 : cluster 0 pgmap v1169: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 454 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 293 op/s; 41577/21648 objects degraded (192.059%); 1286/21648 objects misplaced (5.941%); 3.0 MiB/s, 87 objects/s recovering 2023-09-23T18:54:22.558 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:22 smithi098 bash[79516]: cluster 2023-09-23T18:54:22.222626+0000 mgr.y (mgr.26054) 1092 : cluster 0 pgmap v1169: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 454 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 293 op/s; 41577/21648 objects degraded (192.059%); 1286/21648 objects misplaced (5.941%); 3.0 MiB/s, 87 objects/s recovering 2023-09-23T18:54:22.648 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:22.688 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:22.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:22.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:22.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:22 smithi173 bash[59422]: cluster 2023-09-23T18:54:22.222626+0000 mgr.y (mgr.26054) 1092 : cluster 0 pgmap v1169: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 454 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 293 op/s; 41577/21648 objects degraded (192.059%); 1286/21648 objects misplaced (5.941%); 3.0 MiB/s, 87 objects/s recovering 2023-09-23T18:54:22.792 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:22.816 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:22.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:22.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:22.919 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:22.952 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:22.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:22.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:23.056 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:23.097 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:23.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:23.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:23.204 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:23.224 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:23.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:23.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:23.327 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:23.354 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:23.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:23.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:23.457 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:23.484 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:23.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:23.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:23.589 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:23.623 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:23.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:23.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:23.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:23.757 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:23.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:23.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:23.860 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:23.892 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:23.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:23.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:23.995 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:24.022 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:24.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:24.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:24.125 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:24.152 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:24.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:24.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:24.257 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:24.285 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:24.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:24.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:24.389 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:24.397 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:24 smithi098 bash[79516]: cluster 2023-09-23T18:54:24.223681+0000 mgr.y (mgr.26054) 1093 : cluster 0 pgmap v1170: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 393 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 349 op/s; 41332/18723 objects degraded (220.755%); 1169/18723 objects misplaced (6.244%); 2.0 MiB/s, 89 objects/s recovering 2023-09-23T18:54:24.398 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:24 smithi098 bash[76817]: cluster 2023-09-23T18:54:24.223681+0000 mgr.y (mgr.26054) 1093 : cluster 0 pgmap v1170: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 393 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 349 op/s; 41332/18723 objects degraded (220.755%); 1169/18723 objects misplaced (6.244%); 2.0 MiB/s, 89 objects/s recovering 2023-09-23T18:54:24.426 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:24.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:24.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:24.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:24 smithi173 bash[59422]: cluster 2023-09-23T18:54:24.223681+0000 mgr.y (mgr.26054) 1093 : cluster 0 pgmap v1170: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 393 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 349 op/s; 41332/18723 objects degraded (220.755%); 1169/18723 objects misplaced (6.244%); 2.0 MiB/s, 89 objects/s recovering 2023-09-23T18:54:24.532 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:24.567 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:24.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:24.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:24.675 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:24.718 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:24.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:24.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:24.821 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:24.860 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:24.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:24.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:24.964 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:24.996 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:25.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:25.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:25.106 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:25.134 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:25.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:25.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:25.237 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:25.274 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:25.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:25.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:25.383 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:25.392 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:25 smithi173 bash[59422]: audit 2023-09-23T18:54:24.628071+0000 mon.a (mon.0) 1478 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:54:25.392 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:25 smithi173 bash[59422]: cluster 2023-09-23T18:54:25.223355+0000 mon.a (mon.0) 1479 : cluster 3 Health check update: Degraded data redundancy: 41332/18723 objects degraded (220.755%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:25.405 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:25.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:25.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:25.508 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:25.532 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:25.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:25.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:25.635 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:25.661 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:25 smithi098 bash[79516]: audit 2023-09-23T18:54:24.628071+0000 mon.a (mon.0) 1478 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:54:25.661 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:25 smithi098 bash[79516]: cluster 2023-09-23T18:54:25.223355+0000 mon.a (mon.0) 1479 : cluster 3 Health check update: Degraded data redundancy: 41332/18723 objects degraded (220.755%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:25.662 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:25 smithi098 bash[76817]: audit 2023-09-23T18:54:24.628071+0000 mon.a (mon.0) 1478 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:54:25.662 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:25 smithi098 bash[76817]: cluster 2023-09-23T18:54:25.223355+0000 mon.a (mon.0) 1479 : cluster 3 Health check update: Degraded data redundancy: 41332/18723 objects degraded (220.755%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:25.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:25.663 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:25.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:25.763 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:25.802 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:25.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:25.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:25.910 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:25.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:25.947 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:25.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:26.048 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:26.071 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:26.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:26.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:26.174 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:26.198 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:54:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:54:26] "GET /metrics HTTP/1.1" 200 36631 "" "Prometheus/2.33.4" 2023-09-23T18:54:26.201 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:26.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:26.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:26.305 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:26.316 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:26 smithi098 bash[76817]: cluster 2023-09-23T18:54:26.224505+0000 mgr.y (mgr.26054) 1094 : cluster 0 pgmap v1171: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 333 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 364 op/s; 40574/15843 objects degraded (256.100%); 942/15843 objects misplaced (5.946%); 3.0 MiB/s, 86 objects/s recovering 2023-09-23T18:54:26.316 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:26 smithi098 bash[79516]: cluster 2023-09-23T18:54:26.224505+0000 mgr.y (mgr.26054) 1094 : cluster 0 pgmap v1171: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 333 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 364 op/s; 40574/15843 objects degraded (256.100%); 942/15843 objects misplaced (5.946%); 3.0 MiB/s, 86 objects/s recovering 2023-09-23T18:54:26.335 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:26.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:26.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:26.440 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:26.470 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:26.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:26.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:26.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:26 smithi173 bash[59422]: cluster 2023-09-23T18:54:26.224505+0000 mgr.y (mgr.26054) 1094 : cluster 0 pgmap v1171: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 333 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 364 op/s; 40574/15843 objects degraded (256.100%); 942/15843 objects misplaced (5.946%); 3.0 MiB/s, 86 objects/s recovering 2023-09-23T18:54:26.577 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:26.609 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:26.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:26.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:26.712 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:26.741 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:26.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:26.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:26.845 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:26.869 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:26.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:26.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:26.972 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:26.999 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:27.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:27.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:27.102 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:27.127 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:27.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:27.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:27.233 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:27.268 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:27.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:27.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:27.374 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:27.407 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:27.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:27.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:27.509 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:27.534 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:27.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:27.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:27.641 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:27.678 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:27.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:27.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:27.786 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:27.806 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:27.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:27.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:27.913 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:27.941 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:27.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:27.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:28.048 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:28.099 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:28.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:28.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:28.202 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:28.227 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:28.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:28.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:28.293 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:28 smithi098 bash[76817]: cluster 2023-09-23T18:54:28.225624+0000 mgr.y (mgr.26054) 1095 : cluster 0 pgmap v1172: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 270 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 379 op/s; 40327/12834 objects degraded (314.220%); 820/12834 objects misplaced (6.389%); 2.5 MiB/s, 93 objects/s recovering 2023-09-23T18:54:28.330 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:28.358 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:28.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:28.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:28.467 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:28.497 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:28.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:28.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:28.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:28 smithi173 bash[59422]: cluster 2023-09-23T18:54:28.225624+0000 mgr.y (mgr.26054) 1095 : cluster 0 pgmap v1172: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 270 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 379 op/s; 40327/12834 objects degraded (314.220%); 820/12834 objects misplaced (6.389%); 2.5 MiB/s, 93 objects/s recovering 2023-09-23T18:54:28.607 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:28.636 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:28.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:28.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:28.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:28 smithi098 bash[79516]: cluster 2023-09-23T18:54:28.225624+0000 mgr.y (mgr.26054) 1095 : cluster 0 pgmap v1172: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 270 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 379 op/s; 40327/12834 objects degraded (314.220%); 820/12834 objects misplaced (6.389%); 2.5 MiB/s, 93 objects/s recovering 2023-09-23T18:54:28.740 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:28.771 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:28.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:28.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:28.876 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:28.902 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:28.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:28.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:29.011 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:29.037 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:29.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:29.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:29.145 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:29.174 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:29.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:29.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:29.280 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:29.308 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:29.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:29.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:29.411 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:29.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:29.443 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:29.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:29.544 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:29.567 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:29.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:29.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:29.670 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:29.706 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:29.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:29.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:29.811 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:29.839 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:29.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:29.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:29.950 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:29.978 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:29.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:29.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:30.081 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:30.111 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:30.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:30.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:30.220 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:30.256 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:30.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:30.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:30.365 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:30.412 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:30.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:30.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:30.516 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:30.524 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:30 smithi098 bash[79516]: cluster 2023-09-23T18:54:30.224692+0000 mon.a (mon.0) 1480 : cluster 3 Health check update: Degraded data redundancy: 40327/12834 objects degraded (314.220%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:30.524 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:30 smithi098 bash[79516]: cluster 2023-09-23T18:54:30.226676+0000 mgr.y (mgr.26054) 1096 : cluster 0 pgmap v1173: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 227 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 371 op/s; 39922/10743 objects degraded (371.609%); 676/10743 objects misplaced (6.292%); 2.5 MiB/s, 73 objects/s recovering 2023-09-23T18:54:30.525 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:30 smithi098 bash[76817]: cluster 2023-09-23T18:54:30.224692+0000 mon.a (mon.0) 1480 : cluster 3 Health check update: Degraded data redundancy: 40327/12834 objects degraded (314.220%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:30.525 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:30 smithi098 bash[76817]: cluster 2023-09-23T18:54:30.226676+0000 mgr.y (mgr.26054) 1096 : cluster 0 pgmap v1173: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 227 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 371 op/s; 39922/10743 objects degraded (371.609%); 676/10743 objects misplaced (6.292%); 2.5 MiB/s, 73 objects/s recovering 2023-09-23T18:54:30.545 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:30.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:30.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:30.648 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:30.680 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:30.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:30.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:30.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:30 smithi173 bash[59422]: cluster 2023-09-23T18:54:30.224692+0000 mon.a (mon.0) 1480 : cluster 3 Health check update: Degraded data redundancy: 40327/12834 objects degraded (314.220%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:30.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:30 smithi173 bash[59422]: cluster 2023-09-23T18:54:30.226676+0000 mgr.y (mgr.26054) 1096 : cluster 0 pgmap v1173: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 227 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 371 op/s; 39922/10743 objects degraded (371.609%); 676/10743 objects misplaced (6.292%); 2.5 MiB/s, 73 objects/s recovering 2023-09-23T18:54:30.784 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:30.811 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:30.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:30.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:30.918 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:30.945 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:30.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:30.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:31.050 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:31.077 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:31.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:31.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:31.179 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:31.221 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:31.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:31.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:31.323 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:31.353 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:31.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:31.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:31.460 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:31.500 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:31.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:31.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:31.603 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:31.634 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:31.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:31.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:31.743 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:31.766 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:31.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:31.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:31.870 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:31.897 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:31.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:31.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:32.002 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:32.029 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:32.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:32.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:32.134 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:32.166 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:32.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:32.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:32.272 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:32.301 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:32.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:32.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:32.404 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:32.437 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:32.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:32.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:32.542 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:32.550 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:32 smithi173 bash[59422]: cluster 2023-09-23T18:54:32.227732+0000 mgr.y (mgr.26054) 1097 : cluster 0 pgmap v1174: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 203 MiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 358 op/s; 39695/9615 objects degraded (412.845%); 562/9615 objects misplaced (5.845%); 3.1 MiB/s, 87 objects/s recovering 2023-09-23T18:54:32.564 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:32.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:32.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:32.667 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:32.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:32 smithi098 bash[79516]: cluster 2023-09-23T18:54:32.227732+0000 mgr.y (mgr.26054) 1097 : cluster 0 pgmap v1174: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 203 MiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 358 op/s; 39695/9615 objects degraded (412.845%); 562/9615 objects misplaced (5.845%); 3.1 MiB/s, 87 objects/s recovering 2023-09-23T18:54:32.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:32 smithi098 bash[76817]: cluster 2023-09-23T18:54:32.227732+0000 mgr.y (mgr.26054) 1097 : cluster 0 pgmap v1174: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 203 MiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 358 op/s; 39695/9615 objects degraded (412.845%); 562/9615 objects misplaced (5.845%); 3.1 MiB/s, 87 objects/s recovering 2023-09-23T18:54:32.693 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:32.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:32.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:32.796 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:32.828 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:32.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:32.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:32.936 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:32.961 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:32.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:32.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:33.064 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:33.092 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:33.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:33.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:33.195 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:33.227 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:33.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:33.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:33.333 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:33.360 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:33.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:33.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:33.463 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:33.496 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:33.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:33.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:33.601 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:33.625 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:33.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:33.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:33.730 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:33.753 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:33.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:33.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:33.857 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:33.885 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:33.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:33.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:33.988 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:34.031 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:34.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:34.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:34.137 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:34.178 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:34.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:34.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:34.285 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:34.309 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:34.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:34.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:34.412 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:34.441 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:34.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:34.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:34.544 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:34.557 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:34 smithi173 bash[59422]: cluster 2023-09-23T18:54:34.228874+0000 mgr.y (mgr.26054) 1098 : cluster 0 pgmap v1175: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 138 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 421 op/s; 39392/6468 objects degraded (609.029%); 438/6468 objects misplaced (6.772%); 2.9 MiB/s, 100 objects/s recovering 2023-09-23T18:54:34.576 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:34.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:34.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:34.679 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:34.702 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:34.703 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:34 smithi098 bash[76817]: cluster 2023-09-23T18:54:34.228874+0000 mgr.y (mgr.26054) 1098 : cluster 0 pgmap v1175: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 138 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 421 op/s; 39392/6468 objects degraded (609.029%); 438/6468 objects misplaced (6.772%); 2.9 MiB/s, 100 objects/s recovering 2023-09-23T18:54:34.704 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:34 smithi098 bash[79516]: cluster 2023-09-23T18:54:34.228874+0000 mgr.y (mgr.26054) 1098 : cluster 0 pgmap v1175: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 138 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 421 op/s; 39392/6468 objects degraded (609.029%); 438/6468 objects misplaced (6.772%); 2.9 MiB/s, 100 objects/s recovering 2023-09-23T18:54:34.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:34.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:34.805 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:34.829 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:34.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:34.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:34.932 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:34.959 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:34.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:34.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:35.065 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:35.100 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:35.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:35.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:35.203 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:35.239 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:35.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:35.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:35.346 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:35.355 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:35 smithi098 bash[76817]: cluster 2023-09-23T18:54:35.226181+0000 mon.a (mon.0) 1481 : cluster 3 Health check update: Degraded data redundancy: 39392/6468 objects degraded (609.029%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:35.355 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:35 smithi098 bash[79516]: cluster 2023-09-23T18:54:35.226181+0000 mon.a (mon.0) 1481 : cluster 3 Health check update: Degraded data redundancy: 39392/6468 objects degraded (609.029%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:35.375 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:35.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:35.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:35.479 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:35.506 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:35.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:35.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:35.612 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:35.657 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:35.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:35.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:35.761 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:35.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:35 smithi173 bash[59422]: cluster 2023-09-23T18:54:35.226181+0000 mon.a (mon.0) 1481 : cluster 3 Health check update: Degraded data redundancy: 39392/6468 objects degraded (609.029%), 23 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:35.780 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:35.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:35.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:35.884 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:35.908 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:35.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:35.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:36.012 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:36.036 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:36.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:36.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:36.141 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:36.174 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:36.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:36.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:36.279 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:36.299 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:36.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:36.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:36.403 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:36.408 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Removed 103475 objects 2023-09-23T18:54:36.408 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Clean up completed and total clean up time :291.808 2023-09-23T18:54:36.414 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:54:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:54:36] "GET /metrics HTTP/1.1" 200 36631 "" "Prometheus/2.33.4" 2023-09-23T18:54:36.414 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:36 smithi098 bash[79516]: cluster 2023-09-23T18:54:36.230555+0000 mgr.y (mgr.26054) 1099 : cluster 0 pgmap v1176: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 68 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 432 op/s; 38715/3147 objects degraded (1230.219%); 166/3147 objects misplaced (5.275%); 3.7 MiB/s, 97 objects/s recovering 2023-09-23T18:54:36.414 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:36 smithi098 bash[76817]: cluster 2023-09-23T18:54:36.230555+0000 mgr.y (mgr.26054) 1099 : cluster 0 pgmap v1176: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 68 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 432 op/s; 38715/3147 objects degraded (1230.219%); 166/3147 objects misplaced (5.275%); 3.7 MiB/s, 97 objects/s recovering 2023-09-23T18:54:36.428 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:36.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:36.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:36.532 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:36.554 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:36.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:36.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:36.657 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:36.676 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:36.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:36.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:36.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:36 smithi173 bash[59422]: cluster 2023-09-23T18:54:36.230555+0000 mgr.y (mgr.26054) 1099 : cluster 0 pgmap v1176: 89 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 68 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 432 op/s; 38715/3147 objects degraded (1230.219%); 166/3147 objects misplaced (5.275%); 3.7 MiB/s, 97 objects/s recovering 2023-09-23T18:54:36.779 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:36.799 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:36.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:36.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:36.902 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:36.921 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:36.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:36.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:37.024 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:37.045 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:37.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:37.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:37.148 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:37.170 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:37.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:37.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:37.272 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:37.301 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:37.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:37.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:37.403 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-09-23T18:54:37.404 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-09-23T18:54:37.404 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:37.438 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:37.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:37.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:37.542 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:37.569 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:37.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:37.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:37.673 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:37.706 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:37.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:37.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:37.809 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:37.837 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:37.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:37.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:37.940 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:37.970 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:37.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:37.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:38.074 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:38.097 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:38.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:38.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:38.201 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:38.231 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:38.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:38.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:38.335 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:38.341 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:38 smithi098 bash[79516]: cluster 2023-09-23T18:54:38.231618+0000 mgr.y (mgr.26054) 1100 : cluster 0 pgmap v1177: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 25 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 410 op/s; 38543/1044 objects degraded (3691.858%); 77/1044 objects misplaced (7.375%); 3.1 MiB/s, 116 objects/s recovering 2023-09-23T18:54:38.342 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:38 smithi098 bash[76817]: cluster 2023-09-23T18:54:38.231618+0000 mgr.y (mgr.26054) 1100 : cluster 0 pgmap v1177: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 25 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 410 op/s; 38543/1044 objects degraded (3691.858%); 77/1044 objects misplaced (7.375%); 3.1 MiB/s, 116 objects/s recovering 2023-09-23T18:54:38.344 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:54:38.357 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:38.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:38.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:38.369 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:54:38.389 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:54:38.409 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:54:38.437 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:54:38.461 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:38.464 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:54:38.486 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:38.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:38.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:38.489 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T18:54:38.489 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:54:38.489 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:54:38.489 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:54:38.489 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:54:38.489 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-09-23T18:54:38.489 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:54:38.489 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:54:38.489 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:54:38.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:38 smithi173 bash[59422]: cluster 2023-09-23T18:54:38.231618+0000 mgr.y (mgr.26054) 1100 : cluster 0 pgmap v1177: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 25 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 410 op/s; 38543/1044 objects degraded (3691.858%); 77/1044 objects misplaced (7.375%); 3.1 MiB/s, 116 objects/s recovering 2023-09-23T18:54:38.590 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:38.619 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:38.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:38.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:38.723 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:38.748 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:38.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:38.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:38.852 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:38.878 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:38.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:38.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:38.982 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:39.010 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:39.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:39.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:39.115 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:39.143 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:39.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:39.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:39.246 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:39.267 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:39.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:39.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:39.371 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:39.382 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:39 smithi173 bash[59422]: audit 2023-09-23T18:54:38.312561+0000 mon.a (mon.0) 1482 : audit 0 from='client.? 172.21.15.98:0/313641204' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:54:39.400 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:39.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:39.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:39.504 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:39.511 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:39 smithi098 bash[76817]: audit 2023-09-23T18:54:38.312561+0000 mon.a (mon.0) 1482 : audit 0 from='client.? 172.21.15.98:0/313641204' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:54:39.511 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:39 smithi098 bash[79516]: audit 2023-09-23T18:54:38.312561+0000 mon.a (mon.0) 1482 : audit 0 from='client.? 172.21.15.98:0/313641204' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:54:39.524 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:39.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:39.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:39.627 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:39.648 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:39.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:39.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:39.751 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:39.780 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:39.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:39.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:39.884 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:39.907 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:39.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:39.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:40.010 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:40.041 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:40.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:40.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:40.145 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:40.179 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:40.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:40.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:40.283 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:40.307 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:40.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:40.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:40.411 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:40.435 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:40.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:40.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:40.538 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:40.551 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:40 smithi173 bash[59422]: audit 2023-09-23T18:54:39.628549+0000 mon.a (mon.0) 1483 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:54:40.551 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:40 smithi173 bash[59422]: cluster 2023-09-23T18:54:40.227362+0000 mon.a (mon.0) 1484 : cluster 3 Health check update: Degraded data redundancy: 38543/1044 objects degraded (3691.858%), 22 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:40.551 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:40 smithi173 bash[59422]: cluster 2023-09-23T18:54:40.233157+0000 mgr.y (mgr.26054) 1101 : cluster 0 pgmap v1178: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 9.3 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 347 op/s; 38389/312 objects degraded (12304.167%); 30/312 objects misplaced (9.615%); 2.5 MiB/s, 93 objects/s recovering 2023-09-23T18:54:40.567 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:40.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:40.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:40.671 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:40.700 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:40.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:40.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:40.739 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:40 smithi098 bash[79516]: audit 2023-09-23T18:54:39.628549+0000 mon.a (mon.0) 1483 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:54:40.739 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:40 smithi098 bash[79516]: cluster 2023-09-23T18:54:40.227362+0000 mon.a (mon.0) 1484 : cluster 3 Health check update: Degraded data redundancy: 38543/1044 objects degraded (3691.858%), 22 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:40.739 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:40 smithi098 bash[79516]: cluster 2023-09-23T18:54:40.233157+0000 mgr.y (mgr.26054) 1101 : cluster 0 pgmap v1178: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 9.3 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 347 op/s; 38389/312 objects degraded (12304.167%); 30/312 objects misplaced (9.615%); 2.5 MiB/s, 93 objects/s recovering 2023-09-23T18:54:40.740 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:40 smithi098 bash[76817]: audit 2023-09-23T18:54:39.628549+0000 mon.a (mon.0) 1483 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:54:40.740 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:40 smithi098 bash[76817]: cluster 2023-09-23T18:54:40.227362+0000 mon.a (mon.0) 1484 : cluster 3 Health check update: Degraded data redundancy: 38543/1044 objects degraded (3691.858%), 22 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:40.740 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:40 smithi098 bash[76817]: cluster 2023-09-23T18:54:40.233157+0000 mgr.y (mgr.26054) 1101 : cluster 0 pgmap v1178: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+degraded+remapped, 38 active+clean; 9.3 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 347 op/s; 38389/312 objects degraded (12304.167%); 30/312 objects misplaced (9.615%); 2.5 MiB/s, 93 objects/s recovering 2023-09-23T18:54:40.803 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:40.824 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:40.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:40.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:40.927 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:40.957 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:40.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:40.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:41.061 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:41.086 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:41.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:41.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:41.190 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:41.215 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:41.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:41.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:41.318 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:41.342 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:41.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:41.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:41.445 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:41.473 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:41.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:41.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:41.578 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:41.604 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:41.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:41.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:41.707 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:41.733 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:41.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:41.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:41.836 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:41.866 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:41.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:41.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:41.969 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:41.994 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:41.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:41.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:42.097 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:42.124 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:42.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:42.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:42.227 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:42.255 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:42.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:42.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:42.266 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:54:42.267 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:54:42.359 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:42.386 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:42.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:42.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:42.490 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:42.502 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:42 smithi173 bash[59422]: cluster 2023-09-23T18:54:42.234268+0000 mgr.y (mgr.26054) 1102 : cluster 0 pgmap v1179: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 38 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 295 op/s; 38337/84 objects degraded (45639.286%); 2.7 MiB/s, 115 objects/s recovering 2023-09-23T18:54:42.520 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:42.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:42.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:42.583 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:42 smithi098 bash[76817]: cluster 2023-09-23T18:54:42.234268+0000 mgr.y (mgr.26054) 1102 : cluster 0 pgmap v1179: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 38 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 295 op/s; 38337/84 objects degraded (45639.286%); 2.7 MiB/s, 115 objects/s recovering 2023-09-23T18:54:42.583 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:42 smithi098 bash[79516]: cluster 2023-09-23T18:54:42.234268+0000 mgr.y (mgr.26054) 1102 : cluster 0 pgmap v1179: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 38 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 295 op/s; 38337/84 objects degraded (45639.286%); 2.7 MiB/s, 115 objects/s recovering 2023-09-23T18:54:42.625 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:42.649 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:42.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:42.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:42.753 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:42.779 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:42.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:42.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:42.806 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:54:42.806 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:54:42.882 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:42.903 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:42.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:42.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:42.912 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:54:42.912 INFO:teuthology.orchestra.run.smithi098.stdout: "success": "" 2023-09-23T18:54:42.912 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:54:43.007 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:43.032 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:43.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:43.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:43.135 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:43.161 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:43.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:43.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:43.264 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:43.294 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:43.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:43.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:43.314 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is unset 2023-09-23T18:54:43.398 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:43.417 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:43.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:43.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:43.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:43 smithi098 bash[76817]: audit 2023-09-23T18:54:42.700760+0000 mon.c (mon.2) 121 : audit 1 from='client.? 172.21.15.98:0/2812125821' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:54:43.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:43 smithi098 bash[76817]: audit 2023-09-23T18:54:42.701160+0000 mon.a (mon.0) 1485 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:54:43.520 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:43.548 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:43.549 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:43 smithi173 bash[59422]: audit 2023-09-23T18:54:42.700760+0000 mon.c (mon.2) 121 : audit 1 from='client.? 172.21.15.98:0/2812125821' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:54:43.549 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:43 smithi173 bash[59422]: audit 2023-09-23T18:54:42.701160+0000 mon.a (mon.0) 1485 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:54:43.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:43.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:43.650 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:43.678 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:43.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:43.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:43.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:43 smithi098 bash[79516]: audit 2023-09-23T18:54:42.700760+0000 mon.c (mon.2) 121 : audit 1 from='client.? 172.21.15.98:0/2812125821' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:54:43.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:43 smithi098 bash[79516]: audit 2023-09-23T18:54:42.701160+0000 mon.a (mon.0) 1485 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:54:43.781 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:43.813 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:43.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:43.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:43.916 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:43.946 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:43.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:43.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:44.050 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:44.077 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:44.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:44.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:44.181 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:44.206 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:44.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:44.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:44.268 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:54:44.309 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:44.334 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:44.336 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:44 smithi173 bash[59422]: cluster 2023-09-23T18:54:43.293598+0000 mon.a (mon.0) 1486 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:54:44.336 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:44 smithi173 bash[59422]: cluster 2023-09-23T18:54:43.304554+0000 osd.7 (osd.7) 194 : cluster 3 failed to encode map e270 with expected crc 2023-09-23T18:54:44.336 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:44 smithi173 bash[59422]: audit 2023-09-23T18:54:43.305418+0000 mon.a (mon.0) 1487 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:54:44.336 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:44 smithi173 bash[59422]: cluster 2023-09-23T18:54:43.305691+0000 mon.a (mon.0) 1488 : cluster 0 osdmap e270: 8 total, 7 up, 7 in 2023-09-23T18:54:44.336 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:44 smithi173 bash[59422]: cluster 2023-09-23T18:54:43.310602+0000 osd.6 (osd.6) 230 : cluster 3 failed to encode map e270 with expected crc 2023-09-23T18:54:44.336 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:44 smithi173 bash[59422]: cluster 2023-09-23T18:54:44.235068+0000 mgr.y (mgr.26054) 1103 : cluster 0 pgmap v1181: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 38 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 212 op/s; 38337/84 objects degraded (45639.286%); 1.6 MiB/s, 190 objects/s recovering 2023-09-23T18:54:44.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:44.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:44.438 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:44.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:44 smithi098 bash[79516]: cluster 2023-09-23T18:54:43.293598+0000 mon.a (mon.0) 1486 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:54:44.442 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:44 smithi098 bash[79516]: cluster 2023-09-23T18:54:43.304554+0000 osd.7 (osd.7) 194 : cluster 3 failed to encode map e270 with expected crc 2023-09-23T18:54:44.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:44 smithi098 bash[79516]: audit 2023-09-23T18:54:43.305418+0000 mon.a (mon.0) 1487 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:54:44.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:44 smithi098 bash[79516]: cluster 2023-09-23T18:54:43.305691+0000 mon.a (mon.0) 1488 : cluster 0 osdmap e270: 8 total, 7 up, 7 in 2023-09-23T18:54:44.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:44 smithi098 bash[79516]: cluster 2023-09-23T18:54:43.310602+0000 osd.6 (osd.6) 230 : cluster 3 failed to encode map e270 with expected crc 2023-09-23T18:54:44.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:44 smithi098 bash[79516]: cluster 2023-09-23T18:54:44.235068+0000 mgr.y (mgr.26054) 1103 : cluster 0 pgmap v1181: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 38 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 212 op/s; 38337/84 objects degraded (45639.286%); 1.6 MiB/s, 190 objects/s recovering 2023-09-23T18:54:44.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:44 smithi098 bash[76817]: cluster 2023-09-23T18:54:43.293598+0000 mon.a (mon.0) 1486 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:54:44.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:44 smithi098 bash[76817]: cluster 2023-09-23T18:54:43.304554+0000 osd.7 (osd.7) 194 : cluster 3 failed to encode map e270 with expected crc 2023-09-23T18:54:44.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:44 smithi098 bash[76817]: audit 2023-09-23T18:54:43.305418+0000 mon.a (mon.0) 1487 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:54:44.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:44 smithi098 bash[76817]: cluster 2023-09-23T18:54:43.305691+0000 mon.a (mon.0) 1488 : cluster 0 osdmap e270: 8 total, 7 up, 7 in 2023-09-23T18:54:44.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:44 smithi098 bash[76817]: cluster 2023-09-23T18:54:43.310602+0000 osd.6 (osd.6) 230 : cluster 3 failed to encode map e270 with expected crc 2023-09-23T18:54:44.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:44 smithi098 bash[76817]: cluster 2023-09-23T18:54:44.235068+0000 mgr.y (mgr.26054) 1103 : cluster 0 pgmap v1181: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 38 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 212 op/s; 38337/84 objects degraded (45639.286%); 1.6 MiB/s, 190 objects/s recovering 2023-09-23T18:54:44.465 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:44.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:44.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:44.569 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:44.596 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:44.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:44.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:44.701 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:44.727 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:44.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:44.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:44.830 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:44.859 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:44.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:44.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:44.963 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:44.990 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:44.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:44.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:45.094 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:45.118 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:45.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:45.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:45.222 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:45.248 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:45.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:45.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:45.351 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:45.364 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:45 smithi173 bash[59422]: cluster 2023-09-23T18:54:45.228386+0000 mon.a (mon.0) 1489 : cluster 3 Health check update: Degraded data redundancy: 38337/84 objects degraded (45639.286%), 19 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:45.383 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:45.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:45.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:45.394 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T18:54:45.490 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:45.519 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:45.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:45.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:45.624 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:45.637 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:45 smithi098 bash[79516]: cluster 2023-09-23T18:54:45.228386+0000 mon.a (mon.0) 1489 : cluster 3 Health check update: Degraded data redundancy: 38337/84 objects degraded (45639.286%), 19 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:45.637 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:45 smithi098 bash[76817]: cluster 2023-09-23T18:54:45.228386+0000 mon.a (mon.0) 1489 : cluster 3 Health check update: Degraded data redundancy: 38337/84 objects degraded (45639.286%), 19 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:45.658 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:45.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:45.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:45.761 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:45.787 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:45.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:45.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:45.890 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:45.924 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:45.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:45.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:46.028 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:46.059 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:46.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:46.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:46.163 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:46.175 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:54:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:54:46] "GET /metrics HTTP/1.1" 200 36623 "" "Prometheus/2.33.4" 2023-09-23T18:54:46.194 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:46.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:46.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:46.298 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:46.326 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:46.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:46.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:46.430 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:46.457 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:46.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:46.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:46.561 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:46.584 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:46.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:46.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:46.688 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:46.715 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:46.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:46.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:46.819 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:46.844 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:46.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:46.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:46.902 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:54:46.946 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:46.971 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:46.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:46.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:47.075 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:47.099 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:47.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:47.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:47.202 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:47.231 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:47.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:47.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:47.335 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:47.365 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:47.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:47.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:47.408 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:54:47.417 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:47 smithi098 bash[79516]: cluster 2023-09-23T18:54:46.236798+0000 mgr.y (mgr.26054) 1104 : cluster 0 pgmap v1182: 89 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+clean+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 29 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 38 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 102 op/s; 31265/84 objects degraded (37220.238%); 678 KiB/s, 206 objects/s recovering 2023-09-23T18:54:47.417 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:47 smithi098 bash[76817]: cluster 2023-09-23T18:54:46.236798+0000 mgr.y (mgr.26054) 1104 : cluster 0 pgmap v1182: 89 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+clean+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 29 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 38 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 102 op/s; 31265/84 objects degraded (37220.238%); 678 KiB/s, 206 objects/s recovering 2023-09-23T18:54:47.468 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:47.480 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:47 smithi173 bash[59422]: cluster 2023-09-23T18:54:46.236798+0000 mgr.y (mgr.26054) 1104 : cluster 0 pgmap v1182: 89 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+clean+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 29 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 38 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 102 op/s; 31265/84 objects degraded (37220.238%); 678 KiB/s, 206 objects/s recovering 2023-09-23T18:54:47.494 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:47.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:47.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:47.597 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:47.618 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:47.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:47.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:47.722 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:47.747 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:47.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:47.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:47.850 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:47.876 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:47.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:47.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:47.981 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:48.006 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:48.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:48.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:48.109 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:48.138 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:48.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:48.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:48.241 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:48.270 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:48.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:48.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:48.374 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:48.400 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:48.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:48.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:48.503 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:48.533 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:48.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:48.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:48.637 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:48.661 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:48.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:48.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:48.765 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:48.791 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:48.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:48.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:48.895 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:48.920 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:48.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:48.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:49.023 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:49.049 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:49.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:49.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:49.152 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:49.179 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:49.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:49.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:49.282 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:49.307 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:49.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:49.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:49.410 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:49.423 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:49 smithi173 bash[59422]: cluster 2023-09-23T18:54:48.237688+0000 mgr.y (mgr.26054) 1105 : cluster 0 pgmap v1183: 89 pgs: 1 active+clean+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 29 active+undersized, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 38 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 31 op/s; 31265/84 objects degraded (37220.238%); 192 KiB/s, 163 objects/s recovering 2023-09-23T18:54:49.435 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:54:49.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:49.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:54:49.539 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:49.550 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:49 smithi098 bash[79516]: cluster 2023-09-23T18:54:48.237688+0000 mgr.y (mgr.26054) 1105 : cluster 0 pgmap v1183: 89 pgs: 1 active+clean+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 29 active+undersized, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 38 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 31 op/s; 31265/84 objects degraded (37220.238%); 192 KiB/s, 163 objects/s recovering 2023-09-23T18:54:49.550 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:49 smithi098 bash[76817]: cluster 2023-09-23T18:54:48.237688+0000 mgr.y (mgr.26054) 1105 : cluster 0 pgmap v1183: 89 pgs: 1 active+clean+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 29 active+undersized, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 38 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 31 op/s; 31265/84 objects degraded (37220.238%); 192 KiB/s, 163 objects/s recovering 2023-09-23T18:54:49.564 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:49.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:49.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:49.667 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:49.693 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:49.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:49.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:49.796 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:49.820 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:49.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:49.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:49.923 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:49.951 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:49.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:49.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:50.055 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:50.077 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:50.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:50.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:50.132 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:54:50.132 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:54:50.185 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:50.211 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:50.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:50.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:50.314 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:50.339 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:50.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:50.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:50.415 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [4] live_osds: [0, 2, 3, 5, 6, 7, 1] 2023-09-23T18:54:50.415 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:54:50.416 INFO:tasks.thrashosds.thrasher:Killing osd 7, live_osds are [0, 2, 3, 5, 6, 7, 1] 2023-09-23T18:54:50.416 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-09-23T18:54:50.416 DEBUG:teuthology.orchestra.run.smithi173:> sudo systemctl stop ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.7 2023-09-23T18:54:50.442 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:50 smithi098 bash[76817]: cluster 2023-09-23T18:54:50.229947+0000 mon.a (mon.0) 1490 : cluster 3 Health check update: Degraded data redundancy: 31265/84 objects degraded (37220.238%), 16 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:50.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:50 smithi098 bash[79516]: cluster 2023-09-23T18:54:50.229947+0000 mon.a (mon.0) 1490 : cluster 3 Health check update: Degraded data redundancy: 31265/84 objects degraded (37220.238%), 16 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:50.443 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:50.468 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:50.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:50.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:50.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:50 smithi173 bash[59422]: cluster 2023-09-23T18:54:50.229947+0000 mon.a (mon.0) 1490 : cluster 3 Health check update: Degraded data redundancy: 31265/84 objects degraded (37220.238%), 16 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:50.528 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 18:54:50 smithi173 systemd[1]: Stopping Ceph osd.7 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T18:54:50.571 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:50.599 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:50.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:50.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:50.702 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:50.731 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:50.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:50.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:50.834 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:50.860 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:50.861 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 18:54:50 smithi173 bash[43670]: debug 2023-09-23T18:54:50.601+0000 7fcd1b406700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T18:54:50.861 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 18:54:50 smithi173 bash[43670]: debug 2023-09-23T18:54:50.601+0000 7fcd1b406700 -1 osd.7 270 *** Got signal Terminated *** 2023-09-23T18:54:50.861 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 18:54:50 smithi173 bash[43670]: debug 2023-09-23T18:54:50.601+0000 7fcd1b406700 -1 osd.7 270 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T18:54:50.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:50.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:50.963 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:50.992 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:50.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:50.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:51.095 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:51.118 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:51.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:51.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:51.222 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:51.248 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:51.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:51.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:51.351 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:51.377 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:51.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:51.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:51.481 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:51.504 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:51.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:51.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:51.607 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:51.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:51 smithi173 bash[59422]: cluster 2023-09-23T18:54:50.238432+0000 mgr.y (mgr.26054) 1106 : cluster 0 pgmap v1184: 89 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+clean+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 29 active+undersized, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 38 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 31265/84 objects degraded (37220.238%); 192 KiB/s, 163 objects/s recovering 2023-09-23T18:54:51.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:51 smithi173 bash[59422]: cluster 2023-09-23T18:54:50.602636+0000 mon.a (mon.0) 1491 : cluster 1 osd.7 marked itself down and dead 2023-09-23T18:54:51.641 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:51.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:51.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:51.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:51 smithi098 bash[79516]: cluster 2023-09-23T18:54:50.238432+0000 mgr.y (mgr.26054) 1106 : cluster 0 pgmap v1184: 89 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+clean+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 29 active+undersized, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 38 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 31265/84 objects degraded (37220.238%); 192 KiB/s, 163 objects/s recovering 2023-09-23T18:54:51.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:51 smithi098 bash[79516]: cluster 2023-09-23T18:54:50.602636+0000 mon.a (mon.0) 1491 : cluster 1 osd.7 marked itself down and dead 2023-09-23T18:54:51.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:51 smithi098 bash[76817]: cluster 2023-09-23T18:54:50.238432+0000 mgr.y (mgr.26054) 1106 : cluster 0 pgmap v1184: 89 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+clean+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 29 active+undersized, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 38 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 31265/84 objects degraded (37220.238%); 192 KiB/s, 163 objects/s recovering 2023-09-23T18:54:51.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:51 smithi098 bash[76817]: cluster 2023-09-23T18:54:50.602636+0000 mon.a (mon.0) 1491 : cluster 1 osd.7 marked itself down and dead 2023-09-23T18:54:51.744 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:51.773 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:51.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:51.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:51.876 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:51.897 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:51.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:51.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:51.978 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:54:52.000 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:52.029 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:52.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:52.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:52.132 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:52.155 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:52.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:52.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:52.259 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:52.287 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:52.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:52.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:52.391 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:52.401 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:52 smithi098 bash[79516]: cluster 2023-09-23T18:54:51.316076+0000 mon.a (mon.0) 1492 : cluster 3 Health check update: 2 osds down (OSD_DOWN) 2023-09-23T18:54:52.401 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:52 smithi098 bash[79516]: cluster 2023-09-23T18:54:51.330779+0000 osd.6 (osd.6) 231 : cluster 3 failed to encode map e271 with expected crc 2023-09-23T18:54:52.401 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:52 smithi098 bash[79516]: cluster 2023-09-23T18:54:51.331878+0000 mon.a (mon.0) 1493 : cluster 0 osdmap e271: 8 total, 6 up, 7 in 2023-09-23T18:54:52.401 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:52 smithi098 bash[79516]: cluster 2023-09-23T18:54:51.334913+0000 osd.6 (osd.6) 232 : cluster 3 failed to encode map e271 with expected crc 2023-09-23T18:54:52.401 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:52 smithi098 bash[79516]: cluster 2023-09-23T18:54:51.347937+0000 osd.5 (osd.5) 175 : cluster 3 failed to encode map e271 with expected crc 2023-09-23T18:54:52.401 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:52 smithi098 bash[76817]: cluster 2023-09-23T18:54:51.316076+0000 mon.a (mon.0) 1492 : cluster 3 Health check update: 2 osds down (OSD_DOWN) 2023-09-23T18:54:52.402 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:52 smithi098 bash[76817]: cluster 2023-09-23T18:54:51.330779+0000 osd.6 (osd.6) 231 : cluster 3 failed to encode map e271 with expected crc 2023-09-23T18:54:52.402 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:52 smithi098 bash[76817]: cluster 2023-09-23T18:54:51.331878+0000 mon.a (mon.0) 1493 : cluster 0 osdmap e271: 8 total, 6 up, 7 in 2023-09-23T18:54:52.402 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:52 smithi098 bash[76817]: cluster 2023-09-23T18:54:51.334913+0000 osd.6 (osd.6) 232 : cluster 3 failed to encode map e271 with expected crc 2023-09-23T18:54:52.402 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:52 smithi098 bash[76817]: cluster 2023-09-23T18:54:51.347937+0000 osd.5 (osd.5) 175 : cluster 3 failed to encode map e271 with expected crc 2023-09-23T18:54:52.423 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:52.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:52.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:52.528 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:52.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:52 smithi173 bash[59422]: cluster 2023-09-23T18:54:51.316076+0000 mon.a (mon.0) 1492 : cluster 3 Health check update: 2 osds down (OSD_DOWN) 2023-09-23T18:54:52.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:52 smithi173 bash[59422]: cluster 2023-09-23T18:54:51.330779+0000 osd.6 (osd.6) 231 : cluster 3 failed to encode map e271 with expected crc 2023-09-23T18:54:52.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:52 smithi173 bash[59422]: cluster 2023-09-23T18:54:51.331878+0000 mon.a (mon.0) 1493 : cluster 0 osdmap e271: 8 total, 6 up, 7 in 2023-09-23T18:54:52.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:52 smithi173 bash[59422]: cluster 2023-09-23T18:54:51.334913+0000 osd.6 (osd.6) 232 : cluster 3 failed to encode map e271 with expected crc 2023-09-23T18:54:52.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:52 smithi173 bash[59422]: cluster 2023-09-23T18:54:51.347937+0000 osd.5 (osd.5) 175 : cluster 3 failed to encode map e271 with expected crc 2023-09-23T18:54:52.549 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:52.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:52.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:52.653 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:52.672 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:52.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:52.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:52.775 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:52.800 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:52.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:52.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:52.903 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:52.937 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:52.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:52.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:53.041 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:53.074 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:53.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:53.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:53.178 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:53.198 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:53.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:53.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:53.304 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:53.323 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:53.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:53.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:53.427 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:53.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:53 smithi098 bash[79516]: cluster 2023-09-23T18:54:52.239141+0000 mgr.y (mgr.26054) 1107 : cluster 0 pgmap v1186: 89 pgs: 1 stale+active+recovering+undersized+degraded+remapped, 4 stale+active+undersized+degraded, 5 stale+active+undersized, 9 stale+active+clean, 1 active+undersized+remapped+backfill_wait, 1 active+clean+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 24 active+undersized, 1 active+remapped+backfill_wait, 4 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 29 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 31265/84 objects degraded (37220.238%); 0 B/s, 79 objects/s recovering 2023-09-23T18:54:53.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:53 smithi098 bash[79516]: cluster 2023-09-23T18:54:52.333597+0000 mon.a (mon.0) 1494 : cluster 0 osdmap e272: 8 total, 6 up, 7 in 2023-09-23T18:54:53.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:53 smithi098 bash[79516]: cluster 2023-09-23T18:54:52.334494+0000 osd.5 (osd.5) 176 : cluster 3 failed to encode map e272 with expected crc 2023-09-23T18:54:53.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:53 smithi098 bash[79516]: cluster 2023-09-23T18:54:52.335341+0000 osd.6 (osd.6) 233 : cluster 3 failed to encode map e272 with expected crc 2023-09-23T18:54:53.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:53 smithi098 bash[79516]: cluster 2023-09-23T18:54:52.340359+0000 osd.6 (osd.6) 234 : cluster 3 failed to encode map e272 with expected crc 2023-09-23T18:54:53.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:53 smithi098 bash[79516]: cluster 2023-09-23T18:54:52.342480+0000 osd.5 (osd.5) 177 : cluster 3 failed to encode map e272 with expected crc 2023-09-23T18:54:53.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:53 smithi098 bash[79516]: cluster 2023-09-23T18:54:52.343087+0000 osd.6 (osd.6) 235 : cluster 3 failed to encode map e272 with expected crc 2023-09-23T18:54:53.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:53 smithi098 bash[79516]: cluster 2023-09-23T18:54:52.344312+0000 osd.6 (osd.6) 236 : cluster 3 failed to encode map e272 with expected crc 2023-09-23T18:54:53.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:53 smithi098 bash[76817]: cluster 2023-09-23T18:54:52.239141+0000 mgr.y (mgr.26054) 1107 : cluster 0 pgmap v1186: 89 pgs: 1 stale+active+recovering+undersized+degraded+remapped, 4 stale+active+undersized+degraded, 5 stale+active+undersized, 9 stale+active+clean, 1 active+undersized+remapped+backfill_wait, 1 active+clean+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 24 active+undersized, 1 active+remapped+backfill_wait, 4 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 29 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 31265/84 objects degraded (37220.238%); 0 B/s, 79 objects/s recovering 2023-09-23T18:54:53.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:53 smithi098 bash[76817]: cluster 2023-09-23T18:54:52.333597+0000 mon.a (mon.0) 1494 : cluster 0 osdmap e272: 8 total, 6 up, 7 in 2023-09-23T18:54:53.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:53 smithi098 bash[76817]: cluster 2023-09-23T18:54:52.334494+0000 osd.5 (osd.5) 176 : cluster 3 failed to encode map e272 with expected crc 2023-09-23T18:54:53.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:53 smithi098 bash[76817]: cluster 2023-09-23T18:54:52.335341+0000 osd.6 (osd.6) 233 : cluster 3 failed to encode map e272 with expected crc 2023-09-23T18:54:53.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:53 smithi098 bash[76817]: cluster 2023-09-23T18:54:52.340359+0000 osd.6 (osd.6) 234 : cluster 3 failed to encode map e272 with expected crc 2023-09-23T18:54:53.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:53 smithi098 bash[76817]: cluster 2023-09-23T18:54:52.342480+0000 osd.5 (osd.5) 177 : cluster 3 failed to encode map e272 with expected crc 2023-09-23T18:54:53.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:53 smithi098 bash[76817]: cluster 2023-09-23T18:54:52.343087+0000 osd.6 (osd.6) 235 : cluster 3 failed to encode map e272 with expected crc 2023-09-23T18:54:53.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:53 smithi098 bash[76817]: cluster 2023-09-23T18:54:52.344312+0000 osd.6 (osd.6) 236 : cluster 3 failed to encode map e272 with expected crc 2023-09-23T18:54:53.447 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:53.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:53.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:53.550 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:53.570 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:53.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:53.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:53.674 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:53.682 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:53 smithi173 bash[59422]: cluster 2023-09-23T18:54:52.239141+0000 mgr.y (mgr.26054) 1107 : cluster 0 pgmap v1186: 89 pgs: 1 stale+active+recovering+undersized+degraded+remapped, 4 stale+active+undersized+degraded, 5 stale+active+undersized, 9 stale+active+clean, 1 active+undersized+remapped+backfill_wait, 1 active+clean+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 24 active+undersized, 1 active+remapped+backfill_wait, 4 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 29 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 31265/84 objects degraded (37220.238%); 0 B/s, 79 objects/s recovering 2023-09-23T18:54:53.682 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:53 smithi173 bash[59422]: cluster 2023-09-23T18:54:52.333597+0000 mon.a (mon.0) 1494 : cluster 0 osdmap e272: 8 total, 6 up, 7 in 2023-09-23T18:54:53.682 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:53 smithi173 bash[59422]: cluster 2023-09-23T18:54:52.334494+0000 osd.5 (osd.5) 176 : cluster 3 failed to encode map e272 with expected crc 2023-09-23T18:54:53.682 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:53 smithi173 bash[59422]: cluster 2023-09-23T18:54:52.335341+0000 osd.6 (osd.6) 233 : cluster 3 failed to encode map e272 with expected crc 2023-09-23T18:54:53.682 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:53 smithi173 bash[59422]: cluster 2023-09-23T18:54:52.340359+0000 osd.6 (osd.6) 234 : cluster 3 failed to encode map e272 with expected crc 2023-09-23T18:54:53.683 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:53 smithi173 bash[59422]: cluster 2023-09-23T18:54:52.342480+0000 osd.5 (osd.5) 177 : cluster 3 failed to encode map e272 with expected crc 2023-09-23T18:54:53.683 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:53 smithi173 bash[59422]: cluster 2023-09-23T18:54:52.343087+0000 osd.6 (osd.6) 235 : cluster 3 failed to encode map e272 with expected crc 2023-09-23T18:54:53.683 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:53 smithi173 bash[59422]: cluster 2023-09-23T18:54:52.344312+0000 osd.6 (osd.6) 236 : cluster 3 failed to encode map e272 with expected crc 2023-09-23T18:54:53.699 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:53.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:53.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:53.802 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:53.820 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:53.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:53.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:53.923 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:53.946 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:53.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:53.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:54.049 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:54.072 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:54.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:54.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:54.175 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:54.194 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:54.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:54.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:54.297 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:54.315 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:54.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:54.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:54.350 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 18:54:54 smithi173 bash[127385]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-7 2023-09-23T18:54:54.350 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 18:54:54 smithi173 bash[127573]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-7 2023-09-23T18:54:54.418 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:54.449 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:54.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:54.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:54.552 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:54.585 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:54.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:54.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:54.687 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:54.705 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:54.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:54.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:54.713 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:54 smithi098 bash[79516]: cluster 2023-09-23T18:54:53.334329+0000 mon.a (mon.0) 1495 : cluster 0 osdmap e273: 8 total, 6 up, 7 in 2023-09-23T18:54:54.714 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:54 smithi098 bash[79516]: cluster 2023-09-23T18:54:53.350486+0000 osd.5 (osd.5) 178 : cluster 3 failed to encode map e273 with expected crc 2023-09-23T18:54:54.714 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:54 smithi098 bash[79516]: cluster 2023-09-23T18:54:53.468424+0000 osd.6 (osd.6) 237 : cluster 3 failed to encode map e273 with expected crc 2023-09-23T18:54:54.714 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:54 smithi098 bash[76817]: cluster 2023-09-23T18:54:53.334329+0000 mon.a (mon.0) 1495 : cluster 0 osdmap e273: 8 total, 6 up, 7 in 2023-09-23T18:54:54.714 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:54 smithi098 bash[76817]: cluster 2023-09-23T18:54:53.350486+0000 osd.5 (osd.5) 178 : cluster 3 failed to encode map e273 with expected crc 2023-09-23T18:54:54.714 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:54 smithi098 bash[76817]: cluster 2023-09-23T18:54:53.468424+0000 osd.6 (osd.6) 237 : cluster 3 failed to encode map e273 with expected crc 2023-09-23T18:54:54.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:54 smithi173 bash[59422]: cluster 2023-09-23T18:54:53.334329+0000 mon.a (mon.0) 1495 : cluster 0 osdmap e273: 8 total, 6 up, 7 in 2023-09-23T18:54:54.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:54 smithi173 bash[59422]: cluster 2023-09-23T18:54:53.350486+0000 osd.5 (osd.5) 178 : cluster 3 failed to encode map e273 with expected crc 2023-09-23T18:54:54.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:54 smithi173 bash[59422]: cluster 2023-09-23T18:54:53.468424+0000 osd.6 (osd.6) 237 : cluster 3 failed to encode map e273 with expected crc 2023-09-23T18:54:54.809 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:54.830 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:54.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:54.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:54.933 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:54.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:54.958 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:54.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:55.059 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:55.077 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:55.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:55.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:55.180 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:55.208 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:55.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:55.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:55.256 INFO:teuthology.orchestra.run.smithi098.stderr:pool 'unique_pool_3' removed 2023-09-23T18:54:55.312 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:55.335 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:55.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:55.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:55.438 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:55.469 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:55.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:55.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:55.574 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:55.599 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:55.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:55.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:55.654 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:55 smithi098 bash[79516]: cluster 2023-09-23T18:54:54.239974+0000 mgr.y (mgr.26054) 1108 : cluster 0 pgmap v1189: 89 pgs: 24 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 33 active+undersized, 2 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 stale+active+clean, 1 remapped+peering, 1 undersized+peered, 5 stale+active+undersized, 2 undersized+degraded+peered, 2 stale+active+undersized+degraded, 1 stale+active+recovering+undersized+degraded+remapped; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 24784/84 objects degraded (29504.762%); 0 B/s, 76 objects/s recovering 2023-09-23T18:54:55.654 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:55 smithi098 bash[79516]: audit 2023-09-23T18:54:54.628315+0000 mon.a (mon.0) 1496 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:54:55.654 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:55 smithi098 bash[79516]: audit 2023-09-23T18:54:54.914680+0000 mon.c (mon.2) 122 : audit 1 from='client.? 172.21.15.98:0/3276837564' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:54:55.654 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:55 smithi098 bash[79516]: audit 2023-09-23T18:54:54.915022+0000 mon.a (mon.0) 1497 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:54:55.654 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:55 smithi098 bash[79516]: cluster 2023-09-23T18:54:55.231169+0000 mon.a (mon.0) 1498 : cluster 3 Health check update: Degraded data redundancy: 24784/84 objects degraded (29504.762%), 18 pgs degraded, 41 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:55.655 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:55 smithi098 bash[79516]: audit 2023-09-23T18:54:55.236063+0000 mon.a (mon.0) 1499 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:54:55.655 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:55 smithi098 bash[79516]: cluster 2023-09-23T18:54:55.236136+0000 mon.a (mon.0) 1500 : cluster 0 osdmap e274: 8 total, 6 up, 7 in 2023-09-23T18:54:55.655 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:55 smithi098 bash[76817]: cluster 2023-09-23T18:54:54.239974+0000 mgr.y (mgr.26054) 1108 : cluster 0 pgmap v1189: 89 pgs: 24 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 33 active+undersized, 2 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 stale+active+clean, 1 remapped+peering, 1 undersized+peered, 5 stale+active+undersized, 2 undersized+degraded+peered, 2 stale+active+undersized+degraded, 1 stale+active+recovering+undersized+degraded+remapped; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 24784/84 objects degraded (29504.762%); 0 B/s, 76 objects/s recovering 2023-09-23T18:54:55.655 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:55 smithi098 bash[76817]: audit 2023-09-23T18:54:54.628315+0000 mon.a (mon.0) 1496 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:54:55.655 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:55 smithi098 bash[76817]: audit 2023-09-23T18:54:54.914680+0000 mon.c (mon.2) 122 : audit 1 from='client.? 172.21.15.98:0/3276837564' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:54:55.655 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:55 smithi098 bash[76817]: audit 2023-09-23T18:54:54.915022+0000 mon.a (mon.0) 1497 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:54:55.655 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:55 smithi098 bash[76817]: cluster 2023-09-23T18:54:55.231169+0000 mon.a (mon.0) 1498 : cluster 3 Health check update: Degraded data redundancy: 24784/84 objects degraded (29504.762%), 18 pgs degraded, 41 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:55.656 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:55 smithi098 bash[76817]: audit 2023-09-23T18:54:55.236063+0000 mon.a (mon.0) 1499 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:54:55.656 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:55 smithi098 bash[76817]: cluster 2023-09-23T18:54:55.236136+0000 mon.a (mon.0) 1500 : cluster 0 osdmap e274: 8 total, 6 up, 7 in 2023-09-23T18:54:55.702 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:55.727 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:55.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:55.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:55.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:55 smithi173 bash[59422]: cluster 2023-09-23T18:54:54.239974+0000 mgr.y (mgr.26054) 1108 : cluster 0 pgmap v1189: 89 pgs: 24 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 33 active+undersized, 2 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 stale+active+clean, 1 remapped+peering, 1 undersized+peered, 5 stale+active+undersized, 2 undersized+degraded+peered, 2 stale+active+undersized+degraded, 1 stale+active+recovering+undersized+degraded+remapped; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 24784/84 objects degraded (29504.762%); 0 B/s, 76 objects/s recovering 2023-09-23T18:54:55.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:55 smithi173 bash[59422]: audit 2023-09-23T18:54:54.628315+0000 mon.a (mon.0) 1496 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:54:55.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:55 smithi173 bash[59422]: audit 2023-09-23T18:54:54.914680+0000 mon.c (mon.2) 122 : audit 1 from='client.? 172.21.15.98:0/3276837564' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:54:55.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:55 smithi173 bash[59422]: audit 2023-09-23T18:54:54.915022+0000 mon.a (mon.0) 1497 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:54:55.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:55 smithi173 bash[59422]: cluster 2023-09-23T18:54:55.231169+0000 mon.a (mon.0) 1498 : cluster 3 Health check update: Degraded data redundancy: 24784/84 objects degraded (29504.762%), 18 pgs degraded, 41 pgs undersized (PG_DEGRADED) 2023-09-23T18:54:55.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:55 smithi173 bash[59422]: audit 2023-09-23T18:54:55.236063+0000 mon.a (mon.0) 1499 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:54:55.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:55 smithi173 bash[59422]: cluster 2023-09-23T18:54:55.236136+0000 mon.a (mon.0) 1500 : cluster 0 osdmap e274: 8 total, 6 up, 7 in 2023-09-23T18:54:55.832 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:55.853 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:55.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:55.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:55.956 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:55.988 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:55.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:55.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:56.092 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:56.111 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:56.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:56.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:56.214 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:56.240 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:56.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:56.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:56.343 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:56.352 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:54:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:54:56] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.33.4" 2023-09-23T18:54:56.370 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:56.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:56.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:56.473 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:56.500 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:56.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:56.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:56.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:56 smithi173 bash[59422]: cluster 2023-09-23T18:54:55.251458+0000 osd.6 (osd.6) 238 : cluster 3 failed to encode map e274 with expected crc 2023-09-23T18:54:56.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:56 smithi173 bash[59422]: cluster 2023-09-23T18:54:55.370743+0000 osd.5 (osd.5) 179 : cluster 3 failed to encode map e274 with expected crc 2023-09-23T18:54:56.604 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:56.638 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:56.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:56.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:56.705 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:56 smithi098 bash[79516]: cluster 2023-09-23T18:54:55.251458+0000 osd.6 (osd.6) 238 : cluster 3 failed to encode map e274 with expected crc 2023-09-23T18:54:56.705 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:56 smithi098 bash[79516]: cluster 2023-09-23T18:54:55.370743+0000 osd.5 (osd.5) 179 : cluster 3 failed to encode map e274 with expected crc 2023-09-23T18:54:56.705 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:56 smithi098 bash[76817]: cluster 2023-09-23T18:54:55.251458+0000 osd.6 (osd.6) 238 : cluster 3 failed to encode map e274 with expected crc 2023-09-23T18:54:56.705 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:56 smithi098 bash[76817]: cluster 2023-09-23T18:54:55.370743+0000 osd.5 (osd.5) 179 : cluster 3 failed to encode map e274 with expected crc 2023-09-23T18:54:56.742 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:56.768 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:56.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:56.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:56.801 DEBUG:teuthology.orchestra.run.smithi173:> sudo pkill -f 'journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.7.service' 2023-09-23T18:54:56.836 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T18:54:56.836 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-09-23T18:54:56.872 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:56.902 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:56.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:56.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:57.006 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:57.037 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:57.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:57.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:57.140 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:57.170 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:57.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:57.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:57.274 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:57.304 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:57.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:57.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:57.407 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:57.418 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:57 smithi173 bash[59422]: cluster 2023-09-23T18:54:56.241191+0000 mgr.y (mgr.26054) 1109 : cluster 0 pgmap v1191: 73 pgs: 1 active+clean+remapped, 21 active+clean, 9 active+undersized+degraded, 29 active+undersized, 1 active+recovery_wait+degraded, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 38/84 objects degraded (45.238%); 2/84 objects misplaced (2.381%) 2023-09-23T18:54:57.418 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:57 smithi173 bash[59422]: cluster 2023-09-23T18:54:56.347990+0000 mon.a (mon.0) 1501 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive (PG_AVAILABILITY) 2023-09-23T18:54:57.435 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:57.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:57.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:57.539 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:57.574 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:57.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:57.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:57.677 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:57.704 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:57.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:57.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:57.709 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:57 smithi098 bash[79516]: cluster 2023-09-23T18:54:56.241191+0000 mgr.y (mgr.26054) 1109 : cluster 0 pgmap v1191: 73 pgs: 1 active+clean+remapped, 21 active+clean, 9 active+undersized+degraded, 29 active+undersized, 1 active+recovery_wait+degraded, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 38/84 objects degraded (45.238%); 2/84 objects misplaced (2.381%) 2023-09-23T18:54:57.709 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:57 smithi098 bash[79516]: cluster 2023-09-23T18:54:56.347990+0000 mon.a (mon.0) 1501 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive (PG_AVAILABILITY) 2023-09-23T18:54:57.709 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:57 smithi098 bash[76817]: cluster 2023-09-23T18:54:56.241191+0000 mgr.y (mgr.26054) 1109 : cluster 0 pgmap v1191: 73 pgs: 1 active+clean+remapped, 21 active+clean, 9 active+undersized+degraded, 29 active+undersized, 1 active+recovery_wait+degraded, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 38/84 objects degraded (45.238%); 2/84 objects misplaced (2.381%) 2023-09-23T18:54:57.709 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:57 smithi098 bash[76817]: cluster 2023-09-23T18:54:56.347990+0000 mon.a (mon.0) 1501 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive (PG_AVAILABILITY) 2023-09-23T18:54:57.808 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:57.833 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:57.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:57.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:57.937 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:57.963 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:57.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:57.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:58.066 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:58.095 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:58.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:58.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:58.197 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:58.224 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:58.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:58.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:58.328 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:58.354 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:58.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:58.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:58.457 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:58.487 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:58.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:58.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:58.590 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:58.614 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:58.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:58.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:58.671 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T18:54:58.672 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T18:54:58.672 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-09-23T18:54:58.672 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-09-23T18:54:58.717 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:58.740 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:54:58.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:58.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:54:58.842 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:58.875 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:58.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:58.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:58.979 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:59.009 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:54:59.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:59.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:54:59.113 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:59.140 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:59.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:59.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:59.243 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:59.266 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:59.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:59.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:54:59.369 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:59.381 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:54:59 smithi098 bash[76817]: cluster 2023-09-23T18:54:58.242028+0000 mgr.y (mgr.26054) 1110 : cluster 0 pgmap v1192: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:54:59.381 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:54:59 smithi098 bash[79516]: cluster 2023-09-23T18:54:58.242028+0000 mgr.y (mgr.26054) 1110 : cluster 0 pgmap v1192: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:54:59.394 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:59.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:59.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:59.497 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:59.521 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:54:59.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:59.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:54:59.624 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:59.650 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:54:59.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:59.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:54:59.753 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:59.767 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:54:59 smithi173 bash[59422]: cluster 2023-09-23T18:54:58.242028+0000 mgr.y (mgr.26054) 1110 : cluster 0 pgmap v1192: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:54:59.785 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:54:59.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:59.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:54:59.889 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:59.916 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:54:59.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:54:59.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:00.019 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:00.045 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:00.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:00.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:00.148 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:00.169 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:00.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:00.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:00.272 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:00.293 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:00.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:00.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:00.397 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:00.429 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:00.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:00.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:00.533 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:00.546 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:00 smithi098 bash[79516]: cluster 2023-09-23T18:55:00.232399+0000 mon.a (mon.0) 1502 : cluster 3 Health check update: Degraded data redundancy: 28/84 objects degraded (33.333%), 13 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:55:00.546 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:00 smithi098 bash[76817]: cluster 2023-09-23T18:55:00.232399+0000 mon.a (mon.0) 1502 : cluster 3 Health check update: Degraded data redundancy: 28/84 objects degraded (33.333%), 13 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:55:00.563 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:00.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:00.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:00.667 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:00.679 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:00 smithi173 bash[59422]: cluster 2023-09-23T18:55:00.232399+0000 mon.a (mon.0) 1502 : cluster 3 Health check update: Degraded data redundancy: 28/84 objects degraded (33.333%), 13 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:55:00.699 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:00.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:00.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:00.803 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:00.832 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:00.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:00.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:00.936 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:00.960 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:00.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:00.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:01.063 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:01.094 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:01.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:01.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:01.198 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:01.227 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:01.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:01.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:01.331 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:01.356 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:01.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:01.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:01.460 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:01.495 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:01.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:01.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:01.599 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:01.611 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:01 smithi098 bash[79516]: cluster 2023-09-23T18:55:00.242885+0000 mgr.y (mgr.26054) 1111 : cluster 0 pgmap v1193: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:01.611 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:01 smithi098 bash[79516]: audit 2023-09-23T18:55:00.546846+0000 mon.a (mon.0) 1503 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:55:01.611 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:01 smithi098 bash[76817]: cluster 2023-09-23T18:55:00.242885+0000 mgr.y (mgr.26054) 1111 : cluster 0 pgmap v1193: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:01.611 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:01 smithi098 bash[76817]: audit 2023-09-23T18:55:00.546846+0000 mon.a (mon.0) 1503 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:55:01.627 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:01.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:01.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:01.731 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:01.754 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:01.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:01.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:01.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:01 smithi173 bash[59422]: cluster 2023-09-23T18:55:00.242885+0000 mgr.y (mgr.26054) 1111 : cluster 0 pgmap v1193: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:01.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:01 smithi173 bash[59422]: audit 2023-09-23T18:55:00.546846+0000 mon.a (mon.0) 1503 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:55:01.837 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [4, 7] live_osds: [0, 2, 3, 5, 6, 1] 2023-09-23T18:55:01.838 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:55:01.838 INFO:tasks.thrashosds.thrasher:Adding osd 5 2023-09-23T18:55:01.838 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd in 5 2023-09-23T18:55:01.857 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:01.887 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:01.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:01.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:01.991 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:02.022 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:02.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:02.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:02.126 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:02.156 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:02.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:02.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:02.260 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:02.289 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:02.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:02.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:02.393 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:02.422 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:02.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:02.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:02.525 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:02.547 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:02.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:02.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:02.650 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:02.677 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:02.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:02.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:02.781 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:02.812 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:02.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:02.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:02.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:02.953 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:02.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:02.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:03.057 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:03.078 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:03.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:03.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:03.182 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:03.214 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:03.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:03.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:03.318 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:03.341 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:03.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:03.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:03.444 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:03.455 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:03 smithi173 bash[59422]: cluster 2023-09-23T18:55:02.244172+0000 mgr.y (mgr.26054) 1112 : cluster 0 pgmap v1194: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:03.473 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:03.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:03.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:03.576 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:03.590 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:03 smithi098 bash[76817]: cluster 2023-09-23T18:55:02.244172+0000 mgr.y (mgr.26054) 1112 : cluster 0 pgmap v1194: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:03.590 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:03 smithi098 bash[79516]: cluster 2023-09-23T18:55:02.244172+0000 mgr.y (mgr.26054) 1112 : cluster 0 pgmap v1194: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:03.602 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:03.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:03.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:03.705 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:03.726 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:03.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:03.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:03.811 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:55:03.811 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:55:03.829 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:03.864 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:03.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:03.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:03.971 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:03.998 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:04.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:04.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:04.102 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:04.127 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:04.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:04.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:04.230 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:04.262 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:04.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:04.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:04.366 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:04.395 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:04.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:04.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:04.442 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:55:04.499 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:04.528 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:04.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:04.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:04.632 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:04.664 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:04.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:04.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:04.768 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:04.791 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:04.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:04.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:04.894 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:04.913 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:04.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:04.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:05.016 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:05.047 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:05.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:05.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:05.150 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:05.172 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:05.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:05.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:05.275 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:05.297 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:05.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:05.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:05.400 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:05.419 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:05.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:05.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:05.521 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:05.580 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:05.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:05.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:05.683 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:05.756 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:05.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:05.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:05.860 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:05.881 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:05.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:05.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:05.983 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:06.003 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:06.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:06.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:06.105 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:06.132 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:06.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:06.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:06.192 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:55:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:55:06] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.33.4" 2023-09-23T18:55:06.236 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:06.260 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:06.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:06.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:06.363 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:06.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:06 smithi173 bash[59422]: cluster 2023-09-23T18:55:04.244983+0000 mgr.y (mgr.26054) 1113 : cluster 0 pgmap v1195: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:06.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:06 smithi173 bash[59422]: audit 2023-09-23T18:55:05.530835+0000 mon.a (mon.0) 1504 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:06.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:06 smithi173 bash[59422]: audit 2023-09-23T18:55:05.537599+0000 mon.a (mon.0) 1505 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:06.390 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:06.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:06.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:06.410 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:55:06.430 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:55:06.450 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:55:06.471 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:55:06.495 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:06.498 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:55:06.522 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:55:06.522 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:06.522 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:55:06.522 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:55:06.523 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:55:06.523 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:55:06.523 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-09-23T18:55:06.523 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:55:06.523 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:55:06.523 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T18:55:06.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:06.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:06.626 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:06.635 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:06 smithi098 bash[76817]: cluster 2023-09-23T18:55:04.244983+0000 mgr.y (mgr.26054) 1113 : cluster 0 pgmap v1195: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:06.635 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:06 smithi098 bash[76817]: audit 2023-09-23T18:55:05.530835+0000 mon.a (mon.0) 1504 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:06.635 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:06 smithi098 bash[76817]: audit 2023-09-23T18:55:05.537599+0000 mon.a (mon.0) 1505 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:06.635 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:06 smithi098 bash[79516]: cluster 2023-09-23T18:55:04.244983+0000 mgr.y (mgr.26054) 1113 : cluster 0 pgmap v1195: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:06.636 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:06 smithi098 bash[79516]: audit 2023-09-23T18:55:05.530835+0000 mon.a (mon.0) 1504 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:06.636 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:06 smithi098 bash[79516]: audit 2023-09-23T18:55:05.537599+0000 mon.a (mon.0) 1505 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:06.649 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:06.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:06.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:06.753 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:06.779 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:06.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:06.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:06.882 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:06.908 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:06.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:06.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:07.011 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:07.038 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:07.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:07.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:07.051 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T18:55:07.142 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:07.171 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:07.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:07.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:07.275 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:07.297 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:07.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:07.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:07.400 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:07.434 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:07.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:07.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:07.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:07 smithi098 bash[79516]: audit 2023-09-23T18:55:06.383240+0000 mon.c (mon.2) 123 : audit 0 from='client.? 172.21.15.98:0/1534074896' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:55:07.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:07 smithi098 bash[79516]: audit 2023-09-23T18:55:06.834581+0000 mon.a (mon.0) 1506 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:07.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:07 smithi098 bash[79516]: audit 2023-09-23T18:55:06.841926+0000 mon.a (mon.0) 1507 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:07.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:07 smithi098 bash[79516]: audit 2023-09-23T18:55:07.121753+0000 mon.a (mon.0) 1508 : audit 1 from='client.? 172.21.15.98:0/643704057' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:55:07.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:07 smithi098 bash[76817]: audit 2023-09-23T18:55:06.383240+0000 mon.c (mon.2) 123 : audit 0 from='client.? 172.21.15.98:0/1534074896' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:55:07.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:07 smithi098 bash[76817]: audit 2023-09-23T18:55:06.834581+0000 mon.a (mon.0) 1506 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:07.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:07 smithi098 bash[76817]: audit 2023-09-23T18:55:06.841926+0000 mon.a (mon.0) 1507 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:07.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:07 smithi098 bash[76817]: audit 2023-09-23T18:55:07.121753+0000 mon.a (mon.0) 1508 : audit 1 from='client.? 172.21.15.98:0/643704057' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:55:07.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:07 smithi173 bash[59422]: audit 2023-09-23T18:55:06.383240+0000 mon.c (mon.2) 123 : audit 0 from='client.? 172.21.15.98:0/1534074896' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:55:07.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:07 smithi173 bash[59422]: audit 2023-09-23T18:55:06.834581+0000 mon.a (mon.0) 1506 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:07.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:07 smithi173 bash[59422]: audit 2023-09-23T18:55:06.841926+0000 mon.a (mon.0) 1507 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:07.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:07 smithi173 bash[59422]: audit 2023-09-23T18:55:07.121753+0000 mon.a (mon.0) 1508 : audit 1 from='client.? 172.21.15.98:0/643704057' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:55:07.539 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:07.569 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:07.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:07.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:07.672 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:07.703 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:07.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:07.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:07.806 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:07.833 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:07.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:07.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:07.856 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is unset 2023-09-23T18:55:07.936 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:07.964 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:07.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:07.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:08.067 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:08.096 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:08.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:08.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:08.199 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:08.227 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:08.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:08.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:08.331 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:08.342 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:08 smithi098 bash[79516]: cluster 2023-09-23T18:55:06.245695+0000 mgr.y (mgr.26054) 1114 : cluster 0 pgmap v1196: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%); 372 KiB/s, 1 objects/s recovering 2023-09-23T18:55:08.342 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:08 smithi098 bash[79516]: audit 2023-09-23T18:55:07.681104+0000 mon.a (mon.0) 1509 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:08.342 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:08 smithi098 bash[79516]: audit 2023-09-23T18:55:07.687152+0000 mon.a (mon.0) 1510 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:08.343 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:08 smithi098 bash[79516]: cluster 2023-09-23T18:55:07.839830+0000 mon.a (mon.0) 1511 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:55:08.343 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:08 smithi098 bash[79516]: cluster 2023-09-23T18:55:07.845338+0000 osd.5 (osd.5) 180 : cluster 3 failed to encode map e275 with expected crc 2023-09-23T18:55:08.343 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:08 smithi098 bash[79516]: audit 2023-09-23T18:55:07.848415+0000 mon.a (mon.0) 1512 : audit 1 from='client.? 172.21.15.98:0/643704057' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:55:08.343 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:08 smithi098 bash[79516]: cluster 2023-09-23T18:55:07.848498+0000 mon.a (mon.0) 1513 : cluster 0 osdmap e275: 8 total, 6 up, 7 in 2023-09-23T18:55:08.343 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:08 smithi098 bash[76817]: cluster 2023-09-23T18:55:06.245695+0000 mgr.y (mgr.26054) 1114 : cluster 0 pgmap v1196: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%); 372 KiB/s, 1 objects/s recovering 2023-09-23T18:55:08.343 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:08 smithi098 bash[76817]: audit 2023-09-23T18:55:07.681104+0000 mon.a (mon.0) 1509 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:08.343 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:08 smithi098 bash[76817]: audit 2023-09-23T18:55:07.687152+0000 mon.a (mon.0) 1510 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:08.343 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:08 smithi098 bash[76817]: cluster 2023-09-23T18:55:07.839830+0000 mon.a (mon.0) 1511 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:55:08.343 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:08 smithi098 bash[76817]: cluster 2023-09-23T18:55:07.845338+0000 osd.5 (osd.5) 180 : cluster 3 failed to encode map e275 with expected crc 2023-09-23T18:55:08.344 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:08 smithi098 bash[76817]: audit 2023-09-23T18:55:07.848415+0000 mon.a (mon.0) 1512 : audit 1 from='client.? 172.21.15.98:0/643704057' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:55:08.344 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:08 smithi098 bash[76817]: cluster 2023-09-23T18:55:07.848498+0000 mon.a (mon.0) 1513 : cluster 0 osdmap e275: 8 total, 6 up, 7 in 2023-09-23T18:55:08.359 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:08.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:08.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:08.462 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:08.484 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:08.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:08.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:08.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:08 smithi173 bash[59422]: cluster 2023-09-23T18:55:06.245695+0000 mgr.y (mgr.26054) 1114 : cluster 0 pgmap v1196: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%); 372 KiB/s, 1 objects/s recovering 2023-09-23T18:55:08.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:08 smithi173 bash[59422]: audit 2023-09-23T18:55:07.681104+0000 mon.a (mon.0) 1509 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:08.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:08 smithi173 bash[59422]: audit 2023-09-23T18:55:07.687152+0000 mon.a (mon.0) 1510 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:08.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:08 smithi173 bash[59422]: cluster 2023-09-23T18:55:07.839830+0000 mon.a (mon.0) 1511 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:55:08.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:08 smithi173 bash[59422]: cluster 2023-09-23T18:55:07.845338+0000 osd.5 (osd.5) 180 : cluster 3 failed to encode map e275 with expected crc 2023-09-23T18:55:08.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:08 smithi173 bash[59422]: audit 2023-09-23T18:55:07.848415+0000 mon.a (mon.0) 1512 : audit 1 from='client.? 172.21.15.98:0/643704057' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:55:08.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:08 smithi173 bash[59422]: cluster 2023-09-23T18:55:07.848498+0000 mon.a (mon.0) 1513 : cluster 0 osdmap e275: 8 total, 6 up, 7 in 2023-09-23T18:55:08.587 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:08.609 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:08.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:08.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:08.712 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:08.743 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:08.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:08.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:08.847 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:08.874 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:08.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:08.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:08.977 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:09.004 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:09.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:09.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:09.107 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:09.136 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:09.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:09.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:09.239 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:09.274 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:09.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:09.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:09.378 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:09.388 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:09 smithi098 bash[76817]: cluster 2023-09-23T18:55:08.246453+0000 mgr.y (mgr.26054) 1115 : cluster 0 pgmap v1198: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:09.388 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:09 smithi098 bash[76817]: cluster 2023-09-23T18:55:08.310880+0000 osd.6 (osd.6) 239 : cluster 3 failed to encode map e275 with expected crc 2023-09-23T18:55:09.389 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:09 smithi098 bash[79516]: cluster 2023-09-23T18:55:08.246453+0000 mgr.y (mgr.26054) 1115 : cluster 0 pgmap v1198: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:09.389 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:09 smithi098 bash[79516]: cluster 2023-09-23T18:55:08.310880+0000 osd.6 (osd.6) 239 : cluster 3 failed to encode map e275 with expected crc 2023-09-23T18:55:09.401 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:09.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:09.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:09.505 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:09.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:09 smithi173 bash[59422]: cluster 2023-09-23T18:55:08.246453+0000 mgr.y (mgr.26054) 1115 : cluster 0 pgmap v1198: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:09.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:09 smithi173 bash[59422]: cluster 2023-09-23T18:55:08.310880+0000 osd.6 (osd.6) 239 : cluster 3 failed to encode map e275 with expected crc 2023-09-23T18:55:09.531 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:09.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:09.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:09.635 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:09.661 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:09.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:09.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:09.765 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:09.790 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:09.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:09.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:09.893 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:09.903 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:55:09.903 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:55:09.926 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:09.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:09.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:10.031 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:10.060 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:10.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:10.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:10.164 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:10.191 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:10.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:10.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:10.294 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:10.317 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:10.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:10.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:10.421 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:10.443 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:10.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:10.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:10.546 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:10.573 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:10.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:10.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:10.676 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:10.711 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:10.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:10.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:10.760 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:10 smithi098 bash[76817]: audit 2023-09-23T18:55:09.635269+0000 mon.a (mon.0) 1514 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:10.760 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:10 smithi098 bash[76817]: audit 2023-09-23T18:55:09.636566+0000 mon.a (mon.0) 1515 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:55:10.760 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:10 smithi098 bash[79516]: audit 2023-09-23T18:55:09.635269+0000 mon.a (mon.0) 1514 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:10.760 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:10 smithi098 bash[79516]: audit 2023-09-23T18:55:09.636566+0000 mon.a (mon.0) 1515 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:55:10.814 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:10.841 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:10.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:10.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:10.946 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:10.973 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:10.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:10.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:11.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:10 smithi173 bash[59422]: audit 2023-09-23T18:55:09.635269+0000 mon.a (mon.0) 1514 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:11.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:10 smithi173 bash[59422]: audit 2023-09-23T18:55:09.636566+0000 mon.a (mon.0) 1515 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:55:11.077 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:11.109 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:11.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:11.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:11.212 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:11.237 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:11.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:11.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:11.341 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:11.369 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:11.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:11.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:11.472 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:11.506 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:11.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:11.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:11.540 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:55:11.611 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:11.636 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:11.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:11.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:11.739 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:11.760 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:11.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:11.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:11.863 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:11.888 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:11.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:11.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:11.904 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:55:11.992 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:12.019 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:12.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:12.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:12.121 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:12.153 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:12.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:12.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:12.257 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:12.287 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:12.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:12.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:12.377 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:55:12.392 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:12.441 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:12.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:12.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:12.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:12 smithi173 bash[59422]: cluster 2023-09-23T18:55:10.247073+0000 mgr.y (mgr.26054) 1116 : cluster 0 pgmap v1199: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:12.544 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:12.573 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:12.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:12.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:12.677 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:12.685 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:12 smithi098 bash[79516]: cluster 2023-09-23T18:55:10.247073+0000 mgr.y (mgr.26054) 1116 : cluster 0 pgmap v1199: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:12.686 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:12 smithi098 bash[76817]: cluster 2023-09-23T18:55:10.247073+0000 mgr.y (mgr.26054) 1116 : cluster 0 pgmap v1199: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:12.702 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:12.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:12.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:12.805 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:12.836 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:12.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:12.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:12.940 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:12.968 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:12.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:12.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:13.072 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:13.103 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:13.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:13.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:13.206 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:13.236 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:13.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:13.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:13.340 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:13.352 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:13 smithi173 bash[59422]: cluster 2023-09-23T18:55:12.248115+0000 mgr.y (mgr.26054) 1117 : cluster 0 pgmap v1200: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:13.370 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:13.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:13.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:13.474 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:13.483 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:13 smithi098 bash[79516]: cluster 2023-09-23T18:55:12.248115+0000 mgr.y (mgr.26054) 1117 : cluster 0 pgmap v1200: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:13.483 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:13 smithi098 bash[76817]: cluster 2023-09-23T18:55:12.248115+0000 mgr.y (mgr.26054) 1117 : cluster 0 pgmap v1200: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:13.500 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:13.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:13.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:13.603 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:13.634 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:13.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:13.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:13.737 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:13.769 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:13.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:13.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:13.874 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:13.904 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:55:13.907 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:13.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:13.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:14.011 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:14.044 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:14.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:14.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:14.147 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:14.176 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:14.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:14.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:14.280 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:14.309 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:14.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:14.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:14.412 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:14.433 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:14.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:14.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:14.536 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:14.559 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:14.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:14.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:14.663 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:14.682 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:14.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:14.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:14.785 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:14.810 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:14.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:14.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:14.914 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:14.934 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:14.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:14.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:15.037 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:15.065 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:15.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:15.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:15.169 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:15.191 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:15.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:15.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:15.295 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:15.318 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:15.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:15.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:15.407 INFO:teuthology.orchestra.run.smithi098.stderr:pool 'unique_pool_4' created 2023-09-23T18:55:15.421 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:15.432 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:15 smithi098 bash[79516]: cluster 2023-09-23T18:55:14.249381+0000 mgr.y (mgr.26054) 1118 : cluster 0 pgmap v1201: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:15.432 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:15 smithi098 bash[79516]: audit 2023-09-23T18:55:14.884429+0000 mon.c (mon.2) 124 : audit 1 from='client.? 172.21.15.98:0/1282254963' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T18:55:15.432 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:15 smithi098 bash[79516]: audit 2023-09-23T18:55:14.884942+0000 mon.a (mon.0) 1516 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T18:55:15.433 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:15 smithi098 bash[76817]: cluster 2023-09-23T18:55:14.249381+0000 mgr.y (mgr.26054) 1118 : cluster 0 pgmap v1201: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:15.433 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:15 smithi098 bash[76817]: audit 2023-09-23T18:55:14.884429+0000 mon.c (mon.2) 124 : audit 1 from='client.? 172.21.15.98:0/1282254963' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T18:55:15.433 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:15 smithi098 bash[76817]: audit 2023-09-23T18:55:14.884942+0000 mon.a (mon.0) 1516 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T18:55:15.455 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:15.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:15.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:15.559 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:15.585 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:15.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:15.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:15.688 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:15.713 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:15.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:15.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:15.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:15 smithi173 bash[59422]: cluster 2023-09-23T18:55:14.249381+0000 mgr.y (mgr.26054) 1118 : cluster 0 pgmap v1201: 73 pgs: 1 active+clean+remapped, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:15.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:15 smithi173 bash[59422]: audit 2023-09-23T18:55:14.884429+0000 mon.c (mon.2) 124 : audit 1 from='client.? 172.21.15.98:0/1282254963' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T18:55:15.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:15 smithi173 bash[59422]: audit 2023-09-23T18:55:14.884942+0000 mon.a (mon.0) 1516 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T18:55:15.817 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:15.843 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:15.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:15.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:15.946 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:15.979 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:15.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:15.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:16.084 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:16.111 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:16.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:16.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:16.214 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:16.228 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:55:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:55:16] "GET /metrics HTTP/1.1" 200 34209 "" "Prometheus/2.33.4" 2023-09-23T18:55:16.242 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:16.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:16.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:16.345 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:16.373 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:16.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:16.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:16.476 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:16.506 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:16.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:16.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:16.525 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:16 smithi098 bash[76817]: audit 2023-09-23T18:55:15.393136+0000 mon.a (mon.0) 1517 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T18:55:16.525 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:16 smithi098 bash[76817]: cluster 2023-09-23T18:55:15.393216+0000 mon.a (mon.0) 1518 : cluster 0 osdmap e276: 8 total, 6 up, 7 in 2023-09-23T18:55:16.525 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:16 smithi098 bash[76817]: cluster 2023-09-23T18:55:15.395747+0000 osd.6 (osd.6) 240 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T18:55:16.525 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:16 smithi098 bash[76817]: cluster 2023-09-23T18:55:15.398561+0000 osd.6 (osd.6) 241 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T18:55:16.525 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:16 smithi098 bash[76817]: cluster 2023-09-23T18:55:15.401013+0000 osd.6 (osd.6) 242 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T18:55:16.525 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:16 smithi098 bash[76817]: cluster 2023-09-23T18:55:15.403443+0000 osd.6 (osd.6) 243 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T18:55:16.525 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:16 smithi098 bash[76817]: cluster 2023-09-23T18:55:15.405700+0000 osd.6 (osd.6) 244 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T18:55:16.525 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:16 smithi098 bash[76817]: cluster 2023-09-23T18:55:15.623917+0000 osd.5 (osd.5) 181 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T18:55:16.526 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:16 smithi098 bash[76817]: cluster 2023-09-23T18:55:15.626536+0000 osd.5 (osd.5) 182 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T18:55:16.526 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:16 smithi098 bash[76817]: cluster 2023-09-23T18:55:15.629143+0000 osd.5 (osd.5) 183 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T18:55:16.526 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:16 smithi098 bash[79516]: audit 2023-09-23T18:55:15.393136+0000 mon.a (mon.0) 1517 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T18:55:16.526 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:16 smithi098 bash[79516]: cluster 2023-09-23T18:55:15.393216+0000 mon.a (mon.0) 1518 : cluster 0 osdmap e276: 8 total, 6 up, 7 in 2023-09-23T18:55:16.526 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:16 smithi098 bash[79516]: cluster 2023-09-23T18:55:15.395747+0000 osd.6 (osd.6) 240 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T18:55:16.526 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:16 smithi098 bash[79516]: cluster 2023-09-23T18:55:15.398561+0000 osd.6 (osd.6) 241 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T18:55:16.526 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:16 smithi098 bash[79516]: cluster 2023-09-23T18:55:15.401013+0000 osd.6 (osd.6) 242 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T18:55:16.526 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:16 smithi098 bash[79516]: cluster 2023-09-23T18:55:15.403443+0000 osd.6 (osd.6) 243 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T18:55:16.526 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:16 smithi098 bash[79516]: cluster 2023-09-23T18:55:15.405700+0000 osd.6 (osd.6) 244 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T18:55:16.526 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:16 smithi098 bash[79516]: cluster 2023-09-23T18:55:15.623917+0000 osd.5 (osd.5) 181 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T18:55:16.526 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:16 smithi098 bash[79516]: cluster 2023-09-23T18:55:15.626536+0000 osd.5 (osd.5) 182 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T18:55:16.527 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:16 smithi098 bash[79516]: cluster 2023-09-23T18:55:15.629143+0000 osd.5 (osd.5) 183 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T18:55:16.610 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:16.640 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:16.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:16.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:16.744 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:16.755 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:16 smithi173 bash[59422]: audit 2023-09-23T18:55:15.393136+0000 mon.a (mon.0) 1517 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T18:55:16.755 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:16 smithi173 bash[59422]: cluster 2023-09-23T18:55:15.393216+0000 mon.a (mon.0) 1518 : cluster 0 osdmap e276: 8 total, 6 up, 7 in 2023-09-23T18:55:16.755 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:16 smithi173 bash[59422]: cluster 2023-09-23T18:55:15.395747+0000 osd.6 (osd.6) 240 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T18:55:16.755 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:16 smithi173 bash[59422]: cluster 2023-09-23T18:55:15.398561+0000 osd.6 (osd.6) 241 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T18:55:16.755 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:16 smithi173 bash[59422]: cluster 2023-09-23T18:55:15.401013+0000 osd.6 (osd.6) 242 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T18:55:16.755 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:16 smithi173 bash[59422]: cluster 2023-09-23T18:55:15.403443+0000 osd.6 (osd.6) 243 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T18:55:16.756 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:16 smithi173 bash[59422]: cluster 2023-09-23T18:55:15.405700+0000 osd.6 (osd.6) 244 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T18:55:16.756 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:16 smithi173 bash[59422]: cluster 2023-09-23T18:55:15.623917+0000 osd.5 (osd.5) 181 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T18:55:16.756 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:16 smithi173 bash[59422]: cluster 2023-09-23T18:55:15.626536+0000 osd.5 (osd.5) 182 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T18:55:16.756 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:16 smithi173 bash[59422]: cluster 2023-09-23T18:55:15.629143+0000 osd.5 (osd.5) 183 : cluster 3 failed to encode map e276 with expected crc 2023-09-23T18:55:16.775 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:16.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:16.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:16.879 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:16.904 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:16.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:16.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:17.007 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:17.035 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:17.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:17.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:17.138 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:17.170 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:17.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:17.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:17.274 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:17.300 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:17.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:17.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:17.406 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:17.433 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:17.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:17.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:17.537 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:17.565 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:17.566 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:17 smithi098 bash[79516]: cluster 2023-09-23T18:55:16.250645+0000 mgr.y (mgr.26054) 1119 : cluster 0 pgmap v1203: 89 pgs: 2 creating+peering, 1 active+clean+remapped, 14 unknown, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:17.566 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:17 smithi098 bash[79516]: cluster 2023-09-23T18:55:16.399192+0000 mon.a (mon.0) 1519 : cluster 0 osdmap e277: 8 total, 6 up, 7 in 2023-09-23T18:55:17.567 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:17 smithi098 bash[79516]: cluster 2023-09-23T18:55:16.565413+0000 osd.5 (osd.5) 184 : cluster 3 failed to encode map e277 with expected crc 2023-09-23T18:55:17.567 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:17 smithi098 bash[79516]: cluster 2023-09-23T18:55:16.565531+0000 osd.6 (osd.6) 245 : cluster 3 failed to encode map e277 with expected crc 2023-09-23T18:55:17.567 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:17 smithi098 bash[79516]: cluster 2023-09-23T18:55:17.399325+0000 mon.a (mon.0) 1520 : cluster 0 osdmap e278: 8 total, 6 up, 7 in 2023-09-23T18:55:17.567 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:17 smithi098 bash[76817]: cluster 2023-09-23T18:55:16.250645+0000 mgr.y (mgr.26054) 1119 : cluster 0 pgmap v1203: 89 pgs: 2 creating+peering, 1 active+clean+remapped, 14 unknown, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:17.567 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:17 smithi098 bash[76817]: cluster 2023-09-23T18:55:16.399192+0000 mon.a (mon.0) 1519 : cluster 0 osdmap e277: 8 total, 6 up, 7 in 2023-09-23T18:55:17.567 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:17 smithi098 bash[76817]: cluster 2023-09-23T18:55:16.565413+0000 osd.5 (osd.5) 184 : cluster 3 failed to encode map e277 with expected crc 2023-09-23T18:55:17.567 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:17 smithi098 bash[76817]: cluster 2023-09-23T18:55:16.565531+0000 osd.6 (osd.6) 245 : cluster 3 failed to encode map e277 with expected crc 2023-09-23T18:55:17.567 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:17 smithi098 bash[76817]: cluster 2023-09-23T18:55:17.399325+0000 mon.a (mon.0) 1520 : cluster 0 osdmap e278: 8 total, 6 up, 7 in 2023-09-23T18:55:17.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:17.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:17.669 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:17.677 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:17 smithi173 bash[59422]: cluster 2023-09-23T18:55:16.250645+0000 mgr.y (mgr.26054) 1119 : cluster 0 pgmap v1203: 89 pgs: 2 creating+peering, 1 active+clean+remapped, 14 unknown, 22 active+clean, 9 active+undersized+degraded, 29 active+undersized, 8 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:17.677 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:17 smithi173 bash[59422]: cluster 2023-09-23T18:55:16.399192+0000 mon.a (mon.0) 1519 : cluster 0 osdmap e277: 8 total, 6 up, 7 in 2023-09-23T18:55:17.677 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:17 smithi173 bash[59422]: cluster 2023-09-23T18:55:16.565413+0000 osd.5 (osd.5) 184 : cluster 3 failed to encode map e277 with expected crc 2023-09-23T18:55:17.678 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:17 smithi173 bash[59422]: cluster 2023-09-23T18:55:16.565531+0000 osd.6 (osd.6) 245 : cluster 3 failed to encode map e277 with expected crc 2023-09-23T18:55:17.678 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:17 smithi173 bash[59422]: cluster 2023-09-23T18:55:17.399325+0000 mon.a (mon.0) 1520 : cluster 0 osdmap e278: 8 total, 6 up, 7 in 2023-09-23T18:55:17.691 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:17.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:17.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:17.794 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:17.824 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:17.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:17.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:17.928 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:17.963 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:17.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:17.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:18.068 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:18.098 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:18.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:18.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:18.202 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:18.229 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:18.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:18.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:18.332 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:18.363 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:18.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:18.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:18.468 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:18.478 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:18 smithi098 bash[79516]: cluster 2023-09-23T18:55:17.670898+0000 osd.6 (osd.6) 246 : cluster 3 failed to encode map e278 with expected crc 2023-09-23T18:55:18.479 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:18 smithi098 bash[79516]: cluster 2023-09-23T18:55:17.673812+0000 osd.6 (osd.6) 247 : cluster 3 failed to encode map e278 with expected crc 2023-09-23T18:55:18.479 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:18 smithi098 bash[79516]: audit 2023-09-23T18:55:17.881866+0000 mon.a (mon.0) 1521 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:18.479 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:18 smithi098 bash[79516]: audit 2023-09-23T18:55:17.887698+0000 mon.a (mon.0) 1522 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:18.479 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:18 smithi098 bash[79516]: audit 2023-09-23T18:55:17.888951+0000 mon.a (mon.0) 1523 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:55:18.479 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:18 smithi098 bash[79516]: audit 2023-09-23T18:55:17.889956+0000 mon.a (mon.0) 1524 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:55:18.479 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:18 smithi098 bash[79516]: audit 2023-09-23T18:55:17.896363+0000 mon.a (mon.0) 1525 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:18.479 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:18 smithi098 bash[76817]: cluster 2023-09-23T18:55:17.670898+0000 osd.6 (osd.6) 246 : cluster 3 failed to encode map e278 with expected crc 2023-09-23T18:55:18.479 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:18 smithi098 bash[76817]: cluster 2023-09-23T18:55:17.673812+0000 osd.6 (osd.6) 247 : cluster 3 failed to encode map e278 with expected crc 2023-09-23T18:55:18.479 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:18 smithi098 bash[76817]: audit 2023-09-23T18:55:17.881866+0000 mon.a (mon.0) 1521 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:18.479 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:18 smithi098 bash[76817]: audit 2023-09-23T18:55:17.887698+0000 mon.a (mon.0) 1522 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:18.480 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:18 smithi098 bash[76817]: audit 2023-09-23T18:55:17.888951+0000 mon.a (mon.0) 1523 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:55:18.480 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:18 smithi098 bash[76817]: audit 2023-09-23T18:55:17.889956+0000 mon.a (mon.0) 1524 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:55:18.480 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:18 smithi098 bash[76817]: audit 2023-09-23T18:55:17.896363+0000 mon.a (mon.0) 1525 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:18.496 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:18.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:18.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:18.599 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:18.624 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:18.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:18.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:18.728 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:18.756 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:18.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:18.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:18.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:18 smithi173 bash[59422]: cluster 2023-09-23T18:55:17.670898+0000 osd.6 (osd.6) 246 : cluster 3 failed to encode map e278 with expected crc 2023-09-23T18:55:18.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:18 smithi173 bash[59422]: cluster 2023-09-23T18:55:17.673812+0000 osd.6 (osd.6) 247 : cluster 3 failed to encode map e278 with expected crc 2023-09-23T18:55:18.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:18 smithi173 bash[59422]: audit 2023-09-23T18:55:17.881866+0000 mon.a (mon.0) 1521 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:18.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:18 smithi173 bash[59422]: audit 2023-09-23T18:55:17.887698+0000 mon.a (mon.0) 1522 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:18.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:18 smithi173 bash[59422]: audit 2023-09-23T18:55:17.888951+0000 mon.a (mon.0) 1523 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:55:18.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:18 smithi173 bash[59422]: audit 2023-09-23T18:55:17.889956+0000 mon.a (mon.0) 1524 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:55:18.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:18 smithi173 bash[59422]: audit 2023-09-23T18:55:17.896363+0000 mon.a (mon.0) 1525 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:18.860 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:18.886 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:18.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:18.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:18.990 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:19.020 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:19.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:19.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:19.123 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:19.157 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:19.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:19.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:19.260 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:19.291 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:19.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:19.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:19.395 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:19.423 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:19.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:19.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:19.527 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:19.553 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:19.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:19.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:19.656 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:19.679 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-09-23T18:55:19.693 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:19.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:19.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:19.797 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:19.831 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:19.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:19.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:19.935 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:19.958 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:19.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:19.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:20.002 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:19 smithi098 bash[76817]: cluster 2023-09-23T18:55:18.252259+0000 mgr.y (mgr.26054) 1120 : cluster 0 pgmap v1206: 89 pgs: 2 creating+peering, 1 active+clean+remapped, 3 unknown, 25 active+clean, 9 active+undersized+degraded, 34 active+undersized, 11 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:20.002 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:19 smithi098 bash[76817]: cluster 2023-09-23T18:55:18.912058+0000 osd.5 (osd.5) 185 : cluster 3 failed to encode map e278 with expected crc 2023-09-23T18:55:20.003 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:19 smithi098 bash[79516]: cluster 2023-09-23T18:55:18.252259+0000 mgr.y (mgr.26054) 1120 : cluster 0 pgmap v1206: 89 pgs: 2 creating+peering, 1 active+clean+remapped, 3 unknown, 25 active+clean, 9 active+undersized+degraded, 34 active+undersized, 11 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:20.003 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:19 smithi098 bash[79516]: cluster 2023-09-23T18:55:18.912058+0000 osd.5 (osd.5) 185 : cluster 3 failed to encode map e278 with expected crc 2023-09-23T18:55:20.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:19 smithi173 bash[59422]: cluster 2023-09-23T18:55:18.252259+0000 mgr.y (mgr.26054) 1120 : cluster 0 pgmap v1206: 89 pgs: 2 creating+peering, 1 active+clean+remapped, 3 unknown, 25 active+clean, 9 active+undersized+degraded, 34 active+undersized, 11 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:20.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:19 smithi173 bash[59422]: cluster 2023-09-23T18:55:18.912058+0000 osd.5 (osd.5) 185 : cluster 3 failed to encode map e278 with expected crc 2023-09-23T18:55:20.061 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:20.082 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:20.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:20.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:20.186 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:20.218 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:20.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:20.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:20.322 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:20.345 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:20.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:20.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:20.448 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:20.476 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:20.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:20.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:20.579 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:20.609 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:20.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:20.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:20.713 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:20.743 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:20.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:20.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:20.846 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:20.880 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:20.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:20.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:20.984 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:21.010 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:21.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:21.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:21.114 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:21.142 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:21.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:21.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:21.246 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:21.278 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:21.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:21.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:21.383 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:21.416 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:21.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:21.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:21.520 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:21.551 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:21.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:21.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:21.654 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:21.686 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:21.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:21.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:21.790 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:21.816 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:21.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:21.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:21.900 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:55:21.901 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:55:21.920 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:21.943 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:21.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:21.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:22.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:21 smithi173 bash[59422]: cluster 2023-09-23T18:55:20.253630+0000 mgr.y (mgr.26054) 1121 : cluster 0 pgmap v1207: 89 pgs: 2 creating+peering, 1 active+clean+remapped, 26 active+clean, 9 active+undersized+degraded, 35 active+undersized, 12 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:22.046 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:22.068 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:22.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:22.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:22.172 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:22.175 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:21 smithi098 bash[79516]: cluster 2023-09-23T18:55:20.253630+0000 mgr.y (mgr.26054) 1121 : cluster 0 pgmap v1207: 89 pgs: 2 creating+peering, 1 active+clean+remapped, 26 active+clean, 9 active+undersized+degraded, 35 active+undersized, 12 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:22.175 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:21 smithi098 bash[76817]: cluster 2023-09-23T18:55:20.253630+0000 mgr.y (mgr.26054) 1121 : cluster 0 pgmap v1207: 89 pgs: 2 creating+peering, 1 active+clean+remapped, 26 active+clean, 9 active+undersized+degraded, 35 active+undersized, 12 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:22.200 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:22.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:22.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:22.303 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:22.337 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:22.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:22.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:22.440 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:22.469 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:22.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:22.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:22.573 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:22.595 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:22.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:22.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:22.698 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:22.719 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:22.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:22.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:22.822 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:22.844 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:22.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:22.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:22.938 INFO:teuthology.orchestra.run.smithi098.stderr:marked in osd.5. 2023-09-23T18:55:22.947 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:22.969 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:22.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:22.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:22.984 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:22 smithi098 bash[76817]: debug 2023-09-23T18:55:22.869+0000 7fc8a2a01700 -1 mon.a@0(leader).osd e278 definitely_dead 0 2023-09-23T18:55:22.984 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:22 smithi098 bash[76817]: audit 2023-09-23T18:55:22.872645+0000 mon.a (mon.0) 1526 : audit 0 from='client.? 172.21.15.98:0/85380039' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T18:55:23.073 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:23.104 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:23.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:23.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:23.208 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:23.239 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:23.240 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:22 smithi098 bash[79516]: audit 2023-09-23T18:55:22.872645+0000 mon.a (mon.0) 1526 : audit 0 from='client.? 172.21.15.98:0/85380039' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T18:55:23.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:23.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:23.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:22 smithi173 bash[59422]: audit 2023-09-23T18:55:22.872645+0000 mon.a (mon.0) 1526 : audit 0 from='client.? 172.21.15.98:0/85380039' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T18:55:23.342 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:23.374 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:23.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:23.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:23.478 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:23.503 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:23.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:23.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:23.607 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:23.641 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:23.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:23.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:23.745 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:23.771 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:23.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:23.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:23.874 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:23.910 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:23.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:23.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:24.014 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:24.040 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:24.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:24.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:24.070 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:23 smithi098 bash[79516]: cluster 2023-09-23T18:55:22.254583+0000 mgr.y (mgr.26054) 1122 : cluster 0 pgmap v1208: 89 pgs: 1 active+clean+remapped, 28 active+clean, 9 active+undersized+degraded, 35 active+undersized, 12 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:24.070 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:23 smithi098 bash[79516]: cluster 2023-09-23T18:55:22.924262+0000 osd.6 (osd.6) 248 : cluster 3 failed to encode map e279 with expected crc 2023-09-23T18:55:24.071 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:23 smithi098 bash[79516]: audit 2023-09-23T18:55:22.927126+0000 mon.a (mon.0) 1527 : audit 1 from='client.? 172.21.15.98:0/85380039' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T18:55:24.071 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:23 smithi098 bash[79516]: cluster 2023-09-23T18:55:22.927202+0000 mon.a (mon.0) 1528 : cluster 0 osdmap e279: 8 total, 6 up, 8 in 2023-09-23T18:55:24.071 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:23 smithi098 bash[79516]: cluster 2023-09-23T18:55:22.927320+0000 osd.5 (osd.5) 186 : cluster 3 failed to encode map e279 with expected crc 2023-09-23T18:55:24.071 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:23 smithi098 bash[79516]: cluster 2023-09-23T18:55:22.929710+0000 osd.5 (osd.5) 187 : cluster 3 failed to encode map e279 with expected crc 2023-09-23T18:55:24.071 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:23 smithi098 bash[76817]: cluster 2023-09-23T18:55:22.254583+0000 mgr.y (mgr.26054) 1122 : cluster 0 pgmap v1208: 89 pgs: 1 active+clean+remapped, 28 active+clean, 9 active+undersized+degraded, 35 active+undersized, 12 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:24.071 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:23 smithi098 bash[76817]: cluster 2023-09-23T18:55:22.924262+0000 osd.6 (osd.6) 248 : cluster 3 failed to encode map e279 with expected crc 2023-09-23T18:55:24.071 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:23 smithi098 bash[76817]: audit 2023-09-23T18:55:22.927126+0000 mon.a (mon.0) 1527 : audit 1 from='client.? 172.21.15.98:0/85380039' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T18:55:24.071 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:23 smithi098 bash[76817]: cluster 2023-09-23T18:55:22.927202+0000 mon.a (mon.0) 1528 : cluster 0 osdmap e279: 8 total, 6 up, 8 in 2023-09-23T18:55:24.071 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:23 smithi098 bash[76817]: cluster 2023-09-23T18:55:22.927320+0000 osd.5 (osd.5) 186 : cluster 3 failed to encode map e279 with expected crc 2023-09-23T18:55:24.071 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:23 smithi098 bash[76817]: cluster 2023-09-23T18:55:22.929710+0000 osd.5 (osd.5) 187 : cluster 3 failed to encode map e279 with expected crc 2023-09-23T18:55:24.143 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:24.156 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:23 smithi173 bash[59422]: cluster 2023-09-23T18:55:22.254583+0000 mgr.y (mgr.26054) 1122 : cluster 0 pgmap v1208: 89 pgs: 1 active+clean+remapped, 28 active+clean, 9 active+undersized+degraded, 35 active+undersized, 12 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:24.156 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:23 smithi173 bash[59422]: cluster 2023-09-23T18:55:22.924262+0000 osd.6 (osd.6) 248 : cluster 3 failed to encode map e279 with expected crc 2023-09-23T18:55:24.156 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:23 smithi173 bash[59422]: audit 2023-09-23T18:55:22.927126+0000 mon.a (mon.0) 1527 : audit 1 from='client.? 172.21.15.98:0/85380039' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T18:55:24.156 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:23 smithi173 bash[59422]: cluster 2023-09-23T18:55:22.927202+0000 mon.a (mon.0) 1528 : cluster 0 osdmap e279: 8 total, 6 up, 8 in 2023-09-23T18:55:24.156 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:23 smithi173 bash[59422]: cluster 2023-09-23T18:55:22.927320+0000 osd.5 (osd.5) 186 : cluster 3 failed to encode map e279 with expected crc 2023-09-23T18:55:24.156 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:23 smithi173 bash[59422]: cluster 2023-09-23T18:55:22.929710+0000 osd.5 (osd.5) 187 : cluster 3 failed to encode map e279 with expected crc 2023-09-23T18:55:24.175 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:24.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:24.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:24.280 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:24.299 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:24.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:24.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:24.403 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:24.436 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:24.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:24.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:24.541 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:24.572 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:24.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:24.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:24.676 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:24.702 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:24.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:24.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:24.805 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:24.833 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:24.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:24.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:24.937 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:24.969 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:24.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:24.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:25.072 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:25.104 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:25.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:25.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:25.208 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:25.238 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:25.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:25.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:25.245 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:24 smithi098 bash[79516]: cluster 2023-09-23T18:55:23.925748+0000 osd.6 (osd.6) 249 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T18:55:25.245 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:24 smithi098 bash[79516]: cluster 2023-09-23T18:55:23.927720+0000 mon.a (mon.0) 1529 : cluster 0 osdmap e280: 8 total, 6 up, 8 in 2023-09-23T18:55:25.246 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:24 smithi098 bash[79516]: cluster 2023-09-23T18:55:23.928276+0000 osd.5 (osd.5) 188 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T18:55:25.246 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:24 smithi098 bash[79516]: cluster 2023-09-23T18:55:23.931611+0000 osd.5 (osd.5) 189 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T18:55:25.246 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:24 smithi098 bash[79516]: cluster 2023-09-23T18:55:23.934205+0000 osd.5 (osd.5) 190 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T18:55:25.246 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:24 smithi098 bash[79516]: audit 2023-09-23T18:55:24.630177+0000 mon.a (mon.0) 1530 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:55:25.247 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:24 smithi098 bash[76817]: cluster 2023-09-23T18:55:23.925748+0000 osd.6 (osd.6) 249 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T18:55:25.247 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:24 smithi098 bash[76817]: cluster 2023-09-23T18:55:23.927720+0000 mon.a (mon.0) 1529 : cluster 0 osdmap e280: 8 total, 6 up, 8 in 2023-09-23T18:55:25.247 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:24 smithi098 bash[76817]: cluster 2023-09-23T18:55:23.928276+0000 osd.5 (osd.5) 188 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T18:55:25.247 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:24 smithi098 bash[76817]: cluster 2023-09-23T18:55:23.931611+0000 osd.5 (osd.5) 189 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T18:55:25.247 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:24 smithi098 bash[76817]: cluster 2023-09-23T18:55:23.934205+0000 osd.5 (osd.5) 190 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T18:55:25.247 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:24 smithi098 bash[76817]: audit 2023-09-23T18:55:24.630177+0000 mon.a (mon.0) 1530 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:55:25.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:24 smithi173 bash[59422]: cluster 2023-09-23T18:55:23.925748+0000 osd.6 (osd.6) 249 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T18:55:25.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:24 smithi173 bash[59422]: cluster 2023-09-23T18:55:23.927720+0000 mon.a (mon.0) 1529 : cluster 0 osdmap e280: 8 total, 6 up, 8 in 2023-09-23T18:55:25.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:24 smithi173 bash[59422]: cluster 2023-09-23T18:55:23.928276+0000 osd.5 (osd.5) 188 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T18:55:25.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:24 smithi173 bash[59422]: cluster 2023-09-23T18:55:23.931611+0000 osd.5 (osd.5) 189 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T18:55:25.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:24 smithi173 bash[59422]: cluster 2023-09-23T18:55:23.934205+0000 osd.5 (osd.5) 190 : cluster 3 failed to encode map e280 with expected crc 2023-09-23T18:55:25.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:24 smithi173 bash[59422]: audit 2023-09-23T18:55:24.630177+0000 mon.a (mon.0) 1530 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:55:25.342 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:25.374 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:25.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:25.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:25.478 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:25.506 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:25.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:25.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:25.609 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:25.637 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:25.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:25.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:25.740 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:25.775 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:25.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:25.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:25.878 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:25.904 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:25.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:25.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:26.007 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:26.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:25 smithi173 bash[59422]: cluster 2023-09-23T18:55:24.255311+0000 mgr.y (mgr.26054) 1123 : cluster 0 pgmap v1211: 89 pgs: 1 active+clean+remapped, 28 active+clean, 9 active+undersized+degraded, 35 active+undersized, 12 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:26.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:25 smithi173 bash[59422]: cluster 2023-09-23T18:55:24.943849+0000 osd.6 (osd.6) 250 : cluster 3 failed to encode map e281 with expected crc 2023-09-23T18:55:26.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:25 smithi173 bash[59422]: cluster 2023-09-23T18:55:24.945074+0000 mon.a (mon.0) 1531 : cluster 0 osdmap e281: 8 total, 6 up, 8 in 2023-09-23T18:55:26.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:25 smithi173 bash[59422]: cluster 2023-09-23T18:55:24.947312+0000 osd.5 (osd.5) 191 : cluster 3 failed to encode map e281 with expected crc 2023-09-23T18:55:26.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:25 smithi173 bash[59422]: cluster 2023-09-23T18:55:24.950383+0000 osd.5 (osd.5) 192 : cluster 3 failed to encode map e281 with expected crc 2023-09-23T18:55:26.035 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:26.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:26.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:26.138 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:26.169 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:26.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:26.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:26.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:25 smithi098 bash[79516]: cluster 2023-09-23T18:55:24.255311+0000 mgr.y (mgr.26054) 1123 : cluster 0 pgmap v1211: 89 pgs: 1 active+clean+remapped, 28 active+clean, 9 active+undersized+degraded, 35 active+undersized, 12 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:26.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:25 smithi098 bash[79516]: cluster 2023-09-23T18:55:24.943849+0000 osd.6 (osd.6) 250 : cluster 3 failed to encode map e281 with expected crc 2023-09-23T18:55:26.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:25 smithi098 bash[79516]: cluster 2023-09-23T18:55:24.945074+0000 mon.a (mon.0) 1531 : cluster 0 osdmap e281: 8 total, 6 up, 8 in 2023-09-23T18:55:26.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:25 smithi098 bash[79516]: cluster 2023-09-23T18:55:24.947312+0000 osd.5 (osd.5) 191 : cluster 3 failed to encode map e281 with expected crc 2023-09-23T18:55:26.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:25 smithi098 bash[79516]: cluster 2023-09-23T18:55:24.950383+0000 osd.5 (osd.5) 192 : cluster 3 failed to encode map e281 with expected crc 2023-09-23T18:55:26.193 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:55:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:55:26] "GET /metrics HTTP/1.1" 200 36517 "" "Prometheus/2.33.4" 2023-09-23T18:55:26.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:25 smithi098 bash[76817]: cluster 2023-09-23T18:55:24.255311+0000 mgr.y (mgr.26054) 1123 : cluster 0 pgmap v1211: 89 pgs: 1 active+clean+remapped, 28 active+clean, 9 active+undersized+degraded, 35 active+undersized, 12 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 28/84 objects degraded (33.333%); 2/84 objects misplaced (2.381%) 2023-09-23T18:55:26.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:25 smithi098 bash[76817]: cluster 2023-09-23T18:55:24.943849+0000 osd.6 (osd.6) 250 : cluster 3 failed to encode map e281 with expected crc 2023-09-23T18:55:26.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:25 smithi098 bash[76817]: cluster 2023-09-23T18:55:24.945074+0000 mon.a (mon.0) 1531 : cluster 0 osdmap e281: 8 total, 6 up, 8 in 2023-09-23T18:55:26.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:25 smithi098 bash[76817]: cluster 2023-09-23T18:55:24.947312+0000 osd.5 (osd.5) 191 : cluster 3 failed to encode map e281 with expected crc 2023-09-23T18:55:26.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:25 smithi098 bash[76817]: cluster 2023-09-23T18:55:24.950383+0000 osd.5 (osd.5) 192 : cluster 3 failed to encode map e281 with expected crc 2023-09-23T18:55:26.273 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:26.288 INFO:tasks.thrashosds.thrasher:Added osd 5 2023-09-23T18:55:26.301 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:26.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:26.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:26.405 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:26.428 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:26.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:26.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:26.532 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:26.565 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:26.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:26.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:26.668 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:26.702 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:26.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:26.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:26.757 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:55:26.807 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:26.839 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:26.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:26.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:26.943 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:26.966 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:26.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:26.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:27.066 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:26 smithi098 bash[76817]: cluster 2023-09-23T18:55:24.952242+0000 osd.5 (osd.5) 193 : cluster 3 failed to encode map e281 with expected crc 2023-09-23T18:55:27.066 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:26 smithi098 bash[79516]: cluster 2023-09-23T18:55:24.952242+0000 osd.5 (osd.5) 193 : cluster 3 failed to encode map e281 with expected crc 2023-09-23T18:55:27.069 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:27.080 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:26 smithi173 bash[59422]: cluster 2023-09-23T18:55:24.952242+0000 osd.5 (osd.5) 193 : cluster 3 failed to encode map e281 with expected crc 2023-09-23T18:55:27.100 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:27.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:27.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:27.204 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:27.226 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:27.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:27.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:27.331 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:27.352 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:27.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:27.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:27.455 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:27.484 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:27.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:27.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:27.587 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:27.621 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:27.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:27.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:27.724 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:27.757 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:27.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:27.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:27.861 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:27.893 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:27.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:27.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:27.996 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:28.021 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:28.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:28.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:28.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:27 smithi173 bash[59422]: cluster 2023-09-23T18:55:26.256845+0000 mgr.y (mgr.26054) 1124 : cluster 0 pgmap v1213: 89 pgs: 5 active+clean+remapped, 29 active+clean, 7 active+undersized+degraded, 34 active+undersized, 10 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:28.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:27 smithi173 bash[59422]: cluster 2023-09-23T18:55:26.941507+0000 mon.a (mon.0) 1532 : cluster 3 Health check update: Degraded data redundancy: 26/84 objects degraded (30.952%), 11 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:55:28.124 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:28.152 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:28.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:28.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:28.242 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:27 smithi098 bash[79516]: cluster 2023-09-23T18:55:26.256845+0000 mgr.y (mgr.26054) 1124 : cluster 0 pgmap v1213: 89 pgs: 5 active+clean+remapped, 29 active+clean, 7 active+undersized+degraded, 34 active+undersized, 10 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:28.242 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:27 smithi098 bash[79516]: cluster 2023-09-23T18:55:26.941507+0000 mon.a (mon.0) 1532 : cluster 3 Health check update: Degraded data redundancy: 26/84 objects degraded (30.952%), 11 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:55:28.243 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:27 smithi098 bash[76817]: cluster 2023-09-23T18:55:26.256845+0000 mgr.y (mgr.26054) 1124 : cluster 0 pgmap v1213: 89 pgs: 5 active+clean+remapped, 29 active+clean, 7 active+undersized+degraded, 34 active+undersized, 10 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:28.243 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:27 smithi098 bash[76817]: cluster 2023-09-23T18:55:26.941507+0000 mon.a (mon.0) 1532 : cluster 3 Health check update: Degraded data redundancy: 26/84 objects degraded (30.952%), 11 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:55:28.256 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:28.282 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:28.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:28.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:28.385 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:28.405 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:28.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:28.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:28.460 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:55:28.461 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [], 2023-09-23T18:55:28.461 INFO:teuthology.orchestra.run.smithi098.stdout: "complaint_time": 30, 2023-09-23T18:55:28.461 INFO:teuthology.orchestra.run.smithi098.stdout: "num_blocked_ops": 0 2023-09-23T18:55:28.461 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:55:28.507 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:28.539 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:28.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:28.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:28.642 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:28.677 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:28.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:28.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:28.780 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:28.802 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:28.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:28.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:28.906 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:28.931 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:28.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:28.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:29.035 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:29.057 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:29.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:29.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:29.160 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:29.181 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:29.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:29.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:29.284 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:29.323 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:29.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:29.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:29.427 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:29.459 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:29.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:29.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:29.562 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:29.586 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:29.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:29.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:29.689 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:29.710 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:29.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:29.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:29.813 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:29.838 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:29.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:29.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:29.942 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:29.968 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:29.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:29.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:30.071 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:30.093 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:30.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:30.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:30.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:30 smithi098 bash[76817]: cluster 2023-09-23T18:55:28.257622+0000 mgr.y (mgr.26054) 1125 : cluster 0 pgmap v1214: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:30.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:30 smithi098 bash[79516]: cluster 2023-09-23T18:55:28.257622+0000 mgr.y (mgr.26054) 1125 : cluster 0 pgmap v1214: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:30.195 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:30.219 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:30.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:30.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:30.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:29 smithi173 bash[59422]: cluster 2023-09-23T18:55:28.257622+0000 mgr.y (mgr.26054) 1125 : cluster 0 pgmap v1214: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:30.322 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:30.354 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:30.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:30.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:30.458 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:30.487 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:30.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:30.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:30.591 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:30.622 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:30.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:30.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:30.725 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:30.757 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:30.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:30.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:30.862 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:30.894 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:30.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:30.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:30.997 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:31.027 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:31.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:31.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:31.131 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:31.152 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:31.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:31.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:31.256 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:31.283 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:31.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:31.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:31.288 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7, 5] out_osds: [] dead_osds: [4, 7] live_osds: [0, 2, 3, 5, 6, 1] 2023-09-23T18:55:31.289 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:55:31.289 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-09-23T18:55:31.289 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T18:55:31.289 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T18:55:31.289 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-09-23T18:55:31.386 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:31.414 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:31.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:31.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:31.516 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:31.545 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:31.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:31.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:31.648 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:31.684 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:31.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:31.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:31.787 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:31.822 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:31.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:31.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:31.926 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:31.955 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:31.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:31.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:32.058 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:32.086 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:32.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:32.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:32.189 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:32.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:31 smithi098 bash[79516]: cluster 2023-09-23T18:55:30.258307+0000 mgr.y (mgr.26054) 1126 : cluster 0 pgmap v1215: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:32.192 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:31 smithi098 bash[76817]: cluster 2023-09-23T18:55:30.258307+0000 mgr.y (mgr.26054) 1126 : cluster 0 pgmap v1215: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:32.214 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:32.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:32.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:32.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:31 smithi173 bash[59422]: cluster 2023-09-23T18:55:30.258307+0000 mgr.y (mgr.26054) 1126 : cluster 0 pgmap v1215: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:32.317 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:32.343 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:32.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:32.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:32.447 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:32.477 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:32.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:32.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:32.580 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:32.583 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T18:55:32.601 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:32.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:32.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:32.705 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:32.731 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:32.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:32.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:32.834 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:32.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:32.871 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:32.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:32.972 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:32.995 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:32.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:32.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:33.098 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:33.124 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:33.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:33.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:33.228 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:33.252 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:33.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:33.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:33.355 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:33.379 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:33.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:33.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:33.482 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:33.507 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:33.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:33.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:33.610 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:33.637 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:33.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:33.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:33.741 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:33.770 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:33.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:33.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:33.873 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:33.898 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:33.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:33.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:34.003 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:34.027 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:34.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:33 smithi173 bash[59422]: cluster 2023-09-23T18:55:32.259049+0000 mgr.y (mgr.26054) 1127 : cluster 0 pgmap v1216: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:34.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:34.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:34.130 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:34.160 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:34.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:34.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:34.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:34 smithi098 bash[79516]: cluster 2023-09-23T18:55:32.259049+0000 mgr.y (mgr.26054) 1127 : cluster 0 pgmap v1216: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:34.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:34 smithi098 bash[76817]: cluster 2023-09-23T18:55:32.259049+0000 mgr.y (mgr.26054) 1127 : cluster 0 pgmap v1216: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:34.265 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:34.291 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:34.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:34.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:34.395 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:34.421 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:34.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:34.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:34.524 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:34.549 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:34.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:34.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:34.652 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:34.685 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:34.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:34.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:34.789 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:34.817 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:34.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:34.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:34.920 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:34.948 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:34.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:34.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:35.051 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:35.079 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:35.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:35.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:35.183 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:35.207 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:35.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:35.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:35.311 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:35.342 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:35.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:35.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:35.445 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:35.476 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:35.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:35.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:35.580 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:35.608 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:35.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:35.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:35.711 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:35.737 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:35.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:35.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:35.840 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:35.864 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:35.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:35.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:35.968 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:35.972 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:35 smithi098 bash[76817]: cluster 2023-09-23T18:55:34.259853+0000 mgr.y (mgr.26054) 1128 : cluster 0 pgmap v1217: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:36.000 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:36.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:36.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:36.051 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:55:36.104 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:36.130 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:36.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:36.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:36.233 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:36.244 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:35 smithi098 bash[79516]: cluster 2023-09-23T18:55:34.259853+0000 mgr.y (mgr.26054) 1128 : cluster 0 pgmap v1217: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:36.244 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:55:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:55:36] "GET /metrics HTTP/1.1" 200 36517 "" "Prometheus/2.33.4" 2023-09-23T18:55:36.262 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:36.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:36.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:36.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:35 smithi173 bash[59422]: cluster 2023-09-23T18:55:34.259853+0000 mgr.y (mgr.26054) 1128 : cluster 0 pgmap v1217: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:36.366 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:36.396 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:36.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:36.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:36.499 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:36.525 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:36.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:36.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:36.627 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:36.653 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:36.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:36.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:36.757 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:36.788 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:36.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:36.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:36.892 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:36.911 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:36.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:36.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:36.931 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:55:36.957 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:55:36.985 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:55:36.989 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is set 2023-09-23T18:55:37.009 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:55:37.013 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:37.033 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:55:37.042 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:37.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:37.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:37.052 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:55:37.053 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:55:37.053 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:55:37.053 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:55:37.053 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:55:37.053 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-09-23T18:55:37.054 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:55:37.054 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:55:37.054 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T18:55:37.146 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:37.173 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:37.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:37.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:37.276 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:37.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:36 smithi173 bash[59422]: audit 2023-09-23T18:55:36.792109+0000 mon.c (mon.2) 125 : audit 1 from='client.? 172.21.15.98:0/692668458' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:55:37.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:36 smithi173 bash[59422]: audit 2023-09-23T18:55:36.792525+0000 mon.a (mon.0) 1533 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:55:37.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:36 smithi173 bash[59422]: audit 2023-09-23T18:55:36.899815+0000 mon.c (mon.2) 126 : audit 0 from='client.? 172.21.15.98:0/3218971394' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:55:37.305 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:37.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:37.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:37.383 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:36 smithi098 bash[79516]: audit 2023-09-23T18:55:36.792109+0000 mon.c (mon.2) 125 : audit 1 from='client.? 172.21.15.98:0/692668458' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:55:37.383 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:36 smithi098 bash[79516]: audit 2023-09-23T18:55:36.792525+0000 mon.a (mon.0) 1533 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:55:37.384 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:36 smithi098 bash[79516]: audit 2023-09-23T18:55:36.899815+0000 mon.c (mon.2) 126 : audit 0 from='client.? 172.21.15.98:0/3218971394' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:55:37.384 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:36 smithi098 bash[76817]: audit 2023-09-23T18:55:36.792109+0000 mon.c (mon.2) 125 : audit 1 from='client.? 172.21.15.98:0/692668458' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:55:37.384 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:36 smithi098 bash[76817]: audit 2023-09-23T18:55:36.792525+0000 mon.a (mon.0) 1533 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:55:37.384 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:36 smithi098 bash[76817]: audit 2023-09-23T18:55:36.899815+0000 mon.c (mon.2) 126 : audit 0 from='client.? 172.21.15.98:0/3218971394' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:55:37.408 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:37.433 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:37.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:37.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:37.536 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:37.566 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:37.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:37.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:37.669 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:37.692 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:37.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:37.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:37.796 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:37.823 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:37.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:37.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:37.927 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:37.956 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:37.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:37.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:38.060 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:38.087 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:38.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:38.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:38.191 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:38.216 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:38.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:38.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:38.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:37 smithi173 bash[59422]: cluster 2023-09-23T18:55:36.260608+0000 mgr.y (mgr.26054) 1129 : cluster 0 pgmap v1218: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:38.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:37 smithi173 bash[59422]: cluster 2023-09-23T18:55:36.967869+0000 mon.a (mon.0) 1534 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:55:38.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:37 smithi173 bash[59422]: cluster 2023-09-23T18:55:36.981277+0000 osd.6 (osd.6) 251 : cluster 3 failed to encode map e282 with expected crc 2023-09-23T18:55:38.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:37 smithi173 bash[59422]: audit 2023-09-23T18:55:36.981743+0000 mon.a (mon.0) 1535 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:55:38.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:37 smithi173 bash[59422]: cluster 2023-09-23T18:55:36.981816+0000 mon.a (mon.0) 1536 : cluster 0 osdmap e282: 8 total, 6 up, 8 in 2023-09-23T18:55:38.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:37 smithi173 bash[59422]: cluster 2023-09-23T18:55:37.113747+0000 osd.5 (osd.5) 194 : cluster 3 failed to encode map e282 with expected crc 2023-09-23T18:55:38.320 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:38.354 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:38.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:38.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:38.364 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:37 smithi098 bash[79516]: cluster 2023-09-23T18:55:36.260608+0000 mgr.y (mgr.26054) 1129 : cluster 0 pgmap v1218: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:38.364 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:37 smithi098 bash[79516]: cluster 2023-09-23T18:55:36.967869+0000 mon.a (mon.0) 1534 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:55:38.364 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:37 smithi098 bash[79516]: cluster 2023-09-23T18:55:36.981277+0000 osd.6 (osd.6) 251 : cluster 3 failed to encode map e282 with expected crc 2023-09-23T18:55:38.364 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:37 smithi098 bash[79516]: audit 2023-09-23T18:55:36.981743+0000 mon.a (mon.0) 1535 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:55:38.364 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:37 smithi098 bash[79516]: cluster 2023-09-23T18:55:36.981816+0000 mon.a (mon.0) 1536 : cluster 0 osdmap e282: 8 total, 6 up, 8 in 2023-09-23T18:55:38.364 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:37 smithi098 bash[79516]: cluster 2023-09-23T18:55:37.113747+0000 osd.5 (osd.5) 194 : cluster 3 failed to encode map e282 with expected crc 2023-09-23T18:55:38.364 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:37 smithi098 bash[76817]: cluster 2023-09-23T18:55:36.260608+0000 mgr.y (mgr.26054) 1129 : cluster 0 pgmap v1218: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:38.364 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:37 smithi098 bash[76817]: cluster 2023-09-23T18:55:36.967869+0000 mon.a (mon.0) 1534 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:55:38.364 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:37 smithi098 bash[76817]: cluster 2023-09-23T18:55:36.981277+0000 osd.6 (osd.6) 251 : cluster 3 failed to encode map e282 with expected crc 2023-09-23T18:55:38.365 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:37 smithi098 bash[76817]: audit 2023-09-23T18:55:36.981743+0000 mon.a (mon.0) 1535 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:55:38.365 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:37 smithi098 bash[76817]: cluster 2023-09-23T18:55:36.981816+0000 mon.a (mon.0) 1536 : cluster 0 osdmap e282: 8 total, 6 up, 8 in 2023-09-23T18:55:38.365 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:37 smithi098 bash[76817]: cluster 2023-09-23T18:55:37.113747+0000 osd.5 (osd.5) 194 : cluster 3 failed to encode map e282 with expected crc 2023-09-23T18:55:38.459 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:38.489 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:38.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:38.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:38.593 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:38.620 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:38.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:38.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:38.724 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:38.747 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:38.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:38.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:38.850 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:38.875 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:38.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:38.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:38.980 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:39.007 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:39.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:39.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:39.111 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:39.138 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:39.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:39.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:39.241 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:39.270 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:39.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:39.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:39.373 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:39.398 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:39.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:39.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:39.502 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:39.528 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:39.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:39.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:39.631 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:39.654 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:39.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:39.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:39.756 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:39.777 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:39.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:39.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:39.880 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:39.912 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:39.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:39.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:39.954 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:55:39.954 INFO:teuthology.orchestra.run.smithi173.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T18:55:39.954 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:55:40.016 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:40.039 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:40.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:40.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:40.142 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:40.171 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:40.172 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:39 smithi173 bash[59422]: cluster 2023-09-23T18:55:38.261350+0000 mgr.y (mgr.26054) 1130 : cluster 0 pgmap v1220: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:40.172 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:39 smithi173 bash[59422]: audit 2023-09-23T18:55:39.635914+0000 mon.a (mon.0) 1537 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:40.172 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:39 smithi173 bash[59422]: audit 2023-09-23T18:55:39.637605+0000 mon.a (mon.0) 1538 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:55:40.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:40.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:40.275 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:40.286 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:39 smithi098 bash[79516]: cluster 2023-09-23T18:55:38.261350+0000 mgr.y (mgr.26054) 1130 : cluster 0 pgmap v1220: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:40.286 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:39 smithi098 bash[79516]: audit 2023-09-23T18:55:39.635914+0000 mon.a (mon.0) 1537 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:40.286 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:39 smithi098 bash[79516]: audit 2023-09-23T18:55:39.637605+0000 mon.a (mon.0) 1538 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:55:40.286 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:39 smithi098 bash[76817]: cluster 2023-09-23T18:55:38.261350+0000 mgr.y (mgr.26054) 1130 : cluster 0 pgmap v1220: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:40.287 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:39 smithi098 bash[76817]: audit 2023-09-23T18:55:39.635914+0000 mon.a (mon.0) 1537 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:55:40.287 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:39 smithi098 bash[76817]: audit 2023-09-23T18:55:39.637605+0000 mon.a (mon.0) 1538 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:55:40.306 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:40.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:40.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:40.410 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:40.442 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:40.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:40.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:40.546 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:40.570 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:40.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:40.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:40.604 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:55:40.605 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:55:40.673 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:40.703 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:40.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:40.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:40.806 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:40.840 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:40.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:40.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:40.944 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:40.977 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:40.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:40.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:41.081 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:41.113 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:41.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:41.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:41.216 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:41.248 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:41.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:41.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:41.351 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:41.383 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:41.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:41.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:41.487 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:41.524 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:41.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:41.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:41.627 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:41.662 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:41.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:41.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:41.766 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:41.799 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:41.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:41.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:41.903 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:41.937 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:41.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:41.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:42.001 INFO:teuthology.orchestra.run.smithi098.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-09-23T18:55:42.041 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:42.052 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:41 smithi098 bash[79516]: cluster 2023-09-23T18:55:40.262195+0000 mgr.y (mgr.26054) 1131 : cluster 0 pgmap v1221: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:42.052 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:41 smithi098 bash[79516]: audit 2023-09-23T18:55:41.132064+0000 mon.a (mon.0) 1539 : audit 1 from='client.? 172.21.15.98:0/4327017' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:55:42.052 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:41 smithi098 bash[76817]: cluster 2023-09-23T18:55:40.262195+0000 mgr.y (mgr.26054) 1131 : cluster 0 pgmap v1221: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:42.053 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:41 smithi098 bash[76817]: audit 2023-09-23T18:55:41.132064+0000 mon.a (mon.0) 1539 : audit 1 from='client.? 172.21.15.98:0/4327017' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:55:42.076 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:42.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:42.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:42.179 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:42.211 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:42.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:42.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:42.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:41 smithi173 bash[59422]: cluster 2023-09-23T18:55:40.262195+0000 mgr.y (mgr.26054) 1131 : cluster 0 pgmap v1221: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:42.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:41 smithi173 bash[59422]: audit 2023-09-23T18:55:41.132064+0000 mon.a (mon.0) 1539 : audit 1 from='client.? 172.21.15.98:0/4327017' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:55:42.315 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:42.340 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:42.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:42.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:42.443 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:42.474 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:42.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:42.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:42.576 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:55:42.577 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:42.606 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:55:42.606 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:42.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:42.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:42.710 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:42.741 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:42.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:42.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:42.846 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:42.875 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:42.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:42.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:42.979 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:43.006 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:43.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:43.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:43.109 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:43.138 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:43.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:43.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:43.241 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:43.253 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:42 smithi098 bash[76817]: audit 2023-09-23T18:55:41.991444+0000 mon.a (mon.0) 1540 : audit 1 from='client.? 172.21.15.98:0/4327017' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:55:43.254 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:42 smithi098 bash[76817]: cluster 2023-09-23T18:55:41.991476+0000 osd.6 (osd.6) 252 : cluster 3 failed to encode map e283 with expected crc 2023-09-23T18:55:43.254 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:42 smithi098 bash[76817]: cluster 2023-09-23T18:55:41.991554+0000 mon.a (mon.0) 1541 : cluster 0 osdmap e283: 8 total, 6 up, 8 in 2023-09-23T18:55:43.254 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:42 smithi098 bash[76817]: cluster 2023-09-23T18:55:42.324271+0000 osd.5 (osd.5) 195 : cluster 3 failed to encode map e283 with expected crc 2023-09-23T18:55:43.254 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:42 smithi098 bash[79516]: audit 2023-09-23T18:55:41.991444+0000 mon.a (mon.0) 1540 : audit 1 from='client.? 172.21.15.98:0/4327017' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:55:43.254 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:42 smithi098 bash[79516]: cluster 2023-09-23T18:55:41.991476+0000 osd.6 (osd.6) 252 : cluster 3 failed to encode map e283 with expected crc 2023-09-23T18:55:43.254 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:42 smithi098 bash[79516]: cluster 2023-09-23T18:55:41.991554+0000 mon.a (mon.0) 1541 : cluster 0 osdmap e283: 8 total, 6 up, 8 in 2023-09-23T18:55:43.254 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:42 smithi098 bash[79516]: cluster 2023-09-23T18:55:42.324271+0000 osd.5 (osd.5) 195 : cluster 3 failed to encode map e283 with expected crc 2023-09-23T18:55:43.272 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:43.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:43.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:43.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:42 smithi173 bash[59422]: audit 2023-09-23T18:55:41.991444+0000 mon.a (mon.0) 1540 : audit 1 from='client.? 172.21.15.98:0/4327017' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:55:43.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:42 smithi173 bash[59422]: cluster 2023-09-23T18:55:41.991476+0000 osd.6 (osd.6) 252 : cluster 3 failed to encode map e283 with expected crc 2023-09-23T18:55:43.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:42 smithi173 bash[59422]: cluster 2023-09-23T18:55:41.991554+0000 mon.a (mon.0) 1541 : cluster 0 osdmap e283: 8 total, 6 up, 8 in 2023-09-23T18:55:43.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:42 smithi173 bash[59422]: cluster 2023-09-23T18:55:42.324271+0000 osd.5 (osd.5) 195 : cluster 3 failed to encode map e283 with expected crc 2023-09-23T18:55:43.375 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:43.402 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:43.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:43.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:43.506 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:43.540 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:43.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:43.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:43.644 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:43.675 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:43.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:43.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:43.688 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:55:43.779 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:43.805 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:43.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:43.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:43.909 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:43.943 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:43.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:43.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:44.046 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:44.057 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:43 smithi098 bash[79516]: cluster 2023-09-23T18:55:42.263319+0000 mgr.y (mgr.26054) 1132 : cluster 0 pgmap v1223: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:44.057 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:43 smithi098 bash[76817]: cluster 2023-09-23T18:55:42.263319+0000 mgr.y (mgr.26054) 1132 : cluster 0 pgmap v1223: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:44.072 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:44.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:44.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:44.177 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:44.189 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:44 smithi173 bash[59422]: cluster 2023-09-23T18:55:42.263319+0000 mgr.y (mgr.26054) 1132 : cluster 0 pgmap v1223: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:44.212 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:44.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:44.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:44.267 DEBUG:teuthology.orchestra.run.smithi098:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-09-23T18:55:44.270 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T18:55:44.270 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T18:55:44.316 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:44.319 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:hints = 1 2023-09-23T18:55:44.319 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T18:55:44.319 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Object prefix: benchmark_data_smithi098_206513 2023-09-23T18:55:44.319 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:55:44.319 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 0 0 0 0 0 0 - 0 2023-09-23T18:55:44.341 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:44.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:44.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:44.444 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:44.476 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:44.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:44.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:44.579 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:44.612 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:44.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:44.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:44.716 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:44.744 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:44.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:44.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:44.848 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:44.881 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:44.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:44.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:44.984 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:45.014 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:45.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:45.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:45.118 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:45.145 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:45.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:45.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:45.249 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:45.277 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:45.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:45.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:45.320 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 1 16 131 115 7.1871 7.1875 0.00200611 0.0363185 2023-09-23T18:55:45.381 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:45.414 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:45.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:45.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:45.518 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:45.550 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:45.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:45.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:45.655 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:45.686 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:45.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:45.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:45.789 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:45.811 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:45.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:45.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:45.862 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7, 5] out_osds: [] dead_osds: [4, 7] live_osds: [0, 2, 3, 5, 6, 1] 2023-09-23T18:55:45.862 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:55:45.862 INFO:tasks.thrashosds.thrasher:Reviving osd 4 2023-09-23T18:55:45.862 INFO:tasks.cephadm.osd.4:Restarting osd.4 (starting--it wasn't running)... 2023-09-23T18:55:45.862 DEBUG:teuthology.orchestra.run.smithi173:osd.4> sudo journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.4.service 2023-09-23T18:55:45.865 DEBUG:teuthology.orchestra.run.smithi173:> sudo systemctl start ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.4 2023-09-23T18:55:45.914 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:45.935 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T18:55:45.941 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:45.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:45.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:45.957 INFO:journalctl@ceph.osd.4.smithi173.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T18:55:46.044 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:46.071 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:46.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:46.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:46.174 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:46.206 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:46.207 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:46 smithi098 bash[76817]: cluster 2023-09-23T18:55:44.264425+0000 mgr.y (mgr.26054) 1133 : cluster 0 pgmap v1224: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:46.208 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:55:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:55:46] "GET /metrics HTTP/1.1" 200 36519 "" "Prometheus/2.33.4" 2023-09-23T18:55:46.208 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:46 smithi098 bash[79516]: cluster 2023-09-23T18:55:44.264425+0000 mgr.y (mgr.26054) 1133 : cluster 0 pgmap v1224: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:46.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:46.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:46.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:46 smithi173 bash[59422]: cluster 2023-09-23T18:55:44.264425+0000 mgr.y (mgr.26054) 1133 : cluster 0 pgmap v1224: 89 pgs: 5 active+clean+remapped, 30 active+clean, 7 active+undersized+degraded, 34 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 26/84 objects degraded (30.952%); 4/84 objects misplaced (4.762%) 2023-09-23T18:55:46.310 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:46.320 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 2 16 131 115 3.59337 0 - 0.0363185 2023-09-23T18:55:46.343 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:46.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:46.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:46.446 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:46.477 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:46.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:46.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:46.581 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:46.615 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:46.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:46.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:46.719 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:46.744 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:46.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:46.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:46.848 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:46.877 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:46.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:46.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:46.981 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:47.007 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:47.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:47.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:47.110 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:47.124 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:47 smithi098 bash[79516]: cluster 2023-09-23T18:55:47.001319+0000 mon.a (mon.0) 1542 : cluster 3 Health check update: Degraded data redundancy: 36/189 objects degraded (19.048%), 13 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:55:47.124 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:47 smithi098 bash[76817]: cluster 2023-09-23T18:55:47.001319+0000 mon.a (mon.0) 1542 : cluster 3 Health check update: Degraded data redundancy: 36/189 objects degraded (19.048%), 13 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:55:47.141 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:47.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:47.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:47.245 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:47.273 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:47.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:47.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:47 smithi173 bash[59422]: cluster 2023-09-23T18:55:47.001319+0000 mon.a (mon.0) 1542 : cluster 3 Health check update: Degraded data redundancy: 36/189 objects degraded (19.048%), 13 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:55:47.320 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 3 16 131 115 2.39558 0 - 0.0363185 2023-09-23T18:55:47.376 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:47.404 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:47.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:47.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:47.508 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:47.543 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:47.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:47.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:47.646 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:47.675 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:47.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:47.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:47.778 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:47.800 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:55:47.800 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:55:47.814 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:47.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:47.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:47.919 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:47.946 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:47.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:47.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:48.050 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:48.076 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:48.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:48.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:48.103 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:48 smithi098 bash[76817]: cluster 2023-09-23T18:55:46.265734+0000 mgr.y (mgr.26054) 1134 : cluster 0 pgmap v1225: 89 pgs: 5 active+clean+remapped, 30 active+clean, 9 active+undersized+degraded, 32 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 6.8 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 241 KiB/s wr, 3 op/s; 36/189 objects degraded (19.048%); 4/189 objects misplaced (2.116%) 2023-09-23T18:55:48.103 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:48 smithi098 bash[79516]: cluster 2023-09-23T18:55:46.265734+0000 mgr.y (mgr.26054) 1134 : cluster 0 pgmap v1225: 89 pgs: 5 active+clean+remapped, 30 active+clean, 9 active+undersized+degraded, 32 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 6.8 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 241 KiB/s wr, 3 op/s; 36/189 objects degraded (19.048%); 4/189 objects misplaced (2.116%) 2023-09-23T18:55:48.180 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:48.205 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:48.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:48.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:48.310 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:48.320 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 4 16 131 115 1.79669 0 - 0.0363185 2023-09-23T18:55:48.321 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:48 smithi173 bash[59422]: cluster 2023-09-23T18:55:46.265734+0000 mgr.y (mgr.26054) 1134 : cluster 0 pgmap v1225: 89 pgs: 5 active+clean+remapped, 30 active+clean, 9 active+undersized+degraded, 32 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 6.8 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 241 KiB/s wr, 3 op/s; 36/189 objects degraded (19.048%); 4/189 objects misplaced (2.116%) 2023-09-23T18:55:48.321 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 18:55:48 smithi173 bash[130509]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-09-23T18:55:48.321 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 18:55:48 smithi173 bash[130509]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2023-09-23T18:55:48.322 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 18:55:48 smithi173 bash[130509]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2023-09-23T18:55:48.322 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 18:55:48 smithi173 bash[130509]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-09-23T18:55:48.322 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 18:55:48 smithi173 bash[130509]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2023-09-23T18:55:48.322 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 18:55:48 smithi173 bash[130509]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-09-23T18:55:48.322 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 18:55:48 smithi173 bash[130509]: --> ceph-volume raw activate successful for osd ID: 4 2023-09-23T18:55:48.337 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:48.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:48.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:48.417 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:55:48.417 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:55:48.441 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:48.473 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:48.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:48.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:48.576 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:48.603 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:48.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:48.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:48.707 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:48.736 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:48.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:48.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:48.839 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:48.868 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:48.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:48.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:48.972 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:49.006 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:49.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:49.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:49.110 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:49.133 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:49.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:49.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:49.236 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:49.262 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:49.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:49.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:49.320 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 5 16 131 115 1.43734 0 - 0.0363185 2023-09-23T18:55:49.366 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:49.400 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:49.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:49.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:49.505 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:49.527 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:49.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:49.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:49.631 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:49.659 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:49.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:49.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:49.763 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:49.786 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:49.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:49.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:49.889 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:49.915 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:49.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:49.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:50.019 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:50.045 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:50.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:50.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:50.148 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:50.174 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:50.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:50.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:50.278 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:50.303 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:50.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:50.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:50.320 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 6 16 131 115 1.19778 0 - 0.0363185 2023-09-23T18:55:50.341 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:50 smithi173 bash[59422]: cluster 2023-09-23T18:55:48.267028+0000 mgr.y (mgr.26054) 1135 : cluster 0 pgmap v1226: 89 pgs: 5 active+clean+remapped, 30 active+clean, 15 active+undersized+degraded, 26 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 11 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 608 KiB/s wr, 9 op/s; 81/369 objects degraded (21.951%); 4/369 objects misplaced (1.084%) 2023-09-23T18:55:50.407 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:50.419 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:50 smithi098 bash[79516]: cluster 2023-09-23T18:55:48.267028+0000 mgr.y (mgr.26054) 1135 : cluster 0 pgmap v1226: 89 pgs: 5 active+clean+remapped, 30 active+clean, 15 active+undersized+degraded, 26 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 11 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 608 KiB/s wr, 9 op/s; 81/369 objects degraded (21.951%); 4/369 objects misplaced (1.084%) 2023-09-23T18:55:50.420 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:50 smithi098 bash[76817]: cluster 2023-09-23T18:55:48.267028+0000 mgr.y (mgr.26054) 1135 : cluster 0 pgmap v1226: 89 pgs: 5 active+clean+remapped, 30 active+clean, 15 active+undersized+degraded, 26 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 11 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 608 KiB/s wr, 9 op/s; 81/369 objects degraded (21.951%); 4/369 objects misplaced (1.084%) 2023-09-23T18:55:50.438 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:50.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:50.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:50.542 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:50.571 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:50.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:50.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:50.674 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:50.687 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 18:55:50 smithi173 bash[130836]: debug 2023-09-23T18:55:50.336+0000 7fd750ab1700 1 -- 172.21.15.173:0/2224890696 <== mon.1 v2:172.21.15.173:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c90e9524e0 con 0x55c90e9c5400 2023-09-23T18:55:50.703 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:50.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:50.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:50.806 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:50.838 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:50.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:50.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:50.942 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:50.953 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 18:55:50 smithi173 bash[130836]: debug 2023-09-23T18:55:50.880+0000 7fd75852b200 -1 Falling back to public interface 2023-09-23T18:55:50.971 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:50.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:50.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:51.075 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:51.106 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:51.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:51.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:51.210 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:51.244 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:51.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:51.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:51.270 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T18:55:51.320 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 7 16 131 115 1.02666 0 - 0.0363185 2023-09-23T18:55:51.348 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:51.377 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:51.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:51.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:51.481 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:51.513 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:51.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:51.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:51.617 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:51.646 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:51.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:51.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:51.751 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:51.776 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:51.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:51.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:51.879 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:51.911 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:51.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:51.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:52.014 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:52.048 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:52.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:52.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:52.152 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:52.182 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:52.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:52.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:52.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:52 smithi173 bash[59422]: cluster 2023-09-23T18:55:50.268412+0000 mgr.y (mgr.26054) 1136 : cluster 0 pgmap v1227: 89 pgs: 5 active+clean+remapped, 30 active+clean, 15 active+undersized+degraded, 26 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 12 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 736 KiB/s wr, 11 op/s; 92/429 objects degraded (21.445%); 4/429 objects misplaced (0.932%) 2023-09-23T18:55:52.286 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:52.313 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:52.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:52.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:52.321 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 8 16 131 115 0.898332 0 - 0.0363185 2023-09-23T18:55:52.417 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:52.430 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:52 smithi098 bash[76817]: cluster 2023-09-23T18:55:50.268412+0000 mgr.y (mgr.26054) 1136 : cluster 0 pgmap v1227: 89 pgs: 5 active+clean+remapped, 30 active+clean, 15 active+undersized+degraded, 26 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 12 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 736 KiB/s wr, 11 op/s; 92/429 objects degraded (21.445%); 4/429 objects misplaced (0.932%) 2023-09-23T18:55:52.431 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:52 smithi098 bash[79516]: cluster 2023-09-23T18:55:50.268412+0000 mgr.y (mgr.26054) 1136 : cluster 0 pgmap v1227: 89 pgs: 5 active+clean+remapped, 30 active+clean, 15 active+undersized+degraded, 26 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 12 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 736 KiB/s wr, 11 op/s; 92/429 objects degraded (21.445%); 4/429 objects misplaced (0.932%) 2023-09-23T18:55:52.451 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:52.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:52.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:52.554 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:52.584 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:52.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:52.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:52.605 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:55:52.687 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:52.719 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:52.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:52.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:52.824 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:52.851 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:52.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:52.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:52.954 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:52.983 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:52.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:52.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:53.087 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:53.119 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:53.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:53.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:53.222 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:53.250 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:53.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:53.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:53.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:53 smithi173 bash[59422]: cluster 2023-09-23T18:55:53.064872+0000 mon.a (mon.0) 1543 : cluster 3 Health check update: Reduced data availability: 11 pgs inactive (PG_AVAILABILITY) 2023-09-23T18:55:53.321 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 9 16 131 115 0.798515 0 - 0.0363185 2023-09-23T18:55:53.354 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:53.381 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:53.382 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:53 smithi098 bash[76817]: cluster 2023-09-23T18:55:53.064872+0000 mon.a (mon.0) 1543 : cluster 3 Health check update: Reduced data availability: 11 pgs inactive (PG_AVAILABILITY) 2023-09-23T18:55:53.383 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:53 smithi098 bash[79516]: cluster 2023-09-23T18:55:53.064872+0000 mon.a (mon.0) 1543 : cluster 3 Health check update: Reduced data availability: 11 pgs inactive (PG_AVAILABILITY) 2023-09-23T18:55:53.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:53.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:53.485 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:53.514 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:53.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:53.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:53.617 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:53.647 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:53.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:53.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:53.752 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:53.779 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:53.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:53.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:53.882 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:53.907 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:53.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:53.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:54.011 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:54.038 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:54.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:54.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:54.142 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:54.163 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:54.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:54.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:54.266 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:54.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:54 smithi173 bash[59422]: cluster 2023-09-23T18:55:52.270030+0000 mgr.y (mgr.26054) 1137 : cluster 0 pgmap v1228: 89 pgs: 5 active+clean+remapped, 30 active+clean, 15 active+undersized+degraded, 26 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 12 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 716 KiB/s wr, 11 op/s; 92/429 objects degraded (21.445%); 4/429 objects misplaced (0.932%) 2023-09-23T18:55:54.278 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:54 smithi098 bash[76817]: cluster 2023-09-23T18:55:52.270030+0000 mgr.y (mgr.26054) 1137 : cluster 0 pgmap v1228: 89 pgs: 5 active+clean+remapped, 30 active+clean, 15 active+undersized+degraded, 26 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 12 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 716 KiB/s wr, 11 op/s; 92/429 objects degraded (21.445%); 4/429 objects misplaced (0.932%) 2023-09-23T18:55:54.279 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:54 smithi098 bash[79516]: cluster 2023-09-23T18:55:52.270030+0000 mgr.y (mgr.26054) 1137 : cluster 0 pgmap v1228: 89 pgs: 5 active+clean+remapped, 30 active+clean, 15 active+undersized+degraded, 26 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 12 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 716 KiB/s wr, 11 op/s; 92/429 objects degraded (21.445%); 4/429 objects misplaced (0.932%) 2023-09-23T18:55:54.294 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:54.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:54.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:54.321 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 10 16 131 115 0.718665 0 - 0.0363185 2023-09-23T18:55:54.399 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:54.424 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:54.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:54.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:54.528 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:54.558 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:54.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:54.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:54.662 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:54.687 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:54.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:54.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:54.791 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:54.814 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:54.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:54.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:54.918 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:54.947 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:54.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:54.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:55.051 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:55.081 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:55.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:55.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:55.184 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:55.192 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:55 smithi098 bash[79516]: audit 2023-09-23T18:55:54.630829+0000 mon.a (mon.0) 1544 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:55:55.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:55 smithi098 bash[76817]: audit 2023-09-23T18:55:54.630829+0000 mon.a (mon.0) 1544 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:55:55.212 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:55.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:55.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:55.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:55 smithi173 bash[59422]: audit 2023-09-23T18:55:54.630829+0000 mon.a (mon.0) 1544 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:55:55.315 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:55.321 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 11 16 131 115 0.653333 0 - 0.0363185 2023-09-23T18:55:55.345 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:55.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:55.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:55.448 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:55.482 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:55.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:55.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:55.586 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:55.613 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:55.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:55.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:55.652 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:55:55.716 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:55.734 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:55.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:55.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:55.837 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:55.869 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:55.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:55.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:55.973 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:55.998 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:56.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:56.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:56.101 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:56.113 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:55:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:55:56] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.33.4" 2023-09-23T18:55:56.126 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:56.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:56.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:56.216 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:55:56.229 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:56.242 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:55:56.247 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:56 smithi173 bash[59422]: cluster 2023-09-23T18:55:54.271721+0000 mgr.y (mgr.26054) 1138 : cluster 0 pgmap v1229: 89 pgs: 5 active+clean+remapped, 30 active+clean, 15 active+undersized+degraded, 26 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 12 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 613 KiB/s wr, 9 op/s; 92/429 objects degraded (21.445%); 4/429 objects misplaced (0.932%) 2023-09-23T18:55:56.247 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:56 smithi173 bash[59422]: cluster 2023-09-23T18:55:55.120321+0000 mon.a (mon.0) 1545 : cluster 3 Health check update: Degraded data redundancy: 92/429 objects degraded (21.445%), 19 pgs degraded, 43 pgs undersized (PG_DEGRADED) 2023-09-23T18:55:56.248 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:56.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:56.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:56.270 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:55:56.293 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:55:56.315 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:55:56.321 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 12 16 131 115 0.59889 0 - 0.0363185 2023-09-23T18:55:56.342 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:55:56.343 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:55:56.343 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:55:56.343 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:55:56.343 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:55:56.343 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-09-23T18:55:56.343 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:55:56.343 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:55:56.343 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T18:55:56.351 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:56.378 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:56.380 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:56 smithi098 bash[79516]: cluster 2023-09-23T18:55:54.271721+0000 mgr.y (mgr.26054) 1138 : cluster 0 pgmap v1229: 89 pgs: 5 active+clean+remapped, 30 active+clean, 15 active+undersized+degraded, 26 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 12 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 613 KiB/s wr, 9 op/s; 92/429 objects degraded (21.445%); 4/429 objects misplaced (0.932%) 2023-09-23T18:55:56.380 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:56 smithi098 bash[79516]: cluster 2023-09-23T18:55:55.120321+0000 mon.a (mon.0) 1545 : cluster 3 Health check update: Degraded data redundancy: 92/429 objects degraded (21.445%), 19 pgs degraded, 43 pgs undersized (PG_DEGRADED) 2023-09-23T18:55:56.380 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:56 smithi098 bash[76817]: cluster 2023-09-23T18:55:54.271721+0000 mgr.y (mgr.26054) 1138 : cluster 0 pgmap v1229: 89 pgs: 5 active+clean+remapped, 30 active+clean, 15 active+undersized+degraded, 26 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 12 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 613 KiB/s wr, 9 op/s; 92/429 objects degraded (21.445%); 4/429 objects misplaced (0.932%) 2023-09-23T18:55:56.380 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:56 smithi098 bash[76817]: cluster 2023-09-23T18:55:55.120321+0000 mon.a (mon.0) 1545 : cluster 3 Health check update: Degraded data redundancy: 92/429 objects degraded (21.445%), 19 pgs degraded, 43 pgs undersized (PG_DEGRADED) 2023-09-23T18:55:56.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:56.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:56.482 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:56.505 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:56.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:56.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:56.608 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:56.635 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:56.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:56.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:56.738 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:56.765 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:56.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:56.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:56.870 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:56.894 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:56.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:56.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:56.997 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:57.030 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:57.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:57.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:57.134 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:57.137 INFO:teuthology.orchestra.run.smithi173.stderr:no valid command found; 10 closest matches: 2023-09-23T18:55:57.137 INFO:teuthology.orchestra.run.smithi173.stderr:0 2023-09-23T18:55:57.137 INFO:teuthology.orchestra.run.smithi173.stderr:1 2023-09-23T18:55:57.137 INFO:teuthology.orchestra.run.smithi173.stderr:2 2023-09-23T18:55:57.137 INFO:teuthology.orchestra.run.smithi173.stderr:abort 2023-09-23T18:55:57.137 INFO:teuthology.orchestra.run.smithi173.stderr:assert 2023-09-23T18:55:57.137 INFO:teuthology.orchestra.run.smithi173.stderr:bluefs debug_inject_read_zeros 2023-09-23T18:55:57.137 INFO:teuthology.orchestra.run.smithi173.stderr:bluefs files list 2023-09-23T18:55:57.138 INFO:teuthology.orchestra.run.smithi173.stderr:bluefs stats 2023-09-23T18:55:57.138 INFO:teuthology.orchestra.run.smithi173.stderr:bluestore allocator dump block 2023-09-23T18:55:57.138 INFO:teuthology.orchestra.run.smithi173.stderr:bluestore allocator fragmentation block 2023-09-23T18:55:57.138 INFO:teuthology.orchestra.run.smithi173.stderr:admin_socket: invalid command 2023-09-23T18:55:57.154 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is set 2023-09-23T18:55:57.158 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:57.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:57.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:57.262 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:57.270 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:57 smithi098 bash[79516]: audit 2023-09-23T18:55:56.184822+0000 mon.a (mon.0) 1546 : audit 0 from='client.? 172.21.15.98:0/2222120451' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:55:57.270 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:57 smithi098 bash[79516]: audit 2023-09-23T18:55:56.433956+0000 mon.c (mon.2) 127 : audit 1 from='client.? 172.21.15.98:0/1870269230' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:55:57.270 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:57 smithi098 bash[79516]: audit 2023-09-23T18:55:56.434520+0000 mon.a (mon.0) 1547 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:55:57.270 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:57 smithi098 bash[76817]: audit 2023-09-23T18:55:56.184822+0000 mon.a (mon.0) 1546 : audit 0 from='client.? 172.21.15.98:0/2222120451' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:55:57.271 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:57 smithi098 bash[76817]: audit 2023-09-23T18:55:56.433956+0000 mon.c (mon.2) 127 : audit 1 from='client.? 172.21.15.98:0/1870269230' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:55:57.271 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:57 smithi098 bash[76817]: audit 2023-09-23T18:55:56.434520+0000 mon.a (mon.0) 1547 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:55:57.285 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:57.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:57.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:57.321 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 13 16 131 115 0.552822 0 - 0.0363185 2023-09-23T18:55:57.388 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:57.410 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:57 smithi173 bash[59422]: audit 2023-09-23T18:55:56.184822+0000 mon.a (mon.0) 1546 : audit 0 from='client.? 172.21.15.98:0/2222120451' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:55:57.410 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:57 smithi173 bash[59422]: audit 2023-09-23T18:55:56.433956+0000 mon.c (mon.2) 127 : audit 1 from='client.? 172.21.15.98:0/1870269230' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:55:57.410 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:57 smithi173 bash[59422]: audit 2023-09-23T18:55:56.434520+0000 mon.a (mon.0) 1547 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:55:57.417 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:57.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:57.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:57.520 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:57.546 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:57.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:57.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:57.639 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:55:57.640 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:55:57.649 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:57.671 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:57.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:57.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:57.774 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:57.805 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:57.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:57.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:57.909 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:57.939 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:57.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:57.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:58.043 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:58.066 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:58.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:58.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:58.169 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:58.188 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:58.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:58.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:58.291 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:58.313 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:55:58.315 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:58 smithi173 bash[59422]: cluster 2023-09-23T18:55:56.273575+0000 mgr.y (mgr.26054) 1139 : cluster 0 pgmap v1230: 89 pgs: 5 active+clean+remapped, 30 active+clean, 15 active+undersized+degraded, 26 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 12 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 613 KiB/s wr, 9 op/s; 92/429 objects degraded (21.445%); 4/429 objects misplaced (0.932%) 2023-09-23T18:55:58.315 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:58 smithi173 bash[59422]: cluster 2023-09-23T18:55:57.131090+0000 mon.a (mon.0) 1548 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:55:58.315 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:58 smithi173 bash[59422]: audit 2023-09-23T18:55:57.147140+0000 mon.a (mon.0) 1549 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:55:58.315 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:58 smithi173 bash[59422]: cluster 2023-09-23T18:55:57.147266+0000 mon.a (mon.0) 1550 : cluster 0 osdmap e284: 8 total, 6 up, 8 in 2023-09-23T18:55:58.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:58.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:55:58.321 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 14 16 131 115 0.513332 0 - 0.0363185 2023-09-23T18:55:58.417 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:58.430 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:58 smithi098 bash[76817]: cluster 2023-09-23T18:55:56.273575+0000 mgr.y (mgr.26054) 1139 : cluster 0 pgmap v1230: 89 pgs: 5 active+clean+remapped, 30 active+clean, 15 active+undersized+degraded, 26 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 12 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 613 KiB/s wr, 9 op/s; 92/429 objects degraded (21.445%); 4/429 objects misplaced (0.932%) 2023-09-23T18:55:58.430 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:58 smithi098 bash[76817]: cluster 2023-09-23T18:55:57.131090+0000 mon.a (mon.0) 1548 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:55:58.430 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:58 smithi098 bash[76817]: audit 2023-09-23T18:55:57.147140+0000 mon.a (mon.0) 1549 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:55:58.430 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:58 smithi098 bash[76817]: cluster 2023-09-23T18:55:57.147266+0000 mon.a (mon.0) 1550 : cluster 0 osdmap e284: 8 total, 6 up, 8 in 2023-09-23T18:55:58.431 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:58 smithi098 bash[79516]: cluster 2023-09-23T18:55:56.273575+0000 mgr.y (mgr.26054) 1139 : cluster 0 pgmap v1230: 89 pgs: 5 active+clean+remapped, 30 active+clean, 15 active+undersized+degraded, 26 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 12 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 613 KiB/s wr, 9 op/s; 92/429 objects degraded (21.445%); 4/429 objects misplaced (0.932%) 2023-09-23T18:55:58.431 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:58 smithi098 bash[79516]: cluster 2023-09-23T18:55:57.131090+0000 mon.a (mon.0) 1548 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:55:58.431 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:58 smithi098 bash[79516]: audit 2023-09-23T18:55:57.147140+0000 mon.a (mon.0) 1549 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:55:58.431 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:58 smithi098 bash[79516]: cluster 2023-09-23T18:55:57.147266+0000 mon.a (mon.0) 1550 : cluster 0 osdmap e284: 8 total, 6 up, 8 in 2023-09-23T18:55:58.448 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:55:58.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:58.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:55:58.552 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:58.582 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:58.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:58.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:58.685 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:58.712 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:58.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:58.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:58.817 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:58.850 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:58.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:58.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:58.954 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:58.985 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:58.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:58.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:59.028 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 18:55:58 smithi173 bash[130836]: debug 2023-09-23T18:55:58.583+0000 7fd75852b200 -1 osd.4 262 log_to_monitors {default=true} 2023-09-23T18:55:59.089 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:59.115 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:59.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:59.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:55:59.139 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T18:55:59.139 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-4, ['dump_ops_in_flight'] 2023-09-23T18:55:59.219 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:59.230 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:59 smithi098 bash[76817]: cluster 2023-09-23T18:55:58.403052+0000 osd.6 (osd.6) 253 : cluster 3 failed to encode map e284 with expected crc 2023-09-23T18:55:59.231 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:59 smithi098 bash[76817]: cluster 2023-09-23T18:55:58.427925+0000 osd.5 (osd.5) 196 : cluster 3 failed to encode map e284 with expected crc 2023-09-23T18:55:59.231 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:59 smithi098 bash[76817]: audit 2023-09-23T18:55:58.590200+0000 mon.a (mon.0) 1551 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T18:55:59.231 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:55:59 smithi098 bash[76817]: audit 2023-09-23T18:55:58.591511+0000 mon.b (mon.1) 23 : audit 1 from='osd.4 [v2:172.21.15.173:6800/1430610213,v1:172.21.15.173:6801/1430610213]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T18:55:59.231 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:59 smithi098 bash[79516]: cluster 2023-09-23T18:55:58.403052+0000 osd.6 (osd.6) 253 : cluster 3 failed to encode map e284 with expected crc 2023-09-23T18:55:59.231 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:59 smithi098 bash[79516]: cluster 2023-09-23T18:55:58.427925+0000 osd.5 (osd.5) 196 : cluster 3 failed to encode map e284 with expected crc 2023-09-23T18:55:59.231 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:59 smithi098 bash[79516]: audit 2023-09-23T18:55:58.590200+0000 mon.a (mon.0) 1551 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T18:55:59.231 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:55:59 smithi098 bash[79516]: audit 2023-09-23T18:55:58.591511+0000 mon.b (mon.1) 23 : audit 1 from='osd.4 [v2:172.21.15.173:6800/1430610213,v1:172.21.15.173:6801/1430610213]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T18:55:59.251 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:59.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:59.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:59.321 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 15 16 131 115 0.47911 0 - 0.0363185 2023-09-23T18:55:59.356 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:59.391 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:59.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:59.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:59.495 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:59.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:59 smithi173 bash[59422]: cluster 2023-09-23T18:55:58.403052+0000 osd.6 (osd.6) 253 : cluster 3 failed to encode map e284 with expected crc 2023-09-23T18:55:59.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:59 smithi173 bash[59422]: cluster 2023-09-23T18:55:58.427925+0000 osd.5 (osd.5) 196 : cluster 3 failed to encode map e284 with expected crc 2023-09-23T18:55:59.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:59 smithi173 bash[59422]: audit 2023-09-23T18:55:58.590200+0000 mon.a (mon.0) 1551 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T18:55:59.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:55:59 smithi173 bash[59422]: audit 2023-09-23T18:55:58.591511+0000 mon.b (mon.1) 23 : audit 1 from='osd.4 [v2:172.21.15.173:6800/1430610213,v1:172.21.15.173:6801/1430610213]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T18:55:59.529 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 18:55:59 smithi173 bash[130836]: debug 2023-09-23T18:55:59.175+0000 7fd74b229700 -1 osd.4 262 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T18:55:59.532 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:55:59.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:59.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:55:59.636 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:59.640 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:55:59.669 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:55:59.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:59.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:55:59.773 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:59.800 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:55:59.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:59.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:55:59.904 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:59.926 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:55:59.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:55:59.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:00.030 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:00.061 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:00.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:00.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:00.165 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:00.176 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:00 smithi173 bash[59422]: cluster 2023-09-23T18:55:58.275380+0000 mgr.y (mgr.26054) 1140 : cluster 0 pgmap v1232: 89 pgs: 5 active+clean+remapped, 30 active+clean, 15 active+undersized+degraded, 26 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 12 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 128 KiB/s wr, 1 op/s; 92/429 objects degraded (21.445%); 4/429 objects misplaced (0.932%) 2023-09-23T18:56:00.176 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:00 smithi173 bash[59422]: cluster 2023-09-23T18:55:59.153327+0000 osd.6 (osd.6) 254 : cluster 3 failed to encode map e285 with expected crc 2023-09-23T18:56:00.176 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:00 smithi173 bash[59422]: audit 2023-09-23T18:55:59.154903+0000 mon.a (mon.0) 1552 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T18:56:00.176 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:00 smithi173 bash[59422]: cluster 2023-09-23T18:55:59.155041+0000 mon.a (mon.0) 1553 : cluster 0 osdmap e285: 8 total, 6 up, 8 in 2023-09-23T18:56:00.177 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:00 smithi173 bash[59422]: audit 2023-09-23T18:55:59.157394+0000 mon.a (mon.0) 1554 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:56:00.177 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:00 smithi173 bash[59422]: audit 2023-09-23T18:55:59.158587+0000 mon.b (mon.1) 24 : audit 1 from='osd.4 [v2:172.21.15.173:6800/1430610213,v1:172.21.15.173:6801/1430610213]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:56:00.177 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:00 smithi173 bash[59422]: cluster 2023-09-23T18:55:59.172896+0000 osd.5 (osd.5) 197 : cluster 3 failed to encode map e285 with expected crc 2023-09-23T18:56:00.192 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:00.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:00.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:00.295 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:00.309 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:00 smithi098 bash[76817]: cluster 2023-09-23T18:55:58.275380+0000 mgr.y (mgr.26054) 1140 : cluster 0 pgmap v1232: 89 pgs: 5 active+clean+remapped, 30 active+clean, 15 active+undersized+degraded, 26 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 12 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 128 KiB/s wr, 1 op/s; 92/429 objects degraded (21.445%); 4/429 objects misplaced (0.932%) 2023-09-23T18:56:00.309 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:00 smithi098 bash[76817]: cluster 2023-09-23T18:55:59.153327+0000 osd.6 (osd.6) 254 : cluster 3 failed to encode map e285 with expected crc 2023-09-23T18:56:00.310 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:00 smithi098 bash[76817]: audit 2023-09-23T18:55:59.154903+0000 mon.a (mon.0) 1552 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T18:56:00.310 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:00 smithi098 bash[76817]: cluster 2023-09-23T18:55:59.155041+0000 mon.a (mon.0) 1553 : cluster 0 osdmap e285: 8 total, 6 up, 8 in 2023-09-23T18:56:00.310 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:00 smithi098 bash[76817]: audit 2023-09-23T18:55:59.157394+0000 mon.a (mon.0) 1554 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:56:00.310 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:00 smithi098 bash[76817]: audit 2023-09-23T18:55:59.158587+0000 mon.b (mon.1) 24 : audit 1 from='osd.4 [v2:172.21.15.173:6800/1430610213,v1:172.21.15.173:6801/1430610213]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:56:00.310 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:00 smithi098 bash[76817]: cluster 2023-09-23T18:55:59.172896+0000 osd.5 (osd.5) 197 : cluster 3 failed to encode map e285 with expected crc 2023-09-23T18:56:00.310 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:00 smithi098 bash[79516]: cluster 2023-09-23T18:55:58.275380+0000 mgr.y (mgr.26054) 1140 : cluster 0 pgmap v1232: 89 pgs: 5 active+clean+remapped, 30 active+clean, 15 active+undersized+degraded, 26 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 12 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 128 KiB/s wr, 1 op/s; 92/429 objects degraded (21.445%); 4/429 objects misplaced (0.932%) 2023-09-23T18:56:00.310 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:00 smithi098 bash[79516]: cluster 2023-09-23T18:55:59.153327+0000 osd.6 (osd.6) 254 : cluster 3 failed to encode map e285 with expected crc 2023-09-23T18:56:00.310 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:00 smithi098 bash[79516]: audit 2023-09-23T18:55:59.154903+0000 mon.a (mon.0) 1552 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T18:56:00.310 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:00 smithi098 bash[79516]: cluster 2023-09-23T18:55:59.155041+0000 mon.a (mon.0) 1553 : cluster 0 osdmap e285: 8 total, 6 up, 8 in 2023-09-23T18:56:00.311 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:00 smithi098 bash[79516]: audit 2023-09-23T18:55:59.157394+0000 mon.a (mon.0) 1554 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:56:00.311 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:00 smithi098 bash[79516]: audit 2023-09-23T18:55:59.158587+0000 mon.b (mon.1) 24 : audit 1 from='osd.4 [v2:172.21.15.173:6800/1430610213,v1:172.21.15.173:6801/1430610213]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T18:56:00.311 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:00 smithi098 bash[79516]: cluster 2023-09-23T18:55:59.172896+0000 osd.5 (osd.5) 197 : cluster 3 failed to encode map e285 with expected crc 2023-09-23T18:56:00.321 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 16 16 131 115 0.449166 0 - 0.0363185 2023-09-23T18:56:00.333 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:00.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:00.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:00.437 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:00.463 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:56:00.463 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-09-23T18:56:00.463 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.463 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45602.0:354 9.f 9:feedfdf2:::benchmark_data_smithi098_206513_object353:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-09-23T18:56:00.463 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:56:00.454809+0000", 2023-09-23T18:56:00.463 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.005218487, 2023-09-23T18:56:00.463 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0052430209999999996, 2023-09-23T18:56:00.464 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:00.464 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:56:00.464 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:56:00.464 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45602", 2023-09-23T18:56:00.464 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T18:56:00.464 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 354 2023-09-23T18:56:00.464 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.464 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:00.465 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.465 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:00.465 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.454809+0000", 2023-09-23T18:56:00.465 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:00.465 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.465 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.465 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:00.465 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.454809+0000", 2023-09-23T18:56:00.465 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:56:00.465 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.466 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.466 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:00.466 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.454808+0000", 2023-09-23T18:56:00.466 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.7637000000000003e-05 2023-09-23T18:56:00.466 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.466 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.466 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:00.466 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.454875+0000", 2023-09-23T18:56:00.466 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1504e-05 2023-09-23T18:56:00.467 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.467 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.467 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:00.467 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.454887+0000", 2023-09-23T18:56:00.467 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.5359999999999998e-06 2023-09-23T18:56:00.467 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.467 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.467 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:00.467 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.454894+0000", 2023-09-23T18:56:00.468 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0017883129999999999 2023-09-23T18:56:00.468 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.468 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.468 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:00.468 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.456682+0000", 2023-09-23T18:56:00.468 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.3842000000000001e-05 2023-09-23T18:56:00.468 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.468 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.468 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:00.469 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.456766+0000", 2023-09-23T18:56:00.469 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012574400000000001 2023-09-23T18:56:00.469 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.469 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.469 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3", 2023-09-23T18:56:00.469 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.456891+0000", 2023-09-23T18:56:00.469 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00039417500000000003 2023-09-23T18:56:00.469 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.470 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.470 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:56:00.470 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.457286+0000", 2023-09-23T18:56:00.470 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002187184 2023-09-23T18:56:00.470 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.470 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.470 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:56:00.470 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.459473+0000", 2023-09-23T18:56:00.470 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0046635139999999997 2023-09-23T18:56:00.471 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:00.471 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:00.471 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:00.471 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.471 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.471 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45602.0:366 9.6 9:634fd67e:::benchmark_data_smithi098_206513_object365:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-09-23T18:56:00.472 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:56:00.457890+0000", 2023-09-23T18:56:00.472 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.0021377480000000001, 2023-09-23T18:56:00.472 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002298783, 2023-09-23T18:56:00.472 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:00.472 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:56:00.472 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:56:00.472 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45602", 2023-09-23T18:56:00.472 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T18:56:00.473 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 366 2023-09-23T18:56:00.473 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.473 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:00.473 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.473 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:00.473 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.457890+0000", 2023-09-23T18:56:00.473 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:00.473 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.473 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.474 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:00.474 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.457890+0000", 2023-09-23T18:56:00.474 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:56:00.474 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.474 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.474 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:00.474 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.457889+0000", 2023-09-23T18:56:00.474 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00010215 2023-09-23T18:56:00.474 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.475 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.475 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:00.475 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.457991+0000", 2023-09-23T18:56:00.475 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1770000000000001e-05 2023-09-23T18:56:00.475 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.475 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.475 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:00.475 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.458003+0000", 2023-09-23T18:56:00.475 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.0529999999999997e-06 2023-09-23T18:56:00.475 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.476 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.476 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:00.476 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.458008+0000", 2023-09-23T18:56:00.476 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.2342000000000001e-05 2023-09-23T18:56:00.476 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.476 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.476 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:00.476 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.458060+0000", 2023-09-23T18:56:00.476 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013137199999999999 2023-09-23T18:56:00.476 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.477 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.477 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:00.477 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.458192+0000", 2023-09-23T18:56:00.477 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00026397600000000002 2023-09-23T18:56:00.477 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.477 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.477 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,2", 2023-09-23T18:56:00.477 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.458456+0000", 2023-09-23T18:56:00.477 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00081265700000000005 2023-09-23T18:56:00.478 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.478 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.478 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:56:00.478 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.459268+0000", 2023-09-23T18:56:00.478 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00066790999999999997 2023-09-23T18:56:00.478 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.478 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.478 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:56:00.478 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.459936+0000", 2023-09-23T18:56:00.478 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0020460999999999999 2023-09-23T18:56:00.479 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:00.479 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:00.479 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:00.479 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.479 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.479 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45602.0:367 9.f 9:f7eb05c0:::benchmark_data_smithi098_206513_object366:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-09-23T18:56:00.479 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:56:00.458016+0000", 2023-09-23T18:56:00.479 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.00201125, 2023-09-23T18:56:00.479 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0022849799999999998, 2023-09-23T18:56:00.479 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:00.480 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:56:00.480 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:56:00.480 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45602", 2023-09-23T18:56:00.480 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T18:56:00.480 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 367 2023-09-23T18:56:00.480 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.480 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:00.480 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.480 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:00.480 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.458016+0000", 2023-09-23T18:56:00.481 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:00.481 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.481 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.481 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:00.481 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.458016+0000", 2023-09-23T18:56:00.481 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-09-23T18:56:00.481 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.481 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.481 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:00.481 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.458016+0000", 2023-09-23T18:56:00.481 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.4574999999999999e-05 2023-09-23T18:56:00.482 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.482 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.482 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:00.482 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.458060+0000", 2023-09-23T18:56:00.482 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.8980000000000004e-06 2023-09-23T18:56:00.482 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.482 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.482 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:00.482 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.458065+0000", 2023-09-23T18:56:00.482 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.0749999999999999e-06 2023-09-23T18:56:00.483 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.483 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.483 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:00.483 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.458069+0000", 2023-09-23T18:56:00.483 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.8297999999999999e-05 2023-09-23T18:56:00.483 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.483 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.483 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:00.483 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.458098+0000", 2023-09-23T18:56:00.483 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000107293 2023-09-23T18:56:00.484 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.484 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.484 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:00.484 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.458205+0000", 2023-09-23T18:56:00.484 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013650700000000001 2023-09-23T18:56:00.484 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.484 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.484 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3", 2023-09-23T18:56:00.484 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.458342+0000", 2023-09-23T18:56:00.484 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0011367549999999999 2023-09-23T18:56:00.485 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.485 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.485 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:56:00.485 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.459478+0000", 2023-09-23T18:56:00.485 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001461824 2023-09-23T18:56:00.485 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:00.485 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:00.485 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:00.485 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.485 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.486 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45602.0:370 9.b e287/279)", 2023-09-23T18:56:00.486 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:56:00.459454+0000", 2023-09-23T18:56:00.486 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.00057372, 2023-09-23T18:56:00.486 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000926361, 2023-09-23T18:56:00.486 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:00.486 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "queued for pg", 2023-09-23T18:56:00.486 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:00.486 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.486 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:00.486 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.459454+0000", 2023-09-23T18:56:00.487 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:00.487 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.487 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.487 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:00.487 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.459454+0000", 2023-09-23T18:56:00.487 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999957 2023-09-23T18:56:00.487 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.487 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.487 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:00.487 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.459450+0000", 2023-09-23T18:56:00.488 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.3206000000000003e-05 2023-09-23T18:56:00.488 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.488 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.488 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:00.488 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.459523+0000", 2023-09-23T18:56:00.488 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.452e-05 2023-09-23T18:56:00.488 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.488 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.488 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:00.488 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.459537+0000", 2023-09-23T18:56:00.489 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.8109999999999997e-06 2023-09-23T18:56:00.489 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.489 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.489 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:00.489 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.459544+0000", 2023-09-23T18:56:00.489 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.0146000000000001e-05 2023-09-23T18:56:00.489 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:00.489 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:00.489 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:00.490 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.490 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.490 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45602.0:359 9.d e287/279 9:b7c6320e:::benchmark_data_smithi098_206513_object358:head v 287'24, mlcod=287'22)", 2023-09-23T18:56:00.490 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:56:00.460118+0000", 2023-09-23T18:56:00.490 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 4294967295.9999099, 2023-09-23T18:56:00.490 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00031120900000000001, 2023-09-23T18:56:00.490 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:00.490 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "started", 2023-09-23T18:56:00.490 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:00.490 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.491 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:00.491 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.460118+0000", 2023-09-23T18:56:00.491 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:00.491 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.491 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.491 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:00.491 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.460118+0000", 2023-09-23T18:56:00.491 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:56:00.491 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.491 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.492 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:00.492 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.460117+0000", 2023-09-23T18:56:00.492 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5405000000000002e-05 2023-09-23T18:56:00.492 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.492 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.492 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:00.492 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.460173+0000", 2023-09-23T18:56:00.492 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3033e-05 2023-09-23T18:56:00.492 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.492 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.492 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:00.493 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.460186+0000", 2023-09-23T18:56:00.493 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.5220000000000004e-06 2023-09-23T18:56:00.493 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.493 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.493 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:00.493 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.460192+0000", 2023-09-23T18:56:00.493 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7299999999999997e-05 2023-09-23T18:56:00.493 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.493 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.493 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:00.494 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.460249+0000", 2023-09-23T18:56:00.494 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.5148999999999999e-05 2023-09-23T18:56:00.494 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.494 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.494 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:00.494 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.460275+0000", 2023-09-23T18:56:00.494 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015652799999999999 2023-09-23T18:56:00.494 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:00.494 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:00.495 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:00.495 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.495 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.495 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop_reply(client.45602.0:367 9.f e287/279 ondisk, result = 0)", 2023-09-23T18:56:00.495 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:56:00.460280+0000", 2023-09-23T18:56:00.495 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 4294967295.9997478, 2023-09-23T18:56:00.495 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00030229699999999998, 2023-09-23T18:56:00.495 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:00.495 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "started", 2023-09-23T18:56:00.495 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:00.495 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.496 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:00.496 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.460280+0000", 2023-09-23T18:56:00.496 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:00.496 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.496 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.496 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:00.496 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.460280+0000", 2023-09-23T18:56:00.496 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-09-23T18:56:00.496 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.497 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.497 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:00.497 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.460279+0000", 2023-09-23T18:56:00.497 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.7780000000000001e-06 2023-09-23T18:56:00.497 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.497 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.497 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:00.497 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.460283+0000", 2023-09-23T18:56:00.497 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.0630000000000006e-06 2023-09-23T18:56:00.498 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.498 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.498 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:00.498 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.460291+0000", 2023-09-23T18:56:00.498 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.2354999999999993e-05 2023-09-23T18:56:00.498 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.498 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.498 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:00.498 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.460374+0000", 2023-09-23T18:56:00.499 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00019032200000000001 2023-09-23T18:56:00.499 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.499 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.499 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:00.499 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.460564+0000", 2023-09-23T18:56:00.499 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.5739999999999998e-06 2023-09-23T18:56:00.499 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.499 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.499 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:00.499 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.460571+0000", 2023-09-23T18:56:00.500 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0718e-05 2023-09-23T18:56:00.500 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:00.500 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:00.500 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:56:00.500 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.460582+0000", 2023-09-23T18:56:00.500 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00030229699999999998 2023-09-23T18:56:00.500 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:00.500 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:00.500 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:00.501 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:00.501 INFO:teuthology.orchestra.run.smithi173.stdout: ], 2023-09-23T18:56:00.501 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 6 2023-09-23T18:56:00.501 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:56:00.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:00.502 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:00.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:00.582 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:56:00.604 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:00.656 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:00.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:00.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:00.761 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:00.791 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:00.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:00.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:00.895 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:00.924 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:00.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:00.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:01.027 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:01.053 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:01.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:01.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:01.156 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:01.185 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:01.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:01.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:01.288 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:01.321 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:01.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:01.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:01.326 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 17 16 2720 2704 9.93845 10.1133 0.00646563 0.100582 2023-09-23T18:56:01.425 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:01.435 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:01 smithi098 bash[79516]: cluster 2023-09-23T18:56:00.151969+0000 mon.a (mon.0) 1555 : cluster 3 Health check update: 1 osds down (OSD_DOWN) 2023-09-23T18:56:01.436 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:01 smithi098 bash[79516]: cluster 2023-09-23T18:56:00.164832+0000 osd.5 (osd.5) 198 : cluster 3 failed to encode map e286 with expected crc 2023-09-23T18:56:01.436 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:01 smithi098 bash[79516]: cluster 2023-09-23T18:56:00.164882+0000 osd.6 (osd.6) 255 : cluster 3 failed to encode map e286 with expected crc 2023-09-23T18:56:01.436 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:01 smithi098 bash[79516]: cluster 2023-09-23T18:56:00.166163+0000 mon.a (mon.0) 1556 : cluster 1 osd.4 [v2:172.21.15.173:6800/1430610213,v1:172.21.15.173:6801/1430610213] boot 2023-09-23T18:56:01.436 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:01 smithi098 bash[79516]: cluster 2023-09-23T18:56:00.166222+0000 mon.a (mon.0) 1557 : cluster 0 osdmap e286: 8 total, 7 up, 8 in 2023-09-23T18:56:01.436 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:01 smithi098 bash[79516]: cluster 2023-09-23T18:56:00.171080+0000 osd.5 (osd.5) 199 : cluster 3 failed to encode map e286 with expected crc 2023-09-23T18:56:01.437 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:01 smithi098 bash[79516]: audit 2023-09-23T18:56:00.178269+0000 mon.a (mon.0) 1558 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:56:01.437 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:01 smithi098 bash[79516]: cluster 2023-09-23T18:56:00.353690+0000 osd.5 (osd.5) 200 : cluster 3 failed to encode map e287 with expected crc 2023-09-23T18:56:01.437 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:01 smithi098 bash[79516]: cluster 2023-09-23T18:56:00.356386+0000 mon.a (mon.0) 1559 : cluster 0 osdmap e287: 8 total, 7 up, 8 in 2023-09-23T18:56:01.437 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:01 smithi098 bash[79516]: cluster 2023-09-23T18:56:00.357221+0000 osd.6 (osd.6) 256 : cluster 3 failed to encode map e287 with expected crc 2023-09-23T18:56:01.437 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:01 smithi098 bash[76817]: cluster 2023-09-23T18:56:00.151969+0000 mon.a (mon.0) 1555 : cluster 3 Health check update: 1 osds down (OSD_DOWN) 2023-09-23T18:56:01.438 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:01 smithi098 bash[76817]: cluster 2023-09-23T18:56:00.164832+0000 osd.5 (osd.5) 198 : cluster 3 failed to encode map e286 with expected crc 2023-09-23T18:56:01.438 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:01 smithi098 bash[76817]: cluster 2023-09-23T18:56:00.164882+0000 osd.6 (osd.6) 255 : cluster 3 failed to encode map e286 with expected crc 2023-09-23T18:56:01.438 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:01 smithi098 bash[76817]: cluster 2023-09-23T18:56:00.166163+0000 mon.a (mon.0) 1556 : cluster 1 osd.4 [v2:172.21.15.173:6800/1430610213,v1:172.21.15.173:6801/1430610213] boot 2023-09-23T18:56:01.438 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:01 smithi098 bash[76817]: cluster 2023-09-23T18:56:00.166222+0000 mon.a (mon.0) 1557 : cluster 0 osdmap e286: 8 total, 7 up, 8 in 2023-09-23T18:56:01.438 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:01 smithi098 bash[76817]: cluster 2023-09-23T18:56:00.171080+0000 osd.5 (osd.5) 199 : cluster 3 failed to encode map e286 with expected crc 2023-09-23T18:56:01.438 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:01 smithi098 bash[76817]: audit 2023-09-23T18:56:00.178269+0000 mon.a (mon.0) 1558 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:56:01.439 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:01 smithi098 bash[76817]: cluster 2023-09-23T18:56:00.353690+0000 osd.5 (osd.5) 200 : cluster 3 failed to encode map e287 with expected crc 2023-09-23T18:56:01.439 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:01 smithi098 bash[76817]: cluster 2023-09-23T18:56:00.356386+0000 mon.a (mon.0) 1559 : cluster 0 osdmap e287: 8 total, 7 up, 8 in 2023-09-23T18:56:01.439 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:01 smithi098 bash[76817]: cluster 2023-09-23T18:56:00.357221+0000 osd.6 (osd.6) 256 : cluster 3 failed to encode map e287 with expected crc 2023-09-23T18:56:01.457 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:01.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:01.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:01.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:01 smithi173 bash[59422]: cluster 2023-09-23T18:56:00.151969+0000 mon.a (mon.0) 1555 : cluster 3 Health check update: 1 osds down (OSD_DOWN) 2023-09-23T18:56:01.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:01 smithi173 bash[59422]: cluster 2023-09-23T18:56:00.164832+0000 osd.5 (osd.5) 198 : cluster 3 failed to encode map e286 with expected crc 2023-09-23T18:56:01.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:01 smithi173 bash[59422]: cluster 2023-09-23T18:56:00.164882+0000 osd.6 (osd.6) 255 : cluster 3 failed to encode map e286 with expected crc 2023-09-23T18:56:01.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:01 smithi173 bash[59422]: cluster 2023-09-23T18:56:00.166163+0000 mon.a (mon.0) 1556 : cluster 1 osd.4 [v2:172.21.15.173:6800/1430610213,v1:172.21.15.173:6801/1430610213] boot 2023-09-23T18:56:01.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:01 smithi173 bash[59422]: cluster 2023-09-23T18:56:00.166222+0000 mon.a (mon.0) 1557 : cluster 0 osdmap e286: 8 total, 7 up, 8 in 2023-09-23T18:56:01.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:01 smithi173 bash[59422]: cluster 2023-09-23T18:56:00.171080+0000 osd.5 (osd.5) 199 : cluster 3 failed to encode map e286 with expected crc 2023-09-23T18:56:01.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:01 smithi173 bash[59422]: audit 2023-09-23T18:56:00.178269+0000 mon.a (mon.0) 1558 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T18:56:01.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:01 smithi173 bash[59422]: cluster 2023-09-23T18:56:00.353690+0000 osd.5 (osd.5) 200 : cluster 3 failed to encode map e287 with expected crc 2023-09-23T18:56:01.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:01 smithi173 bash[59422]: cluster 2023-09-23T18:56:00.356386+0000 mon.a (mon.0) 1559 : cluster 0 osdmap e287: 8 total, 7 up, 8 in 2023-09-23T18:56:01.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:01 smithi173 bash[59422]: cluster 2023-09-23T18:56:00.357221+0000 osd.6 (osd.6) 256 : cluster 3 failed to encode map e287 with expected crc 2023-09-23T18:56:01.562 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:01.594 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:01.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:01.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:01.698 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:01.726 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:01.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:01.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:01.757 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T18:56:01.831 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:01.860 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:01.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:01.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:01.965 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:01.996 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:01.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:02.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:02.100 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:02.131 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:02.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:02.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:02.235 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:02.267 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:02.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:02.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:02.324 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 18 16 5419 5403 18.7555 168.688 0.00346529 0.0533063 2023-09-23T18:56:02.372 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:02.381 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:02 smithi098 bash[79516]: cluster 2023-09-23T18:55:59.563344+0000 osd.4 (osd.4) 1 : cluster 3 failed to encode map e263 with expected crc 2023-09-23T18:56:02.382 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:02 smithi098 bash[79516]: cluster 2023-09-23T18:56:00.168832+0000 osd.4 (osd.4) 2 : cluster 3 failed to encode map e286 with expected crc 2023-09-23T18:56:02.382 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:02 smithi098 bash[79516]: cluster 2023-09-23T18:56:00.277131+0000 mgr.y (mgr.26054) 1141 : cluster 0 pgmap v1235: 89 pgs: 5 active+clean+remapped, 30 active+clean, 15 active+undersized+degraded, 26 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 12 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 92/429 objects degraded (21.445%); 4/429 objects misplaced (0.932%) 2023-09-23T18:56:02.382 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:02 smithi098 bash[79516]: cluster 2023-09-23T18:56:00.360905+0000 osd.4 (osd.4) 3 : cluster 3 failed to encode map e287 with expected crc 2023-09-23T18:56:02.382 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:02 smithi098 bash[79516]: cluster 2023-09-23T18:56:00.366634+0000 osd.4 (osd.4) 4 : cluster 3 failed to encode map e287 with expected crc 2023-09-23T18:56:02.382 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:02 smithi098 bash[79516]: cluster 2023-09-23T18:56:01.356492+0000 osd.4 (osd.4) 5 : cluster 3 failed to encode map e288 with expected crc 2023-09-23T18:56:02.382 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:02 smithi098 bash[79516]: cluster 2023-09-23T18:56:01.356992+0000 mon.a (mon.0) 1560 : cluster 0 osdmap e288: 8 total, 7 up, 8 in 2023-09-23T18:56:02.382 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:02 smithi098 bash[79516]: cluster 2023-09-23T18:56:01.359204+0000 osd.5 (osd.5) 201 : cluster 3 failed to encode map e288 with expected crc 2023-09-23T18:56:02.382 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:02 smithi098 bash[79516]: cluster 2023-09-23T18:56:01.359577+0000 osd.4 (osd.4) 6 : cluster 3 failed to encode map e288 with expected crc 2023-09-23T18:56:02.383 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:02 smithi098 bash[76817]: cluster 2023-09-23T18:55:59.563344+0000 osd.4 (osd.4) 1 : cluster 3 failed to encode map e263 with expected crc 2023-09-23T18:56:02.383 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:02 smithi098 bash[76817]: cluster 2023-09-23T18:56:00.168832+0000 osd.4 (osd.4) 2 : cluster 3 failed to encode map e286 with expected crc 2023-09-23T18:56:02.383 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:02 smithi098 bash[76817]: cluster 2023-09-23T18:56:00.277131+0000 mgr.y (mgr.26054) 1141 : cluster 0 pgmap v1235: 89 pgs: 5 active+clean+remapped, 30 active+clean, 15 active+undersized+degraded, 26 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 12 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 92/429 objects degraded (21.445%); 4/429 objects misplaced (0.932%) 2023-09-23T18:56:02.383 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:02 smithi098 bash[76817]: cluster 2023-09-23T18:56:00.360905+0000 osd.4 (osd.4) 3 : cluster 3 failed to encode map e287 with expected crc 2023-09-23T18:56:02.383 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:02 smithi098 bash[76817]: cluster 2023-09-23T18:56:00.366634+0000 osd.4 (osd.4) 4 : cluster 3 failed to encode map e287 with expected crc 2023-09-23T18:56:02.383 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:02 smithi098 bash[76817]: cluster 2023-09-23T18:56:01.356492+0000 osd.4 (osd.4) 5 : cluster 3 failed to encode map e288 with expected crc 2023-09-23T18:56:02.383 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:02 smithi098 bash[76817]: cluster 2023-09-23T18:56:01.356992+0000 mon.a (mon.0) 1560 : cluster 0 osdmap e288: 8 total, 7 up, 8 in 2023-09-23T18:56:02.383 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:02 smithi098 bash[76817]: cluster 2023-09-23T18:56:01.359204+0000 osd.5 (osd.5) 201 : cluster 3 failed to encode map e288 with expected crc 2023-09-23T18:56:02.383 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:02 smithi098 bash[76817]: cluster 2023-09-23T18:56:01.359577+0000 osd.4 (osd.4) 6 : cluster 3 failed to encode map e288 with expected crc 2023-09-23T18:56:02.397 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:02.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:02.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:02.501 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:02.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:02 smithi173 bash[59422]: cluster 2023-09-23T18:55:59.563344+0000 osd.4 (osd.4) 1 : cluster 3 failed to encode map e263 with expected crc 2023-09-23T18:56:02.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:02 smithi173 bash[59422]: cluster 2023-09-23T18:56:00.168832+0000 osd.4 (osd.4) 2 : cluster 3 failed to encode map e286 with expected crc 2023-09-23T18:56:02.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:02 smithi173 bash[59422]: cluster 2023-09-23T18:56:00.277131+0000 mgr.y (mgr.26054) 1141 : cluster 0 pgmap v1235: 89 pgs: 5 active+clean+remapped, 30 active+clean, 15 active+undersized+degraded, 26 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 12 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 92/429 objects degraded (21.445%); 4/429 objects misplaced (0.932%) 2023-09-23T18:56:02.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:02 smithi173 bash[59422]: cluster 2023-09-23T18:56:00.360905+0000 osd.4 (osd.4) 3 : cluster 3 failed to encode map e287 with expected crc 2023-09-23T18:56:02.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:02 smithi173 bash[59422]: cluster 2023-09-23T18:56:00.366634+0000 osd.4 (osd.4) 4 : cluster 3 failed to encode map e287 with expected crc 2023-09-23T18:56:02.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:02 smithi173 bash[59422]: cluster 2023-09-23T18:56:01.356492+0000 osd.4 (osd.4) 5 : cluster 3 failed to encode map e288 with expected crc 2023-09-23T18:56:02.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:02 smithi173 bash[59422]: cluster 2023-09-23T18:56:01.356992+0000 mon.a (mon.0) 1560 : cluster 0 osdmap e288: 8 total, 7 up, 8 in 2023-09-23T18:56:02.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:02 smithi173 bash[59422]: cluster 2023-09-23T18:56:01.359204+0000 osd.5 (osd.5) 201 : cluster 3 failed to encode map e288 with expected crc 2023-09-23T18:56:02.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:02 smithi173 bash[59422]: cluster 2023-09-23T18:56:01.359577+0000 osd.4 (osd.4) 6 : cluster 3 failed to encode map e288 with expected crc 2023-09-23T18:56:02.534 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:02.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:02.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:02.640 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:02.675 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:02.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:02.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:02.780 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:02.810 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:02.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:02.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:02.914 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:02.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:02.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:02.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:03.048 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:03.078 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:03.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:03.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:03.182 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:03.212 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:03.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:03.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:03.317 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:03.324 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 19 16 7935 7919 26.0427 157.25 0.00336898 0.0383835 2023-09-23T18:56:03.326 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:03 smithi098 bash[76817]: cluster 2023-09-23T18:56:03.169739+0000 mon.a (mon.0) 1561 : cluster 3 Health check update: Degraded data redundancy: 120/825 objects degraded (14.545%), 19 pgs degraded, 43 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:03.326 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:03 smithi098 bash[79516]: cluster 2023-09-23T18:56:03.169739+0000 mon.a (mon.0) 1561 : cluster 3 Health check update: Degraded data redundancy: 120/825 objects degraded (14.545%), 19 pgs degraded, 43 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:03.342 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:03.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:03.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:03.447 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:03.474 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:03.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:03.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:03 smithi173 bash[59422]: cluster 2023-09-23T18:56:03.169739+0000 mon.a (mon.0) 1561 : cluster 3 Health check update: Degraded data redundancy: 120/825 objects degraded (14.545%), 19 pgs degraded, 43 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:03.577 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:03.607 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:03.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:03.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:03.710 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:03.737 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:03.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:03.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:03.841 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:03.877 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:03.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:03.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:03.980 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:04.009 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:04.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:04.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:04.113 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:04.140 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T18:56:04.141 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:04.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:04.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:04.244 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:04.271 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:04.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:04.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:04.325 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:2023-09-23T18:56:04.322197+0000 min lat: 0.00116653 max lat: 16.0742 avg lat: 0.0312453 2023-09-23T18:56:04.325 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:56:04.325 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 20 16 10258 10242 31.9983 145.188 0.00382518 0.0312453 2023-09-23T18:56:04.374 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:04.384 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:04 smithi098 bash[79516]: cluster 2023-09-23T18:56:02.278382+0000 mgr.y (mgr.26054) 1142 : cluster 0 pgmap v1238: 89 pgs: 2 peering, 3 active+clean+remapped, 30 active+clean, 15 active+undersized+degraded, 26 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 20 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.1 MiB/s wr, 32 op/s; 120/825 objects degraded (14.545%); 2/825 objects misplaced (0.242%) 2023-09-23T18:56:04.385 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:04 smithi098 bash[76817]: cluster 2023-09-23T18:56:02.278382+0000 mgr.y (mgr.26054) 1142 : cluster 0 pgmap v1238: 89 pgs: 2 peering, 3 active+clean+remapped, 30 active+clean, 15 active+undersized+degraded, 26 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 20 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.1 MiB/s wr, 32 op/s; 120/825 objects degraded (14.545%); 2/825 objects misplaced (0.242%) 2023-09-23T18:56:04.402 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:04.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:04.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:04.451 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:56:04.453 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:56:04.508 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:04.519 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:04 smithi173 bash[59422]: cluster 2023-09-23T18:56:02.278382+0000 mgr.y (mgr.26054) 1142 : cluster 0 pgmap v1238: 89 pgs: 2 peering, 3 active+clean+remapped, 30 active+clean, 15 active+undersized+degraded, 26 active+undersized, 9 undersized+peered, 4 undersized+degraded+peered; 20 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.1 MiB/s wr, 32 op/s; 120/825 objects degraded (14.545%); 2/825 objects misplaced (0.242%) 2023-09-23T18:56:04.539 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:04.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:04.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:04.643 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:04.670 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:04.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:04.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:04.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:04.801 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:04.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:04.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:04.904 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:04.930 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:04.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:04.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:05.033 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:05.061 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:05.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:05.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:05.166 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:05.200 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:05.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:05.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:05.305 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:05.325 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 21 16 12530 12514 37.2345 142 0.00493587 0.0268471 2023-09-23T18:56:05.331 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:05.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:05.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:05.435 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:05.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:05 smithi098 bash[76817]: cluster 2023-09-23T18:56:04.279325+0000 mgr.y (mgr.26054) 1143 : cluster 0 pgmap v1239: 89 pgs: 2 peering, 2 active+clean+remapped, 49 active+clean, 19 active+undersized+degraded, 15 active+undersized, 2 undersized+peered; 298 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 56 MiB/s wr, 892 op/s; 3294/14154 objects degraded (23.273%); 2/14154 objects misplaced (0.014%); 75 KiB/s, 1 objects/s recovering 2023-09-23T18:56:05.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:05 smithi098 bash[76817]: cluster 2023-09-23T18:56:05.223983+0000 mon.a (mon.0) 1562 : cluster 3 Health check update: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2023-09-23T18:56:05.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:05 smithi098 bash[79516]: cluster 2023-09-23T18:56:04.279325+0000 mgr.y (mgr.26054) 1143 : cluster 0 pgmap v1239: 89 pgs: 2 peering, 2 active+clean+remapped, 49 active+clean, 19 active+undersized+degraded, 15 active+undersized, 2 undersized+peered; 298 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 56 MiB/s wr, 892 op/s; 3294/14154 objects degraded (23.273%); 2/14154 objects misplaced (0.014%); 75 KiB/s, 1 objects/s recovering 2023-09-23T18:56:05.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:05 smithi098 bash[79516]: cluster 2023-09-23T18:56:05.223983+0000 mon.a (mon.0) 1562 : cluster 3 Health check update: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2023-09-23T18:56:05.464 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:05.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:05.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:05.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:05 smithi173 bash[59422]: cluster 2023-09-23T18:56:04.279325+0000 mgr.y (mgr.26054) 1143 : cluster 0 pgmap v1239: 89 pgs: 2 peering, 2 active+clean+remapped, 49 active+clean, 19 active+undersized+degraded, 15 active+undersized, 2 undersized+peered; 298 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 56 MiB/s wr, 892 op/s; 3294/14154 objects degraded (23.273%); 2/14154 objects misplaced (0.014%); 75 KiB/s, 1 objects/s recovering 2023-09-23T18:56:05.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:05 smithi173 bash[59422]: cluster 2023-09-23T18:56:05.223983+0000 mon.a (mon.0) 1562 : cluster 3 Health check update: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2023-09-23T18:56:05.567 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:05.596 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:05.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:05.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:05.700 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:05.725 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:05.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:05.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:05.828 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:05.858 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:05.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:05.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:05.961 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:05.990 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:05.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:05.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:06.095 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:06.121 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:06.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:06.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:06.231 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:06.261 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:06.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:06.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:06.325 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 22 16 14783 14767 41.9412 140.812 0.00254868 0.0238344 2023-09-23T18:56:06.365 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:06.374 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:56:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:56:06] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.33.4" 2023-09-23T18:56:06.390 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:06.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:06.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:06.494 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:06.517 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:06.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:06.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:06.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:06 smithi173 bash[59422]: cluster 2023-09-23T18:56:06.352567+0000 mon.a (mon.0) 1563 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive) 2023-09-23T18:56:06.578 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:56:06.624 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:06.633 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:06 smithi098 bash[79516]: cluster 2023-09-23T18:56:06.352567+0000 mon.a (mon.0) 1563 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive) 2023-09-23T18:56:06.633 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:06 smithi098 bash[76817]: cluster 2023-09-23T18:56:06.352567+0000 mon.a (mon.0) 1563 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive) 2023-09-23T18:56:06.651 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:06.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:06.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:06.755 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:06.782 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:06.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:06.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:06.885 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:06.916 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:06.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:06.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:07.021 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:07.050 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:07.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:07.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:07.155 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:07.193 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:07.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:07.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:07.298 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:07.325 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 23 16 16995 16979 46.127 138.25 0.00457358 0.021669 2023-09-23T18:56:07.326 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:07.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:07.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:07.432 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:07.440 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:07 smithi098 bash[79516]: cluster 2023-09-23T18:56:06.280209+0000 mgr.y (mgr.26054) 1144 : cluster 0 pgmap v1240: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 573 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s; 4698/27351 objects degraded (17.177%); 2/27351 objects misplaced (0.007%); 63 KiB/s, 0 objects/s recovering 2023-09-23T18:56:07.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:07 smithi098 bash[76817]: cluster 2023-09-23T18:56:06.280209+0000 mgr.y (mgr.26054) 1144 : cluster 0 pgmap v1240: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 573 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s; 4698/27351 objects degraded (17.177%); 2/27351 objects misplaced (0.007%); 63 KiB/s, 0 objects/s recovering 2023-09-23T18:56:07.455 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:07.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:07.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:07.558 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:07.587 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:07.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:07.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:07.691 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:07.718 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:07.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:07.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:07.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:07 smithi173 bash[59422]: cluster 2023-09-23T18:56:06.280209+0000 mgr.y (mgr.26054) 1144 : cluster 0 pgmap v1240: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 573 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s; 4698/27351 objects degraded (17.177%); 2/27351 objects misplaced (0.007%); 63 KiB/s, 0 objects/s recovering 2023-09-23T18:56:07.822 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:07.920 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:07.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:07.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:08.027 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:08.050 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:08.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:08.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:08.158 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:08.182 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:08.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:08.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:08.285 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:08.315 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:08.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:08.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:08.325 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 24 16 18979 18963 49.3707 124 0.017275 0.0202468 2023-09-23T18:56:08.419 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:08.447 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:08.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:08.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:08.552 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:08.561 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:08 smithi098 bash[76817]: cluster 2023-09-23T18:56:08.407245+0000 mon.a (mon.0) 1564 : cluster 3 Health check update: Degraded data redundancy: 8849/46290 objects degraded (19.116%), 18 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:08.561 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:08 smithi098 bash[79516]: cluster 2023-09-23T18:56:08.407245+0000 mon.a (mon.0) 1564 : cluster 3 Health check update: Degraded data redundancy: 8849/46290 objects degraded (19.116%), 18 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:08.580 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:08.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:08.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:08.603 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:56:08.685 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:08.709 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:08.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:08.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:08.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:08 smithi173 bash[59422]: cluster 2023-09-23T18:56:08.407245+0000 mon.a (mon.0) 1564 : cluster 3 Health check update: Degraded data redundancy: 8849/46290 objects degraded (19.116%), 18 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:08.813 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:08.836 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:08.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:08.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:08.940 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:08.970 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:08.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:08.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:09.074 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:09.100 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:09.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:09.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:09.205 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:09.237 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:09.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:09.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:09.326 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 25 16 20972 20956 52.3775 124.562 0.0104748 0.0190845 2023-09-23T18:56:09.340 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:09.364 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:09.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:09.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:09.468 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:09.493 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:09.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:09.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:09.597 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:09.627 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:09.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:09.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:09.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:09 smithi098 bash[76817]: cluster 2023-09-23T18:56:08.281119+0000 mgr.y (mgr.26054) 1145 : cluster 0 pgmap v1241: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 967 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s; 8849/46290 objects degraded (19.116%); 2/46290 objects misplaced (0.004%); 48 KiB/s, 0 objects/s recovering 2023-09-23T18:56:09.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:09 smithi098 bash[79516]: cluster 2023-09-23T18:56:08.281119+0000 mgr.y (mgr.26054) 1145 : cluster 0 pgmap v1241: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 967 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s; 8849/46290 objects degraded (19.116%); 2/46290 objects misplaced (0.004%); 48 KiB/s, 0 objects/s recovering 2023-09-23T18:56:09.732 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:09.756 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:09.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:09.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:09.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:09 smithi173 bash[59422]: cluster 2023-09-23T18:56:08.281119+0000 mgr.y (mgr.26054) 1145 : cluster 0 pgmap v1241: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 967 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s; 8849/46290 objects degraded (19.116%); 2/46290 objects misplaced (0.004%); 48 KiB/s, 0 objects/s recovering 2023-09-23T18:56:09.859 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:09.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:09.890 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:09.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:09.991 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:10.018 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:10.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:10.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:10.124 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:10.148 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:10.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:10.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:10.251 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:10.280 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:10.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:10.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:10.326 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 26 16 23005 22989 55.2491 127.062 0.00507814 0.0180938 2023-09-23T18:56:10.382 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:10.418 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:10.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:10.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:10.522 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:10.545 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:10.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:10.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:10.648 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:10 smithi098 bash[76817]: audit 2023-09-23T18:56:09.637549+0000 mon.a (mon.0) 1565 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:56:10.648 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:10 smithi098 bash[76817]: audit 2023-09-23T18:56:09.638875+0000 mon.a (mon.0) 1566 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:56:10.648 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:10.674 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:10.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:10.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:10.778 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:10.781 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:10 smithi173 bash[59422]: audit 2023-09-23T18:56:09.637549+0000 mon.a (mon.0) 1565 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:56:10.781 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:10 smithi173 bash[59422]: audit 2023-09-23T18:56:09.638875+0000 mon.a (mon.0) 1566 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:56:10.803 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:10.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:10.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:10.908 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:10.916 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:10 smithi098 bash[79516]: audit 2023-09-23T18:56:09.637549+0000 mon.a (mon.0) 1565 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:56:10.916 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:10 smithi098 bash[79516]: audit 2023-09-23T18:56:09.638875+0000 mon.a (mon.0) 1566 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:56:10.943 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:10.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:10.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:11.047 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:11.076 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:11.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:11.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:11.180 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:11.202 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:11.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:11.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:11.304 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:11.326 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 27 16 24795 24779 57.3457 111.875 0.0055776 0.0174313 2023-09-23T18:56:11.330 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:11.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:11.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:11.433 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:11.462 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:11.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:11.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:11.565 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:11.588 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:11.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:11.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:11.651 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:11 smithi098 bash[76817]: cluster 2023-09-23T18:56:10.281997+0000 mgr.y (mgr.26054) 1146 : cluster 0 pgmap v1242: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 1.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s; 10233/52470 objects degraded (19.503%); 2/52470 objects misplaced (0.004%); 39 KiB/s, 0 objects/s recovering 2023-09-23T18:56:11.651 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:11 smithi098 bash[79516]: cluster 2023-09-23T18:56:10.281997+0000 mgr.y (mgr.26054) 1146 : cluster 0 pgmap v1242: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 1.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s; 10233/52470 objects degraded (19.503%); 2/52470 objects misplaced (0.004%); 39 KiB/s, 0 objects/s recovering 2023-09-23T18:56:11.694 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:11.717 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:11.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:11.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:11.820 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:11.847 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:11.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:11.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:11.952 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:11.978 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:11.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:11.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:12.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:11 smithi173 bash[59422]: cluster 2023-09-23T18:56:10.281997+0000 mgr.y (mgr.26054) 1146 : cluster 0 pgmap v1242: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 1.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s; 10233/52470 objects degraded (19.503%); 2/52470 objects misplaced (0.004%); 39 KiB/s, 0 objects/s recovering 2023-09-23T18:56:12.082 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:12.107 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:12.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:12.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:12.210 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:12.256 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:12.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:12.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:12.326 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 28 16 26703 26687 59.5559 119.25 0.00686062 0.0167849 2023-09-23T18:56:12.360 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:12.396 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:12.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:12.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:12.500 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:12.526 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:12.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:12.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:12.630 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:12.658 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:12.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:12.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:12.766 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:12.793 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:12.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:12.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:12.897 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:12.922 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:12.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:12.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:13.026 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:13.050 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:13.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:13.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:13.155 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:13.178 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:13.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:13.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:13.283 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:13.306 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:13.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:13.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:13.326 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 29 16 28358 28342 61.0685 103.438 0.013898 0.0163675 2023-09-23T18:56:13.409 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:13.435 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:13.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:13.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:13.540 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:13.564 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:13.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:13.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:13.667 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:13.701 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:13.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:13.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:13.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:13 smithi173 bash[59422]: cluster 2023-09-23T18:56:12.282789+0000 mgr.y (mgr.26054) 1147 : cluster 0 pgmap v1243: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s; 10889/60162 objects degraded (18.099%); 2/60162 objects misplaced (0.003%); 35 KiB/s, 0 objects/s recovering 2023-09-23T18:56:13.808 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:13.841 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:13.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:13.846 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:13 smithi098 bash[76817]: cluster 2023-09-23T18:56:12.282789+0000 mgr.y (mgr.26054) 1147 : cluster 0 pgmap v1243: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s; 10889/60162 objects degraded (18.099%); 2/60162 objects misplaced (0.003%); 35 KiB/s, 0 objects/s recovering 2023-09-23T18:56:13.847 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:13 smithi098 bash[79516]: cluster 2023-09-23T18:56:12.282789+0000 mgr.y (mgr.26054) 1147 : cluster 0 pgmap v1243: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s; 10889/60162 objects degraded (18.099%); 2/60162 objects misplaced (0.003%); 35 KiB/s, 0 objects/s recovering 2023-09-23T18:56:13.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:13.948 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:13.980 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:13.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:13.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:14.085 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:14.107 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:14.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:14.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:14.211 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:14.238 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:14.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:14.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:14.326 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 30 16 30090 30074 62.6407 108.25 0.00391627 0.0159574 2023-09-23T18:56:14.343 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:14.370 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:14.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:14.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:14.472 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:14.498 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:14.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:14.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:14.602 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:14.631 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:14.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:14.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:14.734 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:14.760 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:14.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:14.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:14.864 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:14.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:14 smithi173 bash[59422]: cluster 2023-09-23T18:56:14.652057+0000 mon.a (mon.0) 1567 : cluster 3 Health check update: Degraded data redundancy: 15037/78003 objects degraded (19.277%), 18 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:14.887 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:14.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:14.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:14.991 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:15.000 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:14 smithi098 bash[76817]: cluster 2023-09-23T18:56:14.652057+0000 mon.a (mon.0) 1567 : cluster 3 Health check update: Degraded data redundancy: 15037/78003 objects degraded (19.277%), 18 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:15.000 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:14 smithi098 bash[79516]: cluster 2023-09-23T18:56:14.652057+0000 mon.a (mon.0) 1567 : cluster 3 Health check update: Degraded data redundancy: 15037/78003 objects degraded (19.277%), 18 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:15.015 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:15.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:15.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:15.120 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:15.152 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:15.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:15.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:15.256 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:15.280 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:15.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:15.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:15.326 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 31 16 31729 31713 63.924 102.438 0.00681728 0.0156379 2023-09-23T18:56:15.384 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:15.406 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:15.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:15.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:15.510 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:15.535 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:15.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:15.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:15.542 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:56:15.542 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-09-23T18:56:15.542 INFO:teuthology.orchestra.run.smithi173.stdout: "complaint_time": 30, 2023-09-23T18:56:15.542 INFO:teuthology.orchestra.run.smithi173.stdout: "num_blocked_ops": 0 2023-09-23T18:56:15.542 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:56:15.560 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:56:15.561 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-09-23T18:56:15.561 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:15.561 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45602.0:32054 9.e 9:7256187c:::benchmark_data_smithi098_206513_object32053:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e288)", 2023-09-23T18:56:15.561 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:56:15.554009+0000", 2023-09-23T18:56:15.561 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.0052244459999999998, 2023-09-23T18:56:15.561 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0052564530000000003, 2023-09-23T18:56:15.561 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:15.561 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:56:15.561 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:56:15.561 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45602", 2023-09-23T18:56:15.562 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T18:56:15.562 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 32054 2023-09-23T18:56:15.562 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:15.562 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:15.562 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:15.562 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:15.562 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:15.554009+0000", 2023-09-23T18:56:15.562 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:15.562 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:15.562 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:15.562 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:15.562 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:15.554009+0000", 2023-09-23T18:56:15.563 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:56:15.563 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:15.563 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:15.563 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:15.563 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:15.554007+0000", 2023-09-23T18:56:15.563 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.0337000000000001e-05 2023-09-23T18:56:15.563 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:15.563 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:15.563 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:15.563 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:15.554077+0000", 2023-09-23T18:56:15.563 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4337e-05 2023-09-23T18:56:15.563 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:15.564 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:15.564 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:15.564 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:15.554092+0000", 2023-09-23T18:56:15.564 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5583999999999999e-05 2023-09-23T18:56:15.564 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:15.564 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:15.564 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:15.564 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:15.554107+0000", 2023-09-23T18:56:15.564 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.0243e-05 2023-09-23T18:56:15.564 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:15.564 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:15.564 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:15.564 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:15.554157+0000", 2023-09-23T18:56:15.565 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000197911 2023-09-23T18:56:15.565 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:15.565 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:15.565 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:15.565 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:15.554355+0000", 2023-09-23T18:56:15.565 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00020630600000000001 2023-09-23T18:56:15.565 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:15.565 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:15.565 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1", 2023-09-23T18:56:15.565 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:15.554562+0000", 2023-09-23T18:56:15.565 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0013850259999999999 2023-09-23T18:56:15.565 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:15.566 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:15.566 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:56:15.566 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:15.555947+0000", 2023-09-23T18:56:15.566 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0019377260000000001 2023-09-23T18:56:15.566 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:15.566 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:15.566 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:15.566 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:15.566 INFO:teuthology.orchestra.run.smithi173.stdout: ], 2023-09-23T18:56:15.566 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 1 2023-09-23T18:56:15.566 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:56:15.639 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:15.664 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:15.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:15.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:15.767 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:15.794 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:15.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:15.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:15.898 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:15.906 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:15 smithi173 bash[59422]: cluster 2023-09-23T18:56:14.283800+0000 mgr.y (mgr.26054) 1148 : cluster 0 pgmap v1244: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s; 15037/78003 objects degraded (19.277%); 2/78003 objects misplaced (0.003%); 32 KiB/s, 0 objects/s recovering 2023-09-23T18:56:15.921 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:15.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:15.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:16.024 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:16.035 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:15 smithi098 bash[76817]: cluster 2023-09-23T18:56:14.283800+0000 mgr.y (mgr.26054) 1148 : cluster 0 pgmap v1244: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s; 15037/78003 objects degraded (19.277%); 2/78003 objects misplaced (0.003%); 32 KiB/s, 0 objects/s recovering 2023-09-23T18:56:16.035 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:15 smithi098 bash[79516]: cluster 2023-09-23T18:56:14.283800+0000 mgr.y (mgr.26054) 1148 : cluster 0 pgmap v1244: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s; 15037/78003 objects degraded (19.277%); 2/78003 objects misplaced (0.003%); 32 KiB/s, 0 objects/s recovering 2023-09-23T18:56:16.054 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:16.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:16.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:16.160 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:16.183 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:16.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:16.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:16.286 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:16.311 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:16.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:16.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:16.326 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 32 16 33070 33054 64.5452 83.8125 0.0127367 0.0154862 2023-09-23T18:56:16.349 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:56:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:56:16] "GET /metrics HTTP/1.1" 200 36587 "" "Prometheus/2.33.4" 2023-09-23T18:56:16.414 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:16.439 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:16.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:16.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:16.542 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:16.568 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:16.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:16.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:16.673 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:16.698 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:16.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:16.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:16.801 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:16.826 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:16.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:16.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:16.929 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:16.961 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:16.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:16.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:17.064 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:17.091 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:17.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:17.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:17.196 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:17.225 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:17.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:17.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:17.326 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 33 16 34616 34600 65.517 96.625 0.0053512 0.0152554 2023-09-23T18:56:17.329 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:17.336 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T18:56:17.357 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:17.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:17.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:17.464 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:17.489 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:17.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:17.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:17.592 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:17.613 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:17.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:17.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:17.715 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:17.734 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:17.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:17.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:17.837 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:17.859 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:17.860 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:17 smithi098 bash[79516]: cluster 2023-09-23T18:56:16.284693+0000 mgr.y (mgr.26054) 1149 : cluster 0 pgmap v1245: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 1.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s; 16168/87882 objects degraded (18.397%); 2/87882 objects misplaced (0.002%) 2023-09-23T18:56:17.861 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:17 smithi098 bash[76817]: cluster 2023-09-23T18:56:16.284693+0000 mgr.y (mgr.26054) 1149 : cluster 0 pgmap v1245: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 1.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s; 16168/87882 objects degraded (18.397%); 2/87882 objects misplaced (0.002%) 2023-09-23T18:56:17.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:17.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:17.962 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:17.972 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:17 smithi173 bash[59422]: cluster 2023-09-23T18:56:16.284693+0000 mgr.y (mgr.26054) 1149 : cluster 0 pgmap v1245: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 1.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s; 16168/87882 objects degraded (18.397%); 2/87882 objects misplaced (0.002%) 2023-09-23T18:56:17.989 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:17.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:17.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:18.093 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:18.120 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:18.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:18.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:18.224 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:18.249 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:18.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:18.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:18.327 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 34 15 35719 35704 65.6191 69 0.00954921 0.0152345 2023-09-23T18:56:18.353 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:18.381 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:18.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:18.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:18.486 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:18.515 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:18.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:18.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:18.618 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:18.657 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:18.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:18.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:18.762 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:18.771 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:18 smithi098 bash[76817]: audit 2023-09-23T18:56:17.978307+0000 mon.a (mon.0) 1568 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:56:18.772 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:18 smithi098 bash[79516]: audit 2023-09-23T18:56:17.978307+0000 mon.a (mon.0) 1568 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:56:18.798 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:18.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:18.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:18.903 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:18.934 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:18.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:18.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:19.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:18 smithi173 bash[59422]: audit 2023-09-23T18:56:17.978307+0000 mon.a (mon.0) 1568 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:56:19.038 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:19.063 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:19.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:19.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:19.168 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:19.195 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:19.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:19.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:19.299 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:19.327 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 35 16 37282 37266 66.5332 97.625 0.0108099 0.0150249 2023-09-23T18:56:19.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:19.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:19.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:19.432 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:19.459 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:19.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:19.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:19.562 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:19.593 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:19.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:19.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:19.697 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:19.701 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:56:19.732 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:19.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:19.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:19.750 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:56:19.781 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:56:19.813 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:56:19.839 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:19.848 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:56:19.869 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:19.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:19.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:19.884 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:56:19.914 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:56:19.920 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:56:19.920 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:56:19.920 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:56:19.921 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:56:19.921 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:56:19.921 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:56:19.921 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:56:19.921 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T18:56:19.974 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:19.984 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:19 smithi173 bash[59422]: cluster 2023-09-23T18:56:18.285519+0000 mgr.y (mgr.26054) 1150 : cluster 0 pgmap v1246: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s; 18984/100506 objects degraded (18.888%); 2/100506 objects misplaced (0.002%) 2023-09-23T18:56:19.985 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:19 smithi173 bash[59422]: audit 2023-09-23T18:56:19.638182+0000 mon.a (mon.0) 1569 : audit 0 from='client.? 172.21.15.98:0/1067013101' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:56:20.001 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:20.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:20.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:20.099 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:19 smithi098 bash[76817]: cluster 2023-09-23T18:56:18.285519+0000 mgr.y (mgr.26054) 1150 : cluster 0 pgmap v1246: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s; 18984/100506 objects degraded (18.888%); 2/100506 objects misplaced (0.002%) 2023-09-23T18:56:20.099 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:19 smithi098 bash[76817]: audit 2023-09-23T18:56:19.638182+0000 mon.a (mon.0) 1569 : audit 0 from='client.? 172.21.15.98:0/1067013101' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:56:20.099 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:19 smithi098 bash[79516]: cluster 2023-09-23T18:56:18.285519+0000 mgr.y (mgr.26054) 1150 : cluster 0 pgmap v1246: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s; 18984/100506 objects degraded (18.888%); 2/100506 objects misplaced (0.002%) 2023-09-23T18:56:20.099 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:19 smithi098 bash[79516]: audit 2023-09-23T18:56:19.638182+0000 mon.a (mon.0) 1569 : audit 0 from='client.? 172.21.15.98:0/1067013101' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:56:20.104 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:20.137 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:20.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:20.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:20.241 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:20.269 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:20.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:20.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:20.327 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 36 16 38623 38607 67.0129 83.8125 0.0111546 0.0149156 2023-09-23T18:56:20.372 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:20.402 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:20.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:20.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:20.505 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:20.532 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:20.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:20.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:20.634 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:20.659 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:20.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:20.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:20.762 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:20.765 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is unset 2023-09-23T18:56:20.771 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:20 smithi098 bash[76817]: audit 2023-09-23T18:56:20.083653+0000 mon.a (mon.0) 1570 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:56:20.772 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:20 smithi098 bash[76817]: audit 2023-09-23T18:56:20.084116+0000 mon.b (mon.1) 25 : audit 1 from='client.? 172.21.15.98:0/2994365887' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:56:20.772 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:20 smithi098 bash[76817]: cluster 2023-09-23T18:56:20.354644+0000 mon.a (mon.0) 1571 : cluster 3 Health check update: Degraded data redundancy: 18984/100506 objects degraded (18.888%), 18 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:20.772 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:20 smithi098 bash[79516]: audit 2023-09-23T18:56:20.083653+0000 mon.a (mon.0) 1570 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:56:20.772 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:20 smithi098 bash[79516]: audit 2023-09-23T18:56:20.084116+0000 mon.b (mon.1) 25 : audit 1 from='client.? 172.21.15.98:0/2994365887' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:56:20.772 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:20 smithi098 bash[79516]: cluster 2023-09-23T18:56:20.354644+0000 mon.a (mon.0) 1571 : cluster 3 Health check update: Degraded data redundancy: 18984/100506 objects degraded (18.888%), 18 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:20.791 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:20.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:20.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:20.895 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:20.923 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:20.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:20.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:21.027 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:21.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:20 smithi173 bash[59422]: audit 2023-09-23T18:56:20.083653+0000 mon.a (mon.0) 1570 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:56:21.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:20 smithi173 bash[59422]: audit 2023-09-23T18:56:20.084116+0000 mon.b (mon.1) 25 : audit 1 from='client.? 172.21.15.98:0/2994365887' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:56:21.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:20 smithi173 bash[59422]: cluster 2023-09-23T18:56:20.354644+0000 mon.a (mon.0) 1571 : cluster 3 Health check update: Degraded data redundancy: 18984/100506 objects degraded (18.888%), 18 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:21.044 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:21.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:21.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:21.147 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:21.186 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:21.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:21.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:21.290 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:21.317 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:21.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:21.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:21.327 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 37 16 39757 39741 67.1171 70.875 0.00875563 0.0148936 2023-09-23T18:56:21.421 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:21.446 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:21.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:21.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:21.549 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:21.577 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:21.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:21.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:21.683 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:21.717 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:21.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:21.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:21.822 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:21.853 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:21.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:21.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:21.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:21 smithi098 bash[79516]: cluster 2023-09-23T18:56:20.286155+0000 mgr.y (mgr.26054) 1151 : cluster 0 pgmap v1247: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s; 19844/104412 objects degraded (19.005%); 2/104412 objects misplaced (0.002%) 2023-09-23T18:56:21.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:21 smithi098 bash[79516]: cluster 2023-09-23T18:56:20.723487+0000 mon.a (mon.0) 1572 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:56:21.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:21 smithi098 bash[79516]: cluster 2023-09-23T18:56:20.737695+0000 osd.6 (osd.6) 257 : cluster 3 failed to encode map e289 with expected crc 2023-09-23T18:56:21.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:21 smithi098 bash[79516]: cluster 2023-09-23T18:56:20.737899+0000 osd.5 (osd.5) 202 : cluster 3 failed to encode map e289 with expected crc 2023-09-23T18:56:21.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:21 smithi098 bash[79516]: cluster 2023-09-23T18:56:20.743389+0000 osd.4 (osd.4) 7 : cluster 3 failed to encode map e289 with expected crc 2023-09-23T18:56:21.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:21 smithi098 bash[79516]: audit 2023-09-23T18:56:20.747616+0000 mon.a (mon.0) 1573 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:56:21.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:21 smithi098 bash[79516]: cluster 2023-09-23T18:56:20.747630+0000 osd.5 (osd.5) 203 : cluster 3 failed to encode map e289 with expected crc 2023-09-23T18:56:21.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:21 smithi098 bash[79516]: cluster 2023-09-23T18:56:20.747633+0000 osd.4 (osd.4) 8 : cluster 3 failed to encode map e289 with expected crc 2023-09-23T18:56:21.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:21 smithi098 bash[79516]: cluster 2023-09-23T18:56:20.747896+0000 mon.a (mon.0) 1574 : cluster 0 osdmap e289: 8 total, 7 up, 8 in 2023-09-23T18:56:21.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:21 smithi098 bash[76817]: cluster 2023-09-23T18:56:20.286155+0000 mgr.y (mgr.26054) 1151 : cluster 0 pgmap v1247: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s; 19844/104412 objects degraded (19.005%); 2/104412 objects misplaced (0.002%) 2023-09-23T18:56:21.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:21 smithi098 bash[76817]: cluster 2023-09-23T18:56:20.723487+0000 mon.a (mon.0) 1572 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:56:21.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:21 smithi098 bash[76817]: cluster 2023-09-23T18:56:20.737695+0000 osd.6 (osd.6) 257 : cluster 3 failed to encode map e289 with expected crc 2023-09-23T18:56:21.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:21 smithi098 bash[76817]: cluster 2023-09-23T18:56:20.737899+0000 osd.5 (osd.5) 202 : cluster 3 failed to encode map e289 with expected crc 2023-09-23T18:56:21.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:21 smithi098 bash[76817]: cluster 2023-09-23T18:56:20.743389+0000 osd.4 (osd.4) 7 : cluster 3 failed to encode map e289 with expected crc 2023-09-23T18:56:21.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:21 smithi098 bash[76817]: audit 2023-09-23T18:56:20.747616+0000 mon.a (mon.0) 1573 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:56:21.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:21 smithi098 bash[76817]: cluster 2023-09-23T18:56:20.747630+0000 osd.5 (osd.5) 203 : cluster 3 failed to encode map e289 with expected crc 2023-09-23T18:56:21.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:21 smithi098 bash[76817]: cluster 2023-09-23T18:56:20.747633+0000 osd.4 (osd.4) 8 : cluster 3 failed to encode map e289 with expected crc 2023-09-23T18:56:21.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:21 smithi098 bash[76817]: cluster 2023-09-23T18:56:20.747896+0000 mon.a (mon.0) 1574 : cluster 0 osdmap e289: 8 total, 7 up, 8 in 2023-09-23T18:56:21.956 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:21.981 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:21.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:21.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:22.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:21 smithi173 bash[59422]: cluster 2023-09-23T18:56:20.286155+0000 mgr.y (mgr.26054) 1151 : cluster 0 pgmap v1247: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s; 19844/104412 objects degraded (19.005%); 2/104412 objects misplaced (0.002%) 2023-09-23T18:56:22.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:21 smithi173 bash[59422]: cluster 2023-09-23T18:56:20.723487+0000 mon.a (mon.0) 1572 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:56:22.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:21 smithi173 bash[59422]: cluster 2023-09-23T18:56:20.737695+0000 osd.6 (osd.6) 257 : cluster 3 failed to encode map e289 with expected crc 2023-09-23T18:56:22.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:21 smithi173 bash[59422]: cluster 2023-09-23T18:56:20.737899+0000 osd.5 (osd.5) 202 : cluster 3 failed to encode map e289 with expected crc 2023-09-23T18:56:22.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:21 smithi173 bash[59422]: cluster 2023-09-23T18:56:20.743389+0000 osd.4 (osd.4) 7 : cluster 3 failed to encode map e289 with expected crc 2023-09-23T18:56:22.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:21 smithi173 bash[59422]: audit 2023-09-23T18:56:20.747616+0000 mon.a (mon.0) 1573 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:56:22.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:21 smithi173 bash[59422]: cluster 2023-09-23T18:56:20.747630+0000 osd.5 (osd.5) 203 : cluster 3 failed to encode map e289 with expected crc 2023-09-23T18:56:22.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:21 smithi173 bash[59422]: cluster 2023-09-23T18:56:20.747633+0000 osd.4 (osd.4) 8 : cluster 3 failed to encode map e289 with expected crc 2023-09-23T18:56:22.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:21 smithi173 bash[59422]: cluster 2023-09-23T18:56:20.747896+0000 mon.a (mon.0) 1574 : cluster 0 osdmap e289: 8 total, 7 up, 8 in 2023-09-23T18:56:22.086 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:22.114 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:22.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:22.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:22.144 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:56:22.218 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:22.241 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:22.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:22.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:22.327 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 38 16 41241 41225 67.7914 92.75 0.0125848 0.0147456 2023-09-23T18:56:22.346 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:22.350 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7, 5] out_osds: [] dead_osds: [7] live_osds: [0, 2, 3, 5, 6, 1, 4] 2023-09-23T18:56:22.350 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:56:22.351 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-09-23T18:56:22.351 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T18:56:22.351 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T18:56:22.351 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-09-23T18:56:22.371 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:22.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:22.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:22.476 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:22.505 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:22.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:22.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:22.514 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:56:22.514 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:56:22.610 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:22.630 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:22.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:22.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:22.734 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:22.758 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:22.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:22.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:22.861 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:22.897 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:22.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:22.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:23.000 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:23.025 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:23.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:23.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:23.129 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:23.161 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:23.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:23.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:23.266 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:23.291 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:23.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:23.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:23.327 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 39 15 42140 42125 67.4953 56.25 0.00806893 0.0148095 2023-09-23T18:56:23.396 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:23.436 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:23.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:23.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:23.540 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:23.584 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:23.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:23.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:23.689 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:23.718 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:23.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:23.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:23.744 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:23 smithi098 bash[76817]: cluster 2023-09-23T18:56:22.286908+0000 mgr.y (mgr.26054) 1152 : cluster 0 pgmap v1249: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 20268/109587 objects degraded (18.495%); 2/109587 objects misplaced (0.002%) 2023-09-23T18:56:23.821 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:23.846 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:23.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:23.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:23.951 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:23.989 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:23.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:23.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:24.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:23 smithi173 bash[59422]: cluster 2023-09-23T18:56:22.286908+0000 mgr.y (mgr.26054) 1152 : cluster 0 pgmap v1249: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 20268/109587 objects degraded (18.495%); 2/109587 objects misplaced (0.002%) 2023-09-23T18:56:24.092 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:24.103 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:23 smithi098 bash[79516]: cluster 2023-09-23T18:56:22.286908+0000 mgr.y (mgr.26054) 1152 : cluster 0 pgmap v1249: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 20268/109587 objects degraded (18.495%); 2/109587 objects misplaced (0.002%) 2023-09-23T18:56:24.125 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:24.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:24.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:24.228 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:24.261 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:24.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:24.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:24.327 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:2023-09-23T18:56:24.324662+0000 min lat: 0.00116653 max lat: 16.0742 avg lat: 0.0147192 2023-09-23T18:56:24.327 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:56:24.327 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 40 16 43486 43470 67.9093 84.0625 0.0166139 0.0147192 2023-09-23T18:56:24.369 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:24.397 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:24.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:24.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:24.501 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:24.515 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:56:24.531 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:24.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:24.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:24.637 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:24.696 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:24.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:24.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:24.800 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:24.826 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:24.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:24.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:24.929 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:24.939 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:24 smithi098 bash[79516]: audit 2023-09-23T18:56:23.754747+0000 mon.a (mon.0) 1575 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:56:24.939 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:24 smithi098 bash[79516]: audit 2023-09-23T18:56:23.764956+0000 mon.a (mon.0) 1576 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:56:24.939 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:24 smithi098 bash[79516]: audit 2023-09-23T18:56:24.630938+0000 mon.a (mon.0) 1577 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:56:24.939 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:24 smithi098 bash[76817]: audit 2023-09-23T18:56:23.754747+0000 mon.a (mon.0) 1575 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:56:24.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:24 smithi098 bash[76817]: audit 2023-09-23T18:56:23.764956+0000 mon.a (mon.0) 1576 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:56:24.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:24 smithi098 bash[76817]: audit 2023-09-23T18:56:24.630938+0000 mon.a (mon.0) 1577 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:56:24.954 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:24.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:24.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:25.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:24 smithi173 bash[59422]: audit 2023-09-23T18:56:23.754747+0000 mon.a (mon.0) 1575 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:56:25.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:24 smithi173 bash[59422]: audit 2023-09-23T18:56:23.764956+0000 mon.a (mon.0) 1576 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:56:25.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:24 smithi173 bash[59422]: audit 2023-09-23T18:56:24.630938+0000 mon.a (mon.0) 1577 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:56:25.063 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:25.096 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:25.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:25.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:25.202 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:25.230 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:25.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:25.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:25.327 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 41 16 44797 44781 68.2512 81.9375 0.00361851 0.0146443 2023-09-23T18:56:25.333 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:25.364 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:25.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:25.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:25.471 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:25.491 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:56:25.495 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:25.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:25.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:25.598 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:25.634 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:25.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:25.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:25.738 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:25.768 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:25.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:25.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:25.873 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:25.902 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:25.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:25.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:26.007 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:26.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:25 smithi173 bash[59422]: cluster 2023-09-23T18:56:24.287720+0000 mgr.y (mgr.26054) 1153 : cluster 0 pgmap v1250: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s; 23074/121785 objects degraded (18.947%); 2/121785 objects misplaced (0.002%) 2023-09-23T18:56:26.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:25 smithi173 bash[59422]: cluster 2023-09-23T18:56:25.357501+0000 mon.a (mon.0) 1578 : cluster 3 Health check update: Degraded data redundancy: 23074/121785 objects degraded (18.947%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:26.039 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:26.043 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:25 smithi098 bash[79516]: cluster 2023-09-23T18:56:24.287720+0000 mgr.y (mgr.26054) 1153 : cluster 0 pgmap v1250: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s; 23074/121785 objects degraded (18.947%); 2/121785 objects misplaced (0.002%) 2023-09-23T18:56:26.043 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:25 smithi098 bash[79516]: cluster 2023-09-23T18:56:25.357501+0000 mon.a (mon.0) 1578 : cluster 3 Health check update: Degraded data redundancy: 23074/121785 objects degraded (18.947%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:26.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:26.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:26.043 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:25 smithi098 bash[76817]: cluster 2023-09-23T18:56:24.287720+0000 mgr.y (mgr.26054) 1153 : cluster 0 pgmap v1250: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s; 23074/121785 objects degraded (18.947%); 2/121785 objects misplaced (0.002%) 2023-09-23T18:56:26.044 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:25 smithi098 bash[76817]: cluster 2023-09-23T18:56:25.357501+0000 mon.a (mon.0) 1578 : cluster 3 Health check update: Degraded data redundancy: 23074/121785 objects degraded (18.947%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:26.144 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:26.176 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:26.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:26.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:26.279 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:26.311 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:26.314 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:56:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:56:26] "GET /metrics HTTP/1.1" 200 36585 "" "Prometheus/2.33.4" 2023-09-23T18:56:26.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:26.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:26.327 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 42 16 46044 46028 68.4816 77.9375 0.0104275 0.0145966 2023-09-23T18:56:26.416 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:26.456 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:26.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:26.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:26.559 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:26.584 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:26.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:26.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:26.687 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:26.720 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:26.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:26.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:26.826 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:26.851 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:26.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:26.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:26.954 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:26.979 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:26.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:26.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:27.065 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:56:27.082 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:27.132 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:27.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:27.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:27.242 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:27.247 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T18:56:27.248 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:56:27.278 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:27.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:27.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:27.327 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 43 16 47154 47138 68.5023 69.375 0.00939028 0.0145927 2023-09-23T18:56:27.384 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:27.411 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:27.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:27.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:27.519 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:27.555 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:27.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:27.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:27.660 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:27.692 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:27.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:27.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:27.799 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:27.825 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:27.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:27.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:27.927 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:27.936 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:27 smithi098 bash[79516]: cluster 2023-09-23T18:56:26.288763+0000 mgr.y (mgr.26054) 1154 : cluster 0 pgmap v1251: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 23833/128688 objects degraded (18.520%); 2/128688 objects misplaced (0.002%) 2023-09-23T18:56:27.936 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:27 smithi098 bash[79516]: audit 2023-09-23T18:56:27.509389+0000 mon.a (mon.0) 1579 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:56:27.936 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:27 smithi098 bash[79516]: audit 2023-09-23T18:56:27.521529+0000 mon.a (mon.0) 1580 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:56:27.937 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:27 smithi098 bash[76817]: cluster 2023-09-23T18:56:26.288763+0000 mgr.y (mgr.26054) 1154 : cluster 0 pgmap v1251: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 23833/128688 objects degraded (18.520%); 2/128688 objects misplaced (0.002%) 2023-09-23T18:56:27.937 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:27 smithi098 bash[76817]: audit 2023-09-23T18:56:27.509389+0000 mon.a (mon.0) 1579 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:56:27.937 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:27 smithi098 bash[76817]: audit 2023-09-23T18:56:27.521529+0000 mon.a (mon.0) 1580 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:56:27.953 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:27.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:27.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:28.057 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:28.066 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:27 smithi173 bash[59422]: cluster 2023-09-23T18:56:26.288763+0000 mgr.y (mgr.26054) 1154 : cluster 0 pgmap v1251: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 23833/128688 objects degraded (18.520%); 2/128688 objects misplaced (0.002%) 2023-09-23T18:56:28.066 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:27 smithi173 bash[59422]: audit 2023-09-23T18:56:27.509389+0000 mon.a (mon.0) 1579 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:56:28.066 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:27 smithi173 bash[59422]: audit 2023-09-23T18:56:27.521529+0000 mon.a (mon.0) 1580 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:56:28.083 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:28.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:28.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:28.186 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:28.214 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:28.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:28.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:28.247 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T18:56:28.318 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:28.327 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 44 15 48305 48290 68.5816 72 0.0119599 0.0145754 2023-09-23T18:56:28.345 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:28.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:28.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:28.448 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:28.476 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:28.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:28.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:28.579 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:28.605 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:28.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:28.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:28.709 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:28.741 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:28.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:28.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:28.845 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:28.855 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:28 smithi098 bash[76817]: audit 2023-09-23T18:56:28.161564+0000 mon.a (mon.0) 1581 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:56:28.855 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:28 smithi098 bash[76817]: audit 2023-09-23T18:56:28.164456+0000 mon.a (mon.0) 1582 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:56:28.855 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:28 smithi098 bash[76817]: audit 2023-09-23T18:56:28.175911+0000 mon.a (mon.0) 1583 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:56:28.855 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:28 smithi098 bash[79516]: audit 2023-09-23T18:56:28.161564+0000 mon.a (mon.0) 1581 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:56:28.855 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:28 smithi098 bash[79516]: audit 2023-09-23T18:56:28.164456+0000 mon.a (mon.0) 1582 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:56:28.855 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:28 smithi098 bash[79516]: audit 2023-09-23T18:56:28.175911+0000 mon.a (mon.0) 1583 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:56:28.873 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:28.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:28.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:28.981 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:29.006 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:29.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:29.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:29.101 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:56:29.111 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:29.147 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:29.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:29.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:29.253 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:29.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:28 smithi173 bash[59422]: audit 2023-09-23T18:56:28.161564+0000 mon.a (mon.0) 1581 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:56:29.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:28 smithi173 bash[59422]: audit 2023-09-23T18:56:28.164456+0000 mon.a (mon.0) 1582 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:56:29.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:28 smithi173 bash[59422]: audit 2023-09-23T18:56:28.175911+0000 mon.a (mon.0) 1583 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:56:29.285 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:29.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:29.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:29.328 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 45 16 49576 49560 68.8213 79.375 0.0109435 0.0145235 2023-09-23T18:56:29.388 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:29.419 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:29.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:29.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:29.521 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:29.548 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:29.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:29.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:29.651 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:29.678 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:29.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:29.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:29.782 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:29.810 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:29.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:29.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:29.913 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:29.941 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:29.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:29.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:30.044 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:30.073 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:30.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:30.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:30.125 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:56:30.177 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:30.180 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:30 smithi098 bash[76817]: cluster 2023-09-23T18:56:28.289664+0000 mgr.y (mgr.26054) 1155 : cluster 0 pgmap v1252: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 26164/139413 objects degraded (18.767%); 2/139413 objects misplaced (0.001%) 2023-09-23T18:56:30.212 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:30.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:30.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:30.317 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:30.328 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 46 16 50948 50932 69.1889 85.75 0.00567445 0.0144465 2023-09-23T18:56:30.351 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:30.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:30.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:30.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:30 smithi098 bash[79516]: cluster 2023-09-23T18:56:28.289664+0000 mgr.y (mgr.26054) 1155 : cluster 0 pgmap v1252: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 26164/139413 objects degraded (18.767%); 2/139413 objects misplaced (0.001%) 2023-09-23T18:56:30.456 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:30.493 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:30.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:30.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:30.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:30 smithi173 bash[59422]: cluster 2023-09-23T18:56:28.289664+0000 mgr.y (mgr.26054) 1155 : cluster 0 pgmap v1252: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 26164/139413 objects degraded (18.767%); 2/139413 objects misplaced (0.001%) 2023-09-23T18:56:30.596 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:30.616 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:30.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:30.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:30.722 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:30.759 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:30.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:30.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:30.862 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:30.889 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:30.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:30.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:30.991 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:31.023 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:31.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:31.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:31.127 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:31.149 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:31.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:31.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:31.251 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:31.260 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:31 smithi098 bash[79516]: cluster 2023-09-23T18:56:30.359053+0000 mon.a (mon.0) 1584 : cluster 3 Health check update: Degraded data redundancy: 26164/139413 objects degraded (18.767%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:31.261 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:31 smithi098 bash[76817]: cluster 2023-09-23T18:56:30.359053+0000 mon.a (mon.0) 1584 : cluster 3 Health check update: Degraded data redundancy: 26164/139413 objects degraded (18.767%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:31.285 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:31.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:31.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:31.328 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 47 16 52068 52052 69.206 70 0.00654227 0.0144442 2023-09-23T18:56:31.389 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:31.418 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:31.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:31.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:31.526 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:31.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:31 smithi173 bash[59422]: cluster 2023-09-23T18:56:30.359053+0000 mon.a (mon.0) 1584 : cluster 3 Health check update: Degraded data redundancy: 26164/139413 objects degraded (18.767%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:31.570 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:31.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:31.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:31.675 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:31.700 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:31.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:31.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:31.804 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:31.829 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:31.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:31.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:31.933 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:31.951 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:31.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:31.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:32.054 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:32.084 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:32.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:32.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:32.188 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:32.213 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:32.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:32.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:32.316 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:32.325 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:32 smithi098 bash[79516]: cluster 2023-09-23T18:56:30.290512+0000 mgr.y (mgr.26054) 1156 : cluster 0 pgmap v1253: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 26899/142764 objects degraded (18.842%); 2/142764 objects misplaced (0.001%) 2023-09-23T18:56:32.325 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:32 smithi098 bash[76817]: cluster 2023-09-23T18:56:30.290512+0000 mgr.y (mgr.26054) 1156 : cluster 0 pgmap v1253: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 26899/142764 objects degraded (18.842%); 2/142764 objects misplaced (0.001%) 2023-09-23T18:56:32.328 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 48 16 53204 53188 69.2427 71 0.0240404 0.0144358 2023-09-23T18:56:32.348 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:32.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:32.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:32.451 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:32.475 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:32.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:32.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:32.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:32 smithi173 bash[59422]: cluster 2023-09-23T18:56:30.290512+0000 mgr.y (mgr.26054) 1156 : cluster 0 pgmap v1253: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 26899/142764 objects degraded (18.842%); 2/142764 objects misplaced (0.001%) 2023-09-23T18:56:32.580 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:32.605 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:32.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:32.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:32.708 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:32.735 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:32.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:32.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:32.838 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:32.865 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:32.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:32.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:32.968 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:32.992 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:32.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:32.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:33.046 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:56:33.094 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:33.122 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:33.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:33.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:33.226 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:33.254 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:33.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:33.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:33.328 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 49 16 54390 54374 69.3422 74.125 0.00653233 0.0144152 2023-09-23T18:56:33.359 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:33.385 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:33.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:33.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:33.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:33.513 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:33.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:33.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:33.618 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:33.642 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:33.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:33.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:33.748 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:33.775 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:33.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:33.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:33.879 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:33.902 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:33.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:33.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:34.007 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:34.032 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:34.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:34.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:34.134 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:34.157 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:34.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:34.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:34.260 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:34.268 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:34 smithi098 bash[76817]: cluster 2023-09-23T18:56:32.291265+0000 mgr.y (mgr.26054) 1157 : cluster 0 pgmap v1254: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 27294/147420 objects degraded (18.514%); 2/147420 objects misplaced (0.001%) 2023-09-23T18:56:34.268 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:34 smithi098 bash[79516]: cluster 2023-09-23T18:56:32.291265+0000 mgr.y (mgr.26054) 1157 : cluster 0 pgmap v1254: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 27294/147420 objects degraded (18.514%); 2/147420 objects misplaced (0.001%) 2023-09-23T18:56:34.291 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:34.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:34.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:34.329 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 50 16 55734 55718 69.6351 84 0.00502172 0.0143548 2023-09-23T18:56:34.395 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:34.413 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:34.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:34.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:34.515 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:34.525 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:34 smithi173 bash[59422]: cluster 2023-09-23T18:56:32.291265+0000 mgr.y (mgr.26054) 1157 : cluster 0 pgmap v1254: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 27294/147420 objects degraded (18.514%); 2/147420 objects misplaced (0.001%) 2023-09-23T18:56:34.541 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:34.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:34.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:34.644 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:34.672 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:34.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:34.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:34.776 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:34.810 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:34.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:34.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:34.913 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:34.957 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:34.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:34.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:35.059 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:35.083 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:35.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:35.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:35.188 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:35.214 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:35.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:35.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:35.319 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:35.329 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 51 16 56901 56885 69.6996 72.9375 0.0165572 0.0143405 2023-09-23T18:56:35.340 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:35.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:35.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:35.445 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:35.477 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:35.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:35.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:35.581 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:35.605 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:35.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:35.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:35.710 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:35.740 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:35.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:35.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:35.844 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:35.866 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:35.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:35.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:35.969 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:35.995 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:35.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:35.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:36.097 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:36.122 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:36.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:36.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:36.192 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:56:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:56:36] "GET /metrics HTTP/1.1" 200 36585 "" "Prometheus/2.33.4" 2023-09-23T18:56:36.225 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:36.254 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:36.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:36.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:36.318 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:36 smithi098 bash[76817]: cluster 2023-09-23T18:56:34.292066+0000 mgr.y (mgr.26054) 1158 : cluster 0 pgmap v1255: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 30189/160293 objects degraded (18.834%); 2/160293 objects misplaced (0.001%) 2023-09-23T18:56:36.318 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:36 smithi098 bash[76817]: cluster 2023-09-23T18:56:35.360130+0000 mon.a (mon.0) 1585 : cluster 3 Health check update: Degraded data redundancy: 30189/160293 objects degraded (18.834%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:36.318 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:36 smithi098 bash[79516]: cluster 2023-09-23T18:56:34.292066+0000 mgr.y (mgr.26054) 1158 : cluster 0 pgmap v1255: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 30189/160293 objects degraded (18.834%); 2/160293 objects misplaced (0.001%) 2023-09-23T18:56:36.319 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:36 smithi098 bash[79516]: cluster 2023-09-23T18:56:35.360130+0000 mon.a (mon.0) 1585 : cluster 3 Health check update: Degraded data redundancy: 30189/160293 objects degraded (18.834%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:36.329 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 52 16 57926 57910 69.5911 64.0625 0.0110731 0.0143639 2023-09-23T18:56:36.360 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:36.394 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:36.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:36.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:36.497 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:36.523 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:36.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:36.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:36.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:36 smithi173 bash[59422]: cluster 2023-09-23T18:56:34.292066+0000 mgr.y (mgr.26054) 1158 : cluster 0 pgmap v1255: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 30189/160293 objects degraded (18.834%); 2/160293 objects misplaced (0.001%) 2023-09-23T18:56:36.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:36 smithi173 bash[59422]: cluster 2023-09-23T18:56:35.360130+0000 mon.a (mon.0) 1585 : cluster 3 Health check update: Degraded data redundancy: 30189/160293 objects degraded (18.834%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:36.627 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:36.649 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:36.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:36.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:36.752 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:36.781 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:36.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:36.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:36.884 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:36.910 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:36.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:36.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:37.015 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:37.047 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:37.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:37.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:37.151 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:37.173 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:37.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:37.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:37.276 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:37.305 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:37.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:37.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:37.329 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 53 15 58942 58927 69.4773 63.5625 0.0232194 0.0143869 2023-09-23T18:56:37.412 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:37.435 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:37.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:37.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:37.539 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:37.565 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:37.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:37.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:37.669 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:37.693 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:37.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:37.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:37.797 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:37.824 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:37.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:37.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:37.929 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:37.954 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:37.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:37.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:38.061 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:38.086 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:38.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:38.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:38.114 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T18:56:38.114 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:56:38.190 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:38.217 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:38.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:38.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:38.322 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:38.329 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 54 16 60240 60224 69.6916 81.0625 0.0046332 0.0143422 2023-09-23T18:56:38.330 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:38 smithi098 bash[79516]: cluster 2023-09-23T18:56:36.293120+0000 mgr.y (mgr.26054) 1159 : cluster 0 pgmap v1256: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 30565/164901 objects degraded (18.535%); 2/164901 objects misplaced (0.001%) 2023-09-23T18:56:38.330 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:38 smithi098 bash[76817]: cluster 2023-09-23T18:56:36.293120+0000 mgr.y (mgr.26054) 1159 : cluster 0 pgmap v1256: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 30565/164901 objects degraded (18.535%); 2/164901 objects misplaced (0.001%) 2023-09-23T18:56:38.354 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:38.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:38.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:38.458 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:38.482 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:38.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:38.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:38.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:38 smithi173 bash[59422]: cluster 2023-09-23T18:56:36.293120+0000 mgr.y (mgr.26054) 1159 : cluster 0 pgmap v1256: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 30565/164901 objects degraded (18.535%); 2/164901 objects misplaced (0.001%) 2023-09-23T18:56:38.585 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:38.609 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:38.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:38.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:38.716 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:38.741 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:38.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:38.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:38.845 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:38.875 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:38.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:38.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:38.980 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:39.003 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:39.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:39.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:39.107 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:39.132 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:39.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:39.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:39.189 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T18:56:39.237 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:39.261 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:39.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:39.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:39.329 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 55 16 61493 61477 69.8482 78.3125 0.0115301 0.0143105 2023-09-23T18:56:39.365 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:39.395 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:39.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:39.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:39.504 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:39.536 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:39.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:39.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:39.643 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:39.668 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:39.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:39.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:39.771 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:39.802 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:39.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:39.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:39.907 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:39.932 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:39.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:39.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:40.036 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:40.060 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:40.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:40.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:40.164 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:40.188 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:40.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:40.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:40.294 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:40.319 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:40.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:40.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:40.329 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 56 15 62628 62613 69.8687 71 0.03024 0.0143072 2023-09-23T18:56:40.423 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:40.431 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:40 smithi098 bash[79516]: cluster 2023-09-23T18:56:38.293879+0000 mgr.y (mgr.26054) 1160 : cluster 0 pgmap v1257: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s; 33115/176901 objects degraded (18.720%); 2/176901 objects misplaced (0.001%) 2023-09-23T18:56:40.431 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:40 smithi098 bash[79516]: audit 2023-09-23T18:56:39.631222+0000 mon.a (mon.0) 1586 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:56:40.432 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:40 smithi098 bash[76817]: cluster 2023-09-23T18:56:38.293879+0000 mgr.y (mgr.26054) 1160 : cluster 0 pgmap v1257: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s; 33115/176901 objects degraded (18.720%); 2/176901 objects misplaced (0.001%) 2023-09-23T18:56:40.432 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:40 smithi098 bash[76817]: audit 2023-09-23T18:56:39.631222+0000 mon.a (mon.0) 1586 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:56:40.451 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:40.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:40.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:40.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:40 smithi173 bash[59422]: cluster 2023-09-23T18:56:38.293879+0000 mgr.y (mgr.26054) 1160 : cluster 0 pgmap v1257: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s; 33115/176901 objects degraded (18.720%); 2/176901 objects misplaced (0.001%) 2023-09-23T18:56:40.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:40 smithi173 bash[59422]: audit 2023-09-23T18:56:39.631222+0000 mon.a (mon.0) 1586 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:56:40.556 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:40.581 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:40.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:40.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:40.686 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:40.717 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:40.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:40.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:40.819 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:40.846 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:40.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:40.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:40.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:40.979 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:40.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:40.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:41.085 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:41.116 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:41.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:41.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:41.220 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:41.245 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:41.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:41.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:41.329 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 57 16 63834 63818 69.9641 75.3125 0.00698412 0.0142872 2023-09-23T18:56:41.348 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:41.357 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:41 smithi098 bash[79516]: cluster 2023-09-23T18:56:40.361538+0000 mon.a (mon.0) 1587 : cluster 3 Health check update: Degraded data redundancy: 33115/176901 objects degraded (18.720%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:41.357 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:41 smithi098 bash[76817]: cluster 2023-09-23T18:56:40.361538+0000 mon.a (mon.0) 1587 : cluster 3 Health check update: Degraded data redundancy: 33115/176901 objects degraded (18.720%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:41.380 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:41.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:41.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:41.485 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:41.494 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:41 smithi173 bash[59422]: cluster 2023-09-23T18:56:40.361538+0000 mon.a (mon.0) 1587 : cluster 3 Health check update: Degraded data redundancy: 33115/176901 objects degraded (18.720%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:41.511 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:41.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:41.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:41.616 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:41.642 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:41.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:41.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:41.746 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:41.766 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:41.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:41.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:41.869 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:41.893 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:41.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:41.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:41.996 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:42.023 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:42.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:42.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:42.128 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:42.157 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:42.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:42.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:42.263 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:42.294 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:42.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:42.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:42.329 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 58 16 64749 64733 69.7437 57.1875 0.00775163 0.0143321 2023-09-23T18:56:42.399 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:42.407 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:42 smithi098 bash[79516]: cluster 2023-09-23T18:56:40.294713+0000 mgr.y (mgr.26054) 1161 : cluster 0 pgmap v1258: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 33479/177993 objects degraded (18.809%); 2/177993 objects misplaced (0.001%) 2023-09-23T18:56:42.408 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:42 smithi098 bash[76817]: cluster 2023-09-23T18:56:40.294713+0000 mgr.y (mgr.26054) 1161 : cluster 0 pgmap v1258: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 33479/177993 objects degraded (18.809%); 2/177993 objects misplaced (0.001%) 2023-09-23T18:56:42.427 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:42.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:42.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:42.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:42 smithi173 bash[59422]: cluster 2023-09-23T18:56:40.294713+0000 mgr.y (mgr.26054) 1161 : cluster 0 pgmap v1258: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 33479/177993 objects degraded (18.809%); 2/177993 objects misplaced (0.001%) 2023-09-23T18:56:42.531 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:42.563 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:42.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:42.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:42.667 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:42.695 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:42.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:42.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:42.798 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:42.825 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:42.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:42.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:42.928 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:42.953 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:42.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:42.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:43.055 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:43.086 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:43.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:43.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:43.191 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:43.223 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:43.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:43.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:43.327 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:43.330 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 59 16 65734 65718 69.605 61.5625 0.0143094 0.0143617 2023-09-23T18:56:43.350 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:43.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:43.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:43.456 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:43.482 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:43.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:43.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:43.590 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:43.616 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:43.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:43.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:43.720 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:43.739 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:43.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:43.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:43.842 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:43.859 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:43.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:43.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:43.962 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:43.986 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:56:43.993 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:43.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:43.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:44.096 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:44.143 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:44.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:44.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:44.247 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:44.271 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:44.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:44.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:44.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:44 smithi173 bash[59422]: cluster 2023-09-23T18:56:42.295487+0000 mgr.y (mgr.26054) 1162 : cluster 0 pgmap v1259: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 33832/182412 objects degraded (18.547%); 2/182412 objects misplaced (0.001%) 2023-09-23T18:56:44.330 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:2023-09-23T18:56:44.327127+0000 min lat: 0.00116653 max lat: 16.0742 avg lat: 0.0144198 2023-09-23T18:56:44.330 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:56:44.330 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 60 16 66576 66560 69.3219 52.625 0.0121587 0.0144198 2023-09-23T18:56:44.375 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:44.400 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:44.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:44.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:44.415 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:44 smithi098 bash[76817]: cluster 2023-09-23T18:56:42.295487+0000 mgr.y (mgr.26054) 1162 : cluster 0 pgmap v1259: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 33832/182412 objects degraded (18.547%); 2/182412 objects misplaced (0.001%) 2023-09-23T18:56:44.416 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:44 smithi098 bash[79516]: cluster 2023-09-23T18:56:42.295487+0000 mgr.y (mgr.26054) 1162 : cluster 0 pgmap v1259: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 33832/182412 objects degraded (18.547%); 2/182412 objects misplaced (0.001%) 2023-09-23T18:56:44.503 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:44.535 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:44.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:44.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:44.641 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:44.680 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:44.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:44.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:44.784 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:44.805 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:44.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:44.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:44.908 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:44.931 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:44.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:44.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:45.035 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:45.062 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:45.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:45.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:45.167 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:45.193 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:45.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:45.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:45.296 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:45.313 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:45.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:45.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:45.330 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 61 16 67655 67639 69.2909 67.4375 0.00377549 0.014416 2023-09-23T18:56:45.415 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:45.445 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:45.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:45.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:45.549 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:45.573 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:45.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:45.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:45.638 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:56:45.638 INFO:teuthology.orchestra.run.smithi098.stdout: "success": "" 2023-09-23T18:56:45.638 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:56:45.683 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:45.709 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:45.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:45.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:45.812 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:45.840 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:45.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:45.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:45.945 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:45.974 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:45.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:45.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:46.079 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:46.087 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:56:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:56:46] "GET /metrics HTTP/1.1" 200 36584 "" "Prometheus/2.33.4" 2023-09-23T18:56:46.103 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:46.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:46.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:46.207 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:46.233 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:46.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:46.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:46.330 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 62 16 68722 68706 69.2488 66.6875 0.00551049 0.0144346 2023-09-23T18:56:46.336 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:46.345 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:46 smithi098 bash[79516]: cluster 2023-09-23T18:56:44.298274+0000 mgr.y (mgr.26054) 1163 : cluster 0 pgmap v1260: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 36569/194790 objects degraded (18.774%); 2/194790 objects misplaced (0.001%) 2023-09-23T18:56:46.345 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:46 smithi098 bash[79516]: cluster 2023-09-23T18:56:45.362553+0000 mon.a (mon.0) 1588 : cluster 3 Health check update: Degraded data redundancy: 36569/194790 objects degraded (18.774%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:46.345 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:46 smithi098 bash[76817]: cluster 2023-09-23T18:56:44.298274+0000 mgr.y (mgr.26054) 1163 : cluster 0 pgmap v1260: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 36569/194790 objects degraded (18.774%); 2/194790 objects misplaced (0.001%) 2023-09-23T18:56:46.345 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:46 smithi098 bash[76817]: cluster 2023-09-23T18:56:45.362553+0000 mon.a (mon.0) 1588 : cluster 3 Health check update: Degraded data redundancy: 36569/194790 objects degraded (18.774%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:46.377 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:46.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:46.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:46.482 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:46.502 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:46 smithi173 bash[59422]: cluster 2023-09-23T18:56:44.298274+0000 mgr.y (mgr.26054) 1163 : cluster 0 pgmap v1260: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 36569/194790 objects degraded (18.774%); 2/194790 objects misplaced (0.001%) 2023-09-23T18:56:46.503 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:46 smithi173 bash[59422]: cluster 2023-09-23T18:56:45.362553+0000 mon.a (mon.0) 1588 : cluster 3 Health check update: Degraded data redundancy: 36569/194790 objects degraded (18.774%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:46.503 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:46.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:46.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:46.611 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:46.636 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:46.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:46.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:46.741 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:46.773 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:46.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:46.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:46.877 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:46.899 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:46.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:46.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:47.004 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:47.029 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:47.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:47.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:47.134 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:47.162 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:47.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:47.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:47.265 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:47.289 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:47.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:47.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:47.330 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 63 16 69832 69816 69.2507 69.375 0.0261528 0.0144348 2023-09-23T18:56:47.393 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:47.422 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:47.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:47.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:47.529 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:47.558 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:47.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:47.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:47.663 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:47.683 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:47.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:47.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:47.787 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:47.819 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:47.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:47.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:47.926 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:47.972 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:47.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:47.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:48.079 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:48.106 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:48.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:48.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:48.209 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:48.235 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:48.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:48.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:48.330 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 64 16 70947 70931 69.2574 69.6875 0.0078483 0.0144321 2023-09-23T18:56:48.338 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:48.350 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:48 smithi173 bash[59422]: cluster 2023-09-23T18:56:46.299096+0000 mgr.y (mgr.26054) 1164 : cluster 0 pgmap v1261: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 36880/198468 objects degraded (18.582%); 2/198468 objects misplaced (0.001%) 2023-09-23T18:56:48.367 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:48.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:48.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:48.473 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:48.496 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:48.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:48.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:48.560 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T18:56:48.560 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:56:48.601 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:48.610 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:48 smithi098 bash[76817]: cluster 2023-09-23T18:56:46.299096+0000 mgr.y (mgr.26054) 1164 : cluster 0 pgmap v1261: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 36880/198468 objects degraded (18.582%); 2/198468 objects misplaced (0.001%) 2023-09-23T18:56:48.610 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:48 smithi098 bash[79516]: cluster 2023-09-23T18:56:46.299096+0000 mgr.y (mgr.26054) 1164 : cluster 0 pgmap v1261: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 36880/198468 objects degraded (18.582%); 2/198468 objects misplaced (0.001%) 2023-09-23T18:56:48.626 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:48.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:48.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:48.732 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:48.761 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:48.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:48.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:48.867 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:48.896 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:48.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:48.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:48.999 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:49.023 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:49.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:49.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:49.126 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:49.152 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:49.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:49.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:49.256 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:49.259 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:56:49.277 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:49.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:49.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:49.317 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:56:49.330 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 65 16 71929 71913 69.1361 61.375 0.00920224 0.0144577 2023-09-23T18:56:49.353 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:56:49.379 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:49.395 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:56:49.416 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:49.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:49.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:49.435 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:56:49.472 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:56:49.510 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:56:49.513 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:56:49.514 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:56:49.514 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:56:49.514 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:56:49.514 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:56:49.514 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:56:49.514 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:56:49.514 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T18:56:49.522 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:49.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:49 smithi173 bash[59422]: audit 2023-09-23T18:56:49.202841+0000 mon.c (mon.2) 128 : audit 0 from='client.? 172.21.15.98:0/120700381' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:56:49.532 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:49 smithi098 bash[79516]: audit 2023-09-23T18:56:49.202841+0000 mon.c (mon.2) 128 : audit 0 from='client.? 172.21.15.98:0/120700381' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:56:49.532 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:49 smithi098 bash[76817]: audit 2023-09-23T18:56:49.202841+0000 mon.c (mon.2) 128 : audit 0 from='client.? 172.21.15.98:0/120700381' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:56:49.545 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:49.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:49.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:49.650 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:49.675 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:49.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:49.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:49.770 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T18:56:49.780 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:49.807 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:49.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:49.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:49.911 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:49.940 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:49.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:49.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:50.044 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:50.069 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:50.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:50.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:50.173 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:50.206 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:50.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:50.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:50.270 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is unset 2023-09-23T18:56:50.310 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:50.321 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:50 smithi173 bash[59422]: cluster 2023-09-23T18:56:48.299942+0000 mgr.y (mgr.26054) 1165 : cluster 0 pgmap v1262: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 39170/209283 objects degraded (18.716%); 2/209283 objects misplaced (0.001%) 2023-09-23T18:56:50.321 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:50 smithi173 bash[59422]: audit 2023-09-23T18:56:49.410129+0000 mon.c (mon.2) 129 : audit 1 from='client.? 172.21.15.98:0/2631028125' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:56:50.321 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:50 smithi173 bash[59422]: audit 2023-09-23T18:56:49.410919+0000 mon.a (mon.0) 1589 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:56:50.330 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 66 16 73005 72989 69.1074 67.25 0.00457576 0.0144645 2023-09-23T18:56:50.336 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:50.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:50.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:50.440 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:50.465 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:50.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:50.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:50.534 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:50 smithi098 bash[79516]: cluster 2023-09-23T18:56:48.299942+0000 mgr.y (mgr.26054) 1165 : cluster 0 pgmap v1262: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 39170/209283 objects degraded (18.716%); 2/209283 objects misplaced (0.001%) 2023-09-23T18:56:50.534 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:50 smithi098 bash[79516]: audit 2023-09-23T18:56:49.410129+0000 mon.c (mon.2) 129 : audit 1 from='client.? 172.21.15.98:0/2631028125' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:56:50.534 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:50 smithi098 bash[79516]: audit 2023-09-23T18:56:49.410919+0000 mon.a (mon.0) 1589 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:56:50.567 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:50.592 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:50.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:50.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:50.695 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:50.722 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:50.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:50.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:50.825 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:50.834 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:50 smithi098 bash[76817]: cluster 2023-09-23T18:56:48.299942+0000 mgr.y (mgr.26054) 1165 : cluster 0 pgmap v1262: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 39170/209283 objects degraded (18.716%); 2/209283 objects misplaced (0.001%) 2023-09-23T18:56:50.834 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:50 smithi098 bash[76817]: audit 2023-09-23T18:56:49.410129+0000 mon.c (mon.2) 129 : audit 1 from='client.? 172.21.15.98:0/2631028125' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:56:50.835 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:50 smithi098 bash[76817]: audit 2023-09-23T18:56:49.410919+0000 mon.a (mon.0) 1589 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:56:50.877 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:50.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:50.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:50.981 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:51.004 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:51.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:51.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:51.109 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:51.134 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:51.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:51.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:51.238 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:51.265 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:51.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:51.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:51.330 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 67 16 74110 74094 69.1067 69.0625 0.0151457 0.0144649 2023-09-23T18:56:51.370 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:51.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:51 smithi173 bash[59422]: cluster 2023-09-23T18:56:50.245632+0000 mon.a (mon.0) 1590 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:56:51.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:51 smithi173 bash[59422]: cluster 2023-09-23T18:56:50.250132+0000 osd.4 (osd.4) 9 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T18:56:51.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:51 smithi173 bash[59422]: cluster 2023-09-23T18:56:50.255986+0000 osd.5 (osd.5) 204 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T18:56:51.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:51 smithi173 bash[59422]: audit 2023-09-23T18:56:50.258175+0000 mon.a (mon.0) 1591 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:56:51.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:51 smithi173 bash[59422]: cluster 2023-09-23T18:56:50.258521+0000 mon.a (mon.0) 1592 : cluster 0 osdmap e290: 8 total, 7 up, 8 in 2023-09-23T18:56:51.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:51 smithi173 bash[59422]: cluster 2023-09-23T18:56:50.258973+0000 osd.6 (osd.6) 258 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T18:56:51.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:51 smithi173 bash[59422]: cluster 2023-09-23T18:56:50.264424+0000 osd.5 (osd.5) 205 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T18:56:51.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:51 smithi173 bash[59422]: cluster 2023-09-23T18:56:50.270281+0000 osd.6 (osd.6) 259 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T18:56:51.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:51 smithi173 bash[59422]: cluster 2023-09-23T18:56:50.274519+0000 osd.6 (osd.6) 260 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T18:56:51.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:51 smithi173 bash[59422]: cluster 2023-09-23T18:56:50.301912+0000 mgr.y (mgr.26054) 1166 : cluster 0 pgmap v1264: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 39473/210192 objects degraded (18.779%); 2/210192 objects misplaced (0.001%) 2023-09-23T18:56:51.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:51 smithi173 bash[59422]: cluster 2023-09-23T18:56:50.363779+0000 mon.a (mon.0) 1593 : cluster 3 Health check update: Degraded data redundancy: 39170/209283 objects degraded (18.716%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:51.395 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:51.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:51.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:51.499 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:51.524 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:51.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:51.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:51.585 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:51 smithi098 bash[76817]: cluster 2023-09-23T18:56:50.245632+0000 mon.a (mon.0) 1590 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:56:51.585 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:51 smithi098 bash[76817]: cluster 2023-09-23T18:56:50.250132+0000 osd.4 (osd.4) 9 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T18:56:51.585 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:51 smithi098 bash[76817]: cluster 2023-09-23T18:56:50.255986+0000 osd.5 (osd.5) 204 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T18:56:51.585 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:51 smithi098 bash[76817]: audit 2023-09-23T18:56:50.258175+0000 mon.a (mon.0) 1591 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:56:51.585 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:51 smithi098 bash[76817]: cluster 2023-09-23T18:56:50.258521+0000 mon.a (mon.0) 1592 : cluster 0 osdmap e290: 8 total, 7 up, 8 in 2023-09-23T18:56:51.585 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:51 smithi098 bash[76817]: cluster 2023-09-23T18:56:50.258973+0000 osd.6 (osd.6) 258 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T18:56:51.585 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:51 smithi098 bash[76817]: cluster 2023-09-23T18:56:50.264424+0000 osd.5 (osd.5) 205 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T18:56:51.585 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:51 smithi098 bash[76817]: cluster 2023-09-23T18:56:50.270281+0000 osd.6 (osd.6) 259 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T18:56:51.586 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:51 smithi098 bash[76817]: cluster 2023-09-23T18:56:50.274519+0000 osd.6 (osd.6) 260 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T18:56:51.586 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:51 smithi098 bash[76817]: cluster 2023-09-23T18:56:50.301912+0000 mgr.y (mgr.26054) 1166 : cluster 0 pgmap v1264: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 39473/210192 objects degraded (18.779%); 2/210192 objects misplaced (0.001%) 2023-09-23T18:56:51.586 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:51 smithi098 bash[76817]: cluster 2023-09-23T18:56:50.363779+0000 mon.a (mon.0) 1593 : cluster 3 Health check update: Degraded data redundancy: 39170/209283 objects degraded (18.716%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:51.586 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:51 smithi098 bash[79516]: cluster 2023-09-23T18:56:50.245632+0000 mon.a (mon.0) 1590 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:56:51.586 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:51 smithi098 bash[79516]: cluster 2023-09-23T18:56:50.250132+0000 osd.4 (osd.4) 9 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T18:56:51.586 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:51 smithi098 bash[79516]: cluster 2023-09-23T18:56:50.255986+0000 osd.5 (osd.5) 204 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T18:56:51.586 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:51 smithi098 bash[79516]: audit 2023-09-23T18:56:50.258175+0000 mon.a (mon.0) 1591 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:56:51.586 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:51 smithi098 bash[79516]: cluster 2023-09-23T18:56:50.258521+0000 mon.a (mon.0) 1592 : cluster 0 osdmap e290: 8 total, 7 up, 8 in 2023-09-23T18:56:51.586 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:51 smithi098 bash[79516]: cluster 2023-09-23T18:56:50.258973+0000 osd.6 (osd.6) 258 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T18:56:51.586 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:51 smithi098 bash[79516]: cluster 2023-09-23T18:56:50.264424+0000 osd.5 (osd.5) 205 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T18:56:51.586 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:51 smithi098 bash[79516]: cluster 2023-09-23T18:56:50.270281+0000 osd.6 (osd.6) 259 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T18:56:51.587 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:51 smithi098 bash[79516]: cluster 2023-09-23T18:56:50.274519+0000 osd.6 (osd.6) 260 : cluster 3 failed to encode map e290 with expected crc 2023-09-23T18:56:51.587 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:51 smithi098 bash[79516]: cluster 2023-09-23T18:56:50.301912+0000 mgr.y (mgr.26054) 1166 : cluster 0 pgmap v1264: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 39473/210192 objects degraded (18.779%); 2/210192 objects misplaced (0.001%) 2023-09-23T18:56:51.587 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:51 smithi098 bash[79516]: cluster 2023-09-23T18:56:50.363779+0000 mon.a (mon.0) 1593 : cluster 3 Health check update: Degraded data redundancy: 39170/209283 objects degraded (18.716%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:51.587 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:56:51.589 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:56:51.627 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:51.652 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:51.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:51.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:51.755 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:51.786 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:51.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:51.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:51.892 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:51.922 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:51.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:51.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:52.028 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:52.062 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:52.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:52.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:52.166 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:52.187 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:52.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:52.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:52.290 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:52.323 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:52.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:52.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:52.330 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 68 16 75097 75081 68.9975 61.6875 0.0139902 0.0144876 2023-09-23T18:56:52.427 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:52.453 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:52.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:52.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:52.560 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:52.589 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:52.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:52.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:52.696 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:52.723 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:52.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:52.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:52.827 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:52.861 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:52.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:52.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:52.967 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:52.993 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:52.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:52.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:53.096 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:53.122 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:53.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:53.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:53.225 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:53.262 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:53.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:53.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:53.330 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 69 16 76172 76156 68.9711 67.1875 0.00812678 0.0144936 2023-09-23T18:56:53.365 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:53 smithi098 bash[76817]: cluster 2023-09-23T18:56:52.304409+0000 mgr.y (mgr.26054) 1167 : cluster 0 pgmap v1265: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 39802/214146 objects degraded (18.586%); 2/214146 objects misplaced (0.001%) 2023-09-23T18:56:53.365 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:53.397 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:53.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:53.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:53.500 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:53.530 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:53.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:53.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:53.590 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:56:53.633 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:53.645 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:53 smithi098 bash[79516]: cluster 2023-09-23T18:56:52.304409+0000 mgr.y (mgr.26054) 1167 : cluster 0 pgmap v1265: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 39802/214146 objects degraded (18.586%); 2/214146 objects misplaced (0.001%) 2023-09-23T18:56:53.669 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:53.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:53.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:53.773 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:53.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:53 smithi173 bash[59422]: cluster 2023-09-23T18:56:52.304409+0000 mgr.y (mgr.26054) 1167 : cluster 0 pgmap v1265: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 39802/214146 objects degraded (18.586%); 2/214146 objects misplaced (0.001%) 2023-09-23T18:56:53.804 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:53.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:53.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:53.912 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:53.936 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:53.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:53.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:53.960 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7, 5] out_osds: [] dead_osds: [7] live_osds: [0, 2, 3, 5, 6, 1, 4] 2023-09-23T18:56:53.960 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:56:53.960 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T18:56:53.960 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T18:56:53.961 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T18:56:53.961 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T18:56:54.040 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:54.064 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:54.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:54.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:54.167 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:54.196 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:54.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:54.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:54.301 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:54.335 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 70 16 77152 77136 68.8593 61.25 0.0113173 0.0145167 2023-09-23T18:56:54.336 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:54.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:54.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:54.438 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:54.470 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:54.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:54.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:54.553 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:56:54.573 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:54.605 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:54.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:54.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:54.656 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:56:54.711 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:54.748 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:54.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:54.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:54.851 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:54.886 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:54.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:54.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:54.989 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:55.015 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:55.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:55.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:55.124 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:55.147 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:55.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:55.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:55.251 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:55.276 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:55.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:55.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:55.332 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 71 16 78135 78119 68.7547 61.4375 0.0180008 0.0145391 2023-09-23T18:56:55.380 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:55.402 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:55.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:55.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:55.506 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:55.534 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:55.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:55.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:55.639 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:55.664 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:55.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:55.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:55.768 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:55.778 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:55 smithi098 bash[76817]: cluster 2023-09-23T18:56:54.305299+0000 mgr.y (mgr.26054) 1168 : cluster 0 pgmap v1266: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 42432/226077 objects degraded (18.769%); 2/226077 objects misplaced (0.001%) 2023-09-23T18:56:55.778 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:55 smithi098 bash[76817]: audit 2023-09-23T18:56:54.638771+0000 mon.a (mon.0) 1594 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:56:55.778 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:55 smithi098 bash[76817]: audit 2023-09-23T18:56:54.640453+0000 mon.a (mon.0) 1595 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:56:55.778 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:55 smithi098 bash[76817]: cluster 2023-09-23T18:56:55.365006+0000 mon.a (mon.0) 1596 : cluster 3 Health check update: Degraded data redundancy: 42432/226077 objects degraded (18.769%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:55.778 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:55 smithi098 bash[79516]: cluster 2023-09-23T18:56:54.305299+0000 mgr.y (mgr.26054) 1168 : cluster 0 pgmap v1266: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 42432/226077 objects degraded (18.769%); 2/226077 objects misplaced (0.001%) 2023-09-23T18:56:55.778 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:55 smithi098 bash[79516]: audit 2023-09-23T18:56:54.638771+0000 mon.a (mon.0) 1594 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:56:55.778 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:55 smithi098 bash[79516]: audit 2023-09-23T18:56:54.640453+0000 mon.a (mon.0) 1595 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:56:55.779 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:55 smithi098 bash[79516]: cluster 2023-09-23T18:56:55.365006+0000 mon.a (mon.0) 1596 : cluster 3 Health check update: Degraded data redundancy: 42432/226077 objects degraded (18.769%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:55.795 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:55.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:55.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:55.900 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:55 smithi173 bash[59422]: cluster 2023-09-23T18:56:54.305299+0000 mgr.y (mgr.26054) 1168 : cluster 0 pgmap v1266: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 42432/226077 objects degraded (18.769%); 2/226077 objects misplaced (0.001%) 2023-09-23T18:56:55.900 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:55 smithi173 bash[59422]: audit 2023-09-23T18:56:54.638771+0000 mon.a (mon.0) 1594 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:56:55.900 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:55 smithi173 bash[59422]: audit 2023-09-23T18:56:54.640453+0000 mon.a (mon.0) 1595 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:56:55.900 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:55 smithi173 bash[59422]: cluster 2023-09-23T18:56:55.365006+0000 mon.a (mon.0) 1596 : cluster 3 Health check update: Degraded data redundancy: 42432/226077 objects degraded (18.769%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:56:55.904 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:55.927 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:55.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:55.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:56.030 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:56.057 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:56.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:56.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:56.088 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:56:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:56:56] "GET /metrics HTTP/1.1" 200 36584 "" "Prometheus/2.33.4" 2023-09-23T18:56:56.160 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:56.204 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:56.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:56.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:56.308 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:56.332 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 72 16 79129 79113 68.6625 62.125 0.00656844 0.0145586 2023-09-23T18:56:56.333 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:56.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:56.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:56.436 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:56.462 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:56.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:56.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:56.565 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:56.596 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:56.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:56.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:56.700 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:56.725 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:56.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:56.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:56.829 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:56.853 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:56.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:56.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:56.957 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:56.988 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:56.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:56.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:57.094 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:57.124 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:57.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:57.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:57.231 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:57.254 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:57.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:57.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:57.332 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 73 16 80095 80079 68.5489 60.375 0.00677841 0.0145817 2023-09-23T18:56:57.357 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:57.381 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:57.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:57.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:57.485 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:57.514 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:57.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:57.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:57.617 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:57.641 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:57.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:57.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:57.745 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:57.754 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:57 smithi098 bash[76817]: cluster 2023-09-23T18:56:56.306405+0000 mgr.y (mgr.26054) 1169 : cluster 0 pgmap v1267: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 42767/229977 objects degraded (18.596%); 2/229977 objects misplaced (0.001%) 2023-09-23T18:56:57.754 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:57 smithi098 bash[79516]: cluster 2023-09-23T18:56:56.306405+0000 mgr.y (mgr.26054) 1169 : cluster 0 pgmap v1267: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 42767/229977 objects degraded (18.596%); 2/229977 objects misplaced (0.001%) 2023-09-23T18:56:57.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:57.772 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:57.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:57.873 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:57.883 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:57 smithi173 bash[59422]: cluster 2023-09-23T18:56:56.306405+0000 mgr.y (mgr.26054) 1169 : cluster 0 pgmap v1267: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 42767/229977 objects degraded (18.596%); 2/229977 objects misplaced (0.001%) 2023-09-23T18:56:57.896 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:57.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:57.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:57.999 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:58.030 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:58.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:58.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:58.134 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:58.160 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:58.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:58.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:58.264 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:58.287 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:58.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:58.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:58.332 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 74 16 81065 81049 68.4417 60.625 0.0172107 0.014604 2023-09-23T18:56:58.390 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:58.412 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:58.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:58.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:58.433 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:56:58.433 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:56:58.516 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:58.542 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:58.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:58.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:58.644 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:58.665 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:56:58.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:58.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:56:58.768 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:58.792 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:56:58.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:58.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:56:58.894 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:58.933 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:58.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:58.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:56:59.039 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:59.064 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:56:59.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:59.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:56:59.167 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:59.217 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:59.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:59.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:59.324 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:59.332 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 75 16 82003 81987 68.3108 58.625 0.0307435 0.0146328 2023-09-23T18:56:59.352 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:56:59.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:59.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:56:59.449 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:56:59.455 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:59.480 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:59.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:59.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:59.583 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:59.606 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:56:59.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:59.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:56:59.699 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:56:59.699 INFO:teuthology.orchestra.run.smithi173.stdout: "size": 20, 2023-09-23T18:56:59.699 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 600, 2023-09-23T18:56:59.699 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-09-23T18:56:59.700 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.700 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35412.0:159639 8.4 e258/257 8:21b32c99:::benchmark_data_smithi098_176514_object56161:head v 258'9997, mlcod=0'0)", 2023-09-23T18:56:59.700 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:51:59.850986+0000", 2023-09-23T18:56:59.700 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 299.840899272, 2023-09-23T18:56:59.700 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.029942031000000001, 2023-09-23T18:56:59.700 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:59.700 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:56:59.700 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:59.700 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.700 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:59.700 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:51:59.850986+0000", 2023-09-23T18:56:59.700 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:59.701 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.701 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.701 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:59.701 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:51:59.850986+0000", 2023-09-23T18:56:59.701 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:56:59.701 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.701 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.701 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:59.701 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:51:59.850984+0000", 2023-09-23T18:56:59.701 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2683e-05 2023-09-23T18:56:59.701 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.701 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.702 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:59.702 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:51:59.850997+0000", 2023-09-23T18:56:59.702 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3008000000000001e-05 2023-09-23T18:56:59.702 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.702 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.702 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:59.702 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:51:59.851020+0000", 2023-09-23T18:56:59.702 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0417e-05 2023-09-23T18:56:59.702 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.702 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.702 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:59.702 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:51:59.851031+0000", 2023-09-23T18:56:59.703 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.7805000000000003e-05 2023-09-23T18:56:59.703 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.703 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.703 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:59.703 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:51:59.851128+0000", 2023-09-23T18:56:59.703 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.8819000000000001e-05 2023-09-23T18:56:59.703 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.703 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.703 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:59.703 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:51:59.851177+0000", 2023-09-23T18:56:59.703 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.02971149 2023-09-23T18:56:59.703 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.704 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.704 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:56:59.704 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:51:59.880889+0000", 2023-09-23T18:56:59.704 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.9229e-05 2023-09-23T18:56:59.704 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.704 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.704 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:56:59.704 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:51:59.880928+0000", 2023-09-23T18:56:59.704 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.029942031000000001 2023-09-23T18:56:59.704 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.704 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:59.705 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.705 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.705 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.705 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35412.0:175009 8.4 e261/257 8:2e319c62:::benchmark_data_smithi098_176514_object71531:head v 261'10989, mlcod=242'4502)", 2023-09-23T18:56:59.705 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:52:35.078385+0000", 2023-09-23T18:56:59.705 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 264.61349973900002, 2023-09-23T18:56:59.705 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.030839844000000002, 2023-09-23T18:56:59.705 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:59.705 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:56:59.705 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:59.705 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.705 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:59.705 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:35.078385+0000", 2023-09-23T18:56:59.706 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:59.706 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.706 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.706 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:59.706 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:35.078385+0000", 2023-09-23T18:56:59.706 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:56:59.706 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.706 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.706 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:59.706 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:35.078384+0000", 2023-09-23T18:56:59.706 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0305e-05 2023-09-23T18:56:59.707 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.707 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.707 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:59.707 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:35.078395+0000", 2023-09-23T18:56:59.707 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5733999999999999e-05 2023-09-23T18:56:59.707 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.707 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.707 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:59.707 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:35.078410+0000", 2023-09-23T18:56:59.707 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.1580000000000005e-06 2023-09-23T18:56:59.707 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.707 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.707 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:59.708 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:35.078418+0000", 2023-09-23T18:56:59.708 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6193999999999998e-05 2023-09-23T18:56:59.708 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.708 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.708 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:59.708 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:35.078475+0000", 2023-09-23T18:56:59.708 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.4486999999999997e-05 2023-09-23T18:56:59.708 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.708 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.708 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:59.708 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:35.078509+0000", 2023-09-23T18:56:59.708 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.030661005000000002 2023-09-23T18:56:59.709 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.709 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.709 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:56:59.709 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:35.109170+0000", 2023-09-23T18:56:59.709 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5111000000000001e-05 2023-09-23T18:56:59.709 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.709 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.709 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:56:59.709 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:35.109225+0000", 2023-09-23T18:56:59.709 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.030839844000000002 2023-09-23T18:56:59.709 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.709 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:59.710 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.710 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.710 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.710 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35412.0:175933 8.4 e262/257 8:25394559:::benchmark_data_smithi098_176514_object72455:head v 262'11048, mlcod=242'4558)", 2023-09-23T18:56:59.710 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:52:37.068150+0000", 2023-09-23T18:56:59.710 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 262.62373513400001, 2023-09-23T18:56:59.710 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.030974017, 2023-09-23T18:56:59.710 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:59.710 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:56:59.710 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:59.710 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.711 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:59.711 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:37.068150+0000", 2023-09-23T18:56:59.711 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:59.711 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.711 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.711 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:59.711 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:37.068150+0000", 2023-09-23T18:56:59.711 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:56:59.711 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.711 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.711 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:59.711 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:37.068148+0000", 2023-09-23T18:56:59.711 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1938e-05 2023-09-23T18:56:59.712 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.712 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.712 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:59.712 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:37.068160+0000", 2023-09-23T18:56:59.712 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5845000000000002e-05 2023-09-23T18:56:59.712 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.712 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.712 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:59.712 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:37.068176+0000", 2023-09-23T18:56:59.712 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.3969999999999997e-06 2023-09-23T18:56:59.712 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.712 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.712 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:59.713 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:37.068184+0000", 2023-09-23T18:56:59.713 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.4373000000000001e-05 2023-09-23T18:56:59.713 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.713 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.713 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:59.713 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:37.068258+0000", 2023-09-23T18:56:59.713 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.5005000000000001e-05 2023-09-23T18:56:59.713 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.713 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.713 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:59.713 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:37.068293+0000", 2023-09-23T18:56:59.713 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.030795624000000001 2023-09-23T18:56:59.713 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.714 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.714 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:56:59.714 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:37.099089+0000", 2023-09-23T18:56:59.714 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.5352000000000003e-05 2023-09-23T18:56:59.714 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.714 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.714 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:56:59.714 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:37.099124+0000", 2023-09-23T18:56:59.714 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.030974017 2023-09-23T18:56:59.714 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.714 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:59.714 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.715 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.715 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.715 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35412.0:175938 8.4 e262/257 8:2d4c95ae:::benchmark_data_smithi098_176514_object72460:head v 262'11049, mlcod=242'4558)", 2023-09-23T18:56:59.715 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:52:37.070293+0000", 2023-09-23T18:56:59.715 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 262.62159253499999, 2023-09-23T18:56:59.715 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.028850936000000001, 2023-09-23T18:56:59.715 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:59.715 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:56:59.715 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:59.715 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.715 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:59.715 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:37.070293+0000", 2023-09-23T18:56:59.715 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:59.716 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.716 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.716 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:59.716 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:37.070293+0000", 2023-09-23T18:56:59.716 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:56:59.716 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.716 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.716 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:59.716 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:37.070292+0000", 2023-09-23T18:56:59.716 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1951e-05 2023-09-23T18:56:59.716 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.716 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.716 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:59.717 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:37.070303+0000", 2023-09-23T18:56:59.717 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5900999999999999e-05 2023-09-23T18:56:59.717 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.717 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.717 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:59.717 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:37.070319+0000", 2023-09-23T18:56:59.717 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.0700000000000007e-06 2023-09-23T18:56:59.717 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.717 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.717 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:59.717 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:37.070327+0000", 2023-09-23T18:56:59.717 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0054816739999999997 2023-09-23T18:56:59.717 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.718 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.718 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:59.718 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:37.075809+0000", 2023-09-23T18:56:59.718 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.955e-05 2023-09-23T18:56:59.718 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.718 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.718 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:59.718 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:37.075839+0000", 2023-09-23T18:56:59.718 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.023290168999999999 2023-09-23T18:56:59.718 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.718 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.718 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:56:59.719 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:37.099129+0000", 2023-09-23T18:56:59.719 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4662e-05 2023-09-23T18:56:59.719 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.719 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.719 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:56:59.719 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:52:37.099143+0000", 2023-09-23T18:56:59.719 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.028850936000000001 2023-09-23T18:56:59.719 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.719 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:59.719 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.719 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.719 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.720 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35412.0:201712 8.4 e269/264 8:22580e46:::benchmark_data_smithi098_176514_object98234:head v 269'12667, mlcod=244'6179)", 2023-09-23T18:56:59.720 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:54:24.280178+0000", 2023-09-23T18:56:59.720 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 155.41170745100001, 2023-09-23T18:56:59.720 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.017157803999999999, 2023-09-23T18:56:59.720 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:59.720 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:56:59.720 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:59.720 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.720 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:59.720 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:24.280178+0000", 2023-09-23T18:56:59.720 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:59.720 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.720 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.720 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:59.721 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:24.280178+0000", 2023-09-23T18:56:59.721 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:56:59.721 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.721 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.721 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:59.721 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:24.280177+0000", 2023-09-23T18:56:59.721 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.1810000000000007e-06 2023-09-23T18:56:59.721 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.721 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.721 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:59.721 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:24.280186+0000", 2023-09-23T18:56:59.721 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4762e-05 2023-09-23T18:56:59.721 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.722 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.722 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:59.722 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:24.280201+0000", 2023-09-23T18:56:59.722 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.6440000000000003e-06 2023-09-23T18:56:59.722 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.722 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.722 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:59.722 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:24.280207+0000", 2023-09-23T18:56:59.722 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1279999999999997e-05 2023-09-23T18:56:59.722 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.722 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.722 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:59.722 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:24.280259+0000", 2023-09-23T18:56:59.723 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.4106000000000002e-05 2023-09-23T18:56:59.723 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.723 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.723 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:59.723 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:24.280293+0000", 2023-09-23T18:56:59.723 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.016991993 2023-09-23T18:56:59.723 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.723 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.723 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:56:59.723 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:24.297285+0000", 2023-09-23T18:56:59.723 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.0764999999999999e-05 2023-09-23T18:56:59.723 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.723 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.724 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:56:59.724 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:24.297335+0000", 2023-09-23T18:56:59.724 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.017157803999999999 2023-09-23T18:56:59.724 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.724 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:59.724 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.724 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.724 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.724 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35412.0:202606 8.4 e269/264 8:283233f7:::benchmark_data_smithi098_176514_object99128:head v 269'12717, mlcod=248'6225)", 2023-09-23T18:56:59.724 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:54:26.244961+0000", 2023-09-23T18:56:59.724 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 153.44692458, 2023-09-23T18:56:59.724 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.020742759999999999, 2023-09-23T18:56:59.724 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:59.725 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:56:59.725 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:59.725 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.725 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:59.725 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.244961+0000", 2023-09-23T18:56:59.725 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:59.725 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.725 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.725 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:59.725 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.244961+0000", 2023-09-23T18:56:59.725 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:56:59.725 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.725 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.726 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:59.726 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.244959+0000", 2023-09-23T18:56:59.726 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2346e-05 2023-09-23T18:56:59.726 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.726 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.726 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:59.726 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.244972+0000", 2023-09-23T18:56:59.726 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5508000000000001e-05 2023-09-23T18:56:59.726 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.726 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.726 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:59.727 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.244987+0000", 2023-09-23T18:56:59.727 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.6549999999999993e-06 2023-09-23T18:56:59.727 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.727 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.727 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:59.727 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.244996+0000", 2023-09-23T18:56:59.727 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.9994000000000002e-05 2023-09-23T18:56:59.727 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.727 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.727 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:59.727 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.245056+0000", 2023-09-23T18:56:59.727 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.7558999999999997e-05 2023-09-23T18:56:59.727 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.727 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.728 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:59.728 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.245093+0000", 2023-09-23T18:56:59.728 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.020532119000000001 2023-09-23T18:56:59.728 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.728 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.728 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:56:59.728 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.265626+0000", 2023-09-23T18:56:59.728 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.7757999999999997e-05 2023-09-23T18:56:59.728 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.728 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.728 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:56:59.728 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.265703+0000", 2023-09-23T18:56:59.728 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.020742759999999999 2023-09-23T18:56:59.729 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.729 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:59.729 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.729 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.729 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.729 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35412.0:202602 8.4 e269/264 8:2fc490f1:::benchmark_data_smithi098_176514_object99124:head v 269'12718, mlcod=248'6225)", 2023-09-23T18:56:59.729 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:54:26.252461+0000", 2023-09-23T18:56:59.729 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 153.43942398799999, 2023-09-23T18:56:59.729 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.016992696000000002, 2023-09-23T18:56:59.729 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:59.729 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:56:59.729 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:59.729 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.729 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:59.730 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.252461+0000", 2023-09-23T18:56:59.730 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:59.730 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.730 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.730 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:59.730 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.252461+0000", 2023-09-23T18:56:59.730 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:56:59.730 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.730 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.730 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:59.730 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.252460+0000", 2023-09-23T18:56:59.731 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.611e-06 2023-09-23T18:56:59.731 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.731 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.731 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:59.731 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.252469+0000", 2023-09-23T18:56:59.731 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4882000000000001e-05 2023-09-23T18:56:59.731 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.731 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.731 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:59.731 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.252484+0000", 2023-09-23T18:56:59.731 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.8630000000000008e-06 2023-09-23T18:56:59.731 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.732 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.732 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:59.732 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.252492+0000", 2023-09-23T18:56:59.732 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0032284979999999998 2023-09-23T18:56:59.732 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.732 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.732 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:59.732 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.255721+0000", 2023-09-23T18:56:59.732 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6224000000000003e-05 2023-09-23T18:56:59.732 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.732 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.732 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:59.732 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.255767+0000", 2023-09-23T18:56:59.733 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.01361387 2023-09-23T18:56:59.733 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.733 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.733 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:56:59.733 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.269381+0000", 2023-09-23T18:56:59.733 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.3093e-05 2023-09-23T18:56:59.733 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.733 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.733 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:56:59.733 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.269454+0000", 2023-09-23T18:56:59.733 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.016992696000000002 2023-09-23T18:56:59.733 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.733 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:59.734 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.734 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.734 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.734 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35412.0:202838 8.4 e269/264 8:29e7e82a:::benchmark_data_smithi098_176514_object99360:head v 269'12728, mlcod=249'6234)", 2023-09-23T18:56:59.734 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:54:26.762490+0000", 2023-09-23T18:56:59.734 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 152.92939492400001, 2023-09-23T18:56:59.734 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.018098125999999999, 2023-09-23T18:56:59.734 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:59.734 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:56:59.734 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:59.734 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.734 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:59.735 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.762490+0000", 2023-09-23T18:56:59.735 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:59.735 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.735 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.735 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:59.735 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.762490+0000", 2023-09-23T18:56:59.735 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:56:59.735 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.735 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.735 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:59.735 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.762489+0000", 2023-09-23T18:56:59.735 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.6670000000000002e-06 2023-09-23T18:56:59.736 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.736 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.736 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:59.736 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.762498+0000", 2023-09-23T18:56:59.736 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6059e-05 2023-09-23T18:56:59.736 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.736 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.736 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:59.736 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.762514+0000", 2023-09-23T18:56:59.736 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.8800000000000002e-06 2023-09-23T18:56:59.736 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.736 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.737 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:59.737 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.762521+0000", 2023-09-23T18:56:59.737 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0037111879999999998 2023-09-23T18:56:59.737 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.737 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.737 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:59.737 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.766232+0000", 2023-09-23T18:56:59.737 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.8334000000000003e-05 2023-09-23T18:56:59.737 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.737 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.737 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:59.737 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.766270+0000", 2023-09-23T18:56:59.737 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.014275743 2023-09-23T18:56:59.738 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.738 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.738 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:56:59.738 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.780546+0000", 2023-09-23T18:56:59.738 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.2265000000000003e-05 2023-09-23T18:56:59.738 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.738 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.738 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:56:59.738 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:26.780588+0000", 2023-09-23T18:56:59.738 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.018098125999999999 2023-09-23T18:56:59.738 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.739 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:59.739 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.739 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.739 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.739 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35412.0:203096 8.4 e269/264 8:2ad6ae7d:::benchmark_data_smithi098_176514_object99618:head v 269'12742, mlcod=249'6248)", 2023-09-23T18:56:59.739 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:54:27.385056+0000", 2023-09-23T18:56:59.739 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 152.30682954599999, 2023-09-23T18:56:59.739 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.016745149000000001, 2023-09-23T18:56:59.739 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:59.739 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:56:59.739 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:59.739 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.739 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:59.740 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:27.385056+0000", 2023-09-23T18:56:59.740 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:59.740 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.740 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.740 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:59.740 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:27.385056+0000", 2023-09-23T18:56:59.740 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:56:59.740 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.740 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.740 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:59.740 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:27.385055+0000", 2023-09-23T18:56:59.740 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.7299999999999994e-06 2023-09-23T18:56:59.740 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.741 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.741 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:59.741 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:27.385063+0000", 2023-09-23T18:56:59.741 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6249999999999999e-05 2023-09-23T18:56:59.741 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.741 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.741 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:59.741 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:27.385080+0000", 2023-09-23T18:56:59.741 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.3950000000000007e-06 2023-09-23T18:56:59.741 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.741 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.741 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:59.741 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:27.385089+0000", 2023-09-23T18:56:59.742 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00010378399999999999 2023-09-23T18:56:59.742 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.742 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.742 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:59.742 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:27.385193+0000", 2023-09-23T18:56:59.742 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.3582999999999999e-05 2023-09-23T18:56:59.742 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.742 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.742 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:59.742 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:27.385236+0000", 2023-09-23T18:56:59.742 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.016533868 2023-09-23T18:56:59.743 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.743 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.743 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:56:59.743 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:27.401770+0000", 2023-09-23T18:56:59.743 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.0518999999999997e-05 2023-09-23T18:56:59.743 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.743 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.743 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:56:59.743 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:27.401801+0000", 2023-09-23T18:56:59.743 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.016745149000000001 2023-09-23T18:56:59.743 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.743 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:59.744 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.744 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.744 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.744 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35412.0:204210 8.5 e269/263 8:a70785ad:::benchmark_data_smithi098_176514_object100732:head v 269'12608, mlcod=269'12605)", 2023-09-23T18:56:59.744 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:54:29.840546+0000", 2023-09-23T18:56:59.744 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 149.85133876500001, 2023-09-23T18:56:59.744 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.020980518, 2023-09-23T18:56:59.744 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:59.744 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:56:59.744 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:59.744 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.744 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:59.745 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:29.840546+0000", 2023-09-23T18:56:59.745 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:59.745 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.745 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.745 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:59.745 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:29.840546+0000", 2023-09-23T18:56:59.745 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:56:59.745 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.745 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.745 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:59.745 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:29.840545+0000", 2023-09-23T18:56:59.745 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0338e-05 2023-09-23T18:56:59.745 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.746 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.746 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:59.746 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:29.840556+0000", 2023-09-23T18:56:59.746 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2907e-05 2023-09-23T18:56:59.746 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.746 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.746 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:59.746 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:29.840569+0000", 2023-09-23T18:56:59.746 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.652e-06 2023-09-23T18:56:59.746 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.746 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.747 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:59.747 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:29.840575+0000", 2023-09-23T18:56:59.747 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0050637709999999999 2023-09-23T18:56:59.747 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.747 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.747 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:59.747 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:29.845639+0000", 2023-09-23T18:56:59.747 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.5582e-05 2023-09-23T18:56:59.747 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.747 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.747 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:59.747 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:29.845665+0000", 2023-09-23T18:56:59.747 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.015818260000000001 2023-09-23T18:56:59.748 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.748 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.748 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:56:59.748 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:29.861483+0000", 2023-09-23T18:56:59.748 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.4014999999999998e-05 2023-09-23T18:56:59.748 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.748 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.748 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:56:59.748 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:29.861527+0000", 2023-09-23T18:56:59.748 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.020980518 2023-09-23T18:56:59.748 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.748 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:59.748 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.749 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.749 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.749 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35412.0:204200 8.4 e269/264 8:268b42b2:::benchmark_data_smithi098_176514_object100722:head v 269'12802, mlcod=249'6307)", 2023-09-23T18:56:59.749 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:54:29.936232+0000", 2023-09-23T18:56:59.749 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 149.75565278600001, 2023-09-23T18:56:59.749 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.017893246000000002, 2023-09-23T18:56:59.749 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:59.749 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:56:59.749 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:59.749 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.749 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:59.749 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:29.936232+0000", 2023-09-23T18:56:59.749 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:59.749 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.750 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.750 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:59.750 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:29.936232+0000", 2023-09-23T18:56:59.750 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:56:59.750 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.750 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.750 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:59.750 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:29.936231+0000", 2023-09-23T18:56:59.750 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.842e-06 2023-09-23T18:56:59.750 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.750 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.751 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:59.751 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:29.936241+0000", 2023-09-23T18:56:59.751 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.485e-05 2023-09-23T18:56:59.751 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.751 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.751 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:59.751 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:29.936256+0000", 2023-09-23T18:56:59.751 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.2869999999999993e-06 2023-09-23T18:56:59.751 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.751 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.751 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:59.751 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:29.936265+0000", 2023-09-23T18:56:59.751 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.8180000000000003e-05 2023-09-23T18:56:59.752 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.752 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.752 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:59.752 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:29.936313+0000", 2023-09-23T18:56:59.752 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.5307e-05 2023-09-23T18:56:59.752 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.752 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.752 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:59.752 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:29.936339+0000", 2023-09-23T18:56:59.752 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.017759906999999998 2023-09-23T18:56:59.752 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.752 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.752 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:56:59.753 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:29.954099+0000", 2023-09-23T18:56:59.753 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.6937999999999999e-05 2023-09-23T18:56:59.753 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.753 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.753 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:56:59.753 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:29.954126+0000", 2023-09-23T18:56:59.753 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.017893246000000002 2023-09-23T18:56:59.753 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.753 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:59.753 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.753 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.753 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.753 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35412.0:205191 8.6 e269/233 8:68020a97:::benchmark_data_smithi098_176514_object101713:head v 269'13056, mlcod=269'13052)", 2023-09-23T18:56:59.754 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:54:32.183953+0000", 2023-09-23T18:56:59.754 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 147.50793173700001, 2023-09-23T18:56:59.754 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.020283994999999999, 2023-09-23T18:56:59.754 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:59.754 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:56:59.754 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:59.754 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.754 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:59.754 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:32.183953+0000", 2023-09-23T18:56:59.754 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:59.754 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.755 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.755 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:59.755 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:32.183953+0000", 2023-09-23T18:56:59.755 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:56:59.755 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.755 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.755 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:59.755 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:32.183952+0000", 2023-09-23T18:56:59.755 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2565000000000001e-05 2023-09-23T18:56:59.755 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.755 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.755 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:59.755 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:32.183965+0000", 2023-09-23T18:56:59.756 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3295000000000001e-05 2023-09-23T18:56:59.756 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.756 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.756 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:59.756 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:32.183978+0000", 2023-09-23T18:56:59.756 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.4409999999999998e-06 2023-09-23T18:56:59.756 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.756 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.756 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:59.756 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:32.183984+0000", 2023-09-23T18:56:59.756 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0058477449999999997 2023-09-23T18:56:59.756 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.756 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.757 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:59.757 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:32.189832+0000", 2023-09-23T18:56:59.757 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2211000000000001e-05 2023-09-23T18:56:59.757 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.757 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.757 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:59.757 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:32.189854+0000", 2023-09-23T18:56:59.757 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.014352814 2023-09-23T18:56:59.757 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.757 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.757 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:56:59.757 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:32.204207+0000", 2023-09-23T18:56:59.757 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.0171e-05 2023-09-23T18:56:59.758 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.758 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.758 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:56:59.758 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:32.204237+0000", 2023-09-23T18:56:59.758 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.020283994999999999 2023-09-23T18:56:59.758 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.758 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:59.758 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.758 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.758 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.758 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35412.0:206118 8.4 e269/264 8:2ee64445:::benchmark_data_smithi098_176514_object102640:head v 269'12936, mlcod=249'6442)", 2023-09-23T18:56:59.758 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:54:34.418111+0000", 2023-09-23T18:56:59.759 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 145.273774355, 2023-09-23T18:56:59.759 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.01883669, 2023-09-23T18:56:59.759 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:59.759 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:56:59.759 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:59.759 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.759 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:59.759 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.418111+0000", 2023-09-23T18:56:59.759 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:59.759 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.759 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.759 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:59.759 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.418111+0000", 2023-09-23T18:56:59.760 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:56:59.760 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.760 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.760 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:59.760 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.418109+0000", 2023-09-23T18:56:59.760 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.0273999999999999e-05 2023-09-23T18:56:59.760 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.760 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.760 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:59.760 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.418129+0000", 2023-09-23T18:56:59.760 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.4424000000000001e-05 2023-09-23T18:56:59.760 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.760 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.761 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:59.761 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.418164+0000", 2023-09-23T18:56:59.761 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.486e-05 2023-09-23T18:56:59.761 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.761 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.761 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:59.761 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.418178+0000", 2023-09-23T18:56:59.761 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.3425999999999998e-05 2023-09-23T18:56:59.761 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.761 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.761 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:59.761 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.418252+0000", 2023-09-23T18:56:59.761 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.4923999999999999e-05 2023-09-23T18:56:59.762 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.762 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.762 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:59.762 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.418297+0000", 2023-09-23T18:56:59.762 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.018589736999999999 2023-09-23T18:56:59.762 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.762 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.762 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:56:59.762 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.436887+0000", 2023-09-23T18:56:59.762 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.088e-05 2023-09-23T18:56:59.762 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.762 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.762 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:56:59.763 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.436947+0000", 2023-09-23T18:56:59.763 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.01883669 2023-09-23T18:56:59.763 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.763 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:59.763 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.763 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.763 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.763 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35412.0:206241 8.4 e269/264 8:24a8a4e5:::benchmark_data_smithi098_176514_object102763:head v 269'12939, mlcod=249'6445)", 2023-09-23T18:56:59.763 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:54:34.662296+0000", 2023-09-23T18:56:59.763 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 145.02958916899999, 2023-09-23T18:56:59.763 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.027923996, 2023-09-23T18:56:59.763 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:59.764 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:56:59.764 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:59.764 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.764 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:59.764 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.662296+0000", 2023-09-23T18:56:59.764 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:59.764 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.764 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.764 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:59.764 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.662296+0000", 2023-09-23T18:56:59.764 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:56:59.764 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.764 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.765 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:59.765 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.662294+0000", 2023-09-23T18:56:59.765 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5498999999999998e-05 2023-09-23T18:56:59.765 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.765 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.765 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:59.765 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.662310+0000", 2023-09-23T18:56:59.765 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4754000000000001e-05 2023-09-23T18:56:59.765 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.765 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.765 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:59.765 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.662334+0000", 2023-09-23T18:56:59.765 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1051e-05 2023-09-23T18:56:59.766 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.766 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.766 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:59.766 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.662345+0000", 2023-09-23T18:56:59.766 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.0835999999999996e-05 2023-09-23T18:56:59.766 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.766 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.766 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:59.766 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.662436+0000", 2023-09-23T18:56:59.766 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.8312000000000001e-05 2023-09-23T18:56:59.766 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.767 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.767 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:59.767 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.662485+0000", 2023-09-23T18:56:59.767 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.027680349 2023-09-23T18:56:59.767 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.767 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.767 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:56:59.767 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.690165+0000", 2023-09-23T18:56:59.767 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4976999999999999e-05 2023-09-23T18:56:59.767 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.767 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.767 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:56:59.767 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.690220+0000", 2023-09-23T18:56:59.768 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.027923996 2023-09-23T18:56:59.768 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.768 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:59.768 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.768 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.768 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.768 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35412.0:206185 8.4 e269/264 8:20df34b1:::benchmark_data_smithi098_176514_object102707:head v 269'12940, mlcod=249'6445)", 2023-09-23T18:56:59.768 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:54:34.670600+0000", 2023-09-23T18:56:59.768 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 145.02128488700001, 2023-09-23T18:56:59.768 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.019651440999999999, 2023-09-23T18:56:59.768 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:59.768 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:56:59.768 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:59.769 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.769 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:59.769 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.670600+0000", 2023-09-23T18:56:59.769 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:59.769 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.769 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.769 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:59.769 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.670600+0000", 2023-09-23T18:56:59.769 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:56:59.769 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.769 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.769 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:59.769 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.670599+0000", 2023-09-23T18:56:59.770 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2944e-05 2023-09-23T18:56:59.770 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.770 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.770 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:59.770 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.670612+0000", 2023-09-23T18:56:59.770 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3136e-05 2023-09-23T18:56:59.770 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.770 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.770 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:59.770 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.670635+0000", 2023-09-23T18:56:59.770 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.755e-06 2023-09-23T18:56:59.770 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.770 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.771 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:59.771 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.670644+0000", 2023-09-23T18:56:59.771 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001700275 2023-09-23T18:56:59.771 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.771 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.771 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:59.771 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.672344+0000", 2023-09-23T18:56:59.771 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.7613000000000005e-05 2023-09-23T18:56:59.771 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.771 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.771 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:59.771 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.672412+0000", 2023-09-23T18:56:59.771 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.017815660000000001 2023-09-23T18:56:59.771 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.772 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.772 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:56:59.772 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.690227+0000", 2023-09-23T18:56:59.772 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4426999999999998e-05 2023-09-23T18:56:59.772 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.772 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.772 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:56:59.772 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:34.690252+0000", 2023-09-23T18:56:59.772 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.019651440999999999 2023-09-23T18:56:59.772 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.772 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:59.772 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.772 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.773 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.773 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35412.0:206594 8.4 e269/264 8:2c046995:::benchmark_data_smithi098_176514_object103116:head v 269'12958, mlcod=249'6464)", 2023-09-23T18:56:59.773 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:54:35.531873+0000", 2023-09-23T18:56:59.773 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 144.160012438, 2023-09-23T18:56:59.773 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.017866301000000001, 2023-09-23T18:56:59.773 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:59.773 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:56:59.773 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:59.773 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.773 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:59.773 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:35.531873+0000", 2023-09-23T18:56:59.773 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:59.774 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.774 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.774 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:59.774 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:35.531873+0000", 2023-09-23T18:56:59.774 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:56:59.774 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.774 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.774 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:59.774 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:35.531871+0000", 2023-09-23T18:56:59.774 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0794e-05 2023-09-23T18:56:59.774 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.774 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.774 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:59.775 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:35.531882+0000", 2023-09-23T18:56:59.775 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3027000000000001e-05 2023-09-23T18:56:59.775 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.775 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.775 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:59.775 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:35.531895+0000", 2023-09-23T18:56:59.775 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.9190000000000002e-06 2023-09-23T18:56:59.775 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.775 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.775 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:59.775 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:35.531902+0000", 2023-09-23T18:56:59.775 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.7481999999999997e-05 2023-09-23T18:56:59.776 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.776 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.776 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:59.776 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:35.531950+0000", 2023-09-23T18:56:59.776 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2904e-05 2023-09-23T18:56:59.776 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.776 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.776 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:59.776 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:35.531973+0000", 2023-09-23T18:56:59.776 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.017700418999999998 2023-09-23T18:56:59.776 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.776 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.777 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:56:59.777 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:35.549673+0000", 2023-09-23T18:56:59.777 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.5975000000000004e-05 2023-09-23T18:56:59.777 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.777 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.777 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:56:59.777 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:35.549739+0000", 2023-09-23T18:56:59.777 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.017866301000000001 2023-09-23T18:56:59.777 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.777 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:59.777 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.777 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.777 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.778 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35412.0:206694 8.4 e269/264 8:22802799:::benchmark_data_smithi098_176514_object103216:head v 269'12967, mlcod=249'6472)", 2023-09-23T18:56:59.778 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:54:35.747476+0000", 2023-09-23T18:56:59.778 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 143.944409565, 2023-09-23T18:56:59.778 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.019391080000000002, 2023-09-23T18:56:59.778 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:59.778 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:56:59.778 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:59.778 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.778 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:59.778 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:35.747476+0000", 2023-09-23T18:56:59.778 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:59.778 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.779 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.779 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:59.779 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:35.747476+0000", 2023-09-23T18:56:59.779 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:56:59.779 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.779 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.779 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:59.779 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:35.747475+0000", 2023-09-23T18:56:59.779 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1997000000000001e-05 2023-09-23T18:56:59.779 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.779 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.779 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:59.779 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:35.747487+0000", 2023-09-23T18:56:59.779 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.0415999999999999e-05 2023-09-23T18:56:59.780 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.780 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.780 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:59.780 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:35.747507+0000", 2023-09-23T18:56:59.780 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.8380000000000002e-06 2023-09-23T18:56:59.780 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.780 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.780 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:59.780 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:35.747515+0000", 2023-09-23T18:56:59.780 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000105658 2023-09-23T18:56:59.780 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.780 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.780 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:59.781 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:35.747621+0000", 2023-09-23T18:56:59.781 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.9013000000000001e-05 2023-09-23T18:56:59.781 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.781 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.781 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:59.781 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:35.747660+0000", 2023-09-23T18:56:59.781 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.019168152000000001 2023-09-23T18:56:59.781 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.781 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.781 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:56:59.781 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:35.766828+0000", 2023-09-23T18:56:59.781 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.8921999999999998e-05 2023-09-23T18:56:59.781 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.781 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.782 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:56:59.782 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:54:35.766867+0000", 2023-09-23T18:56:59.782 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.019391080000000002 2023-09-23T18:56:59.782 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.782 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:59.782 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.782 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.787 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.787 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45602.0:28568 9.1 e288/276 9:8427f500:::benchmark_data_smithi098_206513_object28567:head v 288'1812, mlcod=288'1808)", 2023-09-23T18:56:59.787 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:56:13.446568+0000", 2023-09-23T18:56:59.787 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 46.245317188000001, 2023-09-23T18:56:59.787 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.018265300000000002, 2023-09-23T18:56:59.787 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:59.787 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:56:59.787 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:59.787 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.788 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:59.788 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:13.446568+0000", 2023-09-23T18:56:59.788 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:59.788 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.788 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.788 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:59.788 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:13.446568+0000", 2023-09-23T18:56:59.788 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:56:59.788 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.788 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.788 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:59.788 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:13.446567+0000", 2023-09-23T18:56:59.788 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000153251 2023-09-23T18:56:59.788 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.789 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.789 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:59.789 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:13.446720+0000", 2023-09-23T18:56:59.789 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6413e-05 2023-09-23T18:56:59.789 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.789 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.789 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:59.789 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:13.446737+0000", 2023-09-23T18:56:59.789 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.6510000000000001e-06 2023-09-23T18:56:59.789 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.789 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.789 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:59.789 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:13.446744+0000", 2023-09-23T18:56:59.789 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.8582999999999999e-05 2023-09-23T18:56:59.790 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.790 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.790 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:59.790 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:13.446793+0000", 2023-09-23T18:56:59.790 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.9090000000000001e-05 2023-09-23T18:56:59.790 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.790 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.790 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:59.790 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:13.446822+0000", 2023-09-23T18:56:59.790 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.017970776000000001 2023-09-23T18:56:59.790 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.790 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.790 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:56:59.791 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:13.464793+0000", 2023-09-23T18:56:59.791 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.0438999999999997e-05 2023-09-23T18:56:59.791 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.791 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.791 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:56:59.791 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:13.464833+0000", 2023-09-23T18:56:59.791 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.018265300000000002 2023-09-23T18:56:59.791 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.791 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:59.791 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.791 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.791 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.791 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45602.0:28579 9.1 e288/276 9:831ac81e:::benchmark_data_smithi098_206513_object28578:head v 288'1813, mlcod=288'1810)", 2023-09-23T18:56:59.792 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:56:13.451933+0000", 2023-09-23T18:56:59.792 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 46.239952252000002, 2023-09-23T18:56:59.792 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.012917756000000001, 2023-09-23T18:56:59.792 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:59.792 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:56:59.792 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:59.792 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.792 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:59.792 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:13.451933+0000", 2023-09-23T18:56:59.792 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:59.792 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.792 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.792 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:59.792 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:13.451933+0000", 2023-09-23T18:56:59.793 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:56:59.793 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.793 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.793 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:59.793 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:13.451932+0000", 2023-09-23T18:56:59.793 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6715999999999997e-05 2023-09-23T18:56:59.793 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.793 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.793 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:59.793 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:13.451988+0000", 2023-09-23T18:56:59.793 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2949e-05 2023-09-23T18:56:59.793 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.793 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.793 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:59.794 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:13.452011+0000", 2023-09-23T18:56:59.794 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.738e-06 2023-09-23T18:56:59.794 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.794 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.794 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:59.794 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:13.452020+0000", 2023-09-23T18:56:59.794 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.004991578 2023-09-23T18:56:59.794 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.794 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.794 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:59.794 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:13.457012+0000", 2023-09-23T18:56:59.794 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.0340000000000001e-05 2023-09-23T18:56:59.794 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.795 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.795 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:59.795 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:13.457042+0000", 2023-09-23T18:56:59.795 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0077964929999999998 2023-09-23T18:56:59.795 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.795 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.795 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:56:59.795 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:13.464838+0000", 2023-09-23T18:56:59.795 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2255e-05 2023-09-23T18:56:59.795 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.795 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.795 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:56:59.796 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:13.464851+0000", 2023-09-23T18:56:59.796 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.012917756000000001 2023-09-23T18:56:59.796 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.796 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:59.796 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.796 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.796 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.796 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45602.0:76345 9.1 e290/276 9:8a4684f6:::benchmark_data_smithi098_206513_object76344:head v 290'4738, mlcod=290'4736)", 2023-09-23T18:56:59.796 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:56:53.514957+0000", 2023-09-23T18:56:59.796 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 6.1769280359999996, 2023-09-23T18:56:59.796 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.013908715, 2023-09-23T18:56:59.796 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:56:59.797 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:56:59.797 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:56:59.797 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.797 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:56:59.797 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:53.514957+0000", 2023-09-23T18:56:59.797 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:56:59.797 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.797 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.797 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:56:59.797 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:53.514957+0000", 2023-09-23T18:56:59.797 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:56:59.797 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.797 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.798 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:56:59.798 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:53.514955+0000", 2023-09-23T18:56:59.798 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.8962999999999998e-05 2023-09-23T18:56:59.798 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.798 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.798 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:56:59.798 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:53.515014+0000", 2023-09-23T18:56:59.798 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.5593999999999999e-05 2023-09-23T18:56:59.798 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.798 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.798 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:56:59.798 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:53.515039+0000", 2023-09-23T18:56:59.799 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1103e-05 2023-09-23T18:56:59.799 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.799 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.799 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:56:59.799 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:53.515051+0000", 2023-09-23T18:56:59.799 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1408e-05 2023-09-23T18:56:59.799 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.799 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.799 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:56:59.799 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:53.515102+0000", 2023-09-23T18:56:59.799 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.0380000000000001e-05 2023-09-23T18:56:59.799 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.799 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.800 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:56:59.800 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:53.515122+0000", 2023-09-23T18:56:59.800 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.013688349000000001 2023-09-23T18:56:59.800 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.800 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.800 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:56:59.800 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:53.528811+0000", 2023-09-23T18:56:59.800 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5037000000000002e-05 2023-09-23T18:56:59.800 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:56:59.800 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:56:59.800 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:56:59.800 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:53.528866+0000", 2023-09-23T18:56:59.800 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.013908715 2023-09-23T18:56:59.801 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.801 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:59.801 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.801 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:56:59.801 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:56:59.801 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:56:59.801 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:59.802 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:56:59 smithi173 bash[59422]: cluster 2023-09-23T18:56:58.308803+0000 mgr.y (mgr.26054) 1170 : cluster 0 pgmap v1268: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 44960/240078 objects degraded (18.727%); 2/240078 objects misplaced (0.001%) 2023-09-23T18:56:59.836 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:56:59.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:59.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:56:59.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:56:59 smithi098 bash[76817]: cluster 2023-09-23T18:56:58.308803+0000 mgr.y (mgr.26054) 1170 : cluster 0 pgmap v1268: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 44960/240078 objects degraded (18.727%); 2/240078 objects misplaced (0.001%) 2023-09-23T18:56:59.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:56:59 smithi098 bash[79516]: cluster 2023-09-23T18:56:58.308803+0000 mgr.y (mgr.26054) 1170 : cluster 0 pgmap v1268: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 44960/240078 objects degraded (18.727%); 2/240078 objects misplaced (0.001%) 2023-09-23T18:56:59.945 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:59.975 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:56:59.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:56:59.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:00.080 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:00.106 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:00.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:00.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:00.209 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:00.234 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:00.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:00.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:00.333 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 76 16 82970 82954 68.2071 60.4375 0.0188071 0.0146557 2023-09-23T18:57:00.337 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:00.368 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:00.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:00.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:00.471 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:00.498 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:00.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:00.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:00.605 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:00.631 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:00.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:00.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:00.734 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:00.770 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:00 smithi173 bash[59422]: cluster 2023-09-23T18:57:00.366273+0000 mon.a (mon.0) 1597 : cluster 3 Health check update: Degraded data redundancy: 44960/240078 objects degraded (18.727%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:00.771 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T18:57:00.772 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:00.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:00.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:00.878 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:00.904 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:00.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:00.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:00.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:00 smithi098 bash[76817]: cluster 2023-09-23T18:57:00.366273+0000 mon.a (mon.0) 1597 : cluster 3 Health check update: Degraded data redundancy: 44960/240078 objects degraded (18.727%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:00.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:00 smithi098 bash[79516]: cluster 2023-09-23T18:57:00.366273+0000 mon.a (mon.0) 1597 : cluster 3 Health check update: Degraded data redundancy: 44960/240078 objects degraded (18.727%), 18 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:01.008 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:01.035 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:01.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:01.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:01.141 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:01.170 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:01.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:01.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:01.273 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:01.306 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:01.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:01.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:01.333 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 77 16 83952 83936 68.1183 61.375 0.0190046 0.0146752 2023-09-23T18:57:01.409 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:01.445 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:01.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:01.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:01.548 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:01.577 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:01.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:01.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:01.681 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:01.690 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:01 smithi098 bash[79516]: cluster 2023-09-23T18:57:00.309586+0000 mgr.y (mgr.26054) 1171 : cluster 0 pgmap v1269: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 45249/240945 objects degraded (18.780%); 2/240945 objects misplaced (0.001%) 2023-09-23T18:57:01.690 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:01 smithi098 bash[76817]: cluster 2023-09-23T18:57:00.309586+0000 mgr.y (mgr.26054) 1171 : cluster 0 pgmap v1269: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 45249/240945 objects degraded (18.780%); 2/240945 objects misplaced (0.001%) 2023-09-23T18:57:01.712 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:01.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:01.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:01.818 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:01.847 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:01.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:01.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:01.950 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:01.957 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:01 smithi173 bash[59422]: cluster 2023-09-23T18:57:00.309586+0000 mgr.y (mgr.26054) 1171 : cluster 0 pgmap v1269: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 45249/240945 objects degraded (18.780%); 2/240945 objects misplaced (0.001%) 2023-09-23T18:57:01.975 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:01.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:01.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:02.083 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:02.106 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:02.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:02.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:02.210 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:02.235 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:02.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:02.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:02.333 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 78 16 84975 84959 68.0646 63.9375 0.00613897 0.0146864 2023-09-23T18:57:02.340 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:02.367 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:02.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:02.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:02.471 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:02.496 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:02.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:02.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:02.600 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:02.626 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:02.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:02.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:02.729 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:02.756 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:02.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:02.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:02.862 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:02.888 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:02.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:02.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:02.991 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:03.023 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:03.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:03.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:03.129 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:03.157 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:03.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:03.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:03.259 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:03.289 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:03.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:03.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:03.333 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 79 16 85887 85871 67.9245 57 0.0212011 0.014717 2023-09-23T18:57:03.392 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:03.430 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:03.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:03.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:03.537 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:03.561 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:03.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:03.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:03.665 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:03.691 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:03.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:03.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:03.795 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:03.805 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:03 smithi098 bash[76817]: cluster 2023-09-23T18:57:02.310432+0000 mgr.y (mgr.26054) 1172 : cluster 0 pgmap v1270: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s; 45558/244590 objects degraded (18.626%); 2/244590 objects misplaced (0.001%) 2023-09-23T18:57:03.805 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:03 smithi098 bash[79516]: cluster 2023-09-23T18:57:02.310432+0000 mgr.y (mgr.26054) 1172 : cluster 0 pgmap v1270: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s; 45558/244590 objects degraded (18.626%); 2/244590 objects misplaced (0.001%) 2023-09-23T18:57:03.833 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:03.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:03.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:03.935 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:03.962 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:03.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:03.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:04.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:03 smithi173 bash[59422]: cluster 2023-09-23T18:57:02.310432+0000 mgr.y (mgr.26054) 1172 : cluster 0 pgmap v1270: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s; 45558/244590 objects degraded (18.626%); 2/244590 objects misplaced (0.001%) 2023-09-23T18:57:04.067 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:04.108 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:04.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:04.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:04.215 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:04.243 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:04.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:04.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:04.333 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:2023-09-23T18:57:04.330593+0000 min lat: 0.00116653 max lat: 16.0742 avg lat: 0.014752 2023-09-23T18:57:04.333 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:57:04.333 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 80 16 86767 86751 67.7629 55 0.0283088 0.014752 2023-09-23T18:57:04.346 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:04.377 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:04.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:04.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:04.480 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:04.515 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:04.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:04.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:04.618 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:04.647 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:04.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:04.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:04.750 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:04.776 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:04.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:04.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:04.879 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:04.909 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:04.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:04.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:05.014 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:05.042 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:05.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:05.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:05.145 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:05.174 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:05.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:05.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:05.278 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:05.310 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:05.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:05.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:05.333 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 81 16 87753 87737 67.6871 61.625 0.00731976 0.0147689 2023-09-23T18:57:05.415 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:05.446 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:05.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:05.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:05.548 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:05.572 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:05.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:05.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:05.678 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:05.708 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:05.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:05.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:05.811 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:05.820 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:05 smithi098 bash[79516]: cluster 2023-09-23T18:57:04.311398+0000 mgr.y (mgr.26054) 1173 : cluster 0 pgmap v1271: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 47943/255432 objects degraded (18.769%); 2/255432 objects misplaced (0.001%) 2023-09-23T18:57:05.820 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:05 smithi098 bash[79516]: cluster 2023-09-23T18:57:05.367487+0000 mon.a (mon.0) 1598 : cluster 3 Health check update: Degraded data redundancy: 47943/255432 objects degraded (18.769%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:05.820 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:05 smithi098 bash[76817]: cluster 2023-09-23T18:57:04.311398+0000 mgr.y (mgr.26054) 1173 : cluster 0 pgmap v1271: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 47943/255432 objects degraded (18.769%); 2/255432 objects misplaced (0.001%) 2023-09-23T18:57:05.820 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:05 smithi098 bash[76817]: cluster 2023-09-23T18:57:05.367487+0000 mon.a (mon.0) 1598 : cluster 3 Health check update: Degraded data redundancy: 47943/255432 objects degraded (18.769%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:05.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:05.854 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:05.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:05.955 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:05.989 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:05.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:05.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:06.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:05 smithi173 bash[59422]: cluster 2023-09-23T18:57:04.311398+0000 mgr.y (mgr.26054) 1173 : cluster 0 pgmap v1271: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 47943/255432 objects degraded (18.769%); 2/255432 objects misplaced (0.001%) 2023-09-23T18:57:06.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:05 smithi173 bash[59422]: cluster 2023-09-23T18:57:05.367487+0000 mon.a (mon.0) 1598 : cluster 3 Health check update: Degraded data redundancy: 47943/255432 objects degraded (18.769%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:06.093 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:06.128 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:06.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:06.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:06.232 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:06.260 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:06.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:06.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:06.333 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 82 16 88682 88666 67.5696 58.0625 0.0139692 0.0147946 2023-09-23T18:57:06.366 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:06.374 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:57:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:57:06] "GET /metrics HTTP/1.1" 200 36584 "" "Prometheus/2.33.4" 2023-09-23T18:57:06.395 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:06.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:06.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:06.501 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:06.526 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:06.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:06.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:06.629 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:06.658 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:06.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:06.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:06.761 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:06.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:06.796 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:06.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:06.897 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:06.924 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:06.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:06.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:07.027 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:07.058 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:07.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:07.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:07.164 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:07.189 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:07.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:07.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:07.291 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:07.313 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:07.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:07.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:07.333 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 83 16 89676 89660 67.504 62.125 0.0154764 0.0148082 2023-09-23T18:57:07.417 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:07.444 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:07.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:07.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:07.550 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:07.573 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:07.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:07.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:07.676 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:07.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:07 smithi098 bash[76817]: cluster 2023-09-23T18:57:06.312313+0000 mgr.y (mgr.26054) 1174 : cluster 0 pgmap v1272: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s; 48247/258945 objects degraded (18.632%); 2/258945 objects misplaced (0.001%) 2023-09-23T18:57:07.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:07 smithi098 bash[79516]: cluster 2023-09-23T18:57:06.312313+0000 mgr.y (mgr.26054) 1174 : cluster 0 pgmap v1272: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s; 48247/258945 objects degraded (18.632%); 2/258945 objects misplaced (0.001%) 2023-09-23T18:57:07.704 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:07.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:07.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:07.808 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:07.839 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:07.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:07.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:07.945 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:07.955 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:07 smithi173 bash[59422]: cluster 2023-09-23T18:57:06.312313+0000 mgr.y (mgr.26054) 1174 : cluster 0 pgmap v1272: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s; 48247/258945 objects degraded (18.632%); 2/258945 objects misplaced (0.001%) 2023-09-23T18:57:07.972 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:07.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:07.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:08.075 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:08.097 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:08.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:08.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:08.201 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:08.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:08.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:08.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:08.244 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:57:08.333 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:08.334 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 84 16 90623 90607 67.4049 59.1875 0.0254272 0.0148307 2023-09-23T18:57:08.360 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:08.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:08.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:08.466 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:08.495 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:08.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:08.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:08.601 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:08.628 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:08.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:08.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:08.733 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:08.754 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:08.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:08.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:08.857 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:08.891 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:08.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:08.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:08.994 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:09.023 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:09.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:09.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:09.127 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:09.152 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:09.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:09.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:09.256 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:09.281 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:09.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:09.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:09.334 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 85 16 91565 91549 67.3045 58.875 0.0203223 0.0148525 2023-09-23T18:57:09.384 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:09.414 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:09.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:09.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:09.517 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:09.540 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:09.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:09.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:09.645 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:09.670 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:09.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:09.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:09.774 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:09.807 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:09.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:09.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:09.912 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:09.937 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:09.940 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:09 smithi098 bash[76817]: cluster 2023-09-23T18:57:08.313265+0000 mgr.y (mgr.26054) 1175 : cluster 0 pgmap v1273: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 50297/268839 objects degraded (18.709%); 2/268839 objects misplaced (0.001%) 2023-09-23T18:57:09.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:09 smithi098 bash[76817]: audit 2023-09-23T18:57:09.631766+0000 mon.a (mon.0) 1599 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:57:09.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:09 smithi098 bash[79516]: cluster 2023-09-23T18:57:08.313265+0000 mgr.y (mgr.26054) 1175 : cluster 0 pgmap v1273: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 50297/268839 objects degraded (18.709%); 2/268839 objects misplaced (0.001%) 2023-09-23T18:57:09.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:09 smithi098 bash[79516]: audit 2023-09-23T18:57:09.631766+0000 mon.a (mon.0) 1599 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:57:09.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:09.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:10.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:09 smithi173 bash[59422]: cluster 2023-09-23T18:57:08.313265+0000 mgr.y (mgr.26054) 1175 : cluster 0 pgmap v1273: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 50297/268839 objects degraded (18.709%); 2/268839 objects misplaced (0.001%) 2023-09-23T18:57:10.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:09 smithi173 bash[59422]: audit 2023-09-23T18:57:09.631766+0000 mon.a (mon.0) 1599 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:57:10.043 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:10.067 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:10.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:10.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:10.172 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:10.197 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:10.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:10.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:10.300 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:10.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:10.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:10.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:10.334 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 86 16 92483 92467 67.1889 57.375 0.00400066 0.0148723 2023-09-23T18:57:10.430 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:10.451 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:10.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:10.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:10.554 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:10.581 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:10.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:10.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:10.685 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:10.704 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:10 smithi098 bash[76817]: cluster 2023-09-23T18:57:10.368300+0000 mon.a (mon.0) 1600 : cluster 3 Health check update: Degraded data redundancy: 50297/268839 objects degraded (18.709%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:10.709 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:10.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:10.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:10.812 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:10.822 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:10 smithi173 bash[59422]: cluster 2023-09-23T18:57:10.368300+0000 mon.a (mon.0) 1600 : cluster 3 Health check update: Degraded data redundancy: 50297/268839 objects degraded (18.709%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:10.842 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:10.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:10.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:10.945 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:10.955 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:10 smithi098 bash[79516]: cluster 2023-09-23T18:57:10.368300+0000 mon.a (mon.0) 1600 : cluster 3 Health check update: Degraded data redundancy: 50297/268839 objects degraded (18.709%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:10.973 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:10.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:10.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:11.076 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:11.106 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:11.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:11.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:11.211 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:11.241 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:11.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:11.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:11.334 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 87 16 93165 93149 66.9066 42.625 0.00380068 0.0149301 2023-09-23T18:57:11.347 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:11.373 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:11.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:11.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:11.478 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:11.503 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:11.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:11.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:11.609 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:11.626 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:11.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:11.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:11.729 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:11.762 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:11.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:11.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:11.788 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:11 smithi098 bash[76817]: cluster 2023-09-23T18:57:10.314582+0000 mgr.y (mgr.26054) 1176 : cluster 0 pgmap v1274: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s; 50602/269754 objects degraded (18.759%); 2/269754 objects misplaced (0.001%) 2023-09-23T18:57:11.866 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:11.901 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:11.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:11.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:12.006 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:12.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:11 smithi173 bash[59422]: cluster 2023-09-23T18:57:10.314582+0000 mgr.y (mgr.26054) 1176 : cluster 0 pgmap v1274: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s; 50602/269754 objects degraded (18.759%); 2/269754 objects misplaced (0.001%) 2023-09-23T18:57:12.030 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:12.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:12.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:12.137 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:12.169 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:12.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:12.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:12.247 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:11 smithi098 bash[79516]: cluster 2023-09-23T18:57:10.314582+0000 mgr.y (mgr.26054) 1176 : cluster 0 pgmap v1274: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s; 50602/269754 objects degraded (18.759%); 2/269754 objects misplaced (0.001%) 2023-09-23T18:57:12.275 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:12.303 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:12.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:12.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:12.334 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 88 16 93932 93916 66.691 47.9375 0.0101962 0.0149891 2023-09-23T18:57:12.405 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:12.429 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:12.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:12.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:12.531 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:12.558 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:12.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:12.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:12.664 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:12.682 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:12.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:12.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:12.784 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:12.813 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:12.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:12.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:12.923 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:12.958 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:12.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:12.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:13.065 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:13.109 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:13.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:13.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:13.178 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:57:13.178 INFO:teuthology.orchestra.run.smithi098.stdout: "success": "" 2023-09-23T18:57:13.178 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:57:13.210 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:13.240 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:13.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:13.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:13.334 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 89 16 94747 94731 66.5139 50.9375 0.0214469 0.0150292 2023-09-23T18:57:13.343 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:13.380 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:13.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:13.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:13.484 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:13.520 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:13.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:13.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:13.626 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:13.655 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:13.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:13.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:13.759 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:13.787 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:13.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:13.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:13.859 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:13 smithi098 bash[79516]: cluster 2023-09-23T18:57:12.315609+0000 mgr.y (mgr.26054) 1177 : cluster 0 pgmap v1275: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 892 op/s; 50873/273105 objects degraded (18.628%); 2/273105 objects misplaced (0.001%) 2023-09-23T18:57:13.860 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:13 smithi098 bash[76817]: cluster 2023-09-23T18:57:12.315609+0000 mgr.y (mgr.26054) 1177 : cluster 0 pgmap v1275: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 892 op/s; 50873/273105 objects degraded (18.628%); 2/273105 objects misplaced (0.001%) 2023-09-23T18:57:13.891 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:13.925 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:13.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:13.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:14.027 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:14.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:13 smithi173 bash[59422]: cluster 2023-09-23T18:57:12.315609+0000 mgr.y (mgr.26054) 1177 : cluster 0 pgmap v1275: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 892 op/s; 50873/273105 objects degraded (18.628%); 2/273105 objects misplaced (0.001%) 2023-09-23T18:57:14.056 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:14.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:14.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:14.134 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:57:14.162 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:14.193 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:14.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:14.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:14.208 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:57:14.249 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:57:14.280 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:57:14.296 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:14.321 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:14.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:14.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:14.325 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:57:14.334 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 90 2 95536 95534 66.3325 50.1875 0.0154667 0.0150697 2023-09-23T18:57:14.349 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:57:14.374 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:57:14.375 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:57:14.375 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:57:14.375 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:57:14.375 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:57:14.375 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:57:14.375 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:57:14.375 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:57:14.375 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T18:57:14.426 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:14.462 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:14.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:14.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:14.565 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:14.600 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:14.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:14.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:14.703 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:14.722 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:14.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:14.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:14.824 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:14.855 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:14.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:14.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:14.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:14 smithi098 bash[79516]: audit 2023-09-23T18:57:14.088202+0000 mon.c (mon.2) 130 : audit 0 from='client.? 172.21.15.98:0/2312843940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:57:14.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:14 smithi098 bash[76817]: audit 2023-09-23T18:57:14.088202+0000 mon.c (mon.2) 130 : audit 0 from='client.? 172.21.15.98:0/2312843940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:57:14.959 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:14.972 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:14 smithi173 bash[59422]: audit 2023-09-23T18:57:14.088202+0000 mon.c (mon.2) 130 : audit 0 from='client.? 172.21.15.98:0/2312843940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:57:14.990 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:14.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:14.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:15.093 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:15.119 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:15.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:15.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:15.223 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:15.254 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:15.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:15.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:15.335 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Total time run: 90.0158 2023-09-23T18:57:15.335 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Total writes made: 95536 2023-09-23T18:57:15.335 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Write size: 65536 2023-09-23T18:57:15.335 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Object size: 65536 2023-09-23T18:57:15.335 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Bandwidth (MB/sec): 66.3328 2023-09-23T18:57:15.335 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Stddev Bandwidth: 39.8892 2023-09-23T18:57:15.335 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Max bandwidth (MB/sec): 168.688 2023-09-23T18:57:15.335 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Min bandwidth (MB/sec): 0 2023-09-23T18:57:15.335 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Average IOPS: 1061 2023-09-23T18:57:15.335 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Stddev IOPS: 638.239 2023-09-23T18:57:15.335 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Max IOPS: 2699 2023-09-23T18:57:15.336 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Min IOPS: 0 2023-09-23T18:57:15.336 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Average Latency(s): 0.0150702 2023-09-23T18:57:15.336 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Stddev Latency(s): 0.20492 2023-09-23T18:57:15.336 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Max latency(s): 16.0742 2023-09-23T18:57:15.336 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Min latency(s): 0.00116653 2023-09-23T18:57:15.342 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T18:57:15.358 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:15.383 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:15.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:15.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:15.486 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:15.513 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:15.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:15.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:15.616 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:15.638 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:15.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:15.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:15.711 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is set 2023-09-23T18:57:15.744 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:15.745 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:15 smithi098 bash[79516]: cluster 2023-09-23T18:57:14.316892+0000 mgr.y (mgr.26054) 1178 : cluster 0 pgmap v1276: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 52908/282420 objects degraded (18.734%); 2/282420 objects misplaced (0.001%) 2023-09-23T18:57:15.746 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:15 smithi098 bash[79516]: audit 2023-09-23T18:57:14.713306+0000 mon.c (mon.2) 131 : audit 1 from='client.? 172.21.15.98:0/2252582634' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:57:15.746 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:15 smithi098 bash[79516]: audit 2023-09-23T18:57:14.713984+0000 mon.a (mon.0) 1601 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:57:15.746 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:15 smithi098 bash[79516]: cluster 2023-09-23T18:57:15.369248+0000 mon.a (mon.0) 1602 : cluster 3 Health check update: Degraded data redundancy: 52908/282420 objects degraded (18.734%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:15.747 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:15 smithi098 bash[76817]: cluster 2023-09-23T18:57:14.316892+0000 mgr.y (mgr.26054) 1178 : cluster 0 pgmap v1276: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 52908/282420 objects degraded (18.734%); 2/282420 objects misplaced (0.001%) 2023-09-23T18:57:15.747 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:15 smithi098 bash[76817]: audit 2023-09-23T18:57:14.713306+0000 mon.c (mon.2) 131 : audit 1 from='client.? 172.21.15.98:0/2252582634' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:57:15.747 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:15 smithi098 bash[76817]: audit 2023-09-23T18:57:14.713984+0000 mon.a (mon.0) 1601 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:57:15.747 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:15 smithi098 bash[76817]: cluster 2023-09-23T18:57:15.369248+0000 mon.a (mon.0) 1602 : cluster 3 Health check update: Degraded data redundancy: 52908/282420 objects degraded (18.734%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:15.767 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:15.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:15.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:15.870 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:15.901 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:15.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:15.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:16.008 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:16.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:15 smithi173 bash[59422]: cluster 2023-09-23T18:57:14.316892+0000 mgr.y (mgr.26054) 1178 : cluster 0 pgmap v1276: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 52908/282420 objects degraded (18.734%); 2/282420 objects misplaced (0.001%) 2023-09-23T18:57:16.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:15 smithi173 bash[59422]: audit 2023-09-23T18:57:14.713306+0000 mon.c (mon.2) 131 : audit 1 from='client.? 172.21.15.98:0/2252582634' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:57:16.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:15 smithi173 bash[59422]: audit 2023-09-23T18:57:14.713984+0000 mon.a (mon.0) 1601 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:57:16.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:15 smithi173 bash[59422]: cluster 2023-09-23T18:57:15.369248+0000 mon.a (mon.0) 1602 : cluster 3 Health check update: Degraded data redundancy: 52908/282420 objects degraded (18.734%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:16.038 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:16.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:16.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:16.144 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:16.171 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:16.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:16.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:16.278 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:16.286 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:57:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:57:16] "GET /metrics HTTP/1.1" 200 36586 "" "Prometheus/2.33.4" 2023-09-23T18:57:16.306 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:16.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:16.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:16.409 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:16.439 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:16.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:16.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:16.544 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:16.575 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:16.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:16.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:16.681 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:16.720 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:16.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:16.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:16.825 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:16.849 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:16.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:16.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:16.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:16.959 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:16 smithi173 bash[59422]: cluster 2023-09-23T18:57:15.692457+0000 mon.a (mon.0) 1603 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:57:16.960 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:16 smithi173 bash[59422]: audit 2023-09-23T18:57:15.698158+0000 mon.a (mon.0) 1604 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:57:16.960 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:16 smithi173 bash[59422]: cluster 2023-09-23T18:57:15.698245+0000 mon.a (mon.0) 1605 : cluster 0 osdmap e291: 8 total, 7 up, 8 in 2023-09-23T18:57:16.960 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:16 smithi173 bash[59422]: cluster 2023-09-23T18:57:15.701769+0000 osd.4 (osd.4) 10 : cluster 3 failed to encode map e291 with expected crc 2023-09-23T18:57:16.960 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:16 smithi173 bash[59422]: cluster 2023-09-23T18:57:15.706356+0000 osd.5 (osd.5) 206 : cluster 3 failed to encode map e291 with expected crc 2023-09-23T18:57:16.960 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:16 smithi173 bash[59422]: cluster 2023-09-23T18:57:15.733882+0000 osd.6 (osd.6) 261 : cluster 3 failed to encode map e291 with expected crc 2023-09-23T18:57:16.975 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:16.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:16.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:17.078 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:17.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:16 smithi098 bash[76817]: cluster 2023-09-23T18:57:15.692457+0000 mon.a (mon.0) 1603 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:57:17.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:16 smithi098 bash[76817]: audit 2023-09-23T18:57:15.698158+0000 mon.a (mon.0) 1604 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:57:17.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:16 smithi098 bash[76817]: cluster 2023-09-23T18:57:15.698245+0000 mon.a (mon.0) 1605 : cluster 0 osdmap e291: 8 total, 7 up, 8 in 2023-09-23T18:57:17.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:16 smithi098 bash[76817]: cluster 2023-09-23T18:57:15.701769+0000 osd.4 (osd.4) 10 : cluster 3 failed to encode map e291 with expected crc 2023-09-23T18:57:17.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:16 smithi098 bash[76817]: cluster 2023-09-23T18:57:15.706356+0000 osd.5 (osd.5) 206 : cluster 3 failed to encode map e291 with expected crc 2023-09-23T18:57:17.088 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:16 smithi098 bash[76817]: cluster 2023-09-23T18:57:15.733882+0000 osd.6 (osd.6) 261 : cluster 3 failed to encode map e291 with expected crc 2023-09-23T18:57:17.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:16 smithi098 bash[79516]: cluster 2023-09-23T18:57:15.692457+0000 mon.a (mon.0) 1603 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:57:17.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:16 smithi098 bash[79516]: audit 2023-09-23T18:57:15.698158+0000 mon.a (mon.0) 1604 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:57:17.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:16 smithi098 bash[79516]: cluster 2023-09-23T18:57:15.698245+0000 mon.a (mon.0) 1605 : cluster 0 osdmap e291: 8 total, 7 up, 8 in 2023-09-23T18:57:17.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:16 smithi098 bash[79516]: cluster 2023-09-23T18:57:15.701769+0000 osd.4 (osd.4) 10 : cluster 3 failed to encode map e291 with expected crc 2023-09-23T18:57:17.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:16 smithi098 bash[79516]: cluster 2023-09-23T18:57:15.706356+0000 osd.5 (osd.5) 206 : cluster 3 failed to encode map e291 with expected crc 2023-09-23T18:57:17.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:16 smithi098 bash[79516]: cluster 2023-09-23T18:57:15.733882+0000 osd.6 (osd.6) 261 : cluster 3 failed to encode map e291 with expected crc 2023-09-23T18:57:17.119 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:17.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:17.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:17.223 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:17.252 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:17.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:17.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:17.356 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:17.380 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:17.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:17.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:17.484 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:17.509 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:17.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:17.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:17.613 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:17.652 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:17.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:17.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:17.754 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:17.781 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:17.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:17.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:17.884 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:17.892 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:17 smithi098 bash[79516]: cluster 2023-09-23T18:57:16.317530+0000 mgr.y (mgr.26054) 1179 : cluster 0 pgmap v1278: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s; 53111/284640 objects degraded (18.659%); 2/284640 objects misplaced (0.001%) 2023-09-23T18:57:17.892 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:17 smithi098 bash[76817]: cluster 2023-09-23T18:57:16.317530+0000 mgr.y (mgr.26054) 1179 : cluster 0 pgmap v1278: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s; 53111/284640 objects degraded (18.659%); 2/284640 objects misplaced (0.001%) 2023-09-23T18:57:17.913 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:17.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:17.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:18.018 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:18.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:17 smithi173 bash[59422]: cluster 2023-09-23T18:57:16.317530+0000 mgr.y (mgr.26054) 1179 : cluster 0 pgmap v1278: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s; 53111/284640 objects degraded (18.659%); 2/284640 objects misplaced (0.001%) 2023-09-23T18:57:18.051 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:18.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:18.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:18.154 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:18.190 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:18.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:18.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:18.295 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:18.322 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:18.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:18.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:18.428 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:18.455 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:18.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:18.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:18.558 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:18.600 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:18.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:18.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:18.710 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:18.735 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:18.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:18.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:18.839 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:18.864 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:18.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:18.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:18.968 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:18.999 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:19.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:19.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:19.104 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:19.139 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:19.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:19.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:19.240 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:19.271 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:19.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:19.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:19.375 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:19.406 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:19.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:19.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:19.508 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:19.538 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:19.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:19.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:19.644 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:19.652 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:57:19.652 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:57:19.683 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:19.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:19.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:19.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:19 smithi173 bash[59422]: cluster 2023-09-23T18:57:18.318456+0000 mgr.y (mgr.26054) 1180 : cluster 0 pgmap v1279: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 37 MiB/s wr, 787 op/s; 52321/280872 objects degraded (18.628%); 2/280872 objects misplaced (0.001%) 2023-09-23T18:57:19.787 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:19.810 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:19.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:19.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:19.914 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:19.940 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:19.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:19.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:20.043 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:20.044 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:19 smithi098 bash[76817]: cluster 2023-09-23T18:57:18.318456+0000 mgr.y (mgr.26054) 1180 : cluster 0 pgmap v1279: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 37 MiB/s wr, 787 op/s; 52321/280872 objects degraded (18.628%); 2/280872 objects misplaced (0.001%) 2023-09-23T18:57:20.044 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:20 smithi098 bash[79516]: cluster 2023-09-23T18:57:18.318456+0000 mgr.y (mgr.26054) 1180 : cluster 0 pgmap v1279: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 37 MiB/s wr, 787 op/s; 52321/280872 objects degraded (18.628%); 2/280872 objects misplaced (0.001%) 2023-09-23T18:57:20.070 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:20.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:20.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:20.175 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:20.198 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:20.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:20.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:20.301 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:20.333 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:20.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:20.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:20.438 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:20.468 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:20.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:20.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:20.573 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:20.603 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:20.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:20.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:20.709 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:20.735 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:20.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:20.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:20.838 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:20.846 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:20 smithi173 bash[59422]: cluster 2023-09-23T18:57:20.370522+0000 mon.a (mon.0) 1606 : cluster 3 Health check update: Degraded data redundancy: 52321/280872 objects degraded (18.628%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:20.862 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:20.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:20.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:20.907 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:20 smithi098 bash[79516]: cluster 2023-09-23T18:57:20.370522+0000 mon.a (mon.0) 1606 : cluster 3 Health check update: Degraded data redundancy: 52321/280872 objects degraded (18.628%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:20.907 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:20 smithi098 bash[76817]: cluster 2023-09-23T18:57:20.370522+0000 mon.a (mon.0) 1606 : cluster 3 Health check update: Degraded data redundancy: 52321/280872 objects degraded (18.628%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:20.965 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:20.988 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:20.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:20.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:21.092 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:21.122 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:21.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:21.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:21.227 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:21.254 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:21.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:21.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:21.358 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:21.387 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:21.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:21.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:21.492 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:21.521 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:21.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:21.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:21.627 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:21.654 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:57:21.655 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:21.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:21.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:21.673 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:57:21.759 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:21.787 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:21.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:21.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:21.840 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:57:21.840 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:57:21.878 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:21 smithi098 bash[76817]: cluster 2023-09-23T18:57:20.319246+0000 mgr.y (mgr.26054) 1181 : cluster 0 pgmap v1280: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 35 MiB/s wr, 781 op/s; 52087/280170 objects degraded (18.591%); 2/280170 objects misplaced (0.001%) 2023-09-23T18:57:21.879 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:21 smithi098 bash[79516]: cluster 2023-09-23T18:57:20.319246+0000 mgr.y (mgr.26054) 1181 : cluster 0 pgmap v1280: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 35 MiB/s wr, 781 op/s; 52087/280170 objects degraded (18.591%); 2/280170 objects misplaced (0.001%) 2023-09-23T18:57:21.891 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:21.899 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:21 smithi173 bash[59422]: cluster 2023-09-23T18:57:20.319246+0000 mgr.y (mgr.26054) 1181 : cluster 0 pgmap v1280: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 35 MiB/s wr, 781 op/s; 52087/280170 objects degraded (18.591%); 2/280170 objects misplaced (0.001%) 2023-09-23T18:57:21.920 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:21.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:21.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:22.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:22.053 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:22.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:22.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:22.156 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:22.184 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:22.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:22.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:22.289 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:22.314 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:22.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:22.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:22.419 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:22.441 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:22.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:22.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:22.544 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:22.574 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:22.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:22.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:22.682 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:22.713 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:22.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:22.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:22.816 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:22.838 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:22.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:22.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:22.946 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:22.974 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:22.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:22.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:23.080 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:23.107 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:23.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:23.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:23.210 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:23.237 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:23.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:23.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:23.324 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7, 5] out_osds: [] dead_osds: [7] live_osds: [0, 2, 3, 5, 6, 1, 4] 2023-09-23T18:57:23.324 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:57:23.324 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-09-23T18:57:23.324 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T18:57:23.324 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T18:57:23.325 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-09-23T18:57:23.340 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:23.343 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T18:57:23.399 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:23.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:23.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:23.501 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:23.531 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:23.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:23.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:23.634 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:23.662 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:23.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:23.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:23.767 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:23.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:23 smithi173 bash[59422]: cluster 2023-09-23T18:57:22.320065+0000 mgr.y (mgr.26054) 1182 : cluster 0 pgmap v1281: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 28 MiB/s wr, 786 op/s; 51819/276687 objects degraded (18.728%); 2/276687 objects misplaced (0.001%) 2023-09-23T18:57:23.796 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:23.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:23.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:23.902 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:23.910 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:23 smithi098 bash[76817]: cluster 2023-09-23T18:57:22.320065+0000 mgr.y (mgr.26054) 1182 : cluster 0 pgmap v1281: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 28 MiB/s wr, 786 op/s; 51819/276687 objects degraded (18.728%); 2/276687 objects misplaced (0.001%) 2023-09-23T18:57:23.910 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:23 smithi098 bash[79516]: cluster 2023-09-23T18:57:22.320065+0000 mgr.y (mgr.26054) 1182 : cluster 0 pgmap v1281: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 28 MiB/s wr, 786 op/s; 51819/276687 objects degraded (18.728%); 2/276687 objects misplaced (0.001%) 2023-09-23T18:57:23.931 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:23.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:23.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:24.036 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:24.061 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:24.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:24.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:24.165 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:24.197 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:24.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:24.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:24.304 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:24.332 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:24.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:24.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:24.437 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:24.466 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:24.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:24.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:24.568 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:24.592 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:24.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:24.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:24.696 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:24.722 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:24.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:24.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:24.829 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:24.857 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:24.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:24.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:24.960 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:24.994 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:24.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:24.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:25.096 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:25.124 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:25.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:25.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:25.229 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:25.267 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:25.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:25.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:25.373 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:25.400 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:25.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:25.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:25.503 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:25.529 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:25.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:25.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:25.633 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:25.641 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:25 smithi173 bash[59422]: cluster 2023-09-23T18:57:24.321097+0000 mgr.y (mgr.26054) 1183 : cluster 0 pgmap v1282: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 8.9 MiB/s wr, 810 op/s; 49610/266649 objects degraded (18.605%); 2/266649 objects misplaced (0.001%) 2023-09-23T18:57:25.641 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:25 smithi173 bash[59422]: audit 2023-09-23T18:57:24.637352+0000 mon.a (mon.0) 1607 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:57:25.641 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:25 smithi173 bash[59422]: audit 2023-09-23T18:57:24.639314+0000 mon.a (mon.0) 1608 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:57:25.659 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:25.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:25.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:25.764 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:25.789 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:25.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:25.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:25.897 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:25.907 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:25 smithi173 bash[59422]: cluster 2023-09-23T18:57:25.374371+0000 mon.a (mon.0) 1609 : cluster 3 Health check update: Degraded data redundancy: 49610/266649 objects degraded (18.605%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:25.924 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:25.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:25.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:25.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:25 smithi098 bash[79516]: cluster 2023-09-23T18:57:24.321097+0000 mgr.y (mgr.26054) 1183 : cluster 0 pgmap v1282: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 8.9 MiB/s wr, 810 op/s; 49610/266649 objects degraded (18.605%); 2/266649 objects misplaced (0.001%) 2023-09-23T18:57:25.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:25 smithi098 bash[79516]: audit 2023-09-23T18:57:24.637352+0000 mon.a (mon.0) 1607 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:57:25.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:25 smithi098 bash[79516]: audit 2023-09-23T18:57:24.639314+0000 mon.a (mon.0) 1608 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:57:25.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:25 smithi098 bash[79516]: cluster 2023-09-23T18:57:25.374371+0000 mon.a (mon.0) 1609 : cluster 3 Health check update: Degraded data redundancy: 49610/266649 objects degraded (18.605%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:25.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:25 smithi098 bash[76817]: cluster 2023-09-23T18:57:24.321097+0000 mgr.y (mgr.26054) 1183 : cluster 0 pgmap v1282: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 8.9 MiB/s wr, 810 op/s; 49610/266649 objects degraded (18.605%); 2/266649 objects misplaced (0.001%) 2023-09-23T18:57:25.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:25 smithi098 bash[76817]: audit 2023-09-23T18:57:24.637352+0000 mon.a (mon.0) 1607 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:57:25.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:25 smithi098 bash[76817]: audit 2023-09-23T18:57:24.639314+0000 mon.a (mon.0) 1608 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:57:25.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:25 smithi098 bash[76817]: cluster 2023-09-23T18:57:25.374371+0000 mon.a (mon.0) 1609 : cluster 3 Health check update: Degraded data redundancy: 49610/266649 objects degraded (18.605%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:26.028 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:26.061 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:26.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:26.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:26.166 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:26.215 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:26.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:26.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:26.321 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:26.330 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:57:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:57:26] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.33.4" 2023-09-23T18:57:26.345 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:26.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:26.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:26.455 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:26.500 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:26.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:26.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:26.515 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:57:26.517 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:57:26.606 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:26.628 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:26.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:26.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:26.735 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:26.760 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:26.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:26.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:26.863 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:26.898 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:26.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:26.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:27.004 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:27.028 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:27.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:27.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:27.135 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:27.183 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:27.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:27.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:27.292 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:27.319 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:27.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:27.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:27.424 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:27.446 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:27.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:27.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:27.541 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:57:27.552 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:27.585 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:27.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:27.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:27.692 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:27.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:27 smithi098 bash[76817]: cluster 2023-09-23T18:57:26.321784+0000 mgr.y (mgr.26054) 1184 : cluster 0 pgmap v1283: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 96 B/s rd, 3.7 MiB/s wr, 791 op/s; 49326/263190 objects degraded (18.742%); 2/263190 objects misplaced (0.001%) 2023-09-23T18:57:27.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:27 smithi098 bash[79516]: cluster 2023-09-23T18:57:26.321784+0000 mgr.y (mgr.26054) 1184 : cluster 0 pgmap v1283: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 96 B/s rd, 3.7 MiB/s wr, 791 op/s; 49326/263190 objects degraded (18.742%); 2/263190 objects misplaced (0.001%) 2023-09-23T18:57:27.728 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:27.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:27.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:27.835 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:27.859 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:27.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:27.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:27.962 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:27.985 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:27.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:27.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:28.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:27 smithi173 bash[59422]: cluster 2023-09-23T18:57:26.321784+0000 mgr.y (mgr.26054) 1184 : cluster 0 pgmap v1283: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 96 B/s rd, 3.7 MiB/s wr, 791 op/s; 49326/263190 objects degraded (18.742%); 2/263190 objects misplaced (0.001%) 2023-09-23T18:57:28.091 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:28.103 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:57:28.120 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:28.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:28.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:28.223 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:28.247 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:28.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:28.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:28.352 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:28.388 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:28.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:28.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:28.491 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:28.534 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:28.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:28.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:28.636 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:28.658 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:28.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:28.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:28.761 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:28.771 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:28 smithi098 bash[79516]: audit 2023-09-23T18:57:28.291940+0000 mon.a (mon.0) 1610 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:57:28.771 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:28 smithi098 bash[76817]: audit 2023-09-23T18:57:28.291940+0000 mon.a (mon.0) 1610 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:57:28.785 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:28.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:28.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:28.890 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:28.926 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:28.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:28.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:29.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:28 smithi173 bash[59422]: audit 2023-09-23T18:57:28.291940+0000 mon.a (mon.0) 1610 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:57:29.034 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:29.055 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:29.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:29.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:29.158 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:29.186 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:29.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:29.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:29.290 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:29.314 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:29.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:29.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:29.417 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:29.444 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:29.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:29.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:29.549 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:29.576 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:29.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:29.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:29.680 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:29.702 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:29.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:29.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:29.808 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:29.850 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:29.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:29.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:29.967 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:29.994 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:29.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:29.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:30.100 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:30.108 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:29 smithi173 bash[59422]: cluster 2023-09-23T18:57:28.322645+0000 mgr.y (mgr.26054) 1185 : cluster 0 pgmap v1284: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 3.3 MiB/s wr, 810 op/s; 48275/259218 objects degraded (18.623%); 2/259218 objects misplaced (0.001%) 2023-09-23T18:57:30.108 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:29 smithi173 bash[59422]: audit 2023-09-23T18:57:28.826107+0000 mon.a (mon.0) 1611 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:57:30.108 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:29 smithi173 bash[59422]: audit 2023-09-23T18:57:28.833424+0000 mon.a (mon.0) 1612 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:57:30.122 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:30.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:30.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:30.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:29 smithi098 bash[79516]: cluster 2023-09-23T18:57:28.322645+0000 mgr.y (mgr.26054) 1185 : cluster 0 pgmap v1284: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 3.3 MiB/s wr, 810 op/s; 48275/259218 objects degraded (18.623%); 2/259218 objects misplaced (0.001%) 2023-09-23T18:57:30.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:29 smithi098 bash[79516]: audit 2023-09-23T18:57:28.826107+0000 mon.a (mon.0) 1611 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:57:30.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:29 smithi098 bash[79516]: audit 2023-09-23T18:57:28.833424+0000 mon.a (mon.0) 1612 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:57:30.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:29 smithi098 bash[76817]: cluster 2023-09-23T18:57:28.322645+0000 mgr.y (mgr.26054) 1185 : cluster 0 pgmap v1284: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 3.3 MiB/s wr, 810 op/s; 48275/259218 objects degraded (18.623%); 2/259218 objects misplaced (0.001%) 2023-09-23T18:57:30.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:29 smithi098 bash[76817]: audit 2023-09-23T18:57:28.826107+0000 mon.a (mon.0) 1611 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:57:30.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:29 smithi098 bash[76817]: audit 2023-09-23T18:57:28.833424+0000 mon.a (mon.0) 1612 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:57:30.229 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:30.266 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:30.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:30.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:30.369 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:30.392 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:30.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:30.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:30.495 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:30.520 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:30.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:30.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:30.625 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:30.660 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:30.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:30.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:30.764 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:30.789 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:30.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:30.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:30.894 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:30.903 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:30 smithi098 bash[79516]: cluster 2023-09-23T18:57:30.375483+0000 mon.a (mon.0) 1613 : cluster 3 Health check update: Degraded data redundancy: 48275/259218 objects degraded (18.623%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:30.903 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:30 smithi098 bash[76817]: cluster 2023-09-23T18:57:30.375483+0000 mon.a (mon.0) 1613 : cluster 3 Health check update: Degraded data redundancy: 48275/259218 objects degraded (18.623%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:30.922 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:30.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:30.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:31.027 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:31.050 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:31.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:31.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:31.158 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:31.183 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:31.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:31.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:31.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:30 smithi173 bash[59422]: cluster 2023-09-23T18:57:30.375483+0000 mon.a (mon.0) 1613 : cluster 3 Health check update: Degraded data redundancy: 48275/259218 objects degraded (18.623%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:31.287 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:31.324 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:31.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:31.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:31.428 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:31.450 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:31.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:31.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:31.553 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:31.576 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:31.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:31.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:31.679 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:31.710 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:31.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:31.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:31.816 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:31.855 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:31.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:31.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:31.960 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:31.969 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:31 smithi098 bash[76817]: cluster 2023-09-23T18:57:30.323611+0000 mgr.y (mgr.26054) 1186 : cluster 0 pgmap v1285: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 27 KiB/s wr, 761 op/s; 47213/253482 objects degraded (18.626%); 2/253482 objects misplaced (0.001%) 2023-09-23T18:57:31.969 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:31 smithi098 bash[79516]: cluster 2023-09-23T18:57:30.323611+0000 mgr.y (mgr.26054) 1186 : cluster 0 pgmap v1285: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 27 KiB/s wr, 761 op/s; 47213/253482 objects degraded (18.626%); 2/253482 objects misplaced (0.001%) 2023-09-23T18:57:31.985 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:31.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:31.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:32.090 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:32.120 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:32.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:32.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:32.218 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:57:32.218 INFO:teuthology.orchestra.run.smithi173.stdout: "success": "" 2023-09-23T18:57:32.219 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:57:32.226 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:32.244 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:31 smithi173 bash[59422]: cluster 2023-09-23T18:57:30.323611+0000 mgr.y (mgr.26054) 1186 : cluster 0 pgmap v1285: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 27 KiB/s wr, 761 op/s; 47213/253482 objects degraded (18.626%); 2/253482 objects misplaced (0.001%) 2023-09-23T18:57:32.255 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:32.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:32.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:32.360 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:32.383 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:32.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:32.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:32.489 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:32.511 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:32.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:32.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:32.614 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:32.635 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:32.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:32.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:32.738 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:32.768 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:32.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:32.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:32.872 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:32.893 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:32.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:32.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:32.996 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:33.022 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:33.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:33.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:33.125 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:33.155 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:33.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:33.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:33.259 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:33.283 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:33.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:33.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:33.386 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:33.422 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:33.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:33.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:33.528 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:33.554 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:33.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:33.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:33.657 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:33.683 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:33.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:33.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:33.788 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:33.813 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:33.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:33.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:33.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:33.958 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:33.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:33.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:34.065 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:34.075 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:33 smithi173 bash[59422]: cluster 2023-09-23T18:57:32.324454+0000 mgr.y (mgr.26054) 1187 : cluster 0 pgmap v1286: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 46969/250530 objects degraded (18.748%); 2/250530 objects misplaced (0.001%) 2023-09-23T18:57:34.091 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:34.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:34.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:34.173 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:33 smithi098 bash[79516]: cluster 2023-09-23T18:57:32.324454+0000 mgr.y (mgr.26054) 1187 : cluster 0 pgmap v1286: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 46969/250530 objects degraded (18.748%); 2/250530 objects misplaced (0.001%) 2023-09-23T18:57:34.174 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:33 smithi098 bash[76817]: cluster 2023-09-23T18:57:32.324454+0000 mgr.y (mgr.26054) 1187 : cluster 0 pgmap v1286: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 46969/250530 objects degraded (18.748%); 2/250530 objects misplaced (0.001%) 2023-09-23T18:57:34.196 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:34.242 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:34.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:34.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:34.345 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:34.374 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:34.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:34.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:34.476 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:34.509 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:34.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:34.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:34.611 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:34.631 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:34.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:34.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:34.733 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:34.770 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:34.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:34.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:34.875 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:34.920 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:34.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:34.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:35.024 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:35.049 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:35.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:35.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:35.154 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:35.187 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:35.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:35.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:35.293 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:35.324 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:35.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:35.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:35.428 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:35.454 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:35.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:35.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:35.557 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:35.591 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:35.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:35.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:35.694 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:35.716 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:35.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:35.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:35.819 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:35.850 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:35.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:35.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:35.957 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:35.978 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:35.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:35.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:36.080 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:36.113 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:36.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:36.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:36.216 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:36.244 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:36.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:36.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:36.348 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:36.373 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:36.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:36.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:36.443 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:57:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:57:36] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.33.4" 2023-09-23T18:57:36.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:36 smithi098 bash[79516]: cluster 2023-09-23T18:57:34.325443+0000 mgr.y (mgr.26054) 1188 : cluster 0 pgmap v1287: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 983 op/s; 44883/241257 objects degraded (18.604%); 2/241257 objects misplaced (0.001%) 2023-09-23T18:57:36.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:36 smithi098 bash[79516]: audit 2023-09-23T18:57:35.204419+0000 mon.a (mon.0) 1614 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:57:36.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:36 smithi098 bash[79516]: audit 2023-09-23T18:57:35.213425+0000 mon.a (mon.0) 1615 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:57:36.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:36 smithi098 bash[79516]: cluster 2023-09-23T18:57:35.377543+0000 mon.a (mon.0) 1616 : cluster 3 Health check update: Degraded data redundancy: 44883/241257 objects degraded (18.604%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:36.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:36 smithi098 bash[76817]: cluster 2023-09-23T18:57:34.325443+0000 mgr.y (mgr.26054) 1188 : cluster 0 pgmap v1287: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 983 op/s; 44883/241257 objects degraded (18.604%); 2/241257 objects misplaced (0.001%) 2023-09-23T18:57:36.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:36 smithi098 bash[76817]: audit 2023-09-23T18:57:35.204419+0000 mon.a (mon.0) 1614 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:57:36.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:36 smithi098 bash[76817]: audit 2023-09-23T18:57:35.213425+0000 mon.a (mon.0) 1615 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:57:36.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:36 smithi098 bash[76817]: cluster 2023-09-23T18:57:35.377543+0000 mon.a (mon.0) 1616 : cluster 3 Health check update: Degraded data redundancy: 44883/241257 objects degraded (18.604%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:36.475 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:36.484 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:36 smithi173 bash[59422]: cluster 2023-09-23T18:57:34.325443+0000 mgr.y (mgr.26054) 1188 : cluster 0 pgmap v1287: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 983 op/s; 44883/241257 objects degraded (18.604%); 2/241257 objects misplaced (0.001%) 2023-09-23T18:57:36.484 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:36 smithi173 bash[59422]: audit 2023-09-23T18:57:35.204419+0000 mon.a (mon.0) 1614 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:57:36.484 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:36 smithi173 bash[59422]: audit 2023-09-23T18:57:35.213425+0000 mon.a (mon.0) 1615 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:57:36.484 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:36 smithi173 bash[59422]: cluster 2023-09-23T18:57:35.377543+0000 mon.a (mon.0) 1616 : cluster 3 Health check update: Degraded data redundancy: 44883/241257 objects degraded (18.604%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:36.505 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:36.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:36.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:36.609 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:36.628 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:36.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:36.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:36.731 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:36.763 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:36.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:36.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:36.865 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:36.889 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:36.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:36.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:36.995 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:37.017 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:37.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:37.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:37.120 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:37.144 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:37.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:37.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:37.249 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:37.281 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:37.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:37.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:37.388 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:37.428 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:37.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:37.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:37.534 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:37.562 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:37.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:37.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:37.665 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:37.710 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:37.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:37.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:37.814 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:37.840 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:37.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:37.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:37.945 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:37.989 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:37.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:37.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:38.093 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:38.122 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:38.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:38.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:38.210 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:38 smithi098 bash[76817]: cluster 2023-09-23T18:57:36.326259+0000 mgr.y (mgr.26054) 1189 : cluster 0 pgmap v1288: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 44622/238059 objects degraded (18.744%); 2/238059 objects misplaced (0.001%) 2023-09-23T18:57:38.215 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:38 smithi098 bash[79516]: cluster 2023-09-23T18:57:36.326259+0000 mgr.y (mgr.26054) 1189 : cluster 0 pgmap v1288: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 44622/238059 objects degraded (18.744%); 2/238059 objects misplaced (0.001%) 2023-09-23T18:57:38.231 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:38.265 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:38.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:38.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:38.330 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7, 5] out_osds: [] dead_osds: [7] live_osds: [0, 2, 3, 5, 6, 1, 4] 2023-09-23T18:57:38.330 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:57:38.330 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [0, 1, 2, 3, 4, 6, 7, 5] 2023-09-23T18:57:38.330 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd out 3 2023-09-23T18:57:38.341 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:57:38.372 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:38.384 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:57:38.427 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:57:38.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:38.438 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:38.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:38.474 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:57:38.526 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:57:38.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:38 smithi173 bash[59422]: cluster 2023-09-23T18:57:36.326259+0000 mgr.y (mgr.26054) 1189 : cluster 0 pgmap v1288: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 44622/238059 objects degraded (18.744%); 2/238059 objects misplaced (0.001%) 2023-09-23T18:57:38.539 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:38.573 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:38.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:38.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:38.593 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:57:38.661 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:57:38.663 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:57:38.663 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:57:38.664 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:57:38.664 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:57:38.664 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:57:38.664 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:57:38.664 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:57:38.664 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T18:57:38.681 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T18:57:38.681 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [], 2023-09-23T18:57:38.681 INFO:teuthology.orchestra.run.smithi098.stdout: "complaint_time": 30, 2023-09-23T18:57:38.681 INFO:teuthology.orchestra.run.smithi098.stdout: "num_blocked_ops": 0 2023-09-23T18:57:38.681 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T18:57:38.682 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:38.707 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:38.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:38.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:38.811 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:38.837 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:38.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:38.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:38.940 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:38.985 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:38.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:38.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:39.088 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:39.114 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:39.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:39.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:39.220 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:39.252 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:39.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:39.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:39.357 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:39.368 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:39 smithi173 bash[59422]: audit 2023-09-23T18:57:38.246215+0000 mon.c (mon.2) 132 : audit 0 from='client.? 172.21.15.98:0/733327064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:57:39.368 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:39 smithi173 bash[59422]: audit 2023-09-23T18:57:38.837300+0000 mon.a (mon.0) 1617 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:57:39.368 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:39 smithi173 bash[59422]: audit 2023-09-23T18:57:38.844758+0000 mon.a (mon.0) 1618 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:57:39.368 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:39 smithi173 bash[59422]: audit 2023-09-23T18:57:38.985221+0000 mon.a (mon.0) 1619 : audit 1 from='client.? 172.21.15.98:0/2539368085' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:57:39.380 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:39 smithi098 bash[76817]: audit 2023-09-23T18:57:38.246215+0000 mon.c (mon.2) 132 : audit 0 from='client.? 172.21.15.98:0/733327064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:57:39.380 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:39 smithi098 bash[76817]: audit 2023-09-23T18:57:38.837300+0000 mon.a (mon.0) 1617 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:57:39.380 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:39 smithi098 bash[76817]: audit 2023-09-23T18:57:38.844758+0000 mon.a (mon.0) 1618 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:57:39.380 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:39 smithi098 bash[76817]: audit 2023-09-23T18:57:38.985221+0000 mon.a (mon.0) 1619 : audit 1 from='client.? 172.21.15.98:0/2539368085' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:57:39.381 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:39 smithi098 bash[79516]: audit 2023-09-23T18:57:38.246215+0000 mon.c (mon.2) 132 : audit 0 from='client.? 172.21.15.98:0/733327064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:57:39.381 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:39 smithi098 bash[79516]: audit 2023-09-23T18:57:38.837300+0000 mon.a (mon.0) 1617 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:57:39.381 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:39 smithi098 bash[79516]: audit 2023-09-23T18:57:38.844758+0000 mon.a (mon.0) 1618 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:57:39.381 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:39 smithi098 bash[79516]: audit 2023-09-23T18:57:38.985221+0000 mon.a (mon.0) 1619 : audit 1 from='client.? 172.21.15.98:0/2539368085' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:57:39.383 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:39.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:39.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:39.490 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:39.523 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:39.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:39.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:39.626 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:39.652 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:39.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:39.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:39.755 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:39.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:39.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:39.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:39.853 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is set 2023-09-23T18:57:39.889 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:39.917 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:39.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:39.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:40.021 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:40.043 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:40.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:40.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:40.148 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:40.176 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:40.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:40.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:40.279 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:40.310 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:40.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:40.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:40.415 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:40.429 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:40 smithi173 bash[59422]: cluster 2023-09-23T18:57:38.329648+0000 mgr.y (mgr.26054) 1190 : cluster 0 pgmap v1289: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 43574/234027 objects degraded (18.619%); 2/234027 objects misplaced (0.001%) 2023-09-23T18:57:40.429 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:40 smithi173 bash[59422]: audit 2023-09-23T18:57:39.480604+0000 mon.a (mon.0) 1620 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:40.429 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:40 smithi173 bash[59422]: audit 2023-09-23T18:57:39.481416+0000 mon.a (mon.0) 1621 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:57:40.429 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:40 smithi173 bash[59422]: audit 2023-09-23T18:57:39.487403+0000 mon.a (mon.0) 1622 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:57:40.429 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:40 smithi173 bash[59422]: audit 2023-09-23T18:57:39.632553+0000 mon.a (mon.0) 1623 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:57:40.429 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:40 smithi173 bash[59422]: cluster 2023-09-23T18:57:39.841403+0000 mon.a (mon.0) 1624 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:57:40.429 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:40 smithi173 bash[59422]: audit 2023-09-23T18:57:39.845747+0000 mon.a (mon.0) 1625 : audit 1 from='client.? 172.21.15.98:0/2539368085' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:57:40.429 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:40 smithi173 bash[59422]: cluster 2023-09-23T18:57:39.845808+0000 mon.a (mon.0) 1626 : cluster 0 osdmap e292: 8 total, 7 up, 8 in 2023-09-23T18:57:40.430 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:40 smithi173 bash[59422]: cluster 2023-09-23T18:57:39.846133+0000 osd.5 (osd.5) 207 : cluster 3 failed to encode map e292 with expected crc 2023-09-23T18:57:40.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:40 smithi098 bash[79516]: cluster 2023-09-23T18:57:38.329648+0000 mgr.y (mgr.26054) 1190 : cluster 0 pgmap v1289: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 43574/234027 objects degraded (18.619%); 2/234027 objects misplaced (0.001%) 2023-09-23T18:57:40.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:40 smithi098 bash[79516]: audit 2023-09-23T18:57:39.480604+0000 mon.a (mon.0) 1620 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:40.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:40 smithi098 bash[79516]: audit 2023-09-23T18:57:39.481416+0000 mon.a (mon.0) 1621 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:57:40.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:40 smithi098 bash[79516]: audit 2023-09-23T18:57:39.487403+0000 mon.a (mon.0) 1622 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:57:40.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:40 smithi098 bash[79516]: audit 2023-09-23T18:57:39.632553+0000 mon.a (mon.0) 1623 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:57:40.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:40 smithi098 bash[79516]: cluster 2023-09-23T18:57:39.841403+0000 mon.a (mon.0) 1624 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:57:40.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:40 smithi098 bash[79516]: audit 2023-09-23T18:57:39.845747+0000 mon.a (mon.0) 1625 : audit 1 from='client.? 172.21.15.98:0/2539368085' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:57:40.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:40 smithi098 bash[79516]: cluster 2023-09-23T18:57:39.845808+0000 mon.a (mon.0) 1626 : cluster 0 osdmap e292: 8 total, 7 up, 8 in 2023-09-23T18:57:40.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:40 smithi098 bash[79516]: cluster 2023-09-23T18:57:39.846133+0000 osd.5 (osd.5) 207 : cluster 3 failed to encode map e292 with expected crc 2023-09-23T18:57:40.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:40 smithi098 bash[76817]: cluster 2023-09-23T18:57:38.329648+0000 mgr.y (mgr.26054) 1190 : cluster 0 pgmap v1289: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 43574/234027 objects degraded (18.619%); 2/234027 objects misplaced (0.001%) 2023-09-23T18:57:40.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:40 smithi098 bash[76817]: audit 2023-09-23T18:57:39.480604+0000 mon.a (mon.0) 1620 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:57:40.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:40 smithi098 bash[76817]: audit 2023-09-23T18:57:39.481416+0000 mon.a (mon.0) 1621 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:57:40.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:40 smithi098 bash[76817]: audit 2023-09-23T18:57:39.487403+0000 mon.a (mon.0) 1622 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:57:40.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:40 smithi098 bash[76817]: audit 2023-09-23T18:57:39.632553+0000 mon.a (mon.0) 1623 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:57:40.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:40 smithi098 bash[76817]: cluster 2023-09-23T18:57:39.841403+0000 mon.a (mon.0) 1624 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:57:40.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:40 smithi098 bash[76817]: audit 2023-09-23T18:57:39.845747+0000 mon.a (mon.0) 1625 : audit 1 from='client.? 172.21.15.98:0/2539368085' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:57:40.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:40 smithi098 bash[76817]: cluster 2023-09-23T18:57:39.845808+0000 mon.a (mon.0) 1626 : cluster 0 osdmap e292: 8 total, 7 up, 8 in 2023-09-23T18:57:40.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:40 smithi098 bash[76817]: cluster 2023-09-23T18:57:39.846133+0000 osd.5 (osd.5) 207 : cluster 3 failed to encode map e292 with expected crc 2023-09-23T18:57:40.445 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:40.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:40.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:40.549 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:40.573 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:40.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:40.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:40.676 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:40.700 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:40.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:40.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:40.804 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:40.834 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:40.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:40.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:40.943 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:40.969 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:40.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:40.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:41.072 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:41.099 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:41.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:41.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:41.202 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:41.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:41.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:41.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:41.329 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:41.344 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:41 smithi173 bash[59422]: cluster 2023-09-23T18:57:39.847276+0000 osd.4 (osd.4) 11 : cluster 3 failed to encode map e292 with expected crc 2023-09-23T18:57:41.344 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:41 smithi173 bash[59422]: cluster 2023-09-23T18:57:40.381233+0000 mon.a (mon.0) 1627 : cluster 3 Health check update: Degraded data redundancy: 43574/234027 objects degraded (18.619%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:41.365 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:41.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:41.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:41.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:41 smithi098 bash[79516]: cluster 2023-09-23T18:57:39.847276+0000 osd.4 (osd.4) 11 : cluster 3 failed to encode map e292 with expected crc 2023-09-23T18:57:41.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:41 smithi098 bash[79516]: cluster 2023-09-23T18:57:40.381233+0000 mon.a (mon.0) 1627 : cluster 3 Health check update: Degraded data redundancy: 43574/234027 objects degraded (18.619%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:41.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:41 smithi098 bash[76817]: cluster 2023-09-23T18:57:39.847276+0000 osd.4 (osd.4) 11 : cluster 3 failed to encode map e292 with expected crc 2023-09-23T18:57:41.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:41 smithi098 bash[76817]: cluster 2023-09-23T18:57:40.381233+0000 mon.a (mon.0) 1627 : cluster 3 Health check update: Degraded data redundancy: 43574/234027 objects degraded (18.619%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:41.470 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:41.496 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:41.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:41.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:41.604 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:41.631 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:41.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:41.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:41.698 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:57:41.698 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:57:41.738 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:41.765 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:41.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:41.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:41.872 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:41.893 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:41.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:41.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:41.999 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:42.027 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:42.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:42.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:42.130 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:42.161 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:42.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:42.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:42.264 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:42.271 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:42 smithi098 bash[76817]: cluster 2023-09-23T18:57:40.331718+0000 mgr.y (mgr.26054) 1191 : cluster 0 pgmap v1291: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 42559/228636 objects degraded (18.614%); 2/228636 objects misplaced (0.001%) 2023-09-23T18:57:42.272 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:42 smithi098 bash[79516]: cluster 2023-09-23T18:57:40.331718+0000 mgr.y (mgr.26054) 1191 : cluster 0 pgmap v1291: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 42559/228636 objects degraded (18.614%); 2/228636 objects misplaced (0.001%) 2023-09-23T18:57:42.285 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:42.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:42.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:42.389 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:42.392 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T18:57:42.396 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:42 smithi173 bash[59422]: cluster 2023-09-23T18:57:40.331718+0000 mgr.y (mgr.26054) 1191 : cluster 0 pgmap v1291: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 42559/228636 objects degraded (18.614%); 2/228636 objects misplaced (0.001%) 2023-09-23T18:57:42.410 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:42.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:42.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:42.514 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:42.541 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:42.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:42.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:42.650 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:42.680 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:42.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:42.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:42.783 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:42.807 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:42.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:42.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:42.913 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:42.942 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:42.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:42.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:43.048 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:43.074 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:43.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:43.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:43.181 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:43.207 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:43.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:43.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:43.313 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:43.338 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:43.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:43.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:43.444 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:43.471 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:43.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:43.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:43.574 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:43.608 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:43.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:43.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:43.699 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:57:43.712 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:43.804 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:43.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:43.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:43.913 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:43.945 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:43.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:43.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:44.053 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:44.096 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:44.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:44.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:44.200 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:44.230 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:44.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:44.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:44.339 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:44.382 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:44.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:44.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:44.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:44.497 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:44 smithi173 bash[59422]: cluster 2023-09-23T18:57:42.332531+0000 mgr.y (mgr.26054) 1192 : cluster 0 pgmap v1292: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 42351/226041 objects degraded (18.736%); 2/226041 objects misplaced (0.001%) 2023-09-23T18:57:44.514 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:44.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:44.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:44.624 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:44.648 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:44.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:44.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:44.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:44 smithi098 bash[76817]: cluster 2023-09-23T18:57:42.332531+0000 mgr.y (mgr.26054) 1192 : cluster 0 pgmap v1292: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 42351/226041 objects degraded (18.736%); 2/226041 objects misplaced (0.001%) 2023-09-23T18:57:44.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:44 smithi098 bash[79516]: cluster 2023-09-23T18:57:42.332531+0000 mgr.y (mgr.26054) 1192 : cluster 0 pgmap v1292: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 42351/226041 objects degraded (18.736%); 2/226041 objects misplaced (0.001%) 2023-09-23T18:57:44.721 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:57:44.751 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:44.794 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:44.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:44.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:44.898 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:44.923 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:44.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:44.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:45.031 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:45.059 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:45.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:45.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:45.165 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:45.190 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:45.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:45.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:45.297 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:45.322 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:45.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:45.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:45.425 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:45.449 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:45.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:45.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:45.553 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:45.581 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:45.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:45.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:45.685 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:45.710 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:45.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:45.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:45.817 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:45.850 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:45.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:45.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:45.953 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:45.980 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:45.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:45.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:46.084 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:46.087 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:57:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:57:46] "GET /metrics HTTP/1.1" 200 36588 "" "Prometheus/2.33.4" 2023-09-23T18:57:46.115 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:46.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:46.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:46.219 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:46.243 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:46.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:46.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:46.264 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:57:46.346 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:46.355 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:46 smithi098 bash[79516]: cluster 2023-09-23T18:57:44.335271+0000 mgr.y (mgr.26054) 1193 : cluster 0 pgmap v1293: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 40500/217803 objects degraded (18.595%); 2/217803 objects misplaced (0.001%) 2023-09-23T18:57:46.355 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:46 smithi098 bash[79516]: cluster 2023-09-23T18:57:45.382633+0000 mon.a (mon.0) 1628 : cluster 3 Health check update: Degraded data redundancy: 40500/217803 objects degraded (18.595%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:46.355 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:46 smithi098 bash[76817]: cluster 2023-09-23T18:57:44.335271+0000 mgr.y (mgr.26054) 1193 : cluster 0 pgmap v1293: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 40500/217803 objects degraded (18.595%); 2/217803 objects misplaced (0.001%) 2023-09-23T18:57:46.355 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:46 smithi098 bash[76817]: cluster 2023-09-23T18:57:45.382633+0000 mon.a (mon.0) 1628 : cluster 3 Health check update: Degraded data redundancy: 40500/217803 objects degraded (18.595%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:46.376 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:46.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:46.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:46.482 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:46.511 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:46.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:46.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:46.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:46 smithi173 bash[59422]: cluster 2023-09-23T18:57:44.335271+0000 mgr.y (mgr.26054) 1193 : cluster 0 pgmap v1293: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 40500/217803 objects degraded (18.595%); 2/217803 objects misplaced (0.001%) 2023-09-23T18:57:46.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:46 smithi173 bash[59422]: cluster 2023-09-23T18:57:45.382633+0000 mon.a (mon.0) 1628 : cluster 3 Health check update: Degraded data redundancy: 40500/217803 objects degraded (18.595%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:46.614 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:46.637 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:46.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:46.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:46.743 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:46.778 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:46.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:46.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:46.881 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:46.912 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:46.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:46.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:47.018 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:47.042 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:47.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:47.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:47.145 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:47.185 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:47.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:47.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:47.284 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:57:47.295 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:47.320 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:47.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:47.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:47.424 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:47.454 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:47.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:47.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:47.556 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:47.585 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:47.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:47.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:47.689 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:47.723 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:47.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:47.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:47.829 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:47.854 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:47.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:47.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:47.958 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:47.990 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:47.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:47.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:48.093 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:48.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:48.129 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:48.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:48.230 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:48.266 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:48.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:48.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:48.301 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:57:48.369 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:48.380 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:48 smithi098 bash[76817]: cluster 2023-09-23T18:57:46.336339+0000 mgr.y (mgr.26054) 1194 : cluster 0 pgmap v1294: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 40223/214941 objects degraded (18.714%); 2/214941 objects misplaced (0.001%) 2023-09-23T18:57:48.380 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:48 smithi098 bash[79516]: cluster 2023-09-23T18:57:46.336339+0000 mgr.y (mgr.26054) 1194 : cluster 0 pgmap v1294: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 40223/214941 objects degraded (18.714%); 2/214941 objects misplaced (0.001%) 2023-09-23T18:57:48.394 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:48.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:48.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:48.496 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:48.523 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:48.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:48 smithi173 bash[59422]: cluster 2023-09-23T18:57:46.336339+0000 mgr.y (mgr.26054) 1194 : cluster 0 pgmap v1294: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 40223/214941 objects degraded (18.714%); 2/214941 objects misplaced (0.001%) 2023-09-23T18:57:48.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:48.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:48.630 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:48.675 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:48.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:48.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:48.780 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:48.808 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:48.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:48.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:48.913 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:48.941 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:48.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:48.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:49.043 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:49.069 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:49.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:49.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:49.177 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:49.213 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:49.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:49.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:49.320 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:57:49.320 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:49.341 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:49.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:49.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:49.445 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:49.474 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:49.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:49.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:49.577 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:49.604 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:49.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:49.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:49.710 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:49.732 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:49.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:49.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:49.836 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:49.866 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:49.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:49.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:49.971 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:49.991 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:49.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:49.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:50.097 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:50.124 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:50.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:50.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:50.227 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:50.257 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:50.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:50.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:50.361 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:50.402 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:50.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:50.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:50.507 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:50.516 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:50 smithi098 bash[76817]: cluster 2023-09-23T18:57:48.337255+0000 mgr.y (mgr.26054) 1195 : cluster 0 pgmap v1295: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 39206/211107 objects degraded (18.572%); 2/211107 objects misplaced (0.001%) 2023-09-23T18:57:50.516 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:50 smithi098 bash[79516]: cluster 2023-09-23T18:57:48.337255+0000 mgr.y (mgr.26054) 1195 : cluster 0 pgmap v1295: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 39206/211107 objects degraded (18.572%); 2/211107 objects misplaced (0.001%) 2023-09-23T18:57:50.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:50 smithi173 bash[59422]: cluster 2023-09-23T18:57:48.337255+0000 mgr.y (mgr.26054) 1195 : cluster 0 pgmap v1295: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 39206/211107 objects degraded (18.572%); 2/211107 objects misplaced (0.001%) 2023-09-23T18:57:50.536 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:50.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:50.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:50.642 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:50.675 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:50.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:50.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:50.782 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:50.804 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:50.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:50.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:50.907 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:50.936 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:50.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:50.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:51.042 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:51.067 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:51.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:51.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:51.174 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:51.199 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:51.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:51.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:51.305 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:51.324 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:51.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:51.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:51.431 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:51.440 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:51 smithi098 bash[76817]: cluster 2023-09-23T18:57:50.383908+0000 mon.a (mon.0) 1629 : cluster 3 Health check update: Degraded data redundancy: 39206/211107 objects degraded (18.572%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:51.441 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:51 smithi098 bash[79516]: cluster 2023-09-23T18:57:50.383908+0000 mon.a (mon.0) 1629 : cluster 3 Health check update: Degraded data redundancy: 39206/211107 objects degraded (18.572%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:51.452 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:51.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:51.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:51.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:51 smithi173 bash[59422]: cluster 2023-09-23T18:57:50.383908+0000 mon.a (mon.0) 1629 : cluster 3 Health check update: Degraded data redundancy: 39206/211107 objects degraded (18.572%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:51.556 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:51.581 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:51.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:51.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:51.683 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:51.710 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:51.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:51.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:51.813 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:51.840 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:51.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:51.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:51.945 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:51.991 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:51.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:51.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:52.094 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:52.125 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:52.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:52.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:52.229 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:52.266 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:52.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:52.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:52.372 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:52.400 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:52.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:52.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:52.499 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:52 smithi098 bash[76817]: cluster 2023-09-23T18:57:50.338401+0000 mgr.y (mgr.26054) 1196 : cluster 0 pgmap v1296: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 38263/206103 objects degraded (18.565%); 2/206103 objects misplaced (0.001%) 2023-09-23T18:57:52.499 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:52 smithi098 bash[79516]: cluster 2023-09-23T18:57:50.338401+0000 mgr.y (mgr.26054) 1196 : cluster 0 pgmap v1296: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 38263/206103 objects degraded (18.565%); 2/206103 objects misplaced (0.001%) 2023-09-23T18:57:52.502 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:52.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:52 smithi173 bash[59422]: cluster 2023-09-23T18:57:50.338401+0000 mgr.y (mgr.26054) 1196 : cluster 0 pgmap v1296: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 38263/206103 objects degraded (18.565%); 2/206103 objects misplaced (0.001%) 2023-09-23T18:57:52.533 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:52.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:52.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:52.642 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:52.688 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:52.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:52.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:52.794 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:52.822 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:57:52.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:52.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:57:52.931 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:52.955 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:52.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:52.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:53.061 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:53.096 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:53.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:53.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:53.203 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:53.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:53.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:53.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:53.333 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:53.358 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:53.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:53.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:53.461 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:53.485 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:53.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:53.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:53.589 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:53.624 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:53.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:53.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:53.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:53.755 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:53.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:53.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:53.861 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:53.883 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:53.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:53.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:53.988 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:54.011 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:54.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:54.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:54.117 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:54.142 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:54.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:54.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:54.248 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:54.273 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:54.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:54.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:54.278 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:54 smithi098 bash[76817]: cluster 2023-09-23T18:57:52.339253+0000 mgr.y (mgr.26054) 1197 : cluster 0 pgmap v1297: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 38016/203199 objects degraded (18.709%); 2/203199 objects misplaced (0.001%) 2023-09-23T18:57:54.378 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:54.408 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:54.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:54.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:54.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:54 smithi098 bash[79516]: cluster 2023-09-23T18:57:52.339253+0000 mgr.y (mgr.26054) 1197 : cluster 0 pgmap v1297: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 38016/203199 objects degraded (18.709%); 2/203199 objects misplaced (0.001%) 2023-09-23T18:57:54.511 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:54.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:54 smithi173 bash[59422]: cluster 2023-09-23T18:57:52.339253+0000 mgr.y (mgr.26054) 1197 : cluster 0 pgmap v1297: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 38016/203199 objects degraded (18.709%); 2/203199 objects misplaced (0.001%) 2023-09-23T18:57:54.536 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:54.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:54.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:54.640 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:54.675 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:54.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:54.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:54.779 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:54.802 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:54.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:54.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:54.912 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:54.938 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:54.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:54.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:55.043 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:55.067 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:55.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:55.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:55.170 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:55.203 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:55.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:55.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:55.288 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:55 smithi098 bash[76817]: audit 2023-09-23T18:57:54.633441+0000 mon.a (mon.0) 1630 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:57:55.310 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:55.335 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:55.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:55.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:55.441 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:55.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:55 smithi098 bash[79516]: audit 2023-09-23T18:57:54.633441+0000 mon.a (mon.0) 1630 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:57:55.468 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:55.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:55.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:55.572 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:55.581 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:55 smithi173 bash[59422]: audit 2023-09-23T18:57:54.633441+0000 mon.a (mon.0) 1630 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:57:55.600 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:55.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:55.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:55.704 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:55.730 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:55.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:55.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:55.836 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:55.860 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:55.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:55.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:55.968 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:55.992 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:55.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:55.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:56.087 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:57:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:57:56] "GET /metrics HTTP/1.1" 200 36590 "" "Prometheus/2.33.4" 2023-09-23T18:57:56.096 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:56.127 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:56.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:56.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:56.232 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:56.275 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:56.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:56.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:56.380 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:56.389 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:56 smithi098 bash[79516]: cluster 2023-09-23T18:57:54.340250+0000 mgr.y (mgr.26054) 1198 : cluster 0 pgmap v1298: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 877 op/s; 36067/194424 objects degraded (18.551%); 2/194424 objects misplaced (0.001%) 2023-09-23T18:57:56.389 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:56 smithi098 bash[79516]: cluster 2023-09-23T18:57:55.386178+0000 mon.a (mon.0) 1631 : cluster 3 Health check update: Degraded data redundancy: 36067/194424 objects degraded (18.551%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:56.389 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:56 smithi098 bash[76817]: cluster 2023-09-23T18:57:54.340250+0000 mgr.y (mgr.26054) 1198 : cluster 0 pgmap v1298: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 877 op/s; 36067/194424 objects degraded (18.551%); 2/194424 objects misplaced (0.001%) 2023-09-23T18:57:56.389 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:56 smithi098 bash[76817]: cluster 2023-09-23T18:57:55.386178+0000 mon.a (mon.0) 1631 : cluster 3 Health check update: Degraded data redundancy: 36067/194424 objects degraded (18.551%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:56.402 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:56.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:56.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:56.506 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:56.536 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:56.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:56.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:56.639 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:56.667 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:56.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:56.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:56.770 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:56.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:56 smithi173 bash[59422]: cluster 2023-09-23T18:57:54.340250+0000 mgr.y (mgr.26054) 1198 : cluster 0 pgmap v1298: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 877 op/s; 36067/194424 objects degraded (18.551%); 2/194424 objects misplaced (0.001%) 2023-09-23T18:57:56.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:56 smithi173 bash[59422]: cluster 2023-09-23T18:57:55.386178+0000 mon.a (mon.0) 1631 : cluster 3 Health check update: Degraded data redundancy: 36067/194424 objects degraded (18.551%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:57:56.793 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:56.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:56.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:56.896 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:56.926 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:56.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:56.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:57.030 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:57.057 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:57.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:57.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:57.164 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:57.192 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:57.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:57.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:57.295 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:57.332 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:57.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:57.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:57.435 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:57.447 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:57 smithi173 bash[59422]: cluster 2023-09-23T18:57:56.341194+0000 mgr.y (mgr.26054) 1199 : cluster 0 pgmap v1299: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 35829/191589 objects degraded (18.701%); 2/191589 objects misplaced (0.001%) 2023-09-23T18:57:57.463 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:57.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:57.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:57.566 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:57.592 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:57.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:57.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:57.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:57 smithi098 bash[79516]: cluster 2023-09-23T18:57:56.341194+0000 mgr.y (mgr.26054) 1199 : cluster 0 pgmap v1299: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 35829/191589 objects degraded (18.701%); 2/191589 objects misplaced (0.001%) 2023-09-23T18:57:57.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:57 smithi098 bash[76817]: cluster 2023-09-23T18:57:56.341194+0000 mgr.y (mgr.26054) 1199 : cluster 0 pgmap v1299: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 35829/191589 objects degraded (18.701%); 2/191589 objects misplaced (0.001%) 2023-09-23T18:57:57.698 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:57.723 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:57.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:57.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:57.827 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:57.855 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:57.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:57.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:57.958 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:57.987 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:57.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:57.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:58.093 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:58.110 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:58.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:58.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:58.214 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:58.239 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:58.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:58.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:58.343 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:58.373 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:58.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:58.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:58.479 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:58.516 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:58.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:58.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:58.621 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:58.671 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:57:58.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:58.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:57:58.776 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:58.804 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:58.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:58.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:58.907 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:58.938 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:57:58.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:58.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:57:59.041 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:59.064 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:59.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:59.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:59.168 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:59.208 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:57:59.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:59.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:57:59.312 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:59.337 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:57:59.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:59.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:57:59.440 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:59.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:57:59 smithi098 bash[79516]: cluster 2023-09-23T18:57:58.342161+0000 mgr.y (mgr.26054) 1200 : cluster 0 pgmap v1300: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 34860/187932 objects degraded (18.549%); 2/187932 objects misplaced (0.001%) 2023-09-23T18:57:59.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:57:59 smithi098 bash[76817]: cluster 2023-09-23T18:57:58.342161+0000 mgr.y (mgr.26054) 1200 : cluster 0 pgmap v1300: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 34860/187932 objects degraded (18.549%); 2/187932 objects misplaced (0.001%) 2023-09-23T18:57:59.462 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:59.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:59.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:57:59.565 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:59.592 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:59.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:59.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:59.697 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:59.718 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:59.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:59.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:59.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:57:59 smithi173 bash[59422]: cluster 2023-09-23T18:57:58.342161+0000 mgr.y (mgr.26054) 1200 : cluster 0 pgmap v1300: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 34860/187932 objects degraded (18.549%); 2/187932 objects misplaced (0.001%) 2023-09-23T18:57:59.820 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:59.862 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:57:59.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:59.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:57:59.968 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:59.995 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:57:59.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:57:59.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:00.099 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:00.129 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:00.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:00.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:00.235 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:00.260 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:00.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:00.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:00.364 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:00.387 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:00.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:00.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:00.492 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:00.501 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:00 smithi173 bash[59422]: cluster 2023-09-23T18:58:00.387467+0000 mon.a (mon.0) 1632 : cluster 3 Health check update: Degraded data redundancy: 34860/187932 objects degraded (18.549%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:00.516 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:00.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:00.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:00.620 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:00.643 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:00.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:00.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:00.679 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:00 smithi098 bash[76817]: cluster 2023-09-23T18:58:00.387467+0000 mon.a (mon.0) 1632 : cluster 3 Health check update: Degraded data redundancy: 34860/187932 objects degraded (18.549%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:00.750 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:00.774 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:00.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:00.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:00.878 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:00.900 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:00.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:00.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:00.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:00 smithi098 bash[79516]: cluster 2023-09-23T18:58:00.387467+0000 mon.a (mon.0) 1632 : cluster 3 Health check update: Degraded data redundancy: 34860/187932 objects degraded (18.549%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:01.003 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:01.036 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:01.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:01.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:01.141 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:01.166 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:01.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:01.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:01.269 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:01.296 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:01.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:01.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:01.400 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:01.424 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:01.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:01.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:01.527 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:01.551 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:01.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:01.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:01.655 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:01.681 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:01.683 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:01 smithi173 bash[59422]: cluster 2023-09-23T18:58:00.343157+0000 mgr.y (mgr.26054) 1201 : cluster 0 pgmap v1301: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 33914/182907 objects degraded (18.542%); 2/182907 objects misplaced (0.001%) 2023-09-23T18:58:01.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:01.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:01.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:01 smithi098 bash[76817]: cluster 2023-09-23T18:58:00.343157+0000 mgr.y (mgr.26054) 1201 : cluster 0 pgmap v1301: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 33914/182907 objects degraded (18.542%); 2/182907 objects misplaced (0.001%) 2023-09-23T18:58:01.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:01 smithi098 bash[79516]: cluster 2023-09-23T18:58:00.343157+0000 mgr.y (mgr.26054) 1201 : cluster 0 pgmap v1301: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 33914/182907 objects degraded (18.542%); 2/182907 objects misplaced (0.001%) 2023-09-23T18:58:01.784 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:01.808 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:01.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:01.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:01.911 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:01.939 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:01.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:01.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:02.044 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:02.072 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:02.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:02.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:02.175 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:02.197 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:02.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:02.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:02.300 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:02.330 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:02.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:02.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:02.433 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:02.462 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:02.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:02.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:02.567 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:02.590 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:02.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:02.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:02.694 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:02.724 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:02.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:02.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:02.827 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:02.851 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:02.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:02.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:02.954 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:02.981 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:02.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:02.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:03.088 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:03.119 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:03.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:03.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:03.222 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:03.254 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:03.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:03.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:03.357 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:03.383 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:03.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:03.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:03.434 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:03 smithi098 bash[76817]: cluster 2023-09-23T18:58:02.346759+0000 mgr.y (mgr.26054) 1202 : cluster 0 pgmap v1302: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 33209/178578 objects degraded (18.596%); 2/178578 objects misplaced (0.001%) 2023-09-23T18:58:03.486 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:03.513 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:03.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:03.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:03.617 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:03.636 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:03.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:03.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:03.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:03 smithi098 bash[79516]: cluster 2023-09-23T18:58:02.346759+0000 mgr.y (mgr.26054) 1202 : cluster 0 pgmap v1302: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 33209/178578 objects degraded (18.596%); 2/178578 objects misplaced (0.001%) 2023-09-23T18:58:03.741 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:03.748 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:03 smithi173 bash[59422]: cluster 2023-09-23T18:58:02.346759+0000 mgr.y (mgr.26054) 1202 : cluster 0 pgmap v1302: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 33209/178578 objects degraded (18.596%); 2/178578 objects misplaced (0.001%) 2023-09-23T18:58:03.769 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:03.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:03.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:03.872 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:03.900 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:03.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:03.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:04.003 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:04.030 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:04.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:04.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:04.134 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:04.171 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:04.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:04.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:04.274 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:04.308 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:04.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:04.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:04.415 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:04.432 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:04.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:04.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:04.535 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:04.565 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:04.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:04.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:04.672 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:04.698 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:04.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:04.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:04.802 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:04.831 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:04.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:04.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:04.935 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:04.963 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:04.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:04.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:05.077 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:05.103 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:05.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:05.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:05.206 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:05.242 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:05.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:05.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:05.347 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:05.382 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:05.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:05.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:05.489 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:05.494 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:58:05.518 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:05.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:05.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:05.534 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:58:05.559 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:58:05.591 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:58:05.621 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:05.626 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:58:05.630 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:05 smithi098 bash[76817]: cluster 2023-09-23T18:58:04.347889+0000 mgr.y (mgr.26054) 1203 : cluster 0 pgmap v1303: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 864 op/s; 32023/172065 objects degraded (18.611%); 2/172065 objects misplaced (0.001%) 2023-09-23T18:58:05.630 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:05 smithi098 bash[76817]: cluster 2023-09-23T18:58:05.388708+0000 mon.a (mon.0) 1633 : cluster 3 Health check update: Degraded data redundancy: 32023/172065 objects degraded (18.611%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:05.631 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:05 smithi098 bash[79516]: cluster 2023-09-23T18:58:04.347889+0000 mgr.y (mgr.26054) 1203 : cluster 0 pgmap v1303: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 864 op/s; 32023/172065 objects degraded (18.611%); 2/172065 objects misplaced (0.001%) 2023-09-23T18:58:05.631 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:05 smithi098 bash[79516]: cluster 2023-09-23T18:58:05.388708+0000 mon.a (mon.0) 1633 : cluster 3 Health check update: Degraded data redundancy: 32023/172065 objects degraded (18.611%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:05.655 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:05.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:05.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:05.662 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:58:05.695 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:58:05.696 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:58:05.696 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:58:05.697 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:58:05.697 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:58:05.697 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:58:05.697 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:58:05.697 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:58:05.697 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T18:58:05.761 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:05.773 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:05 smithi173 bash[59422]: cluster 2023-09-23T18:58:04.347889+0000 mgr.y (mgr.26054) 1203 : cluster 0 pgmap v1303: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 864 op/s; 32023/172065 objects degraded (18.611%); 2/172065 objects misplaced (0.001%) 2023-09-23T18:58:05.774 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:05 smithi173 bash[59422]: cluster 2023-09-23T18:58:05.388708+0000 mon.a (mon.0) 1633 : cluster 3 Health check update: Degraded data redundancy: 32023/172065 objects degraded (18.611%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:05.789 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:05.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:05.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:05.892 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:05.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:05.926 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:05.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:06.027 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:06.054 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:06.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:06.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:06.157 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:06.166 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:58:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:58:06] "GET /metrics HTTP/1.1" 200 36590 "" "Prometheus/2.33.4" 2023-09-23T18:58:06.202 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:06.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:06.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:06.309 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:06.325 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:58:06.325 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:58:06.333 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:06.334 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:06 smithi098 bash[76817]: debug 2023-09-23T18:58:06.307+0000 7fc8a2a01700 -1 mon.a@0(leader).osd e292 definitely_dead 0 2023-09-23T18:58:06.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:06.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:06.436 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:06.472 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:06.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:06.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:06.477 INFO:teuthology.orchestra.run.smithi098.stderr:marked out osd.3. 2023-09-23T18:58:06.578 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:06.609 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:06.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:06.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:06.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:06 smithi098 bash[79516]: audit 2023-09-23T18:58:05.447665+0000 mon.b (mon.1) 26 : audit 0 from='client.? 172.21.15.98:0/2953454955' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:58:06.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:06 smithi098 bash[79516]: audit 2023-09-23T18:58:06.311024+0000 mon.a (mon.0) 1634 : audit 1 from='client.? 172.21.15.98:0/3427718463' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T18:58:06.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:06 smithi098 bash[79516]: cluster 2023-09-23T18:58:06.311096+0000 mon.a (mon.0) 1635 : cluster 1 Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T18:58:06.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:06 smithi098 bash[76817]: audit 2023-09-23T18:58:05.447665+0000 mon.b (mon.1) 26 : audit 0 from='client.? 172.21.15.98:0/2953454955' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:58:06.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:06 smithi098 bash[76817]: audit 2023-09-23T18:58:06.311024+0000 mon.a (mon.0) 1634 : audit 1 from='client.? 172.21.15.98:0/3427718463' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T18:58:06.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:06 smithi098 bash[76817]: cluster 2023-09-23T18:58:06.311096+0000 mon.a (mon.0) 1635 : cluster 1 Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T18:58:06.713 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:06.742 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:06.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:06.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:06.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:06 smithi173 bash[59422]: audit 2023-09-23T18:58:05.447665+0000 mon.b (mon.1) 26 : audit 0 from='client.? 172.21.15.98:0/2953454955' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:58:06.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:06 smithi173 bash[59422]: audit 2023-09-23T18:58:06.311024+0000 mon.a (mon.0) 1634 : audit 1 from='client.? 172.21.15.98:0/3427718463' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T18:58:06.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:06 smithi173 bash[59422]: cluster 2023-09-23T18:58:06.311096+0000 mon.a (mon.0) 1635 : cluster 1 Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T18:58:06.846 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:06.879 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:06.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:06.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:06.982 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:07.014 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:07.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:07.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:07.117 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:07.143 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:07.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:07.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:07.246 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:07.279 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:07.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:07.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:07.382 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:07.415 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:07.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:07.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:07.480 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is unset 2023-09-23T18:58:07.519 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:07.546 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:07.550 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:07 smithi098 bash[76817]: cluster 2023-09-23T18:58:06.348953+0000 mgr.y (mgr.26054) 1204 : cluster 0 pgmap v1304: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 31163/167391 objects degraded (18.617%); 2/167391 objects misplaced (0.001%) 2023-09-23T18:58:07.551 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:07 smithi098 bash[76817]: cluster 2023-09-23T18:58:06.454358+0000 osd.4 (osd.4) 12 : cluster 3 failed to encode map e293 with expected crc 2023-09-23T18:58:07.551 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:07 smithi098 bash[76817]: cluster 2023-09-23T18:58:06.454991+0000 osd.5 (osd.5) 208 : cluster 3 failed to encode map e293 with expected crc 2023-09-23T18:58:07.551 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:07 smithi098 bash[76817]: cluster 2023-09-23T18:58:06.456777+0000 osd.6 (osd.6) 262 : cluster 3 failed to encode map e293 with expected crc 2023-09-23T18:58:07.551 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:07 smithi098 bash[76817]: audit 2023-09-23T18:58:06.461649+0000 mon.a (mon.0) 1636 : audit 1 from='client.? 172.21.15.98:0/3427718463' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T18:58:07.551 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:07 smithi098 bash[76817]: cluster 2023-09-23T18:58:06.461991+0000 mon.a (mon.0) 1637 : cluster 0 osdmap e293: 8 total, 7 up, 7 in 2023-09-23T18:58:07.551 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:07 smithi098 bash[76817]: audit 2023-09-23T18:58:06.643269+0000 mon.a (mon.0) 1638 : audit 1 from='client.? 172.21.15.98:0/3814036173' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:58:07.551 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:07 smithi098 bash[79516]: cluster 2023-09-23T18:58:06.348953+0000 mgr.y (mgr.26054) 1204 : cluster 0 pgmap v1304: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 31163/167391 objects degraded (18.617%); 2/167391 objects misplaced (0.001%) 2023-09-23T18:58:07.551 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:07 smithi098 bash[79516]: cluster 2023-09-23T18:58:06.454358+0000 osd.4 (osd.4) 12 : cluster 3 failed to encode map e293 with expected crc 2023-09-23T18:58:07.551 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:07 smithi098 bash[79516]: cluster 2023-09-23T18:58:06.454991+0000 osd.5 (osd.5) 208 : cluster 3 failed to encode map e293 with expected crc 2023-09-23T18:58:07.551 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:07 smithi098 bash[79516]: cluster 2023-09-23T18:58:06.456777+0000 osd.6 (osd.6) 262 : cluster 3 failed to encode map e293 with expected crc 2023-09-23T18:58:07.552 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:07 smithi098 bash[79516]: audit 2023-09-23T18:58:06.461649+0000 mon.a (mon.0) 1636 : audit 1 from='client.? 172.21.15.98:0/3427718463' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T18:58:07.552 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:07 smithi098 bash[79516]: cluster 2023-09-23T18:58:06.461991+0000 mon.a (mon.0) 1637 : cluster 0 osdmap e293: 8 total, 7 up, 7 in 2023-09-23T18:58:07.552 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:07 smithi098 bash[79516]: audit 2023-09-23T18:58:06.643269+0000 mon.a (mon.0) 1638 : audit 1 from='client.? 172.21.15.98:0/3814036173' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:58:07.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:07.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:07.653 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:07.664 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:07 smithi173 bash[59422]: cluster 2023-09-23T18:58:06.348953+0000 mgr.y (mgr.26054) 1204 : cluster 0 pgmap v1304: 89 pgs: 2 active+clean+remapped, 54 active+clean, 18 active+undersized+degraded, 15 active+undersized; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 31163/167391 objects degraded (18.617%); 2/167391 objects misplaced (0.001%) 2023-09-23T18:58:07.664 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:07 smithi173 bash[59422]: cluster 2023-09-23T18:58:06.454358+0000 osd.4 (osd.4) 12 : cluster 3 failed to encode map e293 with expected crc 2023-09-23T18:58:07.664 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:07 smithi173 bash[59422]: cluster 2023-09-23T18:58:06.454991+0000 osd.5 (osd.5) 208 : cluster 3 failed to encode map e293 with expected crc 2023-09-23T18:58:07.664 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:07 smithi173 bash[59422]: cluster 2023-09-23T18:58:06.456777+0000 osd.6 (osd.6) 262 : cluster 3 failed to encode map e293 with expected crc 2023-09-23T18:58:07.664 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:07 smithi173 bash[59422]: audit 2023-09-23T18:58:06.461649+0000 mon.a (mon.0) 1636 : audit 1 from='client.? 172.21.15.98:0/3427718463' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T18:58:07.664 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:07 smithi173 bash[59422]: cluster 2023-09-23T18:58:06.461991+0000 mon.a (mon.0) 1637 : cluster 0 osdmap e293: 8 total, 7 up, 7 in 2023-09-23T18:58:07.665 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:07 smithi173 bash[59422]: audit 2023-09-23T18:58:06.643269+0000 mon.a (mon.0) 1638 : audit 1 from='client.? 172.21.15.98:0/3814036173' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:58:07.684 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:07.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:07.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:07.787 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:07.811 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:07.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:07.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:07.914 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:07.945 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:07.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:07.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:08.049 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:08.084 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:08.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:08.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:08.188 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:08.215 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:08.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:08.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:08.319 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:08.350 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:08.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:08.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:08.454 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:08.481 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[76817]: cluster 2023-09-23T18:58:07.451151+0000 mon.a (mon.0) 1639 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:58:08.481 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[76817]: cluster 2023-09-23T18:58:07.455866+0000 osd.5 (osd.5) 209 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.481 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[76817]: audit 2023-09-23T18:58:07.456756+0000 mon.a (mon.0) 1640 : audit 1 from='client.? 172.21.15.98:0/3814036173' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:58:08.486 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:08.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:08.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:08.589 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:08.599 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:08 smithi173 bash[59422]: cluster 2023-09-23T18:58:07.451151+0000 mon.a (mon.0) 1639 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:58:08.599 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:08 smithi173 bash[59422]: cluster 2023-09-23T18:58:07.455866+0000 osd.5 (osd.5) 209 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.600 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:08 smithi173 bash[59422]: audit 2023-09-23T18:58:07.456756+0000 mon.a (mon.0) 1640 : audit 1 from='client.? 172.21.15.98:0/3814036173' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:58:08.600 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:08 smithi173 bash[59422]: cluster 2023-09-23T18:58:07.457107+0000 mon.a (mon.0) 1641 : cluster 0 osdmap e294: 8 total, 7 up, 7 in 2023-09-23T18:58:08.600 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:08 smithi173 bash[59422]: cluster 2023-09-23T18:58:07.458694+0000 osd.4 (osd.4) 13 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.600 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:08 smithi173 bash[59422]: cluster 2023-09-23T18:58:07.459751+0000 osd.5 (osd.5) 210 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.600 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:08 smithi173 bash[59422]: cluster 2023-09-23T18:58:07.461027+0000 osd.6 (osd.6) 263 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.600 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:08 smithi173 bash[59422]: cluster 2023-09-23T18:58:07.461228+0000 osd.5 (osd.5) 211 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.600 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:08 smithi173 bash[59422]: cluster 2023-09-23T18:58:07.463478+0000 osd.4 (osd.4) 14 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.600 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:08 smithi173 bash[59422]: cluster 2023-09-23T18:58:07.465714+0000 osd.6 (osd.6) 264 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.600 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:08 smithi173 bash[59422]: cluster 2023-09-23T18:58:07.466686+0000 osd.4 (osd.4) 15 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.600 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:08 smithi173 bash[59422]: cluster 2023-09-23T18:58:07.468370+0000 osd.4 (osd.4) 16 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.601 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:08 smithi173 bash[59422]: cluster 2023-09-23T18:58:07.469008+0000 osd.6 (osd.6) 265 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.601 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:08 smithi173 bash[59422]: cluster 2023-09-23T18:58:07.471192+0000 osd.4 (osd.4) 17 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.601 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:08 smithi173 bash[59422]: cluster 2023-09-23T18:58:08.464582+0000 mon.a (mon.0) 1642 : cluster 0 osdmap e295: 8 total, 7 up, 7 in 2023-09-23T18:58:08.633 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:08.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:08.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:08.736 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:08.744 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[79516]: cluster 2023-09-23T18:58:07.451151+0000 mon.a (mon.0) 1639 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:58:08.745 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[79516]: cluster 2023-09-23T18:58:07.455866+0000 osd.5 (osd.5) 209 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.745 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[79516]: audit 2023-09-23T18:58:07.456756+0000 mon.a (mon.0) 1640 : audit 1 from='client.? 172.21.15.98:0/3814036173' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:58:08.745 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[79516]: cluster 2023-09-23T18:58:07.457107+0000 mon.a (mon.0) 1641 : cluster 0 osdmap e294: 8 total, 7 up, 7 in 2023-09-23T18:58:08.745 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[79516]: cluster 2023-09-23T18:58:07.458694+0000 osd.4 (osd.4) 13 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.745 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[79516]: cluster 2023-09-23T18:58:07.459751+0000 osd.5 (osd.5) 210 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.745 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[79516]: cluster 2023-09-23T18:58:07.461027+0000 osd.6 (osd.6) 263 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.745 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[79516]: cluster 2023-09-23T18:58:07.461228+0000 osd.5 (osd.5) 211 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.745 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[79516]: cluster 2023-09-23T18:58:07.463478+0000 osd.4 (osd.4) 14 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.745 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[79516]: cluster 2023-09-23T18:58:07.465714+0000 osd.6 (osd.6) 264 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.745 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[79516]: cluster 2023-09-23T18:58:07.466686+0000 osd.4 (osd.4) 15 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.746 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[79516]: cluster 2023-09-23T18:58:07.468370+0000 osd.4 (osd.4) 16 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.746 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[79516]: cluster 2023-09-23T18:58:07.469008+0000 osd.6 (osd.6) 265 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.746 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[79516]: cluster 2023-09-23T18:58:07.471192+0000 osd.4 (osd.4) 17 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.746 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[79516]: cluster 2023-09-23T18:58:08.464582+0000 mon.a (mon.0) 1642 : cluster 0 osdmap e295: 8 total, 7 up, 7 in 2023-09-23T18:58:08.746 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[76817]: cluster 2023-09-23T18:58:07.457107+0000 mon.a (mon.0) 1641 : cluster 0 osdmap e294: 8 total, 7 up, 7 in 2023-09-23T18:58:08.746 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[76817]: cluster 2023-09-23T18:58:07.458694+0000 osd.4 (osd.4) 13 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.747 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[76817]: cluster 2023-09-23T18:58:07.459751+0000 osd.5 (osd.5) 210 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.747 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[76817]: cluster 2023-09-23T18:58:07.461027+0000 osd.6 (osd.6) 263 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.747 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[76817]: cluster 2023-09-23T18:58:07.461228+0000 osd.5 (osd.5) 211 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.747 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[76817]: cluster 2023-09-23T18:58:07.463478+0000 osd.4 (osd.4) 14 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.747 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[76817]: cluster 2023-09-23T18:58:07.465714+0000 osd.6 (osd.6) 264 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.748 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[76817]: cluster 2023-09-23T18:58:07.466686+0000 osd.4 (osd.4) 15 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.748 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[76817]: cluster 2023-09-23T18:58:07.468370+0000 osd.4 (osd.4) 16 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.748 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[76817]: cluster 2023-09-23T18:58:07.469008+0000 osd.6 (osd.6) 265 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.748 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[76817]: cluster 2023-09-23T18:58:07.471192+0000 osd.4 (osd.4) 17 : cluster 3 failed to encode map e294 with expected crc 2023-09-23T18:58:08.748 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:08 smithi098 bash[76817]: cluster 2023-09-23T18:58:08.464582+0000 mon.a (mon.0) 1642 : cluster 0 osdmap e295: 8 total, 7 up, 7 in 2023-09-23T18:58:08.765 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:08.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:08.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:08.868 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:08.893 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:08.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:08.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:08.997 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:09.025 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:09.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:09.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:09.129 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:09.148 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:09.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:09.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:09.251 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:09.275 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:09.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:09.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:09.377 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:09.402 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:09.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:09.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:09.505 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:09.526 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:09.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:09.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:09.629 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:09.641 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:09 smithi173 bash[59422]: cluster 2023-09-23T18:58:08.350088+0000 mgr.y (mgr.26054) 1205 : cluster 0 pgmap v1307: 89 pgs: 2 remapped+peering, 7 peering, 2 active+clean+remapped, 47 active+clean, 17 active+undersized+degraded, 14 active+undersized; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 31019/166566 objects degraded (18.623%); 2/166566 objects misplaced (0.001%); 0 B/s, 0 objects/s recovering 2023-09-23T18:58:09.641 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:09 smithi173 bash[59422]: cluster 2023-09-23T18:58:08.463339+0000 osd.1 (osd.1) 6 : cluster 0 9.a starting backfill to osd.2 from (0'0,0'0] MAX to 292'8438 2023-09-23T18:58:09.641 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:09 smithi173 bash[59422]: cluster 2023-09-23T18:58:08.466532+0000 osd.1 (osd.1) 7 : cluster 0 9.4 starting backfill to osd.0 from (0'0,0'0] MAX to 292'8634 2023-09-23T18:58:09.641 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:09 smithi173 bash[59422]: cluster 2023-09-23T18:58:08.467040+0000 mon.a (mon.0) 1643 : cluster 3 Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T18:58:09.642 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:09 smithi173 bash[59422]: cluster 2023-09-23T18:58:08.467133+0000 osd.5 (osd.5) 212 : cluster 3 failed to encode map e295 with expected crc 2023-09-23T18:58:09.642 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:09 smithi173 bash[59422]: cluster 2023-09-23T18:58:08.467234+0000 osd.4 (osd.4) 18 : cluster 3 failed to encode map e295 with expected crc 2023-09-23T18:58:09.642 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:09 smithi173 bash[59422]: cluster 2023-09-23T18:58:08.469846+0000 osd.1 (osd.1) 8 : cluster 0 9.5 starting backfill to osd.6 from (0'0,0'0] MAX to 292'8641 2023-09-23T18:58:09.642 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:09 smithi173 bash[59422]: cluster 2023-09-23T18:58:08.469982+0000 osd.4 (osd.4) 19 : cluster 3 failed to encode map e295 with expected crc 2023-09-23T18:58:09.642 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:09 smithi173 bash[59422]: cluster 2023-09-23T18:58:08.470326+0000 osd.5 (osd.5) 213 : cluster 3 failed to encode map e295 with expected crc 2023-09-23T18:58:09.642 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:09 smithi173 bash[59422]: cluster 2023-09-23T18:58:08.470590+0000 osd.6 (osd.6) 266 : cluster 3 failed to encode map e295 with expected crc 2023-09-23T18:58:09.656 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:09.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:09.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:09.759 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:09.787 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:09.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:09.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:09.893 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:09.900 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:09 smithi098 bash[79516]: cluster 2023-09-23T18:58:08.350088+0000 mgr.y (mgr.26054) 1205 : cluster 0 pgmap v1307: 89 pgs: 2 remapped+peering, 7 peering, 2 active+clean+remapped, 47 active+clean, 17 active+undersized+degraded, 14 active+undersized; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 31019/166566 objects degraded (18.623%); 2/166566 objects misplaced (0.001%); 0 B/s, 0 objects/s recovering 2023-09-23T18:58:09.900 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:09 smithi098 bash[79516]: cluster 2023-09-23T18:58:08.463339+0000 osd.1 (osd.1) 6 : cluster 0 9.a starting backfill to osd.2 from (0'0,0'0] MAX to 292'8438 2023-09-23T18:58:09.900 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:09 smithi098 bash[79516]: cluster 2023-09-23T18:58:08.466532+0000 osd.1 (osd.1) 7 : cluster 0 9.4 starting backfill to osd.0 from (0'0,0'0] MAX to 292'8634 2023-09-23T18:58:09.901 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:09 smithi098 bash[79516]: cluster 2023-09-23T18:58:08.467040+0000 mon.a (mon.0) 1643 : cluster 3 Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T18:58:09.901 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:09 smithi098 bash[79516]: cluster 2023-09-23T18:58:08.467133+0000 osd.5 (osd.5) 212 : cluster 3 failed to encode map e295 with expected crc 2023-09-23T18:58:09.901 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:09 smithi098 bash[79516]: cluster 2023-09-23T18:58:08.467234+0000 osd.4 (osd.4) 18 : cluster 3 failed to encode map e295 with expected crc 2023-09-23T18:58:09.901 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:09 smithi098 bash[79516]: cluster 2023-09-23T18:58:08.469846+0000 osd.1 (osd.1) 8 : cluster 0 9.5 starting backfill to osd.6 from (0'0,0'0] MAX to 292'8641 2023-09-23T18:58:09.901 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:09 smithi098 bash[79516]: cluster 2023-09-23T18:58:08.469982+0000 osd.4 (osd.4) 19 : cluster 3 failed to encode map e295 with expected crc 2023-09-23T18:58:09.901 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:09 smithi098 bash[79516]: cluster 2023-09-23T18:58:08.470326+0000 osd.5 (osd.5) 213 : cluster 3 failed to encode map e295 with expected crc 2023-09-23T18:58:09.901 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:09 smithi098 bash[79516]: cluster 2023-09-23T18:58:08.470590+0000 osd.6 (osd.6) 266 : cluster 3 failed to encode map e295 with expected crc 2023-09-23T18:58:09.901 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:09 smithi098 bash[76817]: cluster 2023-09-23T18:58:08.350088+0000 mgr.y (mgr.26054) 1205 : cluster 0 pgmap v1307: 89 pgs: 2 remapped+peering, 7 peering, 2 active+clean+remapped, 47 active+clean, 17 active+undersized+degraded, 14 active+undersized; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 31019/166566 objects degraded (18.623%); 2/166566 objects misplaced (0.001%); 0 B/s, 0 objects/s recovering 2023-09-23T18:58:09.902 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:09 smithi098 bash[76817]: cluster 2023-09-23T18:58:08.463339+0000 osd.1 (osd.1) 6 : cluster 0 9.a starting backfill to osd.2 from (0'0,0'0] MAX to 292'8438 2023-09-23T18:58:09.902 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:09 smithi098 bash[76817]: cluster 2023-09-23T18:58:08.466532+0000 osd.1 (osd.1) 7 : cluster 0 9.4 starting backfill to osd.0 from (0'0,0'0] MAX to 292'8634 2023-09-23T18:58:09.902 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:09 smithi098 bash[76817]: cluster 2023-09-23T18:58:08.467040+0000 mon.a (mon.0) 1643 : cluster 3 Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T18:58:09.902 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:09 smithi098 bash[76817]: cluster 2023-09-23T18:58:08.467133+0000 osd.5 (osd.5) 212 : cluster 3 failed to encode map e295 with expected crc 2023-09-23T18:58:09.902 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:09 smithi098 bash[76817]: cluster 2023-09-23T18:58:08.467234+0000 osd.4 (osd.4) 18 : cluster 3 failed to encode map e295 with expected crc 2023-09-23T18:58:09.902 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:09 smithi098 bash[76817]: cluster 2023-09-23T18:58:08.469846+0000 osd.1 (osd.1) 8 : cluster 0 9.5 starting backfill to osd.6 from (0'0,0'0] MAX to 292'8641 2023-09-23T18:58:09.902 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:09 smithi098 bash[76817]: cluster 2023-09-23T18:58:08.469982+0000 osd.4 (osd.4) 19 : cluster 3 failed to encode map e295 with expected crc 2023-09-23T18:58:09.902 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:09 smithi098 bash[76817]: cluster 2023-09-23T18:58:08.470326+0000 osd.5 (osd.5) 213 : cluster 3 failed to encode map e295 with expected crc 2023-09-23T18:58:09.903 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:09 smithi098 bash[76817]: cluster 2023-09-23T18:58:08.470590+0000 osd.6 (osd.6) 266 : cluster 3 failed to encode map e295 with expected crc 2023-09-23T18:58:09.912 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:09.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:09.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:09.939 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:58:09.940 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:58:10.016 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:10.037 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:10.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:10.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:10.142 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:10.161 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:10.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:10.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:10.264 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:10.286 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:10.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:10.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:10.389 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:10.412 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:10.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:10.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:10.517 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:10.536 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:10.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:10.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:10.639 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:10.661 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:10.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:10.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:10.764 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:10.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:10 smithi173 bash[59422]: cluster 2023-09-23T18:58:09.487797+0000 osd.6 (osd.6) 267 : cluster 3 failed to encode map e296 with expected crc 2023-09-23T18:58:10.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:10 smithi173 bash[59422]: cluster 2023-09-23T18:58:09.488361+0000 mon.a (mon.0) 1644 : cluster 0 osdmap e296: 8 total, 7 up, 7 in 2023-09-23T18:58:10.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:10 smithi173 bash[59422]: cluster 2023-09-23T18:58:09.489041+0000 osd.4 (osd.4) 20 : cluster 3 failed to encode map e296 with expected crc 2023-09-23T18:58:10.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:10 smithi173 bash[59422]: cluster 2023-09-23T18:58:09.508136+0000 osd.5 (osd.5) 214 : cluster 3 failed to encode map e296 with expected crc 2023-09-23T18:58:10.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:10 smithi173 bash[59422]: cluster 2023-09-23T18:58:09.546826+0000 osd.4 (osd.4) 21 : cluster 3 failed to encode map e296 with expected crc 2023-09-23T18:58:10.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:10 smithi173 bash[59422]: audit 2023-09-23T18:58:09.637006+0000 mon.a (mon.0) 1645 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:10.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:10 smithi173 bash[59422]: audit 2023-09-23T18:58:09.638162+0000 mon.a (mon.0) 1646 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:58:10.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:10 smithi173 bash[59422]: cluster 2023-09-23T18:58:10.390092+0000 mon.a (mon.0) 1647 : cluster 3 Health check update: Degraded data redundancy: 31019/166566 objects degraded (18.623%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:10.791 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:10.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:10.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:10.894 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:10.915 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:10.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:10.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:10.969 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:10 smithi098 bash[79516]: cluster 2023-09-23T18:58:09.487797+0000 osd.6 (osd.6) 267 : cluster 3 failed to encode map e296 with expected crc 2023-09-23T18:58:10.969 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:10 smithi098 bash[79516]: cluster 2023-09-23T18:58:09.488361+0000 mon.a (mon.0) 1644 : cluster 0 osdmap e296: 8 total, 7 up, 7 in 2023-09-23T18:58:10.969 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:10 smithi098 bash[79516]: cluster 2023-09-23T18:58:09.489041+0000 osd.4 (osd.4) 20 : cluster 3 failed to encode map e296 with expected crc 2023-09-23T18:58:10.969 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:10 smithi098 bash[79516]: cluster 2023-09-23T18:58:09.508136+0000 osd.5 (osd.5) 214 : cluster 3 failed to encode map e296 with expected crc 2023-09-23T18:58:10.969 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:10 smithi098 bash[79516]: cluster 2023-09-23T18:58:09.546826+0000 osd.4 (osd.4) 21 : cluster 3 failed to encode map e296 with expected crc 2023-09-23T18:58:10.969 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:10 smithi098 bash[79516]: audit 2023-09-23T18:58:09.637006+0000 mon.a (mon.0) 1645 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:10.969 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:10 smithi098 bash[79516]: audit 2023-09-23T18:58:09.638162+0000 mon.a (mon.0) 1646 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:58:10.969 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:10 smithi098 bash[79516]: cluster 2023-09-23T18:58:10.390092+0000 mon.a (mon.0) 1647 : cluster 3 Health check update: Degraded data redundancy: 31019/166566 objects degraded (18.623%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:10.969 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:10 smithi098 bash[76817]: cluster 2023-09-23T18:58:09.487797+0000 osd.6 (osd.6) 267 : cluster 3 failed to encode map e296 with expected crc 2023-09-23T18:58:10.969 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:10 smithi098 bash[76817]: cluster 2023-09-23T18:58:09.488361+0000 mon.a (mon.0) 1644 : cluster 0 osdmap e296: 8 total, 7 up, 7 in 2023-09-23T18:58:10.970 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:10 smithi098 bash[76817]: cluster 2023-09-23T18:58:09.489041+0000 osd.4 (osd.4) 20 : cluster 3 failed to encode map e296 with expected crc 2023-09-23T18:58:10.970 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:10 smithi098 bash[76817]: cluster 2023-09-23T18:58:09.508136+0000 osd.5 (osd.5) 214 : cluster 3 failed to encode map e296 with expected crc 2023-09-23T18:58:10.970 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:10 smithi098 bash[76817]: cluster 2023-09-23T18:58:09.546826+0000 osd.4 (osd.4) 21 : cluster 3 failed to encode map e296 with expected crc 2023-09-23T18:58:10.970 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:10 smithi098 bash[76817]: audit 2023-09-23T18:58:09.637006+0000 mon.a (mon.0) 1645 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:10.970 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:10 smithi098 bash[76817]: audit 2023-09-23T18:58:09.638162+0000 mon.a (mon.0) 1646 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:58:10.970 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:10 smithi098 bash[76817]: cluster 2023-09-23T18:58:10.390092+0000 mon.a (mon.0) 1647 : cluster 3 Health check update: Degraded data redundancy: 31019/166566 objects degraded (18.623%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:10.996 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T18:58:11.018 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:11.036 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:11.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:11.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:11.139 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:11.158 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:11.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:11.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:11.260 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:11.283 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:11.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:11.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:11.387 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:11.407 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:11.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:11.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:11.510 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:11.536 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:11.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:11.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:11.640 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:11.666 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:11.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:11.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:11.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:11 smithi098 bash[79516]: cluster 2023-09-23T18:58:10.350704+0000 mgr.y (mgr.26054) 1206 : cluster 0 pgmap v1310: 89 pgs: 2 remapped+peering, 7 peering, 2 active+clean+remapped, 47 active+clean, 17 active+undersized+degraded, 14 active+undersized; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 68 op/s; 31019/166566 objects degraded (18.623%); 2/166566 objects misplaced (0.001%); 0 B/s, 0 objects/s recovering 2023-09-23T18:58:11.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:11 smithi098 bash[79516]: cluster 2023-09-23T18:58:10.494604+0000 mon.a (mon.0) 1648 : cluster 0 osdmap e297: 8 total, 7 up, 7 in 2023-09-23T18:58:11.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:11 smithi098 bash[79516]: cluster 2023-09-23T18:58:10.494690+0000 osd.6 (osd.6) 268 : cluster 3 failed to encode map e297 with expected crc 2023-09-23T18:58:11.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:11 smithi098 bash[79516]: cluster 2023-09-23T18:58:10.496083+0000 osd.4 (osd.4) 22 : cluster 3 failed to encode map e297 with expected crc 2023-09-23T18:58:11.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:11 smithi098 bash[79516]: cluster 2023-09-23T18:58:10.499390+0000 osd.6 (osd.6) 269 : cluster 3 failed to encode map e297 with expected crc 2023-09-23T18:58:11.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:11 smithi098 bash[79516]: cluster 2023-09-23T18:58:10.504432+0000 osd.5 (osd.5) 215 : cluster 3 failed to encode map e297 with expected crc 2023-09-23T18:58:11.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:11 smithi098 bash[76817]: cluster 2023-09-23T18:58:10.350704+0000 mgr.y (mgr.26054) 1206 : cluster 0 pgmap v1310: 89 pgs: 2 remapped+peering, 7 peering, 2 active+clean+remapped, 47 active+clean, 17 active+undersized+degraded, 14 active+undersized; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 68 op/s; 31019/166566 objects degraded (18.623%); 2/166566 objects misplaced (0.001%); 0 B/s, 0 objects/s recovering 2023-09-23T18:58:11.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:11 smithi098 bash[76817]: cluster 2023-09-23T18:58:10.494604+0000 mon.a (mon.0) 1648 : cluster 0 osdmap e297: 8 total, 7 up, 7 in 2023-09-23T18:58:11.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:11 smithi098 bash[76817]: cluster 2023-09-23T18:58:10.494690+0000 osd.6 (osd.6) 268 : cluster 3 failed to encode map e297 with expected crc 2023-09-23T18:58:11.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:11 smithi098 bash[76817]: cluster 2023-09-23T18:58:10.496083+0000 osd.4 (osd.4) 22 : cluster 3 failed to encode map e297 with expected crc 2023-09-23T18:58:11.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:11 smithi098 bash[76817]: cluster 2023-09-23T18:58:10.499390+0000 osd.6 (osd.6) 269 : cluster 3 failed to encode map e297 with expected crc 2023-09-23T18:58:11.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:11 smithi098 bash[76817]: cluster 2023-09-23T18:58:10.504432+0000 osd.5 (osd.5) 215 : cluster 3 failed to encode map e297 with expected crc 2023-09-23T18:58:11.769 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:11.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:11 smithi173 bash[59422]: cluster 2023-09-23T18:58:10.350704+0000 mgr.y (mgr.26054) 1206 : cluster 0 pgmap v1310: 89 pgs: 2 remapped+peering, 7 peering, 2 active+clean+remapped, 47 active+clean, 17 active+undersized+degraded, 14 active+undersized; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 68 op/s; 31019/166566 objects degraded (18.623%); 2/166566 objects misplaced (0.001%); 0 B/s, 0 objects/s recovering 2023-09-23T18:58:11.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:11 smithi173 bash[59422]: cluster 2023-09-23T18:58:10.494604+0000 mon.a (mon.0) 1648 : cluster 0 osdmap e297: 8 total, 7 up, 7 in 2023-09-23T18:58:11.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:11 smithi173 bash[59422]: cluster 2023-09-23T18:58:10.494690+0000 osd.6 (osd.6) 268 : cluster 3 failed to encode map e297 with expected crc 2023-09-23T18:58:11.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:11 smithi173 bash[59422]: cluster 2023-09-23T18:58:10.496083+0000 osd.4 (osd.4) 22 : cluster 3 failed to encode map e297 with expected crc 2023-09-23T18:58:11.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:11 smithi173 bash[59422]: cluster 2023-09-23T18:58:10.499390+0000 osd.6 (osd.6) 269 : cluster 3 failed to encode map e297 with expected crc 2023-09-23T18:58:11.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:11 smithi173 bash[59422]: cluster 2023-09-23T18:58:10.504432+0000 osd.5 (osd.5) 215 : cluster 3 failed to encode map e297 with expected crc 2023-09-23T18:58:11.790 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:11.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:11.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:11.894 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:11.921 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:11.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:11.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:11.941 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:58:12.024 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:12.051 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:12.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:12.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:12.154 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:12.176 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:12.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:12.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:12.280 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:12.305 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:12.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:12.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:12.409 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:12.435 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:12.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:12.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:12.538 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:12.548 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:12 smithi098 bash[79516]: cluster 2023-09-23T18:58:11.020334+0000 osd.4 (osd.4) 23 : cluster 3 failed to encode map e297 with expected crc 2023-09-23T18:58:12.548 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:12 smithi098 bash[76817]: cluster 2023-09-23T18:58:11.020334+0000 osd.4 (osd.4) 23 : cluster 3 failed to encode map e297 with expected crc 2023-09-23T18:58:12.562 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:12.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:12.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:12.666 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:12.676 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:12 smithi173 bash[59422]: cluster 2023-09-23T18:58:11.020334+0000 osd.4 (osd.4) 23 : cluster 3 failed to encode map e297 with expected crc 2023-09-23T18:58:12.696 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:12.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:12.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:12.799 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:12.818 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:12.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:12.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:12.921 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:12.953 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:12.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:12.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:13.058 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:13.091 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:13.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:13.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:13.194 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:13.218 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:13.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:13.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:13.321 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:13.349 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:13.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:13.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:13.452 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:13.480 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:13.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:13.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:13.583 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:13.595 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:13 smithi173 bash[59422]: cluster 2023-09-23T18:58:12.351670+0000 mgr.y (mgr.26054) 1207 : cluster 0 pgmap v1312: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 remapped+peering, 3 active+clean+remapped, 53 active+clean, 16 active+undersized+degraded, 13 active+undersized; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 30988/166131 objects degraded (18.653%); 6623/166131 objects misplaced (3.987%); 2.7 MiB/s, 48 objects/s recovering 2023-09-23T18:58:13.610 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:13.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:13.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:13.713 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:13.741 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:13.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:13.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:13.816 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:58:13.828 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:13 smithi098 bash[79516]: cluster 2023-09-23T18:58:12.351670+0000 mgr.y (mgr.26054) 1207 : cluster 0 pgmap v1312: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 remapped+peering, 3 active+clean+remapped, 53 active+clean, 16 active+undersized+degraded, 13 active+undersized; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 30988/166131 objects degraded (18.653%); 6623/166131 objects misplaced (3.987%); 2.7 MiB/s, 48 objects/s recovering 2023-09-23T18:58:13.828 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:13 smithi098 bash[76817]: cluster 2023-09-23T18:58:12.351670+0000 mgr.y (mgr.26054) 1207 : cluster 0 pgmap v1312: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 remapped+peering, 3 active+clean+remapped, 53 active+clean, 16 active+undersized+degraded, 13 active+undersized; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 30988/166131 objects degraded (18.653%); 6623/166131 objects misplaced (3.987%); 2.7 MiB/s, 48 objects/s recovering 2023-09-23T18:58:13.845 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:13.872 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:13.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:13.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:13.975 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:13.997 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:14.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:14.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:14.109 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:14.135 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:14.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:14.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:14.239 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:14.264 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:14.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:14.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:14.367 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:14.394 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:14.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:14.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:14.498 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:14.523 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:14.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:14.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:14.626 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:14.654 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:14.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:14.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:14.757 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:14.768 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:14 smithi173 bash[59422]: cluster 2023-09-23T18:58:14.500648+0000 mon.a (mon.0) 1649 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T18:58:14.785 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:14.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:14.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:14.888 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:14.899 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:14 smithi098 bash[79516]: cluster 2023-09-23T18:58:14.500648+0000 mon.a (mon.0) 1649 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T18:58:14.899 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:14 smithi098 bash[76817]: cluster 2023-09-23T18:58:14.500648+0000 mon.a (mon.0) 1649 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T18:58:14.915 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:14.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:14.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:15.021 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:15.047 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:15.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:15.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:15.150 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:15.177 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:15.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:15.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:15.280 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:15.305 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:15.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:15.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:15.408 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:15.429 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:15.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:15.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:15.533 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:15.557 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:15.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:15.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:15.660 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:15.688 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:15.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:15.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:15.757 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:58:15.791 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:15.799 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:15 smithi173 bash[59422]: cluster 2023-09-23T18:58:14.352454+0000 mgr.y (mgr.26054) 1208 : cluster 0 pgmap v1313: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+clean+remapped, 53 active+clean, 16 active+undersized+degraded, 13 active+undersized; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 24 op/s; 30987/166122 objects degraded (18.653%); 6624/166122 objects misplaced (3.987%); 2.2 MiB/s, 39 objects/s recovering 2023-09-23T18:58:15.799 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:15 smithi173 bash[59422]: cluster 2023-09-23T18:58:15.391043+0000 mon.a (mon.0) 1650 : cluster 3 Health check update: Degraded data redundancy: 30987/166122 objects degraded (18.653%), 18 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:15.813 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:15.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:15.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:15.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:15 smithi098 bash[76817]: cluster 2023-09-23T18:58:14.352454+0000 mgr.y (mgr.26054) 1208 : cluster 0 pgmap v1313: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+clean+remapped, 53 active+clean, 16 active+undersized+degraded, 13 active+undersized; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 24 op/s; 30987/166122 objects degraded (18.653%); 6624/166122 objects misplaced (3.987%); 2.2 MiB/s, 39 objects/s recovering 2023-09-23T18:58:15.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:15 smithi098 bash[76817]: cluster 2023-09-23T18:58:15.391043+0000 mon.a (mon.0) 1650 : cluster 3 Health check update: Degraded data redundancy: 30987/166122 objects degraded (18.653%), 18 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:15.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:15 smithi098 bash[79516]: cluster 2023-09-23T18:58:14.352454+0000 mgr.y (mgr.26054) 1208 : cluster 0 pgmap v1313: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+clean+remapped, 53 active+clean, 16 active+undersized+degraded, 13 active+undersized; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 24 op/s; 30987/166122 objects degraded (18.653%); 6624/166122 objects misplaced (3.987%); 2.2 MiB/s, 39 objects/s recovering 2023-09-23T18:58:15.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:15 smithi098 bash[79516]: cluster 2023-09-23T18:58:15.391043+0000 mon.a (mon.0) 1650 : cluster 3 Health check update: Degraded data redundancy: 30987/166122 objects degraded (18.653%), 18 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:15.945 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:15.968 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:15.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:15.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:16.070 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:16.096 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:16.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:16.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:16.199 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:16.225 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:16.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:16.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:16.330 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:16.357 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:16.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:16.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:16.443 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:58:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:58:16] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.33.4" 2023-09-23T18:58:16.461 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:16.487 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:16.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:16.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:16.590 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:16.617 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:16.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:16.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:16.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:16.721 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:58:16.747 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:16.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:16.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:16.776 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 6, 7, 5] out_osds: [3] dead_osds: [7] live_osds: [0, 2, 3, 5, 6, 1, 4] 2023-09-23T18:58:16.776 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:58:16.776 INFO:tasks.thrashosds.thrasher:Adding osd 3 2023-09-23T18:58:16.776 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd in 3 2023-09-23T18:58:16.850 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:16.871 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:16.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:16.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:16.974 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:16.998 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:17.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:17.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:17.101 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:17.123 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:17.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:17.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:17.226 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:17.252 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:17.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:17.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:17.355 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:17.380 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:17.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:17.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:17.484 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:17.507 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:17.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:17.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:17.610 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:17.617 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:17 smithi098 bash[79516]: cluster 2023-09-23T18:58:16.353446+0000 mgr.y (mgr.26054) 1209 : cluster 0 pgmap v1314: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+clean+remapped, 53 active+clean, 16 active+undersized+degraded, 13 active+undersized; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 18 op/s; 30987/166122 objects degraded (18.653%); 6142/166122 objects misplaced (3.697%); 5.5 MiB/s, 90 objects/s recovering 2023-09-23T18:58:17.617 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:17 smithi098 bash[76817]: cluster 2023-09-23T18:58:16.353446+0000 mgr.y (mgr.26054) 1209 : cluster 0 pgmap v1314: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+clean+remapped, 53 active+clean, 16 active+undersized+degraded, 13 active+undersized; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 18 op/s; 30987/166122 objects degraded (18.653%); 6142/166122 objects misplaced (3.697%); 5.5 MiB/s, 90 objects/s recovering 2023-09-23T18:58:17.638 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:17.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:17.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:17.733 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:58:17.742 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:17.768 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:17.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:17.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:17.871 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:17.896 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:17.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:17.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:17.999 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:18.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:18.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:18.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:18.094 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:17 smithi173 bash[59422]: cluster 2023-09-23T18:58:16.353446+0000 mgr.y (mgr.26054) 1209 : cluster 0 pgmap v1314: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+clean+remapped, 53 active+clean, 16 active+undersized+degraded, 13 active+undersized; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 18 op/s; 30987/166122 objects degraded (18.653%); 6142/166122 objects misplaced (3.697%); 5.5 MiB/s, 90 objects/s recovering 2023-09-23T18:58:18.118 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:18.142 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:18.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:18.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:18.246 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:18.280 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:18.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:18.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:18.386 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:18.424 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:18.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:18.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:18.529 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:18.552 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:18.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:18.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:18.655 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:18.681 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:18.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:18.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:18.788 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:18.807 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:18.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:18.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:18.910 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:18.937 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:18.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:18.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:19.043 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:19.072 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:19.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:19.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:19.175 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:19.198 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:19.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:19.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:19.302 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:19.329 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:19.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:19.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:19.432 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:19.457 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:19.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:19.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:19.560 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:19.585 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:19.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:19.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:19.689 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:19.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:19 smithi098 bash[76817]: cluster 2023-09-23T18:58:18.354446+0000 mgr.y (mgr.26054) 1210 : cluster 0 pgmap v1315: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 activating+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 111 op/s; 30453/163593 objects degraded (18.615%); 28265/163593 objects misplaced (17.278%); 5.4 MiB/s, 88 objects/s recovering 2023-09-23T18:58:19.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:19 smithi098 bash[79516]: cluster 2023-09-23T18:58:18.354446+0000 mgr.y (mgr.26054) 1210 : cluster 0 pgmap v1315: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 activating+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 111 op/s; 30453/163593 objects degraded (18.615%); 28265/163593 objects misplaced (17.278%); 5.4 MiB/s, 88 objects/s recovering 2023-09-23T18:58:19.718 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:19.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:19.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:19.826 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:19.834 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:19 smithi173 bash[59422]: cluster 2023-09-23T18:58:18.354446+0000 mgr.y (mgr.26054) 1210 : cluster 0 pgmap v1315: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 activating+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 111 op/s; 30453/163593 objects degraded (18.615%); 28265/163593 objects misplaced (17.278%); 5.4 MiB/s, 88 objects/s recovering 2023-09-23T18:58:19.850 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:19.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:19.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:19.953 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:19.978 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:19.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:19.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:20.084 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:20.106 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:20.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:20.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:20.212 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:20.235 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:20.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:20.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:20.338 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:20.371 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:20.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:20.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:20.477 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:20.507 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:20.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:20.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:20.612 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:20.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:20 smithi173 bash[59422]: cluster 2023-09-23T18:58:20.392251+0000 mon.a (mon.0) 1651 : cluster 3 Health check update: Degraded data redundancy: 30453/163593 objects degraded (18.615%), 16 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:20.634 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:20.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:20.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:20.744 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:20.752 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:20 smithi098 bash[76817]: cluster 2023-09-23T18:58:20.392251+0000 mon.a (mon.0) 1651 : cluster 3 Health check update: Degraded data redundancy: 30453/163593 objects degraded (18.615%), 16 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:20.753 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:20 smithi098 bash[79516]: cluster 2023-09-23T18:58:20.392251+0000 mon.a (mon.0) 1651 : cluster 3 Health check update: Degraded data redundancy: 30453/163593 objects degraded (18.615%), 16 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:20.779 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:20.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:20.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:20.789 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:58:20.882 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:20.915 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:20.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:20.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:21.019 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:21.054 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:21.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:21.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:21.162 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:21.190 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:21.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:21.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:21.297 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:21.337 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:21.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:21.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:21.440 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:21.464 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:21.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:21.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:21.567 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:21.599 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:21.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:21.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:21.705 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:21.735 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:21.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:21.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:21.843 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:21.857 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:21 smithi173 bash[59422]: cluster 2023-09-23T18:58:20.355330+0000 mgr.y (mgr.26054) 1211 : cluster 0 pgmap v1316: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 activating+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 99 op/s; 30453/163593 objects degraded (18.615%); 28265/163593 objects misplaced (17.278%); 4.8 MiB/s, 78 objects/s recovering 2023-09-23T18:58:21.873 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:21.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:21.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:21.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:21 smithi098 bash[79516]: cluster 2023-09-23T18:58:20.355330+0000 mgr.y (mgr.26054) 1211 : cluster 0 pgmap v1316: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 activating+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 99 op/s; 30453/163593 objects degraded (18.615%); 28265/163593 objects misplaced (17.278%); 4.8 MiB/s, 78 objects/s recovering 2023-09-23T18:58:21.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:21 smithi098 bash[76817]: cluster 2023-09-23T18:58:20.355330+0000 mgr.y (mgr.26054) 1211 : cluster 0 pgmap v1316: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 activating+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 99 op/s; 30453/163593 objects degraded (18.615%); 28265/163593 objects misplaced (17.278%); 4.8 MiB/s, 78 objects/s recovering 2023-09-23T18:58:21.977 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:22.007 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:22.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:22.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:22.112 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:22.138 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:22.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:22.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:22.246 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:22.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:22.287 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:22.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:22.388 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:22.433 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:22.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:22.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:22.539 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:22.565 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:22.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:22.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:22.671 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:22.695 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:22.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:22.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:22.801 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:22.843 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:22.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:22.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:22.951 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:22.985 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:22.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:22.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:23.088 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:23.121 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:23.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:23.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:23.224 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:23.257 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:23.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:23.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:23.366 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:23.396 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:23.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:23.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:23.500 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:23.521 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:23.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:23.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:23.624 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:23.646 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:23.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:23.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:23.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:23 smithi098 bash[76817]: cluster 2023-09-23T18:58:22.356207+0000 mgr.y (mgr.26054) 1212 : cluster 0 pgmap v1317: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 activating+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 3.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 120 op/s; 30160/162282 objects degraded (18.585%); 25141/162282 objects misplaced (15.492%); 7.1 MiB/s, 116 objects/s recovering 2023-09-23T18:58:23.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:23 smithi098 bash[79516]: cluster 2023-09-23T18:58:22.356207+0000 mgr.y (mgr.26054) 1212 : cluster 0 pgmap v1317: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 activating+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 3.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 120 op/s; 30160/162282 objects degraded (18.585%); 25141/162282 objects misplaced (15.492%); 7.1 MiB/s, 116 objects/s recovering 2023-09-23T18:58:23.749 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:23.779 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:23.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:23.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:23.885 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:23.915 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:23.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:23.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:24.020 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:24.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:23 smithi173 bash[59422]: cluster 2023-09-23T18:58:22.356207+0000 mgr.y (mgr.26054) 1212 : cluster 0 pgmap v1317: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 activating+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 3.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 120 op/s; 30160/162282 objects degraded (18.585%); 25141/162282 objects misplaced (15.492%); 7.1 MiB/s, 116 objects/s recovering 2023-09-23T18:58:24.048 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:24.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:24.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:24.153 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:24.181 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:24.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:24.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:24.285 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:24.320 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:24.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:24.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:24.425 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:24.451 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:24.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:24.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:24.556 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:24.588 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:24.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:24.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:24.692 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:24.722 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:24.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:24.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:24.827 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:24.868 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:24.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:24.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:24.973 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:25.015 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:25.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:25.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:25.122 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:25.155 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:25.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:25.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:25.264 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:25.290 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:25.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:25.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:25.340 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T18:58:25.340 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:58:25.400 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:25.423 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:25.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:25.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:25.530 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:25.553 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:25.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:25.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:25.656 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:25.685 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:25.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:25.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:25.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:25 smithi173 bash[59422]: cluster 2023-09-23T18:58:24.357055+0000 mgr.y (mgr.26054) 1213 : cluster 0 pgmap v1318: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 3.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 203 op/s; 29543/158784 objects degraded (18.606%); 24748/158784 objects misplaced (15.586%); 5.9 MiB/s, 95 objects/s recovering 2023-09-23T18:58:25.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:25 smithi173 bash[59422]: audit 2023-09-23T18:58:24.640465+0000 mon.a (mon.0) 1652 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:25.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:25 smithi173 bash[59422]: audit 2023-09-23T18:58:24.641808+0000 mon.a (mon.0) 1653 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:58:25.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:25 smithi173 bash[59422]: cluster 2023-09-23T18:58:25.393242+0000 mon.a (mon.0) 1654 : cluster 3 Health check update: Degraded data redundancy: 29543/158784 objects degraded (18.606%), 16 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:25.788 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:25.808 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:25.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:25.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:25.912 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:25.937 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:25.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:25.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:25.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:25 smithi098 bash[76817]: cluster 2023-09-23T18:58:24.357055+0000 mgr.y (mgr.26054) 1213 : cluster 0 pgmap v1318: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 3.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 203 op/s; 29543/158784 objects degraded (18.606%); 24748/158784 objects misplaced (15.586%); 5.9 MiB/s, 95 objects/s recovering 2023-09-23T18:58:25.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:25 smithi098 bash[76817]: audit 2023-09-23T18:58:24.640465+0000 mon.a (mon.0) 1652 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:25.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:25 smithi098 bash[76817]: audit 2023-09-23T18:58:24.641808+0000 mon.a (mon.0) 1653 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:58:25.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:25 smithi098 bash[76817]: cluster 2023-09-23T18:58:25.393242+0000 mon.a (mon.0) 1654 : cluster 3 Health check update: Degraded data redundancy: 29543/158784 objects degraded (18.606%), 16 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:25.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:25 smithi098 bash[79516]: cluster 2023-09-23T18:58:24.357055+0000 mgr.y (mgr.26054) 1213 : cluster 0 pgmap v1318: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 3.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 203 op/s; 29543/158784 objects degraded (18.606%); 24748/158784 objects misplaced (15.586%); 5.9 MiB/s, 95 objects/s recovering 2023-09-23T18:58:25.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:25 smithi098 bash[79516]: audit 2023-09-23T18:58:24.640465+0000 mon.a (mon.0) 1652 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:25.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:25 smithi098 bash[79516]: audit 2023-09-23T18:58:24.641808+0000 mon.a (mon.0) 1653 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:58:25.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:25 smithi098 bash[79516]: cluster 2023-09-23T18:58:25.393242+0000 mon.a (mon.0) 1654 : cluster 3 Health check update: Degraded data redundancy: 29543/158784 objects degraded (18.606%), 16 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:26.040 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:26.067 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:26.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:26.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:26.170 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:26.200 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:26.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:26.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:26.307 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:26.338 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:26.340 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:58:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:58:26] "GET /metrics HTTP/1.1" 200 36595 "" "Prometheus/2.33.4" 2023-09-23T18:58:26.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:26.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:26.441 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:26.469 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:26.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:26.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:26.577 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:26.614 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:26.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:26.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:26.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:26.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:26.751 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:26.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:26.852 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:26.878 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:26.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:26.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:26.984 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:27.013 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:27.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:27.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:27.116 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:27.138 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:27.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:27.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:27.146 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T18:58:27.241 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:27.260 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:27.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:27.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:27.363 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:27.390 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:27.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:27.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:27.495 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:27.524 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:27.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:27.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:27.627 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:27.652 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:27.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:27.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:27.757 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:27.788 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:27.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:27.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:27.893 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:27.901 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:27 smithi098 bash[76817]: cluster 2023-09-23T18:58:26.357844+0000 mgr.y (mgr.26054) 1214 : cluster 0 pgmap v1319: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 277 op/s; 29203/156114 objects degraded (18.706%); 24470/156114 objects misplaced (15.674%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T18:58:27.901 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:27 smithi098 bash[79516]: cluster 2023-09-23T18:58:26.357844+0000 mgr.y (mgr.26054) 1214 : cluster 0 pgmap v1319: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 277 op/s; 29203/156114 objects degraded (18.706%); 24470/156114 objects misplaced (15.674%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T18:58:27.924 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:27.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:27.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:28.028 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:28.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:27 smithi173 bash[59422]: cluster 2023-09-23T18:58:26.357844+0000 mgr.y (mgr.26054) 1214 : cluster 0 pgmap v1319: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 277 op/s; 29203/156114 objects degraded (18.706%); 24470/156114 objects misplaced (15.674%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T18:58:28.056 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:28.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:28.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:28.161 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:28.191 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:28.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:28.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:28.297 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:28.326 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:28.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:28.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:28.431 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:28.454 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:28.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:28.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:28.560 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:28.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:28.610 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:28.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:28.711 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:28.736 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:28.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:28.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:28.840 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:28.865 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:28.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:28.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:28.970 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:29.001 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:29.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:29.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:29.106 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:29.129 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:29.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:29.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:29.235 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:29.270 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:29.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:29.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:29.376 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:29.400 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:29.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:29.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:29.508 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:29.539 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:29.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:29.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:29.645 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:29.671 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:29.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:29.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:29.775 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:29.800 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:29.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:29.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:29.903 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:29.911 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:29 smithi098 bash[79516]: cluster 2023-09-23T18:58:28.358876+0000 mgr.y (mgr.26054) 1215 : cluster 0 pgmap v1320: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 27846/149580 objects degraded (18.616%); 23701/149580 objects misplaced (15.845%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T18:58:29.912 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:29 smithi098 bash[76817]: cluster 2023-09-23T18:58:28.358876+0000 mgr.y (mgr.26054) 1215 : cluster 0 pgmap v1320: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 27846/149580 objects degraded (18.616%); 23701/149580 objects misplaced (15.845%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T18:58:29.934 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:29.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:29.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:30.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:29 smithi173 bash[59422]: cluster 2023-09-23T18:58:28.358876+0000 mgr.y (mgr.26054) 1215 : cluster 0 pgmap v1320: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 27846/149580 objects degraded (18.616%); 23701/149580 objects misplaced (15.845%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T18:58:30.037 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:30.089 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:30.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:30.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:30.192 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:30.224 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:30.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:30.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:30.328 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:30.361 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:30.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:30.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:30.469 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:30.525 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:30.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:30.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:30.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:30.659 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:30.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:30.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:30.762 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:30.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:30.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:30.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:30.889 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:30.898 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:30 smithi173 bash[59422]: cluster 2023-09-23T18:58:30.394341+0000 mon.a (mon.0) 1655 : cluster 3 Health check update: Degraded data redundancy: 27846/149580 objects degraded (18.616%), 16 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:30.926 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:30.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:30.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:30.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:30 smithi098 bash[76817]: cluster 2023-09-23T18:58:30.394341+0000 mon.a (mon.0) 1655 : cluster 3 Health check update: Degraded data redundancy: 27846/149580 objects degraded (18.616%), 16 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:30.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:30 smithi098 bash[79516]: cluster 2023-09-23T18:58:30.394341+0000 mon.a (mon.0) 1655 : cluster 3 Health check update: Degraded data redundancy: 27846/149580 objects degraded (18.616%), 16 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:31.033 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:31.063 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:31.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:31.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:31.169 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:31.218 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:31.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:31.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:31.323 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:31.349 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:31.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:31.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:31.455 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:31.494 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:31.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:31.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:31.600 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:31.628 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:31.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:31.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:31.733 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:31.740 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:31 smithi173 bash[59422]: cluster 2023-09-23T18:58:30.359573+0000 mgr.y (mgr.26054) 1216 : cluster 0 pgmap v1321: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 27846/149580 objects degraded (18.616%); 23701/149580 objects misplaced (15.845%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T18:58:31.754 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:31.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:31.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:31.857 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:31.866 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:31 smithi098 bash[76817]: cluster 2023-09-23T18:58:30.359573+0000 mgr.y (mgr.26054) 1216 : cluster 0 pgmap v1321: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 27846/149580 objects degraded (18.616%); 23701/149580 objects misplaced (15.845%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T18:58:31.866 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:31 smithi098 bash[79516]: cluster 2023-09-23T18:58:30.359573+0000 mgr.y (mgr.26054) 1216 : cluster 0 pgmap v1321: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 27846/149580 objects degraded (18.616%); 23701/149580 objects misplaced (15.845%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T18:58:31.884 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:31.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:31.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:31.987 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:58:31.990 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:32.026 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:32.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:32.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:32.134 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:32.169 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:32.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:32.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:32.276 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:32.317 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:32.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:32.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:32.422 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:32.448 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:32.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:32.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:32.555 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:32.583 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:32.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:32.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:32.685 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:32.789 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:32.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:32.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:32.892 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:32.927 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:32.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:32.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:33.033 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:33.076 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:33.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:33.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:33.179 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:33.208 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:33.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:33.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:33.317 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:33.398 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:33.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:33.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:33.502 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:33.533 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:33.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:33.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:33.639 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:33.669 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:33.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:33.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:33.777 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:33.801 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:33.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:33.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:33.905 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:33.914 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:33 smithi098 bash[79516]: cluster 2023-09-23T18:58:32.360350+0000 mgr.y (mgr.26054) 1217 : cluster 0 pgmap v1322: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 27033/145368 objects degraded (18.596%); 23319/145368 objects misplaced (16.041%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T18:58:33.914 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:33 smithi098 bash[76817]: cluster 2023-09-23T18:58:32.360350+0000 mgr.y (mgr.26054) 1217 : cluster 0 pgmap v1322: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 27033/145368 objects degraded (18.596%); 23319/145368 objects misplaced (16.041%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T18:58:33.951 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:33.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:33.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:34.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:33 smithi173 bash[59422]: cluster 2023-09-23T18:58:32.360350+0000 mgr.y (mgr.26054) 1217 : cluster 0 pgmap v1322: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 27033/145368 objects degraded (18.596%); 23319/145368 objects misplaced (16.041%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T18:58:34.056 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:34.085 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:34.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:34.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:34.193 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:34.226 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:34.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:34.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:34.331 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:34.367 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:34.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:34.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:34.472 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:34.495 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:34.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:34.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:34.598 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:34.624 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:34.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:34.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:34.727 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:34.751 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:34.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:34.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:34.799 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:58:34.840 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:58:34.854 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:34.873 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:58:34.891 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:34.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:34.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:34.907 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:58:34.941 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:58:34.973 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:58:35.000 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:35.006 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:58:35.011 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:58:35.011 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:58:35.011 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:58:35.011 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:58:35.011 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:58:35.011 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:58:35.011 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:58:35.011 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T18:58:35.025 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:35.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:35.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:35.130 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:35.149 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:35.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:35.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:35.252 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:35.281 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:35.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:35.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:35.387 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:35.410 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:35.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:35.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:35.513 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:35.543 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:35.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:35.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:35.648 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:35.669 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:35.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:35.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:35.776 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:35.784 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:35 smithi098 bash[79516]: cluster 2023-09-23T18:58:34.361380+0000 mgr.y (mgr.26054) 1218 : cluster 0 pgmap v1323: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 26041/140070 objects degraded (18.591%); 22724/140070 objects misplaced (16.223%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T18:58:35.784 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:35 smithi098 bash[79516]: audit 2023-09-23T18:58:34.732823+0000 mon.c (mon.2) 133 : audit 0 from='client.? 172.21.15.98:0/3070307220' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:58:35.784 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:35 smithi098 bash[79516]: cluster 2023-09-23T18:58:35.395417+0000 mon.a (mon.0) 1656 : cluster 3 Health check update: Degraded data redundancy: 26041/140070 objects degraded (18.591%), 16 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:35.784 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:35 smithi098 bash[76817]: cluster 2023-09-23T18:58:34.361380+0000 mgr.y (mgr.26054) 1218 : cluster 0 pgmap v1323: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 26041/140070 objects degraded (18.591%); 22724/140070 objects misplaced (16.223%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T18:58:35.785 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:35 smithi098 bash[76817]: audit 2023-09-23T18:58:34.732823+0000 mon.c (mon.2) 133 : audit 0 from='client.? 172.21.15.98:0/3070307220' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:58:35.785 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:35 smithi098 bash[76817]: cluster 2023-09-23T18:58:35.395417+0000 mon.a (mon.0) 1656 : cluster 3 Health check update: Degraded data redundancy: 26041/140070 objects degraded (18.591%), 16 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:35.804 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:35.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:35.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:35.913 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:35.939 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:35.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:35.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:35.945 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:35 smithi173 bash[59422]: cluster 2023-09-23T18:58:34.361380+0000 mgr.y (mgr.26054) 1218 : cluster 0 pgmap v1323: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 26041/140070 objects degraded (18.591%); 22724/140070 objects misplaced (16.223%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T18:58:35.945 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:35 smithi173 bash[59422]: audit 2023-09-23T18:58:34.732823+0000 mon.c (mon.2) 133 : audit 0 from='client.? 172.21.15.98:0/3070307220' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:58:35.945 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:35 smithi173 bash[59422]: cluster 2023-09-23T18:58:35.395417+0000 mon.a (mon.0) 1656 : cluster 3 Health check update: Degraded data redundancy: 26041/140070 objects degraded (18.591%), 16 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:36.046 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:36.086 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:36.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:36.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:36.192 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:36.218 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:36.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:36.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:36.274 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:58:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:58:36] "GET /metrics HTTP/1.1" 200 36595 "" "Prometheus/2.33.4" 2023-09-23T18:58:36.324 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:36.348 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:36.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:36.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:36.450 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:36.474 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:36.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:36.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:36.583 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:36.613 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:36.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:36.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:36.718 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:36.742 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:36.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:36.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:36.846 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:36.883 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:36.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:36.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:36.985 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:37.031 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:37.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:37.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:37.137 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:37.166 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:37.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:37.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:37.169 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:37 smithi098 bash[76817]: debug 2023-09-23T18:58:37.158+0000 7fc8a2a01700 -1 mon.a@0(leader).osd e297 definitely_dead 0 2023-09-23T18:58:37.269 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:37.296 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:37.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:37.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:37.401 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:37.426 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:37.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:37.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:37.532 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:37.560 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:37.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:37.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:37.664 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:37.701 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:37.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:37.705 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is unset 2023-09-23T18:58:37.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:37.711 INFO:teuthology.orchestra.run.smithi098.stderr:marked in osd.3. 2023-09-23T18:58:37.806 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:37.828 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:37.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:37.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:37.931 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:37.960 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:37.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:37.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:37.969 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:37 smithi098 bash[79516]: cluster 2023-09-23T18:58:36.362181+0000 mgr.y (mgr.26054) 1219 : cluster 0 pgmap v1324: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 2.8 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 25839/137634 objects degraded (18.774%); 22608/137634 objects misplaced (16.426%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T18:58:37.969 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:37 smithi098 bash[79516]: audit 2023-09-23T18:58:36.981982+0000 mon.a (mon.0) 1657 : audit 1 from='client.? 172.21.15.98:0/668058052' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:58:37.969 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:37 smithi098 bash[79516]: audit 2023-09-23T18:58:37.164701+0000 mon.c (mon.2) 134 : audit 0 from='client.? 172.21.15.98:0/3113528243' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T18:58:37.969 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:37 smithi098 bash[79516]: audit 2023-09-23T18:58:37.165466+0000 mon.a (mon.0) 1658 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T18:58:37.970 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:37 smithi098 bash[76817]: cluster 2023-09-23T18:58:36.362181+0000 mgr.y (mgr.26054) 1219 : cluster 0 pgmap v1324: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 2.8 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 25839/137634 objects degraded (18.774%); 22608/137634 objects misplaced (16.426%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T18:58:37.970 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:37 smithi098 bash[76817]: audit 2023-09-23T18:58:36.981982+0000 mon.a (mon.0) 1657 : audit 1 from='client.? 172.21.15.98:0/668058052' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:58:37.970 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:37 smithi098 bash[76817]: audit 2023-09-23T18:58:37.164701+0000 mon.c (mon.2) 134 : audit 0 from='client.? 172.21.15.98:0/3113528243' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T18:58:37.970 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:37 smithi098 bash[76817]: audit 2023-09-23T18:58:37.165466+0000 mon.a (mon.0) 1658 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T18:58:38.024 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:58:38.025 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:58:38.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:37 smithi173 bash[59422]: cluster 2023-09-23T18:58:36.362181+0000 mgr.y (mgr.26054) 1219 : cluster 0 pgmap v1324: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 10 active+clean+remapped, 49 active+clean, 11 active+undersized+degraded, 12 active+undersized; 2.8 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 25839/137634 objects degraded (18.774%); 22608/137634 objects misplaced (16.426%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T18:58:38.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:37 smithi173 bash[59422]: audit 2023-09-23T18:58:36.981982+0000 mon.a (mon.0) 1657 : audit 1 from='client.? 172.21.15.98:0/668058052' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:58:38.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:37 smithi173 bash[59422]: audit 2023-09-23T18:58:37.164701+0000 mon.c (mon.2) 134 : audit 0 from='client.? 172.21.15.98:0/3113528243' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T18:58:38.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:37 smithi173 bash[59422]: audit 2023-09-23T18:58:37.165466+0000 mon.a (mon.0) 1658 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T18:58:38.063 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:38.097 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:38.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:38.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:38.201 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:38.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:38.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:38.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:38.329 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:38.363 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:38.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:38.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:38.467 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:38.501 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:38.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:38.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:38.604 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:38.629 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:38.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:38.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:38.732 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:38.775 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:38.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:38.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:38.881 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:38.909 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:38.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:38.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:38.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:38 smithi098 bash[79516]: cluster 2023-09-23T18:58:37.678531+0000 mon.a (mon.0) 1659 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:58:38.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:38 smithi098 bash[79516]: audit 2023-09-23T18:58:37.692657+0000 mon.a (mon.0) 1660 : audit 1 from='client.? 172.21.15.98:0/668058052' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:58:38.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:38 smithi098 bash[79516]: audit 2023-09-23T18:58:37.692756+0000 mon.a (mon.0) 1661 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T18:58:38.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:38 smithi098 bash[79516]: cluster 2023-09-23T18:58:37.692879+0000 mon.a (mon.0) 1662 : cluster 0 osdmap e298: 8 total, 7 up, 8 in 2023-09-23T18:58:38.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:38 smithi098 bash[79516]: cluster 2023-09-23T18:58:37.692982+0000 osd.4 (osd.4) 24 : cluster 3 failed to encode map e298 with expected crc 2023-09-23T18:58:38.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:38 smithi098 bash[79516]: cluster 2023-09-23T18:58:37.696723+0000 osd.6 (osd.6) 270 : cluster 3 failed to encode map e298 with expected crc 2023-09-23T18:58:38.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:38 smithi098 bash[79516]: cluster 2023-09-23T18:58:37.700305+0000 osd.5 (osd.5) 216 : cluster 3 failed to encode map e298 with expected crc 2023-09-23T18:58:38.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:38 smithi098 bash[76817]: cluster 2023-09-23T18:58:37.678531+0000 mon.a (mon.0) 1659 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:58:38.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:38 smithi098 bash[76817]: audit 2023-09-23T18:58:37.692657+0000 mon.a (mon.0) 1660 : audit 1 from='client.? 172.21.15.98:0/668058052' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:58:38.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:38 smithi098 bash[76817]: audit 2023-09-23T18:58:37.692756+0000 mon.a (mon.0) 1661 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T18:58:38.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:38 smithi098 bash[76817]: cluster 2023-09-23T18:58:37.692879+0000 mon.a (mon.0) 1662 : cluster 0 osdmap e298: 8 total, 7 up, 8 in 2023-09-23T18:58:38.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:38 smithi098 bash[76817]: cluster 2023-09-23T18:58:37.692982+0000 osd.4 (osd.4) 24 : cluster 3 failed to encode map e298 with expected crc 2023-09-23T18:58:38.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:38 smithi098 bash[76817]: cluster 2023-09-23T18:58:37.696723+0000 osd.6 (osd.6) 270 : cluster 3 failed to encode map e298 with expected crc 2023-09-23T18:58:38.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:38 smithi098 bash[76817]: cluster 2023-09-23T18:58:37.700305+0000 osd.5 (osd.5) 216 : cluster 3 failed to encode map e298 with expected crc 2023-09-23T18:58:39.012 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:39.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:38 smithi173 bash[59422]: cluster 2023-09-23T18:58:37.678531+0000 mon.a (mon.0) 1659 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:58:39.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:38 smithi173 bash[59422]: audit 2023-09-23T18:58:37.692657+0000 mon.a (mon.0) 1660 : audit 1 from='client.? 172.21.15.98:0/668058052' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:58:39.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:38 smithi173 bash[59422]: audit 2023-09-23T18:58:37.692756+0000 mon.a (mon.0) 1661 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T18:58:39.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:38 smithi173 bash[59422]: cluster 2023-09-23T18:58:37.692879+0000 mon.a (mon.0) 1662 : cluster 0 osdmap e298: 8 total, 7 up, 8 in 2023-09-23T18:58:39.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:38 smithi173 bash[59422]: cluster 2023-09-23T18:58:37.692982+0000 osd.4 (osd.4) 24 : cluster 3 failed to encode map e298 with expected crc 2023-09-23T18:58:39.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:38 smithi173 bash[59422]: cluster 2023-09-23T18:58:37.696723+0000 osd.6 (osd.6) 270 : cluster 3 failed to encode map e298 with expected crc 2023-09-23T18:58:39.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:38 smithi173 bash[59422]: cluster 2023-09-23T18:58:37.700305+0000 osd.5 (osd.5) 216 : cluster 3 failed to encode map e298 with expected crc 2023-09-23T18:58:39.041 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:39.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:39.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:39.144 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:39.175 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:39.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:39.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:39.278 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:39.303 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:39.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:39.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:39.406 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:39.427 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:58:39.427 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-09-23T18:58:39.427 INFO:teuthology.orchestra.run.smithi173.stdout: "complaint_time": 30, 2023-09-23T18:58:39.427 INFO:teuthology.orchestra.run.smithi173.stdout: "num_blocked_ops": 0 2023-09-23T18:58:39.428 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:58:39.436 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:39.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:39.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:39.540 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:39.565 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:39.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:39.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:39.668 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:39.696 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:39.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:39.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:39.799 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:39.828 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:39.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:39.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:39.927 INFO:tasks.thrashosds.thrasher:Added osd 3 2023-09-23T18:58:39.934 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:39.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[76817]: cluster 2023-09-23T18:58:38.363566+0000 mgr.y (mgr.26054) 1220 : cluster 0 pgmap v1326: 89 pgs: 6 peering, 5 remapped+peering, 3 active+recovering+undersized+degraded+remapped, 5 active+clean+remapped, 47 active+clean, 11 active+undersized+degraded, 12 active+undersized; 2.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 19031/131046 objects degraded (14.522%); 7665/131046 objects misplaced (5.849%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T18:58:39.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[76817]: cluster 2023-09-23T18:58:38.696884+0000 osd.5 (osd.5) 217 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T18:58:39.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[76817]: cluster 2023-09-23T18:58:38.696936+0000 osd.6 (osd.6) 271 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T18:58:39.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[76817]: cluster 2023-09-23T18:58:38.697098+0000 mon.a (mon.0) 1663 : cluster 0 osdmap e299: 8 total, 7 up, 8 in 2023-09-23T18:58:39.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[76817]: cluster 2023-09-23T18:58:38.700442+0000 osd.5 (osd.5) 218 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T18:58:39.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[76817]: cluster 2023-09-23T18:58:38.701626+0000 osd.4 (osd.4) 25 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T18:58:39.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[76817]: cluster 2023-09-23T18:58:38.702769+0000 osd.5 (osd.5) 219 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T18:58:39.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[76817]: cluster 2023-09-23T18:58:38.705286+0000 osd.5 (osd.5) 220 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T18:58:39.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[76817]: cluster 2023-09-23T18:58:38.705781+0000 osd.4 (osd.4) 26 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T18:58:39.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[76817]: cluster 2023-09-23T18:58:38.708295+0000 osd.5 (osd.5) 221 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T18:58:39.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[76817]: audit 2023-09-23T18:58:39.570577+0000 mon.a (mon.0) 1664 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:58:39.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[76817]: audit 2023-09-23T18:58:39.637057+0000 mon.a (mon.0) 1665 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:39.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[76817]: audit 2023-09-23T18:58:39.638037+0000 mon.a (mon.0) 1666 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:58:39.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[79516]: cluster 2023-09-23T18:58:38.363566+0000 mgr.y (mgr.26054) 1220 : cluster 0 pgmap v1326: 89 pgs: 6 peering, 5 remapped+peering, 3 active+recovering+undersized+degraded+remapped, 5 active+clean+remapped, 47 active+clean, 11 active+undersized+degraded, 12 active+undersized; 2.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 19031/131046 objects degraded (14.522%); 7665/131046 objects misplaced (5.849%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T18:58:39.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[79516]: cluster 2023-09-23T18:58:38.696884+0000 osd.5 (osd.5) 217 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T18:58:39.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[79516]: cluster 2023-09-23T18:58:38.696936+0000 osd.6 (osd.6) 271 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T18:58:39.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[79516]: cluster 2023-09-23T18:58:38.697098+0000 mon.a (mon.0) 1663 : cluster 0 osdmap e299: 8 total, 7 up, 8 in 2023-09-23T18:58:39.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[79516]: cluster 2023-09-23T18:58:38.700442+0000 osd.5 (osd.5) 218 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T18:58:39.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[79516]: cluster 2023-09-23T18:58:38.701626+0000 osd.4 (osd.4) 25 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T18:58:39.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[79516]: cluster 2023-09-23T18:58:38.702769+0000 osd.5 (osd.5) 219 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T18:58:39.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[79516]: cluster 2023-09-23T18:58:38.705286+0000 osd.5 (osd.5) 220 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T18:58:39.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[79516]: cluster 2023-09-23T18:58:38.705781+0000 osd.4 (osd.4) 26 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T18:58:39.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[79516]: cluster 2023-09-23T18:58:38.708295+0000 osd.5 (osd.5) 221 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T18:58:39.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[79516]: audit 2023-09-23T18:58:39.570577+0000 mon.a (mon.0) 1664 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:58:39.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[79516]: audit 2023-09-23T18:58:39.637057+0000 mon.a (mon.0) 1665 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:39.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:39 smithi098 bash[79516]: audit 2023-09-23T18:58:39.638037+0000 mon.a (mon.0) 1666 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:58:39.979 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:39.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:39.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:40.026 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:58:40.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:39 smithi173 bash[59422]: cluster 2023-09-23T18:58:38.363566+0000 mgr.y (mgr.26054) 1220 : cluster 0 pgmap v1326: 89 pgs: 6 peering, 5 remapped+peering, 3 active+recovering+undersized+degraded+remapped, 5 active+clean+remapped, 47 active+clean, 11 active+undersized+degraded, 12 active+undersized; 2.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 19031/131046 objects degraded (14.522%); 7665/131046 objects misplaced (5.849%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T18:58:40.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:39 smithi173 bash[59422]: cluster 2023-09-23T18:58:38.696884+0000 osd.5 (osd.5) 217 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T18:58:40.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:39 smithi173 bash[59422]: cluster 2023-09-23T18:58:38.696936+0000 osd.6 (osd.6) 271 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T18:58:40.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:39 smithi173 bash[59422]: cluster 2023-09-23T18:58:38.697098+0000 mon.a (mon.0) 1663 : cluster 0 osdmap e299: 8 total, 7 up, 8 in 2023-09-23T18:58:40.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:39 smithi173 bash[59422]: cluster 2023-09-23T18:58:38.700442+0000 osd.5 (osd.5) 218 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T18:58:40.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:39 smithi173 bash[59422]: cluster 2023-09-23T18:58:38.701626+0000 osd.4 (osd.4) 25 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T18:58:40.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:39 smithi173 bash[59422]: cluster 2023-09-23T18:58:38.702769+0000 osd.5 (osd.5) 219 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T18:58:40.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:39 smithi173 bash[59422]: cluster 2023-09-23T18:58:38.705286+0000 osd.5 (osd.5) 220 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T18:58:40.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:39 smithi173 bash[59422]: cluster 2023-09-23T18:58:38.705781+0000 osd.4 (osd.4) 26 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T18:58:40.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:39 smithi173 bash[59422]: cluster 2023-09-23T18:58:38.708295+0000 osd.5 (osd.5) 221 : cluster 3 failed to encode map e299 with expected crc 2023-09-23T18:58:40.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:39 smithi173 bash[59422]: audit 2023-09-23T18:58:39.570577+0000 mon.a (mon.0) 1664 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:58:40.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:39 smithi173 bash[59422]: audit 2023-09-23T18:58:39.637057+0000 mon.a (mon.0) 1665 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:40.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:39 smithi173 bash[59422]: audit 2023-09-23T18:58:39.638037+0000 mon.a (mon.0) 1666 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:58:40.084 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:40.106 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:40.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:40.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:40.209 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:40.238 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:40.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:40.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:40.355 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:40.387 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:40.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:40.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:40.489 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:40.512 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:40.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:40.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:40.582 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T18:58:40.615 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:40.644 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:40.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:40.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:40.710 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:40 smithi173 bash[59422]: cluster 2023-09-23T18:58:39.699203+0000 osd.6 (osd.6) 272 : cluster 3 failed to encode map e300 with expected crc 2023-09-23T18:58:40.710 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:40 smithi173 bash[59422]: cluster 2023-09-23T18:58:39.699230+0000 osd.4 (osd.4) 27 : cluster 3 failed to encode map e300 with expected crc 2023-09-23T18:58:40.710 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:40 smithi173 bash[59422]: cluster 2023-09-23T18:58:39.700363+0000 mon.a (mon.0) 1667 : cluster 0 osdmap e300: 8 total, 7 up, 8 in 2023-09-23T18:58:40.710 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:40 smithi173 bash[59422]: cluster 2023-09-23T18:58:39.713243+0000 osd.5 (osd.5) 222 : cluster 3 failed to encode map e300 with expected crc 2023-09-23T18:58:40.710 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:40 smithi173 bash[59422]: audit 2023-09-23T18:58:39.992706+0000 mon.a (mon.0) 1668 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:40.710 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:40 smithi173 bash[59422]: audit 2023-09-23T18:58:40.001374+0000 mon.a (mon.0) 1669 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:40.710 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:40 smithi173 bash[59422]: cluster 2023-09-23T18:58:40.396540+0000 mon.a (mon.0) 1670 : cluster 3 Health check update: Degraded data redundancy: 19031/131046 objects degraded (14.522%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:40.753 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:40.782 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:40.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:40.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:40.887 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:40.896 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:40 smithi098 bash[76817]: cluster 2023-09-23T18:58:39.699203+0000 osd.6 (osd.6) 272 : cluster 3 failed to encode map e300 with expected crc 2023-09-23T18:58:40.896 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:40 smithi098 bash[76817]: cluster 2023-09-23T18:58:39.699230+0000 osd.4 (osd.4) 27 : cluster 3 failed to encode map e300 with expected crc 2023-09-23T18:58:40.896 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:40 smithi098 bash[76817]: cluster 2023-09-23T18:58:39.700363+0000 mon.a (mon.0) 1667 : cluster 0 osdmap e300: 8 total, 7 up, 8 in 2023-09-23T18:58:40.896 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:40 smithi098 bash[76817]: cluster 2023-09-23T18:58:39.713243+0000 osd.5 (osd.5) 222 : cluster 3 failed to encode map e300 with expected crc 2023-09-23T18:58:40.897 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:40 smithi098 bash[76817]: audit 2023-09-23T18:58:39.992706+0000 mon.a (mon.0) 1668 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:40.897 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:40 smithi098 bash[76817]: audit 2023-09-23T18:58:40.001374+0000 mon.a (mon.0) 1669 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:40.897 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:40 smithi098 bash[76817]: cluster 2023-09-23T18:58:40.396540+0000 mon.a (mon.0) 1670 : cluster 3 Health check update: Degraded data redundancy: 19031/131046 objects degraded (14.522%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:40.897 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:40 smithi098 bash[79516]: cluster 2023-09-23T18:58:39.699203+0000 osd.6 (osd.6) 272 : cluster 3 failed to encode map e300 with expected crc 2023-09-23T18:58:40.897 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:40 smithi098 bash[79516]: cluster 2023-09-23T18:58:39.699230+0000 osd.4 (osd.4) 27 : cluster 3 failed to encode map e300 with expected crc 2023-09-23T18:58:40.897 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:40 smithi098 bash[79516]: cluster 2023-09-23T18:58:39.700363+0000 mon.a (mon.0) 1667 : cluster 0 osdmap e300: 8 total, 7 up, 8 in 2023-09-23T18:58:40.898 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:40 smithi098 bash[79516]: cluster 2023-09-23T18:58:39.713243+0000 osd.5 (osd.5) 222 : cluster 3 failed to encode map e300 with expected crc 2023-09-23T18:58:40.898 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:40 smithi098 bash[79516]: audit 2023-09-23T18:58:39.992706+0000 mon.a (mon.0) 1668 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:40.898 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:40 smithi098 bash[79516]: audit 2023-09-23T18:58:40.001374+0000 mon.a (mon.0) 1669 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:40.898 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:40 smithi098 bash[79516]: cluster 2023-09-23T18:58:40.396540+0000 mon.a (mon.0) 1670 : cluster 3 Health check update: Degraded data redundancy: 19031/131046 objects degraded (14.522%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:40.910 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:40.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:40.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:41.016 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:41.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:41.042 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:41.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:41.144 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:41.177 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:41.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:41.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:41.284 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:41.314 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:41.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:41.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:41.419 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:41.449 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:41.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:41.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:41.553 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:41.597 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:41.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:41.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:41.704 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:41.733 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:41.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:41.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:41.839 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:41.867 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:41.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:41.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:41.912 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:58:41.973 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:42.013 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:42.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:42.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:42.120 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:42.129 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:42 smithi098 bash[79516]: cluster 2023-09-23T18:58:40.364314+0000 mgr.y (mgr.26054) 1221 : cluster 0 pgmap v1329: 89 pgs: 6 peering, 5 remapped+peering, 3 active+recovering+undersized+degraded+remapped, 5 active+clean+remapped, 47 active+clean, 11 active+undersized+degraded, 12 active+undersized; 2.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 19031/131046 objects degraded (14.522%); 7665/131046 objects misplaced (5.849%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T18:58:42.129 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:42 smithi098 bash[76817]: cluster 2023-09-23T18:58:40.364314+0000 mgr.y (mgr.26054) 1221 : cluster 0 pgmap v1329: 89 pgs: 6 peering, 5 remapped+peering, 3 active+recovering+undersized+degraded+remapped, 5 active+clean+remapped, 47 active+clean, 11 active+undersized+degraded, 12 active+undersized; 2.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 19031/131046 objects degraded (14.522%); 7665/131046 objects misplaced (5.849%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T18:58:42.158 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:42.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:42.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:42.267 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:42.276 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:42 smithi173 bash[59422]: cluster 2023-09-23T18:58:40.364314+0000 mgr.y (mgr.26054) 1221 : cluster 0 pgmap v1329: 89 pgs: 6 peering, 5 remapped+peering, 3 active+recovering+undersized+degraded+remapped, 5 active+clean+remapped, 47 active+clean, 11 active+undersized+degraded, 12 active+undersized; 2.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 19031/131046 objects degraded (14.522%); 7665/131046 objects misplaced (5.849%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T18:58:42.303 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:42.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:42.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:42.409 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:42.442 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:42.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:42.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:42.548 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:42.571 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:42.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:42.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:42.674 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:42.703 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:42.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:42.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:42.807 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:42.832 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:42.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:42.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:42.937 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:42.970 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:42.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:42.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:43.075 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:43.118 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:43.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:43.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:43.221 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:43.250 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:43.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:43.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:43.353 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:43.390 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:43.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:43.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:43.494 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:43.528 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:43.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:43.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:43.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:43.660 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:43.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:43.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:43.768 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:43.794 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:43.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:43.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:43.897 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:43.926 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:43.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:43.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:44.031 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:44.060 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:44.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:44.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:44.166 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:44.174 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:44 smithi173 bash[59422]: cluster 2023-09-23T18:58:42.365320+0000 mgr.y (mgr.26054) 1222 : cluster 0 pgmap v1330: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded+remapped, 6 peering, 1 remapped+peering, 8 active+clean+remapped, 47 active+clean, 11 active+undersized+degraded, 12 active+undersized; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 21222/128475 objects degraded (16.518%); 6/128475 objects misplaced (0.005%); 5.3 MiB/s, 111 objects/s recovering 2023-09-23T18:58:44.191 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:44.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:44.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:44.297 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:44.307 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:44 smithi098 bash[76817]: cluster 2023-09-23T18:58:42.365320+0000 mgr.y (mgr.26054) 1222 : cluster 0 pgmap v1330: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded+remapped, 6 peering, 1 remapped+peering, 8 active+clean+remapped, 47 active+clean, 11 active+undersized+degraded, 12 active+undersized; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 21222/128475 objects degraded (16.518%); 6/128475 objects misplaced (0.005%); 5.3 MiB/s, 111 objects/s recovering 2023-09-23T18:58:44.308 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:44 smithi098 bash[79516]: cluster 2023-09-23T18:58:42.365320+0000 mgr.y (mgr.26054) 1222 : cluster 0 pgmap v1330: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded+remapped, 6 peering, 1 remapped+peering, 8 active+clean+remapped, 47 active+clean, 11 active+undersized+degraded, 12 active+undersized; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 21222/128475 objects degraded (16.518%); 6/128475 objects misplaced (0.005%); 5.3 MiB/s, 111 objects/s recovering 2023-09-23T18:58:44.326 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:44.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:44.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:44.435 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:44.459 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:44.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:44.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:44.562 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:44.590 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:44.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:44.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:44.605 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:58:44.696 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:44.722 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:44.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:44.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:44.826 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:44.854 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:44.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:44.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:44.928 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 6, 7, 5, 3] out_osds: [] dead_osds: [7] live_osds: [0, 2, 3, 5, 6, 1, 4] 2023-09-23T18:58:44.928 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:58:44.928 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [0, 1, 2, 4, 6, 7, 5, 3] 2023-09-23T18:58:44.928 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd out 4 2023-09-23T18:58:44.939 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:58:44.958 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:45.003 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:45.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:45.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:45.109 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:45.132 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:45.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:45.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:45.236 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:45.258 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:45.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:45.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:45.361 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:45.406 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:45.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:45.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:45.511 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:45.558 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:45.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:45.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:45.666 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:45.729 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:45.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:45.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:45.834 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:45.864 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:45.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:45.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:45.968 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:45.993 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:45.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:45.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:46.096 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:46.129 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:46.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:46.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:46.232 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:46.274 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:46.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:46 smithi173 bash[59422]: cluster 2023-09-23T18:58:44.366222+0000 mgr.y (mgr.26054) 1223 : cluster 0 pgmap v1331: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 53 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 23433/125544 objects degraded (18.665%); 6/125544 objects misplaced (0.005%); 2.0 MiB/s, 57 objects/s recovering 2023-09-23T18:58:46.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:46 smithi173 bash[59422]: cluster 2023-09-23T18:58:45.398381+0000 mon.a (mon.0) 1671 : cluster 3 Health check update: Degraded data redundancy: 23433/125544 objects degraded (18.665%), 15 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:46.279 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:46 smithi098 bash[79516]: cluster 2023-09-23T18:58:44.366222+0000 mgr.y (mgr.26054) 1223 : cluster 0 pgmap v1331: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 53 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 23433/125544 objects degraded (18.665%); 6/125544 objects misplaced (0.005%); 2.0 MiB/s, 57 objects/s recovering 2023-09-23T18:58:46.279 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:46 smithi098 bash[79516]: cluster 2023-09-23T18:58:45.398381+0000 mon.a (mon.0) 1671 : cluster 3 Health check update: Degraded data redundancy: 23433/125544 objects degraded (18.665%), 15 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:46.279 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:58:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:58:46] "GET /metrics HTTP/1.1" 200 36604 "" "Prometheus/2.33.4" 2023-09-23T18:58:46.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:46.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:46.280 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:46 smithi098 bash[76817]: cluster 2023-09-23T18:58:44.366222+0000 mgr.y (mgr.26054) 1223 : cluster 0 pgmap v1331: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 53 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 23433/125544 objects degraded (18.665%); 6/125544 objects misplaced (0.005%); 2.0 MiB/s, 57 objects/s recovering 2023-09-23T18:58:46.280 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:46 smithi098 bash[76817]: cluster 2023-09-23T18:58:45.398381+0000 mon.a (mon.0) 1671 : cluster 3 Health check update: Degraded data redundancy: 23433/125544 objects degraded (18.665%), 15 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:46.381 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:46.409 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:46.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:46.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:46.512 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:46.539 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:46.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:46.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:46.638 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:58:46.652 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:46.691 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:46.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:46.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:46.800 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:46.828 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:46.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:46.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:46.933 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:46.980 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:46.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:46.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:47.089 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:47.116 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:47.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:47.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:47.220 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:47.245 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:47.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:47.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:47.348 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:47.375 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:47.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:47.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:47.387 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:47 smithi098 bash[76817]: audit 2023-09-23T18:58:46.234142+0000 mon.a (mon.0) 1672 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:47.387 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:47 smithi098 bash[76817]: audit 2023-09-23T18:58:46.252511+0000 mon.a (mon.0) 1673 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:47.387 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:47 smithi098 bash[76817]: audit 2023-09-23T18:58:46.461526+0000 mon.a (mon.0) 1674 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:47.388 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:47 smithi098 bash[76817]: audit 2023-09-23T18:58:46.471777+0000 mon.a (mon.0) 1675 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:47.484 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:47.492 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:47 smithi173 bash[59422]: audit 2023-09-23T18:58:46.234142+0000 mon.a (mon.0) 1672 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:47.492 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:47 smithi173 bash[59422]: audit 2023-09-23T18:58:46.252511+0000 mon.a (mon.0) 1673 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:47.492 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:47 smithi173 bash[59422]: audit 2023-09-23T18:58:46.461526+0000 mon.a (mon.0) 1674 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:47.492 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:47 smithi173 bash[59422]: audit 2023-09-23T18:58:46.471777+0000 mon.a (mon.0) 1675 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:47.506 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:47.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:47.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:47.609 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:47.650 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:47.655 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:47 smithi098 bash[79516]: audit 2023-09-23T18:58:46.234142+0000 mon.a (mon.0) 1672 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:47.656 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:47 smithi098 bash[79516]: audit 2023-09-23T18:58:46.252511+0000 mon.a (mon.0) 1673 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:47.656 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:47 smithi098 bash[79516]: audit 2023-09-23T18:58:46.461526+0000 mon.a (mon.0) 1674 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:47.656 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:47 smithi098 bash[79516]: audit 2023-09-23T18:58:46.471777+0000 mon.a (mon.0) 1675 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:47.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:47.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:47.757 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:47.788 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:47.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:47.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:47.892 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:47.918 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:47.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:47.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:48.021 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:48.045 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:48.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:48.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:48.148 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:48.176 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:48.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:48.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:48.280 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:48.310 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:48.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:48.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:48.416 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:48.425 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:48 smithi098 bash[76817]: cluster 2023-09-23T18:58:46.366978+0000 mgr.y (mgr.26054) 1224 : cluster 0 pgmap v1332: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 53 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 318 op/s; 23261/123405 objects degraded (18.849%); 6/123405 objects misplaced (0.005%); 1.7 MiB/s, 75 objects/s recovering 2023-09-23T18:58:48.425 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:48 smithi098 bash[79516]: cluster 2023-09-23T18:58:46.366978+0000 mgr.y (mgr.26054) 1224 : cluster 0 pgmap v1332: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 53 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 318 op/s; 23261/123405 objects degraded (18.849%); 6/123405 objects misplaced (0.005%); 1.7 MiB/s, 75 objects/s recovering 2023-09-23T18:58:48.442 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:48.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:48.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:48.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:48 smithi173 bash[59422]: cluster 2023-09-23T18:58:46.366978+0000 mgr.y (mgr.26054) 1224 : cluster 0 pgmap v1332: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 53 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 318 op/s; 23261/123405 objects degraded (18.849%); 6/123405 objects misplaced (0.005%); 1.7 MiB/s, 75 objects/s recovering 2023-09-23T18:58:48.548 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:48.568 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:48.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:48.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:48.672 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:48.710 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:48.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:48.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:48.813 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:48.842 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:48.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:48.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:48.949 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:48.977 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:48.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:48.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:49.081 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:49.104 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:49.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:49.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:49.209 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:49.232 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:49.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:49.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:49.254 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:49 smithi098 bash[76817]: audit 2023-09-23T18:58:48.245545+0000 mon.a (mon.0) 1676 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:49.254 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:49 smithi098 bash[76817]: audit 2023-09-23T18:58:48.261947+0000 mon.a (mon.0) 1677 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:49.342 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:49.349 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:49 smithi173 bash[59422]: audit 2023-09-23T18:58:48.245545+0000 mon.a (mon.0) 1676 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:49.350 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:49 smithi173 bash[59422]: audit 2023-09-23T18:58:48.261947+0000 mon.a (mon.0) 1677 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:49.367 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:49.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:49.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:49.469 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:49.497 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:49.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:49.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:49.600 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:49.629 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:49.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:49.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:49.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:49 smithi098 bash[79516]: audit 2023-09-23T18:58:48.245545+0000 mon.a (mon.0) 1676 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:49.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:49 smithi098 bash[79516]: audit 2023-09-23T18:58:48.261947+0000 mon.a (mon.0) 1677 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:58:49.735 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:49.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:49.769 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:49.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:49.790 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T18:58:49.790 INFO:teuthology.orchestra.run.smithi173.stdout: "size": 20, 2023-09-23T18:58:49.790 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 600, 2023-09-23T18:58:49.790 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-09-23T18:58:49.790 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.790 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45602.0:1 9.e 9:7afb631d:::benchmark_data_smithi098_206513_object0:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e286)", 2023-09-23T18:58:49.790 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:56:00.209236+0000", 2023-09-23T18:58:49.790 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 169.57555128800001, 2023-09-23T18:58:49.790 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.18406957099999999, 2023-09-23T18:58:49.790 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:58:49.791 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:58:49.791 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:58:49.791 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45602", 2023-09-23T18:58:49.791 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T18:58:49.791 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 1 2023-09-23T18:58:49.791 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.791 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:58:49.791 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.791 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:58:49.791 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.209236+0000", 2023-09-23T18:58:49.791 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:58:49.791 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.791 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.791 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:58:49.792 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.209236+0000", 2023-09-23T18:58:49.792 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999971 2023-09-23T18:58:49.792 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.792 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.792 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:58:49.792 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.209233+0000", 2023-09-23T18:58:49.792 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00034027999999999999 2023-09-23T18:58:49.792 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.792 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.792 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:58:49.792 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.209573+0000", 2023-09-23T18:58:49.792 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.1186999999999999e-05 2023-09-23T18:58:49.793 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.793 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.793 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:58:49.793 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.209595+0000", 2023-09-23T18:58:49.793 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2712e-05 2023-09-23T18:58:49.793 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.793 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.793 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:58:49.793 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.209607+0000", 2023-09-23T18:58:49.793 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00045885799999999998 2023-09-23T18:58:49.793 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.793 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.793 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.793 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210066+0000", 2023-09-23T18:58:49.794 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3315999999999998e-05 2023-09-23T18:58:49.794 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.794 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.794 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:58:49.794 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210089+0000", 2023-09-23T18:58:49.794 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.17176863000000001 2023-09-23T18:58:49.794 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.794 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.794 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.794 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.381858+0000", 2023-09-23T18:58:49.794 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000186873 2023-09-23T18:58:49.794 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.795 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.795 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:58:49.795 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.382045+0000", 2023-09-23T18:58:49.795 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00027630100000000001 2023-09-23T18:58:49.795 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.795 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.795 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1", 2023-09-23T18:58:49.795 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.382321+0000", 2023-09-23T18:58:49.795 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0024174410000000002 2023-09-23T18:58:49.795 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.795 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.795 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:58:49.795 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.384739+0000", 2023-09-23T18:58:49.796 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0083751350000000006 2023-09-23T18:58:49.796 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.796 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.796 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:58:49.796 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.393114+0000", 2023-09-23T18:58:49.796 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.6316999999999995e-05 2023-09-23T18:58:49.796 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.796 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.796 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:58:49.796 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.393190+0000", 2023-09-23T18:58:49.796 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000115221 2023-09-23T18:58:49.796 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.797 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.797 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:58:49.797 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.393305+0000", 2023-09-23T18:58:49.797 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.18406957099999999 2023-09-23T18:58:49.797 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.797 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:58:49.797 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.797 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.797 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.797 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45602.0:10 9.e 9:7db389c9:::benchmark_data_smithi098_206513_object9:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e286)", 2023-09-23T18:58:49.797 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:56:00.209621+0000", 2023-09-23T18:58:49.797 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 169.57516553599999, 2023-09-23T18:58:49.798 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.18399531299999999, 2023-09-23T18:58:49.798 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:58:49.798 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:58:49.798 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:58:49.798 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45602", 2023-09-23T18:58:49.798 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T18:58:49.798 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 10 2023-09-23T18:58:49.798 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.798 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:58:49.798 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.798 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:58:49.798 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.209621+0000", 2023-09-23T18:58:49.798 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:58:49.799 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.799 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.799 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:58:49.799 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.209621+0000", 2023-09-23T18:58:49.799 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:58:49.799 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.799 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.799 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:58:49.799 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.209620+0000", 2023-09-23T18:58:49.799 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000102942 2023-09-23T18:58:49.799 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.800 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.800 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:58:49.800 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.209723+0000", 2023-09-23T18:58:49.800 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4348999999999999e-05 2023-09-23T18:58:49.800 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.800 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.800 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:58:49.800 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.209737+0000", 2023-09-23T18:58:49.800 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.5179999999999993e-06 2023-09-23T18:58:49.800 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.800 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.800 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:58:49.800 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.209747+0000", 2023-09-23T18:58:49.801 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00037554100000000001 2023-09-23T18:58:49.801 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.801 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.801 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.801 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210122+0000", 2023-09-23T18:58:49.801 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.3209999999999994e-06 2023-09-23T18:58:49.801 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.801 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.801 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:58:49.801 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210132+0000", 2023-09-23T18:58:49.801 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.17254297299999999 2023-09-23T18:58:49.801 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.802 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.802 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.802 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.382675+0000", 2023-09-23T18:58:49.802 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00010313 2023-09-23T18:58:49.802 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.802 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.802 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:58:49.802 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.382778+0000", 2023-09-23T18:58:49.802 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000107827 2023-09-23T18:58:49.802 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.802 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.802 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1", 2023-09-23T18:58:49.803 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.382886+0000", 2023-09-23T18:58:49.803 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001861583 2023-09-23T18:58:49.803 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.803 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.803 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:58:49.803 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.384747+0000", 2023-09-23T18:58:49.803 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0086107649999999994 2023-09-23T18:58:49.803 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.803 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.803 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:58:49.803 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.393358+0000", 2023-09-23T18:58:49.804 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5019999999999998e-05 2023-09-23T18:58:49.804 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.804 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.804 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:58:49.804 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.393413+0000", 2023-09-23T18:58:49.804 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00020373200000000001 2023-09-23T18:58:49.804 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.804 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.804 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:58:49.804 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.393617+0000", 2023-09-23T18:58:49.804 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.18399531299999999 2023-09-23T18:58:49.804 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.804 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:58:49.805 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.805 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.805 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.805 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45602.0:20 9.e 9:7633e671:::benchmark_data_smithi098_206513_object19:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e286)", 2023-09-23T18:58:49.805 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:56:00.209756+0000", 2023-09-23T18:58:49.805 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 169.57503054099999, 2023-09-23T18:58:49.805 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.18411930200000001, 2023-09-23T18:58:49.805 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:58:49.805 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:58:49.805 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:58:49.805 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45602", 2023-09-23T18:58:49.805 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T18:58:49.806 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 20 2023-09-23T18:58:49.806 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.806 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:58:49.806 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.806 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:58:49.806 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.209756+0000", 2023-09-23T18:58:49.806 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:58:49.806 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.806 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.806 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:58:49.806 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.209756+0000", 2023-09-23T18:58:49.806 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:58:49.807 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.807 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.807 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:58:49.807 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.209755+0000", 2023-09-23T18:58:49.807 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000201867 2023-09-23T18:58:49.807 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.807 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.807 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:58:49.807 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.209957+0000", 2023-09-23T18:58:49.807 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5767e-05 2023-09-23T18:58:49.807 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.807 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.807 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:58:49.808 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.209973+0000", 2023-09-23T18:58:49.808 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.533e-06 2023-09-23T18:58:49.808 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.808 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.808 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:58:49.808 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.209982+0000", 2023-09-23T18:58:49.808 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000181046 2023-09-23T18:58:49.808 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.808 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.808 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.808 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210163+0000", 2023-09-23T18:58:49.808 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.3379999999999999e-06 2023-09-23T18:58:49.809 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.809 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.809 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:58:49.809 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210171+0000", 2023-09-23T18:58:49.809 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.17294853299999999 2023-09-23T18:58:49.809 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.809 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.809 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.809 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.383119+0000", 2023-09-23T18:58:49.809 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.7122999999999999e-05 2023-09-23T18:58:49.809 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.809 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.810 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:58:49.810 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.383206+0000", 2023-09-23T18:58:49.810 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000113223 2023-09-23T18:58:49.810 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.810 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.810 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1", 2023-09-23T18:58:49.810 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.383320+0000", 2023-09-23T18:58:49.810 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00143439 2023-09-23T18:58:49.810 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.810 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.810 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:58:49.811 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.384754+0000", 2023-09-23T18:58:49.811 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0090146640000000004 2023-09-23T18:58:49.811 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.811 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.811 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:58:49.811 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.393769+0000", 2023-09-23T18:58:49.811 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.7110999999999998e-05 2023-09-23T18:58:49.811 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.811 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.811 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:58:49.811 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.393806+0000", 2023-09-23T18:58:49.811 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.9988999999999997e-05 2023-09-23T18:58:49.812 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.812 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.812 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:58:49.812 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.393876+0000", 2023-09-23T18:58:49.812 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.18411930200000001 2023-09-23T18:58:49.812 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.812 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:58:49.812 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.812 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.812 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.812 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45602.0:39 9.e 9:74afb8b3:::benchmark_data_smithi098_206513_object38:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e286)", 2023-09-23T18:58:49.812 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:56:00.209992+0000", 2023-09-23T18:58:49.812 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 169.57479474600001, 2023-09-23T18:58:49.813 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.18405242399999999, 2023-09-23T18:58:49.813 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:58:49.813 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:58:49.813 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:58:49.813 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45602", 2023-09-23T18:58:49.813 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T18:58:49.813 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 39 2023-09-23T18:58:49.813 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.813 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:58:49.813 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.814 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:58:49.814 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.209992+0000", 2023-09-23T18:58:49.814 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:58:49.814 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.814 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.814 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:58:49.814 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.209992+0000", 2023-09-23T18:58:49.814 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:58:49.814 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.814 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.814 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:58:49.814 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.209991+0000", 2023-09-23T18:58:49.815 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000148827 2023-09-23T18:58:49.815 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.815 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.815 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:58:49.815 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210140+0000", 2023-09-23T18:58:49.815 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.235e-05 2023-09-23T18:58:49.815 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.815 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.815 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:58:49.815 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210152+0000", 2023-09-23T18:58:49.815 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.038e-06 2023-09-23T18:58:49.815 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.816 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.816 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:58:49.816 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210160+0000", 2023-09-23T18:58:49.816 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.6792000000000003e-05 2023-09-23T18:58:49.816 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.816 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.816 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.816 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210197+0000", 2023-09-23T18:58:49.816 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.3379999999999999e-06 2023-09-23T18:58:49.816 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.816 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.816 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:58:49.817 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210204+0000", 2023-09-23T18:58:49.817 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.17334285599999999 2023-09-23T18:58:49.817 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.817 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.817 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.817 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.383547+0000", 2023-09-23T18:58:49.817 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000106518 2023-09-23T18:58:49.817 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.817 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.817 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:58:49.817 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.383654+0000", 2023-09-23T18:58:49.817 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00010210200000000001 2023-09-23T18:58:49.817 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.817 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.818 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1", 2023-09-23T18:58:49.818 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.383756+0000", 2023-09-23T18:58:49.818 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001970795 2023-09-23T18:58:49.818 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.818 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.818 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:58:49.818 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.385727+0000", 2023-09-23T18:58:49.818 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0081950070000000007 2023-09-23T18:58:49.818 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.818 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.818 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:58:49.818 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.393922+0000", 2023-09-23T18:58:49.818 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.3980999999999999e-05 2023-09-23T18:58:49.819 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.819 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.819 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:58:49.819 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.393956+0000", 2023-09-23T18:58:49.819 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.9061999999999995e-05 2023-09-23T18:58:49.819 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.819 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.819 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:58:49.819 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.394045+0000", 2023-09-23T18:58:49.819 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.18405242399999999 2023-09-23T18:58:49.819 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.819 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:58:49.819 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.820 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.820 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.820 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45602.0:46 9.e 9:77b8483d:::benchmark_data_smithi098_206513_object45:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e286)", 2023-09-23T18:58:49.820 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:56:00.210169+0000", 2023-09-23T18:58:49.820 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 169.574617757, 2023-09-23T18:58:49.820 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.184085059, 2023-09-23T18:58:49.820 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:58:49.820 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:58:49.820 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:58:49.820 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45602", 2023-09-23T18:58:49.820 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T18:58:49.820 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 46 2023-09-23T18:58:49.820 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.821 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:58:49.821 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.821 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:58:49.821 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210169+0000", 2023-09-23T18:58:49.821 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:58:49.821 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.821 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.821 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:58:49.821 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210169+0000", 2023-09-23T18:58:49.821 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:58:49.821 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.821 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.821 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:58:49.822 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210168+0000", 2023-09-23T18:58:49.822 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.1543000000000003e-05 2023-09-23T18:58:49.822 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.822 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.822 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:58:49.822 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210240+0000", 2023-09-23T18:58:49.822 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3026e-05 2023-09-23T18:58:49.822 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.822 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.822 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:58:49.822 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210253+0000", 2023-09-23T18:58:49.822 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.9359999999999999e-06 2023-09-23T18:58:49.822 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.823 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.823 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:58:49.823 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210261+0000", 2023-09-23T18:58:49.823 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.0159e-05 2023-09-23T18:58:49.823 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.823 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.823 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.823 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210301+0000", 2023-09-23T18:58:49.823 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1900000000000003e-06 2023-09-23T18:58:49.823 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.823 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.823 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:58:49.823 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210306+0000", 2023-09-23T18:58:49.824 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.17364142299999999 2023-09-23T18:58:49.824 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.824 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.824 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.824 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.383947+0000", 2023-09-23T18:58:49.824 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00011234999999999999 2023-09-23T18:58:49.824 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.824 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.824 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:58:49.824 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.384060+0000", 2023-09-23T18:58:49.824 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0001016 2023-09-23T18:58:49.824 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.825 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.825 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1", 2023-09-23T18:58:49.825 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.384161+0000", 2023-09-23T18:58:49.825 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00157143 2023-09-23T18:58:49.825 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.825 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.825 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:58:49.825 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.385733+0000", 2023-09-23T18:58:49.825 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0083538439999999992 2023-09-23T18:58:49.825 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.825 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.825 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:58:49.825 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.394087+0000", 2023-09-23T18:58:49.826 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.0226000000000002e-05 2023-09-23T18:58:49.826 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.826 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.826 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:58:49.826 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.394147+0000", 2023-09-23T18:58:49.826 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000107533 2023-09-23T18:58:49.826 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.826 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.826 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:58:49.826 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.394254+0000", 2023-09-23T18:58:49.826 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.184085059 2023-09-23T18:58:49.826 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.827 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:58:49.827 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.827 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.827 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.827 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45602.0:55 9.e 9:7ad7791b:::benchmark_data_smithi098_206513_object54:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e286)", 2023-09-23T18:58:49.827 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:56:00.210275+0000", 2023-09-23T18:58:49.827 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 169.574511766, 2023-09-23T18:58:49.827 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.184608946, 2023-09-23T18:58:49.827 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:58:49.827 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:58:49.827 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:58:49.827 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45602", 2023-09-23T18:58:49.827 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T18:58:49.828 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 55 2023-09-23T18:58:49.828 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.828 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:58:49.828 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.828 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:58:49.828 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210275+0000", 2023-09-23T18:58:49.828 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:58:49.828 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.828 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.828 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:58:49.828 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210275+0000", 2023-09-23T18:58:49.828 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:58:49.828 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.829 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.829 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:58:49.829 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210274+0000", 2023-09-23T18:58:49.829 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.0093000000000003e-05 2023-09-23T18:58:49.829 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.829 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.829 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:58:49.829 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210324+0000", 2023-09-23T18:58:49.829 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2092e-05 2023-09-23T18:58:49.829 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.829 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.829 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:58:49.829 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210336+0000", 2023-09-23T18:58:49.830 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.7660000000000004e-06 2023-09-23T18:58:49.830 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.830 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.830 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:58:49.830 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210345+0000", 2023-09-23T18:58:49.830 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.9419000000000001e-05 2023-09-23T18:58:49.830 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.830 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.830 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.830 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210374+0000", 2023-09-23T18:58:49.830 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4140000000000002e-06 2023-09-23T18:58:49.831 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.831 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.831 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:58:49.831 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210380+0000", 2023-09-23T18:58:49.831 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.17395244700000001 2023-09-23T18:58:49.831 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.831 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.831 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.831 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.384332+0000", 2023-09-23T18:58:49.831 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.8369000000000006e-05 2023-09-23T18:58:49.831 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.831 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.831 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:58:49.832 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.384431+0000", 2023-09-23T18:58:49.832 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00011340099999999999 2023-09-23T18:58:49.832 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.832 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.832 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1", 2023-09-23T18:58:49.832 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.384544+0000", 2023-09-23T18:58:49.832 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0012687449999999999 2023-09-23T18:58:49.832 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.832 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.832 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:58:49.832 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.385813+0000", 2023-09-23T18:58:49.832 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0088959139999999996 2023-09-23T18:58:49.833 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.833 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.833 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:58:49.833 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.394709+0000", 2023-09-23T18:58:49.833 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.3712000000000003e-05 2023-09-23T18:58:49.833 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.833 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.833 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:58:49.833 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.394772+0000", 2023-09-23T18:58:49.833 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000111743 2023-09-23T18:58:49.833 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.833 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.834 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:58:49.834 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.394884+0000", 2023-09-23T18:58:49.834 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.184608946 2023-09-23T18:58:49.834 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.834 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:58:49.834 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.834 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.834 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.834 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45602.0:81 9.e 9:7f1c200d:::benchmark_data_smithi098_206513_object80:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e286)", 2023-09-23T18:58:49.834 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:56:00.210370+0000", 2023-09-23T18:58:49.834 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 169.57441690799999, 2023-09-23T18:58:49.834 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.18473566, 2023-09-23T18:58:49.835 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:58:49.835 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:58:49.835 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:58:49.835 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45602", 2023-09-23T18:58:49.835 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T18:58:49.835 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 81 2023-09-23T18:58:49.835 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.835 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:58:49.835 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.835 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:58:49.835 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210370+0000", 2023-09-23T18:58:49.835 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:58:49.836 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.836 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.836 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:58:49.836 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210370+0000", 2023-09-23T18:58:49.836 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:58:49.836 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.836 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.836 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:58:49.836 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210369+0000", 2023-09-23T18:58:49.836 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.7970999999999999e-05 2023-09-23T18:58:49.836 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.836 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.836 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:58:49.837 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210417+0000", 2023-09-23T18:58:49.837 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1800000000000001e-05 2023-09-23T18:58:49.837 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.837 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.837 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:58:49.837 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210429+0000", 2023-09-23T18:58:49.837 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.1634e-05 2023-09-23T18:58:49.837 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.837 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.837 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:58:49.837 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210451+0000", 2023-09-23T18:58:49.837 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.7984999999999994e-05 2023-09-23T18:58:49.838 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.838 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.838 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.838 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210519+0000", 2023-09-23T18:58:49.838 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.5196000000000001e-05 2023-09-23T18:58:49.838 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.838 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.838 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:58:49.838 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210544+0000", 2023-09-23T18:58:49.838 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.17423998099999999 2023-09-23T18:58:49.838 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.838 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.838 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.839 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.384784+0000", 2023-09-23T18:58:49.839 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.1134e-05 2023-09-23T18:58:49.839 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.839 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.839 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:58:49.839 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.384855+0000", 2023-09-23T18:58:49.839 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000179481 2023-09-23T18:58:49.839 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.839 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.839 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1", 2023-09-23T18:58:49.839 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.385034+0000", 2023-09-23T18:58:49.839 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0016065579999999999 2023-09-23T18:58:49.840 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.840 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.840 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:58:49.840 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.386641+0000", 2023-09-23T18:58:49.840 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0083137630000000001 2023-09-23T18:58:49.840 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.840 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.840 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:58:49.840 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.394955+0000", 2023-09-23T18:58:49.840 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4988000000000003e-05 2023-09-23T18:58:49.840 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.840 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.841 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:58:49.841 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.395010+0000", 2023-09-23T18:58:49.841 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.6112000000000004e-05 2023-09-23T18:58:49.841 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.841 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.841 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:58:49.841 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.395106+0000", 2023-09-23T18:58:49.841 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.18473566 2023-09-23T18:58:49.841 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.841 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:58:49.841 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.841 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.842 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.842 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45602.0:111 9.e 9:7e8c86d6:::benchmark_data_smithi098_206513_object110:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e286)", 2023-09-23T18:58:49.842 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:56:00.210460+0000", 2023-09-23T18:58:49.842 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 169.57432719900001, 2023-09-23T18:58:49.842 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.18498015600000001, 2023-09-23T18:58:49.842 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:58:49.842 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:58:49.842 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:58:49.842 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45602", 2023-09-23T18:58:49.842 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T18:58:49.842 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 111 2023-09-23T18:58:49.842 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.842 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:58:49.843 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.843 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:58:49.843 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210460+0000", 2023-09-23T18:58:49.843 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:58:49.843 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.843 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.843 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:58:49.843 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210460+0000", 2023-09-23T18:58:49.843 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T18:58:49.843 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.843 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.843 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:58:49.844 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210459+0000", 2023-09-23T18:58:49.844 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.4403999999999997e-05 2023-09-23T18:58:49.844 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.844 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.844 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:58:49.844 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210503+0000", 2023-09-23T18:58:49.844 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0763e-05 2023-09-23T18:58:49.844 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.844 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.844 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:58:49.844 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210514+0000", 2023-09-23T18:58:49.844 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.7619999999999995e-06 2023-09-23T18:58:49.845 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.845 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.845 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:58:49.845 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210522+0000", 2023-09-23T18:58:49.845 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.2527999999999995e-05 2023-09-23T18:58:49.845 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.845 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.845 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.845 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210584+0000", 2023-09-23T18:58:49.845 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0845e-05 2023-09-23T18:58:49.845 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.845 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.846 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:58:49.846 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.210595+0000", 2023-09-23T18:58:49.846 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.17472475800000001 2023-09-23T18:58:49.846 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.846 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.846 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.846 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.385320+0000", 2023-09-23T18:58:49.846 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.9479000000000003e-05 2023-09-23T18:58:49.846 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.846 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.846 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:58:49.847 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.385389+0000", 2023-09-23T18:58:49.847 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012468200000000001 2023-09-23T18:58:49.847 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.847 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.847 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1", 2023-09-23T18:58:49.847 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.385514+0000", 2023-09-23T18:58:49.847 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0013177219999999999 2023-09-23T18:58:49.847 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.847 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.847 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:58:49.847 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.386832+0000", 2023-09-23T18:58:49.847 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.008425742 2023-09-23T18:58:49.848 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.848 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.848 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:58:49.848 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.395258+0000", 2023-09-23T18:58:49.848 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.5468000000000004e-05 2023-09-23T18:58:49.848 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.848 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.848 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:58:49.848 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.395323+0000", 2023-09-23T18:58:49.848 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000116997 2023-09-23T18:58:49.848 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.848 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.848 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:58:49.849 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:56:00.395440+0000", 2023-09-23T18:58:49.849 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.18498015600000001 2023-09-23T18:58:49.849 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.849 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:58:49.849 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.849 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.849 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.849 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45602.0:116225 9.e 9:7f01854e:::benchmark_data_smithi098_206513_object20686:head [delete] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-09-23T18:58:49.849 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:57:39.618074+0000", 2023-09-23T18:58:49.849 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 70.166712872999994, 2023-09-23T18:58:49.849 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.30481356500000001, 2023-09-23T18:58:49.849 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:58:49.850 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:58:49.850 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:58:49.850 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45602", 2023-09-23T18:58:49.850 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T18:58:49.850 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 116225 2023-09-23T18:58:49.850 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.850 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:58:49.850 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.850 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:58:49.850 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.618074+0000", 2023-09-23T18:58:49.850 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:58:49.850 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.851 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.851 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:58:49.851 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.618074+0000", 2023-09-23T18:58:49.851 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:58:49.851 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.851 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.851 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:58:49.851 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.618072+0000", 2023-09-23T18:58:49.851 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.1600000000000004e-06 2023-09-23T18:58:49.851 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.851 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.851 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:58:49.851 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.618082+0000", 2023-09-23T18:58:49.851 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3196e-05 2023-09-23T18:58:49.852 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.852 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.852 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:58:49.852 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.618095+0000", 2023-09-23T18:58:49.852 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.9779999999999999e-06 2023-09-23T18:58:49.852 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.852 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.852 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:58:49.852 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.618102+0000", 2023-09-23T18:58:49.852 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.2113999999999995e-05 2023-09-23T18:58:49.852 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.852 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.852 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.853 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.618164+0000", 2023-09-23T18:58:49.853 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018282800000000001 2023-09-23T18:58:49.853 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.853 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.853 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:58:49.853 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.618347+0000", 2023-09-23T18:58:49.853 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000101718 2023-09-23T18:58:49.853 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.853 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.853 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1", 2023-09-23T18:58:49.853 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.618448+0000", 2023-09-23T18:58:49.853 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0034965859999999999 2023-09-23T18:58:49.854 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.854 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.854 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:58:49.854 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.621945+0000", 2023-09-23T18:58:49.854 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.30056107900000001 2023-09-23T18:58:49.854 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.854 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.854 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:58:49.854 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.922506+0000", 2023-09-23T18:58:49.854 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012484200000000001 2023-09-23T18:58:49.854 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.854 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.855 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:58:49.855 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.922631+0000", 2023-09-23T18:58:49.855 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00025681499999999997 2023-09-23T18:58:49.855 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.855 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.855 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:58:49.855 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.922888+0000", 2023-09-23T18:58:49.855 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.30481356500000001 2023-09-23T18:58:49.855 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.855 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:58:49.855 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.855 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.855 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.856 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45602.0:116245 9.e 9:790244ee:::benchmark_data_smithi098_206513_object20706:head [delete] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-09-23T18:58:49.856 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:57:39.633372+0000", 2023-09-23T18:58:49.856 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 70.151414932999998, 2023-09-23T18:58:49.856 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.30539886799999999, 2023-09-23T18:58:49.856 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:58:49.856 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:58:49.856 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:58:49.856 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45602", 2023-09-23T18:58:49.856 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T18:58:49.856 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 116245 2023-09-23T18:58:49.856 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.856 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:58:49.857 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.857 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:58:49.857 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.633372+0000", 2023-09-23T18:58:49.857 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:58:49.857 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.857 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.857 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:58:49.857 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.633372+0000", 2023-09-23T18:58:49.857 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999971 2023-09-23T18:58:49.857 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.857 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.857 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:58:49.857 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.633369+0000", 2023-09-23T18:58:49.858 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2925000000000001e-05 2023-09-23T18:58:49.858 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.858 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.858 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:58:49.858 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.633382+0000", 2023-09-23T18:58:49.858 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4644e-05 2023-09-23T18:58:49.858 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.858 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.858 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:58:49.858 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.633407+0000", 2023-09-23T18:58:49.858 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0509999999999999e-05 2023-09-23T18:58:49.858 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.858 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.859 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:58:49.859 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.633417+0000", 2023-09-23T18:58:49.859 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.7749000000000003e-05 2023-09-23T18:58:49.859 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.859 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.859 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.859 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.633505+0000", 2023-09-23T18:58:49.859 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000151569 2023-09-23T18:58:49.859 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.859 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.859 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:58:49.859 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.633657+0000", 2023-09-23T18:58:49.859 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013305500000000001 2023-09-23T18:58:49.860 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.860 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.860 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1", 2023-09-23T18:58:49.860 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.633790+0000", 2023-09-23T18:58:49.860 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0042215509999999996 2023-09-23T18:58:49.860 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.860 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.860 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:58:49.860 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.638011+0000", 2023-09-23T18:58:49.860 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.30050648099999999 2023-09-23T18:58:49.860 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.860 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.861 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:58:49.861 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.938518+0000", 2023-09-23T18:58:49.861 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.1428999999999997e-05 2023-09-23T18:58:49.861 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.861 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.861 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:58:49.861 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.938589+0000", 2023-09-23T18:58:49.861 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018179099999999999 2023-09-23T18:58:49.861 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.861 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.861 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:58:49.861 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.938771+0000", 2023-09-23T18:58:49.861 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.30539886799999999 2023-09-23T18:58:49.862 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.862 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:58:49.862 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.862 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.862 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.862 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45602.0:116262 9.e 9:7f8a9e74:::benchmark_data_smithi098_206513_object20723:head [delete] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-09-23T18:58:49.862 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:57:39.652759+0000", 2023-09-23T18:58:49.862 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 70.132028521999999, 2023-09-23T18:58:49.862 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.28846333200000002, 2023-09-23T18:58:49.862 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:58:49.862 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:58:49.862 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:58:49.863 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45602", 2023-09-23T18:58:49.863 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T18:58:49.863 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 116262 2023-09-23T18:58:49.863 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.863 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:58:49.863 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.863 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:58:49.863 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.652759+0000", 2023-09-23T18:58:49.863 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:58:49.863 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.863 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.863 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:58:49.864 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.652759+0000", 2023-09-23T18:58:49.864 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:58:49.864 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.864 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.864 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:58:49.864 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.652756+0000", 2023-09-23T18:58:49.864 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0611000000000001e-05 2023-09-23T18:58:49.864 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.864 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.864 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:58:49.864 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.652767+0000", 2023-09-23T18:58:49.865 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8411999999999999e-05 2023-09-23T18:58:49.865 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.865 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.865 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:58:49.865 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.652785+0000", 2023-09-23T18:58:49.865 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.9740000000000002e-06 2023-09-23T18:58:49.865 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.865 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.865 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:58:49.865 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.652793+0000", 2023-09-23T18:58:49.865 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5819000000000002e-05 2023-09-23T18:58:49.865 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.866 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.866 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.866 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.652849+0000", 2023-09-23T18:58:49.866 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000115534 2023-09-23T18:58:49.866 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.866 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.866 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:58:49.866 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.652965+0000", 2023-09-23T18:58:49.866 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00011252900000000001 2023-09-23T18:58:49.866 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.866 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.866 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1", 2023-09-23T18:58:49.867 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.653077+0000", 2023-09-23T18:58:49.867 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.003303889 2023-09-23T18:58:49.867 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.867 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.867 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:58:49.867 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.656381+0000", 2023-09-23T18:58:49.867 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.28461272900000001 2023-09-23T18:58:49.867 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.867 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.867 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:58:49.867 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.940994+0000", 2023-09-23T18:58:49.867 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.2363999999999999e-05 2023-09-23T18:58:49.868 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.868 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.868 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:58:49.868 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.941076+0000", 2023-09-23T18:58:49.868 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014551700000000001 2023-09-23T18:58:49.868 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.868 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.868 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:58:49.868 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:57:39.941222+0000", 2023-09-23T18:58:49.868 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.28846333200000002 2023-09-23T18:58:49.868 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.868 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:58:49.869 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.869 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.869 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.869 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45602.0:135138 9.e 9:7ffe54f9:::benchmark_data_smithi098_206513_object39599:head [delete] snapc 0=[] ondisk+write+known_if_redirected e292)", 2023-09-23T18:58:49.869 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:58:04.312487+0000", 2023-09-23T18:58:49.869 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 45.472300218000001, 2023-09-23T18:58:49.869 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15359726700000001, 2023-09-23T18:58:49.869 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:58:49.869 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:58:49.869 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:58:49.869 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45602", 2023-09-23T18:58:49.869 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T18:58:49.870 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 135138 2023-09-23T18:58:49.870 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.870 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:58:49.870 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.870 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:58:49.870 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.312487+0000", 2023-09-23T18:58:49.870 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:58:49.870 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.870 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.870 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:58:49.870 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.312487+0000", 2023-09-23T18:58:49.870 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-09-23T18:58:49.870 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.871 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.871 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:58:49.871 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.312485+0000", 2023-09-23T18:58:49.871 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0156999999999999e-05 2023-09-23T18:58:49.871 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.871 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.871 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:58:49.871 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.312495+0000", 2023-09-23T18:58:49.871 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4484000000000001e-05 2023-09-23T18:58:49.871 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.871 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.871 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:58:49.872 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.312509+0000", 2023-09-23T18:58:49.872 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.9270000000000002e-06 2023-09-23T18:58:49.872 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.872 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.872 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:58:49.872 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.312515+0000", 2023-09-23T18:58:49.872 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.7647e-05 2023-09-23T18:58:49.872 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.872 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.872 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.872 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.312563+0000", 2023-09-23T18:58:49.872 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014139999999999999 2023-09-23T18:58:49.872 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.873 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.873 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:58:49.873 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.312704+0000", 2023-09-23T18:58:49.873 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000131505 2023-09-23T18:58:49.873 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.873 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.873 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1", 2023-09-23T18:58:49.873 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.312836+0000", 2023-09-23T18:58:49.873 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0033043059999999999 2023-09-23T18:58:49.873 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.873 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.873 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:58:49.874 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.316140+0000", 2023-09-23T18:58:49.874 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.14972044200000001 2023-09-23T18:58:49.874 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.874 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.874 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:58:49.874 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.465861+0000", 2023-09-23T18:58:49.874 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.6854999999999995e-05 2023-09-23T18:58:49.874 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.874 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.874 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:58:49.874 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.465937+0000", 2023-09-23T18:58:49.874 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014669300000000001 2023-09-23T18:58:49.875 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.875 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.875 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:58:49.875 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.466084+0000", 2023-09-23T18:58:49.875 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15359726700000001 2023-09-23T18:58:49.875 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.875 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:58:49.875 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.875 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.875 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.875 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45602.0:135149 9.e 9:7d9ee60a:::benchmark_data_smithi098_206513_object39610:head [delete] snapc 0=[] ondisk+write+known_if_redirected e292)", 2023-09-23T18:58:49.875 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:58:04.325969+0000", 2023-09-23T18:58:49.875 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 45.458818436999998, 2023-09-23T18:58:49.876 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15356434999999999, 2023-09-23T18:58:49.876 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:58:49.876 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:58:49.876 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:58:49.876 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45602", 2023-09-23T18:58:49.876 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T18:58:49.876 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 135149 2023-09-23T18:58:49.876 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.876 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:58:49.876 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.876 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:58:49.876 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.325969+0000", 2023-09-23T18:58:49.876 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:58:49.877 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.877 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.877 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:58:49.877 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.325969+0000", 2023-09-23T18:58:49.877 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:58:49.877 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.877 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.877 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:58:49.877 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.325967+0000", 2023-09-23T18:58:49.877 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2624e-05 2023-09-23T18:58:49.877 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.877 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.878 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:58:49.878 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.325979+0000", 2023-09-23T18:58:49.878 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7995999999999998e-05 2023-09-23T18:58:49.878 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.878 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.878 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:58:49.878 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.325997+0000", 2023-09-23T18:58:49.878 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.6599999999999998e-06 2023-09-23T18:58:49.878 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.878 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.878 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:58:49.878 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.326004+0000", 2023-09-23T18:58:49.879 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6390000000000001e-05 2023-09-23T18:58:49.879 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.879 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.879 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.879 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.326060+0000", 2023-09-23T18:58:49.879 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013097799999999999 2023-09-23T18:58:49.879 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.879 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.879 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:58:49.879 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.326191+0000", 2023-09-23T18:58:49.879 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000128286 2023-09-23T18:58:49.879 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.879 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.879 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1", 2023-09-23T18:58:49.880 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.326320+0000", 2023-09-23T18:58:49.880 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0047285970000000002 2023-09-23T18:58:49.880 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.880 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.880 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:58:49.880 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.331048+0000", 2023-09-23T18:58:49.880 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.14823182200000001 2023-09-23T18:58:49.880 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.880 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.880 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:58:49.880 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.479280+0000", 2023-09-23T18:58:49.880 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.7844999999999997e-05 2023-09-23T18:58:49.880 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.880 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.881 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T18:58:49.881 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.479368+0000", 2023-09-23T18:58:49.881 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016507199999999999 2023-09-23T18:58:49.881 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.881 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.881 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:58:49.881 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:04.479533+0000", 2023-09-23T18:58:49.881 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15356434999999999 2023-09-23T18:58:49.881 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.881 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:58:49.881 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.881 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.882 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.882 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45602.0:147831 9.c 9:3a351018:::benchmark_data_smithi098_206513_object52292:head [delete] snapc 0=[] ondisk+write+known_if_redirected e297)", 2023-09-23T18:58:49.882 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:58:37.660715+0000", 2023-09-23T18:58:49.882 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 12.124071948999999, 2023-09-23T18:58:49.882 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0518191670000001, 2023-09-23T18:58:49.882 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:58:49.882 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T18:58:49.882 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:58:49.882 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45602", 2023-09-23T18:58:49.882 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T18:58:49.882 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 147831 2023-09-23T18:58:49.882 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.883 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:58:49.883 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.883 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:58:49.883 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.660715+0000", 2023-09-23T18:58:49.883 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:58:49.883 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.883 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.883 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:58:49.883 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.660715+0000", 2023-09-23T18:58:49.883 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:58:49.883 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.883 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.884 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:58:49.884 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.660714+0000", 2023-09-23T18:58:49.884 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.1249999999999993e-06 2023-09-23T18:58:49.884 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.884 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.884 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:58:49.884 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.660722+0000", 2023-09-23T18:58:49.884 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2208999999999999e-05 2023-09-23T18:58:49.884 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.884 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.884 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:58:49.884 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.660734+0000", 2023-09-23T18:58:49.884 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.4120000000000004e-06 2023-09-23T18:58:49.885 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.885 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.885 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:58:49.885 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.660741+0000", 2023-09-23T18:58:49.885 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0039898360000000001 2023-09-23T18:58:49.885 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.885 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.885 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.885 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.664731+0000", 2023-09-23T18:58:49.885 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012436899999999999 2023-09-23T18:58:49.885 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.885 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.885 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for rw locks", 2023-09-23T18:58:49.886 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.664856+0000", 2023-09-23T18:58:49.886 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.040395831 2023-09-23T18:58:49.886 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.886 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.886 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.886 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.705251+0000", 2023-09-23T18:58:49.886 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.4739999999999999e-06 2023-09-23T18:58:49.886 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.886 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.886 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:58:49.886 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.705259+0000", 2023-09-23T18:58:49.886 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0072612670000001 2023-09-23T18:58:49.887 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.887 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.887 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.887 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:38.712520+0000", 2023-09-23T18:58:49.887 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4106e-05 2023-09-23T18:58:49.887 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.887 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.887 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:58:49.887 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:38.712534+0000", 2023-09-23T18:58:49.887 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0518191670000001 2023-09-23T18:58:49.887 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.887 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:58:49.887 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.888 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.888 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.888 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45602.0:147838 9.c 9:34ac855a:::benchmark_data_smithi098_206513_object52299:head [delete] snapc 0=[] ondisk+write+known_if_redirected e297)", 2023-09-23T18:58:49.888 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:58:37.666196+0000", 2023-09-23T18:58:49.888 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 12.118591181999999, 2023-09-23T18:58:49.888 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.045678506, 2023-09-23T18:58:49.888 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:58:49.888 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T18:58:49.888 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:58:49.888 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45602", 2023-09-23T18:58:49.888 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T18:58:49.888 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 147838 2023-09-23T18:58:49.889 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.889 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:58:49.889 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.889 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:58:49.889 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.666196+0000", 2023-09-23T18:58:49.889 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:58:49.889 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.889 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.889 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:58:49.889 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.666196+0000", 2023-09-23T18:58:49.889 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:58:49.889 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.889 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.890 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:58:49.890 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.666194+0000", 2023-09-23T18:58:49.890 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.8380000000000002e-06 2023-09-23T18:58:49.890 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.890 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.890 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:58:49.890 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.666202+0000", 2023-09-23T18:58:49.890 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4725999999999999e-05 2023-09-23T18:58:49.890 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.890 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.890 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:58:49.890 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.666217+0000", 2023-09-23T18:58:49.890 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.5220000000000004e-06 2023-09-23T18:58:49.891 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.891 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.891 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:58:49.891 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.666223+0000", 2023-09-23T18:58:49.891 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0021543819999999998 2023-09-23T18:58:49.891 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.891 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.891 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.891 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.668378+0000", 2023-09-23T18:58:49.891 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00028419099999999999 2023-09-23T18:58:49.891 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.891 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.892 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:58:49.892 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.668662+0000", 2023-09-23T18:58:49.892 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018866900000000001 2023-09-23T18:58:49.892 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.892 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.892 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,3", 2023-09-23T18:58:49.892 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.668850+0000", 2023-09-23T18:58:49.892 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0036318850000000001 2023-09-23T18:58:49.892 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.892 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.892 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:58:49.892 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.672482+0000", 2023-09-23T18:58:49.893 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.017954646000000001 2023-09-23T18:58:49.893 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.893 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.893 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:58:49.893 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.690437+0000", 2023-09-23T18:58:49.893 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.013974222999999999 2023-09-23T18:58:49.893 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.893 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.893 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.893 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.704411+0000", 2023-09-23T18:58:49.893 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0546e-05 2023-09-23T18:58:49.893 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.893 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.894 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:58:49.894 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.704422+0000", 2023-09-23T18:58:49.894 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.007432799 2023-09-23T18:58:49.894 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.894 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.894 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.894 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:38.711854+0000", 2023-09-23T18:58:49.894 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9851e-05 2023-09-23T18:58:49.894 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.894 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.894 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:58:49.894 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:38.711874+0000", 2023-09-23T18:58:49.894 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.045678506 2023-09-23T18:58:49.895 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.895 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:58:49.895 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.895 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.895 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.895 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45602.0:147840 9.c 9:399a3709:::benchmark_data_smithi098_206513_object52301:head [delete] snapc 0=[] ondisk+write+known_if_redirected e297)", 2023-09-23T18:58:49.895 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:58:37.673492+0000", 2023-09-23T18:58:49.895 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 12.111294781, 2023-09-23T18:58:49.895 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0388951820000001, 2023-09-23T18:58:49.895 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:58:49.895 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T18:58:49.895 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:58:49.895 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45602", 2023-09-23T18:58:49.896 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T18:58:49.896 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 147840 2023-09-23T18:58:49.896 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.896 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:58:49.896 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.896 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:58:49.896 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.673492+0000", 2023-09-23T18:58:49.896 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:58:49.896 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.896 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.896 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:58:49.896 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.673492+0000", 2023-09-23T18:58:49.896 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:58:49.897 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.897 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.897 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:58:49.897 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.673490+0000", 2023-09-23T18:58:49.897 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.5920000000000007e-06 2023-09-23T18:58:49.897 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.897 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.897 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:58:49.897 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.673500+0000", 2023-09-23T18:58:49.897 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6132000000000001e-05 2023-09-23T18:58:49.897 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.897 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.897 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:58:49.898 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.673516+0000", 2023-09-23T18:58:49.898 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.7089999999999996e-06 2023-09-23T18:58:49.898 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.898 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.898 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:58:49.898 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.673523+0000", 2023-09-23T18:58:49.898 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4360000000000001e-05 2023-09-23T18:58:49.898 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.898 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.898 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.898 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.673577+0000", 2023-09-23T18:58:49.898 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000189247 2023-09-23T18:58:49.898 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.899 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.899 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:58:49.899 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.673766+0000", 2023-09-23T18:58:49.899 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00022378400000000001 2023-09-23T18:58:49.899 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.899 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.899 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,3", 2023-09-23T18:58:49.899 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.673990+0000", 2023-09-23T18:58:49.899 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0042074460000000001 2023-09-23T18:58:49.899 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.899 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.899 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:58:49.899 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.678198+0000", 2023-09-23T18:58:49.900 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.026942185 2023-09-23T18:58:49.900 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.900 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.900 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.900 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.705140+0000", 2023-09-23T18:58:49.900 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.8300000000000002e-06 2023-09-23T18:58:49.900 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.900 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.900 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:58:49.900 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.705149+0000", 2023-09-23T18:58:49.900 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.007222818 2023-09-23T18:58:49.900 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.901 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.901 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.901 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:38.712371+0000", 2023-09-23T18:58:49.901 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5931999999999999e-05 2023-09-23T18:58:49.901 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.901 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.901 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:58:49.901 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:38.712387+0000", 2023-09-23T18:58:49.901 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0388951820000001 2023-09-23T18:58:49.901 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.901 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:58:49.901 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.902 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.902 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.902 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45602.0:147841 9.c 9:37630b49:::benchmark_data_smithi098_206513_object52302:head [delete] snapc 0=[] ondisk+write+known_if_redirected e297)", 2023-09-23T18:58:49.902 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:58:37.682961+0000", 2023-09-23T18:58:49.902 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 12.101826283999999, 2023-09-23T18:58:49.902 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0294792070000001, 2023-09-23T18:58:49.902 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:58:49.902 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T18:58:49.902 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:58:49.902 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45602", 2023-09-23T18:58:49.902 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T18:58:49.902 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 147841 2023-09-23T18:58:49.902 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.903 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:58:49.903 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.903 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:58:49.903 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.682961+0000", 2023-09-23T18:58:49.903 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:58:49.903 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.903 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.903 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:58:49.903 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.682961+0000", 2023-09-23T18:58:49.903 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T18:58:49.903 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.903 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.904 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:58:49.904 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.682959+0000", 2023-09-23T18:58:49.904 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.3869999999999995e-06 2023-09-23T18:58:49.904 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.904 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.904 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:58:49.904 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.682968+0000", 2023-09-23T18:58:49.904 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3526999999999999e-05 2023-09-23T18:58:49.904 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.904 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.904 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:58:49.904 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.682981+0000", 2023-09-23T18:58:49.904 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.156e-06 2023-09-23T18:58:49.905 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.905 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.905 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:58:49.905 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.682987+0000", 2023-09-23T18:58:49.905 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.74e-05 2023-09-23T18:58:49.905 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.905 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.905 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.905 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.683035+0000", 2023-09-23T18:58:49.905 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000148677 2023-09-23T18:58:49.905 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.905 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.905 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T18:58:49.906 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.683183+0000", 2023-09-23T18:58:49.906 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013945400000000001 2023-09-23T18:58:49.906 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.906 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.906 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,3", 2023-09-23T18:58:49.906 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.683323+0000", 2023-09-23T18:58:49.906 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0032732099999999999 2023-09-23T18:58:49.906 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.906 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.906 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T18:58:49.906 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.686596+0000", 2023-09-23T18:58:49.906 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.018616540000000001 2023-09-23T18:58:49.907 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.907 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.907 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.907 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.705213+0000", 2023-09-23T18:58:49.907 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.7190000000000001e-06 2023-09-23T18:58:49.907 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.907 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.907 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:58:49.907 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.705220+0000", 2023-09-23T18:58:49.907 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.007206466 2023-09-23T18:58:49.907 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.907 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.907 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.908 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:38.712427+0000", 2023-09-23T18:58:49.908 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3190000000000001e-05 2023-09-23T18:58:49.908 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.908 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.908 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:58:49.908 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:38.712440+0000", 2023-09-23T18:58:49.908 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0294792070000001 2023-09-23T18:58:49.908 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.908 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:58:49.908 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.908 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.908 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.909 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45602.0:147853 9.c 9.3bba945c (undecoded) ondisk+write+known_if_redirected e298)", 2023-09-23T18:58:49.909 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:58:37.706605+0000", 2023-09-23T18:58:49.909 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 12.078182444999999, 2023-09-23T18:58:49.909 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.005982999, 2023-09-23T18:58:49.909 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:58:49.909 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T18:58:49.909 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:58:49.909 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45602", 2023-09-23T18:58:49.909 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T18:58:49.909 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 147853 2023-09-23T18:58:49.909 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.909 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:58:49.910 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.910 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:58:49.910 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.706605+0000", 2023-09-23T18:58:49.910 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:58:49.910 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.910 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.910 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:58:49.910 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.706605+0000", 2023-09-23T18:58:49.910 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T18:58:49.910 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.910 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.910 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:58:49.911 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.706603+0000", 2023-09-23T18:58:49.911 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.6580000000000002e-06 2023-09-23T18:58:49.911 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.911 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.911 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:58:49.911 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.706611+0000", 2023-09-23T18:58:49.911 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0634e-05 2023-09-23T18:58:49.911 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.911 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.911 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:58:49.911 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.706621+0000", 2023-09-23T18:58:49.911 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.4560000000000003e-06 2023-09-23T18:58:49.912 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.912 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.912 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:58:49.912 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.706629+0000", 2023-09-23T18:58:49.912 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012869800000000001 2023-09-23T18:58:49.912 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.912 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.912 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.912 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.706757+0000", 2023-09-23T18:58:49.912 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.8980000000000008e-06 2023-09-23T18:58:49.912 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.912 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.912 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:58:49.913 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.706767+0000", 2023-09-23T18:58:49.913 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0057965369999999 2023-09-23T18:58:49.913 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.913 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.913 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.913 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:38.712564+0000", 2023-09-23T18:58:49.913 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3850999999999999e-05 2023-09-23T18:58:49.913 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.913 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.913 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:58:49.913 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:38.712588+0000", 2023-09-23T18:58:49.913 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.005982999 2023-09-23T18:58:49.914 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.914 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:58:49.914 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.914 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.914 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.914 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45602.0:147864 9.c 9.f7994f2c (undecoded) ondisk+write+known_if_redirected e298)", 2023-09-23T18:58:49.914 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:58:37.753155+0000", 2023-09-23T18:58:49.914 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 12.031631708000001, 2023-09-23T18:58:49.914 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.95950417700000001, 2023-09-23T18:58:49.914 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:58:49.914 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T18:58:49.914 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:58:49.915 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45602", 2023-09-23T18:58:49.915 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T18:58:49.915 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 147864 2023-09-23T18:58:49.915 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.915 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:58:49.915 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.915 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:58:49.915 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.753155+0000", 2023-09-23T18:58:49.915 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:58:49.915 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.915 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.915 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:58:49.915 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.753155+0000", 2023-09-23T18:58:49.916 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999967 2023-09-23T18:58:49.916 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.916 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.916 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:58:49.916 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.753152+0000", 2023-09-23T18:58:49.916 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6983e-05 2023-09-23T18:58:49.916 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.916 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.916 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:58:49.916 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.753169+0000", 2023-09-23T18:58:49.916 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.8588000000000001e-05 2023-09-23T18:58:49.916 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.916 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.917 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:58:49.917 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.753198+0000", 2023-09-23T18:58:49.917 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2493999999999999e-05 2023-09-23T18:58:49.917 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.917 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.917 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:58:49.917 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.753210+0000", 2023-09-23T18:58:49.917 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.2695999999999995e-05 2023-09-23T18:58:49.917 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.917 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.917 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.918 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.753303+0000", 2023-09-23T18:58:49.918 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8219e-05 2023-09-23T18:58:49.918 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.918 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.918 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:58:49.918 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.753321+0000", 2023-09-23T18:58:49.918 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.95932390599999995 2023-09-23T18:58:49.918 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.918 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.918 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.918 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:38.712645+0000", 2023-09-23T18:58:49.918 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4613999999999999e-05 2023-09-23T18:58:49.919 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.919 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.919 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:58:49.919 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:38.712659+0000", 2023-09-23T18:58:49.919 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.95950417700000001 2023-09-23T18:58:49.919 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.919 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:58:49.919 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.919 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.919 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.919 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45602.0:147867 9.c 9.887e1b5c (undecoded) ondisk+write+known_if_redirected e298)", 2023-09-23T18:58:49.919 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T18:58:37.763892+0000", 2023-09-23T18:58:49.920 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 12.020894577, 2023-09-23T18:58:49.920 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.94885022399999996, 2023-09-23T18:58:49.920 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T18:58:49.920 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T18:58:49.920 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T18:58:49.920 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45602", 2023-09-23T18:58:49.920 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T18:58:49.920 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 147867 2023-09-23T18:58:49.920 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.920 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T18:58:49.920 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.920 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T18:58:49.921 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.763892+0000", 2023-09-23T18:58:49.921 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T18:58:49.921 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.921 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.921 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T18:58:49.921 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.763892+0000", 2023-09-23T18:58:49.921 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999852 2023-09-23T18:58:49.921 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.921 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.921 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T18:58:49.921 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.763878+0000", 2023-09-23T18:58:49.921 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.0102000000000001e-05 2023-09-23T18:58:49.922 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.922 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.922 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T18:58:49.922 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.763908+0000", 2023-09-23T18:58:49.922 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.8639000000000001e-05 2023-09-23T18:58:49.922 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.922 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.922 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T18:58:49.922 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.763947+0000", 2023-09-23T18:58:49.922 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8731000000000001e-05 2023-09-23T18:58:49.922 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.922 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.923 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T18:58:49.923 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.763965+0000", 2023-09-23T18:58:49.923 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.6301e-05 2023-09-23T18:58:49.923 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.923 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.923 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.923 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.764042+0000", 2023-09-23T18:58:49.923 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.6010000000000001e-06 2023-09-23T18:58:49.923 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.923 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.923 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T18:58:49.923 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:37.764051+0000", 2023-09-23T18:58:49.923 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.94867858900000002 2023-09-23T18:58:49.924 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.924 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.924 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T18:58:49.924 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:38.712730+0000", 2023-09-23T18:58:49.924 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2911999999999999e-05 2023-09-23T18:58:49.924 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T18:58:49.924 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T18:58:49.924 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T18:58:49.924 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T18:58:38.712743+0000", 2023-09-23T18:58:49.924 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.94885022399999996 2023-09-23T18:58:49.924 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.924 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:58:49.924 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.925 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T18:58:49.925 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T18:58:49.925 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T18:58:49.925 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:49.970 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:49.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:49.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:50.078 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:50.117 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:50.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:50.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:50.221 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:50.243 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:50.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:50.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:50.348 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:50.375 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:50.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:50.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:50.477 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:50.504 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:50.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:50.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:50.608 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:50.633 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:50.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:50.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:50.737 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:50.746 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:50 smithi098 bash[79516]: cluster 2023-09-23T18:58:48.367916+0000 mgr.y (mgr.26054) 1225 : cluster 0 pgmap v1333: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 53 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 21800/117024 objects degraded (18.629%); 6/117024 objects misplaced (0.005%); 3.7 MiB/s, 99 objects/s recovering 2023-09-23T18:58:50.746 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:50 smithi098 bash[76817]: cluster 2023-09-23T18:58:48.367916+0000 mgr.y (mgr.26054) 1225 : cluster 0 pgmap v1333: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 53 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 21800/117024 objects degraded (18.629%); 6/117024 objects misplaced (0.005%); 3.7 MiB/s, 99 objects/s recovering 2023-09-23T18:58:50.767 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:50.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:50.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:50.872 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:50.915 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:50.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:50.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:50.971 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:50 smithi173 bash[59422]: cluster 2023-09-23T18:58:48.367916+0000 mgr.y (mgr.26054) 1225 : cluster 0 pgmap v1333: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 53 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 21800/117024 objects degraded (18.629%); 6/117024 objects misplaced (0.005%); 3.7 MiB/s, 99 objects/s recovering 2023-09-23T18:58:50.972 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T18:58:51.021 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:51.044 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:51.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:51.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:51.148 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:51.188 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:51.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:51.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:51.236 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:58:51.292 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:51.319 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:51.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:51.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:51.425 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:51.448 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:51.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:51.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:51.492 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:51 smithi098 bash[79516]: cluster 2023-09-23T18:58:50.399430+0000 mon.a (mon.0) 1678 : cluster 3 Health check update: Degraded data redundancy: 21800/117024 objects degraded (18.629%), 15 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:51.492 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:51 smithi098 bash[76817]: cluster 2023-09-23T18:58:50.399430+0000 mon.a (mon.0) 1678 : cluster 3 Health check update: Degraded data redundancy: 21800/117024 objects degraded (18.629%), 15 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:51.554 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:51.588 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:51.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:51.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:51.693 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:51.716 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:51.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:51.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:51.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:51 smithi173 bash[59422]: cluster 2023-09-23T18:58:50.399430+0000 mon.a (mon.0) 1678 : cluster 3 Health check update: Degraded data redundancy: 21800/117024 objects degraded (18.629%), 15 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:51.819 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:51.850 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:51.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:51.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:51.956 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:51.987 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:51.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:51.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:52.091 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:52.113 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:52.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:52.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:52.217 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:52.254 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:52.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:52.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:52.357 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:58:52.359 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:52.368 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:52 smithi173 bash[59422]: cluster 2023-09-23T18:58:50.368618+0000 mgr.y (mgr.26054) 1226 : cluster 0 pgmap v1334: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 53 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 438 op/s; 21800/117024 objects degraded (18.629%); 6/117024 objects misplaced (0.005%); 3.4 MiB/s, 89 objects/s recovering 2023-09-23T18:58:52.387 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:52.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:52.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:52.402 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:58:52.441 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:58:52.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:52 smithi098 bash[76817]: cluster 2023-09-23T18:58:50.368618+0000 mgr.y (mgr.26054) 1226 : cluster 0 pgmap v1334: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 53 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 438 op/s; 21800/117024 objects degraded (18.629%); 6/117024 objects misplaced (0.005%); 3.4 MiB/s, 89 objects/s recovering 2023-09-23T18:58:52.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:52 smithi098 bash[79516]: cluster 2023-09-23T18:58:50.368618+0000 mgr.y (mgr.26054) 1226 : cluster 0 pgmap v1334: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 53 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 438 op/s; 21800/117024 objects degraded (18.629%); 6/117024 objects misplaced (0.005%); 3.4 MiB/s, 89 objects/s recovering 2023-09-23T18:58:52.486 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:58:52.491 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:52.514 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:52.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:52.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:52.529 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:58:52.560 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:58:52.591 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:58:52.592 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:58:52.592 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:58:52.592 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:58:52.592 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:58:52.592 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:58:52.592 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:58:52.592 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:58:52.592 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T18:58:52.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:52.645 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:52.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:52.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:52.748 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:52.774 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:52.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:52.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:52.877 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:52.903 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:52.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:52.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:53.009 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:53.033 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:53.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:53.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:53.140 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:53.163 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:53.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:53.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:53.266 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:53.284 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:53.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:53.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:53.387 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:53.416 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:53.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:53.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:53.519 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:53.527 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:53 smithi098 bash[76817]: audit 2023-09-23T18:58:52.301741+0000 mon.a (mon.0) 1679 : audit 0 from='client.? 172.21.15.98:0/189722750' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:58:53.527 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:53 smithi098 bash[76817]: cluster 2023-09-23T18:58:52.307268+0000 mon.a (mon.0) 1680 : cluster 0 osdmap e301: 8 total, 7 up, 8 in 2023-09-23T18:58:53.527 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:53 smithi098 bash[76817]: cluster 2023-09-23T18:58:52.309732+0000 osd.5 (osd.5) 223 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T18:58:53.527 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:53 smithi098 bash[76817]: cluster 2023-09-23T18:58:52.311796+0000 osd.4 (osd.4) 28 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T18:58:53.527 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:53 smithi098 bash[76817]: cluster 2023-09-23T18:58:52.313265+0000 osd.4 (osd.4) 29 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T18:58:53.527 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:53 smithi098 bash[76817]: cluster 2023-09-23T18:58:52.313327+0000 osd.5 (osd.5) 224 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T18:58:53.528 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:53 smithi098 bash[76817]: cluster 2023-09-23T18:58:52.316976+0000 osd.5 (osd.5) 225 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T18:58:53.528 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:53 smithi098 bash[76817]: cluster 2023-09-23T18:58:52.321733+0000 osd.6 (osd.6) 273 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T18:58:53.528 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:53 smithi098 bash[76817]: cluster 2023-09-23T18:58:52.323675+0000 osd.6 (osd.6) 274 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T18:58:53.528 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:53 smithi098 bash[76817]: cluster 2023-09-23T18:58:52.369313+0000 mgr.y (mgr.26054) 1227 : cluster 0 pgmap v1336: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 53 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 20907/112548 objects degraded (18.576%); 6/112548 objects misplaced (0.005%); 4.6 MiB/s, 120 objects/s recovering 2023-09-23T18:58:53.528 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:53 smithi098 bash[79516]: audit 2023-09-23T18:58:52.301741+0000 mon.a (mon.0) 1679 : audit 0 from='client.? 172.21.15.98:0/189722750' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:58:53.528 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:53 smithi098 bash[79516]: cluster 2023-09-23T18:58:52.307268+0000 mon.a (mon.0) 1680 : cluster 0 osdmap e301: 8 total, 7 up, 8 in 2023-09-23T18:58:53.528 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:53 smithi098 bash[79516]: cluster 2023-09-23T18:58:52.309732+0000 osd.5 (osd.5) 223 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T18:58:53.528 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:53 smithi098 bash[79516]: cluster 2023-09-23T18:58:52.311796+0000 osd.4 (osd.4) 28 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T18:58:53.529 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:53 smithi098 bash[79516]: cluster 2023-09-23T18:58:52.313265+0000 osd.4 (osd.4) 29 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T18:58:53.529 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:53 smithi098 bash[79516]: cluster 2023-09-23T18:58:52.313327+0000 osd.5 (osd.5) 224 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T18:58:53.529 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:53 smithi098 bash[79516]: cluster 2023-09-23T18:58:52.316976+0000 osd.5 (osd.5) 225 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T18:58:53.529 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:53 smithi098 bash[79516]: cluster 2023-09-23T18:58:52.321733+0000 osd.6 (osd.6) 273 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T18:58:53.529 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:53 smithi098 bash[79516]: cluster 2023-09-23T18:58:52.323675+0000 osd.6 (osd.6) 274 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T18:58:53.529 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:53 smithi098 bash[79516]: cluster 2023-09-23T18:58:52.369313+0000 mgr.y (mgr.26054) 1227 : cluster 0 pgmap v1336: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 53 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 20907/112548 objects degraded (18.576%); 6/112548 objects misplaced (0.005%); 4.6 MiB/s, 120 objects/s recovering 2023-09-23T18:58:53.572 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:53.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:53.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:53.679 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:53.708 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:53.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:53.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:53.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:53 smithi173 bash[59422]: audit 2023-09-23T18:58:52.301741+0000 mon.a (mon.0) 1679 : audit 0 from='client.? 172.21.15.98:0/189722750' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:58:53.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:53 smithi173 bash[59422]: cluster 2023-09-23T18:58:52.307268+0000 mon.a (mon.0) 1680 : cluster 0 osdmap e301: 8 total, 7 up, 8 in 2023-09-23T18:58:53.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:53 smithi173 bash[59422]: cluster 2023-09-23T18:58:52.309732+0000 osd.5 (osd.5) 223 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T18:58:53.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:53 smithi173 bash[59422]: cluster 2023-09-23T18:58:52.311796+0000 osd.4 (osd.4) 28 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T18:58:53.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:53 smithi173 bash[59422]: cluster 2023-09-23T18:58:52.313265+0000 osd.4 (osd.4) 29 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T18:58:53.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:53 smithi173 bash[59422]: cluster 2023-09-23T18:58:52.313327+0000 osd.5 (osd.5) 224 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T18:58:53.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:53 smithi173 bash[59422]: cluster 2023-09-23T18:58:52.316976+0000 osd.5 (osd.5) 225 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T18:58:53.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:53 smithi173 bash[59422]: cluster 2023-09-23T18:58:52.321733+0000 osd.6 (osd.6) 273 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T18:58:53.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:53 smithi173 bash[59422]: cluster 2023-09-23T18:58:52.323675+0000 osd.6 (osd.6) 274 : cluster 3 failed to encode map e301 with expected crc 2023-09-23T18:58:53.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:53 smithi173 bash[59422]: cluster 2023-09-23T18:58:52.369313+0000 mgr.y (mgr.26054) 1227 : cluster 0 pgmap v1336: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 53 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 20907/112548 objects degraded (18.576%); 6/112548 objects misplaced (0.005%); 4.6 MiB/s, 120 objects/s recovering 2023-09-23T18:58:53.812 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:53.838 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:53.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:53.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:53.943 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:53.965 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:53.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:53.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:54.068 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:54.105 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:54.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:54.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:54.210 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:54.236 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:54.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:54.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:54.339 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:54.347 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:54 smithi098 bash[79516]: cluster 2023-09-23T18:58:53.313585+0000 osd.4 (osd.4) 30 : cluster 3 failed to encode map e302 with expected crc 2023-09-23T18:58:54.347 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:54 smithi098 bash[79516]: cluster 2023-09-23T18:58:53.316616+0000 mon.a (mon.0) 1681 : cluster 0 osdmap e302: 8 total, 7 up, 8 in 2023-09-23T18:58:54.347 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:54 smithi098 bash[79516]: cluster 2023-09-23T18:58:53.317815+0000 osd.5 (osd.5) 226 : cluster 3 failed to encode map e302 with expected crc 2023-09-23T18:58:54.348 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:54 smithi098 bash[79516]: cluster 2023-09-23T18:58:53.319364+0000 osd.5 (osd.5) 227 : cluster 3 failed to encode map e302 with expected crc 2023-09-23T18:58:54.348 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:54 smithi098 bash[76817]: cluster 2023-09-23T18:58:53.313585+0000 osd.4 (osd.4) 30 : cluster 3 failed to encode map e302 with expected crc 2023-09-23T18:58:54.348 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:54 smithi098 bash[76817]: cluster 2023-09-23T18:58:53.316616+0000 mon.a (mon.0) 1681 : cluster 0 osdmap e302: 8 total, 7 up, 8 in 2023-09-23T18:58:54.348 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:54 smithi098 bash[76817]: cluster 2023-09-23T18:58:53.317815+0000 osd.5 (osd.5) 226 : cluster 3 failed to encode map e302 with expected crc 2023-09-23T18:58:54.348 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:54 smithi098 bash[76817]: cluster 2023-09-23T18:58:53.319364+0000 osd.5 (osd.5) 227 : cluster 3 failed to encode map e302 with expected crc 2023-09-23T18:58:54.375 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:54.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:54.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:54.480 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:54.491 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:54 smithi173 bash[59422]: cluster 2023-09-23T18:58:53.313585+0000 osd.4 (osd.4) 30 : cluster 3 failed to encode map e302 with expected crc 2023-09-23T18:58:54.491 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:54 smithi173 bash[59422]: cluster 2023-09-23T18:58:53.316616+0000 mon.a (mon.0) 1681 : cluster 0 osdmap e302: 8 total, 7 up, 8 in 2023-09-23T18:58:54.491 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:54 smithi173 bash[59422]: cluster 2023-09-23T18:58:53.317815+0000 osd.5 (osd.5) 226 : cluster 3 failed to encode map e302 with expected crc 2023-09-23T18:58:54.491 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:54 smithi173 bash[59422]: cluster 2023-09-23T18:58:53.319364+0000 osd.5 (osd.5) 227 : cluster 3 failed to encode map e302 with expected crc 2023-09-23T18:58:54.506 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:54.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:54.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:54.610 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:54.640 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:54.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:54.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:54.744 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:54.769 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:54.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:54.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:54.875 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:54.913 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:54.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:54.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:55.020 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:55.051 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:55.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:55.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:55.158 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:55.190 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:55.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:55.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:55.295 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:55.323 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:55.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:55.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:55.428 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:55.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:55 smithi098 bash[79516]: cluster 2023-09-23T18:58:54.370067+0000 mgr.y (mgr.26054) 1228 : cluster 0 pgmap v1338: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 53 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 20212/108273 objects degraded (18.668%); 6/108273 objects misplaced (0.006%); 5.7 MiB/s, 122 objects/s recovering 2023-09-23T18:58:55.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:55 smithi098 bash[79516]: audit 2023-09-23T18:58:54.634183+0000 mon.a (mon.0) 1682 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:58:55.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:55 smithi098 bash[76817]: cluster 2023-09-23T18:58:54.370067+0000 mgr.y (mgr.26054) 1228 : cluster 0 pgmap v1338: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 53 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 20212/108273 objects degraded (18.668%); 6/108273 objects misplaced (0.006%); 5.7 MiB/s, 122 objects/s recovering 2023-09-23T18:58:55.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:55 smithi098 bash[76817]: audit 2023-09-23T18:58:54.634183+0000 mon.a (mon.0) 1682 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:58:55.457 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:55.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:55.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:55.561 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:55.583 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:55.584 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:55 smithi173 bash[59422]: cluster 2023-09-23T18:58:54.370067+0000 mgr.y (mgr.26054) 1228 : cluster 0 pgmap v1338: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 53 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 20212/108273 objects degraded (18.668%); 6/108273 objects misplaced (0.006%); 5.7 MiB/s, 122 objects/s recovering 2023-09-23T18:58:55.584 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:55 smithi173 bash[59422]: audit 2023-09-23T18:58:54.634183+0000 mon.a (mon.0) 1682 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:58:55.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:55.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:55.686 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:55.709 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:55.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:55.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:55.812 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:55.839 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:55.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:55.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:55.942 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:55.969 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:55.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:55.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:56.071 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:56.097 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:56.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:56.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:56.199 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:56.222 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:56.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:56.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:56.326 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:56.350 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:56.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:56.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:56.394 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:56 smithi098 bash[76817]: cluster 2023-09-23T18:58:55.400533+0000 mon.a (mon.0) 1683 : cluster 3 Health check update: Degraded data redundancy: 20212/108273 objects degraded (18.668%), 15 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:56.394 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:58:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:58:56] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.33.4" 2023-09-23T18:58:56.394 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:56 smithi098 bash[79516]: cluster 2023-09-23T18:58:55.400533+0000 mon.a (mon.0) 1683 : cluster 3 Health check update: Degraded data redundancy: 20212/108273 objects degraded (18.668%), 15 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:56.457 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:56.490 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:56.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:56.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:56.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:56 smithi173 bash[59422]: cluster 2023-09-23T18:58:55.400533+0000 mon.a (mon.0) 1683 : cluster 3 Health check update: Degraded data redundancy: 20212/108273 objects degraded (18.668%), 15 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:58:56.528 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:58:56.529 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:58:56.605 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:56.640 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:56.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:56.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:56.744 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:56.775 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:56.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:56.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:56.878 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:56.913 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:56.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:56.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:57.016 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:57.052 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:57.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:57.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:57.156 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:57.185 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:57.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:57.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:57.288 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:57.321 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:57.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:57.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:57.424 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:57.447 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:57.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:57.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:57.550 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:57.558 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:57 smithi098 bash[79516]: cluster 2023-09-23T18:58:56.370887+0000 mgr.y (mgr.26054) 1229 : cluster 0 pgmap v1339: 89 pgs: 3 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 20059/106317 objects degraded (18.867%); 6/106317 objects misplaced (0.006%); 2.9 MiB/s, 86 objects/s recovering 2023-09-23T18:58:57.559 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:57 smithi098 bash[76817]: cluster 2023-09-23T18:58:56.370887+0000 mgr.y (mgr.26054) 1229 : cluster 0 pgmap v1339: 89 pgs: 3 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 20059/106317 objects degraded (18.867%); 6/106317 objects misplaced (0.006%); 2.9 MiB/s, 86 objects/s recovering 2023-09-23T18:58:57.574 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:57.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:57.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:57.682 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:57.707 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:57.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:57.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:57.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:57 smithi173 bash[59422]: cluster 2023-09-23T18:58:56.370887+0000 mgr.y (mgr.26054) 1229 : cluster 0 pgmap v1339: 89 pgs: 3 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 20059/106317 objects degraded (18.867%); 6/106317 objects misplaced (0.006%); 2.9 MiB/s, 86 objects/s recovering 2023-09-23T18:58:57.812 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:57.836 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:57.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:57.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:57.940 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:57.965 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:57.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:57.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:58.069 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:58.087 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:58.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:58.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:58.190 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:58.207 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:58.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:58.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:58.310 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:58.332 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:58.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:58.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:58.435 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:58.464 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:58.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:58.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:58.530 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:58:58.570 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:58.609 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:58:58.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:58.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:58:58.713 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:58.738 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:58:58.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:58.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:58:58.842 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:58.874 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:58:58.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:58.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:58:58.977 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:59.001 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:59.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:59.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:59.107 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:59.140 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:59.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:59.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:58:59.246 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:59.285 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:58:59.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:59.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:58:59.388 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:59.416 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:59.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:59.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:59.520 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:59.545 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:58:59.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:59.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:58:59.649 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:59.688 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:59.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:59.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:59.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:58:59 smithi173 bash[59422]: cluster 2023-09-23T18:58:58.371867+0000 mgr.y (mgr.26054) 1230 : cluster 0 pgmap v1340: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 18793/100566 objects degraded (18.687%); 6/100566 objects misplaced (0.006%); 6.6 MiB/s, 146 objects/s recovering 2023-09-23T18:58:59.784 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:58:59 smithi098 bash[79516]: cluster 2023-09-23T18:58:58.371867+0000 mgr.y (mgr.26054) 1230 : cluster 0 pgmap v1340: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 18793/100566 objects degraded (18.687%); 6/100566 objects misplaced (0.006%); 6.6 MiB/s, 146 objects/s recovering 2023-09-23T18:58:59.784 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:58:59 smithi098 bash[76817]: cluster 2023-09-23T18:58:58.371867+0000 mgr.y (mgr.26054) 1230 : cluster 0 pgmap v1340: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 18793/100566 objects degraded (18.687%); 6/100566 objects misplaced (0.006%); 6.6 MiB/s, 146 objects/s recovering 2023-09-23T18:58:59.795 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:59.819 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:58:59.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:59.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:58:59.924 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:59.963 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:58:59.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:58:59.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:00.071 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:00.099 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:00.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:00.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:00.204 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:00.230 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:00.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:00.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:00.333 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:00.363 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:00.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:00.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:00.471 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:00.480 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:00 smithi098 bash[76817]: cluster 2023-09-23T18:59:00.401588+0000 mon.a (mon.0) 1684 : cluster 3 Health check update: Degraded data redundancy: 18793/100566 objects degraded (18.687%), 15 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:00.481 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:00 smithi098 bash[79516]: cluster 2023-09-23T18:59:00.401588+0000 mon.a (mon.0) 1684 : cluster 3 Health check update: Degraded data redundancy: 18793/100566 objects degraded (18.687%), 15 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:00.504 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:00.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:00.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:00.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:00 smithi173 bash[59422]: cluster 2023-09-23T18:59:00.401588+0000 mon.a (mon.0) 1684 : cluster 3 Health check update: Degraded data redundancy: 18793/100566 objects degraded (18.687%), 15 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:00.608 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:00.632 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:00.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:00.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:00.735 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:00.760 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:00.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:00.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:00.864 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:00.892 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:00.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:00.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:00.997 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:01.019 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:01.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:01.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:01.123 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:01.153 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:01.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:01.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:01.256 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:01.280 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:01.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:01.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:01.383 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:01.407 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:01.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:01.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:01.441 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:01 smithi098 bash[76817]: cluster 2023-09-23T18:59:00.372546+0000 mgr.y (mgr.26054) 1231 : cluster 0 pgmap v1341: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 494 op/s; 18793/100566 objects degraded (18.687%); 6/100566 objects misplaced (0.006%); 3.7 MiB/s, 68 objects/s recovering 2023-09-23T18:59:01.510 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:01.519 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:01 smithi173 bash[59422]: cluster 2023-09-23T18:59:00.372546+0000 mgr.y (mgr.26054) 1231 : cluster 0 pgmap v1341: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 494 op/s; 18793/100566 objects degraded (18.687%); 6/100566 objects misplaced (0.006%); 3.7 MiB/s, 68 objects/s recovering 2023-09-23T18:59:01.536 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:01.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:01.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:01.638 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:01.669 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:01.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:01.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:01.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:01 smithi098 bash[79516]: cluster 2023-09-23T18:59:00.372546+0000 mgr.y (mgr.26054) 1231 : cluster 0 pgmap v1341: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 494 op/s; 18793/100566 objects degraded (18.687%); 6/100566 objects misplaced (0.006%); 3.7 MiB/s, 68 objects/s recovering 2023-09-23T18:59:01.773 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:01.820 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:01.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:01.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:01.927 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:01.952 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:01.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:01.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:02.055 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:02.084 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:02.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:02.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:02.187 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:02.217 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:02.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:02.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:02.322 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:02.346 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:02.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:02.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:02.453 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:02.499 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:02.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:02.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:02.603 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:02.639 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:02.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:02.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:02.744 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:02.768 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:02.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:02.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:02.873 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:02.901 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:02.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:02.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:03.005 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:03.028 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:03.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:03.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:03.132 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:03.161 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:03.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:03.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:03.253 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:59:03.268 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:03.308 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:03.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:03.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:03.412 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:03.437 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:03.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:03.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:03.541 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:03.568 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:03.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:03.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:03.678 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:03.685 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:03 smithi173 bash[59422]: cluster 2023-09-23T18:59:02.373468+0000 mgr.y (mgr.26054) 1232 : cluster 0 pgmap v1342: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 17923/96438 objects degraded (18.585%); 6/96438 objects misplaced (0.006%); 6.3 MiB/s, 107 objects/s recovering 2023-09-23T18:59:03.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:03 smithi098 bash[76817]: cluster 2023-09-23T18:59:02.373468+0000 mgr.y (mgr.26054) 1232 : cluster 0 pgmap v1342: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 17923/96438 objects degraded (18.585%); 6/96438 objects misplaced (0.006%); 6.3 MiB/s, 107 objects/s recovering 2023-09-23T18:59:03.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:03 smithi098 bash[79516]: cluster 2023-09-23T18:59:02.373468+0000 mgr.y (mgr.26054) 1232 : cluster 0 pgmap v1342: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 17923/96438 objects degraded (18.585%); 6/96438 objects misplaced (0.006%); 6.3 MiB/s, 107 objects/s recovering 2023-09-23T18:59:03.699 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:03.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:03.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:03.802 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:03.826 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:03.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:03.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:03.934 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:03.959 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:03.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:03.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:04.064 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:04.098 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:04.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:04.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:04.203 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:04.240 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:04.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:04.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:04.344 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:04.377 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:04.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:04.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:04.484 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:04.517 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:04.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:04.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:04.621 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:04.648 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:04.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:04.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:04.752 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:04.789 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:04.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:04.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:04.892 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:04.916 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:04.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:04.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:05.020 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:05.039 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:05.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:05.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:05.142 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:05.169 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:05.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:05.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:05.272 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:05.302 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:05.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:05.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:05.407 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:05.438 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:05.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:05.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:05.489 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is set 2023-09-23T18:59:05.541 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:05.565 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:05.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:05.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:05.667 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:05.685 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:05.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:05.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:05.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:05 smithi173 bash[59422]: cluster 2023-09-23T18:59:04.374969+0000 mgr.y (mgr.26054) 1233 : cluster 0 pgmap v1343: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 484 op/s; 17225/92193 objects degraded (18.684%); 6/92193 objects misplaced (0.007%); 5.7 MiB/s, 97 objects/s recovering 2023-09-23T18:59:05.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:05 smithi173 bash[59422]: audit 2023-09-23T18:59:04.518732+0000 mon.c (mon.2) 135 : audit 1 from='client.? 172.21.15.98:0/3416840824' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:59:05.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:05 smithi173 bash[59422]: audit 2023-09-23T18:59:04.519300+0000 mon.a (mon.0) 1685 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:59:05.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:05 smithi173 bash[59422]: cluster 2023-09-23T18:59:05.402541+0000 mon.a (mon.0) 1686 : cluster 3 Health check update: Degraded data redundancy: 17225/92193 objects degraded (18.684%), 15 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:05.788 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:05.820 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:05.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:05.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:05.837 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:05 smithi098 bash[76817]: cluster 2023-09-23T18:59:04.374969+0000 mgr.y (mgr.26054) 1233 : cluster 0 pgmap v1343: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 484 op/s; 17225/92193 objects degraded (18.684%); 6/92193 objects misplaced (0.007%); 5.7 MiB/s, 97 objects/s recovering 2023-09-23T18:59:05.837 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:05 smithi098 bash[76817]: audit 2023-09-23T18:59:04.518732+0000 mon.c (mon.2) 135 : audit 1 from='client.? 172.21.15.98:0/3416840824' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:59:05.837 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:05 smithi098 bash[76817]: audit 2023-09-23T18:59:04.519300+0000 mon.a (mon.0) 1685 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:59:05.837 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:05 smithi098 bash[76817]: cluster 2023-09-23T18:59:05.402541+0000 mon.a (mon.0) 1686 : cluster 3 Health check update: Degraded data redundancy: 17225/92193 objects degraded (18.684%), 15 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:05.837 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:05 smithi098 bash[79516]: cluster 2023-09-23T18:59:04.374969+0000 mgr.y (mgr.26054) 1233 : cluster 0 pgmap v1343: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 484 op/s; 17225/92193 objects degraded (18.684%); 6/92193 objects misplaced (0.007%); 5.7 MiB/s, 97 objects/s recovering 2023-09-23T18:59:05.837 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:05 smithi098 bash[79516]: audit 2023-09-23T18:59:04.518732+0000 mon.c (mon.2) 135 : audit 1 from='client.? 172.21.15.98:0/3416840824' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:59:05.837 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:05 smithi098 bash[79516]: audit 2023-09-23T18:59:04.519300+0000 mon.a (mon.0) 1685 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:59:05.837 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:05 smithi098 bash[79516]: cluster 2023-09-23T18:59:05.402541+0000 mon.a (mon.0) 1686 : cluster 3 Health check update: Degraded data redundancy: 17225/92193 objects degraded (18.684%), 15 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:05.924 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:05.951 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:05.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:05.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:06.056 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:06.085 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:06.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:06.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:06.090 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:59:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:59:06] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.33.4" 2023-09-23T18:59:06.190 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:06.216 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:06.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:06.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:06.320 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:06.351 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:06.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:06.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:06.455 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:06.493 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:06.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:06.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:06.597 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:06.606 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:06 smithi098 bash[79516]: cluster 2023-09-23T18:59:05.450443+0000 mon.a (mon.0) 1687 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:59:06.606 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:06 smithi098 bash[79516]: cluster 2023-09-23T18:59:05.469367+0000 osd.6 (osd.6) 275 : cluster 3 failed to encode map e303 with expected crc 2023-09-23T18:59:06.606 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:06 smithi098 bash[79516]: cluster 2023-09-23T18:59:05.473832+0000 osd.5 (osd.5) 228 : cluster 3 failed to encode map e303 with expected crc 2023-09-23T18:59:06.606 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:06 smithi098 bash[79516]: audit 2023-09-23T18:59:05.475196+0000 mon.a (mon.0) 1688 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:59:06.606 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:06 smithi098 bash[79516]: cluster 2023-09-23T18:59:05.475407+0000 mon.a (mon.0) 1689 : cluster 0 osdmap e303: 8 total, 7 up, 8 in 2023-09-23T18:59:06.606 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:06 smithi098 bash[79516]: cluster 2023-09-23T18:59:05.484200+0000 osd.4 (osd.4) 31 : cluster 3 failed to encode map e303 with expected crc 2023-09-23T18:59:06.607 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:06 smithi098 bash[76817]: cluster 2023-09-23T18:59:05.450443+0000 mon.a (mon.0) 1687 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:59:06.607 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:06 smithi098 bash[76817]: cluster 2023-09-23T18:59:05.469367+0000 osd.6 (osd.6) 275 : cluster 3 failed to encode map e303 with expected crc 2023-09-23T18:59:06.607 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:06 smithi098 bash[76817]: cluster 2023-09-23T18:59:05.473832+0000 osd.5 (osd.5) 228 : cluster 3 failed to encode map e303 with expected crc 2023-09-23T18:59:06.607 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:06 smithi098 bash[76817]: audit 2023-09-23T18:59:05.475196+0000 mon.a (mon.0) 1688 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:59:06.607 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:06 smithi098 bash[76817]: cluster 2023-09-23T18:59:05.475407+0000 mon.a (mon.0) 1689 : cluster 0 osdmap e303: 8 total, 7 up, 8 in 2023-09-23T18:59:06.607 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:06 smithi098 bash[76817]: cluster 2023-09-23T18:59:05.484200+0000 osd.4 (osd.4) 31 : cluster 3 failed to encode map e303 with expected crc 2023-09-23T18:59:06.631 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:06.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:06.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:06.736 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:06.744 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:06 smithi173 bash[59422]: cluster 2023-09-23T18:59:05.450443+0000 mon.a (mon.0) 1687 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:59:06.744 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:06 smithi173 bash[59422]: cluster 2023-09-23T18:59:05.469367+0000 osd.6 (osd.6) 275 : cluster 3 failed to encode map e303 with expected crc 2023-09-23T18:59:06.745 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:06 smithi173 bash[59422]: cluster 2023-09-23T18:59:05.473832+0000 osd.5 (osd.5) 228 : cluster 3 failed to encode map e303 with expected crc 2023-09-23T18:59:06.745 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:06 smithi173 bash[59422]: audit 2023-09-23T18:59:05.475196+0000 mon.a (mon.0) 1688 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:59:06.745 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:06 smithi173 bash[59422]: cluster 2023-09-23T18:59:05.475407+0000 mon.a (mon.0) 1689 : cluster 0 osdmap e303: 8 total, 7 up, 8 in 2023-09-23T18:59:06.745 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:06 smithi173 bash[59422]: cluster 2023-09-23T18:59:05.484200+0000 osd.4 (osd.4) 31 : cluster 3 failed to encode map e303 with expected crc 2023-09-23T18:59:06.757 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:06.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:06.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:06.861 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:06.890 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:06.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:06.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:06.998 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:07.038 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:07.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:07.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:07.154 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:07.186 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:07.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:07.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:07.288 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:07.327 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:07.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:07.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:07.428 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:07.454 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:07.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:07.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:07.557 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:07.584 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:07.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:07.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:07.690 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:07.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:07 smithi098 bash[76817]: cluster 2023-09-23T18:59:06.375753+0000 mgr.y (mgr.26054) 1234 : cluster 0 pgmap v1345: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 17049/90135 objects degraded (18.915%); 6/90135 objects misplaced (0.007%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T18:59:07.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:07 smithi098 bash[79516]: cluster 2023-09-23T18:59:06.375753+0000 mgr.y (mgr.26054) 1234 : cluster 0 pgmap v1345: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 17049/90135 objects degraded (18.915%); 6/90135 objects misplaced (0.007%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T18:59:07.698 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:07 smithi173 bash[59422]: cluster 2023-09-23T18:59:06.375753+0000 mgr.y (mgr.26054) 1234 : cluster 0 pgmap v1345: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 17049/90135 objects degraded (18.915%); 6/90135 objects misplaced (0.007%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T18:59:07.715 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:07.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:07.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:07.818 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:07.842 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:07.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:07.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:07.947 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:07.972 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:07.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:07.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:08.076 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:08.098 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:08.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:08.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:08.201 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:08.226 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:08.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:08.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:08.335 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:08.366 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:08.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:08.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:08.473 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:08.508 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:08.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:08.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:08.612 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:08.644 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:08.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:08.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:08.748 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:08.770 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:08.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:08.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:08.878 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:08.903 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:08.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:08.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:09.009 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:09.035 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:09.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:09.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:09.138 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:09.165 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:09.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:09.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:09.268 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:09.288 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:09.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:09.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:09.391 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:09.420 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:09.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:09.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:09.523 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:09.557 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:09.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:09.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:09.662 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:09.697 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:09.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:09.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:09.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:09 smithi173 bash[59422]: cluster 2023-09-23T18:59:08.376609+0000 mgr.y (mgr.26054) 1235 : cluster 0 pgmap v1346: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 1.7 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 15599/83604 objects degraded (18.658%); 6/83604 objects misplaced (0.007%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T18:59:09.802 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:09.809 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:09 smithi098 bash[79516]: cluster 2023-09-23T18:59:08.376609+0000 mgr.y (mgr.26054) 1235 : cluster 0 pgmap v1346: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 1.7 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 15599/83604 objects degraded (18.658%); 6/83604 objects misplaced (0.007%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T18:59:09.810 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:09 smithi098 bash[76817]: cluster 2023-09-23T18:59:08.376609+0000 mgr.y (mgr.26054) 1235 : cluster 0 pgmap v1346: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 1.7 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 15599/83604 objects degraded (18.658%); 6/83604 objects misplaced (0.007%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T18:59:09.825 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:09.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:09.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:09.930 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:09.955 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:09.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:09.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:10.061 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:10.093 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:10.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:10.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:10.196 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:10.221 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:10.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:10.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:10.325 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:10.355 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:10.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:10.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:10.461 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:10.486 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:10.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:10.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:10.589 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:10.636 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:10.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:10.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:10.738 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:10.762 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:10.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:10.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:10.865 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:10.874 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:10 smithi098 bash[76817]: audit 2023-09-23T18:59:09.640756+0000 mon.a (mon.0) 1690 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:59:10.874 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:10 smithi098 bash[76817]: audit 2023-09-23T18:59:09.643626+0000 mon.a (mon.0) 1691 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:59:10.874 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:10 smithi098 bash[76817]: cluster 2023-09-23T18:59:10.403419+0000 mon.a (mon.0) 1692 : cluster 3 Health check update: Degraded data redundancy: 15599/83604 objects degraded (18.658%), 15 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:10.874 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:10 smithi098 bash[79516]: audit 2023-09-23T18:59:09.640756+0000 mon.a (mon.0) 1690 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:59:10.874 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:10 smithi098 bash[79516]: audit 2023-09-23T18:59:09.643626+0000 mon.a (mon.0) 1691 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:59:10.874 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:10 smithi098 bash[79516]: cluster 2023-09-23T18:59:10.403419+0000 mon.a (mon.0) 1692 : cluster 3 Health check update: Degraded data redundancy: 15599/83604 objects degraded (18.658%), 15 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:10.900 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:10.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:10.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:11.005 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:11.014 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:10 smithi173 bash[59422]: audit 2023-09-23T18:59:09.640756+0000 mon.a (mon.0) 1690 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:59:11.014 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:10 smithi173 bash[59422]: audit 2023-09-23T18:59:09.643626+0000 mon.a (mon.0) 1691 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:59:11.014 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:10 smithi173 bash[59422]: cluster 2023-09-23T18:59:10.403419+0000 mon.a (mon.0) 1692 : cluster 3 Health check update: Degraded data redundancy: 15599/83604 objects degraded (18.658%), 15 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:11.029 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:11.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:11.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:11.132 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:11.165 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:11.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:11.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:11.268 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:11.294 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:11.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:11.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:11.397 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:11.431 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:11.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:11.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:11.534 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:11.564 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:11.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:11.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:11.668 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:11 smithi098 bash[76817]: cluster 2023-09-23T18:59:10.377132+0000 mgr.y (mgr.26054) 1236 : cluster 0 pgmap v1347: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 1.7 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 15599/83604 objects degraded (18.658%); 6/83604 objects misplaced (0.007%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T18:59:11.668 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:11 smithi098 bash[79516]: cluster 2023-09-23T18:59:10.377132+0000 mgr.y (mgr.26054) 1236 : cluster 0 pgmap v1347: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 1.7 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 15599/83604 objects degraded (18.658%); 6/83604 objects misplaced (0.007%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T18:59:11.669 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:11.708 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:11.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:11.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:11.813 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:11.854 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:11.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:11.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:11.960 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:11.969 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:11 smithi173 bash[59422]: cluster 2023-09-23T18:59:10.377132+0000 mgr.y (mgr.26054) 1236 : cluster 0 pgmap v1347: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 1.7 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 15599/83604 objects degraded (18.658%); 6/83604 objects misplaced (0.007%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T18:59:11.985 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:11.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:11.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:12.045 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:59:12.090 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:12.131 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:12.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:12.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:12.238 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:12.263 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:12.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:12.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:12.369 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:12.402 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:12.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:12.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:12.508 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:12.539 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:12.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:12.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:12.642 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:12.650 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:12 smithi098 bash[76817]: debug 2023-09-23T18:59:12.270+0000 7fc8a2a01700 -1 mon.a@0(leader).osd e303 definitely_dead 0 2023-09-23T18:59:12.689 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:12.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:12.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:12.694 INFO:teuthology.orchestra.run.smithi098.stderr:marked out osd.4. 2023-09-23T18:59:12.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:12 smithi173 bash[59422]: audit 2023-09-23T18:59:12.274903+0000 mon.a (mon.0) 1693 : audit 1 from='client.? 172.21.15.98:0/2583853654' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T18:59:12.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:12 smithi173 bash[59422]: cluster 2023-09-23T18:59:12.275033+0000 mon.a (mon.0) 1694 : cluster 1 Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T18:59:12.793 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:12.815 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:12.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:12.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:12.918 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:12.925 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:12 smithi098 bash[79516]: audit 2023-09-23T18:59:12.274903+0000 mon.a (mon.0) 1693 : audit 1 from='client.? 172.21.15.98:0/2583853654' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T18:59:12.925 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:12 smithi098 bash[79516]: cluster 2023-09-23T18:59:12.275033+0000 mon.a (mon.0) 1694 : cluster 1 Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T18:59:12.925 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:12 smithi098 bash[76817]: audit 2023-09-23T18:59:12.274903+0000 mon.a (mon.0) 1693 : audit 1 from='client.? 172.21.15.98:0/2583853654' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T18:59:12.925 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:12 smithi098 bash[76817]: cluster 2023-09-23T18:59:12.275033+0000 mon.a (mon.0) 1694 : cluster 1 Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T18:59:12.936 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:12.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:12.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:13.039 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:13.056 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:13.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:13.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:13.160 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:13.181 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:13.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:13.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:13.284 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:13.309 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:13.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:13.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:13.412 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:13.439 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:13.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:13.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:13.541 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:13.564 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:13.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:13.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:13.667 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:13.697 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:59:13.706 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:13.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:13.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:13.813 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:13.844 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:13.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:13.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:13.947 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:13.965 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:13.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:13.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:14.022 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:13 smithi098 bash[79516]: cluster 2023-09-23T18:59:12.377818+0000 mgr.y (mgr.26054) 1237 : cluster 0 pgmap v1348: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 1.6 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 14702/79245 objects degraded (18.553%); 6/79245 objects misplaced (0.008%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T18:59:14.022 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:13 smithi098 bash[79516]: cluster 2023-09-23T18:59:12.675847+0000 osd.4 (osd.4) 32 : cluster 3 failed to encode map e304 with expected crc 2023-09-23T18:59:14.022 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:13 smithi098 bash[79516]: cluster 2023-09-23T18:59:12.676732+0000 osd.5 (osd.5) 229 : cluster 3 failed to encode map e304 with expected crc 2023-09-23T18:59:14.022 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:13 smithi098 bash[79516]: audit 2023-09-23T18:59:12.679060+0000 mon.a (mon.0) 1695 : audit 1 from='client.? 172.21.15.98:0/2583853654' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T18:59:14.022 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:13 smithi098 bash[79516]: cluster 2023-09-23T18:59:12.682208+0000 mon.a (mon.0) 1696 : cluster 0 osdmap e304: 8 total, 7 up, 7 in 2023-09-23T18:59:14.022 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:13 smithi098 bash[76817]: cluster 2023-09-23T18:59:12.377818+0000 mgr.y (mgr.26054) 1237 : cluster 0 pgmap v1348: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 1.6 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 14702/79245 objects degraded (18.553%); 6/79245 objects misplaced (0.008%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T18:59:14.023 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:13 smithi098 bash[76817]: cluster 2023-09-23T18:59:12.675847+0000 osd.4 (osd.4) 32 : cluster 3 failed to encode map e304 with expected crc 2023-09-23T18:59:14.023 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:13 smithi098 bash[76817]: cluster 2023-09-23T18:59:12.676732+0000 osd.5 (osd.5) 229 : cluster 3 failed to encode map e304 with expected crc 2023-09-23T18:59:14.023 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:13 smithi098 bash[76817]: audit 2023-09-23T18:59:12.679060+0000 mon.a (mon.0) 1695 : audit 1 from='client.? 172.21.15.98:0/2583853654' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T18:59:14.023 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:13 smithi098 bash[76817]: cluster 2023-09-23T18:59:12.682208+0000 mon.a (mon.0) 1696 : cluster 0 osdmap e304: 8 total, 7 up, 7 in 2023-09-23T18:59:14.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:13 smithi173 bash[59422]: cluster 2023-09-23T18:59:12.377818+0000 mgr.y (mgr.26054) 1237 : cluster 0 pgmap v1348: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 54 active+clean, 12 active+undersized+degraded, 12 active+undersized; 1.6 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 14702/79245 objects degraded (18.553%); 6/79245 objects misplaced (0.008%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T18:59:14.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:13 smithi173 bash[59422]: cluster 2023-09-23T18:59:12.675847+0000 osd.4 (osd.4) 32 : cluster 3 failed to encode map e304 with expected crc 2023-09-23T18:59:14.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:13 smithi173 bash[59422]: cluster 2023-09-23T18:59:12.676732+0000 osd.5 (osd.5) 229 : cluster 3 failed to encode map e304 with expected crc 2023-09-23T18:59:14.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:13 smithi173 bash[59422]: audit 2023-09-23T18:59:12.679060+0000 mon.a (mon.0) 1695 : audit 1 from='client.? 172.21.15.98:0/2583853654' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T18:59:14.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:13 smithi173 bash[59422]: cluster 2023-09-23T18:59:12.682208+0000 mon.a (mon.0) 1696 : cluster 0 osdmap e304: 8 total, 7 up, 7 in 2023-09-23T18:59:14.068 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:14.090 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:14.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:14.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:14.196 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:14.217 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:14.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:14.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:14.320 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:14.343 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:14.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:14.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:14.448 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:14.475 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:14.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:14.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:14.579 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:14.602 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:14.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:14.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:14.709 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:14.717 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:14 smithi173 bash[59422]: cluster 2023-09-23T18:59:13.678391+0000 osd.4 (osd.4) 33 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T18:59:14.717 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:14 smithi173 bash[59422]: cluster 2023-09-23T18:59:13.680571+0000 mon.a (mon.0) 1697 : cluster 0 osdmap e305: 8 total, 7 up, 7 in 2023-09-23T18:59:14.717 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:14 smithi173 bash[59422]: cluster 2023-09-23T18:59:13.683107+0000 osd.5 (osd.5) 230 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T18:59:14.717 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:14 smithi173 bash[59422]: cluster 2023-09-23T18:59:13.683725+0000 osd.6 (osd.6) 276 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T18:59:14.717 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:14 smithi173 bash[59422]: cluster 2023-09-23T18:59:13.688622+0000 osd.6 (osd.6) 277 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T18:59:14.717 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:14 smithi173 bash[59422]: cluster 2023-09-23T18:59:13.690398+0000 osd.6 (osd.6) 278 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T18:59:14.717 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:14 smithi173 bash[59422]: cluster 2023-09-23T18:59:14.680626+0000 mon.a (mon.0) 1698 : cluster 0 osdmap e306: 8 total, 7 up, 7 in 2023-09-23T18:59:14.738 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:14.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:14.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:14.841 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:14.849 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:14 smithi098 bash[76817]: cluster 2023-09-23T18:59:13.678391+0000 osd.4 (osd.4) 33 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T18:59:14.849 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:14 smithi098 bash[76817]: cluster 2023-09-23T18:59:13.680571+0000 mon.a (mon.0) 1697 : cluster 0 osdmap e305: 8 total, 7 up, 7 in 2023-09-23T18:59:14.849 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:14 smithi098 bash[76817]: cluster 2023-09-23T18:59:13.683107+0000 osd.5 (osd.5) 230 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T18:59:14.849 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:14 smithi098 bash[76817]: cluster 2023-09-23T18:59:13.683725+0000 osd.6 (osd.6) 276 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T18:59:14.849 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:14 smithi098 bash[76817]: cluster 2023-09-23T18:59:13.688622+0000 osd.6 (osd.6) 277 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T18:59:14.849 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:14 smithi098 bash[76817]: cluster 2023-09-23T18:59:13.690398+0000 osd.6 (osd.6) 278 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T18:59:14.849 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:14 smithi098 bash[76817]: cluster 2023-09-23T18:59:14.680626+0000 mon.a (mon.0) 1698 : cluster 0 osdmap e306: 8 total, 7 up, 7 in 2023-09-23T18:59:14.850 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:14 smithi098 bash[79516]: cluster 2023-09-23T18:59:13.678391+0000 osd.4 (osd.4) 33 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T18:59:14.850 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:14 smithi098 bash[79516]: cluster 2023-09-23T18:59:13.680571+0000 mon.a (mon.0) 1697 : cluster 0 osdmap e305: 8 total, 7 up, 7 in 2023-09-23T18:59:14.850 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:14 smithi098 bash[79516]: cluster 2023-09-23T18:59:13.683107+0000 osd.5 (osd.5) 230 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T18:59:14.850 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:14 smithi098 bash[79516]: cluster 2023-09-23T18:59:13.683725+0000 osd.6 (osd.6) 276 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T18:59:14.850 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:14 smithi098 bash[79516]: cluster 2023-09-23T18:59:13.688622+0000 osd.6 (osd.6) 277 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T18:59:14.850 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:14 smithi098 bash[79516]: cluster 2023-09-23T18:59:13.690398+0000 osd.6 (osd.6) 278 : cluster 3 failed to encode map e305 with expected crc 2023-09-23T18:59:14.850 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:14 smithi098 bash[79516]: cluster 2023-09-23T18:59:14.680626+0000 mon.a (mon.0) 1698 : cluster 0 osdmap e306: 8 total, 7 up, 7 in 2023-09-23T18:59:14.866 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:14.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:14.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:14.974 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:15.022 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:15.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:15.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:15.129 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:15.154 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:15.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:15.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:15.260 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:15.295 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:15.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:15.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:15.399 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:15.425 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:15.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:15.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:15.528 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:15.583 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:15.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:15.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:15.688 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:15.715 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:15.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:15.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:15.818 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:15.856 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:15.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:15.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:15.963 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:15.974 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:15 smithi098 bash[79516]: cluster 2023-09-23T18:59:14.378909+0000 mgr.y (mgr.26054) 1238 : cluster 0 pgmap v1351: 89 pgs: 5 unknown, 18 peering, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 39 active+clean, 9 active+undersized+degraded, 7 active+undersized; 1.5 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 12433/74910 objects degraded (16.597%); 6/74910 objects misplaced (0.008%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T18:59:15.974 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:15 smithi098 bash[79516]: cluster 2023-09-23T18:59:14.679509+0000 osd.6 (osd.6) 279 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T18:59:15.974 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:15 smithi098 bash[79516]: cluster 2023-09-23T18:59:14.679515+0000 osd.5 (osd.5) 231 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T18:59:15.974 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:15 smithi098 bash[79516]: cluster 2023-09-23T18:59:14.679762+0000 osd.1 (osd.1) 9 : cluster 0 9.4 starting backfill to osd.0 from (0'0,0'0] MAX to 303'10569 2023-09-23T18:59:15.974 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:15 smithi098 bash[79516]: cluster 2023-09-23T18:59:14.681644+0000 osd.1 (osd.1) 10 : cluster 0 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 303'10428 2023-09-23T18:59:15.974 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:15 smithi098 bash[79516]: cluster 2023-09-23T18:59:14.688083+0000 osd.4 (osd.4) 34 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T18:59:15.975 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:15 smithi098 bash[79516]: cluster 2023-09-23T18:59:14.689300+0000 osd.4 (osd.4) 35 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T18:59:15.975 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:15 smithi098 bash[79516]: cluster 2023-09-23T18:59:14.691703+0000 mon.a (mon.0) 1699 : cluster 3 Health check failed: Reduced data availability: 8 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2023-09-23T18:59:15.975 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:15 smithi098 bash[79516]: cluster 2023-09-23T18:59:15.407358+0000 mon.a (mon.0) 1700 : cluster 3 Health check update: Degraded data redundancy: 12433/74910 objects degraded (16.597%), 12 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:15.975 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:15 smithi098 bash[79516]: cluster 2023-09-23T18:59:15.414791+0000 osd.4 (osd.4) 36 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T18:59:15.975 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:15 smithi098 bash[79516]: cluster 2023-09-23T18:59:15.415546+0000 mon.a (mon.0) 1701 : cluster 0 osdmap e307: 8 total, 7 up, 7 in 2023-09-23T18:59:15.975 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:15 smithi098 bash[76817]: cluster 2023-09-23T18:59:14.378909+0000 mgr.y (mgr.26054) 1238 : cluster 0 pgmap v1351: 89 pgs: 5 unknown, 18 peering, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 39 active+clean, 9 active+undersized+degraded, 7 active+undersized; 1.5 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 12433/74910 objects degraded (16.597%); 6/74910 objects misplaced (0.008%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T18:59:15.976 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:15 smithi098 bash[76817]: cluster 2023-09-23T18:59:14.679509+0000 osd.6 (osd.6) 279 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T18:59:15.976 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:15 smithi098 bash[76817]: cluster 2023-09-23T18:59:14.679515+0000 osd.5 (osd.5) 231 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T18:59:15.976 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:15 smithi098 bash[76817]: cluster 2023-09-23T18:59:14.679762+0000 osd.1 (osd.1) 9 : cluster 0 9.4 starting backfill to osd.0 from (0'0,0'0] MAX to 303'10569 2023-09-23T18:59:15.976 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:15 smithi098 bash[76817]: cluster 2023-09-23T18:59:14.681644+0000 osd.1 (osd.1) 10 : cluster 0 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 303'10428 2023-09-23T18:59:15.976 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:15 smithi098 bash[76817]: cluster 2023-09-23T18:59:14.688083+0000 osd.4 (osd.4) 34 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T18:59:15.976 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:15 smithi098 bash[76817]: cluster 2023-09-23T18:59:14.689300+0000 osd.4 (osd.4) 35 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T18:59:15.976 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:15 smithi098 bash[76817]: cluster 2023-09-23T18:59:14.691703+0000 mon.a (mon.0) 1699 : cluster 3 Health check failed: Reduced data availability: 8 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2023-09-23T18:59:15.976 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:15 smithi098 bash[76817]: cluster 2023-09-23T18:59:15.407358+0000 mon.a (mon.0) 1700 : cluster 3 Health check update: Degraded data redundancy: 12433/74910 objects degraded (16.597%), 12 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:15.976 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:15 smithi098 bash[76817]: cluster 2023-09-23T18:59:15.414791+0000 osd.4 (osd.4) 36 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T18:59:15.976 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:15 smithi098 bash[76817]: cluster 2023-09-23T18:59:15.415546+0000 mon.a (mon.0) 1701 : cluster 0 osdmap e307: 8 total, 7 up, 7 in 2023-09-23T18:59:16.005 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:16.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:16.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:16.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:15 smithi173 bash[59422]: cluster 2023-09-23T18:59:14.378909+0000 mgr.y (mgr.26054) 1238 : cluster 0 pgmap v1351: 89 pgs: 5 unknown, 18 peering, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 39 active+clean, 9 active+undersized+degraded, 7 active+undersized; 1.5 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 12433/74910 objects degraded (16.597%); 6/74910 objects misplaced (0.008%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T18:59:16.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:15 smithi173 bash[59422]: cluster 2023-09-23T18:59:14.679509+0000 osd.6 (osd.6) 279 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T18:59:16.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:15 smithi173 bash[59422]: cluster 2023-09-23T18:59:14.679515+0000 osd.5 (osd.5) 231 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T18:59:16.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:15 smithi173 bash[59422]: cluster 2023-09-23T18:59:14.679762+0000 osd.1 (osd.1) 9 : cluster 0 9.4 starting backfill to osd.0 from (0'0,0'0] MAX to 303'10569 2023-09-23T18:59:16.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:15 smithi173 bash[59422]: cluster 2023-09-23T18:59:14.681644+0000 osd.1 (osd.1) 10 : cluster 0 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 303'10428 2023-09-23T18:59:16.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:15 smithi173 bash[59422]: cluster 2023-09-23T18:59:14.688083+0000 osd.4 (osd.4) 34 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T18:59:16.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:15 smithi173 bash[59422]: cluster 2023-09-23T18:59:14.689300+0000 osd.4 (osd.4) 35 : cluster 3 failed to encode map e306 with expected crc 2023-09-23T18:59:16.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:15 smithi173 bash[59422]: cluster 2023-09-23T18:59:14.691703+0000 mon.a (mon.0) 1699 : cluster 3 Health check failed: Reduced data availability: 8 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2023-09-23T18:59:16.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:15 smithi173 bash[59422]: cluster 2023-09-23T18:59:15.407358+0000 mon.a (mon.0) 1700 : cluster 3 Health check update: Degraded data redundancy: 12433/74910 objects degraded (16.597%), 12 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:16.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:15 smithi173 bash[59422]: cluster 2023-09-23T18:59:15.414791+0000 osd.4 (osd.4) 36 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T18:59:16.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:15 smithi173 bash[59422]: cluster 2023-09-23T18:59:15.415546+0000 mon.a (mon.0) 1701 : cluster 0 osdmap e307: 8 total, 7 up, 7 in 2023-09-23T18:59:16.112 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:16.153 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:16.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:16.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:16.259 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:16.283 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:16.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:16.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:16.335 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:59:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:59:16] "GET /metrics HTTP/1.1" 200 36606 "" "Prometheus/2.33.4" 2023-09-23T18:59:16.390 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:16.415 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:16.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:16.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:16.518 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:16.542 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:16.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:16.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:16.645 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:16.669 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:16.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:16.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:16.777 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:16.803 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:16.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:16.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:16.908 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:16.916 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:16 smithi098 bash[79516]: cluster 2023-09-23T18:59:15.414285+0000 osd.5 (osd.5) 232 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T18:59:16.917 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:16 smithi098 bash[79516]: cluster 2023-09-23T18:59:15.415440+0000 osd.6 (osd.6) 280 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T18:59:16.917 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:16 smithi098 bash[79516]: cluster 2023-09-23T18:59:16.425540+0000 osd.4 (osd.4) 37 : cluster 3 failed to encode map e308 with expected crc 2023-09-23T18:59:16.917 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:16 smithi098 bash[79516]: cluster 2023-09-23T18:59:16.425776+0000 mon.a (mon.0) 1702 : cluster 0 osdmap e308: 8 total, 7 up, 7 in 2023-09-23T18:59:16.917 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:16 smithi098 bash[76817]: cluster 2023-09-23T18:59:15.414285+0000 osd.5 (osd.5) 232 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T18:59:16.917 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:16 smithi098 bash[76817]: cluster 2023-09-23T18:59:15.415440+0000 osd.6 (osd.6) 280 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T18:59:16.917 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:16 smithi098 bash[76817]: cluster 2023-09-23T18:59:16.425540+0000 osd.4 (osd.4) 37 : cluster 3 failed to encode map e308 with expected crc 2023-09-23T18:59:16.917 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:16 smithi098 bash[76817]: cluster 2023-09-23T18:59:16.425776+0000 mon.a (mon.0) 1702 : cluster 0 osdmap e308: 8 total, 7 up, 7 in 2023-09-23T18:59:16.943 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:16.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:16.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:17.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:16 smithi173 bash[59422]: cluster 2023-09-23T18:59:15.414285+0000 osd.5 (osd.5) 232 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T18:59:17.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:16 smithi173 bash[59422]: cluster 2023-09-23T18:59:15.415440+0000 osd.6 (osd.6) 280 : cluster 3 failed to encode map e307 with expected crc 2023-09-23T18:59:17.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:16 smithi173 bash[59422]: cluster 2023-09-23T18:59:16.425540+0000 osd.4 (osd.4) 37 : cluster 3 failed to encode map e308 with expected crc 2023-09-23T18:59:17.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:16 smithi173 bash[59422]: cluster 2023-09-23T18:59:16.425776+0000 mon.a (mon.0) 1702 : cluster 0 osdmap e308: 8 total, 7 up, 7 in 2023-09-23T18:59:17.049 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:17.083 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:17.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:17.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:17.193 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:17.220 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:17.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:17.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:17.324 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:17.352 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:17.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:17.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:17.455 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:17.485 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:17.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:17.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:17.589 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:17.617 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:17.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:17.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:17.722 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:17.755 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:17.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:17.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:17.860 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:17.887 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:17.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:17.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:17.991 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:18.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:18.014 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:18.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:18.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:17 smithi173 bash[59422]: cluster 2023-09-23T18:59:16.379735+0000 mgr.y (mgr.26054) 1239 : cluster 0 pgmap v1354: 89 pgs: 5 unknown, 18 peering, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 39 active+clean, 9 active+undersized+degraded, 7 active+undersized; 1.5 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 462 op/s; 12341/73689 objects degraded (16.747%); 6/73689 objects misplaced (0.008%); 0 B/s, 0 objects/s recovering 2023-09-23T18:59:18.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:17 smithi173 bash[59422]: cluster 2023-09-23T18:59:16.425122+0000 osd.5 (osd.5) 233 : cluster 3 failed to encode map e308 with expected crc 2023-09-23T18:59:18.060 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:17 smithi098 bash[76817]: cluster 2023-09-23T18:59:16.379735+0000 mgr.y (mgr.26054) 1239 : cluster 0 pgmap v1354: 89 pgs: 5 unknown, 18 peering, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 39 active+clean, 9 active+undersized+degraded, 7 active+undersized; 1.5 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 462 op/s; 12341/73689 objects degraded (16.747%); 6/73689 objects misplaced (0.008%); 0 B/s, 0 objects/s recovering 2023-09-23T18:59:18.060 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:17 smithi098 bash[76817]: cluster 2023-09-23T18:59:16.425122+0000 osd.5 (osd.5) 233 : cluster 3 failed to encode map e308 with expected crc 2023-09-23T18:59:18.061 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:17 smithi098 bash[79516]: cluster 2023-09-23T18:59:16.379735+0000 mgr.y (mgr.26054) 1239 : cluster 0 pgmap v1354: 89 pgs: 5 unknown, 18 peering, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 8 active+clean+remapped, 39 active+clean, 9 active+undersized+degraded, 7 active+undersized; 1.5 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 462 op/s; 12341/73689 objects degraded (16.747%); 6/73689 objects misplaced (0.008%); 0 B/s, 0 objects/s recovering 2023-09-23T18:59:18.061 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:17 smithi098 bash[79516]: cluster 2023-09-23T18:59:16.425122+0000 osd.5 (osd.5) 233 : cluster 3 failed to encode map e308 with expected crc 2023-09-23T18:59:18.115 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:18.145 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:18.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:18.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:18.249 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:18.275 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:18.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:18.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:18.382 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:18.409 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:18.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:18.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:18.515 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:18.544 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:18.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:18.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:18.647 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:18.674 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:18.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:18.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:18.781 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:18.808 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:18.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:18.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:18.911 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:18.919 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:18 smithi173 bash[59422]: cluster 2023-09-23T18:59:18.718204+0000 mon.a (mon.0) 1703 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 7 pgs peering) 2023-09-23T18:59:18.936 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:18.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:18.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:19.041 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:19.054 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:18 smithi098 bash[76817]: cluster 2023-09-23T18:59:18.718204+0000 mon.a (mon.0) 1703 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 7 pgs peering) 2023-09-23T18:59:19.054 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:18 smithi098 bash[79516]: cluster 2023-09-23T18:59:18.718204+0000 mon.a (mon.0) 1703 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 7 pgs peering) 2023-09-23T18:59:19.063 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:19.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:19.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:19.167 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:19.196 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:19.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:19.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:19.299 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:19.331 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:19.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:19.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:19.438 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:19.460 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:19.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:19.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:19.563 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:19.602 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:19.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:19.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:19.708 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:19.735 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:19.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:19.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:19.840 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:19.864 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:19.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:19.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:19.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:19 smithi098 bash[79516]: cluster 2023-09-23T18:59:18.380621+0000 mgr.y (mgr.26054) 1240 : cluster 0 pgmap v1356: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 21 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.4 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 12673/69312 objects degraded (18.284%); 6957/69312 objects misplaced (10.037%); 9.6 MiB/s, 2 keys/s, 160 objects/s recovering 2023-09-23T18:59:19.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:19 smithi098 bash[76817]: cluster 2023-09-23T18:59:18.380621+0000 mgr.y (mgr.26054) 1240 : cluster 0 pgmap v1356: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 21 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.4 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 12673/69312 objects degraded (18.284%); 6957/69312 objects misplaced (10.037%); 9.6 MiB/s, 2 keys/s, 160 objects/s recovering 2023-09-23T18:59:19.967 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:20.008 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:20.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:20.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:20.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:19 smithi173 bash[59422]: cluster 2023-09-23T18:59:18.380621+0000 mgr.y (mgr.26054) 1240 : cluster 0 pgmap v1356: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 21 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.4 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 12673/69312 objects degraded (18.284%); 6957/69312 objects misplaced (10.037%); 9.6 MiB/s, 2 keys/s, 160 objects/s recovering 2023-09-23T18:59:20.112 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:20.137 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:20.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:20.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:20.240 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:20.268 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:20.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:20.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:20.373 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:20.399 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:20.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:20.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:20.503 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:20.531 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:20.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:20.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:20.635 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:20.667 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:20.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:20.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:20.772 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:20.797 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:20.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:20.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:20.906 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:20.931 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:20.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:20.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:20.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:20 smithi098 bash[79516]: cluster 2023-09-23T18:59:20.409504+0000 mon.a (mon.0) 1704 : cluster 3 Health check update: Degraded data redundancy: 12673/69312 objects degraded (18.284%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:20.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:20 smithi098 bash[76817]: cluster 2023-09-23T18:59:20.409504+0000 mon.a (mon.0) 1704 : cluster 3 Health check update: Degraded data redundancy: 12673/69312 objects degraded (18.284%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:21.035 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:21.041 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:20 smithi173 bash[59422]: cluster 2023-09-23T18:59:20.409504+0000 mon.a (mon.0) 1704 : cluster 3 Health check update: Degraded data redundancy: 12673/69312 objects degraded (18.284%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:21.058 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:21.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:21.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:21.161 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:21.186 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:21.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:21.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:21.289 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:21.331 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:21.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:21.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:21.439 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:21.471 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:21.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:21.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:21.576 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:21.618 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:21.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:21.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:21.724 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:21.748 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:21.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:21.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:21.851 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:21.860 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:21 smithi098 bash[76817]: cluster 2023-09-23T18:59:20.381199+0000 mgr.y (mgr.26054) 1241 : cluster 0 pgmap v1357: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 21 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.4 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 310 op/s; 12673/69312 objects degraded (18.284%); 6957/69312 objects misplaced (10.037%); 7.5 MiB/s, 1 keys/s, 125 objects/s recovering 2023-09-23T18:59:21.860 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:21 smithi098 bash[79516]: cluster 2023-09-23T18:59:20.381199+0000 mgr.y (mgr.26054) 1241 : cluster 0 pgmap v1357: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 21 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.4 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 310 op/s; 12673/69312 objects degraded (18.284%); 6957/69312 objects misplaced (10.037%); 7.5 MiB/s, 1 keys/s, 125 objects/s recovering 2023-09-23T18:59:21.880 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:21.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:21.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:21.985 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:21.996 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:21 smithi173 bash[59422]: cluster 2023-09-23T18:59:20.381199+0000 mgr.y (mgr.26054) 1241 : cluster 0 pgmap v1357: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 21 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.4 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 310 op/s; 12673/69312 objects degraded (18.284%); 6957/69312 objects misplaced (10.037%); 7.5 MiB/s, 1 keys/s, 125 objects/s recovering 2023-09-23T18:59:22.014 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:22.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:22.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:22.117 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:22.169 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:22.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:22.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:22.276 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:22.302 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:22.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:22.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:22.409 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:22.431 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:22.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:22.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:22.534 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:22.563 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:22.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:22.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:22.667 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:22.691 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:22.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:22.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:22.794 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:22.820 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:22.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:22.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:22.923 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:22.947 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:22.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:22.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:23.051 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:23.079 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:23.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:23.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:23.185 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:23.209 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:23.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:23.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:23.228 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:59:23.316 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:23.336 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:23.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:23.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:23.439 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:23.482 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:23.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:23.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:23.588 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:23.618 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:23.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:23.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:23.722 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:23.752 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:23.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:23.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:23.856 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:23.881 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:23.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:23.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:23.985 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:24.014 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:24.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:24.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:24.085 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:23 smithi098 bash[79516]: cluster 2023-09-23T18:59:22.381943+0000 mgr.y (mgr.26054) 1242 : cluster 0 pgmap v1358: 89 pgs: 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 21 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.3 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 413 op/s; 11906/65346 objects degraded (18.220%); 6770/65346 objects misplaced (10.360%); 10 MiB/s, 1 keys/s, 168 objects/s recovering 2023-09-23T18:59:24.085 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:23 smithi098 bash[76817]: cluster 2023-09-23T18:59:22.381943+0000 mgr.y (mgr.26054) 1242 : cluster 0 pgmap v1358: 89 pgs: 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 21 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.3 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 413 op/s; 11906/65346 objects degraded (18.220%); 6770/65346 objects misplaced (10.360%); 10 MiB/s, 1 keys/s, 168 objects/s recovering 2023-09-23T18:59:24.117 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:24.128 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:23 smithi173 bash[59422]: cluster 2023-09-23T18:59:22.381943+0000 mgr.y (mgr.26054) 1242 : cluster 0 pgmap v1358: 89 pgs: 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 21 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.3 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 413 op/s; 11906/65346 objects degraded (18.220%); 6770/65346 objects misplaced (10.360%); 10 MiB/s, 1 keys/s, 168 objects/s recovering 2023-09-23T18:59:24.145 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:24.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:24.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:24.247 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:24.284 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:24.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:24.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:24.390 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:24.415 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:24.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:24.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:24.518 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:24.546 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:24.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:24.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:24.653 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:24.689 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:24.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:24.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:24.793 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:24.828 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:24.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:24.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:24.932 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:24.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:24 smithi098 bash[76817]: audit 2023-09-23T18:59:24.634832+0000 mon.a (mon.0) 1705 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:59:24.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:24 smithi098 bash[79516]: audit 2023-09-23T18:59:24.634832+0000 mon.a (mon.0) 1705 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:59:24.967 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:24.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:24.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:25.072 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:25.099 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 6, 7, 5, 3] out_osds: [4] dead_osds: [7] live_osds: [0, 2, 3, 5, 6, 1, 4] 2023-09-23T18:59:25.099 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:59:25.100 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-09-23T18:59:25.100 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T18:59:25.100 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T18:59:25.100 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-09-23T18:59:25.112 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:25.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:25.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:25.216 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:25.267 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:25.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:25.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:25.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:24 smithi173 bash[59422]: audit 2023-09-23T18:59:24.634832+0000 mon.a (mon.0) 1705 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:59:25.373 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:25.399 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:25.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:25.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:25.502 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:25.528 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:25.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:25.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:25.631 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:25.661 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:25.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:25.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:25.764 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:25.802 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:25.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:25.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:25.908 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:25.916 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:25 smithi173 bash[59422]: cluster 2023-09-23T18:59:24.382716+0000 mgr.y (mgr.26054) 1243 : cluster 0 pgmap v1359: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 21 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.2 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 545 op/s; 10635/60243 objects degraded (17.654%); 6395/60243 objects misplaced (10.615%); 12 MiB/s, 1 keys/s, 188 objects/s recovering 2023-09-23T18:59:25.916 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:25 smithi173 bash[59422]: cluster 2023-09-23T18:59:25.410613+0000 mon.a (mon.0) 1706 : cluster 3 Health check update: Degraded data redundancy: 10635/60243 objects degraded (17.654%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:25.935 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:25.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:25.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:26.041 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:26.076 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:26.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:26.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:26.096 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:25 smithi098 bash[79516]: cluster 2023-09-23T18:59:24.382716+0000 mgr.y (mgr.26054) 1243 : cluster 0 pgmap v1359: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 21 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.2 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 545 op/s; 10635/60243 objects degraded (17.654%); 6395/60243 objects misplaced (10.615%); 12 MiB/s, 1 keys/s, 188 objects/s recovering 2023-09-23T18:59:26.096 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:25 smithi098 bash[79516]: cluster 2023-09-23T18:59:25.410613+0000 mon.a (mon.0) 1706 : cluster 3 Health check update: Degraded data redundancy: 10635/60243 objects degraded (17.654%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:26.097 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:25 smithi098 bash[76817]: cluster 2023-09-23T18:59:24.382716+0000 mgr.y (mgr.26054) 1243 : cluster 0 pgmap v1359: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 21 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.2 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 545 op/s; 10635/60243 objects degraded (17.654%); 6395/60243 objects misplaced (10.615%); 12 MiB/s, 1 keys/s, 188 objects/s recovering 2023-09-23T18:59:26.097 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:25 smithi098 bash[76817]: cluster 2023-09-23T18:59:25.410613+0000 mon.a (mon.0) 1706 : cluster 3 Health check update: Degraded data redundancy: 10635/60243 objects degraded (17.654%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:26.180 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:26.204 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:26.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:26.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:26.308 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:26.333 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:26.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:26.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:26.350 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:59:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:59:26] "GET /metrics HTTP/1.1" 200 36612 "" "Prometheus/2.33.4" 2023-09-23T18:59:26.438 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:26.472 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:26.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:26.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:26.577 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:26.606 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:26.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:26.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:26.712 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:26.740 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:26.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:26.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:26.843 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:26.870 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:26.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:26.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:26.969 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T18:59:26.969 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:59:26.976 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:27.010 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:27.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:27.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:27.115 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:27.143 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:27.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:27.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:27.250 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:27.281 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:27.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:27.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:27.387 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:27.414 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:27.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:27.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:27.517 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:27.552 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:27.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:27.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:27.657 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:27.682 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:27.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:27.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:27.785 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:27.812 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:27.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:27.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:27.818 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:27 smithi098 bash[76817]: cluster 2023-09-23T18:59:26.383376+0000 mgr.y (mgr.26054) 1244 : cluster 0 pgmap v1360: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 21 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 10451/57960 objects degraded (18.031%); 6395/57960 objects misplaced (11.033%); 10 MiB/s, 0 keys/s, 169 objects/s recovering 2023-09-23T18:59:27.917 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:27.943 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:27.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:27.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:28.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:27 smithi173 bash[59422]: cluster 2023-09-23T18:59:26.383376+0000 mgr.y (mgr.26054) 1244 : cluster 0 pgmap v1360: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 21 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 10451/57960 objects degraded (18.031%); 6395/57960 objects misplaced (11.033%); 10 MiB/s, 0 keys/s, 169 objects/s recovering 2023-09-23T18:59:28.048 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:28.075 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:27 smithi098 bash[79516]: cluster 2023-09-23T18:59:26.383376+0000 mgr.y (mgr.26054) 1244 : cluster 0 pgmap v1360: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 21 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 10451/57960 objects degraded (18.031%); 6395/57960 objects misplaced (11.033%); 10 MiB/s, 0 keys/s, 169 objects/s recovering 2023-09-23T18:59:28.079 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:28.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:28.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:28.183 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:28.211 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:28.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:28.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:28.314 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:28.343 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:28.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:28.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:28.449 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:28.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:28.481 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:28.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:28.582 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:28.613 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:28.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:28.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:28.719 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:28.748 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:28.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:28.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:28.851 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:28.892 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:28.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:28.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:29.001 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:29.026 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:29.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:29.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:29.129 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:29.154 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:29.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:29.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:29.257 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:29.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:29.301 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:29.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:29.402 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:29.425 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:29.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:29.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:29.529 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:29.555 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:29.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:29.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:29.664 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:29.692 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:29.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:29.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:29.795 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:29.818 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:29.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:29.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:29.826 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:29 smithi098 bash[76817]: cluster 2023-09-23T18:59:28.388741+0000 mgr.y (mgr.26054) 1245 : cluster 0 pgmap v1361: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 21 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.0 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 8699/51348 objects degraded (16.941%); 5828/51348 objects misplaced (11.350%); 13 MiB/s, 0 keys/s, 218 objects/s recovering 2023-09-23T18:59:29.921 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:29.950 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:29.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:29.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:30.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:29 smithi173 bash[59422]: cluster 2023-09-23T18:59:28.388741+0000 mgr.y (mgr.26054) 1245 : cluster 0 pgmap v1361: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 21 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.0 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 8699/51348 objects degraded (16.941%); 5828/51348 objects misplaced (11.350%); 13 MiB/s, 0 keys/s, 218 objects/s recovering 2023-09-23T18:59:30.055 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:30.091 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:30.093 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:29 smithi098 bash[79516]: cluster 2023-09-23T18:59:28.388741+0000 mgr.y (mgr.26054) 1245 : cluster 0 pgmap v1361: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 21 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.0 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 8699/51348 objects degraded (16.941%); 5828/51348 objects misplaced (11.350%); 13 MiB/s, 0 keys/s, 218 objects/s recovering 2023-09-23T18:59:30.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:30.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:30.197 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:30.225 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:30.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:30.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:30.328 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:30.353 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:30.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:30.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:30.456 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:30.485 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:30.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:30.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:30.588 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:30.613 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:30.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:30.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:30.719 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:30.754 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:30.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:30.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:30.858 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:30.867 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:30 smithi098 bash[76817]: cluster 2023-09-23T18:59:30.414131+0000 mon.a (mon.0) 1707 : cluster 3 Health check update: Degraded data redundancy: 8699/51348 objects degraded (16.941%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:30.867 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:30 smithi098 bash[79516]: cluster 2023-09-23T18:59:30.414131+0000 mon.a (mon.0) 1707 : cluster 3 Health check update: Degraded data redundancy: 8699/51348 objects degraded (16.941%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:30.897 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:30.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:30.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:31.003 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:31.030 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:31.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:31.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:31.136 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:31.143 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:30 smithi173 bash[59422]: cluster 2023-09-23T18:59:30.414131+0000 mon.a (mon.0) 1707 : cluster 3 Health check update: Degraded data redundancy: 8699/51348 objects degraded (16.941%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:31.160 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:31.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:31.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:31.263 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:31.289 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:31.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:31.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:31.393 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:31.417 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:31.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:31.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:31.523 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:31.555 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:31.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:31.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:31.658 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:31.682 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:31.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:31.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:31.785 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:31.818 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:31.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:31.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:31.924 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:31.948 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:31.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:31.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:32.052 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:32.086 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:32.088 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:31 smithi173 bash[59422]: cluster 2023-09-23T18:59:30.389428+0000 mgr.y (mgr.26054) 1246 : cluster 0 pgmap v1362: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 21 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.0 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 8699/51348 objects degraded (16.941%); 5828/51348 objects misplaced (11.350%); 9.7 MiB/s, 154 objects/s recovering 2023-09-23T18:59:32.088 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:31 smithi173 bash[59422]: cluster 2023-09-23T18:59:31.423709+0000 mon.a (mon.0) 1708 : cluster 0 osdmap e309: 8 total, 7 up, 7 in 2023-09-23T18:59:32.088 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:31 smithi173 bash[59422]: cluster 2023-09-23T18:59:31.431947+0000 osd.4 (osd.4) 38 : cluster 3 failed to encode map e309 with expected crc 2023-09-23T18:59:32.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:32.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:32.190 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:32.210 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:32.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:32.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:32.255 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:31 smithi098 bash[76817]: cluster 2023-09-23T18:59:30.389428+0000 mgr.y (mgr.26054) 1246 : cluster 0 pgmap v1362: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 21 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.0 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 8699/51348 objects degraded (16.941%); 5828/51348 objects misplaced (11.350%); 9.7 MiB/s, 154 objects/s recovering 2023-09-23T18:59:32.255 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:31 smithi098 bash[76817]: cluster 2023-09-23T18:59:31.423709+0000 mon.a (mon.0) 1708 : cluster 0 osdmap e309: 8 total, 7 up, 7 in 2023-09-23T18:59:32.255 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:31 smithi098 bash[76817]: cluster 2023-09-23T18:59:31.431947+0000 osd.4 (osd.4) 38 : cluster 3 failed to encode map e309 with expected crc 2023-09-23T18:59:32.256 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:31 smithi098 bash[79516]: cluster 2023-09-23T18:59:30.389428+0000 mgr.y (mgr.26054) 1246 : cluster 0 pgmap v1362: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 21 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.0 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 8699/51348 objects degraded (16.941%); 5828/51348 objects misplaced (11.350%); 9.7 MiB/s, 154 objects/s recovering 2023-09-23T18:59:32.256 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:31 smithi098 bash[79516]: cluster 2023-09-23T18:59:31.423709+0000 mon.a (mon.0) 1708 : cluster 0 osdmap e309: 8 total, 7 up, 7 in 2023-09-23T18:59:32.256 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:31 smithi098 bash[79516]: cluster 2023-09-23T18:59:31.431947+0000 osd.4 (osd.4) 38 : cluster 3 failed to encode map e309 with expected crc 2023-09-23T18:59:32.313 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:32.337 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:32.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:32.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:32.440 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:32.469 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:32.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:32.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:32.573 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:32.602 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:32.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:32.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:32.707 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:32.730 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:32.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:32.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:32.833 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:32.867 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:32.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:32.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:32.970 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:32.979 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:32 smithi173 bash[59422]: cluster 2023-09-23T18:59:31.422820+0000 osd.5 (osd.5) 234 : cluster 3 failed to encode map e309 with expected crc 2023-09-23T18:59:32.979 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:32 smithi173 bash[59422]: cluster 2023-09-23T18:59:31.423819+0000 osd.6 (osd.6) 281 : cluster 3 failed to encode map e309 with expected crc 2023-09-23T18:59:32.979 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:32 smithi173 bash[59422]: cluster 2023-09-23T18:59:32.424880+0000 mon.a (mon.0) 1709 : cluster 0 osdmap e310: 8 total, 7 up, 7 in 2023-09-23T18:59:32.980 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:32 smithi173 bash[59422]: cluster 2023-09-23T18:59:32.424924+0000 osd.4 (osd.4) 39 : cluster 3 failed to encode map e310 with expected crc 2023-09-23T18:59:32.993 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:32.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:32.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:33.097 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:33.106 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:32 smithi098 bash[76817]: cluster 2023-09-23T18:59:31.422820+0000 osd.5 (osd.5) 234 : cluster 3 failed to encode map e309 with expected crc 2023-09-23T18:59:33.106 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:32 smithi098 bash[76817]: cluster 2023-09-23T18:59:31.423819+0000 osd.6 (osd.6) 281 : cluster 3 failed to encode map e309 with expected crc 2023-09-23T18:59:33.106 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:32 smithi098 bash[76817]: cluster 2023-09-23T18:59:32.424880+0000 mon.a (mon.0) 1709 : cluster 0 osdmap e310: 8 total, 7 up, 7 in 2023-09-23T18:59:33.106 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:32 smithi098 bash[76817]: cluster 2023-09-23T18:59:32.424924+0000 osd.4 (osd.4) 39 : cluster 3 failed to encode map e310 with expected crc 2023-09-23T18:59:33.106 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:32 smithi098 bash[79516]: cluster 2023-09-23T18:59:31.422820+0000 osd.5 (osd.5) 234 : cluster 3 failed to encode map e309 with expected crc 2023-09-23T18:59:33.107 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:32 smithi098 bash[79516]: cluster 2023-09-23T18:59:31.423819+0000 osd.6 (osd.6) 281 : cluster 3 failed to encode map e309 with expected crc 2023-09-23T18:59:33.107 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:32 smithi098 bash[79516]: cluster 2023-09-23T18:59:32.424880+0000 mon.a (mon.0) 1709 : cluster 0 osdmap e310: 8 total, 7 up, 7 in 2023-09-23T18:59:33.107 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:32 smithi098 bash[79516]: cluster 2023-09-23T18:59:32.424924+0000 osd.4 (osd.4) 39 : cluster 3 failed to encode map e310 with expected crc 2023-09-23T18:59:33.132 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:33.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:33.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:33.237 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:33.266 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:33.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:33.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:33.369 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:33.410 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:33.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:33.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:33.516 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:33.550 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:33.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:33.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:33.654 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:33.682 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:33.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:33.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:33.786 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:33.811 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:33.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:33.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:33.916 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:33.925 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:33 smithi098 bash[76817]: cluster 2023-09-23T18:59:32.390223+0000 mgr.y (mgr.26054) 1247 : cluster 0 pgmap v1364: 89 pgs: 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 21 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 992 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 7954/47505 objects degraded (16.744%); 5657/47505 objects misplaced (11.908%); 12 MiB/s, 184 objects/s recovering 2023-09-23T18:59:33.925 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:33 smithi098 bash[76817]: cluster 2023-09-23T18:59:32.424084+0000 osd.6 (osd.6) 282 : cluster 3 failed to encode map e310 with expected crc 2023-09-23T18:59:33.925 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:33 smithi098 bash[76817]: cluster 2023-09-23T18:59:32.438074+0000 osd.5 (osd.5) 235 : cluster 3 failed to encode map e310 with expected crc 2023-09-23T18:59:33.925 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:33 smithi098 bash[79516]: cluster 2023-09-23T18:59:32.390223+0000 mgr.y (mgr.26054) 1247 : cluster 0 pgmap v1364: 89 pgs: 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 21 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 992 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 7954/47505 objects degraded (16.744%); 5657/47505 objects misplaced (11.908%); 12 MiB/s, 184 objects/s recovering 2023-09-23T18:59:33.926 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:33 smithi098 bash[79516]: cluster 2023-09-23T18:59:32.424084+0000 osd.6 (osd.6) 282 : cluster 3 failed to encode map e310 with expected crc 2023-09-23T18:59:33.926 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:33 smithi098 bash[79516]: cluster 2023-09-23T18:59:32.438074+0000 osd.5 (osd.5) 235 : cluster 3 failed to encode map e310 with expected crc 2023-09-23T18:59:33.945 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:33.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:33.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:34.048 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:34.057 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:33 smithi173 bash[59422]: cluster 2023-09-23T18:59:32.390223+0000 mgr.y (mgr.26054) 1247 : cluster 0 pgmap v1364: 89 pgs: 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 21 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 992 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 7954/47505 objects degraded (16.744%); 5657/47505 objects misplaced (11.908%); 12 MiB/s, 184 objects/s recovering 2023-09-23T18:59:34.057 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:33 smithi173 bash[59422]: cluster 2023-09-23T18:59:32.424084+0000 osd.6 (osd.6) 282 : cluster 3 failed to encode map e310 with expected crc 2023-09-23T18:59:34.058 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:33 smithi173 bash[59422]: cluster 2023-09-23T18:59:32.438074+0000 osd.5 (osd.5) 235 : cluster 3 failed to encode map e310 with expected crc 2023-09-23T18:59:34.073 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:34.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:34.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:34.178 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:34.213 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:34.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:34.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:34.320 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:34.347 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:34.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:34.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:34.454 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:34.483 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:34.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:34.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:34.590 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:34.622 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:34.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:34.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:34.725 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:34.761 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:34.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:34.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:34.873 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:34.898 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:34.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:34.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:35.001 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:35.028 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:35.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:35.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:35.132 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:35.154 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:35.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:35.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:35.257 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:35.290 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:35.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:35.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:35.397 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:35.423 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:35.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:35.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:35.526 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:35.559 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:35.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:35.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:35.664 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:35.692 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:35.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:35.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:35.795 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:35.830 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:35.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:35.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:35.935 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:35.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:35 smithi098 bash[79516]: cluster 2023-09-23T18:59:34.391011+0000 mgr.y (mgr.26054) 1248 : cluster 0 pgmap v1366: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 904 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 7121/43248 objects degraded (16.466%); 4973/43248 objects misplaced (11.499%); 16 MiB/s, 252 objects/s recovering 2023-09-23T18:59:35.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:35 smithi098 bash[79516]: cluster 2023-09-23T18:59:35.415522+0000 mon.a (mon.0) 1710 : cluster 3 Health check update: Degraded data redundancy: 7121/43248 objects degraded (16.466%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:35.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:35 smithi098 bash[76817]: cluster 2023-09-23T18:59:34.391011+0000 mgr.y (mgr.26054) 1248 : cluster 0 pgmap v1366: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 904 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 7121/43248 objects degraded (16.466%); 4973/43248 objects misplaced (11.499%); 16 MiB/s, 252 objects/s recovering 2023-09-23T18:59:35.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:35 smithi098 bash[76817]: cluster 2023-09-23T18:59:35.415522+0000 mon.a (mon.0) 1710 : cluster 3 Health check update: Degraded data redundancy: 7121/43248 objects degraded (16.466%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:35.958 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:35.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:35.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:36.061 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:36.088 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:36.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:36.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:36.193 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:59:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:59:36] "GET /metrics HTTP/1.1" 200 36612 "" "Prometheus/2.33.4" 2023-09-23T18:59:36.198 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:36.208 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:35 smithi173 bash[59422]: cluster 2023-09-23T18:59:34.391011+0000 mgr.y (mgr.26054) 1248 : cluster 0 pgmap v1366: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 904 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 7121/43248 objects degraded (16.466%); 4973/43248 objects misplaced (11.499%); 16 MiB/s, 252 objects/s recovering 2023-09-23T18:59:36.208 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:35 smithi173 bash[59422]: cluster 2023-09-23T18:59:35.415522+0000 mon.a (mon.0) 1710 : cluster 3 Health check update: Degraded data redundancy: 7121/43248 objects degraded (16.466%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:36.222 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:36.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:36.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:36.326 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:36.351 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:36.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:36.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:36.462 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:36.490 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:36.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:36.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:36.532 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T18:59:36.596 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:36.623 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:36.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:36.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:36.726 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:36.753 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:36.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:36.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:36.858 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:36.883 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:36.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:36.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:36.986 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:37.016 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:37.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:37.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:37.120 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:37.146 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:37.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:37.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:37.249 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:37.273 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:37.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:37.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:37.377 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:37.401 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:37.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:37.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:37.506 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:37.532 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:37.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:37.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:37.636 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:37.662 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:37.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:37.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:37.766 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:37.785 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:37.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:37.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:37.891 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:37.913 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:37.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:37.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:38.020 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:38.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:37 smithi173 bash[59422]: cluster 2023-09-23T18:59:36.391717+0000 mgr.y (mgr.26054) 1249 : cluster 0 pgmap v1367: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 863 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 419 op/s; 6946/41271 objects degraded (16.830%); 4973/41271 objects misplaced (12.050%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T18:59:38.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:37 smithi173 bash[59422]: audit 2023-09-23T18:59:37.696084+0000 mon.a (mon.0) 1711 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:59:38.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:37 smithi173 bash[59422]: audit 2023-09-23T18:59:37.708624+0000 mon.a (mon.0) 1712 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:59:38.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:37 smithi173 bash[59422]: audit 2023-09-23T18:59:37.711319+0000 mon.a (mon.0) 1713 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:59:38.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:37 smithi173 bash[59422]: audit 2023-09-23T18:59:37.724638+0000 mon.a (mon.0) 1714 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:59:38.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:37 smithi173 bash[59422]: audit 2023-09-23T18:59:37.733485+0000 mon.a (mon.0) 1715 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:59:38.029 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:37 smithi098 bash[76817]: cluster 2023-09-23T18:59:36.391717+0000 mgr.y (mgr.26054) 1249 : cluster 0 pgmap v1367: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 863 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 419 op/s; 6946/41271 objects degraded (16.830%); 4973/41271 objects misplaced (12.050%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T18:59:38.029 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:37 smithi098 bash[76817]: audit 2023-09-23T18:59:37.696084+0000 mon.a (mon.0) 1711 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:59:38.029 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:37 smithi098 bash[76817]: audit 2023-09-23T18:59:37.708624+0000 mon.a (mon.0) 1712 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:59:38.030 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:37 smithi098 bash[76817]: audit 2023-09-23T18:59:37.711319+0000 mon.a (mon.0) 1713 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:59:38.030 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:37 smithi098 bash[76817]: audit 2023-09-23T18:59:37.724638+0000 mon.a (mon.0) 1714 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:59:38.030 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:37 smithi098 bash[76817]: audit 2023-09-23T18:59:37.733485+0000 mon.a (mon.0) 1715 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:59:38.030 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:37 smithi098 bash[79516]: cluster 2023-09-23T18:59:36.391717+0000 mgr.y (mgr.26054) 1249 : cluster 0 pgmap v1367: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 863 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 419 op/s; 6946/41271 objects degraded (16.830%); 4973/41271 objects misplaced (12.050%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T18:59:38.030 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:37 smithi098 bash[79516]: audit 2023-09-23T18:59:37.696084+0000 mon.a (mon.0) 1711 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:59:38.030 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:37 smithi098 bash[79516]: audit 2023-09-23T18:59:37.708624+0000 mon.a (mon.0) 1712 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:59:38.030 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:37 smithi098 bash[79516]: audit 2023-09-23T18:59:37.711319+0000 mon.a (mon.0) 1713 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:59:38.030 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:37 smithi098 bash[79516]: audit 2023-09-23T18:59:37.724638+0000 mon.a (mon.0) 1714 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:59:38.031 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:37 smithi098 bash[79516]: audit 2023-09-23T18:59:37.733485+0000 mon.a (mon.0) 1715 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T18:59:38.047 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:38.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:38.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:38.156 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:38.177 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:38.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:38.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:38.282 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:38.310 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:38.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:38.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:38.413 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:38.448 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:38.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:38.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:38.553 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:38.582 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:38.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:38.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:38.690 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:38.720 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:38.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:38.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:38.822 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:38.856 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:38.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:38.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:38.966 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:38.990 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:38.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:38.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:39.094 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:39.142 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:39.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:39.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:39.245 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:39.271 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:39.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:39.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:39.375 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:39.408 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:39.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:39.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:39.512 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:39.540 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:39.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:39.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:39.644 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:39.682 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:39.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:39.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:39.786 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:39.816 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:39.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:39.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:39.919 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:39.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:39.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:39.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:40.047 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:40.075 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:40.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:40.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:40.183 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:40.193 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:39 smithi173 bash[59422]: cluster 2023-09-23T18:59:38.392702+0000 mgr.y (mgr.26054) 1250 : cluster 0 pgmap v1368: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 728 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 5690/34797 objects degraded (16.352%); 4314/34797 objects misplaced (12.398%); 15 MiB/s, 238 objects/s recovering 2023-09-23T18:59:40.193 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:39 smithi173 bash[59422]: audit 2023-09-23T18:59:39.635835+0000 mon.a (mon.0) 1716 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:59:40.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:39 smithi098 bash[76817]: cluster 2023-09-23T18:59:38.392702+0000 mgr.y (mgr.26054) 1250 : cluster 0 pgmap v1368: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 728 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 5690/34797 objects degraded (16.352%); 4314/34797 objects misplaced (12.398%); 15 MiB/s, 238 objects/s recovering 2023-09-23T18:59:40.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:39 smithi098 bash[76817]: audit 2023-09-23T18:59:39.635835+0000 mon.a (mon.0) 1716 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:59:40.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:39 smithi098 bash[79516]: cluster 2023-09-23T18:59:38.392702+0000 mgr.y (mgr.26054) 1250 : cluster 0 pgmap v1368: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 728 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 5690/34797 objects degraded (16.352%); 4314/34797 objects misplaced (12.398%); 15 MiB/s, 238 objects/s recovering 2023-09-23T18:59:40.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:39 smithi098 bash[79516]: audit 2023-09-23T18:59:39.635835+0000 mon.a (mon.0) 1716 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:59:40.211 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:40.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:40.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:40.314 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:40.340 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:40.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:40.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:40.443 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:40.470 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:40.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:40.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:40.506 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T18:59:40.542 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T18:59:40.573 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:40.579 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T18:59:40.598 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:40.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:40.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:40.630 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T18:59:40.663 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T18:59:40.683 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T18:59:40.704 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:40.726 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T18:59:40.728 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:59:40.728 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:59:40.728 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:59:40.728 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:59:40.728 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:59:40.728 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:59:40.728 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:59:40.728 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T18:59:40.729 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:40.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:40.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:40.833 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:40.861 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:40.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:40.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:40.965 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:40.997 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:41.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:41.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:41.105 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:41.115 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:40 smithi173 bash[59422]: cluster 2023-09-23T18:59:40.416671+0000 mon.a (mon.0) 1717 : cluster 3 Health check update: Degraded data redundancy: 5690/34797 objects degraded (16.352%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:41.115 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:40 smithi173 bash[59422]: audit 2023-09-23T18:59:40.466636+0000 mon.c (mon.2) 136 : audit 0 from='client.? 172.21.15.98:0/3372192821' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:59:41.133 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:41.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:41.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:41.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:40 smithi098 bash[79516]: cluster 2023-09-23T18:59:40.416671+0000 mon.a (mon.0) 1717 : cluster 3 Health check update: Degraded data redundancy: 5690/34797 objects degraded (16.352%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:41.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:40 smithi098 bash[79516]: audit 2023-09-23T18:59:40.466636+0000 mon.c (mon.2) 136 : audit 0 from='client.? 172.21.15.98:0/3372192821' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:59:41.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:40 smithi098 bash[76817]: cluster 2023-09-23T18:59:40.416671+0000 mon.a (mon.0) 1717 : cluster 3 Health check update: Degraded data redundancy: 5690/34797 objects degraded (16.352%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:41.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:40 smithi098 bash[76817]: audit 2023-09-23T18:59:40.466636+0000 mon.c (mon.2) 136 : audit 0 from='client.? 172.21.15.98:0/3372192821' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:59:41.236 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:41.275 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:41.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:41.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:41.380 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:41.404 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:41.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:41.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:41.513 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:41.538 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:41.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:41.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:41.642 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:41.664 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:41.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:41.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:41.767 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:41.796 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:41.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:41.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:41.898 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:41.901 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:41 smithi098 bash[76817]: cluster 2023-09-23T18:59:40.393373+0000 mgr.y (mgr.26054) 1251 : cluster 0 pgmap v1369: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 728 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 5690/34797 objects degraded (16.352%); 4314/34797 objects misplaced (12.398%); 13 MiB/s, 212 objects/s recovering 2023-09-23T18:59:41.932 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:41.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:41.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:42.040 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:42.076 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:42.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:42.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:42.182 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:42.189 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:41 smithi098 bash[79516]: cluster 2023-09-23T18:59:40.393373+0000 mgr.y (mgr.26054) 1251 : cluster 0 pgmap v1369: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 728 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 5690/34797 objects degraded (16.352%); 4314/34797 objects misplaced (12.398%); 13 MiB/s, 212 objects/s recovering 2023-09-23T18:59:42.205 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:42.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:42.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:42.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:41 smithi173 bash[59422]: cluster 2023-09-23T18:59:40.393373+0000 mgr.y (mgr.26054) 1251 : cluster 0 pgmap v1369: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 728 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 5690/34797 objects degraded (16.352%); 4314/34797 objects misplaced (12.398%); 13 MiB/s, 212 objects/s recovering 2023-09-23T18:59:42.311 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:42.340 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:42.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:42.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:42.443 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:42.470 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:42.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:42.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:42.574 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:42.597 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:42.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:42.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:42.699 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:42.726 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:42.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:42.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:42.829 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:42.859 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:42.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:42.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:42.908 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:42 smithi098 bash[76817]: cluster 2023-09-23T18:59:41.894049+0000 osd.6 (osd.6) 283 : cluster 3 failed to encode map e311 with expected crc 2023-09-23T18:59:42.908 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:42 smithi098 bash[76817]: cluster 2023-09-23T18:59:41.894761+0000 mon.a (mon.0) 1718 : cluster 0 osdmap e311: 8 total, 7 up, 7 in 2023-09-23T18:59:42.908 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:42 smithi098 bash[76817]: cluster 2023-09-23T18:59:41.896785+0000 osd.5 (osd.5) 236 : cluster 3 failed to encode map e311 with expected crc 2023-09-23T18:59:42.908 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:42 smithi098 bash[76817]: cluster 2023-09-23T18:59:41.908921+0000 osd.4 (osd.4) 40 : cluster 3 failed to encode map e311 with expected crc 2023-09-23T18:59:42.963 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:42.976 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:42 smithi173 bash[59422]: cluster 2023-09-23T18:59:41.894049+0000 osd.6 (osd.6) 283 : cluster 3 failed to encode map e311 with expected crc 2023-09-23T18:59:42.976 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:42 smithi173 bash[59422]: cluster 2023-09-23T18:59:41.894761+0000 mon.a (mon.0) 1718 : cluster 0 osdmap e311: 8 total, 7 up, 7 in 2023-09-23T18:59:42.976 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:42 smithi173 bash[59422]: cluster 2023-09-23T18:59:41.896785+0000 osd.5 (osd.5) 236 : cluster 3 failed to encode map e311 with expected crc 2023-09-23T18:59:42.976 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:42 smithi173 bash[59422]: cluster 2023-09-23T18:59:41.908921+0000 osd.4 (osd.4) 40 : cluster 3 failed to encode map e311 with expected crc 2023-09-23T18:59:42.993 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:42.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:42.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:43.096 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:43.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:43.124 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:43.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:43.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:42 smithi098 bash[79516]: cluster 2023-09-23T18:59:41.894049+0000 osd.6 (osd.6) 283 : cluster 3 failed to encode map e311 with expected crc 2023-09-23T18:59:43.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:42 smithi098 bash[79516]: cluster 2023-09-23T18:59:41.894761+0000 mon.a (mon.0) 1718 : cluster 0 osdmap e311: 8 total, 7 up, 7 in 2023-09-23T18:59:43.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:42 smithi098 bash[79516]: cluster 2023-09-23T18:59:41.896785+0000 osd.5 (osd.5) 236 : cluster 3 failed to encode map e311 with expected crc 2023-09-23T18:59:43.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:42 smithi098 bash[79516]: cluster 2023-09-23T18:59:41.908921+0000 osd.4 (osd.4) 40 : cluster 3 failed to encode map e311 with expected crc 2023-09-23T18:59:43.225 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:43.250 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:43.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:43.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:43.353 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:43.393 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:43.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:43.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:43.497 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:43.522 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:43.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:43.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:43.626 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:43.651 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:43.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:43.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:43.754 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:43.779 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:43.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:43.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:43.884 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:43.908 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:43.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:43.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:44.014 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:44.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:43 smithi173 bash[59422]: cluster 2023-09-23T18:59:42.393989+0000 mgr.y (mgr.26054) 1252 : cluster 0 pgmap v1371: 89 pgs: 1 active+recovering+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 644 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 4926/30798 objects degraded (15.995%); 4128/30798 objects misplaced (13.403%); 12 MiB/s, 196 objects/s recovering 2023-09-23T18:59:44.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:43 smithi173 bash[59422]: cluster 2023-09-23T18:59:42.906084+0000 osd.5 (osd.5) 237 : cluster 3 failed to encode map e312 with expected crc 2023-09-23T18:59:44.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:43 smithi173 bash[59422]: cluster 2023-09-23T18:59:42.907124+0000 mon.a (mon.0) 1719 : cluster 0 osdmap e312: 8 total, 7 up, 7 in 2023-09-23T18:59:44.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:43 smithi173 bash[59422]: cluster 2023-09-23T18:59:42.908881+0000 osd.4 (osd.4) 41 : cluster 3 failed to encode map e312 with expected crc 2023-09-23T18:59:44.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:43 smithi173 bash[59422]: cluster 2023-09-23T18:59:42.947816+0000 osd.6 (osd.6) 284 : cluster 3 failed to encode map e312 with expected crc 2023-09-23T18:59:44.038 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:44.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:44.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:44.066 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:59:44.067 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:59:44.143 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:44.173 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:44.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:44.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:44.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:43 smithi098 bash[79516]: cluster 2023-09-23T18:59:42.393989+0000 mgr.y (mgr.26054) 1252 : cluster 0 pgmap v1371: 89 pgs: 1 active+recovering+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 644 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 4926/30798 objects degraded (15.995%); 4128/30798 objects misplaced (13.403%); 12 MiB/s, 196 objects/s recovering 2023-09-23T18:59:44.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:43 smithi098 bash[79516]: cluster 2023-09-23T18:59:42.906084+0000 osd.5 (osd.5) 237 : cluster 3 failed to encode map e312 with expected crc 2023-09-23T18:59:44.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:43 smithi098 bash[79516]: cluster 2023-09-23T18:59:42.907124+0000 mon.a (mon.0) 1719 : cluster 0 osdmap e312: 8 total, 7 up, 7 in 2023-09-23T18:59:44.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:43 smithi098 bash[79516]: cluster 2023-09-23T18:59:42.908881+0000 osd.4 (osd.4) 41 : cluster 3 failed to encode map e312 with expected crc 2023-09-23T18:59:44.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:43 smithi098 bash[79516]: cluster 2023-09-23T18:59:42.947816+0000 osd.6 (osd.6) 284 : cluster 3 failed to encode map e312 with expected crc 2023-09-23T18:59:44.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:43 smithi098 bash[76817]: cluster 2023-09-23T18:59:42.393989+0000 mgr.y (mgr.26054) 1252 : cluster 0 pgmap v1371: 89 pgs: 1 active+recovering+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 51 active+clean, 7 active+undersized+degraded, 5 active+undersized; 644 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 4926/30798 objects degraded (15.995%); 4128/30798 objects misplaced (13.403%); 12 MiB/s, 196 objects/s recovering 2023-09-23T18:59:44.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:43 smithi098 bash[76817]: cluster 2023-09-23T18:59:42.906084+0000 osd.5 (osd.5) 237 : cluster 3 failed to encode map e312 with expected crc 2023-09-23T18:59:44.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:43 smithi098 bash[76817]: cluster 2023-09-23T18:59:42.907124+0000 mon.a (mon.0) 1719 : cluster 0 osdmap e312: 8 total, 7 up, 7 in 2023-09-23T18:59:44.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:43 smithi098 bash[76817]: cluster 2023-09-23T18:59:42.908881+0000 osd.4 (osd.4) 41 : cluster 3 failed to encode map e312 with expected crc 2023-09-23T18:59:44.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:43 smithi098 bash[76817]: cluster 2023-09-23T18:59:42.947816+0000 osd.6 (osd.6) 284 : cluster 3 failed to encode map e312 with expected crc 2023-09-23T18:59:44.282 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:44.306 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:44.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:44.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:44.411 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:44.437 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:44.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:44.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:44.537 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:59:44.537 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:59:44.545 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:44.569 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:44.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:44.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:44.672 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:44.722 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:44.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:44.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:44.827 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:44.852 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:44.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:44.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:44.926 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:44 smithi098 bash[76817]: audit 2023-09-23T18:59:44.855369+0000 mon.c (mon.2) 137 : audit 1 from='client.? 172.21.15.98:0/1463760287' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:59:44.926 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:44 smithi098 bash[76817]: audit 2023-09-23T18:59:44.856441+0000 mon.a (mon.0) 1720 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:59:44.926 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:44 smithi098 bash[79516]: audit 2023-09-23T18:59:44.855369+0000 mon.c (mon.2) 137 : audit 1 from='client.? 172.21.15.98:0/1463760287' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:59:44.927 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:44 smithi098 bash[79516]: audit 2023-09-23T18:59:44.856441+0000 mon.a (mon.0) 1720 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:59:44.940 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is set 2023-09-23T18:59:44.957 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:44.997 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:45.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:45.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:45.105 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:45.131 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:45.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:45.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:45.235 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:45.262 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:45.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:45.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:45.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:44 smithi173 bash[59422]: audit 2023-09-23T18:59:44.855369+0000 mon.c (mon.2) 137 : audit 1 from='client.? 172.21.15.98:0/1463760287' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:59:45.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:44 smithi173 bash[59422]: audit 2023-09-23T18:59:44.856441+0000 mon.a (mon.0) 1720 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:59:45.367 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:45.394 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:45.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:45.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:45.497 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:45.519 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:45.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:45.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:45.623 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:45.653 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:45.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:45.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:45.759 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:45.784 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:45.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:45.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:45.887 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:45.913 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:45.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:45.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:45.920 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:45 smithi098 bash[76817]: cluster 2023-09-23T18:59:44.394737+0000 mgr.y (mgr.26054) 1253 : cluster 0 pgmap v1373: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 7 active+undersized+degraded, 5 active+undersized; 550 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 4270/26268 objects degraded (16.256%); 3737/26268 objects misplaced (14.226%); 12 MiB/s, 198 objects/s recovering 2023-09-23T18:59:45.920 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:45 smithi098 bash[76817]: cluster 2023-09-23T18:59:44.907013+0000 mon.a (mon.0) 1721 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:59:46.017 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:46.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:45 smithi173 bash[59422]: cluster 2023-09-23T18:59:44.394737+0000 mgr.y (mgr.26054) 1253 : cluster 0 pgmap v1373: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 7 active+undersized+degraded, 5 active+undersized; 550 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 4270/26268 objects degraded (16.256%); 3737/26268 objects misplaced (14.226%); 12 MiB/s, 198 objects/s recovering 2023-09-23T18:59:46.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:45 smithi173 bash[59422]: cluster 2023-09-23T18:59:44.907013+0000 mon.a (mon.0) 1721 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:59:46.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:45 smithi173 bash[59422]: cluster 2023-09-23T18:59:44.920696+0000 osd.4 (osd.4) 42 : cluster 3 failed to encode map e313 with expected crc 2023-09-23T18:59:46.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:45 smithi173 bash[59422]: cluster 2023-09-23T18:59:44.927554+0000 osd.5 (osd.5) 238 : cluster 3 failed to encode map e313 with expected crc 2023-09-23T18:59:46.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:45 smithi173 bash[59422]: audit 2023-09-23T18:59:44.930265+0000 mon.a (mon.0) 1722 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:59:46.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:45 smithi173 bash[59422]: cluster 2023-09-23T18:59:44.930793+0000 mon.a (mon.0) 1723 : cluster 0 osdmap e313: 8 total, 7 up, 7 in 2023-09-23T18:59:46.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:45 smithi173 bash[59422]: cluster 2023-09-23T18:59:44.946291+0000 osd.6 (osd.6) 285 : cluster 3 failed to encode map e313 with expected crc 2023-09-23T18:59:46.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:45 smithi173 bash[59422]: cluster 2023-09-23T18:59:45.417538+0000 mon.a (mon.0) 1724 : cluster 3 Health check update: Degraded data redundancy: 4270/26268 objects degraded (16.256%), 10 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:46.042 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:46.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:46.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:46.068 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:59:46.145 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:46.171 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:46.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:46.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:46.256 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:45 smithi098 bash[79516]: cluster 2023-09-23T18:59:44.394737+0000 mgr.y (mgr.26054) 1253 : cluster 0 pgmap v1373: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 7 active+undersized+degraded, 5 active+undersized; 550 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 4270/26268 objects degraded (16.256%); 3737/26268 objects misplaced (14.226%); 12 MiB/s, 198 objects/s recovering 2023-09-23T18:59:46.256 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:45 smithi098 bash[79516]: cluster 2023-09-23T18:59:44.907013+0000 mon.a (mon.0) 1721 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:59:46.256 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:45 smithi098 bash[79516]: cluster 2023-09-23T18:59:44.920696+0000 osd.4 (osd.4) 42 : cluster 3 failed to encode map e313 with expected crc 2023-09-23T18:59:46.256 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:45 smithi098 bash[79516]: cluster 2023-09-23T18:59:44.927554+0000 osd.5 (osd.5) 238 : cluster 3 failed to encode map e313 with expected crc 2023-09-23T18:59:46.256 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:45 smithi098 bash[79516]: audit 2023-09-23T18:59:44.930265+0000 mon.a (mon.0) 1722 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:59:46.256 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:45 smithi098 bash[79516]: cluster 2023-09-23T18:59:44.930793+0000 mon.a (mon.0) 1723 : cluster 0 osdmap e313: 8 total, 7 up, 7 in 2023-09-23T18:59:46.256 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:45 smithi098 bash[79516]: cluster 2023-09-23T18:59:44.946291+0000 osd.6 (osd.6) 285 : cluster 3 failed to encode map e313 with expected crc 2023-09-23T18:59:46.257 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:45 smithi098 bash[79516]: cluster 2023-09-23T18:59:45.417538+0000 mon.a (mon.0) 1724 : cluster 3 Health check update: Degraded data redundancy: 4270/26268 objects degraded (16.256%), 10 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:46.257 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:45 smithi098 bash[76817]: cluster 2023-09-23T18:59:44.920696+0000 osd.4 (osd.4) 42 : cluster 3 failed to encode map e313 with expected crc 2023-09-23T18:59:46.257 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:45 smithi098 bash[76817]: cluster 2023-09-23T18:59:44.927554+0000 osd.5 (osd.5) 238 : cluster 3 failed to encode map e313 with expected crc 2023-09-23T18:59:46.257 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:45 smithi098 bash[76817]: audit 2023-09-23T18:59:44.930265+0000 mon.a (mon.0) 1722 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:59:46.257 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:45 smithi098 bash[76817]: cluster 2023-09-23T18:59:44.930793+0000 mon.a (mon.0) 1723 : cluster 0 osdmap e313: 8 total, 7 up, 7 in 2023-09-23T18:59:46.257 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:45 smithi098 bash[76817]: cluster 2023-09-23T18:59:44.946291+0000 osd.6 (osd.6) 285 : cluster 3 failed to encode map e313 with expected crc 2023-09-23T18:59:46.257 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:45 smithi098 bash[76817]: cluster 2023-09-23T18:59:45.417538+0000 mon.a (mon.0) 1724 : cluster 3 Health check update: Degraded data redundancy: 4270/26268 objects degraded (16.256%), 10 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:46.258 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:59:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:59:46] "GET /metrics HTTP/1.1" 200 36597 "" "Prometheus/2.33.4" 2023-09-23T18:59:46.274 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:46.311 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:46.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:46.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:46.417 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:46.449 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:46.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:46.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:46.553 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:46.580 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:46.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:46.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:46.687 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:46.711 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:46.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:46.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:46.814 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:46.839 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:46.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:46.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:46.942 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:46.971 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:46.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:46.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:47.076 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:47.102 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:47.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:47.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:47.211 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:47.237 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:47.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:47.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:47.340 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:47.380 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:47.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:47.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:47.484 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:47.513 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:47.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:47.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:47.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:47.647 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:47.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:47.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:47.751 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:47.800 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:47.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:47.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:47.903 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:47.932 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:47.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:47.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:48.035 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:48.043 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:47 smithi098 bash[76817]: cluster 2023-09-23T18:59:46.395391+0000 mgr.y (mgr.26054) 1254 : cluster 0 pgmap v1375: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 7 active+undersized+degraded, 5 active+undersized; 509 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 4100/24297 objects degraded (16.875%); 3737/24297 objects misplaced (15.380%); 8.2 MiB/s, 130 objects/s recovering 2023-09-23T18:59:48.044 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:47 smithi098 bash[79516]: cluster 2023-09-23T18:59:46.395391+0000 mgr.y (mgr.26054) 1254 : cluster 0 pgmap v1375: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 7 active+undersized+degraded, 5 active+undersized; 509 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 4100/24297 objects degraded (16.875%); 3737/24297 objects misplaced (15.380%); 8.2 MiB/s, 130 objects/s recovering 2023-09-23T18:59:48.072 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:48.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:48.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:48.176 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:48.207 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:48.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:48.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:48.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:47 smithi173 bash[59422]: cluster 2023-09-23T18:59:46.395391+0000 mgr.y (mgr.26054) 1254 : cluster 0 pgmap v1375: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 7 active+undersized+degraded, 5 active+undersized; 509 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 4100/24297 objects degraded (16.875%); 3737/24297 objects misplaced (15.380%); 8.2 MiB/s, 130 objects/s recovering 2023-09-23T18:59:48.313 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:48.343 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:48.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:48.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:48.447 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:48.481 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:48.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:48.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:48.585 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:48.613 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:48.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:48.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:48.716 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:48.741 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:48.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:48.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:48.847 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:48.873 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:48.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:48.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:48.981 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:49.011 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:49.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:49.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:49.115 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:49.147 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:49.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:49.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:49.251 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:49.275 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:49.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:49.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:49.380 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:49.411 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:49.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:49.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:49.516 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:49.543 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:49.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:49.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:49.647 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:49.670 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:49.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:49.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:49.774 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:49.804 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:49.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:49.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:49.907 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:49.952 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:49.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:49.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:50.060 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:50.081 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:50.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:50.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:50.185 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:50.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:49 smithi098 bash[79516]: cluster 2023-09-23T18:59:48.396259+0000 mgr.y (mgr.26054) 1255 : cluster 0 pgmap v1376: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 7 active+undersized+degraded, 5 active+undersized; 366 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 772 op/s; 2823/17433 objects degraded (16.193%); 3370/17433 objects misplaced (19.331%); 13 MiB/s, 211 objects/s recovering 2023-09-23T18:59:50.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:49 smithi098 bash[76817]: cluster 2023-09-23T18:59:48.396259+0000 mgr.y (mgr.26054) 1255 : cluster 0 pgmap v1376: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 7 active+undersized+degraded, 5 active+undersized; 366 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 772 op/s; 2823/17433 objects degraded (16.193%); 3370/17433 objects misplaced (19.331%); 13 MiB/s, 211 objects/s recovering 2023-09-23T18:59:50.223 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:50.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:50.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:50.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:49 smithi173 bash[59422]: cluster 2023-09-23T18:59:48.396259+0000 mgr.y (mgr.26054) 1255 : cluster 0 pgmap v1376: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 7 active+undersized+degraded, 5 active+undersized; 366 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 772 op/s; 2823/17433 objects degraded (16.193%); 3370/17433 objects misplaced (19.331%); 13 MiB/s, 211 objects/s recovering 2023-09-23T18:59:50.331 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:50.358 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:50.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:50.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:50.461 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:50.486 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:50.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:50.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:50.591 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:50.616 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:50.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:50.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:50.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:50.743 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:50.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:50.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:50.847 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:50.869 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:50.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:50.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:50.972 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:51.019 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:51.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:51.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:51.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:50 smithi173 bash[59422]: cluster 2023-09-23T18:59:50.419286+0000 mon.a (mon.0) 1725 : cluster 3 Health check update: Degraded data redundancy: 2823/17433 objects degraded (16.193%), 10 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:51.109 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:50 smithi098 bash[79516]: cluster 2023-09-23T18:59:50.419286+0000 mon.a (mon.0) 1725 : cluster 3 Health check update: Degraded data redundancy: 2823/17433 objects degraded (16.193%), 10 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:51.109 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:50 smithi098 bash[76817]: cluster 2023-09-23T18:59:50.419286+0000 mon.a (mon.0) 1725 : cluster 3 Health check update: Degraded data redundancy: 2823/17433 objects degraded (16.193%), 10 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:51.125 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:51.150 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:51.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:51.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:51.254 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:51.281 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:51.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:51.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:51.385 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:51.411 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:51.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:51.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:51.518 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:51.544 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:51.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:51.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:51.651 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:51.677 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:51.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:51.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:51.782 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:51.814 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:51.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:51.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:51.920 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:51.963 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:51.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:51.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:52.068 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:52.105 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:52.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:52.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:52.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:51 smithi098 bash[79516]: cluster 2023-09-23T18:59:50.396879+0000 mgr.y (mgr.26054) 1256 : cluster 0 pgmap v1377: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 7 active+undersized+degraded, 5 active+undersized; 366 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 2823/17433 objects degraded (16.193%); 3370/17433 objects misplaced (19.331%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T18:59:52.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:51 smithi098 bash[76817]: cluster 2023-09-23T18:59:50.396879+0000 mgr.y (mgr.26054) 1256 : cluster 0 pgmap v1377: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 7 active+undersized+degraded, 5 active+undersized; 366 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 2823/17433 objects degraded (16.193%); 3370/17433 objects misplaced (19.331%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T18:59:52.209 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:52.234 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:52.238 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:59:52.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:52.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:52.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:51 smithi173 bash[59422]: cluster 2023-09-23T18:59:50.396879+0000 mgr.y (mgr.26054) 1256 : cluster 0 pgmap v1377: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 7 active+undersized+degraded, 5 active+undersized; 366 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 2823/17433 objects degraded (16.193%); 3370/17433 objects misplaced (19.331%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T18:59:52.340 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:52.371 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:52.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:52.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:52.476 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:52.514 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:52.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:52.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:52.617 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:52.642 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:52.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:52.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:52.746 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:52.784 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:52.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:52.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:52.889 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:52.918 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:52.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:52.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:53.022 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:53.048 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:53.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:53.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:53.152 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:53.184 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:53.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:53.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:53.288 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:53.314 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:53.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:53.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:53.421 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:53.447 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:53.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:53.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:53.553 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:53.577 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:53.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:53.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:53.680 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:53.720 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:53.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:53.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:53.825 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:53.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:53.856 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:53.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:53.957 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:53.984 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:53.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:53.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:54.089 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:54.117 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:54.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:54.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:54.221 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:54.230 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:53 smithi098 bash[76817]: cluster 2023-09-23T18:59:52.397683+0000 mgr.y (mgr.26054) 1257 : cluster 0 pgmap v1378: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 7 active+undersized+degraded, 5 active+undersized; 283 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 2063/13452 objects degraded (15.336%); 3180/13452 objects misplaced (23.640%); 8.8 MiB/s, 147 objects/s recovering 2023-09-23T18:59:54.230 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:53 smithi098 bash[79516]: cluster 2023-09-23T18:59:52.397683+0000 mgr.y (mgr.26054) 1257 : cluster 0 pgmap v1378: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 7 active+undersized+degraded, 5 active+undersized; 283 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 2063/13452 objects degraded (15.336%); 3180/13452 objects misplaced (23.640%); 8.8 MiB/s, 147 objects/s recovering 2023-09-23T18:59:54.268 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:54.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:54.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:54.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:53 smithi173 bash[59422]: cluster 2023-09-23T18:59:52.397683+0000 mgr.y (mgr.26054) 1257 : cluster 0 pgmap v1378: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 7 active+undersized+degraded, 5 active+undersized; 283 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 2063/13452 objects degraded (15.336%); 3180/13452 objects misplaced (23.640%); 8.8 MiB/s, 147 objects/s recovering 2023-09-23T18:59:54.372 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:54.401 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:54.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:54.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:54.506 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:54.542 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:54.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:54.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:54.648 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:54.670 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:54.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:54.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:54.774 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:54.799 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:54.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:54.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:54.906 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:54.934 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:54.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:54.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:55.038 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:55.059 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:55.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:55.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:55.162 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:55.188 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:55.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:55.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:55.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:54 smithi098 bash[79516]: audit 2023-09-23T18:59:54.635658+0000 mon.a (mon.0) 1726 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:59:55.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:54 smithi098 bash[76817]: audit 2023-09-23T18:59:54.635658+0000 mon.a (mon.0) 1726 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:59:55.227 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:59:55.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:54 smithi173 bash[59422]: audit 2023-09-23T18:59:54.635658+0000 mon.a (mon.0) 1726 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:59:55.294 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:55.328 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:55.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:55.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:55.432 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:55.455 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:55.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:55.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:55.563 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:55.596 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:55.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:55.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:55.699 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:55.727 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:55.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:55.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:55.830 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:55.881 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:55.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:55.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:55.986 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:56.012 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:56.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:56.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:56.115 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:56.152 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:56.154 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 18:59:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:18:59:56] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.33.4" 2023-09-23T18:59:56.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:56.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:56.156 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:55 smithi098 bash[76817]: cluster 2023-09-23T18:59:54.398657+0000 mgr.y (mgr.26054) 1258 : cluster 0 pgmap v1379: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 7 active+undersized+degraded, 5 active+undersized; 164 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 1245/7758 objects degraded (16.048%); 2966/7758 objects misplaced (38.232%); 7.1 MiB/s, 119 objects/s recovering 2023-09-23T18:59:56.156 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:55 smithi098 bash[76817]: cluster 2023-09-23T18:59:55.420673+0000 mon.a (mon.0) 1727 : cluster 3 Health check update: Degraded data redundancy: 1245/7758 objects degraded (16.048%), 10 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:56.156 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:55 smithi098 bash[79516]: cluster 2023-09-23T18:59:54.398657+0000 mgr.y (mgr.26054) 1258 : cluster 0 pgmap v1379: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 7 active+undersized+degraded, 5 active+undersized; 164 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 1245/7758 objects degraded (16.048%); 2966/7758 objects misplaced (38.232%); 7.1 MiB/s, 119 objects/s recovering 2023-09-23T18:59:56.156 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:55 smithi098 bash[79516]: cluster 2023-09-23T18:59:55.420673+0000 mon.a (mon.0) 1727 : cluster 3 Health check update: Degraded data redundancy: 1245/7758 objects degraded (16.048%), 10 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:56.214 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Removed 95536 objects 2023-09-23T18:59:56.214 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Clean up completed and total clean up time :160.871 2023-09-23T18:59:56.256 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:56.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:55 smithi173 bash[59422]: cluster 2023-09-23T18:59:54.398657+0000 mgr.y (mgr.26054) 1258 : cluster 0 pgmap v1379: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 7 active+undersized+degraded, 5 active+undersized; 164 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 1245/7758 objects degraded (16.048%); 2966/7758 objects misplaced (38.232%); 7.1 MiB/s, 119 objects/s recovering 2023-09-23T18:59:56.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:55 smithi173 bash[59422]: cluster 2023-09-23T18:59:55.420673+0000 mon.a (mon.0) 1727 : cluster 3 Health check update: Degraded data redundancy: 1245/7758 objects degraded (16.048%), 10 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T18:59:56.284 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:56.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:56.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:56.387 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:56.412 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:56.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:56.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:56.516 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:56.539 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:56.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:56.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:56.643 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:56.675 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:56.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:56.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:56.779 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:56.804 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:56.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:56.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:56.909 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:56.941 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:56.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:56.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:57.045 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:57.073 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:57.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:57.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:57.177 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:57.201 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:57.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:57.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:57.304 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:57.335 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:57.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:57.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:57.439 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:57.470 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:57.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:57.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:57.573 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:57.599 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:57.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:57.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:57.702 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:57.734 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:57.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:57.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:57.790 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:59:57.838 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:57.871 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:57.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:57.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:57.975 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:57.994 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:57.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:57.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:58.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:57 smithi173 bash[59422]: cluster 2023-09-23T18:59:56.399242+0000 mgr.y (mgr.26054) 1259 : cluster 0 pgmap v1380: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 7 active+undersized+degraded, 5 active+undersized; 109 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 1008/5076 objects degraded (19.858%); 2966/5076 objects misplaced (58.432%); 6.2 MiB/s, 104 objects/s recovering 2023-09-23T18:59:58.097 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:58.120 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:58.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:58.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:58.224 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:58.252 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:59:58.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:58.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:59:58.356 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:58.365 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 18:59:57 smithi098 bash[76817]: cluster 2023-09-23T18:59:56.399242+0000 mgr.y (mgr.26054) 1259 : cluster 0 pgmap v1380: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 7 active+undersized+degraded, 5 active+undersized; 109 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 1008/5076 objects degraded (19.858%); 2966/5076 objects misplaced (58.432%); 6.2 MiB/s, 104 objects/s recovering 2023-09-23T18:59:58.365 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 18:59:57 smithi098 bash[79516]: cluster 2023-09-23T18:59:56.399242+0000 mgr.y (mgr.26054) 1259 : cluster 0 pgmap v1380: 89 pgs: 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 7 active+undersized+degraded, 5 active+undersized; 109 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 1008/5076 objects degraded (19.858%); 2966/5076 objects misplaced (58.432%); 6.2 MiB/s, 104 objects/s recovering 2023-09-23T18:59:58.381 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:58.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:58.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:58.484 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:58.505 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:58.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:58.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:58.609 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:58.634 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:59:58.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:58.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:59:58.738 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:58.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:58.774 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:58.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:58.874 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:58.896 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:58.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:58.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:58.999 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:59.019 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:59.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:59.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:59.122 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:59.146 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:59.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:59.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:59.249 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:59.275 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:59:59.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:59.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:59:59.379 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:59.409 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:59:59.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:59.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:59:59.512 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:59.543 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:59:59.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:59.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:59:59.646 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:59.675 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:59.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:59.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:59:59.778 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:59.800 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:59:59.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:59.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:59:59.903 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:59.926 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:59:59.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:59:59.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:00.029 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:00.056 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:00.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:00.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:00.159 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:00.167 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:00 smithi098 bash[76817]: cluster 2023-09-23T18:59:58.399946+0000 mgr.y (mgr.26054) 1260 : cluster 0 pgmap v1381: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 8.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 20/276 objects degraded (7.246%); 2652/276 objects misplaced (960.870%); 7.8 MiB/s, 154 objects/s recovering 2023-09-23T19:00:00.167 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:00 smithi098 bash[79516]: cluster 2023-09-23T18:59:58.399946+0000 mgr.y (mgr.26054) 1260 : cluster 0 pgmap v1381: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 8.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 20/276 objects degraded (7.246%); 2652/276 objects misplaced (960.870%); 7.8 MiB/s, 154 objects/s recovering 2023-09-23T19:00:00.180 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:00.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:00.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:00.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 18:59:59 smithi173 bash[59422]: cluster 2023-09-23T18:59:58.399946+0000 mgr.y (mgr.26054) 1260 : cluster 0 pgmap v1381: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 8.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 20/276 objects degraded (7.246%); 2652/276 objects misplaced (960.870%); 7.8 MiB/s, 154 objects/s recovering 2023-09-23T19:00:00.283 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:00.311 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:00.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:00.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:00.414 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:00.441 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:00.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:00.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:00.527 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-09-23T19:00:00.528 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-09-23T19:00:00.545 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:00.596 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:00.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:00.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:00.699 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:00.734 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:00.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:00.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:00.837 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:00.857 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:00.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:00.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:00.960 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:00.983 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:00.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:00.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:01.086 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:01.117 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:01.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:01.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:01.220 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:01.234 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:00 smithi173 bash[59422]: cluster 2023-09-23T19:00:00.000114+0000 mon.a (mon.0) 1728 : cluster 3 Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; 1 osds down; Degraded data redundancy: 20/276 objects degraded (7.246%), 8 pgs degraded, 12 pgs undersized 2023-09-23T19:00:01.234 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:00 smithi173 bash[59422]: cluster 2023-09-23T19:00:00.000145+0000 mon.a (mon.0) 1729 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T19:00:01.234 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:00 smithi173 bash[59422]: cluster 2023-09-23T19:00:00.000163+0000 mon.a (mon.0) 1730 : cluster 3 [WRN] OSD_DOWN: 1 osds down 2023-09-23T19:00:01.234 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:00 smithi173 bash[59422]: cluster 2023-09-23T19:00:00.000180+0000 mon.a (mon.0) 1731 : cluster 3 osd.7 (root=default,host=smithi173) is down 2023-09-23T19:00:01.234 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:00 smithi173 bash[59422]: cluster 2023-09-23T19:00:00.000206+0000 mon.a (mon.0) 1732 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 20/276 objects degraded (7.246%), 8 pgs degraded, 12 pgs undersized 2023-09-23T19:00:01.234 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:00 smithi173 bash[59422]: cluster 2023-09-23T19:00:00.000221+0000 mon.a (mon.0) 1733 : cluster 3 pg 1.0 is stuck undersized for 5m, current state active+undersized+degraded, last acting [0,6] 2023-09-23T19:00:01.234 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:00 smithi173 bash[59422]: cluster 2023-09-23T19:00:00.000238+0000 mon.a (mon.0) 1734 : cluster 3 pg 2.0 is stuck undersized for 5m, current state active+undersized, last acting [1,0] 2023-09-23T19:00:01.234 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:00 smithi173 bash[59422]: cluster 2023-09-23T19:00:00.000255+0000 mon.a (mon.0) 1735 : cluster 3 pg 2.4 is stuck undersized for 5m, current state active+undersized+degraded, last acting [1,0] 2023-09-23T19:00:01.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:00 smithi173 bash[59422]: cluster 2023-09-23T19:00:00.000273+0000 mon.a (mon.0) 1736 : cluster 3 pg 3.12 is stuck undersized for 5m, current state active+undersized, last acting [1,0] 2023-09-23T19:00:01.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:00 smithi173 bash[59422]: cluster 2023-09-23T19:00:00.000289+0000 mon.a (mon.0) 1737 : cluster 3 pg 3.16 is stuck undersized for 5m, current state active+undersized, last acting [1,2] 2023-09-23T19:00:01.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:00 smithi173 bash[59422]: cluster 2023-09-23T19:00:00.000306+0000 mon.a (mon.0) 1738 : cluster 3 pg 4.0 is stuck undersized for 5m, current state active+undersized, last acting [1,0] 2023-09-23T19:00:01.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:00 smithi173 bash[59422]: cluster 2023-09-23T19:00:00.000324+0000 mon.a (mon.0) 1739 : cluster 3 pg 4.a is stuck undersized for 5m, current state active+undersized, last acting [2,6] 2023-09-23T19:00:01.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:00 smithi173 bash[59422]: cluster 2023-09-23T19:00:00.000340+0000 mon.a (mon.0) 1740 : cluster 3 pg 9.1 is active+recovery_wait+degraded+remapped, acting [3,6,4] 2023-09-23T19:00:01.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:00 smithi173 bash[59422]: cluster 2023-09-23T19:00:00.000358+0000 mon.a (mon.0) 1741 : cluster 3 pg 9.2 is stuck undersized for 4m, current state active+undersized+degraded, last acting [1,2] 2023-09-23T19:00:01.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:00 smithi173 bash[59422]: cluster 2023-09-23T19:00:00.000376+0000 mon.a (mon.0) 1742 : cluster 3 pg 9.8 is stuck undersized for 4m, current state active+undersized, last acting [0,1] 2023-09-23T19:00:01.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:00 smithi173 bash[59422]: cluster 2023-09-23T19:00:00.000393+0000 mon.a (mon.0) 1743 : cluster 3 pg 9.9 is stuck undersized for 4m, current state active+undersized+degraded, last acting [5,0] 2023-09-23T19:00:01.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:00 smithi173 bash[59422]: cluster 2023-09-23T19:00:00.000410+0000 mon.a (mon.0) 1744 : cluster 3 pg 9.a is stuck undersized for 79s, current state active+undersized+degraded, last acting [1,3] 2023-09-23T19:00:01.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:00 smithi173 bash[59422]: cluster 2023-09-23T19:00:00.000428+0000 mon.a (mon.0) 1745 : cluster 3 pg 9.c is active+recovery_wait+degraded+remapped, acting [3,0,4] 2023-09-23T19:00:01.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:00 smithi173 bash[59422]: cluster 2023-09-23T19:00:00.000444+0000 mon.a (mon.0) 1746 : cluster 3 pg 9.d is stuck undersized for 4m, current state active+undersized+degraded, last acting [1,5] 2023-09-23T19:00:01.236 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:00 smithi173 bash[59422]: cluster 2023-09-23T19:00:00.421786+0000 mon.a (mon.0) 1747 : cluster 3 Health check update: Degraded data redundancy: 20/276 objects degraded (7.246%), 8 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:00:01.255 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:01.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:01.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:01.275 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[76817]: cluster 2023-09-23T19:00:00.000114+0000 mon.a (mon.0) 1728 : cluster 3 Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; 1 osds down; Degraded data redundancy: 20/276 objects degraded (7.246%), 8 pgs degraded, 12 pgs undersized 2023-09-23T19:00:01.275 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[76817]: cluster 2023-09-23T19:00:00.000145+0000 mon.a (mon.0) 1729 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T19:00:01.275 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[76817]: cluster 2023-09-23T19:00:00.000163+0000 mon.a (mon.0) 1730 : cluster 3 [WRN] OSD_DOWN: 1 osds down 2023-09-23T19:00:01.275 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[76817]: cluster 2023-09-23T19:00:00.000180+0000 mon.a (mon.0) 1731 : cluster 3 osd.7 (root=default,host=smithi173) is down 2023-09-23T19:00:01.275 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[76817]: cluster 2023-09-23T19:00:00.000206+0000 mon.a (mon.0) 1732 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 20/276 objects degraded (7.246%), 8 pgs degraded, 12 pgs undersized 2023-09-23T19:00:01.275 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[76817]: cluster 2023-09-23T19:00:00.000221+0000 mon.a (mon.0) 1733 : cluster 3 pg 1.0 is stuck undersized for 5m, current state active+undersized+degraded, last acting [0,6] 2023-09-23T19:00:01.275 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[76817]: cluster 2023-09-23T19:00:00.000238+0000 mon.a (mon.0) 1734 : cluster 3 pg 2.0 is stuck undersized for 5m, current state active+undersized, last acting [1,0] 2023-09-23T19:00:01.276 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[76817]: cluster 2023-09-23T19:00:00.000255+0000 mon.a (mon.0) 1735 : cluster 3 pg 2.4 is stuck undersized for 5m, current state active+undersized+degraded, last acting [1,0] 2023-09-23T19:00:01.276 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[76817]: cluster 2023-09-23T19:00:00.000273+0000 mon.a (mon.0) 1736 : cluster 3 pg 3.12 is stuck undersized for 5m, current state active+undersized, last acting [1,0] 2023-09-23T19:00:01.276 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[76817]: cluster 2023-09-23T19:00:00.000289+0000 mon.a (mon.0) 1737 : cluster 3 pg 3.16 is stuck undersized for 5m, current state active+undersized, last acting [1,2] 2023-09-23T19:00:01.276 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[76817]: cluster 2023-09-23T19:00:00.000306+0000 mon.a (mon.0) 1738 : cluster 3 pg 4.0 is stuck undersized for 5m, current state active+undersized, last acting [1,0] 2023-09-23T19:00:01.276 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[76817]: cluster 2023-09-23T19:00:00.000324+0000 mon.a (mon.0) 1739 : cluster 3 pg 4.a is stuck undersized for 5m, current state active+undersized, last acting [2,6] 2023-09-23T19:00:01.276 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[76817]: cluster 2023-09-23T19:00:00.000340+0000 mon.a (mon.0) 1740 : cluster 3 pg 9.1 is active+recovery_wait+degraded+remapped, acting [3,6,4] 2023-09-23T19:00:01.276 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[76817]: cluster 2023-09-23T19:00:00.000358+0000 mon.a (mon.0) 1741 : cluster 3 pg 9.2 is stuck undersized for 4m, current state active+undersized+degraded, last acting [1,2] 2023-09-23T19:00:01.276 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[76817]: cluster 2023-09-23T19:00:00.000376+0000 mon.a (mon.0) 1742 : cluster 3 pg 9.8 is stuck undersized for 4m, current state active+undersized, last acting [0,1] 2023-09-23T19:00:01.276 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[76817]: cluster 2023-09-23T19:00:00.000393+0000 mon.a (mon.0) 1743 : cluster 3 pg 9.9 is stuck undersized for 4m, current state active+undersized+degraded, last acting [5,0] 2023-09-23T19:00:01.276 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[76817]: cluster 2023-09-23T19:00:00.000410+0000 mon.a (mon.0) 1744 : cluster 3 pg 9.a is stuck undersized for 79s, current state active+undersized+degraded, last acting [1,3] 2023-09-23T19:00:01.276 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[76817]: cluster 2023-09-23T19:00:00.000428+0000 mon.a (mon.0) 1745 : cluster 3 pg 9.c is active+recovery_wait+degraded+remapped, acting [3,0,4] 2023-09-23T19:00:01.277 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[76817]: cluster 2023-09-23T19:00:00.000444+0000 mon.a (mon.0) 1746 : cluster 3 pg 9.d is stuck undersized for 4m, current state active+undersized+degraded, last acting [1,5] 2023-09-23T19:00:01.277 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[76817]: cluster 2023-09-23T19:00:00.421786+0000 mon.a (mon.0) 1747 : cluster 3 Health check update: Degraded data redundancy: 20/276 objects degraded (7.246%), 8 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:00:01.277 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[79516]: cluster 2023-09-23T19:00:00.000114+0000 mon.a (mon.0) 1728 : cluster 3 Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; 1 osds down; Degraded data redundancy: 20/276 objects degraded (7.246%), 8 pgs degraded, 12 pgs undersized 2023-09-23T19:00:01.277 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[79516]: cluster 2023-09-23T19:00:00.000145+0000 mon.a (mon.0) 1729 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T19:00:01.277 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[79516]: cluster 2023-09-23T19:00:00.000163+0000 mon.a (mon.0) 1730 : cluster 3 [WRN] OSD_DOWN: 1 osds down 2023-09-23T19:00:01.277 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[79516]: cluster 2023-09-23T19:00:00.000180+0000 mon.a (mon.0) 1731 : cluster 3 osd.7 (root=default,host=smithi173) is down 2023-09-23T19:00:01.277 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[79516]: cluster 2023-09-23T19:00:00.000206+0000 mon.a (mon.0) 1732 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 20/276 objects degraded (7.246%), 8 pgs degraded, 12 pgs undersized 2023-09-23T19:00:01.277 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[79516]: cluster 2023-09-23T19:00:00.000221+0000 mon.a (mon.0) 1733 : cluster 3 pg 1.0 is stuck undersized for 5m, current state active+undersized+degraded, last acting [0,6] 2023-09-23T19:00:01.277 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[79516]: cluster 2023-09-23T19:00:00.000238+0000 mon.a (mon.0) 1734 : cluster 3 pg 2.0 is stuck undersized for 5m, current state active+undersized, last acting [1,0] 2023-09-23T19:00:01.277 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[79516]: cluster 2023-09-23T19:00:00.000255+0000 mon.a (mon.0) 1735 : cluster 3 pg 2.4 is stuck undersized for 5m, current state active+undersized+degraded, last acting [1,0] 2023-09-23T19:00:01.278 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[79516]: cluster 2023-09-23T19:00:00.000273+0000 mon.a (mon.0) 1736 : cluster 3 pg 3.12 is stuck undersized for 5m, current state active+undersized, last acting [1,0] 2023-09-23T19:00:01.278 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[79516]: cluster 2023-09-23T19:00:00.000289+0000 mon.a (mon.0) 1737 : cluster 3 pg 3.16 is stuck undersized for 5m, current state active+undersized, last acting [1,2] 2023-09-23T19:00:01.278 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[79516]: cluster 2023-09-23T19:00:00.000306+0000 mon.a (mon.0) 1738 : cluster 3 pg 4.0 is stuck undersized for 5m, current state active+undersized, last acting [1,0] 2023-09-23T19:00:01.278 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[79516]: cluster 2023-09-23T19:00:00.000324+0000 mon.a (mon.0) 1739 : cluster 3 pg 4.a is stuck undersized for 5m, current state active+undersized, last acting [2,6] 2023-09-23T19:00:01.278 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[79516]: cluster 2023-09-23T19:00:00.000340+0000 mon.a (mon.0) 1740 : cluster 3 pg 9.1 is active+recovery_wait+degraded+remapped, acting [3,6,4] 2023-09-23T19:00:01.278 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[79516]: cluster 2023-09-23T19:00:00.000358+0000 mon.a (mon.0) 1741 : cluster 3 pg 9.2 is stuck undersized for 4m, current state active+undersized+degraded, last acting [1,2] 2023-09-23T19:00:01.278 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[79516]: cluster 2023-09-23T19:00:00.000376+0000 mon.a (mon.0) 1742 : cluster 3 pg 9.8 is stuck undersized for 4m, current state active+undersized, last acting [0,1] 2023-09-23T19:00:01.278 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[79516]: cluster 2023-09-23T19:00:00.000393+0000 mon.a (mon.0) 1743 : cluster 3 pg 9.9 is stuck undersized for 4m, current state active+undersized+degraded, last acting [5,0] 2023-09-23T19:00:01.278 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[79516]: cluster 2023-09-23T19:00:00.000410+0000 mon.a (mon.0) 1744 : cluster 3 pg 9.a is stuck undersized for 79s, current state active+undersized+degraded, last acting [1,3] 2023-09-23T19:00:01.278 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[79516]: cluster 2023-09-23T19:00:00.000428+0000 mon.a (mon.0) 1745 : cluster 3 pg 9.c is active+recovery_wait+degraded+remapped, acting [3,0,4] 2023-09-23T19:00:01.278 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[79516]: cluster 2023-09-23T19:00:00.000444+0000 mon.a (mon.0) 1746 : cluster 3 pg 9.d is stuck undersized for 4m, current state active+undersized+degraded, last acting [1,5] 2023-09-23T19:00:01.279 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[79516]: cluster 2023-09-23T19:00:00.421786+0000 mon.a (mon.0) 1747 : cluster 3 Health check update: Degraded data redundancy: 20/276 objects degraded (7.246%), 8 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:00:01.359 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:01.384 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:01.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:01.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:01.486 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:01.515 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:01.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:01.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:01.619 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:01.649 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:01.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:01.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:01.752 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:01.776 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:01.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:01.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:01.879 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:01.903 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:01.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:01.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:02.006 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:02.033 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:02.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:02.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:02.114 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[76817]: cluster 2023-09-23T19:00:00.400788+0000 mgr.y (mgr.26054) 1261 : cluster 0 pgmap v1382: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 8.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 20/276 objects degraded (7.246%); 2652/276 objects misplaced (960.870%); 4.7 MiB/s, 105 objects/s recovering 2023-09-23T19:00:02.114 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:01 smithi098 bash[79516]: cluster 2023-09-23T19:00:00.400788+0000 mgr.y (mgr.26054) 1261 : cluster 0 pgmap v1382: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 8.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 20/276 objects degraded (7.246%); 2652/276 objects misplaced (960.870%); 4.7 MiB/s, 105 objects/s recovering 2023-09-23T19:00:02.137 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:02.157 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:02.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:02.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:02.261 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:02.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:01 smithi173 bash[59422]: cluster 2023-09-23T19:00:00.400788+0000 mgr.y (mgr.26054) 1261 : cluster 0 pgmap v1382: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 8.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 20/276 objects degraded (7.246%); 2652/276 objects misplaced (960.870%); 4.7 MiB/s, 105 objects/s recovering 2023-09-23T19:00:02.291 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:02.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:02.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:02.394 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:02.425 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:02.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:02.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:02.529 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:02.553 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:02.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:02.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:02.656 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:02.686 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:02.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:02.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:02.790 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:02.812 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:02.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:02.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:02.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:02.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:02.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:02.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:03.047 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:03.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:03.071 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:03.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:03.171 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:03.197 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:03.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:03.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:03.300 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:03.326 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:03.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:03.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:03.429 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:03.452 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:03.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:03.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:03.555 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:03.581 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:03.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:03.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:03.685 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:03.707 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:03.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:03.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:03.809 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T19:00:03.809 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [], 2023-09-23T19:00:03.809 INFO:teuthology.orchestra.run.smithi098.stdout: "complaint_time": 30, 2023-09-23T19:00:03.809 INFO:teuthology.orchestra.run.smithi098.stdout: "num_blocked_ops": 0 2023-09-23T19:00:03.809 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T19:00:03.810 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:03.841 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:03.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:03.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:03.945 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:03.971 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:03.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:03.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:04.074 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:04.104 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:04.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:04.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:04.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:03 smithi098 bash[79516]: cluster 2023-09-23T19:00:02.402003+0000 mgr.y (mgr.26054) 1262 : cluster 0 pgmap v1383: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 3/84 objects degraded (3.571%); 2652/84 objects misplaced (3157.143%); 4.7 MiB/s, 158 objects/s recovering 2023-09-23T19:00:04.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:03 smithi098 bash[76817]: cluster 2023-09-23T19:00:02.402003+0000 mgr.y (mgr.26054) 1262 : cluster 0 pgmap v1383: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 3/84 objects degraded (3.571%); 2652/84 objects misplaced (3157.143%); 4.7 MiB/s, 158 objects/s recovering 2023-09-23T19:00:04.207 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:04.226 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:04.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:04.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:04.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:03 smithi173 bash[59422]: cluster 2023-09-23T19:00:02.402003+0000 mgr.y (mgr.26054) 1262 : cluster 0 pgmap v1383: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 3/84 objects degraded (3.571%); 2652/84 objects misplaced (3157.143%); 4.7 MiB/s, 158 objects/s recovering 2023-09-23T19:00:04.288 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T19:00:04.288 INFO:teuthology.orchestra.run.smithi098.stdout: "success": "" 2023-09-23T19:00:04.288 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T19:00:04.329 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:04.353 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:04.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:04.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:04.456 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:04.483 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:04.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:04.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:04.588 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:04.608 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:04.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:04.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:04.714 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:04.739 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:04.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:04.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:04.841 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:04.872 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:04.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:04.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:04.976 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:05.006 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:05.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:05.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:05.110 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:05.135 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:05.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:05.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:05.238 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:05.266 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:05.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:05.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:05.370 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:05.396 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:05.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:05.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:05.499 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:05.536 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:05.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:05.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:05.640 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:05.670 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:05.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:05.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:05.774 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:05.798 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:05.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:05.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:05.901 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:05.935 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:05.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:05.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:06.039 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:06.070 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:06.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:06.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:06.174 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:06.196 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:06.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:06.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:06.245 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:00:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:00:06] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.33.4" 2023-09-23T19:00:06.245 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:05 smithi098 bash[79516]: cluster 2023-09-23T19:00:04.402751+0000 mgr.y (mgr.26054) 1263 : cluster 0 pgmap v1384: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 371 op/s; 3/84 objects degraded (3.571%); 2652/84 objects misplaced (3157.143%); 1.8 MiB/s, 108 objects/s recovering 2023-09-23T19:00:06.245 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:05 smithi098 bash[79516]: cluster 2023-09-23T19:00:05.425251+0000 mon.a (mon.0) 1748 : cluster 3 Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 8 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:00:06.245 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:05 smithi098 bash[76817]: cluster 2023-09-23T19:00:04.402751+0000 mgr.y (mgr.26054) 1263 : cluster 0 pgmap v1384: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 371 op/s; 3/84 objects degraded (3.571%); 2652/84 objects misplaced (3157.143%); 1.8 MiB/s, 108 objects/s recovering 2023-09-23T19:00:06.245 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:05 smithi098 bash[76817]: cluster 2023-09-23T19:00:05.425251+0000 mon.a (mon.0) 1748 : cluster 3 Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 8 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:00:06.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:05 smithi173 bash[59422]: cluster 2023-09-23T19:00:04.402751+0000 mgr.y (mgr.26054) 1263 : cluster 0 pgmap v1384: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 371 op/s; 3/84 objects degraded (3.571%); 2652/84 objects misplaced (3157.143%); 1.8 MiB/s, 108 objects/s recovering 2023-09-23T19:00:06.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:05 smithi173 bash[59422]: cluster 2023-09-23T19:00:05.425251+0000 mon.a (mon.0) 1748 : cluster 3 Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 8 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:00:06.300 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:06.332 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:06.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:06.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:06.435 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:06.461 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:06.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:06.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:06.564 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:06.597 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:06.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:06.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:06.700 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:06.731 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:06.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:06.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:06.835 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:06.865 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:06.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:06.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:06.969 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:07.005 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:07.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:07.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:07.108 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:07.134 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:07.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:07.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:07.238 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:07.267 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:07.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:07.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:07.370 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:07.395 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:07.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:07.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:07.499 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:07.531 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:07.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:07.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:07.635 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:07.662 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:07.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:07.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:07.765 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:07.792 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:07.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:07.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:07.894 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:07.927 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:07.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:07.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:08.030 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:08.061 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:08.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:08.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:08.164 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:08.188 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:08.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:08.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:08.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:07 smithi173 bash[59422]: cluster 2023-09-23T19:00:06.403734+0000 mgr.y (mgr.26054) 1264 : cluster 0 pgmap v1385: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 213 op/s; 3/84 objects degraded (3.571%); 2652/84 objects misplaced (3157.143%); 1.8 MiB/s, 108 objects/s recovering 2023-09-23T19:00:08.292 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:08.323 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:08.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:08.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:08.427 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:08.435 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:08 smithi098 bash[79516]: cluster 2023-09-23T19:00:06.403734+0000 mgr.y (mgr.26054) 1264 : cluster 0 pgmap v1385: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 213 op/s; 3/84 objects degraded (3.571%); 2652/84 objects misplaced (3157.143%); 1.8 MiB/s, 108 objects/s recovering 2023-09-23T19:00:08.435 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:07 smithi098 bash[76817]: cluster 2023-09-23T19:00:06.403734+0000 mgr.y (mgr.26054) 1264 : cluster 0 pgmap v1385: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 213 op/s; 3/84 objects degraded (3.571%); 2652/84 objects misplaced (3157.143%); 1.8 MiB/s, 108 objects/s recovering 2023-09-23T19:00:08.453 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:08.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:08.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:08.578 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:08.609 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:08.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:08.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:08.713 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:08.740 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:08.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:08.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:08.844 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:08.876 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:08.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:08.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:08.979 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:09.004 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:09.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:09.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:09.107 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:09.109 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T19:00:09.127 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:09.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:09.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:09.231 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:09.266 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:09.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:09.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:09.370 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:09.402 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:09.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:09.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:09.505 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:09.536 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:09.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:09.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:09.640 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:09.672 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:09.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:09.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:09.691 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:00:09.776 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:09.799 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:09.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:09.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:09.902 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:09.937 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:09.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:09.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:10.041 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:10.070 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:10.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:10.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:10.174 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:10.202 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:10.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:10.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:10.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:09 smithi173 bash[59422]: cluster 2023-09-23T19:00:08.405003+0000 mgr.y (mgr.26054) 1265 : cluster 0 pgmap v1386: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 138 op/s; 3/84 objects degraded (3.571%); 2652/84 objects misplaced (3157.143%); 1.8 MiB/s, 169 objects/s recovering 2023-09-23T19:00:10.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:09 smithi173 bash[59422]: audit 2023-09-23T19:00:09.636148+0000 mon.a (mon.0) 1749 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:00:10.305 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:10.318 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:09 smithi098 bash[79516]: cluster 2023-09-23T19:00:08.405003+0000 mgr.y (mgr.26054) 1265 : cluster 0 pgmap v1386: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 138 op/s; 3/84 objects degraded (3.571%); 2652/84 objects misplaced (3157.143%); 1.8 MiB/s, 169 objects/s recovering 2023-09-23T19:00:10.318 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:09 smithi098 bash[79516]: audit 2023-09-23T19:00:09.636148+0000 mon.a (mon.0) 1749 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:00:10.319 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:09 smithi098 bash[76817]: cluster 2023-09-23T19:00:08.405003+0000 mgr.y (mgr.26054) 1265 : cluster 0 pgmap v1386: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 138 op/s; 3/84 objects degraded (3.571%); 2652/84 objects misplaced (3157.143%); 1.8 MiB/s, 169 objects/s recovering 2023-09-23T19:00:10.319 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:09 smithi098 bash[76817]: audit 2023-09-23T19:00:09.636148+0000 mon.a (mon.0) 1749 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:00:10.335 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:10.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:10.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:10.438 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:10.462 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:10.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:10.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:10.566 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:10.591 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:10.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:10.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:10.694 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:10.727 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:10.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:10.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:10.831 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:10.861 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:10.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:10.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:10.964 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:10.993 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:10.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:10.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:11.096 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:11.119 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:11.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:11.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:11.223 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:11.249 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:11.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:11.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:11.352 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:11.382 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:11.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:11.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:11.485 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:11.514 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:11.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:11.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:11.618 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:11.648 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:11.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:11.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:11.751 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:11.782 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:11.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:11.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:11.885 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:11.918 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:11.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:11.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:12.021 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:12.051 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:12.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:12.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:12.155 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:12.166 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:11 smithi173 bash[59422]: cluster 2023-09-23T19:00:10.405826+0000 mgr.y (mgr.26054) 1266 : cluster 0 pgmap v1387: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 5 op/s; 3/84 objects degraded (3.571%); 2652/84 objects misplaced (3157.143%); 0 B/s, 114 objects/s recovering 2023-09-23T19:00:12.186 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:12.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:12.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:12.292 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:12.323 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:12.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:12.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:12.427 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:12.435 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:11 smithi098 bash[79516]: cluster 2023-09-23T19:00:10.405826+0000 mgr.y (mgr.26054) 1266 : cluster 0 pgmap v1387: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 5 op/s; 3/84 objects degraded (3.571%); 2652/84 objects misplaced (3157.143%); 0 B/s, 114 objects/s recovering 2023-09-23T19:00:12.436 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:11 smithi098 bash[76817]: cluster 2023-09-23T19:00:10.405826+0000 mgr.y (mgr.26054) 1266 : cluster 0 pgmap v1387: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 5 op/s; 3/84 objects degraded (3.571%); 2652/84 objects misplaced (3157.143%); 0 B/s, 114 objects/s recovering 2023-09-23T19:00:12.456 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:12.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:12.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:12.559 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:12.584 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:12.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:12.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:12.688 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:12.720 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:12.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:12.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:12.824 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:12.853 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:12.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:12.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:12.956 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:12.990 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:12.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:12.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:13.093 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:13.124 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:13.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:13.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:13.228 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:13.255 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:13.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:13.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:13.359 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:13.390 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:13.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:13.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:13.495 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:13.518 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:13.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:13.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:13.622 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:13.651 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:13.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:13.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:13.754 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:13.781 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:13.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:13.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:13.802 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T19:00:13.825 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T19:00:13.851 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T19:00:13.878 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T19:00:13.885 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:13.887 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:00:13.903 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T19:00:13.914 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:13.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:13.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:13.930 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T19:00:13.953 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T19:00:13.954 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:00:13.954 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:00:13.954 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:00:13.954 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:00:13.954 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:00:13.954 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:00:13.955 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:00:13.955 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T19:00:14.004 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is unset 2023-09-23T19:00:14.017 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:14.019 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:14 smithi098 bash[76817]: cluster 2023-09-23T19:00:12.407085+0000 mgr.y (mgr.26054) 1267 : cluster 0 pgmap v1388: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 5 op/s; 3/84 objects degraded (3.571%); 2652/84 objects misplaced (3157.143%); 0 B/s, 184 objects/s recovering 2023-09-23T19:00:14.019 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:14 smithi098 bash[76817]: audit 2023-09-23T19:00:13.736842+0000 mon.a (mon.0) 1750 : audit 1 from='client.? 172.21.15.98:0/1863778228' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:00:14.019 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:14 smithi098 bash[76817]: audit 2023-09-23T19:00:13.773056+0000 mon.c (mon.2) 138 : audit 0 from='client.? 172.21.15.98:0/3913177330' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:00:14.040 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:14.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:14.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:14.126 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 6, 7, 5, 3] out_osds: [4] dead_osds: [7] live_osds: [0, 2, 3, 5, 6, 1, 4] 2023-09-23T19:00:14.126 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:00:14.143 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:14.167 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:14.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:14.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:14.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:14 smithi098 bash[79516]: cluster 2023-09-23T19:00:12.407085+0000 mgr.y (mgr.26054) 1267 : cluster 0 pgmap v1388: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 5 op/s; 3/84 objects degraded (3.571%); 2652/84 objects misplaced (3157.143%); 0 B/s, 184 objects/s recovering 2023-09-23T19:00:14.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:14 smithi098 bash[79516]: audit 2023-09-23T19:00:13.736842+0000 mon.a (mon.0) 1750 : audit 1 from='client.? 172.21.15.98:0/1863778228' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:00:14.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:14 smithi098 bash[79516]: audit 2023-09-23T19:00:13.773056+0000 mon.c (mon.2) 138 : audit 0 from='client.? 172.21.15.98:0/3913177330' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:00:14.271 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:14.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:14 smithi173 bash[59422]: cluster 2023-09-23T19:00:12.407085+0000 mgr.y (mgr.26054) 1267 : cluster 0 pgmap v1388: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 5 op/s; 3/84 objects degraded (3.571%); 2652/84 objects misplaced (3157.143%); 0 B/s, 184 objects/s recovering 2023-09-23T19:00:14.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:14 smithi173 bash[59422]: audit 2023-09-23T19:00:13.736842+0000 mon.a (mon.0) 1750 : audit 1 from='client.? 172.21.15.98:0/1863778228' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:00:14.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:14 smithi173 bash[59422]: audit 2023-09-23T19:00:13.773056+0000 mon.c (mon.2) 138 : audit 0 from='client.? 172.21.15.98:0/3913177330' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:00:14.299 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:14.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:14.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:14.402 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:14.432 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:14.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:14.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:14.536 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:14.565 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:14.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:14.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:14.670 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:14.702 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:14.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:14.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:14.805 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:14.829 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:14.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:14.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:14.932 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:14.962 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:14.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:14.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:15.066 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:15.073 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:15 smithi098 bash[79516]: cluster 2023-09-23T19:00:13.992164+0000 mon.a (mon.0) 1751 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:00:15.073 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:15 smithi098 bash[79516]: cluster 2023-09-23T19:00:13.996325+0000 osd.5 (osd.5) 239 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T19:00:15.073 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:15 smithi098 bash[79516]: audit 2023-09-23T19:00:13.996900+0000 mon.a (mon.0) 1752 : audit 1 from='client.? 172.21.15.98:0/1863778228' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:00:15.073 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:15 smithi098 bash[79516]: cluster 2023-09-23T19:00:13.997048+0000 mon.a (mon.0) 1753 : cluster 0 osdmap e314: 8 total, 7 up, 7 in 2023-09-23T19:00:15.073 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:15 smithi098 bash[79516]: cluster 2023-09-23T19:00:14.719911+0000 osd.6 (osd.6) 286 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T19:00:15.074 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:15 smithi098 bash[76817]: cluster 2023-09-23T19:00:13.992164+0000 mon.a (mon.0) 1751 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:00:15.074 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:15 smithi098 bash[76817]: cluster 2023-09-23T19:00:13.996325+0000 osd.5 (osd.5) 239 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T19:00:15.074 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:15 smithi098 bash[76817]: audit 2023-09-23T19:00:13.996900+0000 mon.a (mon.0) 1752 : audit 1 from='client.? 172.21.15.98:0/1863778228' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:00:15.074 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:15 smithi098 bash[76817]: cluster 2023-09-23T19:00:13.997048+0000 mon.a (mon.0) 1753 : cluster 0 osdmap e314: 8 total, 7 up, 7 in 2023-09-23T19:00:15.074 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:15 smithi098 bash[76817]: cluster 2023-09-23T19:00:14.719911+0000 osd.6 (osd.6) 286 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T19:00:15.087 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:15.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:15.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:15.191 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:15.218 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:15.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:15.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:15.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:15 smithi173 bash[59422]: cluster 2023-09-23T19:00:13.992164+0000 mon.a (mon.0) 1751 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:00:15.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:15 smithi173 bash[59422]: cluster 2023-09-23T19:00:13.996325+0000 osd.5 (osd.5) 239 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T19:00:15.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:15 smithi173 bash[59422]: audit 2023-09-23T19:00:13.996900+0000 mon.a (mon.0) 1752 : audit 1 from='client.? 172.21.15.98:0/1863778228' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:00:15.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:15 smithi173 bash[59422]: cluster 2023-09-23T19:00:13.997048+0000 mon.a (mon.0) 1753 : cluster 0 osdmap e314: 8 total, 7 up, 7 in 2023-09-23T19:00:15.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:15 smithi173 bash[59422]: cluster 2023-09-23T19:00:14.719911+0000 osd.6 (osd.6) 286 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T19:00:15.321 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:15.357 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:15.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:15.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:15.459 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:15.492 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:15.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:15.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:15.595 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:15.624 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:15.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:15.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:15.728 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:15.753 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:15.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:15.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:15.856 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:15.883 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:15.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:15.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:15.987 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:16.014 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:16.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:16.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:16.118 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:16.127 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:16 smithi098 bash[79516]: cluster 2023-09-23T19:00:14.407635+0000 mgr.y (mgr.26054) 1268 : cluster 0 pgmap v1390: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 2652/84 objects misplaced (3157.143%); 0 B/s, 157 objects/s recovering 2023-09-23T19:00:16.127 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:16 smithi098 bash[79516]: cluster 2023-09-23T19:00:14.719707+0000 osd.4 (osd.4) 43 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T19:00:16.127 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:00:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:00:16] "GET /metrics HTTP/1.1" 200 36590 "" "Prometheus/2.33.4" 2023-09-23T19:00:16.128 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:16 smithi098 bash[76817]: cluster 2023-09-23T19:00:14.407635+0000 mgr.y (mgr.26054) 1268 : cluster 0 pgmap v1390: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 2652/84 objects misplaced (3157.143%); 0 B/s, 157 objects/s recovering 2023-09-23T19:00:16.128 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:16 smithi098 bash[76817]: cluster 2023-09-23T19:00:14.719707+0000 osd.4 (osd.4) 43 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T19:00:16.145 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:16.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:16.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:16.249 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:16.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:16 smithi173 bash[59422]: cluster 2023-09-23T19:00:14.407635+0000 mgr.y (mgr.26054) 1268 : cluster 0 pgmap v1390: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 6 active+undersized+degraded, 6 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 2652/84 objects misplaced (3157.143%); 0 B/s, 157 objects/s recovering 2023-09-23T19:00:16.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:16 smithi173 bash[59422]: cluster 2023-09-23T19:00:14.719707+0000 osd.4 (osd.4) 43 : cluster 3 failed to encode map e314 with expected crc 2023-09-23T19:00:16.287 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:16.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:16.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:16.390 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:16.421 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:16.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:16.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:16.525 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:16.551 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:16.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:16.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:16.654 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:16.685 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:16.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:16.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:16.789 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:16.816 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:16.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:16.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:16.919 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:16.945 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:16.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:16.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:17.048 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:17.080 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:17.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:17.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:17.185 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:17.212 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:17.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:17.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:17.315 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:17.327 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:17 smithi173 bash[59422]: cluster 2023-09-23T19:00:17.012351+0000 mon.a (mon.0) 1754 : cluster 3 Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:00:17.345 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:17.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:17.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:17.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:17 smithi098 bash[76817]: cluster 2023-09-23T19:00:17.012351+0000 mon.a (mon.0) 1754 : cluster 3 Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:00:17.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:17 smithi098 bash[79516]: cluster 2023-09-23T19:00:17.012351+0000 mon.a (mon.0) 1754 : cluster 3 Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:00:17.450 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:17.486 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:17.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:17.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:17.590 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:17.619 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:17.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:17.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:17.723 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:17.746 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:17.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:17.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:17.848 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:17.876 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:17.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:17.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:17.979 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:18.009 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:18.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:18.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:18.113 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:18.137 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:18.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:18.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:18.241 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:18.267 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:18.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:18.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:18.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:18 smithi173 bash[59422]: cluster 2023-09-23T19:00:16.408256+0000 mgr.y (mgr.26054) 1269 : cluster 0 pgmap v1391: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 5 active+undersized+degraded, 7 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 2652/84 objects misplaced (3157.143%); 0 B/s, 157 objects/s recovering 2023-09-23T19:00:18.313 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:00:18.314 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:00:18.372 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:18.373 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:18 smithi098 bash[79516]: cluster 2023-09-23T19:00:16.408256+0000 mgr.y (mgr.26054) 1269 : cluster 0 pgmap v1391: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 5 active+undersized+degraded, 7 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 2652/84 objects misplaced (3157.143%); 0 B/s, 157 objects/s recovering 2023-09-23T19:00:18.374 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:18 smithi098 bash[76817]: cluster 2023-09-23T19:00:16.408256+0000 mgr.y (mgr.26054) 1269 : cluster 0 pgmap v1391: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 22 active+clean+remapped, 52 active+clean, 5 active+undersized+degraded, 7 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 2652/84 objects misplaced (3157.143%); 0 B/s, 157 objects/s recovering 2023-09-23T19:00:18.402 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:18.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:18.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:18.505 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:18.537 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:18.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:18.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:18.641 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:18.672 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:18.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:18.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:18.775 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:18.798 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:18.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:18.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:18.902 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:18.926 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:18.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:18.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:19.030 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:19.062 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:19.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:19.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:19.166 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:19.187 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:19.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:19.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:19.290 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:19.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:19.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:19.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:19.431 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:19.463 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:19.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:19.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:19.566 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:19.591 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:19.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:19.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:19.695 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:19.717 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:19.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:19.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:19.820 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:19.842 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:19.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:19.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:19.946 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:19.977 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:19.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:19.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:20.080 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:20.109 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:20.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:20.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:20.213 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:20.233 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:20.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:20.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:20.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:20 smithi173 bash[59422]: cluster 2023-09-23T19:00:18.409049+0000 mgr.y (mgr.26054) 1270 : cluster 0 pgmap v1392: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 23 active+clean+remapped, 52 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%); 0 B/s, 181 objects/s recovering 2023-09-23T19:00:20.315 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:00:20.322 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:20 smithi098 bash[79516]: cluster 2023-09-23T19:00:18.409049+0000 mgr.y (mgr.26054) 1270 : cluster 0 pgmap v1392: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 23 active+clean+remapped, 52 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%); 0 B/s, 181 objects/s recovering 2023-09-23T19:00:20.323 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:20 smithi098 bash[76817]: cluster 2023-09-23T19:00:18.409049+0000 mgr.y (mgr.26054) 1270 : cluster 0 pgmap v1392: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 23 active+clean+remapped, 52 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%); 0 B/s, 181 objects/s recovering 2023-09-23T19:00:20.337 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:20.364 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:20.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:20.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:20.376 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:00:20.469 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:20.501 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:20.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:20.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:20.604 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:20.624 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:20.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:20.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:20.727 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:20.757 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:20.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:20.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:20.860 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:20.885 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:20.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:20.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:20.988 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:21.011 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:21.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:21.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:21.114 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:21.144 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:21.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:21.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:21.248 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:21.272 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:21.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:21.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:21.376 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:21.398 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:21.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:21.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:21.501 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:21.532 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:21.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:21.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:21.635 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:21.667 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:21.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:21.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:21.771 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:21.800 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:21.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:21.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:21.904 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:21.933 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:21.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:21.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:22.036 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:22.059 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:22.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:22.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:22.162 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:22.172 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:22 smithi098 bash[79516]: cluster 2023-09-23T19:00:20.409594+0000 mgr.y (mgr.26054) 1271 : cluster 0 pgmap v1393: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 23 active+clean+remapped, 52 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%); 0 B/s, 181 objects/s recovering 2023-09-23T19:00:22.172 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:22 smithi098 bash[76817]: cluster 2023-09-23T19:00:20.409594+0000 mgr.y (mgr.26054) 1271 : cluster 0 pgmap v1393: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 23 active+clean+remapped, 52 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%); 0 B/s, 181 objects/s recovering 2023-09-23T19:00:22.187 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:22.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:22.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:22.290 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:22.315 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:22.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:22.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:22.419 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:22.441 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:22.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:22.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:22.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:22 smithi173 bash[59422]: cluster 2023-09-23T19:00:20.409594+0000 mgr.y (mgr.26054) 1271 : cluster 0 pgmap v1393: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 23 active+clean+remapped, 52 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%); 0 B/s, 181 objects/s recovering 2023-09-23T19:00:22.545 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:22.571 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:22.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:22.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:22.675 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:22.696 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:22.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:22.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:22.800 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:22.824 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:22.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:22.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:22.927 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:22.956 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:22.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:22.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:23.060 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:23.090 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:23.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:23.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:23.193 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:23.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:23.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:23.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:23.330 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:23.354 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:23.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:23.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:23.457 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:23.488 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:23.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:23.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:23.593 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:23.617 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:23.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:23.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:23.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:23.750 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:23.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:23.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:23.854 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:23.879 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:23.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:23.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:23.982 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:24.004 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:24.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:24.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:24.099 INFO:teuthology.orchestra.run.smithi098.stderr:pool 'unique_pool_4' removed 2023-09-23T19:00:24.108 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:24.139 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:24 smithi098 bash[76817]: cluster 2023-09-23T19:00:22.410248+0000 mgr.y (mgr.26054) 1272 : cluster 0 pgmap v1394: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 23 active+clean+remapped, 52 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%); 0 B/s, 176 objects/s recovering 2023-09-23T19:00:24.140 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:24 smithi098 bash[76817]: audit 2023-09-23T19:00:23.287873+0000 mon.a (mon.0) 1755 : audit 1 from='client.? 172.21.15.98:0/1295681921' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:00:24.140 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:24 smithi098 bash[79516]: cluster 2023-09-23T19:00:22.410248+0000 mgr.y (mgr.26054) 1272 : cluster 0 pgmap v1394: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 23 active+clean+remapped, 52 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%); 0 B/s, 176 objects/s recovering 2023-09-23T19:00:24.140 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:24 smithi098 bash[79516]: audit 2023-09-23T19:00:23.287873+0000 mon.a (mon.0) 1755 : audit 1 from='client.? 172.21.15.98:0/1295681921' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:00:24.140 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:24.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:24.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:24.244 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:24.268 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:24.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:24.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:24.372 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:24.394 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:24.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:24.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:24.498 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:24.524 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:24.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:24.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:24.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:24 smithi173 bash[59422]: cluster 2023-09-23T19:00:22.410248+0000 mgr.y (mgr.26054) 1272 : cluster 0 pgmap v1394: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 23 active+clean+remapped, 52 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%); 0 B/s, 176 objects/s recovering 2023-09-23T19:00:24.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:24 smithi173 bash[59422]: audit 2023-09-23T19:00:23.287873+0000 mon.a (mon.0) 1755 : audit 1 from='client.? 172.21.15.98:0/1295681921' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:00:24.628 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:24.660 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:24.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:24.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:24.763 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:24.788 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:24.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:24.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:24.892 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:24.921 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:24.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:24.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:25.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:25.056 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:25.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:25.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:25.160 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:25.188 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:25.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:25.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:25.197 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:25 smithi098 bash[79516]: cluster 2023-09-23T19:00:24.088092+0000 osd.6 (osd.6) 287 : cluster 3 failed to encode map e315 with expected crc 2023-09-23T19:00:25.198 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:25 smithi098 bash[79516]: audit 2023-09-23T19:00:24.088134+0000 mon.a (mon.0) 1756 : audit 1 from='client.? 172.21.15.98:0/1295681921' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:00:25.198 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:25 smithi098 bash[79516]: cluster 2023-09-23T19:00:24.088462+0000 mon.a (mon.0) 1757 : cluster 0 osdmap e315: 8 total, 7 up, 7 in 2023-09-23T19:00:25.198 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:25 smithi098 bash[79516]: cluster 2023-09-23T19:00:24.090103+0000 osd.4 (osd.4) 44 : cluster 3 failed to encode map e315 with expected crc 2023-09-23T19:00:25.198 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:25 smithi098 bash[79516]: cluster 2023-09-23T19:00:24.151764+0000 osd.5 (osd.5) 240 : cluster 3 failed to encode map e315 with expected crc 2023-09-23T19:00:25.198 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:25 smithi098 bash[79516]: audit 2023-09-23T19:00:24.636432+0000 mon.a (mon.0) 1758 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:00:25.198 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:25 smithi098 bash[76817]: cluster 2023-09-23T19:00:24.088092+0000 osd.6 (osd.6) 287 : cluster 3 failed to encode map e315 with expected crc 2023-09-23T19:00:25.198 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:25 smithi098 bash[76817]: audit 2023-09-23T19:00:24.088134+0000 mon.a (mon.0) 1756 : audit 1 from='client.? 172.21.15.98:0/1295681921' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:00:25.198 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:25 smithi098 bash[76817]: cluster 2023-09-23T19:00:24.088462+0000 mon.a (mon.0) 1757 : cluster 0 osdmap e315: 8 total, 7 up, 7 in 2023-09-23T19:00:25.199 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:25 smithi098 bash[76817]: cluster 2023-09-23T19:00:24.090103+0000 osd.4 (osd.4) 44 : cluster 3 failed to encode map e315 with expected crc 2023-09-23T19:00:25.199 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:25 smithi098 bash[76817]: cluster 2023-09-23T19:00:24.151764+0000 osd.5 (osd.5) 240 : cluster 3 failed to encode map e315 with expected crc 2023-09-23T19:00:25.199 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:25 smithi098 bash[76817]: audit 2023-09-23T19:00:24.636432+0000 mon.a (mon.0) 1758 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:00:25.291 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:25.318 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:25.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:25.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:25.421 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:25.444 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:25.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:25.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:25.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:25 smithi173 bash[59422]: cluster 2023-09-23T19:00:24.088092+0000 osd.6 (osd.6) 287 : cluster 3 failed to encode map e315 with expected crc 2023-09-23T19:00:25.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:25 smithi173 bash[59422]: audit 2023-09-23T19:00:24.088134+0000 mon.a (mon.0) 1756 : audit 1 from='client.? 172.21.15.98:0/1295681921' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:00:25.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:25 smithi173 bash[59422]: cluster 2023-09-23T19:00:24.088462+0000 mon.a (mon.0) 1757 : cluster 0 osdmap e315: 8 total, 7 up, 7 in 2023-09-23T19:00:25.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:25 smithi173 bash[59422]: cluster 2023-09-23T19:00:24.090103+0000 osd.4 (osd.4) 44 : cluster 3 failed to encode map e315 with expected crc 2023-09-23T19:00:25.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:25 smithi173 bash[59422]: cluster 2023-09-23T19:00:24.151764+0000 osd.5 (osd.5) 240 : cluster 3 failed to encode map e315 with expected crc 2023-09-23T19:00:25.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:25 smithi173 bash[59422]: audit 2023-09-23T19:00:24.636432+0000 mon.a (mon.0) 1758 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:00:25.546 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:25.571 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:25.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:25.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:25.674 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:25.710 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:25.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:25.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:25.814 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:25.848 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:25.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:25.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:25.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:25.982 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:25.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:25.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:26.086 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:26.106 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T19:00:26.107 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:00:26.114 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:26.115 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:26 smithi098 bash[79516]: cluster 2023-09-23T19:00:24.410898+0000 mgr.y (mgr.26054) 1273 : cluster 0 pgmap v1396: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:26.115 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:26 smithi098 bash[79516]: cluster 2023-09-23T19:00:25.084542+0000 mon.a (mon.0) 1759 : cluster 3 Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:00:26.116 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:00:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:00:26] "GET /metrics HTTP/1.1" 200 34226 "" "Prometheus/2.33.4" 2023-09-23T19:00:26.116 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:26 smithi098 bash[76817]: cluster 2023-09-23T19:00:24.410898+0000 mgr.y (mgr.26054) 1273 : cluster 0 pgmap v1396: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:26.116 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:26 smithi098 bash[76817]: cluster 2023-09-23T19:00:25.084542+0000 mon.a (mon.0) 1759 : cluster 3 Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:00:26.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:26.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:26.197 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T19:00:26.198 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T19:00:26.198 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-09-23T19:00:26.198 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-09-23T19:00:26.218 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:26.230 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:26 smithi173 bash[59422]: cluster 2023-09-23T19:00:24.410898+0000 mgr.y (mgr.26054) 1273 : cluster 0 pgmap v1396: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:26.230 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:26 smithi173 bash[59422]: cluster 2023-09-23T19:00:25.084542+0000 mon.a (mon.0) 1759 : cluster 3 Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:00:26.252 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:26.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:26.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:26.356 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:26.388 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:26.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:26.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:26.491 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:26.517 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:26.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:26.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:26.620 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:26.653 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:26.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:26.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:26.756 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:26.787 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:26.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:26.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:26.891 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:26.922 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:26.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:26.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:27.026 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:27.061 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:27.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:27.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:27.165 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:27.196 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:27.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:27.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:27.273 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T19:00:27.300 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:27.335 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:27.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:27.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:27.439 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:27.468 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:27.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:27.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:27.573 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:27.608 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:27.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:27.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:27.713 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:27.745 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:00:27.745 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:00:27.747 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:27.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:27.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:27.853 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:27.880 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:27.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:27.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:27.983 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:28.018 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:28.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:28.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:28.123 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:28.153 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:28.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:28.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:28.256 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:28.270 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:28 smithi173 bash[59422]: cluster 2023-09-23T19:00:26.411375+0000 mgr.y (mgr.26054) 1274 : cluster 0 pgmap v1397: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:28.291 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:28.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:28.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:28.396 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:28.427 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:28.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:28.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:28.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:28 smithi098 bash[79516]: cluster 2023-09-23T19:00:26.411375+0000 mgr.y (mgr.26054) 1274 : cluster 0 pgmap v1397: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:28.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:28 smithi098 bash[76817]: cluster 2023-09-23T19:00:26.411375+0000 mgr.y (mgr.26054) 1274 : cluster 0 pgmap v1397: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:28.531 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:28.560 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:28.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:28.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:28.664 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:28.696 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:28.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:28.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:28.799 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:28.832 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:28.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:28.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:28.936 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:28.967 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:28.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:28.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:29.071 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:29.104 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:29.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:29.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:29.209 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:29.242 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:29.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:29.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:29.347 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:29.381 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:29.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:29.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:29.484 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:29.519 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:29.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:29.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:29.623 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:29.657 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:29.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:29.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:29.761 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:29.786 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:29.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:29.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:29.889 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:29.915 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:29.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:29.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:30.019 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:30.045 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:30.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:30.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:30.149 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:30.176 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:30.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:30.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:30.226 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:30 smithi098 bash[76817]: cluster 2023-09-23T19:00:28.412577+0000 mgr.y (mgr.26054) 1275 : cluster 0 pgmap v1398: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:30.226 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:30 smithi098 bash[79516]: cluster 2023-09-23T19:00:28.412577+0000 mgr.y (mgr.26054) 1275 : cluster 0 pgmap v1398: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:30.279 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:30.300 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:30.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:30.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:30.404 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:30.416 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:30 smithi173 bash[59422]: cluster 2023-09-23T19:00:28.412577+0000 mgr.y (mgr.26054) 1275 : cluster 0 pgmap v1398: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:30.434 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:30.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:30.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:30.538 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:30.571 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:30.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:30.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:30.675 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:30.705 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:30.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:30.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:30.809 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:30.843 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:30.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:30.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:30.947 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:30.975 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:30.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:30.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:31.078 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:31.111 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:31.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:31.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:31.215 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:31.243 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:31.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:31.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:31.347 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:31.380 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:31.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:31.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:31.484 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:31.509 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:31.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:31.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:31.612 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:31.644 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:31.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:31.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:31.749 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:31.773 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:31.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:31.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:31.877 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:31.899 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:31.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:31.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:32.004 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:32.035 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:32.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:32.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:32.138 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:32.164 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:32.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:32.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:32.268 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:32.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:32 smithi173 bash[59422]: cluster 2023-09-23T19:00:30.413410+0000 mgr.y (mgr.26054) 1276 : cluster 0 pgmap v1399: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:32.298 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:32.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:32.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:32.362 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:00:32.362 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:00:32.387 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:32 smithi098 bash[79516]: cluster 2023-09-23T19:00:30.413410+0000 mgr.y (mgr.26054) 1276 : cluster 0 pgmap v1399: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:32.387 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:32 smithi098 bash[76817]: cluster 2023-09-23T19:00:30.413410+0000 mgr.y (mgr.26054) 1276 : cluster 0 pgmap v1399: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:32.402 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:32.425 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:32.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:32.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:32.528 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:32.559 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:32.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:32.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:32.663 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:32.697 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:32.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:32.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:32.800 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:32.830 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:32.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:32.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:32.933 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:32.955 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:32.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:32.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:33.059 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:33.090 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:33.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:33.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:33.194 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:33.221 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:33.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:33.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:33.324 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:33.351 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:33.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:33.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:33.454 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:33.480 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:33.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:33.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:33.583 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:33.606 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:33.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:33.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:33.709 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:33.740 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:33.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:33.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:33.843 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:33.873 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:33.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:33.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:33.977 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:34.007 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:34.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:34.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:34.111 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:34.143 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:34.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:34.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:34.247 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:34.270 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:34.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:34.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:34.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:34 smithi173 bash[59422]: cluster 2023-09-23T19:00:32.414512+0000 mgr.y (mgr.26054) 1277 : cluster 0 pgmap v1400: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:34.374 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:34.386 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:34 smithi098 bash[76817]: cluster 2023-09-23T19:00:32.414512+0000 mgr.y (mgr.26054) 1277 : cluster 0 pgmap v1400: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:34.387 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:34 smithi098 bash[79516]: cluster 2023-09-23T19:00:32.414512+0000 mgr.y (mgr.26054) 1277 : cluster 0 pgmap v1400: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:34.405 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:34.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:34.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:34.510 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:34.538 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:34.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:34.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:34.642 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:34.675 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:34.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:34.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:34.779 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:34.813 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:34.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:34.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:34.918 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:34.954 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:34.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:34.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:35.059 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:35.089 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:35.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:35.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:35.192 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:35.222 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:35.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:35.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:35.325 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:35.349 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:35.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:35.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:35.453 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:35.484 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:35.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:35.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:35.588 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:35.615 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:35.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:35.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:35.718 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:35.743 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:35.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:35.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:35.811 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T19:00:35.837 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T19:00:35.845 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:35.861 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T19:00:35.867 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:35.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:35.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:35.883 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T19:00:35.906 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T19:00:35.929 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T19:00:35.951 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T19:00:35.952 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:00:35.952 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:00:35.952 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:00:35.952 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:00:35.952 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:00:35.952 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:00:35.952 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:00:35.952 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T19:00:35.971 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:35.996 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:35.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:35.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:36.100 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:36.122 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:36.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:36.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:36.225 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:36.249 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:36.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:36.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:36.295 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:00:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:00:36] "GET /metrics HTTP/1.1" 200 34226 "" "Prometheus/2.33.4" 2023-09-23T19:00:36.353 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:36.362 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:36 smithi098 bash[79516]: cluster 2023-09-23T19:00:34.415196+0000 mgr.y (mgr.26054) 1278 : cluster 0 pgmap v1401: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:36.362 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:36 smithi098 bash[79516]: audit 2023-09-23T19:00:35.674281+0000 mon.c (mon.2) 139 : audit 1 from='client.? 172.21.15.98:0/340887684' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:00:36.362 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:36 smithi098 bash[79516]: audit 2023-09-23T19:00:35.674738+0000 mon.a (mon.0) 1760 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:00:36.362 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:36 smithi098 bash[79516]: audit 2023-09-23T19:00:35.781079+0000 mon.a (mon.0) 1761 : audit 0 from='client.? 172.21.15.98:0/3224339429' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:00:36.362 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:36 smithi098 bash[76817]: cluster 2023-09-23T19:00:34.415196+0000 mgr.y (mgr.26054) 1278 : cluster 0 pgmap v1401: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:36.362 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:36 smithi098 bash[76817]: audit 2023-09-23T19:00:35.674281+0000 mon.c (mon.2) 139 : audit 1 from='client.? 172.21.15.98:0/340887684' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:00:36.363 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:36 smithi098 bash[76817]: audit 2023-09-23T19:00:35.674738+0000 mon.a (mon.0) 1760 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:00:36.363 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:36 smithi098 bash[76817]: audit 2023-09-23T19:00:35.781079+0000 mon.a (mon.0) 1761 : audit 0 from='client.? 172.21.15.98:0/3224339429' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:00:36.375 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:36.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:36.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:36.455 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is unset 2023-09-23T19:00:36.813 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:36.814 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:36 smithi173 bash[59422]: cluster 2023-09-23T19:00:34.415196+0000 mgr.y (mgr.26054) 1278 : cluster 0 pgmap v1401: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:36.814 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:36 smithi173 bash[59422]: audit 2023-09-23T19:00:35.674281+0000 mon.c (mon.2) 139 : audit 1 from='client.? 172.21.15.98:0/340887684' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:00:36.814 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:36 smithi173 bash[59422]: audit 2023-09-23T19:00:35.674738+0000 mon.a (mon.0) 1760 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:00:36.814 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:36 smithi173 bash[59422]: audit 2023-09-23T19:00:35.781079+0000 mon.a (mon.0) 1761 : audit 0 from='client.? 172.21.15.98:0/3224339429' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:00:36.848 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:36.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:36.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:36.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:36.986 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:36.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:36.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:37.091 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:37.118 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:37.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:37.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:37.223 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:37.251 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:37.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:37.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:37.354 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:37.365 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:37 smithi098 bash[76817]: cluster 2023-09-23T19:00:36.436740+0000 mon.a (mon.0) 1762 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:00:37.365 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:37 smithi098 bash[76817]: cluster 2023-09-23T19:00:36.442836+0000 osd.4 (osd.4) 45 : cluster 3 failed to encode map e316 with expected crc 2023-09-23T19:00:37.365 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:37 smithi098 bash[76817]: audit 2023-09-23T19:00:36.444692+0000 mon.a (mon.0) 1763 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:00:37.365 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:37 smithi098 bash[76817]: cluster 2023-09-23T19:00:36.444787+0000 mon.a (mon.0) 1764 : cluster 0 osdmap e316: 8 total, 7 up, 7 in 2023-09-23T19:00:37.365 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:37 smithi098 bash[76817]: cluster 2023-09-23T19:00:36.446822+0000 osd.6 (osd.6) 288 : cluster 3 failed to encode map e316 with expected crc 2023-09-23T19:00:37.365 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:37 smithi098 bash[79516]: cluster 2023-09-23T19:00:36.436740+0000 mon.a (mon.0) 1762 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:00:37.366 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:37 smithi098 bash[79516]: cluster 2023-09-23T19:00:36.442836+0000 osd.4 (osd.4) 45 : cluster 3 failed to encode map e316 with expected crc 2023-09-23T19:00:37.366 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:37 smithi098 bash[79516]: audit 2023-09-23T19:00:36.444692+0000 mon.a (mon.0) 1763 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:00:37.366 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:37 smithi098 bash[79516]: cluster 2023-09-23T19:00:36.444787+0000 mon.a (mon.0) 1764 : cluster 0 osdmap e316: 8 total, 7 up, 7 in 2023-09-23T19:00:37.366 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:37 smithi098 bash[79516]: cluster 2023-09-23T19:00:36.446822+0000 osd.6 (osd.6) 288 : cluster 3 failed to encode map e316 with expected crc 2023-09-23T19:00:37.379 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:37.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:37.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:37.483 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:37.496 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:37 smithi173 bash[59422]: cluster 2023-09-23T19:00:36.436740+0000 mon.a (mon.0) 1762 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:00:37.496 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:37 smithi173 bash[59422]: cluster 2023-09-23T19:00:36.442836+0000 osd.4 (osd.4) 45 : cluster 3 failed to encode map e316 with expected crc 2023-09-23T19:00:37.496 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:37 smithi173 bash[59422]: audit 2023-09-23T19:00:36.444692+0000 mon.a (mon.0) 1763 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:00:37.496 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:37 smithi173 bash[59422]: cluster 2023-09-23T19:00:36.444787+0000 mon.a (mon.0) 1764 : cluster 0 osdmap e316: 8 total, 7 up, 7 in 2023-09-23T19:00:37.496 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:37 smithi173 bash[59422]: cluster 2023-09-23T19:00:36.446822+0000 osd.6 (osd.6) 288 : cluster 3 failed to encode map e316 with expected crc 2023-09-23T19:00:37.517 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:37.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:37.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:37.621 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:37.647 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:37.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:37.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:37.751 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:37.778 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:37.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:37.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:37.881 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:37.914 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:37.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:37.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:38.018 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:38.043 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:38.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:38.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:38.147 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:38.172 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:38.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:38.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:38.274 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:38.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:38 smithi173 bash[59422]: cluster 2023-09-23T19:00:36.415700+0000 mgr.y (mgr.26054) 1279 : cluster 0 pgmap v1402: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:38.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:38 smithi173 bash[59422]: audit 2023-09-23T19:00:38.000016+0000 mon.a (mon.0) 1765 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:00:38.302 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:38.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:38.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:38.405 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:38.418 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:38 smithi098 bash[79516]: cluster 2023-09-23T19:00:36.415700+0000 mgr.y (mgr.26054) 1279 : cluster 0 pgmap v1402: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:38.418 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:38 smithi098 bash[79516]: audit 2023-09-23T19:00:38.000016+0000 mon.a (mon.0) 1765 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:00:38.418 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:38 smithi098 bash[76817]: cluster 2023-09-23T19:00:36.415700+0000 mgr.y (mgr.26054) 1279 : cluster 0 pgmap v1402: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:38.418 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:38 smithi098 bash[76817]: audit 2023-09-23T19:00:38.000016+0000 mon.a (mon.0) 1765 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:00:38.430 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:38.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:38.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:38.533 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:38.559 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:38.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:38.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:38.662 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:38.692 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:38.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:38.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:38.796 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:38.821 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:38.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:38.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:38.925 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:38.960 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:38.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:38.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:39.064 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:39.097 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:39.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:39.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:39.202 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:39.235 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:39.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:39.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:39.339 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:39.365 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:39.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:39.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:39.468 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:39.500 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:39.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:39.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:39.604 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:39.635 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:39.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:39.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:39.739 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:39.765 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:39.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:39.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:39.869 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:39.898 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:39.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:39.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:40.002 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:40.033 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:40.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:40.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:40.137 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:40.163 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:40.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:40.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:40.266 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:40.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:40 smithi173 bash[59422]: cluster 2023-09-23T19:00:38.416297+0000 mgr.y (mgr.26054) 1280 : cluster 0 pgmap v1404: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:40.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:40 smithi173 bash[59422]: audit 2023-09-23T19:00:39.642878+0000 mon.a (mon.0) 1766 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:00:40.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:40 smithi173 bash[59422]: audit 2023-09-23T19:00:39.644139+0000 mon.a (mon.0) 1767 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:00:40.297 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:40.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:40.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:40.401 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:40.402 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:40 smithi098 bash[76817]: cluster 2023-09-23T19:00:38.416297+0000 mgr.y (mgr.26054) 1280 : cluster 0 pgmap v1404: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:40.402 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:40 smithi098 bash[76817]: audit 2023-09-23T19:00:39.642878+0000 mon.a (mon.0) 1766 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:00:40.402 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:40 smithi098 bash[76817]: audit 2023-09-23T19:00:39.644139+0000 mon.a (mon.0) 1767 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:00:40.403 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:40 smithi098 bash[79516]: cluster 2023-09-23T19:00:38.416297+0000 mgr.y (mgr.26054) 1280 : cluster 0 pgmap v1404: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:40.403 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:40 smithi098 bash[79516]: audit 2023-09-23T19:00:39.642878+0000 mon.a (mon.0) 1766 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:00:40.403 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:40 smithi098 bash[79516]: audit 2023-09-23T19:00:39.644139+0000 mon.a (mon.0) 1767 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:00:40.421 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:40.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:40.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:40.524 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:40.546 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:40.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:40.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:40.555 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:00:40.555 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:00:40.650 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:40.679 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:40.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:40.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:40.782 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:40.812 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:40.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:40.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:40.916 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:40.949 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:40.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:40.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:41.052 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:41.082 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:41.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:41.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:41.186 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:41.217 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:41.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:41.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:41.321 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:41.347 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:41.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:41.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:41.450 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:41.475 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:41.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:41.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:41.578 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:41.611 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:41.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:41.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:41.715 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:41.746 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:41.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:41.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:41.850 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:41.876 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:41.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:41.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:41.979 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:42.007 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:42.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:42.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:42.111 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:42.139 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:42.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:42.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:42.243 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:42.275 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:42.277 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:42 smithi098 bash[76817]: cluster 2023-09-23T19:00:40.417043+0000 mgr.y (mgr.26054) 1281 : cluster 0 pgmap v1405: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:42.278 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:42 smithi098 bash[79516]: cluster 2023-09-23T19:00:40.417043+0000 mgr.y (mgr.26054) 1281 : cluster 0 pgmap v1405: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:42.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:42.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:42.379 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:42.392 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:42 smithi173 bash[59422]: cluster 2023-09-23T19:00:40.417043+0000 mgr.y (mgr.26054) 1281 : cluster 0 pgmap v1405: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:42.410 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:42.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:42.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:42.514 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:42.545 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:42.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:42.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:42.557 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:00:42.649 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:42.682 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:42.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:42.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:42.716 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:00:42.767 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T19:00:42.767 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [], 2023-09-23T19:00:42.767 INFO:teuthology.orchestra.run.smithi098.stdout: "num_ops": 0 2023-09-23T19:00:42.767 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T19:00:42.786 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:42.816 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:42.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:42.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:42.919 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:42.942 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:42.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:42.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:43.045 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:43.074 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:43.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:43.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:43.177 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:43.202 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:43.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:43.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:43.306 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:43.326 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:43.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:43.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:43.429 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:43.458 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:43.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:43.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:43.562 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:43.571 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:43 smithi098 bash[76817]: audit 2023-09-23T19:00:43.081692+0000 mon.c (mon.2) 140 : audit 1 from='client.? 172.21.15.98:0/590370474' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T19:00:43.571 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:43 smithi098 bash[79516]: audit 2023-09-23T19:00:43.081692+0000 mon.c (mon.2) 140 : audit 1 from='client.? 172.21.15.98:0/590370474' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T19:00:43.587 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:43.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:43.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:43.625 INFO:teuthology.orchestra.run.smithi098.stderr:pool 'unique_pool_5' created 2023-09-23T19:00:43.691 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:43.713 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:43 smithi173 bash[59422]: audit 2023-09-23T19:00:43.081692+0000 mon.c (mon.2) 140 : audit 1 from='client.? 172.21.15.98:0/590370474' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T19:00:43.733 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:43.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:43.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:43.836 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:43.862 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:43.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:43.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:43.967 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:43.990 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:43.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:43.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:44.094 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:44.113 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T19:00:44.121 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:44.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:44.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:44.224 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:44.243 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:44.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:44.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:44.346 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:44.367 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:44.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:44.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:44.470 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:44.498 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:44.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:44.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:44.602 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:44.633 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:44.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:44.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:44.736 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:44.748 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:44 smithi173 bash[59422]: cluster 2023-09-23T19:00:42.418022+0000 mgr.y (mgr.26054) 1282 : cluster 0 pgmap v1406: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:44.748 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:44 smithi173 bash[59422]: audit 2023-09-23T19:00:43.082151+0000 mon.a (mon.0) 1768 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T19:00:44.748 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:44 smithi173 bash[59422]: audit 2023-09-23T19:00:43.609005+0000 mon.a (mon.0) 1769 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T19:00:44.749 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:44 smithi173 bash[59422]: cluster 2023-09-23T19:00:43.609105+0000 mon.a (mon.0) 1770 : cluster 0 osdmap e317: 8 total, 7 up, 7 in 2023-09-23T19:00:44.749 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:44 smithi173 bash[59422]: cluster 2023-09-23T19:00:43.609151+0000 osd.6 (osd.6) 289 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T19:00:44.749 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:44 smithi173 bash[59422]: cluster 2023-09-23T19:00:43.610837+0000 osd.4 (osd.4) 46 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T19:00:44.749 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:44 smithi173 bash[59422]: cluster 2023-09-23T19:00:43.613343+0000 osd.6 (osd.6) 290 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T19:00:44.749 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:44 smithi173 bash[59422]: cluster 2023-09-23T19:00:43.615056+0000 osd.5 (osd.5) 241 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T19:00:44.749 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:44 smithi173 bash[59422]: cluster 2023-09-23T19:00:43.616262+0000 osd.5 (osd.5) 242 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T19:00:44.749 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:44 smithi173 bash[59422]: cluster 2023-09-23T19:00:43.618257+0000 osd.5 (osd.5) 243 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T19:00:44.749 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:44 smithi173 bash[59422]: audit 2023-09-23T19:00:43.895073+0000 mon.a (mon.0) 1771 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:00:44.749 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:44 smithi173 bash[59422]: audit 2023-09-23T19:00:43.903918+0000 mon.a (mon.0) 1772 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:00:44.749 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:44 smithi173 bash[59422]: audit 2023-09-23T19:00:44.247498+0000 mon.a (mon.0) 1773 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:00:44.750 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:44 smithi173 bash[59422]: audit 2023-09-23T19:00:44.256239+0000 mon.a (mon.0) 1774 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:00:44.762 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:44.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:44.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:44.864 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:44.898 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:44.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:44.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:44.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[79516]: cluster 2023-09-23T19:00:42.418022+0000 mgr.y (mgr.26054) 1282 : cluster 0 pgmap v1406: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:44.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[79516]: audit 2023-09-23T19:00:43.082151+0000 mon.a (mon.0) 1768 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T19:00:44.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[79516]: audit 2023-09-23T19:00:43.609005+0000 mon.a (mon.0) 1769 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T19:00:44.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[79516]: cluster 2023-09-23T19:00:43.609105+0000 mon.a (mon.0) 1770 : cluster 0 osdmap e317: 8 total, 7 up, 7 in 2023-09-23T19:00:44.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[79516]: cluster 2023-09-23T19:00:43.609151+0000 osd.6 (osd.6) 289 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T19:00:44.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[79516]: cluster 2023-09-23T19:00:43.610837+0000 osd.4 (osd.4) 46 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T19:00:44.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[79516]: cluster 2023-09-23T19:00:43.613343+0000 osd.6 (osd.6) 290 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T19:00:44.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[79516]: cluster 2023-09-23T19:00:43.615056+0000 osd.5 (osd.5) 241 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T19:00:44.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[79516]: cluster 2023-09-23T19:00:43.616262+0000 osd.5 (osd.5) 242 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T19:00:44.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[79516]: cluster 2023-09-23T19:00:43.618257+0000 osd.5 (osd.5) 243 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T19:00:44.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[79516]: audit 2023-09-23T19:00:43.895073+0000 mon.a (mon.0) 1771 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:00:44.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[79516]: audit 2023-09-23T19:00:43.903918+0000 mon.a (mon.0) 1772 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:00:44.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[79516]: audit 2023-09-23T19:00:44.247498+0000 mon.a (mon.0) 1773 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:00:44.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[79516]: audit 2023-09-23T19:00:44.256239+0000 mon.a (mon.0) 1774 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:00:44.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[76817]: cluster 2023-09-23T19:00:42.418022+0000 mgr.y (mgr.26054) 1282 : cluster 0 pgmap v1406: 73 pgs: 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:44.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[76817]: audit 2023-09-23T19:00:43.082151+0000 mon.a (mon.0) 1768 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T19:00:44.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[76817]: audit 2023-09-23T19:00:43.609005+0000 mon.a (mon.0) 1769 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T19:00:44.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[76817]: cluster 2023-09-23T19:00:43.609105+0000 mon.a (mon.0) 1770 : cluster 0 osdmap e317: 8 total, 7 up, 7 in 2023-09-23T19:00:44.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[76817]: cluster 2023-09-23T19:00:43.609151+0000 osd.6 (osd.6) 289 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T19:00:44.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[76817]: cluster 2023-09-23T19:00:43.610837+0000 osd.4 (osd.4) 46 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T19:00:44.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[76817]: cluster 2023-09-23T19:00:43.613343+0000 osd.6 (osd.6) 290 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T19:00:44.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[76817]: cluster 2023-09-23T19:00:43.615056+0000 osd.5 (osd.5) 241 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T19:00:44.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[76817]: cluster 2023-09-23T19:00:43.616262+0000 osd.5 (osd.5) 242 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T19:00:44.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[76817]: cluster 2023-09-23T19:00:43.618257+0000 osd.5 (osd.5) 243 : cluster 3 failed to encode map e317 with expected crc 2023-09-23T19:00:44.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[76817]: audit 2023-09-23T19:00:43.895073+0000 mon.a (mon.0) 1771 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:00:44.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[76817]: audit 2023-09-23T19:00:43.903918+0000 mon.a (mon.0) 1772 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:00:44.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[76817]: audit 2023-09-23T19:00:44.247498+0000 mon.a (mon.0) 1773 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:00:44.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:44 smithi098 bash[76817]: audit 2023-09-23T19:00:44.256239+0000 mon.a (mon.0) 1774 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:00:45.002 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:45.036 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:45.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:45.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:45.060 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-09-23T19:00:45.140 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:45.180 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:45.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:45.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:45.283 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:45.313 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:45.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:45.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:45.416 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:45.447 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:45.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:45.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:45.550 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:45.581 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:45.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:45.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:45.686 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:45.718 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:45.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:45.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:45.822 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:45.834 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:45 smithi098 bash[76817]: cluster 2023-09-23T19:00:44.418790+0000 mgr.y (mgr.26054) 1283 : cluster 0 pgmap v1408: 89 pgs: 16 unknown, 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:45.835 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:45 smithi098 bash[76817]: cluster 2023-09-23T19:00:44.611489+0000 osd.4 (osd.4) 47 : cluster 3 failed to encode map e318 with expected crc 2023-09-23T19:00:45.835 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:45 smithi098 bash[76817]: cluster 2023-09-23T19:00:44.613777+0000 osd.6 (osd.6) 291 : cluster 3 failed to encode map e318 with expected crc 2023-09-23T19:00:45.835 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:45 smithi098 bash[76817]: cluster 2023-09-23T19:00:44.614628+0000 mon.a (mon.0) 1775 : cluster 0 osdmap e318: 8 total, 7 up, 7 in 2023-09-23T19:00:45.835 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:45 smithi098 bash[76817]: cluster 2023-09-23T19:00:44.617141+0000 osd.5 (osd.5) 244 : cluster 3 failed to encode map e318 with expected crc 2023-09-23T19:00:45.835 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:45 smithi098 bash[76817]: audit 2023-09-23T19:00:44.846167+0000 mon.a (mon.0) 1776 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:00:45.835 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:45 smithi098 bash[76817]: audit 2023-09-23T19:00:44.847463+0000 mon.a (mon.0) 1777 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:00:45.835 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:45 smithi098 bash[76817]: audit 2023-09-23T19:00:44.855266+0000 mon.a (mon.0) 1778 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:00:45.835 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:45 smithi098 bash[79516]: cluster 2023-09-23T19:00:44.418790+0000 mgr.y (mgr.26054) 1283 : cluster 0 pgmap v1408: 89 pgs: 16 unknown, 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:45.835 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:45 smithi098 bash[79516]: cluster 2023-09-23T19:00:44.611489+0000 osd.4 (osd.4) 47 : cluster 3 failed to encode map e318 with expected crc 2023-09-23T19:00:45.836 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:45 smithi098 bash[79516]: cluster 2023-09-23T19:00:44.613777+0000 osd.6 (osd.6) 291 : cluster 3 failed to encode map e318 with expected crc 2023-09-23T19:00:45.836 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:45 smithi098 bash[79516]: cluster 2023-09-23T19:00:44.614628+0000 mon.a (mon.0) 1775 : cluster 0 osdmap e318: 8 total, 7 up, 7 in 2023-09-23T19:00:45.836 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:45 smithi098 bash[79516]: cluster 2023-09-23T19:00:44.617141+0000 osd.5 (osd.5) 244 : cluster 3 failed to encode map e318 with expected crc 2023-09-23T19:00:45.836 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:45 smithi098 bash[79516]: audit 2023-09-23T19:00:44.846167+0000 mon.a (mon.0) 1776 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:00:45.836 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:45 smithi098 bash[79516]: audit 2023-09-23T19:00:44.847463+0000 mon.a (mon.0) 1777 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:00:45.836 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:45 smithi098 bash[79516]: audit 2023-09-23T19:00:44.855266+0000 mon.a (mon.0) 1778 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:00:45.852 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:45.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:45.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:45.956 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:45.969 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:45 smithi173 bash[59422]: cluster 2023-09-23T19:00:44.418790+0000 mgr.y (mgr.26054) 1283 : cluster 0 pgmap v1408: 89 pgs: 16 unknown, 21 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:45.969 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:45 smithi173 bash[59422]: cluster 2023-09-23T19:00:44.611489+0000 osd.4 (osd.4) 47 : cluster 3 failed to encode map e318 with expected crc 2023-09-23T19:00:45.970 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:45 smithi173 bash[59422]: cluster 2023-09-23T19:00:44.613777+0000 osd.6 (osd.6) 291 : cluster 3 failed to encode map e318 with expected crc 2023-09-23T19:00:45.970 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:45 smithi173 bash[59422]: cluster 2023-09-23T19:00:44.614628+0000 mon.a (mon.0) 1775 : cluster 0 osdmap e318: 8 total, 7 up, 7 in 2023-09-23T19:00:45.970 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:45 smithi173 bash[59422]: cluster 2023-09-23T19:00:44.617141+0000 osd.5 (osd.5) 244 : cluster 3 failed to encode map e318 with expected crc 2023-09-23T19:00:45.970 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:45 smithi173 bash[59422]: audit 2023-09-23T19:00:44.846167+0000 mon.a (mon.0) 1776 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:00:45.970 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:45 smithi173 bash[59422]: audit 2023-09-23T19:00:44.847463+0000 mon.a (mon.0) 1777 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:00:45.970 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:45 smithi173 bash[59422]: audit 2023-09-23T19:00:44.855266+0000 mon.a (mon.0) 1778 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:00:45.991 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:45.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:45.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:46.094 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:46.129 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:46.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:46.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:46.232 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:46.263 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:46.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:46.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:46.367 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:46.378 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:00:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:00:46] "GET /metrics HTTP/1.1" 200 34223 "" "Prometheus/2.33.4" 2023-09-23T19:00:46.396 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:46.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:46.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:46.501 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:46.531 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:46.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:46.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:46.635 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:46.664 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:46.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:46.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:46.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:46 smithi098 bash[76817]: cluster 2023-09-23T19:00:45.615587+0000 osd.6 (osd.6) 292 : cluster 3 failed to encode map e319 with expected crc 2023-09-23T19:00:46.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:46 smithi098 bash[76817]: cluster 2023-09-23T19:00:45.616537+0000 mon.a (mon.0) 1779 : cluster 0 osdmap e319: 8 total, 7 up, 7 in 2023-09-23T19:00:46.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:46 smithi098 bash[79516]: cluster 2023-09-23T19:00:45.615587+0000 osd.6 (osd.6) 292 : cluster 3 failed to encode map e319 with expected crc 2023-09-23T19:00:46.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:46 smithi098 bash[79516]: cluster 2023-09-23T19:00:45.616537+0000 mon.a (mon.0) 1779 : cluster 0 osdmap e319: 8 total, 7 up, 7 in 2023-09-23T19:00:46.768 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:46.801 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:46.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:46.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:46.906 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:46.939 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:46.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:46.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:47.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:46 smithi173 bash[59422]: cluster 2023-09-23T19:00:45.615587+0000 osd.6 (osd.6) 292 : cluster 3 failed to encode map e319 with expected crc 2023-09-23T19:00:47.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:46 smithi173 bash[59422]: cluster 2023-09-23T19:00:45.616537+0000 mon.a (mon.0) 1779 : cluster 0 osdmap e319: 8 total, 7 up, 7 in 2023-09-23T19:00:47.043 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:47.073 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:47.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:47.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:47.177 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:47.208 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:47.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:47.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:47.311 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:47.345 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:47.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:47.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:47.449 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:47.479 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:47.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:47.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:47.583 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:47.612 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:47.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:47.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:47.716 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:47.749 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:47.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:47.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:47.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:47 smithi173 bash[59422]: cluster 2023-09-23T19:00:46.037524+0000 osd.4 (osd.4) 48 : cluster 3 failed to encode map e319 with expected crc 2023-09-23T19:00:47.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:47 smithi173 bash[59422]: cluster 2023-09-23T19:00:46.040599+0000 osd.4 (osd.4) 49 : cluster 3 failed to encode map e319 with expected crc 2023-09-23T19:00:47.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:47 smithi173 bash[59422]: cluster 2023-09-23T19:00:46.419667+0000 mgr.y (mgr.26054) 1284 : cluster 0 pgmap v1411: 89 pgs: 15 unknown, 21 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:47.853 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:47.888 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:47.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:47.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:47.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:47 smithi098 bash[76817]: cluster 2023-09-23T19:00:46.037524+0000 osd.4 (osd.4) 48 : cluster 3 failed to encode map e319 with expected crc 2023-09-23T19:00:47.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:47 smithi098 bash[76817]: cluster 2023-09-23T19:00:46.040599+0000 osd.4 (osd.4) 49 : cluster 3 failed to encode map e319 with expected crc 2023-09-23T19:00:47.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:47 smithi098 bash[76817]: cluster 2023-09-23T19:00:46.419667+0000 mgr.y (mgr.26054) 1284 : cluster 0 pgmap v1411: 89 pgs: 15 unknown, 21 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:47.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:47 smithi098 bash[79516]: cluster 2023-09-23T19:00:46.037524+0000 osd.4 (osd.4) 48 : cluster 3 failed to encode map e319 with expected crc 2023-09-23T19:00:47.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:47 smithi098 bash[79516]: cluster 2023-09-23T19:00:46.040599+0000 osd.4 (osd.4) 49 : cluster 3 failed to encode map e319 with expected crc 2023-09-23T19:00:47.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:47 smithi098 bash[79516]: cluster 2023-09-23T19:00:46.419667+0000 mgr.y (mgr.26054) 1284 : cluster 0 pgmap v1411: 89 pgs: 15 unknown, 21 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded, 5 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:47.991 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:48.023 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:48.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:48.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:48.127 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:48.159 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:48.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:48.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:48.262 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:48.297 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:48.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:48.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:48.401 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:48.438 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:48.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:48.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:48.542 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:48.572 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:48.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:48.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:48.589 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:00:48.590 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:00:48.676 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:48.709 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:48.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:48.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:48.813 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:48.838 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:48.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:48.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:48.941 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:48.975 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:48.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:48.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:49.078 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:49.109 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:49.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:49.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:49.213 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:49.242 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:49.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:49.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:49.346 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:49.380 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:49.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:49.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:49.484 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:49.518 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:49.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:49.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:49.606 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:00:49.606 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:00:49.622 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:49.643 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:49.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:49.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:49.747 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:49.776 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:49.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:49.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:49.880 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:49.915 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:49.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:49.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:49.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:49 smithi098 bash[76817]: cluster 2023-09-23T19:00:48.421028+0000 mgr.y (mgr.26054) 1285 : cluster 0 pgmap v1412: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:49.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:49 smithi098 bash[79516]: cluster 2023-09-23T19:00:48.421028+0000 mgr.y (mgr.26054) 1285 : cluster 0 pgmap v1412: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:50.018 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:50.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:49 smithi173 bash[59422]: cluster 2023-09-23T19:00:48.421028+0000 mgr.y (mgr.26054) 1285 : cluster 0 pgmap v1412: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:50.042 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:50.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:50.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:50.146 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:50.172 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:50.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:50.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:50.275 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:50.299 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:50.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:50.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:50.403 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:50.437 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:50.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:50.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:50.540 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:50.576 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:50.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:50.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:50.679 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:50.705 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:50.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:50.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:50.809 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:50.837 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:50.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:50.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:50.941 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:50.965 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:50.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:50.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:51.068 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:51.090 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:51.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:51.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:51.193 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:51.217 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:51.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:51.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:51.321 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:51.351 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:51.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:51.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:51.454 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:51.487 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:51.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:51.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:51.590 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:51.615 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:51.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:51.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:51.718 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:51.731 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:51 smithi173 bash[59422]: cluster 2023-09-23T19:00:50.421583+0000 mgr.y (mgr.26054) 1286 : cluster 0 pgmap v1413: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:51.749 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:51.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:51.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:51.853 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:51.874 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:51.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:51.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:51.977 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:51.980 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:51 smithi098 bash[76817]: cluster 2023-09-23T19:00:50.421583+0000 mgr.y (mgr.26054) 1286 : cluster 0 pgmap v1413: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:51.980 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:51 smithi098 bash[79516]: cluster 2023-09-23T19:00:50.421583+0000 mgr.y (mgr.26054) 1286 : cluster 0 pgmap v1413: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:52.005 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:52.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:52.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:52.108 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:52.135 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:52.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:52.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:52.238 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:52.266 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:52.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:52.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:52.369 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:52.390 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:52.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:52.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:52.494 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:52.519 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:52.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:52.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:52.623 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:52.654 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:52.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:52.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:52.759 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:52.789 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:52.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:52.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:52.893 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:52.918 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:52.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:52.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:53.022 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:53.043 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:53.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:53.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:53.146 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:53.176 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:53.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:53.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:53.280 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:53.311 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:53.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:53.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:53.415 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:53.444 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:53.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:53.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:53.548 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:53.580 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:53.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:53.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:53.684 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:53.713 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:53.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:53.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:53.817 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:53.830 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:53 smithi173 bash[59422]: cluster 2023-09-23T19:00:52.422464+0000 mgr.y (mgr.26054) 1287 : cluster 0 pgmap v1414: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:53.853 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:53.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:53.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:53.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:53 smithi098 bash[76817]: cluster 2023-09-23T19:00:52.422464+0000 mgr.y (mgr.26054) 1287 : cluster 0 pgmap v1414: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:53.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:53 smithi098 bash[79516]: cluster 2023-09-23T19:00:52.422464+0000 mgr.y (mgr.26054) 1287 : cluster 0 pgmap v1414: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:53.958 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:53.980 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:53.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:53.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:54.083 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:54.115 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:54.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:54.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:54.219 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:54.243 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:54.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:54.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:54.346 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:54.378 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:54.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:54.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:54.483 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:54.515 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:54.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:54.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:54.619 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:54.653 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:54.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:54.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:54.756 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:54.764 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:54 smithi098 bash[76817]: audit 2023-09-23T19:00:54.637106+0000 mon.a (mon.0) 1780 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:00:54.788 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:54.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:54.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:54.892 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:54.905 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:54 smithi098 bash[79516]: audit 2023-09-23T19:00:54.637106+0000 mon.a (mon.0) 1780 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:00:54.919 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:54.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:54.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:55.023 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:55.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:54 smithi173 bash[59422]: audit 2023-09-23T19:00:54.637106+0000 mon.a (mon.0) 1780 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:00:55.043 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:55.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:55.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:55.147 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:55.175 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:55.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:55.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:55.278 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:55.314 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:55.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:55.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:55.418 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:55.446 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:55.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:55.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:55.550 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:55.581 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:55.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:55.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:55.685 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:55.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:55 smithi098 bash[79516]: cluster 2023-09-23T19:00:54.423593+0000 mgr.y (mgr.26054) 1288 : cluster 0 pgmap v1415: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:55.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:55 smithi098 bash[76817]: cluster 2023-09-23T19:00:54.423593+0000 mgr.y (mgr.26054) 1288 : cluster 0 pgmap v1415: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:55.715 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:55.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:55.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:55.820 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:55.832 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:55 smithi173 bash[59422]: cluster 2023-09-23T19:00:54.423593+0000 mgr.y (mgr.26054) 1288 : cluster 0 pgmap v1415: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:55.855 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:55.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:55.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:55.958 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:55.983 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:55.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:55.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:56.086 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:56.117 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:56.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:56.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:56.220 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:56.254 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:56.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:56.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:56.340 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:00:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:00:56] "GET /metrics HTTP/1.1" 200 36573 "" "Prometheus/2.33.4" 2023-09-23T19:00:56.358 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:56.384 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:56.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:56.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:56.487 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:56.512 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:56.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:56.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:56.616 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:56.637 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:56.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:56.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:56.740 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:56.771 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:56.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:56.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:56.875 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:56.904 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:56.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:56.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:57.007 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:57.028 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:57.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:57.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:57.131 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:57.164 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:57.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:57.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:57.268 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:57.298 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:57.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:57.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:57.402 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:57.430 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:57.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:57.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:57.535 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:57.566 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:57.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:57.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:57.669 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:57.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:57 smithi098 bash[79516]: cluster 2023-09-23T19:00:56.424184+0000 mgr.y (mgr.26054) 1289 : cluster 0 pgmap v1416: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:57.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:57 smithi098 bash[76817]: cluster 2023-09-23T19:00:56.424184+0000 mgr.y (mgr.26054) 1289 : cluster 0 pgmap v1416: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:57.702 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:57.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:57.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:57.806 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:57.833 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:57.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:57.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:57.937 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:57.969 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:57.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:57.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:58.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:57 smithi173 bash[59422]: cluster 2023-09-23T19:00:56.424184+0000 mgr.y (mgr.26054) 1289 : cluster 0 pgmap v1416: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:58.072 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:58.103 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:58.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:58.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:58.207 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:58.237 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:58.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:58.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:58.340 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:58.369 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:00:58.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:58.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:00:58.473 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:58.497 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:58.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:58.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:58.601 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:58.626 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:58.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:58.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:58.730 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:58.761 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:00:58.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:58.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:00:58.864 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:58.887 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:00:58.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:58.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:00:58.991 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:59.012 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:59.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:59.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:59.115 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:59.148 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:59.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:59.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:59.251 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:59.279 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:59.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:59.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:59.382 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:59.415 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:59.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:59.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:00:59.518 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:59.544 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:00:59.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:59.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:00:59.648 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:59.672 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:00:59.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:59.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:00:59.775 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:59.797 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:00:59.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:59.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:00:59.863 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:00:59 smithi098 bash[79516]: cluster 2023-09-23T19:00:58.424880+0000 mgr.y (mgr.26054) 1290 : cluster 0 pgmap v1417: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:59.864 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:00:59 smithi098 bash[76817]: cluster 2023-09-23T19:00:58.424880+0000 mgr.y (mgr.26054) 1290 : cluster 0 pgmap v1417: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:59.901 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:59.915 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:00:59 smithi173 bash[59422]: cluster 2023-09-23T19:00:58.424880+0000 mgr.y (mgr.26054) 1290 : cluster 0 pgmap v1417: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:00:59.933 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:00:59.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:00:59.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:00.038 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:00.069 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:00.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:00.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:00.172 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:00.206 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:00.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:00.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:00.309 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:00.331 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:00.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:00.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:00.434 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:00.462 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:00.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:00.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:00.565 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:00.587 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:00.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:00.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:00.690 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:00.722 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:00.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:00.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:00.825 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:00.859 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:00.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:00.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:00.962 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:00.989 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:00.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:00.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:01.092 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:01.126 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:01.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:01.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:01.230 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:01.266 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:01.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:01.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:01.369 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:01.398 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:01.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:01.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:01.501 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:01.535 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:01.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:01.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:01.639 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:01.669 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:01.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:01.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:01.772 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:01.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:01 smithi173 bash[59422]: cluster 2023-09-23T19:01:00.425589+0000 mgr.y (mgr.26054) 1291 : cluster 0 pgmap v1418: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:01:01.807 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:01.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:01.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:01.911 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:01.943 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:01.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:01 smithi098 bash[76817]: cluster 2023-09-23T19:01:00.425589+0000 mgr.y (mgr.26054) 1291 : cluster 0 pgmap v1418: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:01:01.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:01 smithi098 bash[79516]: cluster 2023-09-23T19:01:00.425589+0000 mgr.y (mgr.26054) 1291 : cluster 0 pgmap v1418: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:01:01.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:01.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:02.046 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:02.069 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:02.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:02.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:02.172 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:02.201 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:02.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:02.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:02.305 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:02.329 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:02.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:02.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:02.433 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:02.460 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:02.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:02.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:02.564 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:02.595 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:02.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:02.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:02.699 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:02.731 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:02.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:02.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:02.834 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:02.857 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:02.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:02.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:02.961 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:02.995 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:02.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:02.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:03.098 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:03.126 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:03.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:03.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:03.229 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:03.262 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:03.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:03.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:03.267 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T19:01:03.293 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T19:01:03.315 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T19:01:03.339 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T19:01:03.360 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T19:01:03.365 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:03.390 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:03.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:03.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:03.394 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T19:01:03.419 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T19:01:03.420 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:01:03.420 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:01:03.420 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:01:03.420 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:01:03.420 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:01:03.420 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:01:03.420 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:01:03.420 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T19:01:03.493 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:03.514 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:03.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:03.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:03.617 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:03.648 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:03.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:03.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:03.752 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:03.787 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:03.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:03.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:03.891 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:03.902 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:03 smithi173 bash[59422]: cluster 2023-09-23T19:01:02.426486+0000 mgr.y (mgr.26054) 1292 : cluster 0 pgmap v1419: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:01:03.902 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:03 smithi173 bash[59422]: audit 2023-09-23T19:01:03.242175+0000 mon.a (mon.0) 1781 : audit 0 from='client.? 172.21.15.98:0/235117972' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:01:03.923 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:03.924 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T19:01:03.924 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:01:03.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:03.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:03.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:03 smithi098 bash[76817]: cluster 2023-09-23T19:01:02.426486+0000 mgr.y (mgr.26054) 1292 : cluster 0 pgmap v1419: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:01:03.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:03 smithi098 bash[76817]: audit 2023-09-23T19:01:03.242175+0000 mon.a (mon.0) 1781 : audit 0 from='client.? 172.21.15.98:0/235117972' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:01:03.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:03 smithi098 bash[79516]: cluster 2023-09-23T19:01:02.426486+0000 mgr.y (mgr.26054) 1292 : cluster 0 pgmap v1419: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:01:03.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:03 smithi098 bash[79516]: audit 2023-09-23T19:01:03.242175+0000 mon.a (mon.0) 1781 : audit 0 from='client.? 172.21.15.98:0/235117972' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:01:04.027 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:04.057 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:04.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:04.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:04.161 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:04.183 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:04.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:04.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:04.286 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:04.312 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:04.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:04.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:04.415 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:04.449 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:04.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:04.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:04.552 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:04.584 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:04.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:04.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:04.689 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:04.690 INFO:teuthology.orchestra.run.smithi098.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-09-23T19:01:04.690 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is set 2023-09-23T19:01:04.711 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:04.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:04.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:04.816 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:04.830 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:04 smithi173 bash[59422]: audit 2023-09-23T19:01:03.722372+0000 mon.c (mon.2) 141 : audit 1 from='client.? 172.21.15.98:0/2171534267' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:01:04.830 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:04 smithi173 bash[59422]: audit 2023-09-23T19:01:03.723037+0000 mon.a (mon.0) 1782 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:01:04.830 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:04 smithi173 bash[59422]: audit 2023-09-23T19:01:04.124625+0000 mon.c (mon.2) 142 : audit 1 from='client.? 172.21.15.98:0/538940557' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:01:04.830 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:04 smithi173 bash[59422]: audit 2023-09-23T19:01:04.125444+0000 mon.a (mon.0) 1783 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:01:04.848 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:04.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:04.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:04.889 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:01:04.890 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:01:04.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:04 smithi098 bash[76817]: audit 2023-09-23T19:01:03.722372+0000 mon.c (mon.2) 141 : audit 1 from='client.? 172.21.15.98:0/2171534267' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:01:04.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:04 smithi098 bash[76817]: audit 2023-09-23T19:01:03.723037+0000 mon.a (mon.0) 1782 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:01:04.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:04 smithi098 bash[76817]: audit 2023-09-23T19:01:04.124625+0000 mon.c (mon.2) 142 : audit 1 from='client.? 172.21.15.98:0/538940557' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:01:04.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:04 smithi098 bash[76817]: audit 2023-09-23T19:01:04.125444+0000 mon.a (mon.0) 1783 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:01:04.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:04 smithi098 bash[79516]: audit 2023-09-23T19:01:03.722372+0000 mon.c (mon.2) 141 : audit 1 from='client.? 172.21.15.98:0/2171534267' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:01:04.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:04 smithi098 bash[79516]: audit 2023-09-23T19:01:03.723037+0000 mon.a (mon.0) 1782 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:01:04.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:04 smithi098 bash[79516]: audit 2023-09-23T19:01:04.124625+0000 mon.c (mon.2) 142 : audit 1 from='client.? 172.21.15.98:0/538940557' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:01:04.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:04 smithi098 bash[79516]: audit 2023-09-23T19:01:04.125444+0000 mon.a (mon.0) 1783 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:01:04.952 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:04.980 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:04.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:04.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:05.083 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:05.112 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:05.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:05.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:05.215 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:05.246 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:05.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:05.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:05.350 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:05.382 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:05.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:05.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:05.485 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:05.517 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:05.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:05.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:05.621 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:05.646 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:05.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:05.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:05.749 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:05.776 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:05.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:05.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:05.879 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:05.889 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:05 smithi098 bash[79516]: cluster 2023-09-23T19:01:04.428077+0000 mgr.y (mgr.26054) 1293 : cluster 0 pgmap v1420: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:01:05.889 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:05 smithi098 bash[79516]: cluster 2023-09-23T19:01:04.669229+0000 mon.a (mon.0) 1784 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:01:05.889 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:05 smithi098 bash[79516]: cluster 2023-09-23T19:01:04.679855+0000 osd.4 (osd.4) 50 : cluster 3 failed to encode map e320 with expected crc 2023-09-23T19:01:05.889 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:05 smithi098 bash[79516]: audit 2023-09-23T19:01:04.681135+0000 mon.a (mon.0) 1785 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:01:05.889 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:05 smithi098 bash[79516]: audit 2023-09-23T19:01:04.681327+0000 mon.a (mon.0) 1786 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:01:05.889 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:05 smithi098 bash[79516]: cluster 2023-09-23T19:01:04.681486+0000 mon.a (mon.0) 1787 : cluster 0 osdmap e320: 8 total, 7 up, 7 in 2023-09-23T19:01:05.889 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:05 smithi098 bash[76817]: cluster 2023-09-23T19:01:04.428077+0000 mgr.y (mgr.26054) 1293 : cluster 0 pgmap v1420: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:01:05.889 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:05 smithi098 bash[76817]: cluster 2023-09-23T19:01:04.669229+0000 mon.a (mon.0) 1784 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:01:05.889 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:05 smithi098 bash[76817]: cluster 2023-09-23T19:01:04.679855+0000 osd.4 (osd.4) 50 : cluster 3 failed to encode map e320 with expected crc 2023-09-23T19:01:05.890 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:05 smithi098 bash[76817]: audit 2023-09-23T19:01:04.681135+0000 mon.a (mon.0) 1785 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:01:05.890 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:05 smithi098 bash[76817]: audit 2023-09-23T19:01:04.681327+0000 mon.a (mon.0) 1786 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:01:05.890 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:05 smithi098 bash[76817]: cluster 2023-09-23T19:01:04.681486+0000 mon.a (mon.0) 1787 : cluster 0 osdmap e320: 8 total, 7 up, 7 in 2023-09-23T19:01:05.911 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:05.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:05.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:06.014 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:06.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:05 smithi173 bash[59422]: cluster 2023-09-23T19:01:04.428077+0000 mgr.y (mgr.26054) 1293 : cluster 0 pgmap v1420: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:01:06.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:05 smithi173 bash[59422]: cluster 2023-09-23T19:01:04.669229+0000 mon.a (mon.0) 1784 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:01:06.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:05 smithi173 bash[59422]: cluster 2023-09-23T19:01:04.679855+0000 osd.4 (osd.4) 50 : cluster 3 failed to encode map e320 with expected crc 2023-09-23T19:01:06.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:05 smithi173 bash[59422]: audit 2023-09-23T19:01:04.681135+0000 mon.a (mon.0) 1785 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:01:06.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:05 smithi173 bash[59422]: audit 2023-09-23T19:01:04.681327+0000 mon.a (mon.0) 1786 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T19:01:06.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:05 smithi173 bash[59422]: cluster 2023-09-23T19:01:04.681486+0000 mon.a (mon.0) 1787 : cluster 0 osdmap e320: 8 total, 7 up, 7 in 2023-09-23T19:01:06.049 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:06.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:06.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:06.153 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:06.173 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:01:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:01:06] "GET /metrics HTTP/1.1" 200 36573 "" "Prometheus/2.33.4" 2023-09-23T19:01:06.190 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:06.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:06.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:06.294 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:06.323 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:06.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:06.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:06.427 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:06.461 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:06.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:06.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:06.564 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:06.589 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:06.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:06.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:06.693 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:06.714 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:06.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:06.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:06.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:06 smithi173 bash[59422]: cluster 2023-09-23T19:01:05.266230+0000 osd.5 (osd.5) 245 : cluster 3 failed to encode map e320 with expected crc 2023-09-23T19:01:06.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:06 smithi173 bash[59422]: cluster 2023-09-23T19:01:05.266232+0000 osd.6 (osd.6) 293 : cluster 3 failed to encode map e320 with expected crc 2023-09-23T19:01:06.818 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:06.831 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:06 smithi098 bash[76817]: cluster 2023-09-23T19:01:05.266230+0000 osd.5 (osd.5) 245 : cluster 3 failed to encode map e320 with expected crc 2023-09-23T19:01:06.831 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:06 smithi098 bash[76817]: cluster 2023-09-23T19:01:05.266232+0000 osd.6 (osd.6) 293 : cluster 3 failed to encode map e320 with expected crc 2023-09-23T19:01:06.832 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:06 smithi098 bash[79516]: cluster 2023-09-23T19:01:05.266230+0000 osd.5 (osd.5) 245 : cluster 3 failed to encode map e320 with expected crc 2023-09-23T19:01:06.832 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:06 smithi098 bash[79516]: cluster 2023-09-23T19:01:05.266232+0000 osd.6 (osd.6) 293 : cluster 3 failed to encode map e320 with expected crc 2023-09-23T19:01:06.850 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:06.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:06.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:06.860 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T19:01:06.891 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:01:06.954 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:06.979 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:06.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:06.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:07.082 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:07.104 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:07.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:07.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:07.207 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:07.235 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:07.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:07.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:07.339 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:07.366 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:07.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:07.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:07.470 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:07.501 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:07.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:07.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:07.606 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:07.630 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:07.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:07.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:07.733 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:07.746 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:07 smithi173 bash[59422]: cluster 2023-09-23T19:01:06.429022+0000 mgr.y (mgr.26054) 1294 : cluster 0 pgmap v1422: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:01:07.766 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:07.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:07.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:07.870 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:07.906 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:07.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:07.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:07.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:07 smithi098 bash[79516]: cluster 2023-09-23T19:01:06.429022+0000 mgr.y (mgr.26054) 1294 : cluster 0 pgmap v1422: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:01:07.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:07 smithi098 bash[76817]: cluster 2023-09-23T19:01:06.429022+0000 mgr.y (mgr.26054) 1294 : cluster 0 pgmap v1422: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:01:08.010 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:08.040 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:08.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:08.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:08.143 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:08.174 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:08.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:08.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:08.277 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:08.307 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:08.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:08.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:08.411 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:08.447 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:08.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:08.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:08.550 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:08.576 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:08.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:08.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:08.681 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:08.712 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:08.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:08.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:08.815 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:08.847 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:08.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:08.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:08.951 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:08.983 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:08.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:08.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:09.017 DEBUG:teuthology.orchestra.run.smithi098:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-09-23T19:01:09.018 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-09-23T19:01:09.018 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:01:09.031 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T19:01:09.031 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T19:01:09.077 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:hints = 1 2023-09-23T19:01:09.077 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T19:01:09.077 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Object prefix: benchmark_data_smithi098_230241 2023-09-23T19:01:09.078 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:01:09.078 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 0 0 0 0 0 0 - 0 2023-09-23T19:01:09.087 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:09.145 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:09.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:09.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:09.254 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:09.281 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:09.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:09.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:09.384 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:09.410 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:09.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:09.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:09.514 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:09.544 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:09.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:09.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:09.647 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:09.698 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:09.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:09.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:09.805 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:09.834 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:09.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:09.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:09.939 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:09.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:09 smithi098 bash[79516]: cluster 2023-09-23T19:01:08.429921+0000 mgr.y (mgr.26054) 1295 : cluster 0 pgmap v1423: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:01:09.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:09 smithi098 bash[79516]: audit 2023-09-23T19:01:09.643266+0000 mon.a (mon.0) 1788 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:01:09.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:09 smithi098 bash[79516]: audit 2023-09-23T19:01:09.645243+0000 mon.a (mon.0) 1789 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:01:09.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:09 smithi098 bash[76817]: cluster 2023-09-23T19:01:08.429921+0000 mgr.y (mgr.26054) 1295 : cluster 0 pgmap v1423: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:01:09.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:09 smithi098 bash[76817]: audit 2023-09-23T19:01:09.643266+0000 mon.a (mon.0) 1788 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:01:09.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:09 smithi098 bash[76817]: audit 2023-09-23T19:01:09.645243+0000 mon.a (mon.0) 1789 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:01:09.949 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:09 smithi173 bash[59422]: cluster 2023-09-23T19:01:08.429921+0000 mgr.y (mgr.26054) 1295 : cluster 0 pgmap v1423: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:01:09.949 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:09 smithi173 bash[59422]: audit 2023-09-23T19:01:09.643266+0000 mon.a (mon.0) 1788 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:01:09.949 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:09 smithi173 bash[59422]: audit 2023-09-23T19:01:09.645243+0000 mon.a (mon.0) 1789 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:01:09.967 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:09.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:09.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:10.038 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:01:10.071 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:10.078 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 1 16 2397 2381 148.805 148.812 0.00444538 0.00668652 2023-09-23T19:01:10.102 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:10.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:10.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:10.205 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:10.235 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:10.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:10.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:10.338 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:10.367 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:10.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:10.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:10.472 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:10.508 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:10.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:10.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:10.612 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:10.643 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:10.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:10.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:10.747 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:10.778 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:10.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:10.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:10.881 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:10.913 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:10.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:10.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:11.017 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:11.048 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:11.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:11.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:11.078 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 2 16 4705 4689 146.521 144.25 0.00775132 0.00680662 2023-09-23T19:01:11.152 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:11.180 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:11.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:11.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:11.285 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:11.313 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:11.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:11.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:11.417 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:11.443 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:11.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:11.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:11.548 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:11.581 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:11.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:11.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:11.681 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:01:11.682 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:01:11.685 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:11.713 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:11.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:11.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:11.821 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:11.832 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:11 smithi173 bash[59422]: cluster 2023-09-23T19:01:10.430440+0000 mgr.y (mgr.26054) 1296 : cluster 0 pgmap v1424: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:01:11.851 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:11.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:11.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:11.954 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:11.983 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:11.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:11.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:12.078 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 3 16 7015 6999 145.801 144.375 0.00442777 0.00684324 2023-09-23T19:01:12.086 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:12.094 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:11 smithi098 bash[79516]: cluster 2023-09-23T19:01:10.430440+0000 mgr.y (mgr.26054) 1296 : cluster 0 pgmap v1424: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:01:12.095 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:11 smithi098 bash[76817]: cluster 2023-09-23T19:01:10.430440+0000 mgr.y (mgr.26054) 1296 : cluster 0 pgmap v1424: 89 pgs: 21 active+clean+remapped, 56 active+clean, 2 active+undersized+degraded, 10 active+undersized; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 14/84 objects misplaced (16.667%) 2023-09-23T19:01:12.113 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:12.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:12.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:12.217 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:12.245 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:12.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:12.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:12.348 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:12.377 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:12.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:12.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:12.480 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:12.511 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:12.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:12.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:12.616 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:12.645 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:12.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:12.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:12.748 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:12.777 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:12.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:12.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:12.880 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:12.890 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:12 smithi173 bash[59422]: cluster 2023-09-23T19:01:12.698229+0000 mon.a (mon.0) 1790 : cluster 3 Health check update: Degraded data redundancy: 791/9288 objects degraded (8.516%), 4 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:12.909 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:12.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:12.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:13.013 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:13.043 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:13.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:13.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:13.078 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 4 16 9225 9209 143.879 138.125 0.00905567 0.0069399 2023-09-23T19:01:13.147 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:13.156 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:12 smithi098 bash[79516]: cluster 2023-09-23T19:01:12.698229+0000 mon.a (mon.0) 1790 : cluster 3 Health check update: Degraded data redundancy: 791/9288 objects degraded (8.516%), 4 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:13.157 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:12 smithi098 bash[76817]: cluster 2023-09-23T19:01:12.698229+0000 mon.a (mon.0) 1790 : cluster 3 Health check update: Degraded data redundancy: 791/9288 objects degraded (8.516%), 4 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:13.174 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:13.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:13.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:13.279 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:13.309 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:13.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:13.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:13.412 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:13.439 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:13.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:13.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:13.543 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:13.570 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:13.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:13.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:13.675 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:13.705 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:13.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:13.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:13.728 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:01:13.807 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:13.840 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:13.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:13.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:13.944 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:13.970 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:13.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:13.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:14.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:13 smithi173 bash[59422]: cluster 2023-09-23T19:01:12.431246+0000 mgr.y (mgr.26054) 1297 : cluster 0 pgmap v1425: 89 pgs: 21 active+clean+remapped, 56 active+clean, 4 active+undersized+degraded, 8 active+undersized; 196 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 19 MiB/s wr, 306 op/s; 791/9288 objects degraded (8.516%); 14/9288 objects misplaced (0.151%) 2023-09-23T19:01:14.074 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:14.078 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 5 16 11285 11269 140.851 128.75 0.00727535 0.00708805 2023-09-23T19:01:14.082 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:13 smithi098 bash[76817]: cluster 2023-09-23T19:01:12.431246+0000 mgr.y (mgr.26054) 1297 : cluster 0 pgmap v1425: 89 pgs: 21 active+clean+remapped, 56 active+clean, 4 active+undersized+degraded, 8 active+undersized; 196 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 19 MiB/s wr, 306 op/s; 791/9288 objects degraded (8.516%); 14/9288 objects misplaced (0.151%) 2023-09-23T19:01:14.083 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:13 smithi098 bash[79516]: cluster 2023-09-23T19:01:12.431246+0000 mgr.y (mgr.26054) 1297 : cluster 0 pgmap v1425: 89 pgs: 21 active+clean+remapped, 56 active+clean, 4 active+undersized+degraded, 8 active+undersized; 196 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 19 MiB/s wr, 306 op/s; 791/9288 objects degraded (8.516%); 14/9288 objects misplaced (0.151%) 2023-09-23T19:01:14.101 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:14.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:14.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:14.205 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:14.242 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:14.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:14.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:14.346 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:14.373 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:14.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:14.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:14.477 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:14.504 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:14.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:14.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:14.610 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:14.644 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:14.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:14.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:14.749 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:14.770 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:01:14.785 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:14.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:14.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:14.888 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:14.916 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:14.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:14.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:15.020 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:15.048 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:15.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:15.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:15.078 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 6 16 13258 13242 137.926 123.312 0.00684701 0.00724125 2023-09-23T19:01:15.151 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:15.177 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:15.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:15.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:15.280 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:15.308 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:15.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:15.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:15.413 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:15.442 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:15.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:15.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:15.545 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:15.571 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:15.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:15.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:15.675 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:15.706 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:15.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:15.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:15.809 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:15.817 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:15 smithi098 bash[76817]: cluster 2023-09-23T19:01:14.431965+0000 mgr.y (mgr.26054) 1298 : cluster 0 pgmap v1426: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 491 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 49 MiB/s wr, 777 op/s; 2580/23421 objects degraded (11.016%); 14/23421 objects misplaced (0.060%) 2023-09-23T19:01:15.817 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:15 smithi098 bash[79516]: cluster 2023-09-23T19:01:14.431965+0000 mgr.y (mgr.26054) 1298 : cluster 0 pgmap v1426: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 491 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 49 MiB/s wr, 777 op/s; 2580/23421 objects degraded (11.016%); 14/23421 objects misplaced (0.060%) 2023-09-23T19:01:15.839 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:15.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:15.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:15.943 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:15.972 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:15.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:15.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:16.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:15 smithi173 bash[59422]: cluster 2023-09-23T19:01:14.431965+0000 mgr.y (mgr.26054) 1298 : cluster 0 pgmap v1426: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 491 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 49 MiB/s wr, 777 op/s; 2580/23421 objects degraded (11.016%); 14/23421 objects misplaced (0.060%) 2023-09-23T19:01:16.075 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:16.078 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 7 16 15296 15280 136.416 127.375 0.00683417 0.00732095 2023-09-23T19:01:16.100 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:16.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:16.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:16.204 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:16.228 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:16.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:16.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:16.332 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:16.341 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:01:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:01:16] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.33.4" 2023-09-23T19:01:16.359 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:16.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:16.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:16.463 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:16.492 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:16.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:16.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:16.595 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:16.619 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:16.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:16.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:16.727 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:16.757 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:16.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:16.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:16.862 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:16.890 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:16.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:16.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:16.993 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:17.022 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:17.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:17.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:17.078 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 8 16 17346 17330 135.379 128.125 0.00411317 0.00737649 2023-09-23T19:01:17.125 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:17.155 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:17.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:17.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:17.263 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:17.295 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:17.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:17.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:17.398 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:17.426 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:17.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:17.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:17.529 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:17.557 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:17.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:17.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:17.661 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:17.689 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:17.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:17.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:17.793 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:17.825 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:17.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:17.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:17.928 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:17.958 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:17.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:17.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:18.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:17 smithi173 bash[59422]: cluster 2023-09-23T19:01:16.432586+0000 mgr.y (mgr.26054) 1299 : cluster 0 pgmap v1427: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 532 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 45 MiB/s wr, 717 op/s; 2580/25401 objects degraded (10.157%); 14/25401 objects misplaced (0.055%) 2023-09-23T19:01:18.064 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:18.078 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 9 16 19185 19169 133.106 114.938 0.0035051 0.00750536 2023-09-23T19:01:18.093 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:18.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:18.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:18.160 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:17 smithi098 bash[79516]: cluster 2023-09-23T19:01:16.432586+0000 mgr.y (mgr.26054) 1299 : cluster 0 pgmap v1427: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 532 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 45 MiB/s wr, 717 op/s; 2580/25401 objects degraded (10.157%); 14/25401 objects misplaced (0.055%) 2023-09-23T19:01:18.160 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:17 smithi098 bash[76817]: cluster 2023-09-23T19:01:16.432586+0000 mgr.y (mgr.26054) 1299 : cluster 0 pgmap v1427: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 532 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 45 MiB/s wr, 717 op/s; 2580/25401 objects degraded (10.157%); 14/25401 objects misplaced (0.055%) 2023-09-23T19:01:18.196 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:18.226 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:18.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:18.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:18.331 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:18.358 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:18.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:18.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:18.462 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:18.494 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:18.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:18.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:18.598 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:18.632 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:18.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:18.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:18.736 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:18.763 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:18.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:18.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:18.867 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:18.906 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:18.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:18.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:19.013 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:19.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:18 smithi173 bash[59422]: cluster 2023-09-23T19:01:18.759794+0000 mon.a (mon.0) 1791 : cluster 3 Health check update: Degraded data redundancy: 5765/54024 objects degraded (10.671%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:19.041 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:19.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:19.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:19.078 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 10 16 20333 20317 126.97 71.75 0.00610705 0.00786891 2023-09-23T19:01:19.144 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:19.151 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:18 smithi098 bash[79516]: cluster 2023-09-23T19:01:18.759794+0000 mon.a (mon.0) 1791 : cluster 3 Health check update: Degraded data redundancy: 5765/54024 objects degraded (10.671%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:19.151 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:18 smithi098 bash[76817]: cluster 2023-09-23T19:01:18.759794+0000 mon.a (mon.0) 1791 : cluster 3 Health check update: Degraded data redundancy: 5765/54024 objects degraded (10.671%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:19.163 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:19.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:19.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:19.266 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:19.294 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:19.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:19.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:19.398 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:19.423 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:19.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:19.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:19.529 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:19.559 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:19.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:19.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:19.663 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:19.692 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:19.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:19.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:19.799 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:19.823 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:19.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:19.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:19.929 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:19.963 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:19.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:19.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:20.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:19 smithi173 bash[59422]: cluster 2023-09-23T19:01:18.433328+0000 mgr.y (mgr.26054) 1300 : cluster 0 pgmap v1428: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.1 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s; 5765/54024 objects degraded (10.671%); 14/54024 objects misplaced (0.026%) 2023-09-23T19:01:20.067 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:20.079 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 11 16 22033 22017 125.085 106.25 0.00874524 0.00798789 2023-09-23T19:01:20.095 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:20.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:20.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:20.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:19 smithi098 bash[76817]: cluster 2023-09-23T19:01:18.433328+0000 mgr.y (mgr.26054) 1300 : cluster 0 pgmap v1428: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.1 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s; 5765/54024 objects degraded (10.671%); 14/54024 objects misplaced (0.026%) 2023-09-23T19:01:20.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:19 smithi098 bash[79516]: cluster 2023-09-23T19:01:18.433328+0000 mgr.y (mgr.26054) 1300 : cluster 0 pgmap v1428: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.1 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s; 5765/54024 objects degraded (10.671%); 14/54024 objects misplaced (0.026%) 2023-09-23T19:01:20.199 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:20.224 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:20.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:20.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:20.331 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:20.359 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:20.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:20.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:20.463 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:20.493 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:20.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:20.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:20.596 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:20.626 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:20.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:20.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:20.729 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:20.757 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:20.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:20.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:20.861 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:20.888 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:20.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:20.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:20.991 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:21.013 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:21.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:21.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:21.079 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 12 16 23816 23800 123.947 111.438 0.0099737 0.0080605 2023-09-23T19:01:21.120 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:21.148 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:21.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:21.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:21.251 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:21.277 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:21.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:21.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:21.380 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:21.411 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:21.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:21.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:21.514 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:21.542 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:21.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:21.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:21.649 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:21.674 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:21.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:21.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:21.779 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:21.804 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:21.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:21.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:21.908 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:21.938 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:21.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:21.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:21.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:21 smithi098 bash[76817]: cluster 2023-09-23T19:01:20.433843+0000 mgr.y (mgr.26054) 1301 : cluster 0 pgmap v1429: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.1 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s; 5765/54024 objects degraded (10.671%); 14/54024 objects misplaced (0.026%) 2023-09-23T19:01:21.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:21 smithi098 bash[79516]: cluster 2023-09-23T19:01:20.433843+0000 mgr.y (mgr.26054) 1301 : cluster 0 pgmap v1429: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.1 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s; 5765/54024 objects degraded (10.671%); 14/54024 objects misplaced (0.026%) 2023-09-23T19:01:22.043 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:22.072 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:22.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:22.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:22.079 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 13 16 25533 25517 122.667 107.312 0.00619692 0.00814538 2023-09-23T19:01:22.174 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:22.187 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:21 smithi173 bash[59422]: cluster 2023-09-23T19:01:20.433843+0000 mgr.y (mgr.26054) 1301 : cluster 0 pgmap v1429: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.1 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s; 5765/54024 objects degraded (10.671%); 14/54024 objects misplaced (0.026%) 2023-09-23T19:01:22.204 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:22.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:22.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:22.308 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:22.331 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:22.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:22.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:22.435 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:22.457 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:22.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:22.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:22.561 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:22.591 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:22.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:22.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:22.698 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:22.724 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:22.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:22.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:22.828 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:22.858 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:22.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:22.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:22.963 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:22.986 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:22.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:22.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:23.079 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 14 16 27136 27120 121.06 100.188 0.00255953 0.0081986 2023-09-23T19:01:23.090 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:23.119 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:23.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:23.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:23.222 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:23.250 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:23.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:23.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:23.353 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:23.381 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:23.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:23.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:23.484 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:23.509 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:23.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:23.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:23.613 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:23.641 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:23.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:23.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:23.744 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:23.772 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:23.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:23.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:23.875 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:23.897 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:23.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:23.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:24.000 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:24.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:23 smithi173 bash[59422]: cluster 2023-09-23T19:01:22.434521+0000 mgr.y (mgr.26054) 1302 : cluster 0 pgmap v1430: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 115 MiB/s wr, 1.85k op/s; 6781/66600 objects degraded (10.182%); 14/66600 objects misplaced (0.021%) 2023-09-23T19:01:24.032 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:24.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:24.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:24.079 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 15 16 28414 28398 118.314 79.875 0.00541043 0.00844573 2023-09-23T19:01:24.141 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:24.170 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:24.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:24.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:24.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:23 smithi098 bash[79516]: cluster 2023-09-23T19:01:22.434521+0000 mgr.y (mgr.26054) 1302 : cluster 0 pgmap v1430: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 115 MiB/s wr, 1.85k op/s; 6781/66600 objects degraded (10.182%); 14/66600 objects misplaced (0.021%) 2023-09-23T19:01:24.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:23 smithi098 bash[76817]: cluster 2023-09-23T19:01:22.434521+0000 mgr.y (mgr.26054) 1302 : cluster 0 pgmap v1430: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 115 MiB/s wr, 1.85k op/s; 6781/66600 objects degraded (10.182%); 14/66600 objects misplaced (0.021%) 2023-09-23T19:01:24.274 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:24.303 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:24.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:24.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:24.406 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:24.436 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:24.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:24.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:24.540 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:24.567 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:24.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:24.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:24.670 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:24.704 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:24.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:24.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:24.807 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:24.836 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:24.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:24.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:24.941 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:24.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:24 smithi098 bash[79516]: audit 2023-09-23T19:01:24.637323+0000 mon.a (mon.0) 1792 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:01:24.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:24 smithi098 bash[76817]: audit 2023-09-23T19:01:24.637323+0000 mon.a (mon.0) 1792 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:01:24.964 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:24.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:24.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:25.069 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:25.079 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 16 16 29860 29844 116.568 90.375 0.0072628 0.00857231 2023-09-23T19:01:25.099 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:25.100 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:24 smithi173 bash[59422]: audit 2023-09-23T19:01:24.637323+0000 mon.a (mon.0) 1792 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:01:25.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:25.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:25.202 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:25.228 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:25.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:25.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:25.332 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:25.363 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:25.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:25.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:25.468 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:25.494 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:25.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:25.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:25.597 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:25.622 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:25.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:25.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:25.725 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:25.754 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:25.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:25.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:25.859 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:25.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:25.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:25.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:25.989 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:26.012 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:25 smithi098 bash[79516]: cluster 2023-09-23T19:01:24.435319+0000 mgr.y (mgr.26054) 1303 : cluster 0 pgmap v1431: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.6 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 120 MiB/s wr, 1.91k op/s; 8240/78237 objects degraded (10.532%); 14/78237 objects misplaced (0.018%) 2023-09-23T19:01:26.012 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:25 smithi098 bash[79516]: cluster 2023-09-23T19:01:24.824786+0000 mon.a (mon.0) 1793 : cluster 3 Health check update: Degraded data redundancy: 8240/78237 objects degraded (10.532%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:26.012 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:25 smithi098 bash[76817]: cluster 2023-09-23T19:01:24.435319+0000 mgr.y (mgr.26054) 1303 : cluster 0 pgmap v1431: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.6 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 120 MiB/s wr, 1.91k op/s; 8240/78237 objects degraded (10.532%); 14/78237 objects misplaced (0.018%) 2023-09-23T19:01:26.012 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:25 smithi098 bash[76817]: cluster 2023-09-23T19:01:24.824786+0000 mon.a (mon.0) 1793 : cluster 3 Health check update: Degraded data redundancy: 8240/78237 objects degraded (10.532%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:26.025 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:26.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:26.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:26.079 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 17 16 31298 31282 114.997 89.875 0.00597115 0.00868854 2023-09-23T19:01:26.130 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:26.150 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:26.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:26.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:26.254 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:26.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:25 smithi173 bash[59422]: cluster 2023-09-23T19:01:24.435319+0000 mgr.y (mgr.26054) 1303 : cluster 0 pgmap v1431: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.6 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 120 MiB/s wr, 1.91k op/s; 8240/78237 objects degraded (10.532%); 14/78237 objects misplaced (0.018%) 2023-09-23T19:01:26.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:25 smithi173 bash[59422]: cluster 2023-09-23T19:01:24.824786+0000 mon.a (mon.0) 1793 : cluster 3 Health check update: Degraded data redundancy: 8240/78237 objects degraded (10.532%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:26.286 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:26.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:26.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:26.389 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:26.419 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:26.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:26.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:26.424 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:01:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:01:26] "GET /metrics HTTP/1.1" 200 36643 "" "Prometheus/2.33.4" 2023-09-23T19:01:26.522 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:26.546 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:26.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:26.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:26.649 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:26.678 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:26.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:26.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:26.781 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:26.801 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:26.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:26.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:26.904 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:26.929 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:26.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:26.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:27.033 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:27.061 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:27.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:27.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:27.079 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 18 16 32526 32510 112.872 76.75 0.00562195 0.00885312 2023-09-23T19:01:27.166 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:27.193 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:27.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:27.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:27.297 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:27.323 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:27.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:27.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:27.426 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:27.453 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:27.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:27.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:27.561 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:27.587 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:27.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:27.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:27.690 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:27.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:27.718 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:27.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:27.819 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:27.847 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:27.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:27.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:27.954 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:27.971 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:27.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:27.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:27.996 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:27 smithi098 bash[76817]: cluster 2023-09-23T19:01:26.436415+0000 mgr.y (mgr.26054) 1304 : cluster 0 pgmap v1432: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.6 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 98 MiB/s wr, 1.56k op/s; 8240/79680 objects degraded (10.341%); 14/79680 objects misplaced (0.018%) 2023-09-23T19:01:27.996 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:27 smithi098 bash[79516]: cluster 2023-09-23T19:01:26.436415+0000 mgr.y (mgr.26054) 1304 : cluster 0 pgmap v1432: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.6 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 98 MiB/s wr, 1.56k op/s; 8240/79680 objects degraded (10.341%); 14/79680 objects misplaced (0.018%) 2023-09-23T19:01:28.074 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:28.079 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 19 16 33976 33960 111.701 90.625 0.0135238 0.00894567 2023-09-23T19:01:28.107 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:28.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:28.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:28.214 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:28.236 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:28.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:28.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:28.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:27 smithi173 bash[59422]: cluster 2023-09-23T19:01:26.436415+0000 mgr.y (mgr.26054) 1304 : cluster 0 pgmap v1432: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 1.6 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 98 MiB/s wr, 1.56k op/s; 8240/79680 objects degraded (10.341%); 14/79680 objects misplaced (0.018%) 2023-09-23T19:01:28.340 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:28.367 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:28.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:28.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:28.470 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:28.494 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:28.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:28.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:28.597 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:28.628 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:28.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:28.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:28.731 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:28.756 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:28.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:28.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:28.863 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:28.890 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:28.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:28.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:28.994 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:29.024 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:29.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:29.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:29.080 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:2023-09-23T19:01:29.076756+0000 min lat: 0.00130179 max lat: 0.429637 avg lat: 0.00902328 2023-09-23T19:01:29.080 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:01:29.080 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 20 16 35455 35439 110.737 92.4375 0.00568033 0.00902328 2023-09-23T19:01:29.128 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:29.157 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:29.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:29.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:29.260 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:29.287 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:29.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:29.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:29.391 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:29.416 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:29.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:29.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:29.519 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:29.547 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:29.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:29.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:29.652 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:29.677 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:29.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:29.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:29.781 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:29.811 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:29.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:29.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:29.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:29.941 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:29.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:29 smithi098 bash[79516]: cluster 2023-09-23T19:01:28.437274+0000 mgr.y (mgr.26054) 1305 : cluster 0 pgmap v1433: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 129 MiB/s wr, 2.06k op/s; 10415/99525 objects degraded (10.465%); 14/99525 objects misplaced (0.014%) 2023-09-23T19:01:29.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:29 smithi098 bash[76817]: cluster 2023-09-23T19:01:28.437274+0000 mgr.y (mgr.26054) 1305 : cluster 0 pgmap v1433: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 129 MiB/s wr, 2.06k op/s; 10415/99525 objects degraded (10.465%); 14/99525 objects misplaced (0.014%) 2023-09-23T19:01:29.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:29.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:30.046 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:30.070 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:30.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:30.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:30.079 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 21 16 36927 36911 109.844 92 0.014896 0.00909659 2023-09-23T19:01:30.175 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:30.187 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:29 smithi173 bash[59422]: cluster 2023-09-23T19:01:28.437274+0000 mgr.y (mgr.26054) 1305 : cluster 0 pgmap v1433: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 129 MiB/s wr, 2.06k op/s; 10415/99525 objects degraded (10.465%); 14/99525 objects misplaced (0.014%) 2023-09-23T19:01:30.203 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:30.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:30.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:30.306 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:30.330 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:30.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:30.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:30.433 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:30.464 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:30.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:30.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:30.570 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:30.597 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:30.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:30.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:30.701 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:30.731 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:30.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:30.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:30.834 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:30.862 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:30.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:30.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:30.966 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:30.992 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:30.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:30.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:31.080 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 22 16 38406 38390 109.053 92.4375 0.0158168 0.0091635 2023-09-23T19:01:31.095 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:31.104 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:30 smithi098 bash[76817]: cluster 2023-09-23T19:01:30.448588+0000 mon.a (mon.0) 1794 : cluster 3 Health check update: Degraded data redundancy: 10415/99525 objects degraded (10.465%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:31.104 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:30 smithi098 bash[79516]: cluster 2023-09-23T19:01:30.448588+0000 mon.a (mon.0) 1794 : cluster 3 Health check update: Degraded data redundancy: 10415/99525 objects degraded (10.465%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:31.124 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:31.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:31.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:31.227 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:31.249 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:31.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:31.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:31.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:30 smithi173 bash[59422]: cluster 2023-09-23T19:01:30.448588+0000 mon.a (mon.0) 1794 : cluster 3 Health check update: Degraded data redundancy: 10415/99525 objects degraded (10.465%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:31.356 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:31.383 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:31.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:31.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:31.486 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:31.518 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:31.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:31.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:31.621 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:31.648 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:31.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:31.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:31.752 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:31.778 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:31.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:31.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:31.882 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:31.894 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:31 smithi173 bash[59422]: cluster 2023-09-23T19:01:30.437834+0000 mgr.y (mgr.26054) 1306 : cluster 0 pgmap v1434: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s; 10415/99525 objects degraded (10.465%); 14/99525 objects misplaced (0.014%) 2023-09-23T19:01:31.909 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:31.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:31.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:31.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:31 smithi098 bash[79516]: cluster 2023-09-23T19:01:30.437834+0000 mgr.y (mgr.26054) 1306 : cluster 0 pgmap v1434: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s; 10415/99525 objects degraded (10.465%); 14/99525 objects misplaced (0.014%) 2023-09-23T19:01:31.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:31 smithi098 bash[76817]: cluster 2023-09-23T19:01:30.437834+0000 mgr.y (mgr.26054) 1306 : cluster 0 pgmap v1434: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s; 10415/99525 objects degraded (10.465%); 14/99525 objects misplaced (0.014%) 2023-09-23T19:01:32.014 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:32.046 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:32.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:32.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:32.080 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 23 16 39809 39793 108.123 87.6875 0.00872953 0.00924297 2023-09-23T19:01:32.149 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:32.176 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:32.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:32.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:32.279 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:32.304 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:32.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:32.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:32.408 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:32.440 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:32.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:32.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:32.543 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:32.570 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:32.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:32.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:32.674 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:32.702 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:32.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:32.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:32.806 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:32.842 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:32.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:32.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:32.946 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:32.983 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:32.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:32.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:33.080 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 24 16 40730 40714 106.016 57.5625 0.00467402 0.00942604 2023-09-23T19:01:33.088 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:33.113 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:33.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:33.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:33.216 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:33.242 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:33.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:33.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:33.346 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:33.373 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:33.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:33.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:33.476 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:33.502 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:33.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:33.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:33.607 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:33.661 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:33.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:33.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:33.764 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:33.799 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:33.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:33.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:33.908 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:33.918 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:33 smithi173 bash[59422]: cluster 2023-09-23T19:01:32.438424+0000 mgr.y (mgr.26054) 1307 : cluster 0 pgmap v1435: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 2.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 98 MiB/s wr, 1.56k op/s; 11340/110388 objects degraded (10.273%); 14/110388 objects misplaced (0.013%) 2023-09-23T19:01:33.934 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:33.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:33.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:34.037 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:34.047 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:33 smithi098 bash[76817]: cluster 2023-09-23T19:01:32.438424+0000 mgr.y (mgr.26054) 1307 : cluster 0 pgmap v1435: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 2.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 98 MiB/s wr, 1.56k op/s; 11340/110388 objects degraded (10.273%); 14/110388 objects misplaced (0.013%) 2023-09-23T19:01:34.047 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:33 smithi098 bash[79516]: cluster 2023-09-23T19:01:32.438424+0000 mgr.y (mgr.26054) 1307 : cluster 0 pgmap v1435: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 2.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 98 MiB/s wr, 1.56k op/s; 11340/110388 objects degraded (10.273%); 14/110388 objects misplaced (0.013%) 2023-09-23T19:01:34.072 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:34.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:34.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:34.080 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 25 16 42041 42025 105.053 81.9375 0.0148348 0.00951249 2023-09-23T19:01:34.177 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:34.200 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:34.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:34.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:34.308 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:34.340 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:34.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:34.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:34.445 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:34.478 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:34.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:34.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:34.582 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:34.612 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:34.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:34.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:34.714 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:34.745 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:34.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:34.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:34.848 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:34.881 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:34.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:34.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:34.985 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:35.017 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:35.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:35.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:35.080 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 26 16 43321 43305 104.089 80 0.010201 0.00960007 2023-09-23T19:01:35.122 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:35.148 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:35.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:35.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:35.253 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:35.290 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:35.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:35.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:35.393 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:35.433 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:35.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:35.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:35.537 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:35.563 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:35.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:35.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:35.666 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:35.698 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:35.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:35.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:35.801 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:35.826 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:35.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:35.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:35.929 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:35.952 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:35.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:35.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:36.056 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:36.065 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:35 smithi173 bash[59422]: cluster 2023-09-23T19:01:34.439010+0000 mgr.y (mgr.26054) 1308 : cluster 0 pgmap v1436: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 93 MiB/s wr, 1.50k op/s; 12577/120453 objects degraded (10.441%); 14/120453 objects misplaced (0.012%) 2023-09-23T19:01:36.065 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:35 smithi173 bash[59422]: cluster 2023-09-23T19:01:35.449668+0000 mon.a (mon.0) 1795 : cluster 3 Health check update: Degraded data redundancy: 12577/120453 objects degraded (10.441%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:36.080 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 27 16 44588 44572 103.166 79.1875 0.0198357 0.00968697 2023-09-23T19:01:36.081 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:36.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:36.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:36.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:35 smithi098 bash[76817]: cluster 2023-09-23T19:01:34.439010+0000 mgr.y (mgr.26054) 1308 : cluster 0 pgmap v1436: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 93 MiB/s wr, 1.50k op/s; 12577/120453 objects degraded (10.441%); 14/120453 objects misplaced (0.012%) 2023-09-23T19:01:36.088 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:35 smithi098 bash[76817]: cluster 2023-09-23T19:01:35.449668+0000 mon.a (mon.0) 1795 : cluster 3 Health check update: Degraded data redundancy: 12577/120453 objects degraded (10.441%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:36.088 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:01:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:01:36] "GET /metrics HTTP/1.1" 200 36643 "" "Prometheus/2.33.4" 2023-09-23T19:01:36.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:35 smithi098 bash[79516]: cluster 2023-09-23T19:01:34.439010+0000 mgr.y (mgr.26054) 1308 : cluster 0 pgmap v1436: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 93 MiB/s wr, 1.50k op/s; 12577/120453 objects degraded (10.441%); 14/120453 objects misplaced (0.012%) 2023-09-23T19:01:36.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:35 smithi098 bash[79516]: cluster 2023-09-23T19:01:35.449668+0000 mon.a (mon.0) 1795 : cluster 3 Health check update: Degraded data redundancy: 12577/120453 objects degraded (10.441%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:36.185 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:36.215 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:36.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:36.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:36.319 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:36.339 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:36.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:36.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:36.442 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:36.471 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:36.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:36.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:36.576 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:36.613 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:36.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:36.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:36.716 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:36.745 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:36.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:36.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:36.850 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:36.876 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:36.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:36.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:36.979 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:37.006 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:37.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:37.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:37.080 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 28 16 45893 45877 102.395 81.5625 0.00689623 0.00976016 2023-09-23T19:01:37.109 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:37.138 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:37.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:37.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:37.242 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:37.274 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:37.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:37.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:37.377 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:37.404 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:37.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:37.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:37.506 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:37.527 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:37.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:37.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:37.630 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:37.664 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:37.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:37.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:37.770 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:37.802 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:37.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:37.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:37.887 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:37 smithi098 bash[76817]: cluster 2023-09-23T19:01:36.439602+0000 mgr.y (mgr.26054) 1309 : cluster 0 pgmap v1437: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 75 MiB/s wr, 1.21k op/s; 12577/121641 objects degraded (10.339%); 14/121641 objects misplaced (0.012%) 2023-09-23T19:01:37.905 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:37.935 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:37.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:37.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:38.040 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:38.064 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:38.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:38.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:38.080 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 29 16 46837 46821 100.898 59 0.00226281 0.00988539 2023-09-23T19:01:38.166 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:38.180 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:37 smithi098 bash[79516]: cluster 2023-09-23T19:01:36.439602+0000 mgr.y (mgr.26054) 1309 : cluster 0 pgmap v1437: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 75 MiB/s wr, 1.21k op/s; 12577/121641 objects degraded (10.339%); 14/121641 objects misplaced (0.012%) 2023-09-23T19:01:38.207 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:38.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:38.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:38.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:37 smithi173 bash[59422]: cluster 2023-09-23T19:01:36.439602+0000 mgr.y (mgr.26054) 1309 : cluster 0 pgmap v1437: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 75 MiB/s wr, 1.21k op/s; 12577/121641 objects degraded (10.339%); 14/121641 objects misplaced (0.012%) 2023-09-23T19:01:38.309 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:38.339 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:38.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:38.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:38.444 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:38.471 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:38.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:38.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:38.574 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:38.607 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:38.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:38.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:38.711 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:38.742 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:38.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:38.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:38.846 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:38.871 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:38.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:38.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:38.974 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:38.998 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:39.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:39.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:39.080 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 30 16 47704 47688 99.3409 54.1875 0.0175383 0.0100612 2023-09-23T19:01:39.101 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:39.130 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:39.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:39.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:39.233 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:39.261 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:39.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:39.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:39.363 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:39.398 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:39.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:39.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:39.502 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:39.533 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:39.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:39.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:39.636 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:39.664 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:39.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:39.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:39.770 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:39.799 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:39.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:39.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:39.901 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:39.935 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:39.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:39.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:40.039 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:40.061 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:40.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:40.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:40.080 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 31 16 48654 48638 98.0515 59.375 0.00308508 0.0101813 2023-09-23T19:01:40.142 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:39 smithi098 bash[79516]: cluster 2023-09-23T19:01:38.440512+0000 mgr.y (mgr.26054) 1310 : cluster 0 pgmap v1438: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 103 MiB/s wr, 1.64k op/s; 14502/138777 objects degraded (10.450%); 14/138777 objects misplaced (0.010%) 2023-09-23T19:01:40.142 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:39 smithi098 bash[79516]: audit 2023-09-23T19:01:39.638139+0000 mon.a (mon.0) 1796 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:01:40.142 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:39 smithi098 bash[76817]: cluster 2023-09-23T19:01:38.440512+0000 mgr.y (mgr.26054) 1310 : cluster 0 pgmap v1438: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 103 MiB/s wr, 1.64k op/s; 14502/138777 objects degraded (10.450%); 14/138777 objects misplaced (0.010%) 2023-09-23T19:01:40.142 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:39 smithi098 bash[76817]: audit 2023-09-23T19:01:39.638139+0000 mon.a (mon.0) 1796 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:01:40.165 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:40.194 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:40.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:40.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:40.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:39 smithi173 bash[59422]: cluster 2023-09-23T19:01:38.440512+0000 mgr.y (mgr.26054) 1310 : cluster 0 pgmap v1438: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 103 MiB/s wr, 1.64k op/s; 14502/138777 objects degraded (10.450%); 14/138777 objects misplaced (0.010%) 2023-09-23T19:01:40.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:39 smithi173 bash[59422]: audit 2023-09-23T19:01:39.638139+0000 mon.a (mon.0) 1796 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:01:40.297 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:40.330 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:40.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:40.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:40.435 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:40.457 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:40.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:40.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:40.560 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:40.591 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:40.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:40.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:40.695 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:40.729 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:40.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:40.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:40.837 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:40.863 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:40.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:40.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:40.967 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:40.996 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:40.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:40.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:41.081 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 32 16 49691 49675 97.0125 64.8125 0.00741372 0.0103026 2023-09-23T19:01:41.098 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:41.107 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:40 smithi098 bash[76817]: cluster 2023-09-23T19:01:40.450428+0000 mon.a (mon.0) 1797 : cluster 3 Health check update: Degraded data redundancy: 14502/138777 objects degraded (10.450%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:41.107 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:40 smithi098 bash[79516]: cluster 2023-09-23T19:01:40.450428+0000 mon.a (mon.0) 1797 : cluster 3 Health check update: Degraded data redundancy: 14502/138777 objects degraded (10.450%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:41.132 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:41.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:41.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:41.235 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:41.243 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:40 smithi173 bash[59422]: cluster 2023-09-23T19:01:40.450428+0000 mon.a (mon.0) 1797 : cluster 3 Health check update: Degraded data redundancy: 14502/138777 objects degraded (10.450%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:41.261 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:41.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:41.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:41.364 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:41.384 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:41.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:41.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:41.487 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:41.511 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:41.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:41.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:41.553 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T19:01:41.553 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:01:41.614 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:41.646 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:41.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:41.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:41.753 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:41.780 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:41.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:41.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:41.834 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T19:01:41.884 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:41.885 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T19:01:41.908 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:41 smithi098 bash[76817]: cluster 2023-09-23T19:01:40.440931+0000 mgr.y (mgr.26054) 1311 : cluster 0 pgmap v1439: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 14502/138777 objects degraded (10.450%); 14/138777 objects misplaced (0.010%) 2023-09-23T19:01:41.908 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:41 smithi098 bash[76817]: audit 2023-09-23T19:01:41.770332+0000 mon.a (mon.0) 1798 : audit 0 from='client.? 172.21.15.98:0/1688013445' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:01:41.908 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:41.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:41.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:41.922 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T19:01:41.957 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T19:01:41.990 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T19:01:42.011 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:42.023 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T19:01:42.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:41 smithi173 bash[59422]: cluster 2023-09-23T19:01:40.440931+0000 mgr.y (mgr.26054) 1311 : cluster 0 pgmap v1439: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 14502/138777 objects degraded (10.450%); 14/138777 objects misplaced (0.010%) 2023-09-23T19:01:42.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:41 smithi173 bash[59422]: audit 2023-09-23T19:01:41.770332+0000 mon.a (mon.0) 1798 : audit 0 from='client.? 172.21.15.98:0/1688013445' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:01:42.042 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:42.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:42.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:42.059 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T19:01:42.060 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:01:42.060 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:01:42.060 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:01:42.060 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:01:42.060 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:01:42.060 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:01:42.060 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:01:42.061 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T19:01:42.081 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 33 16 50760 50744 96.0973 66.8125 0.0151366 0.0103999 2023-09-23T19:01:42.145 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:42.190 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:42.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:41 smithi098 bash[79516]: cluster 2023-09-23T19:01:40.440931+0000 mgr.y (mgr.26054) 1311 : cluster 0 pgmap v1439: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 14502/138777 objects degraded (10.450%); 14/138777 objects misplaced (0.010%) 2023-09-23T19:01:42.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:41 smithi098 bash[79516]: audit 2023-09-23T19:01:41.770332+0000 mon.a (mon.0) 1798 : audit 0 from='client.? 172.21.15.98:0/1688013445' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:01:42.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:42.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:42.295 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:42.323 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:42.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:42.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:42.429 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:42.463 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:42.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:42.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:42.567 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:42.592 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:42.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:42.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:42.672 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T19:01:42.689 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-23T19:01:42.695 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:42.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:42.728 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:42.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:42.829 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:42.856 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:42.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:42.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:42.959 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:42.977 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:42.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:42.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:43.081 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 34 16 51705 51689 95.0078 59.0625 0.00794355 0.0105193 2023-09-23T19:01:43.081 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:43.110 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:43.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:43.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:43.213 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:43.233 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:43.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:43.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:43.337 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:43.359 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:43.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:43.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:43.462 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:43.490 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:43.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:43.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:43.597 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:43.621 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:43.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:43.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:43.725 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:43.761 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:43.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:43.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:43.868 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:43.902 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:43.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:43.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:43.925 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:43 smithi098 bash[79516]: cluster 2023-09-23T19:01:42.441709+0000 mgr.y (mgr.26054) 1312 : cluster 0 pgmap v1440: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 3.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s; 15119/146190 objects degraded (10.342%); 14/146190 objects misplaced (0.010%) 2023-09-23T19:01:43.925 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:43 smithi098 bash[79516]: audit 2023-09-23T19:01:42.649823+0000 mgr.y (mgr.26054) 1313 : audit 0 from='client.45788 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:01:43.926 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:43 smithi098 bash[79516]: audit 2023-09-23T19:01:43.330522+0000 mon.c (mon.2) 143 : audit 1 from='client.? 172.21.15.98:0/2087345259' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:01:43.926 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:43 smithi098 bash[79516]: audit 2023-09-23T19:01:43.331599+0000 mon.a (mon.0) 1799 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:01:43.926 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:43 smithi098 bash[76817]: cluster 2023-09-23T19:01:42.441709+0000 mgr.y (mgr.26054) 1312 : cluster 0 pgmap v1440: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 3.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s; 15119/146190 objects degraded (10.342%); 14/146190 objects misplaced (0.010%) 2023-09-23T19:01:43.926 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:43 smithi098 bash[76817]: audit 2023-09-23T19:01:42.649823+0000 mgr.y (mgr.26054) 1313 : audit 0 from='client.45788 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:01:43.926 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:43 smithi098 bash[76817]: audit 2023-09-23T19:01:43.330522+0000 mon.c (mon.2) 143 : audit 1 from='client.? 172.21.15.98:0/2087345259' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:01:43.926 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:43 smithi098 bash[76817]: audit 2023-09-23T19:01:43.331599+0000 mon.a (mon.0) 1799 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:01:43.931 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is set 2023-09-23T19:01:44.007 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:44.032 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:44.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:44.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:44.081 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 35 16 52792 52776 94.2343 67.9375 0.0276275 0.0106059 2023-09-23T19:01:44.137 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:44.147 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:43 smithi173 bash[59422]: cluster 2023-09-23T19:01:42.441709+0000 mgr.y (mgr.26054) 1312 : cluster 0 pgmap v1440: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 3.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s; 15119/146190 objects degraded (10.342%); 14/146190 objects misplaced (0.010%) 2023-09-23T19:01:44.147 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:43 smithi173 bash[59422]: audit 2023-09-23T19:01:42.649823+0000 mgr.y (mgr.26054) 1313 : audit 0 from='client.45788 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:01:44.147 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:43 smithi173 bash[59422]: audit 2023-09-23T19:01:43.330522+0000 mon.c (mon.2) 143 : audit 1 from='client.? 172.21.15.98:0/2087345259' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:01:44.148 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:43 smithi173 bash[59422]: audit 2023-09-23T19:01:43.331599+0000 mon.a (mon.0) 1799 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:01:44.164 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:44.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:44.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:44.269 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:44.295 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:44.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:44.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:44.399 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:44.432 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:44.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:44.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:44.538 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:44.583 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:44.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:44.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:44.688 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:44.716 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:44.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:44.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:44.820 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:44.849 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:44.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:44.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:44.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:44.979 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:44.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:44.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:45.081 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 36 16 53949 53933 93.6251 72.3125 0.0104681 0.010675 2023-09-23T19:01:45.083 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:45.091 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:44 smithi098 bash[76817]: cluster 2023-09-23T19:01:43.908200+0000 mon.a (mon.0) 1800 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:01:45.091 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:44 smithi098 bash[76817]: cluster 2023-09-23T19:01:43.912873+0000 osd.4 (osd.4) 51 : cluster 3 failed to encode map e321 with expected crc 2023-09-23T19:01:45.091 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:44 smithi098 bash[76817]: audit 2023-09-23T19:01:43.913912+0000 mon.a (mon.0) 1801 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:01:45.091 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:44 smithi098 bash[76817]: cluster 2023-09-23T19:01:43.914137+0000 mon.a (mon.0) 1802 : cluster 0 osdmap e321: 8 total, 7 up, 7 in 2023-09-23T19:01:45.091 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:44 smithi098 bash[76817]: cluster 2023-09-23T19:01:43.925703+0000 osd.6 (osd.6) 294 : cluster 3 failed to encode map e321 with expected crc 2023-09-23T19:01:45.091 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:44 smithi098 bash[76817]: cluster 2023-09-23T19:01:43.927894+0000 osd.5 (osd.5) 246 : cluster 3 failed to encode map e321 with expected crc 2023-09-23T19:01:45.092 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:44 smithi098 bash[76817]: cluster 2023-09-23T19:01:43.931082+0000 osd.5 (osd.5) 247 : cluster 3 failed to encode map e321 with expected crc 2023-09-23T19:01:45.092 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:44 smithi098 bash[79516]: cluster 2023-09-23T19:01:43.908200+0000 mon.a (mon.0) 1800 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:01:45.092 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:44 smithi098 bash[79516]: cluster 2023-09-23T19:01:43.912873+0000 osd.4 (osd.4) 51 : cluster 3 failed to encode map e321 with expected crc 2023-09-23T19:01:45.092 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:44 smithi098 bash[79516]: audit 2023-09-23T19:01:43.913912+0000 mon.a (mon.0) 1801 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:01:45.092 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:44 smithi098 bash[79516]: cluster 2023-09-23T19:01:43.914137+0000 mon.a (mon.0) 1802 : cluster 0 osdmap e321: 8 total, 7 up, 7 in 2023-09-23T19:01:45.092 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:44 smithi098 bash[79516]: cluster 2023-09-23T19:01:43.925703+0000 osd.6 (osd.6) 294 : cluster 3 failed to encode map e321 with expected crc 2023-09-23T19:01:45.092 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:44 smithi098 bash[79516]: cluster 2023-09-23T19:01:43.927894+0000 osd.5 (osd.5) 246 : cluster 3 failed to encode map e321 with expected crc 2023-09-23T19:01:45.092 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:44 smithi098 bash[79516]: cluster 2023-09-23T19:01:43.931082+0000 osd.5 (osd.5) 247 : cluster 3 failed to encode map e321 with expected crc 2023-09-23T19:01:45.114 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:45.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:45.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:45.218 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:45.248 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:45.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:45.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:45.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:44 smithi173 bash[59422]: cluster 2023-09-23T19:01:43.908200+0000 mon.a (mon.0) 1800 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:01:45.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:44 smithi173 bash[59422]: cluster 2023-09-23T19:01:43.912873+0000 osd.4 (osd.4) 51 : cluster 3 failed to encode map e321 with expected crc 2023-09-23T19:01:45.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:44 smithi173 bash[59422]: audit 2023-09-23T19:01:43.913912+0000 mon.a (mon.0) 1801 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:01:45.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:44 smithi173 bash[59422]: cluster 2023-09-23T19:01:43.914137+0000 mon.a (mon.0) 1802 : cluster 0 osdmap e321: 8 total, 7 up, 7 in 2023-09-23T19:01:45.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:44 smithi173 bash[59422]: cluster 2023-09-23T19:01:43.925703+0000 osd.6 (osd.6) 294 : cluster 3 failed to encode map e321 with expected crc 2023-09-23T19:01:45.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:44 smithi173 bash[59422]: cluster 2023-09-23T19:01:43.927894+0000 osd.5 (osd.5) 246 : cluster 3 failed to encode map e321 with expected crc 2023-09-23T19:01:45.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:44 smithi173 bash[59422]: cluster 2023-09-23T19:01:43.931082+0000 osd.5 (osd.5) 247 : cluster 3 failed to encode map e321 with expected crc 2023-09-23T19:01:45.351 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:45.376 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:45.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:45.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:45.480 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:45.507 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:45.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:45.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:45.611 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:45.638 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:45.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:45.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:45.741 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:45.775 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:45.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:45.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:45.882 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:45.906 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:45.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:45.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:46.008 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:46.017 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:45 smithi098 bash[76817]: cluster 2023-09-23T19:01:44.442138+0000 mgr.y (mgr.26054) 1314 : cluster 0 pgmap v1442: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 16030/153501 objects degraded (10.443%); 14/153501 objects misplaced (0.009%) 2023-09-23T19:01:46.017 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:45 smithi098 bash[76817]: audit 2023-09-23T19:01:44.937727+0000 mon.a (mon.0) 1803 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:01:46.017 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:45 smithi098 bash[76817]: cluster 2023-09-23T19:01:45.451241+0000 mon.a (mon.0) 1804 : cluster 3 Health check update: Degraded data redundancy: 16030/153501 objects degraded (10.443%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:46.017 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:45 smithi098 bash[79516]: cluster 2023-09-23T19:01:44.442138+0000 mgr.y (mgr.26054) 1314 : cluster 0 pgmap v1442: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 16030/153501 objects degraded (10.443%); 14/153501 objects misplaced (0.009%) 2023-09-23T19:01:46.017 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:45 smithi098 bash[79516]: audit 2023-09-23T19:01:44.937727+0000 mon.a (mon.0) 1803 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:01:46.017 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:45 smithi098 bash[79516]: cluster 2023-09-23T19:01:45.451241+0000 mon.a (mon.0) 1804 : cluster 3 Health check update: Degraded data redundancy: 16030/153501 objects degraded (10.443%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:46.035 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:46.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:46.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:46.081 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 37 16 55089 55073 93.0203 71.25 0.0080176 0.0107449 2023-09-23T19:01:46.142 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:46.170 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:46.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:46.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:46.274 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:46.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:45 smithi173 bash[59422]: cluster 2023-09-23T19:01:44.442138+0000 mgr.y (mgr.26054) 1314 : cluster 0 pgmap v1442: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 16030/153501 objects degraded (10.443%); 14/153501 objects misplaced (0.009%) 2023-09-23T19:01:46.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:45 smithi173 bash[59422]: audit 2023-09-23T19:01:44.937727+0000 mon.a (mon.0) 1803 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:01:46.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:45 smithi173 bash[59422]: cluster 2023-09-23T19:01:45.451241+0000 mon.a (mon.0) 1804 : cluster 3 Health check update: Degraded data redundancy: 16030/153501 objects degraded (10.443%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:46.283 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:01:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:01:46] "GET /metrics HTTP/1.1" 200 36641 "" "Prometheus/2.33.4" 2023-09-23T19:01:46.304 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:46.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:46.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:46.408 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:46.441 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:46.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:46.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:46.544 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:46.571 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:46.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:46.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:46.677 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:46.704 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:46.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:46.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:46.808 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:46.836 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:46.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:46.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:46.939 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:46.966 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:46.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:46.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:47.069 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:47.083 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 38 16 56142 56126 92.2992 65.8125 0.0124384 0.0108277 2023-09-23T19:01:47.109 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:47.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:47.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:47.215 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:47.246 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:47.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:47.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:47.349 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:47.374 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:47.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:47.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:47.477 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:47.504 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:47.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:47.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:47.607 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:47.633 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:47.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:47.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:47.736 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:47.758 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:47.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:47.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:47.860 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:47.884 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:47.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:47.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:47.937 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T19:01:47.989 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:47.997 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:47 smithi098 bash[76817]: cluster 2023-09-23T19:01:46.444586+0000 mgr.y (mgr.26054) 1315 : cluster 0 pgmap v1443: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 16030/154452 objects degraded (10.379%); 14/154452 objects misplaced (0.009%) 2023-09-23T19:01:47.997 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:47 smithi098 bash[79516]: cluster 2023-09-23T19:01:46.444586+0000 mgr.y (mgr.26054) 1315 : cluster 0 pgmap v1443: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 16030/154452 objects degraded (10.379%); 14/154452 objects misplaced (0.009%) 2023-09-23T19:01:48.016 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:48.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:48.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:48.083 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 39 16 57284 57268 91.7625 71.375 0.00400896 0.0108903 2023-09-23T19:01:48.109 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:01:48.110 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:01:48.120 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:48.128 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:47 smithi173 bash[59422]: cluster 2023-09-23T19:01:46.444586+0000 mgr.y (mgr.26054) 1315 : cluster 0 pgmap v1443: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 16030/154452 objects degraded (10.379%); 14/154452 objects misplaced (0.009%) 2023-09-23T19:01:48.138 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":1440,"stamp":"2023-09-23T19:01:42.441113+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3196524467,"num_objects":48730,"num_object_clones":0,"num_object_copies":146190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15119,"num_objects_misplaced":14,"num_objects_unfound":0,"num_objects_dirty":48730,"num_whiteouts":0,"num_read":6231,"num_read_kb":13690,"num_write":49808,"num_write_kb":3132087,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":136,"num_bytes_recovered":8396881,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":49872,"ondisk_log_size":49872,"up":234,"acting":255,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14131628,"kb_used_data":8331892,"kb_used_omap":31,"kb_used_meta":5799584,"kb_avail":642084436,"statfs":{"total":671965249536,"available":657494462464,"internally_reserved":0,"allocated":8531857408,"data_stored":8523895500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32507,"internal_metadata":5938774277},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1019412480,"num_objects":15555,"num_object_clones":0,"num_object_copies":46665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4704,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15555,"num_write_kb":995520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003642"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":27,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:45.617966+0000","last_change":"2023-09-23T18:59:13.709108+0000","last_active":"2023-09-23T19:00:45.617966+0000","last_peered":"2023-09-23T19:00:45.617966+0000","last_clean":"2023-09-23T19:00:45.617966+0000","last_became_active":"2023-09-23T18:59:13.708511+0000","last_became_peered":"2023-09-23T18:59:13.708511+0000","last_unstale":"2023-09-23T19:00:45.617966+0000","last_undegraded":"2023-09-23T19:00:45.617966+0000","last_fullsized":"2023-09-23T19:00:45.617966+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:59.295137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":42,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:04.679931+0000","last_change":"2023-09-23T18:59:13.702575+0000","last_active":"2023-09-23T19:01:04.679931+0000","last_peered":"2023-09-23T19:01:04.679931+0000","last_clean":"2023-09-23T19:01:04.679931+0000","last_became_active":"2023-09-23T18:59:13.701967+0000","last_became_peered":"2023-09-23T18:59:13.701967+0000","last_unstale":"2023-09-23T19:01:04.679931+0000","last_undegraded":"2023-09-23T19:01:04.679931+0000","last_fullsized":"2023-09-23T19:01:04.679931+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:30:12.982518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":308,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:45.617231+0000","last_change":"2023-09-23T18:59:13.707831+0000","last_active":"2023-09-23T19:00:45.617231+0000","last_peered":"2023-09-23T19:00:45.617231+0000","last_clean":"2023-09-23T19:00:45.617231+0000","last_became_active":"2023-09-23T18:59:13.707524+0000","last_became_peered":"2023-09-23T18:59:13.707524+0000","last_unstale":"2023-09-23T19:00:45.617231+0000","last_undegraded":"2023-09-23T19:00:45.617231+0000","last_fullsized":"2023-09-23T19:00:45.617231+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:42.677892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":336,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:46.541795+0000","last_change":"2023-09-23T18:55:23.980716+0000","last_active":"2023-09-23T19:00:46.541795+0000","last_peered":"2023-09-23T19:00:46.541795+0000","last_clean":"2023-09-23T19:00:46.541795+0000","last_became_active":"2023-09-23T18:55:23.980418+0000","last_became_peered":"2023-09-23T18:55:23.980418+0000","last_unstale":"2023-09-23T19:00:46.541795+0000","last_undegraded":"2023-09-23T19:00:46.541795+0000","last_fullsized":"2023-09-23T19:00:46.541795+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:23:14.283875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":33,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:04.678983+0000","last_change":"2023-09-23T18:58:38.718387+0000","last_active":"2023-09-23T19:01:04.678983+0000","last_peered":"2023-09-23T19:01:04.678983+0000","last_clean":"2023-09-23T19:01:04.678983+0000","last_became_active":"2023-09-23T18:58:38.717999+0000","last_became_peered":"2023-09-23T18:58:38.717999+0000","last_unstale":"2023-09-23T19:01:04.678983+0000","last_undegraded":"2023-09-23T19:01:04.678983+0000","last_fullsized":"2023-09-23T19:01:04.678983+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:05.073213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":50,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:46.540534+0000","last_change":"2023-09-23T18:55:23.983933+0000","last_active":"2023-09-23T19:00:46.540534+0000","last_peered":"2023-09-23T19:00:46.540534+0000","last_clean":"2023-09-23T19:00:46.540534+0000","last_became_active":"2023-09-23T18:55:23.982521+0000","last_became_peered":"2023-09-23T18:55:23.982521+0000","last_unstale":"2023-09-23T19:00:46.540534+0000","last_undegraded":"2023-09-23T19:00:46.540534+0000","last_fullsized":"2023-09-23T19:00:46.540534+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":68,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:46.540696+0000","last_change":"2023-09-23T18:58:38.729946+0000","last_active":"2023-09-23T19:00:46.540696+0000","last_peered":"2023-09-23T19:00:46.540696+0000","last_clean":"2023-09-23T19:00:46.540696+0000","last_became_active":"2023-09-23T18:58:38.729577+0000","last_became_peered":"2023-09-23T18:58:38.729577+0000","last_unstale":"2023-09-23T19:00:46.540696+0000","last_undegraded":"2023-09-23T19:00:46.540696+0000","last_fullsized":"2023-09-23T19:00:46.540696+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":33,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:01:02.063037+0000","last_change":"2023-09-23T18:58:38.710920+0000","last_active":"2023-09-23T19:01:02.063037+0000","last_peered":"2023-09-23T19:01:02.063037+0000","last_clean":"2023-09-23T19:01:02.063037+0000","last_became_active":"2023-09-23T18:58:38.710563+0000","last_became_peered":"2023-09-23T18:58:38.710563+0000","last_unstale":"2023-09-23T19:01:02.063037+0000","last_undegraded":"2023-09-23T19:01:02.063037+0000","last_fullsized":"2023-09-23T19:01:02.063037+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:42.386631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":286,"reported_epoch":319,"state":"active+undersized","last_fresh":"2023-09-23T19:00:45.618708+0000","last_change":"2023-09-23T18:54:52.347754+0000","last_active":"2023-09-23T19:00:45.618708+0000","last_peered":"2023-09-23T19:00:45.618708+0000","last_clean":"2023-09-23T18:40:38.019683+0000","last_became_active":"2023-09-23T18:54:52.347754+0000","last_became_peered":"2023-09-23T18:54:52.347754+0000","last_unstale":"2023-09-23T19:00:45.618708+0000","last_undegraded":"2023-09-23T19:00:45.618708+0000","last_fullsized":"2023-09-23T18:54:52.341415+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:00:13.660707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":299,"reported_epoch":319,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:01:02.063080+0000","last_change":"2023-09-23T18:58:39.716217+0000","last_active":"2023-09-23T19:01:02.063080+0000","last_peered":"2023-09-23T19:01:02.063080+0000","last_clean":"2023-09-23T19:01:02.063080+0000","last_became_active":"2023-09-23T18:58:39.715801+0000","last_became_peered":"2023-09-23T18:58:39.715801+0000","last_unstale":"2023-09-23T19:01:02.063080+0000","last_undegraded":"2023-09-23T19:01:02.063080+0000","last_fullsized":"2023-09-23T19:01:02.063080+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:00.811625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6,5],"avail_no_missing":["3","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":87,"reported_epoch":319,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:00:45.615649+0000","last_change":"2023-09-23T18:59:14.702243+0000","last_active":"2023-09-23T19:00:45.615649+0000","last_peered":"2023-09-23T19:00:45.615649+0000","last_clean":"2023-09-23T19:00:45.615649+0000","last_became_active":"2023-09-23T18:59:14.698763+0000","last_became_peered":"2023-09-23T18:59:14.698763+0000","last_unstale":"2023-09-23T19:00:45.615649+0000","last_undegraded":"2023-09-23T19:00:45.615649+0000","last_fullsized":"2023-09-23T19:00:45.615649+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:06.839110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0,4],"avail_no_missing":["3","0","4"],"object_location_counts":[{"shards":"0,3,4","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":262,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:45.617270+0000","last_change":"2023-09-23T18:33:53.850096+0000","last_active":"2023-09-23T19:00:45.617270+0000","last_peered":"2023-09-23T19:00:45.617270+0000","last_clean":"2023-09-23T19:00:45.617270+0000","last_became_active":"2023-09-23T18:33:53.849811+0000","last_became_peered":"2023-09-23T18:33:53.849811+0000","last_unstale":"2023-09-23T19:00:45.617270+0000","last_undegraded":"2023-09-23T19:00:45.617270+0000","last_fullsized":"2023-09-23T19:00:45.617270+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:44.938072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":38,"reported_epoch":320,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:01:04.681086+0000","last_change":"2023-09-23T18:59:14.708879+0000","last_active":"2023-09-23T19:01:04.681086+0000","last_peered":"2023-09-23T19:01:04.681086+0000","last_clean":"2023-09-23T19:01:04.681086+0000","last_became_active":"2023-09-23T18:59:14.691340+0000","last_became_peered":"2023-09-23T18:59:14.691340+0000","last_unstale":"2023-09-23T19:01:04.681086+0000","last_undegraded":"2023-09-23T19:01:04.681086+0000","last_fullsized":"2023-09-23T19:01:04.681086+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:59:30.307834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":4}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":260,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:05.265202+0000","last_change":"2023-09-23T18:59:13.702184+0000","last_active":"2023-09-23T19:01:05.265202+0000","last_peered":"2023-09-23T19:01:05.265202+0000","last_clean":"2023-09-23T19:01:05.265202+0000","last_became_active":"2023-09-23T18:59:13.701822+0000","last_became_peered":"2023-09-23T18:59:13.701822+0000","last_unstale":"2023-09-23T19:01:05.265202+0000","last_undegraded":"2023-09-23T19:01:05.265202+0000","last_fullsized":"2023-09-23T19:01:05.265202+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:23:12.358495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":66,"reported_epoch":320,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:01:04.681652+0000","last_change":"2023-09-23T18:59:14.698497+0000","last_active":"2023-09-23T19:01:04.681652+0000","last_peered":"2023-09-23T19:01:04.681652+0000","last_clean":"2023-09-23T19:01:04.681652+0000","last_became_active":"2023-09-23T18:59:14.691108+0000","last_became_peered":"2023-09-23T18:59:14.691108+0000","last_unstale":"2023-09-23T19:01:04.681652+0000","last_undegraded":"2023-09-23T19:01:04.681652+0000","last_fullsized":"2023-09-23T19:01:04.681652+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:24.885822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":324,"reported_epoch":319,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:01:02.063169+0000","last_change":"2023-09-23T18:58:39.772503+0000","last_active":"2023-09-23T19:01:02.063169+0000","last_peered":"2023-09-23T19:01:02.063169+0000","last_clean":"2023-09-23T19:01:02.063169+0000","last_became_active":"2023-09-23T18:58:39.771729+0000","last_became_peered":"2023-09-23T18:58:39.771729+0000","last_unstale":"2023-09-23T19:01:02.063169+0000","last_undegraded":"2023-09-23T19:01:02.063169+0000","last_fullsized":"2023-09-23T19:01:02.063169+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:02.866230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1,2],"avail_no_missing":["3","1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":176,"reported_epoch":319,"state":"active+undersized","last_fresh":"2023-09-23T19:00:45.618071+0000","last_change":"2023-09-23T18:54:52.352985+0000","last_active":"2023-09-23T19:00:45.618071+0000","last_peered":"2023-09-23T19:00:45.618071+0000","last_clean":"2023-09-23T18:39:19.101686+0000","last_became_active":"2023-09-23T18:54:52.352985+0000","last_became_peered":"2023-09-23T18:54:52.352985+0000","last_unstale":"2023-09-23T19:00:45.618071+0000","last_undegraded":"2023-09-23T19:00:45.618071+0000","last_fullsized":"2023-09-23T18:54:52.343063+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:20:21.326103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":247,"reported_epoch":319,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:00:45.615380+0000","last_change":"2023-09-23T18:58:39.714693+0000","last_active":"2023-09-23T19:00:45.615380+0000","last_peered":"2023-09-23T19:00:45.615380+0000","last_clean":"2023-09-23T19:00:45.615380+0000","last_became_active":"2023-09-23T18:58:39.712663+0000","last_became_peered":"2023-09-23T18:58:39.712663+0000","last_unstale":"2023-09-23T19:00:45.615380+0000","last_undegraded":"2023-09-23T19:00:45.615380+0000","last_fullsized":"2023-09-23T19:00:45.615380+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:01.567495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6,0],"avail_no_missing":["3","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":28,"reported_epoch":320,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:01:05.267729+0000","last_change":"2023-09-23T18:59:14.695207+0000","last_active":"2023-09-23T19:01:05.267729+0000","last_peered":"2023-09-23T19:01:05.267729+0000","last_clean":"2023-09-23T19:01:05.267729+0000","last_became_active":"2023-09-23T18:59:14.693618+0000","last_became_peered":"2023-09-23T18:59:14.693618+0000","last_unstale":"2023-09-23T19:01:05.267729+0000","last_undegraded":"2023-09-23T19:01:05.267729+0000","last_fullsized":"2023-09-23T19:01:05.267729+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:46.279818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,4],"avail_no_missing":["0","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":33,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:04.679462+0000","last_change":"2023-09-23T18:58:38.714235+0000","last_active":"2023-09-23T19:01:04.679462+0000","last_peered":"2023-09-23T19:01:04.679462+0000","last_clean":"2023-09-23T19:01:04.679462+0000","last_became_active":"2023-09-23T18:58:38.713901+0000","last_became_peered":"2023-09-23T18:58:38.713901+0000","last_unstale":"2023-09-23T19:01:04.679462+0000","last_undegraded":"2023-09-23T19:01:04.679462+0000","last_fullsized":"2023-09-23T19:01:04.679462+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:46.862309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":168,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:46.540158+0000","last_change":"2023-09-23T18:58:38.793506+0000","last_active":"2023-09-23T19:00:46.540158+0000","last_peered":"2023-09-23T19:00:46.540158+0000","last_clean":"2023-09-23T19:00:46.540158+0000","last_became_active":"2023-09-23T18:58:38.706537+0000","last_became_peered":"2023-09-23T18:58:38.706537+0000","last_unstale":"2023-09-23T19:00:46.540158+0000","last_undegraded":"2023-09-23T19:00:46.540158+0000","last_fullsized":"2023-09-23T19:00:46.540158+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:59.240510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":270,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:45.619153+0000","last_change":"2023-09-23T18:40:42.058656+0000","last_active":"2023-09-23T19:00:45.619153+0000","last_peered":"2023-09-23T19:00:45.619153+0000","last_clean":"2023-09-23T19:00:45.619153+0000","last_became_active":"2023-09-23T18:40:42.057307+0000","last_became_peered":"2023-09-23T18:40:42.057307+0000","last_unstale":"2023-09-23T19:00:45.619153+0000","last_undegraded":"2023-09-23T19:00:45.619153+0000","last_fullsized":"2023-09-23T19:00:45.619153+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:08:45.979078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.6","version":"320'3166","reported_seq":3177,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:40.818063+0000","last_change":"2023-09-23T19:00:44.628316+0000","last_active":"2023-09-23T19:01:40.818063+0000","last_peered":"2023-09-23T19:01:40.818063+0000","last_clean":"2023-09-23T19:01:40.818063+0000","last_became_active":"2023-09-23T19:00:44.627963+0000","last_became_peered":"2023-09-23T19:00:44.627963+0000","last_unstale":"2023-09-23T19:01:40.818063+0000","last_undegraded":"2023-09-23T19:01:40.818063+0000","last_fullsized":"2023-09-23T19:01:40.818063+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":3166,"log_dups_size":0,"ondisk_log_size":3166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207486976,"num_objects":3166,"num_object_clones":0,"num_object_copies":9498,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3166,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3166,"num_write_kb":202624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":44,"reported_epoch":320,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:01:05.268615+0000","last_change":"2023-09-23T18:59:14.728684+0000","last_active":"2023-09-23T19:01:05.268615+0000","last_peered":"2023-09-23T19:01:05.268615+0000","last_clean":"2023-09-23T19:01:05.268615+0000","last_became_active":"2023-09-23T18:59:14.720265+0000","last_became_peered":"2023-09-23T18:59:14.720265+0000","last_unstale":"2023-09-23T19:01:05.268615+0000","last_undegraded":"2023-09-23T19:01:05.268615+0000","last_fullsized":"2023-09-23T19:01:05.268615+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:03.133014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,2],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":28,"reported_epoch":320,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:01:04.678613+0000","last_change":"2023-09-23T18:59:14.698036+0000","last_active":"2023-09-23T19:01:04.678613+0000","last_peered":"2023-09-23T19:01:04.678613+0000","last_clean":"2023-09-23T19:01:04.678613+0000","last_became_active":"2023-09-23T18:59:14.697547+0000","last_became_peered":"2023-09-23T18:59:14.697547+0000","last_unstale":"2023-09-23T19:01:04.678613+0000","last_undegraded":"2023-09-23T19:01:04.678613+0000","last_fullsized":"2023-09-23T19:01:04.678613+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:56:32.259159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6,4],"avail_no_missing":["3","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.7","version":"320'3087","reported_seq":3099,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:41.756887+0000","last_change":"2023-09-23T19:00:44.634020+0000","last_active":"2023-09-23T19:01:41.756887+0000","last_peered":"2023-09-23T19:01:41.756887+0000","last_clean":"2023-09-23T19:01:41.756887+0000","last_became_active":"2023-09-23T19:00:44.633701+0000","last_became_peered":"2023-09-23T19:00:44.633701+0000","last_unstale":"2023-09-23T19:01:41.756887+0000","last_undegraded":"2023-09-23T19:01:41.756887+0000","last_fullsized":"2023-09-23T19:01:41.756887+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":3087,"log_dups_size":0,"ondisk_log_size":3087,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.208476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202309632,"num_objects":3087,"num_object_clones":0,"num_object_copies":9261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3087,"num_write_kb":197568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":64,"reported_epoch":320,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:01:05.266718+0000","last_change":"2023-09-23T18:59:14.845077+0000","last_active":"2023-09-23T19:01:05.266718+0000","last_peered":"2023-09-23T19:01:05.266718+0000","last_clean":"2023-09-23T19:01:05.266718+0000","last_became_active":"2023-09-23T18:59:14.731245+0000","last_became_peered":"2023-09-23T18:59:14.731245+0000","last_unstale":"2023-09-23T19:01:05.266718+0000","last_undegraded":"2023-09-23T19:01:05.266718+0000","last_fullsized":"2023-09-23T19:01:05.266718+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:36:54.746484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0,4],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":281,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:05.265160+0000","last_change":"2023-09-23T18:59:13.706094+0000","last_active":"2023-09-23T19:01:05.265160+0000","last_peered":"2023-09-23T19:01:05.265160+0000","last_clean":"2023-09-23T19:01:05.265160+0000","last_became_active":"2023-09-23T18:59:13.705831+0000","last_became_peered":"2023-09-23T18:59:13.705831+0000","last_unstale":"2023-09-23T19:01:05.265160+0000","last_undegraded":"2023-09-23T19:01:05.265160+0000","last_fullsized":"2023-09-23T19:01:05.265160+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:11.072246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"320'2914","reported_seq":2926,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:38.301257+0000","last_change":"2023-09-23T19:00:44.631802+0000","last_active":"2023-09-23T19:01:38.301257+0000","last_peered":"2023-09-23T19:01:38.301257+0000","last_clean":"2023-09-23T19:01:38.301257+0000","last_became_active":"2023-09-23T19:00:44.631445+0000","last_became_peered":"2023-09-23T19:00:44.631445+0000","last_unstale":"2023-09-23T19:01:38.301257+0000","last_undegraded":"2023-09-23T19:01:38.301257+0000","last_fullsized":"2023-09-23T19:01:38.301257+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":2914,"log_dups_size":0,"ondisk_log_size":2914,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:19.854112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190971904,"num_objects":2914,"num_object_clones":0,"num_object_copies":8742,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2914,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2914,"num_write_kb":186496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":286,"reported_epoch":320,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:01:04.679780+0000","last_change":"2023-09-23T18:58:39.713533+0000","last_active":"2023-09-23T19:01:04.679780+0000","last_peered":"2023-09-23T19:01:04.679780+0000","last_clean":"2023-09-23T19:01:04.679780+0000","last_became_active":"2023-09-23T18:58:39.711851+0000","last_became_peered":"2023-09-23T18:58:39.711851+0000","last_unstale":"2023-09-23T19:01:04.679780+0000","last_undegraded":"2023-09-23T19:01:04.679780+0000","last_fullsized":"2023-09-23T19:01:04.679780+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:33.450793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":172,"reported_epoch":320,"state":"active+undersized","last_fresh":"2023-09-23T19:01:04.680678+0000","last_change":"2023-09-23T18:54:52.349594+0000","last_active":"2023-09-23T19:01:04.680678+0000","last_peered":"2023-09-23T19:01:04.680678+0000","last_clean":"2023-09-23T18:54:43.304423+0000","last_became_active":"2023-09-23T18:54:52.349594+0000","last_became_peered":"2023-09-23T18:54:52.349594+0000","last_unstale":"2023-09-23T19:01:04.680678+0000","last_undegraded":"2023-09-23T19:01:04.680678+0000","last_fullsized":"2023-09-23T18:54:52.338675+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:49.547901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.5","version":"320'3152","reported_seq":3162,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:41.750419+0000","last_change":"2023-09-23T19:00:44.628649+0000","last_active":"2023-09-23T19:01:41.750419+0000","last_peered":"2023-09-23T19:01:41.750419+0000","last_clean":"2023-09-23T19:01:41.750419+0000","last_became_active":"2023-09-23T19:00:44.628304+0000","last_became_peered":"2023-09-23T19:00:44.628304+0000","last_unstale":"2023-09-23T19:01:41.750419+0000","last_undegraded":"2023-09-23T19:01:41.750419+0000","last_fullsized":"2023-09-23T19:01:41.750419+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":3152,"log_dups_size":0,"ondisk_log_size":3152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:59.345981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206569472,"num_objects":3152,"num_object_clones":0,"num_object_copies":9456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3152,"num_write_kb":201728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":68,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:46.540237+0000","last_change":"2023-09-23T18:58:38.728749+0000","last_active":"2023-09-23T19:00:46.540237+0000","last_peered":"2023-09-23T19:00:46.540237+0000","last_clean":"2023-09-23T19:00:46.540237+0000","last_became_active":"2023-09-23T18:58:38.728177+0000","last_became_peered":"2023-09-23T18:58:38.728177+0000","last_unstale":"2023-09-23T19:00:46.540237+0000","last_undegraded":"2023-09-23T19:00:46.540237+0000","last_fullsized":"2023-09-23T19:00:46.540237+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":319,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:46.540878+0000","last_change":"2023-09-23T18:59:13.690091+0000","last_active":"2023-09-23T19:00:46.540878+0000","last_peered":"2023-09-23T19:00:46.540878+0000","last_clean":"2023-09-23T19:00:46.540878+0000","last_became_active":"2023-09-23T18:59:13.689532+0000","last_became_peered":"2023-09-23T18:59:13.689532+0000","last_unstale":"2023-09-23T19:00:46.540878+0000","last_undegraded":"2023-09-23T19:00:46.540878+0000","last_fullsized":"2023-09-23T19:00:46.540878+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:09.886527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"320'2999","reported_seq":3011,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:38.269338+0000","last_change":"2023-09-23T19:00:44.625986+0000","last_active":"2023-09-23T19:01:38.269338+0000","last_peered":"2023-09-23T19:01:38.269338+0000","last_clean":"2023-09-23T19:01:38.269338+0000","last_became_active":"2023-09-23T19:00:44.625761+0000","last_became_peered":"2023-09-23T19:00:44.625761+0000","last_unstale":"2023-09-23T19:01:38.269338+0000","last_undegraded":"2023-09-23T19:01:38.269338+0000","last_fullsized":"2023-09-23T19:01:38.269338+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":2999,"log_dups_size":0,"ondisk_log_size":2999,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:11:32.923241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196542464,"num_objects":2999,"num_object_clones":0,"num_object_copies":8997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2999,"num_write_kb":191936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":50,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:57.062257+0000","last_change":"2023-09-23T18:59:13.721533+0000","last_active":"2023-09-23T19:00:57.062257+0000","last_peered":"2023-09-23T19:00:57.062257+0000","last_clean":"2023-09-23T19:00:57.062257+0000","last_became_active":"2023-09-23T18:59:13.711121+0000","last_became_peered":"2023-09-23T18:59:13.711121+0000","last_unstale":"2023-09-23T19:00:57.062257+0000","last_undegraded":"2023-09-23T19:00:57.062257+0000","last_fullsized":"2023-09-23T19:00:57.062257+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:51.950774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":253,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:04.679552+0000","last_change":"2023-09-23T18:59:13.702825+0000","last_active":"2023-09-23T19:01:04.679552+0000","last_peered":"2023-09-23T19:01:04.679552+0000","last_clean":"2023-09-23T19:01:04.679552+0000","last_became_active":"2023-09-23T18:59:13.702503+0000","last_became_peered":"2023-09-23T18:59:13.702503+0000","last_unstale":"2023-09-23T19:01:04.679552+0000","last_undegraded":"2023-09-23T19:01:04.679552+0000","last_fullsized":"2023-09-23T19:01:04.679552+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.263337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.3","version":"320'3155","reported_seq":3167,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:41.754652+0000","last_change":"2023-09-23T19:00:44.627952+0000","last_active":"2023-09-23T19:01:41.754652+0000","last_peered":"2023-09-23T19:01:41.754652+0000","last_clean":"2023-09-23T19:01:41.754652+0000","last_became_active":"2023-09-23T19:00:44.627588+0000","last_became_peered":"2023-09-23T19:00:44.627588+0000","last_unstale":"2023-09-23T19:01:41.754652+0000","last_undegraded":"2023-09-23T19:01:41.754652+0000","last_fullsized":"2023-09-23T19:01:41.754652+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":3155,"log_dups_size":0,"ondisk_log_size":3155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:53.912525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206766080,"num_objects":3155,"num_object_clones":0,"num_object_copies":9465,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3155,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3155,"num_write_kb":201920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":157,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:05.265249+0000","last_change":"2023-09-23T18:59:13.708841+0000","last_active":"2023-09-23T19:01:05.265249+0000","last_peered":"2023-09-23T19:01:05.265249+0000","last_clean":"2023-09-23T19:01:05.265249+0000","last_became_active":"2023-09-23T18:59:13.707760+0000","last_became_peered":"2023-09-23T18:59:13.707760+0000","last_unstale":"2023-09-23T19:01:05.265249+0000","last_undegraded":"2023-09-23T19:01:05.265249+0000","last_fullsized":"2023-09-23T19:01:05.265249+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:02.417127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":307,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:04.681972+0000","last_change":"2023-09-23T18:59:13.708710+0000","last_active":"2023-09-23T19:01:04.681972+0000","last_peered":"2023-09-23T19:01:04.681972+0000","last_clean":"2023-09-23T19:01:04.681972+0000","last_became_active":"2023-09-23T18:59:13.708349+0000","last_became_peered":"2023-09-23T18:59:13.708349+0000","last_unstale":"2023-09-23T19:01:04.681972+0000","last_undegraded":"2023-09-23T19:01:04.681972+0000","last_fullsized":"2023-09-23T19:01:04.681972+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:33.033640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"320'2890","reported_seq":2899,"reported_epoch":320,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:01:38.219346+0000","last_change":"2023-09-23T19:01:09.091230+0000","last_active":"2023-09-23T19:01:38.219346+0000","last_peered":"2023-09-23T19:01:38.219346+0000","last_clean":"2023-09-23T19:00:43.602956+0000","last_became_active":"2023-09-23T19:00:44.621693+0000","last_became_peered":"2023-09-23T19:00:44.621693+0000","last_unstale":"2023-09-23T19:01:38.219346+0000","last_undegraded":"2023-09-23T19:01:09.090961+0000","last_fullsized":"2023-09-23T19:00:44.616332+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":2890,"log_dups_size":0,"ondisk_log_size":2890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:50.488964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189399040,"num_objects":2890,"num_object_clones":0,"num_object_copies":8670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2890,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2890,"num_write_kb":184960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":2890}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":222,"reported_epoch":320,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:01:04.680944+0000","last_change":"2023-09-23T18:59:14.692414+0000","last_active":"2023-09-23T19:01:04.680944+0000","last_peered":"2023-09-23T19:01:04.680944+0000","last_clean":"2023-09-23T19:01:04.680944+0000","last_became_active":"2023-09-23T18:59:14.692033+0000","last_became_peered":"2023-09-23T18:59:14.692033+0000","last_unstale":"2023-09-23T19:01:04.680944+0000","last_undegraded":"2023-09-23T19:01:04.680944+0000","last_fullsized":"2023-09-23T19:01:04.680944+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:13.314042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1,4],"avail_no_missing":["2","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":260,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:05.266165+0000","last_change":"2023-09-23T18:59:13.708690+0000","last_active":"2023-09-23T19:01:05.266165+0000","last_peered":"2023-09-23T19:01:05.266165+0000","last_clean":"2023-09-23T19:01:05.266165+0000","last_became_active":"2023-09-23T18:59:13.708248+0000","last_became_peered":"2023-09-23T18:59:13.708248+0000","last_unstale":"2023-09-23T19:01:05.266165+0000","last_undegraded":"2023-09-23T19:01:05.266165+0000","last_fullsized":"2023-09-23T19:01:05.266165+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:43:08.360496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.1","version":"320'2895","reported_seq":2904,"reported_epoch":320,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:01:38.247077+0000","last_change":"2023-09-23T19:01:09.129799+0000","last_active":"2023-09-23T19:01:38.247077+0000","last_peered":"2023-09-23T19:01:38.247077+0000","last_clean":"2023-09-23T19:00:43.602956+0000","last_became_active":"2023-09-23T19:00:44.625145+0000","last_became_peered":"2023-09-23T19:00:44.625145+0000","last_unstale":"2023-09-23T19:01:38.247077+0000","last_undegraded":"2023-09-23T19:01:09.094661+0000","last_fullsized":"2023-09-23T19:00:44.616356+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":2895,"log_dups_size":0,"ondisk_log_size":2895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:11.841704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189726720,"num_objects":2895,"num_object_clones":0,"num_object_copies":8685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2895,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2895,"num_write_kb":185280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":2895}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":307,"reported_epoch":320,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:01:05.265269+0000","last_change":"2023-09-23T18:59:14.722830+0000","last_active":"2023-09-23T19:01:05.265269+0000","last_peered":"2023-09-23T19:01:05.265269+0000","last_clean":"2023-09-23T19:01:05.265269+0000","last_became_active":"2023-09-23T18:59:14.721252+0000","last_became_peered":"2023-09-23T18:59:14.721252+0000","last_unstale":"2023-09-23T19:01:05.265269+0000","last_undegraded":"2023-09-23T19:01:05.265269+0000","last_fullsized":"2023-09-23T19:01:05.265269+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:30.339067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5,4],"avail_no_missing":["1","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":312,"reported_epoch":319,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:01:03.259568+0000","last_change":"2023-09-23T18:59:14.719934+0000","last_active":"2023-09-23T19:01:03.259568+0000","last_peered":"2023-09-23T19:01:03.259568+0000","last_clean":"2023-09-23T19:01:03.259568+0000","last_became_active":"2023-09-23T18:59:14.719375+0000","last_became_peered":"2023-09-23T18:59:14.719375+0000","last_unstale":"2023-09-23T19:01:03.259568+0000","last_undegraded":"2023-09-23T19:01:03.259568+0000","last_fullsized":"2023-09-23T19:01:03.259568+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:20.953751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3,4],"avail_no_missing":["1","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"320'2989","reported_seq":2998,"reported_epoch":320,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:01:38.279336+0000","last_change":"2023-09-23T19:01:09.088024+0000","last_active":"2023-09-23T19:01:38.279336+0000","last_peered":"2023-09-23T19:01:38.279336+0000","last_clean":"2023-09-23T19:00:43.602956+0000","last_became_active":"2023-09-23T19:00:44.628387+0000","last_became_peered":"2023-09-23T19:00:44.628387+0000","last_unstale":"2023-09-23T19:01:38.279336+0000","last_undegraded":"2023-09-23T19:01:09.079699+0000","last_fullsized":"2023-09-23T19:00:44.623577+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":2989,"log_dups_size":0,"ondisk_log_size":2989,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:59.837487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195887104,"num_objects":2989,"num_object_clones":0,"num_object_copies":8967,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2989,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2989,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2989,"num_write_kb":191296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":["0","6"],"object_location_counts":[{"shards":"0,6","objects":2989}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"77'130","reported_seq":800,"reported_epoch":319,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:01:03.259425+0000","last_change":"2023-09-23T18:54:52.351403+0000","last_active":"2023-09-23T19:01:03.259425+0000","last_peered":"2023-09-23T19:01:03.259425+0000","last_clean":"2023-09-23T18:54:47.819753+0000","last_became_active":"2023-09-23T18:54:52.351403+0000","last_became_peered":"2023-09-23T18:54:52.351403+0000","last_unstale":"2023-09-23T19:01:03.259425+0000","last_undegraded":"2023-09-23T18:54:52.342482+0000","last_fullsized":"2023-09-23T18:54:52.342376+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:54:59.818497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":772,"num_read_kb":1712,"num_write":126,"num_write_kb":2126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":68,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:46.540322+0000","last_change":"2023-09-23T18:58:38.774473+0000","last_active":"2023-09-23T19:00:46.540322+0000","last_peered":"2023-09-23T19:00:46.540322+0000","last_clean":"2023-09-23T19:00:46.540322+0000","last_became_active":"2023-09-23T18:58:38.773907+0000","last_became_peered":"2023-09-23T18:58:38.773907+0000","last_unstale":"2023-09-23T19:00:46.540322+0000","last_undegraded":"2023-09-23T19:00:46.540322+0000","last_fullsized":"2023-09-23T19:00:46.540322+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":284,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:58.258440+0000","last_change":"2023-09-23T18:59:13.704997+0000","last_active":"2023-09-23T19:00:58.258440+0000","last_peered":"2023-09-23T19:00:58.258440+0000","last_clean":"2023-09-23T19:00:58.258440+0000","last_became_active":"2023-09-23T18:59:13.703982+0000","last_became_peered":"2023-09-23T18:59:13.703982+0000","last_unstale":"2023-09-23T19:00:58.258440+0000","last_undegraded":"2023-09-23T19:00:58.258440+0000","last_fullsized":"2023-09-23T19:00:58.258440+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:43.668862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"320'2956","reported_seq":2968,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:38.305731+0000","last_change":"2023-09-23T19:00:44.627995+0000","last_active":"2023-09-23T19:01:38.305731+0000","last_peered":"2023-09-23T19:01:38.305731+0000","last_clean":"2023-09-23T19:01:38.305731+0000","last_became_active":"2023-09-23T19:00:44.627677+0000","last_became_peered":"2023-09-23T19:00:44.627677+0000","last_unstale":"2023-09-23T19:01:38.305731+0000","last_undegraded":"2023-09-23T19:01:38.305731+0000","last_fullsized":"2023-09-23T19:01:38.305731+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":2956,"log_dups_size":0,"ondisk_log_size":2956,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:47.360699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193724416,"num_objects":2956,"num_object_clones":0,"num_object_copies":8868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2956,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2956,"num_write_kb":189184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'96","reported_seq":72,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:46.540047+0000","last_change":"2023-09-23T18:55:23.993895+0000","last_active":"2023-09-23T19:00:46.540047+0000","last_peered":"2023-09-23T19:00:46.540047+0000","last_clean":"2023-09-23T19:00:46.540047+0000","last_became_active":"2023-09-23T18:55:23.982623+0000","last_became_peered":"2023-09-23T18:55:23.982623+0000","last_unstale":"2023-09-23T19:00:46.540047+0000","last_undegraded":"2023-09-23T19:00:46.540047+0000","last_fullsized":"2023-09-23T19:00:46.540047+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":96,"log_dups_size":0,"ondisk_log_size":96,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":728,"num_read_kb":1234,"num_write":87,"num_write_kb":338,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":27,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:05.265637+0000","last_change":"2023-09-23T18:59:13.703057+0000","last_active":"2023-09-23T19:01:05.265637+0000","last_peered":"2023-09-23T19:01:05.265637+0000","last_clean":"2023-09-23T19:01:05.265637+0000","last_became_active":"2023-09-23T18:59:13.702759+0000","last_became_peered":"2023-09-23T18:59:13.702759+0000","last_unstale":"2023-09-23T19:01:05.265637+0000","last_undegraded":"2023-09-23T19:01:05.265637+0000","last_fullsized":"2023-09-23T19:01:05.265637+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:53.278738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":329,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:01:03.259395+0000","last_change":"2023-09-23T18:58:38.726546+0000","last_active":"2023-09-23T19:01:03.259395+0000","last_peered":"2023-09-23T19:01:03.259395+0000","last_clean":"2023-09-23T19:01:03.259395+0000","last_became_active":"2023-09-23T18:58:38.724765+0000","last_became_peered":"2023-09-23T18:58:38.724765+0000","last_unstale":"2023-09-23T19:01:03.259395+0000","last_undegraded":"2023-09-23T19:01:03.259395+0000","last_fullsized":"2023-09-23T19:01:03.259395+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:05.322676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"320'3128","reported_seq":3137,"reported_epoch":320,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:01:41.769188+0000","last_change":"2023-09-23T19:01:09.086578+0000","last_active":"2023-09-23T19:01:41.769188+0000","last_peered":"2023-09-23T19:01:41.769188+0000","last_clean":"2023-09-23T19:00:43.602956+0000","last_became_active":"2023-09-23T19:00:44.627664+0000","last_became_peered":"2023-09-23T19:00:44.627664+0000","last_unstale":"2023-09-23T19:01:41.769188+0000","last_undegraded":"2023-09-23T19:01:09.086377+0000","last_fullsized":"2023-09-23T19:00:44.618743+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":3128,"log_dups_size":0,"ondisk_log_size":3128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:18.420065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204996608,"num_objects":3128,"num_object_clones":0,"num_object_copies":9384,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3128,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3128,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3128,"num_write_kb":200192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":3128}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"77'112","reported_seq":766,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:46.538597+0000","last_change":"2023-09-23T18:58:38.708598+0000","last_active":"2023-09-23T19:00:46.538597+0000","last_peered":"2023-09-23T19:00:46.538597+0000","last_clean":"2023-09-23T19:00:46.538597+0000","last_became_active":"2023-09-23T18:58:38.707923+0000","last_became_peered":"2023-09-23T18:58:38.707923+0000","last_unstale":"2023-09-23T19:00:46.538597+0000","last_undegraded":"2023-09-23T19:00:46.538597+0000","last_fullsized":"2023-09-23T19:00:46.538597+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:59.471903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":774,"num_read_kb":596,"num_write":102,"num_write_kb":52,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":264,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:45.617148+0000","last_change":"2023-09-23T18:33:53.849084+0000","last_active":"2023-09-23T19:00:45.617148+0000","last_peered":"2023-09-23T19:00:45.617148+0000","last_clean":"2023-09-23T19:00:45.617148+0000","last_became_active":"2023-09-23T18:33:53.848757+0000","last_became_peered":"2023-09-23T18:33:53.848757+0000","last_unstale":"2023-09-23T19:00:45.617148+0000","last_undegraded":"2023-09-23T19:00:45.617148+0000","last_fullsized":"2023-09-23T19:00:45.617148+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:15.267742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":319,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:45.619015+0000","last_change":"2023-09-23T18:55:23.977983+0000","last_active":"2023-09-23T19:00:45.619015+0000","last_peered":"2023-09-23T19:00:45.619015+0000","last_clean":"2023-09-23T19:00:45.619015+0000","last_became_active":"2023-09-23T18:55:23.977510+0000","last_became_peered":"2023-09-23T18:55:23.977510+0000","last_unstale":"2023-09-23T19:00:45.619015+0000","last_undegraded":"2023-09-23T19:00:45.619015+0000","last_fullsized":"2023-09-23T19:00:45.619015+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:30:14.678512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"320'2969","reported_seq":2981,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:38.260651+0000","last_change":"2023-09-23T19:00:44.626971+0000","last_active":"2023-09-23T19:01:38.260651+0000","last_peered":"2023-09-23T19:01:38.260651+0000","last_clean":"2023-09-23T19:01:38.260651+0000","last_became_active":"2023-09-23T19:00:44.626715+0000","last_became_peered":"2023-09-23T19:00:44.626715+0000","last_unstale":"2023-09-23T19:01:38.260651+0000","last_undegraded":"2023-09-23T19:01:38.260651+0000","last_fullsized":"2023-09-23T19:01:38.260651+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":2969,"log_dups_size":0,"ondisk_log_size":2969,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:22:07.209151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194576384,"num_objects":2969,"num_object_clones":0,"num_object_copies":8907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2969,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2969,"num_write_kb":190016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'147","reported_seq":706,"reported_epoch":319,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:00:46.540484+0000","last_change":"2023-09-23T18:55:24.959884+0000","last_active":"2023-09-23T19:00:46.540484+0000","last_peered":"2023-09-23T19:00:46.540484+0000","last_clean":"2023-09-23T19:00:46.540484+0000","last_became_active":"2023-09-23T18:55:24.959050+0000","last_became_peered":"2023-09-23T18:55:24.959050+0000","last_unstale":"2023-09-23T19:00:46.540484+0000","last_undegraded":"2023-09-23T19:00:46.540484+0000","last_fullsized":"2023-09-23T19:00:46.540484+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":624,"num_read_kb":1868,"num_write":143,"num_write_kb":5368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2,3],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":330,"reported_epoch":319,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:00:58.260778+0000","last_change":"2023-09-23T18:54:52.350132+0000","last_active":"2023-09-23T19:00:58.260778+0000","last_peered":"2023-09-23T19:00:58.260778+0000","last_clean":"2023-09-23T18:21:39.626426+0000","last_became_active":"2023-09-23T18:54:52.350132+0000","last_became_peered":"2023-09-23T18:54:52.350132+0000","last_unstale":"2023-09-23T19:00:58.260778+0000","last_undegraded":"2023-09-23T18:54:52.335600+0000","last_fullsized":"2023-09-23T18:54:52.335485+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_clean_scrub_stamp":"2023-09-23T18:08:35.701374+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:27.569665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":["0","6"],"object_location_counts":[{"shards":"0,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":42,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:47.060317+0000","last_change":"2023-09-23T18:59:13.705466+0000","last_active":"2023-09-23T19:00:47.060317+0000","last_peered":"2023-09-23T19:00:47.060317+0000","last_clean":"2023-09-23T19:00:47.060317+0000","last_became_active":"2023-09-23T18:59:13.705150+0000","last_became_peered":"2023-09-23T18:59:13.705150+0000","last_unstale":"2023-09-23T19:00:47.060317+0000","last_undegraded":"2023-09-23T19:00:47.060317+0000","last_fullsized":"2023-09-23T19:00:47.060317+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:09.675922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":28,"reported_epoch":320,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:01:05.268335+0000","last_change":"2023-09-23T18:59:14.692323+0000","last_active":"2023-09-23T19:01:05.268335+0000","last_peered":"2023-09-23T19:01:05.268335+0000","last_clean":"2023-09-23T19:01:05.268335+0000","last_became_active":"2023-09-23T18:59:14.691887+0000","last_became_peered":"2023-09-23T18:59:14.691887+0000","last_unstale":"2023-09-23T19:01:05.268335+0000","last_undegraded":"2023-09-23T19:01:05.268335+0000","last_fullsized":"2023-09-23T19:01:05.268335+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:30.126169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,4],"avail_no_missing":["0","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.8","version":"320'3214","reported_seq":3223,"reported_epoch":320,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:01:41.723665+0000","last_change":"2023-09-23T19:01:09.082767+0000","last_active":"2023-09-23T19:01:41.723665+0000","last_peered":"2023-09-23T19:01:41.723665+0000","last_clean":"2023-09-23T19:00:43.602956+0000","last_became_active":"2023-09-23T19:00:44.629456+0000","last_became_peered":"2023-09-23T19:00:44.629456+0000","last_unstale":"2023-09-23T19:01:41.723665+0000","last_undegraded":"2023-09-23T19:01:09.082605+0000","last_fullsized":"2023-09-23T19:00:44.620975+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":3214,"log_dups_size":0,"ondisk_log_size":3214,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:26.773324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210632704,"num_objects":3214,"num_object_clones":0,"num_object_copies":9642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3214,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3214,"num_write_kb":205696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":3214}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"77'120","reported_seq":550,"reported_epoch":319,"state":"active+undersized","last_fresh":"2023-09-23T19:00:58.258261+0000","last_change":"2023-09-23T18:54:52.356024+0000","last_active":"2023-09-23T19:00:58.258261+0000","last_peered":"2023-09-23T19:00:58.258261+0000","last_clean":"2023-09-23T18:17:46.566093+0000","last_became_active":"2023-09-23T18:54:52.356024+0000","last_became_peered":"2023-09-23T18:54:52.356024+0000","last_unstale":"2023-09-23T19:00:58.258261+0000","last_undegraded":"2023-09-23T19:00:58.258261+0000","last_fullsized":"2023-09-23T18:54:52.345483+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":120,"log_dups_size":0,"ondisk_log_size":120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:09.374165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":809,"num_read_kb":2749,"num_write":98,"num_write_kb":2087,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":346,"reported_epoch":319,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:00:46.540518+0000","last_change":"2023-09-23T18:59:14.695332+0000","last_active":"2023-09-23T19:00:46.540518+0000","last_peered":"2023-09-23T19:00:46.540518+0000","last_clean":"2023-09-23T19:00:46.540518+0000","last_became_active":"2023-09-23T18:59:14.694467+0000","last_became_peered":"2023-09-23T18:59:14.694467+0000","last_unstale":"2023-09-23T19:00:46.540518+0000","last_undegraded":"2023-09-23T19:00:46.540518+0000","last_fullsized":"2023-09-23T19:00:46.540518+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:39.284652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5,4],"avail_no_missing":["0","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":350,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:05.268656+0000","last_change":"2023-09-23T18:51:43.087834+0000","last_active":"2023-09-23T19:01:05.268656+0000","last_peered":"2023-09-23T19:01:05.268656+0000","last_clean":"2023-09-23T19:01:05.268656+0000","last_became_active":"2023-09-23T18:51:43.087309+0000","last_became_peered":"2023-09-23T18:51:43.087309+0000","last_unstale":"2023-09-23T19:01:05.268656+0000","last_undegraded":"2023-09-23T19:01:05.268656+0000","last_fullsized":"2023-09-23T19:01:05.268656+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:53.913425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"320'2901","reported_seq":2913,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:38.306879+0000","last_change":"2023-09-23T19:00:44.631488+0000","last_active":"2023-09-23T19:01:38.306879+0000","last_peered":"2023-09-23T19:01:38.306879+0000","last_clean":"2023-09-23T19:01:38.306879+0000","last_became_active":"2023-09-23T19:00:44.628764+0000","last_became_peered":"2023-09-23T19:00:44.628764+0000","last_unstale":"2023-09-23T19:01:38.306879+0000","last_undegraded":"2023-09-23T19:01:38.306879+0000","last_fullsized":"2023-09-23T19:01:38.306879+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":2901,"log_dups_size":0,"ondisk_log_size":2901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:01.178133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190119936,"num_objects":2901,"num_object_clones":0,"num_object_copies":8703,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2901,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2901,"num_write_kb":185664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'78","reported_seq":388,"reported_epoch":320,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:01:05.268511+0000","last_change":"2023-09-23T18:59:14.857977+0000","last_active":"2023-09-23T19:01:05.268511+0000","last_peered":"2023-09-23T19:01:05.268511+0000","last_clean":"2023-09-23T19:01:05.268511+0000","last_became_active":"2023-09-23T18:59:14.720066+0000","last_became_peered":"2023-09-23T18:59:14.720066+0000","last_unstale":"2023-09-23T19:01:05.268511+0000","last_undegraded":"2023-09-23T19:01:05.268511+0000","last_fullsized":"2023-09-23T19:01:05.268511+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":78,"log_dups_size":0,"ondisk_log_size":78,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:18.169794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":428,"num_read_kb":1125,"num_write":73,"num_write_kb":431,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,4],"avail_no_missing":["0","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":289,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:01:03.259502+0000","last_change":"2023-09-23T18:49:32.172810+0000","last_active":"2023-09-23T19:01:03.259502+0000","last_peered":"2023-09-23T19:01:03.259502+0000","last_clean":"2023-09-23T19:01:03.259502+0000","last_became_active":"2023-09-23T18:49:32.172207+0000","last_became_peered":"2023-09-23T18:49:32.172207+0000","last_unstale":"2023-09-23T19:01:03.259502+0000","last_undegraded":"2023-09-23T19:01:03.259502+0000","last_fullsized":"2023-09-23T19:01:03.259502+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:08:40.365756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":51,"reported_epoch":319,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:00:46.540493+0000","last_change":"2023-09-23T18:55:25.024282+0000","last_active":"2023-09-23T19:00:46.540493+0000","last_peered":"2023-09-23T19:00:46.540493+0000","last_clean":"2023-09-23T19:00:46.540493+0000","last_became_active":"2023-09-23T18:55:25.023045+0000","last_became_peered":"2023-09-23T18:55:25.023045+0000","last_unstale":"2023-09-23T19:00:46.540493+0000","last_undegraded":"2023-09-23T19:00:46.540493+0000","last_fullsized":"2023-09-23T19:00:46.540493+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0,1],"avail_no_missing":["5","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"320'3087","reported_seq":3099,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:41.746064+0000","last_change":"2023-09-23T19:00:44.628092+0000","last_active":"2023-09-23T19:01:41.746064+0000","last_peered":"2023-09-23T19:01:41.746064+0000","last_clean":"2023-09-23T19:01:41.746064+0000","last_became_active":"2023-09-23T19:00:44.627753+0000","last_became_peered":"2023-09-23T19:00:44.627753+0000","last_unstale":"2023-09-23T19:01:41.746064+0000","last_undegraded":"2023-09-23T19:01:41.746064+0000","last_fullsized":"2023-09-23T19:01:41.746064+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":3087,"log_dups_size":0,"ondisk_log_size":3087,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:18:01.067034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202309632,"num_objects":3087,"num_object_clones":0,"num_object_copies":9261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3087,"num_write_kb":197568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":309,"reported_epoch":319,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:00:52.061223+0000","last_change":"2023-09-23T18:58:39.721145+0000","last_active":"2023-09-23T19:00:52.061223+0000","last_peered":"2023-09-23T19:00:52.061223+0000","last_clean":"2023-09-23T19:00:52.061223+0000","last_became_active":"2023-09-23T18:58:39.720643+0000","last_became_peered":"2023-09-23T18:58:39.720643+0000","last_unstale":"2023-09-23T19:00:52.061223+0000","last_undegraded":"2023-09-23T19:00:52.061223+0000","last_fullsized":"2023-09-23T19:00:52.061223+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:19.116982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0,5],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'181","reported_seq":859,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:45.618569+0000","last_change":"2023-09-23T18:59:16.452945+0000","last_active":"2023-09-23T19:00:45.618569+0000","last_peered":"2023-09-23T19:00:45.618569+0000","last_clean":"2023-09-23T19:00:45.618569+0000","last_became_active":"2023-09-23T18:59:16.452030+0000","last_became_peered":"2023-09-23T18:59:16.452030+0000","last_unstale":"2023-09-23T19:00:45.618569+0000","last_undegraded":"2023-09-23T19:00:45.618569+0000","last_fullsized":"2023-09-23T19:00:45.618569+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":181,"log_dups_size":0,"ondisk_log_size":181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:20.988663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":909,"num_read_kb":734,"num_write":148,"num_write_kb":75,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":121,"reported_epoch":319,"state":"active+undersized","last_fresh":"2023-09-23T19:00:58.258146+0000","last_change":"2023-09-23T18:54:52.355759+0000","last_active":"2023-09-23T19:00:58.258146+0000","last_peered":"2023-09-23T19:00:58.258146+0000","last_clean":"2023-09-23T18:21:39.862208+0000","last_became_active":"2023-09-23T18:54:52.355759+0000","last_became_peered":"2023-09-23T18:54:52.355759+0000","last_unstale":"2023-09-23T19:00:58.258146+0000","last_undegraded":"2023-09-23T19:00:58.258146+0000","last_fullsized":"2023-09-23T18:54:52.345761+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:00.201189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.f","version":"320'3200","reported_seq":3210,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:42.070860+0000","last_change":"2023-09-23T19:00:44.634465+0000","last_active":"2023-09-23T19:01:42.070860+0000","last_peered":"2023-09-23T19:01:42.070860+0000","last_clean":"2023-09-23T19:01:42.070860+0000","last_became_active":"2023-09-23T19:00:44.634053+0000","last_became_peered":"2023-09-23T19:00:44.634053+0000","last_unstale":"2023-09-23T19:01:42.070860+0000","last_undegraded":"2023-09-23T19:01:42.070860+0000","last_fullsized":"2023-09-23T19:01:42.070860+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":3200,"log_dups_size":0,"ondisk_log_size":3200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:25.648826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209715200,"num_objects":3200,"num_object_clones":0,"num_object_copies":9600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3200,"num_write_kb":204800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"77'112","reported_seq":703,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:04.682393+0000","last_change":"2023-09-23T18:58:38.712045+0000","last_active":"2023-09-23T19:01:04.682393+0000","last_peered":"2023-09-23T19:01:04.682393+0000","last_clean":"2023-09-23T19:01:04.682393+0000","last_became_active":"2023-09-23T18:58:38.711280+0000","last_became_peered":"2023-09-23T18:58:38.711280+0000","last_unstale":"2023-09-23T19:01:04.682393+0000","last_undegraded":"2023-09-23T19:01:04.682393+0000","last_fullsized":"2023-09-23T19:01:04.682393+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:35.176831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":893,"num_read_kb":3172,"num_write":69,"num_write_kb":2237,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":330,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:05.268662+0000","last_change":"2023-09-23T18:59:13.714421+0000","last_active":"2023-09-23T19:01:05.268662+0000","last_peered":"2023-09-23T19:01:05.268662+0000","last_clean":"2023-09-23T19:01:05.268662+0000","last_became_active":"2023-09-23T18:59:13.703635+0000","last_became_peered":"2023-09-23T18:59:13.703635+0000","last_unstale":"2023-09-23T19:01:05.268662+0000","last_undegraded":"2023-09-23T19:01:05.268662+0000","last_fullsized":"2023-09-23T19:01:05.268662+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:41.492479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":264,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:46.539858+0000","last_change":"2023-09-23T18:59:13.699373+0000","last_active":"2023-09-23T19:00:46.539858+0000","last_peered":"2023-09-23T19:00:46.539858+0000","last_clean":"2023-09-23T19:00:46.539858+0000","last_became_active":"2023-09-23T18:59:13.698975+0000","last_became_peered":"2023-09-23T18:59:13.698975+0000","last_unstale":"2023-09-23T19:00:46.539858+0000","last_undegraded":"2023-09-23T19:00:46.539858+0000","last_fullsized":"2023-09-23T19:00:46.539858+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":297,"reported_epoch":320,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:01:05.268342+0000","last_change":"2023-09-23T18:59:14.720673+0000","last_active":"2023-09-23T19:01:05.268342+0000","last_peered":"2023-09-23T19:01:05.268342+0000","last_clean":"2023-09-23T19:01:05.268342+0000","last_became_active":"2023-09-23T18:59:14.720069+0000","last_became_peered":"2023-09-23T18:59:14.720069+0000","last_unstale":"2023-09-23T19:01:05.268342+0000","last_undegraded":"2023-09-23T19:01:05.268342+0000","last_fullsized":"2023-09-23T19:01:05.268342+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:42:55.501526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,4],"avail_no_missing":["0","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":107,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:46.539842+0000","last_change":"2023-09-23T18:58:38.715946+0000","last_active":"2023-09-23T19:00:46.539842+0000","last_peered":"2023-09-23T19:00:46.539842+0000","last_clean":"2023-09-23T19:00:46.539842+0000","last_became_active":"2023-09-23T18:58:38.715584+0000","last_became_peered":"2023-09-23T18:58:38.715584+0000","last_unstale":"2023-09-23T19:00:46.539842+0000","last_undegraded":"2023-09-23T19:00:46.539842+0000","last_fullsized":"2023-09-23T19:00:46.539842+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:57.811713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":78,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:46.540143+0000","last_change":"2023-09-23T18:59:13.706536+0000","last_active":"2023-09-23T19:00:46.540143+0000","last_peered":"2023-09-23T19:00:46.540143+0000","last_clean":"2023-09-23T19:00:46.540143+0000","last_became_active":"2023-09-23T18:59:13.702352+0000","last_became_peered":"2023-09-23T18:59:13.702352+0000","last_unstale":"2023-09-23T19:00:46.540143+0000","last_undegraded":"2023-09-23T19:00:46.540143+0000","last_fullsized":"2023-09-23T19:00:46.540143+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":304,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:45.614238+0000","last_change":"2023-09-23T18:59:13.712801+0000","last_active":"2023-09-23T19:00:45.614238+0000","last_peered":"2023-09-23T19:00:45.614238+0000","last_clean":"2023-09-23T19:00:45.614238+0000","last_became_active":"2023-09-23T18:59:13.710748+0000","last_became_peered":"2023-09-23T18:59:13.710748+0000","last_unstale":"2023-09-23T19:00:45.614238+0000","last_undegraded":"2023-09-23T19:00:45.614238+0000","last_fullsized":"2023-09-23T19:00:45.614238+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:26.256173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":78,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:46.540235+0000","last_change":"2023-09-23T18:59:13.712884+0000","last_active":"2023-09-23T19:00:46.540235+0000","last_peered":"2023-09-23T19:00:46.540235+0000","last_clean":"2023-09-23T19:00:46.540235+0000","last_became_active":"2023-09-23T18:59:13.702408+0000","last_became_peered":"2023-09-23T18:59:13.702408+0000","last_unstale":"2023-09-23T19:00:46.540235+0000","last_undegraded":"2023-09-23T19:00:46.540235+0000","last_fullsized":"2023-09-23T19:00:46.540235+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":269,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:45.614444+0000","last_change":"2023-09-23T18:59:13.699034+0000","last_active":"2023-09-23T19:00:45.614444+0000","last_peered":"2023-09-23T19:00:45.614444+0000","last_clean":"2023-09-23T19:00:45.614444+0000","last_became_active":"2023-09-23T18:59:13.698718+0000","last_became_peered":"2023-09-23T18:59:13.698718+0000","last_unstale":"2023-09-23T19:00:45.614444+0000","last_undegraded":"2023-09-23T19:00:45.614444+0000","last_fullsized":"2023-09-23T19:00:45.614444+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:57.449621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":33,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:45.615609+0000","last_change":"2023-09-23T18:58:38.711301+0000","last_active":"2023-09-23T19:00:45.615609+0000","last_peered":"2023-09-23T19:00:45.615609+0000","last_clean":"2023-09-23T19:00:45.615609+0000","last_became_active":"2023-09-23T18:58:38.710923+0000","last_became_peered":"2023-09-23T18:58:38.710923+0000","last_unstale":"2023-09-23T19:00:45.615609+0000","last_undegraded":"2023-09-23T19:00:45.615609+0000","last_fullsized":"2023-09-23T19:00:45.615609+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:17.756147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":33,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:04.678792+0000","last_change":"2023-09-23T18:58:38.711839+0000","last_active":"2023-09-23T19:01:04.678792+0000","last_peered":"2023-09-23T19:01:04.678792+0000","last_clean":"2023-09-23T19:01:04.678792+0000","last_became_active":"2023-09-23T18:58:38.711347+0000","last_became_peered":"2023-09-23T18:58:38.711347+0000","last_unstale":"2023-09-23T19:01:04.678792+0000","last_undegraded":"2023-09-23T19:01:04.678792+0000","last_fullsized":"2023-09-23T19:01:04.678792+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:55.319645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":377,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T19:00:46.541925+0000","last_change":"2023-09-23T18:55:23.985783+0000","last_active":"2023-09-23T19:00:46.541925+0000","last_peered":"2023-09-23T19:00:46.541925+0000","last_clean":"2023-09-23T19:00:46.541925+0000","last_became_active":"2023-09-23T18:55:23.977058+0000","last_became_peered":"2023-09-23T18:55:23.977058+0000","last_unstale":"2023-09-23T19:00:46.541925+0000","last_undegraded":"2023-09-23T19:00:46.541925+0000","last_fullsized":"2023-09-23T19:00:46.541925+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:00.611836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":260,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T19:01:05.265142+0000","last_change":"2023-09-23T18:59:13.704360+0000","last_active":"2023-09-23T19:01:05.265142+0000","last_peered":"2023-09-23T19:01:05.265142+0000","last_clean":"2023-09-23T19:01:05.265142+0000","last_became_active":"2023-09-23T18:59:13.704020+0000","last_became_peered":"2023-09-23T18:59:13.704020+0000","last_unstale":"2023-09-23T19:01:05.265142+0000","last_undegraded":"2023-09-23T19:01:05.265142+0000","last_fullsized":"2023-09-23T19:01:05.265142+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:01.385904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":87,"acting":94,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3191734272,"num_objects":48702,"num_object_clones":0,"num_object_copies":146106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15116,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":48702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":48702,"num_write_kb":3116928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8578007040,"data_stored":8578007040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":48702,"ondisk_log_size":48702,"up":43,"acting":43,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":12,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":8254,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38129,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":82,"acting":94,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":2,"acting":2,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5937,"num_read_kb":13190,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":94,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5621,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":20,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430724,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":914504,"kb_used_data":7168,"kb_used_omap":0,"kb_used_meta":907328,"kb_avail":92830648,"statfs":{"total":95995035648,"available":95058583552,"internally_reserved":0,"allocated":7340032,"data_stored":6683781,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929103872},"hb_peers":[0,1,2,3,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:54:27 2023","interfaces":[{"interface":"back","average":{"1min":5.7489999999999997,"5min":2.3029999999999999,"15min":2.6070000000000002},"min":{"1min":0.29199999999999998,"5min":0.29099999999999998,"15min":0.24299999999999999},"max":{"1min":80.381,"5min":80.381,"15min":97.415999999999997},"last":0.86799999999999999},{"interface":"front","average":{"1min":5.7199999999999998,"5min":2.427,"15min":2.6829999999999998},"min":{"1min":0.376,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":79.781999999999996,"5min":79.781999999999996,"15min":96.581000000000003},"last":0.76300000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:54:27 2023","interfaces":[{"interface":"back","average":{"1min":4.9660000000000002,"5min":2.714,"15min":2.5459999999999998},"min":{"1min":0.29099999999999998,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":80.266000000000005,"5min":80.266000000000005,"15min":94.650999999999996},"last":0.38800000000000001},{"interface":"front","average":{"1min":4.9859999999999998,"5min":2.698,"15min":2.7330000000000001},"min":{"1min":0.38700000000000001,"5min":0.309,"15min":0.29299999999999998},"max":{"1min":80.224999999999994,"5min":80.224999999999994,"15min":94.843999999999994},"last":0.51100000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:54:22 2023","interfaces":[{"interface":"back","average":{"1min":5.2910000000000004,"5min":2.754,"15min":1.881},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.311},"max":{"1min":86.007999999999996,"5min":86.007999999999996,"15min":86.007999999999996},"last":0.84999999999999998},{"interface":"front","average":{"1min":5.6870000000000003,"5min":2.3679999999999999,"15min":1.788},"min":{"1min":0.45600000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":86.832999999999998,"5min":86.832999999999998,"15min":86.832999999999998},"last":0.78800000000000003}]},{"osd":3,"last update":"Sat Sep 23 18:54:48 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":2.7400000000000002,"15min":3.1019999999999999},"min":{"1min":0.38100000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":3.536,"5min":118.07599999999999,"15min":136.345},"last":0.83499999999999996},{"interface":"front","average":{"1min":0.70999999999999996,"5min":2.4710000000000001,"15min":3.1389999999999998},"min":{"1min":0.41599999999999998,"5min":0.34699999999999998,"15min":0.318},"max":{"1min":2.444,"5min":118.69,"15min":136.893},"last":0.73399999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:52:25 2023","interfaces":[{"interface":"back","average":{"1min":1.4450000000000001,"5min":1.1319999999999999,"15min":1.016},"min":{"1min":0.39400000000000002,"5min":0.32500000000000001,"15min":0.29199999999999998},"max":{"1min":6.4809999999999999,"5min":6.4809999999999999,"15min":6.4809999999999999},"last":1.3480000000000001},{"interface":"front","average":{"1min":1.48,"5min":1.73,"15min":1.2849999999999999},"min":{"1min":0.40999999999999998,"5min":0.38900000000000001,"15min":0.35099999999999998},"max":{"1min":6.1100000000000003,"5min":13.010999999999999,"15min":13.010999999999999},"last":0.61299999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:54:27 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.95999999999999996,"15min":1.1479999999999999},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.29799999999999999},"max":{"1min":0.84599999999999997,"5min":6.5780000000000003,"15min":8.7210000000000001},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.1459999999999999,"15min":1.1100000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.1220000000000001,"5min":8.6519999999999992,"15min":8.6519999999999992},"last":0.94599999999999995}]},{"osd":6,"last update":"Sat Sep 23 18:54:27 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":1.0620000000000001,"15min":1.179},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.35799999999999998},"max":{"1min":1.024,"5min":6.4500000000000002,"15min":6.4500000000000002},"last":0.90600000000000003},{"interface":"front","average":{"1min":0.65700000000000003,"5min":1.0609999999999999,"15min":1.179},"min":{"1min":0.40500000000000003,"5min":0.38900000000000001,"15min":0.36199999999999999},"max":{"1min":0.86499999999999999,"5min":7.6029999999999998,"15min":9.2850000000000001},"last":0.89500000000000002}]}]},{"osd":6,"up_from":42,"seq":180388627038,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2664388,"kb_used_data":1764924,"kb_used_omap":0,"kb_used_meta":899456,"kb_avail":91080764,"statfs":{"total":95995035648,"available":93266702336,"internally_reserved":0,"allocated":1807282176,"data_stored":1806516002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921042944},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:00:47 2023","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":1.792,"15min":2.0539999999999998},"min":{"1min":0.34599999999999997,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.141,"5min":88.563000000000002,"15min":103.562},"last":0.307},{"interface":"front","average":{"1min":0.877,"5min":1.8919999999999999,"15min":2.1629999999999998},"min":{"1min":0.42399999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.3009999999999999,"5min":88.611000000000004,"15min":103.643},"last":0.49299999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:00:47 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":2.2250000000000001,"15min":2.9660000000000002},"min":{"1min":0.56599999999999995,"5min":0.32800000000000001,"15min":0.27600000000000002},"max":{"1min":2.0230000000000001,"5min":46.162999999999997,"15min":171.15600000000001},"last":0.434},{"interface":"front","average":{"1min":0.70699999999999996,"5min":1.905,"15min":2.915},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.3360000000000001,"5min":46.430999999999997,"15min":170.52600000000001},"last":0.379}]},{"osd":2,"last update":"Sat Sep 23 19:00:47 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.93100000000000005,"15min":1.2310000000000001},"min":{"1min":0.41999999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.7869999999999999,"5min":7.7080000000000002,"15min":14.039},"last":2.3370000000000002},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.91400000000000003,"15min":1.2390000000000001},"min":{"1min":0.46500000000000002,"5min":0.35299999999999998,"15min":0.32800000000000001},"max":{"1min":4.8070000000000004,"5min":7.6420000000000003,"15min":14.156000000000001},"last":2.3860000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:00:55 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":2.6549999999999998,"15min":2.1110000000000002},"min":{"1min":0.54400000000000004,"5min":0.34000000000000002,"15min":0.30499999999999999},"max":{"1min":1.25,"5min":58.156999999999996,"15min":103.262},"last":1.6990000000000001},{"interface":"front","average":{"1min":0.88300000000000001,"5min":2.7200000000000002,"15min":2.1589999999999998},"min":{"1min":0.61899999999999999,"5min":0.38300000000000001,"15min":0.373},"max":{"1min":1.24,"5min":58.085999999999999,"15min":103.384},"last":1.6850000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:01:07 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":0.77800000000000002,"15min":0.76500000000000001},"min":{"1min":0.72199999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.139,"5min":4.1150000000000002,"15min":4.1150000000000002},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.079,"5min":0.82999999999999996,"15min":0.81000000000000005},"min":{"1min":0.746,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.405,"5min":4.0179999999999998,"15min":4.0179999999999998},"last":0.629}]},{"osd":5,"last update":"Sat Sep 23 19:00:55 2023","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":0.80700000000000005,"15min":1.958},"min":{"1min":0.59799999999999998,"5min":0.373,"15min":0.371},"max":{"1min":1.212,"5min":2.786,"15min":336.26299999999998},"last":0.45900000000000002},{"interface":"front","average":{"1min":1.048,"5min":0.76500000000000001,"15min":1.9790000000000001},"min":{"1min":0.46400000000000002,"5min":0.374,"15min":0.374},"max":{"1min":1.484,"5min":1.7230000000000001,"15min":336.07799999999997},"last":0.54700000000000004}]},{"osd":7,"last update":"Sat Sep 23 18:54:32 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":1.0049999999999999,"15min":1.1839999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.35999999999999999},"max":{"1min":3.3500000000000001,"5min":7.0700000000000003,"15min":7.9320000000000004},"last":1.0640000000000001},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.95299999999999996,"15min":1.264},"min":{"1min":0.45600000000000002,"5min":0.41599999999999998,"15min":0.39900000000000002},"max":{"1min":0.97399999999999998,"5min":5.5149999999999997,"15min":8.9250000000000007},"last":0.91200000000000003}]}]},{"osd":1,"up_from":123,"seq":528280977740,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2040244,"kb_used_data":1128236,"kb_used_omap":7,"kb_used_meta":911928,"kb_avail":91704908,"statfs":{"total":95995035648,"available":93905825792,"internally_reserved":0,"allocated":1155313664,"data_stored":1153757469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":933815054},"hb_peers":[0,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:00:39 2023","interfaces":[{"interface":"back","average":{"1min":1.7010000000000001,"5min":2.4750000000000001,"15min":2.1549999999999998},"min":{"1min":0.251,"5min":0.251,"15min":0.19400000000000001},"max":{"1min":16.959,"5min":64.287999999999997,"15min":64.287999999999997},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.0349999999999999,"5min":2.2799999999999998,"15min":2.3769999999999998},"min":{"1min":0.23999999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":10.798999999999999,"5min":64.343999999999994,"15min":67.034000000000006},"last":0.71099999999999997}]},{"osd":2,"last update":"Sat Sep 23 19:00:39 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":1.5269999999999999,"15min":2.1179999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.20899999999999999},"max":{"1min":7.1369999999999996,"5min":12.800000000000001,"15min":180.578},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.73199999999999998,"5min":1.5640000000000001,"15min":2.5150000000000001},"min":{"1min":0.311,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.8969999999999998,"5min":8.2669999999999995,"15min":180.12299999999999},"last":0.76000000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:00:39 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":2.101,"15min":2.1699999999999999},"min":{"1min":0.36499999999999999,"5min":0.26400000000000001,"15min":0.23899999999999999},"max":{"1min":3.9729999999999999,"5min":9.9039999999999999,"15min":66.908000000000001},"last":0.81299999999999994},{"interface":"front","average":{"1min":0.56999999999999995,"5min":2.028,"15min":1.7390000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":1.246,"5min":13.448,"15min":13.448},"last":0.46400000000000002}]},{"osd":4,"last update":"Sat Sep 23 19:01:09 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":1.649,"15min":1.5289999999999999},"min":{"1min":0.622,"5min":0.314,"15min":0.314},"max":{"1min":1.0509999999999999,"5min":8.3559999999999999,"15min":8.3559999999999999},"last":0.83799999999999997},{"interface":"front","average":{"1min":0.87,"5min":1.617,"15min":1.431},"min":{"1min":0.505,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.214,"5min":9.0549999999999997,"15min":9.0549999999999997},"last":0.56299999999999994}]},{"osd":5,"last update":"Sat Sep 23 19:00:39 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":1.7649999999999999,"15min":2.1200000000000001},"min":{"1min":0.54400000000000004,"5min":0.33400000000000002,"15min":0.33000000000000002},"max":{"1min":4.0380000000000003,"5min":8.391,"15min":66.954999999999998},"last":4.7050000000000001},{"interface":"front","average":{"1min":0.748,"5min":1.7050000000000001,"15min":2.6190000000000002},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.3069999999999999,"5min":9.1349999999999998,"15min":180.52099999999999},"last":0.873}]},{"osd":6,"last update":"Sat Sep 23 19:00:39 2023","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":1.6890000000000001,"15min":1.8029999999999999},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.35899999999999999},"max":{"1min":1.167,"5min":9.1029999999999998,"15min":63.036999999999999},"last":5.4989999999999997},{"interface":"front","average":{"1min":1.7450000000000001,"5min":1.716,"15min":1.671},"min":{"1min":0.60799999999999998,"5min":0.373,"15min":0.35999999999999999},"max":{"1min":12.887,"5min":12.887,"15min":12.887},"last":4.819}]},{"osd":7,"last update":"Sat Sep 23 18:54:28 2023","interfaces":[{"interface":"back","average":{"1min":4.2919999999999998,"5min":2.593,"15min":2.1019999999999999},"min":{"1min":0.432,"5min":0.42199999999999999,"15min":0.33500000000000002},"max":{"1min":62.954999999999998,"5min":62.954999999999998,"15min":62.954999999999998},"last":1.01},{"interface":"front","average":{"1min":4.407,"5min":2.5369999999999999,"15min":1.9339999999999999},"min":{"1min":0.47799999999999998,"5min":0.373,"15min":0.373},"max":{"1min":62.658999999999999,"5min":62.658999999999999,"15min":62.658999999999999},"last":0.96599999999999997}]}]},{"osd":0,"up_from":88,"seq":377957122471,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3141772,"kb_used_data":2243012,"kb_used_omap":8,"kb_used_meta":898743,"kb_avail":90603380,"statfs":{"total":95995035648,"available":92777861120,"internally_reserved":0,"allocated":2296844288,"data_stored":2295440062,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":920313726},"hb_peers":[1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:00:51 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":2.5310000000000001,"15min":2.6669999999999998},"min":{"1min":0.27400000000000002,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":3.2999999999999998,"5min":44.119,"15min":123.81},"last":0.84399999999999997},{"interface":"front","average":{"1min":0.53500000000000003,"5min":2.77,"15min":2.746},"min":{"1min":0.32900000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.099,"5min":44.343000000000004,"15min":121.92400000000001},"last":1.288}]},{"osd":2,"last update":"Sat Sep 23 19:01:35 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.758,"15min":1.861},"min":{"1min":0.22600000000000001,"5min":0.215,"15min":0.215},"max":{"1min":5.6050000000000004,"5min":59.381999999999998,"15min":59.381999999999998},"last":0.88600000000000001},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.6479999999999999,"15min":1.5109999999999999},"min":{"1min":0.35899999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":2.0899999999999999,"5min":59.783000000000001,"15min":59.783000000000001},"last":3.5920000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:01:00 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":3.6459999999999999,"15min":2.7509999999999999},"min":{"1min":0.38100000000000001,"5min":0.27100000000000002,"15min":0.23799999999999999},"max":{"1min":0.88200000000000001,"5min":113.375,"15min":113.375},"last":2.3570000000000002},{"interface":"front","average":{"1min":0.62,"5min":3.5059999999999998,"15min":2.54},"min":{"1min":0.40699999999999997,"5min":0.217,"15min":0.217},"max":{"1min":0.94899999999999995,"5min":113.327,"15min":113.327},"last":3.6360000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:01:07 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":1.204,"15min":1.083},"min":{"1min":0.60899999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.006,"5min":8.1189999999999998,"15min":8.1189999999999998},"last":2.4220000000000002},{"interface":"front","average":{"1min":0.872,"5min":1.0740000000000001,"15min":0.93300000000000005},"min":{"1min":0.58199999999999996,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.173,"5min":5.3810000000000002,"15min":5.3810000000000002},"last":3.6760000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:00:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":1.2230000000000001,"15min":1.597},"min":{"1min":0.56100000000000005,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.4610000000000003,"5min":8.2390000000000008,"15min":9.5879999999999992},"last":2.3929999999999998},{"interface":"front","average":{"1min":0.874,"5min":1.171,"15min":1.3180000000000001},"min":{"1min":0.67100000000000004,"5min":0.35899999999999999,"15min":0.34999999999999998},"max":{"1min":1.1399999999999999,"5min":5.3559999999999999,"15min":8.6720000000000006},"last":3.6930000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:00:51 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":1.2,"15min":1.6120000000000001},"min":{"1min":0.56100000000000005,"5min":0.36799999999999999,"15min":0.34300000000000003},"max":{"1min":1.1060000000000001,"5min":8.2010000000000005,"15min":9.4760000000000009},"last":2.4510000000000001},{"interface":"front","average":{"1min":0.88600000000000001,"5min":1.097,"15min":1.3720000000000001},"min":{"1min":0.58299999999999996,"5min":0.40400000000000003,"15min":0.376},"max":{"1min":1.232,"5min":5.4059999999999997,"15min":8.6300000000000008},"last":3.6579999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:54:41 2023","interfaces":[{"interface":"back","average":{"1min":1.3859999999999999,"5min":1.589,"15min":1.516},"min":{"1min":0.53000000000000003,"5min":0.372,"15min":0.372},"max":{"1min":5.0720000000000001,"5min":8.7040000000000006,"15min":10.191000000000001},"last":0.90900000000000003},{"interface":"front","average":{"1min":1.7629999999999999,"5min":1.5409999999999999,"15min":1.6559999999999999},"min":{"1min":0.47599999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":7.359,"5min":7.649,"15min":10.01},"last":0.75700000000000001}]}]},{"osd":2,"up_from":98,"seq":420906795419,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2225936,"kb_used_data":1389576,"kb_used_omap":11,"kb_used_meta":836340,"kb_avail":91519216,"statfs":{"total":95995035648,"available":93715677184,"internally_reserved":0,"allocated":1422925824,"data_stored":1421521267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11949,"internal_metadata":856412499},"hb_peers":[0,1,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:01:41 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":1.0489999999999999,"15min":1.8500000000000001},"min":{"1min":0.21099999999999999,"5min":0.18099999999999999,"15min":0.16600000000000001},"max":{"1min":3.4430000000000001,"5min":6.0679999999999996,"15min":166.46100000000001},"last":0.505},{"interface":"front","average":{"1min":1.7170000000000001,"5min":1.252,"15min":1.8959999999999999},"min":{"1min":0.307,"5min":0.187,"15min":0.187},"max":{"1min":6.657,"5min":7.3860000000000001,"15min":162.911},"last":2.6000000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:01:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1950000000000001,"5min":3.831,"15min":3.4239999999999999},"min":{"1min":0.26500000000000001,"5min":0.215,"15min":0.20599999999999999},"max":{"1min":3.8780000000000001,"5min":104.43899999999999,"15min":156.35400000000001},"last":1.083},{"interface":"front","average":{"1min":1.5569999999999999,"5min":4.0609999999999999,"15min":3.472},"min":{"1min":0.38200000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":4.2489999999999997,"5min":99.867999999999995,"15min":156.376},"last":2.5169999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:01:12 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":3.6829999999999998,"15min":2.718},"min":{"1min":0.32600000000000001,"5min":0.222,"15min":0.221},"max":{"1min":5.6550000000000002,"5min":85.102999999999994,"15min":157.58099999999999},"last":4.5190000000000001},{"interface":"front","average":{"1min":0.65000000000000002,"5min":3.536,"15min":2.79},"min":{"1min":0.34599999999999997,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.5840000000000001,"5min":84.531999999999996,"15min":159.34},"last":2.827}]},{"osd":4,"last update":"Sat Sep 23 19:01:12 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":0.92600000000000005,"15min":0.81399999999999995},"min":{"1min":0.64800000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.7270000000000001,"5min":5.3239999999999998,"15min":5.3239999999999998},"last":1.1120000000000001},{"interface":"front","average":{"1min":1.1220000000000001,"5min":0.95599999999999996,"15min":1.056},"min":{"1min":0.52500000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.0049999999999999,"5min":6.0049999999999999,"15min":6.0049999999999999},"last":1.484}]},{"osd":5,"last update":"Sat Sep 23 19:01:41 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":0.97699999999999998,"15min":2.157},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.33000000000000002},"max":{"1min":2.3050000000000002,"5min":5.3529999999999998,"15min":296.86200000000002},"last":1.133},{"interface":"front","average":{"1min":1.401,"5min":1.052,"15min":2.3220000000000001},"min":{"1min":0.51300000000000001,"5min":0.36399999999999999,"15min":0.36299999999999999},"max":{"1min":3.5270000000000001,"5min":5.0259999999999998,"15min":296.69499999999999},"last":2.5600000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:01:41 2023","interfaces":[{"interface":"back","average":{"1min":1.5920000000000001,"5min":1.0249999999999999,"15min":1.2330000000000001},"min":{"1min":0.40100000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":5.9649999999999999,"5min":5.9649999999999999,"15min":11.021000000000001},"last":4.4660000000000002},{"interface":"front","average":{"1min":1.077,"5min":1.044,"15min":1.3109999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.35899999999999999},"max":{"1min":2.2759999999999998,"5min":5.3789999999999996,"15min":16.247},"last":1.379}]},{"osd":7,"last update":"Sat Sep 23 18:54:25 2023","interfaces":[{"interface":"back","average":{"1min":1.5700000000000001,"5min":1.623,"15min":1.51},"min":{"1min":0.38300000000000001,"5min":0.374,"15min":0.374},"max":{"1min":5.5880000000000001,"5min":10.971,"15min":16.024000000000001},"last":0.83599999999999997},{"interface":"front","average":{"1min":1.1899999999999999,"5min":1.8149999999999999,"15min":1.4359999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":5.9249999999999998,"5min":16.178999999999998,"15min":16.178999999999998},"last":0.95799999999999996}]}]},{"osd":3,"up_from":103,"seq":442381631894,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1773092,"kb_used_data":1012892,"kb_used_omap":4,"kb_used_meta":760187,"kb_avail":91972060,"statfs":{"total":95995035648,"available":94179389440,"internally_reserved":0,"allocated":1037201408,"data_stored":1035773644,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4826,"internal_metadata":778431782},"hb_peers":[0,1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:01:05 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":4.9779999999999998,"15min":3.476},"min":{"1min":0.217,"5min":0.191,"15min":0.191},"max":{"1min":0.73299999999999998,"5min":296.55399999999997,"15min":296.55399999999997},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.55600000000000005,"5min":5.0110000000000001,"15min":3.4809999999999999},"min":{"1min":0.34799999999999998,"5min":0.191,"15min":0.191},"max":{"1min":0.83699999999999997,"5min":296.36900000000003,"15min":296.36900000000003},"last":0.437}]},{"osd":1,"last update":"Sat Sep 23 19:00:54 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":1.9530000000000001,"15min":2.448},"min":{"1min":0.371,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.83499999999999996,"5min":11.624000000000001,"15min":157.67400000000001},"last":67.965000000000003},{"interface":"front","average":{"1min":0.59999999999999998,"5min":1.698,"15min":2.3719999999999999},"min":{"1min":0.26500000000000001,"5min":0.214,"15min":0.20200000000000001},"max":{"1min":0.87,"5min":18.065999999999999,"15min":157.84700000000001},"last":67.941000000000003}]},{"osd":2,"last update":"Sat Sep 23 19:01:05 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":5.2039999999999997,"15min":3.1269999999999998},"min":{"1min":0.34799999999999998,"5min":0.23699999999999999,"15min":0.22},"max":{"1min":0.90400000000000003,"5min":178.06200000000001,"15min":178.06200000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.64600000000000002,"5min":5.0709999999999997,"15min":2.8109999999999999},"min":{"1min":0.45900000000000002,"5min":0.26300000000000001,"15min":0.23200000000000001},"max":{"1min":1.159,"5min":178.267,"15min":178.267},"last":0.41199999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:01:12 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.2190000000000001,"15min":0.90100000000000002},"min":{"1min":0.47499999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.123,"5min":12.028,"15min":12.028},"last":0.745},{"interface":"front","average":{"1min":0.89100000000000001,"5min":1.4319999999999999,"15min":1.5149999999999999},"min":{"1min":0.50700000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.504,"5min":11.369,"15min":11.369},"last":0.66700000000000004}]},{"osd":5,"last update":"Sat Sep 23 19:01:05 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":1.3380000000000001,"15min":1.3240000000000001},"min":{"1min":0.56399999999999995,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.1299999999999999,"5min":11.978999999999999,"15min":11.978999999999999},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.85699999999999998,"5min":1.7170000000000001,"15min":1.603},"min":{"1min":0.58699999999999997,"5min":0.34300000000000003,"15min":0.33100000000000002},"max":{"1min":1.4099999999999999,"5min":12.958,"15min":12.958},"last":0.56799999999999995}]},{"osd":6,"last update":"Sat Sep 23 19:01:05 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":1.641,"15min":1.637},"min":{"1min":0.60599999999999998,"5min":0.378,"15min":0.36799999999999999},"max":{"1min":1.542,"5min":12.926,"15min":19.242999999999999},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.94199999999999995,"5min":1.5329999999999999,"15min":1.5589999999999999},"min":{"1min":0.71599999999999997,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.4390000000000001,"5min":11.300000000000001,"15min":11.335000000000001},"last":0.752}]},{"osd":7,"last update":"Sat Sep 23 18:53:54 2023","interfaces":[{"interface":"back","average":{"1min":1.4710000000000001,"5min":1.7689999999999999,"15min":1.7250000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.35399999999999998},"max":{"1min":5.0700000000000003,"5min":11.382999999999999,"15min":11.382999999999999},"last":0.76900000000000002},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.5960000000000001,"15min":1.556},"min":{"1min":0.40000000000000002,"5min":0.38900000000000001,"15min":0.36599999999999999},"max":{"1min":3.1640000000000001,"5min":12.170999999999999,"15min":12.170999999999999},"last":0.59899999999999998}]}]},{"osd":4,"up_from":0,"seq":1228360646724,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":36,"seq":154618823272,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1371692,"kb_used_data":786084,"kb_used_omap":0,"kb_used_meta":585600,"kb_avail":92373460,"statfs":{"total":95995035648,"available":94590423040,"internally_reserved":0,"allocated":804950016,"data_stored":804203275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599654400},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:01:39 2023","interfaces":[{"interface":"back","average":{"1min":3.766,"5min":1.97,"15min":2.0960000000000001},"min":{"1min":0.32000000000000001,"5min":0.23499999999999999,"15min":0.23400000000000001},"max":{"1min":64.266000000000005,"5min":64.266000000000005,"15min":154.61099999999999},"last":64.266000000000005},{"interface":"front","average":{"1min":4.46,"5min":2.1779999999999999,"15min":2.3439999999999999},"min":{"1min":0.61399999999999999,"5min":0.251,"15min":0.251},"max":{"1min":64.352999999999994,"5min":64.352999999999994,"15min":154.28800000000001},"last":64.352999999999994}]},{"osd":1,"last update":"Sat Sep 23 19:00:47 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":4.7679999999999998,"15min":3.851},"min":{"1min":0.55900000000000005,"5min":0.246,"15min":0.246},"max":{"1min":1.2789999999999999,"5min":277.226,"15min":277.226},"last":0.439},{"interface":"front","average":{"1min":0.94599999999999995,"5min":4.5830000000000002,"15min":3.7770000000000001},"min":{"1min":0.45200000000000001,"5min":0.32800000000000001,"15min":0.28699999999999998},"max":{"1min":2.2879999999999998,"5min":277.32100000000003,"15min":277.32100000000003},"last":0.42499999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:01:39 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":1.02,"15min":1.1020000000000001},"min":{"1min":0.33900000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":2.2120000000000002,"5min":13.021000000000001,"15min":13.021000000000001},"last":0.33900000000000002},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.181,"15min":1.1619999999999999},"min":{"1min":0.38700000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.23,"5min":15.853999999999999,"15min":15.853999999999999},"last":0.52300000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:01:09 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":1.593,"15min":2.4089999999999998},"min":{"1min":0.44600000000000001,"5min":0.308,"15min":0.308},"max":{"1min":2.5579999999999998,"5min":28.646000000000001,"15min":155.09200000000001},"last":0.58699999999999997},{"interface":"front","average":{"1min":1.024,"5min":1.6870000000000001,"15min":2.2599999999999998},"min":{"1min":0.60799999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":2.6150000000000002,"5min":24.497,"15min":150.37100000000001},"last":0.48399999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:01:15 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.95999999999999996,"15min":0.751},"min":{"1min":0.48899999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.494,"5min":13.484,"15min":13.484},"last":0.65500000000000003},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.78200000000000003,"15min":0.71899999999999997},"min":{"1min":0.55300000000000005,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.415,"5min":4.8979999999999997,"15min":4.8979999999999997},"last":0.60699999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:01:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":0.77400000000000002,"15min":0.90400000000000003},"min":{"1min":0.70299999999999996,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.2969999999999999,"5min":4.0510000000000002,"15min":8.2379999999999995},"last":0.56000000000000005},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.0800000000000001,"15min":1.046},"min":{"1min":0.42899999999999999,"5min":0.378,"15min":0.36099999999999999},"max":{"1min":1.484,"5min":12.977,"15min":12.977},"last":0.73099999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:54:30 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.79700000000000004,"15min":1.149},"min":{"1min":0.39500000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.0009999999999999,"5min":3.4980000000000002,"15min":9.9130000000000003},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.78300000000000003,"5min":0.82599999999999996,"15min":0.96699999999999997},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":2.214,"5min":3.2490000000000001,"15min":6.7779999999999996},"last":0.90700000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4224,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7725,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3429,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11243,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2322137088,"data_stored":2322137088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1158807552,"data_stored":1158807552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1441988608,"data_stored":1441988608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1035337728,"data_stored":1035337728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":811859968,"data_stored":811859968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1807876096,"data_stored":1807876096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:01:48.144 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T19:01:48.144 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 51 2023-09-23T19:01:48.145 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:48.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:48.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:48.248 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:48.280 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:48.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:48.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:48.384 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:48.414 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:48.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:48.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:48.517 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:48.545 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:48.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:48.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:48.649 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:48.678 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:48.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:48.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:48.784 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:48.816 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:48.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:48.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:48.919 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:48.950 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:48.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:48.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:49.056 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:49.082 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:49.085 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:2023-09-23T19:01:49.081980+0000 min lat: 0.00130179 max lat: 0.429637 avg lat: 0.0109669 2023-09-23T19:01:49.085 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:01:49.085 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 40 16 58346 58330 91.1246 66.375 0.00819431 0.0109669 2023-09-23T19:01:49.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:49.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:49.186 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:49.215 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:49.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:49.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:49.322 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:49.352 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:49.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:49.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:49.457 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:49.487 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:49.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:49.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:49.593 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:49.624 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:49.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:49.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:49.727 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:49.756 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:49.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:49.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:49.861 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:49.887 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:49.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:49.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:49.990 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:50.018 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:50.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:50.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:50.085 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 41 16 59389 59373 90.4919 65.1875 0.020511 0.0110438 2023-09-23T19:01:50.110 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:01:50.119 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:49 smithi098 bash[79516]: cluster 2023-09-23T19:01:48.445524+0000 mgr.y (mgr.26054) 1316 : cluster 0 pgmap v1444: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 17793/170031 objects degraded (10.465%); 14/170031 objects misplaced (0.008%) 2023-09-23T19:01:50.120 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:49 smithi098 bash[76817]: cluster 2023-09-23T19:01:48.445524+0000 mgr.y (mgr.26054) 1316 : cluster 0 pgmap v1444: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 17793/170031 objects degraded (10.465%); 14/170031 objects misplaced (0.008%) 2023-09-23T19:01:50.123 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:50.164 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:01:50.167 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:50.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:50.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:50.272 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:50.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:49 smithi173 bash[59422]: cluster 2023-09-23T19:01:48.445524+0000 mgr.y (mgr.26054) 1316 : cluster 0 pgmap v1444: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 17793/170031 objects degraded (10.465%); 14/170031 objects misplaced (0.008%) 2023-09-23T19:01:50.307 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:50.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:50.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:50.417 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:50.468 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:50.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:50.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:50.573 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:50.615 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:50.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:50.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:50.721 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:50.747 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:50.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:50.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:50.851 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:50.885 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:50.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:50.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:50.993 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:51.001 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:50 smithi173 bash[59422]: cluster 2023-09-23T19:01:50.452980+0000 mon.a (mon.0) 1805 : cluster 3 Health check update: Degraded data redundancy: 17793/170031 objects degraded (10.465%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:51.022 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:51.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:51.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:51.085 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 42 16 60317 60301 89.7182 58 0.0161744 0.0111403 2023-09-23T19:01:51.128 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:51.152 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:50 smithi098 bash[76817]: cluster 2023-09-23T19:01:50.452980+0000 mon.a (mon.0) 1805 : cluster 3 Health check update: Degraded data redundancy: 17793/170031 objects degraded (10.465%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:51.153 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:50 smithi098 bash[79516]: cluster 2023-09-23T19:01:50.452980+0000 mon.a (mon.0) 1805 : cluster 3 Health check update: Degraded data redundancy: 17793/170031 objects degraded (10.465%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:51.175 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:51.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:51.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:51.279 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:51.307 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:51.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:51.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:51.410 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:51.436 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:51.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:51.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:51.540 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:51.585 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:51.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:51.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:51.688 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:51.723 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:51.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:51.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:51.830 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:51.857 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:51.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:51.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:51.965 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:51.986 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:01:52.024 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:52.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:52.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:52.087 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 43 16 61149 61133 88.8402 52 0.0126556 0.0112491 2023-09-23T19:01:52.130 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:52.160 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:52.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:52.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:52.299 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:52.326 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:52.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:52.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:52.429 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:52.438 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:52 smithi098 bash[76817]: cluster 2023-09-23T19:01:50.446699+0000 mgr.y (mgr.26054) 1317 : cluster 0 pgmap v1445: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 17793/170031 objects degraded (10.465%); 14/170031 objects misplaced (0.008%) 2023-09-23T19:01:52.438 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:52 smithi098 bash[76817]: audit 2023-09-23T19:01:51.254242+0000 mon.a (mon.0) 1806 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:01:52.438 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:52 smithi098 bash[76817]: audit 2023-09-23T19:01:51.273556+0000 mon.a (mon.0) 1807 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:01:52.439 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:52 smithi098 bash[79516]: cluster 2023-09-23T19:01:50.446699+0000 mgr.y (mgr.26054) 1317 : cluster 0 pgmap v1445: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 17793/170031 objects degraded (10.465%); 14/170031 objects misplaced (0.008%) 2023-09-23T19:01:52.439 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:52 smithi098 bash[79516]: audit 2023-09-23T19:01:51.254242+0000 mon.a (mon.0) 1806 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:01:52.439 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:52 smithi098 bash[79516]: audit 2023-09-23T19:01:51.273556+0000 mon.a (mon.0) 1807 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:01:52.466 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:52.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:52.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:52.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:52 smithi173 bash[59422]: cluster 2023-09-23T19:01:50.446699+0000 mgr.y (mgr.26054) 1317 : cluster 0 pgmap v1445: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 17793/170031 objects degraded (10.465%); 14/170031 objects misplaced (0.008%) 2023-09-23T19:01:52.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:52 smithi173 bash[59422]: audit 2023-09-23T19:01:51.254242+0000 mon.a (mon.0) 1806 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:01:52.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:52 smithi173 bash[59422]: audit 2023-09-23T19:01:51.273556+0000 mon.a (mon.0) 1807 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:01:52.573 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:52.601 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:52.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:52.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:52.704 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:52.746 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:52.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:52.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:52.853 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:52.883 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:52.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:52.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:52.989 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:53.007 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:01:53.018 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:53.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:53.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:53.087 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 44 16 62001 61985 88.0276 53.25 0.014296 0.0113523 2023-09-23T19:01:53.130 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:53.163 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:53.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:53.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:53.266 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:53.290 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:53.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:53.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:53.392 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:53.425 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:53.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:53.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:53.528 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:53.557 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:53.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:53.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:53.660 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:53.688 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:53.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:53.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:53.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:53 smithi098 bash[79516]: cluster 2023-09-23T19:01:52.450006+0000 mgr.y (mgr.26054) 1318 : cluster 0 pgmap v1446: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 18443/177480 objects degraded (10.392%); 14/177480 objects misplaced (0.008%) 2023-09-23T19:01:53.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:53 smithi098 bash[79516]: audit 2023-09-23T19:01:52.482450+0000 mon.a (mon.0) 1808 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:01:53.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:53 smithi098 bash[79516]: audit 2023-09-23T19:01:52.494265+0000 mon.a (mon.0) 1809 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:01:53.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:53 smithi098 bash[79516]: audit 2023-09-23T19:01:53.133593+0000 mon.a (mon.0) 1810 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:01:53.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:53 smithi098 bash[79516]: audit 2023-09-23T19:01:53.135101+0000 mon.a (mon.0) 1811 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:01:53.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:53 smithi098 bash[79516]: audit 2023-09-23T19:01:53.146196+0000 mon.a (mon.0) 1812 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:01:53.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:53 smithi098 bash[76817]: cluster 2023-09-23T19:01:52.450006+0000 mgr.y (mgr.26054) 1318 : cluster 0 pgmap v1446: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 18443/177480 objects degraded (10.392%); 14/177480 objects misplaced (0.008%) 2023-09-23T19:01:53.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:53 smithi098 bash[76817]: audit 2023-09-23T19:01:52.482450+0000 mon.a (mon.0) 1808 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:01:53.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:53 smithi098 bash[76817]: audit 2023-09-23T19:01:52.494265+0000 mon.a (mon.0) 1809 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:01:53.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:53 smithi098 bash[76817]: audit 2023-09-23T19:01:53.133593+0000 mon.a (mon.0) 1810 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:01:53.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:53 smithi098 bash[76817]: audit 2023-09-23T19:01:53.135101+0000 mon.a (mon.0) 1811 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:01:53.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:53 smithi098 bash[76817]: audit 2023-09-23T19:01:53.146196+0000 mon.a (mon.0) 1812 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:01:53.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:53 smithi173 bash[59422]: cluster 2023-09-23T19:01:52.450006+0000 mgr.y (mgr.26054) 1318 : cluster 0 pgmap v1446: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 18443/177480 objects degraded (10.392%); 14/177480 objects misplaced (0.008%) 2023-09-23T19:01:53.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:53 smithi173 bash[59422]: audit 2023-09-23T19:01:52.482450+0000 mon.a (mon.0) 1808 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:01:53.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:53 smithi173 bash[59422]: audit 2023-09-23T19:01:52.494265+0000 mon.a (mon.0) 1809 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:01:53.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:53 smithi173 bash[59422]: audit 2023-09-23T19:01:53.133593+0000 mon.a (mon.0) 1810 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:01:53.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:53 smithi173 bash[59422]: audit 2023-09-23T19:01:53.135101+0000 mon.a (mon.0) 1811 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:01:53.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:53 smithi173 bash[59422]: audit 2023-09-23T19:01:53.146196+0000 mon.a (mon.0) 1812 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:01:53.791 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:53.817 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:53.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:53.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:53.920 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:53.955 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:53.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:53.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:54.022 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:01:54.062 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:54.088 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 45 16 63013 62997 87.4769 63.25 0.0087359 0.0114239 2023-09-23T19:01:54.094 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:54.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:54.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:54.198 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:54.220 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:54.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:54.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:54.324 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:54.349 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:54.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:54.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:54.451 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:54.478 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:54.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:54.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:54.582 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:54.610 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:54.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:54.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:54.713 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:54.747 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:54.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:54.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:54.854 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:54.880 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:54.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:54.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:54.984 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:55.009 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:55.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:55.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:55.037 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:01:55.091 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 46 16 64077 64061 87.0179 66.5 0.0141397 0.0114865 2023-09-23T19:01:55.111 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:55.159 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:55.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:55.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:55.262 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:55.288 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:55.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:55.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:55.393 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:55.418 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:55.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:55.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:55.521 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:55.556 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:55.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:55.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:55.664 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:55.690 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:55.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:55 smithi098 bash[76817]: cluster 2023-09-23T19:01:54.450754+0000 mgr.y (mgr.26054) 1319 : cluster 0 pgmap v1447: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 62 MiB/s wr, 985 op/s; 19356/184665 objects degraded (10.482%); 14/184665 objects misplaced (0.008%) 2023-09-23T19:01:55.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:55 smithi098 bash[76817]: audit 2023-09-23T19:01:54.638337+0000 mon.a (mon.0) 1813 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:01:55.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:55 smithi098 bash[76817]: cluster 2023-09-23T19:01:55.454137+0000 mon.a (mon.0) 1814 : cluster 3 Health check update: Degraded data redundancy: 19356/184665 objects degraded (10.482%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:55.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:55 smithi098 bash[79516]: cluster 2023-09-23T19:01:54.450754+0000 mgr.y (mgr.26054) 1319 : cluster 0 pgmap v1447: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 62 MiB/s wr, 985 op/s; 19356/184665 objects degraded (10.482%); 14/184665 objects misplaced (0.008%) 2023-09-23T19:01:55.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:55 smithi098 bash[79516]: audit 2023-09-23T19:01:54.638337+0000 mon.a (mon.0) 1813 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:01:55.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:55 smithi098 bash[79516]: cluster 2023-09-23T19:01:55.454137+0000 mon.a (mon.0) 1814 : cluster 3 Health check update: Degraded data redundancy: 19356/184665 objects degraded (10.482%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:55.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:55.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:55.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:55 smithi173 bash[59422]: cluster 2023-09-23T19:01:54.450754+0000 mgr.y (mgr.26054) 1319 : cluster 0 pgmap v1447: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 62 MiB/s wr, 985 op/s; 19356/184665 objects degraded (10.482%); 14/184665 objects misplaced (0.008%) 2023-09-23T19:01:55.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:55 smithi173 bash[59422]: audit 2023-09-23T19:01:54.638337+0000 mon.a (mon.0) 1813 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:01:55.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:55 smithi173 bash[59422]: cluster 2023-09-23T19:01:55.454137+0000 mon.a (mon.0) 1814 : cluster 3 Health check update: Degraded data redundancy: 19356/184665 objects degraded (10.482%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:01:55.798 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:55.821 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:55.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:55.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:55.925 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:55.951 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:55.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:55.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:56.055 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:56.084 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:56.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:56.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:56.089 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 47 16 65143 65127 86.584 66.625 0.0158875 0.0115434 2023-09-23T19:01:56.190 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:56.218 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:56.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:56.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:56.321 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:56.330 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:01:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:01:56] "GET /metrics HTTP/1.1" 200 36642 "" "Prometheus/2.33.4" 2023-09-23T19:01:56.356 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:56.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:56.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:56.462 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:56.481 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:56.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:56.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:56.583 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:56.606 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:56.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:56.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:56.709 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:56.740 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:56.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:56.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:56.844 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:56.870 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:56.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:56.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:56.978 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:57.008 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:57.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:57.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:57.089 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 48 16 66138 66122 86.0757 62.1875 0.0172186 0.0116118 2023-09-23T19:01:57.113 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:57.142 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:57.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:57.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:57.245 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:57.273 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:57.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:57.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:57.376 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:57.400 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:57.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:57.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:57.505 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:57.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:57 smithi173 bash[59422]: cluster 2023-09-23T19:01:56.451263+0000 mgr.y (mgr.26054) 1320 : cluster 0 pgmap v1448: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 56 MiB/s wr, 890 op/s; 19356/185589 objects degraded (10.429%); 14/185589 objects misplaced (0.008%) 2023-09-23T19:01:57.534 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:57.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:57.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:57.638 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:57.681 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:57.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:57.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:57.784 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:57.816 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:57.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:57.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:57.919 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:57.928 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:57 smithi098 bash[76817]: cluster 2023-09-23T19:01:56.451263+0000 mgr.y (mgr.26054) 1320 : cluster 0 pgmap v1448: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 56 MiB/s wr, 890 op/s; 19356/185589 objects degraded (10.429%); 14/185589 objects misplaced (0.008%) 2023-09-23T19:01:57.928 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:57 smithi098 bash[79516]: cluster 2023-09-23T19:01:56.451263+0000 mgr.y (mgr.26054) 1320 : cluster 0 pgmap v1448: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 56 MiB/s wr, 890 op/s; 19356/185589 objects degraded (10.429%); 14/185589 objects misplaced (0.008%) 2023-09-23T19:01:57.948 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:57.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:57.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:58.051 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:58.079 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:58.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:58.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:58.089 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 49 16 67089 67073 85.532 59.4375 0.00422361 0.0116852 2023-09-23T19:01:58.182 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:58.208 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:58.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:58.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:58.311 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:58.342 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:01:58.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:58.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:01:58.448 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:58.475 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:58.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:58.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:58.578 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:58.613 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:01:58.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:58.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:01:58.718 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:58.755 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:58.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:58.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:58.861 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:58.889 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:58.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:58.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:58.992 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:59.015 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:59.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:59.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:59.089 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 50 16 67916 67900 84.8551 51.6875 0.00723196 0.0117597 2023-09-23T19:01:59.118 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:59.156 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:01:59.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:59.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:01:59.262 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:59.296 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:01:59.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:59.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:01:59.399 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:59.422 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:59.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:59.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:01:59.526 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:59.554 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:59.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:59.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:59.660 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:59.669 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:01:59 smithi173 bash[59422]: cluster 2023-09-23T19:01:58.451858+0000 mgr.y (mgr.26054) 1321 : cluster 0 pgmap v1449: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s; 20600/195846 objects degraded (10.518%); 14/195846 objects misplaced (0.007%) 2023-09-23T19:01:59.685 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:01:59.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:59.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:01:59.788 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:59.796 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:01:59 smithi098 bash[79516]: cluster 2023-09-23T19:01:58.451858+0000 mgr.y (mgr.26054) 1321 : cluster 0 pgmap v1449: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s; 20600/195846 objects degraded (10.518%); 14/195846 objects misplaced (0.007%) 2023-09-23T19:01:59.797 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:01:59 smithi098 bash[76817]: cluster 2023-09-23T19:01:58.451858+0000 mgr.y (mgr.26054) 1321 : cluster 0 pgmap v1449: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s; 20600/195846 objects degraded (10.518%); 14/195846 objects misplaced (0.007%) 2023-09-23T19:01:59.825 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:01:59.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:59.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:01:59.934 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:59.960 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:01:59.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:01:59.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:00.066 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:00.090 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 51 16 68896 68880 84.3905 61.25 0.0113515 0.0118433 2023-09-23T19:02:00.091 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:00.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:00.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:00.196 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:00.221 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:00.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:00.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:00.325 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:00.359 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:00.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:00.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:00.463 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:00.487 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:00.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:00.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:00.591 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:00.621 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:00.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:00.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:00.724 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:00.732 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:00 smithi098 bash[76817]: cluster 2023-09-23T19:02:00.457383+0000 mon.a (mon.0) 1815 : cluster 3 Health check update: Degraded data redundancy: 20600/195846 objects degraded (10.518%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:00.733 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:00 smithi098 bash[79516]: cluster 2023-09-23T19:02:00.457383+0000 mon.a (mon.0) 1815 : cluster 3 Health check update: Degraded data redundancy: 20600/195846 objects degraded (10.518%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:00.752 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:00.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:00.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:00.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:00 smithi173 bash[59422]: cluster 2023-09-23T19:02:00.457383+0000 mon.a (mon.0) 1815 : cluster 3 Health check update: Degraded data redundancy: 20600/195846 objects degraded (10.518%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:00.859 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:00.892 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:00.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:00.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:00.998 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:01.026 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:01.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:01.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:01.091 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 52 16 69863 69847 83.9299 60.4375 0.00622062 0.0119088 2023-09-23T19:02:01.129 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:01.155 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:01.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:01.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:01.257 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:01.276 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:01.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:01.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:01.378 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:01.404 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:01.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:01.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:01.509 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:01.536 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:01.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:01.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:01.640 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:01.647 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:01 smithi098 bash[79516]: cluster 2023-09-23T19:02:00.453521+0000 mgr.y (mgr.26054) 1322 : cluster 0 pgmap v1450: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.1 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 51 MiB/s wr, 823 op/s; 20943/199701 objects degraded (10.487%); 14/199701 objects misplaced (0.007%) 2023-09-23T19:02:01.647 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:01 smithi098 bash[76817]: cluster 2023-09-23T19:02:00.453521+0000 mgr.y (mgr.26054) 1322 : cluster 0 pgmap v1450: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.1 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 51 MiB/s wr, 823 op/s; 20943/199701 objects degraded (10.487%); 14/199701 objects misplaced (0.007%) 2023-09-23T19:02:01.673 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:01.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:01.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:01.777 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:01.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:01 smithi173 bash[59422]: cluster 2023-09-23T19:02:00.453521+0000 mgr.y (mgr.26054) 1322 : cluster 0 pgmap v1450: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.1 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 51 MiB/s wr, 823 op/s; 20943/199701 objects degraded (10.487%); 14/199701 objects misplaced (0.007%) 2023-09-23T19:02:01.801 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:01.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:01.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:01.905 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:01.932 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:01.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:01.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:02.041 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:02.071 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:02.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:02.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:02.091 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 53 16 70681 70665 83.311 51.125 0.0182998 0.011997 2023-09-23T19:02:02.175 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:02.203 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:02.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:02.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:02.306 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:02.333 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:02.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:02.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:02.436 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:02.464 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:02.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:02.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:02.569 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:02.592 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:02.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:02.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:02.696 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:02.722 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:02.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:02.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:02.825 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:02.850 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:02.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:02.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:02.954 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:02.987 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:02.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:02.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:03.091 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 54 16 71647 71631 82.8862 60.375 0.0127061 0.0120586 2023-09-23T19:02:03.092 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:03.122 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:03.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:03.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:03.225 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:03.260 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:03.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:03.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:03.363 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:03.392 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:03.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:03.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:03.499 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:03.536 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:03.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:03.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:03.640 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:03.669 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:03.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:03.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:03.772 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:03.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:03 smithi173 bash[59422]: cluster 2023-09-23T19:02:02.454228+0000 mgr.y (mgr.26054) 1323 : cluster 0 pgmap v1451: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s; 21531/206745 objects degraded (10.414%); 14/206745 objects misplaced (0.007%) 2023-09-23T19:02:03.781 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:03 smithi098 bash[76817]: cluster 2023-09-23T19:02:02.454228+0000 mgr.y (mgr.26054) 1323 : cluster 0 pgmap v1451: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s; 21531/206745 objects degraded (10.414%); 14/206745 objects misplaced (0.007%) 2023-09-23T19:02:03.782 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:03 smithi098 bash[79516]: cluster 2023-09-23T19:02:02.454228+0000 mgr.y (mgr.26054) 1323 : cluster 0 pgmap v1451: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s; 21531/206745 objects degraded (10.414%); 14/206745 objects misplaced (0.007%) 2023-09-23T19:02:03.803 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:03.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:03.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:03.910 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:03.938 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:03.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:03.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:04.045 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:04.071 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:04.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:04.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:04.091 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 55 16 72538 72522 82.3916 55.6875 0.0182811 0.0121316 2023-09-23T19:02:04.176 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:04.201 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:04.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:04.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:04.304 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:04.341 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:04.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:04.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:04.447 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:04.477 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:04.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:04.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:04.585 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:04.609 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:04.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:04.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:04.712 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:04.736 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:04.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:04.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:04.840 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:04.861 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:04.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:04.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:04.964 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:04.990 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:04.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:04.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:05.091 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 56 16 73533 73517 82.0309 62.1875 0.00615896 0.0121837 2023-09-23T19:02:05.094 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:05.122 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:05.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:05.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:05.225 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:05.253 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:05.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:05.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:05.357 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:05.382 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:05.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:05.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:05.487 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:05.514 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:05.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:05.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:05.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:05.647 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:05.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:05.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:05.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:05 smithi098 bash[79516]: cluster 2023-09-23T19:02:04.455454+0000 mgr.y (mgr.26054) 1324 : cluster 0 pgmap v1452: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 63 MiB/s wr, 1k op/s; 22417/213513 objects degraded (10.499%); 14/213513 objects misplaced (0.007%) 2023-09-23T19:02:05.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:05 smithi098 bash[79516]: cluster 2023-09-23T19:02:05.458647+0000 mon.a (mon.0) 1816 : cluster 3 Health check update: Degraded data redundancy: 22417/213513 objects degraded (10.499%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:05.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:05 smithi098 bash[76817]: cluster 2023-09-23T19:02:04.455454+0000 mgr.y (mgr.26054) 1324 : cluster 0 pgmap v1452: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 63 MiB/s wr, 1k op/s; 22417/213513 objects degraded (10.499%); 14/213513 objects misplaced (0.007%) 2023-09-23T19:02:05.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:05 smithi098 bash[76817]: cluster 2023-09-23T19:02:05.458647+0000 mon.a (mon.0) 1816 : cluster 3 Health check update: Degraded data redundancy: 22417/213513 objects degraded (10.499%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:05.750 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:05.773 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:05.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:05.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:05.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:05 smithi173 bash[59422]: cluster 2023-09-23T19:02:04.455454+0000 mgr.y (mgr.26054) 1324 : cluster 0 pgmap v1452: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 63 MiB/s wr, 1k op/s; 22417/213513 objects degraded (10.499%); 14/213513 objects misplaced (0.007%) 2023-09-23T19:02:05.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:05 smithi173 bash[59422]: cluster 2023-09-23T19:02:05.458647+0000 mon.a (mon.0) 1816 : cluster 3 Health check update: Degraded data redundancy: 22417/213513 objects degraded (10.499%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:05.875 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:05.901 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:05.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:05.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:06.005 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:06.032 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:06.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:06.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:06.091 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 57 15 74550 74535 81.7079 63.625 0.0224524 0.0122334 2023-09-23T19:02:06.136 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:06.161 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:06.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:06.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:06.265 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:06.269 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:02:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:02:06] "GET /metrics HTTP/1.1" 200 36642 "" "Prometheus/2.33.4" 2023-09-23T19:02:06.303 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:06.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:06.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:06.406 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:06.433 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:06.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:06.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:06.536 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:06.572 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:06.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:06.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:06.677 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:06.717 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:06.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:06.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:06.823 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:06.850 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:06.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:06.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:06.954 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:06.979 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:06.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:06.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:07.086 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:07.091 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 58 16 75287 75271 81.0922 46 0.114151 0.0123198 2023-09-23T19:02:07.119 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:07.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:07.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:07.227 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:07.272 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:07.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:07.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:07.377 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:07.408 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:07.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:07.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:07.512 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:07.542 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:07.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:07.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:07.649 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:07.661 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:07 smithi098 bash[76817]: cluster 2023-09-23T19:02:06.456421+0000 mgr.y (mgr.26054) 1325 : cluster 0 pgmap v1453: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 52 MiB/s wr, 825 op/s; 22417/214407 objects degraded (10.455%); 14/214407 objects misplaced (0.007%) 2023-09-23T19:02:07.661 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:07 smithi098 bash[79516]: cluster 2023-09-23T19:02:06.456421+0000 mgr.y (mgr.26054) 1325 : cluster 0 pgmap v1453: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 52 MiB/s wr, 825 op/s; 22417/214407 objects degraded (10.455%); 14/214407 objects misplaced (0.007%) 2023-09-23T19:02:07.681 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:07.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:07.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:07.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:07 smithi173 bash[59422]: cluster 2023-09-23T19:02:06.456421+0000 mgr.y (mgr.26054) 1325 : cluster 0 pgmap v1453: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 52 MiB/s wr, 825 op/s; 22417/214407 objects degraded (10.455%); 14/214407 objects misplaced (0.007%) 2023-09-23T19:02:07.789 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:07.826 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:07.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:07.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:07.929 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:07.963 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:07.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:07.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:08.069 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:08.091 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 59 16 76268 76252 80.7569 61.3125 0.0179455 0.0123774 2023-09-23T19:02:08.102 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:08.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:08.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:08.205 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:08.233 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:08.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:08.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:08.337 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:08.363 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:08.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:08.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:08.466 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:08.499 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:08.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:08.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:08.602 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:08.634 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:08.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:08.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:08.739 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:08.763 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:08.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:08.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:08.869 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:08.894 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:08.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:08.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:08.997 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:09.020 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:09.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:09.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:09.091 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:2023-09-23T19:02:09.088558+0000 min lat: 0.00130179 max lat: 0.429637 avg lat: 0.0124408 2023-09-23T19:02:09.092 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:02:09.092 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 60 16 77165 77149 80.3453 56.0625 0.0118686 0.0124408 2023-09-23T19:02:09.122 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:09.163 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:09.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:09.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:09.266 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:09.318 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:09.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:09.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:09.422 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:09.453 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:09.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:09.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:09.559 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:09.584 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:09.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:09.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:09.604 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:09 smithi098 bash[76817]: cluster 2023-09-23T19:02:08.457190+0000 mgr.y (mgr.26054) 1326 : cluster 0 pgmap v1454: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s; 23576/224013 objects degraded (10.524%); 14/224013 objects misplaced (0.006%) 2023-09-23T19:02:09.605 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:09 smithi098 bash[79516]: cluster 2023-09-23T19:02:08.457190+0000 mgr.y (mgr.26054) 1326 : cluster 0 pgmap v1454: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s; 23576/224013 objects degraded (10.524%); 14/224013 objects misplaced (0.006%) 2023-09-23T19:02:09.688 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:09.715 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:09.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:09.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:09.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:09 smithi173 bash[59422]: cluster 2023-09-23T19:02:08.457190+0000 mgr.y (mgr.26054) 1326 : cluster 0 pgmap v1454: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s; 23576/224013 objects degraded (10.524%); 14/224013 objects misplaced (0.006%) 2023-09-23T19:02:09.818 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:09.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:09.851 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:09.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:09.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:09.979 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:09.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:09.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:10.082 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:10.091 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 61 16 77980 77964 79.8632 50.9375 0.0140251 0.0125154 2023-09-23T19:02:10.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:10.117 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:10.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:10.218 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:10.248 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:10.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:10.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:10.354 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:10.380 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:10.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:10.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:10.484 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:10.510 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:10.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:10.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:10.617 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:10.625 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:10 smithi098 bash[79516]: audit 2023-09-23T19:02:09.638461+0000 mon.a (mon.0) 1817 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:02:10.625 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:10 smithi098 bash[79516]: cluster 2023-09-23T19:02:10.459806+0000 mon.a (mon.0) 1818 : cluster 3 Health check update: Degraded data redundancy: 23576/224013 objects degraded (10.524%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:10.626 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:10 smithi098 bash[76817]: audit 2023-09-23T19:02:09.638461+0000 mon.a (mon.0) 1817 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:02:10.626 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:10 smithi098 bash[76817]: cluster 2023-09-23T19:02:10.459806+0000 mon.a (mon.0) 1818 : cluster 3 Health check update: Degraded data redundancy: 23576/224013 objects degraded (10.524%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:10.641 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:10.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:10.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:10.745 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:10.771 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:10.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:10.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:10.880 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:10.919 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:10.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:10.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:11.023 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:11.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:10 smithi173 bash[59422]: audit 2023-09-23T19:02:09.638461+0000 mon.a (mon.0) 1817 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:02:11.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:10 smithi173 bash[59422]: cluster 2023-09-23T19:02:10.459806+0000 mon.a (mon.0) 1818 : cluster 3 Health check update: Degraded data redundancy: 23576/224013 objects degraded (10.524%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:11.053 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:11.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:11.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:11.092 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 62 16 78897 78881 79.4994 57.3125 0.00742893 0.0125731 2023-09-23T19:02:11.160 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:11.185 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:11.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:11.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:11.293 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:11.321 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:11.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:11.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:11.427 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:11.450 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:11.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:11.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:11.553 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:11.581 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:11.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:11.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:11.685 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:11.714 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:11.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:11.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:11.762 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:11 smithi098 bash[79516]: cluster 2023-09-23T19:02:10.458004+0000 mgr.y (mgr.26054) 1327 : cluster 0 pgmap v1455: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 55 MiB/s wr, 880 op/s; 23881/227565 objects degraded (10.494%); 14/227565 objects misplaced (0.006%) 2023-09-23T19:02:11.762 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:11 smithi098 bash[76817]: cluster 2023-09-23T19:02:10.458004+0000 mgr.y (mgr.26054) 1327 : cluster 0 pgmap v1455: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 55 MiB/s wr, 880 op/s; 23881/227565 objects degraded (10.494%); 14/227565 objects misplaced (0.006%) 2023-09-23T19:02:11.819 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:11.850 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:11.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:11.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:11.956 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:11.997 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:11.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:11.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:12.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:11 smithi173 bash[59422]: cluster 2023-09-23T19:02:10.458004+0000 mgr.y (mgr.26054) 1327 : cluster 0 pgmap v1455: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 55 MiB/s wr, 880 op/s; 23881/227565 objects degraded (10.494%); 14/227565 objects misplaced (0.006%) 2023-09-23T19:02:12.092 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 63 16 79712 79696 79.046 50.9375 0.00939399 0.0126448 2023-09-23T19:02:12.100 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:12.128 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:12.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:12.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:12.232 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:12.256 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:12.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:12.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:12.359 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:12.389 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:12.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:12.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:12.495 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:12.520 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:12.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:12.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:12.624 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:12.650 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:12.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:12.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:12.752 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:12.786 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:12.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:12.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:12.890 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:12.914 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:12.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:12.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:13.017 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:13.043 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:13.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:13.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:13.092 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 64 16 80644 80628 78.721 58.25 0.0242099 0.0126968 2023-09-23T19:02:13.146 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:13.174 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:13.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:13.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:13.281 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:13.307 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:13.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:13.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:13.410 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:13.430 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:13.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:13.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:13.533 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:13.560 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:13.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:13.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:13.664 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:13.689 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:13.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:13.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:13.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:13 smithi098 bash[76817]: cluster 2023-09-23T19:02:12.461734+0000 mgr.y (mgr.26054) 1328 : cluster 0 pgmap v1456: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 60 MiB/s wr, 956 op/s; 24441/234162 objects degraded (10.438%); 14/234162 objects misplaced (0.006%) 2023-09-23T19:02:13.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:13 smithi098 bash[79516]: cluster 2023-09-23T19:02:12.461734+0000 mgr.y (mgr.26054) 1328 : cluster 0 pgmap v1456: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 60 MiB/s wr, 956 op/s; 24441/234162 objects degraded (10.438%); 14/234162 objects misplaced (0.006%) 2023-09-23T19:02:13.793 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:13.822 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:13.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:13.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:13.927 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:13.953 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:13.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:13.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:14.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:13 smithi173 bash[59422]: cluster 2023-09-23T19:02:12.461734+0000 mgr.y (mgr.26054) 1328 : cluster 0 pgmap v1456: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 60 MiB/s wr, 956 op/s; 24441/234162 objects degraded (10.438%); 14/234162 objects misplaced (0.006%) 2023-09-23T19:02:14.057 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:14.085 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:14.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:14.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:14.092 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 65 16 81448 81432 78.283 50.25 0.0211497 0.0127679 2023-09-23T19:02:14.190 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:14.226 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:14.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:14.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:14.329 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:14.352 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:14.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:14.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:14.456 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:14.479 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:14.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:14.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:14.582 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:14.606 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:14.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:14.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:14.709 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:14.732 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:14.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:14.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:14.834 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:14.861 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:14.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:14.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:14.913 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T19:02:14.913 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [ 2023-09-23T19:02:14.914 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.914 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:82186 10.f 10:f89164ff:::benchmark_data_smithi098_230241_object82185:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e321)", 2023-09-23T19:02:14.914 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:14.903362+0000", 2023-09-23T19:02:14.914 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.0060731479999999996, 2023-09-23T19:02:14.914 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0061104000000000002, 2023-09-23T19:02:14.914 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:02:14.914 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "waiting for sub ops", 2023-09-23T19:02:14.914 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:02:14.914 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:02:14.914 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:02:14.914 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 82186 2023-09-23T19:02:14.915 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.915 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:02:14.915 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.915 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:02:14.915 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.903362+0000", 2023-09-23T19:02:14.915 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:02:14.915 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.915 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.915 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:02:14.915 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.903362+0000", 2023-09-23T19:02:14.915 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:02:14.915 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.915 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.916 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:02:14.916 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.903360+0000", 2023-09-23T19:02:14.916 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:02:14.916 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.916 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.916 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:02:14.916 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.903384+0000", 2023-09-23T19:02:14.916 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.3867999999999999e-05 2023-09-23T19:02:14.916 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.916 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.916 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:02:14.916 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.903402+0000", 2023-09-23T19:02:14.916 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.8703999999999999e-05 2023-09-23T19:02:14.917 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.917 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.917 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:02:14.917 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.903412+0000", 2023-09-23T19:02:14.917 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.3070000000000008e-06 2023-09-23T19:02:14.917 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.917 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.917 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:02:14.917 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.903471+0000", 2023-09-23T19:02:14.917 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.9231000000000003e-05 2023-09-23T19:02:14.917 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.917 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.917 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:02:14.918 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.903599+0000", 2023-09-23T19:02:14.918 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00012848800000000001 2023-09-23T19:02:14.918 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.918 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.918 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 2,6", 2023-09-23T19:02:14.918 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.903729+0000", 2023-09-23T19:02:14.918 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00012980700000000001 2023-09-23T19:02:14.918 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:02:14.918 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:02:14.918 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:02:14.918 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.918 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.918 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.45779.0:82179 10.4 e321/317 10:248f10be:::benchmark_data_smithi098_230241_object82178:head v 321'5053, mlcod=321'5050)", 2023-09-23T19:02:14.919 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:14.907219+0000", 2023-09-23T19:02:14.919 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.0022152750000000001, 2023-09-23T19:02:14.919 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.002340684, 2023-09-23T19:02:14.919 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:02:14.919 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T19:02:14.919 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:02:14.919 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.919 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:02:14.919 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.907219+0000", 2023-09-23T19:02:14.919 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:02:14.919 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.919 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.919 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:02:14.920 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.907219+0000", 2023-09-23T19:02:14.920 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:02:14.920 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.920 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.920 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:02:14.920 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.907218+0000", 2023-09-23T19:02:14.920 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T19:02:14.920 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.920 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.920 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:02:14.920 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.907241+0000", 2023-09-23T19:02:14.920 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.2759e-05 2023-09-23T19:02:14.920 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.921 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.921 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:02:14.921 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.907257+0000", 2023-09-23T19:02:14.921 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.6164e-05 2023-09-23T19:02:14.921 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.921 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.921 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:02:14.921 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.907265+0000", 2023-09-23T19:02:14.921 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.1599999999999998e-06 2023-09-23T19:02:14.921 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.921 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.921 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:02:14.921 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.907318+0000", 2023-09-23T19:02:14.922 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.274e-05 2023-09-23T19:02:14.922 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.922 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.922 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:02:14.922 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.907340+0000", 2023-09-23T19:02:14.922 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.1345999999999999e-05 2023-09-23T19:02:14.922 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:02:14.922 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:02:14.922 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:02:14.922 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.922 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.922 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.45779.0:82188 10.d e321/317 10:b1d6db41:::benchmark_data_smithi098_230241_object82187:head v 321'5069, mlcod=321'5066)", 2023-09-23T19:02:14.922 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:14.907284+0000", 2023-09-23T19:02:14.923 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.002150463, 2023-09-23T19:02:14.923 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.002347516, 2023-09-23T19:02:14.923 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:02:14.923 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T19:02:14.923 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:02:14.923 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.923 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:02:14.923 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.907284+0000", 2023-09-23T19:02:14.923 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:02:14.923 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.923 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.923 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:02:14.923 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.907284+0000", 2023-09-23T19:02:14.924 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:02:14.924 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.924 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.924 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:02:14.924 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.907284+0000", 2023-09-23T19:02:14.924 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T19:02:14.924 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.924 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.924 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:02:14.924 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.907522+0000", 2023-09-23T19:02:14.924 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00023784300000000001 2023-09-23T19:02:14.924 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.924 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.925 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:02:14.925 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.907532+0000", 2023-09-23T19:02:14.925 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.9629999999999993e-06 2023-09-23T19:02:14.925 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.925 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.925 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:02:14.925 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.907538+0000", 2023-09-23T19:02:14.925 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.4110000000000001e-06 2023-09-23T19:02:14.925 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.925 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.925 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:02:14.925 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.907605+0000", 2023-09-23T19:02:14.925 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.7005e-05 2023-09-23T19:02:14.926 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.926 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.926 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:02:14.926 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.907624+0000", 2023-09-23T19:02:14.926 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.9276e-05 2023-09-23T19:02:14.926 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:02:14.926 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:02:14.926 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:02:14.926 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.926 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.926 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop_reply(client.45779.0:82186 10.f e321/317)", 2023-09-23T19:02:14.926 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:14.909322+0000", 2023-09-23T19:02:14.927 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 0.000113047, 2023-09-23T19:02:14.927 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00036822800000000001, 2023-09-23T19:02:14.927 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:02:14.927 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "queued for pg", 2023-09-23T19:02:14.927 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:02:14.927 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.927 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:02:14.927 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.909322+0000", 2023-09-23T19:02:14.927 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:02:14.927 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.927 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.927 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:02:14.928 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.909322+0000", 2023-09-23T19:02:14.928 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:02:14.928 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.928 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.928 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:02:14.928 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.909316+0000", 2023-09-23T19:02:14.928 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999948 2023-09-23T19:02:14.928 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.928 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.928 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:02:14.928 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.909330+0000", 2023-09-23T19:02:14.928 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3577000000000001e-05 2023-09-23T19:02:14.928 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.928 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.929 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:02:14.929 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.909353+0000", 2023-09-23T19:02:14.929 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.2639000000000001e-05 2023-09-23T19:02:14.929 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:02:14.929 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:02:14.929 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:02:14.929 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:14.909364+0000", 2023-09-23T19:02:14.929 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1732e-05 2023-09-23T19:02:14.929 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:02:14.929 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:02:14.929 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:02:14.929 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:02:14.930 INFO:teuthology.orchestra.run.smithi098.stdout: ], 2023-09-23T19:02:14.930 INFO:teuthology.orchestra.run.smithi098.stdout: "num_ops": 4 2023-09-23T19:02:14.930 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T19:02:14.964 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:14.988 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:14.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:14.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:15.092 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:15.092 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 66 16 82366 82350 77.9662 57.375 0.0117041 0.0128198 2023-09-23T19:02:15.119 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:15.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:15.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:15.222 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:15.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:15.250 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:15.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:15.350 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:15.379 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:15.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:15.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:15.482 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:15.507 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:15.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:15.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:15.611 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:15.648 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:15.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:15.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:15.754 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:15.767 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:15 smithi173 bash[59422]: cluster 2023-09-23T19:02:14.462561+0000 mgr.y (mgr.26054) 1329 : cluster 0 pgmap v1457: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 942 op/s; 25215/240693 objects degraded (10.476%); 14/240693 objects misplaced (0.006%) 2023-09-23T19:02:15.767 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:15 smithi173 bash[59422]: cluster 2023-09-23T19:02:15.460944+0000 mon.a (mon.0) 1819 : cluster 3 Health check update: Degraded data redundancy: 25215/240693 objects degraded (10.476%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:15.785 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:15 smithi098 bash[76817]: cluster 2023-09-23T19:02:14.462561+0000 mgr.y (mgr.26054) 1329 : cluster 0 pgmap v1457: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 942 op/s; 25215/240693 objects degraded (10.476%); 14/240693 objects misplaced (0.006%) 2023-09-23T19:02:15.785 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:15 smithi098 bash[76817]: cluster 2023-09-23T19:02:15.460944+0000 mon.a (mon.0) 1819 : cluster 3 Health check update: Degraded data redundancy: 25215/240693 objects degraded (10.476%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:15.785 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:15 smithi098 bash[79516]: cluster 2023-09-23T19:02:14.462561+0000 mgr.y (mgr.26054) 1329 : cluster 0 pgmap v1457: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 942 op/s; 25215/240693 objects degraded (10.476%); 14/240693 objects misplaced (0.006%) 2023-09-23T19:02:15.785 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:15 smithi098 bash[79516]: cluster 2023-09-23T19:02:15.460944+0000 mon.a (mon.0) 1819 : cluster 3 Health check update: Degraded data redundancy: 25215/240693 objects degraded (10.476%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:15.787 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:15.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:15.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:15.894 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:15.921 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:15.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:15.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:16.026 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:16.048 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:16.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:16.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:16.092 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 67 16 83158 83142 77.5413 49.5 0.00330769 0.012873 2023-09-23T19:02:16.151 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:16.192 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:16.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:16.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:16.296 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:16.324 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:16.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:16.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:16.427 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:16.438 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:02:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:02:16] "GET /metrics HTTP/1.1" 200 36645 "" "Prometheus/2.33.4" 2023-09-23T19:02:16.451 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:16.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:16.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:16.557 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:16.589 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:16.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:16.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:16.693 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:16.720 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:16.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:16.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:16.823 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:16.849 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:16.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:16.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:16.952 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:16.985 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:16.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:16.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:17.087 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:17.092 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 68 16 84007 83991 77.1813 53.0625 0.0121023 0.0129504 2023-09-23T19:02:17.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:17.131 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:17.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:17.232 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:17.260 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:17.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:17.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:17.363 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:17.397 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:17.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:17.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:17.503 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:17.529 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:17.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:17.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:17.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:17.658 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:17.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:17.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:17.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:17 smithi098 bash[76817]: cluster 2023-09-23T19:02:16.463199+0000 mgr.y (mgr.26054) 1330 : cluster 0 pgmap v1458: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 49 MiB/s wr, 778 op/s; 25215/241545 objects degraded (10.439%); 14/241545 objects misplaced (0.006%) 2023-09-23T19:02:17.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:17 smithi098 bash[79516]: cluster 2023-09-23T19:02:16.463199+0000 mgr.y (mgr.26054) 1330 : cluster 0 pgmap v1458: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 49 MiB/s wr, 778 op/s; 25215/241545 objects degraded (10.439%); 14/241545 objects misplaced (0.006%) 2023-09-23T19:02:17.761 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:17.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:17 smithi173 bash[59422]: cluster 2023-09-23T19:02:16.463199+0000 mgr.y (mgr.26054) 1330 : cluster 0 pgmap v1458: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 49 MiB/s wr, 778 op/s; 25215/241545 objects degraded (10.439%); 14/241545 objects misplaced (0.006%) 2023-09-23T19:02:17.791 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:17.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:17.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:17.895 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:17.920 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:17.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:17.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:18.023 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:18.050 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:18.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:18.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:18.092 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 69 16 84838 84822 76.8154 51.9375 0.00380339 0.0130039 2023-09-23T19:02:18.153 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:18.189 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:18.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:18.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:18.296 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:18.324 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:18.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:18.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:18.427 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:18.461 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:18.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:18.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:18.567 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:18.593 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:18.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:18.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:18.696 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:18.724 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:18.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:18.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:18.828 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:18.856 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:18.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:18.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:18.961 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:18.985 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:18.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:18.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:19.060 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T19:02:19.087 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:19.097 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 70 16 85660 85644 76.4519 51.375 0.01764 0.0130735 2023-09-23T19:02:19.143 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:19.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:19.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:19.249 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:19.294 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:19.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:19.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:19.310 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T19:02:19.373 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T19:02:19.405 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:19.419 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T19:02:19.433 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:19.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:19.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:19.467 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T19:02:19.509 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T19:02:19.538 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:19.554 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T19:02:19.565 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:19 smithi098 bash[76817]: cluster 2023-09-23T19:02:18.463975+0000 mgr.y (mgr.26054) 1331 : cluster 0 pgmap v1459: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 62 MiB/s wr, 992 op/s; 26315/250167 objects degraded (10.519%); 14/250167 objects misplaced (0.006%) 2023-09-23T19:02:19.565 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:19 smithi098 bash[76817]: audit 2023-09-23T19:02:19.237528+0000 mon.b (mon.1) 27 : audit 0 from='client.? 172.21.15.98:0/556665106' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:02:19.566 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:19.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:19.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:19.588 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T19:02:19.594 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:02:19.594 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:02:19.594 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:02:19.594 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:02:19.594 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:02:19.594 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:02:19.594 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:02:19.594 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T19:02:19.671 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:19.680 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:19 smithi173 bash[59422]: cluster 2023-09-23T19:02:18.463975+0000 mgr.y (mgr.26054) 1331 : cluster 0 pgmap v1459: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 62 MiB/s wr, 992 op/s; 26315/250167 objects degraded (10.519%); 14/250167 objects misplaced (0.006%) 2023-09-23T19:02:19.680 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:19 smithi173 bash[59422]: audit 2023-09-23T19:02:19.237528+0000 mon.b (mon.1) 27 : audit 0 from='client.? 172.21.15.98:0/556665106' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:02:19.700 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:19.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:19.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:19.804 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:19.814 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:19 smithi098 bash[79516]: cluster 2023-09-23T19:02:18.463975+0000 mgr.y (mgr.26054) 1331 : cluster 0 pgmap v1459: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 62 MiB/s wr, 992 op/s; 26315/250167 objects degraded (10.519%); 14/250167 objects misplaced (0.006%) 2023-09-23T19:02:19.815 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:19 smithi098 bash[79516]: audit 2023-09-23T19:02:19.237528+0000 mon.b (mon.1) 27 : audit 0 from='client.? 172.21.15.98:0/556665106' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:02:19.849 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:19.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:19.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:19.928 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-09-23T19:02:19.952 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:19.981 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:19.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:19.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:20.089 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:20.092 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 71 16 86345 86329 75.9781 42.8125 0.0306156 0.0131558 2023-09-23T19:02:20.126 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:20.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:20.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:20.230 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:20.260 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:20.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:20.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:20.364 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:20.391 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:20.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:20.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:20.496 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:20.520 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:20.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:20.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:20.586 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is unset 2023-09-23T19:02:20.623 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:20.632 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:20 smithi173 bash[59422]: audit 2023-09-23T19:02:19.921464+0000 mon.a (mon.0) 1820 : audit 1 from='client.? 172.21.15.98:0/3823700572' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T19:02:20.632 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:20 smithi173 bash[59422]: audit 2023-09-23T19:02:19.936571+0000 mon.c (mon.2) 144 : audit 1 from='client.? 172.21.15.98:0/381503093' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:02:20.633 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:20 smithi173 bash[59422]: audit 2023-09-23T19:02:19.937482+0000 mon.a (mon.0) 1821 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:02:20.633 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:20 smithi173 bash[59422]: cluster 2023-09-23T19:02:20.462380+0000 mon.a (mon.0) 1822 : cluster 3 Health check update: Degraded data redundancy: 26315/250167 objects degraded (10.519%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:20.648 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:20.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:20.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:20.753 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:20.781 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:20.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:20.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:20.885 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:20.913 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:20.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:20.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:20.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:20 smithi098 bash[76817]: audit 2023-09-23T19:02:19.921464+0000 mon.a (mon.0) 1820 : audit 1 from='client.? 172.21.15.98:0/3823700572' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T19:02:20.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:20 smithi098 bash[76817]: audit 2023-09-23T19:02:19.936571+0000 mon.c (mon.2) 144 : audit 1 from='client.? 172.21.15.98:0/381503093' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:02:20.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:20 smithi098 bash[76817]: audit 2023-09-23T19:02:19.937482+0000 mon.a (mon.0) 1821 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:02:20.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:20 smithi098 bash[76817]: cluster 2023-09-23T19:02:20.462380+0000 mon.a (mon.0) 1822 : cluster 3 Health check update: Degraded data redundancy: 26315/250167 objects degraded (10.519%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:20.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:20 smithi098 bash[79516]: audit 2023-09-23T19:02:19.921464+0000 mon.a (mon.0) 1820 : audit 1 from='client.? 172.21.15.98:0/3823700572' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T19:02:20.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:20 smithi098 bash[79516]: audit 2023-09-23T19:02:19.936571+0000 mon.c (mon.2) 144 : audit 1 from='client.? 172.21.15.98:0/381503093' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:02:20.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:20 smithi098 bash[79516]: audit 2023-09-23T19:02:19.937482+0000 mon.a (mon.0) 1821 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:02:20.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:20 smithi098 bash[79516]: cluster 2023-09-23T19:02:20.462380+0000 mon.a (mon.0) 1822 : cluster 3 Health check update: Degraded data redundancy: 26315/250167 objects degraded (10.519%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:21.016 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:21.042 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:21.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:21.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:21.093 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 72 16 87219 87203 75.6815 54.625 0.00683575 0.0132038 2023-09-23T19:02:21.144 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:21.176 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:21.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:21.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:21.280 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:21.306 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:21.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:21.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:21.409 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:21.443 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:21.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:21.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:21.548 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:21.575 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:21.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:21.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:21.678 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:21.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:21 smithi098 bash[76817]: cluster 2023-09-23T19:02:20.464561+0000 mgr.y (mgr.26054) 1332 : cluster 0 pgmap v1460: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 5.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 51 MiB/s wr, 810 op/s; 26548/253206 objects degraded (10.485%); 14/253206 objects misplaced (0.006%) 2023-09-23T19:02:21.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:21 smithi098 bash[76817]: cluster 2023-09-23T19:02:20.560205+0000 mon.a (mon.0) 1823 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:02:21.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:21 smithi098 bash[76817]: cluster 2023-09-23T19:02:20.568918+0000 osd.5 (osd.5) 248 : cluster 3 failed to encode map e322 with expected crc 2023-09-23T19:02:21.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:21 smithi098 bash[76817]: audit 2023-09-23T19:02:20.569483+0000 mon.a (mon.0) 1824 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:02:21.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:21 smithi098 bash[76817]: cluster 2023-09-23T19:02:20.569537+0000 mon.a (mon.0) 1825 : cluster 0 osdmap e322: 8 total, 7 up, 7 in 2023-09-23T19:02:21.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:21 smithi098 bash[76817]: cluster 2023-09-23T19:02:20.570179+0000 osd.6 (osd.6) 295 : cluster 3 failed to encode map e322 with expected crc 2023-09-23T19:02:21.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:21 smithi098 bash[76817]: cluster 2023-09-23T19:02:20.585723+0000 osd.4 (osd.4) 52 : cluster 3 failed to encode map e322 with expected crc 2023-09-23T19:02:21.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:21 smithi098 bash[79516]: cluster 2023-09-23T19:02:20.464561+0000 mgr.y (mgr.26054) 1332 : cluster 0 pgmap v1460: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 5.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 51 MiB/s wr, 810 op/s; 26548/253206 objects degraded (10.485%); 14/253206 objects misplaced (0.006%) 2023-09-23T19:02:21.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:21 smithi098 bash[79516]: cluster 2023-09-23T19:02:20.560205+0000 mon.a (mon.0) 1823 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:02:21.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:21 smithi098 bash[79516]: cluster 2023-09-23T19:02:20.568918+0000 osd.5 (osd.5) 248 : cluster 3 failed to encode map e322 with expected crc 2023-09-23T19:02:21.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:21 smithi098 bash[79516]: audit 2023-09-23T19:02:20.569483+0000 mon.a (mon.0) 1824 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:02:21.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:21 smithi098 bash[79516]: cluster 2023-09-23T19:02:20.569537+0000 mon.a (mon.0) 1825 : cluster 0 osdmap e322: 8 total, 7 up, 7 in 2023-09-23T19:02:21.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:21 smithi098 bash[79516]: cluster 2023-09-23T19:02:20.570179+0000 osd.6 (osd.6) 295 : cluster 3 failed to encode map e322 with expected crc 2023-09-23T19:02:21.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:21 smithi098 bash[79516]: cluster 2023-09-23T19:02:20.585723+0000 osd.4 (osd.4) 52 : cluster 3 failed to encode map e322 with expected crc 2023-09-23T19:02:21.706 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:21.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:21.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:21.809 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:21.819 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:21 smithi173 bash[59422]: cluster 2023-09-23T19:02:20.464561+0000 mgr.y (mgr.26054) 1332 : cluster 0 pgmap v1460: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 5.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 51 MiB/s wr, 810 op/s; 26548/253206 objects degraded (10.485%); 14/253206 objects misplaced (0.006%) 2023-09-23T19:02:21.819 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:21 smithi173 bash[59422]: cluster 2023-09-23T19:02:20.560205+0000 mon.a (mon.0) 1823 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:02:21.819 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:21 smithi173 bash[59422]: cluster 2023-09-23T19:02:20.568918+0000 osd.5 (osd.5) 248 : cluster 3 failed to encode map e322 with expected crc 2023-09-23T19:02:21.819 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:21 smithi173 bash[59422]: audit 2023-09-23T19:02:20.569483+0000 mon.a (mon.0) 1824 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:02:21.820 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:21 smithi173 bash[59422]: cluster 2023-09-23T19:02:20.569537+0000 mon.a (mon.0) 1825 : cluster 0 osdmap e322: 8 total, 7 up, 7 in 2023-09-23T19:02:21.820 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:21 smithi173 bash[59422]: cluster 2023-09-23T19:02:20.570179+0000 osd.6 (osd.6) 295 : cluster 3 failed to encode map e322 with expected crc 2023-09-23T19:02:21.820 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:21 smithi173 bash[59422]: cluster 2023-09-23T19:02:20.585723+0000 osd.4 (osd.4) 52 : cluster 3 failed to encode map e322 with expected crc 2023-09-23T19:02:21.833 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:21.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:21.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:21.937 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:21.961 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:21.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:21.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:22.066 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:22.093 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 73 16 88077 88061 75.3794 53.625 0.0182581 0.0132605 2023-09-23T19:02:22.095 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:22.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:22.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:22.200 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:22.225 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:22.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:22.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:22.331 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:22.357 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:22.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:22.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:22.461 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:22.488 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:22.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:22.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:22.593 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:22.611 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:22.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:22.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:22.714 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:22.743 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:22.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:22.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:22.847 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:22.870 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:22.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:22.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:22.973 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:23.004 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:23.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:23.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:23.057 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:02:23.058 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:02:23.093 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 74 16 88792 88776 74.9646 44.6875 0.0105834 0.0133345 2023-09-23T19:02:23.107 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:23.133 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:23.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:23.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:23.238 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:23.268 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:23.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:23.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:23.371 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:23.405 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:23.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:23.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:23.512 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:23.542 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:23.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:23.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:23.646 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:23.655 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:23 smithi098 bash[76817]: cluster 2023-09-23T19:02:22.465332+0000 mgr.y (mgr.26054) 1333 : cluster 0 pgmap v1462: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 53 MiB/s wr, 846 op/s; 27073/259557 objects degraded (10.430%); 14/259557 objects misplaced (0.005%) 2023-09-23T19:02:23.655 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:23 smithi098 bash[79516]: cluster 2023-09-23T19:02:22.465332+0000 mgr.y (mgr.26054) 1333 : cluster 0 pgmap v1462: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 53 MiB/s wr, 846 op/s; 27073/259557 objects degraded (10.430%); 14/259557 objects misplaced (0.005%) 2023-09-23T19:02:23.679 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:23.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:23.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:23.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:23 smithi173 bash[59422]: cluster 2023-09-23T19:02:22.465332+0000 mgr.y (mgr.26054) 1333 : cluster 0 pgmap v1462: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 53 MiB/s wr, 846 op/s; 27073/259557 objects degraded (10.430%); 14/259557 objects misplaced (0.005%) 2023-09-23T19:02:23.783 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:23.809 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:23.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:23.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:23.913 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:23.936 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:23.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:23.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:24.045 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:24.071 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:24.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:24.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:24.093 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 75 16 89658 89642 74.6867 54.125 0.0105217 0.0133834 2023-09-23T19:02:24.173 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:24.205 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:24.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:24.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:24.227 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-09-23T19:02:24.310 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:24.337 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:24.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:24.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:24.440 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:24.471 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:24.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:24.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:24.577 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:24.599 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:24.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:24.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:24.702 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:24.734 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:24.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:24.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:24.838 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:24.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:24.867 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:24.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:24.968 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:24.995 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:24.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:24.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:25.058 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:02:25.093 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 76 16 90508 90492 74.403 53.125 0.0173327 0.0134342 2023-09-23T19:02:25.097 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:25.127 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:25.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:25.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:25.230 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:25.258 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:25.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:25.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:25.364 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:25.406 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:25.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:25.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:25.515 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:25.541 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:25.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:25.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:25.644 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:25.653 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:25 smithi098 bash[79516]: cluster 2023-09-23T19:02:24.465940+0000 mgr.y (mgr.26054) 1334 : cluster 0 pgmap v1463: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 52 MiB/s wr, 825 op/s; 27841/265452 objects degraded (10.488%); 14/265452 objects misplaced (0.005%) 2023-09-23T19:02:25.653 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:25 smithi098 bash[79516]: audit 2023-09-23T19:02:24.639354+0000 mon.a (mon.0) 1826 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:02:25.653 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:25 smithi098 bash[79516]: cluster 2023-09-23T19:02:25.463588+0000 mon.a (mon.0) 1827 : cluster 3 Health check update: Degraded data redundancy: 27841/265452 objects degraded (10.488%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:25.654 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:25 smithi098 bash[76817]: cluster 2023-09-23T19:02:24.465940+0000 mgr.y (mgr.26054) 1334 : cluster 0 pgmap v1463: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 52 MiB/s wr, 825 op/s; 27841/265452 objects degraded (10.488%); 14/265452 objects misplaced (0.005%) 2023-09-23T19:02:25.654 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:25 smithi098 bash[76817]: audit 2023-09-23T19:02:24.639354+0000 mon.a (mon.0) 1826 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:02:25.654 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:25 smithi098 bash[76817]: cluster 2023-09-23T19:02:25.463588+0000 mon.a (mon.0) 1827 : cluster 3 Health check update: Degraded data redundancy: 27841/265452 objects degraded (10.488%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:25.676 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:25.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:25.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:25.785 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:25.815 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:25.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:25.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:25.919 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:25.927 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:25 smithi173 bash[59422]: cluster 2023-09-23T19:02:24.465940+0000 mgr.y (mgr.26054) 1334 : cluster 0 pgmap v1463: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 52 MiB/s wr, 825 op/s; 27841/265452 objects degraded (10.488%); 14/265452 objects misplaced (0.005%) 2023-09-23T19:02:25.928 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:25 smithi173 bash[59422]: audit 2023-09-23T19:02:24.639354+0000 mon.a (mon.0) 1826 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:02:25.928 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:25 smithi173 bash[59422]: cluster 2023-09-23T19:02:25.463588+0000 mon.a (mon.0) 1827 : cluster 3 Health check update: Degraded data redundancy: 27841/265452 objects degraded (10.488%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:25.943 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:25.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:25.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:26.046 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:26.070 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:26.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:26.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:26.093 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 77 16 91337 91321 74.1095 51.8125 0.0168633 0.013487 2023-09-23T19:02:26.176 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:26.205 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:26.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:26.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:26.238 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:02:26.313 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:26.322 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:02:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:02:26] "GET /metrics HTTP/1.1" 200 36644 "" "Prometheus/2.33.4" 2023-09-23T19:02:26.351 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:26.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:26.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:26.457 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:26.493 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:26.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:26.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:26.598 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:26.625 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:26.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:26.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:26.729 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:26.761 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:26.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:26.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:26.865 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:26.893 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:26.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:26.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:26.996 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:27.025 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:27.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:27.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:27.093 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 78 16 92105 92089 73.7748 48 0.0150592 0.0135483 2023-09-23T19:02:27.129 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:27.160 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:27.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:27.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:27.264 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:27.293 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:27.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:27.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:27.397 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:27.423 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:27.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:27.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:27.527 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:27.552 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:27.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:27.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:27.658 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:27.683 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:27.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:27.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:27.761 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:02:27.787 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:27.821 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:27.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:27.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:27.928 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:27.937 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:27 smithi173 bash[59422]: cluster 2023-09-23T19:02:26.466617+0000 mgr.y (mgr.26054) 1335 : cluster 0 pgmap v1464: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 5.4 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 823 op/s; 27841/266244 objects degraded (10.457%); 14/266244 objects misplaced (0.005%) 2023-09-23T19:02:27.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:27 smithi098 bash[76817]: cluster 2023-09-23T19:02:26.466617+0000 mgr.y (mgr.26054) 1335 : cluster 0 pgmap v1464: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 5.4 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 823 op/s; 27841/266244 objects degraded (10.457%); 14/266244 objects misplaced (0.005%) 2023-09-23T19:02:27.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:27 smithi098 bash[79516]: cluster 2023-09-23T19:02:26.466617+0000 mgr.y (mgr.26054) 1335 : cluster 0 pgmap v1464: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 5.4 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 823 op/s; 27841/266244 objects degraded (10.457%); 14/266244 objects misplaced (0.005%) 2023-09-23T19:02:27.954 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:27.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:27.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:28.057 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:28.080 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:28.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:28.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:28.093 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 79 16 92905 92889 73.4738 50 0.00690399 0.0136046 2023-09-23T19:02:28.186 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:28.217 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:28.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:28.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:28.321 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:28.355 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:28.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:28.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:28.465 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:28.494 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:28.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:28.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:28.597 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:28.640 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:28.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:28.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:28.749 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:28.777 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:28.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:28.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:28.879 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:28.911 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:28.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:28.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:29.017 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:29.042 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:29.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:29.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:29.093 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:2023-09-23T19:02:29.090463+0000 min lat: 0.00130179 max lat: 0.429637 avg lat: 0.0136557 2023-09-23T19:02:29.094 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T19:02:29.094 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 80 16 93727 93711 73.1975 51.375 0.0223123 0.0136557 2023-09-23T19:02:29.145 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:29.172 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:02:29.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:29.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:02:29.228 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 6, 7, 5, 3] out_osds: [4] dead_osds: [7] live_osds: [0, 2, 3, 5, 6, 1, 4] 2023-09-23T19:02:29.228 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:02:29.229 INFO:tasks.thrashosds.thrasher:Killing osd 5, live_osds are [0, 2, 3, 5, 6, 1, 4] 2023-09-23T19:02:29.229 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-09-23T19:02:29.229 DEBUG:teuthology.orchestra.run.smithi173:> sudo systemctl stop ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.5 2023-09-23T19:02:29.275 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:29.278 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:02:29 smithi173 systemd[1]: Stopping Ceph osd.5 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T19:02:29.300 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:29.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:29.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:29.404 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:29.435 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:29.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:29.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:29.540 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:29.575 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:29.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:29.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:29.682 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:29.693 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:29 smithi173 bash[59422]: cluster 2023-09-23T19:02:28.469028+0000 mgr.y (mgr.26054) 1336 : cluster 0 pgmap v1465: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s; 28891/274767 objects degraded (10.515%); 14/274767 objects misplaced (0.005%) 2023-09-23T19:02:29.693 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:29 smithi173 bash[59422]: cluster 2023-09-23T19:02:29.392142+0000 mon.a (mon.0) 1828 : cluster 1 osd.5 marked itself down and dead 2023-09-23T19:02:29.693 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:02:29 smithi173 bash[35394]: debug 2023-09-23T19:02:29.391+0000 7f2d73cb4700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T19:02:29.694 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:02:29 smithi173 bash[35394]: debug 2023-09-23T19:02:29.391+0000 7f2d73cb4700 -1 osd.5 322 *** Got signal Terminated *** 2023-09-23T19:02:29.694 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:02:29 smithi173 bash[35394]: debug 2023-09-23T19:02:29.391+0000 7f2d73cb4700 -1 osd.5 322 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T19:02:29.712 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:29.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:29.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:29.810 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:02:29.815 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:29.828 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:29 smithi098 bash[79516]: cluster 2023-09-23T19:02:28.469028+0000 mgr.y (mgr.26054) 1336 : cluster 0 pgmap v1465: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s; 28891/274767 objects degraded (10.515%); 14/274767 objects misplaced (0.005%) 2023-09-23T19:02:29.828 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:29 smithi098 bash[79516]: cluster 2023-09-23T19:02:29.392142+0000 mon.a (mon.0) 1828 : cluster 1 osd.5 marked itself down and dead 2023-09-23T19:02:29.828 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:29 smithi098 bash[76817]: cluster 2023-09-23T19:02:28.469028+0000 mgr.y (mgr.26054) 1336 : cluster 0 pgmap v1465: 89 pgs: 21 active+clean+remapped, 56 active+clean, 7 active+undersized+degraded, 5 active+undersized; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s; 28891/274767 objects degraded (10.515%); 14/274767 objects misplaced (0.005%) 2023-09-23T19:02:29.828 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:29 smithi098 bash[76817]: cluster 2023-09-23T19:02:29.392142+0000 mon.a (mon.0) 1828 : cluster 1 osd.5 marked itself down and dead 2023-09-23T19:02:29.841 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:29.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:29.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:29.944 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:29.973 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:29.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:29.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:30.077 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:30.093 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 81 16 94181 94165 72.6442 28.375 0.005345 0.0136806 2023-09-23T19:02:30.106 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:30.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:30.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:30.210 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:30.239 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:30.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:30.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:30.343 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:30.377 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:30.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:30.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:30.481 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:30.509 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:30.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:30.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:30.613 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:30.648 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:30.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:30.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:30.750 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:30.758 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:30 smithi098 bash[79516]: cluster 2023-09-23T19:02:29.594095+0000 mon.a (mon.0) 1829 : cluster 3 Health check update: 2 osds down (OSD_DOWN) 2023-09-23T19:02:30.758 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:30 smithi098 bash[79516]: cluster 2023-09-23T19:02:29.599680+0000 osd.4 (osd.4) 53 : cluster 3 failed to encode map e323 with expected crc 2023-09-23T19:02:30.758 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:30 smithi098 bash[79516]: cluster 2023-09-23T19:02:29.599744+0000 osd.6 (osd.6) 296 : cluster 3 failed to encode map e323 with expected crc 2023-09-23T19:02:30.758 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:30 smithi098 bash[79516]: cluster 2023-09-23T19:02:29.599891+0000 mon.a (mon.0) 1830 : cluster 0 osdmap e323: 8 total, 6 up, 7 in 2023-09-23T19:02:30.758 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:30 smithi098 bash[79516]: cluster 2023-09-23T19:02:29.601774+0000 osd.6 (osd.6) 297 : cluster 3 failed to encode map e323 with expected crc 2023-09-23T19:02:30.759 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:30 smithi098 bash[79516]: cluster 2023-09-23T19:02:30.464430+0000 mon.a (mon.0) 1831 : cluster 3 Health check update: Degraded data redundancy: 28891/274767 objects degraded (10.515%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:30.759 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:30 smithi098 bash[76817]: cluster 2023-09-23T19:02:29.594095+0000 mon.a (mon.0) 1829 : cluster 3 Health check update: 2 osds down (OSD_DOWN) 2023-09-23T19:02:30.759 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:30 smithi098 bash[76817]: cluster 2023-09-23T19:02:29.599680+0000 osd.4 (osd.4) 53 : cluster 3 failed to encode map e323 with expected crc 2023-09-23T19:02:30.759 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:30 smithi098 bash[76817]: cluster 2023-09-23T19:02:29.599744+0000 osd.6 (osd.6) 296 : cluster 3 failed to encode map e323 with expected crc 2023-09-23T19:02:30.759 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:30 smithi098 bash[76817]: cluster 2023-09-23T19:02:29.599891+0000 mon.a (mon.0) 1830 : cluster 0 osdmap e323: 8 total, 6 up, 7 in 2023-09-23T19:02:30.759 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:30 smithi098 bash[76817]: cluster 2023-09-23T19:02:29.601774+0000 osd.6 (osd.6) 297 : cluster 3 failed to encode map e323 with expected crc 2023-09-23T19:02:30.759 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:30 smithi098 bash[76817]: cluster 2023-09-23T19:02:30.464430+0000 mon.a (mon.0) 1831 : cluster 3 Health check update: Degraded data redundancy: 28891/274767 objects degraded (10.515%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:30.783 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:30.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:30.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:30.823 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:02:30.889 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:30.900 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:30 smithi173 bash[59422]: cluster 2023-09-23T19:02:29.594095+0000 mon.a (mon.0) 1829 : cluster 3 Health check update: 2 osds down (OSD_DOWN) 2023-09-23T19:02:30.900 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:30 smithi173 bash[59422]: cluster 2023-09-23T19:02:29.599680+0000 osd.4 (osd.4) 53 : cluster 3 failed to encode map e323 with expected crc 2023-09-23T19:02:30.900 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:30 smithi173 bash[59422]: cluster 2023-09-23T19:02:29.599744+0000 osd.6 (osd.6) 296 : cluster 3 failed to encode map e323 with expected crc 2023-09-23T19:02:30.900 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:30 smithi173 bash[59422]: cluster 2023-09-23T19:02:29.599891+0000 mon.a (mon.0) 1830 : cluster 0 osdmap e323: 8 total, 6 up, 7 in 2023-09-23T19:02:30.900 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:30 smithi173 bash[59422]: cluster 2023-09-23T19:02:29.601774+0000 osd.6 (osd.6) 297 : cluster 3 failed to encode map e323 with expected crc 2023-09-23T19:02:30.901 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:30 smithi173 bash[59422]: cluster 2023-09-23T19:02:30.464430+0000 mon.a (mon.0) 1831 : cluster 3 Health check update: Degraded data redundancy: 28891/274767 objects degraded (10.515%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:30.914 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:30.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:30.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:31.017 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:31.040 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:31.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:31.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:31.093 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 82 16 94543 94527 72.0342 22.625 0.00432746 0.0138721 2023-09-23T19:02:31.143 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:31.165 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:31.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:31.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:31.272 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:31.302 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:31.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:31.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:31.405 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:31.416 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:02:31 smithi173 bash[147649]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-5 2023-09-23T19:02:31.436 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:31.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:31.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:31.541 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:31.583 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:31.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:31.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:31.685 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:31.713 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:31.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:31.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:31.778 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:02:31 smithi173 bash[147788]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-5 2023-09-23T19:02:31.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:31 smithi173 bash[59422]: cluster 2023-09-23T19:02:30.469892+0000 mgr.y (mgr.26054) 1337 : cluster 0 pgmap v1467: 89 pgs: 2 stale+active+clean+remapped, 9 stale+active+clean, 19 active+clean+remapped, 47 active+clean, 7 active+undersized+degraded, 5 active+undersized; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 822 op/s; 29138/277650 objects degraded (10.495%); 14/277650 objects misplaced (0.005%) 2023-09-23T19:02:31.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:31 smithi173 bash[59422]: cluster 2023-09-23T19:02:30.602566+0000 osd.4 (osd.4) 54 : cluster 3 failed to encode map e324 with expected crc 2023-09-23T19:02:31.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:31 smithi173 bash[59422]: cluster 2023-09-23T19:02:30.603595+0000 mon.a (mon.0) 1832 : cluster 0 osdmap e324: 8 total, 6 up, 7 in 2023-09-23T19:02:31.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:31 smithi173 bash[59422]: cluster 2023-09-23T19:02:30.610811+0000 osd.6 (osd.6) 298 : cluster 3 failed to encode map e324 with expected crc 2023-09-23T19:02:31.816 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:31.825 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:31 smithi098 bash[76817]: cluster 2023-09-23T19:02:30.469892+0000 mgr.y (mgr.26054) 1337 : cluster 0 pgmap v1467: 89 pgs: 2 stale+active+clean+remapped, 9 stale+active+clean, 19 active+clean+remapped, 47 active+clean, 7 active+undersized+degraded, 5 active+undersized; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 822 op/s; 29138/277650 objects degraded (10.495%); 14/277650 objects misplaced (0.005%) 2023-09-23T19:02:31.825 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:31 smithi098 bash[76817]: cluster 2023-09-23T19:02:30.602566+0000 osd.4 (osd.4) 54 : cluster 3 failed to encode map e324 with expected crc 2023-09-23T19:02:31.825 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:31 smithi098 bash[76817]: cluster 2023-09-23T19:02:30.603595+0000 mon.a (mon.0) 1832 : cluster 0 osdmap e324: 8 total, 6 up, 7 in 2023-09-23T19:02:31.825 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:31 smithi098 bash[76817]: cluster 2023-09-23T19:02:30.610811+0000 osd.6 (osd.6) 298 : cluster 3 failed to encode map e324 with expected crc 2023-09-23T19:02:31.825 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:31 smithi098 bash[79516]: cluster 2023-09-23T19:02:30.469892+0000 mgr.y (mgr.26054) 1337 : cluster 0 pgmap v1467: 89 pgs: 2 stale+active+clean+remapped, 9 stale+active+clean, 19 active+clean+remapped, 47 active+clean, 7 active+undersized+degraded, 5 active+undersized; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 822 op/s; 29138/277650 objects degraded (10.495%); 14/277650 objects misplaced (0.005%) 2023-09-23T19:02:31.825 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:31 smithi098 bash[79516]: cluster 2023-09-23T19:02:30.602566+0000 osd.4 (osd.4) 54 : cluster 3 failed to encode map e324 with expected crc 2023-09-23T19:02:31.825 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:31 smithi098 bash[79516]: cluster 2023-09-23T19:02:30.603595+0000 mon.a (mon.0) 1832 : cluster 0 osdmap e324: 8 total, 6 up, 7 in 2023-09-23T19:02:31.826 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:31 smithi098 bash[79516]: cluster 2023-09-23T19:02:30.610811+0000 osd.6 (osd.6) 298 : cluster 3 failed to encode map e324 with expected crc 2023-09-23T19:02:31.847 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:31.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:31.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:31.953 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:31.982 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:31.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:31.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:32.087 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:32.093 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 83 16 95234 95218 71.6866 43.1875 0.00925161 0.0139438 2023-09-23T19:02:32.116 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:32.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:32.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:32.221 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:32.255 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:32.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:32.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:32.358 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:32.386 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:32.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:32.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:32.489 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:32.513 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:32.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:32.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:32.620 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:32.662 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:32.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:32.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:32.767 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:32.801 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:32.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:32.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:32.904 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:32.924 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:32.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:32.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:33.028 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:33.056 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:33.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:33.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:33.094 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 84 16 96037 96021 71.4307 50.1875 0.0227193 0.0139936 2023-09-23T19:02:33.161 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:33.182 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:33.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:33.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:33.286 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:33.316 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:33.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:33.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:33.418 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:33.443 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:33.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:33.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:33.546 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:33.576 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:33.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:33.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:33.683 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:33.705 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:33.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:33.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:33.810 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:33.827 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:33 smithi098 bash[79516]: cluster 2023-09-23T19:02:32.471012+0000 mgr.y (mgr.26054) 1338 : cluster 0 pgmap v1469: 89 pgs: 1 stale+active+clean+remapped, 6 stale+active+clean, 17 active+clean+remapped, 1 active+undersized+remapped, 45 active+clean, 10 active+undersized+degraded, 9 active+undersized; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 42 MiB/s wr, 678 op/s; 41326/281751 objects degraded (14.668%); 15/281751 objects misplaced (0.005%); 520 KiB/s, 1 objects/s recovering 2023-09-23T19:02:33.827 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:33 smithi098 bash[76817]: cluster 2023-09-23T19:02:32.471012+0000 mgr.y (mgr.26054) 1338 : cluster 0 pgmap v1469: 89 pgs: 1 stale+active+clean+remapped, 6 stale+active+clean, 17 active+clean+remapped, 1 active+undersized+remapped, 45 active+clean, 10 active+undersized+degraded, 9 active+undersized; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 42 MiB/s wr, 678 op/s; 41326/281751 objects degraded (14.668%); 15/281751 objects misplaced (0.005%); 520 KiB/s, 1 objects/s recovering 2023-09-23T19:02:33.836 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:33.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:33.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:33.941 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:33.964 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:33.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:33.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:34.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:33 smithi173 bash[59422]: cluster 2023-09-23T19:02:32.471012+0000 mgr.y (mgr.26054) 1338 : cluster 0 pgmap v1469: 89 pgs: 1 stale+active+clean+remapped, 6 stale+active+clean, 17 active+clean+remapped, 1 active+undersized+remapped, 45 active+clean, 10 active+undersized+degraded, 9 active+undersized; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 42 MiB/s wr, 678 op/s; 41326/281751 objects degraded (14.668%); 15/281751 objects misplaced (0.005%); 520 KiB/s, 1 objects/s recovering 2023-09-23T19:02:34.067 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:34.094 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 85 16 96690 96674 71.0704 40.8125 0.0234387 0.0140648 2023-09-23T19:02:34.097 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:34.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:34.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:34.202 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:34.227 DEBUG:teuthology.orchestra.run.smithi173:> sudo pkill -f 'journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.5.service' 2023-09-23T19:02:34.234 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:34.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:34.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:34.253 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T19:02:34.254 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-09-23T19:02:34.342 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:34.377 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:34.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:34.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:34.480 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:34.506 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:34.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:34.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:34.610 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:34.639 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:34.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:34.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:34.745 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:34.778 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:34.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:34.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:34.882 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:34.908 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:34.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:34.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:35.012 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:35.046 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:35.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:35.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:35.094 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 86 16 97451 97435 70.7971 47.5625 0.0118287 0.0141192 2023-09-23T19:02:35.154 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:35.180 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:35.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:35.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:35.284 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:35.325 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:35.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:35.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:35.430 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:35.472 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:35.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:35.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:35.579 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:35.604 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:35.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:35.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:35.710 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:35.735 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:35.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:35.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:35.838 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:35.847 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:35 smithi098 bash[76817]: cluster 2023-09-23T19:02:34.471877+0000 mgr.y (mgr.26054) 1339 : cluster 0 pgmap v1470: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.9 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 55 MiB/s wr, 882 op/s; 54057/287439 objects degraded (18.806%); 15/287439 objects misplaced (0.005%); 528 KiB/s, 1 objects/s recovering 2023-09-23T19:02:35.848 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:35 smithi098 bash[76817]: cluster 2023-09-23T19:02:35.466108+0000 mon.a (mon.0) 1833 : cluster 3 Health check update: Degraded data redundancy: 54057/287439 objects degraded (18.806%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:35.848 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:35 smithi098 bash[79516]: cluster 2023-09-23T19:02:34.471877+0000 mgr.y (mgr.26054) 1339 : cluster 0 pgmap v1470: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.9 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 55 MiB/s wr, 882 op/s; 54057/287439 objects degraded (18.806%); 15/287439 objects misplaced (0.005%); 528 KiB/s, 1 objects/s recovering 2023-09-23T19:02:35.848 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:35 smithi098 bash[79516]: cluster 2023-09-23T19:02:35.466108+0000 mon.a (mon.0) 1833 : cluster 3 Health check update: Degraded data redundancy: 54057/287439 objects degraded (18.806%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:35.863 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:35.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:35.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:35.966 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:35.998 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:36.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:36.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:36.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:35 smithi173 bash[59422]: cluster 2023-09-23T19:02:34.471877+0000 mgr.y (mgr.26054) 1339 : cluster 0 pgmap v1470: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.9 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 55 MiB/s wr, 882 op/s; 54057/287439 objects degraded (18.806%); 15/287439 objects misplaced (0.005%); 528 KiB/s, 1 objects/s recovering 2023-09-23T19:02:36.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:35 smithi173 bash[59422]: cluster 2023-09-23T19:02:35.466108+0000 mon.a (mon.0) 1833 : cluster 3 Health check update: Degraded data redundancy: 54057/287439 objects degraded (18.806%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:36.094 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 87 16 98282 98266 70.5803 51.9375 0.0199876 0.0141625 2023-09-23T19:02:36.101 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:36.111 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:02:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:02:36] "GET /metrics HTTP/1.1" 200 36644 "" "Prometheus/2.33.4" 2023-09-23T19:02:36.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:36.127 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:36.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:36.228 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:36.257 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:36.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:36.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:36.360 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:36.390 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:36.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:36.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:36.494 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:36.521 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:36.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:36.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:36.630 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:36.656 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:36.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:36.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:36.759 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:36.788 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:36.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:36.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:36.891 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:36.909 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:36.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:36.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:37.012 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:37.041 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:37.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:37.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:37.094 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 88 16 99037 99021 70.3144 47.1875 0.0184332 0.0142164 2023-09-23T19:02:37.144 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:37.171 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:37.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:37.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:37.274 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:37.305 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:37.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:37.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:37.408 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:37.438 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:37.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:37.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:37.542 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:37.568 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:37.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:37.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:37.671 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:37.683 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:37 smithi173 bash[59422]: cluster 2023-09-23T19:02:36.472475+0000 mgr.y (mgr.26054) 1340 : cluster 0 pgmap v1471: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 33 MiB/s wr, 527 op/s; 54057/287439 objects degraded (18.806%); 15/287439 objects misplaced (0.005%); 528 KiB/s, 1 objects/s recovering 2023-09-23T19:02:37.699 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:37.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:37.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:37.804 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:37.833 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:37.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:37.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:37.937 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:37.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:37 smithi098 bash[76817]: cluster 2023-09-23T19:02:36.472475+0000 mgr.y (mgr.26054) 1340 : cluster 0 pgmap v1471: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 33 MiB/s wr, 527 op/s; 54057/287439 objects degraded (18.806%); 15/287439 objects misplaced (0.005%); 528 KiB/s, 1 objects/s recovering 2023-09-23T19:02:37.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:37 smithi098 bash[79516]: cluster 2023-09-23T19:02:36.472475+0000 mgr.y (mgr.26054) 1340 : cluster 0 pgmap v1471: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 33 MiB/s wr, 527 op/s; 54057/287439 objects degraded (18.806%); 15/287439 objects misplaced (0.005%); 528 KiB/s, 1 objects/s recovering 2023-09-23T19:02:37.966 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:37.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:37.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:38.069 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:38.092 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:38.094 INFO:tasks.radosbench.radosbench.0.smithi098.stdout: 89 16 99796 99780 70.0574 47.4375 0.0159828 0.0142669 2023-09-23T19:02:38.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:38.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:38.197 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:38.222 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:38.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:38.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:38.328 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:38.358 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:38.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:38.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:38.463 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:38.497 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:38.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:38.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:38.600 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:38.642 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:38.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:38.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:38.746 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:38.788 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:38.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:38.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:38.895 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:38.927 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:38.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:38.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:39.030 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:39.063 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:39.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:39.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:39.094 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Total time run: 90.0089 2023-09-23T19:02:39.094 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Total writes made: 100459 2023-09-23T19:02:39.095 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Write size: 65536 2023-09-23T19:02:39.095 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Object size: 65536 2023-09-23T19:02:39.095 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Bandwidth (MB/sec): 69.7563 2023-09-23T19:02:39.095 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Stddev Bandwidth: 27.0849 2023-09-23T19:02:39.095 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Max bandwidth (MB/sec): 148.812 2023-09-23T19:02:39.095 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Min bandwidth (MB/sec): 22.625 2023-09-23T19:02:39.095 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Average IOPS: 1116 2023-09-23T19:02:39.095 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Stddev IOPS: 433.358 2023-09-23T19:02:39.095 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Max IOPS: 2381 2023-09-23T19:02:39.095 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Min IOPS: 362 2023-09-23T19:02:39.096 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Average Latency(s): 0.0143312 2023-09-23T19:02:39.096 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Stddev Latency(s): 0.0195028 2023-09-23T19:02:39.096 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Max latency(s): 1.09498 2023-09-23T19:02:39.096 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Min latency(s): 0.00130179 2023-09-23T19:02:39.103 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T19:02:39.166 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:39.195 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:39.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:39.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:39.254 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 6, 7, 5, 3] out_osds: [4] dead_osds: [7, 5] live_osds: [0, 2, 3, 6, 1, 4] 2023-09-23T19:02:39.255 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:02:39.255 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T19:02:39.255 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-09-23T19:02:39.297 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:39.354 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:39.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:39.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:39.462 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:39.501 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:39.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:39.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:39.612 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:39.645 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:39.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:39.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:39.752 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:39.780 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:39.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:39.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:39.885 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:39.912 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:39.914 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:39 smithi173 bash[59422]: cluster 2023-09-23T19:02:38.473328+0000 mgr.y (mgr.26054) 1341 : cluster 0 pgmap v1472: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 6.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 42 MiB/s wr, 668 op/s; 55534/295458 objects degraded (18.796%); 15/295458 objects misplaced (0.005%); 476 KiB/s, 1 objects/s recovering 2023-09-23T19:02:39.915 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:39 smithi173 bash[59422]: audit 2023-09-23T19:02:39.639363+0000 mon.a (mon.0) 1834 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:02:39.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:39.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:39.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:39 smithi098 bash[76817]: cluster 2023-09-23T19:02:38.473328+0000 mgr.y (mgr.26054) 1341 : cluster 0 pgmap v1472: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 6.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 42 MiB/s wr, 668 op/s; 55534/295458 objects degraded (18.796%); 15/295458 objects misplaced (0.005%); 476 KiB/s, 1 objects/s recovering 2023-09-23T19:02:39.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:39 smithi098 bash[76817]: audit 2023-09-23T19:02:39.639363+0000 mon.a (mon.0) 1834 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:02:39.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:39 smithi098 bash[79516]: cluster 2023-09-23T19:02:38.473328+0000 mgr.y (mgr.26054) 1341 : cluster 0 pgmap v1472: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 6.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 42 MiB/s wr, 668 op/s; 55534/295458 objects degraded (18.796%); 15/295458 objects misplaced (0.005%); 476 KiB/s, 1 objects/s recovering 2023-09-23T19:02:39.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:39 smithi098 bash[79516]: audit 2023-09-23T19:02:39.639363+0000 mon.a (mon.0) 1834 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:02:40.016 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:40.043 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:40.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:40.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:40.145 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:40.169 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:40.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:40.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:40.273 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:40.310 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:40.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:40.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:40.414 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:40.447 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:40.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:40.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:40.550 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:40.588 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:40.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:40.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:40.691 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:40.733 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:40.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:40.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:40.837 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:40.846 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:40 smithi098 bash[76817]: cluster 2023-09-23T19:02:40.470840+0000 mon.a (mon.0) 1835 : cluster 3 Health check update: Degraded data redundancy: 55534/295458 objects degraded (18.796%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:40.847 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:40 smithi098 bash[79516]: cluster 2023-09-23T19:02:40.470840+0000 mon.a (mon.0) 1835 : cluster 3 Health check update: Degraded data redundancy: 55534/295458 objects degraded (18.796%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:40.860 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:40.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:40.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:40.966 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:40.976 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:40 smithi173 bash[59422]: cluster 2023-09-23T19:02:40.470840+0000 mon.a (mon.0) 1835 : cluster 3 Health check update: Degraded data redundancy: 55534/295458 objects degraded (18.796%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:40.995 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:40.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:40.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:41.099 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:41.123 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:41.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:41.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:41.226 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:41.260 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:41.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:41.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:41.365 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:41.404 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:41.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:41.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:41.507 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:41.547 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:41.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:41.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:41.650 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:41 smithi098 bash[76817]: cluster 2023-09-23T19:02:40.474028+0000 mgr.y (mgr.26054) 1342 : cluster 0 pgmap v1473: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 6.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 44 MiB/s wr, 711 op/s; 56210/299004 objects degraded (18.799%); 15/299004 objects misplaced (0.005%); 422 KiB/s, 0 objects/s recovering 2023-09-23T19:02:41.654 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:41.682 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:41.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:41.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:41.788 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:41.817 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:41.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:41.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:41.920 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:41.930 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:41 smithi173 bash[59422]: cluster 2023-09-23T19:02:40.474028+0000 mgr.y (mgr.26054) 1342 : cluster 0 pgmap v1473: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 6.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 44 MiB/s wr, 711 op/s; 56210/299004 objects degraded (18.799%); 15/299004 objects misplaced (0.005%); 422 KiB/s, 0 objects/s recovering 2023-09-23T19:02:41.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:41 smithi098 bash[79516]: cluster 2023-09-23T19:02:40.474028+0000 mgr.y (mgr.26054) 1342 : cluster 0 pgmap v1473: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 6.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 44 MiB/s wr, 711 op/s; 56210/299004 objects degraded (18.799%); 15/299004 objects misplaced (0.005%); 422 KiB/s, 0 objects/s recovering 2023-09-23T19:02:41.947 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:41.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:41.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:42.050 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:42.077 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:42.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:42.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:42.181 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:42.210 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:42.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:42.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:42.316 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:42.346 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:42.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:42.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:42.450 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:42.479 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:42.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:42.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:42.582 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:42.604 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:42.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:42.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:42.708 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:42.738 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:42.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:42.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:42.841 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:42.867 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:42.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:42.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:42.970 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:43.009 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:43.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:43.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:43.100 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T19:02:43.100 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:02:43.115 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:43.133 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:43.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:43.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:43.235 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:43.266 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:43.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:43.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:43.371 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:43.400 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:43.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:43.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:43.503 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:43.535 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:43.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:43.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:43.638 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:43.670 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:43.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:43.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:43.774 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:43.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:43 smithi173 bash[59422]: cluster 2023-09-23T19:02:42.474855+0000 mgr.y (mgr.26054) 1343 : cluster 0 pgmap v1474: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 6.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 41 MiB/s wr, 744 op/s; 55942/297696 objects degraded (18.792%); 15/297696 objects misplaced (0.005%); 356 KiB/s, 0 objects/s recovering 2023-09-23T19:02:43.810 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:43.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:43.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:43.914 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:43.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:43 smithi098 bash[76817]: cluster 2023-09-23T19:02:42.474855+0000 mgr.y (mgr.26054) 1343 : cluster 0 pgmap v1474: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 6.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 41 MiB/s wr, 744 op/s; 55942/297696 objects degraded (18.792%); 15/297696 objects misplaced (0.005%); 356 KiB/s, 0 objects/s recovering 2023-09-23T19:02:43.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:43 smithi098 bash[79516]: cluster 2023-09-23T19:02:42.474855+0000 mgr.y (mgr.26054) 1343 : cluster 0 pgmap v1474: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 6.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 41 MiB/s wr, 744 op/s; 55942/297696 objects degraded (18.792%); 15/297696 objects misplaced (0.005%); 356 KiB/s, 0 objects/s recovering 2023-09-23T19:02:43.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:43.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:43.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:44.047 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:44.075 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:44.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:44.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:44.096 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:02:44.179 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:44.207 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:44.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:44.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:44.313 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:44.339 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:44.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:44.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:44.447 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:44.472 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:44.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:44.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:44.577 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:44.599 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:44.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:44.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:44.704 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:44.750 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:44.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:44.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:44.857 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:44.881 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:44.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:44.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:44.989 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:45.019 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:45.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:45.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:45.125 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:45.151 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:45.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:45.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:45.254 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:45.286 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:45.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:45.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:45.392 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:45.421 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:45.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:45.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:45.525 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:45.550 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:45.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:45.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:45.653 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:45.659 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:45 smithi098 bash[76817]: cluster 2023-09-23T19:02:44.475644+0000 mgr.y (mgr.26054) 1344 : cluster 0 pgmap v1475: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 85 B/s rd, 34 MiB/s wr, 802 op/s; 54906/292263 objects degraded (18.787%); 15/292263 objects misplaced (0.005%); 5.3 KiB/s, 0 objects/s recovering 2023-09-23T19:02:45.659 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:45 smithi098 bash[76817]: cluster 2023-09-23T19:02:45.472034+0000 mon.a (mon.0) 1836 : cluster 3 Health check update: Degraded data redundancy: 54906/292263 objects degraded (18.787%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:45.687 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:45.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:45.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:45.792 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:45.821 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:45.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:45.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:45.926 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:45.934 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:45 smithi098 bash[79516]: cluster 2023-09-23T19:02:44.475644+0000 mgr.y (mgr.26054) 1344 : cluster 0 pgmap v1475: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 85 B/s rd, 34 MiB/s wr, 802 op/s; 54906/292263 objects degraded (18.787%); 15/292263 objects misplaced (0.005%); 5.3 KiB/s, 0 objects/s recovering 2023-09-23T19:02:45.935 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:45 smithi098 bash[79516]: cluster 2023-09-23T19:02:45.472034+0000 mon.a (mon.0) 1836 : cluster 3 Health check update: Degraded data redundancy: 54906/292263 objects degraded (18.787%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:45.954 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:45.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:45.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:46.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:45 smithi173 bash[59422]: cluster 2023-09-23T19:02:44.475644+0000 mgr.y (mgr.26054) 1344 : cluster 0 pgmap v1475: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 85 B/s rd, 34 MiB/s wr, 802 op/s; 54906/292263 objects degraded (18.787%); 15/292263 objects misplaced (0.005%); 5.3 KiB/s, 0 objects/s recovering 2023-09-23T19:02:46.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:45 smithi173 bash[59422]: cluster 2023-09-23T19:02:45.472034+0000 mon.a (mon.0) 1836 : cluster 3 Health check update: Degraded data redundancy: 54906/292263 objects degraded (18.787%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:46.060 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:46.093 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:46.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:46.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:46.193 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:02:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:02:46] "GET /metrics HTTP/1.1" 200 36664 "" "Prometheus/2.33.4" 2023-09-23T19:02:46.200 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:46.235 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:46.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:46.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:46.343 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:46.374 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:46.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:46.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:46.482 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:46.505 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:46.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:46.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:46.607 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:46.636 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:46.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:46.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:46.740 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:46.767 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:46.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:46.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:46.872 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:46.909 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:46.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:46.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:47.018 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:47.044 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:47.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:47.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:47.148 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:47.174 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:47.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:47.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:47.278 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:47.310 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:47.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:47.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:47.415 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:47.448 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:47.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:47.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:47.551 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:47.577 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:47.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:47.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:47.680 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:47.708 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:47.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:47.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:47.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:47 smithi173 bash[59422]: cluster 2023-09-23T19:02:46.476294+0000 mgr.y (mgr.26054) 1345 : cluster 0 pgmap v1476: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 85 B/s rd, 24 MiB/s wr, 645 op/s; 54906/292263 objects degraded (18.787%); 15/292263 objects misplaced (0.005%) 2023-09-23T19:02:47.817 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:47.842 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:47.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:47.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:47.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:47 smithi098 bash[76817]: cluster 2023-09-23T19:02:46.476294+0000 mgr.y (mgr.26054) 1345 : cluster 0 pgmap v1476: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 85 B/s rd, 24 MiB/s wr, 645 op/s; 54906/292263 objects degraded (18.787%); 15/292263 objects misplaced (0.005%) 2023-09-23T19:02:47.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:47 smithi098 bash[79516]: cluster 2023-09-23T19:02:46.476294+0000 mgr.y (mgr.26054) 1345 : cluster 0 pgmap v1476: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 85 B/s rd, 24 MiB/s wr, 645 op/s; 54906/292263 objects degraded (18.787%); 15/292263 objects misplaced (0.005%) 2023-09-23T19:02:47.947 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:48.041 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:48.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:48.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:48.145 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:48.169 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:48.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:48.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:48.272 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:48.304 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:48.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:48.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:48.410 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:48.436 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:48.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:48.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:48.540 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:48.564 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:48.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:48.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:48.668 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:48.697 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:48.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:48.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:48.803 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:48.824 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:48.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:48.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:48.926 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:48.956 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:48.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:48.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:49.059 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:49.077 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:49.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:49.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:49.179 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:49.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:49.221 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:49.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:49.322 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:49.352 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:49.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:49.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:49.457 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:49.484 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:49.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:49.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:49.590 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:49.619 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:49.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:49.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:49.648 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T19:02:49.722 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:49.757 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:49.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:49.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:49.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:49 smithi173 bash[59422]: cluster 2023-09-23T19:02:48.477140+0000 mgr.y (mgr.26054) 1346 : cluster 0 pgmap v1477: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.8 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 85 B/s rd, 24 MiB/s wr, 865 op/s; 53515/284331 objects degraded (18.821%); 15/284331 objects misplaced (0.005%) 2023-09-23T19:02:49.866 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:49.898 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:49.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:49.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:49.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:49 smithi098 bash[76817]: cluster 2023-09-23T19:02:48.477140+0000 mgr.y (mgr.26054) 1346 : cluster 0 pgmap v1477: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.8 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 85 B/s rd, 24 MiB/s wr, 865 op/s; 53515/284331 objects degraded (18.821%); 15/284331 objects misplaced (0.005%) 2023-09-23T19:02:49.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:49 smithi098 bash[79516]: cluster 2023-09-23T19:02:48.477140+0000 mgr.y (mgr.26054) 1346 : cluster 0 pgmap v1477: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.8 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 85 B/s rd, 24 MiB/s wr, 865 op/s; 53515/284331 objects degraded (18.821%); 15/284331 objects misplaced (0.005%) 2023-09-23T19:02:50.004 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:50.028 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:50.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:50.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:50.134 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:50.164 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:50.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:50.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:50.267 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:50.297 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:50.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:50.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:50.405 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:50.429 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:50.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:50.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:50.532 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:50.571 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:50.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:50.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:50.620 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T19:02:50.670 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T19:02:50.675 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:50.707 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is unset 2023-09-23T19:02:50.719 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:50.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:50.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:50.738 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T19:02:50.782 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T19:02:50.821 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T19:02:50.822 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:50.848 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:50.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:50.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:50.864 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T19:02:50.866 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:02:50.866 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:02:50.866 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:02:50.867 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:02:50.867 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:02:50.867 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T19:02:50.867 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:02:50.867 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T19:02:50.932 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:50 smithi098 bash[76817]: audit 2023-09-23T19:02:50.081808+0000 mon.a (mon.0) 1837 : audit 1 from='client.? 172.21.15.98:0/1890560147' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:02:50.932 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:50 smithi098 bash[76817]: cluster 2023-09-23T19:02:50.473099+0000 mon.a (mon.0) 1838 : cluster 3 Health check update: Degraded data redundancy: 53515/284331 objects degraded (18.821%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:50.932 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:50 smithi098 bash[76817]: audit 2023-09-23T19:02:50.568408+0000 mon.a (mon.0) 1839 : audit 0 from='client.? 172.21.15.98:0/3059777913' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:02:50.933 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:50 smithi098 bash[79516]: audit 2023-09-23T19:02:50.081808+0000 mon.a (mon.0) 1837 : audit 1 from='client.? 172.21.15.98:0/1890560147' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:02:50.933 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:50 smithi098 bash[79516]: cluster 2023-09-23T19:02:50.473099+0000 mon.a (mon.0) 1838 : cluster 3 Health check update: Degraded data redundancy: 53515/284331 objects degraded (18.821%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:50.933 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:50 smithi098 bash[79516]: audit 2023-09-23T19:02:50.568408+0000 mon.a (mon.0) 1839 : audit 0 from='client.? 172.21.15.98:0/3059777913' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:02:50.951 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:50.985 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:50.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:50.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:51.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:50 smithi173 bash[59422]: audit 2023-09-23T19:02:50.081808+0000 mon.a (mon.0) 1837 : audit 1 from='client.? 172.21.15.98:0/1890560147' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:02:51.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:50 smithi173 bash[59422]: cluster 2023-09-23T19:02:50.473099+0000 mon.a (mon.0) 1838 : cluster 3 Health check update: Degraded data redundancy: 53515/284331 objects degraded (18.821%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:51.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:50 smithi173 bash[59422]: audit 2023-09-23T19:02:50.568408+0000 mon.a (mon.0) 1839 : audit 0 from='client.? 172.21.15.98:0/3059777913' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:02:51.088 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:51.119 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:51.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:51.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:51.224 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:51.248 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:51.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:51.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:51.351 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:51.380 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:51.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:51.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:51.483 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:51.519 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:51.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:51.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:51.625 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:51.650 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:51.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:51.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:51.753 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:51.761 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:51 smithi098 bash[79516]: cluster 2023-09-23T19:02:50.477820+0000 mgr.y (mgr.26054) 1347 : cluster 0 pgmap v1478: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 85 B/s rd, 10 MiB/s wr, 737 op/s; 52825/280929 objects degraded (18.804%); 15/280929 objects misplaced (0.005%) 2023-09-23T19:02:51.761 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:51 smithi098 bash[79516]: cluster 2023-09-23T19:02:50.671309+0000 mon.a (mon.0) 1840 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:02:51.762 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:51 smithi098 bash[79516]: cluster 2023-09-23T19:02:50.692150+0000 osd.4 (osd.4) 55 : cluster 3 failed to encode map e325 with expected crc 2023-09-23T19:02:51.762 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:51 smithi098 bash[79516]: audit 2023-09-23T19:02:50.694782+0000 mon.a (mon.0) 1841 : audit 1 from='client.? 172.21.15.98:0/1890560147' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:02:51.762 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:51 smithi098 bash[79516]: cluster 2023-09-23T19:02:50.695010+0000 mon.a (mon.0) 1842 : cluster 0 osdmap e325: 8 total, 6 up, 7 in 2023-09-23T19:02:51.762 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:51 smithi098 bash[79516]: cluster 2023-09-23T19:02:50.721303+0000 osd.6 (osd.6) 299 : cluster 3 failed to encode map e325 with expected crc 2023-09-23T19:02:51.762 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:51 smithi098 bash[79516]: cluster 2023-09-23T19:02:50.724184+0000 osd.6 (osd.6) 300 : cluster 3 failed to encode map e325 with expected crc 2023-09-23T19:02:51.762 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:51 smithi098 bash[76817]: cluster 2023-09-23T19:02:50.477820+0000 mgr.y (mgr.26054) 1347 : cluster 0 pgmap v1478: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 85 B/s rd, 10 MiB/s wr, 737 op/s; 52825/280929 objects degraded (18.804%); 15/280929 objects misplaced (0.005%) 2023-09-23T19:02:51.762 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:51 smithi098 bash[76817]: cluster 2023-09-23T19:02:50.671309+0000 mon.a (mon.0) 1840 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:02:51.763 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:51 smithi098 bash[76817]: cluster 2023-09-23T19:02:50.692150+0000 osd.4 (osd.4) 55 : cluster 3 failed to encode map e325 with expected crc 2023-09-23T19:02:51.763 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:51 smithi098 bash[76817]: audit 2023-09-23T19:02:50.694782+0000 mon.a (mon.0) 1841 : audit 1 from='client.? 172.21.15.98:0/1890560147' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:02:51.763 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:51 smithi098 bash[76817]: cluster 2023-09-23T19:02:50.695010+0000 mon.a (mon.0) 1842 : cluster 0 osdmap e325: 8 total, 6 up, 7 in 2023-09-23T19:02:51.763 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:51 smithi098 bash[76817]: cluster 2023-09-23T19:02:50.721303+0000 osd.6 (osd.6) 299 : cluster 3 failed to encode map e325 with expected crc 2023-09-23T19:02:51.763 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:51 smithi098 bash[76817]: cluster 2023-09-23T19:02:50.724184+0000 osd.6 (osd.6) 300 : cluster 3 failed to encode map e325 with expected crc 2023-09-23T19:02:51.777 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:51.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:51.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:51.883 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:51.918 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:51.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:51.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:52.022 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:52.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:51 smithi173 bash[59422]: cluster 2023-09-23T19:02:50.477820+0000 mgr.y (mgr.26054) 1347 : cluster 0 pgmap v1478: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 85 B/s rd, 10 MiB/s wr, 737 op/s; 52825/280929 objects degraded (18.804%); 15/280929 objects misplaced (0.005%) 2023-09-23T19:02:52.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:51 smithi173 bash[59422]: cluster 2023-09-23T19:02:50.671309+0000 mon.a (mon.0) 1840 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:02:52.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:51 smithi173 bash[59422]: cluster 2023-09-23T19:02:50.692150+0000 osd.4 (osd.4) 55 : cluster 3 failed to encode map e325 with expected crc 2023-09-23T19:02:52.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:51 smithi173 bash[59422]: audit 2023-09-23T19:02:50.694782+0000 mon.a (mon.0) 1841 : audit 1 from='client.? 172.21.15.98:0/1890560147' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:02:52.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:51 smithi173 bash[59422]: cluster 2023-09-23T19:02:50.695010+0000 mon.a (mon.0) 1842 : cluster 0 osdmap e325: 8 total, 6 up, 7 in 2023-09-23T19:02:52.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:51 smithi173 bash[59422]: cluster 2023-09-23T19:02:50.721303+0000 osd.6 (osd.6) 299 : cluster 3 failed to encode map e325 with expected crc 2023-09-23T19:02:52.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:51 smithi173 bash[59422]: cluster 2023-09-23T19:02:50.724184+0000 osd.6 (osd.6) 300 : cluster 3 failed to encode map e325 with expected crc 2023-09-23T19:02:52.057 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:52.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:52.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:52.165 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:52.196 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:52.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:52.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:52.298 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:52.325 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:52.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:52.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:52.429 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:52.460 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:52.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:52.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:52.564 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:52.591 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:52.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:52.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:52.698 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:52.730 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:52.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:52.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:52.835 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:52.865 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:52.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:52.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:52.969 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:53.005 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:53.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:53.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:53.111 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:53.142 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:53.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:53.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:53.247 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:53.270 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:53.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:53.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:53.374 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:53.427 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:53.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:53.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:53.531 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:53.572 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:53.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:53.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:53.679 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:53.710 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:53.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:53.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:53.816 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:53.852 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:53.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:53.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:53.956 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:53.979 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:53 smithi098 bash[76817]: cluster 2023-09-23T19:02:52.478669+0000 mgr.y (mgr.26054) 1348 : cluster 0 pgmap v1480: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 1.1 MiB/s wr, 764 op/s; 51881/275829 objects degraded (18.809%); 15/275829 objects misplaced (0.005%) 2023-09-23T19:02:53.980 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:53 smithi098 bash[76817]: audit 2023-09-23T19:02:53.236515+0000 mon.a (mon.0) 1843 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:02:53.980 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:53 smithi098 bash[79516]: cluster 2023-09-23T19:02:52.478669+0000 mgr.y (mgr.26054) 1348 : cluster 0 pgmap v1480: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 1.1 MiB/s wr, 764 op/s; 51881/275829 objects degraded (18.809%); 15/275829 objects misplaced (0.005%) 2023-09-23T19:02:53.980 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:53 smithi098 bash[79516]: audit 2023-09-23T19:02:53.236515+0000 mon.a (mon.0) 1843 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:02:54.004 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:54.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:54.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:54.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:53 smithi173 bash[59422]: cluster 2023-09-23T19:02:52.478669+0000 mgr.y (mgr.26054) 1348 : cluster 0 pgmap v1480: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 1.1 MiB/s wr, 764 op/s; 51881/275829 objects degraded (18.809%); 15/275829 objects misplaced (0.005%) 2023-09-23T19:02:54.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:53 smithi173 bash[59422]: audit 2023-09-23T19:02:53.236515+0000 mon.a (mon.0) 1843 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:02:54.111 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:54.146 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:54.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:54.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:54.250 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:54.275 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:54.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:54.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:54.379 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:54.419 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:54.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:54.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:54.524 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:54.553 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:54.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:54.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:54.658 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:54.698 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:54.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:54.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:54.803 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:54.845 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:54.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:54.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:54.948 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:54.973 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:54.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:54.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:55.003 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:02:55.003 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:02:55.078 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:55.107 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:55.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:55.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:55.210 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:55.241 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:55.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:55.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:55.344 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:55.367 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:55.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:55.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:55.470 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:55.495 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:55.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:55.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:55.599 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:55.639 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:55.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:55.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:55.747 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:55.776 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:55.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:55.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:55.879 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:55.906 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:55.908 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:55 smithi173 bash[59422]: cluster 2023-09-23T19:02:54.482354+0000 mgr.y (mgr.26054) 1349 : cluster 0 pgmap v1481: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 767 op/s; 50656/269214 objects degraded (18.816%); 15/269214 objects misplaced (0.006%) 2023-09-23T19:02:55.908 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:55 smithi173 bash[59422]: audit 2023-09-23T19:02:54.649943+0000 mon.a (mon.0) 1844 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:02:55.908 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:55 smithi173 bash[59422]: audit 2023-09-23T19:02:54.651275+0000 mon.a (mon.0) 1845 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:02:55.908 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:55 smithi173 bash[59422]: cluster 2023-09-23T19:02:55.474109+0000 mon.a (mon.0) 1846 : cluster 3 Health check update: Degraded data redundancy: 50656/269214 objects degraded (18.816%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:55.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:55.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:55.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:55 smithi098 bash[76817]: cluster 2023-09-23T19:02:54.482354+0000 mgr.y (mgr.26054) 1349 : cluster 0 pgmap v1481: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 767 op/s; 50656/269214 objects degraded (18.816%); 15/269214 objects misplaced (0.006%) 2023-09-23T19:02:55.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:55 smithi098 bash[76817]: audit 2023-09-23T19:02:54.649943+0000 mon.a (mon.0) 1844 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:02:55.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:55 smithi098 bash[76817]: audit 2023-09-23T19:02:54.651275+0000 mon.a (mon.0) 1845 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:02:55.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:55 smithi098 bash[76817]: cluster 2023-09-23T19:02:55.474109+0000 mon.a (mon.0) 1846 : cluster 3 Health check update: Degraded data redundancy: 50656/269214 objects degraded (18.816%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:55.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:55 smithi098 bash[79516]: cluster 2023-09-23T19:02:54.482354+0000 mgr.y (mgr.26054) 1349 : cluster 0 pgmap v1481: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 767 op/s; 50656/269214 objects degraded (18.816%); 15/269214 objects misplaced (0.006%) 2023-09-23T19:02:55.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:55 smithi098 bash[79516]: audit 2023-09-23T19:02:54.649943+0000 mon.a (mon.0) 1844 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:02:55.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:55 smithi098 bash[79516]: audit 2023-09-23T19:02:54.651275+0000 mon.a (mon.0) 1845 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:02:55.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:55 smithi098 bash[79516]: cluster 2023-09-23T19:02:55.474109+0000 mon.a (mon.0) 1846 : cluster 3 Health check update: Degraded data redundancy: 50656/269214 objects degraded (18.816%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:02:56.010 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:56.040 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:56.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:56.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:56.143 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:56.173 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:56.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:56.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:56.277 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:56.310 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:56.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:56.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:56.415 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:56.423 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:02:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:02:56] "GET /metrics HTTP/1.1" 200 36648 "" "Prometheus/2.33.4" 2023-09-23T19:02:56.439 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:56.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:56.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:56.544 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:56.569 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:56.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:56.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:56.673 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:56.698 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:02:56.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:56.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:02:56.802 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:56.831 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:56.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:56.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:56.934 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:56.965 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:56.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:56.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:57.004 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:02:57.005 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:02:57.071 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:57.144 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:57.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:57.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:57.251 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:57.293 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:57.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:57.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:57.411 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:57.444 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:57.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:57.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:57.550 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:57.578 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:57.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:57.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:57.663 INFO:teuthology.orchestra.run.smithi098.stdout:moved 5 / 267 (1.87266%) 2023-09-23T19:02:57.663 INFO:teuthology.orchestra.run.smithi098.stdout:avg 53.4 2023-09-23T19:02:57.663 INFO:teuthology.orchestra.run.smithi098.stdout:stddev 12.088 -> 12.5857 (expected baseline 6.53605) 2023-09-23T19:02:57.663 INFO:teuthology.orchestra.run.smithi098.stdout:min osd.2 with 35 -> 33 pgs (0.655431 -> 0.617977 * mean) 2023-09-23T19:02:57.663 INFO:teuthology.orchestra.run.smithi098.stdout:max osd.1 with 50 -> 49 pgs (0.93633 -> 0.917603 * mean) 2023-09-23T19:02:57.663 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T19:02:57.664 INFO:teuthology.orchestra.run.smithi098.stdout:oload 1000 2023-09-23T19:02:57.664 INFO:teuthology.orchestra.run.smithi098.stdout:max_change 0.05 2023-09-23T19:02:57.664 INFO:teuthology.orchestra.run.smithi098.stdout:max_change_osds 4 2023-09-23T19:02:57.664 INFO:teuthology.orchestra.run.smithi098.stdout:average_utilization 0.0329 2023-09-23T19:02:57.664 INFO:teuthology.orchestra.run.smithi098.stdout:overload_utilization 0.3288 2023-09-23T19:02:57.664 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7 weight 0.8539 -> 0.9039 2023-09-23T19:02:57.664 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3 weight 0.6815 -> 0.7315 2023-09-23T19:02:57.664 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5 weight 0.6437 -> 0.6936 2023-09-23T19:02:57.671 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:57 smithi098 bash[76817]: cluster 2023-09-23T19:02:56.483136+0000 mgr.y (mgr.26054) 1350 : cluster 0 pgmap v1482: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 767 op/s; 50656/269214 objects degraded (18.816%); 15/269214 objects misplaced (0.006%) 2023-09-23T19:02:57.671 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:57 smithi098 bash[76817]: audit 2023-09-23T19:02:57.513162+0000 mon.a (mon.0) 1847 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'47939','5':'45459','7':'59240'}"}]: dispatch 2023-09-23T19:02:57.682 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:57.711 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:57.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:57.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:57.819 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:57.845 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:57.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:57.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:57.920 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:57 smithi098 bash[79516]: cluster 2023-09-23T19:02:56.483136+0000 mgr.y (mgr.26054) 1350 : cluster 0 pgmap v1482: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 767 op/s; 50656/269214 objects degraded (18.816%); 15/269214 objects misplaced (0.006%) 2023-09-23T19:02:57.921 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:57 smithi098 bash[79516]: audit 2023-09-23T19:02:57.513162+0000 mon.a (mon.0) 1847 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'47939','5':'45459','7':'59240'}"}]: dispatch 2023-09-23T19:02:57.948 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:57.961 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:57 smithi173 bash[59422]: cluster 2023-09-23T19:02:56.483136+0000 mgr.y (mgr.26054) 1350 : cluster 0 pgmap v1482: 89 pgs: 15 active+clean+remapped, 4 active+undersized+remapped, 40 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 767 op/s; 50656/269214 objects degraded (18.816%); 15/269214 objects misplaced (0.006%) 2023-09-23T19:02:57.961 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:57 smithi173 bash[59422]: audit 2023-09-23T19:02:57.513162+0000 mon.a (mon.0) 1847 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'47939','5':'45459','7':'59240'}"}]: dispatch 2023-09-23T19:02:57.981 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:57.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:57.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:58.084 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:58.111 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:58.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:58.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:58.214 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:58.242 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:58.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:58.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:58.345 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:58.373 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:58.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:58.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:58.477 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:58.510 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:58.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:58.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:58.612 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:58.648 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:02:58.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:58.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:02:58.682 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:58 smithi098 bash[76817]: audit 2023-09-23T19:02:57.512374+0000 mgr.y (mgr.26054) 1351 : audit 0 from='client.55309 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:02:58.682 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:58 smithi098 bash[76817]: audit 2023-09-23T19:02:57.659225+0000 mon.a (mon.0) 1848 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'3':'47939','5':'45459','7':'59240'}"}]': finished 2023-09-23T19:02:58.682 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:58 smithi098 bash[76817]: cluster 2023-09-23T19:02:57.659345+0000 mon.a (mon.0) 1849 : cluster 0 osdmap e326: 8 total, 6 up, 7 in 2023-09-23T19:02:58.682 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:58 smithi098 bash[76817]: cluster 2023-09-23T19:02:57.661318+0000 osd.4 (osd.4) 56 : cluster 3 failed to encode map e326 with expected crc 2023-09-23T19:02:58.753 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:58.789 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:58.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:58.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:58.893 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:58.920 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:58.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:58.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:58.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:58 smithi098 bash[76817]: cluster 2023-09-23T19:02:57.667386+0000 osd.6 (osd.6) 301 : cluster 3 failed to encode map e326 with expected crc 2023-09-23T19:02:58.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:58 smithi098 bash[76817]: cluster 2023-09-23T19:02:57.669753+0000 osd.6 (osd.6) 302 : cluster 3 failed to encode map e326 with expected crc 2023-09-23T19:02:58.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:58 smithi098 bash[76817]: cluster 2023-09-23T19:02:58.668119+0000 mon.a (mon.0) 1850 : cluster 0 osdmap e327: 8 total, 6 up, 7 in 2023-09-23T19:02:58.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:58 smithi098 bash[79516]: audit 2023-09-23T19:02:57.512374+0000 mgr.y (mgr.26054) 1351 : audit 0 from='client.55309 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:02:58.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:58 smithi098 bash[79516]: audit 2023-09-23T19:02:57.659225+0000 mon.a (mon.0) 1848 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'3':'47939','5':'45459','7':'59240'}"}]': finished 2023-09-23T19:02:58.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:58 smithi098 bash[79516]: cluster 2023-09-23T19:02:57.659345+0000 mon.a (mon.0) 1849 : cluster 0 osdmap e326: 8 total, 6 up, 7 in 2023-09-23T19:02:58.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:58 smithi098 bash[79516]: cluster 2023-09-23T19:02:57.661318+0000 osd.4 (osd.4) 56 : cluster 3 failed to encode map e326 with expected crc 2023-09-23T19:02:58.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:58 smithi098 bash[79516]: cluster 2023-09-23T19:02:57.667386+0000 osd.6 (osd.6) 301 : cluster 3 failed to encode map e326 with expected crc 2023-09-23T19:02:58.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:58 smithi098 bash[79516]: cluster 2023-09-23T19:02:57.669753+0000 osd.6 (osd.6) 302 : cluster 3 failed to encode map e326 with expected crc 2023-09-23T19:02:58.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:58 smithi098 bash[79516]: cluster 2023-09-23T19:02:58.668119+0000 mon.a (mon.0) 1850 : cluster 0 osdmap e327: 8 total, 6 up, 7 in 2023-09-23T19:02:59.023 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:59.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:58 smithi173 bash[59422]: audit 2023-09-23T19:02:57.512374+0000 mgr.y (mgr.26054) 1351 : audit 0 from='client.55309 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:02:59.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:58 smithi173 bash[59422]: audit 2023-09-23T19:02:57.659225+0000 mon.a (mon.0) 1848 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'3':'47939','5':'45459','7':'59240'}"}]': finished 2023-09-23T19:02:59.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:58 smithi173 bash[59422]: cluster 2023-09-23T19:02:57.659345+0000 mon.a (mon.0) 1849 : cluster 0 osdmap e326: 8 total, 6 up, 7 in 2023-09-23T19:02:59.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:58 smithi173 bash[59422]: cluster 2023-09-23T19:02:57.661318+0000 osd.4 (osd.4) 56 : cluster 3 failed to encode map e326 with expected crc 2023-09-23T19:02:59.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:58 smithi173 bash[59422]: cluster 2023-09-23T19:02:57.667386+0000 osd.6 (osd.6) 301 : cluster 3 failed to encode map e326 with expected crc 2023-09-23T19:02:59.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:58 smithi173 bash[59422]: cluster 2023-09-23T19:02:57.669753+0000 osd.6 (osd.6) 302 : cluster 3 failed to encode map e326 with expected crc 2023-09-23T19:02:59.030 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:58 smithi173 bash[59422]: cluster 2023-09-23T19:02:58.668119+0000 mon.a (mon.0) 1850 : cluster 0 osdmap e327: 8 total, 6 up, 7 in 2023-09-23T19:02:59.097 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:59.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:59.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:59.201 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:59.234 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:59.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:59.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:59.318 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T19:02:59.318 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-09-23T19:02:59.337 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:59.394 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:59.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:59.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:59.499 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:59.526 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:02:59.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:59.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:02:59.630 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:59.657 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:02:59.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:59.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:02:59.737 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:02:59.761 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:59.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:59 smithi173 bash[59422]: cluster 2023-09-23T19:02:58.483780+0000 mgr.y (mgr.26054) 1352 : cluster 0 pgmap v1484: 89 pgs: 2 peering, 15 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 49243/261399 objects degraded (18.838%); 15/261399 objects misplaced (0.006%) 2023-09-23T19:02:59.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:59 smithi173 bash[59422]: cluster 2023-09-23T19:02:58.665611+0000 osd.4 (osd.4) 57 : cluster 3 failed to encode map e327 with expected crc 2023-09-23T19:02:59.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:59 smithi173 bash[59422]: cluster 2023-09-23T19:02:58.667210+0000 osd.6 (osd.6) 303 : cluster 3 failed to encode map e327 with expected crc 2023-09-23T19:02:59.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:59 smithi173 bash[59422]: audit 2023-09-23T19:02:59.588294+0000 mon.a (mon.0) 1851 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:02:59.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:59 smithi173 bash[59422]: audit 2023-09-23T19:02:59.595321+0000 mon.a (mon.0) 1852 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:02:59.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:02:59 smithi173 bash[59422]: cluster 2023-09-23T19:02:59.667004+0000 mon.a (mon.0) 1853 : cluster 0 osdmap e328: 8 total, 6 up, 7 in 2023-09-23T19:02:59.789 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:02:59.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:59.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:02:59.892 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:59.924 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:02:59.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:02:59.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:02:59.928 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:59 smithi098 bash[76817]: cluster 2023-09-23T19:02:58.483780+0000 mgr.y (mgr.26054) 1352 : cluster 0 pgmap v1484: 89 pgs: 2 peering, 15 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 49243/261399 objects degraded (18.838%); 15/261399 objects misplaced (0.006%) 2023-09-23T19:02:59.928 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:59 smithi098 bash[76817]: cluster 2023-09-23T19:02:58.665611+0000 osd.4 (osd.4) 57 : cluster 3 failed to encode map e327 with expected crc 2023-09-23T19:02:59.928 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:59 smithi098 bash[76817]: cluster 2023-09-23T19:02:58.667210+0000 osd.6 (osd.6) 303 : cluster 3 failed to encode map e327 with expected crc 2023-09-23T19:02:59.928 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:59 smithi098 bash[76817]: audit 2023-09-23T19:02:59.588294+0000 mon.a (mon.0) 1851 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:02:59.928 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:59 smithi098 bash[76817]: audit 2023-09-23T19:02:59.595321+0000 mon.a (mon.0) 1852 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:02:59.928 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:02:59 smithi098 bash[76817]: cluster 2023-09-23T19:02:59.667004+0000 mon.a (mon.0) 1853 : cluster 0 osdmap e328: 8 total, 6 up, 7 in 2023-09-23T19:02:59.929 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:59 smithi098 bash[79516]: cluster 2023-09-23T19:02:58.483780+0000 mgr.y (mgr.26054) 1352 : cluster 0 pgmap v1484: 89 pgs: 2 peering, 15 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 16 active+undersized+degraded, 14 active+undersized; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 49243/261399 objects degraded (18.838%); 15/261399 objects misplaced (0.006%) 2023-09-23T19:02:59.929 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:59 smithi098 bash[79516]: cluster 2023-09-23T19:02:58.665611+0000 osd.4 (osd.4) 57 : cluster 3 failed to encode map e327 with expected crc 2023-09-23T19:02:59.929 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:59 smithi098 bash[79516]: cluster 2023-09-23T19:02:58.667210+0000 osd.6 (osd.6) 303 : cluster 3 failed to encode map e327 with expected crc 2023-09-23T19:02:59.929 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:59 smithi098 bash[79516]: audit 2023-09-23T19:02:59.588294+0000 mon.a (mon.0) 1851 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:02:59.930 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:59 smithi098 bash[79516]: audit 2023-09-23T19:02:59.595321+0000 mon.a (mon.0) 1852 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:02:59.930 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:02:59 smithi098 bash[79516]: cluster 2023-09-23T19:02:59.667004+0000 mon.a (mon.0) 1853 : cluster 0 osdmap e328: 8 total, 6 up, 7 in 2023-09-23T19:03:00.027 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:00.063 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:00.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:00.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:00.169 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:00.201 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:00.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:00.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:00.305 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:00.336 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:00.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:00.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:00.441 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:00.478 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:00.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:00.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:00.584 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:00.635 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:00.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:00.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:00.740 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:00.748 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:00 smithi098 bash[76817]: cluster 2023-09-23T19:02:59.665681+0000 osd.6 (osd.6) 304 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T19:03:00.749 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:00 smithi098 bash[76817]: cluster 2023-09-23T19:02:59.670213+0000 osd.0 (osd.0) 37 : cluster 0 10.c starting backfill to osd.3 from (0'0,0'0] MAX to 325'7376 2023-09-23T19:03:00.749 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:00 smithi098 bash[76817]: cluster 2023-09-23T19:02:59.701687+0000 osd.4 (osd.4) 58 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T19:03:00.749 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:00 smithi098 bash[76817]: cluster 2023-09-23T19:03:00.475151+0000 mon.a (mon.0) 1854 : cluster 3 Health check update: Degraded data redundancy: 49243/261399 objects degraded (18.838%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:00.749 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:00 smithi098 bash[79516]: cluster 2023-09-23T19:02:59.665681+0000 osd.6 (osd.6) 304 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T19:03:00.749 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:00 smithi098 bash[79516]: cluster 2023-09-23T19:02:59.670213+0000 osd.0 (osd.0) 37 : cluster 0 10.c starting backfill to osd.3 from (0'0,0'0] MAX to 325'7376 2023-09-23T19:03:00.749 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:00 smithi098 bash[79516]: cluster 2023-09-23T19:02:59.701687+0000 osd.4 (osd.4) 58 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T19:03:00.749 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:00 smithi098 bash[79516]: cluster 2023-09-23T19:03:00.475151+0000 mon.a (mon.0) 1854 : cluster 3 Health check update: Degraded data redundancy: 49243/261399 objects degraded (18.838%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:00.772 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:00.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:00.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:00.878 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:00.915 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:00.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:00.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:01.018 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:01.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:00 smithi173 bash[59422]: cluster 2023-09-23T19:02:59.665681+0000 osd.6 (osd.6) 304 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T19:03:01.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:00 smithi173 bash[59422]: cluster 2023-09-23T19:02:59.670213+0000 osd.0 (osd.0) 37 : cluster 0 10.c starting backfill to osd.3 from (0'0,0'0] MAX to 325'7376 2023-09-23T19:03:01.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:00 smithi173 bash[59422]: cluster 2023-09-23T19:02:59.701687+0000 osd.4 (osd.4) 58 : cluster 3 failed to encode map e328 with expected crc 2023-09-23T19:03:01.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:00 smithi173 bash[59422]: cluster 2023-09-23T19:03:00.475151+0000 mon.a (mon.0) 1854 : cluster 3 Health check update: Degraded data redundancy: 49243/261399 objects degraded (18.838%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:01.053 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:01.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:01.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:01.164 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:01.193 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:01.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:01.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:01.296 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:01.329 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:01.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:01.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:01.433 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:01.476 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:01.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:01.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:01.581 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:01.613 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:01.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:01.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:01.718 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:01.746 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:01.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:01.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:01.853 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:01.887 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:01.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:01.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:01.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:01 smithi098 bash[76817]: cluster 2023-09-23T19:03:00.484365+0000 mgr.y (mgr.26054) 1353 : cluster 0 pgmap v1487: 89 pgs: 1 remapped+peering, 2 peering, 15 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 15 active+undersized+degraded, 14 active+undersized; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 43194/258312 objects degraded (16.722%); 15/258312 objects misplaced (0.006%) 2023-09-23T19:03:01.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:01 smithi098 bash[79516]: cluster 2023-09-23T19:03:00.484365+0000 mgr.y (mgr.26054) 1353 : cluster 0 pgmap v1487: 89 pgs: 1 remapped+peering, 2 peering, 15 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 15 active+undersized+degraded, 14 active+undersized; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 43194/258312 objects degraded (16.722%); 15/258312 objects misplaced (0.006%) 2023-09-23T19:03:01.994 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:02.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:01 smithi173 bash[59422]: cluster 2023-09-23T19:03:00.484365+0000 mgr.y (mgr.26054) 1353 : cluster 0 pgmap v1487: 89 pgs: 1 remapped+peering, 2 peering, 15 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 15 active+undersized+degraded, 14 active+undersized; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 43194/258312 objects degraded (16.722%); 15/258312 objects misplaced (0.006%) 2023-09-23T19:03:02.045 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:02.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:02.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:02.150 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:02.179 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:02.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:02.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:02.285 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:02.313 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:02.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:02.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:02.422 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:02.451 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:02.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:02.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:02.557 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:02.586 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:02.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:02.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:02.689 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:02.719 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:02.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:02.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:02.822 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:02.865 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:02.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:02.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:02.972 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:03.009 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:03.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:03.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:03.111 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:03.137 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:03.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:03.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:03.244 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:03.245 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:03 smithi098 bash[79516]: audit 2023-09-23T19:03:02.218867+0000 mon.a (mon.0) 1855 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:03:03.245 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:03 smithi098 bash[79516]: audit 2023-09-23T19:03:02.227421+0000 mon.a (mon.0) 1856 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:03:03.245 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:03 smithi098 bash[79516]: audit 2023-09-23T19:03:02.858408+0000 mon.a (mon.0) 1857 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:03:03.245 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:03 smithi098 bash[79516]: audit 2023-09-23T19:03:02.859767+0000 mon.a (mon.0) 1858 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:03:03.245 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:03 smithi098 bash[79516]: audit 2023-09-23T19:03:02.867137+0000 mon.a (mon.0) 1859 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:03:03.245 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:03 smithi098 bash[76817]: audit 2023-09-23T19:03:02.218867+0000 mon.a (mon.0) 1855 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:03:03.246 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:03 smithi098 bash[76817]: audit 2023-09-23T19:03:02.227421+0000 mon.a (mon.0) 1856 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:03:03.246 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:03 smithi098 bash[76817]: audit 2023-09-23T19:03:02.858408+0000 mon.a (mon.0) 1857 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:03:03.246 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:03 smithi098 bash[76817]: audit 2023-09-23T19:03:02.859767+0000 mon.a (mon.0) 1858 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:03:03.246 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:03 smithi098 bash[76817]: audit 2023-09-23T19:03:02.867137+0000 mon.a (mon.0) 1859 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:03:03.272 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:03.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:03.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:03.375 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:03.408 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:03.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:03.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:03.513 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:03 smithi173 bash[59422]: audit 2023-09-23T19:03:02.218867+0000 mon.a (mon.0) 1855 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:03:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:03 smithi173 bash[59422]: audit 2023-09-23T19:03:02.227421+0000 mon.a (mon.0) 1856 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:03:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:03 smithi173 bash[59422]: audit 2023-09-23T19:03:02.858408+0000 mon.a (mon.0) 1857 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:03:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:03 smithi173 bash[59422]: audit 2023-09-23T19:03:02.859767+0000 mon.a (mon.0) 1858 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:03:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:03 smithi173 bash[59422]: audit 2023-09-23T19:03:02.867137+0000 mon.a (mon.0) 1859 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:03:03.539 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:03.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:03.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:03.643 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:03.664 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:03.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:03.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:03.691 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:03:03.697 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:03:03.768 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:03.797 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:03.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:03.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:03.871 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:03:03.901 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:03.942 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:03.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:03.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:04.050 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:04.073 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:04.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:04.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:04.176 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:04.204 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:04.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:04.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:04.308 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:04.335 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:04.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:04.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:04.438 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:04.469 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:04.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:04.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:04.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:04 smithi173 bash[59422]: cluster 2023-09-23T19:03:02.485130+0000 mgr.y (mgr.26054) 1354 : cluster 0 pgmap v1488: 89 pgs: 1 remapped+peering, 2 peering, 17 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 15 active+undersized+degraded, 14 active+undersized; 5.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 42633/255519 objects degraded (16.685%); 15/255519 objects misplaced (0.006%) 2023-09-23T19:03:04.572 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:04.587 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:04 smithi098 bash[76817]: cluster 2023-09-23T19:03:02.485130+0000 mgr.y (mgr.26054) 1354 : cluster 0 pgmap v1488: 89 pgs: 1 remapped+peering, 2 peering, 17 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 15 active+undersized+degraded, 14 active+undersized; 5.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 42633/255519 objects degraded (16.685%); 15/255519 objects misplaced (0.006%) 2023-09-23T19:03:04.587 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:04 smithi098 bash[79516]: cluster 2023-09-23T19:03:02.485130+0000 mgr.y (mgr.26054) 1354 : cluster 0 pgmap v1488: 89 pgs: 1 remapped+peering, 2 peering, 17 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 15 active+undersized+degraded, 14 active+undersized; 5.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 42633/255519 objects degraded (16.685%); 15/255519 objects misplaced (0.006%) 2023-09-23T19:03:04.595 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:04.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:04.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:04.701 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:04.722 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:04.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:04.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:04.826 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:04.868 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:04.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:04.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:04.972 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:04.995 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:05.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:05.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:05.103 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:05.146 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:05.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:05.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:05.253 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:05.282 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:05.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:05.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:05.384 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:05.410 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:05.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:05.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:05.513 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:05.544 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:05.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:05.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:05.647 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:05.677 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:05.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:05.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:05.782 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:05.807 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:05.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:05.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:05.911 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:05.942 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:05.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:05.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:06.049 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:06.068 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:06.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:06.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:06.172 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:06.193 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:03:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:03:06] "GET /metrics HTTP/1.1" 200 36648 "" "Prometheus/2.33.4" 2023-09-23T19:03:06.203 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:06.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:06.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:06.306 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:06.336 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:06.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:06.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:06.439 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:06.477 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:06.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:06.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:06.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:06 smithi173 bash[59422]: cluster 2023-09-23T19:03:04.485936+0000 mgr.y (mgr.26054) 1355 : cluster 0 pgmap v1489: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 47066/251262 objects degraded (18.732%); 5245/251262 objects misplaced (2.087%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T19:03:06.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:06 smithi173 bash[59422]: cluster 2023-09-23T19:03:05.476197+0000 mon.a (mon.0) 1860 : cluster 3 Health check update: Degraded data redundancy: 47066/251262 objects degraded (18.732%), 16 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:06.585 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:06.612 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:06.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:06.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:06.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:06 smithi098 bash[79516]: cluster 2023-09-23T19:03:04.485936+0000 mgr.y (mgr.26054) 1355 : cluster 0 pgmap v1489: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 47066/251262 objects degraded (18.732%); 5245/251262 objects misplaced (2.087%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T19:03:06.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:06 smithi098 bash[79516]: cluster 2023-09-23T19:03:05.476197+0000 mon.a (mon.0) 1860 : cluster 3 Health check update: Degraded data redundancy: 47066/251262 objects degraded (18.732%), 16 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:06.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:06 smithi098 bash[76817]: cluster 2023-09-23T19:03:04.485936+0000 mgr.y (mgr.26054) 1355 : cluster 0 pgmap v1489: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 47066/251262 objects degraded (18.732%); 5245/251262 objects misplaced (2.087%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T19:03:06.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:06 smithi098 bash[76817]: cluster 2023-09-23T19:03:05.476197+0000 mon.a (mon.0) 1860 : cluster 3 Health check update: Degraded data redundancy: 47066/251262 objects degraded (18.732%), 16 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:06.716 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:06.740 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:06.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:06.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:06.844 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:06.875 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:06.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:06.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:06.980 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:07.009 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:07.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:07.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:07.113 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:07.141 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:07.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:07.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:07.244 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:07.270 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:07.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:07.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:07.373 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:07.396 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:07.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:07.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:07.498 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:07.525 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:07.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:07.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:07.629 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:07.655 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:07.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:07.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:07.760 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:07.787 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:07.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:07.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:07.890 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:07.915 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:07.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:07.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:08.018 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:08.045 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:08.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:08.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:08.149 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:08.184 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:08.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:08.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:08.292 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:08.325 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:08.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:08.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:08.429 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:08.437 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:08 smithi098 bash[76817]: cluster 2023-09-23T19:03:06.486592+0000 mgr.y (mgr.26054) 1356 : cluster 0 pgmap v1490: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 422 op/s; 47066/251262 objects degraded (18.732%); 5245/251262 objects misplaced (2.087%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T19:03:08.437 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:08 smithi098 bash[79516]: cluster 2023-09-23T19:03:06.486592+0000 mgr.y (mgr.26054) 1356 : cluster 0 pgmap v1490: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 422 op/s; 47066/251262 objects degraded (18.732%); 5245/251262 objects misplaced (2.087%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T19:03:08.474 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:08.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:08.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:08.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:08 smithi173 bash[59422]: cluster 2023-09-23T19:03:06.486592+0000 mgr.y (mgr.26054) 1356 : cluster 0 pgmap v1490: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 422 op/s; 47066/251262 objects degraded (18.732%); 5245/251262 objects misplaced (2.087%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T19:03:08.579 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:08.611 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:08.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:08.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:08.718 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:08.745 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:08.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:08.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:08.848 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:08.884 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:08.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:08.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:08.990 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:09.019 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:09.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:09.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:09.041 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T19:03:09.041 INFO:teuthology.orchestra.run.smithi098.stdout: "size": 20, 2023-09-23T19:03:09.042 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 600, 2023-09-23T19:03:09.042 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [ 2023-09-23T19:03:09.042 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.042 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45602.0:147822 9.1 9:8b17f84e:::benchmark_data_smithi098_206513_object52283:head [delete] snapc 0=[] ondisk+write+known_if_redirected e297)", 2023-09-23T19:03:09.042 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:58:37.650423+0000", 2023-09-23T19:03:09.042 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 271.38552503400001, 2023-09-23T19:03:09.042 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.1092193739999998, 2023-09-23T19:03:09.042 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:03:09.042 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:03:09.043 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:03:09.043 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45602", 2023-09-23T19:03:09.043 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T19:03:09.043 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 147822 2023-09-23T19:03:09.043 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.043 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:03:09.043 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.043 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:03:09.043 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.650423+0000", 2023-09-23T19:03:09.043 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.043 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.044 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.044 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:03:09.044 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.650423+0000", 2023-09-23T19:03:09.044 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.044 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.044 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.044 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:03:09.044 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.650420+0000", 2023-09-23T19:03:09.044 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999976 2023-09-23T19:03:09.044 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.045 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.045 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:03:09.045 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.650430+0000", 2023-09-23T19:03:09.045 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.6970000000000005e-06 2023-09-23T19:03:09.045 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.045 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.045 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:03:09.045 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.650448+0000", 2023-09-23T19:03:09.045 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.7727000000000002e-05 2023-09-23T19:03:09.045 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.045 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.046 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:03:09.046 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.650459+0000", 2023-09-23T19:03:09.046 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0893999999999999e-05 2023-09-23T19:03:09.046 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.046 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.046 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.046 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.658636+0000", 2023-09-23T19:03:09.046 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0081769289999999994 2023-09-23T19:03:09.046 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.046 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.047 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:03:09.047 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.658782+0000", 2023-09-23T19:03:09.047 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00014625700000000001 2023-09-23T19:03:09.047 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.047 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.047 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 4,6", 2023-09-23T19:03:09.047 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.658910+0000", 2023-09-23T19:03:09.047 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00012775500000000001 2023-09-23T19:03:09.047 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.047 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.047 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:03:09.048 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.676585+0000", 2023-09-23T19:03:09.048 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.017675290999999999 2023-09-23T19:03:09.048 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.048 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.048 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:03:09.048 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.676614+0000", 2023-09-23T19:03:09.048 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.9512000000000001e-05 2023-09-23T19:03:09.048 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.048 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.048 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.048 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.722618+0000", 2023-09-23T19:03:09.049 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.046003891999999998 2023-09-23T19:03:09.049 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.049 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.049 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:03:09.049 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.722632+0000", 2023-09-23T19:03:09.049 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3726999999999999e-05 2023-09-23T19:03:09.049 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.049 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.049 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.049 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:38.711934+0000", 2023-09-23T19:03:09.049 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.98930178800000002 2023-09-23T19:03:09.050 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.050 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.050 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:03:09.050 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:38.711947+0000", 2023-09-23T19:03:09.050 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3193999999999999e-05 2023-09-23T19:03:09.050 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.050 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.050 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.051 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:39.759559+0000", 2023-09-23T19:03:09.052 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.047612564 2023-09-23T19:03:09.052 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.052 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.052 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:03:09.052 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:39.759642+0000", 2023-09-23T19:03:09.052 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.2540999999999995e-05 2023-09-23T19:03:09.052 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.053 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:03:09.053 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.053 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.053 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.053 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45602.0:147824 9.1 9:8baefee8:::benchmark_data_smithi098_206513_object52285:head [delete] snapc 0=[] ondisk+write+known_if_redirected e297)", 2023-09-23T19:03:09.053 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:58:37.655056+0000", 2023-09-23T19:03:09.053 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 271.38089206199999, 2023-09-23T19:03:09.053 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.1068159070000001, 2023-09-23T19:03:09.053 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:03:09.053 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:03:09.053 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:03:09.054 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45602", 2023-09-23T19:03:09.054 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T19:03:09.054 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 147824 2023-09-23T19:03:09.054 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.054 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:03:09.054 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.054 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:03:09.054 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.655056+0000", 2023-09-23T19:03:09.054 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.054 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.055 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.055 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:03:09.055 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.655056+0000", 2023-09-23T19:03:09.055 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.055 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.055 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.055 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:03:09.055 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.655053+0000", 2023-09-23T19:03:09.055 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999976 2023-09-23T19:03:09.055 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.056 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.056 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:03:09.056 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.655061+0000", 2023-09-23T19:03:09.056 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.695e-06 2023-09-23T19:03:09.056 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.056 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.056 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:03:09.056 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.655144+0000", 2023-09-23T19:03:09.056 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.3135999999999998e-05 2023-09-23T19:03:09.056 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.057 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.057 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:03:09.057 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.655155+0000", 2023-09-23T19:03:09.057 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0496000000000001e-05 2023-09-23T19:03:09.057 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.057 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.057 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.057 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.663693+0000", 2023-09-23T19:03:09.057 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0085381950000000002 2023-09-23T19:03:09.057 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.058 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.058 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:03:09.058 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.663891+0000", 2023-09-23T19:03:09.058 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000198345 2023-09-23T19:03:09.058 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.058 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.058 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 4,6", 2023-09-23T19:03:09.058 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.664083+0000", 2023-09-23T19:03:09.058 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00019180999999999999 2023-09-23T19:03:09.058 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.059 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.059 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:03:09.059 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.676643+0000", 2023-09-23T19:03:09.059 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.012560266 2023-09-23T19:03:09.059 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.059 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.059 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:03:09.059 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.714463+0000", 2023-09-23T19:03:09.059 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.037819879000000001 2023-09-23T19:03:09.059 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.059 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.060 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.060 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.725400+0000", 2023-09-23T19:03:09.060 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.010936634000000001 2023-09-23T19:03:09.060 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.060 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.060 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:03:09.060 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.725414+0000", 2023-09-23T19:03:09.060 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.383e-05 2023-09-23T19:03:09.060 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.060 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.061 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.061 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:38.711979+0000", 2023-09-23T19:03:09.061 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.98656518199999998 2023-09-23T19:03:09.061 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.061 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.061 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:03:09.061 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:38.711990+0000", 2023-09-23T19:03:09.061 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0749e-05 2023-09-23T19:03:09.061 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.061 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.062 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.062 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:39.761803+0000", 2023-09-23T19:03:09.062 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.049813581 2023-09-23T19:03:09.062 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.062 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.062 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:03:09.062 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:39.761872+0000", 2023-09-23T19:03:09.062 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.8394999999999998e-05 2023-09-23T19:03:09.062 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.062 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:03:09.063 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.063 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.063 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.063 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45602.0:147858 9.1 9:82f5e94f:::benchmark_data_smithi098_206513_object52319:head [delete] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-09-23T19:03:09.063 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T18:58:37.734023+0000", 2023-09-23T19:03:09.063 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 271.30192499899999, 2023-09-23T19:03:09.063 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.0654937969999998, 2023-09-23T19:03:09.063 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:03:09.063 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:03:09.063 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:03:09.063 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45602", 2023-09-23T19:03:09.064 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3051714715", 2023-09-23T19:03:09.064 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 147858 2023-09-23T19:03:09.064 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.064 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:03:09.064 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.064 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:03:09.064 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.734023+0000", 2023-09-23T19:03:09.064 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.064 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.064 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.065 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:03:09.065 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.734023+0000", 2023-09-23T19:03:09.065 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.065 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.065 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.065 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:03:09.065 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.734021+0000", 2023-09-23T19:03:09.065 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:03:09.065 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.065 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.066 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:03:09.066 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.734029+0000", 2023-09-23T19:03:09.066 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.9049999999999997e-06 2023-09-23T19:03:09.066 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.066 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.066 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:03:09.066 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.734040+0000", 2023-09-23T19:03:09.066 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1256e-05 2023-09-23T19:03:09.066 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.066 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.066 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:03:09.067 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.734061+0000", 2023-09-23T19:03:09.067 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.1443000000000001e-05 2023-09-23T19:03:09.067 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.067 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.067 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.067 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.734338+0000", 2023-09-23T19:03:09.067 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00027639199999999998 2023-09-23T19:03:09.067 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.067 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.068 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:03:09.068 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:37.734351+0000", 2023-09-23T19:03:09.068 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3077e-05 2023-09-23T19:03:09.068 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.068 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.068 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.068 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:38.712457+0000", 2023-09-23T19:03:09.068 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.97810648099999997 2023-09-23T19:03:09.068 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.068 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.068 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:03:09.069 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:38.712467+0000", 2023-09-23T19:03:09.069 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0149000000000001e-05 2023-09-23T19:03:09.069 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.069 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.069 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.069 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:39.761894+0000", 2023-09-23T19:03:09.069 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0494267660000001 2023-09-23T19:03:09.069 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.069 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.069 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:03:09.070 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:39.762025+0000", 2023-09-23T19:03:09.070 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000130724 2023-09-23T19:03:09.070 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.070 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.070 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 4,6", 2023-09-23T19:03:09.070 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:39.762882+0000", 2023-09-23T19:03:09.070 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00085703099999999996 2023-09-23T19:03:09.070 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.070 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.070 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:03:09.071 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:39.786320+0000", 2023-09-23T19:03:09.071 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.023437746999999998 2023-09-23T19:03:09.071 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.071 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.071 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:03:09.071 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:39.786359+0000", 2023-09-23T19:03:09.071 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.8949999999999998e-05 2023-09-23T19:03:09.071 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.071 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.071 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:03:09.071 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:39.797193+0000", 2023-09-23T19:03:09.072 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.010834773000000001 2023-09-23T19:03:09.072 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.072 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.072 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:03:09.072 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:39.797249+0000", 2023-09-23T19:03:09.072 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.5855e-05 2023-09-23T19:03:09.072 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.072 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.072 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:03:09.072 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T18:58:39.799516+0000", 2023-09-23T19:03:09.073 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0022673089999999999 2023-09-23T19:03:09.073 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.074 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:03:09.075 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.075 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.075 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.075 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:114899 10.c 10.5bfc659c (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:03:09.075 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.675112+0000", 2023-09-23T19:03:09.075 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 11.360836047999999, 2023-09-23T19:03:09.075 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.99592086700000004, 2023-09-23T19:03:09.075 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:03:09.075 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:03:09.076 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:03:09.076 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:03:09.076 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:03:09.076 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 114899 2023-09-23T19:03:09.076 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.076 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:03:09.076 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.076 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:03:09.076 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.675112+0000", 2023-09-23T19:03:09.077 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.077 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.077 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.077 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:03:09.077 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.675112+0000", 2023-09-23T19:03:09.077 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.077 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.077 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.077 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:03:09.077 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.675110+0000", 2023-09-23T19:03:09.077 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:03:09.078 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.078 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.078 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:03:09.078 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.675119+0000", 2023-09-23T19:03:09.078 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.7569999999999993e-06 2023-09-23T19:03:09.078 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.078 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.078 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:03:09.078 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.675134+0000", 2023-09-23T19:03:09.079 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.5721000000000001e-05 2023-09-23T19:03:09.079 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.079 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.079 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:03:09.079 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.675144+0000", 2023-09-23T19:03:09.079 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.7280000000000007e-06 2023-09-23T19:03:09.079 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.079 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.079 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.079 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.679395+0000", 2023-09-23T19:03:09.080 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0042507300000000003 2023-09-23T19:03:09.080 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.080 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.080 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:03:09.080 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.679403+0000", 2023-09-23T19:03:09.080 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.2870000000000004e-06 2023-09-23T19:03:09.080 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.080 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.080 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.080 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671016+0000", 2023-09-23T19:03:09.080 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.99161303899999997 2023-09-23T19:03:09.081 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.081 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.081 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:03:09.081 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671033+0000", 2023-09-23T19:03:09.081 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.6325e-05 2023-09-23T19:03:09.081 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.081 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:03:09.081 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.081 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.082 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.082 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:114902 10.c 10.72deeabc (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:03:09.082 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.689552+0000", 2023-09-23T19:03:09.082 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 11.346395383000001, 2023-09-23T19:03:09.082 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.98151502700000004, 2023-09-23T19:03:09.082 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:03:09.082 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:03:09.082 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:03:09.082 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:03:09.082 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:03:09.083 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 114902 2023-09-23T19:03:09.083 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.083 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:03:09.083 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.083 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:03:09.083 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.689552+0000", 2023-09-23T19:03:09.083 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.083 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.083 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.083 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:03:09.083 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.689552+0000", 2023-09-23T19:03:09.084 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.084 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.084 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.084 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:03:09.084 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.689550+0000", 2023-09-23T19:03:09.084 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:03:09.084 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.084 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.084 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:03:09.084 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.689560+0000", 2023-09-23T19:03:09.085 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.8779999999999993e-06 2023-09-23T19:03:09.085 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.085 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.085 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:03:09.085 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.689578+0000", 2023-09-23T19:03:09.085 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.7966999999999999e-05 2023-09-23T19:03:09.085 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.085 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.085 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:03:09.085 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.689588+0000", 2023-09-23T19:03:09.085 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.7969999999999995e-06 2023-09-23T19:03:09.086 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.086 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.086 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.086 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.689648+0000", 2023-09-23T19:03:09.086 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.9723999999999999e-05 2023-09-23T19:03:09.086 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.086 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.086 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:03:09.086 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.689657+0000", 2023-09-23T19:03:09.086 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.6490000000000003e-06 2023-09-23T19:03:09.087 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.087 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.087 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.087 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671060+0000", 2023-09-23T19:03:09.087 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.98140266799999998 2023-09-23T19:03:09.087 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.087 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.087 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:03:09.087 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671067+0000", 2023-09-23T19:03:09.087 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.4830000000000002e-06 2023-09-23T19:03:09.088 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.088 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:03:09.088 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.088 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.088 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.088 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:114909 10.c 10.dad3dbac (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:03:09.088 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.705355+0000", 2023-09-23T19:03:09.088 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 11.330592606, 2023-09-23T19:03:09.088 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.96574279900000004, 2023-09-23T19:03:09.088 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:03:09.089 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:03:09.089 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:03:09.089 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:03:09.089 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:03:09.089 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 114909 2023-09-23T19:03:09.089 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.089 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:03:09.089 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.089 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:03:09.089 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705355+0000", 2023-09-23T19:03:09.089 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.090 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.090 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.090 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:03:09.090 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705355+0000", 2023-09-23T19:03:09.090 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.090 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.090 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.090 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:03:09.090 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705353+0000", 2023-09-23T19:03:09.091 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:03:09.091 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.091 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.091 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:03:09.091 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705364+0000", 2023-09-23T19:03:09.091 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1090999999999999e-05 2023-09-23T19:03:09.091 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.091 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.091 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:03:09.091 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705386+0000", 2023-09-23T19:03:09.092 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.2014e-05 2023-09-23T19:03:09.092 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.092 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.092 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:03:09.092 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705398+0000", 2023-09-23T19:03:09.092 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.15e-05 2023-09-23T19:03:09.092 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.092 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.092 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.093 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705530+0000", 2023-09-23T19:03:09.093 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00013205899999999999 2023-09-23T19:03:09.093 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.093 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.093 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:03:09.093 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705538+0000", 2023-09-23T19:03:09.093 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.6349999999999995e-06 2023-09-23T19:03:09.093 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.093 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.093 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.094 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671090+0000", 2023-09-23T19:03:09.094 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.96555181099999998 2023-09-23T19:03:09.094 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.094 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.094 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:03:09.094 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671098+0000", 2023-09-23T19:03:09.094 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.6599999999999995e-06 2023-09-23T19:03:09.094 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.094 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:03:09.094 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.095 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.095 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.095 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:114918 10.c 10.4f6f543c (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:03:09.095 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.705455+0000", 2023-09-23T19:03:09.095 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 11.330492459, 2023-09-23T19:03:09.095 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.96567159899999999, 2023-09-23T19:03:09.095 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:03:09.095 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:03:09.095 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:03:09.095 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:03:09.096 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:03:09.096 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 114918 2023-09-23T19:03:09.096 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.096 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:03:09.096 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.096 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:03:09.096 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705455+0000", 2023-09-23T19:03:09.096 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.096 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.097 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.097 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:03:09.097 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705455+0000", 2023-09-23T19:03:09.097 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.097 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.097 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.097 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:03:09.097 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705455+0000", 2023-09-23T19:03:09.097 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T19:03:09.098 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.098 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.098 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:03:09.098 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705458+0000", 2023-09-23T19:03:09.098 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.1319999999999998e-06 2023-09-23T19:03:09.098 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.098 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.098 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:03:09.098 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705478+0000", 2023-09-23T19:03:09.098 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.9837999999999999e-05 2023-09-23T19:03:09.098 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.099 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.099 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:03:09.099 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705484+0000", 2023-09-23T19:03:09.099 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.5230000000000001e-06 2023-09-23T19:03:09.099 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.099 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.099 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.099 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705561+0000", 2023-09-23T19:03:09.099 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.6579e-05 2023-09-23T19:03:09.100 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.100 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.100 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:03:09.100 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705566+0000", 2023-09-23T19:03:09.100 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.2209999999999996e-06 2023-09-23T19:03:09.100 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.100 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.100 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.100 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671119+0000", 2023-09-23T19:03:09.100 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.96555351499999997 2023-09-23T19:03:09.101 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.101 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.101 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:03:09.101 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671127+0000", 2023-09-23T19:03:09.101 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.3479999999999998e-06 2023-09-23T19:03:09.101 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.101 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:03:09.101 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.101 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.101 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.101 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:114946 10.c 10.215a248c (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:03:09.102 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.753910+0000", 2023-09-23T19:03:09.102 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 11.282037504, 2023-09-23T19:03:09.102 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.91724539999999999, 2023-09-23T19:03:09.102 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:03:09.102 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:03:09.102 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:03:09.102 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:03:09.102 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:03:09.102 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 114946 2023-09-23T19:03:09.103 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.103 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:03:09.103 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.103 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:03:09.103 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.753910+0000", 2023-09-23T19:03:09.103 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.103 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.103 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.104 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:03:09.104 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.753910+0000", 2023-09-23T19:03:09.104 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.104 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.104 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.104 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:03:09.104 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.753909+0000", 2023-09-23T19:03:09.104 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T19:03:09.104 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.104 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.104 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:03:09.105 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.753916+0000", 2023-09-23T19:03:09.105 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.6200000000000001e-06 2023-09-23T19:03:09.105 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.105 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.105 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:03:09.105 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.753927+0000", 2023-09-23T19:03:09.105 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1603999999999999e-05 2023-09-23T19:03:09.105 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.105 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.105 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:03:09.105 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.753936+0000", 2023-09-23T19:03:09.106 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.6810000000000004e-06 2023-09-23T19:03:09.106 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.106 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.106 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.106 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.753986+0000", 2023-09-23T19:03:09.106 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.9886000000000003e-05 2023-09-23T19:03:09.106 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.106 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.106 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:03:09.106 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.753993+0000", 2023-09-23T19:03:09.107 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.6459999999999993e-06 2023-09-23T19:03:09.107 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.107 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.107 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.107 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671148+0000", 2023-09-23T19:03:09.107 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.91715488000000001 2023-09-23T19:03:09.107 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.107 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.107 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:03:09.107 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671156+0000", 2023-09-23T19:03:09.108 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.3669999999999999e-06 2023-09-23T19:03:09.108 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.108 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:03:09.108 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.108 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.108 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.108 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:114982 10.c 10.48e0bdac (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:03:09.108 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.803538+0000", 2023-09-23T19:03:09.108 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 11.232409801999999, 2023-09-23T19:03:09.108 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.86764724699999995, 2023-09-23T19:03:09.108 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:03:09.109 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:03:09.109 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:03:09.109 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:03:09.109 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:03:09.109 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 114982 2023-09-23T19:03:09.109 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.109 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:03:09.109 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.109 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:03:09.109 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.803538+0000", 2023-09-23T19:03:09.109 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.110 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.110 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.110 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:03:09.110 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.803538+0000", 2023-09-23T19:03:09.110 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.110 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.110 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.110 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:03:09.110 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.803536+0000", 2023-09-23T19:03:09.110 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:03:09.111 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.111 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.111 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:03:09.111 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.803545+0000", 2023-09-23T19:03:09.111 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.755e-06 2023-09-23T19:03:09.111 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.111 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.111 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:03:09.111 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.803559+0000", 2023-09-23T19:03:09.111 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3834e-05 2023-09-23T19:03:09.112 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.112 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.112 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:03:09.112 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.803568+0000", 2023-09-23T19:03:09.112 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.6190000000000006e-06 2023-09-23T19:03:09.112 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.112 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.112 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.112 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.803647+0000", 2023-09-23T19:03:09.113 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.8288999999999995e-05 2023-09-23T19:03:09.113 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.113 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.113 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:03:09.113 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.803656+0000", 2023-09-23T19:03:09.113 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.0119999999999994e-06 2023-09-23T19:03:09.113 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.113 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.113 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.113 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671178+0000", 2023-09-23T19:03:09.113 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.86752198000000003 2023-09-23T19:03:09.114 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.114 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.114 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:03:09.114 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671185+0000", 2023-09-23T19:03:09.114 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.542e-06 2023-09-23T19:03:09.114 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.114 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:03:09.114 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.114 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.114 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.115 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:114993 10.c 10.7d901f2c (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:03:09.115 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.817716+0000", 2023-09-23T19:03:09.115 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 11.218232175000001, 2023-09-23T19:03:09.115 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.85350135100000002, 2023-09-23T19:03:09.115 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:03:09.115 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:03:09.115 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:03:09.115 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:03:09.115 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:03:09.115 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 114993 2023-09-23T19:03:09.116 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.116 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:03:09.116 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.116 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:03:09.116 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.817716+0000", 2023-09-23T19:03:09.116 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.116 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.116 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.116 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:03:09.116 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.817716+0000", 2023-09-23T19:03:09.116 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.117 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.117 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.117 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:03:09.117 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.817713+0000", 2023-09-23T19:03:09.117 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:03:09.117 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.117 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.117 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:03:09.117 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.817723+0000", 2023-09-23T19:03:09.117 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.5510000000000005e-06 2023-09-23T19:03:09.118 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.118 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.118 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:03:09.118 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.817745+0000", 2023-09-23T19:03:09.118 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.2266999999999999e-05 2023-09-23T19:03:09.118 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.118 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.118 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:03:09.118 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.817756+0000", 2023-09-23T19:03:09.118 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1134e-05 2023-09-23T19:03:09.119 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.119 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.119 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.119 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.817816+0000", 2023-09-23T19:03:09.119 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.9503999999999998e-05 2023-09-23T19:03:09.119 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.119 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.119 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:03:09.119 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.817827+0000", 2023-09-23T19:03:09.119 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0932e-05 2023-09-23T19:03:09.119 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.120 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.120 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.120 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671210+0000", 2023-09-23T19:03:09.120 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.85338271600000004 2023-09-23T19:03:09.120 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.120 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.120 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:03:09.120 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671217+0000", 2023-09-23T19:03:09.120 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.3470000000000001e-06 2023-09-23T19:03:09.120 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.121 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:03:09.121 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.121 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.121 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.121 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:115010 10.c 10.5c53734c (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:03:09.121 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.845185+0000", 2023-09-23T19:03:09.121 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 11.190762347, 2023-09-23T19:03:09.121 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.826060935, 2023-09-23T19:03:09.121 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:03:09.122 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:03:09.122 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:03:09.122 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:03:09.122 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:03:09.122 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 115010 2023-09-23T19:03:09.122 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.122 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:03:09.122 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.122 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:03:09.122 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.845185+0000", 2023-09-23T19:03:09.123 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.123 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.123 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.123 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:03:09.123 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.845185+0000", 2023-09-23T19:03:09.123 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.123 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.123 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.123 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:03:09.123 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.845184+0000", 2023-09-23T19:03:09.124 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:03:09.124 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.124 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.124 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:03:09.124 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.845193+0000", 2023-09-23T19:03:09.124 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.2490000000000007e-06 2023-09-23T19:03:09.124 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.124 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.124 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:03:09.124 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.845209+0000", 2023-09-23T19:03:09.125 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.6053e-05 2023-09-23T19:03:09.125 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.125 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.125 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:03:09.125 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.845219+0000", 2023-09-23T19:03:09.125 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.7850000000000003e-06 2023-09-23T19:03:09.125 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.125 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.125 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.125 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.845280+0000", 2023-09-23T19:03:09.125 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.1080999999999993e-05 2023-09-23T19:03:09.126 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.126 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.126 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:03:09.126 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.845291+0000", 2023-09-23T19:03:09.126 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1334e-05 2023-09-23T19:03:09.126 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.126 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.126 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.126 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671239+0000", 2023-09-23T19:03:09.126 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.82594783400000005 2023-09-23T19:03:09.127 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.127 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.127 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:03:09.127 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671246+0000", 2023-09-23T19:03:09.127 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.357e-06 2023-09-23T19:03:09.127 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.127 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:03:09.127 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.127 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.127 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.127 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:115036 10.c 10.a31d034c (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:03:09.128 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.874971+0000", 2023-09-23T19:03:09.128 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 11.160977129999999, 2023-09-23T19:03:09.128 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.79630574600000004, 2023-09-23T19:03:09.128 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:03:09.128 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:03:09.128 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:03:09.128 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:03:09.128 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:03:09.128 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 115036 2023-09-23T19:03:09.128 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.129 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:03:09.129 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.129 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:03:09.129 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.874971+0000", 2023-09-23T19:03:09.129 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.129 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.129 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.129 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:03:09.129 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.874971+0000", 2023-09-23T19:03:09.130 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.130 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.130 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.130 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:03:09.130 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.874969+0000", 2023-09-23T19:03:09.130 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:03:09.130 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.130 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.130 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:03:09.131 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.874978+0000", 2023-09-23T19:03:09.131 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.7679999999999997e-06 2023-09-23T19:03:09.131 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.131 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.131 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:03:09.131 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.874995+0000", 2023-09-23T19:03:09.131 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.7399999999999999e-05 2023-09-23T19:03:09.131 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.131 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.131 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:03:09.132 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.875006+0000", 2023-09-23T19:03:09.132 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0742e-05 2023-09-23T19:03:09.132 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.132 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.132 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.132 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.875054+0000", 2023-09-23T19:03:09.132 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.8347000000000003e-05 2023-09-23T19:03:09.132 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.132 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.132 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:03:09.133 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.875064+0000", 2023-09-23T19:03:09.133 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0336999999999999e-05 2023-09-23T19:03:09.133 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.133 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.133 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.133 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671269+0000", 2023-09-23T19:03:09.133 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.79620438500000001 2023-09-23T19:03:09.133 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.133 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.133 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:03:09.133 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671276+0000", 2023-09-23T19:03:09.134 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.4610000000000002e-06 2023-09-23T19:03:09.134 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.134 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:03:09.134 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.134 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.134 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.134 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:115037 10.c 10.f007beec (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:03:09.134 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.878380+0000", 2023-09-23T19:03:09.134 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 11.157567265999999, 2023-09-23T19:03:09.134 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.79292473900000005, 2023-09-23T19:03:09.135 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:03:09.135 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:03:09.135 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:03:09.135 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:03:09.135 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:03:09.135 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 115037 2023-09-23T19:03:09.135 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.135 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:03:09.135 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.135 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:03:09.136 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.878380+0000", 2023-09-23T19:03:09.136 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.136 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.136 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.136 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:03:09.136 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.878380+0000", 2023-09-23T19:03:09.136 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.136 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.136 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.136 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:03:09.136 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.878379+0000", 2023-09-23T19:03:09.137 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T19:03:09.137 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.137 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.137 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:03:09.137 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.878387+0000", 2023-09-23T19:03:09.137 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.6040000000000004e-06 2023-09-23T19:03:09.137 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.137 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.137 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:03:09.137 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.878403+0000", 2023-09-23T19:03:09.138 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.6439999999999998e-05 2023-09-23T19:03:09.138 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.138 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.138 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:03:09.138 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.878411+0000", 2023-09-23T19:03:09.138 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.8429999999999993e-06 2023-09-23T19:03:09.138 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.138 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.138 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.138 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.878467+0000", 2023-09-23T19:03:09.138 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.6164e-05 2023-09-23T19:03:09.139 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.139 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.139 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:03:09.139 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.878478+0000", 2023-09-23T19:03:09.139 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0536e-05 2023-09-23T19:03:09.139 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.139 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.139 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.139 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671297+0000", 2023-09-23T19:03:09.139 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.79281938900000004 2023-09-23T19:03:09.140 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.140 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.140 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:03:09.140 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671305+0000", 2023-09-23T19:03:09.140 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.1790000000000008e-06 2023-09-23T19:03:09.140 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.140 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:03:09.140 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.140 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.141 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.141 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:115041 10.c 10.5e5c63ec (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:03:09.141 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.884610+0000", 2023-09-23T19:03:09.141 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 11.151337808999999, 2023-09-23T19:03:09.141 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.786727434, 2023-09-23T19:03:09.141 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:03:09.141 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:03:09.141 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:03:09.141 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:03:09.141 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:03:09.142 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 115041 2023-09-23T19:03:09.142 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.142 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:03:09.142 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.142 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:03:09.142 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.884610+0000", 2023-09-23T19:03:09.142 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.142 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.142 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.143 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:03:09.143 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.884610+0000", 2023-09-23T19:03:09.143 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.143 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.143 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.143 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:03:09.143 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.884608+0000", 2023-09-23T19:03:09.143 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:03:09.143 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.143 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.143 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:03:09.144 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.884617+0000", 2023-09-23T19:03:09.144 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.0280000000000006e-06 2023-09-23T19:03:09.144 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.144 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.144 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:03:09.144 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.884644+0000", 2023-09-23T19:03:09.144 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.7083000000000001e-05 2023-09-23T19:03:09.144 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.144 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.144 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:03:09.145 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.884658+0000", 2023-09-23T19:03:09.145 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3298999999999999e-05 2023-09-23T19:03:09.145 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.145 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.145 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.145 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.884724+0000", 2023-09-23T19:03:09.145 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.6623000000000002e-05 2023-09-23T19:03:09.145 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.145 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.145 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:03:09.146 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.884738+0000", 2023-09-23T19:03:09.146 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3377000000000001e-05 2023-09-23T19:03:09.146 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.146 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.146 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.146 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671330+0000", 2023-09-23T19:03:09.146 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.78659241000000002 2023-09-23T19:03:09.146 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.146 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.147 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:03:09.147 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671337+0000", 2023-09-23T19:03:09.147 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.3599999999999998e-06 2023-09-23T19:03:09.147 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.147 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:03:09.147 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.147 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.147 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.147 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:115065 10.c 10.355e755c (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:03:09.147 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.919459+0000", 2023-09-23T19:03:09.147 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 11.116488356, 2023-09-23T19:03:09.148 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.75190801100000004, 2023-09-23T19:03:09.148 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:03:09.148 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:03:09.148 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:03:09.148 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:03:09.148 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:03:09.148 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 115065 2023-09-23T19:03:09.148 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.148 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:03:09.148 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.149 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:03:09.149 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.919459+0000", 2023-09-23T19:03:09.149 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.149 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.149 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.149 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:03:09.149 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.919459+0000", 2023-09-23T19:03:09.149 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.149 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.149 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.149 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:03:09.150 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.919457+0000", 2023-09-23T19:03:09.150 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:03:09.150 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.150 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.150 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:03:09.150 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.919466+0000", 2023-09-23T19:03:09.150 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.6889999999999993e-06 2023-09-23T19:03:09.150 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.150 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.150 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:03:09.150 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.919484+0000", 2023-09-23T19:03:09.151 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.8042000000000001e-05 2023-09-23T19:03:09.151 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.151 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.151 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:03:09.151 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.919493+0000", 2023-09-23T19:03:09.151 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.4950000000000008e-06 2023-09-23T19:03:09.151 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.151 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.151 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.151 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.919534+0000", 2023-09-23T19:03:09.152 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.0978e-05 2023-09-23T19:03:09.152 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.152 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.152 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:03:09.152 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.919540+0000", 2023-09-23T19:03:09.152 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.7179999999999999e-06 2023-09-23T19:03:09.152 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.152 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.152 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.152 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671360+0000", 2023-09-23T19:03:09.152 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.75181916800000004 2023-09-23T19:03:09.153 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.153 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.153 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:03:09.153 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671367+0000", 2023-09-23T19:03:09.153 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.7589999999999997e-06 2023-09-23T19:03:09.153 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.153 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:03:09.153 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.153 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.153 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.153 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:115076 10.c 10.4cfe028c (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:03:09.154 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.938742+0000", 2023-09-23T19:03:09.154 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 11.097205453999999, 2023-09-23T19:03:09.154 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.73265688399999995, 2023-09-23T19:03:09.154 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:03:09.154 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:03:09.154 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:03:09.154 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:03:09.154 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:03:09.154 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 115076 2023-09-23T19:03:09.155 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.155 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:03:09.155 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.155 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:03:09.155 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938742+0000", 2023-09-23T19:03:09.155 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.155 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.155 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.155 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:03:09.155 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938742+0000", 2023-09-23T19:03:09.155 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.156 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.156 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.156 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:03:09.156 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938740+0000", 2023-09-23T19:03:09.156 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:03:09.156 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.156 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.156 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:03:09.156 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938750+0000", 2023-09-23T19:03:09.156 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.9259999999999995e-06 2023-09-23T19:03:09.157 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.157 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.157 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:03:09.157 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938773+0000", 2023-09-23T19:03:09.157 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.2254000000000001e-05 2023-09-23T19:03:09.157 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.157 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.157 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:03:09.157 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938784+0000", 2023-09-23T19:03:09.157 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.116e-05 2023-09-23T19:03:09.157 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.158 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.158 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.158 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938846+0000", 2023-09-23T19:03:09.158 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.2718999999999994e-05 2023-09-23T19:03:09.158 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.158 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.158 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:03:09.158 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938858+0000", 2023-09-23T19:03:09.158 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1584e-05 2023-09-23T19:03:09.159 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.159 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.159 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.159 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671391+0000", 2023-09-23T19:03:09.159 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.73253294700000005 2023-09-23T19:03:09.159 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.159 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.159 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:03:09.159 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671399+0000", 2023-09-23T19:03:09.159 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.14e-06 2023-09-23T19:03:09.159 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.160 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:03:09.160 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.160 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.160 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.160 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:115077 10.c 10.7476880c (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:03:09.160 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.938879+0000", 2023-09-23T19:03:09.160 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 11.097068654999999, 2023-09-23T19:03:09.160 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.73255084199999998, 2023-09-23T19:03:09.160 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:03:09.160 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:03:09.161 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:03:09.161 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:03:09.161 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:03:09.161 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 115077 2023-09-23T19:03:09.161 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.161 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:03:09.161 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.161 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:03:09.161 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938879+0000", 2023-09-23T19:03:09.161 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.161 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.162 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.162 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:03:09.162 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938879+0000", 2023-09-23T19:03:09.162 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.162 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.162 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.162 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:03:09.162 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938878+0000", 2023-09-23T19:03:09.162 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T19:03:09.163 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.163 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.163 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:03:09.163 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938883+0000", 2023-09-23T19:03:09.163 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.0000000000000004e-06 2023-09-23T19:03:09.163 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.163 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.163 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:03:09.163 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938893+0000", 2023-09-23T19:03:09.163 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.7699999999999996e-06 2023-09-23T19:03:09.163 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.164 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.164 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:03:09.164 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938899+0000", 2023-09-23T19:03:09.164 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.9399999999999999e-06 2023-09-23T19:03:09.164 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.164 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.164 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.164 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938939+0000", 2023-09-23T19:03:09.164 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.9799999999999998e-05 2023-09-23T19:03:09.165 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.165 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.165 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:03:09.165 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938948+0000", 2023-09-23T19:03:09.165 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.2720000000000003e-06 2023-09-23T19:03:09.165 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.165 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.165 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.165 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671422+0000", 2023-09-23T19:03:09.165 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.732474128 2023-09-23T19:03:09.166 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.166 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.166 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:03:09.166 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671430+0000", 2023-09-23T19:03:09.166 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.7100000000000007e-06 2023-09-23T19:03:09.166 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.166 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:03:09.166 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.166 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.166 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.167 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:115098 10.c 10.c815cebc (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:03:09.167 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.996592+0000", 2023-09-23T19:03:09.167 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 11.039355622, 2023-09-23T19:03:09.167 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.67486956499999995, 2023-09-23T19:03:09.167 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:03:09.167 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:03:09.167 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:03:09.167 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:03:09.167 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:03:09.168 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 115098 2023-09-23T19:03:09.168 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.168 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:03:09.168 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.168 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:03:09.168 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.996592+0000", 2023-09-23T19:03:09.168 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.168 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.168 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.168 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:03:09.169 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.996592+0000", 2023-09-23T19:03:09.169 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.169 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.169 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.169 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:03:09.169 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.996591+0000", 2023-09-23T19:03:09.169 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T19:03:09.169 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.169 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.169 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:03:09.170 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.996599+0000", 2023-09-23T19:03:09.170 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.2970000000000004e-06 2023-09-23T19:03:09.170 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.170 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.170 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:03:09.170 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.996612+0000", 2023-09-23T19:03:09.170 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3096e-05 2023-09-23T19:03:09.170 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.170 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.171 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:03:09.171 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.996621+0000", 2023-09-23T19:03:09.171 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.6039999999999993e-06 2023-09-23T19:03:09.171 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.171 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.171 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.171 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.996667+0000", 2023-09-23T19:03:09.171 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.6724000000000002e-05 2023-09-23T19:03:09.171 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.172 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.172 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:03:09.172 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.996677+0000", 2023-09-23T19:03:09.172 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0192e-05 2023-09-23T19:03:09.172 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.172 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.172 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.172 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671454+0000", 2023-09-23T19:03:09.172 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.67477629900000002 2023-09-23T19:03:09.172 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.173 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.173 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:03:09.173 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671462+0000", 2023-09-23T19:03:09.173 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.8390000000000007e-06 2023-09-23T19:03:09.173 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.173 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:03:09.173 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.173 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.173 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.174 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:115102 10.c 10.fc7abffc (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:03:09.174 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:58.016377+0000", 2023-09-23T19:03:09.174 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 11.019570935999999, 2023-09-23T19:03:09.174 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.65511980000000003, 2023-09-23T19:03:09.174 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:03:09.174 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:03:09.174 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:03:09.174 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:03:09.174 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:03:09.174 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 115102 2023-09-23T19:03:09.175 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.175 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:03:09.175 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.175 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:03:09.175 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.016377+0000", 2023-09-23T19:03:09.175 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.175 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.175 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.175 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:03:09.176 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.016377+0000", 2023-09-23T19:03:09.176 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.176 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.176 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.176 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:03:09.176 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.016375+0000", 2023-09-23T19:03:09.176 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:03:09.176 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.176 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.177 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:03:09.177 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.016384+0000", 2023-09-23T19:03:09.177 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.0839999999999998e-06 2023-09-23T19:03:09.177 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.177 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.177 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:03:09.177 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.016401+0000", 2023-09-23T19:03:09.177 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.6518e-05 2023-09-23T19:03:09.177 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.177 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.177 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:03:09.178 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.016409+0000", 2023-09-23T19:03:09.178 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.7830000000000004e-06 2023-09-23T19:03:09.178 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.178 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.178 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.178 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.016454+0000", 2023-09-23T19:03:09.178 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.4481000000000003e-05 2023-09-23T19:03:09.178 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.178 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.179 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:03:09.179 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.016462+0000", 2023-09-23T19:03:09.179 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.7849999999999992e-06 2023-09-23T19:03:09.179 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.179 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.179 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.179 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671488+0000", 2023-09-23T19:03:09.179 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.65502604499999995 2023-09-23T19:03:09.179 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.179 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.179 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:03:09.180 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671497+0000", 2023-09-23T19:03:09.180 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.9309999999999997e-06 2023-09-23T19:03:09.180 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.180 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:03:09.180 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.180 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.180 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.180 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:120622 10.f 10:ffc199c1:::benchmark_data_smithi098_230241_object20160:head [delete] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-09-23T19:03:09.180 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:03:07.741444+0000", 2023-09-23T19:03:09.181 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 1.294503647, 2023-09-23T19:03:09.181 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.16012531499999999, 2023-09-23T19:03:09.181 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:03:09.181 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:03:09.181 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:03:09.181 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:03:09.181 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:03:09.181 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 120622 2023-09-23T19:03:09.181 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.181 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:03:09.181 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.182 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:03:09.182 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:07.741444+0000", 2023-09-23T19:03:09.182 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.182 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.182 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.182 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:03:09.182 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:07.741444+0000", 2023-09-23T19:03:09.182 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:03:09.182 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.182 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.183 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:03:09.183 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:07.741443+0000", 2023-09-23T19:03:09.183 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T19:03:09.183 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.183 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.183 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:03:09.183 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:07.741450+0000", 2023-09-23T19:03:09.183 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.4479999999999997e-06 2023-09-23T19:03:09.183 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.183 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.184 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:03:09.184 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:07.741464+0000", 2023-09-23T19:03:09.184 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3478e-05 2023-09-23T19:03:09.184 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.184 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.184 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:03:09.184 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:07.741471+0000", 2023-09-23T19:03:09.184 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.5830000000000001e-06 2023-09-23T19:03:09.184 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.184 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.185 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:03:09.185 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:07.741506+0000", 2023-09-23T19:03:09.185 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.5021999999999998e-05 2023-09-23T19:03:09.185 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.185 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.185 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:03:09.185 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:07.741621+0000", 2023-09-23T19:03:09.185 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00011506399999999999 2023-09-23T19:03:09.185 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.185 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.185 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 2,6", 2023-09-23T19:03:09.186 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:07.741729+0000", 2023-09-23T19:03:09.186 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00010778800000000001 2023-09-23T19:03:09.186 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.186 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.186 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:03:09.186 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:07.748518+0000", 2023-09-23T19:03:09.186 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0067888030000000004 2023-09-23T19:03:09.186 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.186 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.187 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:03:09.187 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:07.748545+0000", 2023-09-23T19:03:09.187 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.7653000000000001e-05 2023-09-23T19:03:09.187 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.187 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.187 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:03:09.187 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:07.901471+0000", 2023-09-23T19:03:09.187 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.152925955 2023-09-23T19:03:09.187 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.187 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.187 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:03:09.188 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:07.901526+0000", 2023-09-23T19:03:09.188 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.4602999999999999e-05 2023-09-23T19:03:09.188 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:03:09.188 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:03:09.188 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:03:09.188 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:07.901569+0000", 2023-09-23T19:03:09.188 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.3322999999999998e-05 2023-09-23T19:03:09.188 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.188 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:03:09.188 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.189 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:03:09.189 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:03:09.189 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T19:03:09.189 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:09.216 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:09.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:09.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:09.320 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:09.352 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:09.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:09.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:09.455 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:09.486 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:09.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:09.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:09.589 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:09.625 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:09.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:09.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:09.729 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:09.755 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:09.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:09.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:09.858 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:09.884 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:09.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:09.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:09.987 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:10.020 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:10.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:10.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:10.124 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:10.154 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:10.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:10.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:10.258 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:10.288 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:10.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:10.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:10.392 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:10.410 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:10.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:10.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:10.461 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:10 smithi098 bash[79516]: cluster 2023-09-23T19:03:08.487488+0000 mgr.y (mgr.26054) 1357 : cluster 0 pgmap v1491: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 5.0 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 45716/243882 objects degraded (18.745%); 5245/243882 objects misplaced (2.151%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:03:10.461 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:10 smithi098 bash[79516]: audit 2023-09-23T19:03:09.639985+0000 mon.a (mon.0) 1861 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:03:10.461 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:10 smithi098 bash[76817]: cluster 2023-09-23T19:03:08.487488+0000 mgr.y (mgr.26054) 1357 : cluster 0 pgmap v1491: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 5.0 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 45716/243882 objects degraded (18.745%); 5245/243882 objects misplaced (2.151%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:03:10.462 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:10 smithi098 bash[76817]: audit 2023-09-23T19:03:09.639985+0000 mon.a (mon.0) 1861 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:03:10.513 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:10.523 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:10 smithi173 bash[59422]: cluster 2023-09-23T19:03:08.487488+0000 mgr.y (mgr.26054) 1357 : cluster 0 pgmap v1491: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 5.0 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 45716/243882 objects degraded (18.745%); 5245/243882 objects misplaced (2.151%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:03:10.523 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:10 smithi173 bash[59422]: audit 2023-09-23T19:03:09.639985+0000 mon.a (mon.0) 1861 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:03:10.540 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:10.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:10.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:10.644 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:10.661 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:10.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:10.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:10.765 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:10.793 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:10.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:10.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:10.899 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:10.929 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:10.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:10.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:11.033 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:11.059 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:11.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:11.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:11.164 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:11.197 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:11.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:11.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:11.305 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:11.313 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:11 smithi098 bash[76817]: cluster 2023-09-23T19:03:10.477229+0000 mon.a (mon.0) 1862 : cluster 3 Health check update: Degraded data redundancy: 45716/243882 objects degraded (18.745%), 16 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:11.313 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:11 smithi098 bash[79516]: cluster 2023-09-23T19:03:10.477229+0000 mon.a (mon.0) 1862 : cluster 3 Health check update: Degraded data redundancy: 45716/243882 objects degraded (18.745%), 16 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:11.341 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:11.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:11.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:11.449 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:11.480 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:11.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:11.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:11.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:11 smithi173 bash[59422]: cluster 2023-09-23T19:03:10.477229+0000 mon.a (mon.0) 1862 : cluster 3 Health check update: Degraded data redundancy: 45716/243882 objects degraded (18.745%), 16 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:11.584 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:11.617 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:11.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:11.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:11.728 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:11.756 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:11.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:11.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:11.861 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:11.895 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:11.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:11.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:11.999 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:12.038 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:12.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:12.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:12.144 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:12.175 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:12.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:12.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:12.279 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:12.286 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:12 smithi098 bash[79516]: cluster 2023-09-23T19:03:10.488230+0000 mgr.y (mgr.26054) 1358 : cluster 0 pgmap v1492: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 44662/240501 objects degraded (18.570%); 5034/240501 objects misplaced (2.093%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T19:03:12.287 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:12 smithi098 bash[76817]: cluster 2023-09-23T19:03:10.488230+0000 mgr.y (mgr.26054) 1358 : cluster 0 pgmap v1492: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 44662/240501 objects degraded (18.570%); 5034/240501 objects misplaced (2.093%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T19:03:12.304 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:12.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:12.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:12.413 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:12.444 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:12.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:12.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:12.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:12 smithi173 bash[59422]: cluster 2023-09-23T19:03:10.488230+0000 mgr.y (mgr.26054) 1358 : cluster 0 pgmap v1492: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 44662/240501 objects degraded (18.570%); 5034/240501 objects misplaced (2.093%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T19:03:12.547 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:12.576 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:12.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:12.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:12.683 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:12.727 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:12.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:12.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:12.832 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:12.864 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:12.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:12.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:12.967 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:12.985 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:12.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:12.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:13.089 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:13.127 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:13.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:13.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:13.233 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:13.260 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:13.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:13.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:13.363 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:13.389 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:13.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:13.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:13.403 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T19:03:13.498 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:13.527 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:13.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:13.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:13.635 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:13.673 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:13.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:13.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:13.776 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:13.803 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:13.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:13.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:13.906 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:13.932 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:13.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:13.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:14.037 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:14.068 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:14.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:14.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:14.172 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:14.200 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:14.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:14.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:14.305 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:14.332 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:14.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:14.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:14.435 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:14.455 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:14.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:14.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:14.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:14 smithi173 bash[59422]: cluster 2023-09-23T19:03:12.488956+0000 mgr.y (mgr.26054) 1359 : cluster 0 pgmap v1493: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 610 op/s; 43867/236313 objects degraded (18.563%); 5034/236313 objects misplaced (2.130%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T19:03:14.557 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:14.564 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:14 smithi098 bash[79516]: cluster 2023-09-23T19:03:12.488956+0000 mgr.y (mgr.26054) 1359 : cluster 0 pgmap v1493: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 610 op/s; 43867/236313 objects degraded (18.563%); 5034/236313 objects misplaced (2.130%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T19:03:14.565 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:14 smithi098 bash[76817]: cluster 2023-09-23T19:03:12.488956+0000 mgr.y (mgr.26054) 1359 : cluster 0 pgmap v1493: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 610 op/s; 43867/236313 objects degraded (18.563%); 5034/236313 objects misplaced (2.130%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T19:03:14.577 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:14.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:14.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:14.680 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:14.724 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:14.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:14.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:14.827 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:14.850 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:14.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:14.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:14.957 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:14.989 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:14.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:14.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:15.092 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:15.127 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:15.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:15.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:15.234 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:15.285 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:15.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:15.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:15.387 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:15.414 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:15.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:15.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:15.519 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:15.558 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:15.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:15.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:15.661 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:15.688 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:15.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:15.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:15.792 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:15.816 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:15.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:15.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:15.919 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:15.954 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:15.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:15.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:16.062 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:16.091 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:16.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:16.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:16.193 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:03:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:03:16] "GET /metrics HTTP/1.1" 200 36664 "" "Prometheus/2.33.4" 2023-09-23T19:03:16.195 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:16.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:16.234 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:16.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:16.335 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:16.367 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:16.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:16.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:16.466 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:16 smithi098 bash[76817]: cluster 2023-09-23T19:03:14.490214+0000 mgr.y (mgr.26054) 1360 : cluster 0 pgmap v1494: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 42574/230796 objects degraded (18.447%); 4843/230796 objects misplaced (2.098%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T19:03:16.466 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:16 smithi098 bash[76817]: cluster 2023-09-23T19:03:15.478435+0000 mon.a (mon.0) 1863 : cluster 3 Health check update: Degraded data redundancy: 42574/230796 objects degraded (18.447%), 16 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:16.472 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:16.513 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:16.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:16.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:16.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:16 smithi173 bash[59422]: cluster 2023-09-23T19:03:14.490214+0000 mgr.y (mgr.26054) 1360 : cluster 0 pgmap v1494: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 42574/230796 objects degraded (18.447%); 4843/230796 objects misplaced (2.098%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T19:03:16.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:16 smithi173 bash[59422]: cluster 2023-09-23T19:03:15.478435+0000 mon.a (mon.0) 1863 : cluster 3 Health check update: Degraded data redundancy: 42574/230796 objects degraded (18.447%), 16 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:16.620 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:16.649 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:16.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:16.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:16.756 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:16.766 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:16 smithi098 bash[79516]: cluster 2023-09-23T19:03:14.490214+0000 mgr.y (mgr.26054) 1360 : cluster 0 pgmap v1494: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 42574/230796 objects degraded (18.447%); 4843/230796 objects misplaced (2.098%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T19:03:16.766 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:16 smithi098 bash[79516]: cluster 2023-09-23T19:03:15.478435+0000 mon.a (mon.0) 1863 : cluster 3 Health check update: Degraded data redundancy: 42574/230796 objects degraded (18.447%), 16 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:16.803 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:16.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:16.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:16.908 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:16.938 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:16.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:16.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:17.044 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:17.073 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:17.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:17.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:17.177 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:17.200 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:17.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:17.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:17.303 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:17.332 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:17.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:17.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:17.437 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:17.460 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:17.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:17.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:17.565 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:17.594 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:17.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:17.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:17.698 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:17.723 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:17.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:17.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:17.830 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:17.869 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:17.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:17.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:17.972 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:18.004 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:18.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:18.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:18.109 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:18.134 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:18.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:18.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:18.170 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:03:18.241 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:18.284 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:18.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:18.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:18.392 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:18.403 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:18 smithi098 bash[76817]: cluster 2023-09-23T19:03:16.490952+0000 mgr.y (mgr.26054) 1361 : cluster 0 pgmap v1495: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 42574/230796 objects degraded (18.447%); 4843/230796 objects misplaced (2.098%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T19:03:18.404 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:18 smithi098 bash[79516]: cluster 2023-09-23T19:03:16.490952+0000 mgr.y (mgr.26054) 1361 : cluster 0 pgmap v1495: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 42574/230796 objects degraded (18.447%); 4843/230796 objects misplaced (2.098%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T19:03:18.422 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:18.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:18.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:18.528 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:18.554 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:18 smithi173 bash[59422]: cluster 2023-09-23T19:03:16.490952+0000 mgr.y (mgr.26054) 1361 : cluster 0 pgmap v1495: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 42574/230796 objects degraded (18.447%); 4843/230796 objects misplaced (2.098%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T19:03:18.570 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:18.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:18.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:18.677 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:18.703 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:18.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:18.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:18.808 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:18.833 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:18.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:18.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:18.936 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:18.963 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:18.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:18.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:19.066 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:19.087 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:19.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:19.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:19.190 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:19.223 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:19.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:19.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:19.328 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:19.357 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:19.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:19.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:19.464 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:19.487 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:19.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:19.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:19.590 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:19.615 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:19.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:19.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:19.718 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:19.754 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:19.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:19.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:19.859 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:19.885 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:19.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:19.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:19.988 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:20.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:20.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:20.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:20.118 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:20.143 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:20.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:20.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:20.248 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:20.276 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:20.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:20.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:20.382 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:20.400 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:20.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:20.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:20.502 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:20.510 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:20 smithi098 bash[76817]: cluster 2023-09-23T19:03:18.492052+0000 mgr.y (mgr.26054) 1362 : cluster 0 pgmap v1496: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 778 op/s; 41165/223224 objects degraded (18.441%); 4843/223224 objects misplaced (2.170%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T19:03:20.511 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:20 smithi098 bash[79516]: cluster 2023-09-23T19:03:18.492052+0000 mgr.y (mgr.26054) 1362 : cluster 0 pgmap v1496: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 778 op/s; 41165/223224 objects degraded (18.441%); 4843/223224 objects misplaced (2.170%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T19:03:20.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:20.538 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:20.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:20.585 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:20 smithi173 bash[59422]: cluster 2023-09-23T19:03:18.492052+0000 mgr.y (mgr.26054) 1362 : cluster 0 pgmap v1496: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 778 op/s; 41165/223224 objects degraded (18.441%); 4843/223224 objects misplaced (2.170%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T19:03:20.639 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:20.672 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:20.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:20.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:20.775 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:20.805 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:20.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:20.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:20.912 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:20.937 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:20.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:20.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:21.040 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:21.061 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:21.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:21.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:21.166 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:21.196 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:21.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:21.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:21.302 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:21.324 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:21.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:21.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:21.425 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:21 smithi098 bash[76817]: cluster 2023-09-23T19:03:20.480048+0000 mon.a (mon.0) 1864 : cluster 3 Health check update: Degraded data redundancy: 41165/223224 objects degraded (18.441%), 16 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:21.427 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:21.456 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:21.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:21.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:21.563 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:21.598 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:21.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:21.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:21.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:21 smithi098 bash[79516]: cluster 2023-09-23T19:03:20.480048+0000 mon.a (mon.0) 1864 : cluster 3 Health check update: Degraded data redundancy: 41165/223224 objects degraded (18.441%), 16 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:21.702 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:21.726 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:21.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:21.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:21.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:21 smithi173 bash[59422]: cluster 2023-09-23T19:03:20.480048+0000 mon.a (mon.0) 1864 : cluster 3 Health check update: Degraded data redundancy: 41165/223224 objects degraded (18.441%), 16 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:21.833 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:21.856 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:21.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:21.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:21.960 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:21.986 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:21.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:21.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:22.089 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:22.116 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:22.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:22.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:22.219 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:22.257 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:22.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:22.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:22.281 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T19:03:22.281 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-09-23T19:03:22.281 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:03:22.281 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45779.0:130634 10.a e328/323 10:517705de:::benchmark_data_smithi098_230241_object30172:head v 328'8068, mlcod=328'8065)", 2023-09-23T19:03:22.281 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:03:22.278114+0000", 2023-09-23T19:03:22.281 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.0012981749999999999, 2023-09-23T19:03:22.281 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0013218170000000001, 2023-09-23T19:03:22.281 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:03:22.281 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "started", 2023-09-23T19:03:22.282 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:03:22.282 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:03:22.282 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:03:22.282 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:22.278114+0000", 2023-09-23T19:03:22.282 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:03:22.282 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:03:22.282 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:03:22.282 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:03:22.282 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:22.278114+0000", 2023-09-23T19:03:22.282 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T19:03:22.283 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:03:22.283 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:03:22.283 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:03:22.283 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:22.278113+0000", 2023-09-23T19:03:22.283 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5922000000000001e-05 2023-09-23T19:03:22.283 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:03:22.283 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:03:22.283 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:03:22.283 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:22.278129+0000", 2023-09-23T19:03:22.283 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2603e-05 2023-09-23T19:03:22.283 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:03:22.284 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:03:22.284 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:03:22.284 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:22.278151+0000", 2023-09-23T19:03:22.284 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3782e-05 2023-09-23T19:03:22.284 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:03:22.284 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:03:22.284 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:03:22.284 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:22.278165+0000", 2023-09-23T19:03:22.284 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.6871000000000003e-05 2023-09-23T19:03:22.284 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:03:22.284 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:03:22.284 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:03:22.284 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:22.278242+0000", 2023-09-23T19:03:22.285 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.4652e-05 2023-09-23T19:03:22.285 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:03:22.285 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:03:22.285 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:03:22.285 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:22.278277+0000", 2023-09-23T19:03:22.285 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000162168 2023-09-23T19:03:22.285 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:03:22.285 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:03:22.285 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:03:22.285 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:03:22.285 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:03:22.285 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45779.0:130635 10.f e328/317 10:fa8aa2d2:::benchmark_data_smithi098_230241_object30173:head v 328'8246, mlcod=328'8243)", 2023-09-23T19:03:22.285 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:03:22.278448+0000", 2023-09-23T19:03:22.286 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.00096495900000000004, 2023-09-23T19:03:22.286 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0010412729999999999, 2023-09-23T19:03:22.286 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:03:22.286 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "started", 2023-09-23T19:03:22.286 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:03:22.286 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:03:22.286 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:03:22.286 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:22.278448+0000", 2023-09-23T19:03:22.286 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:03:22.286 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:03:22.286 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:03:22.287 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:03:22.287 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:22.278448+0000", 2023-09-23T19:03:22.287 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T19:03:22.287 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:03:22.287 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:03:22.287 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:03:22.287 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:22.278446+0000", 2023-09-23T19:03:22.287 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3619e-05 2023-09-23T19:03:22.287 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:03:22.287 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:03:22.287 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:03:22.287 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:22.278460+0000", 2023-09-23T19:03:22.287 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4545e-05 2023-09-23T19:03:22.288 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:03:22.288 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:03:22.288 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:03:22.288 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:22.278484+0000", 2023-09-23T19:03:22.288 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2068e-05 2023-09-23T19:03:22.288 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:03:22.288 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:03:22.288 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:03:22.288 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:22.278496+0000", 2023-09-23T19:03:22.288 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.4385000000000001e-05 2023-09-23T19:03:22.288 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:03:22.288 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:03:22.288 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:03:22.289 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:22.278561+0000", 2023-09-23T19:03:22.289 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.5361999999999998e-05 2023-09-23T19:03:22.289 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:03:22.289 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:03:22.289 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:03:22.289 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:22.278596+0000", 2023-09-23T19:03:22.289 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000148424 2023-09-23T19:03:22.289 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:03:22.289 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:03:22.289 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:03:22.289 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:03:22.289 INFO:teuthology.orchestra.run.smithi173.stdout: ], 2023-09-23T19:03:22.289 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 2 2023-09-23T19:03:22.290 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T19:03:22.360 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:22.384 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:22.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:22.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:22.404 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:22 smithi098 bash[76817]: cluster 2023-09-23T19:03:20.492781+0000 mgr.y (mgr.26054) 1363 : cluster 0 pgmap v1497: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 40195/219843 objects degraded (18.284%); 4630/219843 objects misplaced (2.106%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T19:03:22.404 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:22 smithi098 bash[79516]: cluster 2023-09-23T19:03:20.492781+0000 mgr.y (mgr.26054) 1363 : cluster 0 pgmap v1497: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 40195/219843 objects degraded (18.284%); 4630/219843 objects misplaced (2.106%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T19:03:22.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:22.513 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:22.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:22.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:22.616 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:22.642 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:22.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:22.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:22.746 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:22.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:22 smithi173 bash[59422]: cluster 2023-09-23T19:03:20.492781+0000 mgr.y (mgr.26054) 1363 : cluster 0 pgmap v1497: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 40195/219843 objects degraded (18.284%); 4630/219843 objects misplaced (2.106%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T19:03:22.782 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:22.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:22.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:22.886 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:22.925 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:22.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:22.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:23.030 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:23.066 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:23.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:23.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:23.124 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T19:03:23.173 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:23.175 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T19:03:23.203 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:23.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:23.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:23.237 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T19:03:23.273 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T19:03:23.308 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T19:03:23.309 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:23.330 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T19:03:23.337 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is set 2023-09-23T19:03:23.343 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:23.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:23.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:23.370 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T19:03:23.375 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:03:23.375 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:03:23.375 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:03:23.375 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:03:23.375 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:03:23.375 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T19:03:23.375 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:03:23.375 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T19:03:23.451 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:23.493 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:23.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:23.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:23.603 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:23.613 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:23 smithi173 bash[59422]: audit 2023-09-23T19:03:23.067117+0000 mon.a (mon.0) 1865 : audit 0 from='client.? 172.21.15.98:0/334919757' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:03:23.614 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:23 smithi173 bash[59422]: audit 2023-09-23T19:03:23.104099+0000 mon.a (mon.0) 1866 : audit 1 from='client.? 172.21.15.98:0/2185911966' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:03:23.632 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:23.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:23.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:23.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:23 smithi098 bash[76817]: audit 2023-09-23T19:03:23.067117+0000 mon.a (mon.0) 1865 : audit 0 from='client.? 172.21.15.98:0/334919757' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:03:23.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:23 smithi098 bash[76817]: audit 2023-09-23T19:03:23.104099+0000 mon.a (mon.0) 1866 : audit 1 from='client.? 172.21.15.98:0/2185911966' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:03:23.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:23 smithi098 bash[79516]: audit 2023-09-23T19:03:23.067117+0000 mon.a (mon.0) 1865 : audit 0 from='client.? 172.21.15.98:0/334919757' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:03:23.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:23 smithi098 bash[79516]: audit 2023-09-23T19:03:23.104099+0000 mon.a (mon.0) 1866 : audit 1 from='client.? 172.21.15.98:0/2185911966' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:03:23.735 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:23.761 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:23.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:23.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:23.865 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:23.896 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:23.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:23.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:24.000 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:24.025 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:24.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:24.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:24.130 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:24.159 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:24.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:24.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:24.262 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:24.291 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:24.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:24.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:24.334 INFO:teuthology.orchestra.run.smithi098.stdout:moved 14 / 267 (5.24345%) 2023-09-23T19:03:24.335 INFO:teuthology.orchestra.run.smithi098.stdout:avg 53.4 2023-09-23T19:03:24.335 INFO:teuthology.orchestra.run.smithi098.stdout:stddev 12.5857 -> 13.8881 (expected baseline 6.53605) 2023-09-23T19:03:24.335 INFO:teuthology.orchestra.run.smithi098.stdout:min osd.2 with 33 -> 31 pgs (0.617977 -> 0.580524 * mean) 2023-09-23T19:03:24.335 INFO:teuthology.orchestra.run.smithi098.stdout:max osd.1 with 49 -> 47 pgs (0.917603 -> 0.88015 * mean) 2023-09-23T19:03:24.335 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T19:03:24.335 INFO:teuthology.orchestra.run.smithi098.stdout:oload 1000 2023-09-23T19:03:24.335 INFO:teuthology.orchestra.run.smithi098.stdout:max_change 1 2023-09-23T19:03:24.335 INFO:teuthology.orchestra.run.smithi098.stdout:max_change_osds 4 2023-09-23T19:03:24.335 INFO:teuthology.orchestra.run.smithi098.stdout:average_utilization 0.0288 2023-09-23T19:03:24.335 INFO:teuthology.orchestra.run.smithi098.stdout:overload_utilization 0.2884 2023-09-23T19:03:24.336 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7 weight 0.9039 -> 1.0000 2023-09-23T19:03:24.336 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3 weight 0.7315 -> 0.8669 2023-09-23T19:03:24.336 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5 weight 0.6936 -> 0.9098 2023-09-23T19:03:24.380 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:24 smithi098 bash[79516]: cluster 2023-09-23T19:03:22.494865+0000 mgr.y (mgr.26054) 1364 : cluster 0 pgmap v1498: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 39326/215352 objects degraded (18.261%); 4630/215352 objects misplaced (2.150%); 3.1 MiB/s, 48 objects/s recovering 2023-09-23T19:03:24.381 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:24 smithi098 bash[79516]: cluster 2023-09-23T19:03:23.287173+0000 mon.a (mon.0) 1867 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:03:24.381 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:24 smithi098 bash[79516]: audit 2023-09-23T19:03:23.319708+0000 mon.a (mon.0) 1868 : audit 1 from='client.? 172.21.15.98:0/2185911966' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:03:24.381 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:24 smithi098 bash[79516]: cluster 2023-09-23T19:03:23.319928+0000 mon.a (mon.0) 1869 : cluster 0 osdmap e329: 8 total, 6 up, 7 in 2023-09-23T19:03:24.381 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:24 smithi098 bash[79516]: cluster 2023-09-23T19:03:23.325847+0000 osd.6 (osd.6) 305 : cluster 3 failed to encode map e329 with expected crc 2023-09-23T19:03:24.381 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:24 smithi098 bash[79516]: cluster 2023-09-23T19:03:23.328929+0000 osd.6 (osd.6) 306 : cluster 3 failed to encode map e329 with expected crc 2023-09-23T19:03:24.382 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:24 smithi098 bash[79516]: audit 2023-09-23T19:03:23.606866+0000 mon.a (mon.0) 1870 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'56814','5':'59624','7':'65536'}"}]: dispatch 2023-09-23T19:03:24.382 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:24 smithi098 bash[76817]: cluster 2023-09-23T19:03:22.494865+0000 mgr.y (mgr.26054) 1364 : cluster 0 pgmap v1498: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 39326/215352 objects degraded (18.261%); 4630/215352 objects misplaced (2.150%); 3.1 MiB/s, 48 objects/s recovering 2023-09-23T19:03:24.382 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:24 smithi098 bash[76817]: cluster 2023-09-23T19:03:23.287173+0000 mon.a (mon.0) 1867 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:03:24.382 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:24 smithi098 bash[76817]: audit 2023-09-23T19:03:23.319708+0000 mon.a (mon.0) 1868 : audit 1 from='client.? 172.21.15.98:0/2185911966' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:03:24.382 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:24 smithi098 bash[76817]: cluster 2023-09-23T19:03:23.319928+0000 mon.a (mon.0) 1869 : cluster 0 osdmap e329: 8 total, 6 up, 7 in 2023-09-23T19:03:24.383 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:24 smithi098 bash[76817]: cluster 2023-09-23T19:03:23.325847+0000 osd.6 (osd.6) 305 : cluster 3 failed to encode map e329 with expected crc 2023-09-23T19:03:24.383 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:24 smithi098 bash[76817]: cluster 2023-09-23T19:03:23.328929+0000 osd.6 (osd.6) 306 : cluster 3 failed to encode map e329 with expected crc 2023-09-23T19:03:24.383 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:24 smithi098 bash[76817]: audit 2023-09-23T19:03:23.606866+0000 mon.a (mon.0) 1870 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'56814','5':'59624','7':'65536'}"}]: dispatch 2023-09-23T19:03:24.435 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:24.459 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:24.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:24.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:24.561 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:24.584 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:24.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:24.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:24.687 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:24.707 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:24.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:24.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:24.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:24 smithi173 bash[59422]: cluster 2023-09-23T19:03:22.494865+0000 mgr.y (mgr.26054) 1364 : cluster 0 pgmap v1498: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 39326/215352 objects degraded (18.261%); 4630/215352 objects misplaced (2.150%); 3.1 MiB/s, 48 objects/s recovering 2023-09-23T19:03:24.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:24 smithi173 bash[59422]: cluster 2023-09-23T19:03:23.287173+0000 mon.a (mon.0) 1867 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:03:24.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:24 smithi173 bash[59422]: audit 2023-09-23T19:03:23.319708+0000 mon.a (mon.0) 1868 : audit 1 from='client.? 172.21.15.98:0/2185911966' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:03:24.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:24 smithi173 bash[59422]: cluster 2023-09-23T19:03:23.319928+0000 mon.a (mon.0) 1869 : cluster 0 osdmap e329: 8 total, 6 up, 7 in 2023-09-23T19:03:24.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:24 smithi173 bash[59422]: cluster 2023-09-23T19:03:23.325847+0000 osd.6 (osd.6) 305 : cluster 3 failed to encode map e329 with expected crc 2023-09-23T19:03:24.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:24 smithi173 bash[59422]: cluster 2023-09-23T19:03:23.328929+0000 osd.6 (osd.6) 306 : cluster 3 failed to encode map e329 with expected crc 2023-09-23T19:03:24.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:24 smithi173 bash[59422]: audit 2023-09-23T19:03:23.606866+0000 mon.a (mon.0) 1870 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'56814','5':'59624','7':'65536'}"}]: dispatch 2023-09-23T19:03:24.811 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:24.834 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:24.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:24.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:24.937 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:24.969 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:24.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:24.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:25.072 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:25.095 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:25.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:25.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:25.198 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:25.221 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:25.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:25.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:25.323 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:25.346 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:25.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:25.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:25.449 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:25.479 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:25.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:25.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:25.583 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:25.590 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:25 smithi098 bash[76817]: audit 2023-09-23T19:03:23.605844+0000 mgr.y (mgr.26054) 1365 : audit 0 from='client.45845 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:03:25.591 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:25 smithi098 bash[76817]: cluster 2023-09-23T19:03:24.309990+0000 osd.4 (osd.4) 59 : cluster 3 failed to encode map e329 with expected crc 2023-09-23T19:03:25.591 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:25 smithi098 bash[76817]: audit 2023-09-23T19:03:24.316109+0000 mon.a (mon.0) 1871 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'3':'56814','5':'59624','7':'65536'}"}]': finished 2023-09-23T19:03:25.591 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:25 smithi098 bash[76817]: cluster 2023-09-23T19:03:24.316179+0000 mon.a (mon.0) 1872 : cluster 0 osdmap e330: 8 total, 6 up, 7 in 2023-09-23T19:03:25.591 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:25 smithi098 bash[76817]: cluster 2023-09-23T19:03:24.325218+0000 osd.6 (osd.6) 307 : cluster 3 failed to encode map e330 with expected crc 2023-09-23T19:03:25.591 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:25 smithi098 bash[76817]: cluster 2023-09-23T19:03:24.327327+0000 osd.6 (osd.6) 308 : cluster 3 failed to encode map e330 with expected crc 2023-09-23T19:03:25.591 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:25 smithi098 bash[76817]: cluster 2023-09-23T19:03:24.331942+0000 osd.6 (osd.6) 309 : cluster 3 failed to encode map e330 with expected crc 2023-09-23T19:03:25.591 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:25 smithi098 bash[76817]: audit 2023-09-23T19:03:24.645022+0000 mon.a (mon.0) 1873 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:03:25.591 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:25 smithi098 bash[76817]: audit 2023-09-23T19:03:24.646165+0000 mon.a (mon.0) 1874 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:03:25.591 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:25 smithi098 bash[76817]: cluster 2023-09-23T19:03:25.315752+0000 mon.a (mon.0) 1875 : cluster 0 osdmap e331: 8 total, 6 up, 7 in 2023-09-23T19:03:25.592 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:25 smithi098 bash[79516]: audit 2023-09-23T19:03:23.605844+0000 mgr.y (mgr.26054) 1365 : audit 0 from='client.45845 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:03:25.592 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:25 smithi098 bash[79516]: cluster 2023-09-23T19:03:24.309990+0000 osd.4 (osd.4) 59 : cluster 3 failed to encode map e329 with expected crc 2023-09-23T19:03:25.592 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:25 smithi098 bash[79516]: audit 2023-09-23T19:03:24.316109+0000 mon.a (mon.0) 1871 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'3':'56814','5':'59624','7':'65536'}"}]': finished 2023-09-23T19:03:25.592 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:25 smithi098 bash[79516]: cluster 2023-09-23T19:03:24.316179+0000 mon.a (mon.0) 1872 : cluster 0 osdmap e330: 8 total, 6 up, 7 in 2023-09-23T19:03:25.592 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:25 smithi098 bash[79516]: cluster 2023-09-23T19:03:24.325218+0000 osd.6 (osd.6) 307 : cluster 3 failed to encode map e330 with expected crc 2023-09-23T19:03:25.592 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:25 smithi098 bash[79516]: cluster 2023-09-23T19:03:24.327327+0000 osd.6 (osd.6) 308 : cluster 3 failed to encode map e330 with expected crc 2023-09-23T19:03:25.592 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:25 smithi098 bash[79516]: cluster 2023-09-23T19:03:24.331942+0000 osd.6 (osd.6) 309 : cluster 3 failed to encode map e330 with expected crc 2023-09-23T19:03:25.592 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:25 smithi098 bash[79516]: audit 2023-09-23T19:03:24.645022+0000 mon.a (mon.0) 1873 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:03:25.592 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:25 smithi098 bash[79516]: audit 2023-09-23T19:03:24.646165+0000 mon.a (mon.0) 1874 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:03:25.592 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:25 smithi098 bash[79516]: cluster 2023-09-23T19:03:25.315752+0000 mon.a (mon.0) 1875 : cluster 0 osdmap e331: 8 total, 6 up, 7 in 2023-09-23T19:03:25.604 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:25.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:25.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:25.707 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:25.731 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:25.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:25.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:25.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:25 smithi173 bash[59422]: audit 2023-09-23T19:03:23.605844+0000 mgr.y (mgr.26054) 1365 : audit 0 from='client.45845 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:03:25.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:25 smithi173 bash[59422]: cluster 2023-09-23T19:03:24.309990+0000 osd.4 (osd.4) 59 : cluster 3 failed to encode map e329 with expected crc 2023-09-23T19:03:25.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:25 smithi173 bash[59422]: audit 2023-09-23T19:03:24.316109+0000 mon.a (mon.0) 1871 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'3':'56814','5':'59624','7':'65536'}"}]': finished 2023-09-23T19:03:25.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:25 smithi173 bash[59422]: cluster 2023-09-23T19:03:24.316179+0000 mon.a (mon.0) 1872 : cluster 0 osdmap e330: 8 total, 6 up, 7 in 2023-09-23T19:03:25.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:25 smithi173 bash[59422]: cluster 2023-09-23T19:03:24.325218+0000 osd.6 (osd.6) 307 : cluster 3 failed to encode map e330 with expected crc 2023-09-23T19:03:25.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:25 smithi173 bash[59422]: cluster 2023-09-23T19:03:24.327327+0000 osd.6 (osd.6) 308 : cluster 3 failed to encode map e330 with expected crc 2023-09-23T19:03:25.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:25 smithi173 bash[59422]: cluster 2023-09-23T19:03:24.331942+0000 osd.6 (osd.6) 309 : cluster 3 failed to encode map e330 with expected crc 2023-09-23T19:03:25.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:25 smithi173 bash[59422]: audit 2023-09-23T19:03:24.645022+0000 mon.a (mon.0) 1873 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:03:25.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:25 smithi173 bash[59422]: audit 2023-09-23T19:03:24.646165+0000 mon.a (mon.0) 1874 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:03:25.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:25 smithi173 bash[59422]: cluster 2023-09-23T19:03:25.315752+0000 mon.a (mon.0) 1875 : cluster 0 osdmap e331: 8 total, 6 up, 7 in 2023-09-23T19:03:25.834 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:25.860 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:25.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:25.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:25.962 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:25.992 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:25.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:25.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:26.095 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:26.120 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:26.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:26.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:26.223 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:26.250 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:26.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:26.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:26.353 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:26.389 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:26.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:26.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:26.443 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:03:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:03:26] "GET /metrics HTTP/1.1" 200 36647 "" "Prometheus/2.33.4" 2023-09-23T19:03:26.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:26 smithi098 bash[79516]: cluster 2023-09-23T19:03:24.495496+0000 mgr.y (mgr.26054) 1366 : cluster 0 pgmap v1501: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 882 op/s; 38079/209601 objects degraded (18.167%); 4434/209601 objects misplaced (2.115%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T19:03:26.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:26 smithi098 bash[79516]: cluster 2023-09-23T19:03:25.312030+0000 osd.6 (osd.6) 310 : cluster 3 failed to encode map e331 with expected crc 2023-09-23T19:03:26.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:26 smithi098 bash[79516]: cluster 2023-09-23T19:03:25.312769+0000 osd.4 (osd.4) 60 : cluster 3 failed to encode map e331 with expected crc 2023-09-23T19:03:26.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:26 smithi098 bash[79516]: cluster 2023-09-23T19:03:25.315829+0000 osd.6 (osd.6) 311 : cluster 3 failed to encode map e331 with expected crc 2023-09-23T19:03:26.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:26 smithi098 bash[79516]: cluster 2023-09-23T19:03:25.316968+0000 osd.4 (osd.4) 61 : cluster 3 failed to encode map e331 with expected crc 2023-09-23T19:03:26.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:26 smithi098 bash[79516]: cluster 2023-09-23T19:03:25.481397+0000 mon.a (mon.0) 1876 : cluster 3 Health check update: Degraded data redundancy: 38079/209601 objects degraded (18.167%), 16 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:26.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:26 smithi098 bash[79516]: cluster 2023-09-23T19:03:26.317217+0000 mon.a (mon.0) 1877 : cluster 0 osdmap e332: 8 total, 6 up, 7 in 2023-09-23T19:03:26.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:26 smithi098 bash[76817]: cluster 2023-09-23T19:03:24.495496+0000 mgr.y (mgr.26054) 1366 : cluster 0 pgmap v1501: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 882 op/s; 38079/209601 objects degraded (18.167%); 4434/209601 objects misplaced (2.115%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T19:03:26.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:26 smithi098 bash[76817]: cluster 2023-09-23T19:03:25.312030+0000 osd.6 (osd.6) 310 : cluster 3 failed to encode map e331 with expected crc 2023-09-23T19:03:26.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:26 smithi098 bash[76817]: cluster 2023-09-23T19:03:25.312769+0000 osd.4 (osd.4) 60 : cluster 3 failed to encode map e331 with expected crc 2023-09-23T19:03:26.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:26 smithi098 bash[76817]: cluster 2023-09-23T19:03:25.315829+0000 osd.6 (osd.6) 311 : cluster 3 failed to encode map e331 with expected crc 2023-09-23T19:03:26.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:26 smithi098 bash[76817]: cluster 2023-09-23T19:03:25.316968+0000 osd.4 (osd.4) 61 : cluster 3 failed to encode map e331 with expected crc 2023-09-23T19:03:26.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:26 smithi098 bash[76817]: cluster 2023-09-23T19:03:25.481397+0000 mon.a (mon.0) 1876 : cluster 3 Health check update: Degraded data redundancy: 38079/209601 objects degraded (18.167%), 16 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:26.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:26 smithi098 bash[76817]: cluster 2023-09-23T19:03:26.317217+0000 mon.a (mon.0) 1877 : cluster 0 osdmap e332: 8 total, 6 up, 7 in 2023-09-23T19:03:26.492 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:26.531 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:26.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:26.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:26.637 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:26.657 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:26.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:26.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:26.764 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:26.773 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:26 smithi173 bash[59422]: cluster 2023-09-23T19:03:24.495496+0000 mgr.y (mgr.26054) 1366 : cluster 0 pgmap v1501: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 18 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 882 op/s; 38079/209601 objects degraded (18.167%); 4434/209601 objects misplaced (2.115%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T19:03:26.774 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:26 smithi173 bash[59422]: cluster 2023-09-23T19:03:25.312030+0000 osd.6 (osd.6) 310 : cluster 3 failed to encode map e331 with expected crc 2023-09-23T19:03:26.774 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:26 smithi173 bash[59422]: cluster 2023-09-23T19:03:25.312769+0000 osd.4 (osd.4) 60 : cluster 3 failed to encode map e331 with expected crc 2023-09-23T19:03:26.774 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:26 smithi173 bash[59422]: cluster 2023-09-23T19:03:25.315829+0000 osd.6 (osd.6) 311 : cluster 3 failed to encode map e331 with expected crc 2023-09-23T19:03:26.774 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:26 smithi173 bash[59422]: cluster 2023-09-23T19:03:25.316968+0000 osd.4 (osd.4) 61 : cluster 3 failed to encode map e331 with expected crc 2023-09-23T19:03:26.774 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:26 smithi173 bash[59422]: cluster 2023-09-23T19:03:25.481397+0000 mon.a (mon.0) 1876 : cluster 3 Health check update: Degraded data redundancy: 38079/209601 objects degraded (18.167%), 16 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:26.774 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:26 smithi173 bash[59422]: cluster 2023-09-23T19:03:26.317217+0000 mon.a (mon.0) 1877 : cluster 0 osdmap e332: 8 total, 6 up, 7 in 2023-09-23T19:03:26.788 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:26.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:26.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:26.893 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:26.920 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:26.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:26.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:27.024 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:27.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:27.065 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:27.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:27.166 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:27.200 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:27.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:27.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:27.305 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:27.329 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:27.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:27.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:27.438 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:27.469 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:27.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:27.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:27.575 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:27.583 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:27 smithi098 bash[79516]: cluster 2023-09-23T19:03:26.315136+0000 osd.6 (osd.6) 312 : cluster 3 failed to encode map e332 with expected crc 2023-09-23T19:03:27.583 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:27 smithi098 bash[79516]: cluster 2023-09-23T19:03:26.315362+0000 osd.4 (osd.4) 62 : cluster 3 failed to encode map e332 with expected crc 2023-09-23T19:03:27.583 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:27 smithi098 bash[79516]: cluster 2023-09-23T19:03:26.318725+0000 osd.6 (osd.6) 313 : cluster 3 failed to encode map e332 with expected crc 2023-09-23T19:03:27.584 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:27 smithi098 bash[79516]: cluster 2023-09-23T19:03:26.321921+0000 osd.6 (osd.6) 314 : cluster 3 failed to encode map e332 with expected crc 2023-09-23T19:03:27.584 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:27 smithi098 bash[79516]: cluster 2023-09-23T19:03:26.323341+0000 osd.1 (osd.1) 11 : cluster 0 10.2 starting backfill to osd.3 from (0'0,0'0] MAX to 329'8364 2023-09-23T19:03:27.584 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:27 smithi098 bash[76817]: cluster 2023-09-23T19:03:26.315136+0000 osd.6 (osd.6) 312 : cluster 3 failed to encode map e332 with expected crc 2023-09-23T19:03:27.584 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:27 smithi098 bash[76817]: cluster 2023-09-23T19:03:26.315362+0000 osd.4 (osd.4) 62 : cluster 3 failed to encode map e332 with expected crc 2023-09-23T19:03:27.584 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:27 smithi098 bash[76817]: cluster 2023-09-23T19:03:26.318725+0000 osd.6 (osd.6) 313 : cluster 3 failed to encode map e332 with expected crc 2023-09-23T19:03:27.584 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:27 smithi098 bash[76817]: cluster 2023-09-23T19:03:26.321921+0000 osd.6 (osd.6) 314 : cluster 3 failed to encode map e332 with expected crc 2023-09-23T19:03:27.584 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:27 smithi098 bash[76817]: cluster 2023-09-23T19:03:26.323341+0000 osd.1 (osd.1) 11 : cluster 0 10.2 starting backfill to osd.3 from (0'0,0'0] MAX to 329'8364 2023-09-23T19:03:27.602 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:27.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:27.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:27.708 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:27.753 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:27.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:27.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:27.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:27 smithi173 bash[59422]: cluster 2023-09-23T19:03:26.315136+0000 osd.6 (osd.6) 312 : cluster 3 failed to encode map e332 with expected crc 2023-09-23T19:03:27.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:27 smithi173 bash[59422]: cluster 2023-09-23T19:03:26.315362+0000 osd.4 (osd.4) 62 : cluster 3 failed to encode map e332 with expected crc 2023-09-23T19:03:27.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:27 smithi173 bash[59422]: cluster 2023-09-23T19:03:26.318725+0000 osd.6 (osd.6) 313 : cluster 3 failed to encode map e332 with expected crc 2023-09-23T19:03:27.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:27 smithi173 bash[59422]: cluster 2023-09-23T19:03:26.321921+0000 osd.6 (osd.6) 314 : cluster 3 failed to encode map e332 with expected crc 2023-09-23T19:03:27.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:27 smithi173 bash[59422]: cluster 2023-09-23T19:03:26.323341+0000 osd.1 (osd.1) 11 : cluster 0 10.2 starting backfill to osd.3 from (0'0,0'0] MAX to 329'8364 2023-09-23T19:03:27.857 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:27.886 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:27.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:27.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:27.940 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:03:27.941 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:03:27.985 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T19:03:27.985 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 0.05 2023-09-23T19:03:27.992 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:28.066 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:28.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:28.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:28.098 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:03:28.175 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:28.210 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:28.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:28.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:28.321 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:28.372 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:28.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:28.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:28.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:28 smithi098 bash[76817]: cluster 2023-09-23T19:03:26.496138+0000 mgr.y (mgr.26054) 1367 : cluster 0 pgmap v1504: 89 pgs: 2 remapped+peering, 1 active+undersized+degraded+remapped+backfilling, 16 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 38079/209601 objects degraded (18.167%); 4433/209601 objects misplaced (2.115%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T19:03:28.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:28 smithi098 bash[76817]: cluster 2023-09-23T19:03:27.335659+0000 mon.a (mon.0) 1878 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T19:03:28.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:28 smithi098 bash[79516]: cluster 2023-09-23T19:03:26.496138+0000 mgr.y (mgr.26054) 1367 : cluster 0 pgmap v1504: 89 pgs: 2 remapped+peering, 1 active+undersized+degraded+remapped+backfilling, 16 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 38079/209601 objects degraded (18.167%); 4433/209601 objects misplaced (2.115%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T19:03:28.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:28 smithi098 bash[79516]: cluster 2023-09-23T19:03:27.335659+0000 mon.a (mon.0) 1878 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T19:03:28.479 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:28.513 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:28.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:28.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:28.617 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:28.635 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:28 smithi173 bash[59422]: cluster 2023-09-23T19:03:26.496138+0000 mgr.y (mgr.26054) 1367 : cluster 0 pgmap v1504: 89 pgs: 2 remapped+peering, 1 active+undersized+degraded+remapped+backfilling, 16 active+clean+remapped, 4 active+undersized+remapped, 37 active+clean, 15 active+undersized+degraded, 14 active+undersized; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 38079/209601 objects degraded (18.167%); 4433/209601 objects misplaced (2.115%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T19:03:28.635 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:28 smithi173 bash[59422]: cluster 2023-09-23T19:03:27.335659+0000 mon.a (mon.0) 1878 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T19:03:28.648 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:28.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:28.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:28.751 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:28.790 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:28.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:28.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:28.896 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:28.919 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:28.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:28.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:29.022 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:29.069 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:29.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:29.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:29.171 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:29.202 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:29.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:29.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:29.306 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:29.349 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:29.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:29.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:29.451 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:29.486 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:29.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:29.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:29.594 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:29.628 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:29.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:29.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:29.732 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:29.762 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:29.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:29.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:29.821 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:03:29.865 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:29.902 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:29.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:29.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:29.942 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:03:30.007 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:30.034 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:30.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:30.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:30.139 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:30.176 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:30.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:30.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:30.280 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:30.334 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:30.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:30.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:30.440 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:30.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:30 smithi098 bash[79516]: cluster 2023-09-23T19:03:28.496990+0000 mgr.y (mgr.26054) 1368 : cluster 0 pgmap v1505: 89 pgs: 1 active+remapped+backfill_wait, 2 remapped+peering, 1 active+undersized+degraded+remapped+backfilling, 21 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 14 active+undersized+degraded, 11 active+undersized; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 33134/206070 objects degraded (16.079%); 17264/206070 objects misplaced (8.378%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T19:03:30.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:30 smithi098 bash[76817]: cluster 2023-09-23T19:03:28.496990+0000 mgr.y (mgr.26054) 1368 : cluster 0 pgmap v1505: 89 pgs: 1 active+remapped+backfill_wait, 2 remapped+peering, 1 active+undersized+degraded+remapped+backfilling, 21 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 14 active+undersized+degraded, 11 active+undersized; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 33134/206070 objects degraded (16.079%); 17264/206070 objects misplaced (8.378%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T19:03:30.479 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:30.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:30.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:30.584 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:30.618 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:30.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:30.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:30.723 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:30.755 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:30.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:30.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:30.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:30 smithi173 bash[59422]: cluster 2023-09-23T19:03:28.496990+0000 mgr.y (mgr.26054) 1368 : cluster 0 pgmap v1505: 89 pgs: 1 active+remapped+backfill_wait, 2 remapped+peering, 1 active+undersized+degraded+remapped+backfilling, 21 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 14 active+undersized+degraded, 11 active+undersized; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 33134/206070 objects degraded (16.079%); 17264/206070 objects misplaced (8.378%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T19:03:30.859 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:30.891 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:30.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:30.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:30.998 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:31.021 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:31.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:31.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:31.127 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:31.162 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:31.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:31.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:31.268 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:31.303 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:31.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:31.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:31.409 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:31.418 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:31 smithi098 bash[79516]: cluster 2023-09-23T19:03:30.482806+0000 mon.a (mon.0) 1879 : cluster 3 Health check update: Degraded data redundancy: 33134/206070 objects degraded (16.079%), 15 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:31.419 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:31 smithi098 bash[76817]: cluster 2023-09-23T19:03:30.482806+0000 mon.a (mon.0) 1879 : cluster 3 Health check update: Degraded data redundancy: 33134/206070 objects degraded (16.079%), 15 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:31.441 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:31.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:31.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:31.544 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:31.569 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:31.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:31.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:31.672 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:31.701 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:31.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:31.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:31.767 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:31 smithi173 bash[59422]: cluster 2023-09-23T19:03:30.482806+0000 mon.a (mon.0) 1879 : cluster 3 Health check update: Degraded data redundancy: 33134/206070 objects degraded (16.079%), 15 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:31.804 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:31.835 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:31.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:31.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:31.941 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:31.966 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:31.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:31.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:31.994 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T19:03:31.994 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:03:32.072 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:32.096 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:32.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:32.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:32.200 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:32.228 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:32.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:32.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:32.335 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:32.369 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:32.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:32.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:32.472 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:32.480 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:32 smithi173 bash[59422]: cluster 2023-09-23T19:03:30.497714+0000 mgr.y (mgr.26054) 1369 : cluster 0 pgmap v1506: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 2 remapped+peering, 22 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 326 op/s; 31803/203547 objects degraded (15.624%); 21336/203547 objects misplaced (10.482%); 10 MiB/s, 164 objects/s recovering 2023-09-23T19:03:32.494 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:32.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:32.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:32.597 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:32.620 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:32.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:32.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:32.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:32 smithi098 bash[79516]: cluster 2023-09-23T19:03:30.497714+0000 mgr.y (mgr.26054) 1369 : cluster 0 pgmap v1506: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 2 remapped+peering, 22 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 326 op/s; 31803/203547 objects degraded (15.624%); 21336/203547 objects misplaced (10.482%); 10 MiB/s, 164 objects/s recovering 2023-09-23T19:03:32.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:32 smithi098 bash[76817]: cluster 2023-09-23T19:03:30.497714+0000 mgr.y (mgr.26054) 1369 : cluster 0 pgmap v1506: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 2 remapped+peering, 22 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 326 op/s; 31803/203547 objects degraded (15.624%); 21336/203547 objects misplaced (10.482%); 10 MiB/s, 164 objects/s recovering 2023-09-23T19:03:32.725 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:32.754 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:32.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:32.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:32.806 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:03:32.857 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:32.903 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:32.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:32.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:33.007 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:33.036 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:33.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:33.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:33.053 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T19:03:33.139 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:33.173 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:33.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:33.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:33.276 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:33.309 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:33.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:33.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:33.414 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:33.447 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:33.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:33.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:33.551 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:33.578 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:33.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:33.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:33.683 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:33.691 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:33 smithi098 bash[76817]: cluster 2023-09-23T19:03:32.498640+0000 mgr.y (mgr.26054) 1370 : cluster 0 pgmap v1507: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 400 op/s; 31220/199977 objects degraded (15.612%); 20954/199977 objects misplaced (10.478%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T19:03:33.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:33 smithi098 bash[76817]: cluster 2023-09-23T19:03:33.359489+0000 mon.a (mon.0) 1880 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T19:03:33.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:33 smithi098 bash[79516]: cluster 2023-09-23T19:03:32.498640+0000 mgr.y (mgr.26054) 1370 : cluster 0 pgmap v1507: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 400 op/s; 31220/199977 objects degraded (15.612%); 20954/199977 objects misplaced (10.478%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T19:03:33.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:33 smithi098 bash[79516]: cluster 2023-09-23T19:03:33.359489+0000 mon.a (mon.0) 1880 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T19:03:33.713 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:33.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:33.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:33.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:33 smithi173 bash[59422]: cluster 2023-09-23T19:03:32.498640+0000 mgr.y (mgr.26054) 1370 : cluster 0 pgmap v1507: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 400 op/s; 31220/199977 objects degraded (15.612%); 20954/199977 objects misplaced (10.478%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T19:03:33.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:33 smithi173 bash[59422]: cluster 2023-09-23T19:03:33.359489+0000 mon.a (mon.0) 1880 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T19:03:33.817 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:33.840 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:03:33.840 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:03:33.845 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:33.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:33.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:33.956 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:33.986 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:33.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:33.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:34.094 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:34.126 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:34.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:34.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:34.231 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:34.268 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:34.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:34.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:34.374 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:34.402 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:34.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:34.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:34.505 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:34.536 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:34.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:34.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:34.638 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:34.667 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:34.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:34.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:34.771 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:34.805 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:34.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:34.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:34.909 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:34.942 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:34.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:34.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:35.045 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:35.074 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:35.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:35.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:35.178 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:35.205 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:35.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:35.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:35.308 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:35.326 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:35.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:35.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:35.429 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:35.449 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:35.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:35.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:35.552 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:35.589 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:35.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:35.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:35.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:35 smithi098 bash[76817]: cluster 2023-09-23T19:03:34.499530+0000 mgr.y (mgr.26054) 1371 : cluster 0 pgmap v1508: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 30335/194106 objects degraded (15.628%); 20478/194106 objects misplaced (10.550%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T19:03:35.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:35 smithi098 bash[76817]: cluster 2023-09-23T19:03:35.484076+0000 mon.a (mon.0) 1881 : cluster 3 Health check update: Degraded data redundancy: 30335/194106 objects degraded (15.628%), 15 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:35.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:35 smithi098 bash[79516]: cluster 2023-09-23T19:03:34.499530+0000 mgr.y (mgr.26054) 1371 : cluster 0 pgmap v1508: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 30335/194106 objects degraded (15.628%); 20478/194106 objects misplaced (10.550%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T19:03:35.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:35 smithi098 bash[79516]: cluster 2023-09-23T19:03:35.484076+0000 mon.a (mon.0) 1881 : cluster 3 Health check update: Degraded data redundancy: 30335/194106 objects degraded (15.628%), 15 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:35.695 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:35.728 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:35.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:35.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:35.831 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:35.843 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:35 smithi173 bash[59422]: cluster 2023-09-23T19:03:34.499530+0000 mgr.y (mgr.26054) 1371 : cluster 0 pgmap v1508: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 30335/194106 objects degraded (15.628%); 20478/194106 objects misplaced (10.550%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T19:03:35.843 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:35 smithi173 bash[59422]: cluster 2023-09-23T19:03:35.484076+0000 mon.a (mon.0) 1881 : cluster 3 Health check update: Degraded data redundancy: 30335/194106 objects degraded (15.628%), 15 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:35.861 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:35.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:35.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:35.964 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:35.987 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:35.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:35.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:36.090 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:36.117 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:36.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:36.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:36.222 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:36.244 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:36.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:36.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:36.350 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:36.369 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:03:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:03:36] "GET /metrics HTTP/1.1" 200 36647 "" "Prometheus/2.33.4" 2023-09-23T19:03:36.383 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:36.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:36.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:36.490 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:36.518 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:36.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:36.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:36.622 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:36.653 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:36.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:36.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:36.757 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:36.784 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:36.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:36.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:36.888 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:36.924 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:36.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:36.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:37.030 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:37.055 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:37.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:37.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:37.158 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:37.187 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:37.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:37.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:37.293 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:37.334 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:37.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:37.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:37.439 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:37.468 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:37.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:37.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:37.566 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:37 smithi098 bash[76817]: cluster 2023-09-23T19:03:36.500138+0000 mgr.y (mgr.26054) 1372 : cluster 0 pgmap v1509: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 30335/194106 objects degraded (15.628%); 20478/194106 objects misplaced (10.550%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T19:03:37.576 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:37.607 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:37.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:37.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:37.713 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:37.743 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:37.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:37.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:37.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:37 smithi173 bash[59422]: cluster 2023-09-23T19:03:36.500138+0000 mgr.y (mgr.26054) 1372 : cluster 0 pgmap v1509: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 30335/194106 objects degraded (15.628%); 20478/194106 objects misplaced (10.550%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T19:03:37.850 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:37.866 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:03:37.883 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:37.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:37.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:37.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:37 smithi098 bash[79516]: cluster 2023-09-23T19:03:36.500138+0000 mgr.y (mgr.26054) 1372 : cluster 0 pgmap v1509: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 30335/194106 objects degraded (15.628%); 20478/194106 objects misplaced (10.550%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T19:03:37.988 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:38.011 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:38.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:38.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:38.114 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:38.151 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:38.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:38.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:38.258 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:38.288 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:38.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:38.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:38.393 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:38.423 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:38.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:38.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:38.530 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:38.583 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:38.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:38.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:38.688 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:38.722 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:38.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:38.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:38.831 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:38.863 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:38.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:38.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:38.967 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:39.001 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:39.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:39.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:39.104 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:39.132 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:39.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:39.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:39.240 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:39.277 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:39.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:39.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:39.384 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:39.411 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:39.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:39.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:39.515 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:39.545 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:39.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:39.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:39.654 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:39.663 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:39 smithi098 bash[76817]: cluster 2023-09-23T19:03:38.501424+0000 mgr.y (mgr.26054) 1373 : cluster 0 pgmap v1510: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 29491/188973 objects degraded (15.606%); 19931/188973 objects misplaced (10.547%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T19:03:39.663 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:39 smithi098 bash[79516]: cluster 2023-09-23T19:03:38.501424+0000 mgr.y (mgr.26054) 1373 : cluster 0 pgmap v1510: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 29491/188973 objects degraded (15.606%); 19931/188973 objects misplaced (10.547%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T19:03:39.680 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:39.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:39.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:39.787 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:39.816 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:39.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:39.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:39.919 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:39.944 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:39.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:39.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:40.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:39 smithi173 bash[59422]: cluster 2023-09-23T19:03:38.501424+0000 mgr.y (mgr.26054) 1373 : cluster 0 pgmap v1510: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 29491/188973 objects degraded (15.606%); 19931/188973 objects misplaced (10.547%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T19:03:40.048 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:40.072 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:40.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:40.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:40.176 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:40.204 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:40.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:40.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:40.309 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:40.334 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:40.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:40.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:40.436 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:40.463 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:40.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:40.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:40.566 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:40.602 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:40.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:40.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:40.705 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:40.714 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:40 smithi098 bash[76817]: audit 2023-09-23T19:03:39.640676+0000 mon.a (mon.0) 1882 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:03:40.714 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:40 smithi098 bash[76817]: cluster 2023-09-23T19:03:40.485453+0000 mon.a (mon.0) 1883 : cluster 3 Health check update: Degraded data redundancy: 29491/188973 objects degraded (15.606%), 15 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:40.714 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:40 smithi098 bash[79516]: audit 2023-09-23T19:03:39.640676+0000 mon.a (mon.0) 1882 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:03:40.715 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:40 smithi098 bash[79516]: cluster 2023-09-23T19:03:40.485453+0000 mon.a (mon.0) 1883 : cluster 3 Health check update: Degraded data redundancy: 29491/188973 objects degraded (15.606%), 15 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:40.731 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:40.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:40.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:40.836 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:40.860 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:40.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:40.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:40.967 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:41.001 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:41.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:41.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:41.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:40 smithi173 bash[59422]: audit 2023-09-23T19:03:39.640676+0000 mon.a (mon.0) 1882 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:03:41.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:40 smithi173 bash[59422]: cluster 2023-09-23T19:03:40.485453+0000 mon.a (mon.0) 1883 : cluster 3 Health check update: Degraded data redundancy: 29491/188973 objects degraded (15.606%), 15 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:41.106 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:41.138 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:41.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:41.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:41.244 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:41.269 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:41.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:41.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:41.372 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:41.399 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:41.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:41.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:41.502 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:41.527 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:41.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:41.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:41.631 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:41.658 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:41.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:41.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:41.764 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:41.771 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T19:03:41.771 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:03:41.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:41 smithi173 bash[59422]: cluster 2023-09-23T19:03:40.502246+0000 mgr.y (mgr.26054) 1374 : cluster 0 pgmap v1511: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 29049/185718 objects degraded (15.641%); 19685/185718 objects misplaced (10.599%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T19:03:41.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:41.796 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:41.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:41.897 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:41.906 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:41 smithi098 bash[79516]: cluster 2023-09-23T19:03:40.502246+0000 mgr.y (mgr.26054) 1374 : cluster 0 pgmap v1511: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 29049/185718 objects degraded (15.641%); 19685/185718 objects misplaced (10.599%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T19:03:41.906 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:41 smithi098 bash[76817]: cluster 2023-09-23T19:03:40.502246+0000 mgr.y (mgr.26054) 1374 : cluster 0 pgmap v1511: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 29049/185718 objects degraded (15.641%); 19685/185718 objects misplaced (10.599%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T19:03:41.929 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:41.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:41.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:42.036 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:42.070 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:42.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:42.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:42.176 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:42.218 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:42.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:42.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:42.323 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:42.352 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:42.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:42.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:42.456 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:42.480 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:42.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:42.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:42.585 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:42.624 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:42.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:42.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:42.725 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T19:03:42.730 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:42.759 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:42.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:42.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:42.862 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:42.887 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:42.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:42.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:42.991 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:43.022 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:43.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:43.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:43.125 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:43.164 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:43.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:43.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:43.268 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:43.308 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:43.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:43.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:43.418 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:43.457 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:43.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:43.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:43.561 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:43.579 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:43 smithi098 bash[76817]: cluster 2023-09-23T19:03:42.506160+0000 mgr.y (mgr.26054) 1375 : cluster 0 pgmap v1512: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 28431/182145 objects degraded (15.609%); 19329/182145 objects misplaced (10.612%); 762 KiB/s, 11 objects/s recovering 2023-09-23T19:03:43.590 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:43.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:43.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:43.693 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:43.722 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:43.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:43.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:43.831 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:43.864 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:43.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:43.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:43.932 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:43 smithi098 bash[79516]: cluster 2023-09-23T19:03:42.506160+0000 mgr.y (mgr.26054) 1375 : cluster 0 pgmap v1512: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 28431/182145 objects degraded (15.609%); 19329/182145 objects misplaced (10.612%); 762 KiB/s, 11 objects/s recovering 2023-09-23T19:03:43.967 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:44.002 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:44.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:44.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:43 smithi173 bash[59422]: cluster 2023-09-23T19:03:42.506160+0000 mgr.y (mgr.26054) 1375 : cluster 0 pgmap v1512: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 28431/182145 objects degraded (15.609%); 19329/182145 objects misplaced (10.612%); 762 KiB/s, 11 objects/s recovering 2023-09-23T19:03:44.112 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:44.137 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:44.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:44.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:44.242 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:44.272 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:44.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:44.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:44.376 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:44.404 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:44.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:44.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:44.509 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:44.537 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:44.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:44.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:44.640 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:44.664 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:44.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:44.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:44.767 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:44.796 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:44.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:44.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:44.899 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:44.933 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:44.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:44.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:45.037 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:45.067 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:45.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:45.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:45.171 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:45.209 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:45.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:45.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:45.312 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:45.353 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:45.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:45.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:45.458 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:45.482 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:45.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:45.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:45.587 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:45 smithi098 bash[76817]: cluster 2023-09-23T19:03:44.507067+0000 mgr.y (mgr.26054) 1376 : cluster 0 pgmap v1513: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 664 op/s; 27510/176034 objects degraded (15.628%); 18832/176034 objects misplaced (10.698%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T19:03:45.587 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:45 smithi098 bash[76817]: cluster 2023-09-23T19:03:45.486616+0000 mon.a (mon.0) 1884 : cluster 3 Health check update: Degraded data redundancy: 27510/176034 objects degraded (15.628%), 15 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:45.588 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:45 smithi098 bash[79516]: cluster 2023-09-23T19:03:44.507067+0000 mgr.y (mgr.26054) 1376 : cluster 0 pgmap v1513: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 664 op/s; 27510/176034 objects degraded (15.628%); 18832/176034 objects misplaced (10.698%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T19:03:45.588 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:45 smithi098 bash[79516]: cluster 2023-09-23T19:03:45.486616+0000 mon.a (mon.0) 1884 : cluster 3 Health check update: Degraded data redundancy: 27510/176034 objects degraded (15.628%), 15 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:45.590 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:45.624 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:45.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:45.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:45.727 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:45.759 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:45.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:45.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:45.863 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:45.898 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:45.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:45.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:46.000 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:46.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:45 smithi173 bash[59422]: cluster 2023-09-23T19:03:44.507067+0000 mgr.y (mgr.26054) 1376 : cluster 0 pgmap v1513: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 664 op/s; 27510/176034 objects degraded (15.628%); 18832/176034 objects misplaced (10.698%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T19:03:46.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:45 smithi173 bash[59422]: cluster 2023-09-23T19:03:45.486616+0000 mon.a (mon.0) 1884 : cluster 3 Health check update: Degraded data redundancy: 27510/176034 objects degraded (15.628%), 15 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:46.035 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:46.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:46.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:46.139 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:46.181 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:46.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:46.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:46.287 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:46.296 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:03:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:03:46] "GET /metrics HTTP/1.1" 200 36651 "" "Prometheus/2.33.4" 2023-09-23T19:03:46.327 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:46.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:46.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:46.433 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:46.465 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:46.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:46.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:46.572 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:46.596 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:46.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:46.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:46.702 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:46.730 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:46.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:46.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:46.836 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:46.870 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:46.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:46.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:46.973 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:47.011 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:47.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:47.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:47.115 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:47.143 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:47.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:47.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:47.246 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:47.275 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:47.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:47.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:47.379 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:47.405 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:47.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:47.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:47.508 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:47.537 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:47.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:47.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:47.580 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:03:47.646 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:47.679 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:47.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:47.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:47.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:47 smithi173 bash[59422]: cluster 2023-09-23T19:03:46.507764+0000 mgr.y (mgr.26054) 1377 : cluster 0 pgmap v1514: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 27510/176034 objects degraded (15.628%); 18832/176034 objects misplaced (10.698%); 783 KiB/s, 12 objects/s recovering 2023-09-23T19:03:47.784 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:47.792 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:47 smithi098 bash[79516]: cluster 2023-09-23T19:03:46.507764+0000 mgr.y (mgr.26054) 1377 : cluster 0 pgmap v1514: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 27510/176034 objects degraded (15.628%); 18832/176034 objects misplaced (10.698%); 783 KiB/s, 12 objects/s recovering 2023-09-23T19:03:47.792 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:47 smithi098 bash[76817]: cluster 2023-09-23T19:03:46.507764+0000 mgr.y (mgr.26054) 1377 : cluster 0 pgmap v1514: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 27510/176034 objects degraded (15.628%); 18832/176034 objects misplaced (10.698%); 783 KiB/s, 12 objects/s recovering 2023-09-23T19:03:47.810 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:47.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:47.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:47.914 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:47.949 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:47.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:47.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:48.053 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:48.083 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:48.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:48.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:48.186 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:48.226 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:48.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:48.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:48.334 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:48.368 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:48.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:48.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:48.472 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:48.521 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:48.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:48.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:48.629 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:48.655 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:48.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:48.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:48.758 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:48.785 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:48.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:48.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:48.888 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:48.920 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:48.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:48.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:49.025 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:49.056 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:49.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:49.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:49.162 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:49.189 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:49.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:49.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:49.291 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:49.319 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:49.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:49.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:49.423 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:49.450 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:49.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:49.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:49.554 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:49.584 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:49.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:49.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:49.691 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:49.701 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:49 smithi173 bash[59422]: cluster 2023-09-23T19:03:48.508760+0000 mgr.y (mgr.26054) 1378 : cluster 0 pgmap v1515: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 669 op/s; 26515/169992 objects degraded (15.598%); 18202/169992 objects misplaced (10.708%); 783 KiB/s, 12 objects/s recovering 2023-09-23T19:03:49.719 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:49.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:49.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:49.824 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:49.832 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:49 smithi098 bash[76817]: cluster 2023-09-23T19:03:48.508760+0000 mgr.y (mgr.26054) 1378 : cluster 0 pgmap v1515: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 669 op/s; 26515/169992 objects degraded (15.598%); 18202/169992 objects misplaced (10.708%); 783 KiB/s, 12 objects/s recovering 2023-09-23T19:03:49.833 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:49 smithi098 bash[79516]: cluster 2023-09-23T19:03:48.508760+0000 mgr.y (mgr.26054) 1378 : cluster 0 pgmap v1515: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 669 op/s; 26515/169992 objects degraded (15.598%); 18202/169992 objects misplaced (10.708%); 783 KiB/s, 12 objects/s recovering 2023-09-23T19:03:49.863 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:49.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:49.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:49.968 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:50.000 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:50.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:50.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:50.104 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:50.134 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:50.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:50.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:50.239 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:50.266 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:50.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:50.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:50.371 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:50.392 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:03:50 smithi098 bash[73085]: debug 2023-09-23T19:03:50.382+0000 7f90a56ad700 -1 mgr.server reply reply (2) No such file or directory no change 2023-09-23T19:03:50.394 INFO:teuthology.orchestra.run.smithi098.stdout:moved 2 / 267 (0.749064%) 2023-09-23T19:03:50.394 INFO:teuthology.orchestra.run.smithi098.stdout:avg 53.4 2023-09-23T19:03:50.394 INFO:teuthology.orchestra.run.smithi098.stdout:stddev 13.8881 -> 14.2295 (expected baseline 6.53605) 2023-09-23T19:03:50.395 INFO:teuthology.orchestra.run.smithi098.stdout:min osd.2 with 31 -> 29 pgs (0.580524 -> 0.543071 * mean) 2023-09-23T19:03:50.395 INFO:teuthology.orchestra.run.smithi098.stdout:max osd.1 with 47 -> 47 pgs (0.88015 -> 0.88015 * mean) 2023-09-23T19:03:50.395 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T19:03:50.395 INFO:teuthology.orchestra.run.smithi098.stdout:oload 1000 2023-09-23T19:03:50.395 INFO:teuthology.orchestra.run.smithi098.stdout:max_change 0.05 2023-09-23T19:03:50.395 INFO:teuthology.orchestra.run.smithi098.stdout:max_change_osds 4 2023-09-23T19:03:50.395 INFO:teuthology.orchestra.run.smithi098.stdout:average_utilization 0.0254 2023-09-23T19:03:50.395 INFO:teuthology.orchestra.run.smithi098.stdout:overload_utilization 0.2541 2023-09-23T19:03:50.395 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3 weight 0.8669 -> 0.9169 2023-09-23T19:03:50.395 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5 weight 0.9098 -> 0.9598 2023-09-23T19:03:50.399 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:50.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:50.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:50.405 INFO:teuthology.orchestra.run.smithi098.stderr:no change 2023-09-23T19:03:50.503 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:50.526 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:50.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:50.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:50.630 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:50.641 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:50 smithi173 bash[59422]: cluster 2023-09-23T19:03:50.487883+0000 mon.a (mon.0) 1885 : cluster 3 Health check update: Degraded data redundancy: 26515/169992 objects degraded (15.598%), 15 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:50.658 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:50.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:50.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:50.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:50 smithi098 bash[76817]: cluster 2023-09-23T19:03:50.487883+0000 mon.a (mon.0) 1885 : cluster 3 Health check update: Degraded data redundancy: 26515/169992 objects degraded (15.598%), 15 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:50.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:50 smithi098 bash[79516]: cluster 2023-09-23T19:03:50.487883+0000 mon.a (mon.0) 1885 : cluster 3 Health check update: Degraded data redundancy: 26515/169992 objects degraded (15.598%), 15 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:50.765 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:50.792 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:50.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:50.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:50.895 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:50.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:50.938 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:50.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:51.039 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:51.078 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:51.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:51.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:51.184 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:51.216 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:51.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:51.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:51.319 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:51.346 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:51.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:51.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:51.452 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:51.491 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:51.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:51.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:51.593 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:51.624 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:51.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:51.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:51.653 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:51 smithi098 bash[79516]: audit 2023-09-23T19:03:50.387195+0000 mgr.y (mgr.26054) 1379 : audit 0 from='client.35862 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:03:51.653 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:51 smithi098 bash[79516]: cluster 2023-09-23T19:03:50.509418+0000 mgr.y (mgr.26054) 1380 : cluster 0 pgmap v1516: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 25976/166200 objects degraded (15.629%); 17909/166200 objects misplaced (10.776%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T19:03:51.654 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:51 smithi098 bash[76817]: audit 2023-09-23T19:03:50.387195+0000 mgr.y (mgr.26054) 1379 : audit 0 from='client.35862 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:03:51.654 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:51 smithi098 bash[76817]: cluster 2023-09-23T19:03:50.509418+0000 mgr.y (mgr.26054) 1380 : cluster 0 pgmap v1516: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 25976/166200 objects degraded (15.629%); 17909/166200 objects misplaced (10.776%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T19:03:51.728 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:51.755 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:51.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:51.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:51.859 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:51.900 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:51.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:51.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:52.007 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:52.019 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:51 smithi173 bash[59422]: audit 2023-09-23T19:03:50.387195+0000 mgr.y (mgr.26054) 1379 : audit 0 from='client.35862 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:03:52.019 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:51 smithi173 bash[59422]: cluster 2023-09-23T19:03:50.509418+0000 mgr.y (mgr.26054) 1380 : cluster 0 pgmap v1516: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 25976/166200 objects degraded (15.629%); 17909/166200 objects misplaced (10.776%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T19:03:52.038 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:52.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:52.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:52.142 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:52.169 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:52.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:52.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:52.274 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:52.301 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:52.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:52.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:52.405 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:52.445 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:52.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:52.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:52.548 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:52.567 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T19:03:52.594 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:52.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:52.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:52.615 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T19:03:52.678 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T19:03:52.703 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:52.712 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:52 smithi098 bash[76817]: audit 2023-09-23T19:03:52.512058+0000 mon.c (mon.2) 145 : audit 0 from='client.? 172.21.15.98:0/1219189668' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:03:52.712 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:52 smithi098 bash[79516]: audit 2023-09-23T19:03:52.512058+0000 mon.c (mon.2) 145 : audit 0 from='client.? 172.21.15.98:0/1219189668' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:03:52.722 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T19:03:52.744 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:52.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:52.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:52.761 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T19:03:52.797 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T19:03:52.800 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:03:52.800 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:03:52.800 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:03:52.800 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:03:52.800 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:03:52.800 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T19:03:52.800 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:03:52.800 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T19:03:52.849 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:52.862 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:52 smithi173 bash[59422]: audit 2023-09-23T19:03:52.512058+0000 mon.c (mon.2) 145 : audit 0 from='client.? 172.21.15.98:0/1219189668' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:03:52.882 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:52.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:52.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:52.986 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:53.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:53.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:53.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:53.118 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:53.145 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:53.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:53.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:53.249 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:53.286 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:53.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:53.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:53.388 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:53.430 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:53.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:53.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:53.537 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:53.570 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:53.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:53.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:53.633 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is set 2023-09-23T19:03:53.673 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:53.701 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:53.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:53.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:53.805 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:53.832 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:53.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:53.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:53.935 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:53.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:53 smithi098 bash[76817]: cluster 2023-09-23T19:03:52.510350+0000 mgr.y (mgr.26054) 1381 : cluster 0 pgmap v1517: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.3 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 25180/161358 objects degraded (15.605%); 17426/161358 objects misplaced (10.800%); 853 KiB/s, 13 objects/s recovering 2023-09-23T19:03:53.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:53 smithi098 bash[76817]: audit 2023-09-23T19:03:52.686479+0000 mon.a (mon.0) 1886 : audit 1 from='client.? 172.21.15.98:0/365597629' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:03:53.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:53 smithi098 bash[79516]: cluster 2023-09-23T19:03:52.510350+0000 mgr.y (mgr.26054) 1381 : cluster 0 pgmap v1517: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.3 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 25180/161358 objects degraded (15.605%); 17426/161358 objects misplaced (10.800%); 853 KiB/s, 13 objects/s recovering 2023-09-23T19:03:53.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:53 smithi098 bash[79516]: audit 2023-09-23T19:03:52.686479+0000 mon.a (mon.0) 1886 : audit 1 from='client.? 172.21.15.98:0/365597629' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:03:53.984 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:53.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:53.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:54.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:53 smithi173 bash[59422]: cluster 2023-09-23T19:03:52.510350+0000 mgr.y (mgr.26054) 1381 : cluster 0 pgmap v1517: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.3 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 25180/161358 objects degraded (15.605%); 17426/161358 objects misplaced (10.800%); 853 KiB/s, 13 objects/s recovering 2023-09-23T19:03:54.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:53 smithi173 bash[59422]: audit 2023-09-23T19:03:52.686479+0000 mon.a (mon.0) 1886 : audit 1 from='client.? 172.21.15.98:0/365597629' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:03:54.089 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:54.132 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:54.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:54.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:54.236 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:54.263 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:54.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:54.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:54.367 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:54.394 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:54.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:54.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:54.497 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:54.526 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:54.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:54.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:54.584 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T19:03:54.584 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-09-23T19:03:54.630 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:54.639 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:54 smithi098 bash[76817]: cluster 2023-09-23T19:03:53.600095+0000 mon.a (mon.0) 1887 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:03:54.639 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:54 smithi098 bash[76817]: cluster 2023-09-23T19:03:53.614176+0000 osd.4 (osd.4) 63 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T19:03:54.640 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:54 smithi098 bash[76817]: audit 2023-09-23T19:03:53.614694+0000 mon.a (mon.0) 1888 : audit 1 from='client.? 172.21.15.98:0/365597629' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:03:54.640 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:54 smithi098 bash[76817]: cluster 2023-09-23T19:03:53.614892+0000 mon.a (mon.0) 1889 : cluster 0 osdmap e333: 8 total, 6 up, 7 in 2023-09-23T19:03:54.640 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:54 smithi098 bash[76817]: cluster 2023-09-23T19:03:53.631255+0000 osd.6 (osd.6) 315 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T19:03:54.640 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:54 smithi098 bash[79516]: cluster 2023-09-23T19:03:53.600095+0000 mon.a (mon.0) 1887 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:03:54.640 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:54 smithi098 bash[79516]: cluster 2023-09-23T19:03:53.614176+0000 osd.4 (osd.4) 63 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T19:03:54.640 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:54 smithi098 bash[79516]: audit 2023-09-23T19:03:53.614694+0000 mon.a (mon.0) 1888 : audit 1 from='client.? 172.21.15.98:0/365597629' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:03:54.640 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:54 smithi098 bash[79516]: cluster 2023-09-23T19:03:53.614892+0000 mon.a (mon.0) 1889 : cluster 0 osdmap e333: 8 total, 6 up, 7 in 2023-09-23T19:03:54.640 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:54 smithi098 bash[79516]: cluster 2023-09-23T19:03:53.631255+0000 osd.6 (osd.6) 315 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T19:03:54.662 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:54.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:54.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:54.768 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:54.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:54 smithi173 bash[59422]: cluster 2023-09-23T19:03:53.600095+0000 mon.a (mon.0) 1887 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:03:54.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:54 smithi173 bash[59422]: cluster 2023-09-23T19:03:53.614176+0000 osd.4 (osd.4) 63 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T19:03:54.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:54 smithi173 bash[59422]: audit 2023-09-23T19:03:53.614694+0000 mon.a (mon.0) 1888 : audit 1 from='client.? 172.21.15.98:0/365597629' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:03:54.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:54 smithi173 bash[59422]: cluster 2023-09-23T19:03:53.614892+0000 mon.a (mon.0) 1889 : cluster 0 osdmap e333: 8 total, 6 up, 7 in 2023-09-23T19:03:54.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:54 smithi173 bash[59422]: cluster 2023-09-23T19:03:53.631255+0000 osd.6 (osd.6) 315 : cluster 3 failed to encode map e333 with expected crc 2023-09-23T19:03:54.796 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:54.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:54.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:54.899 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:54.926 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:54.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:54.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:55.030 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:55.059 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:55.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:55.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:55.163 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:55.188 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:55.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:55.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:55.292 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:55.349 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:55.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:55.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:55.454 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:55.482 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:55.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:55.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:55.585 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:55.615 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:55.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:55.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:55.719 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:55.752 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:55.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:55.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:55.803 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:55 smithi098 bash[79516]: cluster 2023-09-23T19:03:54.511608+0000 mgr.y (mgr.26054) 1382 : cluster 0 pgmap v1519: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 24756/158220 objects degraded (15.647%); 17175/158220 objects misplaced (10.855%); 979 KiB/s, 15 objects/s recovering 2023-09-23T19:03:55.804 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:55 smithi098 bash[79516]: audit 2023-09-23T19:03:54.643600+0000 mon.a (mon.0) 1890 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:03:55.804 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:55 smithi098 bash[79516]: cluster 2023-09-23T19:03:55.489118+0000 mon.a (mon.0) 1891 : cluster 3 Health check update: Degraded data redundancy: 24756/158220 objects degraded (15.647%), 15 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:55.804 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:55 smithi098 bash[76817]: cluster 2023-09-23T19:03:54.511608+0000 mgr.y (mgr.26054) 1382 : cluster 0 pgmap v1519: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 24756/158220 objects degraded (15.647%); 17175/158220 objects misplaced (10.855%); 979 KiB/s, 15 objects/s recovering 2023-09-23T19:03:55.804 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:55 smithi098 bash[76817]: audit 2023-09-23T19:03:54.643600+0000 mon.a (mon.0) 1890 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:03:55.804 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:55 smithi098 bash[76817]: cluster 2023-09-23T19:03:55.489118+0000 mon.a (mon.0) 1891 : cluster 3 Health check update: Degraded data redundancy: 24756/158220 objects degraded (15.647%), 15 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:55.860 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:55.887 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:55.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:55.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:55.992 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:56.006 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:55 smithi173 bash[59422]: cluster 2023-09-23T19:03:54.511608+0000 mgr.y (mgr.26054) 1382 : cluster 0 pgmap v1519: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 24756/158220 objects degraded (15.647%); 17175/158220 objects misplaced (10.855%); 979 KiB/s, 15 objects/s recovering 2023-09-23T19:03:56.006 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:55 smithi173 bash[59422]: audit 2023-09-23T19:03:54.643600+0000 mon.a (mon.0) 1890 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:03:56.007 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:55 smithi173 bash[59422]: cluster 2023-09-23T19:03:55.489118+0000 mon.a (mon.0) 1891 : cluster 3 Health check update: Degraded data redundancy: 24756/158220 objects degraded (15.647%), 15 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T19:03:56.024 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:56.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:56.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:56.127 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:56.145 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:56.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:56.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:56.247 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:56.278 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:56.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:56.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:56.384 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:56.417 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:56.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:56.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:56.524 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:56.553 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:56.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:56.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:56.657 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:56.685 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:56.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:56.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:56.693 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:03:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:03:56] "GET /metrics HTTP/1.1" 200 36650 "" "Prometheus/2.33.4" 2023-09-23T19:03:56.789 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:56.832 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:56.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:56.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:56.941 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:56.973 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:56.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:56.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:57.079 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:57.110 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:57.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:57.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:57.214 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:57.243 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:57.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:57.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:57.349 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:57.401 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:57.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:57.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:57.503 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:57.531 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:57.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:57.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:57.621 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:57 smithi098 bash[76817]: cluster 2023-09-23T19:03:56.512335+0000 mgr.y (mgr.26054) 1383 : cluster 0 pgmap v1520: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 24756/158220 objects degraded (15.647%); 17175/158220 objects misplaced (10.855%); 979 KiB/s, 15 objects/s recovering 2023-09-23T19:03:57.622 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:57 smithi098 bash[79516]: cluster 2023-09-23T19:03:56.512335+0000 mgr.y (mgr.26054) 1383 : cluster 0 pgmap v1520: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 24756/158220 objects degraded (15.647%); 17175/158220 objects misplaced (10.855%); 979 KiB/s, 15 objects/s recovering 2023-09-23T19:03:57.635 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:57.670 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:57.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:57.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:57.775 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:57.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:57 smithi173 bash[59422]: cluster 2023-09-23T19:03:56.512335+0000 mgr.y (mgr.26054) 1383 : cluster 0 pgmap v1520: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 24756/158220 objects degraded (15.647%); 17175/158220 objects misplaced (10.855%); 979 KiB/s, 15 objects/s recovering 2023-09-23T19:03:57.805 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:57.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:57.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:57.910 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:57.944 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:57.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:57.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:58.050 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:58.076 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:58.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:58.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:58.183 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:58.219 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:58.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:58.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:58.327 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:58.328 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:03:58.329 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:03:58.391 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:03:58.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:58.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:03:58.496 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:58.532 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:58.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:58.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:58.636 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:58.663 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:58.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:58.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:58.769 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:58.803 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:58.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:58.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:58.908 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:58.948 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:03:58.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:58.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:03:59.054 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:59.082 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:59.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:59.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:59.186 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:59.214 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:59.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:59.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:59.317 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:59.343 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:03:59.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:59.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:03:59.449 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:59.485 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:59.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:59.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:03:59.591 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:59.638 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:03:59.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:59.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:03:59.744 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:59.771 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:03:59.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:59.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:03:59.874 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:59.884 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:03:59 smithi098 bash[76817]: cluster 2023-09-23T19:03:58.513352+0000 mgr.y (mgr.26054) 1384 : cluster 0 pgmap v1521: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 23832/152757 objects degraded (15.601%); 16639/152757 objects misplaced (10.892%); 979 KiB/s, 15 objects/s recovering 2023-09-23T19:03:59.884 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:03:59 smithi098 bash[79516]: cluster 2023-09-23T19:03:58.513352+0000 mgr.y (mgr.26054) 1384 : cluster 0 pgmap v1521: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 23832/152757 objects degraded (15.601%); 16639/152757 objects misplaced (10.892%); 979 KiB/s, 15 objects/s recovering 2023-09-23T19:03:59.909 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:03:59.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:03:59.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:00.014 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:00.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:03:59 smithi173 bash[59422]: cluster 2023-09-23T19:03:58.513352+0000 mgr.y (mgr.26054) 1384 : cluster 0 pgmap v1521: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 23832/152757 objects degraded (15.601%); 16639/152757 objects misplaced (10.892%); 979 KiB/s, 15 objects/s recovering 2023-09-23T19:04:00.033 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:00.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:00.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:00.135 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:00.174 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:00.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:00.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:00.279 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:00.309 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:00.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:00.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:00.330 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:04:00.353 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:04:00.414 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:00.444 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:00.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:00.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:00.548 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:00.577 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:00.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:00.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:00.681 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:00.710 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:00.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:00.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:00.814 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:00.823 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:00 smithi098 bash[76817]: cluster 2023-09-23T19:04:00.490243+0000 mon.a (mon.0) 1892 : cluster 3 Health check update: Degraded data redundancy: 23832/152757 objects degraded (15.601%), 15 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:00.823 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:00 smithi098 bash[79516]: cluster 2023-09-23T19:04:00.490243+0000 mon.a (mon.0) 1892 : cluster 3 Health check update: Degraded data redundancy: 23832/152757 objects degraded (15.601%), 15 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:00.839 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:00.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:00.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:00.944 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:00.973 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:00.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:00.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:01.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:00 smithi173 bash[59422]: cluster 2023-09-23T19:04:00.490243+0000 mon.a (mon.0) 1892 : cluster 3 Health check update: Degraded data redundancy: 23832/152757 objects degraded (15.601%), 15 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:01.077 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:01.113 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:01.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:01.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:01.220 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:01.237 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T19:04:01.237 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [], 2023-09-23T19:04:01.237 INFO:teuthology.orchestra.run.smithi098.stdout: "num_ops": 0 2023-09-23T19:04:01.237 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T19:04:01.251 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:01.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:01.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:01.354 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:01.395 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:01.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:01.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:01.436 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:04:01.500 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:01.538 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:01.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:01.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:01.631 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:01 smithi098 bash[79516]: cluster 2023-09-23T19:04:00.514209+0000 mgr.y (mgr.26054) 1385 : cluster 0 pgmap v1522: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 569 op/s; 23356/149121 objects degraded (15.662%); 16343/149121 objects misplaced (10.960%); 972 KiB/s, 15 objects/s recovering 2023-09-23T19:04:01.632 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:01 smithi098 bash[76817]: cluster 2023-09-23T19:04:00.514209+0000 mgr.y (mgr.26054) 1385 : cluster 0 pgmap v1522: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 569 op/s; 23356/149121 objects degraded (15.662%); 16343/149121 objects misplaced (10.960%); 972 KiB/s, 15 objects/s recovering 2023-09-23T19:04:01.646 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:01.677 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:01.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:01.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:01.781 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:01.818 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:01.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:01.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:01.921 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:01.949 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:01.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:01.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:02.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:01 smithi173 bash[59422]: cluster 2023-09-23T19:04:00.514209+0000 mgr.y (mgr.26054) 1385 : cluster 0 pgmap v1522: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 569 op/s; 23356/149121 objects degraded (15.662%); 16343/149121 objects misplaced (10.960%); 972 KiB/s, 15 objects/s recovering 2023-09-23T19:04:02.056 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:02.088 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:02.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:02.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:02.192 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:02.229 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:02.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:02.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:02.335 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:02.361 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:02.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:02.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:02.465 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:02.488 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:02.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:02.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:02.592 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:02.620 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:02.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:02.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:02.729 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:02.760 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:02.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:02.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:02.864 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:02.912 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:02.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:02.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:03.016 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:03.051 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:03.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:03.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:03.160 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:03.180 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:03.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:03.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:03.285 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:03.310 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:03.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:03.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:03.413 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:03.453 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:03.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:03.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:03.559 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:03.581 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:03.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:03.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:03.684 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:03.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:03 smithi098 bash[79516]: cluster 2023-09-23T19:04:02.515170+0000 mgr.y (mgr.26054) 1386 : cluster 0 pgmap v1523: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 22344/143061 objects degraded (15.619%); 15730/143061 objects misplaced (10.995%); 972 KiB/s, 15 objects/s recovering 2023-09-23T19:04:03.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:03 smithi098 bash[79516]: audit 2023-09-23T19:04:02.931930+0000 mon.a (mon.0) 1893 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:04:03.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:03 smithi098 bash[76817]: cluster 2023-09-23T19:04:02.515170+0000 mgr.y (mgr.26054) 1386 : cluster 0 pgmap v1523: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 22344/143061 objects degraded (15.619%); 15730/143061 objects misplaced (10.995%); 972 KiB/s, 15 objects/s recovering 2023-09-23T19:04:03.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:03 smithi098 bash[76817]: audit 2023-09-23T19:04:02.931930+0000 mon.a (mon.0) 1893 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:04:03.735 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:03.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:03.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:03.839 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:03.866 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:03.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:03.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:03.973 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:04.004 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:04.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:04.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:04.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:03 smithi173 bash[59422]: cluster 2023-09-23T19:04:02.515170+0000 mgr.y (mgr.26054) 1386 : cluster 0 pgmap v1523: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 22344/143061 objects degraded (15.619%); 15730/143061 objects misplaced (10.995%); 972 KiB/s, 15 objects/s recovering 2023-09-23T19:04:04.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:03 smithi173 bash[59422]: audit 2023-09-23T19:04:02.931930+0000 mon.a (mon.0) 1893 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:04:04.108 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:04.143 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:04.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:04.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:04.247 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:04.283 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:04.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:04.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:04.386 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:04.424 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:04.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:04.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:04.528 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:04.562 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:04.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:04.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:04.669 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:04.711 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:04.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:04.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:04.723 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T19:04:04.818 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:04.846 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:04.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:04.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:04.950 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:04.978 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:04.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:04.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:05.082 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:05.116 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:05.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:05.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:05.221 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:05.248 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:05.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:05.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:05.352 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:05.390 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:05.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:05.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:05.498 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:05.535 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:05.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:05.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:05.638 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:05.665 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:05.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:05.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:05.775 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:05.802 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:05.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:05.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:05.906 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:05.915 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:05 smithi098 bash[76817]: cluster 2023-09-23T19:04:04.516136+0000 mgr.y (mgr.26054) 1387 : cluster 0 pgmap v1524: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 21883/139650 objects degraded (15.670%); 15458/139650 objects misplaced (11.069%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T19:04:05.916 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:05 smithi098 bash[76817]: cluster 2023-09-23T19:04:05.491654+0000 mon.a (mon.0) 1894 : cluster 3 Health check update: Degraded data redundancy: 21883/139650 objects degraded (15.670%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:05.916 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:05 smithi098 bash[79516]: cluster 2023-09-23T19:04:04.516136+0000 mgr.y (mgr.26054) 1387 : cluster 0 pgmap v1524: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 21883/139650 objects degraded (15.670%); 15458/139650 objects misplaced (11.069%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T19:04:05.916 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:05 smithi098 bash[79516]: cluster 2023-09-23T19:04:05.491654+0000 mon.a (mon.0) 1894 : cluster 3 Health check update: Degraded data redundancy: 21883/139650 objects degraded (15.670%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:05.933 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:05.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:05.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:06.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:05 smithi173 bash[59422]: cluster 2023-09-23T19:04:04.516136+0000 mgr.y (mgr.26054) 1387 : cluster 0 pgmap v1524: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 21883/139650 objects degraded (15.670%); 15458/139650 objects misplaced (11.069%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T19:04:06.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:05 smithi173 bash[59422]: cluster 2023-09-23T19:04:05.491654+0000 mon.a (mon.0) 1894 : cluster 3 Health check update: Degraded data redundancy: 21883/139650 objects degraded (15.670%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:06.039 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:06.065 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:06.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:06.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:06.168 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:06.176 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:04:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:04:06] "GET /metrics HTTP/1.1" 200 36650 "" "Prometheus/2.33.4" 2023-09-23T19:04:06.208 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:06.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:06.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:06.313 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:06.337 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:06.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:06.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:06.441 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:06.474 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:06.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:06.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:06.577 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:06.603 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:06.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:06.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:06.706 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:06.735 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:06.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:06.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:06.839 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:06.869 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:06.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:06.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:06.973 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:07.003 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:07.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:07.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:07.108 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:07.134 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:07.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:07.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:07.238 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:07.278 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:07.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:07.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:07.385 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:07.416 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:07.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:07.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:07.519 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:07.545 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:07.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:07.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:07.648 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:07.675 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:07.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:07.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:07.727 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:04:07.728 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:04:07.783 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:07.831 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:07.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:07.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:07.881 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:07 smithi098 bash[76817]: cluster 2023-09-23T19:04:06.516878+0000 mgr.y (mgr.26054) 1388 : cluster 0 pgmap v1525: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 21883/139650 objects degraded (15.670%); 15458/139650 objects misplaced (11.069%); 837 KiB/s, 13 objects/s recovering 2023-09-23T19:04:07.882 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:07 smithi098 bash[79516]: cluster 2023-09-23T19:04:06.516878+0000 mgr.y (mgr.26054) 1388 : cluster 0 pgmap v1525: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 21883/139650 objects degraded (15.670%); 15458/139650 objects misplaced (11.069%); 837 KiB/s, 13 objects/s recovering 2023-09-23T19:04:07.938 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:07.961 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:07.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:07.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:08.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:07 smithi173 bash[59422]: cluster 2023-09-23T19:04:06.516878+0000 mgr.y (mgr.26054) 1388 : cluster 0 pgmap v1525: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 21883/139650 objects degraded (15.670%); 15458/139650 objects misplaced (11.069%); 837 KiB/s, 13 objects/s recovering 2023-09-23T19:04:08.064 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:08.089 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:08.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:08.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:08.193 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:08.218 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:08.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:08.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:08.323 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:08.345 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:08.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:08.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:08.448 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:08.472 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:08.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:08.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:08.577 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:08.601 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:08.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:08.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:08.705 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:08.733 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:08.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:08.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:08.836 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:08.864 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:08.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:08.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:08.968 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:08.996 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:09.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:09.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:09.136 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:09.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:09.176 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:09.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:09.277 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:09.305 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:09.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:09.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:09.412 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:09.434 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:09.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:09.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:09.538 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:09.569 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:09.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:09.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:09.675 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:09.700 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:09.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:09.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:09.806 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:09.830 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:09.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:09.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:09.933 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:09.958 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:09.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:09.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:10.062 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:10.089 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:10.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:10.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:10.195 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:10.227 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:10.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:10.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:10.317 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:04:10.332 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:10.360 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:10.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:10.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:10.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:10 smithi098 bash[76817]: cluster 2023-09-23T19:04:08.517821+0000 mgr.y (mgr.26054) 1389 : cluster 0 pgmap v1526: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 679 op/s; 20906/133755 objects degraded (15.630%); 14878/133755 objects misplaced (11.123%); 837 KiB/s, 13 objects/s recovering 2023-09-23T19:04:10.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:10 smithi098 bash[76817]: audit 2023-09-23T19:04:09.216188+0000 mon.a (mon.0) 1895 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:04:10.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:10 smithi098 bash[76817]: audit 2023-09-23T19:04:09.229546+0000 mon.a (mon.0) 1896 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:04:10.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:10 smithi098 bash[76817]: audit 2023-09-23T19:04:09.644050+0000 mon.a (mon.0) 1897 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:04:10.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:10 smithi098 bash[79516]: cluster 2023-09-23T19:04:08.517821+0000 mgr.y (mgr.26054) 1389 : cluster 0 pgmap v1526: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 679 op/s; 20906/133755 objects degraded (15.630%); 14878/133755 objects misplaced (11.123%); 837 KiB/s, 13 objects/s recovering 2023-09-23T19:04:10.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:10 smithi098 bash[79516]: audit 2023-09-23T19:04:09.216188+0000 mon.a (mon.0) 1895 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:04:10.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:10 smithi098 bash[79516]: audit 2023-09-23T19:04:09.229546+0000 mon.a (mon.0) 1896 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:04:10.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:10 smithi098 bash[79516]: audit 2023-09-23T19:04:09.644050+0000 mon.a (mon.0) 1897 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:04:10.464 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:10.475 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:10 smithi173 bash[59422]: cluster 2023-09-23T19:04:08.517821+0000 mgr.y (mgr.26054) 1389 : cluster 0 pgmap v1526: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 679 op/s; 20906/133755 objects degraded (15.630%); 14878/133755 objects misplaced (11.123%); 837 KiB/s, 13 objects/s recovering 2023-09-23T19:04:10.475 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:10 smithi173 bash[59422]: audit 2023-09-23T19:04:09.216188+0000 mon.a (mon.0) 1895 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:04:10.475 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:10 smithi173 bash[59422]: audit 2023-09-23T19:04:09.229546+0000 mon.a (mon.0) 1896 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:04:10.475 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:10 smithi173 bash[59422]: audit 2023-09-23T19:04:09.644050+0000 mon.a (mon.0) 1897 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:04:10.492 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:10.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:10.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:10.596 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:10.625 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:10.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:10.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:10.731 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:10.777 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:10.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:10.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:10.882 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:10.911 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:10.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:10.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:11.015 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:11.064 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:11.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:11.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:11.189 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:11.234 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:11.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:11.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:11.341 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:11.350 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:11 smithi098 bash[76817]: cluster 2023-09-23T19:04:10.492730+0000 mon.a (mon.0) 1898 : cluster 3 Health check update: Degraded data redundancy: 20906/133755 objects degraded (15.630%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:11.350 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:11 smithi098 bash[79516]: cluster 2023-09-23T19:04:10.492730+0000 mon.a (mon.0) 1898 : cluster 3 Health check update: Degraded data redundancy: 20906/133755 objects degraded (15.630%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:11.373 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:11.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:11.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:11.479 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:11.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:11 smithi173 bash[59422]: cluster 2023-09-23T19:04:10.492730+0000 mon.a (mon.0) 1898 : cluster 3 Health check update: Degraded data redundancy: 20906/133755 objects degraded (15.630%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:11.531 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:11.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:11.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:11.636 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:11.663 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:11.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:11.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:11.766 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:11.794 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:11.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:11.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:11.896 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:11.921 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:11.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:11.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:12.024 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:12.055 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:12.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:12.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:12.161 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:12.198 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:12.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:12.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:12.306 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:12.332 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:12 smithi098 bash[76817]: cluster 2023-09-23T19:04:10.518693+0000 mgr.y (mgr.26054) 1390 : cluster 0 pgmap v1527: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 20367/130152 objects degraded (15.649%); 14568/130152 objects misplaced (11.193%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T19:04:12.332 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:12 smithi098 bash[76817]: audit 2023-09-23T19:04:11.768532+0000 mon.a (mon.0) 1899 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:04:12.333 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:12 smithi098 bash[76817]: audit 2023-09-23T19:04:11.782537+0000 mon.a (mon.0) 1900 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:04:12.341 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:12.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:12.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:12.450 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:12.478 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:12.481 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:12 smithi173 bash[59422]: cluster 2023-09-23T19:04:10.518693+0000 mgr.y (mgr.26054) 1390 : cluster 0 pgmap v1527: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 20367/130152 objects degraded (15.649%); 14568/130152 objects misplaced (11.193%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T19:04:12.481 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:12 smithi173 bash[59422]: audit 2023-09-23T19:04:11.768532+0000 mon.a (mon.0) 1899 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:04:12.481 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:12 smithi173 bash[59422]: audit 2023-09-23T19:04:11.782537+0000 mon.a (mon.0) 1900 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:04:12.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:12.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:12.583 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:12.621 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:12.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:12.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:12.665 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:12 smithi098 bash[79516]: cluster 2023-09-23T19:04:10.518693+0000 mgr.y (mgr.26054) 1390 : cluster 0 pgmap v1527: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 20367/130152 objects degraded (15.649%); 14568/130152 objects misplaced (11.193%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T19:04:12.666 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:12 smithi098 bash[79516]: audit 2023-09-23T19:04:11.768532+0000 mon.a (mon.0) 1899 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:04:12.666 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:12 smithi098 bash[79516]: audit 2023-09-23T19:04:11.782537+0000 mon.a (mon.0) 1900 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:04:12.724 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:12.755 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:12.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:12.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:12.861 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:12.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:12.894 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:12.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:12.995 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:13.026 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:13.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:13.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:13.129 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:13.156 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:13.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:13.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:13.260 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:13.271 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:13 smithi173 bash[59422]: audit 2023-09-23T19:04:12.505060+0000 mon.a (mon.0) 1901 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:04:13.272 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:13 smithi173 bash[59422]: audit 2023-09-23T19:04:12.506143+0000 mon.a (mon.0) 1902 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:04:13.272 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:13 smithi173 bash[59422]: audit 2023-09-23T19:04:12.516422+0000 mon.a (mon.0) 1903 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:04:13.286 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:13.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:13.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:13.389 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:13.418 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:13.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:13.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:13.522 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:13.551 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:13.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:13.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:13.655 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:13.664 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:13 smithi098 bash[76817]: audit 2023-09-23T19:04:12.505060+0000 mon.a (mon.0) 1901 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:04:13.664 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:13 smithi098 bash[76817]: audit 2023-09-23T19:04:12.506143+0000 mon.a (mon.0) 1902 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:04:13.664 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:13 smithi098 bash[76817]: audit 2023-09-23T19:04:12.516422+0000 mon.a (mon.0) 1903 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:04:13.664 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:13 smithi098 bash[79516]: audit 2023-09-23T19:04:12.505060+0000 mon.a (mon.0) 1901 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:04:13.664 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:13 smithi098 bash[79516]: audit 2023-09-23T19:04:12.506143+0000 mon.a (mon.0) 1902 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:04:13.665 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:13 smithi098 bash[79516]: audit 2023-09-23T19:04:12.516422+0000 mon.a (mon.0) 1903 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:04:13.691 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:13.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:13.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:13.798 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:13.837 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:13.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:13.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:13.940 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:13.957 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:13.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:13.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:13.960 INFO:teuthology.orchestra.run.smithi098.stdout:moved 3 / 267 (1.1236%) 2023-09-23T19:04:13.960 INFO:teuthology.orchestra.run.smithi098.stdout:avg 53.4 2023-09-23T19:04:13.960 INFO:teuthology.orchestra.run.smithi098.stdout:stddev 13.8881 -> 13.8607 (expected baseline 6.53605) 2023-09-23T19:04:13.960 INFO:teuthology.orchestra.run.smithi098.stdout:min osd.2 with 31 -> 30 pgs (0.580524 -> 0.561798 * mean) 2023-09-23T19:04:13.960 INFO:teuthology.orchestra.run.smithi098.stdout:max osd.1 with 47 -> 46 pgs (0.88015 -> 0.861423 * mean) 2023-09-23T19:04:13.961 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T19:04:13.961 INFO:teuthology.orchestra.run.smithi098.stdout:oload 1000 2023-09-23T19:04:13.961 INFO:teuthology.orchestra.run.smithi098.stdout:max_change 1 2023-09-23T19:04:13.961 INFO:teuthology.orchestra.run.smithi098.stdout:max_change_osds 4 2023-09-23T19:04:13.961 INFO:teuthology.orchestra.run.smithi098.stdout:average_utilization 0.0218 2023-09-23T19:04:13.961 INFO:teuthology.orchestra.run.smithi098.stdout:overload_utilization 0.2180 2023-09-23T19:04:13.961 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3 weight 0.8669 -> 1.0000 2023-09-23T19:04:13.963 INFO:teuthology.orchestra.run.smithi098.stderr:no change 2023-09-23T19:04:14.060 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:14.094 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:14.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:14.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:14.199 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:14.225 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:14.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:14.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:14.333 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:14.364 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:14.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:14.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:14.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:14 smithi098 bash[76817]: cluster 2023-09-23T19:04:12.519645+0000 mgr.y (mgr.26054) 1391 : cluster 0 pgmap v1528: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 19372/124290 objects degraded (15.586%); 13995/124290 objects misplaced (11.260%); 848 KiB/s, 13 objects/s recovering 2023-09-23T19:04:14.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:04:14 smithi098 bash[73085]: debug 2023-09-23T19:04:13.950+0000 7f90a56ad700 -1 mgr.server reply reply (1) Operation not permitted no change 2023-09-23T19:04:14.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:14 smithi098 bash[79516]: cluster 2023-09-23T19:04:12.519645+0000 mgr.y (mgr.26054) 1391 : cluster 0 pgmap v1528: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 19372/124290 objects degraded (15.586%); 13995/124290 objects misplaced (11.260%); 848 KiB/s, 13 objects/s recovering 2023-09-23T19:04:14.467 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:14.495 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:14.497 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:14 smithi173 bash[59422]: cluster 2023-09-23T19:04:12.519645+0000 mgr.y (mgr.26054) 1391 : cluster 0 pgmap v1528: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 19372/124290 objects degraded (15.586%); 13995/124290 objects misplaced (11.260%); 848 KiB/s, 13 objects/s recovering 2023-09-23T19:04:14.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:14.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:14.598 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:14.621 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:14.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:14.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:14.724 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:14.760 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:14.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:14.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:14.865 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:14.896 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:14.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:14.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:15.000 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:15.027 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:15.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:15.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:15.134 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:15.176 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:15.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:15.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:15.280 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:15.284 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:15 smithi098 bash[79516]: audit 2023-09-23T19:04:13.956632+0000 mgr.y (mgr.26054) 1392 : audit 0 from='client.35874 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:04:15.285 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:15 smithi098 bash[76817]: audit 2023-09-23T19:04:13.956632+0000 mgr.y (mgr.26054) 1392 : audit 0 from='client.35874 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:04:15.308 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:15.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:15.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:15.411 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:15.438 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:15.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:15.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:15.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:15 smithi173 bash[59422]: audit 2023-09-23T19:04:13.956632+0000 mgr.y (mgr.26054) 1392 : audit 0 from='client.35874 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:04:15.541 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:15.576 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:15.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:15.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:15.685 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:15.719 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:15.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:15.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:15.823 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:15.848 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:15.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:15.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:15.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:15.983 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:15.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:15.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:16.086 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:16.087 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:04:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:04:16] "GET /metrics HTTP/1.1" 200 36645 "" "Prometheus/2.33.4" 2023-09-23T19:04:16.117 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:16.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:16.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:16.224 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:16.253 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:16.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:16.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:16.357 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:16.366 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:16 smithi098 bash[76817]: cluster 2023-09-23T19:04:14.520630+0000 mgr.y (mgr.26054) 1393 : cluster 0 pgmap v1529: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 18932/121041 objects degraded (15.641%); 13722/121041 objects misplaced (11.337%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T19:04:16.366 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:16 smithi098 bash[76817]: cluster 2023-09-23T19:04:15.493934+0000 mon.a (mon.0) 1904 : cluster 3 Health check update: Degraded data redundancy: 18932/121041 objects degraded (15.641%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:16.366 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:16 smithi098 bash[79516]: cluster 2023-09-23T19:04:14.520630+0000 mgr.y (mgr.26054) 1393 : cluster 0 pgmap v1529: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 18932/121041 objects degraded (15.641%); 13722/121041 objects misplaced (11.337%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T19:04:16.366 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:16 smithi098 bash[79516]: cluster 2023-09-23T19:04:15.493934+0000 mon.a (mon.0) 1904 : cluster 3 Health check update: Degraded data redundancy: 18932/121041 objects degraded (15.641%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:16.382 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:16.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:16.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:16.487 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:16.510 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:16.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:16.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:16.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:16 smithi173 bash[59422]: cluster 2023-09-23T19:04:14.520630+0000 mgr.y (mgr.26054) 1393 : cluster 0 pgmap v1529: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 18932/121041 objects degraded (15.641%); 13722/121041 objects misplaced (11.337%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T19:04:16.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:16 smithi173 bash[59422]: cluster 2023-09-23T19:04:15.493934+0000 mon.a (mon.0) 1904 : cluster 3 Health check update: Degraded data redundancy: 18932/121041 objects degraded (15.641%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:16.616 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:16.660 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:16.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:16.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:16.764 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:16.792 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:16.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:16.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:16.895 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:16.931 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:16.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:16.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:17.035 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:17.065 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:17.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:17.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:17.168 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:17.201 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:17.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:17.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:17.309 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:17.332 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:17.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:17.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:17.438 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:17.467 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:17.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:17.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:17.577 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:17.616 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:17.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:17.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:17.722 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:17.751 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:17.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:17.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:17.857 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:17.883 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:17.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:17.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:17.990 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:18.025 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:18.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:18.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:18.130 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:18.157 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:18.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:18.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:18.261 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:18.289 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:18.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:18.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:18.392 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:18.402 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:18 smithi173 bash[59422]: cluster 2023-09-23T19:04:16.521528+0000 mgr.y (mgr.26054) 1394 : cluster 0 pgmap v1530: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 18932/121041 objects degraded (15.641%); 13722/121041 objects misplaced (11.337%); 848 KiB/s, 13 objects/s recovering 2023-09-23T19:04:18.420 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:18.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:18.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:18.524 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:18.533 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:18 smithi098 bash[79516]: cluster 2023-09-23T19:04:16.521528+0000 mgr.y (mgr.26054) 1394 : cluster 0 pgmap v1530: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 18932/121041 objects degraded (15.641%); 13722/121041 objects misplaced (11.337%); 848 KiB/s, 13 objects/s recovering 2023-09-23T19:04:18.533 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:18 smithi098 bash[76817]: cluster 2023-09-23T19:04:16.521528+0000 mgr.y (mgr.26054) 1394 : cluster 0 pgmap v1530: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 18932/121041 objects degraded (15.641%); 13722/121041 objects misplaced (11.337%); 848 KiB/s, 13 objects/s recovering 2023-09-23T19:04:18.564 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:18.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:18.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:18.670 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:18.697 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:18.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:18.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:18.801 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:18.839 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:18.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:18.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:18.946 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:18.971 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:18.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:18.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:19.074 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:19.108 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:19.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:19.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:19.214 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:19.242 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:19.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:19.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:19.350 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:19.373 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:19.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:19.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:19.480 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:19.503 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:19.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:19.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:19.613 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:19.649 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:19.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:19.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:19.752 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:19.790 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:19.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:19.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:19.896 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:19.927 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:19.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:19.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:20.031 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:20.055 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:20.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:20.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:20.158 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:20.199 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:20.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:20.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:20.307 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:20.338 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:20.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:20.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:20.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:20 smithi098 bash[79516]: cluster 2023-09-23T19:04:18.522565+0000 mgr.y (mgr.26054) 1395 : cluster 0 pgmap v1531: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 668 op/s; 18048/115590 objects degraded (15.614%); 13175/115590 objects misplaced (11.398%); 848 KiB/s, 13 objects/s recovering 2023-09-23T19:04:20.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:20 smithi098 bash[76817]: cluster 2023-09-23T19:04:18.522565+0000 mgr.y (mgr.26054) 1395 : cluster 0 pgmap v1531: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 668 op/s; 18048/115590 objects degraded (15.614%); 13175/115590 objects misplaced (11.398%); 848 KiB/s, 13 objects/s recovering 2023-09-23T19:04:20.444 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:20.476 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:20.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:20.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:20.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:20 smithi173 bash[59422]: cluster 2023-09-23T19:04:18.522565+0000 mgr.y (mgr.26054) 1395 : cluster 0 pgmap v1531: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 668 op/s; 18048/115590 objects degraded (15.614%); 13175/115590 objects misplaced (11.398%); 848 KiB/s, 13 objects/s recovering 2023-09-23T19:04:20.582 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:20.606 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:20.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:20.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:20.710 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:20.740 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:20.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:20.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:20.846 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:20.874 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:20.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:20.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:20.978 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:21.003 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:21.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:21.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:21.107 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:21.137 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:21.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:21.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:21.242 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:21.271 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:21.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:21.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:21.375 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:21.402 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:21.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:21.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:21.512 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:21.520 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:21 smithi098 bash[76817]: cluster 2023-09-23T19:04:20.495679+0000 mon.a (mon.0) 1905 : cluster 3 Health check update: Degraded data redundancy: 18048/115590 objects degraded (15.614%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:21.520 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:21 smithi098 bash[79516]: cluster 2023-09-23T19:04:20.495679+0000 mon.a (mon.0) 1905 : cluster 3 Health check update: Degraded data redundancy: 18048/115590 objects degraded (15.614%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:21.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:21 smithi173 bash[59422]: cluster 2023-09-23T19:04:20.495679+0000 mon.a (mon.0) 1905 : cluster 3 Health check update: Degraded data redundancy: 18048/115590 objects degraded (15.614%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:21.543 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:21.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:21.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:21.649 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:21.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:21.686 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:21.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:21.787 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:21.814 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:21.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:21.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:21.851 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T19:04:21.852 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-09-23T19:04:21.921 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:21.980 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:21.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:21.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:22.084 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:22.125 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:22.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:22.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:22.230 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:22.280 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:22.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:22.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:22.387 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:22.394 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:22 smithi098 bash[76817]: cluster 2023-09-23T19:04:20.523504+0000 mgr.y (mgr.26054) 1396 : cluster 0 pgmap v1532: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 604 op/s; 17552/111996 objects degraded (15.672%); 12874/111996 objects misplaced (11.495%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T19:04:22.395 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:22 smithi098 bash[79516]: cluster 2023-09-23T19:04:20.523504+0000 mgr.y (mgr.26054) 1396 : cluster 0 pgmap v1532: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 604 op/s; 17552/111996 objects degraded (15.672%); 12874/111996 objects misplaced (11.495%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T19:04:22.415 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:22.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:22.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:22.518 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:22.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:22 smithi173 bash[59422]: cluster 2023-09-23T19:04:20.523504+0000 mgr.y (mgr.26054) 1396 : cluster 0 pgmap v1532: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 604 op/s; 17552/111996 objects degraded (15.672%); 12874/111996 objects misplaced (11.495%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T19:04:22.549 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:22.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:22.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:22.651 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:22.680 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:22.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:22.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:22.784 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:22.816 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:22.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:22.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:22.923 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:22.957 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:22.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:22.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:23.060 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:23.088 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:23.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:23.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:23.193 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:23.220 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:23.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:23.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:23.327 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:23.362 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:23.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:23.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:23.466 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:23.494 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:23.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:23.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:23.598 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:23.638 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:23.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:23.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:23.744 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:23.769 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:23.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:23.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:23.872 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:23.901 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:23.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:23.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:24.004 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:24.030 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:24.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:24.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:24.142 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:24.171 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:24.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:24.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:24.276 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:24.277 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:24 smithi098 bash[76817]: cluster 2023-09-23T19:04:22.524606+0000 mgr.y (mgr.26054) 1397 : cluster 0 pgmap v1533: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 16605/106269 objects degraded (15.625%); 12249/106269 objects misplaced (11.526%); 858 KiB/s, 13 objects/s recovering 2023-09-23T19:04:24.278 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:24 smithi098 bash[79516]: cluster 2023-09-23T19:04:22.524606+0000 mgr.y (mgr.26054) 1397 : cluster 0 pgmap v1533: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 16605/106269 objects degraded (15.625%); 12249/106269 objects misplaced (11.526%); 858 KiB/s, 13 objects/s recovering 2023-09-23T19:04:24.305 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:24.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:24.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:24.409 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:24.420 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:24 smithi173 bash[59422]: cluster 2023-09-23T19:04:22.524606+0000 mgr.y (mgr.26054) 1397 : cluster 0 pgmap v1533: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 16605/106269 objects degraded (15.625%); 12249/106269 objects misplaced (11.526%); 858 KiB/s, 13 objects/s recovering 2023-09-23T19:04:24.438 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:24.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:24.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:24.540 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:24.565 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:24.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:24.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:24.668 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:24.694 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:24.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:24.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:24.797 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:24.835 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:24.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:24.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:24.939 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:24.965 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:24.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:24.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:25.070 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:25.102 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:25.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:25.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:25.207 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:25.244 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:25.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:25.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:25.351 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:25.360 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:25 smithi098 bash[79516]: audit 2023-09-23T19:04:24.644244+0000 mon.a (mon.0) 1906 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:04:25.360 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:25 smithi098 bash[76817]: audit 2023-09-23T19:04:24.644244+0000 mon.a (mon.0) 1906 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:04:25.388 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:25.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:25.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:25.494 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:25.520 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:25.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:25.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:25.624 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:25.654 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:25.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:25.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:25.760 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:25.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:25 smithi173 bash[59422]: audit 2023-09-23T19:04:24.644244+0000 mon.a (mon.0) 1906 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:04:25.787 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:25.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:25.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:25.894 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:25.925 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:25.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:25.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:26.028 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:26.059 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:26.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:26.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:26.164 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:26.189 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:26.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:26.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:26.293 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:26.294 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:26 smithi098 bash[76817]: cluster 2023-09-23T19:04:24.525327+0000 mgr.y (mgr.26054) 1398 : cluster 0 pgmap v1534: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 16143/102996 objects degraded (15.673%); 11966/102996 objects misplaced (11.618%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T19:04:26.294 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:26 smithi098 bash[76817]: cluster 2023-09-23T19:04:25.502006+0000 mon.a (mon.0) 1907 : cluster 3 Health check update: Degraded data redundancy: 16143/102996 objects degraded (15.673%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:26.294 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:04:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:04:26] "GET /metrics HTTP/1.1" 200 36649 "" "Prometheus/2.33.4" 2023-09-23T19:04:26.294 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:26 smithi098 bash[79516]: cluster 2023-09-23T19:04:24.525327+0000 mgr.y (mgr.26054) 1398 : cluster 0 pgmap v1534: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 16143/102996 objects degraded (15.673%); 11966/102996 objects misplaced (11.618%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T19:04:26.295 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:26 smithi098 bash[79516]: cluster 2023-09-23T19:04:25.502006+0000 mon.a (mon.0) 1907 : cluster 3 Health check update: Degraded data redundancy: 16143/102996 objects degraded (15.673%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:26.317 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:26.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:26.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:26.421 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:26.456 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:26.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:26.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:26.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:26 smithi173 bash[59422]: cluster 2023-09-23T19:04:24.525327+0000 mgr.y (mgr.26054) 1398 : cluster 0 pgmap v1534: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 16143/102996 objects degraded (15.673%); 11966/102996 objects misplaced (11.618%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T19:04:26.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:26 smithi173 bash[59422]: cluster 2023-09-23T19:04:25.502006+0000 mon.a (mon.0) 1907 : cluster 3 Health check update: Degraded data redundancy: 16143/102996 objects degraded (15.673%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:26.560 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:26.588 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:26.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:26.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:26.692 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:26.711 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:26.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:26.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:26.813 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:26.844 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:26.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:26.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:26.947 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:26.964 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:26.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:26.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:27.067 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:27.096 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:27.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:27.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:27.200 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:27.222 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:27.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:27.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:27.326 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:27.354 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:27.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:27.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:27.458 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:27.485 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:27.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:27.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:27.589 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:27.626 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:27.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:27.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:27.731 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:27.770 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:27.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:27.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:27.873 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:27.908 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:27.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:27.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:28.013 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:28.037 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:28.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:28.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:28.139 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:28.170 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:28.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:28.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:28.273 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:28.296 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:28.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:28.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:28.399 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:28.429 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:28.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:28.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:28.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:28 smithi173 bash[59422]: cluster 2023-09-23T19:04:26.526073+0000 mgr.y (mgr.26054) 1399 : cluster 0 pgmap v1535: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 16143/102996 objects degraded (15.673%); 11966/102996 objects misplaced (11.618%); 901 KiB/s, 14 objects/s recovering 2023-09-23T19:04:28.532 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:28.541 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:28 smithi098 bash[76817]: cluster 2023-09-23T19:04:26.526073+0000 mgr.y (mgr.26054) 1399 : cluster 0 pgmap v1535: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 16143/102996 objects degraded (15.673%); 11966/102996 objects misplaced (11.618%); 901 KiB/s, 14 objects/s recovering 2023-09-23T19:04:28.541 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:28 smithi098 bash[79516]: cluster 2023-09-23T19:04:26.526073+0000 mgr.y (mgr.26054) 1399 : cluster 0 pgmap v1535: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 16143/102996 objects degraded (15.673%); 11966/102996 objects misplaced (11.618%); 901 KiB/s, 14 objects/s recovering 2023-09-23T19:04:28.565 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:28.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:28.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:28.668 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:28.692 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:28.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:28.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:28.796 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:28.829 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:28.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:28.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:28.939 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:28.957 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:28.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:28.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:29.059 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:29.094 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:29.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:29.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:29.200 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:29.235 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:29.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:29.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:29.339 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:29.398 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:29.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:29.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:29.488 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T19:04:29.501 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:29.528 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:29.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:29.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:29.541 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T19:04:29.593 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T19:04:29.624 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T19:04:29.635 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:29.658 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T19:04:29.673 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:29.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:29.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:29.716 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T19:04:29.721 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:04:29.721 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:04:29.721 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:04:29.721 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:04:29.721 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:04:29.721 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T19:04:29.721 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:04:29.721 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T19:04:29.780 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:29.805 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:29.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:29.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:29.882 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:04:29.909 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:29.949 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:29.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:29.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:30.054 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:30.087 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:30.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:30.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:30.191 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:30.235 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:30.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:30.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:30.342 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:30.388 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:30.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:30.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:30.497 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:30.505 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:30 smithi098 bash[79516]: cluster 2023-09-23T19:04:28.527095+0000 mgr.y (mgr.26054) 1400 : cluster 0 pgmap v1536: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 664 op/s; 15128/97122 objects degraded (15.576%); 11439/97122 objects misplaced (11.778%); 901 KiB/s, 14 objects/s recovering 2023-09-23T19:04:30.505 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:30 smithi098 bash[79516]: audit 2023-09-23T19:04:29.433009+0000 mon.a (mon.0) 1908 : audit 0 from='client.? 172.21.15.98:0/611065534' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:04:30.505 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:30 smithi098 bash[76817]: cluster 2023-09-23T19:04:28.527095+0000 mgr.y (mgr.26054) 1400 : cluster 0 pgmap v1536: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 664 op/s; 15128/97122 objects degraded (15.576%); 11439/97122 objects misplaced (11.778%); 901 KiB/s, 14 objects/s recovering 2023-09-23T19:04:30.505 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:30 smithi098 bash[76817]: audit 2023-09-23T19:04:29.433009+0000 mon.a (mon.0) 1908 : audit 0 from='client.? 172.21.15.98:0/611065534' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:04:30.542 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:30.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:30.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:30.648 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:30.697 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:30.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:30.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:30.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:30 smithi173 bash[59422]: cluster 2023-09-23T19:04:28.527095+0000 mgr.y (mgr.26054) 1400 : cluster 0 pgmap v1536: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 2.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 664 op/s; 15128/97122 objects degraded (15.576%); 11439/97122 objects misplaced (11.778%); 901 KiB/s, 14 objects/s recovering 2023-09-23T19:04:30.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:30 smithi173 bash[59422]: audit 2023-09-23T19:04:29.433009+0000 mon.a (mon.0) 1908 : audit 0 from='client.? 172.21.15.98:0/611065534' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:04:30.801 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:30.823 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:30.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:30.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:30.927 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:30.951 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:30.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:30.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:31.057 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:31.084 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:31.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:31.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:31.188 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:31.216 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:31.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:31.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:31.319 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:31.344 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:31.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:31.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:31.446 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:31.454 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:31 smithi098 bash[79516]: cluster 2023-09-23T19:04:30.506365+0000 mon.a (mon.0) 1909 : cluster 3 Health check update: Degraded data redundancy: 15128/97122 objects degraded (15.576%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:31.454 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:31 smithi098 bash[79516]: audit 2023-09-23T19:04:30.787763+0000 mon.a (mon.0) 1910 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:04:31.455 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:31 smithi098 bash[79516]: audit 2023-09-23T19:04:30.789059+0000 mon.b (mon.1) 28 : audit 1 from='client.? 172.21.15.98:0/2109707264' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:04:31.455 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:31 smithi098 bash[76817]: cluster 2023-09-23T19:04:30.506365+0000 mon.a (mon.0) 1909 : cluster 3 Health check update: Degraded data redundancy: 15128/97122 objects degraded (15.576%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:31.455 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:31 smithi098 bash[76817]: audit 2023-09-23T19:04:30.787763+0000 mon.a (mon.0) 1910 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:04:31.455 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:31 smithi098 bash[76817]: audit 2023-09-23T19:04:30.789059+0000 mon.b (mon.1) 28 : audit 1 from='client.? 172.21.15.98:0/2109707264' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:04:31.483 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:31.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:31.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:31.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:31 smithi173 bash[59422]: cluster 2023-09-23T19:04:30.506365+0000 mon.a (mon.0) 1909 : cluster 3 Health check update: Degraded data redundancy: 15128/97122 objects degraded (15.576%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:31.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:31 smithi173 bash[59422]: audit 2023-09-23T19:04:30.787763+0000 mon.a (mon.0) 1910 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:04:31.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:31 smithi173 bash[59422]: audit 2023-09-23T19:04:30.789059+0000 mon.b (mon.1) 28 : audit 1 from='client.? 172.21.15.98:0/2109707264' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:04:31.543 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is unset 2023-09-23T19:04:31.587 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:31.612 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:31.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:31.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:31.716 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:31.752 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:31.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:31.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:31.859 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:31.887 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:31.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:31.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:31.990 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:32.022 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:32.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:32.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:32.130 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:32.153 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:32.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:32.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:32.255 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:32.282 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:32.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:32.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:32.387 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:32.429 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:32.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:32.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:32.534 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:32.543 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:32 smithi098 bash[76817]: cluster 2023-09-23T19:04:30.527897+0000 mgr.y (mgr.26054) 1401 : cluster 0 pgmap v1537: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 604 op/s; 14642/93807 objects degraded (15.609%); 11162/93807 objects misplaced (11.899%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T19:04:32.544 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:32 smithi098 bash[76817]: cluster 2023-09-23T19:04:31.512549+0000 mon.a (mon.0) 1911 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:04:32.544 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:32 smithi098 bash[76817]: audit 2023-09-23T19:04:31.525449+0000 mon.a (mon.0) 1912 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:04:32.544 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:32 smithi098 bash[76817]: cluster 2023-09-23T19:04:31.525475+0000 osd.4 (osd.4) 64 : cluster 3 failed to encode map e334 with expected crc 2023-09-23T19:04:32.544 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:32 smithi098 bash[76817]: cluster 2023-09-23T19:04:31.525543+0000 mon.a (mon.0) 1913 : cluster 0 osdmap e334: 8 total, 6 up, 7 in 2023-09-23T19:04:32.544 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:32 smithi098 bash[76817]: cluster 2023-09-23T19:04:31.531862+0000 osd.6 (osd.6) 316 : cluster 3 failed to encode map e334 with expected crc 2023-09-23T19:04:32.544 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:32 smithi098 bash[79516]: cluster 2023-09-23T19:04:30.527897+0000 mgr.y (mgr.26054) 1401 : cluster 0 pgmap v1537: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 604 op/s; 14642/93807 objects degraded (15.609%); 11162/93807 objects misplaced (11.899%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T19:04:32.544 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:32 smithi098 bash[79516]: cluster 2023-09-23T19:04:31.512549+0000 mon.a (mon.0) 1911 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:04:32.544 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:32 smithi098 bash[79516]: audit 2023-09-23T19:04:31.525449+0000 mon.a (mon.0) 1912 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:04:32.544 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:32 smithi098 bash[79516]: cluster 2023-09-23T19:04:31.525475+0000 osd.4 (osd.4) 64 : cluster 3 failed to encode map e334 with expected crc 2023-09-23T19:04:32.545 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:32 smithi098 bash[79516]: cluster 2023-09-23T19:04:31.525543+0000 mon.a (mon.0) 1913 : cluster 0 osdmap e334: 8 total, 6 up, 7 in 2023-09-23T19:04:32.545 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:32 smithi098 bash[79516]: cluster 2023-09-23T19:04:31.531862+0000 osd.6 (osd.6) 316 : cluster 3 failed to encode map e334 with expected crc 2023-09-23T19:04:32.566 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:32.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:32.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:32.672 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:32.682 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:32 smithi173 bash[59422]: cluster 2023-09-23T19:04:30.527897+0000 mgr.y (mgr.26054) 1401 : cluster 0 pgmap v1537: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 604 op/s; 14642/93807 objects degraded (15.609%); 11162/93807 objects misplaced (11.899%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T19:04:32.682 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:32 smithi173 bash[59422]: cluster 2023-09-23T19:04:31.512549+0000 mon.a (mon.0) 1911 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:04:32.682 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:32 smithi173 bash[59422]: audit 2023-09-23T19:04:31.525449+0000 mon.a (mon.0) 1912 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:04:32.682 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:32 smithi173 bash[59422]: cluster 2023-09-23T19:04:31.525475+0000 osd.4 (osd.4) 64 : cluster 3 failed to encode map e334 with expected crc 2023-09-23T19:04:32.682 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:32 smithi173 bash[59422]: cluster 2023-09-23T19:04:31.525543+0000 mon.a (mon.0) 1913 : cluster 0 osdmap e334: 8 total, 6 up, 7 in 2023-09-23T19:04:32.683 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:32 smithi173 bash[59422]: cluster 2023-09-23T19:04:31.531862+0000 osd.6 (osd.6) 316 : cluster 3 failed to encode map e334 with expected crc 2023-09-23T19:04:32.700 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:32.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:32.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:32.803 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:32.829 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:32.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:32.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:32.932 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:32.975 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:32.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:32.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:33.079 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:33.114 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:33.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:33.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:33.216 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:33.257 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:33.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:33.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:33.359 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:33.387 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:33.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:33.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:33.490 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:33.517 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:33.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:33.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:33.621 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:33.645 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:33.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:33.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:33.751 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:33.783 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:33.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:33.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:33.886 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:33.918 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:33.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:33.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:34.023 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:34.049 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:34.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:34.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:34.152 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:34.176 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:34.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:34.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:34.280 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:34.312 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:34.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:34.317 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:34 smithi098 bash[76817]: cluster 2023-09-23T19:04:32.528860+0000 mgr.y (mgr.26054) 1402 : cluster 0 pgmap v1539: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 13762/88596 objects degraded (15.533%); 10630/88596 objects misplaced (11.998%); 1024 KiB/s, 15 objects/s recovering 2023-09-23T19:04:34.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:34.418 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:34.440 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:34.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:34.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:34.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:34 smithi173 bash[59422]: cluster 2023-09-23T19:04:32.528860+0000 mgr.y (mgr.26054) 1402 : cluster 0 pgmap v1539: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 13762/88596 objects degraded (15.533%); 10630/88596 objects misplaced (11.998%); 1024 KiB/s, 15 objects/s recovering 2023-09-23T19:04:34.545 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:34.553 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:34 smithi098 bash[79516]: cluster 2023-09-23T19:04:32.528860+0000 mgr.y (mgr.26054) 1402 : cluster 0 pgmap v1539: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 13762/88596 objects degraded (15.533%); 10630/88596 objects misplaced (11.998%); 1024 KiB/s, 15 objects/s recovering 2023-09-23T19:04:34.580 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:34.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:34.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:34.687 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:34.718 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:34.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:34.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:34.822 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:34.864 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:34.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:34.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:34.967 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:34.995 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:34.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:34.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:35.098 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:35.125 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:35.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:35.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:35.231 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:35.266 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:35.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:35.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:35.371 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:35.403 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:35.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:35.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:35.437 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T19:04:35.437 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:04:35.512 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:35.540 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:35.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:35.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:35.644 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:35.669 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:35.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:35.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:35.772 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:35.800 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:35.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:35.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:35.910 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:35.932 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:35.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:35.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:36.034 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:36.060 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:36.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:36.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:36.165 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:36.193 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:36.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:36.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:36.299 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:36.321 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:04:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:04:36] "GET /metrics HTTP/1.1" 200 36649 "" "Prometheus/2.33.4" 2023-09-23T19:04:36.329 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:36.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:36.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:36.343 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:04:36.343 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:04:36.432 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:36.458 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:36.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:36.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:36.561 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:36.567 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:36 smithi098 bash[79516]: cluster 2023-09-23T19:04:34.529684+0000 mgr.y (mgr.26054) 1403 : cluster 0 pgmap v1540: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 13275/85092 objects degraded (15.601%); 10338/85092 objects misplaced (12.149%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:04:36.567 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:36 smithi098 bash[79516]: cluster 2023-09-23T19:04:35.507517+0000 mon.a (mon.0) 1914 : cluster 3 Health check update: Degraded data redundancy: 13275/85092 objects degraded (15.601%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:36.567 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:36 smithi098 bash[76817]: cluster 2023-09-23T19:04:34.529684+0000 mgr.y (mgr.26054) 1403 : cluster 0 pgmap v1540: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 13275/85092 objects degraded (15.601%); 10338/85092 objects misplaced (12.149%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:04:36.567 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:36 smithi098 bash[76817]: cluster 2023-09-23T19:04:35.507517+0000 mon.a (mon.0) 1914 : cluster 3 Health check update: Degraded data redundancy: 13275/85092 objects degraded (15.601%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:36.582 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:36.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:36.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:36.685 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:36.721 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:36.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:36.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:36.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:36 smithi173 bash[59422]: cluster 2023-09-23T19:04:34.529684+0000 mgr.y (mgr.26054) 1403 : cluster 0 pgmap v1540: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 13275/85092 objects degraded (15.601%); 10338/85092 objects misplaced (12.149%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:04:36.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:36 smithi173 bash[59422]: cluster 2023-09-23T19:04:35.507517+0000 mon.a (mon.0) 1914 : cluster 3 Health check update: Degraded data redundancy: 13275/85092 objects degraded (15.601%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:36.828 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:36.870 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:36.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:36.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:36.974 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:37.009 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:37.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:37.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:37.112 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:37.140 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:37.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:37.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:37.245 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:37.277 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:37.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:37.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:37.380 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:37.424 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:37.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:37.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:37.531 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:37.560 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:37.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:37.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:37.667 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:37.687 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:37.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:37.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:37.790 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:37.830 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:37.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:37.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:37.939 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:37.966 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:37.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:37.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:38.069 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:38.095 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:38.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:38.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:38.199 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:38.223 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:38.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:38.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:38.328 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:38.344 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:04:38.357 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:38.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:38.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:38.460 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:38.473 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:38 smithi173 bash[59422]: cluster 2023-09-23T19:04:36.530340+0000 mgr.y (mgr.26054) 1404 : cluster 0 pgmap v1541: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 13275/85092 objects degraded (15.601%); 10338/85092 objects misplaced (12.149%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:04:38.491 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:38.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:38.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:38.594 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:38.605 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:38 smithi098 bash[76817]: cluster 2023-09-23T19:04:36.530340+0000 mgr.y (mgr.26054) 1404 : cluster 0 pgmap v1541: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 13275/85092 objects degraded (15.601%); 10338/85092 objects misplaced (12.149%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:04:38.605 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:38 smithi098 bash[79516]: cluster 2023-09-23T19:04:36.530340+0000 mgr.y (mgr.26054) 1404 : cluster 0 pgmap v1541: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 13275/85092 objects degraded (15.601%); 10338/85092 objects misplaced (12.149%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:04:38.643 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:38.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:38.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:38.751 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:38.781 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:38.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:38.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:38.885 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:38.913 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:38.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:38.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:39.022 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:39.046 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:39.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:39.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:39.149 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:39.184 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:39.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:39.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:39.289 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:39.319 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:39.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:39.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:39.422 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:39.442 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:39.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:39.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:39.544 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:39.578 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:39.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:39.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:39.681 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:39.709 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:39.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:39.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:39.812 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:39.840 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T19:04:39.854 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:39.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:39.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:39.959 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:39.993 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:39.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:39.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:40.098 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:40.126 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:40.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:40.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:40.231 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:40.259 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:40.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:40.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:40.362 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:40.395 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:40.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:40.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:40.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:40 smithi098 bash[76817]: cluster 2023-09-23T19:04:38.531380+0000 mgr.y (mgr.26054) 1405 : cluster 0 pgmap v1542: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.6 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 12333/79539 objects degraded (15.506%); 9786/79539 objects misplaced (12.303%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:04:40.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:40 smithi098 bash[76817]: audit 2023-09-23T19:04:39.644544+0000 mon.a (mon.0) 1915 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:04:40.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:40 smithi098 bash[79516]: cluster 2023-09-23T19:04:38.531380+0000 mgr.y (mgr.26054) 1405 : cluster 0 pgmap v1542: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.6 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 12333/79539 objects degraded (15.506%); 9786/79539 objects misplaced (12.303%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:04:40.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:40 smithi098 bash[79516]: audit 2023-09-23T19:04:39.644544+0000 mon.a (mon.0) 1915 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:04:40.455 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:04:40.499 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:40.510 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:40 smithi173 bash[59422]: cluster 2023-09-23T19:04:38.531380+0000 mgr.y (mgr.26054) 1405 : cluster 0 pgmap v1542: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.6 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 12333/79539 objects degraded (15.506%); 9786/79539 objects misplaced (12.303%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:04:40.510 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:40 smithi173 bash[59422]: audit 2023-09-23T19:04:39.644544+0000 mon.a (mon.0) 1915 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:04:40.531 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:40.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:40.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:40.634 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:40.670 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:40.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:40.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:40.776 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:40.801 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:40.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:40.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:40.904 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:40.953 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:40.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:40.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:41.056 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:41.080 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:41.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:41.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:41.184 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:41.203 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:41.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:41.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:41.307 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:41.339 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:41.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:41.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:41.442 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:41.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:41 smithi098 bash[79516]: cluster 2023-09-23T19:04:40.508720+0000 mon.a (mon.0) 1916 : cluster 3 Health check update: Degraded data redundancy: 12333/79539 objects degraded (15.506%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:41.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:41 smithi098 bash[76817]: cluster 2023-09-23T19:04:40.508720+0000 mon.a (mon.0) 1916 : cluster 3 Health check update: Degraded data redundancy: 12333/79539 objects degraded (15.506%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:41.470 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:41.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:41.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:41.574 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:41.602 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:41.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:41.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:41.706 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:41.718 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:41 smithi173 bash[59422]: cluster 2023-09-23T19:04:40.508720+0000 mon.a (mon.0) 1916 : cluster 3 Health check update: Degraded data redundancy: 12333/79539 objects degraded (15.506%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:41.738 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:41.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:41.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:41.841 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:41.868 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:41.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:41.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:41.972 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:41.996 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:41.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:41.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:42.099 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:42.123 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:42.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:42.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:42.226 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:42.263 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:42.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:42.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:42.369 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:42.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:42 smithi173 bash[59422]: cluster 2023-09-23T19:04:40.532298+0000 mgr.y (mgr.26054) 1406 : cluster 0 pgmap v1543: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 11883/76293 objects degraded (15.575%); 9490/76293 objects misplaced (12.439%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T19:04:42.402 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:42.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:42.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:42.506 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:42.533 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:42.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:42.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:42.637 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:42.667 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:42.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:42.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:42.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:42 smithi098 bash[79516]: cluster 2023-09-23T19:04:40.532298+0000 mgr.y (mgr.26054) 1406 : cluster 0 pgmap v1543: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 11883/76293 objects degraded (15.575%); 9490/76293 objects misplaced (12.439%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T19:04:42.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:42 smithi098 bash[76817]: cluster 2023-09-23T19:04:40.532298+0000 mgr.y (mgr.26054) 1406 : cluster 0 pgmap v1543: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 11883/76293 objects degraded (15.575%); 9490/76293 objects misplaced (12.439%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T19:04:42.770 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:42.808 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:42.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:42.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:42.917 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:42.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:42.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:42.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:43.047 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:43.076 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:43.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:43.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:43.182 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:43.209 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:43.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:43.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:43.312 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:43.343 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:43.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:43.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:43.449 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:43.482 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:43.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:43.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:43.585 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:43.616 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:43.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:43.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:43.720 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:43.743 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:43.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:43.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:43.846 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:43.879 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:43.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:43.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:43.983 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:44.024 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:44.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:44.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:44.129 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:44.162 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:44.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:44.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:44.215 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:04:44.215 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:04:44.266 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:44.307 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:44.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:44.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:44.418 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:44.427 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:44 smithi098 bash[76817]: cluster 2023-09-23T19:04:42.533266+0000 mgr.y (mgr.26054) 1407 : cluster 0 pgmap v1544: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 700 op/s; 10963/70662 objects degraded (15.515%); 8906/70662 objects misplaced (12.604%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:04:44.428 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:44 smithi098 bash[79516]: cluster 2023-09-23T19:04:42.533266+0000 mgr.y (mgr.26054) 1407 : cluster 0 pgmap v1544: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 700 op/s; 10963/70662 objects degraded (15.515%); 8906/70662 objects misplaced (12.604%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:04:44.454 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:44.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:44.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:44.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:44 smithi173 bash[59422]: cluster 2023-09-23T19:04:42.533266+0000 mgr.y (mgr.26054) 1407 : cluster 0 pgmap v1544: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 700 op/s; 10963/70662 objects degraded (15.515%); 8906/70662 objects misplaced (12.604%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:04:44.558 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:44.588 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:44.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:44.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:44.694 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:44.714 INFO:teuthology.orchestra.run.smithi098.stdout:moved 17 / 267 (6.36704%) 2023-09-23T19:04:44.714 INFO:teuthology.orchestra.run.smithi098.stdout:avg 53.4 2023-09-23T19:04:44.714 INFO:teuthology.orchestra.run.smithi098.stdout:stddev 13.8881 -> 14.716 (expected baseline 6.53605) 2023-09-23T19:04:44.714 INFO:teuthology.orchestra.run.smithi098.stdout:min osd.2 with 31 -> 35 pgs (0.580524 -> 0.655431 * mean) 2023-09-23T19:04:44.714 INFO:teuthology.orchestra.run.smithi098.stdout:max osd.1 with 47 -> 48 pgs (0.88015 -> 0.898876 * mean) 2023-09-23T19:04:44.714 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T19:04:44.715 INFO:teuthology.orchestra.run.smithi098.stdout:oload 110 2023-09-23T19:04:44.715 INFO:teuthology.orchestra.run.smithi098.stdout:max_change 1 2023-09-23T19:04:44.715 INFO:teuthology.orchestra.run.smithi098.stdout:max_change_osds 4 2023-09-23T19:04:44.715 INFO:teuthology.orchestra.run.smithi098.stdout:average_utilization 0.0176 2023-09-23T19:04:44.715 INFO:teuthology.orchestra.run.smithi098.stdout:overload_utilization 0.0193 2023-09-23T19:04:44.715 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0 weight 1.0000 -> 0.7269 2023-09-23T19:04:44.715 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3 weight 0.8669 -> 1.0000 2023-09-23T19:04:44.715 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5 weight 0.9098 -> 0.7263 2023-09-23T19:04:44.718 INFO:teuthology.orchestra.run.smithi098.stderr:no change 2023-09-23T19:04:44.727 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:44.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:44.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:44.831 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:44.863 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:44.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:44.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:44.968 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:45.001 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:45.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:45.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:45.104 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:45.113 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:04:44 smithi098 bash[73085]: debug 2023-09-23T19:04:44.710+0000 7f90a56ad700 -1 mgr.server reply reply (3) No such process no change 2023-09-23T19:04:45.152 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:45.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:45.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:45.257 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:45.285 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:45.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:45.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:45.388 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:45.423 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:45.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:45.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:45.528 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:45.557 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:45.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:45.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:45.660 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:45.709 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:45.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:45.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:45.812 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:45.830 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:45.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:45.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:45.932 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:45.950 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:45.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:45.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:46.052 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:46.076 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:46.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:46.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:46.179 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:46.206 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:46.208 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:04:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:04:46] "GET /metrics HTTP/1.1" 200 36647 "" "Prometheus/2.33.4" 2023-09-23T19:04:46.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:46.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:46.310 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:46.334 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:46.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:46.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:46.439 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:46.484 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:46.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:46.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:46.488 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:46 smithi098 bash[76817]: cluster 2023-09-23T19:04:44.533982+0000 mgr.y (mgr.26054) 1408 : cluster 0 pgmap v1545: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 10963/70662 objects degraded (15.515%); 8906/70662 objects misplaced (12.604%); 1018 KiB/s, 15 objects/s recovering 2023-09-23T19:04:46.488 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:46 smithi098 bash[76817]: audit 2023-09-23T19:04:44.710026+0000 mgr.y (mgr.26054) 1409 : audit 0 from='client.45878 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:04:46.488 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:46 smithi098 bash[76817]: cluster 2023-09-23T19:04:45.511718+0000 mon.a (mon.0) 1917 : cluster 3 Health check update: Degraded data redundancy: 10963/70662 objects degraded (15.515%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:46.489 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:46 smithi098 bash[79516]: cluster 2023-09-23T19:04:44.533982+0000 mgr.y (mgr.26054) 1408 : cluster 0 pgmap v1545: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 10963/70662 objects degraded (15.515%); 8906/70662 objects misplaced (12.604%); 1018 KiB/s, 15 objects/s recovering 2023-09-23T19:04:46.489 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:46 smithi098 bash[79516]: audit 2023-09-23T19:04:44.710026+0000 mgr.y (mgr.26054) 1409 : audit 0 from='client.45878 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:04:46.489 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:46 smithi098 bash[79516]: cluster 2023-09-23T19:04:45.511718+0000 mon.a (mon.0) 1917 : cluster 3 Health check update: Degraded data redundancy: 10963/70662 objects degraded (15.515%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:46.589 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:46.599 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:46 smithi173 bash[59422]: cluster 2023-09-23T19:04:44.533982+0000 mgr.y (mgr.26054) 1408 : cluster 0 pgmap v1545: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 10963/70662 objects degraded (15.515%); 8906/70662 objects misplaced (12.604%); 1018 KiB/s, 15 objects/s recovering 2023-09-23T19:04:46.599 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:46 smithi173 bash[59422]: audit 2023-09-23T19:04:44.710026+0000 mgr.y (mgr.26054) 1409 : audit 0 from='client.45878 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:04:46.599 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:46 smithi173 bash[59422]: cluster 2023-09-23T19:04:45.511718+0000 mon.a (mon.0) 1917 : cluster 3 Health check update: Degraded data redundancy: 10963/70662 objects degraded (15.515%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:46.614 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:46.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:46.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:46.720 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:46.783 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:46.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:46.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:46.889 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:46.918 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:46.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:46.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:47.021 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:47.047 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:47.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:47.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:47.151 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:47.182 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:47.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:47.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:47.288 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:47.319 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:47.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:47.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:47.424 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:47.454 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:47.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:47.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:47.557 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:47.586 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:47.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:47.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:47.693 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:47.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:47.725 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:47.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:47.825 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:47.855 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:47.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:47.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:47.959 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:47.993 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:47.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:48.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:48.100 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:48.124 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:48.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:48.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:48.229 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:48.256 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:48.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:48.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:48.353 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:48 smithi098 bash[79516]: cluster 2023-09-23T19:04:46.536655+0000 mgr.y (mgr.26054) 1410 : cluster 0 pgmap v1546: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 10476/67122 objects degraded (15.607%); 8645/67122 objects misplaced (12.880%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:04:48.359 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:48.405 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:48.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:48.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:48.509 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:48.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:48 smithi173 bash[59422]: cluster 2023-09-23T19:04:46.536655+0000 mgr.y (mgr.26054) 1410 : cluster 0 pgmap v1546: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 10476/67122 objects degraded (15.607%); 8645/67122 objects misplaced (12.880%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:04:48.533 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:48.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:48.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:48.636 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:48.663 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:48.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:48.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:48.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:48 smithi098 bash[76817]: cluster 2023-09-23T19:04:46.536655+0000 mgr.y (mgr.26054) 1410 : cluster 0 pgmap v1546: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 10476/67122 objects degraded (15.607%); 8645/67122 objects misplaced (12.880%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:04:48.768 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:48.801 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:48.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:48.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:48.909 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:48.938 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:48.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:48.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:49.040 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:49.077 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:49.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:49.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:49.182 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:49.209 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:49.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:49.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:49.311 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:49.336 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:49.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:49.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:49.440 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:49.466 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:49.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:49.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:49.473 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:04:49.570 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:49.595 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:49.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:49.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:49.698 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:49.724 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:49.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:49.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:49.829 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:49.862 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:49.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:49.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:49.971 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:49.995 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:49.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:49.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:50.099 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:50.125 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:50.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:50.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:50.230 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:50.258 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:50.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:50.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:50.364 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:50.382 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:50.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:50.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:50.485 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:50.498 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:50 smithi173 bash[59422]: cluster 2023-09-23T19:04:48.537617+0000 mgr.y (mgr.26054) 1411 : cluster 0 pgmap v1547: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 9550/61473 objects degraded (15.535%); 8060/61473 objects misplaced (13.111%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:04:50.519 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:50.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:50.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:50.623 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:50.649 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:50.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:50.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:50.757 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:50.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:50.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:50.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:50.891 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:50.916 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:50.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:50.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:50.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:50 smithi098 bash[79516]: cluster 2023-09-23T19:04:48.537617+0000 mgr.y (mgr.26054) 1411 : cluster 0 pgmap v1547: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 9550/61473 objects degraded (15.535%); 8060/61473 objects misplaced (13.111%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:04:50.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:50 smithi098 bash[76817]: cluster 2023-09-23T19:04:48.537617+0000 mgr.y (mgr.26054) 1411 : cluster 0 pgmap v1547: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 9550/61473 objects degraded (15.535%); 8060/61473 objects misplaced (13.111%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T19:04:51.019 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:51.039 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:51.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:51.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:51.142 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:51.171 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:51.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:51.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:51.275 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:51.303 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:51.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:51.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:51.398 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:51 smithi098 bash[76817]: cluster 2023-09-23T19:04:50.512977+0000 mon.a (mon.0) 1918 : cluster 3 Health check update: Degraded data redundancy: 9550/61473 objects degraded (15.535%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:51.398 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:51 smithi098 bash[79516]: cluster 2023-09-23T19:04:50.512977+0000 mon.a (mon.0) 1918 : cluster 3 Health check update: Degraded data redundancy: 9550/61473 objects degraded (15.535%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:51.406 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:51.435 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:51.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:51.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:51.540 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:51.569 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:51.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:51.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:51.674 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:51.703 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:51.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:51.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:51.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:51 smithi173 bash[59422]: cluster 2023-09-23T19:04:50.512977+0000 mon.a (mon.0) 1918 : cluster 3 Health check update: Degraded data redundancy: 9550/61473 objects degraded (15.535%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:51.807 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:51.848 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:51.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:51.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:51.950 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:51.990 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:51.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:51.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:52.095 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:52.125 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:52.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:52.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:52.229 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:52.257 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:52.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:52.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:52.360 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:52.390 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:52.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:52.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:52.494 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:52.524 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:52.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:52.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:52.627 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:52.636 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:52 smithi098 bash[76817]: cluster 2023-09-23T19:04:50.538391+0000 mgr.y (mgr.26054) 1412 : cluster 0 pgmap v1548: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.2 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 603 op/s; 9035/57789 objects degraded (15.634%); 7757/57789 objects misplaced (13.423%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T19:04:52.637 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:52 smithi098 bash[79516]: cluster 2023-09-23T19:04:50.538391+0000 mgr.y (mgr.26054) 1412 : cluster 0 pgmap v1548: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.2 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 603 op/s; 9035/57789 objects degraded (15.634%); 7757/57789 objects misplaced (13.423%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T19:04:52.652 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:52.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:52.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:52.756 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:52.769 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:52 smithi173 bash[59422]: cluster 2023-09-23T19:04:50.538391+0000 mgr.y (mgr.26054) 1412 : cluster 0 pgmap v1548: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.2 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 603 op/s; 9035/57789 objects degraded (15.634%); 7757/57789 objects misplaced (13.423%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T19:04:52.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:52.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:52.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:52.889 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:52.918 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:52.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:52.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:53.022 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:53.046 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:53.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:53.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:53.152 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:53.182 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:53.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:53.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:53.285 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:53.314 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:53.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:53.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:53.417 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:53.446 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:53.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:53.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:53.551 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:53.583 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:53.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:53.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:53.689 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:53.717 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:53.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:53.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:53.821 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:53.876 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:53.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:53.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:53.979 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:54.007 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:54.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:54.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:54.114 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:54.126 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 6, 7, 5, 3] out_osds: [4] dead_osds: [7, 5] live_osds: [0, 2, 3, 6, 1, 4] 2023-09-23T19:04:54.126 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:04:54.126 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-09-23T19:04:54.127 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T19:04:54.127 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T19:04:54.127 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-09-23T19:04:54.145 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:54.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:54.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:54.247 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:54.287 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:54.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:54.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:54.392 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:54.404 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:54 smithi173 bash[59422]: cluster 2023-09-23T19:04:52.541044+0000 mgr.y (mgr.26054) 1413 : cluster 0 pgmap v1549: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 672 op/s; 8068/52083 objects degraded (15.491%); 7185/52083 objects misplaced (13.795%); 970 KiB/s, 15 objects/s recovering 2023-09-23T19:04:54.420 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:54.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:54.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:54.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:54 smithi098 bash[76817]: cluster 2023-09-23T19:04:52.541044+0000 mgr.y (mgr.26054) 1413 : cluster 0 pgmap v1549: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 672 op/s; 8068/52083 objects degraded (15.491%); 7185/52083 objects misplaced (13.795%); 970 KiB/s, 15 objects/s recovering 2023-09-23T19:04:54.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:54 smithi098 bash[79516]: cluster 2023-09-23T19:04:52.541044+0000 mgr.y (mgr.26054) 1413 : cluster 0 pgmap v1549: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 672 op/s; 8068/52083 objects degraded (15.491%); 7185/52083 objects misplaced (13.795%); 970 KiB/s, 15 objects/s recovering 2023-09-23T19:04:54.524 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:54.553 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:54.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:54.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:54.658 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:54.688 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:54.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:54.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:54.793 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:54.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:54.829 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:54.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:54.930 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:54.965 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:54.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:54.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:55.069 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:55.101 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:55.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:55.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:55.204 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:55.232 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:55.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:55.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:55.336 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:55.360 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:55.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:55.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:55.465 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:55.492 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:55.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:55.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:55.596 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:55.627 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:55.629 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:55 smithi173 bash[59422]: audit 2023-09-23T19:04:54.644678+0000 mon.a (mon.0) 1919 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:04:55.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:55.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:55.731 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:55.745 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:55 smithi098 bash[76817]: audit 2023-09-23T19:04:54.644678+0000 mon.a (mon.0) 1919 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:04:55.746 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:55 smithi098 bash[79516]: audit 2023-09-23T19:04:54.644678+0000 mon.a (mon.0) 1919 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:04:55.766 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:55.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:55.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:55.868 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:55.900 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:55.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:55.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:56.004 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:56.032 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:56.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:56.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:56.138 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:56.168 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:56.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:56.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:56.193 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:04:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:04:56] "GET /metrics HTTP/1.1" 200 36641 "" "Prometheus/2.33.4" 2023-09-23T19:04:56.271 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:56.310 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:56.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:56.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:56.415 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:56.453 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:56.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:56.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:56.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:56 smithi173 bash[59422]: cluster 2023-09-23T19:04:54.541776+0000 mgr.y (mgr.26054) 1414 : cluster 0 pgmap v1550: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 8068/52083 objects degraded (15.491%); 7185/52083 objects misplaced (13.795%); 970 KiB/s, 15 objects/s recovering 2023-09-23T19:04:56.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:56 smithi173 bash[59422]: cluster 2023-09-23T19:04:55.518423+0000 mon.a (mon.0) 1920 : cluster 3 Health check update: Degraded data redundancy: 8068/52083 objects degraded (15.491%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:56.560 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:56.568 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:56 smithi098 bash[79516]: cluster 2023-09-23T19:04:54.541776+0000 mgr.y (mgr.26054) 1414 : cluster 0 pgmap v1550: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 8068/52083 objects degraded (15.491%); 7185/52083 objects misplaced (13.795%); 970 KiB/s, 15 objects/s recovering 2023-09-23T19:04:56.568 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:56 smithi098 bash[79516]: cluster 2023-09-23T19:04:55.518423+0000 mon.a (mon.0) 1920 : cluster 3 Health check update: Degraded data redundancy: 8068/52083 objects degraded (15.491%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:56.569 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:56 smithi098 bash[76817]: cluster 2023-09-23T19:04:54.541776+0000 mgr.y (mgr.26054) 1414 : cluster 0 pgmap v1550: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 8068/52083 objects degraded (15.491%); 7185/52083 objects misplaced (13.795%); 970 KiB/s, 15 objects/s recovering 2023-09-23T19:04:56.569 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:56 smithi098 bash[76817]: cluster 2023-09-23T19:04:55.518423+0000 mon.a (mon.0) 1920 : cluster 3 Health check update: Degraded data redundancy: 8068/52083 objects degraded (15.491%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:04:56.592 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:56.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:56.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:56.694 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:56.729 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:56.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:56.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:56.833 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:56.860 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:56.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:56.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:56.967 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:57.001 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:57.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:57.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:57.106 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:57.129 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:57.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:57.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:57.232 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:57.272 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:57.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:57.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:57.377 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:57.402 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:57.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:57.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:57.506 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:57.533 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:57.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:57.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:57.641 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:57.676 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:57.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:57.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:57.780 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:57.814 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:57.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:57.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:57.925 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:57.956 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:57.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:57.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:58.060 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:58.089 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:58.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:58.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:58.195 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:58.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:04:58.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:58.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:04:58.329 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:58.356 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:58.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:58.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:58.459 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:58.487 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:58.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:58.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:58.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:04:58 smithi173 bash[59422]: cluster 2023-09-23T19:04:56.542611+0000 mgr.y (mgr.26054) 1415 : cluster 0 pgmap v1551: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1019 MiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 7598/48786 objects degraded (15.574%); 6936/48786 objects misplaced (14.217%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T19:04:58.596 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:58.605 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:04:58 smithi098 bash[76817]: cluster 2023-09-23T19:04:56.542611+0000 mgr.y (mgr.26054) 1415 : cluster 0 pgmap v1551: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1019 MiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 7598/48786 objects degraded (15.574%); 6936/48786 objects misplaced (14.217%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T19:04:58.605 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:04:58 smithi098 bash[79516]: cluster 2023-09-23T19:04:56.542611+0000 mgr.y (mgr.26054) 1415 : cluster 0 pgmap v1551: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 1019 MiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 7598/48786 objects degraded (15.574%); 6936/48786 objects misplaced (14.217%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T19:04:58.619 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:04:58.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:58.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:04:58.723 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:58.748 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:58.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:58.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:58.854 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:58.893 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:58.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:58.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:58.998 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:59.029 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:59.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:59.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:59.135 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:59.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:59.173 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:59.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:59.274 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:59.302 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:59.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:59.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:04:59.409 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:59.434 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:04:59.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:59.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:04:59.537 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:59.567 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:59.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:59.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:59.673 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:59.700 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:04:59.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:59.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:04:59.803 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:59.834 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:04:59.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:59.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:04:59.939 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:59.958 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:04:59.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:04:59.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:00.061 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:00.089 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:00.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:00.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:00.192 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:00.225 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:00.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:00.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:00.328 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:00.380 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:00.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:00.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:00.482 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:00.513 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:00.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:00.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:00.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:00 smithi173 bash[59422]: cluster 2023-09-23T19:04:58.543551+0000 mgr.y (mgr.26054) 1416 : cluster 0 pgmap v1552: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 901 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 666 op/s; 6673/43113 objects degraded (15.478%); 6354/43113 objects misplaced (14.738%); 975 KiB/s, 15 objects/s recovering 2023-09-23T19:05:00.616 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:00.625 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:00 smithi098 bash[79516]: cluster 2023-09-23T19:04:58.543551+0000 mgr.y (mgr.26054) 1416 : cluster 0 pgmap v1552: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 901 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 666 op/s; 6673/43113 objects degraded (15.478%); 6354/43113 objects misplaced (14.738%); 975 KiB/s, 15 objects/s recovering 2023-09-23T19:05:00.625 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:00 smithi098 bash[76817]: cluster 2023-09-23T19:04:58.543551+0000 mgr.y (mgr.26054) 1416 : cluster 0 pgmap v1552: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 901 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 666 op/s; 6673/43113 objects degraded (15.478%); 6354/43113 objects misplaced (14.738%); 975 KiB/s, 15 objects/s recovering 2023-09-23T19:05:00.657 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:00.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:00.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:00.762 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:00.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:00.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:00.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:00.889 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:00.930 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:05:00.937 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:00.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:00.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:01.044 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:01.069 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:01.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:01.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:01.173 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:01.201 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:01.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:01.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:01.248 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T19:05:01.248 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:05:01.305 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:01.339 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:01.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:01.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:01.443 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:01.471 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:01.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:01.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:01.489 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:01 smithi098 bash[76817]: cluster 2023-09-23T19:05:00.521678+0000 mon.a (mon.0) 1921 : cluster 3 Health check update: Degraded data redundancy: 6673/43113 objects degraded (15.478%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:05:01.490 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:01 smithi098 bash[79516]: cluster 2023-09-23T19:05:00.521678+0000 mon.a (mon.0) 1921 : cluster 3 Health check update: Degraded data redundancy: 6673/43113 objects degraded (15.478%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:05:01.574 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:01.601 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:01.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:01.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:01.706 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:01.740 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:01.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:01.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:01.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:01 smithi173 bash[59422]: cluster 2023-09-23T19:05:00.521678+0000 mon.a (mon.0) 1921 : cluster 3 Health check update: Degraded data redundancy: 6673/43113 objects degraded (15.478%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:05:01.844 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:01.874 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:01.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:01.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:01.977 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:02.006 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:02.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:02.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:02.109 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:02.179 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:02.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:02.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:02.276 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T19:05:02.284 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:02.307 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:02.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:02.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:02.315 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T19:05:02.359 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T19:05:02.406 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T19:05:02.410 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:02.418 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:02 smithi098 bash[79516]: cluster 2023-09-23T19:05:00.544401+0000 mgr.y (mgr.26054) 1417 : cluster 0 pgmap v1553: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 828 MiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 6198/39630 objects degraded (15.640%); 6047/39630 objects misplaced (15.259%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T19:05:02.418 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:02 smithi098 bash[79516]: audit 2023-09-23T19:05:02.236803+0000 mon.a (mon.0) 1922 : audit 0 from='client.? 172.21.15.98:0/535085079' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:05:02.421 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:02 smithi098 bash[76817]: cluster 2023-09-23T19:05:00.544401+0000 mgr.y (mgr.26054) 1417 : cluster 0 pgmap v1553: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 828 MiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 6198/39630 objects degraded (15.640%); 6047/39630 objects misplaced (15.259%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T19:05:02.421 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:02 smithi098 bash[76817]: audit 2023-09-23T19:05:02.236803+0000 mon.a (mon.0) 1922 : audit 0 from='client.? 172.21.15.98:0/535085079' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:05:02.440 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:02.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:02.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:02.447 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T19:05:02.489 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T19:05:02.494 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:05:02.494 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:05:02.494 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:05:02.494 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:05:02.494 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:05:02.494 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T19:05:02.494 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:05:02.494 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T19:05:02.548 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:02.570 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:02.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:02.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:02.676 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:02.690 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:02 smithi173 bash[59422]: cluster 2023-09-23T19:05:00.544401+0000 mgr.y (mgr.26054) 1417 : cluster 0 pgmap v1553: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 828 MiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 6198/39630 objects degraded (15.640%); 6047/39630 objects misplaced (15.259%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T19:05:02.690 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:02 smithi173 bash[59422]: audit 2023-09-23T19:05:02.236803+0000 mon.a (mon.0) 1922 : audit 0 from='client.? 172.21.15.98:0/535085079' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:05:02.707 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:02.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:02.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:02.812 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:02.838 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:02.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:02.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:02.942 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:02.968 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:02.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:02.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:03.071 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:03.103 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:03.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:03.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:03.208 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:03.234 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:03.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:03.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:03.337 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:03.360 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:03.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:03.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:03.463 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:03.497 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:03.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:03.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:03.603 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:03.633 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:03.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:03.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:03.737 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:03.764 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:03.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:03.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:03.869 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:03.909 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:03.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:03.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:04.012 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:04.039 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:04.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:04.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:04.142 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:04.167 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:04.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:04.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:04.271 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:04.303 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:04.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:04.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:04.407 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:04.434 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:04.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:04.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:04.537 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:04.579 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:04.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:04.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:04.684 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:04.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:04 smithi098 bash[76817]: cluster 2023-09-23T19:05:02.545353+0000 mgr.y (mgr.26054) 1418 : cluster 0 pgmap v1554: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 711 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 660 op/s; 5267/34011 objects degraded (15.486%); 5463/34011 objects misplaced (16.062%); 981 KiB/s, 15 objects/s recovering 2023-09-23T19:05:04.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:04 smithi098 bash[79516]: cluster 2023-09-23T19:05:02.545353+0000 mgr.y (mgr.26054) 1418 : cluster 0 pgmap v1554: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 711 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 660 op/s; 5267/34011 objects degraded (15.486%); 5463/34011 objects misplaced (16.062%); 981 KiB/s, 15 objects/s recovering 2023-09-23T19:05:04.694 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:04 smithi173 bash[59422]: cluster 2023-09-23T19:05:02.545353+0000 mgr.y (mgr.26054) 1418 : cluster 0 pgmap v1554: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 711 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 660 op/s; 5267/34011 objects degraded (15.486%); 5463/34011 objects misplaced (16.062%); 981 KiB/s, 15 objects/s recovering 2023-09-23T19:05:04.712 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:04.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:04.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:04.815 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:04.841 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:04.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:04.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:04.945 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:04.972 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:04.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:04.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:05.075 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:05.100 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:05.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:05.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:05.204 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:05.232 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:05.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:05.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:05.335 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:05.360 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:05.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:05.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:05.463 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:05.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:05.494 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:05.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:05.595 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:05.626 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:05.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:05.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:05.733 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:05.764 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:05.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:05.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:05.869 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:05.917 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:05:05.917 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:05:05.918 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:05.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:05.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:05.936 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T19:05:06.020 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:06.050 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:06.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:06.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:06.155 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:06.191 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:06.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:06.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:06.294 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:06.303 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:05:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:05:06] "GET /metrics HTTP/1.1" 200 36641 "" "Prometheus/2.33.4" 2023-09-23T19:05:06.333 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:06.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:06.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:06.440 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:06.467 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:06.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:06.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:06.571 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:06.580 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:06 smithi098 bash[76817]: cluster 2023-09-23T19:05:04.546060+0000 mgr.y (mgr.26054) 1419 : cluster 0 pgmap v1555: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 711 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 5267/34011 objects degraded (15.486%); 5463/34011 objects misplaced (16.062%); 981 KiB/s, 15 objects/s recovering 2023-09-23T19:05:06.580 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:06 smithi098 bash[76817]: cluster 2023-09-23T19:05:05.523281+0000 mon.a (mon.0) 1923 : cluster 3 Health check update: Degraded data redundancy: 5267/34011 objects degraded (15.486%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:05:06.580 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:06 smithi098 bash[79516]: cluster 2023-09-23T19:05:04.546060+0000 mgr.y (mgr.26054) 1419 : cluster 0 pgmap v1555: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 711 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 5267/34011 objects degraded (15.486%); 5463/34011 objects misplaced (16.062%); 981 KiB/s, 15 objects/s recovering 2023-09-23T19:05:06.580 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:06 smithi098 bash[79516]: cluster 2023-09-23T19:05:05.523281+0000 mon.a (mon.0) 1923 : cluster 3 Health check update: Degraded data redundancy: 5267/34011 objects degraded (15.486%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:05:06.604 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:06.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:06.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:06.715 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:06.754 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:06.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:06.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:06.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:06 smithi173 bash[59422]: cluster 2023-09-23T19:05:04.546060+0000 mgr.y (mgr.26054) 1419 : cluster 0 pgmap v1555: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 711 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 5267/34011 objects degraded (15.486%); 5463/34011 objects misplaced (16.062%); 981 KiB/s, 15 objects/s recovering 2023-09-23T19:05:06.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:06 smithi173 bash[59422]: cluster 2023-09-23T19:05:05.523281+0000 mon.a (mon.0) 1923 : cluster 3 Health check update: Degraded data redundancy: 5267/34011 objects degraded (15.486%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:05:06.858 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:06.892 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:06.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:06.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:06.996 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:07.025 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:07.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:07.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:07.129 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:07.158 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:07.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:07.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:07.261 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:07.288 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:07.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:07.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:07.392 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:07.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:07.433 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:07.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:07.442 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is unset 2023-09-23T19:05:07.534 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:07.541 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:07 smithi098 bash[79516]: audit 2023-09-23T19:05:07.075528+0000 mon.a (mon.0) 1924 : audit 1 from='client.? 172.21.15.98:0/1929364320' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:05:07.545 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:07 smithi098 bash[76817]: audit 2023-09-23T19:05:07.075528+0000 mon.a (mon.0) 1924 : audit 1 from='client.? 172.21.15.98:0/1929364320' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:05:07.566 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:07.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:07.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:07.670 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:07.699 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:07.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:07.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:07.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:07 smithi173 bash[59422]: audit 2023-09-23T19:05:07.075528+0000 mon.a (mon.0) 1924 : audit 1 from='client.? 172.21.15.98:0/1929364320' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:05:07.806 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:07.833 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:07.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:07.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:07.919 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:05:07.937 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:08.007 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:08.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:08.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:08.113 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:08.142 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:08.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:08.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:08.246 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:08.278 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:08.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:08.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:08.386 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:08.422 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:08.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:08.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:08.528 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:08.548 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:08.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:08.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:08.651 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:08.656 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:08 smithi098 bash[79516]: cluster 2023-09-23T19:05:06.548856+0000 mgr.y (mgr.26054) 1420 : cluster 0 pgmap v1556: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 641 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 595 op/s; 4805/30651 objects degraded (15.676%); 5180/30651 objects misplaced (16.900%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T19:05:08.656 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:08 smithi098 bash[79516]: cluster 2023-09-23T19:05:07.402810+0000 mon.a (mon.0) 1925 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:05:08.656 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:08 smithi098 bash[79516]: cluster 2023-09-23T19:05:07.424634+0000 osd.4 (osd.4) 65 : cluster 3 failed to encode map e335 with expected crc 2023-09-23T19:05:08.656 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:08 smithi098 bash[79516]: audit 2023-09-23T19:05:07.429667+0000 mon.a (mon.0) 1926 : audit 1 from='client.? 172.21.15.98:0/1929364320' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:05:08.656 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:08 smithi098 bash[79516]: cluster 2023-09-23T19:05:07.429733+0000 mon.a (mon.0) 1927 : cluster 0 osdmap e335: 8 total, 6 up, 7 in 2023-09-23T19:05:08.656 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:08 smithi098 bash[79516]: cluster 2023-09-23T19:05:07.437801+0000 osd.6 (osd.6) 317 : cluster 3 failed to encode map e335 with expected crc 2023-09-23T19:05:08.656 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:08 smithi098 bash[76817]: cluster 2023-09-23T19:05:06.548856+0000 mgr.y (mgr.26054) 1420 : cluster 0 pgmap v1556: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 641 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 595 op/s; 4805/30651 objects degraded (15.676%); 5180/30651 objects misplaced (16.900%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T19:05:08.656 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:08 smithi098 bash[76817]: cluster 2023-09-23T19:05:07.402810+0000 mon.a (mon.0) 1925 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:05:08.657 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:08 smithi098 bash[76817]: cluster 2023-09-23T19:05:07.424634+0000 osd.4 (osd.4) 65 : cluster 3 failed to encode map e335 with expected crc 2023-09-23T19:05:08.657 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:08 smithi098 bash[76817]: audit 2023-09-23T19:05:07.429667+0000 mon.a (mon.0) 1926 : audit 1 from='client.? 172.21.15.98:0/1929364320' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:05:08.657 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:08 smithi098 bash[76817]: cluster 2023-09-23T19:05:07.429733+0000 mon.a (mon.0) 1927 : cluster 0 osdmap e335: 8 total, 6 up, 7 in 2023-09-23T19:05:08.657 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:08 smithi098 bash[76817]: cluster 2023-09-23T19:05:07.437801+0000 osd.6 (osd.6) 317 : cluster 3 failed to encode map e335 with expected crc 2023-09-23T19:05:08.664 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:08 smithi173 bash[59422]: cluster 2023-09-23T19:05:06.548856+0000 mgr.y (mgr.26054) 1420 : cluster 0 pgmap v1556: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 641 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 595 op/s; 4805/30651 objects degraded (15.676%); 5180/30651 objects misplaced (16.900%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T19:05:08.664 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:08 smithi173 bash[59422]: cluster 2023-09-23T19:05:07.402810+0000 mon.a (mon.0) 1925 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:05:08.664 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:08 smithi173 bash[59422]: cluster 2023-09-23T19:05:07.424634+0000 osd.4 (osd.4) 65 : cluster 3 failed to encode map e335 with expected crc 2023-09-23T19:05:08.664 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:08 smithi173 bash[59422]: audit 2023-09-23T19:05:07.429667+0000 mon.a (mon.0) 1926 : audit 1 from='client.? 172.21.15.98:0/1929364320' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:05:08.664 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:08 smithi173 bash[59422]: cluster 2023-09-23T19:05:07.429733+0000 mon.a (mon.0) 1927 : cluster 0 osdmap e335: 8 total, 6 up, 7 in 2023-09-23T19:05:08.664 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:08 smithi173 bash[59422]: cluster 2023-09-23T19:05:07.437801+0000 osd.6 (osd.6) 317 : cluster 3 failed to encode map e335 with expected crc 2023-09-23T19:05:08.686 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:08.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:08.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:08.789 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:08.816 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:08.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:08.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:08.920 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:08.947 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:08.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:08.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:09.049 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:09.083 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:09.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:09.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:09.186 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:09.223 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:09.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:09.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:09.329 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:09.370 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:09.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:09.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:09.462 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:09 smithi098 bash[76817]: cluster 2023-09-23T19:05:08.550047+0000 mgr.y (mgr.26054) 1421 : cluster 0 pgmap v1558: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 517 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 3800/24687 objects degraded (15.393%); 4602/24687 objects misplaced (18.641%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:05:09.473 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:09.499 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:09.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:09.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:09.604 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:09.640 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:09.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:09.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:09.747 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:09.756 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:09 smithi173 bash[59422]: cluster 2023-09-23T19:05:08.550047+0000 mgr.y (mgr.26054) 1421 : cluster 0 pgmap v1558: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 517 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 3800/24687 objects degraded (15.393%); 4602/24687 objects misplaced (18.641%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:05:09.770 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:09.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:09.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:09.875 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:09.898 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:09.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:09.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:09.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:09 smithi098 bash[79516]: cluster 2023-09-23T19:05:08.550047+0000 mgr.y (mgr.26054) 1421 : cluster 0 pgmap v1558: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 517 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 3800/24687 objects degraded (15.393%); 4602/24687 objects misplaced (18.641%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:05:10.002 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:10.038 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:10.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:10.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:10.145 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:10.178 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:10.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:10.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:10.282 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:10.307 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:10.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:10.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:10.411 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:10.445 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:10.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:10.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:10.549 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:10.579 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:10.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:10.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:10.684 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:10.704 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:05:10.724 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:10.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:10.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:10.729 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:10 smithi173 bash[59422]: audit 2023-09-23T19:05:09.649676+0000 mon.a (mon.0) 1928 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:05:10.729 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:10 smithi173 bash[59422]: audit 2023-09-23T19:05:09.654891+0000 mon.a (mon.0) 1929 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:05:10.730 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:10 smithi173 bash[59422]: cluster 2023-09-23T19:05:10.524289+0000 mon.a (mon.0) 1930 : cluster 3 Health check update: Degraded data redundancy: 3800/24687 objects degraded (15.393%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:05:10.830 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:10.857 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:10.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:10.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:10.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:10 smithi098 bash[76817]: audit 2023-09-23T19:05:09.649676+0000 mon.a (mon.0) 1928 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:05:10.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:10 smithi098 bash[76817]: audit 2023-09-23T19:05:09.654891+0000 mon.a (mon.0) 1929 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:05:10.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:10 smithi098 bash[76817]: cluster 2023-09-23T19:05:10.524289+0000 mon.a (mon.0) 1930 : cluster 3 Health check update: Degraded data redundancy: 3800/24687 objects degraded (15.393%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:05:10.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:10 smithi098 bash[79516]: audit 2023-09-23T19:05:09.649676+0000 mon.a (mon.0) 1928 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:05:10.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:10 smithi098 bash[79516]: audit 2023-09-23T19:05:09.654891+0000 mon.a (mon.0) 1929 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:05:10.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:10 smithi098 bash[79516]: cluster 2023-09-23T19:05:10.524289+0000 mon.a (mon.0) 1930 : cluster 3 Health check update: Degraded data redundancy: 3800/24687 objects degraded (15.393%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:05:10.959 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:10.987 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:10.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:10.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:11.094 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:11.121 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:11.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:11.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:11.230 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:11.253 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:11.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:11.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:11.355 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:11.382 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:11.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:11.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:11.485 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:11.520 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:11.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:11.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:11.629 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:11.660 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:11.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:11.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:11.765 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:11.775 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:11 smithi098 bash[76817]: cluster 2023-09-23T19:05:10.550933+0000 mgr.y (mgr.26054) 1422 : cluster 0 pgmap v1559: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 447 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 3309/21333 objects degraded (15.511%); 4342/21333 objects misplaced (20.353%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:05:11.775 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:11 smithi098 bash[79516]: cluster 2023-09-23T19:05:10.550933+0000 mgr.y (mgr.26054) 1422 : cluster 0 pgmap v1559: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 447 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 3309/21333 objects degraded (15.511%); 4342/21333 objects misplaced (20.353%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:05:11.802 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:11.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:11.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:11.909 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:11.939 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:11.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:11.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:12.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:11 smithi173 bash[59422]: cluster 2023-09-23T19:05:10.550933+0000 mgr.y (mgr.26054) 1422 : cluster 0 pgmap v1559: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 447 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 3309/21333 objects degraded (15.511%); 4342/21333 objects misplaced (20.353%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:05:12.046 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:12.086 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:12.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:12.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:12.194 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:12.219 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:12.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:12.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:12.323 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:12.358 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:12.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:12.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:12.462 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:12.516 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:12.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:12.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:12.620 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:12.648 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:12.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:12.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:12.751 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:12.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:12 smithi173 bash[59422]: audit 2023-09-23T19:05:12.600645+0000 mon.a (mon.0) 1931 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:05:12.785 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:12.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:12.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:12.891 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:12.917 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:12.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:12.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:12.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:12 smithi098 bash[76817]: audit 2023-09-23T19:05:12.600645+0000 mon.a (mon.0) 1931 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:05:12.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:12 smithi098 bash[79516]: audit 2023-09-23T19:05:12.600645+0000 mon.a (mon.0) 1931 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:05:13.020 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:13.042 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:13.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:13.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:13.145 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:13.174 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:13.177 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T19:05:13.177 INFO:teuthology.orchestra.run.smithi098.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T19:05:13.177 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T19:05:13.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:13.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:13.281 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:13.311 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:13.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:13.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:13.407 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:05:13.415 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:13.439 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:13.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:13.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:13.543 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:13.581 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:13.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:13.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:13.673 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:13 smithi098 bash[76817]: cluster 2023-09-23T19:05:12.551839+0000 mgr.y (mgr.26054) 1423 : cluster 0 pgmap v1560: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 345 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 2392/16425 objects degraded (14.563%); 3804/16425 objects misplaced (23.160%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:05:13.690 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:13.712 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:13.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:13.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:13.815 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:13.822 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:13 smithi098 bash[79516]: cluster 2023-09-23T19:05:12.551839+0000 mgr.y (mgr.26054) 1423 : cluster 0 pgmap v1560: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 345 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 2392/16425 objects degraded (14.563%); 3804/16425 objects misplaced (23.160%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:05:13.879 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:13.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:13.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:13.983 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:14.017 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:14.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:14.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:14.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:13 smithi173 bash[59422]: cluster 2023-09-23T19:05:12.551839+0000 mgr.y (mgr.26054) 1423 : cluster 0 pgmap v1560: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 345 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 2392/16425 objects degraded (14.563%); 3804/16425 objects misplaced (23.160%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:05:14.120 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:14.145 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:14.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:14.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:14.248 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:14.282 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:14.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:14.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:14.389 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:14.419 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:14.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:14.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:14.522 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:14.553 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:14.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:14.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:14.656 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:14.687 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:14.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:14.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:14.792 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:14.816 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:14.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:14.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:14.910 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T19:05:14.910 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-09-23T19:05:14.910 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 0 2023-09-23T19:05:14.911 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T19:05:14.921 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:14.962 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:14.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:14.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:15.069 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:15.105 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:15.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:15.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:15.209 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:15.242 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:15.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:15.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:15.347 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:15.374 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:15.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:15.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:15.480 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:15.506 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:15.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:15.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:15.609 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:15.637 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:15.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:15.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:15.673 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:05:15.741 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:15.751 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:15 smithi098 bash[76817]: cluster 2023-09-23T19:05:14.552509+0000 mgr.y (mgr.26054) 1424 : cluster 0 pgmap v1561: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 333 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 604 op/s; 2392/15873 objects degraded (15.070%); 3804/15873 objects misplaced (23.965%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:05:15.751 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:15 smithi098 bash[76817]: cluster 2023-09-23T19:05:15.525464+0000 mon.a (mon.0) 1932 : cluster 3 Health check update: Degraded data redundancy: 2392/15873 objects degraded (15.070%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:05:15.751 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:15 smithi098 bash[79516]: cluster 2023-09-23T19:05:14.552509+0000 mgr.y (mgr.26054) 1424 : cluster 0 pgmap v1561: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 333 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 604 op/s; 2392/15873 objects degraded (15.070%); 3804/15873 objects misplaced (23.965%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:05:15.751 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:15 smithi098 bash[79516]: cluster 2023-09-23T19:05:15.525464+0000 mon.a (mon.0) 1932 : cluster 3 Health check update: Degraded data redundancy: 2392/15873 objects degraded (15.070%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:05:15.772 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:15.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:15.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:15.824 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T19:05:15.876 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:15.885 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:15 smithi173 bash[59422]: cluster 2023-09-23T19:05:14.552509+0000 mgr.y (mgr.26054) 1424 : cluster 0 pgmap v1561: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 333 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 604 op/s; 2392/15873 objects degraded (15.070%); 3804/15873 objects misplaced (23.965%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T19:05:15.885 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:15 smithi173 bash[59422]: cluster 2023-09-23T19:05:15.525464+0000 mon.a (mon.0) 1932 : cluster 3 Health check update: Degraded data redundancy: 2392/15873 objects degraded (15.070%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:05:15.899 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:15.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:15.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:16.002 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:16.034 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:16.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:16.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:16.140 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:16.166 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:16.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:16.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:16.269 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:16.277 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:05:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:05:16] "GET /metrics HTTP/1.1" 200 36642 "" "Prometheus/2.33.4" 2023-09-23T19:05:16.297 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:16.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:16.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:16.401 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:16.429 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:16.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:16.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:16.532 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:16.561 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:16.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:16.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:16.665 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:16.709 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:16.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:16.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:16.817 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:16.844 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:16.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:16.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:16.947 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:16.976 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:16.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:16.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:17.084 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:17.110 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:17.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:17.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:17.213 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:17.247 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:17.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:17.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:17.356 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:17.387 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:17.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:17.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:17.491 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:17.528 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:17.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:17.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:17.631 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:17.656 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:17.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:17.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:17.759 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:17.768 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:17 smithi098 bash[76817]: cluster 2023-09-23T19:05:16.553154+0000 mgr.y (mgr.26054) 1425 : cluster 0 pgmap v1562: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 261 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 1926/12417 objects degraded (15.511%); 3502/12417 objects misplaced (28.203%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T19:05:17.768 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:17 smithi098 bash[79516]: cluster 2023-09-23T19:05:16.553154+0000 mgr.y (mgr.26054) 1425 : cluster 0 pgmap v1562: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 261 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 1926/12417 objects degraded (15.511%); 3502/12417 objects misplaced (28.203%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T19:05:17.795 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:17.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:17.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:17.899 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:17.938 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:17.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:17.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:18.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:17 smithi173 bash[59422]: cluster 2023-09-23T19:05:16.553154+0000 mgr.y (mgr.26054) 1425 : cluster 0 pgmap v1562: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 261 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 1926/12417 objects degraded (15.511%); 3502/12417 objects misplaced (28.203%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T19:05:18.043 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:18.074 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:18.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:18.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:18.179 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:18.219 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:18.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:18.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:18.323 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:18.354 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:18.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:18.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:18.457 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:18.508 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:18.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:18.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:18.612 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:18.635 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:18.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:18.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:18.738 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:18.801 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:18.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:18.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:18.905 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:18.931 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:18.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:18.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:19.038 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:19.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:19.072 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:19.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:19.173 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:19.197 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:19.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:19.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:19.301 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:19.342 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:19.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:19.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:19.446 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:19.474 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:19.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:19.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:19.576 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:19.612 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:19.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:19.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:19.700 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Removed 100459 objects 2023-09-23T19:05:19.701 INFO:tasks.radosbench.radosbench.0.smithi098.stdout:Clean up completed and total clean up time :160.597 2023-09-23T19:05:19.718 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:19.739 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:19.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:19.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:19.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:19 smithi173 bash[59422]: cluster 2023-09-23T19:05:18.554131+0000 mgr.y (mgr.26054) 1426 : cluster 0 pgmap v1563: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 135 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 895/6357 objects degraded (14.079%); 2902/6357 objects misplaced (45.650%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T19:05:19.842 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:19.870 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:19.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:19.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:19.941 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:19 smithi098 bash[79516]: cluster 2023-09-23T19:05:18.554131+0000 mgr.y (mgr.26054) 1426 : cluster 0 pgmap v1563: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 135 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 895/6357 objects degraded (14.079%); 2902/6357 objects misplaced (45.650%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T19:05:19.942 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:19 smithi098 bash[76817]: cluster 2023-09-23T19:05:18.554131+0000 mgr.y (mgr.26054) 1426 : cluster 0 pgmap v1563: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 14 active+undersized+degraded, 11 active+undersized; 135 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 895/6357 objects degraded (14.079%); 2902/6357 objects misplaced (45.650%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T19:05:19.973 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:20.000 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:20.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:20.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:20.104 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:20.132 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:20.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:20.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:20.194 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 6, 7, 5, 3] out_osds: [4] dead_osds: [7, 5] live_osds: [0, 2, 3, 6, 1, 4] 2023-09-23T19:05:20.194 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:05:20.194 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_5 2023-09-23T19:05:20.195 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:05:20.236 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:20.266 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:20.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:20.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:20.361 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:05:20.370 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:20.384 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:05:20.395 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:20.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:20.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:20.498 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:20.529 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:20.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:20.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:20.632 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:20.652 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:20.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:20.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:20.755 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:20.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:20.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:20.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:20.890 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:20.921 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:20.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:20.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:20.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:20 smithi098 bash[79516]: audit 2023-09-23T19:05:19.769273+0000 mon.a (mon.0) 1933 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:05:20.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:20 smithi098 bash[79516]: audit 2023-09-23T19:05:19.775132+0000 mon.a (mon.0) 1934 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:05:20.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:20 smithi098 bash[79516]: cluster 2023-09-23T19:05:20.526553+0000 mon.a (mon.0) 1935 : cluster 3 Health check update: Degraded data redundancy: 895/6357 objects degraded (14.079%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:05:20.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:20 smithi098 bash[76817]: audit 2023-09-23T19:05:19.769273+0000 mon.a (mon.0) 1933 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:05:20.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:20 smithi098 bash[76817]: audit 2023-09-23T19:05:19.775132+0000 mon.a (mon.0) 1934 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:05:20.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:20 smithi098 bash[76817]: cluster 2023-09-23T19:05:20.526553+0000 mon.a (mon.0) 1935 : cluster 3 Health check update: Degraded data redundancy: 895/6357 objects degraded (14.079%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:05:21.024 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:21.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:20 smithi173 bash[59422]: audit 2023-09-23T19:05:19.769273+0000 mon.a (mon.0) 1933 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:05:21.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:20 smithi173 bash[59422]: audit 2023-09-23T19:05:19.775132+0000 mon.a (mon.0) 1934 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:05:21.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:20 smithi173 bash[59422]: cluster 2023-09-23T19:05:20.526553+0000 mon.a (mon.0) 1935 : cluster 3 Health check update: Degraded data redundancy: 895/6357 objects degraded (14.079%), 15 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:05:21.047 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:21.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:21.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:21.150 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:21.183 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:21.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:21.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:21.286 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:21.317 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:21.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:21.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:21.421 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:21.443 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:21.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:21.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:21.547 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:21.568 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:21.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:21.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:21.671 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:21.697 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:21.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:21.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:21.801 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:21.821 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:21.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:21.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:21.924 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:21.942 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:21.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:21.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:22.046 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:22.067 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:22.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:22.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:22.170 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:22.200 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:22.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:22.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:22.304 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:22.331 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:22.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:22.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:22.436 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:22.456 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:22.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:22.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:22.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:22 smithi173 bash[59422]: cluster 2023-09-23T19:05:20.554793+0000 mgr.y (mgr.26054) 1427 : cluster 0 pgmap v1564: 89 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 13 active+undersized+degraded, 12 active+undersized; 65 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 602 op/s; 456/2994 objects degraded (15.230%); 2630/2994 objects misplaced (87.842%); 1.6 MiB/s, 27 objects/s recovering 2023-09-23T19:05:22.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:22 smithi173 bash[59422]: audit 2023-09-23T19:05:21.388086+0000 mon.a (mon.0) 1936 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:05:22.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:22 smithi173 bash[59422]: audit 2023-09-23T19:05:21.394531+0000 mon.a (mon.0) 1937 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:05:22.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:22 smithi173 bash[59422]: audit 2023-09-23T19:05:21.864047+0000 mon.a (mon.0) 1938 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:05:22.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:22 smithi173 bash[59422]: audit 2023-09-23T19:05:21.865157+0000 mon.a (mon.0) 1939 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:05:22.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:22 smithi173 bash[59422]: audit 2023-09-23T19:05:21.872855+0000 mon.a (mon.0) 1940 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:05:22.560 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:22.571 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:22 smithi098 bash[76817]: cluster 2023-09-23T19:05:20.554793+0000 mgr.y (mgr.26054) 1427 : cluster 0 pgmap v1564: 89 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 13 active+undersized+degraded, 12 active+undersized; 65 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 602 op/s; 456/2994 objects degraded (15.230%); 2630/2994 objects misplaced (87.842%); 1.6 MiB/s, 27 objects/s recovering 2023-09-23T19:05:22.572 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:22 smithi098 bash[76817]: audit 2023-09-23T19:05:21.388086+0000 mon.a (mon.0) 1936 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:05:22.572 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:22 smithi098 bash[76817]: audit 2023-09-23T19:05:21.394531+0000 mon.a (mon.0) 1937 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:05:22.572 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:22 smithi098 bash[76817]: audit 2023-09-23T19:05:21.864047+0000 mon.a (mon.0) 1938 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:05:22.572 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:22 smithi098 bash[76817]: audit 2023-09-23T19:05:21.865157+0000 mon.a (mon.0) 1939 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:05:22.572 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:22 smithi098 bash[76817]: audit 2023-09-23T19:05:21.872855+0000 mon.a (mon.0) 1940 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:05:22.572 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:22 smithi098 bash[79516]: cluster 2023-09-23T19:05:20.554793+0000 mgr.y (mgr.26054) 1427 : cluster 0 pgmap v1564: 89 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 13 active+undersized+degraded, 12 active+undersized; 65 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 602 op/s; 456/2994 objects degraded (15.230%); 2630/2994 objects misplaced (87.842%); 1.6 MiB/s, 27 objects/s recovering 2023-09-23T19:05:22.572 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:22 smithi098 bash[79516]: audit 2023-09-23T19:05:21.388086+0000 mon.a (mon.0) 1936 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:05:22.572 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:22 smithi098 bash[79516]: audit 2023-09-23T19:05:21.394531+0000 mon.a (mon.0) 1937 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:05:22.572 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:22 smithi098 bash[79516]: audit 2023-09-23T19:05:21.864047+0000 mon.a (mon.0) 1938 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:05:22.572 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:22 smithi098 bash[79516]: audit 2023-09-23T19:05:21.865157+0000 mon.a (mon.0) 1939 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:05:22.573 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:22 smithi098 bash[79516]: audit 2023-09-23T19:05:21.872855+0000 mon.a (mon.0) 1940 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:05:22.588 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:22.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:22.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:22.691 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:22.719 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:22.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:22.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:22.822 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:22.848 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:22.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:22.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:22.951 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:22.980 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:22.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:22.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:23.083 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:23.109 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:23.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:23.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:23.213 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:23.244 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:23.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:23.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:23.346 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:23.375 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:23.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:23.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:23.478 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:23.500 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:23.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:23.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:23.603 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:23.623 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:23.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:23.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:23.727 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:23.748 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:23.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:23.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:23.851 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:23.882 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:23.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:23.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:23.985 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:24.013 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:24.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:24.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:24.116 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:24.150 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:24.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:24.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:24.162 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T19:05:24.162 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-09-23T19:05:24.162 INFO:teuthology.orchestra.run.smithi173.stdout: "complaint_time": 30, 2023-09-23T19:05:24.163 INFO:teuthology.orchestra.run.smithi173.stdout: "num_blocked_ops": 0 2023-09-23T19:05:24.163 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T19:05:24.253 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:24.279 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:24.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:24.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:24.382 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:24.408 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:24.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:24.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:24.512 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:24.523 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:24 smithi098 bash[79516]: cluster 2023-09-23T19:05:22.555530+0000 mgr.y (mgr.26054) 1428 : cluster 0 pgmap v1565: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 11 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 9/387 objects degraded (2.326%); 2331/387 objects misplaced (602.326%); 1.1 MiB/s, 20 objects/s recovering 2023-09-23T19:05:24.523 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:24 smithi098 bash[76817]: cluster 2023-09-23T19:05:22.555530+0000 mgr.y (mgr.26054) 1428 : cluster 0 pgmap v1565: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 11 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 9/387 objects degraded (2.326%); 2331/387 objects misplaced (602.326%); 1.1 MiB/s, 20 objects/s recovering 2023-09-23T19:05:24.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:24 smithi173 bash[59422]: cluster 2023-09-23T19:05:22.555530+0000 mgr.y (mgr.26054) 1428 : cluster 0 pgmap v1565: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 11 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 9/387 objects degraded (2.326%); 2331/387 objects misplaced (602.326%); 1.1 MiB/s, 20 objects/s recovering 2023-09-23T19:05:24.535 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:24.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:24.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:24.638 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:24.664 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:24.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:24.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:24.767 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:24.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:24.799 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:24.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:24.900 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:24.926 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:24.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:24.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:25.030 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:25.055 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:25.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:25.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:25.159 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:25.193 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:25.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:25.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:25.238 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T19:05:25.296 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:25.354 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:25.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:25.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:25.397 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:05:25.447 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T19:05:25.458 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:25.467 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T19:05:25.486 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:25.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:25.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:25.495 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T19:05:25.516 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T19:05:25.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:25 smithi173 bash[59422]: audit 2023-09-23T19:05:24.645623+0000 mon.a (mon.0) 1941 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:05:25.539 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T19:05:25.554 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:25 smithi098 bash[79516]: audit 2023-09-23T19:05:24.645623+0000 mon.a (mon.0) 1941 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:05:25.555 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:25 smithi098 bash[76817]: audit 2023-09-23T19:05:24.645623+0000 mon.a (mon.0) 1941 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:05:25.564 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T19:05:25.565 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:05:25.565 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:05:25.565 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:05:25.565 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:05:25.565 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:05:25.565 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T19:05:25.565 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:05:25.566 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T19:05:25.590 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:25.622 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:25.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:25.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:25.726 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:25.754 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:25.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:25.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:25.857 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:25.880 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:25.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:25.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:25.984 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:26.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:26.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:26.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:26.119 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:26.126 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:05:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:05:26] "GET /metrics HTTP/1.1" 200 36632 "" "Prometheus/2.33.4" 2023-09-23T19:05:26.138 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:26.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:26.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:26.241 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:26.265 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:26.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:26.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:26.368 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:26.389 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:26.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:26.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:26.492 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:26.518 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:26.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:26.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:26.621 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:26.650 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:26.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:26.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:26.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:26 smithi098 bash[76817]: cluster 2023-09-23T19:05:24.556642+0000 mgr.y (mgr.26054) 1429 : cluster 0 pgmap v1566: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 453 op/s; 9/84 objects degraded (10.714%); 2331/84 objects misplaced (2775.000%); 1.1 MiB/s, 20 objects/s recovering 2023-09-23T19:05:26.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:26 smithi098 bash[76817]: audit 2023-09-23T19:05:25.420494+0000 mon.c (mon.2) 146 : audit 0 from='client.? 172.21.15.98:0/1534510835' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:05:26.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:26 smithi098 bash[76817]: cluster 2023-09-23T19:05:25.527531+0000 mon.a (mon.0) 1942 : cluster 3 Health check update: Degraded data redundancy: 9/84 objects degraded (10.714%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:05:26.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:26 smithi098 bash[79516]: cluster 2023-09-23T19:05:24.556642+0000 mgr.y (mgr.26054) 1429 : cluster 0 pgmap v1566: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 453 op/s; 9/84 objects degraded (10.714%); 2331/84 objects misplaced (2775.000%); 1.1 MiB/s, 20 objects/s recovering 2023-09-23T19:05:26.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:26 smithi098 bash[79516]: audit 2023-09-23T19:05:25.420494+0000 mon.c (mon.2) 146 : audit 0 from='client.? 172.21.15.98:0/1534510835' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:05:26.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:26 smithi098 bash[79516]: cluster 2023-09-23T19:05:25.527531+0000 mon.a (mon.0) 1942 : cluster 3 Health check update: Degraded data redundancy: 9/84 objects degraded (10.714%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:05:26.754 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:26.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:26 smithi173 bash[59422]: cluster 2023-09-23T19:05:24.556642+0000 mgr.y (mgr.26054) 1429 : cluster 0 pgmap v1566: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 453 op/s; 9/84 objects degraded (10.714%); 2331/84 objects misplaced (2775.000%); 1.1 MiB/s, 20 objects/s recovering 2023-09-23T19:05:26.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:26 smithi173 bash[59422]: audit 2023-09-23T19:05:25.420494+0000 mon.c (mon.2) 146 : audit 0 from='client.? 172.21.15.98:0/1534510835' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:05:26.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:26 smithi173 bash[59422]: cluster 2023-09-23T19:05:25.527531+0000 mon.a (mon.0) 1942 : cluster 3 Health check update: Degraded data redundancy: 9/84 objects degraded (10.714%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:05:26.780 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:26.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:26.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:26.883 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:26.917 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:26.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:26.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:27.020 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:27.128 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:27.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:27.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:27.231 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:27.252 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:27.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:27.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:27.355 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:27.386 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:27.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:27.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:27.489 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:27.512 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:27.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:27.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:27.614 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:27.643 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:27.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:27.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:27.746 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:27.771 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:27.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:27.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:27.874 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:27.895 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:27.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:27.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:27.998 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:28.024 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:28.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:28.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:28.128 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:28.147 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:28.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:28.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:28.250 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:28.276 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:28.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:28.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:28.379 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:28.412 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:28.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:28.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:28.516 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:28.543 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:28.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:28.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:28.647 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:28.658 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:28 smithi173 bash[59422]: cluster 2023-09-23T19:05:26.557566+0000 mgr.y (mgr.26054) 1430 : cluster 0 pgmap v1567: 89 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 9/84 objects degraded (10.714%); 2331/84 objects misplaced (2775.000%); 1.1 MiB/s, 61 objects/s recovering 2023-09-23T19:05:28.675 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:28.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:28.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:28.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:28 smithi098 bash[76817]: cluster 2023-09-23T19:05:26.557566+0000 mgr.y (mgr.26054) 1430 : cluster 0 pgmap v1567: 89 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 9/84 objects degraded (10.714%); 2331/84 objects misplaced (2775.000%); 1.1 MiB/s, 61 objects/s recovering 2023-09-23T19:05:28.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:28 smithi098 bash[79516]: cluster 2023-09-23T19:05:26.557566+0000 mgr.y (mgr.26054) 1430 : cluster 0 pgmap v1567: 89 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 9/84 objects degraded (10.714%); 2331/84 objects misplaced (2775.000%); 1.1 MiB/s, 61 objects/s recovering 2023-09-23T19:05:28.780 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:28.811 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:28.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:28.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:28.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:28.949 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:28.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:28.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:29.053 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:29.083 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:29.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:29.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:29.186 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:29.220 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:29.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:29.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:29.324 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:29.356 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:29.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:29.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:29.460 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:29.482 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:29.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:29.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:29.585 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:29.608 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:29.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:29.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:29.711 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:29.740 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:29.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:29.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:29.844 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:29.868 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:29.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:29.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:29.971 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:29.995 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:29.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:29.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:30.047 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:05:30.099 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:30.132 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:30.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:30.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:30.235 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:30.262 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:30.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:30.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:30.365 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:30.385 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:30.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:30.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:30.489 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:30.516 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:30.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:30.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:30.619 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:30.631 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:30 smithi098 bash[76817]: cluster 2023-09-23T19:05:28.558660+0000 mgr.y (mgr.26054) 1431 : cluster 0 pgmap v1568: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 342 op/s; 9/84 objects degraded (10.714%); 2331/84 objects misplaced (2775.000%); 533 KiB/s, 51 objects/s recovering 2023-09-23T19:05:30.631 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:30 smithi098 bash[79516]: cluster 2023-09-23T19:05:28.558660+0000 mgr.y (mgr.26054) 1431 : cluster 0 pgmap v1568: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 342 op/s; 9/84 objects degraded (10.714%); 2331/84 objects misplaced (2775.000%); 533 KiB/s, 51 objects/s recovering 2023-09-23T19:05:30.650 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:30.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:30.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:30.754 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:30.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:30 smithi173 bash[59422]: cluster 2023-09-23T19:05:28.558660+0000 mgr.y (mgr.26054) 1431 : cluster 0 pgmap v1568: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 342 op/s; 9/84 objects degraded (10.714%); 2331/84 objects misplaced (2775.000%); 533 KiB/s, 51 objects/s recovering 2023-09-23T19:05:30.780 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:30.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:30.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:30.883 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:30.912 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:30.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:30.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:31.015 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:31.025 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:05:31.026 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:05:31.050 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:31.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:31.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:31.154 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:31.184 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:31.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:31.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:31.287 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:31.318 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:31.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:31.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:31.422 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:31.453 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:31.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:31.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:31.558 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:31.587 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:31.587 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:31 smithi173 bash[59422]: cluster 2023-09-23T19:05:30.559785+0000 mgr.y (mgr.26054) 1432 : cluster 0 pgmap v1569: 89 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 174 op/s; 9/84 objects degraded (10.714%); 2331/84 objects misplaced (2775.000%); 533 KiB/s, 95 objects/s recovering 2023-09-23T19:05:31.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:31.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:31.690 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:31.717 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:31.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:31.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:31.821 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:31.845 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:31.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:31.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:31.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:31 smithi098 bash[79516]: cluster 2023-09-23T19:05:30.559785+0000 mgr.y (mgr.26054) 1432 : cluster 0 pgmap v1569: 89 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 174 op/s; 9/84 objects degraded (10.714%); 2331/84 objects misplaced (2775.000%); 533 KiB/s, 95 objects/s recovering 2023-09-23T19:05:31.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:31 smithi098 bash[76817]: cluster 2023-09-23T19:05:30.559785+0000 mgr.y (mgr.26054) 1432 : cluster 0 pgmap v1569: 89 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 174 op/s; 9/84 objects degraded (10.714%); 2331/84 objects misplaced (2775.000%); 533 KiB/s, 95 objects/s recovering 2023-09-23T19:05:31.948 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:31.968 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:31.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:31.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:32.072 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:32.093 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:32.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:32.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:32.197 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:32.219 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:32.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:32.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:32.322 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:32.351 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:32.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:32.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:32.455 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:32.483 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:32.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:32.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:32.585 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:32.613 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:32.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:32.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:32.716 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:32.737 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:32.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:32.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:32.841 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:32.865 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:32.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:32.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:32.969 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:32.990 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:32.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:32.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:33.027 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:05:33.092 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:33.122 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:33.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:33.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:33.226 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:33.248 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:33.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:33.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:33.351 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:33.375 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:33.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:33.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:33.477 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:33.497 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:33.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:33.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:33.600 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:33.623 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:33.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:33.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:33.726 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:33.749 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:33.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:33.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:33.852 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:33.880 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:33.883 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:33 smithi173 bash[59422]: cluster 2023-09-23T19:05:32.561016+0000 mgr.y (mgr.26054) 1433 : cluster 0 pgmap v1570: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 80 op/s; 9/84 objects degraded (10.714%); 2331/84 objects misplaced (2775.000%); 0 B/s, 85 objects/s recovering 2023-09-23T19:05:33.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:33.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:33.985 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:34.005 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:34.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:34.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:34.107 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:34.134 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:34.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:34.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:34.142 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T19:05:34.142 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:05:34.238 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:34.263 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:34.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:34.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:34.270 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:33 smithi098 bash[76817]: cluster 2023-09-23T19:05:32.561016+0000 mgr.y (mgr.26054) 1433 : cluster 0 pgmap v1570: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 80 op/s; 9/84 objects degraded (10.714%); 2331/84 objects misplaced (2775.000%); 0 B/s, 85 objects/s recovering 2023-09-23T19:05:34.271 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:33 smithi098 bash[79516]: cluster 2023-09-23T19:05:32.561016+0000 mgr.y (mgr.26054) 1433 : cluster 0 pgmap v1570: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 80 op/s; 9/84 objects degraded (10.714%); 2331/84 objects misplaced (2775.000%); 0 B/s, 85 objects/s recovering 2023-09-23T19:05:34.369 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:34.390 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:34.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:34.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:34.493 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:34.519 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:34.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:34.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:34.622 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:34.643 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:34.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:34.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:34.746 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:34.771 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:34.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:34.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:34.874 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:34.897 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:34.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:34.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:35.000 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:35.022 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:35.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:35.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:35.079 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T19:05:35.125 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:35.148 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:35.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:35.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:35.251 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:35.282 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:35.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:35.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:35.385 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:35.412 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:35.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:35.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:35.515 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:35.537 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:35.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:35.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:35.641 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:35.662 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:35.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:35.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:35.766 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:35.790 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:35.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:35.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:35.894 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:35.924 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:35.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:35.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:36.027 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:36.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:35 smithi173 bash[59422]: cluster 2023-09-23T19:05:34.561695+0000 mgr.y (mgr.26054) 1434 : cluster 0 pgmap v1571: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 8 op/s; 9/84 objects degraded (10.714%); 2331/84 objects misplaced (2775.000%); 0 B/s, 85 objects/s recovering 2023-09-23T19:05:36.033 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:35 smithi098 bash[79516]: cluster 2023-09-23T19:05:34.561695+0000 mgr.y (mgr.26054) 1434 : cluster 0 pgmap v1571: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 8 op/s; 9/84 objects degraded (10.714%); 2331/84 objects misplaced (2775.000%); 0 B/s, 85 objects/s recovering 2023-09-23T19:05:36.033 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:35 smithi098 bash[76817]: cluster 2023-09-23T19:05:34.561695+0000 mgr.y (mgr.26054) 1434 : cluster 0 pgmap v1571: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 8 op/s; 9/84 objects degraded (10.714%); 2331/84 objects misplaced (2775.000%); 0 B/s, 85 objects/s recovering 2023-09-23T19:05:36.053 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:36.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:36.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:36.156 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:36.188 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:36.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:36.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:36.292 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:36.303 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:05:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:05:36] "GET /metrics HTTP/1.1" 200 36632 "" "Prometheus/2.33.4" 2023-09-23T19:05:36.320 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:36.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:36.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:36.423 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:36.448 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:36.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:36.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:36.551 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:36.576 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:36.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:36.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:36.679 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:36.706 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:36.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:36.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:36.809 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:36.832 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:36.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:36.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:36.934 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:36.967 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:36.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:36.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:37.070 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:37.094 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:37.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:37.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:37.197 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:37.224 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:37.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:37.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:37.328 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:37.353 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:37.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:37.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:37.457 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:37.479 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:37.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:37.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:37.582 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:37.610 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:37.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:37.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:37.713 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:37.742 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:37.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:37.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:37.846 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:37.855 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:37 smithi098 bash[76817]: cluster 2023-09-23T19:05:36.562404+0000 mgr.y (mgr.26054) 1435 : cluster 0 pgmap v1572: 89 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 2331/84 objects misplaced (2775.000%); 0 B/s, 134 objects/s recovering 2023-09-23T19:05:37.856 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:37 smithi098 bash[79516]: cluster 2023-09-23T19:05:36.562404+0000 mgr.y (mgr.26054) 1435 : cluster 0 pgmap v1572: 89 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 2331/84 objects misplaced (2775.000%); 0 B/s, 134 objects/s recovering 2023-09-23T19:05:37.872 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:37.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:37.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:37.975 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:37.999 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:38.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:38.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:38.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:37 smithi173 bash[59422]: cluster 2023-09-23T19:05:36.562404+0000 mgr.y (mgr.26054) 1435 : cluster 0 pgmap v1572: 89 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 2331/84 objects misplaced (2775.000%); 0 B/s, 134 objects/s recovering 2023-09-23T19:05:38.102 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:38.134 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:38.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:38.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:38.238 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:38.270 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:38.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:38.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:38.374 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:38.408 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:38.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:38.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:38.513 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:38.532 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:38.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:38.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:38.635 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:38.649 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is set 2023-09-23T19:05:38.659 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:38.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:38.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:38.762 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:38.770 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:38 smithi098 bash[76817]: audit 2023-09-23T19:05:38.545567+0000 mon.a (mon.0) 1943 : audit 1 from='client.? 172.21.15.98:0/1627893758' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:05:38.770 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:38 smithi098 bash[79516]: audit 2023-09-23T19:05:38.545567+0000 mon.a (mon.0) 1943 : audit 1 from='client.? 172.21.15.98:0/1627893758' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:05:38.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:38 smithi173 bash[59422]: audit 2023-09-23T19:05:38.545567+0000 mon.a (mon.0) 1943 : audit 1 from='client.? 172.21.15.98:0/1627893758' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:05:38.784 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:38.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:38.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:38.888 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:38.909 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:38.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:38.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:39.012 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:39.046 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:39.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:39.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:39.150 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:39.172 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:39.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:39.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:39.275 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:39.299 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:39.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:39.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:39.403 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:39.422 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:39.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:39.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:39.526 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:39.551 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:39.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:39.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:39.655 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:39.662 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:39 smithi098 bash[79516]: cluster 2023-09-23T19:05:38.563136+0000 mgr.y (mgr.26054) 1436 : cluster 0 pgmap v1573: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 2331/84 objects misplaced (2775.000%); 0 B/s, 93 objects/s recovering 2023-09-23T19:05:39.662 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:39 smithi098 bash[79516]: cluster 2023-09-23T19:05:38.622334+0000 mon.a (mon.0) 1944 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:05:39.662 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:39 smithi098 bash[79516]: cluster 2023-09-23T19:05:38.639945+0000 osd.4 (osd.4) 66 : cluster 3 failed to encode map e336 with expected crc 2023-09-23T19:05:39.662 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:39 smithi098 bash[79516]: audit 2023-09-23T19:05:38.640510+0000 mon.a (mon.0) 1945 : audit 1 from='client.? 172.21.15.98:0/1627893758' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:05:39.662 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:39 smithi098 bash[79516]: cluster 2023-09-23T19:05:38.640597+0000 mon.a (mon.0) 1946 : cluster 0 osdmap e336: 8 total, 6 up, 7 in 2023-09-23T19:05:39.663 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:39 smithi098 bash[76817]: cluster 2023-09-23T19:05:38.563136+0000 mgr.y (mgr.26054) 1436 : cluster 0 pgmap v1573: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 2331/84 objects misplaced (2775.000%); 0 B/s, 93 objects/s recovering 2023-09-23T19:05:39.663 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:39 smithi098 bash[76817]: cluster 2023-09-23T19:05:38.622334+0000 mon.a (mon.0) 1944 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:05:39.663 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:39 smithi098 bash[76817]: cluster 2023-09-23T19:05:38.639945+0000 osd.4 (osd.4) 66 : cluster 3 failed to encode map e336 with expected crc 2023-09-23T19:05:39.663 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:39 smithi098 bash[76817]: audit 2023-09-23T19:05:38.640510+0000 mon.a (mon.0) 1945 : audit 1 from='client.? 172.21.15.98:0/1627893758' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:05:39.663 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:39 smithi098 bash[76817]: cluster 2023-09-23T19:05:38.640597+0000 mon.a (mon.0) 1946 : cluster 0 osdmap e336: 8 total, 6 up, 7 in 2023-09-23T19:05:39.676 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:39.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:39.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:39.779 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:39.801 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:39.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:39.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:39.904 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:39.929 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:39.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:39.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:40.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:39 smithi173 bash[59422]: cluster 2023-09-23T19:05:38.563136+0000 mgr.y (mgr.26054) 1436 : cluster 0 pgmap v1573: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 11 active+undersized+degraded, 14 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 2331/84 objects misplaced (2775.000%); 0 B/s, 93 objects/s recovering 2023-09-23T19:05:40.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:39 smithi173 bash[59422]: cluster 2023-09-23T19:05:38.622334+0000 mon.a (mon.0) 1944 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:05:40.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:39 smithi173 bash[59422]: cluster 2023-09-23T19:05:38.639945+0000 osd.4 (osd.4) 66 : cluster 3 failed to encode map e336 with expected crc 2023-09-23T19:05:40.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:39 smithi173 bash[59422]: audit 2023-09-23T19:05:38.640510+0000 mon.a (mon.0) 1945 : audit 1 from='client.? 172.21.15.98:0/1627893758' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:05:40.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:39 smithi173 bash[59422]: cluster 2023-09-23T19:05:38.640597+0000 mon.a (mon.0) 1946 : cluster 0 osdmap e336: 8 total, 6 up, 7 in 2023-09-23T19:05:40.033 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:40.062 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:40.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:40.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:40.166 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:40.196 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:40.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:40.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:40.300 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:40.318 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:40.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:40.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:40.420 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:40.448 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:40.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:40.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:40.552 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:40.582 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:40.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:40.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:40.686 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:40.716 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:40.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:40.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:40.820 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:40.851 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:40.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:40.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:40.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:40 smithi098 bash[76817]: audit 2023-09-23T19:05:39.649318+0000 mon.a (mon.0) 1947 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:05:40.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:40 smithi098 bash[76817]: audit 2023-09-23T19:05:39.650500+0000 mon.a (mon.0) 1948 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:05:40.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:40 smithi098 bash[79516]: audit 2023-09-23T19:05:39.649318+0000 mon.a (mon.0) 1947 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:05:40.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:40 smithi098 bash[79516]: audit 2023-09-23T19:05:39.650500+0000 mon.a (mon.0) 1948 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:05:40.956 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:40.978 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:40.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:40.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:41.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:40 smithi173 bash[59422]: audit 2023-09-23T19:05:39.649318+0000 mon.a (mon.0) 1947 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:05:41.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:40 smithi173 bash[59422]: audit 2023-09-23T19:05:39.650500+0000 mon.a (mon.0) 1948 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:05:41.080 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:41.104 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:41.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:41.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:41.208 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:41.233 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:41.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:41.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:41.335 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:41.360 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:41.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:41.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:41.464 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:41.485 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:41.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:41.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:41.588 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:41.616 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:41.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:41.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:41.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:41.752 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:41.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:41.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:41.856 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:41.864 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:41 smithi098 bash[79516]: cluster 2023-09-23T19:05:40.564462+0000 mgr.y (mgr.26054) 1437 : cluster 0 pgmap v1575: 89 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 10 active+undersized+degraded, 15 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 1745/84 objects misplaced (2077.381%); 0 B/s, 149 objects/s recovering 2023-09-23T19:05:41.864 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:41 smithi098 bash[79516]: cluster 2023-09-23T19:05:41.533738+0000 mon.a (mon.0) 1949 : cluster 3 Health check update: Degraded data redundancy: 9/84 objects degraded (10.714%), 10 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:05:41.864 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:41 smithi098 bash[76817]: cluster 2023-09-23T19:05:40.564462+0000 mgr.y (mgr.26054) 1437 : cluster 0 pgmap v1575: 89 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 10 active+undersized+degraded, 15 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 1745/84 objects misplaced (2077.381%); 0 B/s, 149 objects/s recovering 2023-09-23T19:05:41.864 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:41 smithi098 bash[76817]: cluster 2023-09-23T19:05:41.533738+0000 mon.a (mon.0) 1949 : cluster 3 Health check update: Degraded data redundancy: 9/84 objects degraded (10.714%), 10 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:05:41.877 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:41.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:41.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:41.981 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:42.003 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:42.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:42.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:42.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:41 smithi173 bash[59422]: cluster 2023-09-23T19:05:40.564462+0000 mgr.y (mgr.26054) 1437 : cluster 0 pgmap v1575: 89 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 10 active+undersized+degraded, 15 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 1745/84 objects misplaced (2077.381%); 0 B/s, 149 objects/s recovering 2023-09-23T19:05:42.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:41 smithi173 bash[59422]: cluster 2023-09-23T19:05:41.533738+0000 mon.a (mon.0) 1949 : cluster 3 Health check update: Degraded data redundancy: 9/84 objects degraded (10.714%), 10 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T19:05:42.106 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:42.127 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:42.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:42.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:42.230 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:42.264 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:42.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:42.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:42.367 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:42.394 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:42.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:42.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:42.497 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:42.521 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:42.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:42.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:42.624 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:42.645 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:42.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:42.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:42.748 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:42.781 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:42.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:42.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:42.886 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:42.907 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:42.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:42.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:43.010 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:43.036 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:43.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:43.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:43.140 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:43.161 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:43.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:43.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:43.264 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:43.296 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:43.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:43.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:43.389 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T19:05:43.391 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-23T19:05:43.399 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:43.434 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:43.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:43.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:43.538 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:43.569 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:43.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:43.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:43.672 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:43.695 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:43.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:43.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:43.798 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:43.808 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:43 smithi098 bash[79516]: cluster 2023-09-23T19:05:42.565311+0000 mgr.y (mgr.26054) 1438 : cluster 0 pgmap v1576: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 1745/84 objects misplaced (2077.381%); 0 B/s, 149 objects/s recovering 2023-09-23T19:05:43.808 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:43 smithi098 bash[79516]: audit 2023-09-23T19:05:43.368512+0000 mgr.y (mgr.26054) 1439 : audit 0 from='client.35919 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:05:43.808 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:43 smithi098 bash[76817]: cluster 2023-09-23T19:05:42.565311+0000 mgr.y (mgr.26054) 1438 : cluster 0 pgmap v1576: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 1745/84 objects misplaced (2077.381%); 0 B/s, 149 objects/s recovering 2023-09-23T19:05:43.809 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:43 smithi098 bash[76817]: audit 2023-09-23T19:05:43.368512+0000 mgr.y (mgr.26054) 1439 : audit 0 from='client.35919 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:05:43.827 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:43.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:43.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:43.931 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:43.954 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:43.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:43.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:43 smithi173 bash[59422]: cluster 2023-09-23T19:05:42.565311+0000 mgr.y (mgr.26054) 1438 : cluster 0 pgmap v1576: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 1745/84 objects misplaced (2077.381%); 0 B/s, 149 objects/s recovering 2023-09-23T19:05:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:43 smithi173 bash[59422]: audit 2023-09-23T19:05:43.368512+0000 mgr.y (mgr.26054) 1439 : audit 0 from='client.35919 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:05:44.058 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:44.081 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:44.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:44.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:44.185 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:44.209 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:44.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:44.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:44.313 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:44.346 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:44.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:44.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:44.450 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:44.482 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:44.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:44.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:44.585 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:44.616 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:44.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:44.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:44.720 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:44.742 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:44.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:44.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:44.845 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:44.870 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:44.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:44.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:44.974 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:45.008 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:45.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:45.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:45.112 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:45.139 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:45.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:45.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:45.242 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:45.265 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:45.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:45.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:45.369 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:45.397 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:45.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:45.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:45.406 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":1576,"stamp":"2023-09-23T19:05:42.564687+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":1745,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6232,"num_read_kb":13691,"num_write":202026,"num_write_kb":6444536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7034,"num_bytes_recovered":298065861,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111915,"ondisk_log_size":111915,"up":203,"acting":236,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":7874020,"kb_used_data":1499372,"kb_used_omap":31,"kb_used_meta":6374560,"kb_avail":648342044,"statfs":{"total":671965249536,"available":663902253056,"internally_reserved":0,"allocated":1535356928,"data_stored":1526880500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32507,"internal_metadata":6527549701},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-586,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1491,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004651"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":45,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T19:05:07.448890+0000","last_change":"2023-09-23T18:59:13.709108+0000","last_active":"2023-09-23T19:05:07.448890+0000","last_peered":"2023-09-23T19:05:07.448890+0000","last_clean":"2023-09-23T19:05:07.448890+0000","last_became_active":"2023-09-23T18:59:13.708511+0000","last_became_peered":"2023-09-23T18:59:13.708511+0000","last_unstale":"2023-09-23T19:05:07.448890+0000","last_undegraded":"2023-09-23T19:05:07.448890+0000","last_fullsized":"2023-09-23T19:05:07.448890+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:59.295137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":58,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-23T19:04:31.529652+0000","last_change":"2023-09-23T18:59:13.702575+0000","last_active":"2023-09-23T19:04:31.529652+0000","last_peered":"2023-09-23T19:04:31.529652+0000","last_clean":"2023-09-23T19:04:31.529652+0000","last_became_active":"2023-09-23T18:59:13.701967+0000","last_became_peered":"2023-09-23T18:59:13.701967+0000","last_unstale":"2023-09-23T19:04:31.529652+0000","last_undegraded":"2023-09-23T19:04:31.529652+0000","last_fullsized":"2023-09-23T19:04:31.529652+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:30:12.982518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":326,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T19:05:36.970479+0000","last_change":"2023-09-23T18:59:13.707831+0000","last_active":"2023-09-23T19:05:36.970479+0000","last_peered":"2023-09-23T19:05:36.970479+0000","last_clean":"2023-09-23T19:05:36.970479+0000","last_became_active":"2023-09-23T18:59:13.707524+0000","last_became_peered":"2023-09-23T18:59:13.707524+0000","last_unstale":"2023-09-23T19:05:36.970479+0000","last_undegraded":"2023-09-23T19:05:36.970479+0000","last_fullsized":"2023-09-23T19:05:36.970479+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:42.677892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":360,"reported_epoch":335,"state":"active+undersized","last_fresh":"2023-09-23T19:05:07.427294+0000","last_change":"2023-09-23T19:02:30.622965+0000","last_active":"2023-09-23T19:05:07.427294+0000","last_peered":"2023-09-23T19:05:07.427294+0000","last_clean":"2023-09-23T19:02:20.581966+0000","last_became_active":"2023-09-23T19:02:30.622965+0000","last_became_peered":"2023-09-23T19:02:30.622965+0000","last_unstale":"2023-09-23T19:05:07.427294+0000","last_undegraded":"2023-09-23T19:05:07.427294+0000","last_fullsized":"2023-09-23T19:02:30.614276+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:23.666704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":49,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-23T19:04:31.524936+0000","last_change":"2023-09-23T18:58:38.718387+0000","last_active":"2023-09-23T19:04:31.524936+0000","last_peered":"2023-09-23T19:04:31.524936+0000","last_clean":"2023-09-23T19:04:31.524936+0000","last_became_active":"2023-09-23T18:58:38.717999+0000","last_became_peered":"2023-09-23T18:58:38.717999+0000","last_unstale":"2023-09-23T19:04:31.524936+0000","last_undegraded":"2023-09-23T19:04:31.524936+0000","last_fullsized":"2023-09-23T19:04:31.524936+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:05.073213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":278,"reported_epoch":335,"state":"active+undersized","last_fresh":"2023-09-23T19:05:07.447785+0000","last_change":"2023-09-23T19:02:30.628877+0000","last_active":"2023-09-23T19:05:07.447785+0000","last_peered":"2023-09-23T19:05:07.447785+0000","last_clean":"2023-09-23T18:47:23.057197+0000","last_became_active":"2023-09-23T19:02:30.628877+0000","last_became_peered":"2023-09-23T19:02:30.628877+0000","last_unstale":"2023-09-23T19:05:07.447785+0000","last_undegraded":"2023-09-23T19:05:07.447785+0000","last_fullsized":"2023-09-23T19:02:30.608793+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:08:57.956352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":59,"reported_epoch":334,"state":"active+undersized","last_fresh":"2023-09-23T19:04:57.520105+0000","last_change":"2023-09-23T19:02:30.624068+0000","last_active":"2023-09-23T19:04:57.520105+0000","last_peered":"2023-09-23T19:04:57.520105+0000","last_clean":"2023-09-23T18:58:10.508100+0000","last_became_active":"2023-09-23T19:02:30.624068+0000","last_became_peered":"2023-09-23T19:02:30.624068+0000","last_unstale":"2023-09-23T19:04:57.520105+0000","last_undegraded":"2023-09-23T19:04:57.520105+0000","last_fullsized":"2023-09-23T19:02:30.604686+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:21.468834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":49,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-23T19:04:31.524805+0000","last_change":"2023-09-23T18:58:38.710920+0000","last_active":"2023-09-23T19:04:31.524805+0000","last_peered":"2023-09-23T19:04:31.524805+0000","last_clean":"2023-09-23T19:04:31.524805+0000","last_became_active":"2023-09-23T18:58:38.710563+0000","last_became_peered":"2023-09-23T18:58:38.710563+0000","last_unstale":"2023-09-23T19:04:31.524805+0000","last_undegraded":"2023-09-23T19:04:31.524805+0000","last_fullsized":"2023-09-23T19:04:31.524805+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:42.386631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":311,"reported_epoch":335,"state":"active+undersized+remapped","last_fresh":"2023-09-23T19:05:07.440158+0000","last_change":"2023-09-23T19:03:26.329844+0000","last_active":"2023-09-23T19:05:07.440158+0000","last_peered":"2023-09-23T19:05:07.440158+0000","last_clean":"2023-09-23T18:40:38.019683+0000","last_became_active":"2023-09-23T19:03:26.329844+0000","last_became_peered":"2023-09-23T19:03:26.329844+0000","last_unstale":"2023-09-23T19:05:07.440158+0000","last_undegraded":"2023-09-23T19:05:07.440158+0000","last_fullsized":"2023-09-23T19:03:26.323016+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:29.074261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":321,"reported_epoch":334,"state":"active+undersized","last_fresh":"2023-09-23T19:04:31.529365+0000","last_change":"2023-09-23T19:02:30.626319+0000","last_active":"2023-09-23T19:04:31.529365+0000","last_peered":"2023-09-23T19:04:31.529365+0000","last_clean":"2023-09-23T19:02:20.594240+0000","last_became_active":"2023-09-23T19:02:30.626319+0000","last_became_peered":"2023-09-23T19:02:30.626319+0000","last_unstale":"2023-09-23T19:04:31.529365+0000","last_undegraded":"2023-09-23T19:04:31.529365+0000","last_fullsized":"2023-09-23T19:02:30.606198+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:59:13.925319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":104,"reported_epoch":334,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:04:32.481845+0000","last_change":"2023-09-23T18:59:14.702243+0000","last_active":"2023-09-23T19:04:32.481845+0000","last_peered":"2023-09-23T19:04:32.481845+0000","last_clean":"2023-09-23T19:04:32.481845+0000","last_became_active":"2023-09-23T18:59:14.698763+0000","last_became_peered":"2023-09-23T18:59:14.698763+0000","last_unstale":"2023-09-23T19:04:32.481845+0000","last_undegraded":"2023-09-23T19:04:32.481845+0000","last_fullsized":"2023-09-23T19:04:32.481845+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:06.839110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0,4],"avail_no_missing":["3","0","4"],"object_location_counts":[{"shards":"0,3,4","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":280,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T19:05:07.448449+0000","last_change":"2023-09-23T18:33:53.850096+0000","last_active":"2023-09-23T19:05:07.448449+0000","last_peered":"2023-09-23T19:05:07.448449+0000","last_clean":"2023-09-23T19:05:07.448449+0000","last_became_active":"2023-09-23T18:33:53.849811+0000","last_became_peered":"2023-09-23T18:33:53.849811+0000","last_unstale":"2023-09-23T19:05:07.448449+0000","last_undegraded":"2023-09-23T19:05:07.448449+0000","last_fullsized":"2023-09-23T19:05:07.448449+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:44.938072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":55,"reported_epoch":335,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:05:07.436175+0000","last_change":"2023-09-23T18:59:14.708879+0000","last_active":"2023-09-23T19:05:07.436175+0000","last_peered":"2023-09-23T19:05:07.436175+0000","last_clean":"2023-09-23T19:05:07.436175+0000","last_became_active":"2023-09-23T18:59:14.691340+0000","last_became_peered":"2023-09-23T18:59:14.691340+0000","last_unstale":"2023-09-23T19:05:07.436175+0000","last_undegraded":"2023-09-23T19:05:07.436175+0000","last_fullsized":"2023-09-23T19:05:07.436175+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:59:30.307834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":4}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":283,"reported_epoch":335,"state":"active+undersized","last_fresh":"2023-09-23T19:05:07.447929+0000","last_change":"2023-09-23T19:02:30.625696+0000","last_active":"2023-09-23T19:05:07.447929+0000","last_peered":"2023-09-23T19:05:07.447929+0000","last_clean":"2023-09-23T19:02:20.573807+0000","last_became_active":"2023-09-23T19:02:30.625696+0000","last_became_peered":"2023-09-23T19:02:30.625696+0000","last_unstale":"2023-09-23T19:05:07.447929+0000","last_undegraded":"2023-09-23T19:05:07.447929+0000","last_fullsized":"2023-09-23T19:02:30.607959+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:19.914239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":83,"reported_epoch":335,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:05:07.442263+0000","last_change":"2023-09-23T18:59:14.698497+0000","last_active":"2023-09-23T19:05:07.442263+0000","last_peered":"2023-09-23T19:05:07.442263+0000","last_clean":"2023-09-23T19:05:07.442263+0000","last_became_active":"2023-09-23T18:59:14.691108+0000","last_became_peered":"2023-09-23T18:59:14.691108+0000","last_unstale":"2023-09-23T19:05:07.442263+0000","last_undegraded":"2023-09-23T19:05:07.442263+0000","last_fullsized":"2023-09-23T19:05:07.442263+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:24.885822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":340,"reported_epoch":334,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:04:31.526551+0000","last_change":"2023-09-23T18:58:39.772503+0000","last_active":"2023-09-23T19:04:31.526551+0000","last_peered":"2023-09-23T19:04:31.526551+0000","last_clean":"2023-09-23T19:04:31.526551+0000","last_became_active":"2023-09-23T18:58:39.771729+0000","last_became_peered":"2023-09-23T18:58:39.771729+0000","last_unstale":"2023-09-23T19:04:31.526551+0000","last_undegraded":"2023-09-23T19:04:31.526551+0000","last_fullsized":"2023-09-23T19:04:31.526551+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:02.866230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1,2],"avail_no_missing":["3","1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":16,"reported_epoch":334,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:04:31.532431+0000","last_change":"2023-09-23T19:03:26.354464+0000","last_active":"2023-09-23T19:04:31.532431+0000","last_peered":"2023-09-23T19:04:31.532431+0000","last_clean":"2023-09-23T19:04:31.532431+0000","last_became_active":"2023-09-23T19:03:26.354012+0000","last_became_peered":"2023-09-23T19:03:26.354012+0000","last_unstale":"2023-09-23T19:04:31.532431+0000","last_undegraded":"2023-09-23T19:04:31.532431+0000","last_fullsized":"2023-09-23T19:04:31.532431+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:04.272355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0,1],"avail_no_missing":["3","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":273,"reported_epoch":334,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:04:32.481771+0000","last_change":"2023-09-23T19:03:25.326773+0000","last_active":"2023-09-23T19:04:32.481771+0000","last_peered":"2023-09-23T19:04:32.481771+0000","last_clean":"2023-09-23T19:04:32.481771+0000","last_became_active":"2023-09-23T19:03:25.326155+0000","last_became_peered":"2023-09-23T19:03:25.326155+0000","last_unstale":"2023-09-23T19:04:32.481771+0000","last_undegraded":"2023-09-23T19:04:32.481771+0000","last_fullsized":"2023-09-23T19:04:32.481771+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:14.608502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3],"acting":[3,6,0],"avail_no_missing":["3","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":45,"reported_epoch":335,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:05:07.437942+0000","last_change":"2023-09-23T18:59:14.695207+0000","last_active":"2023-09-23T19:05:07.437942+0000","last_peered":"2023-09-23T19:05:07.437942+0000","last_clean":"2023-09-23T19:05:07.437942+0000","last_became_active":"2023-09-23T18:59:14.693618+0000","last_became_peered":"2023-09-23T18:59:14.693618+0000","last_unstale":"2023-09-23T19:05:07.437942+0000","last_undegraded":"2023-09-23T19:05:07.437942+0000","last_fullsized":"2023-09-23T19:05:07.437942+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:46.279818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,4],"avail_no_missing":["0","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":59,"reported_epoch":334,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:04:31.526170+0000","last_change":"2023-09-23T19:02:59.704883+0000","last_active":"2023-09-23T19:04:31.526170+0000","last_peered":"2023-09-23T19:04:31.526170+0000","last_clean":"2023-09-23T19:04:31.526170+0000","last_became_active":"2023-09-23T19:02:59.704031+0000","last_became_peered":"2023-09-23T19:02:59.704031+0000","last_unstale":"2023-09-23T19:04:31.526170+0000","last_undegraded":"2023-09-23T19:04:31.526170+0000","last_fullsized":"2023-09-23T19:04:31.526170+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:56.818608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0,2],"avail_no_missing":["3","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":186,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T19:05:07.427506+0000","last_change":"2023-09-23T18:58:38.793506+0000","last_active":"2023-09-23T19:05:07.427506+0000","last_peered":"2023-09-23T19:05:07.427506+0000","last_clean":"2023-09-23T19:05:07.427506+0000","last_became_active":"2023-09-23T18:58:38.706537+0000","last_became_peered":"2023-09-23T18:58:38.706537+0000","last_unstale":"2023-09-23T19:05:07.427506+0000","last_undegraded":"2023-09-23T19:05:07.427506+0000","last_fullsized":"2023-09-23T19:05:07.427506+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:59.240510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":298,"reported_epoch":335,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:05:07.448097+0000","last_change":"2023-09-23T19:02:59.703185+0000","last_active":"2023-09-23T19:05:07.448097+0000","last_peered":"2023-09-23T19:05:07.448097+0000","last_clean":"2023-09-23T19:05:07.448097+0000","last_became_active":"2023-09-23T19:02:59.702510+0000","last_became_peered":"2023-09-23T19:02:59.702510+0000","last_unstale":"2023-09-23T19:05:07.448097+0000","last_undegraded":"2023-09-23T19:05:07.448097+0000","last_fullsized":"2023-09-23T19:05:07.448097+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:06.519928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2,6],"avail_no_missing":["1","2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.6","version":"335'12834","reported_seq":12865,"reported_epoch":335,"state":"active+undersized","last_fresh":"2023-09-23T19:05:19.679634+0000","last_change":"2023-09-23T19:05:19.679561+0000","last_active":"2023-09-23T19:05:19.679634+0000","last_peered":"2023-09-23T19:05:19.679634+0000","last_clean":"2023-09-23T19:02:29.534597+0000","last_became_active":"2023-09-23T19:02:30.622901+0000","last_became_peered":"2023-09-23T19:02:30.622901+0000","last_unstale":"2023-09-23T19:05:19.679634+0000","last_undegraded":"2023-09-23T19:05:19.679634+0000","last_fullsized":"2023-09-23T19:02:30.614700+0000","mapping_epoch":323,"log_start":"324'6400","ondisk_log_start":"324'6400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":6434,"log_dups_size":0,"ondisk_log_size":6434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:04.404317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12834,"num_write_kb":410688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":61,"reported_epoch":335,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:05:07.448495+0000","last_change":"2023-09-23T18:59:14.728684+0000","last_active":"2023-09-23T19:05:07.448495+0000","last_peered":"2023-09-23T19:05:07.448495+0000","last_clean":"2023-09-23T19:05:07.448495+0000","last_became_active":"2023-09-23T18:59:14.720265+0000","last_became_peered":"2023-09-23T18:59:14.720265+0000","last_unstale":"2023-09-23T19:05:07.448495+0000","last_undegraded":"2023-09-23T19:05:07.448495+0000","last_fullsized":"2023-09-23T19:05:07.448495+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:03.133014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,2],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":190,"reported_epoch":335,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:05:07.442350+0000","last_change":"2023-09-23T19:03:26.327803+0000","last_active":"2023-09-23T19:05:07.442350+0000","last_peered":"2023-09-23T19:05:07.442350+0000","last_clean":"2023-09-23T19:05:07.442350+0000","last_became_active":"2023-09-23T19:03:26.327084+0000","last_became_peered":"2023-09-23T19:03:26.327084+0000","last_unstale":"2023-09-23T19:05:07.442350+0000","last_undegraded":"2023-09-23T19:05:07.442350+0000","last_fullsized":"2023-09-23T19:05:07.442350+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6],"acting":[6,3,4],"avail_no_missing":["6","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.7","version":"335'12364","reported_seq":21309,"reported_epoch":336,"state":"active+recovering+remapped","last_fresh":"2023-09-23T19:05:39.942560+0000","last_change":"2023-09-23T19:03:26.795580+0000","last_active":"2023-09-23T19:05:39.942560+0000","last_peered":"2023-09-23T19:05:39.942560+0000","last_clean":"2023-09-23T19:03:24.271725+0000","last_became_active":"2023-09-23T19:03:26.672791+0000","last_became_peered":"2023-09-23T19:03:26.672791+0000","last_unstale":"2023-09-23T19:05:39.942560+0000","last_undegraded":"2023-09-23T19:05:39.942560+0000","last_fullsized":"2023-09-23T19:05:39.942560+0000","mapping_epoch":331,"log_start":"324'5951","ondisk_log_start":"324'5951","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":6413,"log_dups_size":0,"ondisk_log_size":6413,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:20.660309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1728,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12364,"num_write_kb":395648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4454,"num_bytes_recovered":125632512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[0,6,2],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":-1728}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":81,"reported_epoch":335,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:05:07.440368+0000","last_change":"2023-09-23T18:59:14.845077+0000","last_active":"2023-09-23T19:05:07.440368+0000","last_peered":"2023-09-23T19:05:07.440368+0000","last_clean":"2023-09-23T19:05:07.440368+0000","last_became_active":"2023-09-23T18:59:14.731245+0000","last_became_peered":"2023-09-23T18:59:14.731245+0000","last_unstale":"2023-09-23T19:05:07.440368+0000","last_undegraded":"2023-09-23T19:05:07.440368+0000","last_fullsized":"2023-09-23T19:05:07.440368+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:36:54.746484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0,4],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":307,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T19:05:07.448125+0000","last_change":"2023-09-23T19:03:25.324518+0000","last_active":"2023-09-23T19:05:07.448125+0000","last_peered":"2023-09-23T19:05:07.448125+0000","last_clean":"2023-09-23T19:05:07.448125+0000","last_became_active":"2023-09-23T19:03:25.324166+0000","last_became_peered":"2023-09-23T19:03:25.324166+0000","last_unstale":"2023-09-23T19:05:07.448125+0000","last_undegraded":"2023-09-23T19:05:07.448125+0000","last_fullsized":"2023-09-23T19:05:07.448125+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:24.379778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"335'12384","reported_seq":12414,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T19:05:19.663060+0000","last_change":"2023-09-23T19:00:44.631802+0000","last_active":"2023-09-23T19:05:19.663060+0000","last_peered":"2023-09-23T19:05:19.663060+0000","last_clean":"2023-09-23T19:05:19.663060+0000","last_became_active":"2023-09-23T19:00:44.631445+0000","last_became_peered":"2023-09-23T19:00:44.631445+0000","last_unstale":"2023-09-23T19:05:19.663060+0000","last_undegraded":"2023-09-23T19:05:19.663060+0000","last_fullsized":"2023-09-23T19:05:19.663060+0000","mapping_epoch":317,"log_start":"324'6000","ondisk_log_start":"324'6000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":6384,"log_dups_size":0,"ondisk_log_size":6384,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:19.854112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12384,"num_write_kb":396288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":27,"reported_epoch":335,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:05:07.443310+0000","last_change":"2023-09-23T19:03:26.327574+0000","last_active":"2023-09-23T19:05:07.443310+0000","last_peered":"2023-09-23T19:05:07.443310+0000","last_clean":"2023-09-23T19:05:07.443310+0000","last_became_active":"2023-09-23T19:03:26.326969+0000","last_became_peered":"2023-09-23T19:03:26.326969+0000","last_unstale":"2023-09-23T19:05:07.443310+0000","last_undegraded":"2023-09-23T19:05:07.443310+0000","last_fullsized":"2023-09-23T19:05:07.443310+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6],"acting":[6,3,4],"avail_no_missing":["6","3","4"],"object_location_counts":[{"shards":"3,4,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":190,"reported_epoch":335,"state":"active+undersized","last_fresh":"2023-09-23T19:05:27.520354+0000","last_change":"2023-09-23T18:54:52.349594+0000","last_active":"2023-09-23T19:05:27.520354+0000","last_peered":"2023-09-23T19:05:27.520354+0000","last_clean":"2023-09-23T18:54:43.304423+0000","last_became_active":"2023-09-23T18:54:52.349594+0000","last_became_peered":"2023-09-23T18:54:52.349594+0000","last_unstale":"2023-09-23T19:05:27.520354+0000","last_undegraded":"2023-09-23T19:05:27.520354+0000","last_fullsized":"2023-09-23T18:54:52.338675+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:49.547901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.5","version":"335'12418","reported_seq":12448,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T19:05:19.695481+0000","last_change":"2023-09-23T19:00:44.628649+0000","last_active":"2023-09-23T19:05:19.695481+0000","last_peered":"2023-09-23T19:05:19.695481+0000","last_clean":"2023-09-23T19:05:19.695481+0000","last_became_active":"2023-09-23T19:00:44.628304+0000","last_became_peered":"2023-09-23T19:00:44.628304+0000","last_unstale":"2023-09-23T19:05:19.695481+0000","last_undegraded":"2023-09-23T19:05:19.695481+0000","last_fullsized":"2023-09-23T19:05:19.695481+0000","mapping_epoch":317,"log_start":"321'4000","ondisk_log_start":"321'4000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":8418,"log_dups_size":0,"ondisk_log_size":8418,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:59.345981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12418,"num_write_kb":397376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":59,"reported_epoch":334,"state":"active+undersized","last_fresh":"2023-09-23T19:04:47.509580+0000","last_change":"2023-09-23T19:02:30.625752+0000","last_active":"2023-09-23T19:04:47.509580+0000","last_peered":"2023-09-23T19:04:47.509580+0000","last_clean":"2023-09-23T18:58:10.507610+0000","last_became_active":"2023-09-23T19:02:30.625752+0000","last_became_peered":"2023-09-23T19:02:30.625752+0000","last_unstale":"2023-09-23T19:04:47.509580+0000","last_undegraded":"2023-09-23T19:04:47.509580+0000","last_fullsized":"2023-09-23T19:02:30.604740+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:57.670803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":337,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T19:05:07.427052+0000","last_change":"2023-09-23T18:59:13.690091+0000","last_active":"2023-09-23T19:05:07.427052+0000","last_peered":"2023-09-23T19:05:07.427052+0000","last_clean":"2023-09-23T19:05:07.427052+0000","last_became_active":"2023-09-23T18:59:13.689532+0000","last_became_peered":"2023-09-23T18:59:13.689532+0000","last_unstale":"2023-09-23T19:05:07.427052+0000","last_undegraded":"2023-09-23T19:05:07.427052+0000","last_fullsized":"2023-09-23T19:05:07.427052+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:09.886527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"335'12746","reported_seq":12803,"reported_epoch":335,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T19:05:19.682916+0000","last_change":"2023-09-23T19:03:26.750306+0000","last_active":"2023-09-23T19:05:19.682916+0000","last_peered":"2023-09-23T19:05:19.682916+0000","last_clean":"2023-09-23T19:03:24.314992+0000","last_became_active":"2023-09-23T19:03:26.463415+0000","last_became_peered":"2023-09-23T19:03:26.463415+0000","last_unstale":"2023-09-23T19:05:19.682916+0000","last_undegraded":"2023-09-23T19:05:19.682916+0000","last_fullsized":"2023-09-23T19:05:19.682916+0000","mapping_epoch":331,"log_start":"324'6800","ondisk_log_start":"324'6800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":5946,"log_dups_size":0,"ondisk_log_size":5946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:13.918706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12746,"num_write_kb":407872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,6],"avail_no_missing":["1","0","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":67,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T19:05:07.432579+0000","last_change":"2023-09-23T18:59:13.721533+0000","last_active":"2023-09-23T19:05:07.432579+0000","last_peered":"2023-09-23T19:05:07.432579+0000","last_clean":"2023-09-23T19:05:07.432579+0000","last_became_active":"2023-09-23T18:59:13.711121+0000","last_became_peered":"2023-09-23T18:59:13.711121+0000","last_unstale":"2023-09-23T19:05:07.432579+0000","last_undegraded":"2023-09-23T19:05:07.432579+0000","last_fullsized":"2023-09-23T19:05:07.432579+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:51.950774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":269,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-23T19:04:31.526382+0000","last_change":"2023-09-23T18:59:13.702825+0000","last_active":"2023-09-23T19:04:31.526382+0000","last_peered":"2023-09-23T19:04:31.526382+0000","last_clean":"2023-09-23T19:04:31.526382+0000","last_became_active":"2023-09-23T18:59:13.702503+0000","last_became_peered":"2023-09-23T18:59:13.702503+0000","last_unstale":"2023-09-23T19:04:31.526382+0000","last_undegraded":"2023-09-23T19:04:31.526382+0000","last_fullsized":"2023-09-23T19:04:31.526382+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.263337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.3","version":"335'12488","reported_seq":12523,"reported_epoch":335,"state":"active+undersized","last_fresh":"2023-09-23T19:05:19.693463+0000","last_change":"2023-09-23T19:05:38.639821+0000","last_active":"2023-09-23T19:05:19.693463+0000","last_peered":"2023-09-23T19:05:19.693463+0000","last_clean":"2023-09-23T19:02:29.558711+0000","last_became_active":"2023-09-23T19:02:30.624617+0000","last_became_peered":"2023-09-23T19:02:30.624617+0000","last_unstale":"2023-09-23T19:05:19.693463+0000","last_undegraded":"2023-09-23T19:02:30.608827+0000","last_fullsized":"2023-09-23T19:02:30.608638+0000","mapping_epoch":323,"log_start":"321'4100","ondisk_log_start":"321'4100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":8388,"log_dups_size":0,"ondisk_log_size":8388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:27:21.404337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12488,"num_write_kb":399616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":174,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T19:05:07.448633+0000","last_change":"2023-09-23T18:59:13.708841+0000","last_active":"2023-09-23T19:05:07.448633+0000","last_peered":"2023-09-23T19:05:07.448633+0000","last_clean":"2023-09-23T19:05:07.448633+0000","last_became_active":"2023-09-23T18:59:13.707760+0000","last_became_peered":"2023-09-23T18:59:13.707760+0000","last_unstale":"2023-09-23T19:05:07.448633+0000","last_undegraded":"2023-09-23T19:05:07.448633+0000","last_fullsized":"2023-09-23T19:05:07.448633+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:02.417127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":324,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T19:05:07.436879+0000","last_change":"2023-09-23T18:59:13.708710+0000","last_active":"2023-09-23T19:05:07.436879+0000","last_peered":"2023-09-23T19:05:07.436879+0000","last_clean":"2023-09-23T19:05:07.436879+0000","last_became_active":"2023-09-23T18:59:13.708349+0000","last_became_peered":"2023-09-23T18:59:13.708349+0000","last_unstale":"2023-09-23T19:05:07.436879+0000","last_undegraded":"2023-09-23T19:05:07.436879+0000","last_fullsized":"2023-09-23T19:05:07.436879+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:33.033640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"335'12428","reported_seq":12455,"reported_epoch":335,"state":"active+undersized","last_fresh":"2023-09-23T19:05:19.693019+0000","last_change":"2023-09-23T19:05:39.238501+0000","last_active":"2023-09-23T19:05:19.693019+0000","last_peered":"2023-09-23T19:05:19.693019+0000","last_clean":"2023-09-23T19:00:43.602956+0000","last_became_active":"2023-09-23T19:00:44.621693+0000","last_became_peered":"2023-09-23T19:00:44.621693+0000","last_unstale":"2023-09-23T19:05:19.693019+0000","last_undegraded":"2023-09-23T19:01:09.090961+0000","last_fullsized":"2023-09-23T19:00:44.616332+0000","mapping_epoch":317,"log_start":"324'6500","ondisk_log_start":"324'6500","created":317,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":5928,"log_dups_size":0,"ondisk_log_size":5928,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:50.488964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12428,"num_write_kb":397696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":239,"reported_epoch":335,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:05:07.436200+0000","last_change":"2023-09-23T18:59:14.692414+0000","last_active":"2023-09-23T19:05:07.436200+0000","last_peered":"2023-09-23T19:05:07.436200+0000","last_clean":"2023-09-23T19:05:07.436200+0000","last_became_active":"2023-09-23T18:59:14.692033+0000","last_became_peered":"2023-09-23T18:59:14.692033+0000","last_unstale":"2023-09-23T19:05:07.436200+0000","last_undegraded":"2023-09-23T19:05:07.436200+0000","last_fullsized":"2023-09-23T19:05:07.436200+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:13.314042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1,4],"avail_no_missing":["2","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":277,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T19:05:07.439981+0000","last_change":"2023-09-23T18:59:13.708690+0000","last_active":"2023-09-23T19:05:07.439981+0000","last_peered":"2023-09-23T19:05:07.439981+0000","last_clean":"2023-09-23T19:05:07.439981+0000","last_became_active":"2023-09-23T18:59:13.708248+0000","last_became_peered":"2023-09-23T18:59:13.708248+0000","last_unstale":"2023-09-23T19:05:07.439981+0000","last_undegraded":"2023-09-23T19:05:07.439981+0000","last_fullsized":"2023-09-23T19:05:07.439981+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:43:08.360496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.1","version":"335'12462","reported_seq":12489,"reported_epoch":335,"state":"active+undersized","last_fresh":"2023-09-23T19:05:19.693149+0000","last_change":"2023-09-23T19:05:39.238219+0000","last_active":"2023-09-23T19:05:19.693149+0000","last_peered":"2023-09-23T19:05:19.693149+0000","last_clean":"2023-09-23T19:00:43.602956+0000","last_became_active":"2023-09-23T19:00:44.625145+0000","last_became_peered":"2023-09-23T19:00:44.625145+0000","last_unstale":"2023-09-23T19:05:19.693149+0000","last_undegraded":"2023-09-23T19:01:09.094661+0000","last_fullsized":"2023-09-23T19:00:44.616356+0000","mapping_epoch":317,"log_start":"324'6500","ondisk_log_start":"324'6500","created":317,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":5962,"log_dups_size":0,"ondisk_log_size":5962,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:11.841704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12462,"num_write_kb":398784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T19:05:07.432026+0000","last_change":"2023-09-23T19:03:26.331668+0000","last_active":"2023-09-23T19:05:07.432026+0000","last_peered":"2023-09-23T19:05:07.432026+0000","last_clean":"2023-09-23T19:05:07.432026+0000","last_became_active":"2023-09-23T19:03:26.330833+0000","last_became_peered":"2023-09-23T19:03:26.330833+0000","last_unstale":"2023-09-23T19:05:07.432026+0000","last_undegraded":"2023-09-23T19:05:07.432026+0000","last_fullsized":"2023-09-23T19:05:07.432026+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:39.740161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":330,"reported_epoch":335,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:05:36.971069+0000","last_change":"2023-09-23T18:59:14.719934+0000","last_active":"2023-09-23T19:05:36.971069+0000","last_peered":"2023-09-23T19:05:36.971069+0000","last_clean":"2023-09-23T19:05:36.971069+0000","last_became_active":"2023-09-23T18:59:14.719375+0000","last_became_peered":"2023-09-23T18:59:14.719375+0000","last_unstale":"2023-09-23T19:05:36.971069+0000","last_undegraded":"2023-09-23T19:05:36.971069+0000","last_fullsized":"2023-09-23T19:05:36.971069+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:20.953751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3,4],"avail_no_missing":["1","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"335'12788","reported_seq":17688,"reported_epoch":335,"state":"active+undersized+remapped+backfill_wait","last_fresh":"2023-09-23T19:05:19.933912+0000","last_change":"2023-09-23T19:05:19.697322+0000","last_active":"2023-09-23T19:05:19.933912+0000","last_peered":"2023-09-23T19:05:19.933912+0000","last_clean":"2023-09-23T19:00:43.602956+0000","last_became_active":"2023-09-23T19:02:59.772782+0000","last_became_peered":"2023-09-23T19:02:59.772782+0000","last_unstale":"2023-09-23T19:05:19.933912+0000","last_undegraded":"2023-09-23T19:05:19.933912+0000","last_fullsized":"2023-09-23T19:02:59.670100+0000","mapping_epoch":327,"log_start":"324'6400","ondisk_log_start":"324'6400","created":317,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":6388,"log_dups_size":0,"ondisk_log_size":6388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:47.015388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12788,"num_write_kb":409216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2430,"num_bytes_recovered":159252480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[0,6],"avail_no_missing":["0","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"77'130","reported_seq":818,"reported_epoch":335,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:05:16.966698+0000","last_change":"2023-09-23T18:54:52.351403+0000","last_active":"2023-09-23T19:05:16.966698+0000","last_peered":"2023-09-23T19:05:16.966698+0000","last_clean":"2023-09-23T18:54:47.819753+0000","last_became_active":"2023-09-23T18:54:52.351403+0000","last_became_peered":"2023-09-23T18:54:52.351403+0000","last_unstale":"2023-09-23T19:05:16.966698+0000","last_undegraded":"2023-09-23T18:54:52.342482+0000","last_fullsized":"2023-09-23T18:54:52.342376+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:54:59.818497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":772,"num_read_kb":1712,"num_write":126,"num_write_kb":2126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":59,"reported_epoch":334,"state":"active+undersized","last_fresh":"2023-09-23T19:04:37.482623+0000","last_change":"2023-09-23T19:02:30.614415+0000","last_active":"2023-09-23T19:04:37.482623+0000","last_peered":"2023-09-23T19:04:37.482623+0000","last_clean":"2023-09-23T18:58:10.507904+0000","last_became_active":"2023-09-23T19:02:30.614415+0000","last_became_peered":"2023-09-23T19:02:30.614415+0000","last_unstale":"2023-09-23T19:04:37.482623+0000","last_undegraded":"2023-09-23T19:04:37.482623+0000","last_fullsized":"2023-09-23T19:02:30.605923+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:17.947214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":312,"reported_epoch":335,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:05:07.448520+0000","last_change":"2023-09-23T19:03:26.348218+0000","last_active":"2023-09-23T19:05:07.448520+0000","last_peered":"2023-09-23T19:05:07.448520+0000","last_clean":"2023-09-23T19:05:07.448520+0000","last_became_active":"2023-09-23T19:03:26.347806+0000","last_became_peered":"2023-09-23T19:03:26.347806+0000","last_unstale":"2023-09-23T19:05:07.448520+0000","last_undegraded":"2023-09-23T19:05:07.448520+0000","last_fullsized":"2023-09-23T19:05:07.448520+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:41:48.025238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0,2],"avail_no_missing":["1","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"335'12432","reported_seq":12468,"reported_epoch":335,"state":"active+undersized","last_fresh":"2023-09-23T19:05:19.675216+0000","last_change":"2023-09-23T19:05:38.650537+0000","last_active":"2023-09-23T19:05:19.675216+0000","last_peered":"2023-09-23T19:05:19.675216+0000","last_clean":"2023-09-23T19:02:29.537539+0000","last_became_active":"2023-09-23T19:02:30.640299+0000","last_became_peered":"2023-09-23T19:02:30.640299+0000","last_unstale":"2023-09-23T19:05:19.675216+0000","last_undegraded":"2023-09-23T19:02:30.625838+0000","last_fullsized":"2023-09-23T19:02:30.613828+0000","mapping_epoch":323,"log_start":"324'6000","ondisk_log_start":"324'6000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":6432,"log_dups_size":0,"ondisk_log_size":6432,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:29.198990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12432,"num_write_kb":397824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":["0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'96","reported_seq":714,"reported_epoch":335,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:05:07.449033+0000","last_change":"2023-09-23T19:02:30.626876+0000","last_active":"2023-09-23T19:05:07.449033+0000","last_peered":"2023-09-23T19:05:07.449033+0000","last_clean":"2023-09-23T18:55:17.398523+0000","last_became_active":"2023-09-23T19:02:30.626876+0000","last_became_peered":"2023-09-23T19:02:30.626876+0000","last_unstale":"2023-09-23T19:05:07.449033+0000","last_undegraded":"2023-09-23T19:02:30.609918+0000","last_fullsized":"2023-09-23T19:02:30.609795+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":96,"log_dups_size":0,"ondisk_log_size":96,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:16.157139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":728,"num_read_kb":1234,"num_write":87,"num_write_kb":338,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":45,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T19:05:26.968594+0000","last_change":"2023-09-23T18:59:13.703057+0000","last_active":"2023-09-23T19:05:26.968594+0000","last_peered":"2023-09-23T19:05:26.968594+0000","last_clean":"2023-09-23T19:05:26.968594+0000","last_became_active":"2023-09-23T18:59:13.702759+0000","last_became_peered":"2023-09-23T18:59:13.702759+0000","last_unstale":"2023-09-23T19:05:26.968594+0000","last_undegraded":"2023-09-23T19:05:26.968594+0000","last_fullsized":"2023-09-23T19:05:26.968594+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:53.278738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":347,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T19:05:16.966680+0000","last_change":"2023-09-23T18:58:38.726546+0000","last_active":"2023-09-23T19:05:16.966680+0000","last_peered":"2023-09-23T19:05:16.966680+0000","last_clean":"2023-09-23T19:05:16.966680+0000","last_became_active":"2023-09-23T18:58:38.724765+0000","last_became_peered":"2023-09-23T18:58:38.724765+0000","last_unstale":"2023-09-23T19:05:16.966680+0000","last_undegraded":"2023-09-23T19:05:16.966680+0000","last_fullsized":"2023-09-23T19:05:16.966680+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:05.322676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"335'12544","reported_seq":12578,"reported_epoch":335,"state":"active+undersized+remapped","last_fresh":"2023-09-23T19:05:19.691966+0000","last_change":"2023-09-23T19:03:26.318882+0000","last_active":"2023-09-23T19:05:19.691966+0000","last_peered":"2023-09-23T19:05:19.691966+0000","last_clean":"2023-09-23T19:00:43.602956+0000","last_became_active":"2023-09-23T19:03:26.318882+0000","last_became_peered":"2023-09-23T19:03:26.318882+0000","last_unstale":"2023-09-23T19:05:19.691966+0000","last_undegraded":"2023-09-23T19:05:19.691966+0000","last_fullsized":"2023-09-23T19:03:26.316357+0000","mapping_epoch":331,"log_start":"321'4200","ondisk_log_start":"321'4200","created":317,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":8344,"log_dups_size":0,"ondisk_log_size":8344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:28.715984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12544,"num_write_kb":401408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"77'112","reported_seq":784,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T19:05:07.436145+0000","last_change":"2023-09-23T18:58:38.708598+0000","last_active":"2023-09-23T19:05:07.436145+0000","last_peered":"2023-09-23T19:05:07.436145+0000","last_clean":"2023-09-23T19:05:07.436145+0000","last_became_active":"2023-09-23T18:58:38.707923+0000","last_became_peered":"2023-09-23T18:58:38.707923+0000","last_unstale":"2023-09-23T19:05:07.436145+0000","last_undegraded":"2023-09-23T19:05:07.436145+0000","last_fullsized":"2023-09-23T19:05:07.436145+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:59.471903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":774,"num_read_kb":596,"num_write":102,"num_write_kb":52,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":282,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T19:05:07.449584+0000","last_change":"2023-09-23T18:33:53.849084+0000","last_active":"2023-09-23T19:05:07.449584+0000","last_peered":"2023-09-23T19:05:07.449584+0000","last_clean":"2023-09-23T19:05:07.449584+0000","last_became_active":"2023-09-23T18:33:53.848757+0000","last_became_peered":"2023-09-23T18:33:53.848757+0000","last_unstale":"2023-09-23T19:05:07.449584+0000","last_undegraded":"2023-09-23T19:05:07.449584+0000","last_fullsized":"2023-09-23T19:05:07.449584+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:15.267742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":343,"reported_epoch":335,"state":"active+undersized","last_fresh":"2023-09-23T19:05:07.447936+0000","last_change":"2023-09-23T19:02:30.625647+0000","last_active":"2023-09-23T19:05:07.447936+0000","last_peered":"2023-09-23T19:05:07.447936+0000","last_clean":"2023-09-23T19:02:20.581899+0000","last_became_active":"2023-09-23T19:02:30.625647+0000","last_became_peered":"2023-09-23T19:02:30.625647+0000","last_unstale":"2023-09-23T19:05:07.447936+0000","last_undegraded":"2023-09-23T19:05:07.447936+0000","last_fullsized":"2023-09-23T19:02:30.608980+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:50.131120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"335'12632","reported_seq":12662,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T19:05:19.657967+0000","last_change":"2023-09-23T19:00:44.626971+0000","last_active":"2023-09-23T19:05:19.657967+0000","last_peered":"2023-09-23T19:05:19.657967+0000","last_clean":"2023-09-23T19:05:19.657967+0000","last_became_active":"2023-09-23T19:00:44.626715+0000","last_became_peered":"2023-09-23T19:00:44.626715+0000","last_unstale":"2023-09-23T19:05:19.657967+0000","last_undegraded":"2023-09-23T19:05:19.657967+0000","last_fullsized":"2023-09-23T19:05:19.657967+0000","mapping_epoch":317,"log_start":"324'6700","ondisk_log_start":"324'6700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":5932,"log_dups_size":0,"ondisk_log_size":5932,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:22:07.209151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12632,"num_write_kb":404224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'147","reported_seq":737,"reported_epoch":335,"state":"active+undersized+remapped","last_fresh":"2023-09-23T19:05:07.436342+0000","last_change":"2023-09-23T19:02:30.615652+0000","last_active":"2023-09-23T19:05:07.436342+0000","last_peered":"2023-09-23T19:05:07.436342+0000","last_clean":"2023-09-23T18:55:17.397785+0000","last_became_active":"2023-09-23T19:02:30.615652+0000","last_became_peered":"2023-09-23T19:02:30.615652+0000","last_unstale":"2023-09-23T19:05:07.436342+0000","last_undegraded":"2023-09-23T19:05:07.436342+0000","last_fullsized":"2023-09-23T19:02:30.607368+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:55:03.566290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":624,"num_read_kb":1868,"num_write":143,"num_write_kb":5368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":348,"reported_epoch":335,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:05:07.426682+0000","last_change":"2023-09-23T18:54:52.350132+0000","last_active":"2023-09-23T19:05:07.426682+0000","last_peered":"2023-09-23T19:05:07.426682+0000","last_clean":"2023-09-23T18:21:39.626426+0000","last_became_active":"2023-09-23T18:54:52.350132+0000","last_became_peered":"2023-09-23T18:54:52.350132+0000","last_unstale":"2023-09-23T19:05:07.426682+0000","last_undegraded":"2023-09-23T18:54:52.335600+0000","last_fullsized":"2023-09-23T18:54:52.335485+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_clean_scrub_stamp":"2023-09-23T18:08:35.701374+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:27.569665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":["0","6"],"object_location_counts":[{"shards":"0,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":69,"reported_epoch":334,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:04:57.519886+0000","last_change":"2023-09-23T19:02:59.675237+0000","last_active":"2023-09-23T19:04:57.519886+0000","last_peered":"2023-09-23T19:04:57.519886+0000","last_clean":"2023-09-23T19:04:57.519886+0000","last_became_active":"2023-09-23T19:02:59.674730+0000","last_became_peered":"2023-09-23T19:02:59.674730+0000","last_unstale":"2023-09-23T19:04:57.519886+0000","last_undegraded":"2023-09-23T19:04:57.519886+0000","last_fullsized":"2023-09-23T19:04:57.519886+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:40.310570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6,1],"avail_no_missing":["3","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":46,"reported_epoch":335,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:05:29.937245+0000","last_change":"2023-09-23T18:59:14.692323+0000","last_active":"2023-09-23T19:05:29.937245+0000","last_peered":"2023-09-23T19:05:29.937245+0000","last_clean":"2023-09-23T19:05:29.937245+0000","last_became_active":"2023-09-23T18:59:14.691887+0000","last_became_peered":"2023-09-23T18:59:14.691887+0000","last_unstale":"2023-09-23T19:05:29.937245+0000","last_undegraded":"2023-09-23T19:05:29.937245+0000","last_fullsized":"2023-09-23T19:05:29.937245+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:30.126169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,4],"avail_no_missing":["0","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.8","version":"335'12880","reported_seq":12907,"reported_epoch":335,"state":"active+undersized","last_fresh":"2023-09-23T19:05:19.682523+0000","last_change":"2023-09-23T19:05:19.682523+0000","last_active":"2023-09-23T19:05:19.682523+0000","last_peered":"2023-09-23T19:05:19.682523+0000","last_clean":"2023-09-23T19:00:43.602956+0000","last_became_active":"2023-09-23T19:00:44.629456+0000","last_became_peered":"2023-09-23T19:00:44.629456+0000","last_unstale":"2023-09-23T19:05:19.682523+0000","last_undegraded":"2023-09-23T19:05:19.682523+0000","last_fullsized":"2023-09-23T19:00:44.620975+0000","mapping_epoch":317,"log_start":"321'4500","ondisk_log_start":"321'4500","created":317,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":8380,"log_dups_size":0,"ondisk_log_size":8380,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:26.773324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12880,"num_write_kb":412160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"77'120","reported_seq":568,"reported_epoch":335,"state":"active+undersized","last_fresh":"2023-09-23T19:05:07.449674+0000","last_change":"2023-09-23T18:54:52.356024+0000","last_active":"2023-09-23T19:05:07.449674+0000","last_peered":"2023-09-23T19:05:07.449674+0000","last_clean":"2023-09-23T18:17:46.566093+0000","last_became_active":"2023-09-23T18:54:52.356024+0000","last_became_peered":"2023-09-23T18:54:52.356024+0000","last_unstale":"2023-09-23T19:05:07.449674+0000","last_undegraded":"2023-09-23T19:05:07.449674+0000","last_fullsized":"2023-09-23T18:54:52.345483+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":120,"log_dups_size":0,"ondisk_log_size":120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:09.374165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":809,"num_read_kb":2749,"num_write":98,"num_write_kb":2087,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":370,"reported_epoch":335,"state":"active+undersized+remapped","last_fresh":"2023-09-23T19:05:07.437726+0000","last_change":"2023-09-23T19:02:30.619496+0000","last_active":"2023-09-23T19:05:07.437726+0000","last_peered":"2023-09-23T19:05:07.437726+0000","last_clean":"2023-09-23T19:02:20.582983+0000","last_became_active":"2023-09-23T19:02:30.619496+0000","last_became_peered":"2023-09-23T19:02:30.619496+0000","last_unstale":"2023-09-23T19:05:07.437726+0000","last_undegraded":"2023-09-23T19:05:07.437726+0000","last_fullsized":"2023-09-23T19:02:30.613352+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:55:33.179195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":368,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T19:05:29.937430+0000","last_change":"2023-09-23T18:51:43.087834+0000","last_active":"2023-09-23T19:05:29.937430+0000","last_peered":"2023-09-23T19:05:29.937430+0000","last_clean":"2023-09-23T19:05:29.937430+0000","last_became_active":"2023-09-23T18:51:43.087309+0000","last_became_peered":"2023-09-23T18:51:43.087309+0000","last_unstale":"2023-09-23T19:05:29.937430+0000","last_undegraded":"2023-09-23T19:05:29.937430+0000","last_fullsized":"2023-09-23T19:05:29.937430+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:53.913425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"335'12396","reported_seq":12427,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T19:05:19.934138+0000","last_change":"2023-09-23T19:00:44.631488+0000","last_active":"2023-09-23T19:05:19.934138+0000","last_peered":"2023-09-23T19:05:19.934138+0000","last_clean":"2023-09-23T19:05:19.934138+0000","last_became_active":"2023-09-23T19:00:44.628764+0000","last_became_peered":"2023-09-23T19:00:44.628764+0000","last_unstale":"2023-09-23T19:05:19.934138+0000","last_undegraded":"2023-09-23T19:05:19.934138+0000","last_fullsized":"2023-09-23T19:05:19.934138+0000","mapping_epoch":317,"log_start":"324'6000","ondisk_log_start":"324'6000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":6396,"log_dups_size":0,"ondisk_log_size":6396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:01.178133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12396,"num_write_kb":396609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'78","reported_seq":406,"reported_epoch":335,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:05:24.935053+0000","last_change":"2023-09-23T18:59:14.857977+0000","last_active":"2023-09-23T19:05:24.935053+0000","last_peered":"2023-09-23T19:05:24.935053+0000","last_clean":"2023-09-23T19:05:24.935053+0000","last_became_active":"2023-09-23T18:59:14.720066+0000","last_became_peered":"2023-09-23T18:59:14.720066+0000","last_unstale":"2023-09-23T19:05:24.935053+0000","last_undegraded":"2023-09-23T19:05:24.935053+0000","last_fullsized":"2023-09-23T19:05:24.935053+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":78,"log_dups_size":0,"ondisk_log_size":78,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:18.169794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":428,"num_read_kb":1125,"num_write":73,"num_write_kb":431,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,4],"avail_no_missing":["0","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":317,"reported_epoch":335,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:05:16.966774+0000","last_change":"2023-09-23T19:03:26.336256+0000","last_active":"2023-09-23T19:05:16.966774+0000","last_peered":"2023-09-23T19:05:16.966774+0000","last_clean":"2023-09-23T19:05:16.966774+0000","last_became_active":"2023-09-23T19:03:26.335803+0000","last_became_peered":"2023-09-23T19:03:26.335803+0000","last_unstale":"2023-09-23T19:05:16.966774+0000","last_undegraded":"2023-09-23T19:05:16.966774+0000","last_fullsized":"2023-09-23T19:05:16.966774+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:52.934062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2,0],"avail_no_missing":["1","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":263,"reported_epoch":335,"state":"active+undersized+remapped","last_fresh":"2023-09-23T19:05:07.427317+0000","last_change":"2023-09-23T19:02:30.622990+0000","last_active":"2023-09-23T19:05:07.427317+0000","last_peered":"2023-09-23T19:05:07.427317+0000","last_clean":"2023-09-23T18:51:34.082714+0000","last_became_active":"2023-09-23T19:02:30.622990+0000","last_became_peered":"2023-09-23T19:02:30.622990+0000","last_unstale":"2023-09-23T19:05:07.427317+0000","last_undegraded":"2023-09-23T19:05:07.427317+0000","last_fullsized":"2023-09-23T19:02:30.617856+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:35:00.270059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"335'12420","reported_seq":12460,"reported_epoch":335,"state":"active+undersized","last_fresh":"2023-09-23T19:05:19.644985+0000","last_change":"2023-09-23T19:05:19.644985+0000","last_active":"2023-09-23T19:05:19.644985+0000","last_peered":"2023-09-23T19:05:19.644985+0000","last_clean":"2023-09-23T19:02:29.601318+0000","last_became_active":"2023-09-23T19:02:30.634999+0000","last_became_peered":"2023-09-23T19:02:30.634999+0000","last_unstale":"2023-09-23T19:05:19.644985+0000","last_undegraded":"2023-09-23T19:05:19.644985+0000","last_fullsized":"2023-09-23T19:02:30.610107+0000","mapping_epoch":323,"log_start":"321'4000","ondisk_log_start":"321'4000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":8420,"log_dups_size":0,"ondisk_log_size":8420,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:42.741753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12420,"num_write_kb":397440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":332,"reported_epoch":335,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:05:07.434551+0000","last_change":"2023-09-23T19:02:30.624148+0000","last_active":"2023-09-23T19:05:07.434551+0000","last_peered":"2023-09-23T19:05:07.434551+0000","last_clean":"2023-09-23T19:02:20.598593+0000","last_became_active":"2023-09-23T19:02:30.624148+0000","last_became_peered":"2023-09-23T19:02:30.624148+0000","last_unstale":"2023-09-23T19:05:07.434551+0000","last_undegraded":"2023-09-23T19:02:30.606836+0000","last_fullsized":"2023-09-23T19:02:30.606707+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:37:10.194825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'181","reported_seq":877,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T19:05:07.440059+0000","last_change":"2023-09-23T18:59:16.452945+0000","last_active":"2023-09-23T19:05:07.440059+0000","last_peered":"2023-09-23T19:05:07.440059+0000","last_clean":"2023-09-23T19:05:07.440059+0000","last_became_active":"2023-09-23T18:59:16.452030+0000","last_became_peered":"2023-09-23T18:59:16.452030+0000","last_unstale":"2023-09-23T19:05:07.440059+0000","last_undegraded":"2023-09-23T19:05:07.440059+0000","last_fullsized":"2023-09-23T19:05:07.440059+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":181,"log_dups_size":0,"ondisk_log_size":181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:20.988663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":909,"num_read_kb":734,"num_write":148,"num_write_kb":75,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":17,"reported_epoch":335,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:05:07.431873+0000","last_change":"2023-09-23T19:03:26.331922+0000","last_active":"2023-09-23T19:05:07.431873+0000","last_peered":"2023-09-23T19:05:07.431873+0000","last_clean":"2023-09-23T19:05:07.431873+0000","last_became_active":"2023-09-23T19:03:26.331272+0000","last_became_peered":"2023-09-23T19:03:26.331272+0000","last_unstale":"2023-09-23T19:05:07.431873+0000","last_undegraded":"2023-09-23T19:05:07.431873+0000","last_fullsized":"2023-09-23T19:05:07.431873+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:04.741357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0,1],"avail_no_missing":["3","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"335'12704","reported_seq":12733,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T19:05:19.641995+0000","last_change":"2023-09-23T19:00:44.634465+0000","last_active":"2023-09-23T19:05:19.641995+0000","last_peered":"2023-09-23T19:05:19.641995+0000","last_clean":"2023-09-23T19:05:19.641995+0000","last_became_active":"2023-09-23T19:00:44.634053+0000","last_became_peered":"2023-09-23T19:00:44.634053+0000","last_unstale":"2023-09-23T19:05:19.641995+0000","last_undegraded":"2023-09-23T19:05:19.641995+0000","last_fullsized":"2023-09-23T19:05:19.641995+0000","mapping_epoch":317,"log_start":"324'6124","ondisk_log_start":"324'6124","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T19:00:43.602956+0000","last_clean_scrub_stamp":"2023-09-23T19:00:43.602956+0000","objects_scrubbed":0,"log_size":6580,"log_dups_size":0,"ondisk_log_size":6580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:25.648826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12704,"num_write_kb":406528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"77'112","reported_seq":731,"reported_epoch":335,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:05:27.520522+0000","last_change":"2023-09-23T19:03:26.329608+0000","last_active":"2023-09-23T19:05:27.520522+0000","last_peered":"2023-09-23T19:05:27.520522+0000","last_clean":"2023-09-23T19:05:27.520522+0000","last_became_active":"2023-09-23T19:03:26.328839+0000","last_became_peered":"2023-09-23T19:03:26.328839+0000","last_unstale":"2023-09-23T19:05:27.520522+0000","last_undegraded":"2023-09-23T19:05:27.520522+0000","last_fullsized":"2023-09-23T19:05:27.520522+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:13.821966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":893,"num_read_kb":3172,"num_write":69,"num_write_kb":2237,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6,3],"avail_no_missing":["2","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":358,"reported_epoch":335,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:05:29.937463+0000","last_change":"2023-09-23T19:03:26.333232+0000","last_active":"2023-09-23T19:05:29.937463+0000","last_peered":"2023-09-23T19:05:29.937463+0000","last_clean":"2023-09-23T19:05:29.937463+0000","last_became_active":"2023-09-23T19:03:26.332703+0000","last_became_peered":"2023-09-23T19:03:26.332703+0000","last_unstale":"2023-09-23T19:05:29.937463+0000","last_undegraded":"2023-09-23T19:05:29.937463+0000","last_fullsized":"2023-09-23T19:05:29.937463+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:55.358005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,6],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":262,"reported_epoch":335,"state":"active+undersized","last_fresh":"2023-09-23T19:05:07.437476+0000","last_change":"2023-09-23T19:02:30.627687+0000","last_active":"2023-09-23T19:05:07.437476+0000","last_peered":"2023-09-23T19:05:07.437476+0000","last_clean":"2023-09-23T18:51:14.259249+0000","last_became_active":"2023-09-23T19:02:30.627687+0000","last_became_peered":"2023-09-23T19:02:30.627687+0000","last_unstale":"2023-09-23T19:05:07.437476+0000","last_undegraded":"2023-09-23T19:05:07.437476+0000","last_fullsized":"2023-09-23T19:02:30.613928+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:30.165582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":["0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":314,"reported_epoch":335,"state":"active+clean+remapped","last_fresh":"2023-09-23T19:05:07.427187+0000","last_change":"2023-09-23T18:59:14.720673+0000","last_active":"2023-09-23T19:05:07.427187+0000","last_peered":"2023-09-23T19:05:07.427187+0000","last_clean":"2023-09-23T19:05:07.427187+0000","last_became_active":"2023-09-23T18:59:14.720069+0000","last_became_peered":"2023-09-23T18:59:14.720069+0000","last_unstale":"2023-09-23T19:05:07.427187+0000","last_undegraded":"2023-09-23T19:05:07.427187+0000","last_fullsized":"2023-09-23T19:05:07.427187+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:42:55.501526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,4],"avail_no_missing":["0","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":125,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T19:05:07.436725+0000","last_change":"2023-09-23T18:58:38.715946+0000","last_active":"2023-09-23T19:05:07.436725+0000","last_peered":"2023-09-23T19:05:07.436725+0000","last_clean":"2023-09-23T19:05:07.436725+0000","last_became_active":"2023-09-23T18:58:38.715584+0000","last_became_peered":"2023-09-23T18:58:38.715584+0000","last_unstale":"2023-09-23T19:05:07.436725+0000","last_undegraded":"2023-09-23T19:05:07.436725+0000","last_fullsized":"2023-09-23T19:05:07.436725+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:57.811713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":56,"reported_epoch":335,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:05:07.447415+0000","last_change":"2023-09-23T19:02:30.626707+0000","last_active":"2023-09-23T19:05:07.447415+0000","last_peered":"2023-09-23T19:05:07.447415+0000","last_clean":"2023-09-23T18:15:12.312222+0000","last_became_active":"2023-09-23T19:02:30.626707+0000","last_became_peered":"2023-09-23T19:02:30.626707+0000","last_unstale":"2023-09-23T19:05:07.447415+0000","last_undegraded":"2023-09-23T19:02:30.612650+0000","last_fullsized":"2023-09-23T19:02:30.610389+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:28.560216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":321,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-23T19:04:31.525051+0000","last_change":"2023-09-23T18:59:13.712801+0000","last_active":"2023-09-23T19:04:31.525051+0000","last_peered":"2023-09-23T19:04:31.525051+0000","last_clean":"2023-09-23T19:04:31.525051+0000","last_became_active":"2023-09-23T18:59:13.710748+0000","last_became_peered":"2023-09-23T18:59:13.710748+0000","last_unstale":"2023-09-23T19:04:31.525051+0000","last_undegraded":"2023-09-23T19:04:31.525051+0000","last_fullsized":"2023-09-23T19:04:31.525051+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:26.256173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":72,"reported_epoch":335,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:05:07.439614+0000","last_change":"2023-09-23T19:02:30.626598+0000","last_active":"2023-09-23T19:05:07.439614+0000","last_peered":"2023-09-23T19:05:07.439614+0000","last_clean":"2023-09-23T18:59:05.479280+0000","last_became_active":"2023-09-23T19:02:30.626598+0000","last_became_peered":"2023-09-23T19:02:30.626598+0000","last_unstale":"2023-09-23T19:05:07.439614+0000","last_undegraded":"2023-09-23T19:02:30.608116+0000","last_fullsized":"2023-09-23T19:02:30.607988+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:33.450577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":286,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-23T19:04:31.532644+0000","last_change":"2023-09-23T18:59:13.699034+0000","last_active":"2023-09-23T19:04:31.532644+0000","last_peered":"2023-09-23T19:04:31.532644+0000","last_clean":"2023-09-23T19:04:31.532644+0000","last_became_active":"2023-09-23T18:59:13.698718+0000","last_became_peered":"2023-09-23T18:59:13.698718+0000","last_unstale":"2023-09-23T19:04:31.532644+0000","last_undegraded":"2023-09-23T19:04:31.532644+0000","last_fullsized":"2023-09-23T19:04:31.532644+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:57.449621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":50,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-23T19:04:42.501722+0000","last_change":"2023-09-23T18:58:38.711301+0000","last_active":"2023-09-23T19:04:42.501722+0000","last_peered":"2023-09-23T19:04:42.501722+0000","last_clean":"2023-09-23T19:04:42.501722+0000","last_became_active":"2023-09-23T18:58:38.710923+0000","last_became_peered":"2023-09-23T18:58:38.710923+0000","last_unstale":"2023-09-23T19:04:42.501722+0000","last_undegraded":"2023-09-23T19:04:42.501722+0000","last_fullsized":"2023-09-23T19:04:42.501722+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:17.756147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":49,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-23T19:04:31.526089+0000","last_change":"2023-09-23T18:58:38.711839+0000","last_active":"2023-09-23T19:04:31.526089+0000","last_peered":"2023-09-23T19:04:31.526089+0000","last_clean":"2023-09-23T19:04:31.526089+0000","last_became_active":"2023-09-23T18:58:38.711347+0000","last_became_peered":"2023-09-23T18:58:38.711347+0000","last_unstale":"2023-09-23T19:04:31.526089+0000","last_undegraded":"2023-09-23T19:04:31.526089+0000","last_fullsized":"2023-09-23T19:04:31.526089+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:55.319645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":401,"reported_epoch":335,"state":"active+undersized+degraded","last_fresh":"2023-09-23T19:05:07.448760+0000","last_change":"2023-09-23T19:02:30.624794+0000","last_active":"2023-09-23T19:05:07.448760+0000","last_peered":"2023-09-23T19:05:07.448760+0000","last_clean":"2023-09-23T19:02:20.581699+0000","last_became_active":"2023-09-23T19:02:30.624794+0000","last_became_peered":"2023-09-23T19:02:30.624794+0000","last_unstale":"2023-09-23T19:05:07.448760+0000","last_undegraded":"2023-09-23T19:02:30.616363+0000","last_fullsized":"2023-09-23T19:02:30.616248+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:51:49.970790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":286,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-09-23T19:05:07.449792+0000","last_change":"2023-09-23T19:02:58.680806+0000","last_active":"2023-09-23T19:05:07.449792+0000","last_peered":"2023-09-23T19:05:07.449792+0000","last_clean":"2023-09-23T19:05:07.449792+0000","last_became_active":"2023-09-23T19:02:58.680426+0000","last_became_peered":"2023-09-23T19:02:58.680426+0000","last_unstale":"2023-09-23T19:05:07.449792+0000","last_undegraded":"2023-09-23T19:05:07.449792+0000","last_fullsized":"2023-09-23T19:05:07.449792+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:16:15.128164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":76,"acting":88,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1728,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":200920,"num_write_kb":6429377,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6893,"num_bytes_recovered":285474816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1492975616,"data_stored":1492975616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":110745,"ondisk_log_size":110745,"up":38,"acting":39,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5,"num_objects_misplaced":13,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":8114,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38129,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":70,"acting":87,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":2,"acting":2,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5937,"num_read_kb":13190,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":104,"num_bytes_recovered":12582931,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5621,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":17,"acting":20,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430724,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":914504,"kb_used_data":7168,"kb_used_omap":0,"kb_used_meta":907328,"kb_avail":92830648,"statfs":{"total":95995035648,"available":95058583552,"internally_reserved":0,"allocated":7340032,"data_stored":6683781,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929103872},"hb_peers":[0,1,2,3,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:54:27 2023","interfaces":[{"interface":"back","average":{"1min":5.7489999999999997,"5min":2.3029999999999999,"15min":2.6070000000000002},"min":{"1min":0.29199999999999998,"5min":0.29099999999999998,"15min":0.24299999999999999},"max":{"1min":80.381,"5min":80.381,"15min":97.415999999999997},"last":0.86799999999999999},{"interface":"front","average":{"1min":5.7199999999999998,"5min":2.427,"15min":2.6829999999999998},"min":{"1min":0.376,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":79.781999999999996,"5min":79.781999999999996,"15min":96.581000000000003},"last":0.76300000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:54:27 2023","interfaces":[{"interface":"back","average":{"1min":4.9660000000000002,"5min":2.714,"15min":2.5459999999999998},"min":{"1min":0.29099999999999998,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":80.266000000000005,"5min":80.266000000000005,"15min":94.650999999999996},"last":0.38800000000000001},{"interface":"front","average":{"1min":4.9859999999999998,"5min":2.698,"15min":2.7330000000000001},"min":{"1min":0.38700000000000001,"5min":0.309,"15min":0.29299999999999998},"max":{"1min":80.224999999999994,"5min":80.224999999999994,"15min":94.843999999999994},"last":0.51100000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:54:22 2023","interfaces":[{"interface":"back","average":{"1min":5.2910000000000004,"5min":2.754,"15min":1.881},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.311},"max":{"1min":86.007999999999996,"5min":86.007999999999996,"15min":86.007999999999996},"last":0.84999999999999998},{"interface":"front","average":{"1min":5.6870000000000003,"5min":2.3679999999999999,"15min":1.788},"min":{"1min":0.45600000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":86.832999999999998,"5min":86.832999999999998,"15min":86.832999999999998},"last":0.78800000000000003}]},{"osd":3,"last update":"Sat Sep 23 18:54:48 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":2.7400000000000002,"15min":3.1019999999999999},"min":{"1min":0.38100000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":3.536,"5min":118.07599999999999,"15min":136.345},"last":0.83499999999999996},{"interface":"front","average":{"1min":0.70999999999999996,"5min":2.4710000000000001,"15min":3.1389999999999998},"min":{"1min":0.41599999999999998,"5min":0.34699999999999998,"15min":0.318},"max":{"1min":2.444,"5min":118.69,"15min":136.893},"last":0.73399999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:52:25 2023","interfaces":[{"interface":"back","average":{"1min":1.4450000000000001,"5min":1.1319999999999999,"15min":1.016},"min":{"1min":0.39400000000000002,"5min":0.32500000000000001,"15min":0.29199999999999998},"max":{"1min":6.4809999999999999,"5min":6.4809999999999999,"15min":6.4809999999999999},"last":1.3480000000000001},{"interface":"front","average":{"1min":1.48,"5min":1.73,"15min":1.2849999999999999},"min":{"1min":0.40999999999999998,"5min":0.38900000000000001,"15min":0.35099999999999998},"max":{"1min":6.1100000000000003,"5min":13.010999999999999,"15min":13.010999999999999},"last":0.61299999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:54:27 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.95999999999999996,"15min":1.1479999999999999},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.29799999999999999},"max":{"1min":0.84599999999999997,"5min":6.5780000000000003,"15min":8.7210000000000001},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.1459999999999999,"15min":1.1100000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.1220000000000001,"5min":8.6519999999999992,"15min":8.6519999999999992},"last":0.94599999999999995}]},{"osd":6,"last update":"Sat Sep 23 18:54:27 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":1.0620000000000001,"15min":1.179},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.35799999999999998},"max":{"1min":1.024,"5min":6.4500000000000002,"15min":6.4500000000000002},"last":0.90600000000000003},{"interface":"front","average":{"1min":0.65700000000000003,"5min":1.0609999999999999,"15min":1.179},"min":{"1min":0.40500000000000003,"5min":0.38900000000000001,"15min":0.36199999999999999},"max":{"1min":0.86499999999999999,"5min":7.6029999999999998,"15min":9.2850000000000001},"last":0.89500000000000002}]}]},{"osd":6,"up_from":42,"seq":180388627086,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":911428,"kb_used_data":3772,"kb_used_omap":0,"kb_used_meta":907648,"kb_avail":92833724,"statfs":{"total":95995035648,"available":95061733376,"internally_reserved":0,"allocated":3862528,"data_stored":3037634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929431552},"hb_peers":[0,1,2,3,4],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:04:53 2023","interfaces":[{"interface":"back","average":{"1min":2.7839999999999998,"5min":2.9969999999999999,"15min":2.0310000000000001},"min":{"1min":0.32400000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":20.850000000000001,"5min":100.648,"15min":100.648},"last":0.496},{"interface":"front","average":{"1min":2.843,"5min":3.0920000000000001,"15min":2.1309999999999998},"min":{"1min":0.373,"5min":0.317,"15min":0.317},"max":{"1min":20.794,"5min":100.70399999999999,"15min":100.70399999999999},"last":0.59299999999999997}]},{"osd":1,"last update":"Sat Sep 23 19:04:53 2023","interfaces":[{"interface":"back","average":{"1min":4.0519999999999996,"5min":1.8200000000000001,"15min":2.262},"min":{"1min":0.35399999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":32.149999999999999,"5min":32.149999999999999,"15min":107.88800000000001},"last":0.70099999999999996},{"interface":"front","average":{"1min":4.4109999999999996,"5min":1.643,"15min":2.1120000000000001},"min":{"1min":0.311,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":32.814,"5min":32.814,"15min":106.474},"last":0.81699999999999995}]},{"osd":2,"last update":"Sat Sep 23 19:04:53 2023","interfaces":[{"interface":"back","average":{"1min":5.0410000000000004,"5min":3.0049999999999999,"15min":1.819},"min":{"1min":0.35499999999999998,"5min":0.33200000000000002,"15min":0.29599999999999999},"max":{"1min":66.587999999999994,"5min":66.587999999999994,"15min":66.587999999999994},"last":0.63},{"interface":"front","average":{"1min":5.0860000000000003,"5min":3.0249999999999999,"15min":1.831},"min":{"1min":0.46300000000000002,"5min":0.372,"15min":0.35299999999999998},"max":{"1min":66.614000000000004,"5min":66.614000000000004,"15min":66.614000000000004},"last":0.65400000000000003}]},{"osd":3,"last update":"Sat Sep 23 19:05:04 2023","interfaces":[{"interface":"back","average":{"1min":2.2509999999999999,"5min":2.1419999999999999,"15min":2.1499999999999999},"min":{"1min":0.39700000000000002,"5min":0.38100000000000001,"15min":0.34000000000000002},"max":{"1min":10.878,"5min":93.456000000000003,"15min":93.456000000000003},"last":0.78900000000000003},{"interface":"front","average":{"1min":2.2869999999999999,"5min":2.145,"15min":2.2050000000000001},"min":{"1min":0.42099999999999999,"5min":0.38900000000000001,"15min":0.38300000000000001},"max":{"1min":10.849,"5min":93.472999999999999,"15min":93.472999999999999},"last":0.91400000000000003}]},{"osd":4,"last update":"Sat Sep 23 19:05:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.72299999999999998,"15min":0.74399999999999999},"min":{"1min":0.42199999999999999,"5min":0.39800000000000002,"15min":0.32200000000000001},"max":{"1min":0.78800000000000003,"5min":3.1899999999999999,"15min":4.1150000000000002},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.70599999999999996,"5min":0.78400000000000003,"15min":0.78600000000000003},"min":{"1min":0.54000000000000004,"5min":0.41799999999999998,"15min":0.35799999999999998},"max":{"1min":0.88700000000000001,"5min":1.405,"15min":4.0179999999999998},"last":0.98299999999999998}]},{"osd":5,"last update":"Sat Sep 23 19:01:56 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":0.83899999999999997,"15min":1.946},"min":{"1min":0.434,"5min":0.373,"15min":0.371},"max":{"1min":3.3500000000000001,"5min":3.3500000000000001,"15min":336.26299999999998},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.78300000000000003,"5min":0.78100000000000003,"15min":1.9650000000000001},"min":{"1min":0.43099999999999999,"5min":0.374,"15min":0.374},"max":{"1min":1.768,"5min":1.768,"15min":336.07799999999997},"last":0.64700000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:54:32 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":1.0049999999999999,"15min":1.1839999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.35999999999999999},"max":{"1min":3.3500000000000001,"5min":7.0700000000000003,"15min":7.9320000000000004},"last":1.0640000000000001},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.95299999999999996,"15min":1.264},"min":{"1min":0.45600000000000002,"5min":0.41599999999999998,"15min":0.39900000000000002},"max":{"1min":0.97399999999999998,"5min":5.5149999999999997,"15min":8.9250000000000007},"last":0.91200000000000003}]}]},{"osd":1,"up_from":123,"seq":528280977789,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":978224,"kb_used_data":4136,"kb_used_omap":7,"kb_used_meta":974072,"kb_avail":92766928,"statfs":{"total":95995035648,"available":94993334272,"internally_reserved":0,"allocated":4235264,"data_stored":2564951,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":997450510},"hb_peers":[0,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:04:52 2023","interfaces":[{"interface":"back","average":{"1min":3.9049999999999998,"5min":3.96,"15min":2.8889999999999998},"min":{"1min":0.23899999999999999,"5min":0.23499999999999999,"15min":0.21299999999999999},"max":{"1min":10.311,"5min":90.492000000000004,"15min":90.492000000000004},"last":0.90400000000000003},{"interface":"front","average":{"1min":2.98,"5min":3.3239999999999998,"15min":2.915},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.17399999999999999},"max":{"1min":11.773,"5min":90.552999999999997,"15min":90.552999999999997},"last":0.91800000000000004}]},{"osd":2,"last update":"Sat Sep 23 19:04:52 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":3.1459999999999999,"15min":2.7330000000000001},"min":{"1min":0.34000000000000002,"5min":0.219,"15min":0.219},"max":{"1min":6.6890000000000001,"5min":87.832999999999998,"15min":180.578},"last":0.39400000000000002},{"interface":"front","average":{"1min":3.5750000000000002,"5min":3.6259999999999999,"15min":3.262},"min":{"1min":0.28799999999999998,"5min":0.27800000000000002,"15min":0.22800000000000001},"max":{"1min":12.784000000000001,"5min":87.846999999999994,"15min":180.12299999999999},"last":0.41499999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:04:52 2023","interfaces":[{"interface":"back","average":{"1min":2.5659999999999998,"5min":2.5070000000000001,"15min":2.5270000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.23899999999999999},"max":{"1min":11.74,"5min":99.126000000000005,"15min":99.126000000000005},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.089,"5min":2.4089999999999998,"15min":2.1589999999999998},"min":{"1min":0.35099999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":6.742,"5min":99.165999999999997,"15min":99.165999999999997},"last":0.34200000000000003}]},{"osd":4,"last update":"Sat Sep 23 19:05:15 2023","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":1.1739999999999999,"15min":1.472},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.314},"max":{"1min":4.5209999999999999,"5min":6.1879999999999997,"15min":8.3559999999999999},"last":0.65300000000000002},{"interface":"front","average":{"1min":1.4890000000000001,"5min":1.2050000000000001,"15min":1.4179999999999999},"min":{"1min":0.45400000000000001,"5min":0.40699999999999997,"15min":0.34200000000000003},"max":{"1min":6.8070000000000004,"5min":7.5640000000000001,"15min":9.0549999999999997},"last":0.69199999999999995}]},{"osd":5,"last update":"Sat Sep 23 19:01:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.756,"15min":2.0150000000000001},"min":{"1min":0.39200000000000002,"5min":0.34399999999999997,"15min":0.33000000000000002},"max":{"1min":4.7050000000000001,"5min":8.391,"15min":66.954999999999998},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.85499999999999998,"5min":1.6719999999999999,"15min":2.5830000000000002},"min":{"1min":0.44800000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.925,"5min":9.1349999999999998,"15min":180.52099999999999},"last":0.63}]},{"osd":6,"last update":"Sat Sep 23 19:04:52 2023","interfaces":[{"interface":"back","average":{"1min":1.2250000000000001,"5min":1.1830000000000001,"15min":1.7829999999999999},"min":{"1min":0.46300000000000002,"5min":0.38100000000000001,"15min":0.35899999999999999},"max":{"1min":6.7789999999999999,"5min":7.6299999999999999,"15min":63.036999999999999},"last":0.72199999999999998},{"interface":"front","average":{"1min":2.2090000000000001,"5min":1.6970000000000001,"15min":1.6950000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.36199999999999999},"max":{"1min":10.351000000000001,"5min":12.887,"15min":12.887},"last":0.61799999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:54:28 2023","interfaces":[{"interface":"back","average":{"1min":4.2919999999999998,"5min":2.593,"15min":2.1019999999999999},"min":{"1min":0.432,"5min":0.42199999999999999,"15min":0.33500000000000002},"max":{"1min":62.954999999999998,"5min":62.954999999999998,"15min":62.954999999999998},"last":1.01},{"interface":"front","average":{"1min":4.407,"5min":2.5369999999999999,"15min":1.9339999999999999},"min":{"1min":0.47799999999999998,"5min":0.373,"15min":0.373},"max":{"1min":62.658999999999999,"5min":62.658999999999999,"15min":62.658999999999999},"last":0.96599999999999997}]}]},{"osd":0,"up_from":88,"seq":377957122519,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1110148,"kb_used_data":4540,"kb_used_omap":8,"kb_used_meta":1105591,"kb_avail":92635004,"statfs":{"total":95995035648,"available":94858244096,"internally_reserved":0,"allocated":4648960,"data_stored":3134349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":1132126078},"hb_peers":[1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:05:05 2023","interfaces":[{"interface":"back","average":{"1min":4.915,"5min":4.0129999999999999,"15min":3.0169999999999999},"min":{"1min":0.38800000000000001,"5min":0.21099999999999999,"15min":0.19800000000000001},"max":{"1min":31.420000000000002,"5min":49.168999999999997,"15min":49.168999999999997},"last":0.32200000000000001},{"interface":"front","average":{"1min":4.0090000000000003,"5min":4.0839999999999996,"15min":3.234},"min":{"1min":0.28299999999999997,"5min":0.25700000000000001,"15min":0.20499999999999999},"max":{"1min":22.198,"5min":50.673000000000002,"15min":50.673000000000002},"last":0.36199999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:04:39 2023","interfaces":[{"interface":"back","average":{"1min":4.6269999999999998,"5min":3.2309999999999999,"15min":2.4590000000000001},"min":{"1min":0.249,"5min":0.22600000000000001,"15min":0.215},"max":{"1min":31.288,"5min":52.914000000000001,"15min":59.381999999999998},"last":0.41799999999999998},{"interface":"front","average":{"1min":3.8700000000000001,"5min":2.5150000000000001,"15min":2.0110000000000001},"min":{"1min":0.23499999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":16.507000000000001,"5min":48.963999999999999,"15min":59.783000000000001},"last":0.308}]},{"osd":3,"last update":"Sat Sep 23 19:05:09 2023","interfaces":[{"interface":"back","average":{"1min":3.1219999999999999,"5min":2.0459999999999998,"15min":2.9350000000000001},"min":{"1min":0.29299999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":10.301,"5min":12.904,"15min":113.375},"last":0.39500000000000002},{"interface":"front","average":{"1min":2.327,"5min":2.2450000000000001,"15min":2.8639999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.217},"max":{"1min":7.8170000000000002,"5min":14.91,"15min":113.327},"last":0.64600000000000002}]},{"osd":4,"last update":"Sat Sep 23 19:05:12 2023","interfaces":[{"interface":"back","average":{"1min":2.431,"5min":1.962,"15min":1.4079999999999999},"min":{"1min":0.41999999999999998,"5min":0.375,"15min":0.33200000000000002},"max":{"1min":7.6109999999999998,"5min":55.710999999999999,"15min":55.710999999999999},"last":0.83299999999999996},{"interface":"front","average":{"1min":1.5669999999999999,"5min":2.0910000000000002,"15min":1.3420000000000001},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.30299999999999999},"max":{"1min":7.7869999999999999,"5min":46.509,"15min":46.509},"last":0.92000000000000004}]},{"osd":5,"last update":"Sat Sep 23 19:01:53 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.2330000000000001,"15min":1.4890000000000001},"min":{"1min":0.36499999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":3.044,"5min":8.2390000000000008,"15min":9.5879999999999992},"last":0.73399999999999999},{"interface":"front","average":{"1min":1.8360000000000001,"5min":1.381,"15min":1.329},"min":{"1min":0.442,"5min":0.40899999999999997,"15min":0.34999999999999998},"max":{"1min":10.407999999999999,"5min":10.407999999999999,"15min":10.407999999999999},"last":1.0469999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:05:05 2023","interfaces":[{"interface":"back","average":{"1min":2.359,"5min":1.827,"15min":1.6930000000000001},"min":{"1min":0.38500000000000001,"5min":0.378,"15min":0.36799999999999999},"max":{"1min":7.6390000000000002,"5min":55.796999999999997,"15min":55.796999999999997},"last":0.877},{"interface":"front","average":{"1min":1.5860000000000001,"5min":2.1099999999999999,"15min":1.6180000000000001},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":7.7400000000000002,"5min":46.411999999999999,"15min":46.411999999999999},"last":0.78700000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:54:41 2023","interfaces":[{"interface":"back","average":{"1min":1.3859999999999999,"5min":1.589,"15min":1.516},"min":{"1min":0.53000000000000003,"5min":0.372,"15min":0.372},"max":{"1min":5.0720000000000001,"5min":8.7040000000000006,"15min":10.191000000000001},"last":0.90900000000000003},{"interface":"front","average":{"1min":1.7629999999999999,"5min":1.5409999999999999,"15min":1.6559999999999999},"min":{"1min":0.47599999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":7.359,"5min":7.649,"15min":10.01},"last":0.75700000000000001}]}]},{"osd":2,"up_from":98,"seq":420906795467,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":985548,"kb_used_data":8068,"kb_used_omap":11,"kb_used_meta":977460,"kb_avail":92759604,"statfs":{"total":95995035648,"available":94985834496,"internally_reserved":0,"allocated":8261632,"data_stored":6743525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11949,"internal_metadata":1000919379},"hb_peers":[0,1,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:04:44 2023","interfaces":[{"interface":"back","average":{"1min":3.371,"5min":6.625,"15min":3.6379999999999999},"min":{"1min":0.312,"5min":0.184,"15min":0.16600000000000001},"max":{"1min":10.208,"5min":143.048,"15min":166.46100000000001},"last":0.35699999999999998},{"interface":"front","average":{"1min":3.4750000000000001,"5min":6.8150000000000004,"15min":3.738},"min":{"1min":0.24099999999999999,"5min":0.189,"15min":0.187},"max":{"1min":11.044,"5min":143.25200000000001,"15min":162.911},"last":0.27700000000000002}]},{"osd":1,"last update":"Sat Sep 23 19:04:44 2023","interfaces":[{"interface":"back","average":{"1min":2.4700000000000002,"5min":2.2189999999999999,"15min":3.4220000000000002},"min":{"1min":0.33100000000000002,"5min":0.24299999999999999,"15min":0.20599999999999999},"max":{"1min":10.260999999999999,"5min":46.601999999999997,"15min":156.35400000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":3.1899999999999999,"5min":2.6760000000000002,"15min":3.6120000000000001},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23799999999999999},"max":{"1min":10.327999999999999,"5min":49.411999999999999,"15min":156.376},"last":0.52800000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:05:22 2023","interfaces":[{"interface":"back","average":{"1min":1.859,"5min":1.4490000000000001,"15min":2.3140000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.222},"max":{"1min":7.1669999999999998,"5min":7.9619999999999997,"15min":85.102999999999994},"last":0.44500000000000001},{"interface":"front","average":{"1min":2.4350000000000001,"5min":1.4590000000000001,"15min":2.3849999999999998},"min":{"1min":0.33800000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":9.1679999999999993,"5min":9.1679999999999993,"15min":84.531999999999996},"last":0.46899999999999997}]},{"osd":4,"last update":"Sat Sep 23 19:05:22 2023","interfaces":[{"interface":"back","average":{"1min":2.1960000000000002,"5min":1.423,"15min":1.0229999999999999},"min":{"1min":0.42499999999999999,"5min":0.38800000000000001,"15min":0.31900000000000001},"max":{"1min":8.9969999999999999,"5min":8.9969999999999999,"15min":8.9969999999999999},"last":0.57499999999999996},{"interface":"front","average":{"1min":1.3580000000000001,"5min":1.24,"15min":1.1000000000000001},"min":{"1min":0.441,"5min":0.40899999999999997,"15min":0.32000000000000001},"max":{"1min":5.8470000000000004,"5min":6.0049999999999999,"15min":6.0049999999999999},"last":0.69499999999999995}]},{"osd":5,"last update":"Sat Sep 23 19:01:41 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":0.97699999999999998,"15min":2.157},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.33000000000000002},"max":{"1min":2.3050000000000002,"5min":5.3529999999999998,"15min":296.86200000000002},"last":0.443},{"interface":"front","average":{"1min":1.401,"5min":1.052,"15min":2.3220000000000001},"min":{"1min":0.51300000000000001,"5min":0.36399999999999999,"15min":0.36299999999999999},"max":{"1min":3.5270000000000001,"5min":5.0259999999999998,"15min":296.69499999999999},"last":0.60299999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:04:44 2023","interfaces":[{"interface":"back","average":{"1min":1.282,"5min":1.129,"15min":1.2470000000000001},"min":{"1min":0.378,"5min":0.35999999999999999,"15min":0.32800000000000001},"max":{"1min":4.2709999999999999,"5min":5.9649999999999999,"15min":11.021000000000001},"last":0.78000000000000003},{"interface":"front","average":{"1min":1.577,"5min":1.256,"15min":1.377},"min":{"1min":0.502,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.4359999999999999,"5min":6.4359999999999999,"15min":16.247},"last":0.68600000000000005}]},{"osd":7,"last update":"Sat Sep 23 18:54:25 2023","interfaces":[{"interface":"back","average":{"1min":1.5700000000000001,"5min":1.623,"15min":1.51},"min":{"1min":0.38300000000000001,"5min":0.374,"15min":0.374},"max":{"1min":5.5880000000000001,"5min":10.971,"15min":16.024000000000001},"last":0.83599999999999997},{"interface":"front","average":{"1min":1.1899999999999999,"5min":1.8149999999999999,"15min":1.4359999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":5.9249999999999998,"5min":16.178999999999998,"15min":16.178999999999998},"last":0.95799999999999996}]}]},{"osd":3,"up_from":103,"seq":442381631941,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":912660,"kb_used_data":8076,"kb_used_omap":4,"kb_used_meta":904571,"kb_avail":92832492,"statfs":{"total":95995035648,"available":95060471808,"internally_reserved":0,"allocated":8269824,"data_stored":6731939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4826,"internal_metadata":926280998},"hb_peers":[0,1,2,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:05:18 2023","interfaces":[{"interface":"back","average":{"1min":4.6749999999999998,"5min":2.6419999999999999,"15min":3.73},"min":{"1min":0.20799999999999999,"5min":0.19700000000000001,"15min":0.191},"max":{"1min":35.985999999999997,"5min":40.652000000000001,"15min":296.55399999999997},"last":0.35699999999999998},{"interface":"front","average":{"1min":4.306,"5min":2.8799999999999999,"15min":3.7570000000000001},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":36.289000000000001,"5min":42.091999999999999,"15min":296.36900000000003},"last":0.42399999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:05:05 2023","interfaces":[{"interface":"back","average":{"1min":9.5779999999999994,"5min":7.8419999999999996,"15min":4.133},"min":{"1min":0.33700000000000002,"5min":0.28299999999999997,"15min":0.22500000000000001},"max":{"1min":129.91300000000001,"5min":343.79300000000001,"15min":343.79300000000001},"last":0.38},{"interface":"front","average":{"1min":10.154999999999999,"5min":7.9889999999999999,"15min":4.1669999999999998},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20200000000000001},"max":{"1min":133.62799999999999,"5min":343.73000000000002,"15min":343.73000000000002},"last":0.40200000000000002}]},{"osd":2,"last update":"Sat Sep 23 19:05:18 2023","interfaces":[{"interface":"back","average":{"1min":3.3900000000000001,"5min":3.173,"15min":3.661},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.23699999999999999},"max":{"1min":10.42,"5min":61.334000000000003,"15min":178.06200000000001},"last":0.499},{"interface":"front","average":{"1min":1.4059999999999999,"5min":2.7570000000000001,"15min":3.2639999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.25600000000000001},"max":{"1min":3.415,"5min":52.493000000000002,"15min":178.267},"last":0.45500000000000002}]},{"osd":4,"last update":"Sat Sep 23 19:05:23 2023","interfaces":[{"interface":"back","average":{"1min":2.1360000000000001,"5min":1.4159999999999999,"15min":1.1220000000000001},"min":{"1min":0.42899999999999999,"5min":0.41399999999999998,"15min":0.33100000000000002},"max":{"1min":7.9119999999999999,"5min":7.9119999999999999,"15min":12.028},"last":0.67400000000000004},{"interface":"front","average":{"1min":1.3819999999999999,"5min":1.208,"15min":1.4430000000000001},"min":{"1min":0.432,"5min":0.42399999999999999,"15min":0.34300000000000003},"max":{"1min":3.5579999999999998,"5min":5.0949999999999998,"15min":11.369},"last":0.75800000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:02:08 2023","interfaces":[{"interface":"back","average":{"1min":1.216,"5min":1.359,"15min":1.304},"min":{"1min":0.37,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":6.4630000000000001,"5min":11.978999999999999,"15min":11.978999999999999},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.595,"15min":1.5509999999999999},"min":{"1min":0.34399999999999997,"5min":0.34300000000000003,"15min":0.33100000000000002},"max":{"1min":2.9249999999999998,"5min":12.958,"15min":12.958},"last":3.9790000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:05:18 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":1.345,"15min":1.573},"min":{"1min":0.38,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":8.4169999999999998,"5min":8.4169999999999998,"15min":12.926},"last":0.68799999999999994},{"interface":"front","average":{"1min":1.3620000000000001,"5min":1.278,"15min":1.47},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.38500000000000001},"max":{"1min":3.5209999999999999,"5min":5.125,"15min":11.335000000000001},"last":0.85699999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:53:54 2023","interfaces":[{"interface":"back","average":{"1min":1.4710000000000001,"5min":1.7689999999999999,"15min":1.7250000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.35399999999999998},"max":{"1min":5.0700000000000003,"5min":11.382999999999999,"15min":11.382999999999999},"last":0.76900000000000002},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.5960000000000001,"15min":1.556},"min":{"1min":0.40000000000000002,"5min":0.38900000000000001,"15min":0.36599999999999999},"max":{"1min":3.1640000000000001,"5min":12.170999999999999,"15min":12.170999999999999},"last":0.59899999999999998}]}]},{"osd":4,"up_from":0,"seq":1228360646772,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":36,"seq":154618823281,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2061508,"kb_used_data":1463612,"kb_used_omap":0,"kb_used_meta":597888,"kb_avail":91683644,"statfs":{"total":95995035648,"available":93884051456,"internally_reserved":0,"allocated":1498738688,"data_stored":1497984321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612237312},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:01:39 2023","interfaces":[{"interface":"back","average":{"1min":3.766,"5min":1.97,"15min":2.0960000000000001},"min":{"1min":0.32000000000000001,"5min":0.23499999999999999,"15min":0.23400000000000001},"max":{"1min":64.266000000000005,"5min":64.266000000000005,"15min":154.61099999999999},"last":0.80900000000000005},{"interface":"front","average":{"1min":4.46,"5min":2.1779999999999999,"15min":2.3439999999999999},"min":{"1min":0.61399999999999999,"5min":0.251,"15min":0.251},"max":{"1min":64.352999999999994,"5min":64.352999999999994,"15min":154.28800000000001},"last":6.1619999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:01:48 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":4.7770000000000001,"15min":3.3029999999999999},"min":{"1min":0.28799999999999998,"5min":0.246,"15min":0.246},"max":{"1min":1.647,"5min":277.226,"15min":277.226},"last":1.0329999999999999},{"interface":"front","average":{"1min":0.76800000000000002,"5min":4.5910000000000002,"15min":3.2320000000000002},"min":{"1min":0.34300000000000003,"5min":0.34200000000000003,"15min":0.28699999999999998},"max":{"1min":1.6639999999999999,"5min":277.32100000000003,"15min":277.32100000000003},"last":0.93500000000000005}]},{"osd":2,"last update":"Sat Sep 23 19:01:39 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":1.02,"15min":1.1020000000000001},"min":{"1min":0.33900000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":2.2120000000000002,"5min":13.021000000000001,"15min":13.021000000000001},"last":0.442},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.181,"15min":1.1619999999999999},"min":{"1min":0.38700000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.23,"5min":15.853999999999999,"15min":15.853999999999999},"last":0.85299999999999998}]},{"osd":3,"last update":"Sat Sep 23 19:02:10 2023","interfaces":[{"interface":"back","average":{"1min":3.3940000000000001,"5min":2.0760000000000001,"15min":2.0310000000000001},"min":{"1min":0.36399999999999999,"5min":0.308,"15min":0.308},"max":{"1min":49.286999999999999,"5min":49.286999999999999,"15min":149.83500000000001},"last":0.83399999999999996},{"interface":"front","average":{"1min":3.27,"5min":2.133,"15min":1.9430000000000001},"min":{"1min":0.44700000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":48.726999999999997,"5min":48.726999999999997,"15min":149.00399999999999},"last":2.444}]},{"osd":4,"last update":"Sat Sep 23 19:02:15 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.95799999999999996,"15min":0.75},"min":{"1min":0.36399999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.1200000000000001,"5min":13.484,"15min":13.484},"last":1.1399999999999999},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.77900000000000003,"15min":0.71799999999999997},"min":{"1min":0.38,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.099,"5min":4.8979999999999997,"15min":4.8979999999999997},"last":0.59799999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:02:10 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":0.80500000000000005,"15min":0.89300000000000002},"min":{"1min":0.34000000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":3.008,"5min":4.0510000000000002,"15min":8.2379999999999995},"last":1.329},{"interface":"front","average":{"1min":0.83999999999999997,"5min":1.1140000000000001,"15min":1.0449999999999999},"min":{"1min":0.43099999999999999,"5min":0.378,"15min":0.36099999999999999},"max":{"1min":3.0470000000000002,"5min":12.977,"15min":12.977},"last":1.3460000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:54:30 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.79700000000000004,"15min":1.149},"min":{"1min":0.39500000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.0009999999999999,"5min":3.4980000000000002,"15min":9.9130000000000003},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.78300000000000003,"5min":0.82599999999999996,"15min":0.96699999999999997},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":2.214,"5min":3.2490000000000001,"15min":6.7779999999999996},"last":0.90700000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4224,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7725,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3429,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11243,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1492975616,"data_stored":1492975616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:05:45.415 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T19:05:45.416 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_5 pg_num 26 2023-09-23T19:05:45.501 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:45.530 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:45.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:45.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:45.634 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:45.657 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:45.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:45.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:45.669 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:45 smithi098 bash[76817]: cluster 2023-09-23T19:05:44.566078+0000 mgr.y (mgr.26054) 1440 : cluster 0 pgmap v1577: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 1745/84 objects misplaced (2077.381%); 0 B/s, 149 objects/s recovering 2023-09-23T19:05:45.669 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:45 smithi098 bash[76817]: cluster 2023-09-23T19:05:44.668094+0000 mon.a (mon.0) 1950 : cluster 0 osdmap e337: 8 total, 6 up, 7 in 2023-09-23T19:05:45.669 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:45 smithi098 bash[76817]: cluster 2023-09-23T19:05:44.678597+0000 osd.6 (osd.6) 318 : cluster 3 failed to encode map e337 with expected crc 2023-09-23T19:05:45.669 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:45 smithi098 bash[76817]: cluster 2023-09-23T19:05:45.186298+0000 osd.4 (osd.4) 67 : cluster 3 failed to encode map e337 with expected crc 2023-09-23T19:05:45.669 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:45 smithi098 bash[76817]: cluster 2023-09-23T19:05:45.189600+0000 osd.4 (osd.4) 68 : cluster 3 failed to encode map e337 with expected crc 2023-09-23T19:05:45.669 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:45 smithi098 bash[76817]: cluster 2023-09-23T19:05:45.542088+0000 osd.4 (osd.4) 69 : cluster 3 failed to encode map e338 with expected crc 2023-09-23T19:05:45.760 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:45.774 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:45 smithi098 bash[76817]: cluster 2023-09-23T19:05:45.542940+0000 mon.a (mon.0) 1951 : cluster 0 osdmap e338: 8 total, 6 up, 7 in 2023-09-23T19:05:45.774 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:45 smithi098 bash[79516]: cluster 2023-09-23T19:05:44.566078+0000 mgr.y (mgr.26054) 1440 : cluster 0 pgmap v1577: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 1745/84 objects misplaced (2077.381%); 0 B/s, 149 objects/s recovering 2023-09-23T19:05:45.774 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:45 smithi098 bash[79516]: cluster 2023-09-23T19:05:44.668094+0000 mon.a (mon.0) 1950 : cluster 0 osdmap e337: 8 total, 6 up, 7 in 2023-09-23T19:05:45.774 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:45 smithi098 bash[79516]: cluster 2023-09-23T19:05:44.678597+0000 osd.6 (osd.6) 318 : cluster 3 failed to encode map e337 with expected crc 2023-09-23T19:05:45.774 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:45 smithi098 bash[79516]: cluster 2023-09-23T19:05:45.186298+0000 osd.4 (osd.4) 67 : cluster 3 failed to encode map e337 with expected crc 2023-09-23T19:05:45.774 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:45 smithi098 bash[79516]: cluster 2023-09-23T19:05:45.189600+0000 osd.4 (osd.4) 68 : cluster 3 failed to encode map e337 with expected crc 2023-09-23T19:05:45.775 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:45 smithi098 bash[79516]: cluster 2023-09-23T19:05:45.542088+0000 osd.4 (osd.4) 69 : cluster 3 failed to encode map e338 with expected crc 2023-09-23T19:05:45.775 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:45 smithi098 bash[79516]: cluster 2023-09-23T19:05:45.542940+0000 mon.a (mon.0) 1951 : cluster 0 osdmap e338: 8 total, 6 up, 7 in 2023-09-23T19:05:45.788 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:45.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:45.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:45.840 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:05:45.877 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:05:45.878 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:05:45.893 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:45.919 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:45.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:45.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:46.023 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:46.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:45 smithi173 bash[59422]: cluster 2023-09-23T19:05:44.566078+0000 mgr.y (mgr.26054) 1440 : cluster 0 pgmap v1577: 89 pgs: 1 active+remapped+backfill_wait, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 5 active+undersized+remapped, 32 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 1745/84 objects misplaced (2077.381%); 0 B/s, 149 objects/s recovering 2023-09-23T19:05:46.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:45 smithi173 bash[59422]: cluster 2023-09-23T19:05:44.668094+0000 mon.a (mon.0) 1950 : cluster 0 osdmap e337: 8 total, 6 up, 7 in 2023-09-23T19:05:46.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:45 smithi173 bash[59422]: cluster 2023-09-23T19:05:44.678597+0000 osd.6 (osd.6) 318 : cluster 3 failed to encode map e337 with expected crc 2023-09-23T19:05:46.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:45 smithi173 bash[59422]: cluster 2023-09-23T19:05:45.186298+0000 osd.4 (osd.4) 67 : cluster 3 failed to encode map e337 with expected crc 2023-09-23T19:05:46.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:45 smithi173 bash[59422]: cluster 2023-09-23T19:05:45.189600+0000 osd.4 (osd.4) 68 : cluster 3 failed to encode map e337 with expected crc 2023-09-23T19:05:46.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:45 smithi173 bash[59422]: cluster 2023-09-23T19:05:45.542088+0000 osd.4 (osd.4) 69 : cluster 3 failed to encode map e338 with expected crc 2023-09-23T19:05:46.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:45 smithi173 bash[59422]: cluster 2023-09-23T19:05:45.542940+0000 mon.a (mon.0) 1951 : cluster 0 osdmap e338: 8 total, 6 up, 7 in 2023-09-23T19:05:46.045 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:46.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:46.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:46.149 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:46.182 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:46.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:46.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:46.193 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:05:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:05:46] "GET /metrics HTTP/1.1" 200 36626 "" "Prometheus/2.33.4" 2023-09-23T19:05:46.287 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:46.319 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:46.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:46.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:46.422 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:46.451 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:46.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:46.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:46.555 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:46.580 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:46.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:46.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:46.684 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:46.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:46 smithi098 bash[76817]: cluster 2023-09-23T19:05:45.547760+0000 osd.6 (osd.6) 319 : cluster 3 failed to encode map e338 with expected crc 2023-09-23T19:05:46.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:46 smithi098 bash[76817]: cluster 2023-09-23T19:05:46.544879+0000 osd.4 (osd.4) 70 : cluster 3 failed to encode map e339 with expected crc 2023-09-23T19:05:46.692 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:46 smithi098 bash[76817]: cluster 2023-09-23T19:05:46.546183+0000 mon.a (mon.0) 1952 : cluster 0 osdmap e339: 8 total, 6 up, 7 in 2023-09-23T19:05:46.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:46 smithi098 bash[79516]: cluster 2023-09-23T19:05:45.547760+0000 osd.6 (osd.6) 319 : cluster 3 failed to encode map e338 with expected crc 2023-09-23T19:05:46.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:46 smithi098 bash[79516]: cluster 2023-09-23T19:05:46.544879+0000 osd.4 (osd.4) 70 : cluster 3 failed to encode map e339 with expected crc 2023-09-23T19:05:46.692 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:46 smithi098 bash[79516]: cluster 2023-09-23T19:05:46.546183+0000 mon.a (mon.0) 1952 : cluster 0 osdmap e339: 8 total, 6 up, 7 in 2023-09-23T19:05:46.707 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:46.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:46.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:46.811 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:46.842 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:46.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:46.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:46.946 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:46.978 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:46.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:46.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:47.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:46 smithi173 bash[59422]: cluster 2023-09-23T19:05:45.547760+0000 osd.6 (osd.6) 319 : cluster 3 failed to encode map e338 with expected crc 2023-09-23T19:05:47.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:46 smithi173 bash[59422]: cluster 2023-09-23T19:05:46.544879+0000 osd.4 (osd.4) 70 : cluster 3 failed to encode map e339 with expected crc 2023-09-23T19:05:47.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:46 smithi173 bash[59422]: cluster 2023-09-23T19:05:46.546183+0000 mon.a (mon.0) 1952 : cluster 0 osdmap e339: 8 total, 6 up, 7 in 2023-09-23T19:05:47.081 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:47.114 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:47.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:47.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:47.218 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:47.251 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:47.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:47.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:47.356 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:47.382 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:47.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:47.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:47.486 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:47.517 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:47.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:47.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:47.621 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:47.655 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:47.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:47.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:47.758 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:47.771 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:47 smithi098 bash[76817]: cluster 2023-09-23T19:05:46.544998+0000 osd.6 (osd.6) 320 : cluster 3 failed to encode map e339 with expected crc 2023-09-23T19:05:47.771 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:47 smithi098 bash[76817]: cluster 2023-09-23T19:05:46.566925+0000 mgr.y (mgr.26054) 1441 : cluster 0 pgmap v1581: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 6 active+undersized+remapped, 32 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 1745/84 objects misplaced (2077.381%) 2023-09-23T19:05:47.771 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:47 smithi098 bash[79516]: cluster 2023-09-23T19:05:46.544998+0000 osd.6 (osd.6) 320 : cluster 3 failed to encode map e339 with expected crc 2023-09-23T19:05:47.771 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:47 smithi098 bash[79516]: cluster 2023-09-23T19:05:46.566925+0000 mgr.y (mgr.26054) 1441 : cluster 0 pgmap v1581: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 6 active+undersized+remapped, 32 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 1745/84 objects misplaced (2077.381%) 2023-09-23T19:05:47.787 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:47.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:47.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:47.891 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:47.923 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:47.925 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:47 smithi173 bash[59422]: cluster 2023-09-23T19:05:46.544998+0000 osd.6 (osd.6) 320 : cluster 3 failed to encode map e339 with expected crc 2023-09-23T19:05:47.925 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:47 smithi173 bash[59422]: cluster 2023-09-23T19:05:46.566925+0000 mgr.y (mgr.26054) 1441 : cluster 0 pgmap v1581: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+remapped, 24 active+clean+remapped, 6 active+undersized+remapped, 32 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 1745/84 objects misplaced (2077.381%) 2023-09-23T19:05:47.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:47.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:48.027 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:48.055 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:48.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:48.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:48.159 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:48.184 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:48.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:48.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:48.288 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:48.318 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:48.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:48.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:48.422 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:48.451 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:48.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:48.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:48.555 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:48.588 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:48.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:48.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:48.692 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:48.722 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:48.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:48.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:48.826 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:48.839 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:48 smithi173 bash[59422]: cluster 2023-09-23T19:05:48.667603+0000 mon.a (mon.0) 1953 : cluster 3 Health check update: Degraded data redundancy: 9/84 objects degraded (10.714%), 7 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-09-23T19:05:48.848 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:48 smithi098 bash[79516]: cluster 2023-09-23T19:05:48.667603+0000 mon.a (mon.0) 1953 : cluster 3 Health check update: Degraded data redundancy: 9/84 objects degraded (10.714%), 7 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-09-23T19:05:48.849 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:48 smithi098 bash[76817]: cluster 2023-09-23T19:05:48.667603+0000 mon.a (mon.0) 1953 : cluster 3 Health check update: Degraded data redundancy: 9/84 objects degraded (10.714%), 7 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-09-23T19:05:48.860 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:48.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:48.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:48.964 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:48.986 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:48.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:48.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:49.092 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:49.115 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:49.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:49.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:49.218 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:49.248 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:49.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:49.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:49.352 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:49.383 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:49.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:49.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:49.486 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:49.518 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:49.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:49.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:49.622 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:49.649 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:49.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:49.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:49.753 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:49.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:49.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:49.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:49.889 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:49.902 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:49 smithi098 bash[79516]: cluster 2023-09-23T19:05:48.567874+0000 mgr.y (mgr.26054) 1442 : cluster 0 pgmap v1582: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%); 0 B/s, 287 objects/s recovering 2023-09-23T19:05:49.902 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:49 smithi098 bash[76817]: cluster 2023-09-23T19:05:48.567874+0000 mgr.y (mgr.26054) 1442 : cluster 0 pgmap v1582: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%); 0 B/s, 287 objects/s recovering 2023-09-23T19:05:49.917 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:49.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:49.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:50.019 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:50.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:49 smithi173 bash[59422]: cluster 2023-09-23T19:05:48.567874+0000 mgr.y (mgr.26054) 1442 : cluster 0 pgmap v1582: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%); 0 B/s, 287 objects/s recovering 2023-09-23T19:05:50.046 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:50.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:50.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:50.150 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:50.178 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:50.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:50.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:50.282 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:50.315 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:50.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:50.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:50.420 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:50.454 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:50.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:50.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:50.556 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:50.592 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:50.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:50.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:50.697 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:50.707 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:05:50.708 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:05:50.725 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:50.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:50.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:50.829 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:50.861 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:50.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:50.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:50.965 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:50.994 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:50.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:50.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:51.098 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:51.122 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:51.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:51.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:51.225 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:51.254 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:51.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:51.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:51.357 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:51.383 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:51.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:51.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:51.486 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:51.516 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:51.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:51.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:51.619 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:51.632 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:51 smithi098 bash[79516]: cluster 2023-09-23T19:05:50.568954+0000 mgr.y (mgr.26054) 1443 : cluster 0 pgmap v1583: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%); 0 B/s, 287 objects/s recovering 2023-09-23T19:05:51.632 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:51 smithi098 bash[76817]: cluster 2023-09-23T19:05:50.568954+0000 mgr.y (mgr.26054) 1443 : cluster 0 pgmap v1583: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%); 0 B/s, 287 objects/s recovering 2023-09-23T19:05:51.650 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:51.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:51.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:51.753 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:51.791 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:51.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:51.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:51.895 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:51.907 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:51 smithi173 bash[59422]: cluster 2023-09-23T19:05:50.568954+0000 mgr.y (mgr.26054) 1443 : cluster 0 pgmap v1583: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%); 0 B/s, 287 objects/s recovering 2023-09-23T19:05:51.929 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:51.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:51.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:52.033 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:52.059 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:52.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:52.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:52.163 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:52.193 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:52.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:52.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:52.297 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:52.325 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:52.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:52.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:52.429 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:52.463 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:52.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:52.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:52.566 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:52.593 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:52.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:52.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:52.696 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:52.731 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:52.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:52.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:52.835 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:52.866 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:52.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:52.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:52.969 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:53.006 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:53.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:53.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:53.109 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:53.143 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:53.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:53.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:53.246 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:53.281 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:53.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:53.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:53.404 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:53.439 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:53.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:53.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:53.542 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:53.574 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:53.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:53.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:53.677 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:53.688 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:53 smithi098 bash[76817]: cluster 2023-09-23T19:05:52.570348+0000 mgr.y (mgr.26054) 1444 : cluster 0 pgmap v1584: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%); 0 B/s, 218 objects/s recovering 2023-09-23T19:05:53.688 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:53 smithi098 bash[79516]: cluster 2023-09-23T19:05:52.570348+0000 mgr.y (mgr.26054) 1444 : cluster 0 pgmap v1584: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%); 0 B/s, 218 objects/s recovering 2023-09-23T19:05:53.704 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:53.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:53.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:53.808 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:53.832 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:53.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:53.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:53.936 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:53.949 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:53 smithi173 bash[59422]: cluster 2023-09-23T19:05:52.570348+0000 mgr.y (mgr.26054) 1444 : cluster 0 pgmap v1584: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%); 0 B/s, 218 objects/s recovering 2023-09-23T19:05:53.971 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:53.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:53.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:54.075 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:54.100 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:54.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:54.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:54.203 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:54.234 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:54.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:54.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:54.338 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:54.366 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:54.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:54.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:54.470 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:54.496 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:54.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:54.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:54.599 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:54.624 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:54.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:54.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:54.727 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:54.739 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T19:05:54.739 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:05:54.750 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:54.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:54.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:54.853 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:54.879 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:54.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:54.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:54.983 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:55.009 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:55.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:55.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:55.111 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:55.134 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:55.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:55.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:55.237 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:55.259 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:55.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:55.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:55.297 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T19:05:55.320 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T19:05:55.340 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T19:05:55.360 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T19:05:55.363 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:55.384 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:55.385 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T19:05:55.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:55.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:55.412 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T19:05:55.412 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:05:55.412 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:05:55.412 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:05:55.412 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:05:55.412 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:05:55.413 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T19:05:55.413 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:05:55.413 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T19:05:55.489 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:55.513 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:55.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:55.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:55.616 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:55.647 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:55.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:55.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:55.707 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:55 smithi098 bash[76817]: cluster 2023-09-23T19:05:54.571834+0000 mgr.y (mgr.26054) 1445 : cluster 0 pgmap v1585: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%); 0 B/s, 191 objects/s recovering 2023-09-23T19:05:55.708 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:55 smithi098 bash[76817]: audit 2023-09-23T19:05:54.645879+0000 mon.a (mon.0) 1954 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:05:55.708 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:55 smithi098 bash[76817]: audit 2023-09-23T19:05:55.270328+0000 mon.c (mon.2) 147 : audit 0 from='client.? 172.21.15.98:0/4140502963' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:05:55.751 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:55.784 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:55.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:55.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:55.887 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:55.913 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:55.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:55.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:56.017 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:56.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:55 smithi173 bash[59422]: cluster 2023-09-23T19:05:54.571834+0000 mgr.y (mgr.26054) 1445 : cluster 0 pgmap v1585: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%); 0 B/s, 191 objects/s recovering 2023-09-23T19:05:56.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:55 smithi173 bash[59422]: audit 2023-09-23T19:05:54.645879+0000 mon.a (mon.0) 1954 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:05:56.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:55 smithi173 bash[59422]: audit 2023-09-23T19:05:55.270328+0000 mon.c (mon.2) 147 : audit 0 from='client.? 172.21.15.98:0/4140502963' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:05:56.029 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:55 smithi098 bash[79516]: cluster 2023-09-23T19:05:54.571834+0000 mgr.y (mgr.26054) 1445 : cluster 0 pgmap v1585: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%); 0 B/s, 191 objects/s recovering 2023-09-23T19:05:56.029 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:55 smithi098 bash[79516]: audit 2023-09-23T19:05:54.645879+0000 mon.a (mon.0) 1954 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:05:56.029 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:55 smithi098 bash[79516]: audit 2023-09-23T19:05:55.270328+0000 mon.c (mon.2) 147 : audit 0 from='client.? 172.21.15.98:0/4140502963' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:05:56.044 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:56.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:56.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:56.147 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:56.174 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:56.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:56.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:56.277 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:56.290 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:05:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:05:56] "GET /metrics HTTP/1.1" 200 36622 "" "Prometheus/2.33.4" 2023-09-23T19:05:56.305 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:56.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:56.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:56.409 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:56.430 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:56.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:56.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:56.533 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:56.557 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:56.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:56.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:56.661 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:56.691 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:56.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:56.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:56.795 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:56.823 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:56.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:56.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:56.926 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:56.951 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:56.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:56.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:57.054 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:57.088 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:57.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:57.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:57.192 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:57.218 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:57.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:57.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:57.322 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:57.357 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:57.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:57.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:57.461 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:57.487 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:57.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:57.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:57.590 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:57.615 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:57.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:57.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:57.718 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:57.729 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:57 smithi098 bash[79516]: cluster 2023-09-23T19:05:56.573232+0000 mgr.y (mgr.26054) 1446 : cluster 0 pgmap v1586: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%); 0 B/s, 172 objects/s recovering 2023-09-23T19:05:57.729 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:57 smithi098 bash[76817]: cluster 2023-09-23T19:05:56.573232+0000 mgr.y (mgr.26054) 1446 : cluster 0 pgmap v1586: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%); 0 B/s, 172 objects/s recovering 2023-09-23T19:05:57.743 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:57.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:57.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:57.846 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:57.878 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:57.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:57.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:57.982 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:58.008 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:58.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:58.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:58.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:57 smithi173 bash[59422]: cluster 2023-09-23T19:05:56.573232+0000 mgr.y (mgr.26054) 1446 : cluster 0 pgmap v1586: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%); 0 B/s, 172 objects/s recovering 2023-09-23T19:05:58.111 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:58.142 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:58.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:58.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:58.245 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:58.278 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:05:58.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:58.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:05:58.381 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:58.414 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:58.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:58.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:58.517 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:58.539 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:58.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:58.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:58.641 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:58.664 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:58.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:58.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:58.767 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:58.794 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:05:58.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:58.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:05:58.896 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:58.921 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:05:58.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:58.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:05:59.023 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:59.057 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:59.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:59.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:59.160 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:59.188 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:59.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:59.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:59.291 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:59.322 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:59.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:59.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:59.426 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:59.451 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:59.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:59.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:59.554 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:59.585 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:05:59.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:59.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:05:59.690 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:59.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:05:59 smithi098 bash[76817]: cluster 2023-09-23T19:05:58.574320+0000 mgr.y (mgr.26054) 1447 : cluster 0 pgmap v1587: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%); 0 B/s, 143 objects/s recovering 2023-09-23T19:05:59.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:05:59 smithi098 bash[79516]: cluster 2023-09-23T19:05:58.574320+0000 mgr.y (mgr.26054) 1447 : cluster 0 pgmap v1587: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%); 0 B/s, 143 objects/s recovering 2023-09-23T19:05:59.699 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T19:05:59.715 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:59.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:59.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:05:59.818 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:59.838 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:05:59.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:59.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:05:59.941 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:59.967 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:05:59.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:05:59.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:00.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:05:59 smithi173 bash[59422]: cluster 2023-09-23T19:05:58.574320+0000 mgr.y (mgr.26054) 1447 : cluster 0 pgmap v1587: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%); 0 B/s, 143 objects/s recovering 2023-09-23T19:06:00.070 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:00.101 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:00.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:00.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:00.184 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:06:00.184 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:06:00.204 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:00.234 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:00.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:00.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:00.338 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:00.363 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:00.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:00.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:00.466 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:00.492 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:00.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:00.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:00.595 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:00.623 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:00.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:00.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:00.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:00.758 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:00.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:00.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:00.861 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:00.897 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:00.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:00.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:01.001 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:01.027 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:01.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:01.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:01.129 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:01.156 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:01.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:01.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:01.260 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:01.289 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:01.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:01.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:01.393 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:01.423 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:01.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:01.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:01.525 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:01.558 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:01.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:01.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:01.661 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:01.698 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:01.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:01.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:01.802 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:01.810 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:01 smithi098 bash[76817]: cluster 2023-09-23T19:06:00.574986+0000 mgr.y (mgr.26054) 1448 : cluster 0 pgmap v1588: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:01.810 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:01 smithi098 bash[79516]: cluster 2023-09-23T19:06:00.574986+0000 mgr.y (mgr.26054) 1448 : cluster 0 pgmap v1588: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:01.824 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:01.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:01.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:01.927 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:01.953 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:01.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:01.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:02.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:01 smithi173 bash[59422]: cluster 2023-09-23T19:06:00.574986+0000 mgr.y (mgr.26054) 1448 : cluster 0 pgmap v1588: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:02.058 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:02.089 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:02.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:02.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:02.185 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:06:02.193 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:02.257 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:02.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:02.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:02.361 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:02.386 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:02.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:02.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:02.488 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:02.518 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:02.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:02.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:02.623 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:02.649 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:02.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:02.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:02.752 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:02.777 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:02.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:02.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:02.881 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:02.908 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:02.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:02.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:03.011 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:03.043 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:03.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:03.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:03.146 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:03.180 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:03.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:03.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:03.284 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:03.314 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:03.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:03.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:03.418 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:03.451 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:03.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:03.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:03.555 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:03.589 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:03.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:03.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:03.656 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is set 2023-09-23T19:06:03.656 INFO:teuthology.orchestra.run.smithi098.stderr:set pool 10 pg_num to 26 2023-09-23T19:06:03.680 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:03 smithi098 bash[76817]: cluster 2023-09-23T19:06:02.576431+0000 mgr.y (mgr.26054) 1449 : cluster 0 pgmap v1589: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:03.681 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:03 smithi098 bash[76817]: audit 2023-09-23T19:06:02.995630+0000 mon.a (mon.0) 1955 : audit 1 from='client.? 172.21.15.98:0/2309552478' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:06:03.681 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:03 smithi098 bash[76817]: audit 2023-09-23T19:06:03.245859+0000 mon.c (mon.2) 148 : audit 1 from='client.? 172.21.15.98:0/372276497' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T19:06:03.681 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:03 smithi098 bash[76817]: audit 2023-09-23T19:06:03.246429+0000 mon.a (mon.0) 1956 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T19:06:03.681 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:03 smithi098 bash[79516]: cluster 2023-09-23T19:06:02.576431+0000 mgr.y (mgr.26054) 1449 : cluster 0 pgmap v1589: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:03.681 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:03 smithi098 bash[79516]: audit 2023-09-23T19:06:02.995630+0000 mon.a (mon.0) 1955 : audit 1 from='client.? 172.21.15.98:0/2309552478' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:06:03.681 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:03 smithi098 bash[79516]: audit 2023-09-23T19:06:03.245859+0000 mon.c (mon.2) 148 : audit 1 from='client.? 172.21.15.98:0/372276497' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T19:06:03.681 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:03 smithi098 bash[79516]: audit 2023-09-23T19:06:03.246429+0000 mon.a (mon.0) 1956 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T19:06:03.693 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:03.726 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:03.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:03.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:03.829 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:03.863 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:03.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:03.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:03.966 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:03.997 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:04.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:04.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:04.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:03 smithi173 bash[59422]: cluster 2023-09-23T19:06:02.576431+0000 mgr.y (mgr.26054) 1449 : cluster 0 pgmap v1589: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:04.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:03 smithi173 bash[59422]: audit 2023-09-23T19:06:02.995630+0000 mon.a (mon.0) 1955 : audit 1 from='client.? 172.21.15.98:0/2309552478' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:06:04.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:03 smithi173 bash[59422]: audit 2023-09-23T19:06:03.245859+0000 mon.c (mon.2) 148 : audit 1 from='client.? 172.21.15.98:0/372276497' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T19:06:04.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:03 smithi173 bash[59422]: audit 2023-09-23T19:06:03.246429+0000 mon.a (mon.0) 1956 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T19:06:04.102 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:04.125 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:04.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:04.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:04.228 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:04.260 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:04.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:04.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:04.363 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:04.396 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:04.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:04.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:04.499 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:04.520 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:04.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:04.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:04.624 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:04.656 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:04.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:04.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:04.759 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:04.766 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:04 smithi098 bash[76817]: cluster 2023-09-23T19:06:03.631397+0000 mon.a (mon.0) 1957 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:06:04.766 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:04 smithi098 bash[76817]: audit 2023-09-23T19:06:03.646969+0000 mon.a (mon.0) 1958 : audit 1 from='client.? 172.21.15.98:0/2309552478' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:06:04.766 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:04 smithi098 bash[76817]: audit 2023-09-23T19:06:03.647090+0000 mon.a (mon.0) 1959 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-09-23T19:06:04.766 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:04 smithi098 bash[76817]: cluster 2023-09-23T19:06:03.647207+0000 mon.a (mon.0) 1960 : cluster 0 osdmap e340: 8 total, 6 up, 7 in 2023-09-23T19:06:04.766 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:04 smithi098 bash[76817]: cluster 2023-09-23T19:06:03.800025+0000 osd.6 (osd.6) 321 : cluster 3 failed to encode map e340 with expected crc 2023-09-23T19:06:04.767 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:04 smithi098 bash[76817]: audit 2023-09-23T19:06:04.578941+0000 mon.a (mon.0) 1961 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T19:06:04.767 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:04 smithi098 bash[79516]: cluster 2023-09-23T19:06:03.631397+0000 mon.a (mon.0) 1957 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:06:04.767 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:04 smithi098 bash[79516]: audit 2023-09-23T19:06:03.646969+0000 mon.a (mon.0) 1958 : audit 1 from='client.? 172.21.15.98:0/2309552478' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:06:04.767 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:04 smithi098 bash[79516]: audit 2023-09-23T19:06:03.647090+0000 mon.a (mon.0) 1959 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-09-23T19:06:04.767 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:04 smithi098 bash[79516]: cluster 2023-09-23T19:06:03.647207+0000 mon.a (mon.0) 1960 : cluster 0 osdmap e340: 8 total, 6 up, 7 in 2023-09-23T19:06:04.767 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:04 smithi098 bash[79516]: cluster 2023-09-23T19:06:03.800025+0000 osd.6 (osd.6) 321 : cluster 3 failed to encode map e340 with expected crc 2023-09-23T19:06:04.767 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:04 smithi098 bash[79516]: audit 2023-09-23T19:06:04.578941+0000 mon.a (mon.0) 1961 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T19:06:04.778 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:04.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:04.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:04.880 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:04.894 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:04 smithi173 bash[59422]: cluster 2023-09-23T19:06:03.631397+0000 mon.a (mon.0) 1957 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:06:04.895 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:04 smithi173 bash[59422]: audit 2023-09-23T19:06:03.646969+0000 mon.a (mon.0) 1958 : audit 1 from='client.? 172.21.15.98:0/2309552478' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:06:04.895 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:04 smithi173 bash[59422]: audit 2023-09-23T19:06:03.647090+0000 mon.a (mon.0) 1959 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-09-23T19:06:04.895 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:04 smithi173 bash[59422]: cluster 2023-09-23T19:06:03.647207+0000 mon.a (mon.0) 1960 : cluster 0 osdmap e340: 8 total, 6 up, 7 in 2023-09-23T19:06:04.895 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:04 smithi173 bash[59422]: cluster 2023-09-23T19:06:03.800025+0000 osd.6 (osd.6) 321 : cluster 3 failed to encode map e340 with expected crc 2023-09-23T19:06:04.895 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:04 smithi173 bash[59422]: audit 2023-09-23T19:06:04.578941+0000 mon.a (mon.0) 1961 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T19:06:04.914 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:04.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:04.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:05.017 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:05.049 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:05.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:05.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:05.154 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:05.183 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:05.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:05.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:05.287 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:05.507 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:05.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:05.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:05.610 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:05.641 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:05.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:05.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:05.745 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:05.775 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:05.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:05.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:05.824 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-09-23T19:06:05.825 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-09-23T19:06:05.879 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:05.908 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:05.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:05.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:05.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:05 smithi098 bash[79516]: cluster 2023-09-23T19:06:03.799978+0000 osd.4 (osd.4) 71 : cluster 3 failed to encode map e340 with expected crc 2023-09-23T19:06:05.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:05 smithi098 bash[79516]: cluster 2023-09-23T19:06:04.577118+0000 mgr.y (mgr.26054) 1450 : cluster 0 pgmap v1591: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:05.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:05 smithi098 bash[79516]: audit 2023-09-23T19:06:04.648772+0000 mon.a (mon.0) 1962 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T19:06:05.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:05 smithi098 bash[79516]: cluster 2023-09-23T19:06:04.648878+0000 mon.a (mon.0) 1963 : cluster 0 osdmap e341: 8 total, 6 up, 7 in 2023-09-23T19:06:05.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:05 smithi098 bash[79516]: cluster 2023-09-23T19:06:04.649572+0000 osd.4 (osd.4) 72 : cluster 3 failed to encode map e341 with expected crc 2023-09-23T19:06:05.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:05 smithi098 bash[79516]: cluster 2023-09-23T19:06:04.875019+0000 osd.6 (osd.6) 322 : cluster 3 failed to encode map e341 with expected crc 2023-09-23T19:06:05.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:05 smithi098 bash[76817]: cluster 2023-09-23T19:06:03.799978+0000 osd.4 (osd.4) 71 : cluster 3 failed to encode map e340 with expected crc 2023-09-23T19:06:05.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:05 smithi098 bash[76817]: cluster 2023-09-23T19:06:04.577118+0000 mgr.y (mgr.26054) 1450 : cluster 0 pgmap v1591: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:05.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:05 smithi098 bash[76817]: audit 2023-09-23T19:06:04.648772+0000 mon.a (mon.0) 1962 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T19:06:05.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:05 smithi098 bash[76817]: cluster 2023-09-23T19:06:04.648878+0000 mon.a (mon.0) 1963 : cluster 0 osdmap e341: 8 total, 6 up, 7 in 2023-09-23T19:06:05.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:05 smithi098 bash[76817]: cluster 2023-09-23T19:06:04.649572+0000 osd.4 (osd.4) 72 : cluster 3 failed to encode map e341 with expected crc 2023-09-23T19:06:05.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:05 smithi098 bash[76817]: cluster 2023-09-23T19:06:04.875019+0000 osd.6 (osd.6) 322 : cluster 3 failed to encode map e341 with expected crc 2023-09-23T19:06:06.011 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:06.023 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:05 smithi173 bash[59422]: cluster 2023-09-23T19:06:03.799978+0000 osd.4 (osd.4) 71 : cluster 3 failed to encode map e340 with expected crc 2023-09-23T19:06:06.024 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:05 smithi173 bash[59422]: cluster 2023-09-23T19:06:04.577118+0000 mgr.y (mgr.26054) 1450 : cluster 0 pgmap v1591: 89 pgs: 25 active+clean+remapped, 5 active+undersized+remapped, 34 active+clean, 7 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:06.024 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:05 smithi173 bash[59422]: audit 2023-09-23T19:06:04.648772+0000 mon.a (mon.0) 1962 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T19:06:06.024 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:05 smithi173 bash[59422]: cluster 2023-09-23T19:06:04.648878+0000 mon.a (mon.0) 1963 : cluster 0 osdmap e341: 8 total, 6 up, 7 in 2023-09-23T19:06:06.024 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:05 smithi173 bash[59422]: cluster 2023-09-23T19:06:04.649572+0000 osd.4 (osd.4) 72 : cluster 3 failed to encode map e341 with expected crc 2023-09-23T19:06:06.024 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:05 smithi173 bash[59422]: cluster 2023-09-23T19:06:04.875019+0000 osd.6 (osd.6) 322 : cluster 3 failed to encode map e341 with expected crc 2023-09-23T19:06:06.038 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:06.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:06.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:06.142 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:06.174 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:06.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:06.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:06.277 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:06.309 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:06.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:06.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:06.314 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:06.339 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:06:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:06:06] "GET /metrics HTTP/1.1" 200 36622 "" "Prometheus/2.33.4" 2023-09-23T19:06:06.411 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:06.441 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:06.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:06.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:06.545 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:06.574 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:06.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:06.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:06.678 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:06.700 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:06.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:06.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:06.803 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:06.834 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:06.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:06.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:06.937 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:06.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:06 smithi098 bash[79516]: cluster 2023-09-23T19:06:05.651311+0000 osd.6 (osd.6) 323 : cluster 3 failed to encode map e342 with expected crc 2023-09-23T19:06:06.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:06 smithi098 bash[79516]: cluster 2023-09-23T19:06:05.652705+0000 mon.a (mon.0) 1964 : cluster 0 osdmap e342: 8 total, 6 up, 7 in 2023-09-23T19:06:06.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:06 smithi098 bash[79516]: cluster 2023-09-23T19:06:05.659961+0000 osd.6 (osd.6) 324 : cluster 3 failed to encode map e342 with expected crc 2023-09-23T19:06:06.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:06 smithi098 bash[76817]: cluster 2023-09-23T19:06:05.651311+0000 osd.6 (osd.6) 323 : cluster 3 failed to encode map e342 with expected crc 2023-09-23T19:06:06.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:06 smithi098 bash[76817]: cluster 2023-09-23T19:06:05.652705+0000 mon.a (mon.0) 1964 : cluster 0 osdmap e342: 8 total, 6 up, 7 in 2023-09-23T19:06:06.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:06 smithi098 bash[76817]: cluster 2023-09-23T19:06:05.659961+0000 osd.6 (osd.6) 324 : cluster 3 failed to encode map e342 with expected crc 2023-09-23T19:06:06.968 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:06.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:06.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:07.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:06 smithi173 bash[59422]: cluster 2023-09-23T19:06:05.651311+0000 osd.6 (osd.6) 323 : cluster 3 failed to encode map e342 with expected crc 2023-09-23T19:06:07.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:06 smithi173 bash[59422]: cluster 2023-09-23T19:06:05.652705+0000 mon.a (mon.0) 1964 : cluster 0 osdmap e342: 8 total, 6 up, 7 in 2023-09-23T19:06:07.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:06 smithi173 bash[59422]: cluster 2023-09-23T19:06:05.659961+0000 osd.6 (osd.6) 324 : cluster 3 failed to encode map e342 with expected crc 2023-09-23T19:06:07.072 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:07.105 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:07.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:07.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:07.207 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:07.238 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:07.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:07.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:07.342 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:07.367 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:07.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:07.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:07.471 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:07.502 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:07.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:07.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:07.605 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:07.637 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:07.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:07.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:07.740 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:07.753 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:07 smithi098 bash[76817]: cluster 2023-09-23T19:06:06.341788+0000 osd.4 (osd.4) 73 : cluster 3 failed to encode map e342 with expected crc 2023-09-23T19:06:07.753 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:07 smithi098 bash[76817]: cluster 2023-09-23T19:06:06.578312+0000 mgr.y (mgr.26054) 1451 : cluster 0 pgmap v1594: 99 pgs: 2 peering, 10 unknown, 25 active+clean+remapped, 5 active+undersized+remapped, 33 active+clean, 7 active+undersized+degraded, 17 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:07.753 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:07 smithi098 bash[76817]: cluster 2023-09-23T19:06:06.657859+0000 mon.a (mon.0) 1965 : cluster 3 Health check update: Degraded data redundancy: 9/84 objects degraded (10.714%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T19:06:07.753 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:07 smithi098 bash[76817]: cluster 2023-09-23T19:06:06.670569+0000 osd.4 (osd.4) 74 : cluster 3 failed to encode map e343 with expected crc 2023-09-23T19:06:07.753 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:07 smithi098 bash[76817]: cluster 2023-09-23T19:06:06.677637+0000 mon.a (mon.0) 1966 : cluster 0 osdmap e343: 8 total, 6 up, 7 in 2023-09-23T19:06:07.753 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:07 smithi098 bash[79516]: cluster 2023-09-23T19:06:06.341788+0000 osd.4 (osd.4) 73 : cluster 3 failed to encode map e342 with expected crc 2023-09-23T19:06:07.753 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:07 smithi098 bash[79516]: cluster 2023-09-23T19:06:06.578312+0000 mgr.y (mgr.26054) 1451 : cluster 0 pgmap v1594: 99 pgs: 2 peering, 10 unknown, 25 active+clean+remapped, 5 active+undersized+remapped, 33 active+clean, 7 active+undersized+degraded, 17 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:07.753 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:07 smithi098 bash[79516]: cluster 2023-09-23T19:06:06.657859+0000 mon.a (mon.0) 1965 : cluster 3 Health check update: Degraded data redundancy: 9/84 objects degraded (10.714%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T19:06:07.754 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:07 smithi098 bash[79516]: cluster 2023-09-23T19:06:06.670569+0000 osd.4 (osd.4) 74 : cluster 3 failed to encode map e343 with expected crc 2023-09-23T19:06:07.754 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:07 smithi098 bash[79516]: cluster 2023-09-23T19:06:06.677637+0000 mon.a (mon.0) 1966 : cluster 0 osdmap e343: 8 total, 6 up, 7 in 2023-09-23T19:06:07.768 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:07.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:07.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:07.832 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:06:07.871 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:07.906 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:07.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:07.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:08.010 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:08.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:07 smithi173 bash[59422]: cluster 2023-09-23T19:06:06.341788+0000 osd.4 (osd.4) 73 : cluster 3 failed to encode map e342 with expected crc 2023-09-23T19:06:08.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:07 smithi173 bash[59422]: cluster 2023-09-23T19:06:06.578312+0000 mgr.y (mgr.26054) 1451 : cluster 0 pgmap v1594: 99 pgs: 2 peering, 10 unknown, 25 active+clean+remapped, 5 active+undersized+remapped, 33 active+clean, 7 active+undersized+degraded, 17 active+undersized; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:08.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:07 smithi173 bash[59422]: cluster 2023-09-23T19:06:06.657859+0000 mon.a (mon.0) 1965 : cluster 3 Health check update: Degraded data redundancy: 9/84 objects degraded (10.714%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T19:06:08.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:07 smithi173 bash[59422]: cluster 2023-09-23T19:06:06.670569+0000 osd.4 (osd.4) 74 : cluster 3 failed to encode map e343 with expected crc 2023-09-23T19:06:08.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:07 smithi173 bash[59422]: cluster 2023-09-23T19:06:06.677637+0000 mon.a (mon.0) 1966 : cluster 0 osdmap e343: 8 total, 6 up, 7 in 2023-09-23T19:06:08.035 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:08.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:08.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:08.139 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:08.168 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:08.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:08.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:08.272 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:08.304 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:08.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:08.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:08.354 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:08.408 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:08.446 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:08.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:08.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:08.550 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:08.574 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:08.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:08.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:08.678 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:08.702 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:08.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:08.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:08.805 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:08.813 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:08 smithi098 bash[76817]: cluster 2023-09-23T19:06:06.989013+0000 osd.6 (osd.6) 325 : cluster 3 failed to encode map e343 with expected crc 2023-09-23T19:06:08.813 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:08 smithi098 bash[76817]: cluster 2023-09-23T19:06:07.680437+0000 mon.a (mon.0) 1967 : cluster 0 osdmap e344: 8 total, 6 up, 7 in 2023-09-23T19:06:08.813 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:08 smithi098 bash[76817]: cluster 2023-09-23T19:06:07.680620+0000 osd.4 (osd.4) 75 : cluster 3 failed to encode map e344 with expected crc 2023-09-23T19:06:08.813 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:08 smithi098 bash[79516]: cluster 2023-09-23T19:06:06.989013+0000 osd.6 (osd.6) 325 : cluster 3 failed to encode map e343 with expected crc 2023-09-23T19:06:08.813 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:08 smithi098 bash[79516]: cluster 2023-09-23T19:06:07.680437+0000 mon.a (mon.0) 1967 : cluster 0 osdmap e344: 8 total, 6 up, 7 in 2023-09-23T19:06:08.813 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:08 smithi098 bash[79516]: cluster 2023-09-23T19:06:07.680620+0000 osd.4 (osd.4) 75 : cluster 3 failed to encode map e344 with expected crc 2023-09-23T19:06:08.830 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:08.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:08.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:08.933 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:08.958 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:08.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:08.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:09.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:08 smithi173 bash[59422]: cluster 2023-09-23T19:06:06.989013+0000 osd.6 (osd.6) 325 : cluster 3 failed to encode map e343 with expected crc 2023-09-23T19:06:09.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:08 smithi173 bash[59422]: cluster 2023-09-23T19:06:07.680437+0000 mon.a (mon.0) 1967 : cluster 0 osdmap e344: 8 total, 6 up, 7 in 2023-09-23T19:06:09.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:08 smithi173 bash[59422]: cluster 2023-09-23T19:06:07.680620+0000 osd.4 (osd.4) 75 : cluster 3 failed to encode map e344 with expected crc 2023-09-23T19:06:09.063 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:09.094 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:09.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:09.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:09.197 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:09.231 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:09.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:09.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:09.334 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:09.369 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:09.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:09.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:09.472 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:09.503 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:09.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:09.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:09.606 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:09.640 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:09.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:09.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:09.744 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:09.757 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:09 smithi173 bash[59422]: cluster 2023-09-23T19:06:08.579257+0000 mgr.y (mgr.26054) 1452 : cluster 0 pgmap v1597: 99 pgs: 1 remapped+peering, 2 peering, 2 unknown, 25 active+clean+remapped, 5 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:09.758 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:09 smithi173 bash[59422]: audit 2023-09-23T19:06:09.646562+0000 mon.a (mon.0) 1968 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:06:09.778 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:09.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:09.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:09.882 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:09.889 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:09 smithi098 bash[79516]: cluster 2023-09-23T19:06:08.579257+0000 mgr.y (mgr.26054) 1452 : cluster 0 pgmap v1597: 99 pgs: 1 remapped+peering, 2 peering, 2 unknown, 25 active+clean+remapped, 5 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:09.890 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:09 smithi098 bash[79516]: audit 2023-09-23T19:06:09.646562+0000 mon.a (mon.0) 1968 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:06:09.890 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:09 smithi098 bash[76817]: cluster 2023-09-23T19:06:08.579257+0000 mgr.y (mgr.26054) 1452 : cluster 0 pgmap v1597: 99 pgs: 1 remapped+peering, 2 peering, 2 unknown, 25 active+clean+remapped, 5 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:09.890 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:09 smithi098 bash[76817]: audit 2023-09-23T19:06:09.646562+0000 mon.a (mon.0) 1968 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:06:09.906 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:09.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:09.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:10.009 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:10.037 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:10.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:10.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:10.140 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:10.169 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:10.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:10.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:10.272 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:10.301 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:10.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:10.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:10.405 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:10.435 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:10.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:10.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:10.533 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:10.538 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:10.571 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:10.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:10.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:10.675 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:10.696 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:10.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:10.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:10.799 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:10.825 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 6, 7, 5, 3] out_osds: [4] dead_osds: [7, 5] live_osds: [0, 2, 3, 6, 1, 4] 2023-09-23T19:06:10.825 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:06:10.825 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-09-23T19:06:10.825 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T19:06:10.825 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T19:06:10.825 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-09-23T19:06:10.826 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:10.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:10.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:10.929 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:10.949 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:10.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:10.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:11.052 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:11.080 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:11.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:11.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:11.183 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:11.214 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:11.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:11.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:11.318 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:11.350 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:11.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:11.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:11.454 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:11.486 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:11.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:11.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:11.589 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:11.621 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:11.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:11.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:11.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:11 smithi098 bash[76817]: cluster 2023-09-23T19:06:10.580471+0000 mgr.y (mgr.26054) 1453 : cluster 0 pgmap v1598: 99 pgs: 1 remapped+peering, 25 active+clean+remapped, 5 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:11.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:11 smithi098 bash[79516]: cluster 2023-09-23T19:06:10.580471+0000 mgr.y (mgr.26054) 1453 : cluster 0 pgmap v1598: 99 pgs: 1 remapped+peering, 25 active+clean+remapped, 5 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:11.725 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:11.754 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:11.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:11.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:11.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:11 smithi173 bash[59422]: cluster 2023-09-23T19:06:10.580471+0000 mgr.y (mgr.26054) 1453 : cluster 0 pgmap v1598: 99 pgs: 1 remapped+peering, 25 active+clean+remapped, 5 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:11.857 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:11.884 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:11.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:11.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:11.988 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:12.014 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:12.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:12.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:12.120 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:12.145 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:12.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:12.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:12.249 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:12.279 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:12.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:12.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:12.383 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:12.412 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:12.414 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:12.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:12.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:12.517 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:12.553 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:12.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:12.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:12.657 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:12.679 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:12.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:12.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:12.782 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:12.813 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:12.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:12.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:12.916 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:12.940 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:12.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:12.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:13.043 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:13.065 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:13.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:13.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:13.168 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:13.193 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:13.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:13.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:13.297 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:13.325 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:13.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:13.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:13.429 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:13.459 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:13.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:13.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:13.562 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:13.584 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:13.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:13.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:13.687 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:13.715 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:13.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:13.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:13.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:13 smithi173 bash[59422]: cluster 2023-09-23T19:06:12.581792+0000 mgr.y (mgr.26054) 1454 : cluster 0 pgmap v1599: 99 pgs: 25 active+clean+remapped, 6 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:13.819 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:13.827 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:13 smithi098 bash[76817]: cluster 2023-09-23T19:06:12.581792+0000 mgr.y (mgr.26054) 1454 : cluster 0 pgmap v1599: 99 pgs: 25 active+clean+remapped, 6 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:13.827 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:13 smithi098 bash[79516]: cluster 2023-09-23T19:06:12.581792+0000 mgr.y (mgr.26054) 1454 : cluster 0 pgmap v1599: 99 pgs: 25 active+clean+remapped, 6 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:13.840 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:13.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:13.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:13.944 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:13.975 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:13.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:13.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:14.078 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:14.099 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:14.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:14.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:14.202 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:14.232 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:14.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:14.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:14.336 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:14.358 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:14.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:14.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:14.461 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:14.494 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:14.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:14.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:14.598 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:14.630 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:14.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:14.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:14.732 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:14.753 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:14.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:14.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:14.857 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:14.881 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:14.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:14.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:14.984 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:15.018 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:15.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:15.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:15.123 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:15.150 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:15.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:15.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:15.253 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:15.285 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:15.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:15.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:15.387 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:15.412 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:15.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:15.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:15.516 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:15.537 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:15.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:15.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:15.591 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:06:15.641 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:15.662 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:15.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:15.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:15.765 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:15.800 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:15.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:15.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:15.904 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:15.931 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:15.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:15.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:16.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:15 smithi173 bash[59422]: cluster 2023-09-23T19:06:14.582549+0000 mgr.y (mgr.26054) 1455 : cluster 0 pgmap v1600: 99 pgs: 25 active+clean+remapped, 6 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:16.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:15 smithi173 bash[59422]: cluster 2023-09-23T19:06:15.541156+0000 mon.a (mon.0) 1969 : cluster 3 Health check update: Degraded data redundancy: 9/84 objects degraded (10.714%), 7 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T19:06:16.035 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:16.044 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:15 smithi098 bash[76817]: cluster 2023-09-23T19:06:14.582549+0000 mgr.y (mgr.26054) 1455 : cluster 0 pgmap v1600: 99 pgs: 25 active+clean+remapped, 6 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:16.044 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:15 smithi098 bash[76817]: cluster 2023-09-23T19:06:15.541156+0000 mon.a (mon.0) 1969 : cluster 3 Health check update: Degraded data redundancy: 9/84 objects degraded (10.714%), 7 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T19:06:16.044 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:15 smithi098 bash[79516]: cluster 2023-09-23T19:06:14.582549+0000 mgr.y (mgr.26054) 1455 : cluster 0 pgmap v1600: 99 pgs: 25 active+clean+remapped, 6 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:16.044 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:15 smithi098 bash[79516]: cluster 2023-09-23T19:06:15.541156+0000 mon.a (mon.0) 1969 : cluster 3 Health check update: Degraded data redundancy: 9/84 objects degraded (10.714%), 7 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T19:06:16.057 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:16.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:16.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:16.161 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:16.187 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:16.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:16.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:16.291 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:16.311 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:06:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:06:16] "GET /metrics HTTP/1.1" 200 36624 "" "Prometheus/2.33.4" 2023-09-23T19:06:16.326 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:16.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:16.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:16.430 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:16.454 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:16.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:16.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:16.557 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:16.579 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:16.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:16.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:16.683 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:16.708 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:16.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:16.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:16.812 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:16.839 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:16.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:16.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:16.942 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:16.969 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:16.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:16.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:17.072 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:17.094 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:17.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:17.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:17.198 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:17.225 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:17.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:17.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:17.329 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:17.353 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:17.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:17.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:17.376 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T19:06:17.376 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [], 2023-09-23T19:06:17.376 INFO:teuthology.orchestra.run.smithi098.stdout: "complaint_time": 30, 2023-09-23T19:06:17.376 INFO:teuthology.orchestra.run.smithi098.stdout: "num_blocked_ops": 0 2023-09-23T19:06:17.376 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T19:06:17.457 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:17.486 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:17.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:17.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:17.590 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:17.617 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:17.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:17.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:17.720 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:17.746 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:17.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:17.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:17.849 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:17.857 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:17 smithi098 bash[79516]: cluster 2023-09-23T19:06:16.583632+0000 mgr.y (mgr.26054) 1456 : cluster 0 pgmap v1601: 99 pgs: 25 active+clean+remapped, 6 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:17.858 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:17 smithi098 bash[76817]: cluster 2023-09-23T19:06:16.583632+0000 mgr.y (mgr.26054) 1456 : cluster 0 pgmap v1601: 99 pgs: 25 active+clean+remapped, 6 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:17.872 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:17.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:17.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:17.975 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:18.001 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:18.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:18.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:18.019 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:17 smithi173 bash[59422]: cluster 2023-09-23T19:06:16.583632+0000 mgr.y (mgr.26054) 1456 : cluster 0 pgmap v1601: 99 pgs: 25 active+clean+remapped, 6 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:18.104 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:18.132 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:18.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:18.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:18.235 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:18.259 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:18.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:18.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:18.362 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:18.386 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:18.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:18.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:18.490 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:18.523 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:18.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:18.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:18.627 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:18.648 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:18.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:18.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:18.752 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:18.779 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:18.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:18.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:18.882 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:18.903 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:18.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:18.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:19.005 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:19.027 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:19.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:19.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:19.131 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:19.153 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:19.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:19.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:19.201 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T19:06:19.201 INFO:teuthology.orchestra.run.smithi173.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T19:06:19.201 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T19:06:19.257 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:19.282 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:19.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:19.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:19.386 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:19.418 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:19.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:19.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:19.523 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:19.548 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:19.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:19.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:19.651 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:19.674 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:19.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:19.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:19.777 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:19.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:19 smithi173 bash[59422]: cluster 2023-09-23T19:06:18.585320+0000 mgr.y (mgr.26054) 1457 : cluster 0 pgmap v1602: 99 pgs: 25 active+clean+remapped, 6 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:19.812 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:19.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:19.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:19.916 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:19.929 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:19 smithi098 bash[79516]: cluster 2023-09-23T19:06:18.585320+0000 mgr.y (mgr.26054) 1457 : cluster 0 pgmap v1602: 99 pgs: 25 active+clean+remapped, 6 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:19.930 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:19 smithi098 bash[76817]: cluster 2023-09-23T19:06:18.585320+0000 mgr.y (mgr.26054) 1457 : cluster 0 pgmap v1602: 99 pgs: 25 active+clean+remapped, 6 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:19.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:19.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:19.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:20.048 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:20.073 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:20.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:20.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:20.175 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:20.201 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:20.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:20.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:20.304 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:20.331 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:20.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:20.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:20.435 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:20.462 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:20.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:20.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:20.565 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:20.589 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:20.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:20.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:20.692 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:20.715 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:20.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:20.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:20.818 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:20.843 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:20.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:20.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:20.947 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:20.978 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:20.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:20.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:21.077 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T19:06:21.081 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:21.149 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:21.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:21.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:21.252 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:21.284 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:21.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:21.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:21.388 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:21.419 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:21.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:21.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:21.524 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:21.549 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:21.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:21.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:21.652 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:21.677 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:21.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:21.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:21.780 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:21.791 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:21 smithi098 bash[76817]: cluster 2023-09-23T19:06:20.586093+0000 mgr.y (mgr.26054) 1458 : cluster 0 pgmap v1603: 99 pgs: 25 active+clean+remapped, 6 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:21.791 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:21 smithi098 bash[79516]: cluster 2023-09-23T19:06:20.586093+0000 mgr.y (mgr.26054) 1458 : cluster 0 pgmap v1603: 99 pgs: 25 active+clean+remapped, 6 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:21.807 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:21.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:21.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:21.911 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:21.924 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:21 smithi173 bash[59422]: cluster 2023-09-23T19:06:20.586093+0000 mgr.y (mgr.26054) 1458 : cluster 0 pgmap v1603: 99 pgs: 25 active+clean+remapped, 6 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:21.945 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:21.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:21.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:22.048 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:22.072 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:22.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:22.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:22.175 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:22.202 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:22.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:22.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:22.305 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:22.331 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:22.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:22.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:22.434 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:22.459 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:22.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:22.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:22.563 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:22.593 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:22.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:22.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:22.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:22 smithi098 bash[76817]: audit 2023-09-23T19:06:21.948750+0000 mon.a (mon.0) 1970 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:06:22.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:22 smithi098 bash[79516]: audit 2023-09-23T19:06:21.948750+0000 mon.a (mon.0) 1970 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:06:22.697 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:22.724 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:22.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:22.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:22.827 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:22.849 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:22.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:22.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:22.952 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:22.974 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:22.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:22.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:23.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:22 smithi173 bash[59422]: audit 2023-09-23T19:06:21.948750+0000 mon.a (mon.0) 1970 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:06:23.077 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:23.111 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:23.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:23.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:23.215 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:23.245 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:23.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:23.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:23.329 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T19:06:23.349 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:23.355 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T19:06:23.374 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:23.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:23.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:23.385 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T19:06:23.409 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T19:06:23.431 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T19:06:23.460 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T19:06:23.461 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:06:23.461 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:06:23.461 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:06:23.461 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:06:23.461 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:06:23.461 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T19:06:23.461 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:06:23.461 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T19:06:23.477 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:23.509 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:23.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:23.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:23.613 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:23.624 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:23 smithi098 bash[79516]: cluster 2023-09-23T19:06:22.586927+0000 mgr.y (mgr.26054) 1459 : cluster 0 pgmap v1604: 99 pgs: 25 active+clean+remapped, 6 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:23.624 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:23 smithi098 bash[79516]: audit 2023-09-23T19:06:23.299093+0000 mon.a (mon.0) 1971 : audit 0 from='client.? 172.21.15.98:0/4096973172' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:06:23.625 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:23 smithi098 bash[76817]: cluster 2023-09-23T19:06:22.586927+0000 mgr.y (mgr.26054) 1459 : cluster 0 pgmap v1604: 99 pgs: 25 active+clean+remapped, 6 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:23.625 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:23 smithi098 bash[76817]: audit 2023-09-23T19:06:23.299093+0000 mon.a (mon.0) 1971 : audit 0 from='client.? 172.21.15.98:0/4096973172' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:06:23.642 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:23.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:23.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:23.745 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:23.759 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:23 smithi173 bash[59422]: cluster 2023-09-23T19:06:22.586927+0000 mgr.y (mgr.26054) 1459 : cluster 0 pgmap v1604: 99 pgs: 25 active+clean+remapped, 6 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:23.759 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:23 smithi173 bash[59422]: audit 2023-09-23T19:06:23.299093+0000 mon.a (mon.0) 1971 : audit 0 from='client.? 172.21.15.98:0/4096973172' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:06:23.779 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:23.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:23.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:23.884 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:23.914 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:23.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:23.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:24.017 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:24.043 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:24.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:24.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:24.146 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:24.173 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:24.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:24.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:24.277 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:24.307 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:24.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:24.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:24.411 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:24.435 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:24.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:24.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:24.539 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:24.566 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:24.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:24.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:24.670 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:24.701 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:24.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:24.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:24.806 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:24.832 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:24.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:24.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:24.935 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:24.964 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:24.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:24.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:25.068 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:25.091 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:25.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:25.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:25.094 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 6, 7, 5, 3] out_osds: [4] dead_osds: [7, 5] live_osds: [0, 2, 3, 6, 1, 4] 2023-09-23T19:06:25.094 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:06:25.095 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T19:06:25.095 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-09-23T19:06:25.195 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:25.216 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:25.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:25.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:25.320 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:25.339 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:25.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:25.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:25.442 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:25.463 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:25.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:25.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:25.566 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:25.587 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:25.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:25.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:25.690 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:25.712 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:25.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:25.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:25.815 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:25.843 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:25.843 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:25 smithi098 bash[76817]: cluster 2023-09-23T19:06:24.588201+0000 mgr.y (mgr.26054) 1460 : cluster 0 pgmap v1605: 99 pgs: 25 active+clean+remapped, 6 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:25.843 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:25 smithi098 bash[76817]: audit 2023-09-23T19:06:24.647530+0000 mon.a (mon.0) 1972 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:06:25.844 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:25 smithi098 bash[79516]: cluster 2023-09-23T19:06:24.588201+0000 mgr.y (mgr.26054) 1460 : cluster 0 pgmap v1605: 99 pgs: 25 active+clean+remapped, 6 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:25.844 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:25 smithi098 bash[79516]: audit 2023-09-23T19:06:24.647530+0000 mon.a (mon.0) 1972 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:06:25.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:25.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:25.947 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:25.972 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:25.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:25.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:26.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:25 smithi173 bash[59422]: cluster 2023-09-23T19:06:24.588201+0000 mgr.y (mgr.26054) 1460 : cluster 0 pgmap v1605: 99 pgs: 25 active+clean+remapped, 6 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:26.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:25 smithi173 bash[59422]: audit 2023-09-23T19:06:24.647530+0000 mon.a (mon.0) 1972 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:06:26.075 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:26.100 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:26.102 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:06:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:06:26] "GET /metrics HTTP/1.1" 200 36618 "" "Prometheus/2.33.4" 2023-09-23T19:06:26.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:26.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:26.204 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:26.231 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:26.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:26.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:26.335 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:26.366 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:26.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:26.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:26.470 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:26.491 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:26.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:26.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:26.594 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:26.621 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:26.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:26.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:26.724 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:26.753 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:26.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:26.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:26.855 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:26.885 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:26.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:26.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:26.892 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:06:26.892 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:06:26.989 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:27.017 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:27.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:27.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:27.120 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:27.150 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:27.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:27.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:27.253 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:27.279 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:27.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:27.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:27.346 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:27.384 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:27.408 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:27.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:27.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:27.511 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:27.541 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:27.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:27.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:27.644 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:27.670 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:27.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:27.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:27.688 INFO:teuthology.orchestra.run.smithi098.stderr:pool 'unique_pool_5' removed 2023-09-23T19:06:27.692 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is unset 2023-09-23T19:06:27.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:27.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:27 smithi173 bash[59422]: cluster 2023-09-23T19:06:26.589399+0000 mgr.y (mgr.26054) 1461 : cluster 0 pgmap v1606: 99 pgs: 25 active+clean+remapped, 6 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:27.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:27 smithi173 bash[59422]: audit 2023-09-23T19:06:27.188462+0000 mon.c (mon.2) 149 : audit 1 from='client.? 172.21.15.98:0/2943952191' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:06:27.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:27 smithi173 bash[59422]: audit 2023-09-23T19:06:27.189054+0000 mon.a (mon.0) 1973 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:06:27.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:27 smithi173 bash[59422]: audit 2023-09-23T19:06:27.577374+0000 mon.a (mon.0) 1974 : audit 1 from='client.? 172.21.15.98:0/2701530934' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:06:27.801 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:27.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:27.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:27.904 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:27.915 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:27 smithi098 bash[79516]: cluster 2023-09-23T19:06:26.589399+0000 mgr.y (mgr.26054) 1461 : cluster 0 pgmap v1606: 99 pgs: 25 active+clean+remapped, 6 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:27.915 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:27 smithi098 bash[79516]: audit 2023-09-23T19:06:27.188462+0000 mon.c (mon.2) 149 : audit 1 from='client.? 172.21.15.98:0/2943952191' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:06:27.915 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:27 smithi098 bash[79516]: audit 2023-09-23T19:06:27.189054+0000 mon.a (mon.0) 1973 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:06:27.915 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:27 smithi098 bash[79516]: audit 2023-09-23T19:06:27.577374+0000 mon.a (mon.0) 1974 : audit 1 from='client.? 172.21.15.98:0/2701530934' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:06:27.916 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:27 smithi098 bash[76817]: cluster 2023-09-23T19:06:26.589399+0000 mgr.y (mgr.26054) 1461 : cluster 0 pgmap v1606: 99 pgs: 25 active+clean+remapped, 6 active+undersized+remapped, 38 active+clean, 7 active+undersized+degraded, 23 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:27.916 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:27 smithi098 bash[76817]: audit 2023-09-23T19:06:27.188462+0000 mon.c (mon.2) 149 : audit 1 from='client.? 172.21.15.98:0/2943952191' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:06:27.916 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:27 smithi098 bash[76817]: audit 2023-09-23T19:06:27.189054+0000 mon.a (mon.0) 1973 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T19:06:27.916 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:27 smithi098 bash[76817]: audit 2023-09-23T19:06:27.577374+0000 mon.a (mon.0) 1974 : audit 1 from='client.? 172.21.15.98:0/2701530934' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:06:27.931 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:27.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:27.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:28.037 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:28.102 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:28.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:28.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:28.207 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:28.232 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:28.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:28.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:28.335 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:28.364 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:28.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:28.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:28.468 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:28.494 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:28.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:28.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:28.597 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:28.622 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:28.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:28.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:28.726 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:28.759 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:28.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:28.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:28.863 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:28.877 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:28 smithi098 bash[79516]: cluster 2023-09-23T19:06:27.646426+0000 mon.a (mon.0) 1975 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:06:28.877 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:28 smithi098 bash[79516]: cluster 2023-09-23T19:06:27.659115+0000 osd.6 (osd.6) 326 : cluster 3 failed to encode map e345 with expected crc 2023-09-23T19:06:28.877 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:28 smithi098 bash[79516]: audit 2023-09-23T19:06:27.661470+0000 mon.a (mon.0) 1976 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:06:28.877 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:28 smithi098 bash[79516]: audit 2023-09-23T19:06:27.661709+0000 mon.a (mon.0) 1977 : audit 1 from='client.? 172.21.15.98:0/2701530934' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:06:28.877 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:28 smithi098 bash[79516]: cluster 2023-09-23T19:06:27.661963+0000 mon.a (mon.0) 1978 : cluster 0 osdmap e345: 8 total, 6 up, 7 in 2023-09-23T19:06:28.877 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:28 smithi098 bash[79516]: audit 2023-09-23T19:06:28.240418+0000 mon.a (mon.0) 1979 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:06:28.877 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:28 smithi098 bash[79516]: audit 2023-09-23T19:06:28.245736+0000 mon.a (mon.0) 1980 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:06:28.877 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:28 smithi098 bash[76817]: cluster 2023-09-23T19:06:27.646426+0000 mon.a (mon.0) 1975 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:06:28.877 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:28 smithi098 bash[76817]: cluster 2023-09-23T19:06:27.659115+0000 osd.6 (osd.6) 326 : cluster 3 failed to encode map e345 with expected crc 2023-09-23T19:06:28.878 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:28 smithi098 bash[76817]: audit 2023-09-23T19:06:27.661470+0000 mon.a (mon.0) 1976 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:06:28.878 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:28 smithi098 bash[76817]: audit 2023-09-23T19:06:27.661709+0000 mon.a (mon.0) 1977 : audit 1 from='client.? 172.21.15.98:0/2701530934' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:06:28.878 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:28 smithi098 bash[76817]: cluster 2023-09-23T19:06:27.661963+0000 mon.a (mon.0) 1978 : cluster 0 osdmap e345: 8 total, 6 up, 7 in 2023-09-23T19:06:28.878 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:28 smithi098 bash[76817]: audit 2023-09-23T19:06:28.240418+0000 mon.a (mon.0) 1979 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:06:28.878 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:28 smithi098 bash[76817]: audit 2023-09-23T19:06:28.245736+0000 mon.a (mon.0) 1980 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:06:28.892 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:28.893 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:06:28.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:28.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:28.996 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:29.015 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:29.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:29.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:29.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:28 smithi173 bash[59422]: cluster 2023-09-23T19:06:27.646426+0000 mon.a (mon.0) 1975 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:06:29.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:28 smithi173 bash[59422]: cluster 2023-09-23T19:06:27.659115+0000 osd.6 (osd.6) 326 : cluster 3 failed to encode map e345 with expected crc 2023-09-23T19:06:29.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:28 smithi173 bash[59422]: audit 2023-09-23T19:06:27.661470+0000 mon.a (mon.0) 1976 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T19:06:29.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:28 smithi173 bash[59422]: audit 2023-09-23T19:06:27.661709+0000 mon.a (mon.0) 1977 : audit 1 from='client.? 172.21.15.98:0/2701530934' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:06:29.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:28 smithi173 bash[59422]: cluster 2023-09-23T19:06:27.661963+0000 mon.a (mon.0) 1978 : cluster 0 osdmap e345: 8 total, 6 up, 7 in 2023-09-23T19:06:29.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:28 smithi173 bash[59422]: audit 2023-09-23T19:06:28.240418+0000 mon.a (mon.0) 1979 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:06:29.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:28 smithi173 bash[59422]: audit 2023-09-23T19:06:28.245736+0000 mon.a (mon.0) 1980 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:06:29.118 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:29.137 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:29.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:29.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:29.240 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:29.261 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:29.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:29.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:29.364 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:29.398 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:29.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:29.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:29.502 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:29.529 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:29.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:29.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:29.633 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:29.666 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:29.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:29.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:29.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:29 smithi098 bash[76817]: cluster 2023-09-23T19:06:27.944542+0000 osd.4 (osd.4) 76 : cluster 3 failed to encode map e345 with expected crc 2023-09-23T19:06:29.693 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:29 smithi098 bash[76817]: cluster 2023-09-23T19:06:28.590433+0000 mgr.y (mgr.26054) 1462 : cluster 0 pgmap v1608: 73 pgs: 24 active+clean+remapped, 4 active+undersized+remapped, 27 active+clean, 7 active+undersized+degraded, 11 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:29.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:29 smithi098 bash[76817]: cluster 2023-09-23T19:06:29.243905+0000 mon.a (mon.0) 1981 : cluster 3 Health check update: Degraded data redundancy: 9/84 objects degraded (10.714%), 7 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-09-23T19:06:29.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:29 smithi098 bash[79516]: cluster 2023-09-23T19:06:27.944542+0000 osd.4 (osd.4) 76 : cluster 3 failed to encode map e345 with expected crc 2023-09-23T19:06:29.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:29 smithi098 bash[79516]: cluster 2023-09-23T19:06:28.590433+0000 mgr.y (mgr.26054) 1462 : cluster 0 pgmap v1608: 73 pgs: 24 active+clean+remapped, 4 active+undersized+remapped, 27 active+clean, 7 active+undersized+degraded, 11 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:29.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:29 smithi098 bash[79516]: cluster 2023-09-23T19:06:29.243905+0000 mon.a (mon.0) 1981 : cluster 3 Health check update: Degraded data redundancy: 9/84 objects degraded (10.714%), 7 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-09-23T19:06:29.770 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:29.794 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:29.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:29.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:29.897 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:29.928 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:29.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:29.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:30.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:29 smithi173 bash[59422]: cluster 2023-09-23T19:06:27.944542+0000 osd.4 (osd.4) 76 : cluster 3 failed to encode map e345 with expected crc 2023-09-23T19:06:30.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:29 smithi173 bash[59422]: cluster 2023-09-23T19:06:28.590433+0000 mgr.y (mgr.26054) 1462 : cluster 0 pgmap v1608: 73 pgs: 24 active+clean+remapped, 4 active+undersized+remapped, 27 active+clean, 7 active+undersized+degraded, 11 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:30.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:29 smithi173 bash[59422]: cluster 2023-09-23T19:06:29.243905+0000 mon.a (mon.0) 1981 : cluster 3 Health check update: Degraded data redundancy: 9/84 objects degraded (10.714%), 7 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-09-23T19:06:30.032 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:30.062 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:30.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:30.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:30.166 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:30.196 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:30.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:30.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:30.299 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:30.331 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:30.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:30.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:30.434 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:30.466 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:30.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:30.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:30.570 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:30.601 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:30.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:30.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:30.705 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:30.730 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:30.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:30.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:30.833 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:30.862 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:30.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:30.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:30.966 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:30.994 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:30.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:30.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:31.097 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:31.126 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:31.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:31.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:31.171 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T19:06:31.171 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-09-23T19:06:31.171 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T19:06:31.172 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-09-23T19:06:31.172 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T19:06:31.172 DEBUG:teuthology.orchestra.run.smithi098:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T19:06:31.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:31.227 INFO:teuthology.orchestra.run.smithi098.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T19:06:31.228 DEBUG:teuthology.orchestra.run.smithi098:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T19:06:31.230 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:31.262 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:31.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:31.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:31.301 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T19:06:31.301 DEBUG:teuthology.orchestra.run.smithi098:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T19:06:31.352 INFO:tasks.workunit:timeout=3h 2023-09-23T19:06:31.352 INFO:tasks.workunit:cleanup=True 2023-09-23T19:06:31.353 DEBUG:teuthology.orchestra.run.smithi098:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T19:06:31.365 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:31.420 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:31.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:31.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:31.481 INFO:tasks.workunit.client.0.smithi098.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T19:06:31.524 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:31.563 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:31.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:31.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:31.601 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:31 smithi098 bash[79516]: cluster 2023-09-23T19:06:30.591415+0000 mgr.y (mgr.26054) 1463 : cluster 0 pgmap v1609: 73 pgs: 24 active+clean+remapped, 4 active+undersized+remapped, 27 active+clean, 7 active+undersized+degraded, 11 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:31.602 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:31 smithi098 bash[76817]: cluster 2023-09-23T19:06:30.591415+0000 mgr.y (mgr.26054) 1463 : cluster 0 pgmap v1609: 73 pgs: 24 active+clean+remapped, 4 active+undersized+remapped, 27 active+clean, 7 active+undersized+degraded, 11 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:31.668 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:31.703 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:31.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:31.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:31.807 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:31.818 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:31 smithi173 bash[59422]: cluster 2023-09-23T19:06:30.591415+0000 mgr.y (mgr.26054) 1463 : cluster 0 pgmap v1609: 73 pgs: 24 active+clean+remapped, 4 active+undersized+remapped, 27 active+clean, 7 active+undersized+degraded, 11 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:31.843 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:31.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:31.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:31.947 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:31.973 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:31.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:31.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:32.076 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:32.098 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:32.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:32.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:32.201 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:32.235 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:32.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:32.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:32.339 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:32.372 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:32.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:32.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:32.476 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:32.505 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:32.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:32.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:32.608 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:32.633 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:32.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:32.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:32.737 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:32.760 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:32.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:32.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:32.864 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:32.885 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:32.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:32.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:32.989 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:33.021 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:33.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:33.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:33.125 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:33.153 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:06:33.155 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:33.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:33.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:33.260 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:33.312 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:33.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:33.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:33.417 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:33.446 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:33.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:33.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:33.549 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:33.584 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:33.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:33.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:33.675 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:33.688 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:33.714 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:33 smithi098 bash[76817]: cluster 2023-09-23T19:06:32.592328+0000 mgr.y (mgr.26054) 1464 : cluster 0 pgmap v1610: 73 pgs: 24 active+clean+remapped, 4 active+undersized+remapped, 27 active+clean, 7 active+undersized+degraded, 11 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:33.714 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:33 smithi098 bash[79516]: cluster 2023-09-23T19:06:32.592328+0000 mgr.y (mgr.26054) 1464 : cluster 0 pgmap v1610: 73 pgs: 24 active+clean+remapped, 4 active+undersized+remapped, 27 active+clean, 7 active+undersized+degraded, 11 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:33.733 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:33.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:33.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:33.836 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:33.867 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:33.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:33.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:33.970 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:34.014 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:34.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:34.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:34.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:33 smithi173 bash[59422]: cluster 2023-09-23T19:06:32.592328+0000 mgr.y (mgr.26054) 1464 : cluster 0 pgmap v1610: 73 pgs: 24 active+clean+remapped, 4 active+undersized+remapped, 27 active+clean, 7 active+undersized+degraded, 11 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:34.117 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:34.150 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:34.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:34.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:34.253 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:34.274 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:34.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:34.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:34.378 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:34.401 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:34.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:34.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:34.504 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:34.535 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:34.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:34.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:34.638 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:34.659 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:34.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:34.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:34.762 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:34.784 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:34.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:34.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:34.887 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:34.911 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:34.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:34.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:35.014 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:35.046 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:35.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:35.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:35.150 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:35.177 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:35.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:35.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:35.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:35 smithi173 bash[59422]: audit 2023-09-23T19:06:34.199384+0000 mon.a (mon.0) 1982 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:06:35.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:35 smithi173 bash[59422]: audit 2023-09-23T19:06:34.206260+0000 mon.a (mon.0) 1983 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:06:35.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:35 smithi173 bash[59422]: audit 2023-09-23T19:06:34.627369+0000 mon.a (mon.0) 1984 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:06:35.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:35 smithi173 bash[59422]: audit 2023-09-23T19:06:34.628565+0000 mon.a (mon.0) 1985 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:06:35.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:35 smithi173 bash[59422]: audit 2023-09-23T19:06:34.635545+0000 mon.a (mon.0) 1986 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:06:35.281 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:35.311 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:35.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:35.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:35.414 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:35.441 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:35.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:35.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:35.545 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:35.553 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:35 smithi098 bash[76817]: audit 2023-09-23T19:06:34.199384+0000 mon.a (mon.0) 1982 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:06:35.553 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:35 smithi098 bash[76817]: audit 2023-09-23T19:06:34.206260+0000 mon.a (mon.0) 1983 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:06:35.554 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:35 smithi098 bash[76817]: audit 2023-09-23T19:06:34.627369+0000 mon.a (mon.0) 1984 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:06:35.554 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:35 smithi098 bash[76817]: audit 2023-09-23T19:06:34.628565+0000 mon.a (mon.0) 1985 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:06:35.554 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:35 smithi098 bash[76817]: audit 2023-09-23T19:06:34.635545+0000 mon.a (mon.0) 1986 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:06:35.554 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:35 smithi098 bash[79516]: audit 2023-09-23T19:06:34.199384+0000 mon.a (mon.0) 1982 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:06:35.554 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:35 smithi098 bash[79516]: audit 2023-09-23T19:06:34.206260+0000 mon.a (mon.0) 1983 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:06:35.554 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:35 smithi098 bash[79516]: audit 2023-09-23T19:06:34.627369+0000 mon.a (mon.0) 1984 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:06:35.554 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:35 smithi098 bash[79516]: audit 2023-09-23T19:06:34.628565+0000 mon.a (mon.0) 1985 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:06:35.554 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:35 smithi098 bash[79516]: audit 2023-09-23T19:06:34.635545+0000 mon.a (mon.0) 1986 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:06:35.571 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:35.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:35.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:35.674 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:35.698 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:35.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:35.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:35.702 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:35.802 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:35.822 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:35.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:35.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:35.925 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:35.946 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:35.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:35.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:36.049 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:36.073 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:36.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:36.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:36.176 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:36.197 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:36.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:36.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:36.301 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:36.323 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:36.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:36.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:36.376 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:36 smithi098 bash[76817]: cluster 2023-09-23T19:06:34.592924+0000 mgr.y (mgr.26054) 1465 : cluster 0 pgmap v1611: 73 pgs: 24 active+clean+remapped, 4 active+undersized+remapped, 27 active+clean, 7 active+undersized+degraded, 11 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:36.377 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:36 smithi098 bash[79516]: cluster 2023-09-23T19:06:34.592924+0000 mgr.y (mgr.26054) 1465 : cluster 0 pgmap v1611: 73 pgs: 24 active+clean+remapped, 4 active+undersized+remapped, 27 active+clean, 7 active+undersized+degraded, 11 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:36.377 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:06:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:06:36] "GET /metrics HTTP/1.1" 200 36618 "" "Prometheus/2.33.4" 2023-09-23T19:06:36.426 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:36.452 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:36.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:36.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:36.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:36 smithi173 bash[59422]: cluster 2023-09-23T19:06:34.592924+0000 mgr.y (mgr.26054) 1465 : cluster 0 pgmap v1611: 73 pgs: 24 active+clean+remapped, 4 active+undersized+remapped, 27 active+clean, 7 active+undersized+degraded, 11 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:36.556 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:36.575 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:36.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:36.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:36.637 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T19:06:36.637 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:06:36.677 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:36.699 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:36.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:36.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:36.802 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:36.831 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:36.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:36.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:36.935 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:36.953 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:36.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:36.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:37.056 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:37.076 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:37.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:37.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:37.180 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:37.212 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:37.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:37.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:37.316 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:37.341 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:37.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:37.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:37.445 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:37.475 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:37.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:37.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:37.579 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:37.604 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:37.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:37.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:37.708 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:37.735 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:37.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:37.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:37.838 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:37.873 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:37.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:37.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:37.976 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:37.997 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:37.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:38.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:38.100 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:38.132 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:38.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:38.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:38.235 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:38.268 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:38.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:38.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:38.372 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:38.384 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:38 smithi098 bash[76817]: cluster 2023-09-23T19:06:36.593486+0000 mgr.y (mgr.26054) 1466 : cluster 0 pgmap v1612: 73 pgs: 24 active+clean+remapped, 4 active+undersized+remapped, 27 active+clean, 7 active+undersized+degraded, 11 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:38.384 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:38 smithi098 bash[79516]: cluster 2023-09-23T19:06:36.593486+0000 mgr.y (mgr.26054) 1466 : cluster 0 pgmap v1612: 73 pgs: 24 active+clean+remapped, 4 active+undersized+remapped, 27 active+clean, 7 active+undersized+degraded, 11 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:38.395 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:38.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:38.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:38.499 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:38.522 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:38.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:38.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:38.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:38 smithi173 bash[59422]: cluster 2023-09-23T19:06:36.593486+0000 mgr.y (mgr.26054) 1466 : cluster 0 pgmap v1612: 73 pgs: 24 active+clean+remapped, 4 active+undersized+remapped, 27 active+clean, 7 active+undersized+degraded, 11 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:38.625 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:38.650 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:38.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:38.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:38.753 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:38.761 DEBUG:teuthology.orchestra.run.smithi098:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T19:06:38.778 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:38.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:38.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:38.782 INFO:tasks.workunit.client.0.smithi098.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T19:06:38.784 INFO:tasks.workunit.client.0.smithi098.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T19:06:38.784 INFO:tasks.workunit.client.0.smithi098.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T19:06:38.881 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:38.914 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:38.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:38.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:39.018 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:39.044 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:39.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:39.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:39.091 INFO:tasks.workunit.client.0.smithi098.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T19:06:39.148 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:39.168 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:39.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:39.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:39.220 INFO:tasks.workunit.client.0.smithi098.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T19:06:39.272 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:39.297 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:39.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:39.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:39.352 INFO:tasks.workunit.client.0.smithi098.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T19:06:39.355 INFO:tasks.workunit.client.0.smithi098.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T19:06:39.355 INFO:tasks.workunit.client.0.smithi098.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T19:06:39.399 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:39.418 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:39.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:39.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:39.520 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:39.545 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:39.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:39.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:39.648 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:39.677 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:39.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:39.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:39.761 INFO:tasks.workunit.client.0.smithi098.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T19:06:39.767 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-23T19:06:39.767 DEBUG:teuthology.orchestra.run.smithi098:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T19:06:39.782 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:39.810 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:39.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:39.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:39.878 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-09-23T19:06:39.879 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-09-23T19:06:39.879 DEBUG:teuthology.orchestra.run.smithi098:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-09-23T19:06:39.913 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:39.949 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:39.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:39.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:40.053 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:40.080 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:40.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:40.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:40.183 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:40.214 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:40.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:40.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:40.318 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:40.344 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:40.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:40.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:40.448 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:40.460 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:40 smithi173 bash[59422]: cluster 2023-09-23T19:06:38.594349+0000 mgr.y (mgr.26054) 1467 : cluster 0 pgmap v1613: 73 pgs: 24 active+clean+remapped, 4 active+undersized+remapped, 27 active+clean, 7 active+undersized+degraded, 11 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:40.460 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:40 smithi173 bash[59422]: audit 2023-09-23T19:06:39.647101+0000 mon.a (mon.0) 1987 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:06:40.477 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:40.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:40.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:40.581 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:40.588 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:40 smithi098 bash[79516]: cluster 2023-09-23T19:06:38.594349+0000 mgr.y (mgr.26054) 1467 : cluster 0 pgmap v1613: 73 pgs: 24 active+clean+remapped, 4 active+undersized+remapped, 27 active+clean, 7 active+undersized+degraded, 11 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:40.588 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:40 smithi098 bash[79516]: audit 2023-09-23T19:06:39.647101+0000 mon.a (mon.0) 1987 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:06:40.589 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:40 smithi098 bash[76817]: cluster 2023-09-23T19:06:38.594349+0000 mgr.y (mgr.26054) 1467 : cluster 0 pgmap v1613: 73 pgs: 24 active+clean+remapped, 4 active+undersized+remapped, 27 active+clean, 7 active+undersized+degraded, 11 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:40.589 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:40 smithi098 bash[76817]: audit 2023-09-23T19:06:39.647101+0000 mon.a (mon.0) 1987 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:06:40.607 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:40.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:40.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:40.710 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:40.741 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:40.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:40.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:40.844 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:40.877 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:40.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:40.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:40.980 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:41.015 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:41.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:41.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:41.042 INFO:tasks.workunit.client.0.smithi098.stdout:Running main() from gmock_main.cc 2023-09-23T19:06:41.042 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] Running 70 tests from 1 test suite. 2023-09-23T19:06:41.042 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment set-up. 2023-09-23T19:06:41.042 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 70 tests from TestClsRbd 2023-09-23T19:06:41.119 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:41.141 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:41.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:41.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:41.245 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:41.263 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:41.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:41.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:41.367 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:41.391 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:41.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:41.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:41.493 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:41.517 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:41.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:41.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:41.620 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:41.640 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:41.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:41.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:41.743 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:41.776 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:41.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:41.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:41.879 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:41.892 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:41 smithi173 bash[59422]: cluster 2023-09-23T19:06:40.594967+0000 mgr.y (mgr.26054) 1468 : cluster 0 pgmap v1614: 73 pgs: 24 active+clean+remapped, 4 active+undersized+remapped, 27 active+clean, 7 active+undersized+degraded, 11 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:41.912 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:41.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:41.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:42.005 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:41 smithi098 bash[76817]: cluster 2023-09-23T19:06:40.594967+0000 mgr.y (mgr.26054) 1468 : cluster 0 pgmap v1614: 73 pgs: 24 active+clean+remapped, 4 active+undersized+remapped, 27 active+clean, 7 active+undersized+degraded, 11 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:42.016 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:42.040 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:42.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:42.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:42.143 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:42.176 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:42.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:42.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:42.280 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:42.310 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:42.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:42.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:42.413 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:42.426 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:42 smithi098 bash[79516]: cluster 2023-09-23T19:06:40.594967+0000 mgr.y (mgr.26054) 1468 : cluster 0 pgmap v1614: 73 pgs: 24 active+clean+remapped, 4 active+undersized+remapped, 27 active+clean, 7 active+undersized+degraded, 11 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:42.440 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:42.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:42.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:42.543 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:42.572 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:42.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:42.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:42.582 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.get_all_features 2023-09-23T19:06:42.603 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.get_all_features (21 ms) 2023-09-23T19:06:42.603 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.copyup 2023-09-23T19:06:42.676 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:42.685 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:42 smithi098 bash[79516]: cluster 2023-09-23T19:06:41.567360+0000 mon.a (mon.0) 1988 : cluster 0 osdmap e346: 8 total, 6 up, 7 in 2023-09-23T19:06:42.686 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:42 smithi098 bash[79516]: cluster 2023-09-23T19:06:41.568433+0000 osd.6 (osd.6) 327 : cluster 3 failed to encode map e346 with expected crc 2023-09-23T19:06:42.686 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:42 smithi098 bash[79516]: cluster 2023-09-23T19:06:41.568571+0000 osd.4 (osd.4) 77 : cluster 3 failed to encode map e346 with expected crc 2023-09-23T19:06:42.686 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:42 smithi098 bash[79516]: cluster 2023-09-23T19:06:41.572345+0000 osd.6 (osd.6) 328 : cluster 3 failed to encode map e346 with expected crc 2023-09-23T19:06:42.686 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:42 smithi098 bash[79516]: audit 2023-09-23T19:06:41.999527+0000 mon.c (mon.2) 150 : audit 1 from='client.? 172.21.15.98:0/1050517665' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi098-254928-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:06:42.686 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:42 smithi098 bash[79516]: audit 2023-09-23T19:06:41.999985+0000 mon.a (mon.0) 1989 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi098-254928-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:06:42.686 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:42 smithi098 bash[76817]: cluster 2023-09-23T19:06:41.567360+0000 mon.a (mon.0) 1988 : cluster 0 osdmap e346: 8 total, 6 up, 7 in 2023-09-23T19:06:42.686 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:42 smithi098 bash[76817]: cluster 2023-09-23T19:06:41.568433+0000 osd.6 (osd.6) 327 : cluster 3 failed to encode map e346 with expected crc 2023-09-23T19:06:42.686 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:42 smithi098 bash[76817]: cluster 2023-09-23T19:06:41.568571+0000 osd.4 (osd.4) 77 : cluster 3 failed to encode map e346 with expected crc 2023-09-23T19:06:42.687 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:42 smithi098 bash[76817]: cluster 2023-09-23T19:06:41.572345+0000 osd.6 (osd.6) 328 : cluster 3 failed to encode map e346 with expected crc 2023-09-23T19:06:42.687 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:42 smithi098 bash[76817]: audit 2023-09-23T19:06:41.999527+0000 mon.c (mon.2) 150 : audit 1 from='client.? 172.21.15.98:0/1050517665' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi098-254928-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:06:42.687 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:42 smithi098 bash[76817]: audit 2023-09-23T19:06:41.999985+0000 mon.a (mon.0) 1989 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi098-254928-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:06:42.698 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:42.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:42.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:42.802 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:42.829 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:42.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:42.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:42.932 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:42.946 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:42 smithi173 bash[59422]: cluster 2023-09-23T19:06:41.567360+0000 mon.a (mon.0) 1988 : cluster 0 osdmap e346: 8 total, 6 up, 7 in 2023-09-23T19:06:42.946 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:42 smithi173 bash[59422]: cluster 2023-09-23T19:06:41.568433+0000 osd.6 (osd.6) 327 : cluster 3 failed to encode map e346 with expected crc 2023-09-23T19:06:42.947 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:42 smithi173 bash[59422]: cluster 2023-09-23T19:06:41.568571+0000 osd.4 (osd.4) 77 : cluster 3 failed to encode map e346 with expected crc 2023-09-23T19:06:42.947 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:42 smithi173 bash[59422]: cluster 2023-09-23T19:06:41.572345+0000 osd.6 (osd.6) 328 : cluster 3 failed to encode map e346 with expected crc 2023-09-23T19:06:42.947 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:42 smithi173 bash[59422]: audit 2023-09-23T19:06:41.999527+0000 mon.c (mon.2) 150 : audit 1 from='client.? 172.21.15.98:0/1050517665' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi098-254928-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:06:42.947 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:42 smithi173 bash[59422]: audit 2023-09-23T19:06:41.999985+0000 mon.a (mon.0) 1989 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi098-254928-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T19:06:42.963 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:42.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:42.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:43.067 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:43.094 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:43.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:43.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:43.197 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:43.217 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.copyup (614 ms) 2023-09-23T19:06:43.217 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-09-23T19:06:43.219 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:43.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:43.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:43.322 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:43.351 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:43.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:43.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:43.373 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T19:06:43.454 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:43.486 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:43.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:43.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:43.590 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:43.611 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:43.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:43.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:43.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:43 smithi098 bash[79516]: audit 2023-09-23T19:06:42.574229+0000 mon.a (mon.0) 1990 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi098-254928-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T19:06:43.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:43 smithi098 bash[79516]: cluster 2023-09-23T19:06:42.574352+0000 mon.a (mon.0) 1991 : cluster 0 osdmap e347: 8 total, 6 up, 7 in 2023-09-23T19:06:43.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:43 smithi098 bash[79516]: cluster 2023-09-23T19:06:42.575036+0000 osd.6 (osd.6) 329 : cluster 3 failed to encode map e347 with expected crc 2023-09-23T19:06:43.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:43 smithi098 bash[79516]: cluster 2023-09-23T19:06:42.578171+0000 osd.6 (osd.6) 330 : cluster 3 failed to encode map e347 with expected crc 2023-09-23T19:06:43.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:43 smithi098 bash[79516]: cluster 2023-09-23T19:06:42.595656+0000 mgr.y (mgr.26054) 1469 : cluster 0 pgmap v1617: 105 pgs: 19 creating+peering, 13 unknown, 24 active+clean+remapped, 4 active+undersized+remapped, 27 active+clean, 7 active+undersized+degraded, 11 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:43.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:43 smithi098 bash[76817]: audit 2023-09-23T19:06:42.574229+0000 mon.a (mon.0) 1990 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi098-254928-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T19:06:43.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:43 smithi098 bash[76817]: cluster 2023-09-23T19:06:42.574352+0000 mon.a (mon.0) 1991 : cluster 0 osdmap e347: 8 total, 6 up, 7 in 2023-09-23T19:06:43.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:43 smithi098 bash[76817]: cluster 2023-09-23T19:06:42.575036+0000 osd.6 (osd.6) 329 : cluster 3 failed to encode map e347 with expected crc 2023-09-23T19:06:43.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:43 smithi098 bash[76817]: cluster 2023-09-23T19:06:42.578171+0000 osd.6 (osd.6) 330 : cluster 3 failed to encode map e347 with expected crc 2023-09-23T19:06:43.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:43 smithi098 bash[76817]: cluster 2023-09-23T19:06:42.595656+0000 mgr.y (mgr.26054) 1469 : cluster 0 pgmap v1617: 105 pgs: 19 creating+peering, 13 unknown, 24 active+clean+remapped, 4 active+undersized+remapped, 27 active+clean, 7 active+undersized+degraded, 11 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:43.715 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:43.729 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:43 smithi173 bash[59422]: audit 2023-09-23T19:06:42.574229+0000 mon.a (mon.0) 1990 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi098-254928-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T19:06:43.729 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:43 smithi173 bash[59422]: cluster 2023-09-23T19:06:42.574352+0000 mon.a (mon.0) 1991 : cluster 0 osdmap e347: 8 total, 6 up, 7 in 2023-09-23T19:06:43.729 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:43 smithi173 bash[59422]: cluster 2023-09-23T19:06:42.575036+0000 osd.6 (osd.6) 329 : cluster 3 failed to encode map e347 with expected crc 2023-09-23T19:06:43.730 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:43 smithi173 bash[59422]: cluster 2023-09-23T19:06:42.578171+0000 osd.6 (osd.6) 330 : cluster 3 failed to encode map e347 with expected crc 2023-09-23T19:06:43.730 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:43 smithi173 bash[59422]: cluster 2023-09-23T19:06:42.595656+0000 mgr.y (mgr.26054) 1469 : cluster 0 pgmap v1617: 105 pgs: 19 creating+peering, 13 unknown, 24 active+clean+remapped, 4 active+undersized+remapped, 27 active+clean, 7 active+undersized+degraded, 11 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:43.747 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:43.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:43.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:43.851 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:43.873 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:43.876 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:43.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:43.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:43.977 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:44.001 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:44.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:44.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:44.105 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:44.139 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:44.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:44.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:44.242 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:44.273 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:44.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:44.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:44.376 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:44.408 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:44.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:44.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:44.511 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:44.541 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:44.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:44.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:44.644 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:44.658 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:44 smithi173 bash[59422]: cluster 2023-09-23T19:06:43.277878+0000 osd.4 (osd.4) 78 : cluster 3 failed to encode map e347 with expected crc 2023-09-23T19:06:44.658 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:44 smithi173 bash[59422]: cluster 2023-09-23T19:06:43.591618+0000 mon.a (mon.0) 1992 : cluster 0 osdmap e348: 8 total, 6 up, 7 in 2023-09-23T19:06:44.658 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:44 smithi173 bash[59422]: cluster 2023-09-23T19:06:43.591902+0000 osd.6 (osd.6) 331 : cluster 3 failed to encode map e348 with expected crc 2023-09-23T19:06:44.681 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:44.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:44.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:44.785 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:44.818 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:44.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:44.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:44.921 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:44.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:44 smithi098 bash[76817]: cluster 2023-09-23T19:06:43.277878+0000 osd.4 (osd.4) 78 : cluster 3 failed to encode map e347 with expected crc 2023-09-23T19:06:44.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:44 smithi098 bash[76817]: cluster 2023-09-23T19:06:43.591618+0000 mon.a (mon.0) 1992 : cluster 0 osdmap e348: 8 total, 6 up, 7 in 2023-09-23T19:06:44.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:44 smithi098 bash[76817]: cluster 2023-09-23T19:06:43.591902+0000 osd.6 (osd.6) 331 : cluster 3 failed to encode map e348 with expected crc 2023-09-23T19:06:44.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:44 smithi098 bash[79516]: cluster 2023-09-23T19:06:43.277878+0000 osd.4 (osd.4) 78 : cluster 3 failed to encode map e347 with expected crc 2023-09-23T19:06:44.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:44 smithi098 bash[79516]: cluster 2023-09-23T19:06:43.591618+0000 mon.a (mon.0) 1992 : cluster 0 osdmap e348: 8 total, 6 up, 7 in 2023-09-23T19:06:44.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:44 smithi098 bash[79516]: cluster 2023-09-23T19:06:43.591902+0000 osd.6 (osd.6) 331 : cluster 3 failed to encode map e348 with expected crc 2023-09-23T19:06:44.958 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:44.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:44.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:45.062 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:45.088 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:45.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:45.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:45.191 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:45.217 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:45.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:45.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:45.320 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:45.351 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:45.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:45.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:45.455 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:45.489 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:45.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:45.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:45.593 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:45.623 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:45.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:45.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:45.726 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:45.739 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:45 smithi173 bash[59422]: cluster 2023-09-23T19:06:44.247541+0000 osd.4 (osd.4) 79 : cluster 3 failed to encode map e348 with expected crc 2023-09-23T19:06:45.739 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:45 smithi173 bash[59422]: cluster 2023-09-23T19:06:44.596607+0000 mgr.y (mgr.26054) 1470 : cluster 0 pgmap v1619: 105 pgs: 19 creating+peering, 8 unknown, 24 active+clean+remapped, 4 active+undersized+remapped, 31 active+clean, 7 active+undersized+degraded, 12 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:45.760 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:45.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:45.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:45.865 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:45.876 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:45 smithi098 bash[76817]: cluster 2023-09-23T19:06:44.247541+0000 osd.4 (osd.4) 79 : cluster 3 failed to encode map e348 with expected crc 2023-09-23T19:06:45.876 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:45 smithi098 bash[76817]: cluster 2023-09-23T19:06:44.596607+0000 mgr.y (mgr.26054) 1470 : cluster 0 pgmap v1619: 105 pgs: 19 creating+peering, 8 unknown, 24 active+clean+remapped, 4 active+undersized+remapped, 31 active+clean, 7 active+undersized+degraded, 12 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:45.876 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:45 smithi098 bash[79516]: cluster 2023-09-23T19:06:44.247541+0000 osd.4 (osd.4) 79 : cluster 3 failed to encode map e348 with expected crc 2023-09-23T19:06:45.876 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:45 smithi098 bash[79516]: cluster 2023-09-23T19:06:44.596607+0000 mgr.y (mgr.26054) 1470 : cluster 0 pgmap v1619: 105 pgs: 19 creating+peering, 8 unknown, 24 active+clean+remapped, 4 active+undersized+remapped, 31 active+clean, 7 active+undersized+degraded, 12 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:45.892 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:45.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:45.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:45.995 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:46.029 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:46.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:46.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:46.134 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:46.145 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:06:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:06:46] "GET /metrics HTTP/1.1" 200 34210 "" "Prometheus/2.33.4" 2023-09-23T19:06:46.162 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:46.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:46.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:46.266 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:46.296 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:46.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:46.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:46.399 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:46.430 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:46.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:46.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:46.534 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:46.563 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:46.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:46.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:46.667 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:46.697 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:46.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:46.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:46.800 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:46.836 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:46.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:46.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:46.939 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:46.967 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:46.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:46.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:47.070 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:47.104 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:47.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:47.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:47.209 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:47.236 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:47.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:47.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:47.338 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:47.367 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:47.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:47.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:47.471 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:47.502 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:47.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:47.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:47.605 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:47.631 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:47.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:47.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:47.734 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:47.747 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:47 smithi173 bash[59422]: cluster 2023-09-23T19:06:46.597795+0000 mgr.y (mgr.26054) 1471 : cluster 0 pgmap v1620: 105 pgs: 2 undersized+peered, 19 creating+peering, 24 active+clean+remapped, 4 active+undersized+remapped, 33 active+clean, 7 active+undersized+degraded, 16 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:47.766 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:47.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:47.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:47.869 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:47.880 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:47 smithi098 bash[79516]: cluster 2023-09-23T19:06:46.597795+0000 mgr.y (mgr.26054) 1471 : cluster 0 pgmap v1620: 105 pgs: 2 undersized+peered, 19 creating+peering, 24 active+clean+remapped, 4 active+undersized+remapped, 33 active+clean, 7 active+undersized+degraded, 16 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:47.880 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:47 smithi098 bash[76817]: cluster 2023-09-23T19:06:46.597795+0000 mgr.y (mgr.26054) 1471 : cluster 0 pgmap v1620: 105 pgs: 2 undersized+peered, 19 creating+peering, 24 active+clean+remapped, 4 active+undersized+remapped, 33 active+clean, 7 active+undersized+degraded, 16 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/84 objects degraded (10.714%); 17/84 objects misplaced (20.238%) 2023-09-23T19:06:47.896 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:47.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:47.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:48.000 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:48.026 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:48.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:48.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:48.129 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:48.162 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:48.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:48.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:48.265 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:48.289 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:48.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:48.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:48.393 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:48.421 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:48.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:48.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:48.524 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:48.554 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:48.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:48.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:48.658 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:48.693 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:48.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:48.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:48.796 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:48.824 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:48.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:48.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:48.928 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:48.940 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:48 smithi173 bash[59422]: cluster 2023-09-23T19:06:48.653926+0000 mon.a (mon.0) 1993 : cluster 3 Health check update: Degraded data redundancy: 9/87 objects degraded (10.345%), 7 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-09-23T19:06:48.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:48 smithi098 bash[76817]: cluster 2023-09-23T19:06:48.653926+0000 mon.a (mon.0) 1993 : cluster 3 Health check update: Degraded data redundancy: 9/87 objects degraded (10.345%), 7 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-09-23T19:06:48.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:48 smithi098 bash[79516]: cluster 2023-09-23T19:06:48.653926+0000 mon.a (mon.0) 1993 : cluster 3 Health check update: Degraded data redundancy: 9/87 objects degraded (10.345%), 7 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-09-23T19:06:48.964 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:48.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:48.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:49.069 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:49.096 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:49.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:49.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:49.200 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:49.227 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:49.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:49.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:49.331 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:49.360 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:49.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:49.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:49.464 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:49.496 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:49.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:49.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:49.600 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:49.622 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:49.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:49.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:49.726 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:49.745 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:49.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:49.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:49.790 INFO:teuthology.orchestra.run.smithi098.stdout:moved 3 / 315 (0.952381%) 2023-09-23T19:06:49.790 INFO:teuthology.orchestra.run.smithi098.stdout:avg 63 2023-09-23T19:06:49.790 INFO:teuthology.orchestra.run.smithi098.stdout:stddev 16.6012 -> 13.784 (expected baseline 7.0993) 2023-09-23T19:06:49.790 INFO:teuthology.orchestra.run.smithi098.stdout:min osd.2 with 42 -> 43 pgs (0.666667 -> 0.68254 * mean) 2023-09-23T19:06:49.790 INFO:teuthology.orchestra.run.smithi098.stdout:max osd.1 with 55 -> 54 pgs (0.873016 -> 0.857143 * mean) 2023-09-23T19:06:49.790 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T19:06:49.790 INFO:teuthology.orchestra.run.smithi098.stdout:oload 110 2023-09-23T19:06:49.791 INFO:teuthology.orchestra.run.smithi098.stdout:max_change 1 2023-09-23T19:06:49.791 INFO:teuthology.orchestra.run.smithi098.stdout:max_change_osds 4 2023-09-23T19:06:49.791 INFO:teuthology.orchestra.run.smithi098.stdout:average_utilization 0.0121 2023-09-23T19:06:49.791 INFO:teuthology.orchestra.run.smithi098.stdout:overload_utilization 0.0133 2023-09-23T19:06:49.791 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5 weight 0.9098 -> 0.5015 2023-09-23T19:06:49.791 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6 weight 0.8251 -> 1.0000 2023-09-23T19:06:49.791 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3 weight 0.8669 -> 1.0000 2023-09-23T19:06:49.793 INFO:teuthology.orchestra.run.smithi098.stderr:no change 2023-09-23T19:06:49.848 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:49.871 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:49.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:49.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:49.926 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:49 smithi098 bash[79516]: cluster 2023-09-23T19:06:48.598767+0000 mgr.y (mgr.26054) 1472 : cluster 0 pgmap v1621: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 1.1 MiB/s rd, 582 KiB/s wr, 1 op/s; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:06:49.927 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:06:49 smithi098 bash[73085]: debug 2023-09-23T19:06:49.780+0000 7f90a56ad700 -1 mgr.server reply reply (3) No such process no change 2023-09-23T19:06:49.927 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:49 smithi098 bash[76817]: cluster 2023-09-23T19:06:48.598767+0000 mgr.y (mgr.26054) 1472 : cluster 0 pgmap v1621: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 1.1 MiB/s rd, 582 KiB/s wr, 1 op/s; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:06:49.974 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:50.000 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:50.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:50.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:50.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:49 smithi173 bash[59422]: cluster 2023-09-23T19:06:48.598767+0000 mgr.y (mgr.26054) 1472 : cluster 0 pgmap v1621: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 1.1 MiB/s rd, 582 KiB/s wr, 1 op/s; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:06:50.104 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:50.129 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:50.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:50.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:50.232 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:50.251 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:50.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:50.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:50.277 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T19:06:50.297 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T19:06:50.318 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T19:06:50.341 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T19:06:50.355 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:50.366 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T19:06:50.374 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:50.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:50.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:50.390 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T19:06:50.390 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:06:50.391 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:06:50.391 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:06:50.391 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:06:50.391 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:06:50.391 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T19:06:50.391 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:06:50.391 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T19:06:50.477 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:50.503 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:50.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:50.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:50.606 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:50.630 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:50.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:50.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:50.733 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:50.744 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:50 smithi098 bash[79516]: audit 2023-09-23T19:06:49.784837+0000 mgr.y (mgr.26054) 1473 : audit 0 from='client.35967 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:06:50.745 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:50 smithi098 bash[79516]: audit 2023-09-23T19:06:50.250733+0000 mon.a (mon.0) 1994 : audit 0 from='client.? 172.21.15.98:0/2147182283' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:06:50.745 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:50 smithi098 bash[76817]: audit 2023-09-23T19:06:49.784837+0000 mgr.y (mgr.26054) 1473 : audit 0 from='client.35967 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:06:50.745 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:50 smithi098 bash[76817]: audit 2023-09-23T19:06:50.250733+0000 mon.a (mon.0) 1994 : audit 0 from='client.? 172.21.15.98:0/2147182283' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:06:50.754 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:50.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:50.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:50.857 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:50.879 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:50.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:50.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:50.982 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:51.015 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:51.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:51.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:51.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:50 smithi173 bash[59422]: audit 2023-09-23T19:06:49.784837+0000 mgr.y (mgr.26054) 1473 : audit 0 from='client.35967 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:06:51.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:50 smithi173 bash[59422]: audit 2023-09-23T19:06:50.250733+0000 mon.a (mon.0) 1994 : audit 0 from='client.? 172.21.15.98:0/2147182283' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:06:51.118 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:51.150 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:51.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:51.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:51.254 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:51.281 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:51.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:51.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:51.385 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:51.414 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:51.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:51.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:51.518 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:51.543 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:51.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:51.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:51.646 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:51.672 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:51.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:51.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:51.775 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:51.786 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:51 smithi098 bash[76817]: cluster 2023-09-23T19:06:50.599434+0000 mgr.y (mgr.26054) 1474 : cluster 0 pgmap v1622: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 1020 KiB/s rd, 510 KiB/s wr, 1 op/s; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:06:51.787 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:51 smithi098 bash[79516]: cluster 2023-09-23T19:06:50.599434+0000 mgr.y (mgr.26054) 1474 : cluster 0 pgmap v1622: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 1020 KiB/s rd, 510 KiB/s wr, 1 op/s; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:06:51.803 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:51.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:51.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:51.908 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:51.934 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:51.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:51.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:52.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:51 smithi173 bash[59422]: cluster 2023-09-23T19:06:50.599434+0000 mgr.y (mgr.26054) 1474 : cluster 0 pgmap v1622: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 1020 KiB/s rd, 510 KiB/s wr, 1 op/s; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:06:52.038 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:52.068 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:52.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:52.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:52.171 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:52.202 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:52.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:52.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:52.305 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:52.330 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:52.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:52.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:52.434 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:52.462 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:52.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:52.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:52.565 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:52.598 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:52.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:52.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:52.701 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:52.735 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:52.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:52.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:52.838 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:52.867 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:52.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:52.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:52.970 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:52.999 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:53.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:53.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:53.103 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:53.135 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:53.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:53.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:53.238 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:53.265 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:53.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:53.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:53.282 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T19:06:53.282 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 3.0 2023-09-23T19:06:53.368 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:53.391 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:53.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:53.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:53.430 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:06:53.430 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:06:53.495 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:53.528 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:53.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:53.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:53.632 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:53.659 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:53.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:53.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:53.762 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:53.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:53 smithi173 bash[59422]: cluster 2023-09-23T19:06:52.600437+0000 mgr.y (mgr.26054) 1475 : cluster 0 pgmap v1623: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 819 KiB/s rd, 409 KiB/s wr, 0 op/s; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:06:53.791 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:53.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:53.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:53.894 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:53.907 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:53 smithi098 bash[79516]: cluster 2023-09-23T19:06:52.600437+0000 mgr.y (mgr.26054) 1475 : cluster 0 pgmap v1623: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 819 KiB/s rd, 409 KiB/s wr, 0 op/s; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:06:53.907 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:53 smithi098 bash[76817]: cluster 2023-09-23T19:06:52.600437+0000 mgr.y (mgr.26054) 1475 : cluster 0 pgmap v1623: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 819 KiB/s rd, 409 KiB/s wr, 0 op/s; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:06:53.907 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:06:53.923 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:53.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:53.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:54.027 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:54.055 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:54.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:54.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:54.159 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:54.182 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:54.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:54.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:54.285 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:54.320 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:54.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:54.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:54.423 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:54.455 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:54.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:54.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:54.560 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:54.591 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:54.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:54.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:54.694 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:54.724 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:54.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:54.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:54.846 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:54.858 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:54 smithi098 bash[79516]: audit 2023-09-23T19:06:54.647464+0000 mon.a (mon.0) 1995 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:06:54.859 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:54 smithi098 bash[76817]: audit 2023-09-23T19:06:54.647464+0000 mon.a (mon.0) 1995 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:06:54.876 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:54.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:54.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:54.979 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:55.005 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:55.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:55.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:55.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:54 smithi173 bash[59422]: audit 2023-09-23T19:06:54.647464+0000 mon.a (mon.0) 1995 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:06:55.109 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:55.139 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:55.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:55.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:55.243 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:55.273 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:55.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:55.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:55.378 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:55.405 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:55.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:55.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:55.431 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:06:55.509 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:55.536 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:55.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:55.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:55.639 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:55.670 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:55.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:55.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:55.773 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:55.790 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:55 smithi098 bash[76817]: cluster 2023-09-23T19:06:54.601657+0000 mgr.y (mgr.26054) 1476 : cluster 0 pgmap v1624: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 744 KiB/s rd, 372 KiB/s wr, 0 op/s; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:06:55.800 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:55.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:55.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:55.903 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:55.924 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:55.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:55.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:56.027 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:56.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:55 smithi173 bash[59422]: cluster 2023-09-23T19:06:54.601657+0000 mgr.y (mgr.26054) 1476 : cluster 0 pgmap v1624: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 744 KiB/s rd, 372 KiB/s wr, 0 op/s; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:06:56.050 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:56.052 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:55 smithi098 bash[79516]: cluster 2023-09-23T19:06:54.601657+0000 mgr.y (mgr.26054) 1476 : cluster 0 pgmap v1624: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 744 KiB/s rd, 372 KiB/s wr, 0 op/s; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:06:56.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:56.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:56.154 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:56.183 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:56.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:56.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:56.286 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:56.318 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:56.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:56.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:56.422 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:56.434 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:06:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:06:56] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.33.4" 2023-09-23T19:06:56.451 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:56.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:56.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:56.554 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:56.586 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:56.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:56.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:56.690 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:56.714 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:56.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:56.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:56.818 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:56.843 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:56.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:56.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:56.947 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:56.974 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:56.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:56.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:57.078 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:57.105 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:57.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:57.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:57.209 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:57.234 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:57.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:57.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:57.338 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:57.366 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:57.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:57.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:57.469 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:57.498 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:57.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:57.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:57.602 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:57.633 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:57.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:57.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:57.736 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:57.756 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is unset 2023-09-23T19:06:57.763 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:57.765 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:57 smithi098 bash[79516]: cluster 2023-09-23T19:06:56.603045+0000 mgr.y (mgr.26054) 1477 : cluster 0 pgmap v1625: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 682 KiB/s rd, 341 KiB/s wr, 0 op/s; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:06:57.765 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:57 smithi098 bash[79516]: audit 2023-09-23T19:06:57.091992+0000 mon.a (mon.0) 1996 : audit 1 from='client.? 172.21.15.98:0/1528995368' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:06:57.765 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:57 smithi098 bash[76817]: cluster 2023-09-23T19:06:56.603045+0000 mgr.y (mgr.26054) 1477 : cluster 0 pgmap v1625: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 682 KiB/s rd, 341 KiB/s wr, 0 op/s; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:06:57.765 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:57 smithi098 bash[76817]: audit 2023-09-23T19:06:57.091992+0000 mon.a (mon.0) 1996 : audit 1 from='client.? 172.21.15.98:0/1528995368' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:06:57.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:57.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:57.867 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:57.899 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:57.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:57.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:58.002 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:58.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:57 smithi173 bash[59422]: cluster 2023-09-23T19:06:56.603045+0000 mgr.y (mgr.26054) 1477 : cluster 0 pgmap v1625: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 682 KiB/s rd, 341 KiB/s wr, 0 op/s; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:06:58.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:57 smithi173 bash[59422]: audit 2023-09-23T19:06:57.091992+0000 mon.a (mon.0) 1996 : audit 1 from='client.? 172.21.15.98:0/1528995368' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:06:58.028 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:58.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:58.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:58.132 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:58.158 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:58.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:58.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:58.262 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:58.293 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:58.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:58.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:58.397 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:58.425 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:58.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:58.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:58.529 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:58.556 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:58.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:58.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:58.659 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:58.689 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:58.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:58.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:58.792 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:58.818 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:58.820 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:58 smithi098 bash[76817]: cluster 2023-09-23T19:06:57.729769+0000 mon.a (mon.0) 1997 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:06:58.820 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:58 smithi098 bash[76817]: cluster 2023-09-23T19:06:57.744008+0000 osd.4 (osd.4) 80 : cluster 3 failed to encode map e349 with expected crc 2023-09-23T19:06:58.820 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:58 smithi098 bash[76817]: audit 2023-09-23T19:06:57.745119+0000 mon.a (mon.0) 1998 : audit 1 from='client.? 172.21.15.98:0/1528995368' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:06:58.820 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:58 smithi098 bash[76817]: cluster 2023-09-23T19:06:57.745215+0000 mon.a (mon.0) 1999 : cluster 0 osdmap e349: 8 total, 6 up, 7 in 2023-09-23T19:06:58.821 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:58 smithi098 bash[79516]: cluster 2023-09-23T19:06:57.729769+0000 mon.a (mon.0) 1997 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:06:58.821 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:58 smithi098 bash[79516]: cluster 2023-09-23T19:06:57.744008+0000 osd.4 (osd.4) 80 : cluster 3 failed to encode map e349 with expected crc 2023-09-23T19:06:58.821 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:58 smithi098 bash[79516]: audit 2023-09-23T19:06:57.745119+0000 mon.a (mon.0) 1998 : audit 1 from='client.? 172.21.15.98:0/1528995368' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:06:58.821 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:58 smithi098 bash[79516]: cluster 2023-09-23T19:06:57.745215+0000 mon.a (mon.0) 1999 : cluster 0 osdmap e349: 8 total, 6 up, 7 in 2023-09-23T19:06:58.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:58.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:58.922 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:58.961 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:58.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:58.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:59.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:58 smithi173 bash[59422]: cluster 2023-09-23T19:06:57.729769+0000 mon.a (mon.0) 1997 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:06:59.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:58 smithi173 bash[59422]: cluster 2023-09-23T19:06:57.744008+0000 osd.4 (osd.4) 80 : cluster 3 failed to encode map e349 with expected crc 2023-09-23T19:06:59.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:58 smithi173 bash[59422]: audit 2023-09-23T19:06:57.745119+0000 mon.a (mon.0) 1998 : audit 1 from='client.? 172.21.15.98:0/1528995368' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:06:59.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:58 smithi173 bash[59422]: cluster 2023-09-23T19:06:57.745215+0000 mon.a (mon.0) 1999 : cluster 0 osdmap e349: 8 total, 6 up, 7 in 2023-09-23T19:06:59.065 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:59.090 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:06:59.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:59.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:06:59.193 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:59.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:06:59.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:59.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:06:59.330 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:59.361 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:06:59.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:59.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:06:59.464 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:59.490 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:59.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:59.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:59.594 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:59.624 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:06:59.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:59.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:06:59.728 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:59.764 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:06:59.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:59.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:06:59.867 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:59.878 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:59 smithi098 bash[79516]: cluster 2023-09-23T19:06:58.083921+0000 osd.6 (osd.6) 332 : cluster 3 failed to encode map e349 with expected crc 2023-09-23T19:06:59.879 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:06:59 smithi098 bash[79516]: cluster 2023-09-23T19:06:58.603757+0000 mgr.y (mgr.26054) 1478 : cluster 0 pgmap v1627: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:06:59.879 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:59 smithi098 bash[76817]: cluster 2023-09-23T19:06:58.083921+0000 osd.6 (osd.6) 332 : cluster 3 failed to encode map e349 with expected crc 2023-09-23T19:06:59.879 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:06:59 smithi098 bash[76817]: cluster 2023-09-23T19:06:58.603757+0000 mgr.y (mgr.26054) 1478 : cluster 0 pgmap v1627: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:06:59.896 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:06:59.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:06:59.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:06:59.999 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:00.012 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:59 smithi173 bash[59422]: cluster 2023-09-23T19:06:58.083921+0000 osd.6 (osd.6) 332 : cluster 3 failed to encode map e349 with expected crc 2023-09-23T19:07:00.012 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:06:59 smithi173 bash[59422]: cluster 2023-09-23T19:06:58.603757+0000 mgr.y (mgr.26054) 1478 : cluster 0 pgmap v1627: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:00.032 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:00.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:00.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:00.135 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:00.166 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:00.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:00.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:00.270 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:00.293 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:00.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:00.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:00.396 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:00.423 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:00.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:00.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:00.526 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:00.554 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:00.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:00.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:00.585 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:07:00.657 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:00.679 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:00.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:00.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:00.782 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:00.813 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:00.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:00.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:00.916 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:00.948 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:00.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:00.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:01.052 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:01.080 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:01.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:01.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:01.184 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:01.217 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:01.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:01.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:01.321 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:01.349 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:01.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:01.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:01.452 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:01.483 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:01.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:01.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:01.586 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:01.598 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:01 smithi098 bash[76817]: cluster 2023-09-23T19:07:00.604656+0000 mgr.y (mgr.26054) 1479 : cluster 0 pgmap v1628: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:01.598 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:01 smithi098 bash[79516]: cluster 2023-09-23T19:07:00.604656+0000 mgr.y (mgr.26054) 1479 : cluster 0 pgmap v1628: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:01.615 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:01.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:01.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:01.719 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:01.748 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:01.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:01.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:01.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:01 smithi173 bash[59422]: cluster 2023-09-23T19:07:00.604656+0000 mgr.y (mgr.26054) 1479 : cluster 0 pgmap v1628: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:01.850 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:01.887 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:01.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:01.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:01.990 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:02.022 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:02.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:02.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:02.125 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:02.158 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:02.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:02.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:02.262 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:02.287 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:02.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:02.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:02.353 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T19:07:02.354 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [], 2023-09-23T19:07:02.354 INFO:teuthology.orchestra.run.smithi098.stdout: "num_ops": 0 2023-09-23T19:07:02.354 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T19:07:02.390 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:02.421 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:02.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:02.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:02.469 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:07:02.525 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:02.549 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:02.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:02.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:02.652 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:02.674 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:02.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:02.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:02.777 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:02.814 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:02.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:02.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:02.917 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:02.944 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:02.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:02.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:03.047 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:03.076 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:03.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:03.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:03.179 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:03.208 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:03.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:03.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:03.312 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:03.340 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:03.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:03.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:03.443 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:03.467 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:03.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:03.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:03.571 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:03.594 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:03.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:03.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:03.698 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:03.709 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:03 smithi098 bash[79516]: cluster 2023-09-23T19:07:02.605972+0000 mgr.y (mgr.26054) 1480 : cluster 0 pgmap v1629: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:03.709 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:03 smithi098 bash[76817]: cluster 2023-09-23T19:07:02.605972+0000 mgr.y (mgr.26054) 1480 : cluster 0 pgmap v1629: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:03.726 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:03.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:03.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:03.830 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:03.862 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:03.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:03.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:03.966 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:03.993 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:03.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:03.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:04.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:03 smithi173 bash[59422]: cluster 2023-09-23T19:07:02.605972+0000 mgr.y (mgr.26054) 1480 : cluster 0 pgmap v1629: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:04.097 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:04.125 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:04.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:04.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:04.229 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:04.262 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:04.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:04.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:04.336 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T19:07:04.366 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:04.408 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:04.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:04.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:04.512 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:04.547 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:04.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:04.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:04.651 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:04.678 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:04.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:04.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:04.781 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:04.806 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:04.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:04.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:04.909 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:04.939 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:04.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:04.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:05.043 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:05.068 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:05.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:05.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:05.172 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:05.201 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:05.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:05.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:05.305 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:05.335 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:05.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:05.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:05.439 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:05.468 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:05.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:05.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:05.571 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:05.600 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:05.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:05.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:05.702 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:05.715 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:05 smithi098 bash[79516]: cluster 2023-09-23T19:07:04.606998+0000 mgr.y (mgr.26054) 1481 : cluster 0 pgmap v1630: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:05.715 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:05 smithi098 bash[76817]: cluster 2023-09-23T19:07:04.606998+0000 mgr.y (mgr.26054) 1481 : cluster 0 pgmap v1630: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:05.737 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:05.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:05.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:05.785 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:07:05.841 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:05.875 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:05.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:05.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:05.978 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:06.005 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:06.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:06.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:06.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:05 smithi173 bash[59422]: cluster 2023-09-23T19:07:04.606998+0000 mgr.y (mgr.26054) 1481 : cluster 0 pgmap v1630: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:06.108 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:06.134 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:06.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:06.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:06.238 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:06.266 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:06.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:06.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:06.370 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:06.379 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:07:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:07:06] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.33.4" 2023-09-23T19:07:06.402 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:06.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:06.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:06.505 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:06.529 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:06.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:06.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:06.633 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:06.664 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:06.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:06.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:06.768 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:06.797 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:06.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:06.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:06.901 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:06.932 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:06.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:06.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:07.036 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:07.057 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:07.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:07.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:07.160 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:07.192 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:07.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:07.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:07.296 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:07.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:07.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:07.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:07.431 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:07.462 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:07.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:07.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:07.566 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:07.595 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:07.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:07.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:07.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:07 smithi098 bash[79516]: cluster 2023-09-23T19:07:06.608197+0000 mgr.y (mgr.26054) 1482 : cluster 0 pgmap v1631: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:07.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:07 smithi098 bash[76817]: cluster 2023-09-23T19:07:06.608197+0000 mgr.y (mgr.26054) 1482 : cluster 0 pgmap v1631: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:07.699 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:07.730 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:07.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:07.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:07.833 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:07.860 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:07.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:07.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:07.964 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:07.994 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:07.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:07.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:08.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:07 smithi173 bash[59422]: cluster 2023-09-23T19:07:06.608197+0000 mgr.y (mgr.26054) 1482 : cluster 0 pgmap v1631: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:08.043 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:07:08.099 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:08.120 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:08.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:08.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:08.224 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:08.244 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:08.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:08.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:08.347 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:08.369 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:08.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:08.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:08.449 INFO:teuthology.orchestra.run.smithi098.stdout:moved 6 / 315 (1.90476%) 2023-09-23T19:07:08.449 INFO:teuthology.orchestra.run.smithi098.stdout:avg 63 2023-09-23T19:07:08.449 INFO:teuthology.orchestra.run.smithi098.stdout:stddev 16.6012 -> 16.0873 (expected baseline 7.0993) 2023-09-23T19:07:08.449 INFO:teuthology.orchestra.run.smithi098.stdout:min osd.2 with 42 -> 41 pgs (0.666667 -> 0.650794 * mean) 2023-09-23T19:07:08.449 INFO:teuthology.orchestra.run.smithi098.stdout:max osd.1 with 55 -> 52 pgs (0.873016 -> 0.825397 * mean) 2023-09-23T19:07:08.449 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T19:07:08.449 INFO:teuthology.orchestra.run.smithi098.stdout:oload 1000 2023-09-23T19:07:08.449 INFO:teuthology.orchestra.run.smithi098.stdout:max_change 3 2023-09-23T19:07:08.449 INFO:teuthology.orchestra.run.smithi098.stdout:max_change_osds 4 2023-09-23T19:07:08.449 INFO:teuthology.orchestra.run.smithi098.stdout:average_utilization 0.0121 2023-09-23T19:07:08.449 INFO:teuthology.orchestra.run.smithi098.stdout:overload_utilization 0.1212 2023-09-23T19:07:08.450 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6 weight 0.8251 -> 1.0000 2023-09-23T19:07:08.450 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3 weight 0.8669 -> 1.0000 2023-09-23T19:07:08.452 INFO:teuthology.orchestra.run.smithi098.stderr:no change 2023-09-23T19:07:08.472 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:08.492 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:08.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:08.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:08.595 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:08.622 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:08.623 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:07:08 smithi098 bash[73085]: debug 2023-09-23T19:07:08.440+0000 7f90a56ad700 -1 mgr.server reply reply (2) No such file or directory no change 2023-09-23T19:07:08.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:08.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:08.724 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:08.755 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:08.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:08.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:08.859 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:08.895 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:08.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:08.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:08.998 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:09.029 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:09.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:09.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:09.057 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:07:09.133 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:09.164 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:09.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:09.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:09.268 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:09.300 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:09.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:09.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:09.404 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:09.436 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:09.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:09.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:09.540 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:09.565 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:09.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:09.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:09.669 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:09.690 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:09.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:09.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:09.793 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:09.817 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:09.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:09.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:09.920 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:09.946 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:09.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:09.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:10.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:09 smithi173 bash[59422]: audit 2023-09-23T19:07:08.444602+0000 mgr.y (mgr.26054) 1483 : audit 0 from='client.35985 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:07:10.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:09 smithi173 bash[59422]: cluster 2023-09-23T19:07:08.609699+0000 mgr.y (mgr.26054) 1484 : cluster 0 pgmap v1632: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:10.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:09 smithi173 bash[59422]: audit 2023-09-23T19:07:09.652278+0000 mon.a (mon.0) 2000 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:10.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:09 smithi173 bash[59422]: audit 2023-09-23T19:07:09.653361+0000 mon.a (mon.0) 2001 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:07:10.049 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:10.081 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:10.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:10.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:10.184 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:10.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:09 smithi098 bash[79516]: audit 2023-09-23T19:07:08.444602+0000 mgr.y (mgr.26054) 1483 : audit 0 from='client.35985 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:07:10.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:09 smithi098 bash[79516]: cluster 2023-09-23T19:07:08.609699+0000 mgr.y (mgr.26054) 1484 : cluster 0 pgmap v1632: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:10.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:09 smithi098 bash[79516]: audit 2023-09-23T19:07:09.652278+0000 mon.a (mon.0) 2000 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:10.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:09 smithi098 bash[79516]: audit 2023-09-23T19:07:09.653361+0000 mon.a (mon.0) 2001 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:07:10.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:09 smithi098 bash[76817]: audit 2023-09-23T19:07:08.444602+0000 mgr.y (mgr.26054) 1483 : audit 0 from='client.35985 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:07:10.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:09 smithi098 bash[76817]: cluster 2023-09-23T19:07:08.609699+0000 mgr.y (mgr.26054) 1484 : cluster 0 pgmap v1632: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:10.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:09 smithi098 bash[76817]: audit 2023-09-23T19:07:09.652278+0000 mon.a (mon.0) 2000 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:10.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:09 smithi098 bash[76817]: audit 2023-09-23T19:07:09.653361+0000 mon.a (mon.0) 2001 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:07:10.218 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:10.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:10.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:10.322 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:10.345 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:10.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:10.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:10.448 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:10.479 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:10.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:10.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:10.582 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:10.610 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:10.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:10.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:10.713 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:10.740 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:10.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:10.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:10.843 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:10.868 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:10.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:10.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:10.972 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:11.000 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:11.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:11.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:11.103 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:11.134 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:11.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:11.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:11.238 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:11.259 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:11.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:11.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:11.362 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:11.395 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:11.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:11.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:11.499 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:11.521 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:11.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:11.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:11.624 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:11.636 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:11 smithi173 bash[59422]: cluster 2023-09-23T19:07:10.610687+0000 mgr.y (mgr.26054) 1485 : cluster 0 pgmap v1633: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:11.658 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:11.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:11.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:11.761 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:11.769 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:11 smithi098 bash[76817]: cluster 2023-09-23T19:07:10.610687+0000 mgr.y (mgr.26054) 1485 : cluster 0 pgmap v1633: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:11.769 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:11 smithi098 bash[79516]: cluster 2023-09-23T19:07:10.610687+0000 mgr.y (mgr.26054) 1485 : cluster 0 pgmap v1633: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:11.786 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:11.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:11.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:11.889 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:11.911 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:11.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:11.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:12.015 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:12.048 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:12.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:12.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:12.151 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:12.174 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:12.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:12.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:12.277 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:12.303 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:12.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:12.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:12.406 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:12.440 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:12.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:12.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:12.543 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:12.570 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:12.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:12.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:12.673 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:12.703 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:12.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:12.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:12.807 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:12.832 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:12.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:12.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:12.936 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:12.962 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:12.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:12.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:13.065 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:13.093 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:13.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:13.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:13.196 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:13.214 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T19:07:13.214 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-09-23T19:07:13.223 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:13.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:13.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:13.326 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:13.355 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:13.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:13.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:13.458 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:13.483 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:13.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:13.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:13.586 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:13.613 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:13.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:13.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:13.717 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:13.748 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:13.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:13.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:13.852 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:13.863 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:13 smithi098 bash[76817]: cluster 2023-09-23T19:07:12.611440+0000 mgr.y (mgr.26054) 1486 : cluster 0 pgmap v1634: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:13.864 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:13 smithi098 bash[79516]: cluster 2023-09-23T19:07:12.611440+0000 mgr.y (mgr.26054) 1486 : cluster 0 pgmap v1634: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:13.878 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:13.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:13.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:13.982 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:14.004 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:14.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:14.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:14.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:13 smithi173 bash[59422]: cluster 2023-09-23T19:07:12.611440+0000 mgr.y (mgr.26054) 1486 : cluster 0 pgmap v1634: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:14.107 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:14.131 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:14.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:14.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:14.234 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:14.256 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:14.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:14.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:14.358 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:14.380 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:14.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:14.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:14.483 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:14.513 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:14.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:14.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:14.617 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:14.640 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:14.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:14.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:14.743 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:14.768 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:14.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:14.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:14.871 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:14.903 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:14.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:14.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:15.006 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:15.031 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:15.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:15.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:15.133 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:15.162 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:15.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:15.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:15.265 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:15.290 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:15.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:15.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:15.394 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:15.425 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:15.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:15.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:15.527 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:15.557 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:15.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:15.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:15.661 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:15.693 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:15.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:15.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:15.796 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:15.810 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:15 smithi098 bash[76817]: cluster 2023-09-23T19:07:14.611991+0000 mgr.y (mgr.26054) 1487 : cluster 0 pgmap v1635: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:15.810 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:15 smithi098 bash[79516]: cluster 2023-09-23T19:07:14.611991+0000 mgr.y (mgr.26054) 1487 : cluster 0 pgmap v1635: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:15.829 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:15.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:15.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:15.932 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:15.956 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:15.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:15.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:16.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:15 smithi173 bash[59422]: cluster 2023-09-23T19:07:14.611991+0000 mgr.y (mgr.26054) 1487 : cluster 0 pgmap v1635: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:16.060 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:16.085 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:16.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:16.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:16.188 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:16.216 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:16.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:16.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:16.319 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:16.350 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:16.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:16.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:16.384 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:07:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:07:16] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.33.4" 2023-09-23T19:07:16.454 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:16.486 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:16.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:16.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:16.589 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:16.619 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:16.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:16.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:16.724 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:16.752 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:16.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:16.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:16.854 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:16.883 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:16.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:16.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:16.987 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:17.009 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:17.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:17.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:17.112 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:17.145 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:17.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:17.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:17.247 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:17.274 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:17.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:17.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:17.377 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:17.410 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:17.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:17.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:17.514 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:17.541 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:17.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:17.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:17.645 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:17.675 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:17.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:17.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:17.780 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:17.811 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:17.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:17.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:17.913 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:17.936 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:17.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:17.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:18.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:17 smithi173 bash[59422]: cluster 2023-09-23T19:07:16.612898+0000 mgr.y (mgr.26054) 1488 : cluster 0 pgmap v1636: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:18.038 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:18.051 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:17 smithi098 bash[76817]: cluster 2023-09-23T19:07:16.612898+0000 mgr.y (mgr.26054) 1488 : cluster 0 pgmap v1636: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:18.051 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:17 smithi098 bash[79516]: cluster 2023-09-23T19:07:16.612898+0000 mgr.y (mgr.26054) 1488 : cluster 0 pgmap v1636: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:18.067 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:18.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:18.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:18.171 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:18.196 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:18.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:18.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:18.299 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:18.327 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:18.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:18.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:18.430 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:18.456 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:18.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:18.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:18.560 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:18.589 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:18.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:18.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:18.693 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:18.718 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:18.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:18.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:18.821 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:18.853 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:18.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:18.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:18.956 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:18.983 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:18.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:18.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:19.086 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:19.121 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:19.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:19.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:19.205 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T19:07:19.224 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:19.225 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T19:07:19.246 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T19:07:19.248 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:19.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:19.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:19.269 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T19:07:19.292 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T19:07:19.314 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T19:07:19.315 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:07:19.315 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:07:19.315 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:07:19.315 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:07:19.315 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:07:19.315 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T19:07:19.315 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:07:19.315 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T19:07:19.352 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:19.381 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:19.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:19.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:19.485 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:19.517 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:19.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:19.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:19.620 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:19.646 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:19.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:19.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:19.749 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:19.758 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:19 smithi098 bash[79516]: cluster 2023-09-23T19:07:18.614417+0000 mgr.y (mgr.26054) 1489 : cluster 0 pgmap v1637: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:19.759 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:19 smithi098 bash[79516]: audit 2023-09-23T19:07:19.177930+0000 mon.a (mon.0) 2002 : audit 0 from='client.? 172.21.15.98:0/4119563410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:07:19.759 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:19 smithi098 bash[76817]: cluster 2023-09-23T19:07:18.614417+0000 mgr.y (mgr.26054) 1489 : cluster 0 pgmap v1637: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:19.759 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:19 smithi098 bash[76817]: audit 2023-09-23T19:07:19.177930+0000 mon.a (mon.0) 2002 : audit 0 from='client.? 172.21.15.98:0/4119563410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:07:19.776 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:19.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:19.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:19.880 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:19.913 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:19.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:19.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:20.016 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:20.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:19 smithi173 bash[59422]: cluster 2023-09-23T19:07:18.614417+0000 mgr.y (mgr.26054) 1489 : cluster 0 pgmap v1637: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:20.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:19 smithi173 bash[59422]: audit 2023-09-23T19:07:19.177930+0000 mon.a (mon.0) 2002 : audit 0 from='client.? 172.21.15.98:0/4119563410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:07:20.044 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:20.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:20.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:20.149 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:20.180 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:20.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:20.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:20.284 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:20.319 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:20.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:20.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:20.423 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:20.458 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:20.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:20.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:20.561 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:20.588 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:20.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:20.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:20.692 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:20.721 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:20.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:20.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:20.825 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:20.851 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:20.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:20.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:20.954 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:20.984 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:20.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:20.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:21.088 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:21.120 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:21.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:21.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:21.224 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:21.250 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:21.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:21.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:21.353 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:21.387 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:21.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:21.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:21.491 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:21.519 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:21.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:21.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:21.592 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:07:21.592 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:07:21.623 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:21.650 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:21.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:21.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:21.754 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:21.761 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:21 smithi098 bash[76817]: cluster 2023-09-23T19:07:20.615648+0000 mgr.y (mgr.26054) 1490 : cluster 0 pgmap v1638: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:21.762 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:21 smithi098 bash[79516]: cluster 2023-09-23T19:07:20.615648+0000 mgr.y (mgr.26054) 1490 : cluster 0 pgmap v1638: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:21.775 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:21.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:21.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:21.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:21 smithi173 bash[59422]: cluster 2023-09-23T19:07:20.615648+0000 mgr.y (mgr.26054) 1490 : cluster 0 pgmap v1638: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:21.849 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T19:07:21.849 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:07:21.878 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:21.910 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:21.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:21.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:22.014 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:22.042 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:22.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:22.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:22.060 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:07:22.145 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:22.177 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:22.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:22.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:22.281 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:22.304 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:22.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:22.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:22.407 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:22.429 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:22.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:22.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:22.533 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:22.556 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:22.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:22.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:22.608 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is set 2023-09-23T19:07:22.659 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:22.692 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:22.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:22.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:22.796 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:22.808 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:22 smithi098 bash[79516]: audit 2023-09-23T19:07:22.056140+0000 mon.a (mon.0) 2003 : audit 1 from='client.? 172.21.15.98:0/1874099872' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:07:22.808 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:22 smithi098 bash[76817]: audit 2023-09-23T19:07:22.056140+0000 mon.a (mon.0) 2003 : audit 1 from='client.? 172.21.15.98:0/1874099872' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:07:22.825 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:22.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:22.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:22.929 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:22.943 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:22 smithi173 bash[59422]: audit 2023-09-23T19:07:22.056140+0000 mon.a (mon.0) 2003 : audit 1 from='client.? 172.21.15.98:0/1874099872' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:07:22.962 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:22.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:22.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:23.066 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:23.094 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:23.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:23.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:23.198 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:23.219 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:23.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:23.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:23.323 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:23.354 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:23.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:23.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:23.458 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:23.480 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:23.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:23.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:23.583 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:23.593 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:07:23.607 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:23.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:23.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:23.710 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:23.717 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:23 smithi098 bash[76817]: cluster 2023-09-23T19:07:22.578084+0000 mon.a (mon.0) 2004 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:07:23.717 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:23 smithi098 bash[76817]: cluster 2023-09-23T19:07:22.592700+0000 osd.4 (osd.4) 81 : cluster 3 failed to encode map e350 with expected crc 2023-09-23T19:07:23.717 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:23 smithi098 bash[76817]: audit 2023-09-23T19:07:22.594480+0000 mon.a (mon.0) 2005 : audit 1 from='client.? 172.21.15.98:0/1874099872' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:07:23.717 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:23 smithi098 bash[76817]: cluster 2023-09-23T19:07:22.594562+0000 mon.a (mon.0) 2006 : cluster 0 osdmap e350: 8 total, 6 up, 7 in 2023-09-23T19:07:23.717 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:23 smithi098 bash[76817]: cluster 2023-09-23T19:07:22.616869+0000 mgr.y (mgr.26054) 1491 : cluster 0 pgmap v1640: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:23.717 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:23 smithi098 bash[79516]: cluster 2023-09-23T19:07:22.578084+0000 mon.a (mon.0) 2004 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:07:23.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:23 smithi098 bash[79516]: cluster 2023-09-23T19:07:22.592700+0000 osd.4 (osd.4) 81 : cluster 3 failed to encode map e350 with expected crc 2023-09-23T19:07:23.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:23 smithi098 bash[79516]: audit 2023-09-23T19:07:22.594480+0000 mon.a (mon.0) 2005 : audit 1 from='client.? 172.21.15.98:0/1874099872' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:07:23.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:23 smithi098 bash[79516]: cluster 2023-09-23T19:07:22.594562+0000 mon.a (mon.0) 2006 : cluster 0 osdmap e350: 8 total, 6 up, 7 in 2023-09-23T19:07:23.718 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:23 smithi098 bash[79516]: cluster 2023-09-23T19:07:22.616869+0000 mgr.y (mgr.26054) 1491 : cluster 0 pgmap v1640: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:23.732 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:23.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:23.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:23.834 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:23.859 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:23.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:23.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:23.962 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:23.975 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:23 smithi173 bash[59422]: cluster 2023-09-23T19:07:22.578084+0000 mon.a (mon.0) 2004 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:07:23.975 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:23 smithi173 bash[59422]: cluster 2023-09-23T19:07:22.592700+0000 osd.4 (osd.4) 81 : cluster 3 failed to encode map e350 with expected crc 2023-09-23T19:07:23.975 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:23 smithi173 bash[59422]: audit 2023-09-23T19:07:22.594480+0000 mon.a (mon.0) 2005 : audit 1 from='client.? 172.21.15.98:0/1874099872' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:07:23.975 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:23 smithi173 bash[59422]: cluster 2023-09-23T19:07:22.594562+0000 mon.a (mon.0) 2006 : cluster 0 osdmap e350: 8 total, 6 up, 7 in 2023-09-23T19:07:23.976 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:23 smithi173 bash[59422]: cluster 2023-09-23T19:07:22.616869+0000 mgr.y (mgr.26054) 1491 : cluster 0 pgmap v1640: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:23.999 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:24.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:24.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:24.104 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:24.132 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:24.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:24.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:24.235 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:24.265 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:24.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:24.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:24.316 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T19:07:24.370 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:24.394 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:24.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:24.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:24.497 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:24.518 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:24.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:24.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:24.621 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:24.656 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:24.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:24.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:24.759 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:24.787 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:24.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:24.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:24.891 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:24.903 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:24 smithi098 bash[79516]: cluster 2023-09-23T19:07:23.088022+0000 osd.6 (osd.6) 333 : cluster 3 failed to encode map e350 with expected crc 2023-09-23T19:07:24.903 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:24 smithi098 bash[76817]: cluster 2023-09-23T19:07:23.088022+0000 osd.6 (osd.6) 333 : cluster 3 failed to encode map e350 with expected crc 2023-09-23T19:07:24.921 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:24.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:24.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:25.025 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:25.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:24 smithi173 bash[59422]: cluster 2023-09-23T19:07:23.088022+0000 osd.6 (osd.6) 333 : cluster 3 failed to encode map e350 with expected crc 2023-09-23T19:07:25.056 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:25.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:25.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:25.160 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:25.189 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:25.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:25.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:25.293 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:25.324 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:25.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:25.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:25.426 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:25.457 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:25.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:25.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:25.584 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:25.611 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:25.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:25.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:25.716 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:25.729 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:25 smithi173 bash[59422]: cluster 2023-09-23T19:07:24.617932+0000 mgr.y (mgr.26054) 1492 : cluster 0 pgmap v1641: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:25.729 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:25 smithi173 bash[59422]: audit 2023-09-23T19:07:24.652570+0000 mon.a (mon.0) 2007 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:25.729 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:25 smithi173 bash[59422]: audit 2023-09-23T19:07:24.654430+0000 mon.a (mon.0) 2008 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:07:25.747 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:25.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:25.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:25.850 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:25.858 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:25 smithi098 bash[76817]: cluster 2023-09-23T19:07:24.617932+0000 mgr.y (mgr.26054) 1492 : cluster 0 pgmap v1641: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:25.858 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:25 smithi098 bash[76817]: audit 2023-09-23T19:07:24.652570+0000 mon.a (mon.0) 2007 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:25.858 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:25 smithi098 bash[76817]: audit 2023-09-23T19:07:24.654430+0000 mon.a (mon.0) 2008 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:07:25.858 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:25 smithi098 bash[79516]: cluster 2023-09-23T19:07:24.617932+0000 mgr.y (mgr.26054) 1492 : cluster 0 pgmap v1641: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:25.858 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:25 smithi098 bash[79516]: audit 2023-09-23T19:07:24.652570+0000 mon.a (mon.0) 2007 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:25.859 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:25 smithi098 bash[79516]: audit 2023-09-23T19:07:24.654430+0000 mon.a (mon.0) 2008 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:07:25.877 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:25.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:25.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:25.981 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:26.013 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:26.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:26.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:26.117 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:26.130 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:07:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:07:26] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.33.4" 2023-09-23T19:07:26.149 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:26.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:26.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:26.252 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:26.280 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:26.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:26.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:26.382 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:26.413 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:26.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:26.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:26.516 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:26.548 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:26.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:26.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:26.652 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:26.684 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:26.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:26.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:26.787 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:26.809 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:26.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:26.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:26.913 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:26.938 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:26.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:26.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:27.041 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:27.070 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:27.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:27.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:27.175 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:27.195 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:07:27.209 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:27.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:27.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:27.313 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:27.336 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:27.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:27.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:27.439 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:27.464 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:27.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:27.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:27.568 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:27.597 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:27.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:27.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:27.701 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:27.732 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:27.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:27.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:27.836 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:27.869 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:27.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:27.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:27.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:27 smithi098 bash[76817]: cluster 2023-09-23T19:07:26.619201+0000 mgr.y (mgr.26054) 1493 : cluster 0 pgmap v1642: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:27.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:27 smithi098 bash[79516]: cluster 2023-09-23T19:07:26.619201+0000 mgr.y (mgr.26054) 1493 : cluster 0 pgmap v1642: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:27.973 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:28.004 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:28.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:28.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:28.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:27 smithi173 bash[59422]: cluster 2023-09-23T19:07:26.619201+0000 mgr.y (mgr.26054) 1493 : cluster 0 pgmap v1642: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:28.108 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:28.138 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:28.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:28.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:28.242 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:28.268 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:28.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:28.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:28.372 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:28.399 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:28.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:28.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:28.503 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:28.539 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:28.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:28.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:28.643 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:28.670 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:28.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:28.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:28.773 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:28.804 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:28.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:28.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:28.908 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:28.939 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:28.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:28.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:29.043 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:29.074 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:29.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:29.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:29.178 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:29.213 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:29.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:29.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:29.317 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:29.348 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:29.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:29.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:29.452 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:29.487 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:29.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:29.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:29.591 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:29.623 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:29.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:29.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:29.678 INFO:teuthology.orchestra.run.smithi098.stdout:moved 6 / 315 (1.90476%) 2023-09-23T19:07:29.678 INFO:teuthology.orchestra.run.smithi098.stdout:avg 63 2023-09-23T19:07:29.678 INFO:teuthology.orchestra.run.smithi098.stdout:stddev 16.6012 -> 16.0873 (expected baseline 7.0993) 2023-09-23T19:07:29.678 INFO:teuthology.orchestra.run.smithi098.stdout:min osd.2 with 42 -> 41 pgs (0.666667 -> 0.650794 * mean) 2023-09-23T19:07:29.678 INFO:teuthology.orchestra.run.smithi098.stdout:max osd.1 with 55 -> 52 pgs (0.873016 -> 0.825397 * mean) 2023-09-23T19:07:29.678 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T19:07:29.678 INFO:teuthology.orchestra.run.smithi098.stdout:oload 1000 2023-09-23T19:07:29.678 INFO:teuthology.orchestra.run.smithi098.stdout:max_change 1 2023-09-23T19:07:29.679 INFO:teuthology.orchestra.run.smithi098.stdout:max_change_osds 4 2023-09-23T19:07:29.679 INFO:teuthology.orchestra.run.smithi098.stdout:average_utilization 0.0121 2023-09-23T19:07:29.679 INFO:teuthology.orchestra.run.smithi098.stdout:overload_utilization 0.1212 2023-09-23T19:07:29.679 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6 weight 0.8251 -> 1.0000 2023-09-23T19:07:29.679 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3 weight 0.8669 -> 1.0000 2023-09-23T19:07:29.727 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:29.740 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:29 smithi098 bash[76817]: cluster 2023-09-23T19:07:28.620235+0000 mgr.y (mgr.26054) 1494 : cluster 0 pgmap v1643: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:29.741 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:29 smithi098 bash[76817]: audit 2023-09-23T19:07:28.663452+0000 mgr.y (mgr.26054) 1495 : audit 0 from='client.36003 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:07:29.741 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:29 smithi098 bash[76817]: audit 2023-09-23T19:07:28.664609+0000 mon.a (mon.0) 2009 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'65536','6':'65536'}"}]: dispatch 2023-09-23T19:07:29.741 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:29 smithi098 bash[79516]: cluster 2023-09-23T19:07:28.620235+0000 mgr.y (mgr.26054) 1494 : cluster 0 pgmap v1643: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:29.741 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:29 smithi098 bash[79516]: audit 2023-09-23T19:07:28.663452+0000 mgr.y (mgr.26054) 1495 : audit 0 from='client.36003 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:07:29.741 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:29 smithi098 bash[79516]: audit 2023-09-23T19:07:28.664609+0000 mon.a (mon.0) 2009 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'65536','6':'65536'}"}]: dispatch 2023-09-23T19:07:29.756 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:29.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:29.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:29.860 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:29.874 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:29 smithi173 bash[59422]: cluster 2023-09-23T19:07:28.620235+0000 mgr.y (mgr.26054) 1494 : cluster 0 pgmap v1643: 105 pgs: 3 undersized+peered, 24 active+clean+remapped, 4 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 31 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:29.874 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:29 smithi173 bash[59422]: audit 2023-09-23T19:07:28.663452+0000 mgr.y (mgr.26054) 1495 : audit 0 from='client.36003 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:07:29.874 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:29 smithi173 bash[59422]: audit 2023-09-23T19:07:28.664609+0000 mon.a (mon.0) 2009 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'65536','6':'65536'}"}]: dispatch 2023-09-23T19:07:29.894 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:29.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:29.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:29.997 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:30.031 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:30.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:30.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:30.135 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:30.166 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:30.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:30.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:30.270 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:30.298 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:30.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:30.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:30.401 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:30.432 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:30.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:30.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:30.536 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:30.566 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:30.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:30.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:30.669 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:30.698 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:30.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:30.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:30.788 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T19:07:30.789 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-09-23T19:07:30.802 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:30.845 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:30 smithi098 bash[76817]: audit 2023-09-23T19:07:29.667096+0000 mon.a (mon.0) 2010 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'3':'65536','6':'65536'}"}]': finished 2023-09-23T19:07:30.845 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:30 smithi098 bash[76817]: cluster 2023-09-23T19:07:29.667244+0000 mon.a (mon.0) 2011 : cluster 0 osdmap e351: 8 total, 6 up, 7 in 2023-09-23T19:07:30.845 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:30 smithi098 bash[76817]: cluster 2023-09-23T19:07:29.671211+0000 osd.6 (osd.6) 334 : cluster 3 failed to encode map e351 with expected crc 2023-09-23T19:07:30.845 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:30 smithi098 bash[76817]: cluster 2023-09-23T19:07:29.675305+0000 osd.4 (osd.4) 82 : cluster 3 failed to encode map e351 with expected crc 2023-09-23T19:07:30.845 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:30 smithi098 bash[76817]: cluster 2023-09-23T19:07:29.675711+0000 osd.6 (osd.6) 335 : cluster 3 failed to encode map e351 with expected crc 2023-09-23T19:07:30.845 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:30 smithi098 bash[76817]: cluster 2023-09-23T19:07:29.679132+0000 osd.6 (osd.6) 336 : cluster 3 failed to encode map e351 with expected crc 2023-09-23T19:07:30.845 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:30 smithi098 bash[76817]: cluster 2023-09-23T19:07:29.680978+0000 osd.6 (osd.6) 337 : cluster 3 failed to encode map e351 with expected crc 2023-09-23T19:07:30.845 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:30 smithi098 bash[76817]: cluster 2023-09-23T19:07:30.579305+0000 mon.a (mon.0) 2012 : cluster 0 osdmap e352: 8 total, 6 up, 7 in 2023-09-23T19:07:30.846 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:30 smithi098 bash[79516]: audit 2023-09-23T19:07:29.667096+0000 mon.a (mon.0) 2010 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'3':'65536','6':'65536'}"}]': finished 2023-09-23T19:07:30.846 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:30 smithi098 bash[79516]: cluster 2023-09-23T19:07:29.667244+0000 mon.a (mon.0) 2011 : cluster 0 osdmap e351: 8 total, 6 up, 7 in 2023-09-23T19:07:30.846 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:30 smithi098 bash[79516]: cluster 2023-09-23T19:07:29.671211+0000 osd.6 (osd.6) 334 : cluster 3 failed to encode map e351 with expected crc 2023-09-23T19:07:30.846 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:30 smithi098 bash[79516]: cluster 2023-09-23T19:07:29.675305+0000 osd.4 (osd.4) 82 : cluster 3 failed to encode map e351 with expected crc 2023-09-23T19:07:30.846 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:30 smithi098 bash[79516]: cluster 2023-09-23T19:07:29.675711+0000 osd.6 (osd.6) 335 : cluster 3 failed to encode map e351 with expected crc 2023-09-23T19:07:30.846 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:30 smithi098 bash[79516]: cluster 2023-09-23T19:07:29.679132+0000 osd.6 (osd.6) 336 : cluster 3 failed to encode map e351 with expected crc 2023-09-23T19:07:30.846 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:30 smithi098 bash[79516]: cluster 2023-09-23T19:07:29.680978+0000 osd.6 (osd.6) 337 : cluster 3 failed to encode map e351 with expected crc 2023-09-23T19:07:30.846 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:30 smithi098 bash[79516]: cluster 2023-09-23T19:07:30.579305+0000 mon.a (mon.0) 2012 : cluster 0 osdmap e352: 8 total, 6 up, 7 in 2023-09-23T19:07:30.859 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:30.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:30.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:30.963 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:30.992 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:30.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:30.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:31.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:30 smithi173 bash[59422]: audit 2023-09-23T19:07:29.667096+0000 mon.a (mon.0) 2010 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'3':'65536','6':'65536'}"}]': finished 2023-09-23T19:07:31.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:30 smithi173 bash[59422]: cluster 2023-09-23T19:07:29.667244+0000 mon.a (mon.0) 2011 : cluster 0 osdmap e351: 8 total, 6 up, 7 in 2023-09-23T19:07:31.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:30 smithi173 bash[59422]: cluster 2023-09-23T19:07:29.671211+0000 osd.6 (osd.6) 334 : cluster 3 failed to encode map e351 with expected crc 2023-09-23T19:07:31.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:30 smithi173 bash[59422]: cluster 2023-09-23T19:07:29.675305+0000 osd.4 (osd.4) 82 : cluster 3 failed to encode map e351 with expected crc 2023-09-23T19:07:31.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:30 smithi173 bash[59422]: cluster 2023-09-23T19:07:29.675711+0000 osd.6 (osd.6) 335 : cluster 3 failed to encode map e351 with expected crc 2023-09-23T19:07:31.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:30 smithi173 bash[59422]: cluster 2023-09-23T19:07:29.679132+0000 osd.6 (osd.6) 336 : cluster 3 failed to encode map e351 with expected crc 2023-09-23T19:07:31.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:30 smithi173 bash[59422]: cluster 2023-09-23T19:07:29.680978+0000 osd.6 (osd.6) 337 : cluster 3 failed to encode map e351 with expected crc 2023-09-23T19:07:31.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:30 smithi173 bash[59422]: cluster 2023-09-23T19:07:30.579305+0000 mon.a (mon.0) 2012 : cluster 0 osdmap e352: 8 total, 6 up, 7 in 2023-09-23T19:07:31.096 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:31.128 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:31.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:31.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:31.207 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:07:31.209 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:07:31.232 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:31.257 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:31.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:31.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:31.360 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:31.382 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:31.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:31.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:31.485 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:31.512 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:31.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:31.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:31.616 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:31.644 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:31.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:31.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:31.748 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:31.775 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:31.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:31.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:31.878 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:31.891 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:31 smithi098 bash[79516]: cluster 2023-09-23T19:07:30.575147+0000 osd.4 (osd.4) 83 : cluster 3 failed to encode map e352 with expected crc 2023-09-23T19:07:31.891 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:31 smithi098 bash[79516]: cluster 2023-09-23T19:07:30.583979+0000 osd.6 (osd.6) 338 : cluster 3 failed to encode map e352 with expected crc 2023-09-23T19:07:31.891 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:31 smithi098 bash[79516]: cluster 2023-09-23T19:07:30.587147+0000 osd.6 (osd.6) 339 : cluster 3 failed to encode map e352 with expected crc 2023-09-23T19:07:31.891 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:31 smithi098 bash[79516]: cluster 2023-09-23T19:07:30.620809+0000 mgr.y (mgr.26054) 1496 : cluster 0 pgmap v1646: 105 pgs: 1 peering, 1 remapped+peering, 3 undersized+peered, 24 active+clean+remapped, 3 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 30 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:31.891 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:31 smithi098 bash[79516]: cluster 2023-09-23T19:07:31.577789+0000 mon.a (mon.0) 2013 : cluster 0 osdmap e353: 8 total, 6 up, 7 in 2023-09-23T19:07:31.891 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:31 smithi098 bash[79516]: cluster 2023-09-23T19:07:31.579567+0000 osd.4 (osd.4) 84 : cluster 3 failed to encode map e353 with expected crc 2023-09-23T19:07:31.891 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:31 smithi098 bash[79516]: cluster 2023-09-23T19:07:31.584119+0000 mon.a (mon.0) 2014 : cluster 3 Health check update: Degraded data redundancy: 9/87 objects degraded (10.345%), 7 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T19:07:31.892 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:31 smithi098 bash[76817]: cluster 2023-09-23T19:07:30.575147+0000 osd.4 (osd.4) 83 : cluster 3 failed to encode map e352 with expected crc 2023-09-23T19:07:31.892 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:31 smithi098 bash[76817]: cluster 2023-09-23T19:07:30.583979+0000 osd.6 (osd.6) 338 : cluster 3 failed to encode map e352 with expected crc 2023-09-23T19:07:31.892 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:31 smithi098 bash[76817]: cluster 2023-09-23T19:07:30.587147+0000 osd.6 (osd.6) 339 : cluster 3 failed to encode map e352 with expected crc 2023-09-23T19:07:31.892 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:31 smithi098 bash[76817]: cluster 2023-09-23T19:07:30.620809+0000 mgr.y (mgr.26054) 1496 : cluster 0 pgmap v1646: 105 pgs: 1 peering, 1 remapped+peering, 3 undersized+peered, 24 active+clean+remapped, 3 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 30 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:31.892 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:31 smithi098 bash[76817]: cluster 2023-09-23T19:07:31.577789+0000 mon.a (mon.0) 2013 : cluster 0 osdmap e353: 8 total, 6 up, 7 in 2023-09-23T19:07:31.892 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:31 smithi098 bash[76817]: cluster 2023-09-23T19:07:31.579567+0000 osd.4 (osd.4) 84 : cluster 3 failed to encode map e353 with expected crc 2023-09-23T19:07:31.892 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:31 smithi098 bash[76817]: cluster 2023-09-23T19:07:31.584119+0000 mon.a (mon.0) 2014 : cluster 3 Health check update: Degraded data redundancy: 9/87 objects degraded (10.345%), 7 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T19:07:31.905 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:31.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:31.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:32.009 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:32.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:31 smithi173 bash[59422]: cluster 2023-09-23T19:07:30.575147+0000 osd.4 (osd.4) 83 : cluster 3 failed to encode map e352 with expected crc 2023-09-23T19:07:32.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:31 smithi173 bash[59422]: cluster 2023-09-23T19:07:30.583979+0000 osd.6 (osd.6) 338 : cluster 3 failed to encode map e352 with expected crc 2023-09-23T19:07:32.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:31 smithi173 bash[59422]: cluster 2023-09-23T19:07:30.587147+0000 osd.6 (osd.6) 339 : cluster 3 failed to encode map e352 with expected crc 2023-09-23T19:07:32.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:31 smithi173 bash[59422]: cluster 2023-09-23T19:07:30.620809+0000 mgr.y (mgr.26054) 1496 : cluster 0 pgmap v1646: 105 pgs: 1 peering, 1 remapped+peering, 3 undersized+peered, 24 active+clean+remapped, 3 active+undersized+remapped, 36 active+clean, 7 active+undersized+degraded, 30 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9/87 objects degraded (10.345%); 17/87 objects misplaced (19.540%) 2023-09-23T19:07:32.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:31 smithi173 bash[59422]: cluster 2023-09-23T19:07:31.577789+0000 mon.a (mon.0) 2013 : cluster 0 osdmap e353: 8 total, 6 up, 7 in 2023-09-23T19:07:32.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:31 smithi173 bash[59422]: cluster 2023-09-23T19:07:31.579567+0000 osd.4 (osd.4) 84 : cluster 3 failed to encode map e353 with expected crc 2023-09-23T19:07:32.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:31 smithi173 bash[59422]: cluster 2023-09-23T19:07:31.584119+0000 mon.a (mon.0) 2014 : cluster 3 Health check update: Degraded data redundancy: 9/87 objects degraded (10.345%), 7 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T19:07:32.036 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:32.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:32.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:32.139 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:32.173 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:32.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:32.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:32.277 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:32.307 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:32.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:32.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:32.358 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.sparse_copyup (49141 ms) 2023-09-23T19:07:32.358 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-09-23T19:07:32.411 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:32.438 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:32.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:32.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:32.457 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.get_and_set_id (100 ms) 2023-09-23T19:07:32.457 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-09-23T19:07:32.530 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.add_remove_child (72 ms) 2023-09-23T19:07:32.530 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.directory_methods 2023-09-23T19:07:32.542 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:32.564 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:32.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:32.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:32.667 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:32.696 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:32.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:32.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:32.768 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.directory_methods (238 ms) 2023-09-23T19:07:32.768 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.create 2023-09-23T19:07:32.801 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:32.833 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:32.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:32.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:32.937 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:32.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:32 smithi098 bash[76817]: cluster 2023-09-23T19:07:31.623712+0000 osd.6 (osd.6) 340 : cluster 3 failed to encode map e353 with expected crc 2023-09-23T19:07:32.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:32 smithi098 bash[79516]: cluster 2023-09-23T19:07:31.623712+0000 osd.6 (osd.6) 340 : cluster 3 failed to encode map e353 with expected crc 2023-09-23T19:07:32.951 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:32 smithi173 bash[59422]: cluster 2023-09-23T19:07:31.623712+0000 osd.6 (osd.6) 340 : cluster 3 failed to encode map e353 with expected crc 2023-09-23T19:07:32.970 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:32.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:32.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:32.981 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.create (213 ms) 2023-09-23T19:07:32.981 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.get_features 2023-09-23T19:07:32.983 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.get_features (3 ms) 2023-09-23T19:07:32.984 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-09-23T19:07:32.985 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2023-09-23T19:07:32.986 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-09-23T19:07:32.987 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2023-09-23T19:07:32.987 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-09-23T19:07:32.990 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.get_access_timestamp (2 ms) 2023-09-23T19:07:32.990 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-09-23T19:07:32.992 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.get_modify_timestamp (2 ms) 2023-09-23T19:07:32.992 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-09-23T19:07:32.996 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.get_data_pool (4 ms) 2023-09-23T19:07:32.996 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.get_size 2023-09-23T19:07:33.002 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.get_size (6 ms) 2023-09-23T19:07:33.002 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.set_size 2023-09-23T19:07:33.007 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.set_size (6 ms) 2023-09-23T19:07:33.007 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.protection_status 2023-09-23T19:07:33.074 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:33.100 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:33.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:33.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:33.204 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:33.233 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:33.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:33.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:33.289 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:07:33.336 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:33.369 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:33.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:33.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:33.473 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:33.501 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:33.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:33.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:33.604 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:33.624 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:33.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:33.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:33.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:33.759 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:33.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:33.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:33.864 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:33.896 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:33.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:33.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:33.999 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:34.011 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:33 smithi173 bash[59422]: cluster 2023-09-23T19:07:32.621818+0000 mgr.y (mgr.26054) 1497 : cluster 0 pgmap v1648: 105 pgs: 1 peering, 1 remapped+peering, 2 undersized+peered, 25 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 8 active+undersized+degraded, 26 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 7.3 KiB/s rd, 2.8 KiB/s wr, 6 op/s; 10/96 objects degraded (10.417%); 19/96 objects misplaced (19.792%) 2023-09-23T19:07:34.029 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:34.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:34.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:34.133 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:34.164 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:34.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:34.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:34.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:33 smithi098 bash[76817]: cluster 2023-09-23T19:07:32.621818+0000 mgr.y (mgr.26054) 1497 : cluster 0 pgmap v1648: 105 pgs: 1 peering, 1 remapped+peering, 2 undersized+peered, 25 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 8 active+undersized+degraded, 26 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 7.3 KiB/s rd, 2.8 KiB/s wr, 6 op/s; 10/96 objects degraded (10.417%); 19/96 objects misplaced (19.792%) 2023-09-23T19:07:34.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:33 smithi098 bash[79516]: cluster 2023-09-23T19:07:32.621818+0000 mgr.y (mgr.26054) 1497 : cluster 0 pgmap v1648: 105 pgs: 1 peering, 1 remapped+peering, 2 undersized+peered, 25 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 8 active+undersized+degraded, 26 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 7.3 KiB/s rd, 2.8 KiB/s wr, 6 op/s; 10/96 objects degraded (10.417%); 19/96 objects misplaced (19.792%) 2023-09-23T19:07:34.268 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:34.297 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:34.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:34.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:34.401 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:34.425 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:34.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:34.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:34.528 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:34.560 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:34.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:34.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:34.664 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:34.689 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:34.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:34.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:34.793 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:34.823 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:34.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:34.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:34.926 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:34.956 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:34.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:34.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:35.060 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:35.093 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:35.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:35.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:35.197 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:35.219 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:35.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:35.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:35.316 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:07:35.322 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:35.342 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:35.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:35.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:35.445 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:35.466 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:35.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:35.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:35.569 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:35.602 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:35.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:35.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:35.705 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:35.737 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:35.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:35.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:35.841 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:35.853 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:35 smithi098 bash[79516]: cluster 2023-09-23T19:07:34.622674+0000 mgr.y (mgr.26054) 1498 : cluster 0 pgmap v1649: 105 pgs: 1 peering, 1 remapped+peering, 2 undersized+peered, 25 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 8 active+undersized+degraded, 26 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 7.3 KiB/s rd, 2.8 KiB/s wr, 6 op/s; 10/96 objects degraded (10.417%); 19/96 objects misplaced (19.792%) 2023-09-23T19:07:35.853 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:35 smithi098 bash[79516]: audit 2023-09-23T19:07:34.713915+0000 mon.a (mon.0) 2015 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:07:35.853 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:35 smithi098 bash[79516]: audit 2023-09-23T19:07:35.091905+0000 mon.a (mon.0) 2016 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:35.853 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:35 smithi098 bash[79516]: audit 2023-09-23T19:07:35.098957+0000 mon.a (mon.0) 2017 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:35.853 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:35 smithi098 bash[76817]: cluster 2023-09-23T19:07:34.622674+0000 mgr.y (mgr.26054) 1498 : cluster 0 pgmap v1649: 105 pgs: 1 peering, 1 remapped+peering, 2 undersized+peered, 25 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 8 active+undersized+degraded, 26 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 7.3 KiB/s rd, 2.8 KiB/s wr, 6 op/s; 10/96 objects degraded (10.417%); 19/96 objects misplaced (19.792%) 2023-09-23T19:07:35.853 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:35 smithi098 bash[76817]: audit 2023-09-23T19:07:34.713915+0000 mon.a (mon.0) 2015 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:07:35.853 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:35 smithi098 bash[76817]: audit 2023-09-23T19:07:35.091905+0000 mon.a (mon.0) 2016 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:35.853 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:35 smithi098 bash[76817]: audit 2023-09-23T19:07:35.098957+0000 mon.a (mon.0) 2017 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:35.866 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:35.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:35.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:35.969 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:35.994 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:35.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:35.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:36.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:35 smithi173 bash[59422]: cluster 2023-09-23T19:07:34.622674+0000 mgr.y (mgr.26054) 1498 : cluster 0 pgmap v1649: 105 pgs: 1 peering, 1 remapped+peering, 2 undersized+peered, 25 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 8 active+undersized+degraded, 26 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 7.3 KiB/s rd, 2.8 KiB/s wr, 6 op/s; 10/96 objects degraded (10.417%); 19/96 objects misplaced (19.792%) 2023-09-23T19:07:36.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:35 smithi173 bash[59422]: audit 2023-09-23T19:07:34.713915+0000 mon.a (mon.0) 2015 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:07:36.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:35 smithi173 bash[59422]: audit 2023-09-23T19:07:35.091905+0000 mon.a (mon.0) 2016 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:36.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:35 smithi173 bash[59422]: audit 2023-09-23T19:07:35.098957+0000 mon.a (mon.0) 2017 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:36.098 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:36.106 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:07:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:07:36] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.33.4" 2023-09-23T19:07:36.120 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:36.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:36.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:36.223 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:36.245 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:36.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:36.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:36.348 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:36.373 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:36.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:36.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:36.477 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:36.514 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:36.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:36.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:36.618 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:36.648 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:36.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:36.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:36.752 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:36.780 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:36.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:36.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:36.883 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:36.904 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:36.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:36.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:37.006 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:37.015 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:36 smithi098 bash[76817]: cluster 2023-09-23T19:07:36.698040+0000 mon.a (mon.0) 2018 : cluster 3 Health check update: Degraded data redundancy: 10/99 objects degraded (10.101%), 8 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T19:07:37.015 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:36 smithi098 bash[79516]: cluster 2023-09-23T19:07:36.698040+0000 mon.a (mon.0) 2018 : cluster 3 Health check update: Degraded data redundancy: 10/99 objects degraded (10.101%), 8 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T19:07:37.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:36 smithi173 bash[59422]: cluster 2023-09-23T19:07:36.698040+0000 mon.a (mon.0) 2018 : cluster 3 Health check update: Degraded data redundancy: 10/99 objects degraded (10.101%), 8 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T19:07:37.028 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:37.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:37.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:37.131 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:37.159 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:37.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:37.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:37.262 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:37.296 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:37.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:37.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:37.400 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:37.425 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:37.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:37.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:37.528 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:37.564 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:37.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:37.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:37.668 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:37.694 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:37.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:37.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:37.797 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:37.820 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:37.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:37.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:37.923 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:37.936 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:37 smithi173 bash[59422]: cluster 2023-09-23T19:07:36.623332+0000 mgr.y (mgr.26054) 1499 : cluster 0 pgmap v1650: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 8 active+undersized+degraded, 26 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 6.6 KiB/s rd, 2.6 KiB/s wr, 6 op/s; 10/99 objects degraded (10.101%); 19/99 objects misplaced (19.192%) 2023-09-23T19:07:37.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:37 smithi098 bash[79516]: cluster 2023-09-23T19:07:36.623332+0000 mgr.y (mgr.26054) 1499 : cluster 0 pgmap v1650: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 8 active+undersized+degraded, 26 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 6.6 KiB/s rd, 2.6 KiB/s wr, 6 op/s; 10/99 objects degraded (10.101%); 19/99 objects misplaced (19.192%) 2023-09-23T19:07:37.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:37 smithi098 bash[76817]: cluster 2023-09-23T19:07:36.623332+0000 mgr.y (mgr.26054) 1499 : cluster 0 pgmap v1650: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 8 active+undersized+degraded, 26 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 6.6 KiB/s rd, 2.6 KiB/s wr, 6 op/s; 10/99 objects degraded (10.101%); 19/99 objects misplaced (19.192%) 2023-09-23T19:07:37.954 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:37.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:37.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:38.058 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:38.083 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:38.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:38.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:38.186 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:38.207 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:38.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:38.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:38.311 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:38.337 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:38.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:38.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:38.440 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:38.464 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:38.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:38.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:38.568 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:38.600 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:38.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:38.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:38.704 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:38.729 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:38.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:38.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:38.833 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:38.866 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:38.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:38.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:38.970 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:39.002 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:39.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:39.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:39.105 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:39.134 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:39.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:39.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:39.239 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:39.270 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:39.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:39.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:39.374 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:39.405 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:39.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:39.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:39.508 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:39.540 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:39.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:39.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:39.645 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:39.666 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:39.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:39.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:39.760 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:39 smithi098 bash[76817]: cluster 2023-09-23T19:07:38.624649+0000 mgr.y (mgr.26054) 1500 : cluster 0 pgmap v1651: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 12 active+undersized+degraded, 22 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 14 KiB/s rd, 4.7 KiB/s wr, 18 op/s; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:39.761 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:39 smithi098 bash[76817]: audit 2023-09-23T19:07:39.649002+0000 mon.a (mon.0) 2019 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:07:39.769 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:39.799 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:39.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:39.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:39.902 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:39.931 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:39.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:39.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:40.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:39 smithi173 bash[59422]: cluster 2023-09-23T19:07:38.624649+0000 mgr.y (mgr.26054) 1500 : cluster 0 pgmap v1651: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 12 active+undersized+degraded, 22 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 14 KiB/s rd, 4.7 KiB/s wr, 18 op/s; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:40.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:39 smithi173 bash[59422]: audit 2023-09-23T19:07:39.649002+0000 mon.a (mon.0) 2019 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:07:40.034 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:40.042 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:39 smithi098 bash[79516]: cluster 2023-09-23T19:07:38.624649+0000 mgr.y (mgr.26054) 1500 : cluster 0 pgmap v1651: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 12 active+undersized+degraded, 22 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 14 KiB/s rd, 4.7 KiB/s wr, 18 op/s; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:40.042 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:39 smithi098 bash[79516]: audit 2023-09-23T19:07:39.649002+0000 mon.a (mon.0) 2019 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:07:40.059 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:40.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:40.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:40.163 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:40.189 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:40.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:40.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:40.292 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:40.323 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:40.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:40.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:40.427 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:40.455 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:40.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:40.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:40.559 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:40.588 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:40.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:40.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:40.691 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:40.717 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:40.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:40.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:40.821 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:40.848 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:40.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:40.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:40.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:40.977 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:40.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:40.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:41.081 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:41.101 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:41.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:41.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:41.205 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:41.235 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:41.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:41.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:41.339 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:41.364 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:41.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:41.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:41.468 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:41.498 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:41.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:41.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:41.602 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:41.630 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:41.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:41.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:41.734 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:41.756 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:41.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:41.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:41.859 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:41.881 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:41.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:41.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:41.984 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:42.008 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:42.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:42.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:42.111 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:42.133 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:42.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:42.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:42.236 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:42.258 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:42.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:42.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:42.361 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:42.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:42 smithi173 bash[59422]: cluster 2023-09-23T19:07:40.625920+0000 mgr.y (mgr.26054) 1501 : cluster 0 pgmap v1652: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 12 active+undersized+degraded, 22 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 11 KiB/s rd, 3.8 KiB/s wr, 14 op/s; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:42.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:42 smithi173 bash[59422]: audit 2023-09-23T19:07:41.054672+0000 mon.a (mon.0) 2020 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:42.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:42 smithi173 bash[59422]: audit 2023-09-23T19:07:41.060635+0000 mon.a (mon.0) 2021 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:42.390 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:42.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:42.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:42.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:42 smithi098 bash[79516]: cluster 2023-09-23T19:07:40.625920+0000 mgr.y (mgr.26054) 1501 : cluster 0 pgmap v1652: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 12 active+undersized+degraded, 22 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 11 KiB/s rd, 3.8 KiB/s wr, 14 op/s; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:42.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:42 smithi098 bash[79516]: audit 2023-09-23T19:07:41.054672+0000 mon.a (mon.0) 2020 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:42.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:42 smithi098 bash[79516]: audit 2023-09-23T19:07:41.060635+0000 mon.a (mon.0) 2021 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:42.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:42 smithi098 bash[76817]: cluster 2023-09-23T19:07:40.625920+0000 mgr.y (mgr.26054) 1501 : cluster 0 pgmap v1652: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 12 active+undersized+degraded, 22 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 11 KiB/s rd, 3.8 KiB/s wr, 14 op/s; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:42.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:42 smithi098 bash[76817]: audit 2023-09-23T19:07:41.054672+0000 mon.a (mon.0) 2020 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:42.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:42 smithi098 bash[76817]: audit 2023-09-23T19:07:41.060635+0000 mon.a (mon.0) 2021 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:42.494 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:42.518 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:42.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:42.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:42.621 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:42.644 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:42.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:42.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:42.747 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:42.782 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:42.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:42.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:42.886 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:42.920 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:42.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:42.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:43.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:43.057 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:43.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:43.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:43.160 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:43.185 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:43.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:43.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:43.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:43 smithi173 bash[59422]: cluster 2023-09-23T19:07:43.052251+0000 mon.a (mon.0) 2022 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2023-09-23T19:07:43.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:43 smithi173 bash[59422]: cluster 2023-09-23T19:07:43.052337+0000 mon.a (mon.0) 2023 : cluster 3 Health check update: Degraded data redundancy: 15/126 objects degraded (11.905%), 12 pgs degraded, 39 pgs undersized (PG_DEGRADED) 2023-09-23T19:07:43.289 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:43.317 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:43.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:43.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:43.420 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:43.432 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:43 smithi098 bash[79516]: cluster 2023-09-23T19:07:43.052251+0000 mon.a (mon.0) 2022 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2023-09-23T19:07:43.432 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:43 smithi098 bash[79516]: cluster 2023-09-23T19:07:43.052337+0000 mon.a (mon.0) 2023 : cluster 3 Health check update: Degraded data redundancy: 15/126 objects degraded (11.905%), 12 pgs degraded, 39 pgs undersized (PG_DEGRADED) 2023-09-23T19:07:43.432 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:43 smithi098 bash[76817]: cluster 2023-09-23T19:07:43.052251+0000 mon.a (mon.0) 2022 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2023-09-23T19:07:43.432 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:43 smithi098 bash[76817]: cluster 2023-09-23T19:07:43.052337+0000 mon.a (mon.0) 2023 : cluster 3 Health check update: Degraded data redundancy: 15/126 objects degraded (11.905%), 12 pgs degraded, 39 pgs undersized (PG_DEGRADED) 2023-09-23T19:07:43.446 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:43.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:43.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:43.549 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:43.575 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:43.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:43.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:43.678 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:43.703 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:43.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:43.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:43.807 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:43.832 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:43.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:43.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:43.935 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:43.962 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:43.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:43.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:44.065 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:44.089 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:44.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:44.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:44.191 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:44.214 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:44.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:44.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:44.317 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:44.345 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:44.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:44.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:44.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:44 smithi098 bash[79516]: cluster 2023-09-23T19:07:42.626937+0000 mgr.y (mgr.26054) 1502 : cluster 0 pgmap v1653: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 12 active+undersized+degraded, 22 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 10 KiB/s rd, 3.4 KiB/s wr, 13 op/s; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:44.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:44 smithi098 bash[76817]: cluster 2023-09-23T19:07:42.626937+0000 mgr.y (mgr.26054) 1502 : cluster 0 pgmap v1653: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 12 active+undersized+degraded, 22 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 10 KiB/s rd, 3.4 KiB/s wr, 13 op/s; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:44.448 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:44.463 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:44 smithi173 bash[59422]: cluster 2023-09-23T19:07:42.626937+0000 mgr.y (mgr.26054) 1502 : cluster 0 pgmap v1653: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 12 active+undersized+degraded, 22 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 10 KiB/s rd, 3.4 KiB/s wr, 13 op/s; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:44.482 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:44.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:44.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:44.586 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:44.618 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:44.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:44.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:44.723 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:44.751 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:44.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:44.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:44.855 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:44.887 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:44.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:44.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:44.992 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:45.022 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:45.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:45.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:45.125 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:45.158 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:45.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:45.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:45.261 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:45.291 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:45.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:45.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:45.394 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:45.415 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:45.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:45.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:45.519 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:45.544 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:45.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:45.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:45.647 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:45.670 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:45.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:45.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:45.774 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:45.806 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:45.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:45.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:45.910 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:45.935 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:45.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:45.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:46.038 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:46.059 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:46.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:46.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:46.163 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:46.194 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:46.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:46.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:46.265 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:07:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:07:46] "GET /metrics HTTP/1.1" 200 36588 "" "Prometheus/2.33.4" 2023-09-23T19:07:46.265 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:46 smithi098 bash[79516]: cluster 2023-09-23T19:07:44.627600+0000 mgr.y (mgr.26054) 1503 : cluster 0 pgmap v1654: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 12 active+undersized+degraded, 22 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 5.9 KiB/s rd, 1.7 KiB/s wr, 9 op/s; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:46.266 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:46 smithi098 bash[76817]: cluster 2023-09-23T19:07:44.627600+0000 mgr.y (mgr.26054) 1503 : cluster 0 pgmap v1654: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 12 active+undersized+degraded, 22 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 5.9 KiB/s rd, 1.7 KiB/s wr, 9 op/s; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:46.298 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:46.328 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:46.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:46.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:46.432 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:46.455 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:46.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:46.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:46.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:46 smithi173 bash[59422]: cluster 2023-09-23T19:07:44.627600+0000 mgr.y (mgr.26054) 1503 : cluster 0 pgmap v1654: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 12 active+undersized+degraded, 22 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 5.9 KiB/s rd, 1.7 KiB/s wr, 9 op/s; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:46.532 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T19:07:46.552 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T19:07:46.558 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:46.572 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T19:07:46.592 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:46.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:46.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:46.606 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T19:07:46.632 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T19:07:46.659 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T19:07:46.660 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:07:46.660 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:07:46.660 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:07:46.660 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:07:46.660 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:07:46.660 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T19:07:46.661 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:07:46.661 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T19:07:46.696 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:46.727 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:46.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:46.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:46.830 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:46.852 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:46.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:46.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:46.955 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:46.989 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:46.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:46.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:47.093 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:47.128 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:47.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:47.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:47.231 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:47.255 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:47.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:47.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:47 smithi173 bash[59422]: audit 2023-09-23T19:07:46.507555+0000 mon.a (mon.0) 2024 : audit 0 from='client.? 172.21.15.98:0/1734931803' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:07:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:47 smithi173 bash[59422]: audit 2023-09-23T19:07:46.757103+0000 mon.a (mon.0) 2025 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:47 smithi173 bash[59422]: audit 2023-09-23T19:07:46.763991+0000 mon.a (mon.0) 2026 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:47.330 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:47 smithi098 bash[76817]: audit 2023-09-23T19:07:46.507555+0000 mon.a (mon.0) 2024 : audit 0 from='client.? 172.21.15.98:0/1734931803' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:07:47.330 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:47 smithi098 bash[76817]: audit 2023-09-23T19:07:46.757103+0000 mon.a (mon.0) 2025 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:47.330 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:47 smithi098 bash[76817]: audit 2023-09-23T19:07:46.763991+0000 mon.a (mon.0) 2026 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:47.358 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:47.383 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:47.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:47.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:47.486 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:47.520 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:47.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:47.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:47.624 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:47.657 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:47.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:47.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:47.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:47 smithi098 bash[79516]: audit 2023-09-23T19:07:46.507555+0000 mon.a (mon.0) 2024 : audit 0 from='client.? 172.21.15.98:0/1734931803' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:07:47.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:47 smithi098 bash[79516]: audit 2023-09-23T19:07:46.757103+0000 mon.a (mon.0) 2025 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:47.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:47 smithi098 bash[79516]: audit 2023-09-23T19:07:46.763991+0000 mon.a (mon.0) 2026 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:47.761 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:47.792 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:47.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:47.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:47.896 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:47.920 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:47.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:47.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:48.023 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:48.027 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T19:07:48.028 INFO:teuthology.orchestra.run.smithi098.stdout: "size": 20, 2023-09-23T19:07:48.028 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 600, 2023-09-23T19:07:48.028 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [ 2023-09-23T19:07:48.028 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.028 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:122384 10.3 10:cea32f60:::benchmark_data_smithi098_230241_object21922:head [delete] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-09-23T19:07:48.028 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:03:10.266658+0000", 2023-09-23T19:07:48.028 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 277.75222776099997, 2023-09-23T19:07:48.028 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.192546304, 2023-09-23T19:07:48.028 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:07:48.028 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:07:48.029 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:07:48.029 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:07:48.029 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:07:48.029 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 122384 2023-09-23T19:07:48.029 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.029 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:07:48.029 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.029 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:07:48.029 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:10.266658+0000", 2023-09-23T19:07:48.029 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.029 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.029 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.029 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:07:48.030 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:10.266658+0000", 2023-09-23T19:07:48.030 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.030 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.030 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.030 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:07:48.030 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:10.266658+0000", 2023-09-23T19:07:48.030 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T19:07:48.030 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.030 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.030 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:07:48.030 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:10.266660+0000", 2023-09-23T19:07:48.030 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.03e-06 2023-09-23T19:07:48.031 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.031 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.031 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:07:48.031 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:10.266664+0000", 2023-09-23T19:07:48.031 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.8659999999999999e-06 2023-09-23T19:07:48.031 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.031 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.031 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:07:48.031 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:10.266667+0000", 2023-09-23T19:07:48.031 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.2090000000000001e-06 2023-09-23T19:07:48.031 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.031 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.032 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:07:48.032 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:10.275399+0000", 2023-09-23T19:07:48.032 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0087323399999999999 2023-09-23T19:07:48.032 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.032 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.032 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:07:48.032 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:10.275522+0000", 2023-09-23T19:07:48.032 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00012243399999999999 2023-09-23T19:07:48.032 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.032 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.032 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0", 2023-09-23T19:07:48.032 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:10.275634+0000", 2023-09-23T19:07:48.032 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00011199100000000001 2023-09-23T19:07:48.033 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.033 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.033 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:07:48.033 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:10.286688+0000", 2023-09-23T19:07:48.033 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.011054454 2023-09-23T19:07:48.033 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.033 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.033 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.033 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:10.459047+0000", 2023-09-23T19:07:48.033 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.17235920299999999 2023-09-23T19:07:48.033 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.033 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.033 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:07:48.034 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:10.459105+0000", 2023-09-23T19:07:48.034 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.7357e-05 2023-09-23T19:07:48.034 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.034 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.034 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:07:48.034 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:10.459204+0000", 2023-09-23T19:07:48.034 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.9722999999999994e-05 2023-09-23T19:07:48.034 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.034 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:07:48.034 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.034 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.035 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.035 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:123345 10.9 10:9180f393:::benchmark_data_smithi098_230241_object22883:head [delete] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-09-23T19:07:48.035 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:03:11.815257+0000", 2023-09-23T19:07:48.035 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 276.20362900399999, 2023-09-23T19:07:48.035 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.19659358599999999, 2023-09-23T19:07:48.035 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:07:48.035 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:07:48.035 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:07:48.035 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:07:48.035 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:07:48.035 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 123345 2023-09-23T19:07:48.035 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.035 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:07:48.036 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.036 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:07:48.036 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.815257+0000", 2023-09-23T19:07:48.036 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.036 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.036 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.036 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:07:48.036 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.815257+0000", 2023-09-23T19:07:48.036 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.036 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.036 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.036 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:07:48.037 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.815256+0000", 2023-09-23T19:07:48.037 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T19:07:48.037 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.037 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.037 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:07:48.037 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.815262+0000", 2023-09-23T19:07:48.037 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.178e-06 2023-09-23T19:07:48.037 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.037 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.037 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:07:48.037 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.815272+0000", 2023-09-23T19:07:48.037 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.6630000000000005e-06 2023-09-23T19:07:48.037 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.038 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.038 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:07:48.038 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.815280+0000", 2023-09-23T19:07:48.038 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.3750000000000003e-06 2023-09-23T19:07:48.038 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.038 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.038 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:07:48.038 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.815325+0000", 2023-09-23T19:07:48.038 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.4960999999999999e-05 2023-09-23T19:07:48.038 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.038 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.038 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:07:48.038 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.815439+0000", 2023-09-23T19:07:48.039 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000114497 2023-09-23T19:07:48.039 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.039 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.039 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 3", 2023-09-23T19:07:48.039 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.815594+0000", 2023-09-23T19:07:48.039 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000154783 2023-09-23T19:07:48.039 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.039 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.039 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:07:48.039 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.822373+0000", 2023-09-23T19:07:48.039 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0067789419999999996 2023-09-23T19:07:48.039 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.039 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.039 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.040 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.011736+0000", 2023-09-23T19:07:48.040 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.18936270399999999 2023-09-23T19:07:48.040 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.040 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.040 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:07:48.040 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.011778+0000", 2023-09-23T19:07:48.040 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.2168e-05 2023-09-23T19:07:48.040 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.040 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.040 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:07:48.040 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.011850+0000", 2023-09-23T19:07:48.040 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.2472999999999995e-05 2023-09-23T19:07:48.040 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.041 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:07:48.041 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.041 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.041 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.041 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:123350 10.8 10:1fcc1dc5:::benchmark_data_smithi098_230241_object22888:head [delete] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-09-23T19:07:48.041 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:03:11.817165+0000", 2023-09-23T19:07:48.041 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 276.201720572, 2023-09-23T19:07:48.041 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.19348372999999999, 2023-09-23T19:07:48.041 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:07:48.041 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:07:48.041 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:07:48.041 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:07:48.041 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:07:48.042 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 123350 2023-09-23T19:07:48.042 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.042 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:07:48.042 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.042 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:07:48.042 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.817165+0000", 2023-09-23T19:07:48.042 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.042 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.042 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.042 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:07:48.042 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.817165+0000", 2023-09-23T19:07:48.042 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.042 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.043 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.043 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:07:48.043 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.817164+0000", 2023-09-23T19:07:48.043 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T19:07:48.043 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.043 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.043 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:07:48.043 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.817170+0000", 2023-09-23T19:07:48.043 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.5220000000000004e-06 2023-09-23T19:07:48.043 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.043 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.043 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:07:48.043 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.817182+0000", 2023-09-23T19:07:48.044 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1956e-05 2023-09-23T19:07:48.044 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.044 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.044 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:07:48.044 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.817190+0000", 2023-09-23T19:07:48.044 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.5809999999999999e-06 2023-09-23T19:07:48.044 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.044 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.044 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:07:48.044 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.817325+0000", 2023-09-23T19:07:48.044 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000134902 2023-09-23T19:07:48.044 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.044 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.045 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:07:48.045 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.817438+0000", 2023-09-23T19:07:48.045 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00011352 2023-09-23T19:07:48.045 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.045 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.045 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0", 2023-09-23T19:07:48.045 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.817537+0000", 2023-09-23T19:07:48.045 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.8770000000000005e-05 2023-09-23T19:07:48.045 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.045 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.045 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:07:48.045 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.826222+0000", 2023-09-23T19:07:48.045 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0086849860000000004 2023-09-23T19:07:48.046 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.046 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.046 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.046 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.831658+0000", 2023-09-23T19:07:48.046 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.005435718 2023-09-23T19:07:48.046 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.046 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.046 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:07:48.046 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.831705+0000", 2023-09-23T19:07:48.046 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.7410000000000002e-05 2023-09-23T19:07:48.046 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.046 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.046 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:07:48.047 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.010649+0000", 2023-09-23T19:07:48.047 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.17894382 2023-09-23T19:07:48.047 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.047 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:07:48.047 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.047 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.047 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.047 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:123351 10.8 10:1cce510d:::benchmark_data_smithi098_230241_object22889:head [delete] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-09-23T19:07:48.047 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:03:11.817292+0000", 2023-09-23T19:07:48.047 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 276.201593704, 2023-09-23T19:07:48.047 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.199075902, 2023-09-23T19:07:48.047 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:07:48.048 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:07:48.048 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:07:48.048 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:07:48.048 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:07:48.048 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 123351 2023-09-23T19:07:48.048 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.048 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:07:48.048 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.048 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:07:48.048 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.817292+0000", 2023-09-23T19:07:48.048 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.048 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.049 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.049 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:07:48.049 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.817292+0000", 2023-09-23T19:07:48.049 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.049 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.049 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.049 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:07:48.049 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.817291+0000", 2023-09-23T19:07:48.049 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T19:07:48.049 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.049 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.049 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:07:48.049 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.817296+0000", 2023-09-23T19:07:48.050 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.2919999999999997e-06 2023-09-23T19:07:48.050 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.050 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.050 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:07:48.050 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.817304+0000", 2023-09-23T19:07:48.050 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.4700000000000002e-06 2023-09-23T19:07:48.050 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.050 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.050 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:07:48.050 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.817310+0000", 2023-09-23T19:07:48.050 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.9669999999999998e-06 2023-09-23T19:07:48.050 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.051 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.051 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:07:48.051 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.822041+0000", 2023-09-23T19:07:48.051 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0047308690000000004 2023-09-23T19:07:48.051 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.051 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.051 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:07:48.051 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.822165+0000", 2023-09-23T19:07:48.051 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00012402800000000001 2023-09-23T19:07:48.051 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.051 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.051 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0", 2023-09-23T19:07:48.051 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.822279+0000", 2023-09-23T19:07:48.052 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000113517 2023-09-23T19:07:48.052 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.052 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.052 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:07:48.052 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.015984+0000", 2023-09-23T19:07:48.052 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.193705397 2023-09-23T19:07:48.052 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.052 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.052 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.052 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.016063+0000", 2023-09-23T19:07:48.052 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.8633000000000004e-05 2023-09-23T19:07:48.052 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.052 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.052 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:07:48.053 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.016128+0000", 2023-09-23T19:07:48.053 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.5806e-05 2023-09-23T19:07:48.053 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.053 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.053 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:07:48.053 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.016368+0000", 2023-09-23T19:07:48.053 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000239647 2023-09-23T19:07:48.053 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.053 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:07:48.053 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.053 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.053 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.053 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:123352 10.7 10:e67d0523:::benchmark_data_smithi098_230241_object22890:head [delete] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-09-23T19:07:48.054 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:03:11.818557+0000", 2023-09-23T19:07:48.054 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 276.20032899300003, 2023-09-23T19:07:48.054 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.203146093, 2023-09-23T19:07:48.054 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:07:48.054 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:07:48.054 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:07:48.054 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:07:48.054 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:07:48.054 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 123352 2023-09-23T19:07:48.054 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.054 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:07:48.054 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.054 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:07:48.055 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.818557+0000", 2023-09-23T19:07:48.055 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.055 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.055 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.055 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:07:48.055 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.818557+0000", 2023-09-23T19:07:48.055 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.055 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.055 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.055 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:07:48.055 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.818556+0000", 2023-09-23T19:07:48.055 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T19:07:48.055 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.056 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.056 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:07:48.056 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.818561+0000", 2023-09-23T19:07:48.056 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.0220000000000004e-06 2023-09-23T19:07:48.056 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.056 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.056 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:07:48.056 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.818572+0000", 2023-09-23T19:07:48.056 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1012999999999999e-05 2023-09-23T19:07:48.056 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.056 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.056 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:07:48.056 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.818579+0000", 2023-09-23T19:07:48.057 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.1450000000000002e-06 2023-09-23T19:07:48.057 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.057 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.057 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:07:48.057 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.822813+0000", 2023-09-23T19:07:48.057 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0042334959999999998 2023-09-23T19:07:48.057 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.057 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.057 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:07:48.057 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.822921+0000", 2023-09-23T19:07:48.057 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000108485 2023-09-23T19:07:48.057 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.057 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.057 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,6", 2023-09-23T19:07:48.058 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.823012+0000", 2023-09-23T19:07:48.058 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.1230999999999994e-05 2023-09-23T19:07:48.058 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.058 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.058 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.058 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.827178+0000", 2023-09-23T19:07:48.058 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0041655809999999998 2023-09-23T19:07:48.058 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.058 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.058 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:07:48.058 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.827473+0000", 2023-09-23T19:07:48.058 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00029500700000000002 2023-09-23T19:07:48.059 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.059 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.059 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.059 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.021630+0000", 2023-09-23T19:07:48.059 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.19415655100000001 2023-09-23T19:07:48.059 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.059 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.059 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:07:48.059 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.021669+0000", 2023-09-23T19:07:48.059 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.9579000000000002e-05 2023-09-23T19:07:48.059 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.060 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.060 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:07:48.060 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.021703+0000", 2023-09-23T19:07:48.060 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.3864e-05 2023-09-23T19:07:48.060 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.060 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:07:48.060 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.060 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.060 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.060 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:123353 10.9 10:9f9093bb:::benchmark_data_smithi098_230241_object22891:head [delete] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-09-23T19:07:48.060 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:03:11.822910+0000", 2023-09-23T19:07:48.060 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 276.19597547400002, 2023-09-23T19:07:48.060 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.192667369, 2023-09-23T19:07:48.061 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:07:48.061 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:07:48.061 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:07:48.061 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:07:48.061 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:07:48.061 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 123353 2023-09-23T19:07:48.061 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.061 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:07:48.061 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.061 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:07:48.061 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.822910+0000", 2023-09-23T19:07:48.061 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.062 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.062 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.062 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:07:48.062 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.822910+0000", 2023-09-23T19:07:48.062 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.062 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.062 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.062 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:07:48.062 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.822909+0000", 2023-09-23T19:07:48.062 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T19:07:48.062 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.062 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.062 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:07:48.063 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.822915+0000", 2023-09-23T19:07:48.063 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.0560000000000001e-06 2023-09-23T19:07:48.063 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.063 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.063 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:07:48.063 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.822926+0000", 2023-09-23T19:07:48.063 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0984e-05 2023-09-23T19:07:48.063 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.063 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.063 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:07:48.063 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.822934+0000", 2023-09-23T19:07:48.063 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.61e-06 2023-09-23T19:07:48.063 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.064 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.064 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:07:48.064 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.823003+0000", 2023-09-23T19:07:48.064 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.9628000000000005e-05 2023-09-23T19:07:48.064 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.064 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.064 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:07:48.064 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.823165+0000", 2023-09-23T19:07:48.064 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000161791 2023-09-23T19:07:48.064 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.064 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.064 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 3", 2023-09-23T19:07:48.064 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.823259+0000", 2023-09-23T19:07:48.065 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.3703999999999996e-05 2023-09-23T19:07:48.065 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.065 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.065 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:07:48.065 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.831199+0000", 2023-09-23T19:07:48.065 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0079399610000000006 2023-09-23T19:07:48.065 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.065 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.065 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.065 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.011887+0000", 2023-09-23T19:07:48.065 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.180687598 2023-09-23T19:07:48.065 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.065 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.065 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:07:48.066 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.015493+0000", 2023-09-23T19:07:48.066 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0036068770000000001 2023-09-23T19:07:48.066 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.066 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.066 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:07:48.066 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.015578+0000", 2023-09-23T19:07:48.066 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.4345999999999995e-05 2023-09-23T19:07:48.066 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.066 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:07:48.066 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.066 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.066 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.067 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:123355 10.3 10:c95d0e1f:::benchmark_data_smithi098_230241_object22893:head [delete] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-09-23T19:07:48.067 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:03:11.826670+0000", 2023-09-23T19:07:48.067 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 276.19221621200001, 2023-09-23T19:07:48.067 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.19179468, 2023-09-23T19:07:48.067 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:07:48.067 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:07:48.067 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:07:48.067 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:07:48.067 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:07:48.067 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 123355 2023-09-23T19:07:48.067 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.067 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:07:48.067 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.068 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:07:48.068 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.826670+0000", 2023-09-23T19:07:48.068 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.068 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.068 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.068 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:07:48.068 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.826670+0000", 2023-09-23T19:07:48.068 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.068 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.068 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.068 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:07:48.068 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.826668+0000", 2023-09-23T19:07:48.068 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T19:07:48.068 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.069 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.069 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:07:48.069 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.826675+0000", 2023-09-23T19:07:48.069 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.9070000000000001e-06 2023-09-23T19:07:48.069 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.069 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.069 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:07:48.069 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.826687+0000", 2023-09-23T19:07:48.069 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.2153999999999999e-05 2023-09-23T19:07:48.069 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.069 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.069 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:07:48.069 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.826695+0000", 2023-09-23T19:07:48.070 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.7279999999999995e-06 2023-09-23T19:07:48.070 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.070 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.070 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:07:48.070 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.826801+0000", 2023-09-23T19:07:48.070 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000106186 2023-09-23T19:07:48.070 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.070 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.070 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:07:48.070 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.826924+0000", 2023-09-23T19:07:48.070 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00012316200000000001 2023-09-23T19:07:48.070 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.070 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.071 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0", 2023-09-23T19:07:48.071 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.827033+0000", 2023-09-23T19:07:48.071 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00010894 2023-09-23T19:07:48.071 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.071 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.071 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:07:48.071 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.831719+0000", 2023-09-23T19:07:48.071 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0046853440000000001 2023-09-23T19:07:48.071 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.071 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.071 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.071 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.018261+0000", 2023-09-23T19:07:48.071 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.18654232100000001 2023-09-23T19:07:48.072 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.072 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.072 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:07:48.072 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.018373+0000", 2023-09-23T19:07:48.072 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00011159599999999999 2023-09-23T19:07:48.072 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.072 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.072 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:07:48.072 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.018464+0000", 2023-09-23T19:07:48.072 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.1731999999999995e-05 2023-09-23T19:07:48.072 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.072 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:07:48.072 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.072 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.073 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.073 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:123357 10.5 10:a46d6ed0:::benchmark_data_smithi098_230241_object22895:head [delete] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-09-23T19:07:48.073 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:03:11.827836+0000", 2023-09-23T19:07:48.073 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 276.19104992799998, 2023-09-23T19:07:48.073 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.193745626, 2023-09-23T19:07:48.073 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:07:48.073 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:07:48.073 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:07:48.073 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:07:48.073 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:07:48.073 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 123357 2023-09-23T19:07:48.073 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.074 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:07:48.074 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.074 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:07:48.074 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.827836+0000", 2023-09-23T19:07:48.074 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.074 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.074 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.074 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:07:48.074 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.827836+0000", 2023-09-23T19:07:48.074 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.074 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.074 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.074 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:07:48.075 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.827835+0000", 2023-09-23T19:07:48.075 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T19:07:48.075 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.075 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.075 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:07:48.075 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.827839+0000", 2023-09-23T19:07:48.075 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.8770000000000003e-06 2023-09-23T19:07:48.075 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.075 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.075 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:07:48.075 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.827847+0000", 2023-09-23T19:07:48.075 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.4989999999999997e-06 2023-09-23T19:07:48.075 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.076 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.076 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:07:48.076 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.827853+0000", 2023-09-23T19:07:48.076 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.0299999999999999e-06 2023-09-23T19:07:48.076 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.076 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.076 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:07:48.076 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.827903+0000", 2023-09-23T19:07:48.076 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.0207999999999998e-05 2023-09-23T19:07:48.076 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.076 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.076 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:07:48.076 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.828007+0000", 2023-09-23T19:07:48.077 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000104394 2023-09-23T19:07:48.077 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.077 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.077 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,3", 2023-09-23T19:07:48.077 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.828104+0000", 2023-09-23T19:07:48.077 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.7183999999999997e-05 2023-09-23T19:07:48.077 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.077 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.077 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:07:48.077 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.015966+0000", 2023-09-23T19:07:48.077 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.187861158 2023-09-23T19:07:48.077 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.077 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.078 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.078 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.016023+0000", 2023-09-23T19:07:48.078 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.6882999999999997e-05 2023-09-23T19:07:48.078 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.078 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.078 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.078 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.021484+0000", 2023-09-23T19:07:48.078 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.005461161 2023-09-23T19:07:48.078 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.078 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.078 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:07:48.078 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.021541+0000", 2023-09-23T19:07:48.079 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.7370000000000001e-05 2023-09-23T19:07:48.079 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.079 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.079 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:07:48.079 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.021582+0000", 2023-09-23T19:07:48.079 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.0528000000000002e-05 2023-09-23T19:07:48.079 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.079 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:07:48.079 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.079 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.079 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.079 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:123358 10.5 10:aab3a3d6:::benchmark_data_smithi098_230241_object22896:head [delete] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-09-23T19:07:48.079 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:03:11.830454+0000", 2023-09-23T19:07:48.079 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 276.18843225500001, 2023-09-23T19:07:48.080 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.21056792099999999, 2023-09-23T19:07:48.080 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:07:48.080 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:07:48.080 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:07:48.080 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:07:48.080 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:07:48.080 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 123358 2023-09-23T19:07:48.080 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.080 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:07:48.080 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.080 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:07:48.080 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.830454+0000", 2023-09-23T19:07:48.080 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.081 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.081 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.081 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:07:48.081 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.830454+0000", 2023-09-23T19:07:48.081 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.081 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.081 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.081 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:07:48.081 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.830452+0000", 2023-09-23T19:07:48.081 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T19:07:48.081 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.081 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.081 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:07:48.082 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.830459+0000", 2023-09-23T19:07:48.082 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.1300000000000003e-06 2023-09-23T19:07:48.082 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.082 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.082 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:07:48.082 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.830472+0000", 2023-09-23T19:07:48.082 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.2398000000000001e-05 2023-09-23T19:07:48.082 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.082 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.082 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:07:48.082 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.830480+0000", 2023-09-23T19:07:48.082 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.1030000000000002e-06 2023-09-23T19:07:48.082 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.083 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.083 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:07:48.083 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.831712+0000", 2023-09-23T19:07:48.083 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.001232627 2023-09-23T19:07:48.083 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.083 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.083 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:07:48.083 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.011646+0000", 2023-09-23T19:07:48.083 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.17993382599999999 2023-09-23T19:07:48.083 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.083 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.083 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,3", 2023-09-23T19:07:48.083 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.011762+0000", 2023-09-23T19:07:48.083 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000115449 2023-09-23T19:07:48.084 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.084 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.084 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:07:48.084 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.016209+0000", 2023-09-23T19:07:48.084 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0044468989999999998 2023-09-23T19:07:48.084 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.084 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.084 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.084 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.040885+0000", 2023-09-23T19:07:48.084 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.024675965000000001 2023-09-23T19:07:48.084 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.084 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.085 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.085 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.040940+0000", 2023-09-23T19:07:48.085 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.5581999999999997e-05 2023-09-23T19:07:48.085 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.085 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.085 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:07:48.085 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.040984+0000", 2023-09-23T19:07:48.085 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.3964999999999997e-05 2023-09-23T19:07:48.085 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.085 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.085 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:07:48.085 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.041022+0000", 2023-09-23T19:07:48.085 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.7370000000000003e-05 2023-09-23T19:07:48.085 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.086 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:07:48.086 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.086 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.086 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.086 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:123363 10.9 10:99e00805:::benchmark_data_smithi098_230241_object22901:head [delete] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-09-23T19:07:48.086 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:03:11.832678+0000", 2023-09-23T19:07:48.086 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 276.18620798299997, 2023-09-23T19:07:48.086 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.202782672, 2023-09-23T19:07:48.086 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:07:48.086 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:07:48.086 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:07:48.086 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:07:48.086 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:07:48.087 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 123363 2023-09-23T19:07:48.087 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.087 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:07:48.087 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.087 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:07:48.087 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.832678+0000", 2023-09-23T19:07:48.087 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.087 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.087 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.087 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:07:48.087 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.832678+0000", 2023-09-23T19:07:48.087 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.088 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.088 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.088 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:07:48.088 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.832677+0000", 2023-09-23T19:07:48.088 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T19:07:48.088 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.088 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.088 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:07:48.088 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:11.832683+0000", 2023-09-23T19:07:48.088 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.2709999999999998e-06 2023-09-23T19:07:48.088 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.088 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.088 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:07:48.089 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.010632+0000", 2023-09-23T19:07:48.089 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.17794901199999999 2023-09-23T19:07:48.089 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.089 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.089 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:07:48.089 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.010655+0000", 2023-09-23T19:07:48.089 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.2983e-05 2023-09-23T19:07:48.089 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.089 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.089 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:07:48.089 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.015617+0000", 2023-09-23T19:07:48.090 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0049618179999999998 2023-09-23T19:07:48.090 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.090 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.090 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:07:48.090 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.015768+0000", 2023-09-23T19:07:48.090 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000151142 2023-09-23T19:07:48.090 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.090 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.090 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 3", 2023-09-23T19:07:48.090 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.015892+0000", 2023-09-23T19:07:48.090 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000124208 2023-09-23T19:07:48.090 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.091 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.091 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:07:48.091 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.027369+0000", 2023-09-23T19:07:48.091 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.01147726 2023-09-23T19:07:48.091 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.091 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.091 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.091 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.034955+0000", 2023-09-23T19:07:48.091 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0075852100000000002 2023-09-23T19:07:48.091 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.091 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.091 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:07:48.091 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.035114+0000", 2023-09-23T19:07:48.092 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000159541 2023-09-23T19:07:48.092 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.092 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.092 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:07:48.092 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:12.035461+0000", 2023-09-23T19:07:48.092 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00034651400000000002 2023-09-23T19:07:48.092 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.092 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:07:48.092 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.092 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.092 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.092 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:150140 10.3 10:c0617e29:::benchmark_data_smithi098_230241_object49678:head [delete] snapc 0=[] ondisk+write+known_if_redirected e333)", 2023-09-23T19:07:48.092 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:03:56.040532+0000", 2023-09-23T19:07:48.093 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 231.97835400100001, 2023-09-23T19:07:48.093 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.18967495300000001, 2023-09-23T19:07:48.093 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:07:48.093 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:07:48.093 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:07:48.093 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:07:48.093 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:07:48.093 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 150140 2023-09-23T19:07:48.093 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.093 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:07:48.093 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.093 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:07:48.093 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:56.040532+0000", 2023-09-23T19:07:48.094 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.094 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.094 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.094 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:07:48.094 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:56.040532+0000", 2023-09-23T19:07:48.094 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.094 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.094 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.094 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:07:48.094 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:56.040531+0000", 2023-09-23T19:07:48.094 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T19:07:48.094 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.094 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.095 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:07:48.095 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:56.040536+0000", 2023-09-23T19:07:48.095 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.0069999999999997e-06 2023-09-23T19:07:48.095 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.095 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.095 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:07:48.095 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:56.040546+0000", 2023-09-23T19:07:48.095 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.5699999999999999e-06 2023-09-23T19:07:48.095 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.095 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.095 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:07:48.095 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:56.040553+0000", 2023-09-23T19:07:48.095 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.9949999999999999e-06 2023-09-23T19:07:48.096 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.096 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.096 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:07:48.096 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:56.051624+0000", 2023-09-23T19:07:48.096 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.011071397 2023-09-23T19:07:48.096 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.096 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.096 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:07:48.096 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:56.051738+0000", 2023-09-23T19:07:48.096 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00011350599999999999 2023-09-23T19:07:48.096 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.096 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.096 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0", 2023-09-23T19:07:48.097 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:56.051848+0000", 2023-09-23T19:07:48.097 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00011061399999999999 2023-09-23T19:07:48.097 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.097 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.097 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:07:48.097 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:56.062316+0000", 2023-09-23T19:07:48.097 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.010468023999999999 2023-09-23T19:07:48.097 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.097 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.097 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.097 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:56.230078+0000", 2023-09-23T19:07:48.097 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.16776133400000001 2023-09-23T19:07:48.097 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.098 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.098 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:07:48.098 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:56.230128+0000", 2023-09-23T19:07:48.098 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.0454000000000002e-05 2023-09-23T19:07:48.098 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.098 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.098 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:07:48.098 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:03:56.230207+0000", 2023-09-23T19:07:48.098 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.8788999999999994e-05 2023-09-23T19:07:48.098 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.098 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:07:48.098 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.099 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.099 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.099 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:169187 10.5 10:ad293c91:::benchmark_data_smithi098_230241_object68725:head [delete] snapc 0=[] ondisk+write+known_if_redirected e333)", 2023-09-23T19:07:48.099 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:04:26.909209+0000", 2023-09-23T19:07:48.099 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 201.10967637799999, 2023-09-23T19:07:48.099 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.213487965, 2023-09-23T19:07:48.099 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:07:48.099 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:07:48.099 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:07:48.099 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:07:48.099 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:07:48.099 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 169187 2023-09-23T19:07:48.099 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.099 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:07:48.100 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.100 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:07:48.100 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.909209+0000", 2023-09-23T19:07:48.100 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.100 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.100 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.100 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:07:48.100 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.909209+0000", 2023-09-23T19:07:48.100 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.100 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.100 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.100 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:07:48.100 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.909208+0000", 2023-09-23T19:07:48.101 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T19:07:48.101 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.101 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.101 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:07:48.101 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.909215+0000", 2023-09-23T19:07:48.101 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.4200000000000004e-06 2023-09-23T19:07:48.101 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.101 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.101 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:07:48.101 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.909226+0000", 2023-09-23T19:07:48.101 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1768e-05 2023-09-23T19:07:48.101 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.101 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.102 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:07:48.102 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.909234+0000", 2023-09-23T19:07:48.102 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.6750000000000002e-06 2023-09-23T19:07:48.102 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.102 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.102 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:07:48.102 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.909286+0000", 2023-09-23T19:07:48.102 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.2076000000000001e-05 2023-09-23T19:07:48.102 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.102 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.102 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:07:48.102 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.909386+0000", 2023-09-23T19:07:48.102 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.9902000000000007e-05 2023-09-23T19:07:48.103 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.103 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.103 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,3", 2023-09-23T19:07:48.103 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.909478+0000", 2023-09-23T19:07:48.103 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.1596999999999997e-05 2023-09-23T19:07:48.103 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.103 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.103 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:07:48.103 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.920429+0000", 2023-09-23T19:07:48.103 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.010951447 2023-09-23T19:07:48.103 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.103 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.103 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.104 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.928240+0000", 2023-09-23T19:07:48.104 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0078108980000000001 2023-09-23T19:07:48.104 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.104 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.104 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.104 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:27.122616+0000", 2023-09-23T19:07:48.104 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.19437653899999999 2023-09-23T19:07:48.104 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.104 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.104 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:07:48.104 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:27.122662+0000", 2023-09-23T19:07:48.104 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.5067000000000001e-05 2023-09-23T19:07:48.104 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.105 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.105 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:07:48.105 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:27.122697+0000", 2023-09-23T19:07:48.105 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.5899999999999998e-05 2023-09-23T19:07:48.105 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.105 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:07:48.105 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.105 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.105 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.105 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:169190 10.5 10:a408261e:::benchmark_data_smithi098_230241_object68728:head [delete] snapc 0=[] ondisk+write+known_if_redirected e333)", 2023-09-23T19:07:48.105 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:04:26.913118+0000", 2023-09-23T19:07:48.105 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 201.105768205, 2023-09-23T19:07:48.105 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.20967871699999999, 2023-09-23T19:07:48.105 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:07:48.106 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:07:48.106 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:07:48.106 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:07:48.106 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:07:48.106 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 169190 2023-09-23T19:07:48.106 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.106 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:07:48.106 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.106 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:07:48.106 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.913118+0000", 2023-09-23T19:07:48.106 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.106 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.107 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.107 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:07:48.107 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.913118+0000", 2023-09-23T19:07:48.107 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.107 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.107 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.107 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:07:48.107 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.913117+0000", 2023-09-23T19:07:48.107 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T19:07:48.107 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.107 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.107 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:07:48.107 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.913120+0000", 2023-09-23T19:07:48.107 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.2930000000000001e-06 2023-09-23T19:07:48.108 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.108 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.108 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:07:48.108 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.913127+0000", 2023-09-23T19:07:48.108 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.7390000000000002e-06 2023-09-23T19:07:48.108 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.108 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.108 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:07:48.108 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.913132+0000", 2023-09-23T19:07:48.108 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.1989999999999997e-06 2023-09-23T19:07:48.108 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.108 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.108 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:07:48.109 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.913732+0000", 2023-09-23T19:07:48.109 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00060011500000000002 2023-09-23T19:07:48.109 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.109 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.109 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:07:48.109 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.913839+0000", 2023-09-23T19:07:48.109 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000106619 2023-09-23T19:07:48.109 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.109 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.109 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,3", 2023-09-23T19:07:48.109 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.913936+0000", 2023-09-23T19:07:48.109 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.7343000000000006e-05 2023-09-23T19:07:48.109 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.110 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.110 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:07:48.110 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.923100+0000", 2023-09-23T19:07:48.110 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.009163717 2023-09-23T19:07:48.110 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.110 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.110 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.110 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.928428+0000", 2023-09-23T19:07:48.110 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0053280940000000002 2023-09-23T19:07:48.110 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.110 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.110 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.110 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:27.122734+0000", 2023-09-23T19:07:48.111 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.19430576499999999 2023-09-23T19:07:48.111 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.111 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.111 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:07:48.111 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:27.122767+0000", 2023-09-23T19:07:48.111 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.2583999999999998e-05 2023-09-23T19:07:48.111 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.111 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.111 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:07:48.111 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:27.122796+0000", 2023-09-23T19:07:48.111 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.9771999999999999e-05 2023-09-23T19:07:48.111 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.112 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:07:48.112 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.112 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.112 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.112 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:169192 10.3 10:c6c18ba3:::benchmark_data_smithi098_230241_object68730:head [delete] snapc 0=[] ondisk+write+known_if_redirected e333)", 2023-09-23T19:07:48.112 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:04:26.917837+0000", 2023-09-23T19:07:48.112 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 201.10104847900001, 2023-09-23T19:07:48.112 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.20535513899999999, 2023-09-23T19:07:48.112 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:07:48.112 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:07:48.112 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:07:48.112 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:07:48.112 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:07:48.113 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 169192 2023-09-23T19:07:48.113 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.113 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:07:48.113 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.113 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:07:48.113 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.917837+0000", 2023-09-23T19:07:48.113 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.113 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.113 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.113 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:07:48.113 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.917837+0000", 2023-09-23T19:07:48.113 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.114 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.114 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.114 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:07:48.114 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.917836+0000", 2023-09-23T19:07:48.114 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:07:48.114 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.114 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.114 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:07:48.114 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.917844+0000", 2023-09-23T19:07:48.114 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.3119999999999994e-06 2023-09-23T19:07:48.114 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.114 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.114 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:07:48.115 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.917972+0000", 2023-09-23T19:07:48.115 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00012807399999999999 2023-09-23T19:07:48.115 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.115 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.115 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:07:48.115 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.917985+0000", 2023-09-23T19:07:48.115 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3278999999999999e-05 2023-09-23T19:07:48.115 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.115 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.115 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:07:48.115 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.918042+0000", 2023-09-23T19:07:48.115 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.643e-05 2023-09-23T19:07:48.115 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.116 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.116 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:07:48.116 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.918367+0000", 2023-09-23T19:07:48.116 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00032543699999999998 2023-09-23T19:07:48.116 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.116 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.116 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0", 2023-09-23T19:07:48.116 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.918565+0000", 2023-09-23T19:07:48.116 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00019828700000000001 2023-09-23T19:07:48.116 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.116 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.116 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:07:48.116 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.923377+0000", 2023-09-23T19:07:48.116 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0048112129999999999 2023-09-23T19:07:48.117 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.117 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.117 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.117 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:27.123068+0000", 2023-09-23T19:07:48.117 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.19969142300000001 2023-09-23T19:07:48.117 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.117 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.117 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:07:48.117 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:27.123114+0000", 2023-09-23T19:07:48.117 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.5500999999999997e-05 2023-09-23T19:07:48.117 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.117 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.117 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:07:48.118 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:27.123193+0000", 2023-09-23T19:07:48.118 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.8897e-05 2023-09-23T19:07:48.118 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.118 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:07:48.118 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.118 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.118 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.118 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:169197 10.5 10:a6f6304a:::benchmark_data_smithi098_230241_object68735:head [delete] snapc 0=[] ondisk+write+known_if_redirected e333)", 2023-09-23T19:07:48.118 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:04:26.930360+0000", 2023-09-23T19:07:48.118 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 201.08852603299999, 2023-09-23T19:07:48.118 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.201852163, 2023-09-23T19:07:48.118 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:07:48.119 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:07:48.119 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:07:48.119 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:07:48.119 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:07:48.119 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 169197 2023-09-23T19:07:48.119 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.119 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:07:48.119 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.119 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:07:48.119 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.930360+0000", 2023-09-23T19:07:48.119 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.119 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.119 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.120 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:07:48.120 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.930360+0000", 2023-09-23T19:07:48.120 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.120 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.120 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.120 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:07:48.120 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.930359+0000", 2023-09-23T19:07:48.120 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T19:07:48.120 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.120 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.120 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:07:48.120 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.930364+0000", 2023-09-23T19:07:48.121 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.5249999999999996e-06 2023-09-23T19:07:48.121 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.121 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.121 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:07:48.121 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.930375+0000", 2023-09-23T19:07:48.121 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1215e-05 2023-09-23T19:07:48.121 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.121 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.121 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:07:48.121 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.930383+0000", 2023-09-23T19:07:48.121 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.8720000000000002e-06 2023-09-23T19:07:48.121 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.121 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.122 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:07:48.122 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.930490+0000", 2023-09-23T19:07:48.122 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000106601 2023-09-23T19:07:48.122 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.122 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.122 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:07:48.122 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.930636+0000", 2023-09-23T19:07:48.122 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000146296 2023-09-23T19:07:48.122 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.122 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.122 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,3", 2023-09-23T19:07:48.122 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.930776+0000", 2023-09-23T19:07:48.122 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000139758 2023-09-23T19:07:48.123 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.123 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.123 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:07:48.123 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.934227+0000", 2023-09-23T19:07:48.123 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0034514810000000002 2023-09-23T19:07:48.123 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.123 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.123 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.123 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.934372+0000", 2023-09-23T19:07:48.123 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00014492800000000001 2023-09-23T19:07:48.123 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.123 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.123 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.124 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:27.129539+0000", 2023-09-23T19:07:48.124 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.19516707799999999 2023-09-23T19:07:48.124 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.124 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.124 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:07:48.124 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:27.129902+0000", 2023-09-23T19:07:48.124 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00036271300000000002 2023-09-23T19:07:48.124 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.124 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.124 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:07:48.124 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:27.132212+0000", 2023-09-23T19:07:48.124 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0023098070000000001 2023-09-23T19:07:48.124 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.125 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:07:48.125 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.125 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.125 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.125 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:194005 10.5 10:aa06fb2c:::benchmark_data_smithi098_230241_object93543:head [delete] snapc 0=[] ondisk+write+known_if_redirected e335)", 2023-09-23T19:07:48.125 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:05:08.424737+0000", 2023-09-23T19:07:48.125 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 159.594148459, 2023-09-23T19:07:48.125 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.24417540700000001, 2023-09-23T19:07:48.125 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:07:48.125 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:07:48.125 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:07:48.125 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:07:48.125 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:07:48.126 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 194005 2023-09-23T19:07:48.126 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.126 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:07:48.126 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.126 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:07:48.126 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.424737+0000", 2023-09-23T19:07:48.126 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.126 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.126 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.126 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:07:48.126 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.424737+0000", 2023-09-23T19:07:48.126 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.126 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.127 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.127 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:07:48.127 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.424736+0000", 2023-09-23T19:07:48.127 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T19:07:48.127 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.127 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.127 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:07:48.127 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.424742+0000", 2023-09-23T19:07:48.127 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.6110000000000001e-06 2023-09-23T19:07:48.127 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.127 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.127 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:07:48.127 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.424752+0000", 2023-09-23T19:07:48.128 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0389e-05 2023-09-23T19:07:48.128 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.128 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.128 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:07:48.128 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.424758+0000", 2023-09-23T19:07:48.128 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.2920000000000001e-06 2023-09-23T19:07:48.128 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.128 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.128 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:07:48.128 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.424867+0000", 2023-09-23T19:07:48.128 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00010849300000000001 2023-09-23T19:07:48.128 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.128 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.128 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:07:48.129 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.424998+0000", 2023-09-23T19:07:48.129 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000131001 2023-09-23T19:07:48.129 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.129 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.129 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,3", 2023-09-23T19:07:48.129 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.425111+0000", 2023-09-23T19:07:48.129 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000112946 2023-09-23T19:07:48.129 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.129 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.129 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:07:48.129 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.435634+0000", 2023-09-23T19:07:48.129 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.010522795 2023-09-23T19:07:48.129 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.130 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.130 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.130 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.444372+0000", 2023-09-23T19:07:48.130 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0087387379999999994 2023-09-23T19:07:48.130 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.130 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.130 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.130 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.667952+0000", 2023-09-23T19:07:48.130 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.223579109 2023-09-23T19:07:48.130 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.130 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.130 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:07:48.131 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.668146+0000", 2023-09-23T19:07:48.131 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00019406100000000001 2023-09-23T19:07:48.131 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.131 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.131 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:07:48.131 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.668913+0000", 2023-09-23T19:07:48.131 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00076725999999999997 2023-09-23T19:07:48.131 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.131 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:07:48.131 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.131 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.131 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.131 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:194007 10.5 10:aea4ea8b:::benchmark_data_smithi098_230241_object93545:head [delete] snapc 0=[] ondisk+write+known_if_redirected e335)", 2023-09-23T19:07:48.131 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:05:08.428674+0000", 2023-09-23T19:07:48.132 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 159.590212372, 2023-09-23T19:07:48.132 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.246817916, 2023-09-23T19:07:48.132 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:07:48.132 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:07:48.132 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:07:48.132 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:07:48.132 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:07:48.132 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 194007 2023-09-23T19:07:48.132 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.132 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:07:48.132 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.132 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:07:48.132 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.428674+0000", 2023-09-23T19:07:48.133 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.133 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.133 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.133 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:07:48.133 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.428674+0000", 2023-09-23T19:07:48.133 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.133 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.133 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.133 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:07:48.133 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.428672+0000", 2023-09-23T19:07:48.133 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T19:07:48.133 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.134 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.134 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:07:48.134 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.428678+0000", 2023-09-23T19:07:48.134 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.1749999999999996e-06 2023-09-23T19:07:48.134 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.134 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.134 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:07:48.134 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.428687+0000", 2023-09-23T19:07:48.134 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.6749999999999997e-06 2023-09-23T19:07:48.134 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.134 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.134 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:07:48.135 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.428693+0000", 2023-09-23T19:07:48.135 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.8680000000000004e-06 2023-09-23T19:07:48.135 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.135 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.135 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:07:48.135 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.430967+0000", 2023-09-23T19:07:48.135 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0022735339999999998 2023-09-23T19:07:48.135 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.135 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.135 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:07:48.135 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.431066+0000", 2023-09-23T19:07:48.135 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.8844000000000004e-05 2023-09-23T19:07:48.135 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.136 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.136 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,3", 2023-09-23T19:07:48.136 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.431165+0000", 2023-09-23T19:07:48.136 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.9636000000000006e-05 2023-09-23T19:07:48.136 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.136 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.136 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.136 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.446056+0000", 2023-09-23T19:07:48.136 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.014891065 2023-09-23T19:07:48.136 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.136 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.136 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:07:48.136 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.446064+0000", 2023-09-23T19:07:48.137 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.2960000000000001e-06 2023-09-23T19:07:48.137 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.137 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.137 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.137 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.675392+0000", 2023-09-23T19:07:48.137 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.22932818299999999 2023-09-23T19:07:48.137 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.137 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.137 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:07:48.137 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.675447+0000", 2023-09-23T19:07:48.137 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.4784999999999998e-05 2023-09-23T19:07:48.137 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.137 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.138 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:07:48.138 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.675491+0000", 2023-09-23T19:07:48.138 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.4879999999999997e-05 2023-09-23T19:07:48.138 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.138 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:07:48.138 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.138 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.138 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.138 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:194012 10.5 10:aa04c153:::benchmark_data_smithi098_230241_object93550:head [delete] snapc 0=[] ondisk+write+known_if_redirected e335)", 2023-09-23T19:07:48.138 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:05:08.431077+0000", 2023-09-23T19:07:48.138 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 159.58780914299999, 2023-09-23T19:07:48.138 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.255528584, 2023-09-23T19:07:48.139 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:07:48.139 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:07:48.139 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:07:48.139 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:07:48.139 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:07:48.139 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 194012 2023-09-23T19:07:48.139 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.139 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:07:48.139 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.139 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:07:48.139 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.431077+0000", 2023-09-23T19:07:48.139 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.139 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.139 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.140 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:07:48.140 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.431077+0000", 2023-09-23T19:07:48.140 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.140 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.140 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.140 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:07:48.140 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.431076+0000", 2023-09-23T19:07:48.140 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T19:07:48.140 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.140 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.140 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:07:48.140 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.431081+0000", 2023-09-23T19:07:48.141 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.1359999999999996e-06 2023-09-23T19:07:48.141 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.141 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.141 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:07:48.141 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.431091+0000", 2023-09-23T19:07:48.141 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0190999999999999e-05 2023-09-23T19:07:48.141 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.141 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.141 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:07:48.141 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.431099+0000", 2023-09-23T19:07:48.141 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.4490000000000002e-06 2023-09-23T19:07:48.141 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.141 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.142 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:07:48.142 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.439043+0000", 2023-09-23T19:07:48.142 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0079445880000000007 2023-09-23T19:07:48.142 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.142 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.142 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:07:48.142 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.439160+0000", 2023-09-23T19:07:48.142 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00011636 2023-09-23T19:07:48.142 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.142 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.142 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,3", 2023-09-23T19:07:48.142 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.439267+0000", 2023-09-23T19:07:48.142 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000107769 2023-09-23T19:07:48.143 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.143 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.143 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:07:48.143 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.446067+0000", 2023-09-23T19:07:48.143 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0067992520000000004 2023-09-23T19:07:48.143 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.143 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.143 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.143 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.449679+0000", 2023-09-23T19:07:48.143 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.003612559 2023-09-23T19:07:48.143 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.143 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.143 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.144 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.686529+0000", 2023-09-23T19:07:48.144 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.23684951200000001 2023-09-23T19:07:48.144 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.144 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.144 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:07:48.144 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.686572+0000", 2023-09-23T19:07:48.144 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.2861000000000002e-05 2023-09-23T19:07:48.144 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.144 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.144 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:07:48.144 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.686605+0000", 2023-09-23T19:07:48.144 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.3766000000000002e-05 2023-09-23T19:07:48.144 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.145 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:07:48.145 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.145 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.145 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.145 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop(client.45779.0:194009 10.f e335/317 10:fd5f9473:::benchmark_data_smithi098_230241_object93547:head v 335'12256, mlcod=335'12253)", 2023-09-23T19:07:48.145 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:05:08.447760+0000", 2023-09-23T19:07:48.145 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 159.57112546499999, 2023-09-23T19:07:48.145 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.20660645699999999, 2023-09-23T19:07:48.145 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:07:48.145 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:07:48.145 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:07:48.145 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.145 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:07:48.146 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.447760+0000", 2023-09-23T19:07:48.146 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.146 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.146 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.146 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:07:48.146 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.447760+0000", 2023-09-23T19:07:48.146 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.146 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.146 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.146 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:07:48.146 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.447759+0000", 2023-09-23T19:07:48.146 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T19:07:48.146 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.147 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.147 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:07:48.147 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.447768+0000", 2023-09-23T19:07:48.147 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.4339999999999992e-06 2023-09-23T19:07:48.147 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.147 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.147 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:07:48.147 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.447778+0000", 2023-09-23T19:07:48.147 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.7489999999999993e-06 2023-09-23T19:07:48.147 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.147 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.147 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:07:48.148 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.447785+0000", 2023-09-23T19:07:48.148 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.0199999999999997e-06 2023-09-23T19:07:48.148 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.148 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.148 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:07:48.148 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.447851+0000", 2023-09-23T19:07:48.148 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.6478999999999998e-05 2023-09-23T19:07:48.148 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.148 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.148 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:07:48.148 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.447871+0000", 2023-09-23T19:07:48.148 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.9613999999999999e-05 2023-09-23T19:07:48.148 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.149 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.149 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:07:48.149 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.654321+0000", 2023-09-23T19:07:48.149 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.206450258 2023-09-23T19:07:48.149 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.149 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.149 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:07:48.149 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.654367+0000", 2023-09-23T19:07:48.149 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.5908000000000002e-05 2023-09-23T19:07:48.149 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.149 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:07:48.149 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.149 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.150 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.150 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:194022 10.9 10:98de083a:::benchmark_data_smithi098_230241_object93560:head [delete] snapc 0=[] ondisk+write+known_if_redirected e335)", 2023-09-23T19:07:48.150 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:05:08.449928+0000", 2023-09-23T19:07:48.150 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 159.56895794100001, 2023-09-23T19:07:48.150 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.208747031, 2023-09-23T19:07:48.150 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:07:48.150 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:07:48.150 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:07:48.150 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:07:48.150 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:07:48.150 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 194022 2023-09-23T19:07:48.150 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.150 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:07:48.151 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.151 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:07:48.151 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.449928+0000", 2023-09-23T19:07:48.151 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.151 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.151 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.151 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:07:48.151 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.449928+0000", 2023-09-23T19:07:48.151 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:07:48.151 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.151 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.151 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:07:48.152 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.449926+0000", 2023-09-23T19:07:48.152 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T19:07:48.152 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.152 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.152 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:07:48.152 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.449933+0000", 2023-09-23T19:07:48.152 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.669e-06 2023-09-23T19:07:48.152 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.152 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.152 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:07:48.152 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.449943+0000", 2023-09-23T19:07:48.152 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0186999999999999e-05 2023-09-23T19:07:48.152 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.153 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.153 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:07:48.153 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.449951+0000", 2023-09-23T19:07:48.153 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.6459999999999993e-06 2023-09-23T19:07:48.153 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.153 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.153 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:07:48.153 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.450003+0000", 2023-09-23T19:07:48.153 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.2329e-05 2023-09-23T19:07:48.153 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.153 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.153 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:07:48.153 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.450139+0000", 2023-09-23T19:07:48.154 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00013575300000000001 2023-09-23T19:07:48.154 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.154 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.154 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 3", 2023-09-23T19:07:48.154 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.652475+0000", 2023-09-23T19:07:48.154 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.202336085 2023-09-23T19:07:48.154 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.154 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.154 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:07:48.154 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.658134+0000", 2023-09-23T19:07:48.154 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0056590140000000004 2023-09-23T19:07:48.154 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.154 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.155 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:07:48.155 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.658497+0000", 2023-09-23T19:07:48.155 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00036271300000000002 2023-09-23T19:07:48.155 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.155 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.155 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:07:48.155 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.658559+0000", 2023-09-23T19:07:48.155 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.1911999999999999e-05 2023-09-23T19:07:48.155 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:07:48.155 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:07:48.155 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:07:48.155 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.658675+0000", 2023-09-23T19:07:48.155 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000116261 2023-09-23T19:07:48.156 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.156 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:07:48.156 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.156 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:07:48.156 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:07:48.156 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T19:07:48.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:48.157 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:48.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:48.258 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:48.281 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:48.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:48.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:48.384 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:48.396 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:48 smithi173 bash[59422]: cluster 2023-09-23T19:07:46.628334+0000 mgr.y (mgr.26054) 1504 : cluster 0 pgmap v1655: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 12 active+undersized+degraded, 22 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 5.9 KiB/s rd, 1.7 KiB/s wr, 9 op/s; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:48.396 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:48 smithi173 bash[59422]: audit 2023-09-23T19:07:47.232392+0000 mon.a (mon.0) 2027 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:07:48.396 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:48 smithi173 bash[59422]: audit 2023-09-23T19:07:47.233230+0000 mon.a (mon.0) 2028 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:07:48.396 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:48 smithi173 bash[59422]: audit 2023-09-23T19:07:47.240832+0000 mon.a (mon.0) 2029 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:48.414 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:48.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:48.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:48.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:48 smithi098 bash[76817]: cluster 2023-09-23T19:07:46.628334+0000 mgr.y (mgr.26054) 1504 : cluster 0 pgmap v1655: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 12 active+undersized+degraded, 22 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 5.9 KiB/s rd, 1.7 KiB/s wr, 9 op/s; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:48.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:48 smithi098 bash[76817]: audit 2023-09-23T19:07:47.232392+0000 mon.a (mon.0) 2027 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:07:48.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:48 smithi098 bash[76817]: audit 2023-09-23T19:07:47.233230+0000 mon.a (mon.0) 2028 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:07:48.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:48 smithi098 bash[76817]: audit 2023-09-23T19:07:47.240832+0000 mon.a (mon.0) 2029 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:48.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:48 smithi098 bash[79516]: cluster 2023-09-23T19:07:46.628334+0000 mgr.y (mgr.26054) 1504 : cluster 0 pgmap v1655: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 12 active+undersized+degraded, 22 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 5.9 KiB/s rd, 1.7 KiB/s wr, 9 op/s; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:48.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:48 smithi098 bash[79516]: audit 2023-09-23T19:07:47.232392+0000 mon.a (mon.0) 2027 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:07:48.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:48 smithi098 bash[79516]: audit 2023-09-23T19:07:47.233230+0000 mon.a (mon.0) 2028 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:07:48.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:48 smithi098 bash[79516]: audit 2023-09-23T19:07:47.240832+0000 mon.a (mon.0) 2029 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:48.519 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:48.540 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:48.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:48.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:48.643 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:48.669 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:48.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:48.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:48.772 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:48.804 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:48.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:48.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:48.907 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:48.939 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:48.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:48.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:49.043 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:49.067 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:49.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:49.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:49.147 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is set 2023-09-23T19:07:49.171 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:49.183 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:49 smithi173 bash[59422]: audit 2023-09-23T19:07:48.496121+0000 mon.c (mon.2) 151 : audit 1 from='client.? 172.21.15.98:0/531447952' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:07:49.183 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:49 smithi173 bash[59422]: audit 2023-09-23T19:07:48.496445+0000 mon.a (mon.0) 2030 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:07:49.203 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:49.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:49.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:49.308 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:49.321 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:49 smithi098 bash[79516]: audit 2023-09-23T19:07:48.496121+0000 mon.c (mon.2) 151 : audit 1 from='client.? 172.21.15.98:0/531447952' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:07:49.321 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:49 smithi098 bash[79516]: audit 2023-09-23T19:07:48.496445+0000 mon.a (mon.0) 2030 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:07:49.321 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:49 smithi098 bash[76817]: audit 2023-09-23T19:07:48.496121+0000 mon.c (mon.2) 151 : audit 1 from='client.? 172.21.15.98:0/531447952' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:07:49.322 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:49 smithi098 bash[76817]: audit 2023-09-23T19:07:48.496445+0000 mon.a (mon.0) 2030 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:07:49.336 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:49.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:49.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:49.439 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:49.465 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:49.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:49.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:49.569 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:49.591 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:49.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:49.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:49.695 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:49.724 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:49.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:49.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:49.828 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:49.858 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:49.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:49.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:49.962 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:49.987 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:49.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:49.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:50.090 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:50.110 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:07:50.111 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:07:50.111 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:50.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:50.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:50.213 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:50.240 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:50.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:50.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:50.344 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:50.366 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:50.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:50.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:50.373 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:50 smithi098 bash[76817]: cluster 2023-09-23T19:07:48.630177+0000 mgr.y (mgr.26054) 1505 : cluster 0 pgmap v1656: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 12 active+undersized+degraded, 22 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 5.7 KiB/s rd, 1.7 KiB/s wr, 8 op/s; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:50.373 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:50 smithi098 bash[76817]: cluster 2023-09-23T19:07:49.120059+0000 mon.a (mon.0) 2031 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:07:50.374 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:50 smithi098 bash[76817]: audit 2023-09-23T19:07:49.132716+0000 mon.a (mon.0) 2032 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:07:50.374 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:50 smithi098 bash[76817]: cluster 2023-09-23T19:07:49.132794+0000 mon.a (mon.0) 2033 : cluster 0 osdmap e354: 8 total, 6 up, 7 in 2023-09-23T19:07:50.374 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:50 smithi098 bash[76817]: cluster 2023-09-23T19:07:49.134595+0000 osd.4 (osd.4) 85 : cluster 3 failed to encode map e354 with expected crc 2023-09-23T19:07:50.374 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:50 smithi098 bash[76817]: cluster 2023-09-23T19:07:49.770573+0000 osd.6 (osd.6) 341 : cluster 3 failed to encode map e354 with expected crc 2023-09-23T19:07:50.374 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:50 smithi098 bash[79516]: cluster 2023-09-23T19:07:48.630177+0000 mgr.y (mgr.26054) 1505 : cluster 0 pgmap v1656: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 12 active+undersized+degraded, 22 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 5.7 KiB/s rd, 1.7 KiB/s wr, 8 op/s; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:50.374 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:50 smithi098 bash[79516]: cluster 2023-09-23T19:07:49.120059+0000 mon.a (mon.0) 2031 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:07:50.374 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:50 smithi098 bash[79516]: audit 2023-09-23T19:07:49.132716+0000 mon.a (mon.0) 2032 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:07:50.374 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:50 smithi098 bash[79516]: cluster 2023-09-23T19:07:49.132794+0000 mon.a (mon.0) 2033 : cluster 0 osdmap e354: 8 total, 6 up, 7 in 2023-09-23T19:07:50.374 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:50 smithi098 bash[79516]: cluster 2023-09-23T19:07:49.134595+0000 osd.4 (osd.4) 85 : cluster 3 failed to encode map e354 with expected crc 2023-09-23T19:07:50.375 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:50 smithi098 bash[79516]: cluster 2023-09-23T19:07:49.770573+0000 osd.6 (osd.6) 341 : cluster 3 failed to encode map e354 with expected crc 2023-09-23T19:07:50.469 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:50.489 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:50.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:50.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:50.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:50 smithi173 bash[59422]: cluster 2023-09-23T19:07:48.630177+0000 mgr.y (mgr.26054) 1505 : cluster 0 pgmap v1656: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 12 active+undersized+degraded, 22 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 5.7 KiB/s rd, 1.7 KiB/s wr, 8 op/s; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:50.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:50 smithi173 bash[59422]: cluster 2023-09-23T19:07:49.120059+0000 mon.a (mon.0) 2031 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:07:50.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:50 smithi173 bash[59422]: audit 2023-09-23T19:07:49.132716+0000 mon.a (mon.0) 2032 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:07:50.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:50 smithi173 bash[59422]: cluster 2023-09-23T19:07:49.132794+0000 mon.a (mon.0) 2033 : cluster 0 osdmap e354: 8 total, 6 up, 7 in 2023-09-23T19:07:50.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:50 smithi173 bash[59422]: cluster 2023-09-23T19:07:49.134595+0000 osd.4 (osd.4) 85 : cluster 3 failed to encode map e354 with expected crc 2023-09-23T19:07:50.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:50 smithi173 bash[59422]: cluster 2023-09-23T19:07:49.770573+0000 osd.6 (osd.6) 341 : cluster 3 failed to encode map e354 with expected crc 2023-09-23T19:07:50.594 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:50.617 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:50.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:50.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:50.720 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:50.747 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:50.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:50.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:50.815 INFO:teuthology.orchestra.run.smithi098.stdout:moved 0 / 315 (0%) 2023-09-23T19:07:50.816 INFO:teuthology.orchestra.run.smithi098.stdout:avg 63 2023-09-23T19:07:50.816 INFO:teuthology.orchestra.run.smithi098.stdout:stddev 16.0873 -> 13.784 (expected baseline 7.0993) 2023-09-23T19:07:50.816 INFO:teuthology.orchestra.run.smithi098.stdout:min osd.2 with 41 -> 43 pgs (0.650794 -> 0.68254 * mean) 2023-09-23T19:07:50.816 INFO:teuthology.orchestra.run.smithi098.stdout:max osd.1 with 52 -> 54 pgs (0.825397 -> 0.857143 * mean) 2023-09-23T19:07:50.816 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T19:07:50.816 INFO:teuthology.orchestra.run.smithi098.stdout:oload 110 2023-09-23T19:07:50.816 INFO:teuthology.orchestra.run.smithi098.stdout:max_change 1 2023-09-23T19:07:50.816 INFO:teuthology.orchestra.run.smithi098.stdout:max_change_osds 4 2023-09-23T19:07:50.816 INFO:teuthology.orchestra.run.smithi098.stdout:average_utilization 0.0121 2023-09-23T19:07:50.816 INFO:teuthology.orchestra.run.smithi098.stdout:overload_utilization 0.0133 2023-09-23T19:07:50.817 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5 weight 0.9098 -> 0.5016 2023-09-23T19:07:50.819 INFO:teuthology.orchestra.run.smithi098.stderr:no change 2023-09-23T19:07:50.851 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:50.882 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:50.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:50.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:50.986 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:51.019 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:51.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:51.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:51.122 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:51.135 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:07:50 smithi098 bash[73085]: debug 2023-09-23T19:07:50.807+0000 7f90a56ad700 -1 mgr.server reply reply (1) Operation not permitted no change 2023-09-23T19:07:51.149 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:51.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:51.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:51.252 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:51.282 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:51.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:51.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:51.385 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:51.412 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:51.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:51.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:51.515 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:51.547 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:51.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:51.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:51.651 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:51.659 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:51 smithi098 bash[79516]: cluster 2023-09-23T19:07:50.630946+0000 mgr.y (mgr.26054) 1506 : cluster 0 pgmap v1658: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 12 active+undersized+degraded, 22 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:51.659 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:51 smithi098 bash[79516]: audit 2023-09-23T19:07:50.811327+0000 mgr.y (mgr.26054) 1507 : audit 0 from='client.36021 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:07:51.659 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:51 smithi098 bash[76817]: cluster 2023-09-23T19:07:50.630946+0000 mgr.y (mgr.26054) 1506 : cluster 0 pgmap v1658: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 12 active+undersized+degraded, 22 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:51.659 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:51 smithi098 bash[76817]: audit 2023-09-23T19:07:50.811327+0000 mgr.y (mgr.26054) 1507 : audit 0 from='client.36021 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:07:51.676 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:51.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:51.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:51.767 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T19:07:51.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:51 smithi173 bash[59422]: cluster 2023-09-23T19:07:50.630946+0000 mgr.y (mgr.26054) 1506 : cluster 0 pgmap v1658: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 12 active+undersized+degraded, 22 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:51.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:51 smithi173 bash[59422]: audit 2023-09-23T19:07:50.811327+0000 mgr.y (mgr.26054) 1507 : audit 0 from='client.36021 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:07:51.779 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:51.861 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:51.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:51.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:51.966 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:51.991 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:51.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:51.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:52.094 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:52.111 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:07:52.123 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:52.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:52.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:52.226 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:52.257 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:52.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:52.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:52.361 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:52.392 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:52.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:52.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:52.495 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:52.523 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:52.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:52.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:52.626 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:52.654 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:52.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:52.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:52.698 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T19:07:52.698 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 1.0 2023-09-23T19:07:52.758 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:52.823 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:52.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:52.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:52.926 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:52.936 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:52 smithi098 bash[79516]: cluster 2023-09-23T19:07:52.685190+0000 mon.a (mon.0) 2034 : cluster 3 Health check update: Degraded data redundancy: 15/126 objects degraded (11.905%), 13 pgs degraded, 39 pgs undersized (PG_DEGRADED) 2023-09-23T19:07:52.936 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:52 smithi098 bash[76817]: cluster 2023-09-23T19:07:52.685190+0000 mon.a (mon.0) 2034 : cluster 3 Health check update: Degraded data redundancy: 15/126 objects degraded (11.905%), 13 pgs degraded, 39 pgs undersized (PG_DEGRADED) 2023-09-23T19:07:52.950 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:52.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:52.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:53.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:52 smithi173 bash[59422]: cluster 2023-09-23T19:07:52.685190+0000 mon.a (mon.0) 2034 : cluster 3 Health check update: Degraded data redundancy: 15/126 objects degraded (11.905%), 13 pgs degraded, 39 pgs undersized (PG_DEGRADED) 2023-09-23T19:07:53.053 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:53.087 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:53.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:53.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:53.192 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:53.224 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:53.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:53.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:53.328 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:53.362 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:53.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:53.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:53.466 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:53.500 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:53.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:53.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:53.604 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:53.631 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:53.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:53.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:53.734 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:53.767 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:53.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:53.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:53.792 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:07:53.871 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:53.902 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:53.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:53.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:54.006 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:54.020 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:53 smithi173 bash[59422]: cluster 2023-09-23T19:07:52.633314+0000 mgr.y (mgr.26054) 1508 : cluster 0 pgmap v1659: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:54.039 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:54.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:54.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:54.142 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:54.155 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:53 smithi098 bash[76817]: cluster 2023-09-23T19:07:52.633314+0000 mgr.y (mgr.26054) 1508 : cluster 0 pgmap v1659: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:54.155 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:53 smithi098 bash[79516]: cluster 2023-09-23T19:07:52.633314+0000 mgr.y (mgr.26054) 1508 : cluster 0 pgmap v1659: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:54.173 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:54.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:54.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:54.277 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:54.310 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:54.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:54.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:54.414 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:54.446 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:54.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:54.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:54.550 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:54.585 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:54.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:54.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:54.689 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:54.719 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:54.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:54.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:54.823 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:54.854 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:54.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:54.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:54.958 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:54.987 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:54.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:54.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:55.091 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:55.122 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:55.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:55.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:55.226 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:55.257 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:55.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:55.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:55.361 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:55.392 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:55.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:55.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:55.497 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:55.529 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:55.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:55.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:55.632 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:55.664 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:55.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:55.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:55.768 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:55.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:55 smithi173 bash[59422]: cluster 2023-09-23T19:07:54.634512+0000 mgr.y (mgr.26054) 1509 : cluster 0 pgmap v1660: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:55.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:55 smithi173 bash[59422]: audit 2023-09-23T19:07:54.656329+0000 mon.a (mon.0) 2035 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:55.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:55 smithi173 bash[59422]: audit 2023-09-23T19:07:54.657886+0000 mon.a (mon.0) 2036 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:07:55.803 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:55.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:55.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:55.907 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:55.939 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:55.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:55.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:55.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:55 smithi098 bash[76817]: cluster 2023-09-23T19:07:54.634512+0000 mgr.y (mgr.26054) 1509 : cluster 0 pgmap v1660: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:55.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:55 smithi098 bash[76817]: audit 2023-09-23T19:07:54.656329+0000 mon.a (mon.0) 2035 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:55.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:55 smithi098 bash[76817]: audit 2023-09-23T19:07:54.657886+0000 mon.a (mon.0) 2036 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:07:55.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:55 smithi098 bash[79516]: cluster 2023-09-23T19:07:54.634512+0000 mgr.y (mgr.26054) 1509 : cluster 0 pgmap v1660: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:55.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:55 smithi098 bash[79516]: audit 2023-09-23T19:07:54.656329+0000 mon.a (mon.0) 2035 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:07:55.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:55 smithi098 bash[79516]: audit 2023-09-23T19:07:54.657886+0000 mon.a (mon.0) 2036 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:07:56.042 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:56.071 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:56.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:56.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:56.175 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:56.215 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:56.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:56.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:56.320 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:56.330 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:07:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:07:56] "GET /metrics HTTP/1.1" 200 36593 "" "Prometheus/2.33.4" 2023-09-23T19:07:56.350 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:56.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:56.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:56.454 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:56.486 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:56.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:56.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:56.531 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:07:56.531 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:07:56.533 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:07:56.591 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:56.622 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:56.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:56.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:56.725 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:56.756 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:56.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:56.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:56.859 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:56.889 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:56.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:56.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:56.991 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:57.014 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:57.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:57.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:57.117 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:57.141 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:57.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:57.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:57.243 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:57.271 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:57.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:57.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:57.374 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:57.402 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:57.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:57.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:57.505 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:57.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:57.538 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:57.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:57.639 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:57.666 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:07:57.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:57.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:07:57.770 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:57.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:57 smithi173 bash[59422]: cluster 2023-09-23T19:07:56.635965+0000 mgr.y (mgr.26054) 1510 : cluster 0 pgmap v1661: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:57.803 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:57.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:57.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:57.907 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:57.921 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:57 smithi098 bash[76817]: cluster 2023-09-23T19:07:56.635965+0000 mgr.y (mgr.26054) 1510 : cluster 0 pgmap v1661: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:57.921 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:57 smithi098 bash[79516]: cluster 2023-09-23T19:07:56.635965+0000 mgr.y (mgr.26054) 1510 : cluster 0 pgmap v1661: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:57.939 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:57.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:57.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:58.043 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:58.069 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:58.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:58.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:58.173 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:58.209 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:58.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:58.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:58.313 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:58.348 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:58.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:58.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:58.452 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:58.481 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:58.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:58.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:58.556 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:07:58.586 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:58.620 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:58.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:58.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:58.724 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:58.758 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:58.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:58.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:58.862 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:58.895 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:58.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:58.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:07:58.999 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:59.024 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:59.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:59.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:59.127 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:59.160 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:59.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:59.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:59.264 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:59.297 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:59.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:59.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:59.400 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:59.422 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:07:59.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:59.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:07:59.525 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:59.549 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:07:59.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:59.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:07:59.652 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:59.686 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:07:59.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:59.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:07:59.789 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:59.816 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:07:59.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:59.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:07:59.919 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:59.932 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:07:59 smithi173 bash[59422]: cluster 2023-09-23T19:07:58.637932+0000 mgr.y (mgr.26054) 1511 : cluster 0 pgmap v1662: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:59.943 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:07:59 smithi098 bash[79516]: cluster 2023-09-23T19:07:58.637932+0000 mgr.y (mgr.26054) 1511 : cluster 0 pgmap v1662: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:59.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:07:59 smithi098 bash[76817]: cluster 2023-09-23T19:07:58.637932+0000 mgr.y (mgr.26054) 1511 : cluster 0 pgmap v1662: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:07:59.951 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:07:59.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:07:59.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:00.056 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:00.077 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:00.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:00.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:00.180 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:00.201 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:00.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:00.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:00.304 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:00.332 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:00.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:00.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:00.436 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:00.458 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:00.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:00.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:00.562 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:00.592 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:00.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:00.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:00.696 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:00.731 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:00.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:00.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:00.836 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:00.866 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:00.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:00.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:00.970 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:01.003 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:01.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:01.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:01.106 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:01.131 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:01.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:01.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:01.234 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:01.260 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:01.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:01.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:01.364 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:01.391 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:01.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:01.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:01.495 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:01.526 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:01.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:01.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:01.630 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:01.641 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:01 smithi098 bash[76817]: cluster 2023-09-23T19:08:00.638812+0000 mgr.y (mgr.26054) 1512 : cluster 0 pgmap v1663: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:08:01.641 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:01 smithi098 bash[79516]: cluster 2023-09-23T19:08:00.638812+0000 mgr.y (mgr.26054) 1512 : cluster 0 pgmap v1663: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:08:01.657 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:01.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:01.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:01.760 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:01.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:01 smithi173 bash[59422]: cluster 2023-09-23T19:08:00.638812+0000 mgr.y (mgr.26054) 1512 : cluster 0 pgmap v1663: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:08:01.788 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:01.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:01.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:01.892 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:01.928 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:01.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:01.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:02.032 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:02.056 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:02.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:02.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:02.159 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:02.187 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:02.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:02.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:02.290 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:02.321 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:02.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:02.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:02.425 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:02.451 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:02.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:02.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:02.554 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:02.585 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:02.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:02.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:02.689 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:02.718 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:02.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:02.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:02.821 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:02.847 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:02.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:02.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:02.951 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:02.977 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:02.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:02.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:03.081 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:03.114 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:03.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:03.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:03.218 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:03.254 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:03.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:03.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:03.358 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:03.389 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:03.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:03.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:03.493 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:03.516 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:03.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:03.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:03.619 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:03.652 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:03.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:03.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:03.755 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:03.789 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:03.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:03.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:03.894 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:03.922 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:03.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:03.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:03.964 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:03 smithi098 bash[76817]: cluster 2023-09-23T19:08:02.639869+0000 mgr.y (mgr.26054) 1513 : cluster 0 pgmap v1664: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:08:03.964 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:03 smithi098 bash[79516]: cluster 2023-09-23T19:08:02.639869+0000 mgr.y (mgr.26054) 1513 : cluster 0 pgmap v1664: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:08:04.027 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:04.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:03 smithi173 bash[59422]: cluster 2023-09-23T19:08:02.639869+0000 mgr.y (mgr.26054) 1513 : cluster 0 pgmap v1664: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:08:04.058 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:04.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:04.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:04.161 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:04.183 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:04.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:04.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:04.286 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:04.308 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:04.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:04.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:04.411 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:04.444 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:04.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:04.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:04.548 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:04.574 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:04.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:04.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:04.677 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:04.714 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:04.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:04.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:04.817 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:04.838 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:04.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:04.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:04.941 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:04.963 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:04.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:04.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:05.066 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:05.088 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:05.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:05.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:05.191 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:05.228 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:05.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:05.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:05.332 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:05.363 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:05.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:05.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:05.466 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:05.495 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:05.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:05.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:05.599 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:05.633 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:05.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:05.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:05.737 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:05.758 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:05.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:05.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:05.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:05 smithi173 bash[59422]: cluster 2023-09-23T19:08:04.641230+0000 mgr.y (mgr.26054) 1514 : cluster 0 pgmap v1665: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:08:05.862 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:05.887 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:05.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:05.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:05.990 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:06.003 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:05 smithi098 bash[79516]: cluster 2023-09-23T19:08:04.641230+0000 mgr.y (mgr.26054) 1514 : cluster 0 pgmap v1665: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:08:06.003 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:05 smithi098 bash[76817]: cluster 2023-09-23T19:08:04.641230+0000 mgr.y (mgr.26054) 1514 : cluster 0 pgmap v1665: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:08:06.020 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:06.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:06.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:06.123 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:06.146 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:06.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:06.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:06.250 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:06.263 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:08:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:08:06] "GET /metrics HTTP/1.1" 200 36593 "" "Prometheus/2.33.4" 2023-09-23T19:08:06.283 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:06.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:06.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:06.387 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:06.419 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:06.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:06.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:06.522 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:06.549 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:06.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:06.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:06.653 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:06.675 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:06.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:06.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:06.778 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:06.804 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:06.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:06.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:06.907 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:06.930 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:06.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:06.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:07.033 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:07.065 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:07.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:07.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:07.168 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:07.194 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:07.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:07.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:07.298 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:07.329 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:07.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:07.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:07.433 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:07.454 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:07.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:07.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:07.557 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:07.592 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:07.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:07.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:07.696 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:07.731 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:07.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:07.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:07.834 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:07.863 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:07.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:07.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:07.967 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:07.986 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:07.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:07.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:08.020 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T19:08:08.020 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [], 2023-09-23T19:08:08.020 INFO:teuthology.orchestra.run.smithi098.stdout: "num_ops": 0 2023-09-23T19:08:08.020 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T19:08:08.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:07 smithi173 bash[59422]: cluster 2023-09-23T19:08:06.642165+0000 mgr.y (mgr.26054) 1515 : cluster 0 pgmap v1666: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:08:08.090 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:08.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:07 smithi098 bash[79516]: cluster 2023-09-23T19:08:06.642165+0000 mgr.y (mgr.26054) 1515 : cluster 0 pgmap v1666: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:08:08.099 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:07 smithi098 bash[76817]: cluster 2023-09-23T19:08:06.642165+0000 mgr.y (mgr.26054) 1515 : cluster 0 pgmap v1666: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:08:08.122 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:08.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:08.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:08.226 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:08.252 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:08.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:08.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:08.355 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:08.385 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:08.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:08.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:08.488 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:08.515 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:08.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:08.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:08.618 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:08.650 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:08.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:08.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:08.753 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:08.785 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:08.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:08.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:08.889 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:08.913 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:08.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:08.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:09.016 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:09.052 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:09.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:09.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:09.157 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:09.191 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:09.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:09.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:09.294 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:09.324 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:09.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:09.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:09.427 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:09.453 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:09.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:09.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:09.556 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:09.581 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:09.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:09.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:09.684 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:09.710 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:09.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:09.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:09.814 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:09.845 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:09.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:09.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:09.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:09 smithi098 bash[76817]: cluster 2023-09-23T19:08:08.643302+0000 mgr.y (mgr.26054) 1516 : cluster 0 pgmap v1667: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:08:09.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:09 smithi098 bash[76817]: audit 2023-09-23T19:08:09.650181+0000 mon.a (mon.0) 2037 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:08:09.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:09 smithi098 bash[79516]: cluster 2023-09-23T19:08:08.643302+0000 mgr.y (mgr.26054) 1516 : cluster 0 pgmap v1667: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:08:09.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:09 smithi098 bash[79516]: audit 2023-09-23T19:08:09.650181+0000 mon.a (mon.0) 2037 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:08:09.948 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:09.981 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:09.983 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:09 smithi173 bash[59422]: cluster 2023-09-23T19:08:08.643302+0000 mgr.y (mgr.26054) 1516 : cluster 0 pgmap v1667: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:08:09.983 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:09 smithi173 bash[59422]: audit 2023-09-23T19:08:09.650181+0000 mon.a (mon.0) 2037 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:08:09.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:09.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:10.085 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:10.111 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:10.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:10.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:10.214 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:10.243 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:10.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:10.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:10.347 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:10.388 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:10.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:10.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:10.492 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:10.517 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:10.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:10.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:10.619 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:10.649 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:10.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:10.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:10.753 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:10.780 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:10.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:10.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:10.883 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:10.907 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:10.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:10.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:11.010 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:11.044 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:11.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:11.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:11.147 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:11.177 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:11.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:11.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:11.279 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:11.306 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:11.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:11.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:11.410 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:11.442 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:11.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:11.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:11.538 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T19:08:11.546 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:11.608 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:11.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:11.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:11.712 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:11.725 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:11 smithi173 bash[59422]: cluster 2023-09-23T19:08:10.644113+0000 mgr.y (mgr.26054) 1517 : cluster 0 pgmap v1668: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:08:11.744 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:11.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:11.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:11.848 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:11.880 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:11.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:11.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:11.895 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T19:08:11.920 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T19:08:11.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:11 smithi098 bash[76817]: cluster 2023-09-23T19:08:10.644113+0000 mgr.y (mgr.26054) 1517 : cluster 0 pgmap v1668: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:08:11.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:11 smithi098 bash[79516]: cluster 2023-09-23T19:08:10.644113+0000 mgr.y (mgr.26054) 1517 : cluster 0 pgmap v1668: 105 pgs: 2 undersized+peered, 27 active+clean+remapped, 4 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 23/126 objects misplaced (18.254%) 2023-09-23T19:08:11.946 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T19:08:11.972 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T19:08:11.984 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:11.997 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T19:08:12.015 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:12.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:12.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:12.025 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T19:08:12.025 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:08:12.025 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:08:12.025 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:08:12.025 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:08:12.026 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:08:12.026 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T19:08:12.026 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:08:12.026 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T19:08:12.119 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:12.143 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:12.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:12.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:12.246 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:12.273 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:12.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:12.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:12.377 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:12.411 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:12.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:12.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:12.514 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:12.547 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:12.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:12.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:12.614 INFO:teuthology.orchestra.run.smithi098.stdout:moved 0 / 315 (0%) 2023-09-23T19:08:12.614 INFO:teuthology.orchestra.run.smithi098.stdout:avg 63 2023-09-23T19:08:12.614 INFO:teuthology.orchestra.run.smithi098.stdout:stddev 16.0873 -> 13.784 (expected baseline 7.0993) 2023-09-23T19:08:12.614 INFO:teuthology.orchestra.run.smithi098.stdout:min osd.2 with 41 -> 43 pgs (0.650794 -> 0.68254 * mean) 2023-09-23T19:08:12.614 INFO:teuthology.orchestra.run.smithi098.stdout:max osd.1 with 52 -> 54 pgs (0.825397 -> 0.857143 * mean) 2023-09-23T19:08:12.614 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T19:08:12.614 INFO:teuthology.orchestra.run.smithi098.stdout:oload 110 2023-09-23T19:08:12.615 INFO:teuthology.orchestra.run.smithi098.stdout:max_change 1 2023-09-23T19:08:12.615 INFO:teuthology.orchestra.run.smithi098.stdout:max_change_osds 4 2023-09-23T19:08:12.615 INFO:teuthology.orchestra.run.smithi098.stdout:average_utilization 0.0121 2023-09-23T19:08:12.615 INFO:teuthology.orchestra.run.smithi098.stdout:overload_utilization 0.0133 2023-09-23T19:08:12.615 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5 weight 0.9098 -> 0.5016 2023-09-23T19:08:12.628 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is unset 2023-09-23T19:08:12.650 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:12.682 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:12.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:12.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:12.787 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:12.795 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:12 smithi098 bash[76817]: audit 2023-09-23T19:08:11.817555+0000 mgr.y (mgr.26054) 1518 : audit 0 from='client.46010 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:08:12.795 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:12 smithi098 bash[76817]: audit 2023-09-23T19:08:11.819175+0000 mon.a (mon.0) 2038 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'5':'32871'}"}]: dispatch 2023-09-23T19:08:12.795 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:12 smithi098 bash[76817]: audit 2023-09-23T19:08:11.863844+0000 mon.a (mon.0) 2039 : audit 0 from='client.? 172.21.15.98:0/1754219423' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:08:12.795 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:12 smithi098 bash[76817]: audit 2023-09-23T19:08:12.114367+0000 mon.c (mon.2) 152 : audit 1 from='client.? 172.21.15.98:0/964739237' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:08:12.795 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:12 smithi098 bash[76817]: audit 2023-09-23T19:08:12.115047+0000 mon.a (mon.0) 2040 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:08:12.796 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:12 smithi098 bash[79516]: audit 2023-09-23T19:08:11.817555+0000 mgr.y (mgr.26054) 1518 : audit 0 from='client.46010 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:08:12.796 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:12 smithi098 bash[79516]: audit 2023-09-23T19:08:11.819175+0000 mon.a (mon.0) 2038 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'5':'32871'}"}]: dispatch 2023-09-23T19:08:12.796 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:12 smithi098 bash[79516]: audit 2023-09-23T19:08:11.863844+0000 mon.a (mon.0) 2039 : audit 0 from='client.? 172.21.15.98:0/1754219423' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:08:12.796 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:12 smithi098 bash[79516]: audit 2023-09-23T19:08:12.114367+0000 mon.c (mon.2) 152 : audit 1 from='client.? 172.21.15.98:0/964739237' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:08:12.796 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:12 smithi098 bash[79516]: audit 2023-09-23T19:08:12.115047+0000 mon.a (mon.0) 2040 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:08:12.811 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:12.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:12.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:12.916 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:12.929 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:12 smithi173 bash[59422]: audit 2023-09-23T19:08:11.817555+0000 mgr.y (mgr.26054) 1518 : audit 0 from='client.46010 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T19:08:12.929 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:12 smithi173 bash[59422]: audit 2023-09-23T19:08:11.819175+0000 mon.a (mon.0) 2038 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'5':'32871'}"}]: dispatch 2023-09-23T19:08:12.929 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:12 smithi173 bash[59422]: audit 2023-09-23T19:08:11.863844+0000 mon.a (mon.0) 2039 : audit 0 from='client.? 172.21.15.98:0/1754219423' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:08:12.929 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:12 smithi173 bash[59422]: audit 2023-09-23T19:08:12.114367+0000 mon.c (mon.2) 152 : audit 1 from='client.? 172.21.15.98:0/964739237' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:08:12.929 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:12 smithi173 bash[59422]: audit 2023-09-23T19:08:12.115047+0000 mon.a (mon.0) 2040 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:08:12.947 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:12.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:12.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:13.050 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:13.078 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:13.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:13.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:13.181 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:13.205 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:13.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:13.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:13.309 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:13.331 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:13.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:13.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:13.434 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:13.466 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:13.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:13.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:13.569 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:13.598 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:13.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:13.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:13.703 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:13.708 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:08:13.708 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:08:13.728 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:13.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:13.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:13.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:13 smithi173 bash[59422]: cluster 2023-09-23T19:08:12.590878+0000 mon.a (mon.0) 2041 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:08:13.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:13 smithi173 bash[59422]: audit 2023-09-23T19:08:12.607450+0000 mon.a (mon.0) 2042 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'5':'32871'}"}]': finished 2023-09-23T19:08:13.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:13 smithi173 bash[59422]: audit 2023-09-23T19:08:12.607540+0000 mon.a (mon.0) 2043 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:08:13.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:13 smithi173 bash[59422]: cluster 2023-09-23T19:08:12.607609+0000 mon.a (mon.0) 2044 : cluster 0 osdmap e355: 8 total, 6 up, 7 in 2023-09-23T19:08:13.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:13 smithi173 bash[59422]: cluster 2023-09-23T19:08:12.608794+0000 osd.4 (osd.4) 86 : cluster 3 failed to encode map e355 with expected crc 2023-09-23T19:08:13.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:13 smithi173 bash[59422]: cluster 2023-09-23T19:08:12.612004+0000 osd.6 (osd.6) 342 : cluster 3 failed to encode map e355 with expected crc 2023-09-23T19:08:13.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:13 smithi173 bash[59422]: cluster 2023-09-23T19:08:12.645020+0000 mgr.y (mgr.26054) 1519 : cluster 0 pgmap v1670: 105 pgs: 3 peering, 1 remapped+peering, 2 undersized+peered, 24 active+clean+remapped, 3 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 17/126 objects misplaced (13.492%) 2023-09-23T19:08:13.830 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:13.846 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:13 smithi098 bash[79516]: cluster 2023-09-23T19:08:12.590878+0000 mon.a (mon.0) 2041 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:08:13.846 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:13 smithi098 bash[79516]: audit 2023-09-23T19:08:12.607450+0000 mon.a (mon.0) 2042 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'5':'32871'}"}]': finished 2023-09-23T19:08:13.846 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:13 smithi098 bash[79516]: audit 2023-09-23T19:08:12.607540+0000 mon.a (mon.0) 2043 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:08:13.846 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:13 smithi098 bash[79516]: cluster 2023-09-23T19:08:12.607609+0000 mon.a (mon.0) 2044 : cluster 0 osdmap e355: 8 total, 6 up, 7 in 2023-09-23T19:08:13.846 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:13 smithi098 bash[79516]: cluster 2023-09-23T19:08:12.608794+0000 osd.4 (osd.4) 86 : cluster 3 failed to encode map e355 with expected crc 2023-09-23T19:08:13.846 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:13 smithi098 bash[79516]: cluster 2023-09-23T19:08:12.612004+0000 osd.6 (osd.6) 342 : cluster 3 failed to encode map e355 with expected crc 2023-09-23T19:08:13.846 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:13 smithi098 bash[79516]: cluster 2023-09-23T19:08:12.645020+0000 mgr.y (mgr.26054) 1519 : cluster 0 pgmap v1670: 105 pgs: 3 peering, 1 remapped+peering, 2 undersized+peered, 24 active+clean+remapped, 3 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 17/126 objects misplaced (13.492%) 2023-09-23T19:08:13.847 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:13 smithi098 bash[76817]: cluster 2023-09-23T19:08:12.590878+0000 mon.a (mon.0) 2041 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:08:13.847 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:13 smithi098 bash[76817]: audit 2023-09-23T19:08:12.607450+0000 mon.a (mon.0) 2042 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'5':'32871'}"}]': finished 2023-09-23T19:08:13.847 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:13 smithi098 bash[76817]: audit 2023-09-23T19:08:12.607540+0000 mon.a (mon.0) 2043 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:08:13.847 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:13 smithi098 bash[76817]: cluster 2023-09-23T19:08:12.607609+0000 mon.a (mon.0) 2044 : cluster 0 osdmap e355: 8 total, 6 up, 7 in 2023-09-23T19:08:13.847 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:13 smithi098 bash[76817]: cluster 2023-09-23T19:08:12.608794+0000 osd.4 (osd.4) 86 : cluster 3 failed to encode map e355 with expected crc 2023-09-23T19:08:13.847 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:13 smithi098 bash[76817]: cluster 2023-09-23T19:08:12.612004+0000 osd.6 (osd.6) 342 : cluster 3 failed to encode map e355 with expected crc 2023-09-23T19:08:13.847 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:13 smithi098 bash[76817]: cluster 2023-09-23T19:08:12.645020+0000 mgr.y (mgr.26054) 1519 : cluster 0 pgmap v1670: 105 pgs: 3 peering, 1 remapped+peering, 2 undersized+peered, 24 active+clean+remapped, 3 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 17/126 objects misplaced (13.492%) 2023-09-23T19:08:13.857 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:13.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:13.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:13.961 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:13.998 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:14.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:14.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:14.101 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:14.128 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:14.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:14.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:14.232 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:14.263 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:14.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:14.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:14.366 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:14.387 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:14.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:14.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:14.491 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:14.519 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:14.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:14.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:14.623 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:14.645 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:14.647 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:14 smithi098 bash[76817]: cluster 2023-09-23T19:08:13.614111+0000 mon.a (mon.0) 2045 : cluster 0 osdmap e356: 8 total, 6 up, 7 in 2023-09-23T19:08:14.647 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:14 smithi098 bash[76817]: cluster 2023-09-23T19:08:13.614623+0000 osd.4 (osd.4) 87 : cluster 3 failed to encode map e356 with expected crc 2023-09-23T19:08:14.647 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:14 smithi098 bash[76817]: cluster 2023-09-23T19:08:13.617036+0000 osd.6 (osd.6) 343 : cluster 3 failed to encode map e356 with expected crc 2023-09-23T19:08:14.647 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:14 smithi098 bash[76817]: cluster 2023-09-23T19:08:13.617610+0000 osd.4 (osd.4) 88 : cluster 3 failed to encode map e356 with expected crc 2023-09-23T19:08:14.647 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:14 smithi098 bash[76817]: cluster 2023-09-23T19:08:13.619113+0000 osd.6 (osd.6) 344 : cluster 3 failed to encode map e356 with expected crc 2023-09-23T19:08:14.647 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:14 smithi098 bash[79516]: cluster 2023-09-23T19:08:13.614111+0000 mon.a (mon.0) 2045 : cluster 0 osdmap e356: 8 total, 6 up, 7 in 2023-09-23T19:08:14.647 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:14 smithi098 bash[79516]: cluster 2023-09-23T19:08:13.614623+0000 osd.4 (osd.4) 87 : cluster 3 failed to encode map e356 with expected crc 2023-09-23T19:08:14.647 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:14 smithi098 bash[79516]: cluster 2023-09-23T19:08:13.617036+0000 osd.6 (osd.6) 343 : cluster 3 failed to encode map e356 with expected crc 2023-09-23T19:08:14.647 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:14 smithi098 bash[79516]: cluster 2023-09-23T19:08:13.617610+0000 osd.4 (osd.4) 88 : cluster 3 failed to encode map e356 with expected crc 2023-09-23T19:08:14.647 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:14 smithi098 bash[79516]: cluster 2023-09-23T19:08:13.619113+0000 osd.6 (osd.6) 344 : cluster 3 failed to encode map e356 with expected crc 2023-09-23T19:08:14.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:14.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:14.749 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:14.761 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:14 smithi173 bash[59422]: cluster 2023-09-23T19:08:13.614111+0000 mon.a (mon.0) 2045 : cluster 0 osdmap e356: 8 total, 6 up, 7 in 2023-09-23T19:08:14.761 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:14 smithi173 bash[59422]: cluster 2023-09-23T19:08:13.614623+0000 osd.4 (osd.4) 87 : cluster 3 failed to encode map e356 with expected crc 2023-09-23T19:08:14.761 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:14 smithi173 bash[59422]: cluster 2023-09-23T19:08:13.617036+0000 osd.6 (osd.6) 343 : cluster 3 failed to encode map e356 with expected crc 2023-09-23T19:08:14.761 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:14 smithi173 bash[59422]: cluster 2023-09-23T19:08:13.617610+0000 osd.4 (osd.4) 88 : cluster 3 failed to encode map e356 with expected crc 2023-09-23T19:08:14.761 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:14 smithi173 bash[59422]: cluster 2023-09-23T19:08:13.619113+0000 osd.6 (osd.6) 344 : cluster 3 failed to encode map e356 with expected crc 2023-09-23T19:08:14.780 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:14.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:14.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:14.884 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:14.916 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:14.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:14.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:15.020 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:15.052 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:15.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:15.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:15.156 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:15.192 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:15.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:15.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:15.295 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:15.326 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:15.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:15.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:15.430 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:15.459 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:15.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:15.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:15.562 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:15.592 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:15.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:15.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:15.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:15 smithi098 bash[76817]: cluster 2023-09-23T19:08:14.618071+0000 osd.4 (osd.4) 89 : cluster 3 failed to encode map e357 with expected crc 2023-09-23T19:08:15.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:15 smithi098 bash[76817]: cluster 2023-09-23T19:08:14.618071+0000 osd.6 (osd.6) 345 : cluster 3 failed to encode map e357 with expected crc 2023-09-23T19:08:15.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:15 smithi098 bash[76817]: cluster 2023-09-23T19:08:14.618129+0000 mon.a (mon.0) 2046 : cluster 0 osdmap e357: 8 total, 6 up, 7 in 2023-09-23T19:08:15.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:15 smithi098 bash[76817]: cluster 2023-09-23T19:08:14.624511+0000 osd.4 (osd.4) 90 : cluster 3 failed to encode map e357 with expected crc 2023-09-23T19:08:15.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:15 smithi098 bash[76817]: cluster 2023-09-23T19:08:14.624679+0000 osd.6 (osd.6) 346 : cluster 3 failed to encode map e357 with expected crc 2023-09-23T19:08:15.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:15 smithi098 bash[76817]: cluster 2023-09-23T19:08:14.646044+0000 mgr.y (mgr.26054) 1520 : cluster 0 pgmap v1673: 105 pgs: 3 peering, 1 remapped+peering, 2 undersized+peered, 24 active+clean+remapped, 3 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 17/126 objects misplaced (13.492%) 2023-09-23T19:08:15.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:15 smithi098 bash[79516]: cluster 2023-09-23T19:08:14.618071+0000 osd.4 (osd.4) 89 : cluster 3 failed to encode map e357 with expected crc 2023-09-23T19:08:15.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:15 smithi098 bash[79516]: cluster 2023-09-23T19:08:14.618071+0000 osd.6 (osd.6) 345 : cluster 3 failed to encode map e357 with expected crc 2023-09-23T19:08:15.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:15 smithi098 bash[79516]: cluster 2023-09-23T19:08:14.618129+0000 mon.a (mon.0) 2046 : cluster 0 osdmap e357: 8 total, 6 up, 7 in 2023-09-23T19:08:15.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:15 smithi098 bash[79516]: cluster 2023-09-23T19:08:14.624511+0000 osd.4 (osd.4) 90 : cluster 3 failed to encode map e357 with expected crc 2023-09-23T19:08:15.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:15 smithi098 bash[79516]: cluster 2023-09-23T19:08:14.624679+0000 osd.6 (osd.6) 346 : cluster 3 failed to encode map e357 with expected crc 2023-09-23T19:08:15.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:15 smithi098 bash[79516]: cluster 2023-09-23T19:08:14.646044+0000 mgr.y (mgr.26054) 1520 : cluster 0 pgmap v1673: 105 pgs: 3 peering, 1 remapped+peering, 2 undersized+peered, 24 active+clean+remapped, 3 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 17/126 objects misplaced (13.492%) 2023-09-23T19:08:15.697 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:15.709 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:08:15.730 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:15.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:15.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:15.834 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:15.863 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:15.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:15.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:15.967 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:16.001 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:16.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:16.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:16.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:15 smithi173 bash[59422]: cluster 2023-09-23T19:08:14.618071+0000 osd.4 (osd.4) 89 : cluster 3 failed to encode map e357 with expected crc 2023-09-23T19:08:16.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:15 smithi173 bash[59422]: cluster 2023-09-23T19:08:14.618071+0000 osd.6 (osd.6) 345 : cluster 3 failed to encode map e357 with expected crc 2023-09-23T19:08:16.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:15 smithi173 bash[59422]: cluster 2023-09-23T19:08:14.618129+0000 mon.a (mon.0) 2046 : cluster 0 osdmap e357: 8 total, 6 up, 7 in 2023-09-23T19:08:16.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:15 smithi173 bash[59422]: cluster 2023-09-23T19:08:14.624511+0000 osd.4 (osd.4) 90 : cluster 3 failed to encode map e357 with expected crc 2023-09-23T19:08:16.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:15 smithi173 bash[59422]: cluster 2023-09-23T19:08:14.624679+0000 osd.6 (osd.6) 346 : cluster 3 failed to encode map e357 with expected crc 2023-09-23T19:08:16.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:15 smithi173 bash[59422]: cluster 2023-09-23T19:08:14.646044+0000 mgr.y (mgr.26054) 1520 : cluster 0 pgmap v1673: 105 pgs: 3 peering, 1 remapped+peering, 2 undersized+peered, 24 active+clean+remapped, 3 active+undersized+remapped, 38 active+clean, 13 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 17/126 objects misplaced (13.492%) 2023-09-23T19:08:16.105 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:16.136 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:16.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:16.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:16.240 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:16.273 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:16.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:16.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:16.377 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:16.407 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:16.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:16.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:16.443 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:08:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:08:16] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.33.4" 2023-09-23T19:08:16.510 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:16.539 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:16.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:16.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:16.644 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:16.671 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:16.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:16.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:16.775 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:16.788 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:16 smithi098 bash[79516]: cluster 2023-09-23T19:08:16.699156+0000 mon.a (mon.0) 2047 : cluster 3 Health check update: Degraded data redundancy: 15/126 objects degraded (11.905%), 13 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T19:08:16.789 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:16 smithi098 bash[76817]: cluster 2023-09-23T19:08:16.699156+0000 mon.a (mon.0) 2047 : cluster 3 Health check update: Degraded data redundancy: 15/126 objects degraded (11.905%), 13 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T19:08:16.807 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:16.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:16.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:16.911 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:16.942 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:16.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:16.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:17.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:16 smithi173 bash[59422]: cluster 2023-09-23T19:08:16.699156+0000 mon.a (mon.0) 2047 : cluster 3 Health check update: Degraded data redundancy: 15/126 objects degraded (11.905%), 13 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T19:08:17.045 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:17.080 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:17.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:17.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:17.111 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:08:17.184 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:17.221 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:17.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:17.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:17.325 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:17.356 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:17.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:17.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:17.460 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:17.493 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:17.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:17.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:17.596 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:17.631 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:17.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:17.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:17.734 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:17.769 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:17.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:17.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:17.872 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:17.884 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:17 smithi173 bash[59422]: cluster 2023-09-23T19:08:16.647418+0000 mgr.y (mgr.26054) 1521 : cluster 0 pgmap v1674: 105 pgs: 3 peering, 1 remapped+peering, 2 undersized+peered, 24 active+clean+remapped, 3 active+undersized+remapped, 39 active+clean, 13 active+undersized+degraded, 20 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 17/126 objects misplaced (13.492%) 2023-09-23T19:08:17.903 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:17.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:17.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:18.007 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:18.041 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:18.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:18.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:18.145 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:18.157 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:17 smithi098 bash[76817]: cluster 2023-09-23T19:08:16.647418+0000 mgr.y (mgr.26054) 1521 : cluster 0 pgmap v1674: 105 pgs: 3 peering, 1 remapped+peering, 2 undersized+peered, 24 active+clean+remapped, 3 active+undersized+remapped, 39 active+clean, 13 active+undersized+degraded, 20 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 17/126 objects misplaced (13.492%) 2023-09-23T19:08:18.157 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:17 smithi098 bash[79516]: cluster 2023-09-23T19:08:16.647418+0000 mgr.y (mgr.26054) 1521 : cluster 0 pgmap v1674: 105 pgs: 3 peering, 1 remapped+peering, 2 undersized+peered, 24 active+clean+remapped, 3 active+undersized+remapped, 39 active+clean, 13 active+undersized+degraded, 20 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 17/126 objects misplaced (13.492%) 2023-09-23T19:08:18.175 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:18.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:18.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:18.279 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:18.308 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:18.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:18.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:18.412 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:18.448 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:18.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:18.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:18.552 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:18.583 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:18.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:18.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:18.637 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:08:18.688 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:18.714 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:18.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:18.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:18.818 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:18.841 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:18.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:18.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:18.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:18 smithi098 bash[76817]: cluster 2023-09-23T19:08:18.755993+0000 mon.a (mon.0) 2048 : cluster 3 Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T19:08:18.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:18 smithi098 bash[79516]: cluster 2023-09-23T19:08:18.755993+0000 mon.a (mon.0) 2048 : cluster 3 Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T19:08:18.945 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:18.968 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:18.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:18.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:19.071 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:19.094 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:19.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:19.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:19.197 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:19.211 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:18 smithi173 bash[59422]: cluster 2023-09-23T19:08:18.755993+0000 mon.a (mon.0) 2048 : cluster 3 Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T19:08:19.229 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:19.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:19.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:19.333 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:19.359 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:19.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:19.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:19.462 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:19.486 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:19.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:19.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:19.590 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:19.611 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:19.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:19.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:19.654 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:08:19.715 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:19.750 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:19.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:19.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:19.853 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:19.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:19.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:19.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:19.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:19 smithi098 bash[79516]: cluster 2023-09-23T19:08:18.648940+0000 mgr.y (mgr.26054) 1522 : cluster 0 pgmap v1675: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%); 23 B/s, 0 objects/s recovering 2023-09-23T19:08:19.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:19 smithi098 bash[76817]: cluster 2023-09-23T19:08:18.648940+0000 mgr.y (mgr.26054) 1522 : cluster 0 pgmap v1675: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%); 23 B/s, 0 objects/s recovering 2023-09-23T19:08:19.990 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:20.003 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:19 smithi173 bash[59422]: cluster 2023-09-23T19:08:18.648940+0000 mgr.y (mgr.26054) 1522 : cluster 0 pgmap v1675: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%); 23 B/s, 0 objects/s recovering 2023-09-23T19:08:20.017 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:20.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:20.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:20.099 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 6, 7, 5, 3] out_osds: [4] dead_osds: [7, 5] live_osds: [0, 2, 3, 6, 1, 4] 2023-09-23T19:08:20.099 INFO:tasks.thrashosds.thrasher:Reviving osd 5 2023-09-23T19:08:20.099 INFO:tasks.cephadm.osd.5:Restarting osd.5 (starting--it wasn't running)... 2023-09-23T19:08:20.099 DEBUG:teuthology.orchestra.run.smithi173:osd.5> sudo journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.5.service 2023-09-23T19:08:20.102 DEBUG:teuthology.orchestra.run.smithi173:> sudo systemctl start ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.5 2023-09-23T19:08:20.121 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:20.165 INFO:journalctl@ceph.osd.5.smithi173.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T19:08:20.167 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T19:08:20.174 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:20.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:20.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:20.278 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:20.278 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:08:20 smithi173 systemd[1]: Started Ceph osd.5 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T19:08:20.306 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:20.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:20.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:20.410 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:20.437 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:20.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:20.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:20.540 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:20.565 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:20.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:20.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:20.668 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:20.691 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:20.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:20.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:20.794 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:20.821 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:20.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:20.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:20.924 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:20.947 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:20.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:20.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:21.050 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:21.072 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:21.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:21.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:21.176 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:21.200 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:21.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:21.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:21.304 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:21.326 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:21.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:21.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:21.429 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:21.454 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:21.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:21.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:21.557 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:21.582 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:21.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:21.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:21.681 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:08:21.686 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:21.707 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:21.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:21.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:21.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:21 smithi173 bash[59422]: cluster 2023-09-23T19:08:20.650528+0000 mgr.y (mgr.26054) 1523 : cluster 0 pgmap v1676: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%); 17 B/s, 0 objects/s recovering 2023-09-23T19:08:21.811 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:21.819 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:21 smithi098 bash[79516]: cluster 2023-09-23T19:08:20.650528+0000 mgr.y (mgr.26054) 1523 : cluster 0 pgmap v1676: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%); 17 B/s, 0 objects/s recovering 2023-09-23T19:08:21.819 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:21 smithi098 bash[76817]: cluster 2023-09-23T19:08:20.650528+0000 mgr.y (mgr.26054) 1523 : cluster 0 pgmap v1676: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%); 17 B/s, 0 objects/s recovering 2023-09-23T19:08:21.834 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:21.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:21.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:21.937 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:21.964 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:21.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:21.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:22.068 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:22.088 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:22.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:22.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:22.191 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:22.215 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:22.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:22.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:22.318 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:22.347 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:22.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:22.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:22.450 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:22.472 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:22.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:22.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:22.528 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:08:22 smithi173 bash[160191]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2023-09-23T19:08:22.528 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:08:22 smithi173 bash[160191]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2023-09-23T19:08:22.528 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:08:22 smithi173 bash[160191]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2023-09-23T19:08:22.528 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:08:22 smithi173 bash[160191]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-09-23T19:08:22.528 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:08:22 smithi173 bash[160191]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2023-09-23T19:08:22.528 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:08:22 smithi173 bash[160191]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2023-09-23T19:08:22.528 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:08:22 smithi173 bash[160191]: --> ceph-volume raw activate successful for osd ID: 5 2023-09-23T19:08:22.576 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:22.600 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:22.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:22.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:22.703 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:22.730 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:22.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:22.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:22.833 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:22.855 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:22.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:22.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:22.958 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:22.982 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:22.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:22.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:23.085 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:23.109 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:23.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:23.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:23.212 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:23.237 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:23.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:23.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:23.340 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:23.374 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:23.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:23.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:23.477 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:23.498 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:23.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:23.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:23.601 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:23.629 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:23.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:23.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:23.733 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:23.755 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:23.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:23.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:23.858 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:23.867 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:23 smithi098 bash[76817]: cluster 2023-09-23T19:08:22.651415+0000 mgr.y (mgr.26054) 1524 : cluster 0 pgmap v1677: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%); 15 B/s, 0 objects/s recovering 2023-09-23T19:08:23.867 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:23 smithi098 bash[79516]: cluster 2023-09-23T19:08:22.651415+0000 mgr.y (mgr.26054) 1524 : cluster 0 pgmap v1677: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%); 15 B/s, 0 objects/s recovering 2023-09-23T19:08:23.886 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:23.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:23.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:23.989 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:24.021 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:24.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:24.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:24.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:23 smithi173 bash[59422]: cluster 2023-09-23T19:08:22.651415+0000 mgr.y (mgr.26054) 1524 : cluster 0 pgmap v1677: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%); 15 B/s, 0 objects/s recovering 2023-09-23T19:08:24.125 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:24.150 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:24.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:24.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:24.254 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:24.281 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:24.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:24.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:24.384 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:24.412 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:24.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:24.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:24.515 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:24.547 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:24.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:24.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:24.651 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:24.684 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:24.686 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:08:24 smithi173 bash[160505]: debug 2023-09-23T19:08:24.420+0000 7f88d4c20700 1 -- 172.21.15.173:0/1582023014 <== mon.2 v2:172.21.15.98:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563542d6e4e0 con 0x563542de1400 2023-09-23T19:08:24.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:24.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:24.788 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:24.817 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:24.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:24.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:24.920 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:24.940 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:24.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:24.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:24.966 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:24 smithi173 bash[59422]: audit 2023-09-23T19:08:24.649689+0000 mon.a (mon.0) 2049 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:08:25.043 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:25.050 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:24 smithi098 bash[76817]: audit 2023-09-23T19:08:24.649689+0000 mon.a (mon.0) 2049 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:08:25.050 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:24 smithi098 bash[79516]: audit 2023-09-23T19:08:24.649689+0000 mon.a (mon.0) 2049 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:08:25.063 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:25.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:25.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:25.166 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:25.188 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:25.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:25.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:25.278 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:08:24 smithi173 bash[160505]: debug 2023-09-23T19:08:24.960+0000 7f88dc69a200 -1 Falling back to public interface 2023-09-23T19:08:25.291 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:25.317 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:25.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:25.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:25.420 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:25.445 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:25.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:25.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:25.548 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:25.580 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:25.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:25.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:25.683 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:25.710 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:25.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:25.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:25.813 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:25.835 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:25.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:25.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:25.938 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:25.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:25 smithi098 bash[76817]: cluster 2023-09-23T19:08:24.652717+0000 mgr.y (mgr.26054) 1525 : cluster 0 pgmap v1678: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%); 13 B/s, 0 objects/s recovering 2023-09-23T19:08:25.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:25 smithi098 bash[76817]: cluster 2023-09-23T19:08:25.585439+0000 mon.a (mon.0) 2050 : cluster 3 Health check update: Degraded data redundancy: 16/126 objects degraded (12.698%), 14 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:08:25.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:25 smithi098 bash[79516]: cluster 2023-09-23T19:08:24.652717+0000 mgr.y (mgr.26054) 1525 : cluster 0 pgmap v1678: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%); 13 B/s, 0 objects/s recovering 2023-09-23T19:08:25.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:25 smithi098 bash[79516]: cluster 2023-09-23T19:08:25.585439+0000 mon.a (mon.0) 2050 : cluster 3 Health check update: Degraded data redundancy: 16/126 objects degraded (12.698%), 14 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:08:25.959 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:25.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:25.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:26.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:25 smithi173 bash[59422]: cluster 2023-09-23T19:08:24.652717+0000 mgr.y (mgr.26054) 1525 : cluster 0 pgmap v1678: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%); 13 B/s, 0 objects/s recovering 2023-09-23T19:08:26.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:25 smithi173 bash[59422]: cluster 2023-09-23T19:08:25.585439+0000 mon.a (mon.0) 2050 : cluster 3 Health check update: Degraded data redundancy: 16/126 objects degraded (12.698%), 14 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T19:08:26.062 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:26.084 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:26.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:26.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:26.187 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:26.199 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:08:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:08:26] "GET /metrics HTTP/1.1" 200 36590 "" "Prometheus/2.33.4" 2023-09-23T19:08:26.212 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:26.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:26.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:26.316 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:26.351 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:26.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:26.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:26.455 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:26.488 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:26.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:26.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:26.591 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:26.621 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:26.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:26.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:26.657 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:08:26.726 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:26.756 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:26.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:26.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:26.860 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:26.889 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:26.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:26.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:26.993 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:27.023 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:27.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:27.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:27.126 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:27.155 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:27.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:27.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:27.259 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:27.292 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:27.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:27.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:27.394 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:27.422 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:27.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:27.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:27.525 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:27.552 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:27.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:27.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:27.656 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:27.678 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:27.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:27.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:27.781 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:27.811 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:27.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:27.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:27.914 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:27.926 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:27 smithi173 bash[59422]: cluster 2023-09-23T19:08:26.653696+0000 mgr.y (mgr.26054) 1526 : cluster 0 pgmap v1679: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%); 11 B/s, 0 objects/s recovering 2023-09-23T19:08:27.942 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:27.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:27.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:27.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:27 smithi098 bash[79516]: cluster 2023-09-23T19:08:26.653696+0000 mgr.y (mgr.26054) 1526 : cluster 0 pgmap v1679: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%); 11 B/s, 0 objects/s recovering 2023-09-23T19:08:27.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:27 smithi098 bash[76817]: cluster 2023-09-23T19:08:26.653696+0000 mgr.y (mgr.26054) 1526 : cluster 0 pgmap v1679: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%); 11 B/s, 0 objects/s recovering 2023-09-23T19:08:28.044 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:28.069 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:28.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:28.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:28.172 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:28.193 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:28.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:28.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:28.296 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:28.325 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:28.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:28.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:28.429 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:28.448 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:28.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:28.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:28.551 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:28.574 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:28.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:28.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:28.676 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:28.706 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:28.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:28.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:28.810 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:28.840 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:28.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:28.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:28.943 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:28.968 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:28.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:28.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:29.071 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:29.093 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:29.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:29.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:29.196 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:29.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:29.218 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:29.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:29.319 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:29.338 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:29.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:29.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:29.441 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:29.463 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:29.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:29.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:29.566 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:29.596 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:29.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:29.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:29.700 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:29.723 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:29.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:29.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:29.826 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:29.853 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:29.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:29.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:29.956 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:29.967 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:29 smithi173 bash[59422]: cluster 2023-09-23T19:08:28.655511+0000 mgr.y (mgr.26054) 1527 : cluster 0 pgmap v1680: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%); 11 B/s, 0 objects/s recovering 2023-09-23T19:08:29.979 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:29.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:29.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:30.083 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:30.095 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:29 smithi098 bash[76817]: cluster 2023-09-23T19:08:28.655511+0000 mgr.y (mgr.26054) 1527 : cluster 0 pgmap v1680: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%); 11 B/s, 0 objects/s recovering 2023-09-23T19:08:30.095 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:29 smithi098 bash[79516]: cluster 2023-09-23T19:08:28.655511+0000 mgr.y (mgr.26054) 1527 : cluster 0 pgmap v1680: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%); 11 B/s, 0 objects/s recovering 2023-09-23T19:08:30.112 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:30.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:30.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:30.215 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:30.249 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:30.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:30.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:30.353 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:30.375 INFO:teuthology.orchestra.run.smithi173.stderr:no valid command found; 10 closest matches: 2023-09-23T19:08:30.375 INFO:teuthology.orchestra.run.smithi173.stderr:0 2023-09-23T19:08:30.375 INFO:teuthology.orchestra.run.smithi173.stderr:1 2023-09-23T19:08:30.376 INFO:teuthology.orchestra.run.smithi173.stderr:2 2023-09-23T19:08:30.376 INFO:teuthology.orchestra.run.smithi173.stderr:abort 2023-09-23T19:08:30.376 INFO:teuthology.orchestra.run.smithi173.stderr:assert 2023-09-23T19:08:30.376 INFO:teuthology.orchestra.run.smithi173.stderr:bluefs debug_inject_read_zeros 2023-09-23T19:08:30.376 INFO:teuthology.orchestra.run.smithi173.stderr:bluefs files list 2023-09-23T19:08:30.376 INFO:teuthology.orchestra.run.smithi173.stderr:bluefs stats 2023-09-23T19:08:30.376 INFO:teuthology.orchestra.run.smithi173.stderr:bluestore bluefs device info [] 2023-09-23T19:08:30.376 INFO:teuthology.orchestra.run.smithi173.stderr:config diff 2023-09-23T19:08:30.376 INFO:teuthology.orchestra.run.smithi173.stderr:admin_socket: invalid command 2023-09-23T19:08:30.386 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:30.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:30.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:30.489 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:30.516 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:30.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:30.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:30.620 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:30.647 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:30.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:30.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:30.749 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:30.771 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:30.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:30.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:30.874 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:30.899 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:30.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:30.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:31.001 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:31.029 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:31.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:31.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:31.133 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:31.155 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:31.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:31.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:31.258 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:31.278 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:31.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:31.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:31.345 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T19:08:31.346 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-5, ['dump_ops_in_flight'] 2023-09-23T19:08:31.382 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:31.411 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:31.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:31.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:31.514 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:31.542 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:31.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:31.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:31.645 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:31.676 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:31.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:31.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:31.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:31 smithi098 bash[76817]: cluster 2023-09-23T19:08:30.656983+0000 mgr.y (mgr.26054) 1528 : cluster 0 pgmap v1681: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%) 2023-09-23T19:08:31.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:31 smithi098 bash[79516]: cluster 2023-09-23T19:08:30.656983+0000 mgr.y (mgr.26054) 1528 : cluster 0 pgmap v1681: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%) 2023-09-23T19:08:31.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:31 smithi173 bash[59422]: cluster 2023-09-23T19:08:30.656983+0000 mgr.y (mgr.26054) 1528 : cluster 0 pgmap v1681: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%) 2023-09-23T19:08:31.780 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:31.811 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:31.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:31.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:31.914 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:31.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:31.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:31.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:32.048 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:32.079 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:32.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:32.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:32.183 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:32.211 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:32.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:32.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:32.249 INFO:teuthology.orchestra.run.smithi098.stdout:ERROR: (22) Invalid argument 2023-09-23T19:08:32.249 INFO:teuthology.orchestra.run.smithi098.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:08:32.315 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:32.344 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:32.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:32.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:32.447 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:32.476 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:32.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:32.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:32.579 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:32.613 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:32.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:32.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:32.716 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:32.741 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:32.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:32.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:32.844 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:32.872 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:32.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:32.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:32.976 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:33.004 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:33.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:33.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:33.108 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:33.128 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:33.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:33.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:33.231 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:33.259 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:33.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:33.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:33.271 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T19:08:33.291 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T19:08:33.314 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T19:08:33.333 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T19:08:33.356 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T19:08:33.362 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:33.387 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T19:08:33.387 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:33.388 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:08:33.388 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:08:33.388 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:08:33.388 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:08:33.388 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:08:33.388 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T19:08:33.388 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:08:33.389 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T19:08:33.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:33.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:33.492 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:33.523 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:33.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:33.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:33.626 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:33.656 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:33.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:33.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:33.760 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:33.779 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:33.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:33.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:33.882 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:33.904 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:33.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:33.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:34.007 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:34.015 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:33 smithi098 bash[76817]: cluster 2023-09-23T19:08:32.658936+0000 mgr.y (mgr.26054) 1529 : cluster 0 pgmap v1682: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%) 2023-09-23T19:08:34.015 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:33 smithi098 bash[76817]: audit 2023-09-23T19:08:33.244127+0000 mon.a (mon.0) 2051 : audit 0 from='client.? 172.21.15.98:0/2236631303' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:08:34.015 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:33 smithi098 bash[79516]: cluster 2023-09-23T19:08:32.658936+0000 mgr.y (mgr.26054) 1529 : cluster 0 pgmap v1682: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%) 2023-09-23T19:08:34.016 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:33 smithi098 bash[79516]: audit 2023-09-23T19:08:33.244127+0000 mon.a (mon.0) 2051 : audit 0 from='client.? 172.21.15.98:0/2236631303' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:08:34.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:33 smithi173 bash[59422]: cluster 2023-09-23T19:08:32.658936+0000 mgr.y (mgr.26054) 1529 : cluster 0 pgmap v1682: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%) 2023-09-23T19:08:34.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:33 smithi173 bash[59422]: audit 2023-09-23T19:08:33.244127+0000 mon.a (mon.0) 2051 : audit 0 from='client.? 172.21.15.98:0/2236631303' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:08:34.032 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:34.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:34.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:34.135 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:34.159 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:34.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:34.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:34.255 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T19:08:34.262 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:34.323 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:34.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:34.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:34.427 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:34.451 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:34.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:34.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:34.554 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:34.585 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:34.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:34.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:34.689 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:34.721 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:34.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:34.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:34.736 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is unset 2023-09-23T19:08:34.826 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:34.852 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:34.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:34.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:34.955 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:34.967 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:34 smithi098 bash[79516]: audit 2023-09-23T19:08:33.945138+0000 mon.a (mon.0) 2052 : audit 1 from='client.? 172.21.15.98:0/3437660213' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:08:34.968 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:34 smithi098 bash[76817]: audit 2023-09-23T19:08:33.945138+0000 mon.a (mon.0) 2052 : audit 1 from='client.? 172.21.15.98:0/3437660213' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:08:34.983 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:34.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:34.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:35.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:34 smithi173 bash[59422]: audit 2023-09-23T19:08:33.945138+0000 mon.a (mon.0) 2052 : audit 1 from='client.? 172.21.15.98:0/3437660213' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:08:35.087 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:35.118 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:35.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:35.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:35.222 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:35.246 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:35.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:35.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:35.322 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T19:08:35.323 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T19:08:35.349 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:35.380 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:35.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:35.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:35.483 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:35.512 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:35.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:35.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:35.616 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:35.629 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:08:35 smithi173 bash[160505]: debug 2023-09-23T19:08:35.532+0000 7f88dc69a200 -1 osd.5 322 log_to_monitors {default=true} 2023-09-23T19:08:35.646 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:35.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:35.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:35.750 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:35.775 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:35.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:35.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:35.878 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:35.901 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:35.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:35.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:36.004 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:36.018 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:08:35 smithi173 bash[160505]: debug 2023-09-23T19:08:35.748+0000 7f88ccb93700 -1 osd.5 322 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T19:08:36.018 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:35 smithi173 bash[59422]: cluster 2023-09-23T19:08:34.660311+0000 mgr.y (mgr.26054) 1530 : cluster 0 pgmap v1683: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%) 2023-09-23T19:08:36.019 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:35 smithi173 bash[59422]: cluster 2023-09-23T19:08:34.714192+0000 mon.a (mon.0) 2053 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:08:36.019 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:35 smithi173 bash[59422]: audit 2023-09-23T19:08:34.726152+0000 mon.a (mon.0) 2054 : audit 1 from='client.? 172.21.15.98:0/3437660213' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:08:36.019 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:35 smithi173 bash[59422]: cluster 2023-09-23T19:08:34.726211+0000 mon.a (mon.0) 2055 : cluster 0 osdmap e358: 8 total, 6 up, 7 in 2023-09-23T19:08:36.019 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:35 smithi173 bash[59422]: cluster 2023-09-23T19:08:34.816559+0000 osd.6 (osd.6) 347 : cluster 3 failed to encode map e358 with expected crc 2023-09-23T19:08:36.019 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:35 smithi173 bash[59422]: cluster 2023-09-23T19:08:34.818686+0000 osd.6 (osd.6) 348 : cluster 3 failed to encode map e358 with expected crc 2023-09-23T19:08:36.019 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:35 smithi173 bash[59422]: cluster 2023-09-23T19:08:34.820771+0000 osd.6 (osd.6) 349 : cluster 3 failed to encode map e358 with expected crc 2023-09-23T19:08:36.019 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:35 smithi173 bash[59422]: cluster 2023-09-23T19:08:35.099497+0000 osd.4 (osd.4) 91 : cluster 3 failed to encode map e358 with expected crc 2023-09-23T19:08:36.019 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:35 smithi173 bash[59422]: audit 2023-09-23T19:08:35.538824+0000 mon.a (mon.0) 2056 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T19:08:36.019 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:35 smithi173 bash[59422]: audit 2023-09-23T19:08:35.541068+0000 mon.b (mon.1) 29 : audit 1 from='osd.5 [v2:172.21.15.173:6808/1502690846,v1:172.21.15.173:6809/1502690846]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T19:08:36.035 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:36.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:36.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:36.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:35 smithi098 bash[76817]: cluster 2023-09-23T19:08:34.660311+0000 mgr.y (mgr.26054) 1530 : cluster 0 pgmap v1683: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%) 2023-09-23T19:08:36.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:35 smithi098 bash[76817]: cluster 2023-09-23T19:08:34.714192+0000 mon.a (mon.0) 2053 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:08:36.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:35 smithi098 bash[76817]: audit 2023-09-23T19:08:34.726152+0000 mon.a (mon.0) 2054 : audit 1 from='client.? 172.21.15.98:0/3437660213' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:08:36.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:35 smithi098 bash[76817]: cluster 2023-09-23T19:08:34.726211+0000 mon.a (mon.0) 2055 : cluster 0 osdmap e358: 8 total, 6 up, 7 in 2023-09-23T19:08:36.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:35 smithi098 bash[76817]: cluster 2023-09-23T19:08:34.816559+0000 osd.6 (osd.6) 347 : cluster 3 failed to encode map e358 with expected crc 2023-09-23T19:08:36.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:35 smithi098 bash[76817]: cluster 2023-09-23T19:08:34.818686+0000 osd.6 (osd.6) 348 : cluster 3 failed to encode map e358 with expected crc 2023-09-23T19:08:36.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:35 smithi098 bash[76817]: cluster 2023-09-23T19:08:34.820771+0000 osd.6 (osd.6) 349 : cluster 3 failed to encode map e358 with expected crc 2023-09-23T19:08:36.088 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:35 smithi098 bash[76817]: cluster 2023-09-23T19:08:35.099497+0000 osd.4 (osd.4) 91 : cluster 3 failed to encode map e358 with expected crc 2023-09-23T19:08:36.088 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:35 smithi098 bash[76817]: audit 2023-09-23T19:08:35.538824+0000 mon.a (mon.0) 2056 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T19:08:36.088 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:35 smithi098 bash[76817]: audit 2023-09-23T19:08:35.541068+0000 mon.b (mon.1) 29 : audit 1 from='osd.5 [v2:172.21.15.173:6808/1502690846,v1:172.21.15.173:6809/1502690846]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T19:08:36.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:35 smithi098 bash[79516]: cluster 2023-09-23T19:08:34.660311+0000 mgr.y (mgr.26054) 1530 : cluster 0 pgmap v1683: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%) 2023-09-23T19:08:36.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:35 smithi098 bash[79516]: cluster 2023-09-23T19:08:34.714192+0000 mon.a (mon.0) 2053 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:08:36.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:35 smithi098 bash[79516]: audit 2023-09-23T19:08:34.726152+0000 mon.a (mon.0) 2054 : audit 1 from='client.? 172.21.15.98:0/3437660213' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:08:36.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:35 smithi098 bash[79516]: cluster 2023-09-23T19:08:34.726211+0000 mon.a (mon.0) 2055 : cluster 0 osdmap e358: 8 total, 6 up, 7 in 2023-09-23T19:08:36.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:35 smithi098 bash[79516]: cluster 2023-09-23T19:08:34.816559+0000 osd.6 (osd.6) 347 : cluster 3 failed to encode map e358 with expected crc 2023-09-23T19:08:36.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:35 smithi098 bash[79516]: cluster 2023-09-23T19:08:34.818686+0000 osd.6 (osd.6) 348 : cluster 3 failed to encode map e358 with expected crc 2023-09-23T19:08:36.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:35 smithi098 bash[79516]: cluster 2023-09-23T19:08:34.820771+0000 osd.6 (osd.6) 349 : cluster 3 failed to encode map e358 with expected crc 2023-09-23T19:08:36.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:35 smithi098 bash[79516]: cluster 2023-09-23T19:08:35.099497+0000 osd.4 (osd.4) 91 : cluster 3 failed to encode map e358 with expected crc 2023-09-23T19:08:36.089 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:35 smithi098 bash[79516]: audit 2023-09-23T19:08:35.538824+0000 mon.a (mon.0) 2056 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T19:08:36.089 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:35 smithi098 bash[79516]: audit 2023-09-23T19:08:35.541068+0000 mon.b (mon.1) 29 : audit 1 from='osd.5 [v2:172.21.15.173:6808/1502690846,v1:172.21.15.173:6809/1502690846]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T19:08:36.139 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:36.211 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:36.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:36.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:36.317 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:36.347 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T19:08:36.350 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:36.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:36.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:36.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:08:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:08:36] "GET /metrics HTTP/1.1" 200 36590 "" "Prometheus/2.33.4" 2023-09-23T19:08:36.452 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:36.484 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:36.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:36.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:36.588 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:36.620 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:36.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:36.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:36.724 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:36.762 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:36.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:36.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:36.868 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:36.903 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:36.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:36.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:37.006 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:37.019 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:36 smithi098 bash[79516]: audit 2023-09-23T19:08:35.734864+0000 mon.a (mon.0) 2057 : audit 1 from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T19:08:37.019 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:36 smithi098 bash[79516]: cluster 2023-09-23T19:08:35.734951+0000 mon.a (mon.0) 2058 : cluster 0 osdmap e359: 8 total, 6 up, 7 in 2023-09-23T19:08:37.019 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:36 smithi098 bash[79516]: cluster 2023-09-23T19:08:35.735431+0000 osd.4 (osd.4) 92 : cluster 3 failed to encode map e359 with expected crc 2023-09-23T19:08:37.019 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:36 smithi098 bash[79516]: cluster 2023-09-23T19:08:35.736060+0000 osd.6 (osd.6) 350 : cluster 3 failed to encode map e359 with expected crc 2023-09-23T19:08:37.019 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:36 smithi098 bash[79516]: audit 2023-09-23T19:08:35.736461+0000 mon.a (mon.0) 2059 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T19:08:37.020 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:36 smithi098 bash[79516]: audit 2023-09-23T19:08:35.738948+0000 mon.b (mon.1) 30 : audit 1 from='osd.5 [v2:172.21.15.173:6808/1502690846,v1:172.21.15.173:6809/1502690846]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T19:08:37.020 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:36 smithi098 bash[76817]: audit 2023-09-23T19:08:35.734864+0000 mon.a (mon.0) 2057 : audit 1 from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T19:08:37.020 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:36 smithi098 bash[76817]: cluster 2023-09-23T19:08:35.734951+0000 mon.a (mon.0) 2058 : cluster 0 osdmap e359: 8 total, 6 up, 7 in 2023-09-23T19:08:37.020 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:36 smithi098 bash[76817]: cluster 2023-09-23T19:08:35.735431+0000 osd.4 (osd.4) 92 : cluster 3 failed to encode map e359 with expected crc 2023-09-23T19:08:37.020 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:36 smithi098 bash[76817]: cluster 2023-09-23T19:08:35.736060+0000 osd.6 (osd.6) 350 : cluster 3 failed to encode map e359 with expected crc 2023-09-23T19:08:37.020 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:36 smithi098 bash[76817]: audit 2023-09-23T19:08:35.736461+0000 mon.a (mon.0) 2059 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T19:08:37.020 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:36 smithi098 bash[76817]: audit 2023-09-23T19:08:35.738948+0000 mon.b (mon.1) 30 : audit 1 from='osd.5 [v2:172.21.15.173:6808/1502690846,v1:172.21.15.173:6809/1502690846]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T19:08:37.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:36 smithi173 bash[59422]: audit 2023-09-23T19:08:35.734864+0000 mon.a (mon.0) 2057 : audit 1 from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T19:08:37.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:36 smithi173 bash[59422]: cluster 2023-09-23T19:08:35.734951+0000 mon.a (mon.0) 2058 : cluster 0 osdmap e359: 8 total, 6 up, 7 in 2023-09-23T19:08:37.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:36 smithi173 bash[59422]: cluster 2023-09-23T19:08:35.735431+0000 osd.4 (osd.4) 92 : cluster 3 failed to encode map e359 with expected crc 2023-09-23T19:08:37.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:36 smithi173 bash[59422]: cluster 2023-09-23T19:08:35.736060+0000 osd.6 (osd.6) 350 : cluster 3 failed to encode map e359 with expected crc 2023-09-23T19:08:37.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:36 smithi173 bash[59422]: audit 2023-09-23T19:08:35.736461+0000 mon.a (mon.0) 2059 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T19:08:37.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:36 smithi173 bash[59422]: audit 2023-09-23T19:08:35.738948+0000 mon.b (mon.1) 30 : audit 1 from='osd.5 [v2:172.21.15.173:6808/1502690846,v1:172.21.15.173:6809/1502690846]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T19:08:37.041 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:37.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:37.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:37.145 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:37.175 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:37.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:37.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:37.278 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:37.309 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:37.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:37.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:37.324 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T19:08:37.414 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:37.446 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:37.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:37.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:37.550 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:37.584 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:37.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:37.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:37.687 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:37.717 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:37.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:37.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:37.820 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.protection_status (64812 ms) 2023-09-23T19:08:37.821 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-09-23T19:08:37.821 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:37.845 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.snapshot_limits (26 ms) 2023-09-23T19:08:37.846 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.parents_v1 2023-09-23T19:08:37.852 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:37.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:37.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:37.914 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.parents_v1 (68 ms) 2023-09-23T19:08:37.914 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.parents_v2 2023-09-23T19:08:37.955 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:37.966 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:37 smithi098 bash[76817]: cluster 2023-09-23T19:08:36.661750+0000 mgr.y (mgr.26054) 1531 : cluster 0 pgmap v1686: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%) 2023-09-23T19:08:37.967 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:37 smithi098 bash[76817]: cluster 2023-09-23T19:08:36.733139+0000 mon.a (mon.0) 2060 : cluster 3 Health check update: 1 osds down (OSD_DOWN) 2023-09-23T19:08:37.967 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:37 smithi098 bash[76817]: cluster 2023-09-23T19:08:36.746786+0000 mon.a (mon.0) 2061 : cluster 1 osd.5 [v2:172.21.15.173:6808/1502690846,v1:172.21.15.173:6809/1502690846] boot 2023-09-23T19:08:37.967 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:37 smithi098 bash[76817]: cluster 2023-09-23T19:08:36.746881+0000 mon.a (mon.0) 2062 : cluster 0 osdmap e360: 8 total, 7 up, 7 in 2023-09-23T19:08:37.967 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:37 smithi098 bash[76817]: cluster 2023-09-23T19:08:36.747082+0000 osd.4 (osd.4) 93 : cluster 3 failed to encode map e360 with expected crc 2023-09-23T19:08:37.967 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:37 smithi098 bash[76817]: cluster 2023-09-23T19:08:36.747141+0000 osd.6 (osd.6) 351 : cluster 3 failed to encode map e360 with expected crc 2023-09-23T19:08:37.967 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:37 smithi098 bash[76817]: audit 2023-09-23T19:08:36.747337+0000 mon.a (mon.0) 2063 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:08:37.967 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:37 smithi098 bash[76817]: cluster 2023-09-23T19:08:36.750473+0000 osd.6 (osd.6) 352 : cluster 3 failed to encode map e360 with expected crc 2023-09-23T19:08:37.967 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:37 smithi098 bash[79516]: cluster 2023-09-23T19:08:36.661750+0000 mgr.y (mgr.26054) 1531 : cluster 0 pgmap v1686: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%) 2023-09-23T19:08:37.967 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:37 smithi098 bash[79516]: cluster 2023-09-23T19:08:36.733139+0000 mon.a (mon.0) 2060 : cluster 3 Health check update: 1 osds down (OSD_DOWN) 2023-09-23T19:08:37.967 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:37 smithi098 bash[79516]: cluster 2023-09-23T19:08:36.746786+0000 mon.a (mon.0) 2061 : cluster 1 osd.5 [v2:172.21.15.173:6808/1502690846,v1:172.21.15.173:6809/1502690846] boot 2023-09-23T19:08:37.967 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:37 smithi098 bash[79516]: cluster 2023-09-23T19:08:36.746881+0000 mon.a (mon.0) 2062 : cluster 0 osdmap e360: 8 total, 7 up, 7 in 2023-09-23T19:08:37.968 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:37 smithi098 bash[79516]: cluster 2023-09-23T19:08:36.747082+0000 osd.4 (osd.4) 93 : cluster 3 failed to encode map e360 with expected crc 2023-09-23T19:08:37.968 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:37 smithi098 bash[79516]: cluster 2023-09-23T19:08:36.747141+0000 osd.6 (osd.6) 351 : cluster 3 failed to encode map e360 with expected crc 2023-09-23T19:08:37.968 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:37 smithi098 bash[79516]: audit 2023-09-23T19:08:36.747337+0000 mon.a (mon.0) 2063 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:08:37.968 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:37 smithi098 bash[79516]: cluster 2023-09-23T19:08:36.750473+0000 osd.6 (osd.6) 352 : cluster 3 failed to encode map e360 with expected crc 2023-09-23T19:08:37.972 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.parents_v2 (59 ms) 2023-09-23T19:08:37.973 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.snapshots 2023-09-23T19:08:37.984 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:37.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:37.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:38.012 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.snapshots (39 ms) 2023-09-23T19:08:38.012 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.snapid_race 2023-09-23T19:08:38.021 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.snapid_race (10 ms) 2023-09-23T19:08:38.021 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.stripingv2 2023-09-23T19:08:38.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:37 smithi173 bash[59422]: cluster 2023-09-23T19:08:36.661750+0000 mgr.y (mgr.26054) 1531 : cluster 0 pgmap v1686: 105 pgs: 1 undersized+peered, 23 active+clean+remapped, 2 active+undersized+remapped, 44 active+clean, 14 active+undersized+degraded, 21 active+undersized; 4.6 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 16/126 objects degraded (12.698%); 16/126 objects misplaced (12.698%) 2023-09-23T19:08:38.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:37 smithi173 bash[59422]: cluster 2023-09-23T19:08:36.733139+0000 mon.a (mon.0) 2060 : cluster 3 Health check update: 1 osds down (OSD_DOWN) 2023-09-23T19:08:38.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:37 smithi173 bash[59422]: cluster 2023-09-23T19:08:36.746786+0000 mon.a (mon.0) 2061 : cluster 1 osd.5 [v2:172.21.15.173:6808/1502690846,v1:172.21.15.173:6809/1502690846] boot 2023-09-23T19:08:38.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:37 smithi173 bash[59422]: cluster 2023-09-23T19:08:36.746881+0000 mon.a (mon.0) 2062 : cluster 0 osdmap e360: 8 total, 7 up, 7 in 2023-09-23T19:08:38.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:37 smithi173 bash[59422]: cluster 2023-09-23T19:08:36.747082+0000 osd.4 (osd.4) 93 : cluster 3 failed to encode map e360 with expected crc 2023-09-23T19:08:38.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:37 smithi173 bash[59422]: cluster 2023-09-23T19:08:36.747141+0000 osd.6 (osd.6) 351 : cluster 3 failed to encode map e360 with expected crc 2023-09-23T19:08:38.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:37 smithi173 bash[59422]: audit 2023-09-23T19:08:36.747337+0000 mon.a (mon.0) 2063 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T19:08:38.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:37 smithi173 bash[59422]: cluster 2023-09-23T19:08:36.750473+0000 osd.6 (osd.6) 352 : cluster 3 failed to encode map e360 with expected crc 2023-09-23T19:08:38.036 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.stripingv2 (14 ms) 2023-09-23T19:08:38.036 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.object_map_save 2023-09-23T19:08:38.038 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.object_map_save (3 ms) 2023-09-23T19:08:38.039 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-09-23T19:08:38.047 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.object_map_resize (8 ms) 2023-09-23T19:08:38.047 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.object_map_update 2023-09-23T19:08:38.057 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.object_map_update (10 ms) 2023-09-23T19:08:38.057 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-09-23T19:08:38.057 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-09-23T19:08:38.057 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-09-23T19:08:38.068 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.object_map_snap_add (12 ms) 2023-09-23T19:08:38.069 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-09-23T19:08:38.076 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.object_map_snap_remove (7 ms) 2023-09-23T19:08:38.076 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.flags 2023-09-23T19:08:38.085 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.flags (10 ms) 2023-09-23T19:08:38.086 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.metadata 2023-09-23T19:08:38.088 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:38.121 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:38.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:38.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:38.141 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T19:08:38.156 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.metadata (70 ms) 2023-09-23T19:08:38.156 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.set_features 2023-09-23T19:08:38.167 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.set_features (11 ms) 2023-09-23T19:08:38.167 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.mirror 2023-09-23T19:08:38.219 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.mirror (52 ms) 2023-09-23T19:08:38.219 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.mirror_image 2023-09-23T19:08:38.225 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:38.242 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.mirror_image (23 ms) 2023-09-23T19:08:38.242 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-09-23T19:08:38.250 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:38.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:38.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:38.353 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:38.384 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:38.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:38.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:38.487 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:38.517 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:38.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:38.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:38.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:38.657 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:38.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:38.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:38.760 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:38.789 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:38.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:38.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:38.892 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:38.920 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:38.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:38.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:39.024 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:39.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:38 smithi173 bash[59422]: cluster 2023-09-23T19:08:36.545248+0000 osd.5 (osd.5) 1 : cluster 3 failed to encode map e323 with expected crc 2023-09-23T19:08:39.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:38 smithi173 bash[59422]: cluster 2023-09-23T19:08:36.750885+0000 osd.5 (osd.5) 2 : cluster 3 failed to encode map e360 with expected crc 2023-09-23T19:08:39.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:38 smithi173 bash[59422]: cluster 2023-09-23T19:08:37.757256+0000 osd.5 (osd.5) 3 : cluster 3 failed to encode map e361 with expected crc 2023-09-23T19:08:39.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:38 smithi173 bash[59422]: cluster 2023-09-23T19:08:37.758813+0000 osd.6 (osd.6) 353 : cluster 3 failed to encode map e361 with expected crc 2023-09-23T19:08:39.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:38 smithi173 bash[59422]: cluster 2023-09-23T19:08:37.759213+0000 mon.a (mon.0) 2064 : cluster 0 osdmap e361: 8 total, 7 up, 7 in 2023-09-23T19:08:39.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:38 smithi173 bash[59422]: cluster 2023-09-23T19:08:37.762028+0000 osd.5 (osd.5) 4 : cluster 3 failed to encode map e361 with expected crc 2023-09-23T19:08:39.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:38 smithi173 bash[59422]: cluster 2023-09-23T19:08:37.770343+0000 osd.4 (osd.4) 94 : cluster 3 failed to encode map e361 with expected crc 2023-09-23T19:08:39.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:38 smithi173 bash[59422]: cluster 2023-09-23T19:08:37.773263+0000 osd.4 (osd.4) 95 : cluster 3 failed to encode map e361 with expected crc 2023-09-23T19:08:39.034 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:38 smithi098 bash[76817]: cluster 2023-09-23T19:08:36.545248+0000 osd.5 (osd.5) 1 : cluster 3 failed to encode map e323 with expected crc 2023-09-23T19:08:39.034 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:38 smithi098 bash[76817]: cluster 2023-09-23T19:08:36.750885+0000 osd.5 (osd.5) 2 : cluster 3 failed to encode map e360 with expected crc 2023-09-23T19:08:39.034 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:38 smithi098 bash[76817]: cluster 2023-09-23T19:08:37.757256+0000 osd.5 (osd.5) 3 : cluster 3 failed to encode map e361 with expected crc 2023-09-23T19:08:39.035 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:38 smithi098 bash[76817]: cluster 2023-09-23T19:08:37.758813+0000 osd.6 (osd.6) 353 : cluster 3 failed to encode map e361 with expected crc 2023-09-23T19:08:39.035 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:38 smithi098 bash[76817]: cluster 2023-09-23T19:08:37.759213+0000 mon.a (mon.0) 2064 : cluster 0 osdmap e361: 8 total, 7 up, 7 in 2023-09-23T19:08:39.035 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:38 smithi098 bash[76817]: cluster 2023-09-23T19:08:37.762028+0000 osd.5 (osd.5) 4 : cluster 3 failed to encode map e361 with expected crc 2023-09-23T19:08:39.035 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:38 smithi098 bash[76817]: cluster 2023-09-23T19:08:37.770343+0000 osd.4 (osd.4) 94 : cluster 3 failed to encode map e361 with expected crc 2023-09-23T19:08:39.035 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:38 smithi098 bash[76817]: cluster 2023-09-23T19:08:37.773263+0000 osd.4 (osd.4) 95 : cluster 3 failed to encode map e361 with expected crc 2023-09-23T19:08:39.035 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:38 smithi098 bash[79516]: cluster 2023-09-23T19:08:36.545248+0000 osd.5 (osd.5) 1 : cluster 3 failed to encode map e323 with expected crc 2023-09-23T19:08:39.035 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:38 smithi098 bash[79516]: cluster 2023-09-23T19:08:36.750885+0000 osd.5 (osd.5) 2 : cluster 3 failed to encode map e360 with expected crc 2023-09-23T19:08:39.035 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:38 smithi098 bash[79516]: cluster 2023-09-23T19:08:37.757256+0000 osd.5 (osd.5) 3 : cluster 3 failed to encode map e361 with expected crc 2023-09-23T19:08:39.035 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:38 smithi098 bash[79516]: cluster 2023-09-23T19:08:37.758813+0000 osd.6 (osd.6) 353 : cluster 3 failed to encode map e361 with expected crc 2023-09-23T19:08:39.035 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:38 smithi098 bash[79516]: cluster 2023-09-23T19:08:37.759213+0000 mon.a (mon.0) 2064 : cluster 0 osdmap e361: 8 total, 7 up, 7 in 2023-09-23T19:08:39.036 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:38 smithi098 bash[79516]: cluster 2023-09-23T19:08:37.762028+0000 osd.5 (osd.5) 4 : cluster 3 failed to encode map e361 with expected crc 2023-09-23T19:08:39.036 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:38 smithi098 bash[79516]: cluster 2023-09-23T19:08:37.770343+0000 osd.4 (osd.4) 94 : cluster 3 failed to encode map e361 with expected crc 2023-09-23T19:08:39.036 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:38 smithi098 bash[79516]: cluster 2023-09-23T19:08:37.773263+0000 osd.4 (osd.4) 95 : cluster 3 failed to encode map e361 with expected crc 2023-09-23T19:08:39.054 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:39.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:39.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:39.157 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:39.184 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:39.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:39.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:39.288 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:39.319 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:39.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:39.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:39.422 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:39.451 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:39.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:39.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:39.555 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:39.582 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:39.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:39.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:39.661 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:08:39.686 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:39.717 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:39.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:39.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:39.820 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:39.852 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:39.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:39.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:39.943 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:39 smithi098 bash[76817]: cluster 2023-09-23T19:08:38.662847+0000 mgr.y (mgr.26054) 1532 : cluster 0 pgmap v1689: 105 pgs: 3 peering, 1 remapped+peering, 1 undersized+peered, 23 active+clean+remapped, 1 active+undersized+remapped, 44 active+clean, 13 active+undersized+degraded, 19 active+undersized; 4.6 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 12/126 objects misplaced (9.524%) 2023-09-23T19:08:39.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:39 smithi098 bash[76817]: cluster 2023-09-23T19:08:38.753537+0000 mon.a (mon.0) 2065 : cluster 3 Health check update: Degraded data redundancy: 15/126 objects degraded (11.905%), 13 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2023-09-23T19:08:39.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:39 smithi098 bash[76817]: cluster 2023-09-23T19:08:38.761947+0000 osd.4 (osd.4) 96 : cluster 3 failed to encode map e362 with expected crc 2023-09-23T19:08:39.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:39 smithi098 bash[76817]: cluster 2023-09-23T19:08:38.762278+0000 mon.a (mon.0) 2066 : cluster 0 osdmap e362: 8 total, 7 up, 7 in 2023-09-23T19:08:39.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:39 smithi098 bash[76817]: cluster 2023-09-23T19:08:38.765689+0000 osd.5 (osd.5) 5 : cluster 3 failed to encode map e362 with expected crc 2023-09-23T19:08:39.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:39 smithi098 bash[76817]: cluster 2023-09-23T19:08:38.769337+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e362 with expected crc 2023-09-23T19:08:39.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:39 smithi098 bash[76817]: cluster 2023-09-23T19:08:38.770839+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e362 with expected crc 2023-09-23T19:08:39.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:39 smithi098 bash[76817]: cluster 2023-09-23T19:08:38.774072+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e362 with expected crc 2023-09-23T19:08:39.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:39 smithi098 bash[76817]: audit 2023-09-23T19:08:39.657382+0000 mon.a (mon.0) 2067 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:08:39.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:39 smithi098 bash[76817]: audit 2023-09-23T19:08:39.658860+0000 mon.a (mon.0) 2068 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:08:39.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:39 smithi098 bash[79516]: cluster 2023-09-23T19:08:38.662847+0000 mgr.y (mgr.26054) 1532 : cluster 0 pgmap v1689: 105 pgs: 3 peering, 1 remapped+peering, 1 undersized+peered, 23 active+clean+remapped, 1 active+undersized+remapped, 44 active+clean, 13 active+undersized+degraded, 19 active+undersized; 4.6 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 12/126 objects misplaced (9.524%) 2023-09-23T19:08:39.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:39 smithi098 bash[79516]: cluster 2023-09-23T19:08:38.753537+0000 mon.a (mon.0) 2065 : cluster 3 Health check update: Degraded data redundancy: 15/126 objects degraded (11.905%), 13 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2023-09-23T19:08:39.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:39 smithi098 bash[79516]: cluster 2023-09-23T19:08:38.761947+0000 osd.4 (osd.4) 96 : cluster 3 failed to encode map e362 with expected crc 2023-09-23T19:08:39.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:39 smithi098 bash[79516]: cluster 2023-09-23T19:08:38.762278+0000 mon.a (mon.0) 2066 : cluster 0 osdmap e362: 8 total, 7 up, 7 in 2023-09-23T19:08:39.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:39 smithi098 bash[79516]: cluster 2023-09-23T19:08:38.765689+0000 osd.5 (osd.5) 5 : cluster 3 failed to encode map e362 with expected crc 2023-09-23T19:08:39.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:39 smithi098 bash[79516]: cluster 2023-09-23T19:08:38.769337+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e362 with expected crc 2023-09-23T19:08:39.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:39 smithi098 bash[79516]: cluster 2023-09-23T19:08:38.770839+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e362 with expected crc 2023-09-23T19:08:39.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:39 smithi098 bash[79516]: cluster 2023-09-23T19:08:38.774072+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e362 with expected crc 2023-09-23T19:08:39.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:39 smithi098 bash[79516]: audit 2023-09-23T19:08:39.657382+0000 mon.a (mon.0) 2067 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:08:39.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:39 smithi098 bash[79516]: audit 2023-09-23T19:08:39.658860+0000 mon.a (mon.0) 2068 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:08:39.956 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:39.981 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:39.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:39.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:40.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:39 smithi173 bash[59422]: cluster 2023-09-23T19:08:38.662847+0000 mgr.y (mgr.26054) 1532 : cluster 0 pgmap v1689: 105 pgs: 3 peering, 1 remapped+peering, 1 undersized+peered, 23 active+clean+remapped, 1 active+undersized+remapped, 44 active+clean, 13 active+undersized+degraded, 19 active+undersized; 4.6 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 15/126 objects degraded (11.905%); 12/126 objects misplaced (9.524%) 2023-09-23T19:08:40.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:39 smithi173 bash[59422]: cluster 2023-09-23T19:08:38.753537+0000 mon.a (mon.0) 2065 : cluster 3 Health check update: Degraded data redundancy: 15/126 objects degraded (11.905%), 13 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2023-09-23T19:08:40.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:39 smithi173 bash[59422]: cluster 2023-09-23T19:08:38.761947+0000 osd.4 (osd.4) 96 : cluster 3 failed to encode map e362 with expected crc 2023-09-23T19:08:40.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:39 smithi173 bash[59422]: cluster 2023-09-23T19:08:38.762278+0000 mon.a (mon.0) 2066 : cluster 0 osdmap e362: 8 total, 7 up, 7 in 2023-09-23T19:08:40.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:39 smithi173 bash[59422]: cluster 2023-09-23T19:08:38.765689+0000 osd.5 (osd.5) 5 : cluster 3 failed to encode map e362 with expected crc 2023-09-23T19:08:40.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:39 smithi173 bash[59422]: cluster 2023-09-23T19:08:38.769337+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e362 with expected crc 2023-09-23T19:08:40.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:39 smithi173 bash[59422]: cluster 2023-09-23T19:08:38.770839+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e362 with expected crc 2023-09-23T19:08:40.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:39 smithi173 bash[59422]: cluster 2023-09-23T19:08:38.774072+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e362 with expected crc 2023-09-23T19:08:40.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:39 smithi173 bash[59422]: audit 2023-09-23T19:08:39.657382+0000 mon.a (mon.0) 2067 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:08:40.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:39 smithi173 bash[59422]: audit 2023-09-23T19:08:39.658860+0000 mon.a (mon.0) 2068 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:08:40.084 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:40.111 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:40.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:40.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:40.214 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:40.245 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:40.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:40.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:40.349 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:40.370 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:40.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:40.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:40.474 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:40.505 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:40.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:40.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:40.609 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:40.637 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:40.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:40.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:40.740 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:40.764 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:40.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:40.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:40.868 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:40.891 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:40.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:40.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:40.993 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:41.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:40 smithi173 bash[59422]: cluster 2023-09-23T19:08:39.586043+0000 osd.6 (osd.6) 354 : cluster 3 failed to encode map e362 with expected crc 2023-09-23T19:08:41.029 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:41.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:41.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:41.118 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:08:41.134 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:41.161 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:41.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:41.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:41.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:40 smithi098 bash[76817]: cluster 2023-09-23T19:08:39.586043+0000 osd.6 (osd.6) 354 : cluster 3 failed to encode map e362 with expected crc 2023-09-23T19:08:41.193 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:40 smithi098 bash[79516]: cluster 2023-09-23T19:08:39.586043+0000 osd.6 (osd.6) 354 : cluster 3 failed to encode map e362 with expected crc 2023-09-23T19:08:41.265 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:41.290 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:41.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:41.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:41.393 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:41.417 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:41.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:41.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:41.520 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:41.541 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:41.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:41.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:41.644 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:41.670 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:41.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:41.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:41.687 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:08:41.774 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:41.797 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:41.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:41.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:41.814 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.mirror_image_status (3572 ms) 2023-09-23T19:08:41.814 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-09-23T19:08:41.901 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:41.911 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:41 smithi098 bash[79516]: cluster 2023-09-23T19:08:40.663769+0000 mgr.y (mgr.26054) 1533 : cluster 0 pgmap v1691: 105 pgs: 3 peering, 1 remapped+peering, 1 undersized+peered, 22 active+clean+remapped, 1 active+undersized+remapped, 46 active+clean, 13 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 552 KiB/s rd, 386 KiB/s wr, 952 op/s; 14/135 objects degraded (10.370%); 12/135 objects misplaced (8.889%) 2023-09-23T19:08:41.911 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:41 smithi098 bash[76817]: cluster 2023-09-23T19:08:40.663769+0000 mgr.y (mgr.26054) 1533 : cluster 0 pgmap v1691: 105 pgs: 3 peering, 1 remapped+peering, 1 undersized+peered, 22 active+clean+remapped, 1 active+undersized+remapped, 46 active+clean, 13 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 552 KiB/s rd, 386 KiB/s wr, 952 op/s; 14/135 objects degraded (10.370%); 12/135 objects misplaced (8.889%) 2023-09-23T19:08:41.919 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.mirror_image_map (106 ms) 2023-09-23T19:08:41.919 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-09-23T19:08:41.925 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:41.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:41.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:42.027 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:42.053 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:42.054 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.mirror_instances (134 ms) 2023-09-23T19:08:42.054 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-09-23T19:08:42.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:42.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:42.070 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.mirror_snapshot (16 ms) 2023-09-23T19:08:42.070 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-09-23T19:08:42.072 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2023-09-23T19:08:42.072 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-09-23T19:08:42.074 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.group_dir_add (3 ms) 2023-09-23T19:08:42.074 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-09-23T19:08:42.078 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2023-09-23T19:08:42.078 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-09-23T19:08:42.085 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.group_dir_rename (8 ms) 2023-09-23T19:08:42.086 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-09-23T19:08:42.090 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.group_dir_remove (4 ms) 2023-09-23T19:08:42.090 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-09-23T19:08:42.095 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (6 ms) 2023-09-23T19:08:42.095 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.group_image_add 2023-09-23T19:08:42.098 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2023-09-23T19:08:42.098 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-09-23T19:08:42.103 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.group_image_remove (6 ms) 2023-09-23T19:08:42.104 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.group_image_list 2023-09-23T19:08:42.109 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.group_image_list (5 ms) 2023-09-23T19:08:42.109 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-09-23T19:08:42.115 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.group_image_clean (6 ms) 2023-09-23T19:08:42.115 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.image_group_add 2023-09-23T19:08:42.118 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.image_group_add (4 ms) 2023-09-23T19:08:42.119 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-09-23T19:08:42.123 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.image_group_remove (4 ms) 2023-09-23T19:08:42.123 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.image_group_get 2023-09-23T19:08:42.127 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.image_group_get (4 ms) 2023-09-23T19:08:42.127 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-09-23T19:08:42.129 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (3 ms) 2023-09-23T19:08:42.129 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-09-23T19:08:42.132 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2023-09-23T19:08:42.132 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-09-23T19:08:42.136 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (4 ms) 2023-09-23T19:08:42.136 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-09-23T19:08:42.141 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (5 ms) 2023-09-23T19:08:42.141 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-09-23T19:08:42.145 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.group_snap_set (4 ms) 2023-09-23T19:08:42.145 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-09-23T19:08:42.149 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.group_snap_list (4 ms) 2023-09-23T19:08:42.149 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-09-23T19:08:42.156 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:42.172 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (23 ms) 2023-09-23T19:08:42.172 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-09-23T19:08:42.180 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:42.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:42.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:42.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:41 smithi173 bash[59422]: cluster 2023-09-23T19:08:40.663769+0000 mgr.y (mgr.26054) 1533 : cluster 0 pgmap v1691: 105 pgs: 3 peering, 1 remapped+peering, 1 undersized+peered, 22 active+clean+remapped, 1 active+undersized+remapped, 46 active+clean, 13 active+undersized+degraded, 18 active+undersized; 4.6 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 552 KiB/s rd, 386 KiB/s wr, 952 op/s; 14/135 objects degraded (10.370%); 12/135 objects misplaced (8.889%) 2023-09-23T19:08:42.283 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:42.308 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:42.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:42.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:42.412 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:42.441 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:42.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:42.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:42.449 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (278 ms) 2023-09-23T19:08:42.449 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-09-23T19:08:42.454 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.group_snap_remove (4 ms) 2023-09-23T19:08:42.454 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-09-23T19:08:42.457 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (3 ms) 2023-09-23T19:08:42.457 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.trash_methods 2023-09-23T19:08:42.485 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.trash_methods (28 ms) 2023-09-23T19:08:42.485 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.op_features 2023-09-23T19:08:42.495 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.op_features (10 ms) 2023-09-23T19:08:42.495 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.clone_parent 2023-09-23T19:08:42.525 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.clone_parent (31 ms) 2023-09-23T19:08:42.526 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-09-23T19:08:42.540 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.clone_parent_ns (14 ms) 2023-09-23T19:08:42.540 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.clone_child 2023-09-23T19:08:42.544 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:42.558 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.clone_child (18 ms) 2023-09-23T19:08:42.558 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-09-23T19:08:42.567 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.namespace_methods (9 ms) 2023-09-23T19:08:42.567 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.migration 2023-09-23T19:08:42.570 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:42.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:42.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:42.579 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.migration (12 ms) 2023-09-23T19:08:42.579 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.migration_v1 2023-09-23T19:08:42.592 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.migration_v1 (14 ms) 2023-09-23T19:08:42.592 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-09-23T19:08:42.673 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:42.699 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:42.700 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:08:42.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:42.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:42.802 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:42.825 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:42.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:42.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:42.928 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:42.954 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:42.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:42.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:43.057 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:43.069 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:42 smithi173 bash[59422]: cluster 2023-09-23T19:08:42.773841+0000 mon.a (mon.0) 2069 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T19:08:43.069 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:42 smithi173 bash[59422]: cluster 2023-09-23T19:08:42.785864+0000 mon.a (mon.0) 2070 : cluster 0 osdmap e363: 8 total, 7 up, 7 in 2023-09-23T19:08:43.088 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:43.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:43.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:43.191 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:43.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:42 smithi098 bash[79516]: cluster 2023-09-23T19:08:42.773841+0000 mon.a (mon.0) 2069 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T19:08:43.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:42 smithi098 bash[79516]: cluster 2023-09-23T19:08:42.785864+0000 mon.a (mon.0) 2070 : cluster 0 osdmap e363: 8 total, 7 up, 7 in 2023-09-23T19:08:43.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:42 smithi098 bash[76817]: cluster 2023-09-23T19:08:42.773841+0000 mon.a (mon.0) 2069 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T19:08:43.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:42 smithi098 bash[76817]: cluster 2023-09-23T19:08:42.785864+0000 mon.a (mon.0) 2070 : cluster 0 osdmap e363: 8 total, 7 up, 7 in 2023-09-23T19:08:43.221 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:43.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:43.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:43.324 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:43.357 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:43.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:43.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:43.460 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:43.488 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:43.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:43.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:43.591 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:43.614 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:43.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:43.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:43.718 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:43.740 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:43.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:43.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:43.798 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1205 ms) 2023-09-23T19:08:43.798 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestClsRbd.sparsify 2023-09-23T19:08:43.835 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestClsRbd.sparsify (37 ms) 2023-09-23T19:08:43.843 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:43.851 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:43 smithi098 bash[79516]: cluster 2023-09-23T19:08:42.664672+0000 mgr.y (mgr.26054) 1534 : cluster 0 pgmap v1692: 105 pgs: 26 active+clean+remapped, 57 active+clean, 13 active+undersized+degraded, 9 active+undersized; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 755 KiB/s rd, 369 KiB/s wr, 1.01k op/s; 19/216 objects degraded (8.796%); 15/216 objects misplaced (6.944%); 0 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T19:08:43.851 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:43 smithi098 bash[79516]: cluster 2023-09-23T19:08:42.786813+0000 osd.6 (osd.6) 355 : cluster 3 failed to encode map e363 with expected crc 2023-09-23T19:08:43.851 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:43 smithi098 bash[79516]: cluster 2023-09-23T19:08:43.788321+0000 mon.a (mon.0) 2071 : cluster 0 osdmap e364: 8 total, 7 up, 7 in 2023-09-23T19:08:43.851 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:43 smithi098 bash[76817]: cluster 2023-09-23T19:08:42.664672+0000 mgr.y (mgr.26054) 1534 : cluster 0 pgmap v1692: 105 pgs: 26 active+clean+remapped, 57 active+clean, 13 active+undersized+degraded, 9 active+undersized; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 755 KiB/s rd, 369 KiB/s wr, 1.01k op/s; 19/216 objects degraded (8.796%); 15/216 objects misplaced (6.944%); 0 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T19:08:43.851 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:43 smithi098 bash[76817]: cluster 2023-09-23T19:08:42.786813+0000 osd.6 (osd.6) 355 : cluster 3 failed to encode map e363 with expected crc 2023-09-23T19:08:43.851 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:43 smithi098 bash[76817]: cluster 2023-09-23T19:08:43.788321+0000 mon.a (mon.0) 2071 : cluster 0 osdmap e364: 8 total, 7 up, 7 in 2023-09-23T19:08:43.866 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:43.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:43.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:43.972 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:44.001 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:44.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:44.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:44.105 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:44.129 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:44.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:44.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:44.233 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:44.257 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:44.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:44.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:44.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:43 smithi173 bash[59422]: cluster 2023-09-23T19:08:42.664672+0000 mgr.y (mgr.26054) 1534 : cluster 0 pgmap v1692: 105 pgs: 26 active+clean+remapped, 57 active+clean, 13 active+undersized+degraded, 9 active+undersized; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 755 KiB/s rd, 369 KiB/s wr, 1.01k op/s; 19/216 objects degraded (8.796%); 15/216 objects misplaced (6.944%); 0 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T19:08:44.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:43 smithi173 bash[59422]: cluster 2023-09-23T19:08:42.786813+0000 osd.6 (osd.6) 355 : cluster 3 failed to encode map e363 with expected crc 2023-09-23T19:08:44.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:43 smithi173 bash[59422]: cluster 2023-09-23T19:08:43.788321+0000 mon.a (mon.0) 2071 : cluster 0 osdmap e364: 8 total, 7 up, 7 in 2023-09-23T19:08:44.361 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:44.392 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:44.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:44.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:44.495 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:44.518 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:44.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:44.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:44.621 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:44.647 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:44.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:44.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:44.751 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:44.775 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:44.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:44.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:44.843 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:44 smithi098 bash[76817]: cluster 2023-09-23T19:08:43.789340+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e364 with expected crc 2023-09-23T19:08:44.843 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:44 smithi098 bash[76817]: cluster 2023-09-23T19:08:43.798629+0000 osd.6 (osd.6) 356 : cluster 3 failed to encode map e364 with expected crc 2023-09-23T19:08:44.844 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:44 smithi098 bash[76817]: cluster 2023-09-23T19:08:44.401211+0000 osd.4 (osd.4) 97 : cluster 3 failed to encode map e364 with expected crc 2023-09-23T19:08:44.844 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:44 smithi098 bash[79516]: cluster 2023-09-23T19:08:43.789340+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e364 with expected crc 2023-09-23T19:08:44.844 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:44 smithi098 bash[79516]: cluster 2023-09-23T19:08:43.798629+0000 osd.6 (osd.6) 356 : cluster 3 failed to encode map e364 with expected crc 2023-09-23T19:08:44.844 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:44 smithi098 bash[79516]: cluster 2023-09-23T19:08:44.401211+0000 osd.4 (osd.4) 97 : cluster 3 failed to encode map e364 with expected crc 2023-09-23T19:08:44.848 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 70 tests from TestClsRbd (121253 ms total) 2023-09-23T19:08:44.848 INFO:tasks.workunit.client.0.smithi098.stdout: 2023-09-23T19:08:44.848 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment tear-down 2023-09-23T19:08:44.848 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] 70 tests from 1 test suite ran. (123806 ms total) 2023-09-23T19:08:44.849 INFO:tasks.workunit.client.0.smithi098.stdout:[ PASSED ] 70 tests. 2023-09-23T19:08:44.851 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T19:08:44.851 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T19:08:44.861 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-09-23T19:08:44.861 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T19:08:44.879 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:44.932 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:44.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:44.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:45.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:44 smithi173 bash[59422]: cluster 2023-09-23T19:08:43.789340+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e364 with expected crc 2023-09-23T19:08:45.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:44 smithi173 bash[59422]: cluster 2023-09-23T19:08:43.798629+0000 osd.6 (osd.6) 356 : cluster 3 failed to encode map e364 with expected crc 2023-09-23T19:08:45.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:44 smithi173 bash[59422]: cluster 2023-09-23T19:08:44.401211+0000 osd.4 (osd.4) 97 : cluster 3 failed to encode map e364 with expected crc 2023-09-23T19:08:45.037 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:45.068 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:45.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:45.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:45.137 DEBUG:teuthology.parallel:result is None 2023-09-23T19:08:45.137 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T19:08:45.145 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T19:08:45.145 DEBUG:teuthology.orchestra.run.smithi098:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T19:08:45.172 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:45.193 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T19:08:45.193 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T19:08:45.194 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-09-23T19:08:45.194 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T19:08:45.194 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-09-23T19:08:45.194 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T19:08:45.195 DEBUG:teuthology.orchestra.run.smithi098:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T19:08:45.211 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:45.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:45.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:45.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:45.217 INFO:teuthology.orchestra.run.smithi098.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T19:08:45.217 DEBUG:teuthology.orchestra.run.smithi098:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T19:08:45.314 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:45.320 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T19:08:45.321 DEBUG:teuthology.orchestra.run.smithi098:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T19:08:45.336 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:45.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:45.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:45.340 INFO:tasks.workunit:timeout=3h 2023-09-23T19:08:45.340 INFO:tasks.workunit:cleanup=True 2023-09-23T19:08:45.341 DEBUG:teuthology.orchestra.run.smithi098:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T19:08:45.439 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:45.454 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T19:08:45.454 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-09-23T19:08:45.455 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 0 2023-09-23T19:08:45.455 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T19:08:45.465 INFO:tasks.workunit.client.0.smithi098.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T19:08:45.469 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:45.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:45.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:45.574 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:45.607 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:45.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:45.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:45.710 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:45.741 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:45.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:45.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:45.845 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:45.871 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:45.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:45.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:45.974 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:46.002 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:46.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:46.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:46.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:45 smithi173 bash[59422]: cluster 2023-09-23T19:08:44.665239+0000 mgr.y (mgr.26054) 1535 : cluster 0 pgmap v1695: 105 pgs: 26 active+clean+remapped, 57 active+clean, 13 active+undersized+degraded, 9 active+undersized; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 755 KiB/s rd, 369 KiB/s wr, 1.01k op/s; 19/216 objects degraded (8.796%); 15/216 objects misplaced (6.944%); 0 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T19:08:46.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:45 smithi173 bash[59422]: cluster 2023-09-23T19:08:44.834187+0000 mon.a (mon.0) 2072 : cluster 0 osdmap e365: 8 total, 7 up, 7 in 2023-09-23T19:08:46.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:45 smithi173 bash[59422]: cluster 2023-09-23T19:08:44.835302+0000 osd.6 (osd.6) 357 : cluster 3 failed to encode map e365 with expected crc 2023-09-23T19:08:46.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:45 smithi173 bash[59422]: cluster 2023-09-23T19:08:44.835496+0000 osd.4 (osd.4) 98 : cluster 3 failed to encode map e365 with expected crc 2023-09-23T19:08:46.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:45 smithi173 bash[59422]: cluster 2023-09-23T19:08:44.842146+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e365 with expected crc 2023-09-23T19:08:46.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:45 smithi173 bash[59422]: cluster 2023-09-23T19:08:45.592047+0000 mon.a (mon.0) 2073 : cluster 3 Health check update: Degraded data redundancy: 19/216 objects degraded (8.796%), 13 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T19:08:46.106 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:46.131 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:46.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:46.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:46.220 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:08:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:08:46] "GET /metrics HTTP/1.1" 200 36444 "" "Prometheus/2.33.4" 2023-09-23T19:08:46.220 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:45 smithi098 bash[79516]: cluster 2023-09-23T19:08:44.665239+0000 mgr.y (mgr.26054) 1535 : cluster 0 pgmap v1695: 105 pgs: 26 active+clean+remapped, 57 active+clean, 13 active+undersized+degraded, 9 active+undersized; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 755 KiB/s rd, 369 KiB/s wr, 1.01k op/s; 19/216 objects degraded (8.796%); 15/216 objects misplaced (6.944%); 0 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T19:08:46.220 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:45 smithi098 bash[79516]: cluster 2023-09-23T19:08:44.834187+0000 mon.a (mon.0) 2072 : cluster 0 osdmap e365: 8 total, 7 up, 7 in 2023-09-23T19:08:46.220 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:45 smithi098 bash[79516]: cluster 2023-09-23T19:08:44.835302+0000 osd.6 (osd.6) 357 : cluster 3 failed to encode map e365 with expected crc 2023-09-23T19:08:46.220 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:45 smithi098 bash[79516]: cluster 2023-09-23T19:08:44.835496+0000 osd.4 (osd.4) 98 : cluster 3 failed to encode map e365 with expected crc 2023-09-23T19:08:46.220 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:45 smithi098 bash[79516]: cluster 2023-09-23T19:08:44.842146+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e365 with expected crc 2023-09-23T19:08:46.221 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:45 smithi098 bash[79516]: cluster 2023-09-23T19:08:45.592047+0000 mon.a (mon.0) 2073 : cluster 3 Health check update: Degraded data redundancy: 19/216 objects degraded (8.796%), 13 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T19:08:46.221 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:45 smithi098 bash[76817]: cluster 2023-09-23T19:08:44.665239+0000 mgr.y (mgr.26054) 1535 : cluster 0 pgmap v1695: 105 pgs: 26 active+clean+remapped, 57 active+clean, 13 active+undersized+degraded, 9 active+undersized; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 755 KiB/s rd, 369 KiB/s wr, 1.01k op/s; 19/216 objects degraded (8.796%); 15/216 objects misplaced (6.944%); 0 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T19:08:46.221 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:45 smithi098 bash[76817]: cluster 2023-09-23T19:08:44.834187+0000 mon.a (mon.0) 2072 : cluster 0 osdmap e365: 8 total, 7 up, 7 in 2023-09-23T19:08:46.221 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:45 smithi098 bash[76817]: cluster 2023-09-23T19:08:44.835302+0000 osd.6 (osd.6) 357 : cluster 3 failed to encode map e365 with expected crc 2023-09-23T19:08:46.221 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:45 smithi098 bash[76817]: cluster 2023-09-23T19:08:44.835496+0000 osd.4 (osd.4) 98 : cluster 3 failed to encode map e365 with expected crc 2023-09-23T19:08:46.221 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:45 smithi098 bash[76817]: cluster 2023-09-23T19:08:44.842146+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e365 with expected crc 2023-09-23T19:08:46.221 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:45 smithi098 bash[76817]: cluster 2023-09-23T19:08:45.592047+0000 mon.a (mon.0) 2073 : cluster 3 Health check update: Degraded data redundancy: 19/216 objects degraded (8.796%), 13 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T19:08:46.235 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:46.268 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:46.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:46.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:46.373 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:46.391 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:46.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:46.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:46.405 INFO:tasks.thrashosds.thrasher:Reviving osd 7 2023-09-23T19:08:46.405 INFO:tasks.cephadm.osd.7:Restarting osd.7 (starting--it wasn't running)... 2023-09-23T19:08:46.405 DEBUG:teuthology.orchestra.run.smithi173:osd.7> sudo journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.7.service 2023-09-23T19:08:46.408 DEBUG:teuthology.orchestra.run.smithi173:> sudo systemctl start ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.7 2023-09-23T19:08:46.423 INFO:journalctl@ceph.osd.7.smithi173.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-09-23T19:08:46.446 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T19:08:46.494 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:46.522 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:46.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:46.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:46.625 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:46.656 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:46.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:46.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:46.759 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:46.771 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:08:46 smithi173 systemd[1]: Started Ceph osd.7 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T19:08:46.785 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:46.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:46.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:46.888 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:46.915 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:46.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:46.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:47.018 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:08:47.045 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:08:47.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:47.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:08:47.148 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:47.181 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:47.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:47.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:47.284 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:47.317 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:47.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:47.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:47.420 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:08:47.444 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:08:47.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:47.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:08:47.548 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:47.576 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:47.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:47.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:47.679 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:47.707 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:47.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:47.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:47.810 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:47.832 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:47.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:47.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:47.934 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:47.956 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:47.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:47.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:47.962 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:47 smithi098 bash[79516]: cluster 2023-09-23T19:08:46.666029+0000 mgr.y (mgr.26054) 1536 : cluster 0 pgmap v1697: 73 pgs: 26 active+clean+remapped, 43 active+clean, 2 active+undersized+degraded, 2 active+undersized; 4.6 MiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 15/84 objects misplaced (17.857%) 2023-09-23T19:08:47.962 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:47 smithi098 bash[79516]: audit 2023-09-23T19:08:47.320784+0000 mon.a (mon.0) 2074 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:08:47.963 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:47 smithi098 bash[79516]: audit 2023-09-23T19:08:47.695172+0000 mon.a (mon.0) 2075 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:08:47.963 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:47 smithi098 bash[79516]: audit 2023-09-23T19:08:47.701036+0000 mon.a (mon.0) 2076 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:08:47.963 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:47 smithi098 bash[76817]: cluster 2023-09-23T19:08:46.666029+0000 mgr.y (mgr.26054) 1536 : cluster 0 pgmap v1697: 73 pgs: 26 active+clean+remapped, 43 active+clean, 2 active+undersized+degraded, 2 active+undersized; 4.6 MiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 15/84 objects misplaced (17.857%) 2023-09-23T19:08:47.963 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:47 smithi098 bash[76817]: audit 2023-09-23T19:08:47.320784+0000 mon.a (mon.0) 2074 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:08:47.963 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:47 smithi098 bash[76817]: audit 2023-09-23T19:08:47.695172+0000 mon.a (mon.0) 2075 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:08:47.963 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:47 smithi098 bash[76817]: audit 2023-09-23T19:08:47.701036+0000 mon.a (mon.0) 2076 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:08:48.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:47 smithi173 bash[59422]: cluster 2023-09-23T19:08:46.666029+0000 mgr.y (mgr.26054) 1536 : cluster 0 pgmap v1697: 73 pgs: 26 active+clean+remapped, 43 active+clean, 2 active+undersized+degraded, 2 active+undersized; 4.6 MiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 15/84 objects misplaced (17.857%) 2023-09-23T19:08:48.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:47 smithi173 bash[59422]: audit 2023-09-23T19:08:47.320784+0000 mon.a (mon.0) 2074 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:08:48.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:47 smithi173 bash[59422]: audit 2023-09-23T19:08:47.695172+0000 mon.a (mon.0) 2075 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:08:48.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:47 smithi173 bash[59422]: audit 2023-09-23T19:08:47.701036+0000 mon.a (mon.0) 2076 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:08:48.059 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:48.091 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:48.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:48.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:48.194 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:48.217 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:48.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:48.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:48.320 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:48.343 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:48.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:48.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:48.445 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:48.477 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:48.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:48.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:48.581 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:48.604 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:48.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:48.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:48.615 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:08:48 smithi173 bash[162149]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2023-09-23T19:08:48.615 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:08:48 smithi173 bash[162149]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2023-09-23T19:08:48.615 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:08:48 smithi173 bash[162149]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2023-09-23T19:08:48.616 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:08:48 smithi173 bash[162149]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-09-23T19:08:48.616 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:08:48 smithi173 bash[162149]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2023-09-23T19:08:48.616 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:08:48 smithi173 bash[162149]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2023-09-23T19:08:48.616 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:08:48 smithi173 bash[162149]: --> ceph-volume raw activate successful for osd ID: 7 2023-09-23T19:08:48.706 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:48.731 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:48.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:48.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:48.834 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:48.857 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:48.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:48.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:48.960 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:08:48.989 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:08:48.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:48.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:08:49.092 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:49.120 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:49.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:49.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:49.224 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:49.249 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:49.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:49.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:49.352 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:49.385 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:49.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:49.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:49.487 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:49.517 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:49.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:49.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:49.621 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:49.645 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:49.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:49.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:49.747 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:08:49.768 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:08:49.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:49.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:08:49.871 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:49.883 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:49 smithi098 bash[76817]: cluster 2023-09-23T19:08:48.667081+0000 mgr.y (mgr.26054) 1537 : cluster 0 pgmap v1698: 73 pgs: 26 active+clean+remapped, 43 active+clean, 2 active+undersized+degraded, 2 active+undersized; 4.6 MiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 15/84 objects misplaced (17.857%) 2023-09-23T19:08:49.883 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:49 smithi098 bash[79516]: cluster 2023-09-23T19:08:48.667081+0000 mgr.y (mgr.26054) 1537 : cluster 0 pgmap v1698: 73 pgs: 26 active+clean+remapped, 43 active+clean, 2 active+undersized+degraded, 2 active+undersized; 4.6 MiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 15/84 objects misplaced (17.857%) 2023-09-23T19:08:49.894 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:49.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:49.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:49.997 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:50.009 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:49 smithi173 bash[59422]: cluster 2023-09-23T19:08:48.667081+0000 mgr.y (mgr.26054) 1537 : cluster 0 pgmap v1698: 73 pgs: 26 active+clean+remapped, 43 active+clean, 2 active+undersized+degraded, 2 active+undersized; 4.6 MiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 15/84 objects misplaced (17.857%) 2023-09-23T19:08:50.031 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:50.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:50.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:50.134 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:50.159 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:50.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:50.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:50.263 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:50.294 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:50.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:50.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:50.398 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:50.425 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:50.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:50.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:50.530 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:50.549 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:50.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:50.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:50.653 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:50.681 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:50.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:50.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:50.784 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:50.812 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:50.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:50.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:50.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:50.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:50.946 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:50 smithi173 bash[59422]: cluster 2023-09-23T19:08:50.593689+0000 mon.a (mon.0) 2077 : cluster 3 Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:08:50.946 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:08:50 smithi173 bash[162608]: debug 2023-09-23T19:08:50.804+0000 7fee4aead700 1 -- 172.21.15.173:0/3964947220 <== mon.1 v2:172.21.15.173:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a6aa21a4e0 con 0x55a6aa28d400 2023-09-23T19:08:50.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:50.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:51.005 DEBUG:teuthology.orchestra.run.smithi098:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T19:08:51.010 INFO:tasks.workunit.client.0.smithi098.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T19:08:51.013 INFO:tasks.workunit.client.0.smithi098.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T19:08:51.013 INFO:tasks.workunit.client.0.smithi098.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T19:08:51.047 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:51.054 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:50 smithi098 bash[76817]: cluster 2023-09-23T19:08:50.593689+0000 mon.a (mon.0) 2077 : cluster 3 Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:08:51.054 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:50 smithi098 bash[79516]: cluster 2023-09-23T19:08:50.593689+0000 mon.a (mon.0) 2077 : cluster 3 Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:08:51.067 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:51.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:51.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:51.169 INFO:tasks.workunit.client.0.smithi098.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T19:08:51.170 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:51.192 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:51.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:51.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:51.295 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:51.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:51.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:51.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:51.345 INFO:tasks.workunit.client.0.smithi098.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T19:08:51.430 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:51.454 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:51.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:51.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:51.557 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:51.570 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:08:51 smithi173 bash[162608]: debug 2023-09-23T19:08:51.384+0000 7fee52927200 -1 Falling back to public interface 2023-09-23T19:08:51.592 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:51.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:51.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:51.677 INFO:tasks.workunit.client.0.smithi098.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T19:08:51.680 INFO:tasks.workunit.client.0.smithi098.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T19:08:51.680 INFO:tasks.workunit.client.0.smithi098.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T19:08:51.695 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:51.727 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:51.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:51.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:51.831 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:51.834 INFO:tasks.workunit.client.0.smithi098.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T19:08:51.839 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-23T19:08:51.840 DEBUG:teuthology.orchestra.run.smithi098:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T19:08:51.845 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-09-23T19:08:51.846 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-09-23T19:08:51.847 DEBUG:teuthology.orchestra.run.smithi098:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-09-23T19:08:51.860 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:51.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:51.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:51.897 INFO:tasks.workunit.client.0.smithi098.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-09-23T19:08:51.897 INFO:tasks.workunit.client.0.smithi098.stderr:+ TMPDIR=/tmp/rbd_import_export_266403 2023-09-23T19:08:51.897 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm -rf /tmp/rbd_import_export_266403 2023-09-23T19:08:51.898 INFO:tasks.workunit.client.0.smithi098.stderr:+ mkdir /tmp/rbd_import_export_266403 2023-09-23T19:08:51.899 INFO:tasks.workunit.client.0.smithi098.stderr:+ trap rm -rf /tmp/rbd_import_export_266403 INT TERM EXIT 2023-09-23T19:08:51.899 INFO:tasks.workunit.client.0.smithi098.stderr:+ mkdir foo.266403 2023-09-23T19:08:51.901 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import foo.266403 foo.dir 2023-09-23T19:08:51.964 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:51.990 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:51.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:51.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:52.093 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:08:52.107 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:51 smithi173 bash[59422]: cluster 2023-09-23T19:08:50.667666+0000 mgr.y (mgr.26054) 1538 : cluster 0 pgmap v1699: 73 pgs: 26 active+clean+remapped, 43 active+clean, 2 active+undersized+degraded, 2 active+undersized; 4.6 MiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 15/84 objects misplaced (17.857%) 2023-09-23T19:08:52.124 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:08:52.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:52.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:08:52.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:51 smithi098 bash[76817]: cluster 2023-09-23T19:08:50.667666+0000 mgr.y (mgr.26054) 1538 : cluster 0 pgmap v1699: 73 pgs: 26 active+clean+remapped, 43 active+clean, 2 active+undersized+degraded, 2 active+undersized; 4.6 MiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 15/84 objects misplaced (17.857%) 2023-09-23T19:08:52.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:51 smithi098 bash[79516]: cluster 2023-09-23T19:08:50.667666+0000 mgr.y (mgr.26054) 1538 : cluster 0 pgmap v1699: 73 pgs: 26 active+clean+remapped, 43 active+clean, 2 active+undersized+degraded, 2 active+undersized; 4.6 MiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 15/84 objects misplaced (17.857%) 2023-09-23T19:08:52.228 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:52.253 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:52.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:52.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:52.291 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: cannot import a directory 2023-09-23T19:08:52.291 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 0% complete...failed. 2023-09-23T19:08:52.291 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: import failed: (21) Is a directory 2023-09-23T19:08:52.296 INFO:tasks.workunit.client.0.smithi098.stderr:+ true 2023-09-23T19:08:52.296 INFO:tasks.workunit.client.0.smithi098.stderr:+ rmdir foo.266403 2023-09-23T19:08:52.296 INFO:tasks.workunit.client.0.smithi098.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_266403/img bs=1k count=1 seek=10 2023-09-23T19:08:52.297 INFO:tasks.workunit.client.0.smithi098.stderr:1+0 records in 2023-09-23T19:08:52.297 INFO:tasks.workunit.client.0.smithi098.stderr:1+0 records out 2023-09-23T19:08:52.297 INFO:tasks.workunit.client.0.smithi098.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 0.000105538 s, 9.7 MB/s 2023-09-23T19:08:52.297 INFO:tasks.workunit.client.0.smithi098.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_266403/img bs=1k count=10 seek=100 2023-09-23T19:08:52.298 INFO:tasks.workunit.client.0.smithi098.stderr:10+0 records in 2023-09-23T19:08:52.298 INFO:tasks.workunit.client.0.smithi098.stderr:10+0 records out 2023-09-23T19:08:52.298 INFO:tasks.workunit.client.0.smithi098.stderr:10240 bytes (10 kB, 10 KiB) copied, 0.00024565 s, 41.7 MB/s 2023-09-23T19:08:52.299 INFO:tasks.workunit.client.0.smithi098.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_266403/img bs=1k count=100 seek=1000 2023-09-23T19:08:52.300 INFO:tasks.workunit.client.0.smithi098.stderr:70+1 records in 2023-09-23T19:08:52.300 INFO:tasks.workunit.client.0.smithi098.stderr:70+1 records out 2023-09-23T19:08:52.300 INFO:tasks.workunit.client.0.smithi098.stderr:72056 bytes (72 kB, 70 KiB) copied, 0.000561048 s, 128 MB/s 2023-09-23T19:08:52.300 INFO:tasks.workunit.client.0.smithi098.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_266403/img bs=1k seek=10000 2023-09-23T19:08:52.301 INFO:tasks.workunit.client.0.smithi098.stderr:138+1 records in 2023-09-23T19:08:52.301 INFO:tasks.workunit.client.0.smithi098.stderr:138+1 records out 2023-09-23T19:08:52.302 INFO:tasks.workunit.client.0.smithi098.stderr:142144 bytes (142 kB, 139 KiB) copied, 0.000543659 s, 261 MB/s 2023-09-23T19:08:52.302 INFO:tasks.workunit.client.0.smithi098.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_266403/img bs=1k seek=100000 2023-09-23T19:08:52.357 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:52.376 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:52.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:52.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:52.429 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T19:08:52.429 INFO:teuthology.orchestra.run.smithi098.stdout: "size": 20, 2023-09-23T19:08:52.429 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 600, 2023-09-23T19:08:52.429 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [ 2023-09-23T19:08:52.430 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.430 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:114899 10.c 10.5bfc659c (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:08:52.430 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.675112+0000", 2023-09-23T19:08:52.430 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 354.74795253600001, 2023-09-23T19:08:52.430 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.99592086700000004, 2023-09-23T19:08:52.430 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:08:52.430 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:08:52.430 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:08:52.430 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:08:52.430 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:08:52.430 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 114899 2023-09-23T19:08:52.430 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.431 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:08:52.431 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.431 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:08:52.431 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.675112+0000", 2023-09-23T19:08:52.431 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.431 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.431 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.431 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:08:52.431 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.675112+0000", 2023-09-23T19:08:52.431 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.431 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.431 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.432 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:08:52.432 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.675110+0000", 2023-09-23T19:08:52.432 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:08:52.432 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.432 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.432 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:08:52.432 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.675119+0000", 2023-09-23T19:08:52.432 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.7569999999999993e-06 2023-09-23T19:08:52.432 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.432 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.432 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:08:52.432 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.675134+0000", 2023-09-23T19:08:52.432 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.5721000000000001e-05 2023-09-23T19:08:52.433 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.433 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.433 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:08:52.433 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.675144+0000", 2023-09-23T19:08:52.433 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.7280000000000007e-06 2023-09-23T19:08:52.433 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.433 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.433 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.433 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.679395+0000", 2023-09-23T19:08:52.433 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0042507300000000003 2023-09-23T19:08:52.433 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.433 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.433 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:08:52.434 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.679403+0000", 2023-09-23T19:08:52.434 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.2870000000000004e-06 2023-09-23T19:08:52.434 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.434 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.434 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.434 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671016+0000", 2023-09-23T19:08:52.434 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.99161303899999997 2023-09-23T19:08:52.434 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.434 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.434 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:08:52.434 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671033+0000", 2023-09-23T19:08:52.435 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.6325e-05 2023-09-23T19:08:52.435 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.435 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:08:52.435 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.435 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.435 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.435 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:114902 10.c 10.72deeabc (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:08:52.435 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.689552+0000", 2023-09-23T19:08:52.435 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 354.73351187100002, 2023-09-23T19:08:52.435 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.98151502700000004, 2023-09-23T19:08:52.435 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:08:52.435 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:08:52.435 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:08:52.436 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:08:52.436 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:08:52.436 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 114902 2023-09-23T19:08:52.436 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.436 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:08:52.436 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.436 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:08:52.436 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.689552+0000", 2023-09-23T19:08:52.436 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.436 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.436 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.436 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:08:52.436 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.689552+0000", 2023-09-23T19:08:52.437 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.437 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.437 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.437 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:08:52.437 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.689550+0000", 2023-09-23T19:08:52.437 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:08:52.437 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.437 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.437 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:08:52.437 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.689560+0000", 2023-09-23T19:08:52.437 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.8779999999999993e-06 2023-09-23T19:08:52.437 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.437 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.438 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:08:52.438 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.689578+0000", 2023-09-23T19:08:52.438 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.7966999999999999e-05 2023-09-23T19:08:52.438 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.438 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.438 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:08:52.438 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.689588+0000", 2023-09-23T19:08:52.438 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.7969999999999995e-06 2023-09-23T19:08:52.438 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.438 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.438 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.438 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.689648+0000", 2023-09-23T19:08:52.439 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.9723999999999999e-05 2023-09-23T19:08:52.439 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.439 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.439 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:08:52.439 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.689657+0000", 2023-09-23T19:08:52.439 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.6490000000000003e-06 2023-09-23T19:08:52.439 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.439 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.439 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.439 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671060+0000", 2023-09-23T19:08:52.439 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.98140266799999998 2023-09-23T19:08:52.439 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.439 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.440 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:08:52.440 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671067+0000", 2023-09-23T19:08:52.440 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.4830000000000002e-06 2023-09-23T19:08:52.440 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.440 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:08:52.440 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.440 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.440 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.440 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:114909 10.c 10.dad3dbac (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:08:52.440 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.705355+0000", 2023-09-23T19:08:52.440 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 354.71770909399999, 2023-09-23T19:08:52.440 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.96574279900000004, 2023-09-23T19:08:52.441 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:08:52.441 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:08:52.441 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:08:52.441 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:08:52.441 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:08:52.441 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 114909 2023-09-23T19:08:52.441 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.441 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:08:52.441 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.441 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:08:52.441 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705355+0000", 2023-09-23T19:08:52.441 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.442 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.442 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.442 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:08:52.442 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705355+0000", 2023-09-23T19:08:52.442 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.442 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.442 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.442 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:08:52.442 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705353+0000", 2023-09-23T19:08:52.442 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:08:52.442 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.442 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.443 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:08:52.443 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705364+0000", 2023-09-23T19:08:52.443 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1090999999999999e-05 2023-09-23T19:08:52.443 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.443 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.443 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:08:52.443 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705386+0000", 2023-09-23T19:08:52.443 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.2014e-05 2023-09-23T19:08:52.443 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.443 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.444 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:08:52.444 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705398+0000", 2023-09-23T19:08:52.444 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.15e-05 2023-09-23T19:08:52.444 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.444 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.444 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.444 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705530+0000", 2023-09-23T19:08:52.444 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00013205899999999999 2023-09-23T19:08:52.444 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.444 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.444 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:08:52.444 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705538+0000", 2023-09-23T19:08:52.445 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.6349999999999995e-06 2023-09-23T19:08:52.445 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.445 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.445 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.445 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671090+0000", 2023-09-23T19:08:52.445 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.96555181099999998 2023-09-23T19:08:52.445 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.445 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.445 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:08:52.445 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671098+0000", 2023-09-23T19:08:52.445 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.6599999999999995e-06 2023-09-23T19:08:52.446 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.446 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:08:52.446 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.446 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.446 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.446 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:114918 10.c 10.4f6f543c (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:08:52.446 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.705455+0000", 2023-09-23T19:08:52.446 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 354.71760894699997, 2023-09-23T19:08:52.446 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.96567159899999999, 2023-09-23T19:08:52.446 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:08:52.446 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:08:52.447 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:08:52.447 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:08:52.447 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:08:52.447 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 114918 2023-09-23T19:08:52.447 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.447 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:08:52.447 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.447 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:08:52.447 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705455+0000", 2023-09-23T19:08:52.447 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.447 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.447 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.448 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:08:52.448 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705455+0000", 2023-09-23T19:08:52.448 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.448 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.448 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.448 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:08:52.448 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705455+0000", 2023-09-23T19:08:52.448 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T19:08:52.448 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.448 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.448 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:08:52.448 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705458+0000", 2023-09-23T19:08:52.448 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.1319999999999998e-06 2023-09-23T19:08:52.449 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.449 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.449 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:08:52.449 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705478+0000", 2023-09-23T19:08:52.449 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.9837999999999999e-05 2023-09-23T19:08:52.449 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.449 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.449 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:08:52.449 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705484+0000", 2023-09-23T19:08:52.449 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.5230000000000001e-06 2023-09-23T19:08:52.449 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.450 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.450 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.450 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705561+0000", 2023-09-23T19:08:52.450 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.6579e-05 2023-09-23T19:08:52.450 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.450 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.450 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:08:52.450 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.705566+0000", 2023-09-23T19:08:52.450 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.2209999999999996e-06 2023-09-23T19:08:52.450 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.450 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.450 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.451 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671119+0000", 2023-09-23T19:08:52.451 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.96555351499999997 2023-09-23T19:08:52.451 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.451 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.451 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:08:52.451 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671127+0000", 2023-09-23T19:08:52.451 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.3479999999999998e-06 2023-09-23T19:08:52.451 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.451 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:08:52.451 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.451 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.451 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.452 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:114946 10.c 10.215a248c (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:08:52.452 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.753910+0000", 2023-09-23T19:08:52.452 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 354.66915399200002, 2023-09-23T19:08:52.452 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.91724539999999999, 2023-09-23T19:08:52.452 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:08:52.452 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:08:52.452 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:08:52.452 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:08:52.452 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:08:52.452 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 114946 2023-09-23T19:08:52.452 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.453 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:08:52.453 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.453 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:08:52.453 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.753910+0000", 2023-09-23T19:08:52.453 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.453 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.453 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.453 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:08:52.453 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.753910+0000", 2023-09-23T19:08:52.453 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.453 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.453 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.454 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:08:52.454 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.753909+0000", 2023-09-23T19:08:52.454 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T19:08:52.454 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.454 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.454 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:08:52.454 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.753916+0000", 2023-09-23T19:08:52.454 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.6200000000000001e-06 2023-09-23T19:08:52.454 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.454 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.454 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:08:52.454 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.753927+0000", 2023-09-23T19:08:52.455 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1603999999999999e-05 2023-09-23T19:08:52.455 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.455 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.455 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:08:52.455 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.753936+0000", 2023-09-23T19:08:52.455 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.6810000000000004e-06 2023-09-23T19:08:52.455 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.455 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.455 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.455 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.753986+0000", 2023-09-23T19:08:52.455 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.9886000000000003e-05 2023-09-23T19:08:52.455 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.456 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.456 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:08:52.456 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.753993+0000", 2023-09-23T19:08:52.456 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.6459999999999993e-06 2023-09-23T19:08:52.456 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.456 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.456 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.456 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671148+0000", 2023-09-23T19:08:52.456 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.91715488000000001 2023-09-23T19:08:52.456 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.456 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.457 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:08:52.457 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671156+0000", 2023-09-23T19:08:52.457 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.3669999999999999e-06 2023-09-23T19:08:52.457 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.457 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:08:52.457 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.457 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.457 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.457 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:114982 10.c 10.48e0bdac (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:08:52.457 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.803538+0000", 2023-09-23T19:08:52.457 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 354.61952629000001, 2023-09-23T19:08:52.457 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.86764724699999995, 2023-09-23T19:08:52.457 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:08:52.458 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:08:52.458 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:08:52.458 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:08:52.458 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:08:52.458 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 114982 2023-09-23T19:08:52.458 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.458 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:08:52.458 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.458 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:08:52.458 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.803538+0000", 2023-09-23T19:08:52.458 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.458 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.459 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.459 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:08:52.459 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.803538+0000", 2023-09-23T19:08:52.459 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.459 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.459 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.459 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:08:52.459 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.803536+0000", 2023-09-23T19:08:52.459 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:08:52.459 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.459 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.459 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:08:52.460 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.803545+0000", 2023-09-23T19:08:52.460 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.755e-06 2023-09-23T19:08:52.460 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.460 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.460 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:08:52.460 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.803559+0000", 2023-09-23T19:08:52.460 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3834e-05 2023-09-23T19:08:52.460 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.460 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.460 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:08:52.460 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.803568+0000", 2023-09-23T19:08:52.460 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.6190000000000006e-06 2023-09-23T19:08:52.460 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.460 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.461 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.461 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.803647+0000", 2023-09-23T19:08:52.461 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.8288999999999995e-05 2023-09-23T19:08:52.461 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.461 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.461 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:08:52.461 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.803656+0000", 2023-09-23T19:08:52.461 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.0119999999999994e-06 2023-09-23T19:08:52.461 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.461 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.461 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.461 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671178+0000", 2023-09-23T19:08:52.462 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.86752198000000003 2023-09-23T19:08:52.462 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.462 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.462 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:08:52.462 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671185+0000", 2023-09-23T19:08:52.462 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.542e-06 2023-09-23T19:08:52.462 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.462 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:08:52.462 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.462 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.462 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.463 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:114993 10.c 10.7d901f2c (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:08:52.463 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.817716+0000", 2023-09-23T19:08:52.463 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 354.60534866299997, 2023-09-23T19:08:52.463 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.85350135100000002, 2023-09-23T19:08:52.463 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:08:52.463 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:08:52.463 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:08:52.463 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:08:52.463 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:08:52.463 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 114993 2023-09-23T19:08:52.463 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.463 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:08:52.463 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.464 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:08:52.464 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.817716+0000", 2023-09-23T19:08:52.464 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.464 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.464 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.464 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:08:52.464 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.817716+0000", 2023-09-23T19:08:52.464 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.464 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.464 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.464 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:08:52.464 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.817713+0000", 2023-09-23T19:08:52.465 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:08:52.465 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.465 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.465 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:08:52.465 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.817723+0000", 2023-09-23T19:08:52.465 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.5510000000000005e-06 2023-09-23T19:08:52.465 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.465 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.465 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:08:52.465 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.817745+0000", 2023-09-23T19:08:52.465 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.2266999999999999e-05 2023-09-23T19:08:52.465 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.465 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.466 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:08:52.466 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.817756+0000", 2023-09-23T19:08:52.466 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1134e-05 2023-09-23T19:08:52.466 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.466 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.466 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.466 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.817816+0000", 2023-09-23T19:08:52.466 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.9503999999999998e-05 2023-09-23T19:08:52.466 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.466 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.466 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:08:52.466 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.817827+0000", 2023-09-23T19:08:52.467 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0932e-05 2023-09-23T19:08:52.467 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.467 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.467 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.467 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671210+0000", 2023-09-23T19:08:52.467 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.85338271600000004 2023-09-23T19:08:52.467 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.467 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.467 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:08:52.467 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671217+0000", 2023-09-23T19:08:52.467 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.3470000000000001e-06 2023-09-23T19:08:52.467 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.467 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:08:52.468 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.468 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.468 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.468 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:115010 10.c 10.5c53734c (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:08:52.468 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.845185+0000", 2023-09-23T19:08:52.468 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 354.57787883499998, 2023-09-23T19:08:52.468 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.826060935, 2023-09-23T19:08:52.468 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:08:52.468 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:08:52.468 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:08:52.468 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:08:52.468 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:08:52.468 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 115010 2023-09-23T19:08:52.468 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.469 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:08:52.469 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.469 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:08:52.469 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.845185+0000", 2023-09-23T19:08:52.469 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.469 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.469 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.469 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:08:52.469 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.845185+0000", 2023-09-23T19:08:52.469 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.469 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.469 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.469 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:08:52.470 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.845184+0000", 2023-09-23T19:08:52.470 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:08:52.470 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.470 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.470 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:08:52.470 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.845193+0000", 2023-09-23T19:08:52.470 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.2490000000000007e-06 2023-09-23T19:08:52.470 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.470 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.470 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:08:52.470 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.845209+0000", 2023-09-23T19:08:52.471 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.6053e-05 2023-09-23T19:08:52.471 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.471 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.471 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:08:52.471 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.845219+0000", 2023-09-23T19:08:52.471 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.7850000000000003e-06 2023-09-23T19:08:52.471 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.471 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.471 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.471 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.845280+0000", 2023-09-23T19:08:52.471 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.1080999999999993e-05 2023-09-23T19:08:52.471 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.471 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.472 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:08:52.472 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.845291+0000", 2023-09-23T19:08:52.472 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1334e-05 2023-09-23T19:08:52.472 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.472 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.472 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.472 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671239+0000", 2023-09-23T19:08:52.472 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.82594783400000005 2023-09-23T19:08:52.472 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.472 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.472 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:08:52.472 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671246+0000", 2023-09-23T19:08:52.473 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.357e-06 2023-09-23T19:08:52.473 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.473 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:08:52.473 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.473 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.473 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.473 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:115036 10.c 10.a31d034c (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:08:52.473 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.874971+0000", 2023-09-23T19:08:52.473 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 354.548093618, 2023-09-23T19:08:52.473 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.79630574600000004, 2023-09-23T19:08:52.473 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:08:52.473 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:08:52.474 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:08:52.474 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:08:52.474 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:08:52.474 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 115036 2023-09-23T19:08:52.474 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.474 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:08:52.474 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.474 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:08:52.474 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.874971+0000", 2023-09-23T19:08:52.474 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.474 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.474 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.475 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:08:52.475 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.874971+0000", 2023-09-23T19:08:52.475 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.475 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.475 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.475 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:08:52.475 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.874969+0000", 2023-09-23T19:08:52.475 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:08:52.475 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.475 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.475 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:08:52.475 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.874978+0000", 2023-09-23T19:08:52.476 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.7679999999999997e-06 2023-09-23T19:08:52.476 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.476 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.476 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:08:52.476 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.874995+0000", 2023-09-23T19:08:52.476 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.7399999999999999e-05 2023-09-23T19:08:52.476 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.476 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.476 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:08:52.476 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.875006+0000", 2023-09-23T19:08:52.476 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0742e-05 2023-09-23T19:08:52.476 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.476 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.477 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.477 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.875054+0000", 2023-09-23T19:08:52.477 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.8347000000000003e-05 2023-09-23T19:08:52.477 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.477 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.477 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:08:52.477 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.875064+0000", 2023-09-23T19:08:52.477 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0336999999999999e-05 2023-09-23T19:08:52.477 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.477 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.477 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.477 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671269+0000", 2023-09-23T19:08:52.477 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.79620438500000001 2023-09-23T19:08:52.478 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.478 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.478 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:08:52.478 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671276+0000", 2023-09-23T19:08:52.478 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.4610000000000002e-06 2023-09-23T19:08:52.478 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.478 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:08:52.478 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.478 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.478 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.478 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:115037 10.c 10.f007beec (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:08:52.478 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.878380+0000", 2023-09-23T19:08:52.479 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 354.544683754, 2023-09-23T19:08:52.479 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.79292473900000005, 2023-09-23T19:08:52.479 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:08:52.479 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:08:52.479 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:08:52.479 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:08:52.479 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:08:52.479 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 115037 2023-09-23T19:08:52.479 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.479 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:08:52.479 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.479 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:08:52.479 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.878380+0000", 2023-09-23T19:08:52.480 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.480 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.480 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.480 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:08:52.480 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.878380+0000", 2023-09-23T19:08:52.480 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.480 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.480 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.480 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:08:52.480 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.878379+0000", 2023-09-23T19:08:52.480 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T19:08:52.480 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.480 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.481 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:08:52.481 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.878387+0000", 2023-09-23T19:08:52.481 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.6040000000000004e-06 2023-09-23T19:08:52.481 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.481 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.481 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:08:52.481 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.878403+0000", 2023-09-23T19:08:52.481 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.6439999999999998e-05 2023-09-23T19:08:52.481 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.481 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.481 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:08:52.482 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.878411+0000", 2023-09-23T19:08:52.482 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.8429999999999993e-06 2023-09-23T19:08:52.482 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.482 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.482 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.482 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.878467+0000", 2023-09-23T19:08:52.482 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.6164e-05 2023-09-23T19:08:52.482 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.482 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.482 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:08:52.482 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.878478+0000", 2023-09-23T19:08:52.482 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0536e-05 2023-09-23T19:08:52.482 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.483 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.483 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.483 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671297+0000", 2023-09-23T19:08:52.483 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.79281938900000004 2023-09-23T19:08:52.483 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.483 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.483 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:08:52.483 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671305+0000", 2023-09-23T19:08:52.483 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.1790000000000008e-06 2023-09-23T19:08:52.483 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.483 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:08:52.483 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.483 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.484 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.484 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:115041 10.c 10.5e5c63ec (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:08:52.484 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.884610+0000", 2023-09-23T19:08:52.484 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 354.53845429699999, 2023-09-23T19:08:52.484 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.786727434, 2023-09-23T19:08:52.484 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:08:52.484 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:08:52.484 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:08:52.484 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:08:52.484 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:08:52.484 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 115041 2023-09-23T19:08:52.484 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.485 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:08:52.485 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.485 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:08:52.485 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.884610+0000", 2023-09-23T19:08:52.485 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.485 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.485 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.485 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:08:52.485 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.884610+0000", 2023-09-23T19:08:52.485 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.485 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.485 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.486 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:08:52.486 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.884608+0000", 2023-09-23T19:08:52.486 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:08:52.486 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.486 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.486 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:08:52.486 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.884617+0000", 2023-09-23T19:08:52.486 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.0280000000000006e-06 2023-09-23T19:08:52.486 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.486 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.486 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:08:52.487 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.884644+0000", 2023-09-23T19:08:52.487 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.7083000000000001e-05 2023-09-23T19:08:52.487 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.487 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.487 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:08:52.487 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.884658+0000", 2023-09-23T19:08:52.487 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3298999999999999e-05 2023-09-23T19:08:52.487 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.487 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.487 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.487 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.884724+0000", 2023-09-23T19:08:52.488 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.6623000000000002e-05 2023-09-23T19:08:52.488 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.488 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.488 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:08:52.488 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.884738+0000", 2023-09-23T19:08:52.488 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3377000000000001e-05 2023-09-23T19:08:52.488 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.488 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.488 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.488 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671330+0000", 2023-09-23T19:08:52.488 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.78659241000000002 2023-09-23T19:08:52.488 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.488 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.489 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:08:52.489 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671337+0000", 2023-09-23T19:08:52.489 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.3599999999999998e-06 2023-09-23T19:08:52.489 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.489 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:08:52.489 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.489 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.489 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.489 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:115065 10.c 10.355e755c (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:08:52.489 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.919459+0000", 2023-09-23T19:08:52.489 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 354.50360484399999, 2023-09-23T19:08:52.490 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.75190801100000004, 2023-09-23T19:08:52.490 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:08:52.490 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:08:52.490 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:08:52.490 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:08:52.490 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:08:52.490 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 115065 2023-09-23T19:08:52.490 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.490 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:08:52.490 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.490 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:08:52.491 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.919459+0000", 2023-09-23T19:08:52.491 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.491 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.491 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.491 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:08:52.491 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.919459+0000", 2023-09-23T19:08:52.491 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.491 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.491 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.491 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:08:52.491 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.919457+0000", 2023-09-23T19:08:52.491 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:08:52.492 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.492 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.492 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:08:52.492 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.919466+0000", 2023-09-23T19:08:52.492 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.6889999999999993e-06 2023-09-23T19:08:52.492 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.492 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.492 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:08:52.492 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.919484+0000", 2023-09-23T19:08:52.492 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.8042000000000001e-05 2023-09-23T19:08:52.492 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.492 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.493 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:08:52.493 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.919493+0000", 2023-09-23T19:08:52.493 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.4950000000000008e-06 2023-09-23T19:08:52.493 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.493 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.493 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.493 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.919534+0000", 2023-09-23T19:08:52.493 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.0978e-05 2023-09-23T19:08:52.493 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.493 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.493 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:08:52.493 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.919540+0000", 2023-09-23T19:08:52.494 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.7179999999999999e-06 2023-09-23T19:08:52.494 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.494 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.494 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.494 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671360+0000", 2023-09-23T19:08:52.494 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.75181916800000004 2023-09-23T19:08:52.494 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.494 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.494 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:08:52.494 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671367+0000", 2023-09-23T19:08:52.494 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.7589999999999997e-06 2023-09-23T19:08:52.494 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.495 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:08:52.495 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.495 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.495 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.495 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:115076 10.c 10.4cfe028c (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:08:52.495 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.938742+0000", 2023-09-23T19:08:52.495 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 354.48432194200001, 2023-09-23T19:08:52.495 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.73265688399999995, 2023-09-23T19:08:52.495 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:08:52.495 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:08:52.495 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:08:52.495 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:08:52.496 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:08:52.496 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 115076 2023-09-23T19:08:52.496 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.496 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:08:52.496 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.496 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:08:52.496 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938742+0000", 2023-09-23T19:08:52.496 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.496 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.496 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.496 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:08:52.496 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938742+0000", 2023-09-23T19:08:52.497 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.497 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.497 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.497 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:08:52.497 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938740+0000", 2023-09-23T19:08:52.497 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:08:52.497 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.497 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.497 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:08:52.497 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938750+0000", 2023-09-23T19:08:52.497 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.9259999999999995e-06 2023-09-23T19:08:52.497 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.497 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.498 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:08:52.498 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938773+0000", 2023-09-23T19:08:52.498 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.2254000000000001e-05 2023-09-23T19:08:52.498 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.498 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.498 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:08:52.498 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938784+0000", 2023-09-23T19:08:52.498 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.116e-05 2023-09-23T19:08:52.498 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.498 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.498 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.498 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938846+0000", 2023-09-23T19:08:52.498 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.2718999999999994e-05 2023-09-23T19:08:52.499 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.499 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.499 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:08:52.499 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938858+0000", 2023-09-23T19:08:52.499 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1584e-05 2023-09-23T19:08:52.499 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.499 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.499 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.499 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671391+0000", 2023-09-23T19:08:52.499 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.73253294700000005 2023-09-23T19:08:52.499 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.499 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.499 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:08:52.500 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671399+0000", 2023-09-23T19:08:52.500 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.14e-06 2023-09-23T19:08:52.500 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.500 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:08:52.500 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.500 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.500 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.500 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:115077 10.c 10.7476880c (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:08:52.500 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.938879+0000", 2023-09-23T19:08:52.500 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 354.48418514299999, 2023-09-23T19:08:52.500 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.73255084199999998, 2023-09-23T19:08:52.500 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:08:52.500 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:08:52.501 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:08:52.501 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:08:52.501 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:08:52.501 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 115077 2023-09-23T19:08:52.501 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.501 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:08:52.501 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.501 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:08:52.501 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938879+0000", 2023-09-23T19:08:52.501 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.501 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.501 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.502 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:08:52.502 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938879+0000", 2023-09-23T19:08:52.502 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.502 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.502 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.502 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:08:52.502 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938878+0000", 2023-09-23T19:08:52.502 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T19:08:52.502 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.502 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.502 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:08:52.502 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938883+0000", 2023-09-23T19:08:52.503 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.0000000000000004e-06 2023-09-23T19:08:52.503 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.503 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.503 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:08:52.503 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938893+0000", 2023-09-23T19:08:52.503 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.7699999999999996e-06 2023-09-23T19:08:52.503 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.503 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.503 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:08:52.503 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938899+0000", 2023-09-23T19:08:52.503 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.9399999999999999e-06 2023-09-23T19:08:52.503 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.503 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.504 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.504 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938939+0000", 2023-09-23T19:08:52.504 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.9799999999999998e-05 2023-09-23T19:08:52.504 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.504 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.504 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:08:52.504 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.938948+0000", 2023-09-23T19:08:52.504 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.2720000000000003e-06 2023-09-23T19:08:52.504 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.504 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.504 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.504 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671422+0000", 2023-09-23T19:08:52.504 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.732474128 2023-09-23T19:08:52.504 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.505 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.505 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:08:52.505 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671430+0000", 2023-09-23T19:08:52.505 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.7100000000000007e-06 2023-09-23T19:08:52.505 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.505 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:08:52.505 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.505 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.505 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.505 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:115098 10.c 10.c815cebc (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:08:52.505 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:57.996592+0000", 2023-09-23T19:08:52.505 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 354.42647211000002, 2023-09-23T19:08:52.505 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.67486956499999995, 2023-09-23T19:08:52.506 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:08:52.506 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:08:52.506 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:08:52.506 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:08:52.506 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:08:52.506 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 115098 2023-09-23T19:08:52.506 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.506 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:08:52.506 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.506 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:08:52.506 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.996592+0000", 2023-09-23T19:08:52.506 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.506 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.507 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.507 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:08:52.507 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.996592+0000", 2023-09-23T19:08:52.507 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.507 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.507 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.507 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:08:52.507 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.996591+0000", 2023-09-23T19:08:52.507 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T19:08:52.507 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.507 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.507 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:08:52.507 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.996599+0000", 2023-09-23T19:08:52.507 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.2970000000000004e-06 2023-09-23T19:08:52.508 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.508 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.508 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:08:52.508 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.996612+0000", 2023-09-23T19:08:52.508 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3096e-05 2023-09-23T19:08:52.508 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.508 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.508 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:08:52.508 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.996621+0000", 2023-09-23T19:08:52.508 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.6039999999999993e-06 2023-09-23T19:08:52.508 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.508 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.508 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.508 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.996667+0000", 2023-09-23T19:08:52.509 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.6724000000000002e-05 2023-09-23T19:08:52.509 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.509 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.509 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:08:52.509 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:57.996677+0000", 2023-09-23T19:08:52.509 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0192e-05 2023-09-23T19:08:52.509 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.509 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.509 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.509 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671454+0000", 2023-09-23T19:08:52.509 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.67477629900000002 2023-09-23T19:08:52.509 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.510 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.510 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:08:52.510 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671462+0000", 2023-09-23T19:08:52.510 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.8390000000000007e-06 2023-09-23T19:08:52.510 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.510 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:08:52.510 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.510 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.510 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.510 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:115102 10.c 10.fc7abffc (undecoded) ondisk+write+known_if_redirected e326)", 2023-09-23T19:08:52.510 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:58.016377+0000", 2023-09-23T19:08:52.510 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 354.40668742399998, 2023-09-23T19:08:52.511 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.65511980000000003, 2023-09-23T19:08:52.511 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:08:52.511 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "reached pg", 2023-09-23T19:08:52.511 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:08:52.511 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:08:52.511 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:08:52.511 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 115102 2023-09-23T19:08:52.511 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.511 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:08:52.511 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.511 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:08:52.511 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.016377+0000", 2023-09-23T19:08:52.511 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.511 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.512 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.512 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:08:52.512 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.016377+0000", 2023-09-23T19:08:52.512 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.512 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.512 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.512 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:08:52.512 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.016375+0000", 2023-09-23T19:08:52.512 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:08:52.512 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.512 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.512 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:08:52.512 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.016384+0000", 2023-09-23T19:08:52.513 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.0839999999999998e-06 2023-09-23T19:08:52.513 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.513 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.513 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:08:52.513 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.016401+0000", 2023-09-23T19:08:52.513 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.6518e-05 2023-09-23T19:08:52.513 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.513 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.513 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:08:52.513 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.016409+0000", 2023-09-23T19:08:52.513 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.7830000000000004e-06 2023-09-23T19:08:52.513 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.513 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.513 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.514 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.016454+0000", 2023-09-23T19:08:52.514 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.4481000000000003e-05 2023-09-23T19:08:52.514 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.514 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.514 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:08:52.514 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.016462+0000", 2023-09-23T19:08:52.514 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.7849999999999992e-06 2023-09-23T19:08:52.514 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.514 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.514 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.514 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671488+0000", 2023-09-23T19:08:52.514 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.65502604499999995 2023-09-23T19:08:52.515 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.515 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.515 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:08:52.515 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:58.671497+0000", 2023-09-23T19:08:52.515 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.9309999999999997e-06 2023-09-23T19:08:52.515 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.515 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:08:52.515 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.515 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.515 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.515 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:194009 10.f 10:fd5f9473:::benchmark_data_smithi098_230241_object93547:head [delete] snapc 0=[] ondisk+write+known_if_redirected e335)", 2023-09-23T19:08:52.515 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:05:08.430031+0000", 2023-09-23T19:08:52.516 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 223.99303366699999, 2023-09-23T19:08:52.516 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.22738197099999999, 2023-09-23T19:08:52.516 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:08:52.516 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:08:52.516 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:08:52.516 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:08:52.516 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:08:52.516 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 194009 2023-09-23T19:08:52.516 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.516 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:08:52.517 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.517 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:08:52.517 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.430031+0000", 2023-09-23T19:08:52.517 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.517 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.517 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.517 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:08:52.517 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.430031+0000", 2023-09-23T19:08:52.517 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.517 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.517 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.517 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:08:52.517 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.430029+0000", 2023-09-23T19:08:52.518 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T19:08:52.518 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.518 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.518 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:08:52.518 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.430036+0000", 2023-09-23T19:08:52.518 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.8800000000000002e-06 2023-09-23T19:08:52.518 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.518 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.518 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:08:52.518 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.430051+0000", 2023-09-23T19:08:52.518 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.4423000000000001e-05 2023-09-23T19:08:52.518 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.519 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.519 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:08:52.519 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.430059+0000", 2023-09-23T19:08:52.519 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.7169999999999997e-06 2023-09-23T19:08:52.519 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.519 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.519 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.519 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.446531+0000", 2023-09-23T19:08:52.519 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.016472189000000002 2023-09-23T19:08:52.519 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.519 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.519 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:08:52.519 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.446637+0000", 2023-09-23T19:08:52.520 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00010542199999999999 2023-09-23T19:08:52.520 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.520 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.520 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 2,6", 2023-09-23T19:08:52.520 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.446744+0000", 2023-09-23T19:08:52.520 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000107637 2023-09-23T19:08:52.520 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.520 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.520 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:08:52.520 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.449733+0000", 2023-09-23T19:08:52.520 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0029885369999999999 2023-09-23T19:08:52.520 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.521 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.521 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:08:52.521 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.454465+0000", 2023-09-23T19:08:52.521 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0047316880000000004 2023-09-23T19:08:52.521 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.521 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.521 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:08:52.521 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.657265+0000", 2023-09-23T19:08:52.521 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.20280068900000001 2023-09-23T19:08:52.521 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.521 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.521 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:08:52.522 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.657367+0000", 2023-09-23T19:08:52.522 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000101743 2023-09-23T19:08:52.522 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.522 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.522 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:08:52.522 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.657412+0000", 2023-09-23T19:08:52.522 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.5334999999999999e-05 2023-09-23T19:08:52.522 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.522 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:08:52.522 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.522 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.523 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.523 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45944.0:162 11.a 11:57642022:::image19:head [call rbd.snapshot_add in=15b] snapc 0=[] ondisk+write+known_if_redirected e361)", 2023-09-23T19:08:52.523 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:08:37.782679+0000", 2023-09-23T19:08:52.523 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 14.6403847, 2023-09-23T19:08:52.523 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0087987819999999998, 2023-09-23T19:08:52.523 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:08:52.523 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:08:52.523 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:08:52.523 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45944", 2023-09-23T19:08:52.523 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/1050517665", 2023-09-23T19:08:52.523 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 162 2023-09-23T19:08:52.523 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.524 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:08:52.524 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.524 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:08:52.524 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.782679+0000", 2023-09-23T19:08:52.524 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.524 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.524 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.524 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:08:52.524 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.782679+0000", 2023-09-23T19:08:52.524 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.524 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.524 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.525 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:08:52.525 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.782679+0000", 2023-09-23T19:08:52.525 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T19:08:52.525 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.525 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.525 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:08:52.525 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.782684+0000", 2023-09-23T19:08:52.525 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.6160000000000001e-06 2023-09-23T19:08:52.525 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.525 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.525 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:08:52.525 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.782695+0000", 2023-09-23T19:08:52.526 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1177e-05 2023-09-23T19:08:52.526 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.526 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.526 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:08:52.526 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.782705+0000", 2023-09-23T19:08:52.526 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.2019999999999993e-06 2023-09-23T19:08:52.526 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.526 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.526 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.526 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.782751+0000", 2023-09-23T19:08:52.527 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.6338999999999998e-05 2023-09-23T19:08:52.527 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.527 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.527 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:08:52.527 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.782817+0000", 2023-09-23T19:08:52.527 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.6204999999999994e-05 2023-09-23T19:08:52.527 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.527 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.527 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 5,6", 2023-09-23T19:08:52.527 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.783075+0000", 2023-09-23T19:08:52.527 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00025753499999999998 2023-09-23T19:08:52.528 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.528 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.528 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:08:52.528 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.790359+0000", 2023-09-23T19:08:52.528 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0072839460000000003 2023-09-23T19:08:52.528 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.528 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.528 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:08:52.528 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.791360+0000", 2023-09-23T19:08:52.528 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.001001818 2023-09-23T19:08:52.529 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.529 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.529 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:08:52.529 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.791398+0000", 2023-09-23T19:08:52.529 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.7826e-05 2023-09-23T19:08:52.529 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.529 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.529 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:08:52.529 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.791443+0000", 2023-09-23T19:08:52.529 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.4963000000000003e-05 2023-09-23T19:08:52.529 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.530 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.530 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:08:52.530 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.791478+0000", 2023-09-23T19:08:52.530 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.5052000000000003e-05 2023-09-23T19:08:52.530 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.530 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:08:52.530 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.530 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.530 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.530 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45944.0:164 11.a 11:57642022:::image19:head [call rbd.set_protection_status in=24b] snapc 0=[] ondisk+write+known_if_redirected e361)", 2023-09-23T19:08:52.530 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:08:37.792125+0000", 2023-09-23T19:08:52.530 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 14.630939254999999, 2023-09-23T19:08:52.533 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.010218023, 2023-09-23T19:08:52.534 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:08:52.534 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:08:52.534 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:08:52.534 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45944", 2023-09-23T19:08:52.534 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/1050517665", 2023-09-23T19:08:52.534 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 164 2023-09-23T19:08:52.534 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.534 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:08:52.534 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.534 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:08:52.534 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.792125+0000", 2023-09-23T19:08:52.535 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.535 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.535 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.535 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:08:52.535 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.792125+0000", 2023-09-23T19:08:52.535 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.535 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.535 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.535 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:08:52.535 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.792124+0000", 2023-09-23T19:08:52.535 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T19:08:52.535 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.535 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.536 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:08:52.536 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.792129+0000", 2023-09-23T19:08:52.536 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.4469999999999997e-06 2023-09-23T19:08:52.536 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.536 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.536 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:08:52.536 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.792140+0000", 2023-09-23T19:08:52.536 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0917999999999999e-05 2023-09-23T19:08:52.536 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.536 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.536 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:08:52.536 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.792150+0000", 2023-09-23T19:08:52.537 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.5950000000000005e-06 2023-09-23T19:08:52.537 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.537 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.537 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.537 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.792211+0000", 2023-09-23T19:08:52.537 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.1204000000000006e-05 2023-09-23T19:08:52.537 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.537 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.537 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:08:52.537 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.792297+0000", 2023-09-23T19:08:52.537 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.6253000000000002e-05 2023-09-23T19:08:52.537 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.537 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.538 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 5,6", 2023-09-23T19:08:52.538 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.792512+0000", 2023-09-23T19:08:52.538 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000214305 2023-09-23T19:08:52.538 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.538 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.538 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:08:52.538 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.793585+0000", 2023-09-23T19:08:52.538 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0010731099999999999 2023-09-23T19:08:52.538 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.538 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.538 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:08:52.538 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.793632+0000", 2023-09-23T19:08:52.539 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.7290999999999998e-05 2023-09-23T19:08:52.539 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.539 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.539 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:08:52.539 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.802195+0000", 2023-09-23T19:08:52.539 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0085627829999999992 2023-09-23T19:08:52.539 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.539 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.539 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:08:52.539 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.802264+0000", 2023-09-23T19:08:52.539 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.9338999999999995e-05 2023-09-23T19:08:52.539 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.540 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.540 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:08:52.540 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:37.802343+0000", 2023-09-23T19:08:52.540 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.8678999999999997e-05 2023-09-23T19:08:52.540 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.540 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:08:52.540 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.540 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.540 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.540 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45944.0:2920 11.9 11:94ddb4a8:::group_id_snap_list_max_read:head [call rbd.group_snap_set in=17b] snapc 0=[] ondisk+read+known_if_redirected e362)", 2023-09-23T19:08:52.540 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:08:42.396464+0000", 2023-09-23T19:08:52.540 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 10.026600074999999, 2023-09-23T19:08:52.541 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0035825399999999999, 2023-09-23T19:08:52.541 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:08:52.541 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:08:52.541 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:08:52.541 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45944", 2023-09-23T19:08:52.541 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/1050517665", 2023-09-23T19:08:52.541 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 2920 2023-09-23T19:08:52.541 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.541 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:08:52.541 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.541 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:08:52.541 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:42.396464+0000", 2023-09-23T19:08:52.542 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.542 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.542 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.542 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:08:52.542 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:42.396464+0000", 2023-09-23T19:08:52.542 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:08:52.542 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.542 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.542 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:08:52.542 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:42.396463+0000", 2023-09-23T19:08:52.542 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T19:08:52.542 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.543 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.543 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:08:52.543 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:42.396469+0000", 2023-09-23T19:08:52.543 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.4249999999999997e-06 2023-09-23T19:08:52.543 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.543 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.543 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:08:52.543 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:42.396479+0000", 2023-09-23T19:08:52.543 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0475000000000001e-05 2023-09-23T19:08:52.543 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.543 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.544 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:08:52.544 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:42.396485+0000", 2023-09-23T19:08:52.544 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.7200000000000003e-06 2023-09-23T19:08:52.544 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.544 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.544 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:08:52.544 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:42.396549+0000", 2023-09-23T19:08:52.544 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.4490000000000001e-05 2023-09-23T19:08:52.544 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.544 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.544 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:08:52.544 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:42.396711+0000", 2023-09-23T19:08:52.545 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000161974 2023-09-23T19:08:52.545 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.545 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.545 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,6", 2023-09-23T19:08:52.545 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:42.399025+0000", 2023-09-23T19:08:52.545 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.002313776 2023-09-23T19:08:52.545 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.545 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.545 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:08:52.545 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:42.399505+0000", 2023-09-23T19:08:52.545 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00048007900000000001 2023-09-23T19:08:52.545 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.546 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.546 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:08:52.546 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:42.399708+0000", 2023-09-23T19:08:52.546 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000203191 2023-09-23T19:08:52.546 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.546 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.546 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:08:52.546 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:42.399983+0000", 2023-09-23T19:08:52.546 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00027469300000000001 2023-09-23T19:08:52.546 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.546 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.546 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:08:52.546 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:42.400012+0000", 2023-09-23T19:08:52.547 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.8685000000000001e-05 2023-09-23T19:08:52.547 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:08:52.547 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:08:52.547 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:08:52.547 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:42.400047+0000", 2023-09-23T19:08:52.547 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.4904999999999999e-05 2023-09-23T19:08:52.547 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.547 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:08:52.547 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.547 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:08:52.547 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:08:52.547 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T19:08:52.548 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:52.579 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:52.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:52.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:52.683 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:52.711 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:52.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:52.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:52.793 INFO:tasks.workunit.client.0.smithi098.stderr:74+1 records in 2023-09-23T19:08:52.793 INFO:tasks.workunit.client.0.smithi098.stderr:74+1 records out 2023-09-23T19:08:52.793 INFO:tasks.workunit.client.0.smithi098.stderr:76160 bytes (76 kB, 74 KiB) copied, 0.469131 s, 162 kB/s 2023-09-23T19:08:52.793 INFO:tasks.workunit.client.0.smithi098.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_266403/img bs=1k seek=1000000 2023-09-23T19:08:52.815 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:52.842 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:52.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:52.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:52.925 INFO:tasks.workunit.client.0.smithi098.stderr:194+1 records in 2023-09-23T19:08:52.925 INFO:tasks.workunit.client.0.smithi098.stderr:194+1 records out 2023-09-23T19:08:52.925 INFO:tasks.workunit.client.0.smithi098.stderr:199136 bytes (199 kB, 194 KiB) copied, 0.130419 s, 1.5 MB/s 2023-09-23T19:08:52.925 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm testimg 2023-09-23T19:08:52.937 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:08:52.945 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:08:52.973 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:08:52.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:52.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:08:53.076 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:53.097 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:53.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:53.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:53.201 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:53.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:53.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:53.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:53.330 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:53.352 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 0% complete...failed. 2023-09-23T19:08:53.352 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T19:08:53.356 INFO:tasks.workunit.client.0.smithi098.stderr:+ true 2023-09-23T19:08:53.356 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --new-format /tmp/rbd_import_export_266403/img testimg 2023-09-23T19:08:53.366 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:53.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:53.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:53.470 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:08:53.490 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:08:53.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:53.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:08:53.594 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:53.620 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:53.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:53.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:53.725 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:53.750 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:53.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:53.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:53.854 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:53.866 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:53 smithi098 bash[79516]: cluster 2023-09-23T19:08:52.668755+0000 mgr.y (mgr.26054) 1539 : cluster 0 pgmap v1700: 73 pgs: 26 active+clean+remapped, 43 active+clean, 2 active+undersized+degraded, 2 active+undersized; 4.6 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 15/84 objects misplaced (17.857%) 2023-09-23T19:08:53.900 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:53.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:53.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:54.004 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:54.026 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:54.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:54.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:54.131 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:08:54.138 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:53 smithi173 bash[59422]: cluster 2023-09-23T19:08:52.668755+0000 mgr.y (mgr.26054) 1539 : cluster 0 pgmap v1700: 73 pgs: 26 active+clean+remapped, 43 active+clean, 2 active+undersized+degraded, 2 active+undersized; 4.6 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 15/84 objects misplaced (17.857%) 2023-09-23T19:08:54.156 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:08:54.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:54.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:08:54.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:53 smithi098 bash[76817]: cluster 2023-09-23T19:08:52.668755+0000 mgr.y (mgr.26054) 1539 : cluster 0 pgmap v1700: 73 pgs: 26 active+clean+remapped, 43 active+clean, 2 active+undersized+degraded, 2 active+undersized; 4.6 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 15/84 objects misplaced (17.857%) 2023-09-23T19:08:54.206 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-09-23T19:08:54.208 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T19:08:54.210 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export testimg /tmp/rbd_import_export_266403/img2 2023-09-23T19:08:54.260 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:54.282 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:54.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:54.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:54.386 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:54.543 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:54.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:54.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:54.550 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T19:08:54.554 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export testimg - 2023-09-23T19:08:54.647 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:54.669 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:54.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:54.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:54.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:54.796 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:54.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:54.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:54.903 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:54.928 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:54.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:54.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:55.032 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:55.051 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:55.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:55.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:55.154 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:55.180 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:55.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:55.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:55.285 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:08:55.315 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:08:55.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:55.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:08:55.417 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:55.437 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:55.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:55.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:55.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:55 smithi098 bash[79516]: audit 2023-09-23T19:08:54.189909+0000 mon.a (mon.0) 2078 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:08:55.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:55 smithi098 bash[79516]: audit 2023-09-23T19:08:54.198352+0000 mon.a (mon.0) 2079 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:08:55.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:55 smithi098 bash[79516]: audit 2023-09-23T19:08:54.654305+0000 mon.a (mon.0) 2080 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:08:55.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:55 smithi098 bash[79516]: audit 2023-09-23T19:08:54.655388+0000 mon.a (mon.0) 2081 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:08:55.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:55 smithi098 bash[76817]: audit 2023-09-23T19:08:54.189909+0000 mon.a (mon.0) 2078 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:08:55.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:55 smithi098 bash[76817]: audit 2023-09-23T19:08:54.198352+0000 mon.a (mon.0) 2079 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:08:55.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:55 smithi098 bash[76817]: audit 2023-09-23T19:08:54.654305+0000 mon.a (mon.0) 2080 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:08:55.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:55 smithi098 bash[76817]: audit 2023-09-23T19:08:54.655388+0000 mon.a (mon.0) 2081 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:08:55.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:55 smithi173 bash[59422]: audit 2023-09-23T19:08:54.189909+0000 mon.a (mon.0) 2078 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:08:55.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:55 smithi173 bash[59422]: audit 2023-09-23T19:08:54.198352+0000 mon.a (mon.0) 2079 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:08:55.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:55 smithi173 bash[59422]: audit 2023-09-23T19:08:54.654305+0000 mon.a (mon.0) 2080 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:08:55.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:55 smithi173 bash[59422]: audit 2023-09-23T19:08:54.655388+0000 mon.a (mon.0) 2081 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:08:55.540 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:55.547 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T19:08:55.551 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm testimg 2023-09-23T19:08:55.562 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:55.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:55.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:55.665 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:55.690 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:55.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:55.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:55.793 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:55.823 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:55.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:55.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:55.926 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:55.953 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:55.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:55.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:56.062 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:56.086 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:56.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:56.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:56.095 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T19:08:56.099 INFO:tasks.workunit.client.0.smithi098.stderr:+ cmp /tmp/rbd_import_export_266403/img /tmp/rbd_import_export_266403/img2 2023-09-23T19:08:56.190 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:56.221 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:56.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:56.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:56.324 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:08:56.350 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:08:56.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:56.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:08:56.443 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:56 smithi098 bash[79516]: cluster 2023-09-23T19:08:54.672171+0000 mgr.y (mgr.26054) 1540 : cluster 0 pgmap v1701: 73 pgs: 26 active+clean+remapped, 43 active+clean, 2 active+undersized+degraded, 2 active+undersized; 4.6 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 15/84 objects misplaced (17.857%) 2023-09-23T19:08:56.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:56 smithi098 bash[76817]: cluster 2023-09-23T19:08:54.672171+0000 mgr.y (mgr.26054) 1540 : cluster 0 pgmap v1701: 73 pgs: 26 active+clean+remapped, 43 active+clean, 2 active+undersized+degraded, 2 active+undersized; 4.6 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 15/84 objects misplaced (17.857%) 2023-09-23T19:08:56.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:08:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:08:56] "GET /metrics HTTP/1.1" 200 34182 "" "Prometheus/2.33.4" 2023-09-23T19:08:56.454 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:56.473 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:56.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:56.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:56.527 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:56 smithi173 bash[59422]: cluster 2023-09-23T19:08:54.672171+0000 mgr.y (mgr.26054) 1540 : cluster 0 pgmap v1701: 73 pgs: 26 active+clean+remapped, 43 active+clean, 2 active+undersized+degraded, 2 active+undersized; 4.6 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 15/84 objects misplaced (17.857%) 2023-09-23T19:08:56.577 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:56.600 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:56.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:56.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:56.703 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:56.730 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:56.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:56.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:56.834 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:56.858 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:08:56.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:56.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:08:56.937 INFO:tasks.workunit.client.0.smithi098.stderr:+ cmp /tmp/rbd_import_export_266403/img /tmp/rbd_import_export_266403/img3 2023-09-23T19:08:56.962 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:56.987 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:56.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:56.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:57.064 INFO:teuthology.orchestra.run.smithi173.stderr:no valid command found; 10 closest matches: 2023-09-23T19:08:57.064 INFO:teuthology.orchestra.run.smithi173.stderr:0 2023-09-23T19:08:57.064 INFO:teuthology.orchestra.run.smithi173.stderr:1 2023-09-23T19:08:57.065 INFO:teuthology.orchestra.run.smithi173.stderr:2 2023-09-23T19:08:57.065 INFO:teuthology.orchestra.run.smithi173.stderr:abort 2023-09-23T19:08:57.065 INFO:teuthology.orchestra.run.smithi173.stderr:assert 2023-09-23T19:08:57.065 INFO:teuthology.orchestra.run.smithi173.stderr:bluefs debug_inject_read_zeros 2023-09-23T19:08:57.065 INFO:teuthology.orchestra.run.smithi173.stderr:bluefs files list 2023-09-23T19:08:57.065 INFO:teuthology.orchestra.run.smithi173.stderr:bluefs stats 2023-09-23T19:08:57.065 INFO:teuthology.orchestra.run.smithi173.stderr:bluestore allocator dump block 2023-09-23T19:08:57.066 INFO:teuthology.orchestra.run.smithi173.stderr:bluestore allocator fragmentation block 2023-09-23T19:08:57.066 INFO:teuthology.orchestra.run.smithi173.stderr:admin_socket: invalid command 2023-09-23T19:08:57.097 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:57.124 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:57.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:57.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:57.227 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:57.253 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:57.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:57.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:57.356 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:57.370 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:57 smithi098 bash[79516]: cluster 2023-09-23T19:08:57.190555+0000 mon.a (mon.0) 2082 : cluster 3 Health check update: Degraded data redundancy: 3/87 objects degraded (3.448%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:08:57.370 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:57 smithi098 bash[76817]: cluster 2023-09-23T19:08:57.190555+0000 mon.a (mon.0) 2082 : cluster 3 Health check update: Degraded data redundancy: 3/87 objects degraded (3.448%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:08:57.385 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:57.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:57.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:57.489 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:57.494 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_266403/img2 /tmp/rbd_import_export_266403/img3 2023-09-23T19:08:57.496 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:57 smithi173 bash[59422]: cluster 2023-09-23T19:08:57.190555+0000 mon.a (mon.0) 2082 : cluster 3 Health check update: Degraded data redundancy: 3/87 objects degraded (3.448%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T19:08:57.512 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:57.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:57.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:57.614 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:57.682 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:57.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:57.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:57.786 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:08:57.810 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:08:57.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:57.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:08:57.913 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:57.941 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:57.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:57.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:58.005 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T19:08:58.006 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-7, ['dump_ops_in_flight'] 2023-09-23T19:08:58.044 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:58.075 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:58.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:58.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:58.178 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:58.213 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:08:58.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:58.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:08:58.316 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:58.330 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:08:58 smithi173 bash[59422]: cluster 2023-09-23T19:08:56.673168+0000 mgr.y (mgr.26054) 1541 : cluster 0 pgmap v1702: 73 pgs: 26 active+clean+remapped, 43 active+clean, 2 active+undersized+degraded, 2 active+undersized; 5.6 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 15 KiB/s rd, 7.4 KiB/s wr, 0 op/s; 3/87 objects degraded (3.448%); 15/87 objects misplaced (17.241%) 2023-09-23T19:08:58.346 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:58.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:58.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:58.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:08:58 smithi098 bash[76817]: cluster 2023-09-23T19:08:56.673168+0000 mgr.y (mgr.26054) 1541 : cluster 0 pgmap v1702: 73 pgs: 26 active+clean+remapped, 43 active+clean, 2 active+undersized+degraded, 2 active+undersized; 5.6 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 15 KiB/s rd, 7.4 KiB/s wr, 0 op/s; 3/87 objects degraded (3.448%); 15/87 objects misplaced (17.241%) 2023-09-23T19:08:58.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:08:58 smithi098 bash[79516]: cluster 2023-09-23T19:08:56.673168+0000 mgr.y (mgr.26054) 1541 : cluster 0 pgmap v1702: 73 pgs: 26 active+clean+remapped, 43 active+clean, 2 active+undersized+degraded, 2 active+undersized; 5.6 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 15 KiB/s rd, 7.4 KiB/s wr, 0 op/s; 3/87 objects degraded (3.448%); 15/87 objects misplaced (17.241%) 2023-09-23T19:08:58.449 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:58.480 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:58.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:58.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:58.584 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:58.613 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:58.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:58.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:58.716 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:58.744 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:58.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:58.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:58.847 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:58.877 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:58.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:58.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:08:58.980 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:59.012 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:59.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:59.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:59.116 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:59.151 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:59.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:59.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:59.255 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:59.283 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:08:59.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:59.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:08:59.387 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:59.418 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:59.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:59.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:59.521 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:59.549 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:59.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:59.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:59.654 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:59.688 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:08:59.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:59.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:08:59.778 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:08:59 smithi173 bash[162608]: debug 2023-09-23T19:08:59.519+0000 7fee52927200 -1 osd.7 270 log_to_monitors {default=true} 2023-09-23T19:08:59.791 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:59.821 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:08:59.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:59.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:08:59.924 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:59.957 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:08:59.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:08:59.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:00.061 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:00.093 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:00.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:00.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:00.197 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:00.230 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:00.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:00.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:00.333 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:00.345 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:00 smithi098 bash[79516]: cluster 2023-09-23T19:08:58.674188+0000 mgr.y (mgr.26054) 1542 : cluster 0 pgmap v1703: 73 pgs: 26 active+clean+remapped, 43 active+clean, 2 active+undersized+degraded, 2 active+undersized; 5.6 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 100 KiB/s rd, 43 KiB/s wr, 24 op/s; 3/87 objects degraded (3.448%); 15/87 objects misplaced (17.241%) 2023-09-23T19:09:00.346 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:00 smithi098 bash[79516]: audit 2023-09-23T19:08:59.524930+0000 mon.a (mon.0) 2083 : audit 1 from='osd.7 [v2:172.21.15.173:6824/1212718676,v1:172.21.15.173:6825/1212718676]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T19:09:00.346 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:00 smithi098 bash[76817]: cluster 2023-09-23T19:08:58.674188+0000 mgr.y (mgr.26054) 1542 : cluster 0 pgmap v1703: 73 pgs: 26 active+clean+remapped, 43 active+clean, 2 active+undersized+degraded, 2 active+undersized; 5.6 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 100 KiB/s rd, 43 KiB/s wr, 24 op/s; 3/87 objects degraded (3.448%); 15/87 objects misplaced (17.241%) 2023-09-23T19:09:00.346 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:00 smithi098 bash[76817]: audit 2023-09-23T19:08:59.524930+0000 mon.a (mon.0) 2083 : audit 1 from='osd.7 [v2:172.21.15.173:6824/1212718676,v1:172.21.15.173:6825/1212718676]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T19:09:00.360 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:00.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:00.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:00.464 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:00.492 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:00.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:00.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:00.528 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:09:00 smithi173 bash[162608]: debug 2023-09-23T19:09:00.383+0000 7fee4161d700 -1 osd.7 366 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T19:09:00.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:00 smithi173 bash[59422]: cluster 2023-09-23T19:08:58.674188+0000 mgr.y (mgr.26054) 1542 : cluster 0 pgmap v1703: 73 pgs: 26 active+clean+remapped, 43 active+clean, 2 active+undersized+degraded, 2 active+undersized; 5.6 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 100 KiB/s rd, 43 KiB/s wr, 24 op/s; 3/87 objects degraded (3.448%); 15/87 objects misplaced (17.241%) 2023-09-23T19:09:00.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:00 smithi173 bash[59422]: audit 2023-09-23T19:08:59.524930+0000 mon.a (mon.0) 2083 : audit 1 from='osd.7 [v2:172.21.15.173:6824/1212718676,v1:172.21.15.173:6825/1212718676]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T19:09:00.594 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:00.621 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:00.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:00.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:00.725 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:00.757 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:00.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:00.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:00.861 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:00.889 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:00.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:00.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:00.993 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:01.026 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:01.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:01.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:01.129 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:01.162 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:01.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:01.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:01.266 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:01.297 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:01.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:01.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:01.402 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:01.431 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:01.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:01.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:01.535 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:01.550 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:01 smithi098 bash[79516]: cluster 2023-09-23T19:09:00.266073+0000 osd.4 (osd.4) 99 : cluster 3 failed to encode map e366 with expected crc 2023-09-23T19:09:01.550 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:01 smithi098 bash[79516]: audit 2023-09-23T19:09:00.271481+0000 mon.a (mon.0) 2084 : audit 1 from='osd.7 [v2:172.21.15.173:6824/1212718676,v1:172.21.15.173:6825/1212718676]' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T19:09:01.550 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:01 smithi098 bash[79516]: cluster 2023-09-23T19:09:00.271577+0000 mon.a (mon.0) 2085 : cluster 0 osdmap e366: 8 total, 7 up, 7 in 2023-09-23T19:09:01.550 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:01 smithi098 bash[79516]: audit 2023-09-23T19:09:00.272433+0000 mon.a (mon.0) 2086 : audit 1 from='osd.7 [v2:172.21.15.173:6824/1212718676,v1:172.21.15.173:6825/1212718676]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T19:09:01.551 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:01 smithi098 bash[76817]: cluster 2023-09-23T19:09:00.266073+0000 osd.4 (osd.4) 99 : cluster 3 failed to encode map e366 with expected crc 2023-09-23T19:09:01.551 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:01 smithi098 bash[76817]: audit 2023-09-23T19:09:00.271481+0000 mon.a (mon.0) 2084 : audit 1 from='osd.7 [v2:172.21.15.173:6824/1212718676,v1:172.21.15.173:6825/1212718676]' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T19:09:01.551 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:01 smithi098 bash[76817]: cluster 2023-09-23T19:09:00.271577+0000 mon.a (mon.0) 2085 : cluster 0 osdmap e366: 8 total, 7 up, 7 in 2023-09-23T19:09:01.551 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:01 smithi098 bash[76817]: audit 2023-09-23T19:09:00.272433+0000 mon.a (mon.0) 2086 : audit 1 from='osd.7 [v2:172.21.15.173:6824/1212718676,v1:172.21.15.173:6825/1212718676]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T19:09:01.556 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:01.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:01.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:01.660 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:01.674 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:01 smithi173 bash[59422]: cluster 2023-09-23T19:09:00.266073+0000 osd.4 (osd.4) 99 : cluster 3 failed to encode map e366 with expected crc 2023-09-23T19:09:01.674 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:01 smithi173 bash[59422]: audit 2023-09-23T19:09:00.271481+0000 mon.a (mon.0) 2084 : audit 1 from='osd.7 [v2:172.21.15.173:6824/1212718676,v1:172.21.15.173:6825/1212718676]' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T19:09:01.674 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:01 smithi173 bash[59422]: cluster 2023-09-23T19:09:00.271577+0000 mon.a (mon.0) 2085 : cluster 0 osdmap e366: 8 total, 7 up, 7 in 2023-09-23T19:09:01.674 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:01 smithi173 bash[59422]: audit 2023-09-23T19:09:00.272433+0000 mon.a (mon.0) 2086 : audit 1 from='osd.7 [v2:172.21.15.173:6824/1212718676,v1:172.21.15.173:6825/1212718676]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-09-23T19:09:01.683 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --new-format - testimg 2023-09-23T19:09:01.691 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:01.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:01.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:01.796 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:01.826 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:01.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:01.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:01.929 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:01.966 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:01.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:01.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:02.069 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:02.098 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:02.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:02.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:02.202 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:02.230 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:02.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:02.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:02.334 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:02.358 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:02.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:02.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:02.462 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:02.493 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:02.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:02.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:02.596 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:02.610 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:02 smithi173 bash[59422]: cluster 2023-09-23T19:09:00.675099+0000 mgr.y (mgr.26054) 1543 : cluster 0 pgmap v1705: 73 pgs: 26 active+clean+remapped, 43 active+clean, 2 active+undersized+degraded, 2 active+undersized; 5.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 120 KiB/s rd, 51 KiB/s wr, 29 op/s; 3/87 objects degraded (3.448%); 15/87 objects misplaced (17.241%) 2023-09-23T19:09:02.611 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:02 smithi173 bash[59422]: cluster 2023-09-23T19:09:01.262192+0000 mon.a (mon.0) 2087 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:09:02.611 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:02 smithi173 bash[59422]: cluster 2023-09-23T19:09:01.269818+0000 osd.4 (osd.4) 100 : cluster 3 failed to encode map e367 with expected crc 2023-09-23T19:09:02.611 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:02 smithi173 bash[59422]: cluster 2023-09-23T19:09:01.271550+0000 osd.6 (osd.6) 358 : cluster 3 failed to encode map e367 with expected crc 2023-09-23T19:09:02.611 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:02 smithi173 bash[59422]: cluster 2023-09-23T19:09:01.271635+0000 mon.a (mon.0) 2088 : cluster 1 osd.7 [v2:172.21.15.173:6824/1212718676,v1:172.21.15.173:6825/1212718676] boot 2023-09-23T19:09:02.611 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:02 smithi173 bash[59422]: cluster 2023-09-23T19:09:01.271706+0000 mon.a (mon.0) 2089 : cluster 0 osdmap e367: 8 total, 8 up, 7 in 2023-09-23T19:09:02.611 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:02 smithi173 bash[59422]: audit 2023-09-23T19:09:01.272587+0000 mon.a (mon.0) 2090 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:09:02.611 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:02 smithi173 bash[59422]: cluster 2023-09-23T19:09:01.289866+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e366 with expected crc 2023-09-23T19:09:02.628 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:02.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:02.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:02.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:02 smithi098 bash[76817]: cluster 2023-09-23T19:09:00.675099+0000 mgr.y (mgr.26054) 1543 : cluster 0 pgmap v1705: 73 pgs: 26 active+clean+remapped, 43 active+clean, 2 active+undersized+degraded, 2 active+undersized; 5.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 120 KiB/s rd, 51 KiB/s wr, 29 op/s; 3/87 objects degraded (3.448%); 15/87 objects misplaced (17.241%) 2023-09-23T19:09:02.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:02 smithi098 bash[76817]: cluster 2023-09-23T19:09:01.262192+0000 mon.a (mon.0) 2087 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:09:02.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:02 smithi098 bash[76817]: cluster 2023-09-23T19:09:01.269818+0000 osd.4 (osd.4) 100 : cluster 3 failed to encode map e367 with expected crc 2023-09-23T19:09:02.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:02 smithi098 bash[76817]: cluster 2023-09-23T19:09:01.271550+0000 osd.6 (osd.6) 358 : cluster 3 failed to encode map e367 with expected crc 2023-09-23T19:09:02.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:02 smithi098 bash[76817]: cluster 2023-09-23T19:09:01.271635+0000 mon.a (mon.0) 2088 : cluster 1 osd.7 [v2:172.21.15.173:6824/1212718676,v1:172.21.15.173:6825/1212718676] boot 2023-09-23T19:09:02.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:02 smithi098 bash[76817]: cluster 2023-09-23T19:09:01.271706+0000 mon.a (mon.0) 2089 : cluster 0 osdmap e367: 8 total, 8 up, 7 in 2023-09-23T19:09:02.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:02 smithi098 bash[76817]: audit 2023-09-23T19:09:01.272587+0000 mon.a (mon.0) 2090 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:09:02.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:02 smithi098 bash[76817]: cluster 2023-09-23T19:09:01.289866+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e366 with expected crc 2023-09-23T19:09:02.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:02 smithi098 bash[79516]: cluster 2023-09-23T19:09:00.675099+0000 mgr.y (mgr.26054) 1543 : cluster 0 pgmap v1705: 73 pgs: 26 active+clean+remapped, 43 active+clean, 2 active+undersized+degraded, 2 active+undersized; 5.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 120 KiB/s rd, 51 KiB/s wr, 29 op/s; 3/87 objects degraded (3.448%); 15/87 objects misplaced (17.241%) 2023-09-23T19:09:02.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:02 smithi098 bash[79516]: cluster 2023-09-23T19:09:01.262192+0000 mon.a (mon.0) 2087 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T19:09:02.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:02 smithi098 bash[79516]: cluster 2023-09-23T19:09:01.269818+0000 osd.4 (osd.4) 100 : cluster 3 failed to encode map e367 with expected crc 2023-09-23T19:09:02.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:02 smithi098 bash[79516]: cluster 2023-09-23T19:09:01.271550+0000 osd.6 (osd.6) 358 : cluster 3 failed to encode map e367 with expected crc 2023-09-23T19:09:02.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:02 smithi098 bash[79516]: cluster 2023-09-23T19:09:01.271635+0000 mon.a (mon.0) 2088 : cluster 1 osd.7 [v2:172.21.15.173:6824/1212718676,v1:172.21.15.173:6825/1212718676] boot 2023-09-23T19:09:02.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:02 smithi098 bash[79516]: cluster 2023-09-23T19:09:01.271706+0000 mon.a (mon.0) 2089 : cluster 0 osdmap e367: 8 total, 8 up, 7 in 2023-09-23T19:09:02.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:02 smithi098 bash[79516]: audit 2023-09-23T19:09:01.272587+0000 mon.a (mon.0) 2090 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T19:09:02.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:02 smithi098 bash[79516]: cluster 2023-09-23T19:09:01.289866+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e366 with expected crc 2023-09-23T19:09:02.732 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:02.765 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:02.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:02.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:02.869 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:02.902 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:02.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:02.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:03.006 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:03.006 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T19:09:03.039 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:03.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:03.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:03.142 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:03.169 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:03.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:03.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:03.271 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:03.297 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:03.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:03.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:03.400 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:03.423 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:03.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:03.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:03.527 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:03 smithi173 bash[59422]: cluster 2023-09-23T19:09:00.291316+0000 osd.7 (osd.7) 1 : cluster 3 failed to encode map e271 with expected crc 2023-09-23T19:09:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:03 smithi173 bash[59422]: cluster 2023-09-23T19:09:00.301844+0000 osd.7 (osd.7) 2 : cluster 3 failed to encode map e311 with expected crc 2023-09-23T19:09:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:03 smithi173 bash[59422]: cluster 2023-09-23T19:09:00.331701+0000 osd.7 (osd.7) 3 : cluster 3 failed to encode map e351 with expected crc 2023-09-23T19:09:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:03 smithi173 bash[59422]: cluster 2023-09-23T19:09:01.270899+0000 osd.7 (osd.7) 4 : cluster 3 failed to encode map e367 with expected crc 2023-09-23T19:09:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:03 smithi173 bash[59422]: cluster 2023-09-23T19:09:01.274179+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e367 with expected crc 2023-09-23T19:09:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:03 smithi173 bash[59422]: cluster 2023-09-23T19:09:02.286363+0000 mon.a (mon.0) 2091 : cluster 0 osdmap e368: 8 total, 8 up, 7 in 2023-09-23T19:09:03.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:03 smithi173 bash[59422]: cluster 2023-09-23T19:09:02.286577+0000 osd.4 (osd.4) 101 : cluster 3 failed to encode map e368 with expected crc 2023-09-23T19:09:03.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:03 smithi173 bash[59422]: cluster 2023-09-23T19:09:02.287196+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e368 with expected crc 2023-09-23T19:09:03.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:03 smithi173 bash[59422]: cluster 2023-09-23T19:09:02.287309+0000 osd.6 (osd.6) 359 : cluster 3 failed to encode map e368 with expected crc 2023-09-23T19:09:03.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:03 smithi173 bash[59422]: cluster 2023-09-23T19:09:02.289734+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e368 with expected crc 2023-09-23T19:09:03.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:03 smithi173 bash[59422]: cluster 2023-09-23T19:09:02.289995+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e368 with expected crc 2023-09-23T19:09:03.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:03 smithi173 bash[59422]: cluster 2023-09-23T19:09:02.293870+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e368 with expected crc 2023-09-23T19:09:03.549 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:03.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:03.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:03.552 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:03 smithi098 bash[76817]: cluster 2023-09-23T19:09:00.291316+0000 osd.7 (osd.7) 1 : cluster 3 failed to encode map e271 with expected crc 2023-09-23T19:09:03.552 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:03 smithi098 bash[76817]: cluster 2023-09-23T19:09:00.301844+0000 osd.7 (osd.7) 2 : cluster 3 failed to encode map e311 with expected crc 2023-09-23T19:09:03.552 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:03 smithi098 bash[76817]: cluster 2023-09-23T19:09:00.331701+0000 osd.7 (osd.7) 3 : cluster 3 failed to encode map e351 with expected crc 2023-09-23T19:09:03.552 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:03 smithi098 bash[76817]: cluster 2023-09-23T19:09:01.270899+0000 osd.7 (osd.7) 4 : cluster 3 failed to encode map e367 with expected crc 2023-09-23T19:09:03.552 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:03 smithi098 bash[76817]: cluster 2023-09-23T19:09:01.274179+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e367 with expected crc 2023-09-23T19:09:03.553 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:03 smithi098 bash[76817]: cluster 2023-09-23T19:09:02.286363+0000 mon.a (mon.0) 2091 : cluster 0 osdmap e368: 8 total, 8 up, 7 in 2023-09-23T19:09:03.553 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:03 smithi098 bash[76817]: cluster 2023-09-23T19:09:02.286577+0000 osd.4 (osd.4) 101 : cluster 3 failed to encode map e368 with expected crc 2023-09-23T19:09:03.553 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:03 smithi098 bash[76817]: cluster 2023-09-23T19:09:02.287196+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e368 with expected crc 2023-09-23T19:09:03.553 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:03 smithi098 bash[76817]: cluster 2023-09-23T19:09:02.287309+0000 osd.6 (osd.6) 359 : cluster 3 failed to encode map e368 with expected crc 2023-09-23T19:09:03.553 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:03 smithi098 bash[76817]: cluster 2023-09-23T19:09:02.289734+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e368 with expected crc 2023-09-23T19:09:03.553 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:03 smithi098 bash[76817]: cluster 2023-09-23T19:09:02.289995+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e368 with expected crc 2023-09-23T19:09:03.553 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:03 smithi098 bash[76817]: cluster 2023-09-23T19:09:02.293870+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e368 with expected crc 2023-09-23T19:09:03.553 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:03 smithi098 bash[79516]: cluster 2023-09-23T19:09:00.291316+0000 osd.7 (osd.7) 1 : cluster 3 failed to encode map e271 with expected crc 2023-09-23T19:09:03.553 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:03 smithi098 bash[79516]: cluster 2023-09-23T19:09:00.301844+0000 osd.7 (osd.7) 2 : cluster 3 failed to encode map e311 with expected crc 2023-09-23T19:09:03.553 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:03 smithi098 bash[79516]: cluster 2023-09-23T19:09:00.331701+0000 osd.7 (osd.7) 3 : cluster 3 failed to encode map e351 with expected crc 2023-09-23T19:09:03.554 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:03 smithi098 bash[79516]: cluster 2023-09-23T19:09:01.270899+0000 osd.7 (osd.7) 4 : cluster 3 failed to encode map e367 with expected crc 2023-09-23T19:09:03.554 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:03 smithi098 bash[79516]: cluster 2023-09-23T19:09:01.274179+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e367 with expected crc 2023-09-23T19:09:03.554 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:03 smithi098 bash[79516]: cluster 2023-09-23T19:09:02.286363+0000 mon.a (mon.0) 2091 : cluster 0 osdmap e368: 8 total, 8 up, 7 in 2023-09-23T19:09:03.554 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:03 smithi098 bash[79516]: cluster 2023-09-23T19:09:02.286577+0000 osd.4 (osd.4) 101 : cluster 3 failed to encode map e368 with expected crc 2023-09-23T19:09:03.554 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:03 smithi098 bash[79516]: cluster 2023-09-23T19:09:02.287196+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e368 with expected crc 2023-09-23T19:09:03.554 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:03 smithi098 bash[79516]: cluster 2023-09-23T19:09:02.287309+0000 osd.6 (osd.6) 359 : cluster 3 failed to encode map e368 with expected crc 2023-09-23T19:09:03.554 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:03 smithi098 bash[79516]: cluster 2023-09-23T19:09:02.289734+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e368 with expected crc 2023-09-23T19:09:03.554 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:03 smithi098 bash[79516]: cluster 2023-09-23T19:09:02.289995+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e368 with expected crc 2023-09-23T19:09:03.554 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:03 smithi098 bash[79516]: cluster 2023-09-23T19:09:02.293870+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e368 with expected crc 2023-09-23T19:09:03.652 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:03.661 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T19:09:03.678 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:03.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:03.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:03.781 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:03.807 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:03.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:03.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:03.911 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:03.943 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:03.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:03.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:03.963 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 100% complete...done. 2023-09-23T19:09:03.964 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T19:09:03.968 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export testimg /tmp/rbd_import_export_266403/img2 2023-09-23T19:09:04.048 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:04.075 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:04.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:04.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:04.178 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:04.211 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:04.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:04.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:04.315 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:04.318 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T19:09:04.321 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export testimg - 2023-09-23T19:09:04.343 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:04.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:04.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:04.447 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:04.459 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:04 smithi173 bash[59422]: cluster 2023-09-23T19:09:02.675977+0000 mgr.y (mgr.26054) 1544 : cluster 0 pgmap v1708: 73 pgs: 4 remapped+peering, 9 peering, 16 active+clean+remapped, 42 active+clean, 1 active+undersized+degraded, 1 active+undersized; 4.6 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 171 KiB/s rd, 72 KiB/s wr, 49 op/s; 2/87 objects degraded (2.299%); 13/87 objects misplaced (14.943%) 2023-09-23T19:09:04.460 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:04 smithi173 bash[59422]: cluster 2023-09-23T19:09:03.281935+0000 mon.a (mon.0) 2092 : cluster 3 Health check update: Degraded data redundancy: 2/87 objects degraded (2.299%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T19:09:04.460 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:04 smithi173 bash[59422]: cluster 2023-09-23T19:09:03.294874+0000 osd.4 (osd.4) 102 : cluster 3 failed to encode map e369 with expected crc 2023-09-23T19:09:04.460 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:04 smithi173 bash[59422]: cluster 2023-09-23T19:09:03.295237+0000 osd.6 (osd.6) 360 : cluster 3 failed to encode map e369 with expected crc 2023-09-23T19:09:04.460 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:04 smithi173 bash[59422]: cluster 2023-09-23T19:09:03.295320+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e369 with expected crc 2023-09-23T19:09:04.460 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:04 smithi173 bash[59422]: cluster 2023-09-23T19:09:03.296072+0000 mon.a (mon.0) 2093 : cluster 0 osdmap e369: 8 total, 8 up, 7 in 2023-09-23T19:09:04.460 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:04 smithi173 bash[59422]: cluster 2023-09-23T19:09:03.299416+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e369 with expected crc 2023-09-23T19:09:04.460 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:04 smithi173 bash[59422]: cluster 2023-09-23T19:09:03.300075+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e369 with expected crc 2023-09-23T19:09:04.478 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:04.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:04.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:04.581 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:04.612 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:04.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:04.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:04.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:04 smithi098 bash[76817]: cluster 2023-09-23T19:09:02.675977+0000 mgr.y (mgr.26054) 1544 : cluster 0 pgmap v1708: 73 pgs: 4 remapped+peering, 9 peering, 16 active+clean+remapped, 42 active+clean, 1 active+undersized+degraded, 1 active+undersized; 4.6 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 171 KiB/s rd, 72 KiB/s wr, 49 op/s; 2/87 objects degraded (2.299%); 13/87 objects misplaced (14.943%) 2023-09-23T19:09:04.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:04 smithi098 bash[76817]: cluster 2023-09-23T19:09:03.281935+0000 mon.a (mon.0) 2092 : cluster 3 Health check update: Degraded data redundancy: 2/87 objects degraded (2.299%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T19:09:04.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:04 smithi098 bash[76817]: cluster 2023-09-23T19:09:03.294874+0000 osd.4 (osd.4) 102 : cluster 3 failed to encode map e369 with expected crc 2023-09-23T19:09:04.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:04 smithi098 bash[76817]: cluster 2023-09-23T19:09:03.295237+0000 osd.6 (osd.6) 360 : cluster 3 failed to encode map e369 with expected crc 2023-09-23T19:09:04.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:04 smithi098 bash[76817]: cluster 2023-09-23T19:09:03.295320+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e369 with expected crc 2023-09-23T19:09:04.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:04 smithi098 bash[76817]: cluster 2023-09-23T19:09:03.296072+0000 mon.a (mon.0) 2093 : cluster 0 osdmap e369: 8 total, 8 up, 7 in 2023-09-23T19:09:04.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:04 smithi098 bash[76817]: cluster 2023-09-23T19:09:03.299416+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e369 with expected crc 2023-09-23T19:09:04.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:04 smithi098 bash[76817]: cluster 2023-09-23T19:09:03.300075+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e369 with expected crc 2023-09-23T19:09:04.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:04 smithi098 bash[79516]: cluster 2023-09-23T19:09:02.675977+0000 mgr.y (mgr.26054) 1544 : cluster 0 pgmap v1708: 73 pgs: 4 remapped+peering, 9 peering, 16 active+clean+remapped, 42 active+clean, 1 active+undersized+degraded, 1 active+undersized; 4.6 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 171 KiB/s rd, 72 KiB/s wr, 49 op/s; 2/87 objects degraded (2.299%); 13/87 objects misplaced (14.943%) 2023-09-23T19:09:04.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:04 smithi098 bash[79516]: cluster 2023-09-23T19:09:03.281935+0000 mon.a (mon.0) 2092 : cluster 3 Health check update: Degraded data redundancy: 2/87 objects degraded (2.299%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T19:09:04.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:04 smithi098 bash[79516]: cluster 2023-09-23T19:09:03.294874+0000 osd.4 (osd.4) 102 : cluster 3 failed to encode map e369 with expected crc 2023-09-23T19:09:04.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:04 smithi098 bash[79516]: cluster 2023-09-23T19:09:03.295237+0000 osd.6 (osd.6) 360 : cluster 3 failed to encode map e369 with expected crc 2023-09-23T19:09:04.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:04 smithi098 bash[79516]: cluster 2023-09-23T19:09:03.295320+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e369 with expected crc 2023-09-23T19:09:04.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:04 smithi098 bash[79516]: cluster 2023-09-23T19:09:03.296072+0000 mon.a (mon.0) 2093 : cluster 0 osdmap e369: 8 total, 8 up, 7 in 2023-09-23T19:09:04.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:04 smithi098 bash[79516]: cluster 2023-09-23T19:09:03.299416+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e369 with expected crc 2023-09-23T19:09:04.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:04 smithi098 bash[79516]: cluster 2023-09-23T19:09:03.300075+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e369 with expected crc 2023-09-23T19:09:04.717 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:04.741 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:04.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:04.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:04.844 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:04.866 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:04.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:04.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:04.970 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:04.995 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:04.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:04.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:05.098 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:05.126 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:05.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:05.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:05.229 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:05.257 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:05.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:05.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:05.361 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:05.388 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:05.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:05.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:05.492 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:05.520 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:05.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:05.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:05.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:05 smithi173 bash[59422]: cluster 2023-09-23T19:09:04.301420+0000 mon.a (mon.0) 2094 : cluster 0 osdmap e370: 8 total, 8 up, 7 in 2023-09-23T19:09:05.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:05 smithi173 bash[59422]: cluster 2023-09-23T19:09:04.302067+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e370 with expected crc 2023-09-23T19:09:05.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:05 smithi173 bash[59422]: cluster 2023-09-23T19:09:04.305415+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e370 with expected crc 2023-09-23T19:09:05.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:05 smithi173 bash[59422]: cluster 2023-09-23T19:09:04.307664+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e370 with expected crc 2023-09-23T19:09:05.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:05 smithi173 bash[59422]: cluster 2023-09-23T19:09:04.673984+0000 osd.4 (osd.4) 103 : cluster 3 failed to encode map e370 with expected crc 2023-09-23T19:09:05.623 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:05.630 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:05 smithi098 bash[79516]: cluster 2023-09-23T19:09:04.301420+0000 mon.a (mon.0) 2094 : cluster 0 osdmap e370: 8 total, 8 up, 7 in 2023-09-23T19:09:05.630 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:05 smithi098 bash[79516]: cluster 2023-09-23T19:09:04.302067+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e370 with expected crc 2023-09-23T19:09:05.631 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:05 smithi098 bash[79516]: cluster 2023-09-23T19:09:04.305415+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e370 with expected crc 2023-09-23T19:09:05.631 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:05 smithi098 bash[79516]: cluster 2023-09-23T19:09:04.307664+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e370 with expected crc 2023-09-23T19:09:05.631 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:05 smithi098 bash[79516]: cluster 2023-09-23T19:09:04.673984+0000 osd.4 (osd.4) 103 : cluster 3 failed to encode map e370 with expected crc 2023-09-23T19:09:05.631 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:05 smithi098 bash[76817]: cluster 2023-09-23T19:09:04.301420+0000 mon.a (mon.0) 2094 : cluster 0 osdmap e370: 8 total, 8 up, 7 in 2023-09-23T19:09:05.631 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:05 smithi098 bash[76817]: cluster 2023-09-23T19:09:04.302067+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e370 with expected crc 2023-09-23T19:09:05.631 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:05 smithi098 bash[76817]: cluster 2023-09-23T19:09:04.305415+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e370 with expected crc 2023-09-23T19:09:05.631 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:05 smithi098 bash[76817]: cluster 2023-09-23T19:09:04.307664+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e370 with expected crc 2023-09-23T19:09:05.631 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:05 smithi098 bash[76817]: cluster 2023-09-23T19:09:04.673984+0000 osd.4 (osd.4) 103 : cluster 3 failed to encode map e370 with expected crc 2023-09-23T19:09:05.644 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:05.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:05.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:05.648 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T19:09:05.670 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T19:09:05.694 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T19:09:05.722 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T19:09:05.748 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T19:09:05.749 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:05.773 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:05.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:05.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:05.779 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T19:09:05.801 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T19:09:05.826 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T19:09:05.828 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T19:09:05.828 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T19:09:05.829 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T19:09:05.829 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T19:09:05.829 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T19:09:05.829 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T19:09:05.829 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T19:09:05.829 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T19:09:05.876 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:05.907 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:05.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:05.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:06.011 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:06.032 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:06.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:06.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:06.137 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:06.165 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T19:09:06.168 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:06.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:06.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:06.172 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm testimg 2023-09-23T19:09:06.272 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:06.283 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:09:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:09:06] "GET /metrics HTTP/1.1" 200 34182 "" "Prometheus/2.33.4" 2023-09-23T19:09:06.300 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:06.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:06.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:06.310 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T19:09:06.316 INFO:tasks.workunit.client.0.smithi098.stderr:+ cmp /tmp/rbd_import_export_266403/img /tmp/rbd_import_export_266403/img2 2023-09-23T19:09:06.404 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:06.432 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:06.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:06.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:06.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:06 smithi173 bash[59422]: cluster 2023-09-23T19:09:04.676684+0000 mgr.y (mgr.26054) 1545 : cluster 0 pgmap v1711: 73 pgs: 4 remapped+peering, 9 peering, 16 active+clean+remapped, 42 active+clean, 1 active+undersized+degraded, 1 active+undersized; 4.6 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 2.5 KiB/s rd, 1023 B/s wr, 4 op/s; 2/87 objects degraded (2.299%); 13/87 objects misplaced (14.943%) 2023-09-23T19:09:06.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:06 smithi173 bash[59422]: cluster 2023-09-23T19:09:05.309366+0000 mon.a (mon.0) 2095 : cluster 0 osdmap e371: 8 total, 8 up, 7 in 2023-09-23T19:09:06.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:06 smithi173 bash[59422]: cluster 2023-09-23T19:09:05.309401+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e371 with expected crc 2023-09-23T19:09:06.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:06 smithi173 bash[59422]: cluster 2023-09-23T19:09:05.310677+0000 osd.4 (osd.4) 104 : cluster 3 failed to encode map e371 with expected crc 2023-09-23T19:09:06.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:06 smithi173 bash[59422]: cluster 2023-09-23T19:09:05.310801+0000 osd.6 (osd.6) 361 : cluster 3 failed to encode map e371 with expected crc 2023-09-23T19:09:06.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:06 smithi173 bash[59422]: cluster 2023-09-23T19:09:05.313233+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e371 with expected crc 2023-09-23T19:09:06.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:06 smithi173 bash[59422]: cluster 2023-09-23T19:09:05.361692+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e371 with expected crc 2023-09-23T19:09:06.529 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:06 smithi173 bash[59422]: audit 2023-09-23T19:09:05.618787+0000 mon.c (mon.2) 153 : audit 0 from='client.? 172.21.15.98:0/1094823489' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:09:06.536 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:06.566 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:06.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:06.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:06.669 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:06.677 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:06 smithi098 bash[79516]: cluster 2023-09-23T19:09:04.676684+0000 mgr.y (mgr.26054) 1545 : cluster 0 pgmap v1711: 73 pgs: 4 remapped+peering, 9 peering, 16 active+clean+remapped, 42 active+clean, 1 active+undersized+degraded, 1 active+undersized; 4.6 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 2.5 KiB/s rd, 1023 B/s wr, 4 op/s; 2/87 objects degraded (2.299%); 13/87 objects misplaced (14.943%) 2023-09-23T19:09:06.677 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:06 smithi098 bash[79516]: cluster 2023-09-23T19:09:05.309366+0000 mon.a (mon.0) 2095 : cluster 0 osdmap e371: 8 total, 8 up, 7 in 2023-09-23T19:09:06.677 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:06 smithi098 bash[79516]: cluster 2023-09-23T19:09:05.309401+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e371 with expected crc 2023-09-23T19:09:06.677 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:06 smithi098 bash[79516]: cluster 2023-09-23T19:09:05.310677+0000 osd.4 (osd.4) 104 : cluster 3 failed to encode map e371 with expected crc 2023-09-23T19:09:06.677 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:06 smithi098 bash[79516]: cluster 2023-09-23T19:09:05.310801+0000 osd.6 (osd.6) 361 : cluster 3 failed to encode map e371 with expected crc 2023-09-23T19:09:06.677 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:06 smithi098 bash[79516]: cluster 2023-09-23T19:09:05.313233+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e371 with expected crc 2023-09-23T19:09:06.678 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:06 smithi098 bash[79516]: cluster 2023-09-23T19:09:05.361692+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e371 with expected crc 2023-09-23T19:09:06.678 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:06 smithi098 bash[79516]: audit 2023-09-23T19:09:05.618787+0000 mon.c (mon.2) 153 : audit 0 from='client.? 172.21.15.98:0/1094823489' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:09:06.678 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:06 smithi098 bash[76817]: cluster 2023-09-23T19:09:04.676684+0000 mgr.y (mgr.26054) 1545 : cluster 0 pgmap v1711: 73 pgs: 4 remapped+peering, 9 peering, 16 active+clean+remapped, 42 active+clean, 1 active+undersized+degraded, 1 active+undersized; 4.6 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 2.5 KiB/s rd, 1023 B/s wr, 4 op/s; 2/87 objects degraded (2.299%); 13/87 objects misplaced (14.943%) 2023-09-23T19:09:06.678 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:06 smithi098 bash[76817]: cluster 2023-09-23T19:09:05.309366+0000 mon.a (mon.0) 2095 : cluster 0 osdmap e371: 8 total, 8 up, 7 in 2023-09-23T19:09:06.678 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:06 smithi098 bash[76817]: cluster 2023-09-23T19:09:05.309401+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e371 with expected crc 2023-09-23T19:09:06.678 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:06 smithi098 bash[76817]: cluster 2023-09-23T19:09:05.310677+0000 osd.4 (osd.4) 104 : cluster 3 failed to encode map e371 with expected crc 2023-09-23T19:09:06.678 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:06 smithi098 bash[76817]: cluster 2023-09-23T19:09:05.310801+0000 osd.6 (osd.6) 361 : cluster 3 failed to encode map e371 with expected crc 2023-09-23T19:09:06.678 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:06 smithi098 bash[76817]: cluster 2023-09-23T19:09:05.313233+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e371 with expected crc 2023-09-23T19:09:06.678 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:06 smithi098 bash[76817]: cluster 2023-09-23T19:09:05.361692+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e371 with expected crc 2023-09-23T19:09:06.678 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:06 smithi098 bash[76817]: audit 2023-09-23T19:09:05.618787+0000 mon.c (mon.2) 153 : audit 0 from='client.? 172.21.15.98:0/1094823489' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:09:06.690 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:06.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:06.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:06.793 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:06.813 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:06.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:06.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:06.916 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:06.940 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:06.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:06.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:07.044 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:07.068 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:07.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:07.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:07.076 INFO:tasks.workunit.client.0.smithi098.stderr:+ cmp /tmp/rbd_import_export_266403/img /tmp/rbd_import_export_266403/img3 2023-09-23T19:09:07.172 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:07.200 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:07.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:07.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:07.303 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:07.328 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:07.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:07.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:07.431 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:07.443 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:07 smithi098 bash[76817]: cluster 2023-09-23T19:09:07.306779+0000 mon.a (mon.0) 2096 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T19:09:07.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:07 smithi098 bash[76817]: cluster 2023-09-23T19:09:07.306860+0000 mon.a (mon.0) 2097 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/87 objects degraded (2.299%), 1 pg degraded, 2 pgs undersized) 2023-09-23T19:09:07.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:07 smithi098 bash[79516]: cluster 2023-09-23T19:09:07.306779+0000 mon.a (mon.0) 2096 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T19:09:07.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:07 smithi098 bash[79516]: cluster 2023-09-23T19:09:07.306860+0000 mon.a (mon.0) 2097 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/87 objects degraded (2.299%), 1 pg degraded, 2 pgs undersized) 2023-09-23T19:09:07.463 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:07.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:07.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:07.566 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:07.597 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:07.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:07.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:07.611 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_266403/img /tmp/rbd_import_export_266403/img2 /tmp/rbd_import_export_266403/img3 2023-09-23T19:09:07.700 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:07.719 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:07.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:07.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:07.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:07 smithi173 bash[59422]: cluster 2023-09-23T19:09:07.306779+0000 mon.a (mon.0) 2096 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T19:09:07.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:07 smithi173 bash[59422]: cluster 2023-09-23T19:09:07.306860+0000 mon.a (mon.0) 2097 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/87 objects degraded (2.299%), 1 pg degraded, 2 pgs undersized) 2023-09-23T19:09:07.786 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:09:07.787 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:09:07.822 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:07.847 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:07.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:07.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:07.950 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:07.973 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:07.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:07.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:08.077 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:08.105 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:08.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:08.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:08.209 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:08.241 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:08.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:08.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:08.345 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:08.377 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:08.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:08.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:08.481 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:08.492 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:08 smithi098 bash[76817]: cluster 2023-09-23T19:09:06.677429+0000 mgr.y (mgr.26054) 1546 : cluster 0 pgmap v1713: 73 pgs: 1 creating+peering, 1 remapped+peering, 9 peering, 62 active+clean; 8.2 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 110 KiB/s rd, 52 KiB/s wr, 55 op/s; 0 B/s, 3 objects/s recovering 2023-09-23T19:09:08.493 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:08 smithi098 bash[79516]: cluster 2023-09-23T19:09:06.677429+0000 mgr.y (mgr.26054) 1546 : cluster 0 pgmap v1713: 73 pgs: 1 creating+peering, 1 remapped+peering, 9 peering, 62 active+clean; 8.2 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 110 KiB/s rd, 52 KiB/s wr, 55 op/s; 0 B/s, 3 objects/s recovering 2023-09-23T19:09:08.511 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:08.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:08.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:08.615 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:08.646 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:08.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:08.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:08.750 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:08.758 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:08 smithi173 bash[59422]: cluster 2023-09-23T19:09:06.677429+0000 mgr.y (mgr.26054) 1546 : cluster 0 pgmap v1713: 73 pgs: 1 creating+peering, 1 remapped+peering, 9 peering, 62 active+clean; 8.2 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 110 KiB/s rd, 52 KiB/s wr, 55 op/s; 0 B/s, 3 objects/s recovering 2023-09-23T19:09:08.772 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:08.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:08.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:08.875 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:08.906 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:08.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:08.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:09.009 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:09.035 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:09.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:09.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:09.140 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:09.174 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:09.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:09.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:09.277 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:09.307 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:09.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:09.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:09.410 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:09.446 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:09.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:09.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:09.550 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:09.584 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:09.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:09.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:09.688 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:09.716 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:09.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:09.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:09.820 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:09.854 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:09.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:09.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:09.958 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:09.985 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:09.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:09.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:10.089 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:10.120 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:10.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:10.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:10.223 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:10.253 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:10.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:10.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:10.356 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:10.386 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:10.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:10.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:10.490 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:10.503 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:10 smithi098 bash[79516]: cluster 2023-09-23T19:09:08.678539+0000 mgr.y (mgr.26054) 1547 : cluster 0 pgmap v1714: 73 pgs: 1 creating+peering, 72 active+clean; 8.2 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 186 KiB/s rd, 89 KiB/s wr, 64 op/s; 0 B/s, 2 objects/s recovering 2023-09-23T19:09:10.503 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:10 smithi098 bash[79516]: audit 2023-09-23T19:09:09.657256+0000 mon.a (mon.0) 2098 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:09:10.503 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:10 smithi098 bash[79516]: audit 2023-09-23T19:09:09.658947+0000 mon.a (mon.0) 2099 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:09:10.503 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:10 smithi098 bash[76817]: cluster 2023-09-23T19:09:08.678539+0000 mgr.y (mgr.26054) 1547 : cluster 0 pgmap v1714: 73 pgs: 1 creating+peering, 72 active+clean; 8.2 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 186 KiB/s rd, 89 KiB/s wr, 64 op/s; 0 B/s, 2 objects/s recovering 2023-09-23T19:09:10.503 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:10 smithi098 bash[76817]: audit 2023-09-23T19:09:09.657256+0000 mon.a (mon.0) 2098 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:09:10.504 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:10 smithi098 bash[76817]: audit 2023-09-23T19:09:09.658947+0000 mon.a (mon.0) 2099 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:09:10.516 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:10.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:10.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:10.613 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep -q export-format 2023-09-23T19:09:10.613 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd help export 2023-09-23T19:09:10.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:10.633 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:10 smithi173 bash[59422]: cluster 2023-09-23T19:09:08.678539+0000 mgr.y (mgr.26054) 1547 : cluster 0 pgmap v1714: 73 pgs: 1 creating+peering, 72 active+clean; 8.2 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 186 KiB/s rd, 89 KiB/s wr, 64 op/s; 0 B/s, 2 objects/s recovering 2023-09-23T19:09:10.633 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:10 smithi173 bash[59422]: audit 2023-09-23T19:09:09.657256+0000 mon.a (mon.0) 2098 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:09:10.633 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:10 smithi173 bash[59422]: audit 2023-09-23T19:09:09.658947+0000 mon.a (mon.0) 2099 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:09:10.651 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:10.651 INFO:tasks.workunit.client.0.smithi098.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_266403/img bs=1k count=10 seek=100 2023-09-23T19:09:10.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:10.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:10.665 INFO:tasks.workunit.client.0.smithi098.stderr:10+0 records in 2023-09-23T19:09:10.665 INFO:tasks.workunit.client.0.smithi098.stderr:10+0 records out 2023-09-23T19:09:10.666 INFO:tasks.workunit.client.0.smithi098.stderr:10240 bytes (10 kB, 10 KiB) copied, 0.000106229 s, 96.4 MB/s 2023-09-23T19:09:10.666 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --new-format /tmp/rbd_import_export_266403/img testimg 2023-09-23T19:09:10.755 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:10.785 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:10.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:10.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:10.852 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 100% complete...done. 2023-09-23T19:09:10.853 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T19:09:10.856 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd snap create testimg@snap 2023-09-23T19:09:10.888 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:10.916 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:10.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:10.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:11.020 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:11.048 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:11.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:11.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:11.151 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:11.177 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:11.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:11.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:11.280 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:11.311 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:11.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:11.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:11.415 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:11.441 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:11.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:11.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:11.545 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:11.571 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:11.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:11.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:11.635 INFO:tasks.workunit.client.0.smithi098.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T19:09:11.644 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd image-meta set testimg key1 value1 2023-09-23T19:09:11.674 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:11.693 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:11 smithi098 bash[79516]: cluster 2023-09-23T19:09:10.679026+0000 mgr.y (mgr.26054) 1548 : cluster 0 pgmap v1715: 73 pgs: 73 active+clean; 4.6 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 161 KiB/s rd, 72 KiB/s wr, 52 op/s; 0 B/s, 2 objects/s recovering 2023-09-23T19:09:11.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:11 smithi098 bash[76817]: cluster 2023-09-23T19:09:10.679026+0000 mgr.y (mgr.26054) 1548 : cluster 0 pgmap v1715: 73 pgs: 73 active+clean; 4.6 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 161 KiB/s rd, 72 KiB/s wr, 52 op/s; 0 B/s, 2 objects/s recovering 2023-09-23T19:09:11.702 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:11.703 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd image-meta list testimg 2023-09-23T19:09:11.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:11.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:11.749 INFO:tasks.workunit.client.0.smithi098.stderr:+ IMAGEMETA_BEFORE=There is 1 metadatum on this image: 2023-09-23T19:09:11.749 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-23T19:09:11.750 INFO:tasks.workunit.client.0.smithi098.stderr:Key Value 2023-09-23T19:09:11.750 INFO:tasks.workunit.client.0.smithi098.stderr:key1 value1 2023-09-23T19:09:11.750 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_266403/img_v2 2023-09-23T19:09:11.796 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T19:09:11.799 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_266403/img_v2 testimg_import 2023-09-23T19:09:11.806 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:11.831 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:11.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:11.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:11.935 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:11.943 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:11 smithi173 bash[59422]: cluster 2023-09-23T19:09:10.679026+0000 mgr.y (mgr.26054) 1548 : cluster 0 pgmap v1715: 73 pgs: 73 active+clean; 4.6 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 161 KiB/s rd, 72 KiB/s wr, 52 op/s; 0 B/s, 2 objects/s recovering 2023-09-23T19:09:11.960 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:11.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:11.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:12.063 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:12.091 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:12.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:12.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:12.195 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:12.222 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:12.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:12.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:12.325 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:12.353 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:12.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:12.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:12.457 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:12.484 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:12.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:12.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:12.588 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:12.613 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:12.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:12.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:12.650 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T19:09:12.654 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info testimg_import 2023-09-23T19:09:12.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:12 smithi098 bash[79516]: cluster 2023-09-23T19:09:11.601383+0000 mon.a (mon.0) 2100 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T19:09:12.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:12 smithi098 bash[79516]: cluster 2023-09-23T19:09:11.601446+0000 mon.a (mon.0) 2101 : cluster 1 Cluster is now healthy 2023-09-23T19:09:12.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:12 smithi098 bash[79516]: cluster 2023-09-23T19:09:11.614036+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e372 with expected crc 2023-09-23T19:09:12.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:12 smithi098 bash[79516]: cluster 2023-09-23T19:09:11.614141+0000 mon.a (mon.0) 2102 : cluster 0 osdmap e372: 8 total, 8 up, 7 in 2023-09-23T19:09:12.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:12 smithi098 bash[79516]: cluster 2023-09-23T19:09:11.614525+0000 osd.4 (osd.4) 105 : cluster 3 failed to encode map e372 with expected crc 2023-09-23T19:09:12.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:12 smithi098 bash[79516]: cluster 2023-09-23T19:09:11.614796+0000 osd.6 (osd.6) 362 : cluster 3 failed to encode map e372 with expected crc 2023-09-23T19:09:12.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:12 smithi098 bash[79516]: cluster 2023-09-23T19:09:11.620057+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e372 with expected crc 2023-09-23T19:09:12.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:12 smithi098 bash[76817]: cluster 2023-09-23T19:09:11.601383+0000 mon.a (mon.0) 2100 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T19:09:12.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:12 smithi098 bash[76817]: cluster 2023-09-23T19:09:11.601446+0000 mon.a (mon.0) 2101 : cluster 1 Cluster is now healthy 2023-09-23T19:09:12.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:12 smithi098 bash[76817]: cluster 2023-09-23T19:09:11.614036+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e372 with expected crc 2023-09-23T19:09:12.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:12 smithi098 bash[76817]: cluster 2023-09-23T19:09:11.614141+0000 mon.a (mon.0) 2102 : cluster 0 osdmap e372: 8 total, 8 up, 7 in 2023-09-23T19:09:12.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:12 smithi098 bash[76817]: cluster 2023-09-23T19:09:11.614525+0000 osd.4 (osd.4) 105 : cluster 3 failed to encode map e372 with expected crc 2023-09-23T19:09:12.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:12 smithi098 bash[76817]: cluster 2023-09-23T19:09:11.614796+0000 osd.6 (osd.6) 362 : cluster 3 failed to encode map e372 with expected crc 2023-09-23T19:09:12.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:12 smithi098 bash[76817]: cluster 2023-09-23T19:09:11.620057+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e372 with expected crc 2023-09-23T19:09:12.699 INFO:tasks.workunit.client.0.smithi098.stdout:rbd image 'testimg_import': 2023-09-23T19:09:12.700 INFO:tasks.workunit.client.0.smithi098.stdout: size 110 KiB in 1 objects 2023-09-23T19:09:12.700 INFO:tasks.workunit.client.0.smithi098.stdout: order 22 (4 MiB objects) 2023-09-23T19:09:12.700 INFO:tasks.workunit.client.0.smithi098.stdout: snapshot_count: 1 2023-09-23T19:09:12.700 INFO:tasks.workunit.client.0.smithi098.stdout: id: b444f3683a5 2023-09-23T19:09:12.700 INFO:tasks.workunit.client.0.smithi098.stdout: block_name_prefix: rbd_data.b444f3683a5 2023-09-23T19:09:12.700 INFO:tasks.workunit.client.0.smithi098.stdout: format: 2 2023-09-23T19:09:12.700 INFO:tasks.workunit.client.0.smithi098.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T19:09:12.700 INFO:tasks.workunit.client.0.smithi098.stdout: op_features: 2023-09-23T19:09:12.700 INFO:tasks.workunit.client.0.smithi098.stdout: flags: 2023-09-23T19:09:12.700 INFO:tasks.workunit.client.0.smithi098.stdout: create_timestamp: Sat Sep 23 19:09:11 2023 2023-09-23T19:09:12.700 INFO:tasks.workunit.client.0.smithi098.stdout: access_timestamp: Sat Sep 23 19:09:11 2023 2023-09-23T19:09:12.700 INFO:tasks.workunit.client.0.smithi098.stdout: modify_timestamp: Sat Sep 23 19:09:11 2023 2023-09-23T19:09:12.703 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info testimg_import@snap 2023-09-23T19:09:12.717 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:12.744 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:12.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:12.748 INFO:tasks.workunit.client.0.smithi098.stdout:rbd image 'testimg_import': 2023-09-23T19:09:12.748 INFO:tasks.workunit.client.0.smithi098.stdout: size 110 KiB in 1 objects 2023-09-23T19:09:12.748 INFO:tasks.workunit.client.0.smithi098.stdout: order 22 (4 MiB objects) 2023-09-23T19:09:12.748 INFO:tasks.workunit.client.0.smithi098.stdout: snapshot_count: 1 2023-09-23T19:09:12.748 INFO:tasks.workunit.client.0.smithi098.stdout: id: b444f3683a5 2023-09-23T19:09:12.748 INFO:tasks.workunit.client.0.smithi098.stdout: block_name_prefix: rbd_data.b444f3683a5 2023-09-23T19:09:12.748 INFO:tasks.workunit.client.0.smithi098.stdout: format: 2 2023-09-23T19:09:12.749 INFO:tasks.workunit.client.0.smithi098.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T19:09:12.749 INFO:tasks.workunit.client.0.smithi098.stdout: op_features: 2023-09-23T19:09:12.749 INFO:tasks.workunit.client.0.smithi098.stdout: flags: 2023-09-23T19:09:12.749 INFO:tasks.workunit.client.0.smithi098.stdout: create_timestamp: Sat Sep 23 19:09:11 2023 2023-09-23T19:09:12.749 INFO:tasks.workunit.client.0.smithi098.stdout: access_timestamp: Sat Sep 23 19:09:11 2023 2023-09-23T19:09:12.749 INFO:tasks.workunit.client.0.smithi098.stdout: modify_timestamp: Sat Sep 23 19:09:11 2023 2023-09-23T19:09:12.749 INFO:tasks.workunit.client.0.smithi098.stdout: protected: False 2023-09-23T19:09:12.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:12.752 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd image-meta list testimg_import 2023-09-23T19:09:12.802 INFO:tasks.workunit.client.0.smithi098.stderr:+ IMAGEMETA_AFTER=There is 1 metadatum on this image: 2023-09-23T19:09:12.802 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-23T19:09:12.802 INFO:tasks.workunit.client.0.smithi098.stderr:Key Value 2023-09-23T19:09:12.802 INFO:tasks.workunit.client.0.smithi098.stderr:key1 value1 2023-09-23T19:09:12.802 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ There is 1 metadatum on this image: 2023-09-23T19:09:12.803 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-23T19:09:12.803 INFO:tasks.workunit.client.0.smithi098.stderr:Key Value 2023-09-23T19:09:12.803 INFO:tasks.workunit.client.0.smithi098.stderr:key1 value1 = There is 1 metadatum on this image: 2023-09-23T19:09:12.803 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-23T19:09:12.803 INFO:tasks.workunit.client.0.smithi098.stderr:Key Value 2023-09-23T19:09:12.803 INFO:tasks.workunit.client.0.smithi098.stderr:key1 value1 ] 2023-09-23T19:09:12.803 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export testimg_import /tmp/rbd_import_export_266403/img_import 2023-09-23T19:09:12.848 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 100% complete...done. 2023-09-23T19:09:12.850 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:12.853 INFO:tasks.workunit.client.0.smithi098.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_266403/img /tmp/rbd_import_export_266403/img_import 2023-09-23T19:09:12.853 INFO:tasks.workunit.client.0.smithi098.stderr:+ cmp -l /tmp/rbd_import_export_266403/img /tmp/rbd_import_export_266403/img_import 2023-09-23T19:09:12.854 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_266403/img --format %b 2023-09-23T19:09:12.863 INFO:tasks.workunit.client.0.smithi098.stderr:+ origsize=24 2023-09-23T19:09:12.864 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_266403/img_import --format %b 2023-09-23T19:09:12.864 INFO:tasks.workunit.client.0.smithi098.stderr:+ exportsize=224 2023-09-23T19:09:12.864 INFO:tasks.workunit.client.0.smithi098.stderr:+ difference=200 2023-09-23T19:09:12.864 INFO:tasks.workunit.client.0.smithi098.stderr:+ difference=200 2023-09-23T19:09:12.864 INFO:tasks.workunit.client.0.smithi098.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-09-23T19:09:12.865 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_266403/img_snap 2023-09-23T19:09:12.877 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:12.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:12.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:12.908 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 100% complete...done. 2023-09-23T19:09:12.913 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_266403/img_snap_import 2023-09-23T19:09:12.960 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 100% complete...done. 2023-09-23T19:09:12.965 INFO:tasks.workunit.client.0.smithi098.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_266403/img_snap /tmp/rbd_import_export_266403/img_snap_import 2023-09-23T19:09:12.965 INFO:tasks.workunit.client.0.smithi098.stderr:+ cmp -l /tmp/rbd_import_export_266403/img_snap /tmp/rbd_import_export_266403/img_snap_import 2023-09-23T19:09:12.966 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_266403/img_snap --format %b 2023-09-23T19:09:12.968 INFO:tasks.workunit.client.0.smithi098.stderr:+ origsize=224 2023-09-23T19:09:12.968 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_266403/img_snap_import --format %b 2023-09-23T19:09:12.969 INFO:tasks.workunit.client.0.smithi098.stderr:+ exportsize=224 2023-09-23T19:09:12.969 INFO:tasks.workunit.client.0.smithi098.stderr:+ difference=0 2023-09-23T19:09:12.969 INFO:tasks.workunit.client.0.smithi098.stderr:+ difference=0 2023-09-23T19:09:12.969 INFO:tasks.workunit.client.0.smithi098.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T19:09:12.969 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_266403/img_v2 2023-09-23T19:09:12.970 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_266403/img_import 2023-09-23T19:09:12.971 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_266403/img_snap 2023-09-23T19:09:12.972 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_266403/img_snap_import 2023-09-23T19:09:12.973 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd snap rm testimg_import@snap 2023-09-23T19:09:12.981 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:12.994 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:12 smithi173 bash[59422]: cluster 2023-09-23T19:09:11.601383+0000 mon.a (mon.0) 2100 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T19:09:12.994 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:12 smithi173 bash[59422]: cluster 2023-09-23T19:09:11.601446+0000 mon.a (mon.0) 2101 : cluster 1 Cluster is now healthy 2023-09-23T19:09:12.994 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:12 smithi173 bash[59422]: cluster 2023-09-23T19:09:11.614036+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e372 with expected crc 2023-09-23T19:09:12.994 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:12 smithi173 bash[59422]: cluster 2023-09-23T19:09:11.614141+0000 mon.a (mon.0) 2102 : cluster 0 osdmap e372: 8 total, 8 up, 7 in 2023-09-23T19:09:12.994 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:12 smithi173 bash[59422]: cluster 2023-09-23T19:09:11.614525+0000 osd.4 (osd.4) 105 : cluster 3 failed to encode map e372 with expected crc 2023-09-23T19:09:12.994 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:12 smithi173 bash[59422]: cluster 2023-09-23T19:09:11.614796+0000 osd.6 (osd.6) 362 : cluster 3 failed to encode map e372 with expected crc 2023-09-23T19:09:12.994 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:12 smithi173 bash[59422]: cluster 2023-09-23T19:09:11.620057+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e372 with expected crc 2023-09-23T19:09:13.007 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:13.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:13.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:13.112 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:13.138 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:13.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:13.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:13.242 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:13.268 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:13.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:13.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:13.371 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:13.396 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:13.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:13.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:13.499 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:13.526 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:13.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:13.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:13.631 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:13.642 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is set 2023-09-23T19:09:13.651 INFO:tasks.workunit.client.0.smithi098.stderr: Removing snap: 100% complete...done. 2023-09-23T19:09:13.654 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:13.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:13.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:13.661 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd remove testimg_import 2023-09-23T19:09:13.758 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:13.798 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 100% complete...done. 2023-09-23T19:09:13.803 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:13.804 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd snap rm testimg@snap 2023-09-23T19:09:13.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:13.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:13.904 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:13 smithi098 bash[79516]: cluster 2023-09-23T19:09:12.622102+0000 mon.a (mon.0) 2103 : cluster 0 osdmap e373: 8 total, 8 up, 7 in 2023-09-23T19:09:13.905 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:13 smithi098 bash[79516]: cluster 2023-09-23T19:09:12.622758+0000 osd.6 (osd.6) 363 : cluster 3 failed to encode map e373 with expected crc 2023-09-23T19:09:13.905 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:13 smithi098 bash[79516]: cluster 2023-09-23T19:09:12.631979+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e373 with expected crc 2023-09-23T19:09:13.905 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:13 smithi098 bash[79516]: cluster 2023-09-23T19:09:12.673521+0000 osd.4 (osd.4) 106 : cluster 3 failed to encode map e373 with expected crc 2023-09-23T19:09:13.905 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:13 smithi098 bash[79516]: cluster 2023-09-23T19:09:12.673653+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e373 with expected crc 2023-09-23T19:09:13.905 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:13 smithi098 bash[79516]: cluster 2023-09-23T19:09:12.679734+0000 mgr.y (mgr.26054) 1549 : cluster 0 pgmap v1718: 73 pgs: 73 active+clean; 4.6 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 151 KiB/s rd, 46 KiB/s wr, 91 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T19:09:13.905 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:13 smithi098 bash[79516]: audit 2023-09-23T19:09:12.798618+0000 mon.a (mon.0) 2104 : audit 1 from='client.? 172.21.15.98:0/2258839127' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:09:13.905 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:13 smithi098 bash[76817]: cluster 2023-09-23T19:09:12.622102+0000 mon.a (mon.0) 2103 : cluster 0 osdmap e373: 8 total, 8 up, 7 in 2023-09-23T19:09:13.905 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:13 smithi098 bash[76817]: cluster 2023-09-23T19:09:12.622758+0000 osd.6 (osd.6) 363 : cluster 3 failed to encode map e373 with expected crc 2023-09-23T19:09:13.905 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:13 smithi098 bash[76817]: cluster 2023-09-23T19:09:12.631979+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e373 with expected crc 2023-09-23T19:09:13.906 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:13 smithi098 bash[76817]: cluster 2023-09-23T19:09:12.673521+0000 osd.4 (osd.4) 106 : cluster 3 failed to encode map e373 with expected crc 2023-09-23T19:09:13.906 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:13 smithi098 bash[76817]: cluster 2023-09-23T19:09:12.673653+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e373 with expected crc 2023-09-23T19:09:13.906 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:13 smithi098 bash[76817]: cluster 2023-09-23T19:09:12.679734+0000 mgr.y (mgr.26054) 1549 : cluster 0 pgmap v1718: 73 pgs: 73 active+clean; 4.6 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 151 KiB/s rd, 46 KiB/s wr, 91 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T19:09:13.906 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:13 smithi098 bash[76817]: audit 2023-09-23T19:09:12.798618+0000 mon.a (mon.0) 2104 : audit 1 from='client.? 172.21.15.98:0/2258839127' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:09:13.907 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:13.920 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:13 smithi173 bash[59422]: cluster 2023-09-23T19:09:12.622102+0000 mon.a (mon.0) 2103 : cluster 0 osdmap e373: 8 total, 8 up, 7 in 2023-09-23T19:09:13.920 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:13 smithi173 bash[59422]: cluster 2023-09-23T19:09:12.622758+0000 osd.6 (osd.6) 363 : cluster 3 failed to encode map e373 with expected crc 2023-09-23T19:09:13.920 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:13 smithi173 bash[59422]: cluster 2023-09-23T19:09:12.631979+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e373 with expected crc 2023-09-23T19:09:13.920 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:13 smithi173 bash[59422]: cluster 2023-09-23T19:09:12.673521+0000 osd.4 (osd.4) 106 : cluster 3 failed to encode map e373 with expected crc 2023-09-23T19:09:13.920 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:13 smithi173 bash[59422]: cluster 2023-09-23T19:09:12.673653+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e373 with expected crc 2023-09-23T19:09:13.920 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:13 smithi173 bash[59422]: cluster 2023-09-23T19:09:12.679734+0000 mgr.y (mgr.26054) 1549 : cluster 0 pgmap v1718: 73 pgs: 73 active+clean; 4.6 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 151 KiB/s rd, 46 KiB/s wr, 91 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T19:09:13.920 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:13 smithi173 bash[59422]: audit 2023-09-23T19:09:12.798618+0000 mon.a (mon.0) 2104 : audit 1 from='client.? 172.21.15.98:0/2258839127' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T19:09:13.936 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:13.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:13.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:14.039 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:14.063 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:14.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:14.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:14.168 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:14.200 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:14.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:14.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:14.303 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:14.327 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:14.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:14.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:14.392 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T19:09:14.430 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:14.458 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:14.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:14.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:14.561 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:14.586 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:14.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:14.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:14.589 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T19:09:14.589 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:09:14.648 INFO:tasks.workunit.client.0.smithi098.stderr: Removing snap: 100% complete...done. 2023-09-23T19:09:14.658 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm testimg 2023-09-23T19:09:14.690 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:14.705 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T19:09:14.706 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:09:14.718 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:14.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:14.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:14.773 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 100% complete...done. 2023-09-23T19:09:14.778 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --order 20 /tmp/rbd_import_export_266403/img testimg 2023-09-23T19:09:14.821 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:14.829 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:14 smithi098 bash[79516]: cluster 2023-09-23T19:09:13.618817+0000 mon.a (mon.0) 2105 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:09:14.829 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:14 smithi098 bash[79516]: cluster 2023-09-23T19:09:13.631726+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e374 with expected crc 2023-09-23T19:09:14.829 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:14 smithi098 bash[79516]: audit 2023-09-23T19:09:13.632682+0000 mon.a (mon.0) 2106 : audit 1 from='client.? 172.21.15.98:0/2258839127' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:09:14.829 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:14 smithi098 bash[79516]: cluster 2023-09-23T19:09:13.632837+0000 mon.a (mon.0) 2107 : cluster 0 osdmap e374: 8 total, 8 up, 7 in 2023-09-23T19:09:14.829 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:14 smithi098 bash[79516]: cluster 2023-09-23T19:09:13.635102+0000 osd.6 (osd.6) 364 : cluster 3 failed to encode map e374 with expected crc 2023-09-23T19:09:14.830 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:14 smithi098 bash[79516]: cluster 2023-09-23T19:09:13.700807+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e374 with expected crc 2023-09-23T19:09:14.830 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:14 smithi098 bash[79516]: audit 2023-09-23T19:09:14.013033+0000 mon.a (mon.0) 2108 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:09:14.830 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:14 smithi098 bash[79516]: audit 2023-09-23T19:09:14.019296+0000 mon.a (mon.0) 2109 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:09:14.830 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:14 smithi098 bash[76817]: cluster 2023-09-23T19:09:13.618817+0000 mon.a (mon.0) 2105 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:09:14.831 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:14 smithi098 bash[76817]: cluster 2023-09-23T19:09:13.631726+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e374 with expected crc 2023-09-23T19:09:14.831 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:14 smithi098 bash[76817]: audit 2023-09-23T19:09:13.632682+0000 mon.a (mon.0) 2106 : audit 1 from='client.? 172.21.15.98:0/2258839127' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:09:14.831 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:14 smithi098 bash[76817]: cluster 2023-09-23T19:09:13.632837+0000 mon.a (mon.0) 2107 : cluster 0 osdmap e374: 8 total, 8 up, 7 in 2023-09-23T19:09:14.831 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:14 smithi098 bash[76817]: cluster 2023-09-23T19:09:13.635102+0000 osd.6 (osd.6) 364 : cluster 3 failed to encode map e374 with expected crc 2023-09-23T19:09:14.831 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:14 smithi098 bash[76817]: cluster 2023-09-23T19:09:13.700807+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e374 with expected crc 2023-09-23T19:09:14.831 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:14 smithi098 bash[76817]: audit 2023-09-23T19:09:14.013033+0000 mon.a (mon.0) 2108 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:09:14.831 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:14 smithi098 bash[76817]: audit 2023-09-23T19:09:14.019296+0000 mon.a (mon.0) 2109 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:09:14.842 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:14.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:14.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:14.849 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 100% complete...done. 2023-09-23T19:09:14.850 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T19:09:14.853 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_266403/img_v2 2023-09-23T19:09:14.904 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T19:09:14.907 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_266403/img_v2 testimg_import 2023-09-23T19:09:14.946 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:14.958 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:14 smithi173 bash[59422]: cluster 2023-09-23T19:09:13.618817+0000 mon.a (mon.0) 2105 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:09:14.958 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:14 smithi173 bash[59422]: cluster 2023-09-23T19:09:13.631726+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e374 with expected crc 2023-09-23T19:09:14.959 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:14 smithi173 bash[59422]: audit 2023-09-23T19:09:13.632682+0000 mon.a (mon.0) 2106 : audit 1 from='client.? 172.21.15.98:0/2258839127' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T19:09:14.959 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:14 smithi173 bash[59422]: cluster 2023-09-23T19:09:13.632837+0000 mon.a (mon.0) 2107 : cluster 0 osdmap e374: 8 total, 8 up, 7 in 2023-09-23T19:09:14.959 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:14 smithi173 bash[59422]: cluster 2023-09-23T19:09:13.635102+0000 osd.6 (osd.6) 364 : cluster 3 failed to encode map e374 with expected crc 2023-09-23T19:09:14.959 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:14 smithi173 bash[59422]: cluster 2023-09-23T19:09:13.700807+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e374 with expected crc 2023-09-23T19:09:14.959 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:14 smithi173 bash[59422]: audit 2023-09-23T19:09:14.013033+0000 mon.a (mon.0) 2108 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:09:14.959 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:14 smithi173 bash[59422]: audit 2023-09-23T19:09:14.019296+0000 mon.a (mon.0) 2109 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:09:14.976 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:14.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:14.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:14.990 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T19:09:14.996 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info testimg_import 2023-09-23T19:09:14.996 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep order 2023-09-23T19:09:14.996 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk {print $2} 2023-09-23T19:09:14.996 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep 20 2023-09-23T19:09:15.049 INFO:tasks.workunit.client.0.smithi098.stdout:20 2023-09-23T19:09:15.050 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_266403/img_v2 2023-09-23T19:09:15.050 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd remove testimg_import 2023-09-23T19:09:15.079 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:15.104 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:15.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:15.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:15.197 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 100% complete...done. 2023-09-23T19:09:15.203 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd remove testimg 2023-09-23T19:09:15.208 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:15.239 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:15.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:15.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:15.343 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:15.344 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 100% complete...done. 2023-09-23T19:09:15.349 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_266403/img testimg 2023-09-23T19:09:15.364 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:15.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:15.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:15.421 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 100% complete...done. 2023-09-23T19:09:15.427 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info testimg 2023-09-23T19:09:15.427 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep features 2023-09-23T19:09:15.467 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:15.484 INFO:tasks.workunit.client.0.smithi098.stderr:+ FEATURES_BEFORE= features: layering 2023-09-23T19:09:15.484 INFO:tasks.workunit.client.0.smithi098.stderr: op_features: 2023-09-23T19:09:15.484 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_266403/img_v2 2023-09-23T19:09:15.491 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:15.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:15.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:15.531 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T19:09:15.537 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_266403/img_v2 testimg_import 2023-09-23T19:09:15.595 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:15.613 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T19:09:15.618 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info testimg_import 2023-09-23T19:09:15.618 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep features 2023-09-23T19:09:15.624 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:15.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:15.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:15.666 INFO:tasks.workunit.client.0.smithi098.stderr:+ FEATURES_AFTER= features: layering 2023-09-23T19:09:15.667 INFO:tasks.workunit.client.0.smithi098.stderr: op_features: 2023-09-23T19:09:15.667 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ features: layering 2023-09-23T19:09:15.667 INFO:tasks.workunit.client.0.smithi098.stderr: op_features: != features: layering 2023-09-23T19:09:15.667 INFO:tasks.workunit.client.0.smithi098.stderr: op_features: ] 2023-09-23T19:09:15.667 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_266403/img_v2 2023-09-23T19:09:15.667 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd remove testimg_import 2023-09-23T19:09:15.729 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:15.755 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:15.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:15.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:15.775 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 100% complete...done. 2023-09-23T19:09:15.781 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd remove testimg 2023-09-23T19:09:15.858 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:15.885 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:15.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:15.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:15.898 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 100% complete...done. 2023-09-23T19:09:15.903 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_266403/img testimg 2023-09-23T19:09:15.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:15 smithi098 bash[76817]: cluster 2023-09-23T19:09:13.876874+0000 osd.4 (osd.4) 107 : cluster 3 failed to encode map e374 with expected crc 2023-09-23T19:09:15.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:15 smithi098 bash[76817]: cluster 2023-09-23T19:09:14.633005+0000 mon.a (mon.0) 2110 : cluster 0 osdmap e375: 8 total, 8 up, 7 in 2023-09-23T19:09:15.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:15 smithi098 bash[76817]: cluster 2023-09-23T19:09:14.633363+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e375 with expected crc 2023-09-23T19:09:15.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:15 smithi098 bash[76817]: cluster 2023-09-23T19:09:14.633829+0000 osd.6 (osd.6) 365 : cluster 3 failed to encode map e375 with expected crc 2023-09-23T19:09:15.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:15 smithi098 bash[76817]: cluster 2023-09-23T19:09:14.639831+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e375 with expected crc 2023-09-23T19:09:15.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:15 smithi098 bash[76817]: cluster 2023-09-23T19:09:14.680187+0000 mgr.y (mgr.26054) 1550 : cluster 0 pgmap v1721: 73 pgs: 73 active+clean; 4.6 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 50 KiB/s rd, 5.0 KiB/s wr, 71 op/s 2023-09-23T19:09:15.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:15 smithi098 bash[76817]: audit 2023-09-23T19:09:14.841563+0000 mon.a (mon.0) 2111 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:09:15.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:15 smithi098 bash[76817]: audit 2023-09-23T19:09:14.842643+0000 mon.a (mon.0) 2112 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:09:15.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:15 smithi098 bash[76817]: audit 2023-09-23T19:09:14.849915+0000 mon.a (mon.0) 2113 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:09:15.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:15 smithi098 bash[79516]: cluster 2023-09-23T19:09:13.876874+0000 osd.4 (osd.4) 107 : cluster 3 failed to encode map e374 with expected crc 2023-09-23T19:09:15.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:15 smithi098 bash[79516]: cluster 2023-09-23T19:09:14.633005+0000 mon.a (mon.0) 2110 : cluster 0 osdmap e375: 8 total, 8 up, 7 in 2023-09-23T19:09:15.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:15 smithi098 bash[79516]: cluster 2023-09-23T19:09:14.633363+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e375 with expected crc 2023-09-23T19:09:15.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:15 smithi098 bash[79516]: cluster 2023-09-23T19:09:14.633829+0000 osd.6 (osd.6) 365 : cluster 3 failed to encode map e375 with expected crc 2023-09-23T19:09:15.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:15 smithi098 bash[79516]: cluster 2023-09-23T19:09:14.639831+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e375 with expected crc 2023-09-23T19:09:15.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:15 smithi098 bash[79516]: cluster 2023-09-23T19:09:14.680187+0000 mgr.y (mgr.26054) 1550 : cluster 0 pgmap v1721: 73 pgs: 73 active+clean; 4.6 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 50 KiB/s rd, 5.0 KiB/s wr, 71 op/s 2023-09-23T19:09:15.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:15 smithi098 bash[79516]: audit 2023-09-23T19:09:14.841563+0000 mon.a (mon.0) 2111 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:09:15.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:15 smithi098 bash[79516]: audit 2023-09-23T19:09:14.842643+0000 mon.a (mon.0) 2112 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:09:15.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:15 smithi098 bash[79516]: audit 2023-09-23T19:09:14.849915+0000 mon.a (mon.0) 2113 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:09:15.988 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:15.994 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 100% complete...done. 2023-09-23T19:09:15.998 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_266403/img_v2 2023-09-23T19:09:16.014 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:16.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:16.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:16.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:15 smithi173 bash[59422]: cluster 2023-09-23T19:09:13.876874+0000 osd.4 (osd.4) 107 : cluster 3 failed to encode map e374 with expected crc 2023-09-23T19:09:16.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:15 smithi173 bash[59422]: cluster 2023-09-23T19:09:14.633005+0000 mon.a (mon.0) 2110 : cluster 0 osdmap e375: 8 total, 8 up, 7 in 2023-09-23T19:09:16.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:15 smithi173 bash[59422]: cluster 2023-09-23T19:09:14.633363+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e375 with expected crc 2023-09-23T19:09:16.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:15 smithi173 bash[59422]: cluster 2023-09-23T19:09:14.633829+0000 osd.6 (osd.6) 365 : cluster 3 failed to encode map e375 with expected crc 2023-09-23T19:09:16.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:15 smithi173 bash[59422]: cluster 2023-09-23T19:09:14.639831+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e375 with expected crc 2023-09-23T19:09:16.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:15 smithi173 bash[59422]: cluster 2023-09-23T19:09:14.680187+0000 mgr.y (mgr.26054) 1550 : cluster 0 pgmap v1721: 73 pgs: 73 active+clean; 4.6 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 50 KiB/s rd, 5.0 KiB/s wr, 71 op/s 2023-09-23T19:09:16.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:15 smithi173 bash[59422]: audit 2023-09-23T19:09:14.841563+0000 mon.a (mon.0) 2111 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:09:16.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:15 smithi173 bash[59422]: audit 2023-09-23T19:09:14.842643+0000 mon.a (mon.0) 2112 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:09:16.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:15 smithi173 bash[59422]: audit 2023-09-23T19:09:14.849915+0000 mon.a (mon.0) 2113 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:09:16.064 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T19:09:16.070 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_266403/img_v2 testimg_import 2023-09-23T19:09:16.118 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:16.150 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:16.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:16.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:16.168 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T19:09:16.173 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info testimg_import 2023-09-23T19:09:16.174 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep stripe unit 2023-09-23T19:09:16.174 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep -Ei (4 KiB|4096) 2023-09-23T19:09:16.225 INFO:tasks.workunit.client.0.smithi098.stdout: stripe unit: 4 KiB 2023-09-23T19:09:16.225 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info testimg_import 2023-09-23T19:09:16.226 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep stripe count 2023-09-23T19:09:16.226 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk {print $3} 2023-09-23T19:09:16.226 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep 1000 2023-09-23T19:09:16.253 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:16.265 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:09:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:09:16] "GET /metrics HTTP/1.1" 200 34177 "" "Prometheus/2.33.4" 2023-09-23T19:09:16.276 INFO:tasks.workunit.client.0.smithi098.stdout:1000 2023-09-23T19:09:16.277 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_266403/img_v2 2023-09-23T19:09:16.277 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd remove testimg_import 2023-09-23T19:09:16.284 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:16.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:16.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:16.388 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:16.417 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:16.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:16.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:16.421 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T19:09:16.426 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd remove testimg 2023-09-23T19:09:16.520 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:16.550 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:16.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:16.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:16.567 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T19:09:16.572 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_266403/img testimg 2023-09-23T19:09:16.654 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:16.655 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 100% complete...done. 2023-09-23T19:09:16.659 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd snap create testimg@snap1 2023-09-23T19:09:16.684 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:16.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:16.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:16.744 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T19:09:16.759 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T19:09:16.786 INFO:tasks.workunit.client.0.smithi098.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T19:09:16.788 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:16.795 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd snap create testimg@snap2 2023-09-23T19:09:16.816 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:16.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:16.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:16.921 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:16.950 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:16.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:16.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:17.006 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T19:09:17.053 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:17.084 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:17.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:17.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:17.187 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:17.216 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:17.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:17.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:17.320 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:17.350 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:17.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:17.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:17.453 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:17.488 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:17.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:17.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:17.593 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:17.621 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:17.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:17.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:17.725 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:17.755 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:17.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:17.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:17.794 INFO:tasks.workunit.client.0.smithi098.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T19:09:17.803 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd snap protect testimg@snap2 2023-09-23T19:09:17.859 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:17.866 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_266403/snap_protect 2023-09-23T19:09:17.869 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:17 smithi098 bash[79516]: cluster 2023-09-23T19:09:16.681231+0000 mgr.y (mgr.26054) 1551 : cluster 0 pgmap v1722: 73 pgs: 73 active+clean; 4.6 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 150 KiB/s rd, 16 KiB/s wr, 186 op/s 2023-09-23T19:09:17.869 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:17 smithi098 bash[79516]: cluster 2023-09-23T19:09:16.767425+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e376 with expected crc 2023-09-23T19:09:17.869 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:17 smithi098 bash[79516]: cluster 2023-09-23T19:09:16.768059+0000 osd.6 (osd.6) 366 : cluster 3 failed to encode map e376 with expected crc 2023-09-23T19:09:17.869 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:17 smithi098 bash[79516]: cluster 2023-09-23T19:09:16.768308+0000 mon.a (mon.0) 2114 : cluster 0 osdmap e376: 8 total, 8 up, 7 in 2023-09-23T19:09:17.869 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:17 smithi098 bash[79516]: cluster 2023-09-23T19:09:16.770378+0000 osd.4 (osd.4) 108 : cluster 3 failed to encode map e376 with expected crc 2023-09-23T19:09:17.869 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:17 smithi098 bash[79516]: cluster 2023-09-23T19:09:16.840967+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e376 with expected crc 2023-09-23T19:09:17.869 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:17 smithi098 bash[76817]: cluster 2023-09-23T19:09:16.681231+0000 mgr.y (mgr.26054) 1551 : cluster 0 pgmap v1722: 73 pgs: 73 active+clean; 4.6 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 150 KiB/s rd, 16 KiB/s wr, 186 op/s 2023-09-23T19:09:17.870 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:17 smithi098 bash[76817]: cluster 2023-09-23T19:09:16.767425+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e376 with expected crc 2023-09-23T19:09:17.870 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:17 smithi098 bash[76817]: cluster 2023-09-23T19:09:16.768059+0000 osd.6 (osd.6) 366 : cluster 3 failed to encode map e376 with expected crc 2023-09-23T19:09:17.870 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:17 smithi098 bash[76817]: cluster 2023-09-23T19:09:16.768308+0000 mon.a (mon.0) 2114 : cluster 0 osdmap e376: 8 total, 8 up, 7 in 2023-09-23T19:09:17.870 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:17 smithi098 bash[76817]: cluster 2023-09-23T19:09:16.770378+0000 osd.4 (osd.4) 108 : cluster 3 failed to encode map e376 with expected crc 2023-09-23T19:09:17.870 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:17 smithi098 bash[76817]: cluster 2023-09-23T19:09:16.840967+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e376 with expected crc 2023-09-23T19:09:17.886 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:17.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:17.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:17.917 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T19:09:17.922 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_266403/snap_protect testimg_import 2023-09-23T19:09:17.990 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:18.022 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:18.024 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:17 smithi173 bash[59422]: cluster 2023-09-23T19:09:16.681231+0000 mgr.y (mgr.26054) 1551 : cluster 0 pgmap v1722: 73 pgs: 73 active+clean; 4.6 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 150 KiB/s rd, 16 KiB/s wr, 186 op/s 2023-09-23T19:09:18.024 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:17 smithi173 bash[59422]: cluster 2023-09-23T19:09:16.767425+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e376 with expected crc 2023-09-23T19:09:18.024 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:17 smithi173 bash[59422]: cluster 2023-09-23T19:09:16.768059+0000 osd.6 (osd.6) 366 : cluster 3 failed to encode map e376 with expected crc 2023-09-23T19:09:18.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:17 smithi173 bash[59422]: cluster 2023-09-23T19:09:16.768308+0000 mon.a (mon.0) 2114 : cluster 0 osdmap e376: 8 total, 8 up, 7 in 2023-09-23T19:09:18.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:17 smithi173 bash[59422]: cluster 2023-09-23T19:09:16.770378+0000 osd.4 (osd.4) 108 : cluster 3 failed to encode map e376 with expected crc 2023-09-23T19:09:18.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:17 smithi173 bash[59422]: cluster 2023-09-23T19:09:16.840967+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e376 with expected crc 2023-09-23T19:09:18.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:18.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:18.126 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:18.160 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:18.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:18.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:18.264 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:18.300 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:18.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:18.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:18.403 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:09:18.434 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:09:18.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:18.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:09:18.538 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:18.571 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:18.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:18.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:18.675 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:18.704 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:18.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:18.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:18.808 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:18.818 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:18 smithi098 bash[79516]: cluster 2023-09-23T19:09:17.776684+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e377 with expected crc 2023-09-23T19:09:18.819 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:18 smithi098 bash[79516]: cluster 2023-09-23T19:09:17.776949+0000 mon.a (mon.0) 2115 : cluster 0 osdmap e377: 8 total, 8 up, 7 in 2023-09-23T19:09:18.819 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:18 smithi098 bash[79516]: cluster 2023-09-23T19:09:17.777402+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e377 with expected crc 2023-09-23T19:09:18.819 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:18 smithi098 bash[79516]: cluster 2023-09-23T19:09:17.785128+0000 osd.6 (osd.6) 367 : cluster 3 failed to encode map e377 with expected crc 2023-09-23T19:09:18.819 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:18 smithi098 bash[79516]: cluster 2023-09-23T19:09:17.932315+0000 osd.4 (osd.4) 109 : cluster 3 failed to encode map e377 with expected crc 2023-09-23T19:09:18.819 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:18 smithi098 bash[76817]: cluster 2023-09-23T19:09:17.776684+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e377 with expected crc 2023-09-23T19:09:18.819 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:18 smithi098 bash[76817]: cluster 2023-09-23T19:09:17.776949+0000 mon.a (mon.0) 2115 : cluster 0 osdmap e377: 8 total, 8 up, 7 in 2023-09-23T19:09:18.819 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:18 smithi098 bash[76817]: cluster 2023-09-23T19:09:17.777402+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e377 with expected crc 2023-09-23T19:09:18.819 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:18 smithi098 bash[76817]: cluster 2023-09-23T19:09:17.785128+0000 osd.6 (osd.6) 367 : cluster 3 failed to encode map e377 with expected crc 2023-09-23T19:09:18.819 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:18 smithi098 bash[76817]: cluster 2023-09-23T19:09:17.932315+0000 osd.4 (osd.4) 109 : cluster 3 failed to encode map e377 with expected crc 2023-09-23T19:09:18.837 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:18.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:18.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:18.940 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:18.969 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:18.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:18.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:19.073 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:19.105 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:19.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:19.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:19.209 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:09:19.223 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:18 smithi173 bash[59422]: cluster 2023-09-23T19:09:17.776684+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e377 with expected crc 2023-09-23T19:09:19.223 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:18 smithi173 bash[59422]: cluster 2023-09-23T19:09:17.776949+0000 mon.a (mon.0) 2115 : cluster 0 osdmap e377: 8 total, 8 up, 7 in 2023-09-23T19:09:19.223 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:18 smithi173 bash[59422]: cluster 2023-09-23T19:09:17.777402+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e377 with expected crc 2023-09-23T19:09:19.223 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:18 smithi173 bash[59422]: cluster 2023-09-23T19:09:17.785128+0000 osd.6 (osd.6) 367 : cluster 3 failed to encode map e377 with expected crc 2023-09-23T19:09:19.223 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:18 smithi173 bash[59422]: cluster 2023-09-23T19:09:17.932315+0000 osd.4 (osd.4) 109 : cluster 3 failed to encode map e377 with expected crc 2023-09-23T19:09:19.242 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:09:19.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:19.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:09:19.346 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:19.376 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:19.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:19.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:19.471 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:09:19.479 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:19.511 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:19.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:19.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:19.615 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:19.645 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:19.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:19.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:19.749 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:19.780 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:19.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:19.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:19.807 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:19 smithi098 bash[76817]: cluster 2023-09-23T19:09:18.681892+0000 mgr.y (mgr.26054) 1552 : cluster 0 pgmap v1725: 73 pgs: 73 active+clean; 4.7 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 337 KiB/s rd, 34 KiB/s wr, 439 op/s 2023-09-23T19:09:19.807 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:19 smithi098 bash[76817]: cluster 2023-09-23T19:09:18.786363+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e378 with expected crc 2023-09-23T19:09:19.808 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:19 smithi098 bash[76817]: cluster 2023-09-23T19:09:18.786598+0000 mon.a (mon.0) 2116 : cluster 0 osdmap e378: 8 total, 8 up, 7 in 2023-09-23T19:09:19.808 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:19 smithi098 bash[76817]: cluster 2023-09-23T19:09:18.802336+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e378 with expected crc 2023-09-23T19:09:19.837 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T19:09:19.843 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info testimg_import@snap1 2023-09-23T19:09:19.843 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep protected: False 2023-09-23T19:09:19.884 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:19.895 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:19 smithi098 bash[79516]: cluster 2023-09-23T19:09:18.681892+0000 mgr.y (mgr.26054) 1552 : cluster 0 pgmap v1725: 73 pgs: 73 active+clean; 4.7 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 337 KiB/s rd, 34 KiB/s wr, 439 op/s 2023-09-23T19:09:19.895 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:19 smithi098 bash[79516]: cluster 2023-09-23T19:09:18.786363+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e378 with expected crc 2023-09-23T19:09:19.895 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:19 smithi098 bash[79516]: cluster 2023-09-23T19:09:18.786598+0000 mon.a (mon.0) 2116 : cluster 0 osdmap e378: 8 total, 8 up, 7 in 2023-09-23T19:09:19.896 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:19 smithi098 bash[79516]: cluster 2023-09-23T19:09:18.802336+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e378 with expected crc 2023-09-23T19:09:19.901 INFO:tasks.workunit.client.0.smithi098.stdout: protected: False 2023-09-23T19:09:19.902 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info+ testimg_import@snap2 2023-09-23T19:09:19.902 INFO:tasks.workunit.client.0.smithi098.stderr:grep protected: True 2023-09-23T19:09:19.912 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:19.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:19.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:19.956 INFO:tasks.workunit.client.0.smithi098.stdout: protected: True 2023-09-23T19:09:19.957 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_266403/snap_protect 2023-09-23T19:09:19.958 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd snap unprotect testimg@snap2 2023-09-23T19:09:20.015 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:20.020 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd snap unprotect testimg_import@snap2 2023-09-23T19:09:20.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:19 smithi173 bash[59422]: cluster 2023-09-23T19:09:18.681892+0000 mgr.y (mgr.26054) 1552 : cluster 0 pgmap v1725: 73 pgs: 73 active+clean; 4.7 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 337 KiB/s rd, 34 KiB/s wr, 439 op/s 2023-09-23T19:09:20.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:19 smithi173 bash[59422]: cluster 2023-09-23T19:09:18.786363+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e378 with expected crc 2023-09-23T19:09:20.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:19 smithi173 bash[59422]: cluster 2023-09-23T19:09:18.786598+0000 mon.a (mon.0) 2116 : cluster 0 osdmap e378: 8 total, 8 up, 7 in 2023-09-23T19:09:20.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:19 smithi173 bash[59422]: cluster 2023-09-23T19:09:18.802336+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e378 with expected crc 2023-09-23T19:09:20.040 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:20.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:20.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:20.077 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd snap purge testimg 2023-09-23T19:09:20.144 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:20.170 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:20.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:20.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:20.274 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:20.306 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:20.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:20.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:20.409 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:20.441 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:20.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:20.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:20.545 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:09:20.577 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:09:20.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:20.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:09:20.679 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:20.709 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:20.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:20.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:20.812 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:20.841 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:20.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:20.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:20.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:20 smithi098 bash[79516]: cluster 2023-09-23T19:09:19.792372+0000 mon.a (mon.0) 2117 : cluster 0 osdmap e379: 8 total, 8 up, 7 in 2023-09-23T19:09:20.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:20 smithi098 bash[79516]: cluster 2023-09-23T19:09:19.793982+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e379 with expected crc 2023-09-23T19:09:20.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:20 smithi098 bash[79516]: cluster 2023-09-23T19:09:19.794064+0000 osd.4 (osd.4) 110 : cluster 3 failed to encode map e379 with expected crc 2023-09-23T19:09:20.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:20 smithi098 bash[79516]: cluster 2023-09-23T19:09:19.794154+0000 osd.6 (osd.6) 368 : cluster 3 failed to encode map e379 with expected crc 2023-09-23T19:09:20.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:20 smithi098 bash[79516]: cluster 2023-09-23T19:09:19.815688+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e379 with expected crc 2023-09-23T19:09:20.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:20 smithi098 bash[79516]: audit 2023-09-23T19:09:20.559781+0000 mon.a (mon.0) 2118 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:09:20.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:20 smithi098 bash[79516]: cluster 2023-09-23T19:09:20.612059+0000 osd.4 (osd.4) 111 : cluster 3 failed to encode map e380 with expected crc 2023-09-23T19:09:20.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:20 smithi098 bash[79516]: cluster 2023-09-23T19:09:20.613208+0000 mon.a (mon.0) 2119 : cluster 0 osdmap e380: 8 total, 8 up, 7 in 2023-09-23T19:09:20.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:20 smithi098 bash[76817]: cluster 2023-09-23T19:09:19.792372+0000 mon.a (mon.0) 2117 : cluster 0 osdmap e379: 8 total, 8 up, 7 in 2023-09-23T19:09:20.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:20 smithi098 bash[76817]: cluster 2023-09-23T19:09:19.793982+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e379 with expected crc 2023-09-23T19:09:20.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:20 smithi098 bash[76817]: cluster 2023-09-23T19:09:19.794064+0000 osd.4 (osd.4) 110 : cluster 3 failed to encode map e379 with expected crc 2023-09-23T19:09:20.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:20 smithi098 bash[76817]: cluster 2023-09-23T19:09:19.794154+0000 osd.6 (osd.6) 368 : cluster 3 failed to encode map e379 with expected crc 2023-09-23T19:09:20.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:20 smithi098 bash[76817]: cluster 2023-09-23T19:09:19.815688+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e379 with expected crc 2023-09-23T19:09:20.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:20 smithi098 bash[76817]: audit 2023-09-23T19:09:20.559781+0000 mon.a (mon.0) 2118 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:09:20.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:20 smithi098 bash[76817]: cluster 2023-09-23T19:09:20.612059+0000 osd.4 (osd.4) 111 : cluster 3 failed to encode map e380 with expected crc 2023-09-23T19:09:20.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:20 smithi098 bash[76817]: cluster 2023-09-23T19:09:20.613208+0000 mon.a (mon.0) 2119 : cluster 0 osdmap e380: 8 total, 8 up, 7 in 2023-09-23T19:09:20.946 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:20.979 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:20.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:20.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:21.084 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:21.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:20 smithi173 bash[59422]: cluster 2023-09-23T19:09:19.792372+0000 mon.a (mon.0) 2117 : cluster 0 osdmap e379: 8 total, 8 up, 7 in 2023-09-23T19:09:21.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:20 smithi173 bash[59422]: cluster 2023-09-23T19:09:19.793982+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e379 with expected crc 2023-09-23T19:09:21.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:20 smithi173 bash[59422]: cluster 2023-09-23T19:09:19.794064+0000 osd.4 (osd.4) 110 : cluster 3 failed to encode map e379 with expected crc 2023-09-23T19:09:21.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:20 smithi173 bash[59422]: cluster 2023-09-23T19:09:19.794154+0000 osd.6 (osd.6) 368 : cluster 3 failed to encode map e379 with expected crc 2023-09-23T19:09:21.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:20 smithi173 bash[59422]: cluster 2023-09-23T19:09:19.815688+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e379 with expected crc 2023-09-23T19:09:21.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:20 smithi173 bash[59422]: audit 2023-09-23T19:09:20.559781+0000 mon.a (mon.0) 2118 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:09:21.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:20 smithi173 bash[59422]: cluster 2023-09-23T19:09:20.612059+0000 osd.4 (osd.4) 111 : cluster 3 failed to encode map e380 with expected crc 2023-09-23T19:09:21.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:20 smithi173 bash[59422]: cluster 2023-09-23T19:09:20.613208+0000 mon.a (mon.0) 2119 : cluster 0 osdmap e380: 8 total, 8 up, 7 in 2023-09-23T19:09:21.116 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:21.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:21.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:21.219 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:21.247 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:21.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:21.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:21.350 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:21.380 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:21.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:21.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:21.484 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:09:21.498 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:09:21.501 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:09:21.512 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:09:21.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:21.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:09:21.581 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:09:21.616 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:21.635 INFO:tasks.workunit.client.0.smithi098.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T19:09:21.644 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:21.644 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd snap purge testimg_import 2023-09-23T19:09:21.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:21.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:21.747 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:21.777 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:21.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:21.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:21.881 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:21.911 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:21.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:21.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:22.015 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:22.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:21 smithi173 bash[59422]: cluster 2023-09-23T19:09:20.611461+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e380 with expected crc 2023-09-23T19:09:22.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:21 smithi173 bash[59422]: cluster 2023-09-23T19:09:20.625377+0000 osd.6 (osd.6) 369 : cluster 3 failed to encode map e380 with expected crc 2023-09-23T19:09:22.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:21 smithi173 bash[59422]: cluster 2023-09-23T19:09:20.630463+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e380 with expected crc 2023-09-23T19:09:22.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:21 smithi173 bash[59422]: cluster 2023-09-23T19:09:20.682305+0000 mgr.y (mgr.26054) 1553 : cluster 0 pgmap v1729: 73 pgs: 73 active+clean; 4.7 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 366 KiB/s rd, 36 KiB/s wr, 494 op/s 2023-09-23T19:09:22.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:21 smithi173 bash[59422]: cluster 2023-09-23T19:09:21.625388+0000 mon.a (mon.0) 2120 : cluster 0 osdmap e381: 8 total, 8 up, 7 in 2023-09-23T19:09:22.042 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:22.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:22.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:22.145 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:22.172 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:22.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:22.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:22.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:21 smithi098 bash[76817]: cluster 2023-09-23T19:09:20.611461+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e380 with expected crc 2023-09-23T19:09:22.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:21 smithi098 bash[76817]: cluster 2023-09-23T19:09:20.625377+0000 osd.6 (osd.6) 369 : cluster 3 failed to encode map e380 with expected crc 2023-09-23T19:09:22.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:21 smithi098 bash[76817]: cluster 2023-09-23T19:09:20.630463+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e380 with expected crc 2023-09-23T19:09:22.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:21 smithi098 bash[76817]: cluster 2023-09-23T19:09:20.682305+0000 mgr.y (mgr.26054) 1553 : cluster 0 pgmap v1729: 73 pgs: 73 active+clean; 4.7 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 366 KiB/s rd, 36 KiB/s wr, 494 op/s 2023-09-23T19:09:22.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:21 smithi098 bash[76817]: cluster 2023-09-23T19:09:21.625388+0000 mon.a (mon.0) 2120 : cluster 0 osdmap e381: 8 total, 8 up, 7 in 2023-09-23T19:09:22.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:21 smithi098 bash[79516]: cluster 2023-09-23T19:09:20.611461+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e380 with expected crc 2023-09-23T19:09:22.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:21 smithi098 bash[79516]: cluster 2023-09-23T19:09:20.625377+0000 osd.6 (osd.6) 369 : cluster 3 failed to encode map e380 with expected crc 2023-09-23T19:09:22.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:21 smithi098 bash[79516]: cluster 2023-09-23T19:09:20.630463+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e380 with expected crc 2023-09-23T19:09:22.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:21 smithi098 bash[79516]: cluster 2023-09-23T19:09:20.682305+0000 mgr.y (mgr.26054) 1553 : cluster 0 pgmap v1729: 73 pgs: 73 active+clean; 4.7 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 366 KiB/s rd, 36 KiB/s wr, 494 op/s 2023-09-23T19:09:22.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:21 smithi098 bash[79516]: cluster 2023-09-23T19:09:21.625388+0000 mon.a (mon.0) 2120 : cluster 0 osdmap e381: 8 total, 8 up, 7 in 2023-09-23T19:09:22.275 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:22.307 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:22.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:22.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:22.410 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:22.436 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:22.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:22.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:22.540 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:22.570 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:22.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:22.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:22.674 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:22.699 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:22.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:22.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:22.803 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:22.813 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:22 smithi098 bash[76817]: cluster 2023-09-23T19:09:21.626865+0000 osd.7 (osd.7) 23 : cluster 3 failed to encode map e381 with expected crc 2023-09-23T19:09:22.813 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:22 smithi098 bash[76817]: cluster 2023-09-23T19:09:21.627470+0000 osd.6 (osd.6) 370 : cluster 3 failed to encode map e381 with expected crc 2023-09-23T19:09:22.813 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:22 smithi098 bash[76817]: cluster 2023-09-23T19:09:21.683407+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e381 with expected crc 2023-09-23T19:09:22.813 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:22 smithi098 bash[76817]: cluster 2023-09-23T19:09:22.308487+0000 osd.4 (osd.4) 112 : cluster 3 failed to encode map e381 with expected crc 2023-09-23T19:09:22.813 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:22 smithi098 bash[76817]: cluster 2023-09-23T19:09:22.629015+0000 mon.a (mon.0) 2121 : cluster 0 osdmap e382: 8 total, 8 up, 7 in 2023-09-23T19:09:22.813 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:22 smithi098 bash[79516]: cluster 2023-09-23T19:09:21.626865+0000 osd.7 (osd.7) 23 : cluster 3 failed to encode map e381 with expected crc 2023-09-23T19:09:22.814 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:22 smithi098 bash[79516]: cluster 2023-09-23T19:09:21.627470+0000 osd.6 (osd.6) 370 : cluster 3 failed to encode map e381 with expected crc 2023-09-23T19:09:22.814 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:22 smithi098 bash[79516]: cluster 2023-09-23T19:09:21.683407+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e381 with expected crc 2023-09-23T19:09:22.814 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:22 smithi098 bash[79516]: cluster 2023-09-23T19:09:22.308487+0000 osd.4 (osd.4) 112 : cluster 3 failed to encode map e381 with expected crc 2023-09-23T19:09:22.814 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:22 smithi098 bash[79516]: cluster 2023-09-23T19:09:22.629015+0000 mon.a (mon.0) 2121 : cluster 0 osdmap e382: 8 total, 8 up, 7 in 2023-09-23T19:09:22.831 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:22.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:22.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:22.933 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:22.959 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:22.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:22.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:23.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:22 smithi173 bash[59422]: cluster 2023-09-23T19:09:21.626865+0000 osd.7 (osd.7) 23 : cluster 3 failed to encode map e381 with expected crc 2023-09-23T19:09:23.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:22 smithi173 bash[59422]: cluster 2023-09-23T19:09:21.627470+0000 osd.6 (osd.6) 370 : cluster 3 failed to encode map e381 with expected crc 2023-09-23T19:09:23.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:22 smithi173 bash[59422]: cluster 2023-09-23T19:09:21.683407+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e381 with expected crc 2023-09-23T19:09:23.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:22 smithi173 bash[59422]: cluster 2023-09-23T19:09:22.308487+0000 osd.4 (osd.4) 112 : cluster 3 failed to encode map e381 with expected crc 2023-09-23T19:09:23.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:22 smithi173 bash[59422]: cluster 2023-09-23T19:09:22.629015+0000 mon.a (mon.0) 2121 : cluster 0 osdmap e382: 8 total, 8 up, 7 in 2023-09-23T19:09:23.062 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:23.088 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:23.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:23.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:23.191 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:23.222 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:23.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:23.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:23.326 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:23.353 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:23.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:23.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:23.457 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:23.478 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:23.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:23.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:23.582 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:23.608 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:23.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:23.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:23.648 INFO:tasks.workunit.client.0.smithi098.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T19:09:23.659 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd remove testimg 2023-09-23T19:09:23.711 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:23.737 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:23.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:23.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:23.785 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 100% complete...done. 2023-09-23T19:09:23.789 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd remove testimg_import 2023-09-23T19:09:23.841 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:23.866 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:23.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:23.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:23.923 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 100% complete...done. 2023-09-23T19:09:23.929 INFO:tasks.workunit.client.0.smithi098.stderr:+ tiered=0 2023-09-23T19:09:23.929 INFO:tasks.workunit.client.0.smithi098.stderr:+ ceph osd dump 2023-09-23T19:09:23.929 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep ^pool 2023-09-23T19:09:23.929 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep 'rbd' 2023-09-23T19:09:23.929 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep tier 2023-09-23T19:09:23.970 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:23.981 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:23 smithi098 bash[79516]: cluster 2023-09-23T19:09:22.630968+0000 osd.6 (osd.6) 371 : cluster 3 failed to encode map e382 with expected crc 2023-09-23T19:09:23.981 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:23 smithi098 bash[79516]: cluster 2023-09-23T19:09:22.634559+0000 osd.7 (osd.7) 24 : cluster 3 failed to encode map e382 with expected crc 2023-09-23T19:09:23.981 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:23 smithi098 bash[79516]: cluster 2023-09-23T19:09:22.644838+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e382 with expected crc 2023-09-23T19:09:23.981 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:23 smithi098 bash[79516]: cluster 2023-09-23T19:09:22.682816+0000 mgr.y (mgr.26054) 1554 : cluster 0 pgmap v1732: 73 pgs: 73 active+clean; 4.8 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 183 KiB/s rd, 22 KiB/s wr, 241 op/s 2023-09-23T19:09:23.981 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:23 smithi098 bash[79516]: cluster 2023-09-23T19:09:23.636445+0000 mon.a (mon.0) 2122 : cluster 0 osdmap e383: 8 total, 8 up, 7 in 2023-09-23T19:09:23.981 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:23 smithi098 bash[76817]: cluster 2023-09-23T19:09:22.630968+0000 osd.6 (osd.6) 371 : cluster 3 failed to encode map e382 with expected crc 2023-09-23T19:09:23.981 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:23 smithi098 bash[76817]: cluster 2023-09-23T19:09:22.634559+0000 osd.7 (osd.7) 24 : cluster 3 failed to encode map e382 with expected crc 2023-09-23T19:09:23.981 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:23 smithi098 bash[76817]: cluster 2023-09-23T19:09:22.644838+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e382 with expected crc 2023-09-23T19:09:23.982 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:23 smithi098 bash[76817]: cluster 2023-09-23T19:09:22.682816+0000 mgr.y (mgr.26054) 1554 : cluster 0 pgmap v1732: 73 pgs: 73 active+clean; 4.8 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 183 KiB/s rd, 22 KiB/s wr, 241 op/s 2023-09-23T19:09:23.982 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:23 smithi098 bash[76817]: cluster 2023-09-23T19:09:23.636445+0000 mon.a (mon.0) 2122 : cluster 0 osdmap e383: 8 total, 8 up, 7 in 2023-09-23T19:09:23.995 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:23.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:23.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:24.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:23 smithi173 bash[59422]: cluster 2023-09-23T19:09:22.630968+0000 osd.6 (osd.6) 371 : cluster 3 failed to encode map e382 with expected crc 2023-09-23T19:09:24.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:23 smithi173 bash[59422]: cluster 2023-09-23T19:09:22.634559+0000 osd.7 (osd.7) 24 : cluster 3 failed to encode map e382 with expected crc 2023-09-23T19:09:24.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:23 smithi173 bash[59422]: cluster 2023-09-23T19:09:22.644838+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e382 with expected crc 2023-09-23T19:09:24.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:23 smithi173 bash[59422]: cluster 2023-09-23T19:09:22.682816+0000 mgr.y (mgr.26054) 1554 : cluster 0 pgmap v1732: 73 pgs: 73 active+clean; 4.8 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 183 KiB/s rd, 22 KiB/s wr, 241 op/s 2023-09-23T19:09:24.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:23 smithi173 bash[59422]: cluster 2023-09-23T19:09:23.636445+0000 mon.a (mon.0) 2122 : cluster 0 osdmap e383: 8 total, 8 up, 7 in 2023-09-23T19:09:24.100 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:24.129 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:24.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:24.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:24.233 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:24.268 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:24.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:24.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:24.371 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:24.402 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:24.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:24.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:24.505 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:24.534 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:24.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:24.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:24.638 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:09:24.659 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:09:24.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:24.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:09:24.693 INFO:tasks.workunit.client.0.smithi098.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_266403/sparse1 2023-09-23T19:09:24.697 INFO:tasks.workunit.client.0.smithi098.stderr:1+0 records in 2023-09-23T19:09:24.697 INFO:tasks.workunit.client.0.smithi098.stderr:1+0 records out 2023-09-23T19:09:24.697 INFO:tasks.workunit.client.0.smithi098.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00355677 s, 295 MB/s 2023-09-23T19:09:24.697 INFO:tasks.workunit.client.0.smithi098.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_266403/sparse2 2023-09-23T19:09:24.701 INFO:tasks.workunit.client.0.smithi098.stderr:1+0 records in 2023-09-23T19:09:24.701 INFO:tasks.workunit.client.0.smithi098.stderr:1+0 records out 2023-09-23T19:09:24.701 INFO:tasks.workunit.client.0.smithi098.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00353717 s, 296 MB/s 2023-09-23T19:09:24.702 INFO:tasks.workunit.client.0.smithi098.stderr:+ truncate /tmp/rbd_import_export_266403/sparse2 -s 2M 2023-09-23T19:09:24.728 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm sparse1 2023-09-23T19:09:24.763 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:24.773 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 0% complete...failed. 2023-09-23T19:09:24.773 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T19:09:24.776 INFO:tasks.workunit.client.0.smithi098.stderr:+ true 2023-09-23T19:09:24.777 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_266403/sparse1 2023-09-23T19:09:24.789 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:24.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:24.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:24.863 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T19:09:24.863 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T19:09:24.863 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T19:09:24.866 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd ls -l 2023-09-23T19:09:24.866 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep sparse1 2023-09-23T19:09:24.867 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep -Ei (2 MiB|2048k) 2023-09-23T19:09:24.893 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:24.907 INFO:tasks.workunit.client.0.smithi098.stdout:sparse1 2 MiB 2 2023-09-23T19:09:24.908 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects sparse1 2023-09-23T19:09:24.908 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse1 2023-09-23T19:09:24.908 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse1 2023-09-23T19:09:24.908 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep block_name_prefix 2023-09-23T19:09:24.908 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk {print $NF;} 2023-09-23T19:09:24.912 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:24.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:24.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:24.950 INFO:tasks.workunit.client.0.smithi098.stderr:+ prefix=rbd_data.8e5fb0a3a1d2 2023-09-23T19:09:24.950 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep rbd_data.8e5fb0a3a1d2 2023-09-23T19:09:24.950 INFO:tasks.workunit.client.0.smithi098.stderr:+ get_image_data_pool sparse1 2023-09-23T19:09:24.950 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse1 2023-09-23T19:09:24.950 INFO:tasks.workunit.client.0.smithi098.stderr:+ sed -e s/rbd_data.8e5fb0a3a1d2\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-09-23T19:09:24.950 INFO:tasks.workunit.client.0.smithi098.stderr:+ sort -u 2023-09-23T19:09:24.950 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse1 2023-09-23T19:09:24.950 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep data_pool: 2023-09-23T19:09:24.950 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk -F: { print $NF } 2023-09-23T19:09:24.989 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool= 2023-09-23T19:09:24.989 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ -z ] 2023-09-23T19:09:24.989 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool=rbd 2023-09-23T19:09:24.989 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo rbd 2023-09-23T19:09:24.989 INFO:tasks.workunit.client.0.smithi098.stderr:+ rados ls -p rbd 2023-09-23T19:09:25.015 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:25.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:24 smithi173 bash[59422]: cluster 2023-09-23T19:09:23.637895+0000 osd.7 (osd.7) 25 : cluster 3 failed to encode map e383 with expected crc 2023-09-23T19:09:25.026 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:24 smithi173 bash[59422]: cluster 2023-09-23T19:09:23.638119+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e383 with expected crc 2023-09-23T19:09:25.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:24 smithi173 bash[59422]: audit 2023-09-23T19:09:24.569628+0000 mon.a (mon.0) 2123 : audit 0 from='client.? 172.21.15.98:0/730389981' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T19:09:25.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:24 smithi173 bash[59422]: audit 2023-09-23T19:09:24.655245+0000 mon.a (mon.0) 2124 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:09:25.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:24 smithi173 bash[59422]: audit 2023-09-23T19:09:24.656736+0000 mon.a (mon.0) 2125 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:09:25.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:24 smithi173 bash[59422]: audit 2023-09-23T19:09:24.673996+0000 mon.a (mon.0) 2126 : audit 0 from='client.? 172.21.15.98:0/730389981' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T19:09:25.045 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:25.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:25.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:25.104 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects=1 2023-09-23T19:09:25.104 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo 1 2023-09-23T19:09:25.104 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-09-23T19:09:25.104 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export sparse1 /tmp/rbd_import_export_266403/sparse1.out 2023-09-23T19:09:25.149 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:09:25.153 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T19:09:25.158 INFO:tasks.workunit.client.0.smithi098.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_266403/sparse1 /tmp/rbd_import_export_266403/sparse1.out 2023-09-23T19:09:25.158 INFO:tasks.workunit.client.0.smithi098.stderr:+ cmp -l /tmp/rbd_import_export_266403/sparse1 /tmp/rbd_import_export_266403/sparse1.out 2023-09-23T19:09:25.161 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_266403/sparse1 --format %b 2023-09-23T19:09:25.162 INFO:tasks.workunit.client.0.smithi098.stderr:+ origsize=2048 2023-09-23T19:09:25.162 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_266403/sparse1.out --format %b 2023-09-23T19:09:25.163 INFO:tasks.workunit.client.0.smithi098.stderr:+ exportsize=2048 2023-09-23T19:09:25.164 INFO:tasks.workunit.client.0.smithi098.stderr:+ difference=0 2023-09-23T19:09:25.164 INFO:tasks.workunit.client.0.smithi098.stderr:+ difference=0 2023-09-23T19:09:25.164 INFO:tasks.workunit.client.0.smithi098.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T19:09:25.164 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_266403/sparse1.out 2023-09-23T19:09:25.164 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm sparse1 2023-09-23T19:09:25.179 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:09:25.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:25.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:09:25.193 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:24 smithi098 bash[76817]: cluster 2023-09-23T19:09:23.637895+0000 osd.7 (osd.7) 25 : cluster 3 failed to encode map e383 with expected crc 2023-09-23T19:09:25.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:24 smithi098 bash[76817]: cluster 2023-09-23T19:09:23.638119+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e383 with expected crc 2023-09-23T19:09:25.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:24 smithi098 bash[76817]: audit 2023-09-23T19:09:24.569628+0000 mon.a (mon.0) 2123 : audit 0 from='client.? 172.21.15.98:0/730389981' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T19:09:25.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:24 smithi098 bash[76817]: audit 2023-09-23T19:09:24.655245+0000 mon.a (mon.0) 2124 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:09:25.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:24 smithi098 bash[76817]: audit 2023-09-23T19:09:24.656736+0000 mon.a (mon.0) 2125 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:09:25.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:24 smithi098 bash[76817]: audit 2023-09-23T19:09:24.673996+0000 mon.a (mon.0) 2126 : audit 0 from='client.? 172.21.15.98:0/730389981' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T19:09:25.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:24 smithi098 bash[79516]: cluster 2023-09-23T19:09:23.637895+0000 osd.7 (osd.7) 25 : cluster 3 failed to encode map e383 with expected crc 2023-09-23T19:09:25.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:24 smithi098 bash[79516]: cluster 2023-09-23T19:09:23.638119+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e383 with expected crc 2023-09-23T19:09:25.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:24 smithi098 bash[79516]: audit 2023-09-23T19:09:24.569628+0000 mon.a (mon.0) 2123 : audit 0 from='client.? 172.21.15.98:0/730389981' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T19:09:25.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:24 smithi098 bash[79516]: audit 2023-09-23T19:09:24.655245+0000 mon.a (mon.0) 2124 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:09:25.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:24 smithi098 bash[79516]: audit 2023-09-23T19:09:24.656736+0000 mon.a (mon.0) 2125 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:09:25.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:24 smithi098 bash[79516]: audit 2023-09-23T19:09:24.673996+0000 mon.a (mon.0) 2126 : audit 0 from='client.? 172.21.15.98:0/730389981' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T19:09:25.282 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:25.303 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T19:09:25.308 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:25.308 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm sparse2 2023-09-23T19:09:25.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:25.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:25.363 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 0% complete...failed. 2023-09-23T19:09:25.363 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T19:09:25.366 INFO:tasks.workunit.client.0.smithi098.stderr:+ true 2023-09-23T19:09:25.366 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_266403/sparse2 2023-09-23T19:09:25.411 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:25.435 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:25.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:25.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:25.447 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T19:09:25.448 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T19:09:25.448 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T19:09:25.451 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd ls -l 2023-09-23T19:09:25.451 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep -Ei (2 MiB|2048k) 2023-09-23T19:09:25.451 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep sparse2 2023-09-23T19:09:25.503 INFO:tasks.workunit.client.0.smithi098.stdout:sparse2 2 MiB 2 2023-09-23T19:09:25.504 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects sparse2 2023-09-23T19:09:25.504 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse2 2023-09-23T19:09:25.504 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse2 2023-09-23T19:09:25.504 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep block_name_prefix 2023-09-23T19:09:25.505 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk {print $NF;} 2023-09-23T19:09:25.540 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:25.558 INFO:tasks.workunit.client.0.smithi098.stderr:+ prefix=rbd_data.b5979a0c8808 2023-09-23T19:09:25.560 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep rbd_data.b5979a0c8808 2023-09-23T19:09:25.560 INFO:tasks.workunit.client.0.smithi098.stderr:+ sed -e s/rbd_data.b5979a0c8808\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-09-23T19:09:25.560 INFO:tasks.workunit.client.0.smithi098.stderr:+ get_image_data_pool sparse2 2023-09-23T19:09:25.560 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse2 2023-09-23T19:09:25.560 INFO:tasks.workunit.client.0.smithi098.stderr:+ sort -u 2023-09-23T19:09:25.560 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse2 2023-09-23T19:09:25.560 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep data_pool: 2023-09-23T19:09:25.560 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk -F: { print $NF } 2023-09-23T19:09:25.569 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:25.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:25.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:25.611 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool= 2023-09-23T19:09:25.611 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ -z ] 2023-09-23T19:09:25.611 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool=rbd 2023-09-23T19:09:25.611 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo rbd 2023-09-23T19:09:25.611 INFO:tasks.workunit.client.0.smithi098.stderr:+ rados ls -p rbd 2023-09-23T19:09:25.650 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects=0 2023-09-23T19:09:25.650 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo 0 2023-09-23T19:09:25.650 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2023-09-23T19:09:25.650 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export sparse2 /tmp/rbd_import_export_266403/sparse2.out 2023-09-23T19:09:25.674 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:09:25.696 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T19:09:25.698 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:09:25.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:25.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:09:25.701 INFO:tasks.workunit.client.0.smithi098.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_266403/sparse2 /tmp/rbd_import_export_266403/sparse2.out 2023-09-23T19:09:25.701 INFO:tasks.workunit.client.0.smithi098.stderr:+ cmp -l /tmp/rbd_import_export_266403/sparse2 /tmp/rbd_import_export_266403/sparse2.out 2023-09-23T19:09:25.705 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_266403/sparse2 --format %b 2023-09-23T19:09:25.706 INFO:tasks.workunit.client.0.smithi098.stderr:+ origsize=2048 2023-09-23T19:09:25.706 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_266403/sparse2.out --format %b 2023-09-23T19:09:25.707 INFO:tasks.workunit.client.0.smithi098.stderr:+ exportsize=2048 2023-09-23T19:09:25.707 INFO:tasks.workunit.client.0.smithi098.stderr:+ difference=0 2023-09-23T19:09:25.707 INFO:tasks.workunit.client.0.smithi098.stderr:+ difference=0 2023-09-23T19:09:25.707 INFO:tasks.workunit.client.0.smithi098.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T19:09:25.708 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_266403/sparse2.out 2023-09-23T19:09:25.709 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm sparse2 2023-09-23T19:09:25.802 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:25.829 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:25.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:25.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:25.847 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T19:09:25.853 INFO:tasks.workunit.client.0.smithi098.stderr:+ truncate /tmp/rbd_import_export_266403/sparse1 -s 10M 2023-09-23T19:09:25.853 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --new-format --order 20 - sparse1 2023-09-23T19:09:25.933 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:25.962 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T19:09:25.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:25.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T19:09:25.978 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 100% complete...done. 2023-09-23T19:09:25.979 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T19:09:25.979 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T19:09:25.983 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd ls -l 2023-09-23T19:09:25.984 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep sparse1 2023-09-23T19:09:25.984 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep -Ei (10 MiB|10240k) 2023-09-23T19:09:26.031 INFO:tasks.workunit.client.0.smithi098.stdout:sparse1 10 MiB 2 2023-09-23T19:09:26.032 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects sparse1 2023-09-23T19:09:26.033 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse1 2023-09-23T19:09:26.033 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse1 2023-09-23T19:09:26.033 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep block_name_prefix 2023-09-23T19:09:26.033 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk {print $NF;} 2023-09-23T19:09:26.067 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:26.076 INFO:tasks.workunit.client.0.smithi098.stderr:+ prefix=rbd_data.8ebf55e31aac 2023-09-23T19:09:26.077 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep rbd_data.8ebf55e31aac 2023-09-23T19:09:26.077 INFO:tasks.workunit.client.0.smithi098.stderr:+ sed -e s/rbd_data.8ebf55e31aac\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-09-23T19:09:26.077 INFO:tasks.workunit.client.0.smithi098.stderr:+ get_image_data_pool sparse1 2023-09-23T19:09:26.077 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse1 2023-09-23T19:09:26.077 INFO:tasks.workunit.client.0.smithi098.stderr:+ sort -u 2023-09-23T19:09:26.077 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:25 smithi098 bash[76817]: cluster 2023-09-23T19:09:24.683151+0000 mgr.y (mgr.26054) 1555 : cluster 0 pgmap v1734: 73 pgs: 73 active+clean; 4.8 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 76 KiB/s rd, 12 KiB/s wr, 106 op/s 2023-09-23T19:09:26.078 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:25 smithi098 bash[79516]: cluster 2023-09-23T19:09:24.683151+0000 mgr.y (mgr.26054) 1555 : cluster 0 pgmap v1734: 73 pgs: 73 active+clean; 4.8 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 76 KiB/s rd, 12 KiB/s wr, 106 op/s 2023-09-23T19:09:26.078 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse1 2023-09-23T19:09:26.078 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep data_pool: 2023-09-23T19:09:26.078 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk -F: { print $NF } 2023-09-23T19:09:26.090 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:26.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:26.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:26.115 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-09-23T19:09:26.115 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T19:09:26.120 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool= 2023-09-23T19:09:26.121 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ -z ] 2023-09-23T19:09:26.121 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool=rbd 2023-09-23T19:09:26.121 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo rbd 2023-09-23T19:09:26.121 INFO:tasks.workunit.client.0.smithi098.stderr:+ rados ls -p rbd 2023-09-23T19:09:26.155 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects=1 2023-09-23T19:09:26.156 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo 1 2023-09-23T19:09:26.156 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-09-23T19:09:26.156 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export sparse1 /tmp/rbd_import_export_266403/sparse1.out 2023-09-23T19:09:26.160 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:25 smithi173 bash[59422]: cluster 2023-09-23T19:09:24.683151+0000 mgr.y (mgr.26054) 1555 : cluster 0 pgmap v1734: 73 pgs: 73 active+clean; 4.8 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 76 KiB/s rd, 12 KiB/s wr, 106 op/s 2023-09-23T19:09:26.194 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:26.196 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-09-23T19:09:26.200 INFO:tasks.workunit.client.0.smithi098.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_266403/sparse1 /tmp/rbd_import_export_266403/sparse1.out 2023-09-23T19:09:26.200 INFO:tasks.workunit.client.0.smithi098.stderr:+ cmp -l /tmp/rbd_import_export_266403/sparse1 /tmp/rbd_import_export_266403/sparse1.out 2023-09-23T19:09:26.209 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_266403/sparse1 --format %b 2023-09-23T19:09:26.209 INFO:tasks.workunit.client.0.smithi098.stderr:+ origsize=2048 2023-09-23T19:09:26.210 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_266403/sparse1.out --format %b 2023-09-23T19:09:26.210 INFO:tasks.workunit.client.0.smithi098.stderr:+ exportsize=2048 2023-09-23T19:09:26.211 INFO:tasks.workunit.client.0.smithi098.stderr:+ difference=0 2023-09-23T19:09:26.211 INFO:tasks.workunit.client.0.smithi098.stderr:+ difference=0 2023-09-23T19:09:26.211 INFO:tasks.workunit.client.0.smithi098.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T19:09:26.211 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_266403/sparse1.out 2023-09-23T19:09:26.212 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm sparse1 2023-09-23T19:09:26.213 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:26.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:26.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:26.317 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:26.319 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T19:09:26.323 INFO:tasks.workunit.client.0.smithi098.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_266403/sparse2 oflag=append conv=notrunc 2023-09-23T19:09:26.332 INFO:tasks.workunit.client.0.smithi098.stderr:1+0 records in 2023-09-23T19:09:26.332 INFO:tasks.workunit.client.0.smithi098.stderr:1+0 records out 2023-09-23T19:09:26.332 INFO:tasks.workunit.client.0.smithi098.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.0073764 s, 284 MB/s 2023-09-23T19:09:26.332 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --new-format --order 20 - sparse2 2023-09-23T19:09:26.346 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:26.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:26.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:26.449 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:26.474 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:26.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:26.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:26.483 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:09:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:09:26] "GET /metrics HTTP/1.1" 200 34177 "" "Prometheus/2.33.4" 2023-09-23T19:09:26.530 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 100% complete...done. 2023-09-23T19:09:26.530 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T19:09:26.530 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T19:09:26.534 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd ls -l 2023-09-23T19:09:26.534 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep sparse2 2023-09-23T19:09:26.534 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep -Ei (4 MiB|4096k) 2023-09-23T19:09:26.578 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:26.581 INFO:tasks.workunit.client.0.smithi098.stdout:sparse2 4 MiB 2 2023-09-23T19:09:26.582 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects sparse2 2023-09-23T19:09:26.582 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse2 2023-09-23T19:09:26.582 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse2 2023-09-23T19:09:26.582 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep block_name_prefix 2023-09-23T19:09:26.582 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk {print $NF;} 2023-09-23T19:09:26.601 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:26.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:26.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:26.632 INFO:tasks.workunit.client.0.smithi098.stderr:+ prefix=rbd_data.b5f1f281c4e1 2023-09-23T19:09:26.633 INFO:tasks.workunit.client.0.smithi098.stderr:+ + grep rbd_data.b5f1f281c4e1 2023-09-23T19:09:26.633 INFO:tasks.workunit.client.0.smithi098.stderr:get_image_data_pool sparse2 2023-09-23T19:09:26.633 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse2 2023-09-23T19:09:26.633 INFO:tasks.workunit.client.0.smithi098.stderr:+ sed -e s/rbd_data.b5f1f281c4e1\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-09-23T19:09:26.633 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse2 2023-09-23T19:09:26.634 INFO:tasks.workunit.client.0.smithi098.stderr:+ sort -u 2023-09-23T19:09:26.634 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep data_pool: 2023-09-23T19:09:26.634 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk -F: { print $NF } 2023-09-23T19:09:26.679 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool= 2023-09-23T19:09:26.679 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ -z ] 2023-09-23T19:09:26.679 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool=rbd 2023-09-23T19:09:26.679 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo rbd 2023-09-23T19:09:26.679 INFO:tasks.workunit.client.0.smithi098.stderr:+ rados ls -p rbd 2023-09-23T19:09:26.704 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:26.719 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects=0 2023-09-23T19:09:26.719 INFO:tasks.workunit.client.0.smithi098.stderr:2 2023-09-23T19:09:26.720 INFO:tasks.workunit.client.0.smithi098.stderr:3 2023-09-23T19:09:26.720 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo 0 2 3 2023-09-23T19:09:26.720 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ 0 -eq 1 -o 0 2 3 = 0 2 3 ] 2023-09-23T19:09:26.720 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export sparse2 /tmp/rbd_import_export_266403/sparse2.out 2023-09-23T19:09:26.734 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:26.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:26.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:26.767 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-09-23T19:09:26.771 INFO:tasks.workunit.client.0.smithi098.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_266403/sparse2 /tmp/rbd_import_export_266403/sparse2.out 2023-09-23T19:09:26.772 INFO:tasks.workunit.client.0.smithi098.stderr:+ cmp -l /tmp/rbd_import_export_266403/sparse2 /tmp/rbd_import_export_266403/sparse2.out 2023-09-23T19:09:26.775 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_266403/sparse2 --format %b 2023-09-23T19:09:26.776 INFO:tasks.workunit.client.0.smithi098.stderr:+ origsize=6144 2023-09-23T19:09:26.776 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_266403/sparse2.out --format %b 2023-09-23T19:09:26.778 INFO:tasks.workunit.client.0.smithi098.stderr:+ exportsize=6144 2023-09-23T19:09:26.778 INFO:tasks.workunit.client.0.smithi098.stderr:+ difference=0 2023-09-23T19:09:26.778 INFO:tasks.workunit.client.0.smithi098.stderr:+ difference=0 2023-09-23T19:09:26.778 INFO:tasks.workunit.client.0.smithi098.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T19:09:26.778 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_266403/sparse2.out 2023-09-23T19:09:26.779 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm sparse2 2023-09-23T19:09:26.837 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:26.864 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:26.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:26.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:26.913 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-09-23T19:09:26.917 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo partially-sparse file imports to partially-sparse image 2023-09-23T19:09:26.917 INFO:tasks.workunit.client.0.smithi098.stdout:partially-sparse file imports to partially-sparse image 2023-09-23T19:09:26.917 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_266403/sparse1 sparse 2023-09-23T19:09:26.969 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:26.996 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:26.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:26.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:27.000 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-09-23T19:09:27.000 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T19:09:27.000 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T19:09:27.005 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects sparse 2023-09-23T19:09:27.005 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse 2023-09-23T19:09:27.005 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse 2023-09-23T19:09:27.005 INFO:tasks.workunit.client.0.smithi098.stderr:+ + grep block_name_prefix 2023-09-23T19:09:27.005 INFO:tasks.workunit.client.0.smithi098.stderr:awk {print $NF;} 2023-09-23T19:09:27.047 INFO:tasks.workunit.client.0.smithi098.stderr:+ prefix=rbd_data.b61e1b7138bb 2023-09-23T19:09:27.049 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep rbd_data.b61e1b7138bb 2023-09-23T19:09:27.049 INFO:tasks.workunit.client.0.smithi098.stderr:+ get_image_data_pool sparse 2023-09-23T19:09:27.049 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse 2023-09-23T19:09:27.049 INFO:tasks.workunit.client.0.smithi098.stderr:+ sed -e s/rbd_data.b61e1b7138bb\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-09-23T19:09:27.049 INFO:tasks.workunit.client.0.smithi098.stderr:+ sort -u 2023-09-23T19:09:27.049 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse 2023-09-23T19:09:27.050 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep data_pool: 2023-09-23T19:09:27.050 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk -F: { print $NF } 2023-09-23T19:09:27.091 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool= 2023-09-23T19:09:27.091 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ -z ] 2023-09-23T19:09:27.091 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool=rbd 2023-09-23T19:09:27.091 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo rbd 2023-09-23T19:09:27.092 INFO:tasks.workunit.client.0.smithi098.stderr:+ rados ls -p rbd 2023-09-23T19:09:27.100 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:27.125 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T19:09:27.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:27.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T19:09:27.134 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects=1 2023-09-23T19:09:27.134 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo 1 2023-09-23T19:09:27.134 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-09-23T19:09:27.134 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm sparse 2023-09-23T19:09:27.168 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T19:09:27.229 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:09:27.255 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T19:09:27.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:27.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T19:09:27.270 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T19:09:27.275 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo zeros import through stdin to sparse image 2023-09-23T19:09:27.276 INFO:tasks.workunit.client.0.smithi098.stdout:zeros import through stdin to sparse image 2023-09-23T19:09:27.276 INFO:tasks.workunit.client.0.smithi098.stderr:+ dd if=/dev/zero bs=1M count=4 2023-09-23T19:09:27.276 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd import --new-format - sparse 2023-09-23T19:09:27.339 INFO:tasks.workunit.client.0.smithi098.stderr:4+0 records in 2023-09-23T19:09:27.339 INFO:tasks.workunit.client.0.smithi098.stderr:4+0 records out 2023-09-23T19:09:27.339 INFO:tasks.workunit.client.0.smithi098.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0620104 s, 67.6 MB/s 2023-09-23T19:09:27.354 INFO:tasks.workunit.client.0.smithi098.stderr: Importing image: 100% complete...done. 2023-09-23T19:09:27.355 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T19:09:27.359 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:27.360 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects sparse 2023-09-23T19:09:27.360 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse 2023-09-23T19:09:27.360 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse 2023-09-23T19:09:27.360 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep block_name_prefix 2023-09-23T19:09:27.360 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk {print $NF;} 2023-09-23T19:09:27.381 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:27.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:27.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:27.401 INFO:tasks.workunit.client.0.smithi098.stderr:+ prefix=rbd_data.b63fbb85092d 2023-09-23T19:09:27.402 INFO:tasks.workunit.client.0.smithi098.stderr:+ + get_image_data_pool sparse 2023-09-23T19:09:27.402 INFO:tasks.workunit.client.0.smithi098.stderr:grep+ rbd_data.b63fbb85092d 2023-09-23T19:09:27.402 INFO:tasks.workunit.client.0.smithi098.stderr:image=sparse 2023-09-23T19:09:27.402 INFO:tasks.workunit.client.0.smithi098.stderr:+ sed -e s/rbd_data.b63fbb85092d\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-09-23T19:09:27.402 INFO:tasks.workunit.client.0.smithi098.stderr:+ sort -u 2023-09-23T19:09:27.402 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse 2023-09-23T19:09:27.402 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep data_pool: 2023-09-23T19:09:27.402 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk -F: { print $NF } 2023-09-23T19:09:27.445 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool= 2023-09-23T19:09:27.445 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ -z ] 2023-09-23T19:09:27.445 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool=rbd 2023-09-23T19:09:27.445 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo rbd 2023-09-23T19:09:27.445 INFO:tasks.workunit.client.0.smithi098.stderr:+ rados ls -p rbd 2023-09-23T19:09:27.484 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects= 2023-09-23T19:09:27.484 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo 2023-09-23T19:09:27.484 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ 0 -eq 1 -o = ] 2023-09-23T19:09:27.484 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm sparse 2023-09-23T19:09:27.486 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:27.505 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:27.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:27.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:27.608 INFO:teuthology.orchestra.run.smithi098.stdout:osd.0: {} 2023-09-23T19:09:27.608 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:27.636 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T19:09:27.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:27.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T19:09:27.665 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 100% complete...done. 2023-09-23T19:09:27.669 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo zeros export to sparse file 2023-09-23T19:09:27.669 INFO:tasks.workunit.client.0.smithi098.stdout:zeros export to sparse file 2023-09-23T19:09:27.670 INFO:teuthology.orchestra.run.smithi098.stdout:osd.1: {} 2023-09-23T19:09:27.694 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd create --new-format sparse --size 4 2023-09-23T19:09:27.697 INFO:teuthology.orchestra.run.smithi098.stdout:osd.2: {} 2023-09-23T19:09:27.722 INFO:teuthology.orchestra.run.smithi098.stdout:osd.3: {} 2023-09-23T19:09:27.738 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:27.741 INFO:tasks.workunit.client.0.smithi098.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T19:09:27.744 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse 2023-09-23T19:09:27.744 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep block_name_prefix 2023-09-23T19:09:27.744 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk {print $NF;} 2023-09-23T19:09:27.746 INFO:teuthology.orchestra.run.smithi098.stdout:osd.4: {} 2023-09-23T19:09:27.762 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:27.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:27.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:27.771 INFO:teuthology.orchestra.run.smithi098.stdout:osd.5: {} 2023-09-23T19:09:27.787 INFO:tasks.workunit.client.0.smithi098.stderr:+ prefix=rbd_data.b6661fc69d7d 2023-09-23T19:09:27.787 INFO:tasks.workunit.client.0.smithi098.stderr:+ dd if=/dev/zero bs=4M count=1 2023-09-23T19:09:27.787 INFO:tasks.workunit.client.0.smithi098.stderr:+ get_image_data_pool sparse 2023-09-23T19:09:27.787 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse 2023-09-23T19:09:27.787 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse 2023-09-23T19:09:27.787 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep data_pool: 2023-09-23T19:09:27.788 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk -F: { print $NF } 2023-09-23T19:09:27.798 INFO:teuthology.orchestra.run.smithi098.stdout:osd.6: {} 2023-09-23T19:09:27.824 INFO:teuthology.orchestra.run.smithi098.stdout:osd.7: {} 2023-09-23T19:09:27.826 INFO:teuthology.orchestra.run.smithi098.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T19:09:27.826 INFO:teuthology.orchestra.run.smithi098.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T19:09:27.826 INFO:teuthology.orchestra.run.smithi098.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T19:09:27.826 INFO:teuthology.orchestra.run.smithi098.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T19:09:27.826 INFO:teuthology.orchestra.run.smithi098.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T19:09:27.827 INFO:teuthology.orchestra.run.smithi098.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T19:09:27.827 INFO:teuthology.orchestra.run.smithi098.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T19:09:27.827 INFO:teuthology.orchestra.run.smithi098.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T19:09:27.830 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool= 2023-09-23T19:09:27.830 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ -z ] 2023-09-23T19:09:27.830 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool=rbd 2023-09-23T19:09:27.831 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo rbd 2023-09-23T19:09:27.831 INFO:tasks.workunit.client.0.smithi098.stderr:+ rados -p rbd put rbd_data.b6661fc69d7d.000000000000 - 2023-09-23T19:09:27.852 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:27 smithi098 bash[79516]: cluster 2023-09-23T19:09:26.683643+0000 mgr.y (mgr.26054) 1556 : cluster 0 pgmap v1735: 73 pgs: 73 active+clean; 4.8 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 289 KiB/s rd, 182 KiB/s wr, 155 op/s 2023-09-23T19:09:27.852 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:27 smithi098 bash[79516]: audit 2023-09-23T19:09:27.580185+0000 mon.c (mon.2) 154 : audit 0 from='client.? 172.21.15.98:0/479319723' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:09:27.852 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:27 smithi098 bash[76817]: cluster 2023-09-23T19:09:26.683643+0000 mgr.y (mgr.26054) 1556 : cluster 0 pgmap v1735: 73 pgs: 73 active+clean; 4.8 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 289 KiB/s rd, 182 KiB/s wr, 155 op/s 2023-09-23T19:09:27.852 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:27 smithi098 bash[76817]: audit 2023-09-23T19:09:27.580185+0000 mon.c (mon.2) 154 : audit 0 from='client.? 172.21.15.98:0/479319723' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:09:27.866 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:27.867 INFO:tasks.workunit.client.0.smithi098.stderr:1+0 records in 2023-09-23T19:09:27.867 INFO:tasks.workunit.client.0.smithi098.stderr:1+0 records out 2023-09-23T19:09:27.867 INFO:tasks.workunit.client.0.smithi098.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0800153 s, 52.4 MB/s 2023-09-23T19:09:27.878 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:27 smithi173 bash[59422]: cluster 2023-09-23T19:09:26.683643+0000 mgr.y (mgr.26054) 1556 : cluster 0 pgmap v1735: 73 pgs: 73 active+clean; 4.8 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 289 KiB/s rd, 182 KiB/s wr, 155 op/s 2023-09-23T19:09:27.879 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:27 smithi173 bash[59422]: audit 2023-09-23T19:09:27.580185+0000 mon.c (mon.2) 154 : audit 0 from='client.? 172.21.15.98:0/479319723' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T19:09:27.894 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:27.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:27.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:27.901 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects sparse 2023-09-23T19:09:27.901 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse 2023-09-23T19:09:27.901 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse 2023-09-23T19:09:27.901 INFO:tasks.workunit.client.0.smithi098.stderr:+ awk {print $NF;} 2023-09-23T19:09:27.901 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep block_name_prefix 2023-09-23T19:09:27.944 INFO:tasks.workunit.client.0.smithi098.stderr:+ prefix=rbd_data.b6661fc69d7d 2023-09-23T19:09:27.944 INFO:tasks.workunit.client.0.smithi098.stderr:+ grep rbd_data.b6661fc69d7d 2023-09-23T19:09:27.944 INFO:tasks.workunit.client.0.smithi098.stderr:+ get_image_data_pool sparse 2023-09-23T19:09:27.944 INFO:tasks.workunit.client.0.smithi098.stderr:+ image=sparse 2023-09-23T19:09:27.944 INFO:tasks.workunit.client.0.smithi098.stderr:+ sed -e s/rbd_data.b6661fc69d7d\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-09-23T19:09:27.944 INFO:tasks.workunit.client.0.smithi098.stderr:+ sort -u 2023-09-23T19:09:27.944 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd info sparse 2023-09-23T19:09:27.944 INFO:tasks.workunit.client.0.smithi098.stderr:+ + grep data_pool: 2023-09-23T19:09:27.944 INFO:tasks.workunit.client.0.smithi098.stderr:awk -F: { print $NF } 2023-09-23T19:09:27.983 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool= 2023-09-23T19:09:27.984 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ -z ] 2023-09-23T19:09:27.984 INFO:tasks.workunit.client.0.smithi098.stderr:+ data_pool=rbd 2023-09-23T19:09:27.984 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo rbd 2023-09-23T19:09:27.984 INFO:tasks.workunit.client.0.smithi098.stderr:+ rados ls -p rbd 2023-09-23T19:09:27.996 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:28.024 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:28.026 INFO:tasks.workunit.client.0.smithi098.stderr:+ objects=0 2023-09-23T19:09:28.026 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo 0 2023-09-23T19:09:28.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:28.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:28.028 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2023-09-23T19:09:28.028 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_266403/sparse 2023-09-23T19:09:28.028 INFO:tasks.workunit.client.0.smithi098.stderr:rm: cannot remove '/tmp/rbd_import_export_266403/sparse': No such file or directory 2023-09-23T19:09:28.028 INFO:tasks.workunit.client.0.smithi098.stderr:+ true 2023-09-23T19:09:28.028 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd export sparse /tmp/rbd_import_export_266403/sparse 2023-09-23T19:09:28.067 INFO:tasks.workunit.client.0.smithi098.stderr: Exporting image: 100% complete...done. 2023-09-23T19:09:28.071 INFO:tasks.workunit.client.0.smithi098.stderr:+ stat /tmp/rbd_import_export_266403/sparse --format=%b 2023-09-23T19:09:28.073 INFO:tasks.workunit.client.0.smithi098.stderr:+ [ 0 = 0 ] 2023-09-23T19:09:28.073 INFO:tasks.workunit.client.0.smithi098.stderr:+ rbd rm sparse 2023-09-23T19:09:28.128 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:28.154 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:28.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:28.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:28.199 INFO:tasks.workunit.client.0.smithi098.stderr: Removing image: 100% complete...done. 2023-09-23T19:09:28.203 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm /tmp/rbd_import_export_266403/sparse /tmp/rbd_import_export_266403/sparse1 /tmp/rbd_import_export_266403/sparse2 /tmp/rbd_import_export_266403/sparse3 2023-09-23T19:09:28.209 INFO:tasks.workunit.client.0.smithi098.stderr:rm: cannot remove '/tmp/rbd_import_export_266403/sparse3': No such file or directory 2023-09-23T19:09:28.209 INFO:tasks.workunit.client.0.smithi098.stderr:+ true 2023-09-23T19:09:28.209 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo OK 2023-09-23T19:09:28.210 INFO:tasks.workunit.client.0.smithi098.stdout:OK 2023-09-23T19:09:28.210 INFO:tasks.workunit.client.0.smithi098.stderr:+ rm -rf /tmp/rbd_import_export_266403 2023-09-23T19:09:28.211 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T19:09:28.211 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T19:09:28.220 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-09-23T19:09:28.221 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T19:09:28.256 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:28.291 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T19:09:28.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:28.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T19:09:28.395 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:28.423 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T19:09:28.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:28.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T19:09:28.474 DEBUG:teuthology.parallel:result is None 2023-09-23T19:09:28.474 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T19:09:28.483 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T19:09:28.483 DEBUG:teuthology.orchestra.run.smithi098:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T19:09:28.527 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:28.557 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T19:09:28.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:28.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T19:09:28.583 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T19:09:28.583 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T19:09:28.584 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-09-23T19:09:28.584 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T19:09:28.584 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-09-23T19:09:28.584 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T19:09:28.584 DEBUG:teuthology.orchestra.run.smithi098:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T19:09:28.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:28.589 INFO:teuthology.orchestra.run.smithi098.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T19:09:28.589 DEBUG:teuthology.orchestra.run.smithi098:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T19:09:28.661 DEBUG:teuthology.orchestra.run.smithi098:> sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:28.675 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T19:09:28.675 DEBUG:teuthology.orchestra.run.smithi098:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T19:09:28.725 INFO:tasks.thrashosds:joining thrashosds 2023-09-23T19:09:28.746 INFO:teuthology.orchestra.run.smithi098.stderr:Error response from daemon: Cannot kill container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T19:09:28.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:09:28.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi098 with status 1: 'sudo docker kill -s 1 ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T19:09:30.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:29 smithi098 bash[76817]: cluster 2023-09-23T19:09:28.684516+0000 mgr.y (mgr.26054) 1557 : cluster 0 pgmap v1736: 73 pgs: 73 active+clean; 5.6 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 740 KiB/s rd, 739 KiB/s wr, 222 op/s 2023-09-23T19:09:30.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:29 smithi098 bash[79516]: cluster 2023-09-23T19:09:28.684516+0000 mgr.y (mgr.26054) 1557 : cluster 0 pgmap v1736: 73 pgs: 73 active+clean; 5.6 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 740 KiB/s rd, 739 KiB/s wr, 222 op/s 2023-09-23T19:09:30.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:29 smithi173 bash[59422]: cluster 2023-09-23T19:09:28.684516+0000 mgr.y (mgr.26054) 1557 : cluster 0 pgmap v1736: 73 pgs: 73 active+clean; 5.6 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 740 KiB/s rd, 739 KiB/s wr, 222 op/s 2023-09-23T19:09:31.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:31 smithi098 bash[76817]: cluster 2023-09-23T19:09:30.612641+0000 mon.a (mon.0) 2127 : cluster 0 osdmap e384: 8 total, 8 up, 7 in 2023-09-23T19:09:31.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:31 smithi098 bash[76817]: cluster 2023-09-23T19:09:30.612658+0000 osd.7 (osd.7) 26 : cluster 3 failed to encode map e384 with expected crc 2023-09-23T19:09:31.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:31 smithi098 bash[76817]: cluster 2023-09-23T19:09:30.613632+0000 osd.4 (osd.4) 113 : cluster 3 failed to encode map e384 with expected crc 2023-09-23T19:09:31.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:31 smithi098 bash[76817]: cluster 2023-09-23T19:09:30.614099+0000 osd.6 (osd.6) 372 : cluster 3 failed to encode map e384 with expected crc 2023-09-23T19:09:31.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:31 smithi098 bash[76817]: cluster 2023-09-23T19:09:30.685196+0000 mgr.y (mgr.26054) 1558 : cluster 0 pgmap v1738: 73 pgs: 73 active+clean; 9.6 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 812 KiB/s rd, 1.3 MiB/s wr, 231 op/s 2023-09-23T19:09:31.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:31 smithi098 bash[79516]: cluster 2023-09-23T19:09:30.612641+0000 mon.a (mon.0) 2127 : cluster 0 osdmap e384: 8 total, 8 up, 7 in 2023-09-23T19:09:31.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:31 smithi098 bash[79516]: cluster 2023-09-23T19:09:30.612658+0000 osd.7 (osd.7) 26 : cluster 3 failed to encode map e384 with expected crc 2023-09-23T19:09:31.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:31 smithi098 bash[79516]: cluster 2023-09-23T19:09:30.613632+0000 osd.4 (osd.4) 113 : cluster 3 failed to encode map e384 with expected crc 2023-09-23T19:09:31.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:31 smithi098 bash[79516]: cluster 2023-09-23T19:09:30.614099+0000 osd.6 (osd.6) 372 : cluster 3 failed to encode map e384 with expected crc 2023-09-23T19:09:31.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:31 smithi098 bash[79516]: cluster 2023-09-23T19:09:30.685196+0000 mgr.y (mgr.26054) 1558 : cluster 0 pgmap v1738: 73 pgs: 73 active+clean; 9.6 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 812 KiB/s rd, 1.3 MiB/s wr, 231 op/s 2023-09-23T19:09:31.957 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:09:31.991 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:31 smithi173 bash[59422]: cluster 2023-09-23T19:09:30.612641+0000 mon.a (mon.0) 2127 : cluster 0 osdmap e384: 8 total, 8 up, 7 in 2023-09-23T19:09:31.991 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:31 smithi173 bash[59422]: cluster 2023-09-23T19:09:30.612658+0000 osd.7 (osd.7) 26 : cluster 3 failed to encode map e384 with expected crc 2023-09-23T19:09:31.991 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:31 smithi173 bash[59422]: cluster 2023-09-23T19:09:30.613632+0000 osd.4 (osd.4) 113 : cluster 3 failed to encode map e384 with expected crc 2023-09-23T19:09:31.992 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:31 smithi173 bash[59422]: cluster 2023-09-23T19:09:30.614099+0000 osd.6 (osd.6) 372 : cluster 3 failed to encode map e384 with expected crc 2023-09-23T19:09:31.992 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:31 smithi173 bash[59422]: cluster 2023-09-23T19:09:30.685196+0000 mgr.y (mgr.26054) 1558 : cluster 0 pgmap v1738: 73 pgs: 73 active+clean; 9.6 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 812 KiB/s rd, 1.3 MiB/s wr, 231 op/s 2023-09-23T19:09:32.641 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is set 2023-09-23T19:09:32.642 INFO:teuthology.orchestra.run.smithi098.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T19:09:32.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:32 smithi098 bash[76817]: audit 2023-09-23T19:09:31.951099+0000 mon.c (mon.2) 155 : audit 1 from='client.? 172.21.15.98:0/1868422180' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:09:32.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:32 smithi098 bash[76817]: audit 2023-09-23T19:09:31.951721+0000 mon.a (mon.0) 2128 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:09:32.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:32 smithi098 bash[76817]: audit 2023-09-23T19:09:31.977649+0000 mon.c (mon.2) 156 : audit 1 from='client.? 172.21.15.98:0/188036145' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:09:32.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:32 smithi098 bash[76817]: audit 2023-09-23T19:09:31.978326+0000 mon.a (mon.0) 2129 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:09:32.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:32 smithi098 bash[79516]: audit 2023-09-23T19:09:31.951099+0000 mon.c (mon.2) 155 : audit 1 from='client.? 172.21.15.98:0/1868422180' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:09:32.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:32 smithi098 bash[79516]: audit 2023-09-23T19:09:31.951721+0000 mon.a (mon.0) 2128 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:09:32.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:32 smithi098 bash[79516]: audit 2023-09-23T19:09:31.977649+0000 mon.c (mon.2) 156 : audit 1 from='client.? 172.21.15.98:0/188036145' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:09:32.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:32 smithi098 bash[79516]: audit 2023-09-23T19:09:31.978326+0000 mon.a (mon.0) 2129 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:09:33.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:32 smithi173 bash[59422]: audit 2023-09-23T19:09:31.951099+0000 mon.c (mon.2) 155 : audit 1 from='client.? 172.21.15.98:0/1868422180' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:09:33.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:32 smithi173 bash[59422]: audit 2023-09-23T19:09:31.951721+0000 mon.a (mon.0) 2128 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:09:33.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:32 smithi173 bash[59422]: audit 2023-09-23T19:09:31.977649+0000 mon.c (mon.2) 156 : audit 1 from='client.? 172.21.15.98:0/188036145' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:09:33.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:32 smithi173 bash[59422]: audit 2023-09-23T19:09:31.978326+0000 mon.a (mon.0) 2129 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:09:33.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:33 smithi173 bash[59422]: cluster 2023-09-23T19:09:32.619365+0000 mon.a (mon.0) 2130 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:09:33.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:33 smithi173 bash[59422]: audit 2023-09-23T19:09:32.632049+0000 mon.a (mon.0) 2131 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:09:33.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:33 smithi173 bash[59422]: audit 2023-09-23T19:09:32.632170+0000 mon.a (mon.0) 2132 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:09:33.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:33 smithi173 bash[59422]: cluster 2023-09-23T19:09:32.632261+0000 mon.a (mon.0) 2133 : cluster 0 osdmap e385: 8 total, 8 up, 7 in 2023-09-23T19:09:33.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:33 smithi173 bash[59422]: cluster 2023-09-23T19:09:32.633357+0000 osd.6 (osd.6) 373 : cluster 3 failed to encode map e385 with expected crc 2023-09-23T19:09:33.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:33 smithi173 bash[59422]: cluster 2023-09-23T19:09:32.633538+0000 osd.4 (osd.4) 114 : cluster 3 failed to encode map e385 with expected crc 2023-09-23T19:09:33.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:33 smithi173 bash[59422]: cluster 2023-09-23T19:09:32.634380+0000 osd.7 (osd.7) 27 : cluster 3 failed to encode map e385 with expected crc 2023-09-23T19:09:33.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:33 smithi173 bash[59422]: cluster 2023-09-23T19:09:32.685957+0000 mgr.y (mgr.26054) 1559 : cluster 0 pgmap v1740: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 1019 KiB/s rd, 1.4 MiB/s wr, 336 op/s 2023-09-23T19:09:33.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:33 smithi098 bash[79516]: cluster 2023-09-23T19:09:32.619365+0000 mon.a (mon.0) 2130 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:09:33.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:33 smithi098 bash[79516]: audit 2023-09-23T19:09:32.632049+0000 mon.a (mon.0) 2131 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:09:33.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:33 smithi098 bash[79516]: audit 2023-09-23T19:09:32.632170+0000 mon.a (mon.0) 2132 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:09:33.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:33 smithi098 bash[79516]: cluster 2023-09-23T19:09:32.632261+0000 mon.a (mon.0) 2133 : cluster 0 osdmap e385: 8 total, 8 up, 7 in 2023-09-23T19:09:33.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:33 smithi098 bash[79516]: cluster 2023-09-23T19:09:32.633357+0000 osd.6 (osd.6) 373 : cluster 3 failed to encode map e385 with expected crc 2023-09-23T19:09:33.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:33 smithi098 bash[79516]: cluster 2023-09-23T19:09:32.633538+0000 osd.4 (osd.4) 114 : cluster 3 failed to encode map e385 with expected crc 2023-09-23T19:09:33.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:33 smithi098 bash[79516]: cluster 2023-09-23T19:09:32.634380+0000 osd.7 (osd.7) 27 : cluster 3 failed to encode map e385 with expected crc 2023-09-23T19:09:33.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:33 smithi098 bash[79516]: cluster 2023-09-23T19:09:32.685957+0000 mgr.y (mgr.26054) 1559 : cluster 0 pgmap v1740: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 1019 KiB/s rd, 1.4 MiB/s wr, 336 op/s 2023-09-23T19:09:33.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:33 smithi098 bash[76817]: cluster 2023-09-23T19:09:32.619365+0000 mon.a (mon.0) 2130 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:09:33.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:33 smithi098 bash[76817]: audit 2023-09-23T19:09:32.632049+0000 mon.a (mon.0) 2131 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T19:09:33.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:33 smithi098 bash[76817]: audit 2023-09-23T19:09:32.632170+0000 mon.a (mon.0) 2132 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:09:33.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:33 smithi098 bash[76817]: cluster 2023-09-23T19:09:32.632261+0000 mon.a (mon.0) 2133 : cluster 0 osdmap e385: 8 total, 8 up, 7 in 2023-09-23T19:09:33.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:33 smithi098 bash[76817]: cluster 2023-09-23T19:09:32.633357+0000 osd.6 (osd.6) 373 : cluster 3 failed to encode map e385 with expected crc 2023-09-23T19:09:33.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:33 smithi098 bash[76817]: cluster 2023-09-23T19:09:32.633538+0000 osd.4 (osd.4) 114 : cluster 3 failed to encode map e385 with expected crc 2023-09-23T19:09:33.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:33 smithi098 bash[76817]: cluster 2023-09-23T19:09:32.634380+0000 osd.7 (osd.7) 27 : cluster 3 failed to encode map e385 with expected crc 2023-09-23T19:09:33.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:33 smithi098 bash[76817]: cluster 2023-09-23T19:09:32.685957+0000 mgr.y (mgr.26054) 1559 : cluster 0 pgmap v1740: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 1019 KiB/s rd, 1.4 MiB/s wr, 336 op/s 2023-09-23T19:09:33.988 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T19:09:35.569 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T19:09:35.569 INFO:teuthology.orchestra.run.smithi173.stdout: "size": 20, 2023-09-23T19:09:35.569 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 600, 2023-09-23T19:09:35.569 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-09-23T19:09:35.569 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.569 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45779.0:122373 10.f e328/317 10:f79e126a:::benchmark_data_smithi098_230241_object21911:head v 328'7745, mlcod=328'7743)", 2023-09-23T19:09:35.569 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:03:10.264865+0000", 2023-09-23T19:09:35.569 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 385.29902296699998, 2023-09-23T19:09:35.569 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.017547726, 2023-09-23T19:09:35.570 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:09:35.570 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:09:35.570 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:09:35.570 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.570 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:09:35.570 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:10.264865+0000", 2023-09-23T19:09:35.570 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:09:35.570 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.570 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.570 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:09:35.570 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:10.264865+0000", 2023-09-23T19:09:35.570 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T19:09:35.571 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.571 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.571 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:09:35.571 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:10.264863+0000", 2023-09-23T19:09:35.571 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4534e-05 2023-09-23T19:09:35.571 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.571 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.571 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:09:35.571 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:10.264878+0000", 2023-09-23T19:09:35.571 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.6545000000000001e-05 2023-09-23T19:09:35.571 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.571 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.572 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:09:35.572 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:10.264904+0000", 2023-09-23T19:09:35.572 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2099999999999999e-05 2023-09-23T19:09:35.572 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.572 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.572 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:09:35.572 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:10.264916+0000", 2023-09-23T19:09:35.572 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7005000000000001e-05 2023-09-23T19:09:35.572 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.572 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.572 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:09:35.572 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:10.264973+0000", 2023-09-23T19:09:35.573 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.0826999999999999e-05 2023-09-23T19:09:35.573 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.573 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.573 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:09:35.573 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:10.264994+0000", 2023-09-23T19:09:35.573 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.017373960000000001 2023-09-23T19:09:35.573 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.573 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.573 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:09:35.573 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:10.282368+0000", 2023-09-23T19:09:35.573 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.4691999999999998e-05 2023-09-23T19:09:35.573 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.573 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.574 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:09:35.574 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:10.282413+0000", 2023-09-23T19:09:35.574 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.017547726 2023-09-23T19:09:35.574 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.574 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:09:35.574 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.574 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.574 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.574 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45779.0:129480 10.2 e328/317 10:419b34b9:::benchmark_data_smithi098_230241_object29018:head v 328'8183, mlcod=328'8181)", 2023-09-23T19:09:35.574 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:03:20.570796+0000", 2023-09-23T19:09:35.575 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 374.99309215900001, 2023-09-23T19:09:35.575 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.022283013000000001, 2023-09-23T19:09:35.575 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:09:35.575 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:09:35.575 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:09:35.575 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.575 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:09:35.575 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.570796+0000", 2023-09-23T19:09:35.575 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:09:35.575 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.575 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.575 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:09:35.575 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.570796+0000", 2023-09-23T19:09:35.576 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T19:09:35.576 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.576 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.576 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:09:35.576 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.570795+0000", 2023-09-23T19:09:35.576 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.841e-06 2023-09-23T19:09:35.576 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.576 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.576 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:09:35.576 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.570803+0000", 2023-09-23T19:09:35.576 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0277e-05 2023-09-23T19:09:35.576 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.576 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.577 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:09:35.577 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.570813+0000", 2023-09-23T19:09:35.577 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6570000000000002e-06 2023-09-23T19:09:35.577 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.577 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.577 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:09:35.577 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.570819+0000", 2023-09-23T19:09:35.577 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.8004000000000003e-05 2023-09-23T19:09:35.577 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.577 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.577 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:09:35.577 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.570867+0000", 2023-09-23T19:09:35.577 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2534000000000001e-05 2023-09-23T19:09:35.578 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.578 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.578 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:09:35.578 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.570889+0000", 2023-09-23T19:09:35.578 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.022151438999999998 2023-09-23T19:09:35.578 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.578 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.578 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:09:35.578 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.593041+0000", 2023-09-23T19:09:35.578 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.8108000000000002e-05 2023-09-23T19:09:35.578 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.578 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.579 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:09:35.579 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.593079+0000", 2023-09-23T19:09:35.579 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.022283013000000001 2023-09-23T19:09:35.579 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.579 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:09:35.579 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.579 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.579 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.579 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45779.0:129481 10.2 e328/317 10:4b5c8e9e:::benchmark_data_smithi098_230241_object29019:head v 328'8184, mlcod=328'8181)", 2023-09-23T19:09:35.579 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:03:20.575023+0000", 2023-09-23T19:09:35.579 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 374.98886481900001, 2023-09-23T19:09:35.579 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.018079886999999999, 2023-09-23T19:09:35.579 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:09:35.580 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:09:35.580 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:09:35.580 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.580 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:09:35.580 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.575023+0000", 2023-09-23T19:09:35.580 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:09:35.580 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.580 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.580 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:09:35.580 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.575023+0000", 2023-09-23T19:09:35.580 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T19:09:35.580 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.580 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.581 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:09:35.581 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.575021+0000", 2023-09-23T19:09:35.581 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5763000000000001e-05 2023-09-23T19:09:35.581 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.581 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.581 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:09:35.581 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.575037+0000", 2023-09-23T19:09:35.581 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0089645499999999999 2023-09-23T19:09:35.581 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.581 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.581 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:09:35.581 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.584002+0000", 2023-09-23T19:09:35.581 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5160000000000001e-05 2023-09-23T19:09:35.582 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.582 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.582 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:09:35.582 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.584017+0000", 2023-09-23T19:09:35.582 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0033862509999999998 2023-09-23T19:09:35.582 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.582 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.582 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:09:35.582 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.587403+0000", 2023-09-23T19:09:35.582 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.6061e-05 2023-09-23T19:09:35.582 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.582 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.583 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:09:35.583 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.587429+0000", 2023-09-23T19:09:35.583 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.005652659 2023-09-23T19:09:35.583 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.583 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.583 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:09:35.583 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.593082+0000", 2023-09-23T19:09:35.583 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.1353e-05 2023-09-23T19:09:35.583 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.583 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.583 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:09:35.583 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.593103+0000", 2023-09-23T19:09:35.583 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.018079886999999999 2023-09-23T19:09:35.584 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.584 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:09:35.584 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.584 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.584 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.584 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45779.0:129476 10.e e328/323 10:75175d40:::benchmark_data_smithi098_230241_object29014:head v 328'8014, mlcod=328'8012)", 2023-09-23T19:09:35.584 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:03:20.576597+0000", 2023-09-23T19:09:35.584 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 374.98729097299997, 2023-09-23T19:09:35.584 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.016803358000000001, 2023-09-23T19:09:35.584 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:09:35.584 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:09:35.584 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:09:35.584 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.585 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:09:35.585 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.576597+0000", 2023-09-23T19:09:35.585 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:09:35.585 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.585 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.585 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:09:35.585 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.576597+0000", 2023-09-23T19:09:35.585 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T19:09:35.585 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.585 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.585 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:09:35.585 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.576596+0000", 2023-09-23T19:09:35.585 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4579999999999999e-05 2023-09-23T19:09:35.586 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.586 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.586 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:09:35.586 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.576610+0000", 2023-09-23T19:09:35.586 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0073725250000000004 2023-09-23T19:09:35.586 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.586 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.586 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:09:35.586 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.583983+0000", 2023-09-23T19:09:35.586 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.1106000000000003e-05 2023-09-23T19:09:35.586 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.586 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.586 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:09:35.587 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.584014+0000", 2023-09-23T19:09:35.587 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012727100000000001 2023-09-23T19:09:35.587 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.587 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.587 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:09:35.587 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.584141+0000", 2023-09-23T19:09:35.587 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.7009999999999996e-05 2023-09-23T19:09:35.587 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.587 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.587 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:09:35.587 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.584218+0000", 2023-09-23T19:09:35.587 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0091583710000000002 2023-09-23T19:09:35.587 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.588 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.588 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:09:35.588 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.593376+0000", 2023-09-23T19:09:35.588 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4079000000000001e-05 2023-09-23T19:09:35.588 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.588 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.588 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:09:35.588 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:03:20.593401+0000", 2023-09-23T19:09:35.588 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.016803358000000001 2023-09-23T19:09:35.588 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.588 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:09:35.588 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.589 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.589 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.589 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45779.0:168984 10.a e333/323 10:5ad56134:::benchmark_data_smithi098_230241_object68522:head v 333'10471, mlcod=333'10469)", 2023-09-23T19:09:35.589 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:04:26.571999+0000", 2023-09-23T19:09:35.589 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 308.99188875599998, 2023-09-23T19:09:35.589 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.014849387, 2023-09-23T19:09:35.589 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:09:35.589 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:09:35.589 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:09:35.589 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.589 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:09:35.589 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:04:26.571999+0000", 2023-09-23T19:09:35.589 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:09:35.590 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.590 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.590 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:09:35.590 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:04:26.571999+0000", 2023-09-23T19:09:35.590 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T19:09:35.590 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.590 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.590 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:09:35.590 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:04:26.571998+0000", 2023-09-23T19:09:35.590 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5923e-05 2023-09-23T19:09:35.590 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.590 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.590 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:09:35.591 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:04:26.572014+0000", 2023-09-23T19:09:35.591 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4757e-05 2023-09-23T19:09:35.591 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.591 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.591 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:09:35.591 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:04:26.572038+0000", 2023-09-23T19:09:35.591 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2531000000000001e-05 2023-09-23T19:09:35.591 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.591 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.591 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:09:35.591 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:04:26.572051+0000", 2023-09-23T19:09:35.591 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.0643999999999997e-05 2023-09-23T19:09:35.592 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.592 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.592 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:09:35.592 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:04:26.572122+0000", 2023-09-23T19:09:35.592 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.5401000000000002e-05 2023-09-23T19:09:35.592 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.592 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.592 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:09:35.592 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:04:26.572157+0000", 2023-09-23T19:09:35.592 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.014640594999999999 2023-09-23T19:09:35.592 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.592 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.592 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:09:35.593 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:04:26.586798+0000", 2023-09-23T19:09:35.593 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1226e-05 2023-09-23T19:09:35.593 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.593 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.593 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:09:35.593 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:04:26.586849+0000", 2023-09-23T19:09:35.593 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.014849387 2023-09-23T19:09:35.593 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.593 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:09:35.593 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.593 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.593 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.594 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45779.0:168989 10.2 e333/331 10:4cd7c789:::benchmark_data_smithi098_230241_object68527:head v 333'10733, mlcod=333'10731)", 2023-09-23T19:09:35.594 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:04:26.575557+0000", 2023-09-23T19:09:35.594 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 308.988331152, 2023-09-23T19:09:35.594 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.013142025999999999, 2023-09-23T19:09:35.594 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:09:35.594 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:09:35.594 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:09:35.594 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.594 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:09:35.594 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:04:26.575557+0000", 2023-09-23T19:09:35.594 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:09:35.594 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.595 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.595 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:09:35.595 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:04:26.575557+0000", 2023-09-23T19:09:35.595 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999928 2023-09-23T19:09:35.595 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.595 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.595 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:09:35.595 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:04:26.575550+0000", 2023-09-23T19:09:35.595 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.1894000000000001e-05 2023-09-23T19:09:35.595 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.595 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.595 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:09:35.596 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:04:26.575572+0000", 2023-09-23T19:09:35.596 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3416000000000001e-05 2023-09-23T19:09:35.596 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.596 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.596 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:09:35.596 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:04:26.575595+0000", 2023-09-23T19:09:35.596 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0141e-05 2023-09-23T19:09:35.596 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.596 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.596 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:09:35.596 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:04:26.575605+0000", 2023-09-23T19:09:35.596 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.2557000000000002e-05 2023-09-23T19:09:35.596 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.597 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.597 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:09:35.597 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:04:26.575698+0000", 2023-09-23T19:09:35.597 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.6636e-05 2023-09-23T19:09:35.597 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.597 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.597 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:09:35.597 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:04:26.575735+0000", 2023-09-23T19:09:35.597 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.012923028 2023-09-23T19:09:35.597 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.597 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.597 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:09:35.597 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:04:26.588658+0000", 2023-09-23T19:09:35.598 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.1415999999999998e-05 2023-09-23T19:09:35.598 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.598 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.598 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:09:35.598 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:04:26.588699+0000", 2023-09-23T19:09:35.598 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.013142025999999999 2023-09-23T19:09:35.598 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.598 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:09:35.598 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.598 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.598 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.598 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45779.0:191275 10.e e334/323 10:7c528780:::benchmark_data_smithi098_230241_object90813:head v 334'11836, mlcod=334'11834)", 2023-09-23T19:09:35.599 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:05:04.072212+0000", 2023-09-23T19:09:35.599 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 271.49167598700001, 2023-09-23T19:09:35.599 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.013028421, 2023-09-23T19:09:35.599 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:09:35.599 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:09:35.599 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:09:35.599 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.599 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:09:35.599 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:04.072212+0000", 2023-09-23T19:09:35.599 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:09:35.599 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.599 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.599 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:09:35.600 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:04.072212+0000", 2023-09-23T19:09:35.600 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T19:09:35.600 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.600 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.600 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:09:35.600 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:04.072211+0000", 2023-09-23T19:09:35.600 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9528999999999999e-05 2023-09-23T19:09:35.600 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.600 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.600 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:09:35.600 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:04.072230+0000", 2023-09-23T19:09:35.600 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5707000000000001e-05 2023-09-23T19:09:35.600 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.601 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.601 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:09:35.601 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:04.072246+0000", 2023-09-23T19:09:35.601 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.4110000000000001e-06 2023-09-23T19:09:35.601 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.601 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.601 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:09:35.601 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:04.072252+0000", 2023-09-23T19:09:35.601 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.5425999999999993e-05 2023-09-23T19:09:35.601 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.601 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.601 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:09:35.601 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:04.072318+0000", 2023-09-23T19:09:35.602 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.2236000000000001e-05 2023-09-23T19:09:35.602 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.602 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.602 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:09:35.602 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:04.072350+0000", 2023-09-23T19:09:35.602 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.012845468000000001 2023-09-23T19:09:35.602 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.602 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.602 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:09:35.602 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:04.085196+0000", 2023-09-23T19:09:35.602 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.5018000000000002e-05 2023-09-23T19:09:35.602 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.602 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.603 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:09:35.603 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:04.085241+0000", 2023-09-23T19:09:35.603 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.013028421 2023-09-23T19:09:35.603 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.603 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:09:35.603 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.603 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.603 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.603 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45779.0:192121 10.d e334/317 10:bacf841d:::benchmark_data_smithi098_230241_object91659:head v 334'11837, mlcod=334'11835)", 2023-09-23T19:09:35.603 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:05:05.378530+0000", 2023-09-23T19:09:35.603 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 270.18535779899997, 2023-09-23T19:09:35.603 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.015560684999999999, 2023-09-23T19:09:35.603 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:09:35.604 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:09:35.604 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:09:35.604 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.604 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:09:35.604 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.378530+0000", 2023-09-23T19:09:35.604 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:09:35.604 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.604 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.604 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:09:35.604 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.378530+0000", 2023-09-23T19:09:35.604 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T19:09:35.604 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.604 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.605 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:09:35.605 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.378529+0000", 2023-09-23T19:09:35.605 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2982e-05 2023-09-23T19:09:35.605 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.605 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.605 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:09:35.605 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.378542+0000", 2023-09-23T19:09:35.605 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6390999999999999e-05 2023-09-23T19:09:35.605 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.605 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.605 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:09:35.605 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.378558+0000", 2023-09-23T19:09:35.605 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.5889999999999997e-06 2023-09-23T19:09:35.605 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.606 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.606 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:09:35.606 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.378566+0000", 2023-09-23T19:09:35.606 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00011982200000000001 2023-09-23T19:09:35.606 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.606 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.606 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:09:35.606 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.378686+0000", 2023-09-23T19:09:35.606 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.7499000000000001e-05 2023-09-23T19:09:35.606 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.606 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.606 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:09:35.607 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.378723+0000", 2023-09-23T19:09:35.607 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.015336061 2023-09-23T19:09:35.607 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.607 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.607 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:09:35.607 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.394059+0000", 2023-09-23T19:09:35.607 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.1634999999999998e-05 2023-09-23T19:09:35.607 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.607 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.607 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:09:35.607 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.394091+0000", 2023-09-23T19:09:35.607 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.015560684999999999 2023-09-23T19:09:35.608 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.608 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:09:35.608 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.608 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.608 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.608 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45779.0:192122 10.d e334/317 10:bee6e81e:::benchmark_data_smithi098_230241_object91660:head v 334'11838, mlcod=334'11835)", 2023-09-23T19:09:35.608 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:05:05.378578+0000", 2023-09-23T19:09:35.608 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 270.185310367, 2023-09-23T19:09:35.608 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.015533416, 2023-09-23T19:09:35.608 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:09:35.608 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:09:35.608 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:09:35.608 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.609 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:09:35.609 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.378578+0000", 2023-09-23T19:09:35.609 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:09:35.609 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.609 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.609 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:09:35.609 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.378578+0000", 2023-09-23T19:09:35.609 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-09-23T19:09:35.609 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.609 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.609 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:09:35.609 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.378577+0000", 2023-09-23T19:09:35.610 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.8239999999999999e-06 2023-09-23T19:09:35.610 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.610 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.610 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:09:35.610 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.378582+0000", 2023-09-23T19:09:35.610 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.9790000000000001e-06 2023-09-23T19:09:35.610 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.610 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.610 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:09:35.610 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.378587+0000", 2023-09-23T19:09:35.610 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.1219999999999999e-06 2023-09-23T19:09:35.610 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.610 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.611 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:09:35.611 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.378590+0000", 2023-09-23T19:09:35.611 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.005286458 2023-09-23T19:09:35.611 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.611 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.611 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:09:35.611 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.383877+0000", 2023-09-23T19:09:35.611 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.3939e-05 2023-09-23T19:09:35.611 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.611 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.611 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:09:35.611 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.383931+0000", 2023-09-23T19:09:35.611 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010165742 2023-09-23T19:09:35.612 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.612 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.612 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:09:35.612 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.394096+0000", 2023-09-23T19:09:35.612 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4795e-05 2023-09-23T19:09:35.612 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.612 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.612 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:09:35.612 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.394111+0000", 2023-09-23T19:09:35.612 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.015533416 2023-09-23T19:09:35.612 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.612 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:09:35.613 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.613 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.613 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.613 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45779.0:192126 10.d e334/317 10:b94d289e:::benchmark_data_smithi098_230241_object91664:head v 334'11839, mlcod=334'11835)", 2023-09-23T19:09:35.613 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:05:05.378593+0000", 2023-09-23T19:09:35.613 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 270.185294779, 2023-09-23T19:09:35.613 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.015534249999999999, 2023-09-23T19:09:35.613 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:09:35.613 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:09:35.613 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:09:35.613 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.613 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:09:35.613 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.378593+0000", 2023-09-23T19:09:35.613 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:09:35.614 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.614 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.614 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:09:35.614 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.378593+0000", 2023-09-23T19:09:35.614 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-09-23T19:09:35.614 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.614 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.614 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:09:35.614 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.378593+0000", 2023-09-23T19:09:35.614 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.0879999999999995e-06 2023-09-23T19:09:35.614 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.614 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.615 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:09:35.615 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.378601+0000", 2023-09-23T19:09:35.615 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.3409999999999996e-06 2023-09-23T19:09:35.615 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.615 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.615 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:09:35.615 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.378606+0000", 2023-09-23T19:09:35.615 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.9390000000000002e-06 2023-09-23T19:09:35.615 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.615 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.615 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:09:35.615 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.378609+0000", 2023-09-23T19:09:35.615 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.011630279 2023-09-23T19:09:35.616 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.616 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.616 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:09:35.616 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.390239+0000", 2023-09-23T19:09:35.616 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.6841999999999997e-05 2023-09-23T19:09:35.616 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.616 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.616 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:09:35.616 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.390276+0000", 2023-09-23T19:09:35.616 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.003837975 2023-09-23T19:09:35.616 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.616 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.616 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:09:35.617 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.394114+0000", 2023-09-23T19:09:35.617 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4027e-05 2023-09-23T19:09:35.617 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.617 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.617 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:09:35.617 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.394128+0000", 2023-09-23T19:09:35.617 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.015534249999999999 2023-09-23T19:09:35.617 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.617 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:09:35.617 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.617 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.617 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.617 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45779.0:192136 10.d e334/317 10:bd1dfc24:::benchmark_data_smithi098_230241_object91674:head v 334'11840, mlcod=334'11835)", 2023-09-23T19:09:35.618 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:05:05.381352+0000", 2023-09-23T19:09:35.618 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 270.18253591199999, 2023-09-23T19:09:35.618 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.014270788, 2023-09-23T19:09:35.618 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:09:35.618 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:09:35.618 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:09:35.618 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.618 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:09:35.618 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.381352+0000", 2023-09-23T19:09:35.618 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:09:35.618 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.618 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.618 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:09:35.619 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.381352+0000", 2023-09-23T19:09:35.619 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T19:09:35.619 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.619 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.619 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:09:35.619 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.381351+0000", 2023-09-23T19:09:35.619 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3901e-05 2023-09-23T19:09:35.619 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.619 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.619 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:09:35.619 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.381365+0000", 2023-09-23T19:09:35.619 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7173999999999999e-05 2023-09-23T19:09:35.619 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.620 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.620 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:09:35.620 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.381382+0000", 2023-09-23T19:09:35.620 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.9410000000000007e-06 2023-09-23T19:09:35.620 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.620 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.620 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:09:35.620 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.381390+0000", 2023-09-23T19:09:35.620 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.011233411 2023-09-23T19:09:35.620 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.620 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.620 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:09:35.620 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.392624+0000", 2023-09-23T19:09:35.621 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4166e-05 2023-09-23T19:09:35.621 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.621 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.621 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:09:35.621 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.392638+0000", 2023-09-23T19:09:35.621 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0029611949999999998 2023-09-23T19:09:35.621 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.621 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.621 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:09:35.621 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.395599+0000", 2023-09-23T19:09:35.621 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4065000000000001e-05 2023-09-23T19:09:35.621 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.621 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.622 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:09:35.622 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:05.395623+0000", 2023-09-23T19:09:35.622 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.014270788 2023-09-23T19:09:35.622 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.622 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:09:35.622 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.622 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.622 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.622 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45779.0:193319 10.b e334/317 10:dd8b6f86:::benchmark_data_smithi098_230241_object92857:head v 334'12189, mlcod=334'12187)", 2023-09-23T19:09:35.622 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:05:07.294238+0000", 2023-09-23T19:09:35.622 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 268.26964968999999, 2023-09-23T19:09:35.622 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.014364684000000001, 2023-09-23T19:09:35.622 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:09:35.623 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:09:35.623 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:09:35.623 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.623 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:09:35.623 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:07.294238+0000", 2023-09-23T19:09:35.623 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:09:35.623 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.623 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.623 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:09:35.623 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:07.294238+0000", 2023-09-23T19:09:35.623 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T19:09:35.623 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.623 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.624 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:09:35.624 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:07.294236+0000", 2023-09-23T19:09:35.624 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4888e-05 2023-09-23T19:09:35.624 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.624 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.624 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:09:35.624 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:07.294251+0000", 2023-09-23T19:09:35.624 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.62e-05 2023-09-23T19:09:35.624 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.624 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.624 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:09:35.624 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:07.294278+0000", 2023-09-23T19:09:35.624 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2091e-05 2023-09-23T19:09:35.624 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.625 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.625 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:09:35.625 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:07.294290+0000", 2023-09-23T19:09:35.625 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00011066200000000001 2023-09-23T19:09:35.625 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.625 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.625 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:09:35.625 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:07.294400+0000", 2023-09-23T19:09:35.625 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.0321999999999997e-05 2023-09-23T19:09:35.625 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.625 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.625 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:09:35.625 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:07.294451+0000", 2023-09-23T19:09:35.626 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.014100998 2023-09-23T19:09:35.626 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.626 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.626 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:09:35.626 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:07.308552+0000", 2023-09-23T19:09:35.626 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1548999999999997e-05 2023-09-23T19:09:35.626 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.626 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.626 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:09:35.626 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:07.308603+0000", 2023-09-23T19:09:35.626 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.014364684000000001 2023-09-23T19:09:35.626 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.626 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:09:35.627 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.627 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.627 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.627 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45779.0:193320 10.b e334/317 10:dc2cf1c2:::benchmark_data_smithi098_230241_object92858:head v 334'12190, mlcod=334'12187)", 2023-09-23T19:09:35.627 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:05:07.294306+0000", 2023-09-23T19:09:35.627 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 268.26958190900001, 2023-09-23T19:09:35.627 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.014328728000000001, 2023-09-23T19:09:35.627 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:09:35.627 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:09:35.627 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:09:35.627 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.627 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:09:35.627 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:07.294306+0000", 2023-09-23T19:09:35.628 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:09:35.628 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.628 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.628 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:09:35.628 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:07.294306+0000", 2023-09-23T19:09:35.628 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-09-23T19:09:35.628 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.628 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.628 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:09:35.628 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:07.294306+0000", 2023-09-23T19:09:35.628 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.0110000000000003e-06 2023-09-23T19:09:35.628 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.628 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.629 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:09:35.629 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:07.294313+0000", 2023-09-23T19:09:35.629 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.8200000000000003e-06 2023-09-23T19:09:35.629 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.629 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.629 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:09:35.629 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:07.294321+0000", 2023-09-23T19:09:35.629 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1739999999999997e-06 2023-09-23T19:09:35.629 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.629 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.629 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:09:35.629 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:07.294328+0000", 2023-09-23T19:09:35.629 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0064448980000000001 2023-09-23T19:09:35.630 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.630 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.630 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:09:35.630 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:07.300772+0000", 2023-09-23T19:09:35.630 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.9490000000000002e-05 2023-09-23T19:09:35.630 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.630 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.630 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:09:35.630 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:07.300822+0000", 2023-09-23T19:09:35.630 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0077849579999999998 2023-09-23T19:09:35.630 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.630 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.630 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:09:35.631 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:07.308607+0000", 2023-09-23T19:09:35.631 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.8065e-05 2023-09-23T19:09:35.631 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.631 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.631 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:09:35.631 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:07.308635+0000", 2023-09-23T19:09:35.631 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.014328728000000001 2023-09-23T19:09:35.631 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.631 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:09:35.631 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.631 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.631 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.631 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45779.0:194209 10.c e335/327 10:325ef377:::benchmark_data_smithi098_230241_object93747:head v 335'12401, mlcod=335'12396)", 2023-09-23T19:09:35.632 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:05:09.160661+0000", 2023-09-23T19:09:35.632 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 266.40322730399998, 2023-09-23T19:09:35.632 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.014243749999999999, 2023-09-23T19:09:35.632 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:09:35.632 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:09:35.632 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:09:35.632 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.632 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:09:35.632 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:09.160661+0000", 2023-09-23T19:09:35.632 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:09:35.632 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.632 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.632 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:09:35.633 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:09.160661+0000", 2023-09-23T19:09:35.633 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-09-23T19:09:35.633 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.633 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.633 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:09:35.633 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:09.160660+0000", 2023-09-23T19:09:35.633 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.144e-06 2023-09-23T19:09:35.633 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.633 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.633 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:09:35.633 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:09.160666+0000", 2023-09-23T19:09:35.633 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.2660000000000001e-06 2023-09-23T19:09:35.633 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.634 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.634 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:09:35.634 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:09.160675+0000", 2023-09-23T19:09:35.634 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5670000000000002e-06 2023-09-23T19:09:35.634 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.634 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.634 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:09:35.634 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:09.160680+0000", 2023-09-23T19:09:35.634 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0079511570000000004 2023-09-23T19:09:35.634 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.634 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.634 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:09:35.635 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:09.168631+0000", 2023-09-23T19:09:35.635 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.6701000000000001e-05 2023-09-23T19:09:35.635 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.635 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.635 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:09:35.635 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:09.168658+0000", 2023-09-23T19:09:35.635 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0061944110000000004 2023-09-23T19:09:35.635 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.635 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.635 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:09:35.635 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:09.174853+0000", 2023-09-23T19:09:35.635 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.2027999999999997e-05 2023-09-23T19:09:35.636 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.636 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.636 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:09:35.636 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:09.174905+0000", 2023-09-23T19:09:35.636 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.014243749999999999 2023-09-23T19:09:35.636 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.636 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:09:35.636 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.636 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.636 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.636 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45779.0:200369 10.f e335/317 10:f62d77fa:::benchmark_data_smithi098_230241_object99907:head v 335'12668, mlcod=335'12665)", 2023-09-23T19:09:35.636 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:05:18.887987+0000", 2023-09-23T19:09:35.636 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 256.67590069900001, 2023-09-23T19:09:35.637 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.013615471000000001, 2023-09-23T19:09:35.637 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:09:35.637 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:09:35.637 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:09:35.637 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.637 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:09:35.637 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:18.887987+0000", 2023-09-23T19:09:35.637 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:09:35.637 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.637 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.637 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:09:35.637 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:18.887987+0000", 2023-09-23T19:09:35.637 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T19:09:35.638 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.638 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.638 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:09:35.638 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:18.887987+0000", 2023-09-23T19:09:35.638 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.9420000000000007e-06 2023-09-23T19:09:35.638 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.638 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.638 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:09:35.638 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:18.887997+0000", 2023-09-23T19:09:35.638 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5204e-05 2023-09-23T19:09:35.638 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.638 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.638 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:09:35.639 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:18.888012+0000", 2023-09-23T19:09:35.639 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5470000000000004e-06 2023-09-23T19:09:35.639 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.639 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.639 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:09:35.639 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:18.888017+0000", 2023-09-23T19:09:35.639 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.0574000000000002e-05 2023-09-23T19:09:35.639 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.639 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.639 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:09:35.639 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:18.888058+0000", 2023-09-23T19:09:35.639 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9165999999999999e-05 2023-09-23T19:09:35.639 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.640 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.640 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:09:35.640 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:18.888077+0000", 2023-09-23T19:09:35.640 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.013444149000000001 2023-09-23T19:09:35.640 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.640 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.640 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:09:35.640 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:18.901521+0000", 2023-09-23T19:09:35.640 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.1735000000000003e-05 2023-09-23T19:09:35.640 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.640 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.640 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:09:35.640 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:05:18.901603+0000", 2023-09-23T19:09:35.641 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.013615471000000001 2023-09-23T19:09:35.641 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.641 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:09:35.641 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.641 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.641 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.641 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45944.0:162 11.a e361/360 11:57642022:::image19:head v 361'5, mlcod=361'3)", 2023-09-23T19:09:35.641 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:08:37.786256+0000", 2023-09-23T19:09:35.641 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 57.777632566999998, 2023-09-23T19:09:35.641 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0079360090000000008, 2023-09-23T19:09:35.641 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:09:35.641 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:09:35.641 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:09:35.641 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.642 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:09:35.642 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.786256+0000", 2023-09-23T19:09:35.642 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:09:35.642 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.642 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.642 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:09:35.642 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.786256+0000", 2023-09-23T19:09:35.642 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T19:09:35.642 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.642 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.642 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:09:35.642 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.786255+0000", 2023-09-23T19:09:35.643 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.2280000000000003e-06 2023-09-23T19:09:35.643 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.643 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.643 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:09:35.643 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.786261+0000", 2023-09-23T19:09:35.643 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0977e-05 2023-09-23T19:09:35.643 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.643 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.643 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:09:35.643 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.786272+0000", 2023-09-23T19:09:35.643 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.618e-06 2023-09-23T19:09:35.643 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.643 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.643 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:09:35.644 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.786279+0000", 2023-09-23T19:09:35.644 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.0238999999999999e-05 2023-09-23T19:09:35.644 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.644 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.644 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:09:35.644 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.786319+0000", 2023-09-23T19:09:35.644 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2194000000000001e-05 2023-09-23T19:09:35.644 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.644 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.644 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:09:35.644 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.786341+0000", 2023-09-23T19:09:35.644 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0077581730000000002 2023-09-23T19:09:35.644 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.645 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.645 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:09:35.645 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.794099+0000", 2023-09-23T19:09:35.645 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.2298000000000003e-05 2023-09-23T19:09:35.645 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.645 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.645 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:09:35.645 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.794192+0000", 2023-09-23T19:09:35.645 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0079360090000000008 2023-09-23T19:09:35.645 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.645 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:09:35.645 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.645 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.646 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.646 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45944.0:375 11.d e361/346 11:b7102ad3:::image35:head v 361'29, mlcod=361'27)", 2023-09-23T19:09:35.646 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:08:38.060790+0000", 2023-09-23T19:09:35.646 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 57.503098125000001, 2023-09-23T19:09:35.646 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0039316739999999996, 2023-09-23T19:09:35.646 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:09:35.646 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:09:35.646 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:09:35.646 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.646 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:09:35.646 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:38.060790+0000", 2023-09-23T19:09:35.646 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:09:35.646 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.647 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.647 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:09:35.647 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:38.060790+0000", 2023-09-23T19:09:35.647 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T19:09:35.647 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.647 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.647 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:09:35.647 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:38.060788+0000", 2023-09-23T19:09:35.647 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6351999999999999e-05 2023-09-23T19:09:35.647 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.647 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.647 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:09:35.647 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:38.060804+0000", 2023-09-23T19:09:35.648 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.8357e-05 2023-09-23T19:09:35.648 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.648 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.648 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:09:35.648 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:38.060833+0000", 2023-09-23T19:09:35.648 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.261e-05 2023-09-23T19:09:35.648 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.648 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.648 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:09:35.648 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:38.060845+0000", 2023-09-23T19:09:35.648 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000144036 2023-09-23T19:09:35.648 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.648 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.649 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:09:35.649 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:38.060989+0000", 2023-09-23T19:09:35.649 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.6538999999999997e-05 2023-09-23T19:09:35.649 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.649 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.649 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:09:35.649 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:38.061026+0000", 2023-09-23T19:09:35.649 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0036318000000000001 2023-09-23T19:09:35.649 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.649 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.649 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:09:35.649 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:38.064658+0000", 2023-09-23T19:09:35.649 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.3953000000000002e-05 2023-09-23T19:09:35.650 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.650 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.650 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:09:35.650 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:38.064722+0000", 2023-09-23T19:09:35.650 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0039316739999999996 2023-09-23T19:09:35.650 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.650 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:09:35.650 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.650 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.650 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.650 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45944.0:2794 11.9 e362/346 11:94ddb4a8:::group_id_snap_list_max_read:head v 362'11, mlcod=362'9)", 2023-09-23T19:09:35.650 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:08:42.182573+0000", 2023-09-23T19:09:35.650 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 53.3813149, 2023-09-23T19:09:35.651 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0021085650000000002, 2023-09-23T19:09:35.651 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:09:35.651 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:09:35.651 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:09:35.651 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.651 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:09:35.651 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:42.182573+0000", 2023-09-23T19:09:35.651 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:09:35.651 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.651 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.651 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:09:35.651 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:42.182573+0000", 2023-09-23T19:09:35.651 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T19:09:35.652 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.652 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.652 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:09:35.652 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:42.182572+0000", 2023-09-23T19:09:35.652 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0851e-05 2023-09-23T19:09:35.652 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.652 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.652 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:09:35.652 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:42.182583+0000", 2023-09-23T19:09:35.652 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8374000000000001e-05 2023-09-23T19:09:35.652 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.652 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.652 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:09:35.653 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:42.182601+0000", 2023-09-23T19:09:35.653 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.9740000000000008e-06 2023-09-23T19:09:35.653 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.653 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.653 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:09:35.653 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:42.182610+0000", 2023-09-23T19:09:35.653 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00098152799999999996 2023-09-23T19:09:35.653 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.653 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.653 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:09:35.653 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:42.183592+0000", 2023-09-23T19:09:35.653 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.0143999999999996e-05 2023-09-23T19:09:35.654 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.654 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.654 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:09:35.654 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:42.183682+0000", 2023-09-23T19:09:35.654 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00094625600000000003 2023-09-23T19:09:35.654 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.654 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.654 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:09:35.654 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:42.184628+0000", 2023-09-23T19:09:35.654 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.3674000000000001e-05 2023-09-23T19:09:35.654 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.654 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.654 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:09:35.655 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:42.184682+0000", 2023-09-23T19:09:35.655 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0021085650000000002 2023-09-23T19:09:35.655 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.655 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:09:35.655 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.655 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.655 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.655 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46061.0:27 2.6 e365/307 2:6c9cdb26:::rbd_object_map.b3ed9942bcee:head v 365'185, mlcod=365'183)", 2023-09-23T19:09:35.655 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:08:53.646530+0000", 2023-09-23T19:09:35.655 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 41.91735843, 2023-09-23T19:09:35.655 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0041273680000000002, 2023-09-23T19:09:35.655 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:09:35.655 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:09:35.656 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:09:35.656 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.656 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:09:35.656 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.646530+0000", 2023-09-23T19:09:35.656 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:09:35.656 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.656 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.656 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:09:35.656 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.646530+0000", 2023-09-23T19:09:35.656 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T19:09:35.656 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.656 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.656 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:09:35.657 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.646528+0000", 2023-09-23T19:09:35.657 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6370000000000001e-05 2023-09-23T19:09:35.657 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.657 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.657 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:09:35.657 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.646544+0000", 2023-09-23T19:09:35.657 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3906000000000001e-05 2023-09-23T19:09:35.657 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.657 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.657 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:09:35.657 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.646568+0000", 2023-09-23T19:09:35.657 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.9399999999999997e-06 2023-09-23T19:09:35.657 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.658 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.658 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:09:35.658 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.646578+0000", 2023-09-23T19:09:35.658 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5535000000000003e-05 2023-09-23T19:09:35.658 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.658 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.658 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:09:35.658 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.646634+0000", 2023-09-23T19:09:35.658 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.6228e-05 2023-09-23T19:09:35.658 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.658 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.658 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:09:35.658 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.646660+0000", 2023-09-23T19:09:35.659 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0039254720000000002 2023-09-23T19:09:35.659 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.659 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.659 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:09:35.659 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.650585+0000", 2023-09-23T19:09:35.659 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.1709000000000001e-05 2023-09-23T19:09:35.659 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.659 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.659 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:09:35.659 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.650657+0000", 2023-09-23T19:09:35.659 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0041273680000000002 2023-09-23T19:09:35.659 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.659 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:09:35.660 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.660 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.660 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.660 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46088.0:80 2.7 e369/368 2:e1b06f48:::rbd_data.b408e23a9739.0000000000000018:head v 369'113, mlcod=0'0)", 2023-09-23T19:09:35.660 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:03.591399+0000", 2023-09-23T19:09:35.660 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 31.972488865999999, 2023-09-23T19:09:35.660 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001502857, 2023-09-23T19:09:35.660 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:09:35.660 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:09:35.660 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:09:35.660 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.660 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:09:35.660 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.591399+0000", 2023-09-23T19:09:35.661 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:09:35.661 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.661 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.661 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:09:35.661 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.591399+0000", 2023-09-23T19:09:35.661 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-09-23T19:09:35.661 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.661 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.661 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:09:35.661 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.591397+0000", 2023-09-23T19:09:35.661 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000270692 2023-09-23T19:09:35.661 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.661 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.662 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:09:35.662 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.591667+0000", 2023-09-23T19:09:35.662 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.8396999999999999e-05 2023-09-23T19:09:35.662 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.662 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.662 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:09:35.662 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.591696+0000", 2023-09-23T19:09:35.662 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5756999999999998e-05 2023-09-23T19:09:35.662 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.662 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.662 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:09:35.662 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.591712+0000", 2023-09-23T19:09:35.662 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016961 2023-09-23T19:09:35.663 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.663 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.663 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:09:35.663 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.591881+0000", 2023-09-23T19:09:35.663 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.0998999999999998e-05 2023-09-23T19:09:35.663 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.663 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.663 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:09:35.663 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.591922+0000", 2023-09-23T19:09:35.663 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000882284 2023-09-23T19:09:35.663 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.663 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.663 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:09:35.663 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.592805+0000", 2023-09-23T19:09:35.664 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.7644000000000002e-05 2023-09-23T19:09:35.664 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:09:35.664 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:09:35.664 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:09:35.664 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.592902+0000", 2023-09-23T19:09:35.664 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001502857 2023-09-23T19:09:35.664 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.664 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:09:35.664 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.664 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:09:35.664 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:09:35.664 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T19:09:35.972 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T19:09:36.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:35 smithi173 bash[59422]: cluster 2023-09-23T19:09:34.686735+0000 mgr.y (mgr.26054) 1560 : cluster 0 pgmap v1741: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 842 KiB/s rd, 1.3 MiB/s wr, 274 op/s 2023-09-23T19:09:36.087 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:35 smithi098 bash[79516]: cluster 2023-09-23T19:09:34.686735+0000 mgr.y (mgr.26054) 1560 : cluster 0 pgmap v1741: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 842 KiB/s rd, 1.3 MiB/s wr, 274 op/s 2023-09-23T19:09:36.089 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:35 smithi098 bash[76817]: cluster 2023-09-23T19:09:34.686735+0000 mgr.y (mgr.26054) 1560 : cluster 0 pgmap v1741: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 842 KiB/s rd, 1.3 MiB/s wr, 274 op/s 2023-09-23T19:09:36.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:09:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:09:36] "GET /metrics HTTP/1.1" 200 34177 "" "Prometheus/2.33.4" 2023-09-23T19:09:36.498 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T19:09:38.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:37 smithi173 bash[59422]: cluster 2023-09-23T19:09:36.687594+0000 mgr.y (mgr.26054) 1561 : cluster 0 pgmap v1742: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 396 KiB/s rd, 775 KiB/s wr, 182 op/s 2023-09-23T19:09:38.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:37 smithi098 bash[76817]: cluster 2023-09-23T19:09:36.687594+0000 mgr.y (mgr.26054) 1561 : cluster 0 pgmap v1742: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 396 KiB/s rd, 775 KiB/s wr, 182 op/s 2023-09-23T19:09:38.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:37 smithi098 bash[79516]: cluster 2023-09-23T19:09:36.687594+0000 mgr.y (mgr.26054) 1561 : cluster 0 pgmap v1742: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 396 KiB/s rd, 775 KiB/s wr, 182 op/s 2023-09-23T19:09:38.810 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:09:39.827 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:09:40.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:39 smithi173 bash[59422]: cluster 2023-09-23T19:09:38.688489+0000 mgr.y (mgr.26054) 1562 : cluster 0 pgmap v1743: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 206 KiB/s rd, 131 KiB/s wr, 103 op/s 2023-09-23T19:09:40.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:39 smithi173 bash[59422]: audit 2023-09-23T19:09:39.651640+0000 mon.a (mon.0) 2134 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:09:40.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:39 smithi098 bash[79516]: cluster 2023-09-23T19:09:38.688489+0000 mgr.y (mgr.26054) 1562 : cluster 0 pgmap v1743: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 206 KiB/s rd, 131 KiB/s wr, 103 op/s 2023-09-23T19:09:40.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:39 smithi098 bash[79516]: audit 2023-09-23T19:09:39.651640+0000 mon.a (mon.0) 2134 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:09:40.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:39 smithi098 bash[76817]: cluster 2023-09-23T19:09:38.688489+0000 mgr.y (mgr.26054) 1562 : cluster 0 pgmap v1743: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 206 KiB/s rd, 131 KiB/s wr, 103 op/s 2023-09-23T19:09:40.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:39 smithi098 bash[76817]: audit 2023-09-23T19:09:39.651640+0000 mon.a (mon.0) 2134 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:09:40.840 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:09:41.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:41 smithi098 bash[76817]: cluster 2023-09-23T19:09:40.689090+0000 mgr.y (mgr.26054) 1563 : cluster 0 pgmap v1744: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 166 KiB/s rd, 105 KiB/s wr, 83 op/s 2023-09-23T19:09:41.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:41 smithi098 bash[76817]: audit 2023-09-23T19:09:40.755426+0000 mon.a (mon.0) 2135 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.4", "id": [1, 6]}]: dispatch 2023-09-23T19:09:41.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:41 smithi098 bash[76817]: audit 2023-09-23T19:09:40.755810+0000 mon.a (mon.0) 2136 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.16", "id": [1, 2]}]: dispatch 2023-09-23T19:09:41.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:41 smithi098 bash[76817]: audit 2023-09-23T19:09:40.756102+0000 mon.a (mon.0) 2137 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2, 3, 7]}]: dispatch 2023-09-23T19:09:41.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:41 smithi098 bash[76817]: audit 2023-09-23T19:09:40.756418+0000 mon.a (mon.0) 2138 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 0]}]: dispatch 2023-09-23T19:09:41.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:41 smithi098 bash[79516]: cluster 2023-09-23T19:09:40.689090+0000 mgr.y (mgr.26054) 1563 : cluster 0 pgmap v1744: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 166 KiB/s rd, 105 KiB/s wr, 83 op/s 2023-09-23T19:09:41.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:41 smithi098 bash[79516]: audit 2023-09-23T19:09:40.755426+0000 mon.a (mon.0) 2135 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.4", "id": [1, 6]}]: dispatch 2023-09-23T19:09:41.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:41 smithi098 bash[79516]: audit 2023-09-23T19:09:40.755810+0000 mon.a (mon.0) 2136 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.16", "id": [1, 2]}]: dispatch 2023-09-23T19:09:41.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:41 smithi098 bash[79516]: audit 2023-09-23T19:09:40.756102+0000 mon.a (mon.0) 2137 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2, 3, 7]}]: dispatch 2023-09-23T19:09:41.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:41 smithi098 bash[79516]: audit 2023-09-23T19:09:40.756418+0000 mon.a (mon.0) 2138 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 0]}]: dispatch 2023-09-23T19:09:42.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:41 smithi173 bash[59422]: cluster 2023-09-23T19:09:40.689090+0000 mgr.y (mgr.26054) 1563 : cluster 0 pgmap v1744: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 166 KiB/s rd, 105 KiB/s wr, 83 op/s 2023-09-23T19:09:42.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:41 smithi173 bash[59422]: audit 2023-09-23T19:09:40.755426+0000 mon.a (mon.0) 2135 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.4", "id": [1, 6]}]: dispatch 2023-09-23T19:09:42.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:41 smithi173 bash[59422]: audit 2023-09-23T19:09:40.755810+0000 mon.a (mon.0) 2136 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.16", "id": [1, 2]}]: dispatch 2023-09-23T19:09:42.030 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:41 smithi173 bash[59422]: audit 2023-09-23T19:09:40.756102+0000 mon.a (mon.0) 2137 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2, 3, 7]}]: dispatch 2023-09-23T19:09:42.030 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:41 smithi173 bash[59422]: audit 2023-09-23T19:09:40.756418+0000 mon.a (mon.0) 2138 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 0]}]: dispatch 2023-09-23T19:09:42.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:42 smithi098 bash[76817]: audit 2023-09-23T19:09:41.623252+0000 mon.a (mon.0) 2139 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.4", "id": [1, 6]}]': finished 2023-09-23T19:09:42.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:42 smithi098 bash[76817]: audit 2023-09-23T19:09:41.623302+0000 mon.a (mon.0) 2140 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.16", "id": [1, 2]}]': finished 2023-09-23T19:09:42.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:42 smithi098 bash[76817]: audit 2023-09-23T19:09:41.623344+0000 mon.a (mon.0) 2141 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2, 3, 7]}]': finished 2023-09-23T19:09:42.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:42 smithi098 bash[76817]: audit 2023-09-23T19:09:41.623379+0000 mon.a (mon.0) 2142 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 0]}]': finished 2023-09-23T19:09:42.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:42 smithi098 bash[76817]: cluster 2023-09-23T19:09:41.623416+0000 mon.a (mon.0) 2143 : cluster 0 osdmap e386: 8 total, 8 up, 7 in 2023-09-23T19:09:42.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:42 smithi098 bash[76817]: cluster 2023-09-23T19:09:41.624808+0000 osd.7 (osd.7) 28 : cluster 3 failed to encode map e386 with expected crc 2023-09-23T19:09:42.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:42 smithi098 bash[76817]: cluster 2023-09-23T19:09:41.625941+0000 osd.6 (osd.6) 374 : cluster 3 failed to encode map e386 with expected crc 2023-09-23T19:09:42.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:42 smithi098 bash[79516]: audit 2023-09-23T19:09:41.623252+0000 mon.a (mon.0) 2139 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.4", "id": [1, 6]}]': finished 2023-09-23T19:09:42.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:42 smithi098 bash[79516]: audit 2023-09-23T19:09:41.623302+0000 mon.a (mon.0) 2140 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.16", "id": [1, 2]}]': finished 2023-09-23T19:09:42.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:42 smithi098 bash[79516]: audit 2023-09-23T19:09:41.623344+0000 mon.a (mon.0) 2141 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2, 3, 7]}]': finished 2023-09-23T19:09:42.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:42 smithi098 bash[79516]: audit 2023-09-23T19:09:41.623379+0000 mon.a (mon.0) 2142 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 0]}]': finished 2023-09-23T19:09:42.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:42 smithi098 bash[79516]: cluster 2023-09-23T19:09:41.623416+0000 mon.a (mon.0) 2143 : cluster 0 osdmap e386: 8 total, 8 up, 7 in 2023-09-23T19:09:42.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:42 smithi098 bash[79516]: cluster 2023-09-23T19:09:41.624808+0000 osd.7 (osd.7) 28 : cluster 3 failed to encode map e386 with expected crc 2023-09-23T19:09:42.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:42 smithi098 bash[79516]: cluster 2023-09-23T19:09:41.625941+0000 osd.6 (osd.6) 374 : cluster 3 failed to encode map e386 with expected crc 2023-09-23T19:09:43.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:42 smithi173 bash[59422]: audit 2023-09-23T19:09:41.623252+0000 mon.a (mon.0) 2139 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.4", "id": [1, 6]}]': finished 2023-09-23T19:09:43.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:42 smithi173 bash[59422]: audit 2023-09-23T19:09:41.623302+0000 mon.a (mon.0) 2140 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.16", "id": [1, 2]}]': finished 2023-09-23T19:09:43.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:42 smithi173 bash[59422]: audit 2023-09-23T19:09:41.623344+0000 mon.a (mon.0) 2141 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2, 3, 7]}]': finished 2023-09-23T19:09:43.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:42 smithi173 bash[59422]: audit 2023-09-23T19:09:41.623379+0000 mon.a (mon.0) 2142 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 0]}]': finished 2023-09-23T19:09:43.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:42 smithi173 bash[59422]: cluster 2023-09-23T19:09:41.623416+0000 mon.a (mon.0) 2143 : cluster 0 osdmap e386: 8 total, 8 up, 7 in 2023-09-23T19:09:43.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:42 smithi173 bash[59422]: cluster 2023-09-23T19:09:41.624808+0000 osd.7 (osd.7) 28 : cluster 3 failed to encode map e386 with expected crc 2023-09-23T19:09:43.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:42 smithi173 bash[59422]: cluster 2023-09-23T19:09:41.625941+0000 osd.6 (osd.6) 374 : cluster 3 failed to encode map e386 with expected crc 2023-09-23T19:09:43.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:43 smithi098 bash[79516]: cluster 2023-09-23T19:09:42.625695+0000 mon.a (mon.0) 2144 : cluster 0 osdmap e387: 8 total, 8 up, 7 in 2023-09-23T19:09:43.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:43 smithi098 bash[79516]: cluster 2023-09-23T19:09:42.627499+0000 osd.6 (osd.6) 375 : cluster 3 failed to encode map e387 with expected crc 2023-09-23T19:09:43.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:43 smithi098 bash[79516]: cluster 2023-09-23T19:09:42.627781+0000 osd.4 (osd.4) 115 : cluster 3 failed to encode map e387 with expected crc 2023-09-23T19:09:43.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:43 smithi098 bash[79516]: cluster 2023-09-23T19:09:42.629165+0000 osd.7 (osd.7) 29 : cluster 3 failed to encode map e387 with expected crc 2023-09-23T19:09:43.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:43 smithi098 bash[79516]: cluster 2023-09-23T19:09:42.631936+0000 osd.7 (osd.7) 30 : cluster 3 failed to encode map e387 with expected crc 2023-09-23T19:09:43.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:43 smithi098 bash[79516]: cluster 2023-09-23T19:09:42.689698+0000 mgr.y (mgr.26054) 1564 : cluster 0 pgmap v1747: 73 pgs: 1 peering, 72 active+clean; 8.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:09:43.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:43 smithi098 bash[76817]: cluster 2023-09-23T19:09:42.625695+0000 mon.a (mon.0) 2144 : cluster 0 osdmap e387: 8 total, 8 up, 7 in 2023-09-23T19:09:43.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:43 smithi098 bash[76817]: cluster 2023-09-23T19:09:42.627499+0000 osd.6 (osd.6) 375 : cluster 3 failed to encode map e387 with expected crc 2023-09-23T19:09:43.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:43 smithi098 bash[76817]: cluster 2023-09-23T19:09:42.627781+0000 osd.4 (osd.4) 115 : cluster 3 failed to encode map e387 with expected crc 2023-09-23T19:09:43.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:43 smithi098 bash[76817]: cluster 2023-09-23T19:09:42.629165+0000 osd.7 (osd.7) 29 : cluster 3 failed to encode map e387 with expected crc 2023-09-23T19:09:43.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:43 smithi098 bash[76817]: cluster 2023-09-23T19:09:42.631936+0000 osd.7 (osd.7) 30 : cluster 3 failed to encode map e387 with expected crc 2023-09-23T19:09:43.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:43 smithi098 bash[76817]: cluster 2023-09-23T19:09:42.689698+0000 mgr.y (mgr.26054) 1564 : cluster 0 pgmap v1747: 73 pgs: 1 peering, 72 active+clean; 8.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:09:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:43 smithi173 bash[59422]: cluster 2023-09-23T19:09:42.625695+0000 mon.a (mon.0) 2144 : cluster 0 osdmap e387: 8 total, 8 up, 7 in 2023-09-23T19:09:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:43 smithi173 bash[59422]: cluster 2023-09-23T19:09:42.627499+0000 osd.6 (osd.6) 375 : cluster 3 failed to encode map e387 with expected crc 2023-09-23T19:09:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:43 smithi173 bash[59422]: cluster 2023-09-23T19:09:42.627781+0000 osd.4 (osd.4) 115 : cluster 3 failed to encode map e387 with expected crc 2023-09-23T19:09:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:43 smithi173 bash[59422]: cluster 2023-09-23T19:09:42.629165+0000 osd.7 (osd.7) 29 : cluster 3 failed to encode map e387 with expected crc 2023-09-23T19:09:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:43 smithi173 bash[59422]: cluster 2023-09-23T19:09:42.631936+0000 osd.7 (osd.7) 30 : cluster 3 failed to encode map e387 with expected crc 2023-09-23T19:09:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:43 smithi173 bash[59422]: cluster 2023-09-23T19:09:42.689698+0000 mgr.y (mgr.26054) 1564 : cluster 0 pgmap v1747: 73 pgs: 1 peering, 72 active+clean; 8.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:09:44.912 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:44 smithi098 bash[79516]: cluster 2023-09-23T19:09:43.628557+0000 mon.a (mon.0) 2145 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T19:09:44.912 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:44 smithi098 bash[76817]: cluster 2023-09-23T19:09:43.628557+0000 mon.a (mon.0) 2145 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T19:09:45.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:44 smithi173 bash[59422]: cluster 2023-09-23T19:09:43.628557+0000 mon.a (mon.0) 2145 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T19:09:45.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:45 smithi098 bash[76817]: cluster 2023-09-23T19:09:44.690419+0000 mgr.y (mgr.26054) 1565 : cluster 0 pgmap v1748: 73 pgs: 1 peering, 72 active+clean; 8.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:09:45.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:45 smithi098 bash[79516]: cluster 2023-09-23T19:09:44.690419+0000 mgr.y (mgr.26054) 1565 : cluster 0 pgmap v1748: 73 pgs: 1 peering, 72 active+clean; 8.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:09:46.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:45 smithi173 bash[59422]: cluster 2023-09-23T19:09:44.690419+0000 mgr.y (mgr.26054) 1565 : cluster 0 pgmap v1748: 73 pgs: 1 peering, 72 active+clean; 8.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:09:46.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:09:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:09:46] "GET /metrics HTTP/1.1" 200 34177 "" "Prometheus/2.33.4" 2023-09-23T19:09:47.759 INFO:teuthology.orchestra.run.smithi098.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T19:09:48.011 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:47 smithi098 bash[76817]: cluster 2023-09-23T19:09:46.690950+0000 mgr.y (mgr.26054) 1566 : cluster 0 pgmap v1749: 73 pgs: 1 peering, 72 active+clean; 8.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:09:48.011 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:47 smithi098 bash[76817]: audit 2023-09-23T19:09:47.513248+0000 mon.c (mon.2) 157 : audit 1 from='client.? 172.21.15.98:0/4271472141' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:09:48.011 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:47 smithi098 bash[76817]: audit 2023-09-23T19:09:47.513791+0000 mon.a (mon.0) 2146 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:09:48.011 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:47 smithi098 bash[79516]: cluster 2023-09-23T19:09:46.690950+0000 mgr.y (mgr.26054) 1566 : cluster 0 pgmap v1749: 73 pgs: 1 peering, 72 active+clean; 8.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:09:48.011 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:47 smithi098 bash[79516]: audit 2023-09-23T19:09:47.513248+0000 mon.c (mon.2) 157 : audit 1 from='client.? 172.21.15.98:0/4271472141' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:09:48.011 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:47 smithi098 bash[79516]: audit 2023-09-23T19:09:47.513791+0000 mon.a (mon.0) 2146 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:09:48.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:47 smithi173 bash[59422]: cluster 2023-09-23T19:09:46.690950+0000 mgr.y (mgr.26054) 1566 : cluster 0 pgmap v1749: 73 pgs: 1 peering, 72 active+clean; 8.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:09:48.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:47 smithi173 bash[59422]: audit 2023-09-23T19:09:47.513248+0000 mon.c (mon.2) 157 : audit 1 from='client.? 172.21.15.98:0/4271472141' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:09:48.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:47 smithi173 bash[59422]: audit 2023-09-23T19:09:47.513791+0000 mon.a (mon.0) 2146 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:09:48.322 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T19:09:48.322 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [], 2023-09-23T19:09:48.322 INFO:teuthology.orchestra.run.smithi098.stdout: "num_ops": 0 2023-09-23T19:09:48.323 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T19:09:48.777 INFO:teuthology.orchestra.run.smithi098.stderr:noscrub is unset 2023-09-23T19:09:49.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:48 smithi173 bash[59422]: audit 2023-09-23T19:09:47.750207+0000 mon.a (mon.0) 2147 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:09:49.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:48 smithi173 bash[59422]: cluster 2023-09-23T19:09:47.750449+0000 osd.7 (osd.7) 31 : cluster 3 failed to encode map e388 with expected crc 2023-09-23T19:09:49.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:48 smithi173 bash[59422]: cluster 2023-09-23T19:09:47.750475+0000 mon.a (mon.0) 2148 : cluster 0 osdmap e388: 8 total, 8 up, 7 in 2023-09-23T19:09:49.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:48 smithi173 bash[59422]: cluster 2023-09-23T19:09:47.753231+0000 osd.4 (osd.4) 116 : cluster 3 failed to encode map e388 with expected crc 2023-09-23T19:09:49.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:48 smithi173 bash[59422]: audit 2023-09-23T19:09:48.178260+0000 mon.a (mon.0) 2149 : audit 1 from='client.? 172.21.15.98:0/4103889305' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:09:49.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:48 smithi173 bash[59422]: cluster 2023-09-23T19:09:48.281505+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e388 with expected crc 2023-09-23T19:09:49.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:48 smithi098 bash[76817]: audit 2023-09-23T19:09:47.750207+0000 mon.a (mon.0) 2147 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:09:49.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:48 smithi098 bash[76817]: cluster 2023-09-23T19:09:47.750449+0000 osd.7 (osd.7) 31 : cluster 3 failed to encode map e388 with expected crc 2023-09-23T19:09:49.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:48 smithi098 bash[76817]: cluster 2023-09-23T19:09:47.750475+0000 mon.a (mon.0) 2148 : cluster 0 osdmap e388: 8 total, 8 up, 7 in 2023-09-23T19:09:49.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:48 smithi098 bash[76817]: cluster 2023-09-23T19:09:47.753231+0000 osd.4 (osd.4) 116 : cluster 3 failed to encode map e388 with expected crc 2023-09-23T19:09:49.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:48 smithi098 bash[76817]: audit 2023-09-23T19:09:48.178260+0000 mon.a (mon.0) 2149 : audit 1 from='client.? 172.21.15.98:0/4103889305' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:09:49.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:48 smithi098 bash[76817]: cluster 2023-09-23T19:09:48.281505+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e388 with expected crc 2023-09-23T19:09:49.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:48 smithi098 bash[79516]: audit 2023-09-23T19:09:47.750207+0000 mon.a (mon.0) 2147 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:09:49.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:48 smithi098 bash[79516]: cluster 2023-09-23T19:09:47.750449+0000 osd.7 (osd.7) 31 : cluster 3 failed to encode map e388 with expected crc 2023-09-23T19:09:49.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:48 smithi098 bash[79516]: cluster 2023-09-23T19:09:47.750475+0000 mon.a (mon.0) 2148 : cluster 0 osdmap e388: 8 total, 8 up, 7 in 2023-09-23T19:09:49.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:48 smithi098 bash[79516]: cluster 2023-09-23T19:09:47.753231+0000 osd.4 (osd.4) 116 : cluster 3 failed to encode map e388 with expected crc 2023-09-23T19:09:49.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:48 smithi098 bash[79516]: audit 2023-09-23T19:09:48.178260+0000 mon.a (mon.0) 2149 : audit 1 from='client.? 172.21.15.98:0/4103889305' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T19:09:49.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:48 smithi098 bash[79516]: cluster 2023-09-23T19:09:48.281505+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e388 with expected crc 2023-09-23T19:09:49.620 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T19:09:49.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:49 smithi098 bash[79516]: cluster 2023-09-23T19:09:48.281508+0000 osd.6 (osd.6) 376 : cluster 3 failed to encode map e388 with expected crc 2023-09-23T19:09:49.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:49 smithi098 bash[79516]: cluster 2023-09-23T19:09:48.691273+0000 mgr.y (mgr.26054) 1567 : cluster 0 pgmap v1751: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:09:49.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:49 smithi098 bash[79516]: cluster 2023-09-23T19:09:48.755276+0000 mon.a (mon.0) 2150 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T19:09:49.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:49 smithi098 bash[79516]: cluster 2023-09-23T19:09:48.755998+0000 mon.a (mon.0) 2151 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:09:49.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:49 smithi098 bash[79516]: cluster 2023-09-23T19:09:48.768175+0000 osd.6 (osd.6) 377 : cluster 3 failed to encode map e389 with expected crc 2023-09-23T19:09:49.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:49 smithi098 bash[79516]: cluster 2023-09-23T19:09:48.768278+0000 osd.4 (osd.4) 117 : cluster 3 failed to encode map e389 with expected crc 2023-09-23T19:09:49.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:49 smithi098 bash[79516]: audit 2023-09-23T19:09:48.768482+0000 mon.a (mon.0) 2152 : audit 1 from='client.? 172.21.15.98:0/4103889305' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:09:49.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:49 smithi098 bash[79516]: cluster 2023-09-23T19:09:48.768543+0000 mon.a (mon.0) 2153 : cluster 0 osdmap e389: 8 total, 8 up, 7 in 2023-09-23T19:09:49.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:49 smithi098 bash[79516]: cluster 2023-09-23T19:09:48.838106+0000 osd.7 (osd.7) 32 : cluster 3 failed to encode map e389 with expected crc 2023-09-23T19:09:49.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:49 smithi098 bash[79516]: cluster 2023-09-23T19:09:49.384682+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e389 with expected crc 2023-09-23T19:09:49.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:49 smithi098 bash[76817]: cluster 2023-09-23T19:09:48.281508+0000 osd.6 (osd.6) 376 : cluster 3 failed to encode map e388 with expected crc 2023-09-23T19:09:49.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:49 smithi098 bash[76817]: cluster 2023-09-23T19:09:48.691273+0000 mgr.y (mgr.26054) 1567 : cluster 0 pgmap v1751: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:09:49.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:49 smithi098 bash[76817]: cluster 2023-09-23T19:09:48.755276+0000 mon.a (mon.0) 2150 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T19:09:49.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:49 smithi098 bash[76817]: cluster 2023-09-23T19:09:48.755998+0000 mon.a (mon.0) 2151 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:09:49.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:49 smithi098 bash[76817]: cluster 2023-09-23T19:09:48.768175+0000 osd.6 (osd.6) 377 : cluster 3 failed to encode map e389 with expected crc 2023-09-23T19:09:49.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:49 smithi098 bash[76817]: cluster 2023-09-23T19:09:48.768278+0000 osd.4 (osd.4) 117 : cluster 3 failed to encode map e389 with expected crc 2023-09-23T19:09:49.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:49 smithi098 bash[76817]: audit 2023-09-23T19:09:48.768482+0000 mon.a (mon.0) 2152 : audit 1 from='client.? 172.21.15.98:0/4103889305' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:09:49.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:49 smithi098 bash[76817]: cluster 2023-09-23T19:09:48.768543+0000 mon.a (mon.0) 2153 : cluster 0 osdmap e389: 8 total, 8 up, 7 in 2023-09-23T19:09:49.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:49 smithi098 bash[76817]: cluster 2023-09-23T19:09:48.838106+0000 osd.7 (osd.7) 32 : cluster 3 failed to encode map e389 with expected crc 2023-09-23T19:09:49.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:49 smithi098 bash[76817]: cluster 2023-09-23T19:09:49.384682+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e389 with expected crc 2023-09-23T19:09:50.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:49 smithi173 bash[59422]: cluster 2023-09-23T19:09:48.281508+0000 osd.6 (osd.6) 376 : cluster 3 failed to encode map e388 with expected crc 2023-09-23T19:09:50.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:49 smithi173 bash[59422]: cluster 2023-09-23T19:09:48.691273+0000 mgr.y (mgr.26054) 1567 : cluster 0 pgmap v1751: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:09:50.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:49 smithi173 bash[59422]: cluster 2023-09-23T19:09:48.755276+0000 mon.a (mon.0) 2150 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T19:09:50.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:49 smithi173 bash[59422]: cluster 2023-09-23T19:09:48.755998+0000 mon.a (mon.0) 2151 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T19:09:50.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:49 smithi173 bash[59422]: cluster 2023-09-23T19:09:48.768175+0000 osd.6 (osd.6) 377 : cluster 3 failed to encode map e389 with expected crc 2023-09-23T19:09:50.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:49 smithi173 bash[59422]: cluster 2023-09-23T19:09:48.768278+0000 osd.4 (osd.4) 117 : cluster 3 failed to encode map e389 with expected crc 2023-09-23T19:09:50.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:49 smithi173 bash[59422]: audit 2023-09-23T19:09:48.768482+0000 mon.a (mon.0) 2152 : audit 1 from='client.? 172.21.15.98:0/4103889305' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T19:09:50.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:49 smithi173 bash[59422]: cluster 2023-09-23T19:09:48.768543+0000 mon.a (mon.0) 2153 : cluster 0 osdmap e389: 8 total, 8 up, 7 in 2023-09-23T19:09:50.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:49 smithi173 bash[59422]: cluster 2023-09-23T19:09:48.838106+0000 osd.7 (osd.7) 32 : cluster 3 failed to encode map e389 with expected crc 2023-09-23T19:09:50.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:49 smithi173 bash[59422]: cluster 2023-09-23T19:09:49.384682+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e389 with expected crc 2023-09-23T19:09:51.048 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T19:09:51.140 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T19:09:51.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:51 smithi098 bash[76817]: cluster 2023-09-23T19:09:50.692042+0000 mgr.y (mgr.26054) 1568 : cluster 0 pgmap v1753: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:09:51.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:51 smithi098 bash[79516]: cluster 2023-09-23T19:09:50.692042+0000 mgr.y (mgr.26054) 1568 : cluster 0 pgmap v1753: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:09:52.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:51 smithi173 bash[59422]: cluster 2023-09-23T19:09:50.692042+0000 mgr.y (mgr.26054) 1568 : cluster 0 pgmap v1753: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:09:54.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:53 smithi173 bash[59422]: cluster 2023-09-23T19:09:52.692957+0000 mgr.y (mgr.26054) 1569 : cluster 0 pgmap v1754: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:09:54.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:53 smithi098 bash[76817]: cluster 2023-09-23T19:09:52.692957+0000 mgr.y (mgr.26054) 1569 : cluster 0 pgmap v1754: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:09:54.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:53 smithi098 bash[79516]: cluster 2023-09-23T19:09:52.692957+0000 mgr.y (mgr.26054) 1569 : cluster 0 pgmap v1754: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:09:54.387 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:09:55.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:54 smithi173 bash[59422]: audit 2023-09-23T19:09:54.652388+0000 mon.a (mon.0) 2154 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:09:55.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:54 smithi098 bash[76817]: audit 2023-09-23T19:09:54.652388+0000 mon.a (mon.0) 2154 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:09:55.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:54 smithi098 bash[79516]: audit 2023-09-23T19:09:54.652388+0000 mon.a (mon.0) 2154 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:09:55.406 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:09:55.406 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:09:56.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:55 smithi173 bash[59422]: cluster 2023-09-23T19:09:54.693561+0000 mgr.y (mgr.26054) 1570 : cluster 0 pgmap v1755: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:09:56.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:55 smithi173 bash[59422]: audit 2023-09-23T19:09:55.584899+0000 mon.a (mon.0) 2155 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:09:56.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:55 smithi098 bash[76817]: cluster 2023-09-23T19:09:54.693561+0000 mgr.y (mgr.26054) 1570 : cluster 0 pgmap v1755: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:09:56.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:55 smithi098 bash[76817]: audit 2023-09-23T19:09:55.584899+0000 mon.a (mon.0) 2155 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:09:56.087 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:55 smithi098 bash[79516]: cluster 2023-09-23T19:09:54.693561+0000 mgr.y (mgr.26054) 1570 : cluster 0 pgmap v1755: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:09:56.087 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:55 smithi098 bash[79516]: audit 2023-09-23T19:09:55.584899+0000 mon.a (mon.0) 2155 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:09:56.443 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:09:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:09:56] "GET /metrics HTTP/1.1" 200 34181 "" "Prometheus/2.33.4" 2023-09-23T19:09:57.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:57 smithi098 bash[76817]: cluster 2023-09-23T19:09:56.694533+0000 mgr.y (mgr.26054) 1571 : cluster 0 pgmap v1756: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:09:57.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:57 smithi098 bash[79516]: cluster 2023-09-23T19:09:56.694533+0000 mgr.y (mgr.26054) 1571 : cluster 0 pgmap v1756: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:09:58.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:57 smithi173 bash[59422]: cluster 2023-09-23T19:09:56.694533+0000 mgr.y (mgr.26054) 1571 : cluster 0 pgmap v1756: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:00.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:09:59 smithi173 bash[59422]: cluster 2023-09-23T19:09:58.695096+0000 mgr.y (mgr.26054) 1572 : cluster 0 pgmap v1757: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:00.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:09:59 smithi098 bash[79516]: cluster 2023-09-23T19:09:58.695096+0000 mgr.y (mgr.26054) 1572 : cluster 0 pgmap v1757: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:00.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:09:59 smithi098 bash[76817]: cluster 2023-09-23T19:09:58.695096+0000 mgr.y (mgr.26054) 1572 : cluster 0 pgmap v1757: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:00.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:00 smithi098 bash[79516]: cluster 2023-09-23T19:10:00.000133+0000 mon.a (mon.0) 2156 : cluster 3 Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-09-23T19:10:00.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:00 smithi098 bash[79516]: cluster 2023-09-23T19:10:00.000179+0000 mon.a (mon.0) 2157 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T19:10:00.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:00 smithi098 bash[76817]: cluster 2023-09-23T19:10:00.000133+0000 mon.a (mon.0) 2156 : cluster 3 Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-09-23T19:10:00.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:00 smithi098 bash[76817]: cluster 2023-09-23T19:10:00.000179+0000 mon.a (mon.0) 2157 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T19:10:01.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:00 smithi173 bash[59422]: cluster 2023-09-23T19:10:00.000133+0000 mon.a (mon.0) 2156 : cluster 3 Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-09-23T19:10:01.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:00 smithi173 bash[59422]: cluster 2023-09-23T19:10:00.000179+0000 mon.a (mon.0) 2157 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T19:10:02.094 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:01 smithi098 bash[76817]: cluster 2023-09-23T19:10:00.695843+0000 mgr.y (mgr.26054) 1573 : cluster 0 pgmap v1758: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:02.094 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:01 smithi098 bash[79516]: cluster 2023-09-23T19:10:00.695843+0000 mgr.y (mgr.26054) 1573 : cluster 0 pgmap v1758: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:02.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:01 smithi173 bash[59422]: cluster 2023-09-23T19:10:00.695843+0000 mgr.y (mgr.26054) 1573 : cluster 0 pgmap v1758: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:02.792 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:02 smithi098 bash[76817]: audit 2023-09-23T19:10:02.519792+0000 mon.c (mon.2) 158 : audit 1 from='client.? 172.21.15.98:0/3288118880' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:10:02.801 INFO:teuthology.orchestra.run.smithi098.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T19:10:02.879 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T19:10:02.879 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [], 2023-09-23T19:10:02.879 INFO:teuthology.orchestra.run.smithi098.stdout: "complaint_time": 30, 2023-09-23T19:10:02.879 INFO:teuthology.orchestra.run.smithi098.stdout: "num_blocked_ops": 0 2023-09-23T19:10:02.879 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T19:10:03.098 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:02 smithi098 bash[76817]: audit 2023-09-23T19:10:02.520381+0000 mon.a (mon.0) 2158 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:10:03.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:02 smithi098 bash[79516]: audit 2023-09-23T19:10:02.519792+0000 mon.c (mon.2) 158 : audit 1 from='client.? 172.21.15.98:0/3288118880' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:10:03.098 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:02 smithi098 bash[79516]: audit 2023-09-23T19:10:02.520381+0000 mon.a (mon.0) 2158 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:10:03.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:02 smithi173 bash[59422]: audit 2023-09-23T19:10:02.519792+0000 mon.c (mon.2) 158 : audit 1 from='client.? 172.21.15.98:0/3288118880' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:10:03.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:02 smithi173 bash[59422]: audit 2023-09-23T19:10:02.520381+0000 mon.a (mon.0) 2158 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:10:03.813 INFO:teuthology.orchestra.run.smithi098.stderr:nodeep-scrub is unset 2023-09-23T19:10:04.133 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:03 smithi098 bash[79516]: cluster 2023-09-23T19:10:02.696489+0000 mgr.y (mgr.26054) 1574 : cluster 0 pgmap v1759: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:04.133 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:03 smithi098 bash[79516]: audit 2023-09-23T19:10:02.792260+0000 mon.a (mon.0) 2159 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:10:04.133 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:03 smithi098 bash[79516]: cluster 2023-09-23T19:10:02.792378+0000 mon.a (mon.0) 2160 : cluster 0 osdmap e390: 8 total, 8 up, 7 in 2023-09-23T19:10:04.133 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:03 smithi098 bash[79516]: cluster 2023-09-23T19:10:02.792695+0000 osd.6 (osd.6) 378 : cluster 3 failed to encode map e390 with expected crc 2023-09-23T19:10:04.133 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:03 smithi098 bash[79516]: audit 2023-09-23T19:10:02.996837+0000 mon.c (mon.2) 159 : audit 1 from='client.? 172.21.15.98:0/3010389425' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:10:04.133 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:03 smithi098 bash[79516]: audit 2023-09-23T19:10:02.997691+0000 mon.a (mon.0) 2161 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:10:04.133 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:03 smithi098 bash[76817]: cluster 2023-09-23T19:10:02.696489+0000 mgr.y (mgr.26054) 1574 : cluster 0 pgmap v1759: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:04.134 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:03 smithi098 bash[76817]: audit 2023-09-23T19:10:02.792260+0000 mon.a (mon.0) 2159 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:10:04.134 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:03 smithi098 bash[76817]: cluster 2023-09-23T19:10:02.792378+0000 mon.a (mon.0) 2160 : cluster 0 osdmap e390: 8 total, 8 up, 7 in 2023-09-23T19:10:04.134 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:03 smithi098 bash[76817]: cluster 2023-09-23T19:10:02.792695+0000 osd.6 (osd.6) 378 : cluster 3 failed to encode map e390 with expected crc 2023-09-23T19:10:04.134 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:03 smithi098 bash[76817]: audit 2023-09-23T19:10:02.996837+0000 mon.c (mon.2) 159 : audit 1 from='client.? 172.21.15.98:0/3010389425' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:10:04.134 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:03 smithi098 bash[76817]: audit 2023-09-23T19:10:02.997691+0000 mon.a (mon.0) 2161 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:10:04.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:03 smithi173 bash[59422]: cluster 2023-09-23T19:10:02.696489+0000 mgr.y (mgr.26054) 1574 : cluster 0 pgmap v1759: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:04.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:03 smithi173 bash[59422]: audit 2023-09-23T19:10:02.792260+0000 mon.a (mon.0) 2159 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:10:04.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:03 smithi173 bash[59422]: cluster 2023-09-23T19:10:02.792378+0000 mon.a (mon.0) 2160 : cluster 0 osdmap e390: 8 total, 8 up, 7 in 2023-09-23T19:10:04.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:03 smithi173 bash[59422]: cluster 2023-09-23T19:10:02.792695+0000 osd.6 (osd.6) 378 : cluster 3 failed to encode map e390 with expected crc 2023-09-23T19:10:04.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:03 smithi173 bash[59422]: audit 2023-09-23T19:10:02.996837+0000 mon.c (mon.2) 159 : audit 1 from='client.? 172.21.15.98:0/3010389425' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:10:04.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:03 smithi173 bash[59422]: audit 2023-09-23T19:10:02.997691+0000 mon.a (mon.0) 2161 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T19:10:04.742 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T19:10:04.943 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T19:10:05.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:04 smithi098 bash[76817]: cluster 2023-09-23T19:10:03.788024+0000 mon.a (mon.0) 2162 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:10:05.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:04 smithi098 bash[76817]: cluster 2023-09-23T19:10:03.788074+0000 mon.a (mon.0) 2163 : cluster 1 Cluster is now healthy 2023-09-23T19:10:05.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:04 smithi098 bash[76817]: audit 2023-09-23T19:10:03.804624+0000 mon.a (mon.0) 2164 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:10:05.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:04 smithi098 bash[76817]: cluster 2023-09-23T19:10:03.804753+0000 mon.a (mon.0) 2165 : cluster 0 osdmap e391: 8 total, 8 up, 7 in 2023-09-23T19:10:05.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:04 smithi098 bash[76817]: cluster 2023-09-23T19:10:03.805223+0000 osd.4 (osd.4) 118 : cluster 3 failed to encode map e391 with expected crc 2023-09-23T19:10:05.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:04 smithi098 bash[76817]: cluster 2023-09-23T19:10:03.806132+0000 osd.7 (osd.7) 33 : cluster 3 failed to encode map e391 with expected crc 2023-09-23T19:10:05.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:04 smithi098 bash[76817]: cluster 2023-09-23T19:10:03.806404+0000 osd.6 (osd.6) 379 : cluster 3 failed to encode map e391 with expected crc 2023-09-23T19:10:05.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:04 smithi098 bash[79516]: cluster 2023-09-23T19:10:03.788024+0000 mon.a (mon.0) 2162 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:10:05.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:04 smithi098 bash[79516]: cluster 2023-09-23T19:10:03.788074+0000 mon.a (mon.0) 2163 : cluster 1 Cluster is now healthy 2023-09-23T19:10:05.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:04 smithi098 bash[79516]: audit 2023-09-23T19:10:03.804624+0000 mon.a (mon.0) 2164 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:10:05.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:04 smithi098 bash[79516]: cluster 2023-09-23T19:10:03.804753+0000 mon.a (mon.0) 2165 : cluster 0 osdmap e391: 8 total, 8 up, 7 in 2023-09-23T19:10:05.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:04 smithi098 bash[79516]: cluster 2023-09-23T19:10:03.805223+0000 osd.4 (osd.4) 118 : cluster 3 failed to encode map e391 with expected crc 2023-09-23T19:10:05.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:04 smithi098 bash[79516]: cluster 2023-09-23T19:10:03.806132+0000 osd.7 (osd.7) 33 : cluster 3 failed to encode map e391 with expected crc 2023-09-23T19:10:05.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:04 smithi098 bash[79516]: cluster 2023-09-23T19:10:03.806404+0000 osd.6 (osd.6) 379 : cluster 3 failed to encode map e391 with expected crc 2023-09-23T19:10:05.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:04 smithi173 bash[59422]: cluster 2023-09-23T19:10:03.788024+0000 mon.a (mon.0) 2162 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T19:10:05.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:04 smithi173 bash[59422]: cluster 2023-09-23T19:10:03.788074+0000 mon.a (mon.0) 2163 : cluster 1 Cluster is now healthy 2023-09-23T19:10:05.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:04 smithi173 bash[59422]: audit 2023-09-23T19:10:03.804624+0000 mon.a (mon.0) 2164 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T19:10:05.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:04 smithi173 bash[59422]: cluster 2023-09-23T19:10:03.804753+0000 mon.a (mon.0) 2165 : cluster 0 osdmap e391: 8 total, 8 up, 7 in 2023-09-23T19:10:05.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:04 smithi173 bash[59422]: cluster 2023-09-23T19:10:03.805223+0000 osd.4 (osd.4) 118 : cluster 3 failed to encode map e391 with expected crc 2023-09-23T19:10:05.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:04 smithi173 bash[59422]: cluster 2023-09-23T19:10:03.806132+0000 osd.7 (osd.7) 33 : cluster 3 failed to encode map e391 with expected crc 2023-09-23T19:10:05.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:04 smithi173 bash[59422]: cluster 2023-09-23T19:10:03.806404+0000 osd.6 (osd.6) 379 : cluster 3 failed to encode map e391 with expected crc 2023-09-23T19:10:05.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:05 smithi098 bash[76817]: cluster 2023-09-23T19:10:04.697038+0000 mgr.y (mgr.26054) 1575 : cluster 0 pgmap v1762: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:05.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:05 smithi098 bash[79516]: cluster 2023-09-23T19:10:04.697038+0000 mgr.y (mgr.26054) 1575 : cluster 0 pgmap v1762: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:06.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:05 smithi173 bash[59422]: cluster 2023-09-23T19:10:04.697038+0000 mgr.y (mgr.26054) 1575 : cluster 0 pgmap v1762: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:06.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:10:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:10:06] "GET /metrics HTTP/1.1" 200 34181 "" "Prometheus/2.33.4" 2023-09-23T19:10:08.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:07 smithi098 bash[76817]: cluster 2023-09-23T19:10:06.697935+0000 mgr.y (mgr.26054) 1576 : cluster 0 pgmap v1763: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:08.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:07 smithi098 bash[79516]: cluster 2023-09-23T19:10:06.697935+0000 mgr.y (mgr.26054) 1576 : cluster 0 pgmap v1763: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:08.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:07 smithi173 bash[59422]: cluster 2023-09-23T19:10:06.697935+0000 mgr.y (mgr.26054) 1576 : cluster 0 pgmap v1763: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:09.510 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:10:09.510 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:10:09.816 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:09 smithi098 bash[76817]: cluster 2023-09-23T19:10:08.698881+0000 mgr.y (mgr.26054) 1577 : cluster 0 pgmap v1764: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:09.817 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:09 smithi098 bash[76817]: audit 2023-09-23T19:10:09.658648+0000 mon.a (mon.0) 2166 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:10:09.817 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:09 smithi098 bash[76817]: audit 2023-09-23T19:10:09.660080+0000 mon.a (mon.0) 2167 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:10:10.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:09 smithi098 bash[79516]: cluster 2023-09-23T19:10:08.698881+0000 mgr.y (mgr.26054) 1577 : cluster 0 pgmap v1764: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:10.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:09 smithi098 bash[79516]: audit 2023-09-23T19:10:09.658648+0000 mon.a (mon.0) 2166 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:10:10.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:09 smithi098 bash[79516]: audit 2023-09-23T19:10:09.660080+0000 mon.a (mon.0) 2167 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:10:10.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:09 smithi173 bash[59422]: cluster 2023-09-23T19:10:08.698881+0000 mgr.y (mgr.26054) 1577 : cluster 0 pgmap v1764: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:10.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:09 smithi173 bash[59422]: audit 2023-09-23T19:10:09.658648+0000 mon.a (mon.0) 2166 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:10:10.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:09 smithi173 bash[59422]: audit 2023-09-23T19:10:09.660080+0000 mon.a (mon.0) 2167 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:10:11.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:11 smithi098 bash[76817]: cluster 2023-09-23T19:10:10.699710+0000 mgr.y (mgr.26054) 1578 : cluster 0 pgmap v1765: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:11.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:11 smithi098 bash[79516]: cluster 2023-09-23T19:10:10.699710+0000 mgr.y (mgr.26054) 1578 : cluster 0 pgmap v1765: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:12.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:11 smithi173 bash[59422]: cluster 2023-09-23T19:10:10.699710+0000 mgr.y (mgr.26054) 1578 : cluster 0 pgmap v1765: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:13.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:13 smithi098 bash[79516]: cluster 2023-09-23T19:10:12.700259+0000 mgr.y (mgr.26054) 1579 : cluster 0 pgmap v1766: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:13.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:13 smithi098 bash[76817]: cluster 2023-09-23T19:10:12.700259+0000 mgr.y (mgr.26054) 1579 : cluster 0 pgmap v1766: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:14.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:13 smithi173 bash[59422]: cluster 2023-09-23T19:10:12.700259+0000 mgr.y (mgr.26054) 1579 : cluster 0 pgmap v1766: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:15.253 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-23T19:10:15.253 INFO:teuthology.orchestra.run.smithi098.stdout: "size": 20, 2023-09-23T19:10:15.253 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 600, 2023-09-23T19:10:15.253 INFO:teuthology.orchestra.run.smithi098.stdout: "ops": [ 2023-09-23T19:10:15.254 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.254 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:69972 10.1 10:8dcc2fab:::benchmark_data_smithi098_230241_object69971:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e321)", 2023-09-23T19:10:15.254 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:01.190404+0000", 2023-09-23T19:10:15.254 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 494.05575142100002, 2023-09-23T19:10:15.254 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.21232968799999999, 2023-09-23T19:10:15.254 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:10:15.254 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:10:15.254 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:10:15.254 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:10:15.254 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:10:15.254 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 69972 2023-09-23T19:10:15.254 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.255 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:10:15.255 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.255 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:10:15.255 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.190404+0000", 2023-09-23T19:10:15.255 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.255 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.255 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.255 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:10:15.255 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.190404+0000", 2023-09-23T19:10:15.255 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.255 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.255 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.255 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:10:15.256 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.190402+0000", 2023-09-23T19:10:15.256 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T19:10:15.256 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.256 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.256 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:10:15.256 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.190450+0000", 2023-09-23T19:10:15.256 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.7657999999999997e-05 2023-09-23T19:10:15.256 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.256 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.256 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:10:15.256 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.190460+0000", 2023-09-23T19:10:15.256 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0328e-05 2023-09-23T19:10:15.257 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.257 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.257 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:10:15.257 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.190467+0000", 2023-09-23T19:10:15.257 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.9500000000000004e-06 2023-09-23T19:10:15.257 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.257 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.257 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:10:15.257 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.193484+0000", 2023-09-23T19:10:15.257 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0030171059999999999 2023-09-23T19:10:15.257 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.257 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.257 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:10:15.258 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.193632+0000", 2023-09-23T19:10:15.258 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00014761099999999999 2023-09-23T19:10:15.258 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.258 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.258 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0", 2023-09-23T19:10:15.258 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.193989+0000", 2023-09-23T19:10:15.258 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00035690500000000003 2023-09-23T19:10:15.258 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.258 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.258 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:10:15.258 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.200441+0000", 2023-09-23T19:10:15.258 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0064521810000000004 2023-09-23T19:10:15.259 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.259 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.259 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:10:15.259 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.402582+0000", 2023-09-23T19:10:15.259 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.20214072 2023-09-23T19:10:15.259 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.259 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.259 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:10:15.259 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.402644+0000", 2023-09-23T19:10:15.259 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.198e-05 2023-09-23T19:10:15.259 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.259 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.259 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:10:15.260 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.402733+0000", 2023-09-23T19:10:15.260 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.9518000000000006e-05 2023-09-23T19:10:15.260 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.260 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:10:15.260 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.260 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.260 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.260 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:69986 10.1 10:8aab0511:::benchmark_data_smithi098_230241_object69985:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e321)", 2023-09-23T19:10:15.260 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:01.205827+0000", 2023-09-23T19:10:15.260 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 494.04032767199999, 2023-09-23T19:10:15.260 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.208237853, 2023-09-23T19:10:15.260 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:10:15.260 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:10:15.261 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:10:15.261 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:10:15.261 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:10:15.261 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 69986 2023-09-23T19:10:15.261 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.261 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:10:15.261 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.261 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:10:15.261 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.205827+0000", 2023-09-23T19:10:15.261 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.261 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.261 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.261 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:10:15.262 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.205827+0000", 2023-09-23T19:10:15.262 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.262 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.262 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.262 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:10:15.262 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.205826+0000", 2023-09-23T19:10:15.262 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T19:10:15.262 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.262 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.262 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:10:15.263 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.205880+0000", 2023-09-23T19:10:15.263 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.3476000000000001e-05 2023-09-23T19:10:15.263 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.263 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.263 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:10:15.263 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.205894+0000", 2023-09-23T19:10:15.263 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.4902999999999999e-05 2023-09-23T19:10:15.263 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.263 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.263 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:10:15.263 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.205904+0000", 2023-09-23T19:10:15.264 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.2199999999999998e-06 2023-09-23T19:10:15.264 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.264 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.264 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:10:15.264 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.206222+0000", 2023-09-23T19:10:15.264 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00031817599999999998 2023-09-23T19:10:15.264 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.264 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.264 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:10:15.264 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.206391+0000", 2023-09-23T19:10:15.264 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00016934099999999999 2023-09-23T19:10:15.264 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.264 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.265 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0", 2023-09-23T19:10:15.265 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.206533+0000", 2023-09-23T19:10:15.265 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00014168300000000001 2023-09-23T19:10:15.265 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.265 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.265 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:10:15.265 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.210580+0000", 2023-09-23T19:10:15.265 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0040473920000000004 2023-09-23T19:10:15.265 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.265 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.265 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:10:15.265 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.413916+0000", 2023-09-23T19:10:15.265 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.20333542600000001 2023-09-23T19:10:15.266 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.266 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.266 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:10:15.266 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.413972+0000", 2023-09-23T19:10:15.266 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.63e-05 2023-09-23T19:10:15.266 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.266 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.266 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:10:15.266 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.414065+0000", 2023-09-23T19:10:15.266 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.3305000000000002e-05 2023-09-23T19:10:15.266 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.267 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:10:15.267 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.267 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.267 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.267 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:69987 10.1 10:8d5610c7:::benchmark_data_smithi098_230241_object69986:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e321)", 2023-09-23T19:10:15.267 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:01.205914+0000", 2023-09-23T19:10:15.267 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 494.04024138900002, 2023-09-23T19:10:15.267 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.208333517, 2023-09-23T19:10:15.267 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:10:15.267 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:10:15.267 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:10:15.268 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:10:15.268 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:10:15.268 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 69987 2023-09-23T19:10:15.268 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.268 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:10:15.268 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.268 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:10:15.268 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.205914+0000", 2023-09-23T19:10:15.268 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.268 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.268 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.268 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:10:15.269 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.205914+0000", 2023-09-23T19:10:15.269 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.269 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.269 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.269 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:10:15.269 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.205913+0000", 2023-09-23T19:10:15.269 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T19:10:15.269 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.269 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.269 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:10:15.269 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.205966+0000", 2023-09-23T19:10:15.269 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.3279000000000003e-05 2023-09-23T19:10:15.270 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.270 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.270 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:10:15.270 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.205978+0000", 2023-09-23T19:10:15.270 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1265e-05 2023-09-23T19:10:15.270 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.270 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.270 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:10:15.270 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.205985+0000", 2023-09-23T19:10:15.270 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.4390000000000003e-06 2023-09-23T19:10:15.271 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.271 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.271 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:10:15.271 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.210839+0000", 2023-09-23T19:10:15.271 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0048536710000000004 2023-09-23T19:10:15.271 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.271 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.271 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:10:15.271 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.210973+0000", 2023-09-23T19:10:15.271 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00013452700000000001 2023-09-23T19:10:15.271 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.272 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.272 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0", 2023-09-23T19:10:15.272 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.211102+0000", 2023-09-23T19:10:15.272 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000128534 2023-09-23T19:10:15.272 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.272 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.272 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:10:15.272 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.218696+0000", 2023-09-23T19:10:15.272 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0075943649999999996 2023-09-23T19:10:15.272 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.272 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.272 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:10:15.273 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.414104+0000", 2023-09-23T19:10:15.273 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.195408369 2023-09-23T19:10:15.273 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.273 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.273 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:10:15.273 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.414165+0000", 2023-09-23T19:10:15.273 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.0248999999999999e-05 2023-09-23T19:10:15.273 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.273 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.273 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:10:15.273 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:01.414247+0000", 2023-09-23T19:10:15.273 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.2478000000000006e-05 2023-09-23T19:10:15.273 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.274 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:10:15.274 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.274 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.274 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.274 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:74840 10.2 10:40ff22f0:::benchmark_data_smithi098_230241_object74839:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e321)", 2023-09-23T19:10:15.274 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:06.385275+0000", 2023-09-23T19:10:15.274 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 488.86087989599997, 2023-09-23T19:10:15.274 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.20857925399999999, 2023-09-23T19:10:15.274 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:10:15.274 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:10:15.274 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:10:15.275 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:10:15.275 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:10:15.275 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 74840 2023-09-23T19:10:15.275 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.275 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:10:15.275 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.275 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:10:15.275 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:06.385275+0000", 2023-09-23T19:10:15.275 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.275 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.275 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.275 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:10:15.275 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:06.385275+0000", 2023-09-23T19:10:15.276 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.276 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.276 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.276 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:10:15.276 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:06.385274+0000", 2023-09-23T19:10:15.276 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T19:10:15.276 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.276 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.276 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:10:15.276 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:06.385297+0000", 2023-09-23T19:10:15.276 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.2450999999999999e-05 2023-09-23T19:10:15.276 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.277 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.277 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:10:15.277 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:06.385311+0000", 2023-09-23T19:10:15.277 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3994e-05 2023-09-23T19:10:15.277 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.277 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.277 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:10:15.277 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:06.385318+0000", 2023-09-23T19:10:15.277 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.0550000000000002e-06 2023-09-23T19:10:15.277 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.277 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.278 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:10:15.278 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:06.388001+0000", 2023-09-23T19:10:15.278 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0026830859999999999 2023-09-23T19:10:15.278 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.278 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.278 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:10:15.278 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:06.388165+0000", 2023-09-23T19:10:15.278 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000164079 2023-09-23T19:10:15.278 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.278 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.278 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,6", 2023-09-23T19:10:15.278 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:06.388553+0000", 2023-09-23T19:10:15.279 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00038800500000000002 2023-09-23T19:10:15.279 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.279 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.279 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:10:15.279 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:06.393211+0000", 2023-09-23T19:10:15.279 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0046576439999999998 2023-09-23T19:10:15.279 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.279 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.279 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:10:15.279 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:06.393227+0000", 2023-09-23T19:10:15.279 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.6192e-05 2023-09-23T19:10:15.279 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.279 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.280 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:10:15.280 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:06.593608+0000", 2023-09-23T19:10:15.280 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.200381754 2023-09-23T19:10:15.280 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.280 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.280 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:10:15.280 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:06.593751+0000", 2023-09-23T19:10:15.280 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000142819 2023-09-23T19:10:15.280 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.280 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.280 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:10:15.280 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:06.593854+0000", 2023-09-23T19:10:15.280 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00010312200000000001 2023-09-23T19:10:15.281 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.281 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:10:15.281 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.281 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.281 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.281 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:83106 10.2 10:4f4e5c75:::benchmark_data_smithi098_230241_object83105:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e321)", 2023-09-23T19:10:15.281 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:15.955336+0000", 2023-09-23T19:10:15.281 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 479.290819274, 2023-09-23T19:10:15.281 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.21028454199999999, 2023-09-23T19:10:15.281 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:10:15.281 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:10:15.281 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:10:15.282 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:10:15.282 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:10:15.282 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 83106 2023-09-23T19:10:15.282 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.282 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:10:15.282 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.282 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:10:15.282 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:15.955336+0000", 2023-09-23T19:10:15.282 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.282 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.282 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.282 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:10:15.283 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:15.955336+0000", 2023-09-23T19:10:15.283 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.283 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.283 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.283 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:10:15.283 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:15.955335+0000", 2023-09-23T19:10:15.283 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T19:10:15.283 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.283 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.283 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:10:15.283 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:15.955390+0000", 2023-09-23T19:10:15.283 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.5009999999999997e-05 2023-09-23T19:10:15.283 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.284 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.284 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:10:15.284 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:15.955403+0000", 2023-09-23T19:10:15.284 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.3118e-05 2023-09-23T19:10:15.284 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.284 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.284 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:10:15.284 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:15.955411+0000", 2023-09-23T19:10:15.284 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.3389999999999993e-06 2023-09-23T19:10:15.284 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.284 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.284 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:10:15.284 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:15.958605+0000", 2023-09-23T19:10:15.285 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0031939529999999998 2023-09-23T19:10:15.285 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.285 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.285 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:10:15.285 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:15.958742+0000", 2023-09-23T19:10:15.285 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000136515 2023-09-23T19:10:15.285 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.285 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.285 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,6", 2023-09-23T19:10:15.285 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:15.958880+0000", 2023-09-23T19:10:15.285 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00013828700000000001 2023-09-23T19:10:15.286 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.286 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.286 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:10:15.286 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:15.963776+0000", 2023-09-23T19:10:15.286 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0048955320000000002 2023-09-23T19:10:15.286 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.286 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.286 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:10:15.286 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:15.967596+0000", 2023-09-23T19:10:15.286 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0038203859999999998 2023-09-23T19:10:15.286 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.286 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.287 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:10:15.287 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:16.165515+0000", 2023-09-23T19:10:15.287 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.19791905800000001 2023-09-23T19:10:15.287 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.287 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.287 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:10:15.287 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:16.165574+0000", 2023-09-23T19:10:15.287 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.9419999999999997e-05 2023-09-23T19:10:15.287 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.287 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.287 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:10:15.287 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:16.165620+0000", 2023-09-23T19:10:15.288 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.5927000000000003e-05 2023-09-23T19:10:15.288 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.288 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:10:15.288 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.288 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.288 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.288 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:83122 10.b 10:d2ccb8e2:::benchmark_data_smithi098_230241_object83121:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e321)", 2023-09-23T19:10:15.288 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:02:15.975922+0000", 2023-09-23T19:10:15.288 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 479.27023293600001, 2023-09-23T19:10:15.288 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.21157658200000001, 2023-09-23T19:10:15.288 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:10:15.288 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:10:15.289 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:10:15.289 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:10:15.289 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:10:15.289 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 83122 2023-09-23T19:10:15.289 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.289 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:10:15.289 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.289 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:10:15.289 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:15.975922+0000", 2023-09-23T19:10:15.289 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.290 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.290 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.290 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:10:15.290 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:15.975922+0000", 2023-09-23T19:10:15.290 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.290 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.290 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.290 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:10:15.291 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:15.975920+0000", 2023-09-23T19:10:15.291 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:10:15.291 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.291 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.291 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:10:15.291 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:15.975973+0000", 2023-09-23T19:10:15.291 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.2677999999999999e-05 2023-09-23T19:10:15.291 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.291 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.291 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:10:15.291 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:15.975984+0000", 2023-09-23T19:10:15.291 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0713000000000001e-05 2023-09-23T19:10:15.292 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.292 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.292 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:10:15.292 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:15.975990+0000", 2023-09-23T19:10:15.292 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.083e-06 2023-09-23T19:10:15.292 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.292 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.292 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:10:15.292 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:16.165416+0000", 2023-09-23T19:10:15.292 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.18942595100000001 2023-09-23T19:10:15.292 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.292 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.293 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:10:15.293 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:16.165606+0000", 2023-09-23T19:10:15.293 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000190347 2023-09-23T19:10:15.293 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.293 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.293 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,6", 2023-09-23T19:10:15.293 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:16.165846+0000", 2023-09-23T19:10:15.293 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000240437 2023-09-23T19:10:15.293 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.293 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.293 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:10:15.293 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:16.187360+0000", 2023-09-23T19:10:15.293 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.021513971 2023-09-23T19:10:15.294 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.294 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.294 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:10:15.294 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:16.187406+0000", 2023-09-23T19:10:15.294 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.6048999999999999e-05 2023-09-23T19:10:15.294 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.294 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.294 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:10:15.294 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:16.187416+0000", 2023-09-23T19:10:15.294 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.5610000000000004e-06 2023-09-23T19:10:15.294 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.295 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.295 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:10:15.295 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:16.187463+0000", 2023-09-23T19:10:15.295 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.6737000000000003e-05 2023-09-23T19:10:15.295 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.295 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.295 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:10:15.295 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:02:16.187499+0000", 2023-09-23T19:10:15.295 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.5979000000000002e-05 2023-09-23T19:10:15.295 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.295 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:10:15.295 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.296 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.296 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.296 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:169183 10.b 10:df26a6a4:::benchmark_data_smithi098_230241_object68721:head [delete] snapc 0=[] ondisk+write+known_if_redirected e333)", 2023-09-23T19:10:15.296 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:04:26.902224+0000", 2023-09-23T19:10:15.296 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 348.34393076600003, 2023-09-23T19:10:15.296 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.21950240500000001, 2023-09-23T19:10:15.296 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:10:15.296 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:10:15.296 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:10:15.296 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:10:15.297 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:10:15.297 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 169183 2023-09-23T19:10:15.297 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.297 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:10:15.297 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.297 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:10:15.297 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.902224+0000", 2023-09-23T19:10:15.297 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.297 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.297 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.297 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:10:15.297 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.902224+0000", 2023-09-23T19:10:15.298 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.298 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.298 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.298 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:10:15.298 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.902223+0000", 2023-09-23T19:10:15.298 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T19:10:15.298 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.298 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.298 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:10:15.298 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.902231+0000", 2023-09-23T19:10:15.298 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.9629999999999998e-06 2023-09-23T19:10:15.298 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.299 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.299 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:10:15.299 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.902243+0000", 2023-09-23T19:10:15.299 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.2291e-05 2023-09-23T19:10:15.299 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.299 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.299 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:10:15.299 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.902252+0000", 2023-09-23T19:10:15.299 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.7309999999999999e-06 2023-09-23T19:10:15.299 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.299 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.299 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:10:15.300 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.908684+0000", 2023-09-23T19:10:15.300 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0064324350000000002 2023-09-23T19:10:15.300 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.300 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.300 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:10:15.300 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.908803+0000", 2023-09-23T19:10:15.300 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00011913699999999999 2023-09-23T19:10:15.300 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.300 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.300 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,6", 2023-09-23T19:10:15.300 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.908910+0000", 2023-09-23T19:10:15.300 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000107105 2023-09-23T19:10:15.301 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.301 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.301 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:10:15.301 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.917949+0000", 2023-09-23T19:10:15.301 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0090386530000000007 2023-09-23T19:10:15.301 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.301 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.301 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:10:15.301 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:26.917981+0000", 2023-09-23T19:10:15.301 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.1647999999999999e-05 2023-09-23T19:10:15.301 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.301 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.302 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:10:15.302 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:27.120878+0000", 2023-09-23T19:10:15.302 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.202897358 2023-09-23T19:10:15.302 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.302 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.302 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:10:15.302 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:27.121679+0000", 2023-09-23T19:10:15.302 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00080118099999999999 2023-09-23T19:10:15.303 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.303 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.303 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:10:15.303 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:04:27.121727+0000", 2023-09-23T19:10:15.303 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.7422000000000001e-05 2023-09-23T19:10:15.303 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.303 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:10:15.303 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.303 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.303 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.303 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:194016 10.b 10:dc990bf3:::benchmark_data_smithi098_230241_object93554:head [delete] snapc 0=[] ondisk+write+known_if_redirected e335)", 2023-09-23T19:10:15.304 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:05:08.442940+0000", 2023-09-23T19:10:15.304 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 306.80321487200001, 2023-09-23T19:10:15.304 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.218175219, 2023-09-23T19:10:15.304 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:10:15.304 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:10:15.304 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:10:15.304 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:10:15.304 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:10:15.304 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 194016 2023-09-23T19:10:15.304 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.304 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:10:15.304 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.304 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:10:15.305 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.442940+0000", 2023-09-23T19:10:15.305 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.305 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.305 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.305 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:10:15.305 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.442940+0000", 2023-09-23T19:10:15.305 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.305 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.305 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.305 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:10:15.305 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.442939+0000", 2023-09-23T19:10:15.306 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999986 2023-09-23T19:10:15.306 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.306 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.306 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:10:15.306 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.442946+0000", 2023-09-23T19:10:15.306 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.8379999999999996e-06 2023-09-23T19:10:15.306 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.306 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.306 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:10:15.306 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.442956+0000", 2023-09-23T19:10:15.306 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.0079e-05 2023-09-23T19:10:15.306 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.307 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.307 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:10:15.307 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.442963+0000", 2023-09-23T19:10:15.307 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.2490000000000004e-06 2023-09-23T19:10:15.307 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.307 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.307 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:10:15.307 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.443014+0000", 2023-09-23T19:10:15.307 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.1313000000000002e-05 2023-09-23T19:10:15.307 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.307 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.307 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:10:15.307 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.443152+0000", 2023-09-23T19:10:15.308 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000137515 2023-09-23T19:10:15.308 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.308 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.308 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,6", 2023-09-23T19:10:15.308 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.443280+0000", 2023-09-23T19:10:15.308 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00012869499999999999 2023-09-23T19:10:15.308 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.308 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.308 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:10:15.308 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.446512+0000", 2023-09-23T19:10:15.308 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0032317919999999998 2023-09-23T19:10:15.308 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.308 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.309 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:10:15.309 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.447350+0000", 2023-09-23T19:10:15.309 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00083812000000000003 2023-09-23T19:10:15.309 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.309 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.309 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:10:15.309 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.660966+0000", 2023-09-23T19:10:15.309 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.21361591199999999 2023-09-23T19:10:15.309 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.309 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.309 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:10:15.309 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.661070+0000", 2023-09-23T19:10:15.310 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000103943 2023-09-23T19:10:15.310 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.310 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.310 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:10:15.310 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.661115+0000", 2023-09-23T19:10:15.310 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.5238000000000003e-05 2023-09-23T19:10:15.310 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.310 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:10:15.310 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.310 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.310 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.311 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.45779.0:194017 10.2 10:4b71006e:::benchmark_data_smithi098_230241_object93555:head [delete] snapc 0=[] ondisk+write+known_if_redirected e335)", 2023-09-23T19:10:15.311 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:05:08.443094+0000", 2023-09-23T19:10:15.311 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 306.80306092500001, 2023-09-23T19:10:15.311 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.21803172900000001, 2023-09-23T19:10:15.311 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:10:15.311 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:10:15.311 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:10:15.311 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.45779", 2023-09-23T19:10:15.311 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3684029926", 2023-09-23T19:10:15.311 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 194017 2023-09-23T19:10:15.311 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.311 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:10:15.312 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.312 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:10:15.312 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.443094+0000", 2023-09-23T19:10:15.312 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.312 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.312 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.312 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:10:15.312 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.443094+0000", 2023-09-23T19:10:15.312 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.312 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.312 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.312 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:10:15.312 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.443093+0000", 2023-09-23T19:10:15.313 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T19:10:15.313 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.313 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.313 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:10:15.313 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.443098+0000", 2023-09-23T19:10:15.313 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.882e-06 2023-09-23T19:10:15.313 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.313 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.313 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:10:15.313 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.443108+0000", 2023-09-23T19:10:15.313 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.7370000000000001e-06 2023-09-23T19:10:15.313 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.314 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.314 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:10:15.314 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.443115+0000", 2023-09-23T19:10:15.314 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.0330000000000003e-06 2023-09-23T19:10:15.314 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.314 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.314 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:10:15.314 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.446258+0000", 2023-09-23T19:10:15.314 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0031430310000000001 2023-09-23T19:10:15.314 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.314 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.314 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:10:15.314 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.446386+0000", 2023-09-23T19:10:15.315 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00012786000000000001 2023-09-23T19:10:15.315 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.315 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.315 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,3,6", 2023-09-23T19:10:15.315 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.446471+0000", 2023-09-23T19:10:15.315 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.5377999999999995e-05 2023-09-23T19:10:15.315 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.315 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.315 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:10:15.315 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.449900+0000", 2023-09-23T19:10:15.315 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0034290459999999998 2023-09-23T19:10:15.315 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.316 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.316 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:10:15.316 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.449994+0000", 2023-09-23T19:10:15.316 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.3790999999999997e-05 2023-09-23T19:10:15.316 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.316 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.316 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:10:15.316 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.453762+0000", 2023-09-23T19:10:15.316 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0037681059999999998 2023-09-23T19:10:15.316 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.316 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.316 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:10:15.317 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.660986+0000", 2023-09-23T19:10:15.317 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.20722374199999999 2023-09-23T19:10:15.317 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.317 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.317 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:10:15.317 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.661028+0000", 2023-09-23T19:10:15.317 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.2366000000000001e-05 2023-09-23T19:10:15.317 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.317 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.317 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:10:15.317 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:05:08.661126+0000", 2023-09-23T19:10:15.317 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 9.7504999999999996e-05 2023-09-23T19:10:15.318 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.318 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:10:15.318 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.318 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.318 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.318 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.46055.0:5 2.4 2:3831950c:::rbd_directory:head [tmapup] snapc 0=[] ondisk+write+known_if_redirected+full_try e365)", 2023-09-23T19:10:15.318 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:08:52.968339+0000", 2023-09-23T19:10:15.318 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 82.277815583000006, 2023-09-23T19:10:15.318 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.263450721, 2023-09-23T19:10:15.318 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:10:15.318 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T19:10:15.318 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:10:15.319 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.46055", 2023-09-23T19:10:15.319 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/3914642451", 2023-09-23T19:10:15.319 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 5 2023-09-23T19:10:15.319 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.319 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:10:15.319 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.319 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:10:15.319 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:52.968339+0000", 2023-09-23T19:10:15.319 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.319 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.319 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.319 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:10:15.320 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:52.968339+0000", 2023-09-23T19:10:15.320 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.320 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.320 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.320 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:10:15.320 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:52.968338+0000", 2023-09-23T19:10:15.320 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:10:15.320 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.320 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.320 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:10:15.320 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:52.968345+0000", 2023-09-23T19:10:15.320 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.1219999999999997e-06 2023-09-23T19:10:15.321 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.321 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.321 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:10:15.321 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:52.968356+0000", 2023-09-23T19:10:15.321 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.135e-05 2023-09-23T19:10:15.321 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.321 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.321 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:10:15.321 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:52.968364+0000", 2023-09-23T19:10:15.321 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.8199999999999997e-06 2023-09-23T19:10:15.321 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.321 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.322 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:10:15.322 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:52.968420+0000", 2023-09-23T19:10:15.322 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.5720000000000002e-05 2023-09-23T19:10:15.322 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.322 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.322 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:10:15.322 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:53.159016+0000", 2023-09-23T19:10:15.322 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.190596186 2023-09-23T19:10:15.322 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.322 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.322 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:10:15.323 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:53.231790+0000", 2023-09-23T19:10:15.323 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.072774484 2023-09-23T19:10:15.323 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.323 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:10:15.323 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.323 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.323 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.323 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_repop_reply(client.46079.0:87 2.0 e365/271 ondisk, result = 0)", 2023-09-23T19:10:15.323 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:08:55.892294+0000", 2023-09-23T19:10:15.323 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 79.353861160999998, 2023-09-23T19:10:15.323 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.14808570400000001, 2023-09-23T19:10:15.323 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:10:15.323 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T19:10:15.324 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:10:15.324 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.324 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:10:15.324 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:55.892294+0000", 2023-09-23T19:10:15.324 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.324 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.324 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.324 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:10:15.324 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:55.892294+0000", 2023-09-23T19:10:15.324 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.324 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.324 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.325 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:10:15.325 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:55.892293+0000", 2023-09-23T19:10:15.325 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999995 2023-09-23T19:10:15.325 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.325 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.325 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:10:15.325 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:55.892297+0000", 2023-09-23T19:10:15.325 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.5930000000000002e-06 2023-09-23T19:10:15.325 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.325 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.325 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:10:15.325 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:55.892304+0000", 2023-09-23T19:10:15.325 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.6789999999999999e-06 2023-09-23T19:10:15.326 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.326 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.326 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:10:15.326 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:55.892308+0000", 2023-09-23T19:10:15.326 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.5709999999999999e-06 2023-09-23T19:10:15.326 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.326 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.326 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:10:15.326 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:55.892343+0000", 2023-09-23T19:10:15.326 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.5132000000000002e-05 2023-09-23T19:10:15.326 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.327 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.327 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:10:15.327 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:55.892351+0000", 2023-09-23T19:10:15.327 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 7.807e-06 2023-09-23T19:10:15.327 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.327 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.327 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:10:15.327 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:56.040380+0000", 2023-09-23T19:10:15.327 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.14802839700000001 2023-09-23T19:10:15.327 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.327 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:10:15.327 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.327 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.328 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.328 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.46079.0:88 2.0 2:08a1c67d:::rbd_header.b3ed9942bcee:head [notify cookie 139962314209648] snapc 0=[] ondisk+read+known_if_redirected+full_try e365)", 2023-09-23T19:10:15.328 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:08:55.892574+0000", 2023-09-23T19:10:15.328 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 79.353581390000002, 2023-09-23T19:10:15.328 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.148308621, 2023-09-23T19:10:15.328 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:10:15.328 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "started", 2023-09-23T19:10:15.328 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:10:15.328 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.46079", 2023-09-23T19:10:15.328 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/811283827", 2023-09-23T19:10:15.328 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 88 2023-09-23T19:10:15.328 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.329 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:10:15.329 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.329 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:10:15.329 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:55.892574+0000", 2023-09-23T19:10:15.329 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.329 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.329 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.329 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:10:15.329 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:55.892574+0000", 2023-09-23T19:10:15.329 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.329 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.329 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.329 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:10:15.330 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:55.892573+0000", 2023-09-23T19:10:15.330 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.999999 2023-09-23T19:10:15.330 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.330 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.330 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:10:15.330 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:55.892579+0000", 2023-09-23T19:10:15.330 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.9900000000000002e-06 2023-09-23T19:10:15.330 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.330 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.330 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:10:15.330 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:56.040352+0000", 2023-09-23T19:10:15.330 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.14777373099999999 2023-09-23T19:10:15.331 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.331 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.331 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:10:15.331 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:56.040386+0000", 2023-09-23T19:10:15.331 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 3.3234000000000001e-05 2023-09-23T19:10:15.331 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.331 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.331 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:10:15.331 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:56.040511+0000", 2023-09-23T19:10:15.331 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00012583399999999999 2023-09-23T19:10:15.331 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.331 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.331 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:10:15.332 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:56.040700+0000", 2023-09-23T19:10:15.332 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000188858 2023-09-23T19:10:15.332 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.332 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.332 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:10:15.332 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:08:56.040882+0000", 2023-09-23T19:10:15.332 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000181953 2023-09-23T19:10:15.332 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.332 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:10:15.332 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.332 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.332 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.333 INFO:teuthology.orchestra.run.smithi098.stdout: "description": "osd_op(client.46088.0:5 2.4 2:3831950c:::rbd_directory:head [call rbd.dir_add_image in=16b] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-09-23T19:10:15.333 INFO:teuthology.orchestra.run.smithi098.stdout: "initiated_at": "2023-09-23T19:09:01.841423+0000", 2023-09-23T19:10:15.333 INFO:teuthology.orchestra.run.smithi098.stdout: "age": 73.404731808999998, 2023-09-23T19:10:15.333 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.46925335499999998, 2023-09-23T19:10:15.333 INFO:teuthology.orchestra.run.smithi098.stdout: "type_data": { 2023-09-23T19:10:15.333 INFO:teuthology.orchestra.run.smithi098.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:10:15.333 INFO:teuthology.orchestra.run.smithi098.stdout: "client_info": { 2023-09-23T19:10:15.333 INFO:teuthology.orchestra.run.smithi098.stdout: "client": "client.46088", 2023-09-23T19:10:15.333 INFO:teuthology.orchestra.run.smithi098.stdout: "client_addr": "172.21.15.98:0/1258423516", 2023-09-23T19:10:15.333 INFO:teuthology.orchestra.run.smithi098.stdout: "tid": 5 2023-09-23T19:10:15.333 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.333 INFO:teuthology.orchestra.run.smithi098.stdout: "events": [ 2023-09-23T19:10:15.333 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.334 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "initiated", 2023-09-23T19:10:15.334 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:09:01.841423+0000", 2023-09-23T19:10:15.334 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.334 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.334 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.334 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "throttled", 2023-09-23T19:10:15.334 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:09:01.841423+0000", 2023-09-23T19:10:15.334 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0 2023-09-23T19:10:15.334 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.334 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.334 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "header_read", 2023-09-23T19:10:15.334 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:09:01.841421+0000", 2023-09-23T19:10:15.335 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4294967295.9999981 2023-09-23T19:10:15.335 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.335 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.335 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "all_read", 2023-09-23T19:10:15.335 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:09:01.841430+0000", 2023-09-23T19:10:15.335 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 8.9080000000000001e-06 2023-09-23T19:10:15.335 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.335 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.335 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "dispatched", 2023-09-23T19:10:15.335 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:09:01.841445+0000", 2023-09-23T19:10:15.335 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.5075000000000001e-05 2023-09-23T19:10:15.335 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.335 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.336 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "queued_for_pg", 2023-09-23T19:10:15.336 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:09:01.841457+0000", 2023-09-23T19:10:15.336 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 1.1678000000000001e-05 2023-09-23T19:10:15.336 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.336 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.336 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:10:15.336 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:09:01.841568+0000", 2023-09-23T19:10:15.336 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000111094 2023-09-23T19:10:15.336 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.336 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.336 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for peered", 2023-09-23T19:10:15.336 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:09:01.841589+0000", 2023-09-23T19:10:15.337 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 2.0594999999999998e-05 2023-09-23T19:10:15.337 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.337 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.337 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:10:15.337 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:09:02.307228+0000", 2023-09-23T19:10:15.337 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.46563912899999998 2023-09-23T19:10:15.337 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.337 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.337 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for degraded object", 2023-09-23T19:10:15.337 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:09:02.307639+0000", 2023-09-23T19:10:15.337 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00041122400000000002 2023-09-23T19:10:15.338 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.338 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.338 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "reached_pg", 2023-09-23T19:10:15.338 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:09:02.309296+0000", 2023-09-23T19:10:15.338 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.0016567369999999999 2023-09-23T19:10:15.338 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.338 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.338 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "started", 2023-09-23T19:10:15.338 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:09:02.309362+0000", 2023-09-23T19:10:15.338 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 6.6064999999999999e-05 2023-09-23T19:10:15.338 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.338 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.338 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "waiting for subops from 0,7", 2023-09-23T19:10:15.339 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:09:02.309552+0000", 2023-09-23T19:10:15.339 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000189877 2023-09-23T19:10:15.339 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.339 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.339 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "op_commit", 2023-09-23T19:10:15.339 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:09:02.310103+0000", 2023-09-23T19:10:15.339 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00055121900000000004 2023-09-23T19:10:15.339 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.339 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.339 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:10:15.339 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:09:02.310255+0000", 2023-09-23T19:10:15.339 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.000152597 2023-09-23T19:10:15.339 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.340 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.340 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:10:15.340 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:09:02.310584+0000", 2023-09-23T19:10:15.340 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 0.00032868699999999998 2023-09-23T19:10:15.340 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.340 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.340 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "commit_sent", 2023-09-23T19:10:15.340 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:09:02.310634+0000", 2023-09-23T19:10:15.340 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 5.0259000000000001e-05 2023-09-23T19:10:15.340 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-23T19:10:15.340 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-23T19:10:15.340 INFO:teuthology.orchestra.run.smithi098.stdout: "event": "done", 2023-09-23T19:10:15.341 INFO:teuthology.orchestra.run.smithi098.stdout: "time": "2023-09-23T19:09:02.310677+0000", 2023-09-23T19:10:15.341 INFO:teuthology.orchestra.run.smithi098.stdout: "duration": 4.2187000000000001e-05 2023-09-23T19:10:15.341 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.341 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:10:15.341 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.341 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-23T19:10:15.341 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-23T19:10:15.341 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-23T19:10:15.781 INFO:teuthology.orchestra.run.smithi098.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T19:10:15.809 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:15 smithi098 bash[76817]: cluster 2023-09-23T19:10:14.700902+0000 mgr.y (mgr.26054) 1580 : cluster 0 pgmap v1767: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:15.809 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:15 smithi098 bash[76817]: audit 2023-09-23T19:10:14.931808+0000 mon.a (mon.0) 2168 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:10:15.809 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:15 smithi098 bash[76817]: audit 2023-09-23T19:10:15.350764+0000 mon.c (mon.2) 160 : audit 1 from='client.? 172.21.15.98:0/936289015' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:10:15.809 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:15 smithi098 bash[76817]: audit 2023-09-23T19:10:15.351402+0000 mon.a (mon.0) 2169 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:10:15.809 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:15 smithi098 bash[79516]: cluster 2023-09-23T19:10:14.700902+0000 mgr.y (mgr.26054) 1580 : cluster 0 pgmap v1767: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:15.809 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:15 smithi098 bash[79516]: audit 2023-09-23T19:10:14.931808+0000 mon.a (mon.0) 2168 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:10:15.809 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:15 smithi098 bash[79516]: audit 2023-09-23T19:10:15.350764+0000 mon.c (mon.2) 160 : audit 1 from='client.? 172.21.15.98:0/936289015' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:10:15.809 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:15 smithi098 bash[79516]: audit 2023-09-23T19:10:15.351402+0000 mon.a (mon.0) 2169 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:10:16.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:15 smithi173 bash[59422]: cluster 2023-09-23T19:10:14.700902+0000 mgr.y (mgr.26054) 1580 : cluster 0 pgmap v1767: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:16.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:15 smithi173 bash[59422]: audit 2023-09-23T19:10:14.931808+0000 mon.a (mon.0) 2168 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:10:16.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:15 smithi173 bash[59422]: audit 2023-09-23T19:10:15.350764+0000 mon.c (mon.2) 160 : audit 1 from='client.? 172.21.15.98:0/936289015' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:10:16.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:15 smithi173 bash[59422]: audit 2023-09-23T19:10:15.351402+0000 mon.a (mon.0) 2169 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:10:16.355 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:10:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:10:16] "GET /metrics HTTP/1.1" 200 34178 "" "Prometheus/2.33.4" 2023-09-23T19:10:16.585 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T19:10:17.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:16 smithi173 bash[59422]: audit 2023-09-23T19:10:15.770055+0000 mon.a (mon.0) 2170 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:10:17.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:16 smithi173 bash[59422]: cluster 2023-09-23T19:10:15.770164+0000 mon.a (mon.0) 2171 : cluster 0 osdmap e392: 8 total, 8 up, 7 in 2023-09-23T19:10:17.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:16 smithi173 bash[59422]: cluster 2023-09-23T19:10:15.771603+0000 osd.7 (osd.7) 34 : cluster 3 failed to encode map e392 with expected crc 2023-09-23T19:10:17.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:16 smithi173 bash[59422]: cluster 2023-09-23T19:10:15.771671+0000 osd.6 (osd.6) 380 : cluster 3 failed to encode map e392 with expected crc 2023-09-23T19:10:17.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:16 smithi173 bash[59422]: cluster 2023-09-23T19:10:16.442002+0000 osd.4 (osd.4) 119 : cluster 3 failed to encode map e392 with expected crc 2023-09-23T19:10:17.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:16 smithi098 bash[79516]: audit 2023-09-23T19:10:15.770055+0000 mon.a (mon.0) 2170 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:10:17.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:16 smithi098 bash[79516]: cluster 2023-09-23T19:10:15.770164+0000 mon.a (mon.0) 2171 : cluster 0 osdmap e392: 8 total, 8 up, 7 in 2023-09-23T19:10:17.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:16 smithi098 bash[79516]: cluster 2023-09-23T19:10:15.771603+0000 osd.7 (osd.7) 34 : cluster 3 failed to encode map e392 with expected crc 2023-09-23T19:10:17.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:16 smithi098 bash[79516]: cluster 2023-09-23T19:10:15.771671+0000 osd.6 (osd.6) 380 : cluster 3 failed to encode map e392 with expected crc 2023-09-23T19:10:17.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:16 smithi098 bash[79516]: cluster 2023-09-23T19:10:16.442002+0000 osd.4 (osd.4) 119 : cluster 3 failed to encode map e392 with expected crc 2023-09-23T19:10:17.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:16 smithi098 bash[76817]: audit 2023-09-23T19:10:15.770055+0000 mon.a (mon.0) 2170 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:10:17.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:16 smithi098 bash[76817]: cluster 2023-09-23T19:10:15.770164+0000 mon.a (mon.0) 2171 : cluster 0 osdmap e392: 8 total, 8 up, 7 in 2023-09-23T19:10:17.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:16 smithi098 bash[76817]: cluster 2023-09-23T19:10:15.771603+0000 osd.7 (osd.7) 34 : cluster 3 failed to encode map e392 with expected crc 2023-09-23T19:10:17.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:16 smithi098 bash[76817]: cluster 2023-09-23T19:10:15.771671+0000 osd.6 (osd.6) 380 : cluster 3 failed to encode map e392 with expected crc 2023-09-23T19:10:17.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:16 smithi098 bash[76817]: cluster 2023-09-23T19:10:16.442002+0000 osd.4 (osd.4) 119 : cluster 3 failed to encode map e392 with expected crc 2023-09-23T19:10:17.244 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T19:10:18.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:17 smithi098 bash[76817]: cluster 2023-09-23T19:10:16.701482+0000 mgr.y (mgr.26054) 1581 : cluster 0 pgmap v1769: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:18.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:17 smithi098 bash[76817]: cluster 2023-09-23T19:10:17.091920+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e392 with expected crc 2023-09-23T19:10:18.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:17 smithi098 bash[79516]: cluster 2023-09-23T19:10:16.701482+0000 mgr.y (mgr.26054) 1581 : cluster 0 pgmap v1769: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:18.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:17 smithi098 bash[79516]: cluster 2023-09-23T19:10:17.091920+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e392 with expected crc 2023-09-23T19:10:18.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:17 smithi173 bash[59422]: cluster 2023-09-23T19:10:16.701482+0000 mgr.y (mgr.26054) 1581 : cluster 0 pgmap v1769: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:18.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:17 smithi173 bash[59422]: cluster 2023-09-23T19:10:17.091920+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e392 with expected crc 2023-09-23T19:10:20.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:19 smithi098 bash[76817]: cluster 2023-09-23T19:10:18.702404+0000 mgr.y (mgr.26054) 1582 : cluster 0 pgmap v1770: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:20.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:19 smithi098 bash[79516]: cluster 2023-09-23T19:10:18.702404+0000 mgr.y (mgr.26054) 1582 : cluster 0 pgmap v1770: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:20.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:19 smithi173 bash[59422]: cluster 2023-09-23T19:10:18.702404+0000 mgr.y (mgr.26054) 1582 : cluster 0 pgmap v1770: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:20.827 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:10:21.791 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:10:22.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:22 smithi098 bash[76817]: cluster 2023-09-23T19:10:20.703180+0000 mgr.y (mgr.26054) 1583 : cluster 0 pgmap v1771: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:22.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:22 smithi098 bash[76817]: audit 2023-09-23T19:10:21.285456+0000 mon.a (mon.0) 2172 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:10:22.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:22 smithi098 bash[76817]: audit 2023-09-23T19:10:21.292102+0000 mon.a (mon.0) 2173 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:10:22.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:22 smithi098 bash[76817]: audit 2023-09-23T19:10:21.930234+0000 mon.a (mon.0) 2174 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:10:22.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:22 smithi098 bash[76817]: audit 2023-09-23T19:10:21.938426+0000 mon.a (mon.0) 2175 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:10:22.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:22 smithi098 bash[79516]: cluster 2023-09-23T19:10:20.703180+0000 mgr.y (mgr.26054) 1583 : cluster 0 pgmap v1771: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:22.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:22 smithi098 bash[79516]: audit 2023-09-23T19:10:21.285456+0000 mon.a (mon.0) 2172 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:10:22.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:22 smithi098 bash[79516]: audit 2023-09-23T19:10:21.292102+0000 mon.a (mon.0) 2173 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:10:22.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:22 smithi098 bash[79516]: audit 2023-09-23T19:10:21.930234+0000 mon.a (mon.0) 2174 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:10:22.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:22 smithi098 bash[79516]: audit 2023-09-23T19:10:21.938426+0000 mon.a (mon.0) 2175 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:10:22.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:22 smithi173 bash[59422]: cluster 2023-09-23T19:10:20.703180+0000 mgr.y (mgr.26054) 1583 : cluster 0 pgmap v1771: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:22.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:22 smithi173 bash[59422]: audit 2023-09-23T19:10:21.285456+0000 mon.a (mon.0) 2172 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:10:22.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:22 smithi173 bash[59422]: audit 2023-09-23T19:10:21.292102+0000 mon.a (mon.0) 2173 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:10:22.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:22 smithi173 bash[59422]: audit 2023-09-23T19:10:21.930234+0000 mon.a (mon.0) 2174 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:10:22.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:22 smithi173 bash[59422]: audit 2023-09-23T19:10:21.938426+0000 mon.a (mon.0) 2175 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:10:23.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:23 smithi098 bash[79516]: audit 2023-09-23T19:10:22.408977+0000 mon.a (mon.0) 2176 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:10:23.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:23 smithi098 bash[79516]: audit 2023-09-23T19:10:22.410561+0000 mon.a (mon.0) 2177 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:10:23.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:23 smithi098 bash[79516]: audit 2023-09-23T19:10:22.420263+0000 mon.a (mon.0) 2178 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:10:23.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:23 smithi098 bash[76817]: audit 2023-09-23T19:10:22.408977+0000 mon.a (mon.0) 2176 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:10:23.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:23 smithi098 bash[76817]: audit 2023-09-23T19:10:22.410561+0000 mon.a (mon.0) 2177 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:10:23.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:23 smithi098 bash[76817]: audit 2023-09-23T19:10:22.420263+0000 mon.a (mon.0) 2178 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:10:23.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:23 smithi173 bash[59422]: audit 2023-09-23T19:10:22.408977+0000 mon.a (mon.0) 2176 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:10:23.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:23 smithi173 bash[59422]: audit 2023-09-23T19:10:22.410561+0000 mon.a (mon.0) 2177 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:10:23.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:23 smithi173 bash[59422]: audit 2023-09-23T19:10:22.420263+0000 mon.a (mon.0) 2178 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:10:24.189 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T19:10:24.189 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-09-23T19:10:24.189 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 0 2023-09-23T19:10:24.189 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T19:10:24.528 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:24 smithi173 bash[59422]: cluster 2023-09-23T19:10:22.704103+0000 mgr.y (mgr.26054) 1584 : cluster 0 pgmap v1772: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:24.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:24 smithi098 bash[76817]: cluster 2023-09-23T19:10:22.704103+0000 mgr.y (mgr.26054) 1584 : cluster 0 pgmap v1772: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:24.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:24 smithi098 bash[79516]: cluster 2023-09-23T19:10:22.704103+0000 mgr.y (mgr.26054) 1584 : cluster 0 pgmap v1772: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:25.091 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T19:10:25.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:25 smithi098 bash[79516]: audit 2023-09-23T19:10:24.653074+0000 mon.a (mon.0) 2179 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:10:25.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:25 smithi098 bash[76817]: audit 2023-09-23T19:10:24.653074+0000 mon.a (mon.0) 2179 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:10:25.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:25 smithi173 bash[59422]: audit 2023-09-23T19:10:24.653074+0000 mon.a (mon.0) 2179 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:10:26.437 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:10:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:10:26] "GET /metrics HTTP/1.1" 200 34184 "" "Prometheus/2.33.4" 2023-09-23T19:10:26.645 INFO:teuthology.orchestra.run.smithi098.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T19:10:26.689 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:26 smithi098 bash[79516]: cluster 2023-09-23T19:10:24.704687+0000 mgr.y (mgr.26054) 1585 : cluster 0 pgmap v1773: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:26.690 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:26 smithi098 bash[79516]: audit 2023-09-23T19:10:25.729906+0000 mon.c (mon.2) 161 : audit 1 from='client.? 172.21.15.98:0/3499899226' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:10:26.690 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:26 smithi098 bash[79516]: audit 2023-09-23T19:10:25.730595+0000 mon.a (mon.0) 2180 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:10:26.690 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:26 smithi098 bash[76817]: cluster 2023-09-23T19:10:24.704687+0000 mgr.y (mgr.26054) 1585 : cluster 0 pgmap v1773: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:26.690 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:26 smithi098 bash[76817]: audit 2023-09-23T19:10:25.729906+0000 mon.c (mon.2) 161 : audit 1 from='client.? 172.21.15.98:0/3499899226' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:10:26.690 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:26 smithi098 bash[76817]: audit 2023-09-23T19:10:25.730595+0000 mon.a (mon.0) 2180 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:10:26.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:26 smithi173 bash[59422]: cluster 2023-09-23T19:10:24.704687+0000 mgr.y (mgr.26054) 1585 : cluster 0 pgmap v1773: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:26.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:26 smithi173 bash[59422]: audit 2023-09-23T19:10:25.729906+0000 mon.c (mon.2) 161 : audit 1 from='client.? 172.21.15.98:0/3499899226' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:10:26.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:26 smithi173 bash[59422]: audit 2023-09-23T19:10:25.730595+0000 mon.a (mon.0) 2180 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:10:27.661 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T19:10:27.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:27 smithi098 bash[79516]: audit 2023-09-23T19:10:26.634575+0000 mon.a (mon.0) 2181 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:10:27.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:27 smithi098 bash[79516]: cluster 2023-09-23T19:10:26.634696+0000 mon.a (mon.0) 2182 : cluster 0 osdmap e393: 8 total, 8 up, 7 in 2023-09-23T19:10:27.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:27 smithi098 bash[79516]: cluster 2023-09-23T19:10:26.705345+0000 mgr.y (mgr.26054) 1586 : cluster 0 pgmap v1775: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:27.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:27 smithi098 bash[79516]: cluster 2023-09-23T19:10:27.324549+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e393 with expected crc 2023-09-23T19:10:27.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:27 smithi098 bash[79516]: cluster 2023-09-23T19:10:27.324590+0000 osd.7 (osd.7) 35 : cluster 3 failed to encode map e393 with expected crc 2023-09-23T19:10:27.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:27 smithi098 bash[76817]: audit 2023-09-23T19:10:26.634575+0000 mon.a (mon.0) 2181 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:10:27.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:27 smithi098 bash[76817]: cluster 2023-09-23T19:10:26.634696+0000 mon.a (mon.0) 2182 : cluster 0 osdmap e393: 8 total, 8 up, 7 in 2023-09-23T19:10:27.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:27 smithi098 bash[76817]: cluster 2023-09-23T19:10:26.705345+0000 mgr.y (mgr.26054) 1586 : cluster 0 pgmap v1775: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:27.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:27 smithi098 bash[76817]: cluster 2023-09-23T19:10:27.324549+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e393 with expected crc 2023-09-23T19:10:27.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:27 smithi098 bash[76817]: cluster 2023-09-23T19:10:27.324590+0000 osd.7 (osd.7) 35 : cluster 3 failed to encode map e393 with expected crc 2023-09-23T19:10:28.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:27 smithi173 bash[59422]: audit 2023-09-23T19:10:26.634575+0000 mon.a (mon.0) 2181 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:10:28.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:27 smithi173 bash[59422]: cluster 2023-09-23T19:10:26.634696+0000 mon.a (mon.0) 2182 : cluster 0 osdmap e393: 8 total, 8 up, 7 in 2023-09-23T19:10:28.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:27 smithi173 bash[59422]: cluster 2023-09-23T19:10:26.705345+0000 mgr.y (mgr.26054) 1586 : cluster 0 pgmap v1775: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:28.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:27 smithi173 bash[59422]: cluster 2023-09-23T19:10:27.324549+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e393 with expected crc 2023-09-23T19:10:28.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:27 smithi173 bash[59422]: cluster 2023-09-23T19:10:27.324590+0000 osd.7 (osd.7) 35 : cluster 3 failed to encode map e393 with expected crc 2023-09-23T19:10:28.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:28 smithi098 bash[79516]: cluster 2023-09-23T19:10:27.043105+0000 osd.4 (osd.4) 120 : cluster 3 failed to encode map e393 with expected crc 2023-09-23T19:10:28.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:28 smithi098 bash[79516]: cluster 2023-09-23T19:10:27.046047+0000 osd.4 (osd.4) 121 : cluster 3 failed to encode map e393 with expected crc 2023-09-23T19:10:28.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:28 smithi098 bash[79516]: cluster 2023-09-23T19:10:27.048917+0000 osd.4 (osd.4) 122 : cluster 3 failed to encode map e393 with expected crc 2023-09-23T19:10:28.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:28 smithi098 bash[79516]: cluster 2023-09-23T19:10:27.324306+0000 osd.6 (osd.6) 381 : cluster 3 failed to encode map e393 with expected crc 2023-09-23T19:10:28.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:28 smithi098 bash[76817]: cluster 2023-09-23T19:10:27.043105+0000 osd.4 (osd.4) 120 : cluster 3 failed to encode map e393 with expected crc 2023-09-23T19:10:28.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:28 smithi098 bash[76817]: cluster 2023-09-23T19:10:27.046047+0000 osd.4 (osd.4) 121 : cluster 3 failed to encode map e393 with expected crc 2023-09-23T19:10:28.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:28 smithi098 bash[76817]: cluster 2023-09-23T19:10:27.048917+0000 osd.4 (osd.4) 122 : cluster 3 failed to encode map e393 with expected crc 2023-09-23T19:10:28.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:28 smithi098 bash[76817]: cluster 2023-09-23T19:10:27.324306+0000 osd.6 (osd.6) 381 : cluster 3 failed to encode map e393 with expected crc 2023-09-23T19:10:29.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:28 smithi173 bash[59422]: cluster 2023-09-23T19:10:27.043105+0000 osd.4 (osd.4) 120 : cluster 3 failed to encode map e393 with expected crc 2023-09-23T19:10:29.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:28 smithi173 bash[59422]: cluster 2023-09-23T19:10:27.046047+0000 osd.4 (osd.4) 121 : cluster 3 failed to encode map e393 with expected crc 2023-09-23T19:10:29.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:28 smithi173 bash[59422]: cluster 2023-09-23T19:10:27.048917+0000 osd.4 (osd.4) 122 : cluster 3 failed to encode map e393 with expected crc 2023-09-23T19:10:29.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:28 smithi173 bash[59422]: cluster 2023-09-23T19:10:27.324306+0000 osd.6 (osd.6) 381 : cluster 3 failed to encode map e393 with expected crc 2023-09-23T19:10:29.864 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:10:29.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:29 smithi098 bash[76817]: cluster 2023-09-23T19:10:28.706204+0000 mgr.y (mgr.26054) 1587 : cluster 0 pgmap v1776: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:29.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:29 smithi098 bash[79516]: cluster 2023-09-23T19:10:28.706204+0000 mgr.y (mgr.26054) 1587 : cluster 0 pgmap v1776: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:30.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:29 smithi173 bash[59422]: cluster 2023-09-23T19:10:28.706204+0000 mgr.y (mgr.26054) 1587 : cluster 0 pgmap v1776: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:31.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:31 smithi173 bash[59422]: cluster 2023-09-23T19:10:30.706979+0000 mgr.y (mgr.26054) 1588 : cluster 0 pgmap v1777: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:31.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:31 smithi098 bash[79516]: cluster 2023-09-23T19:10:30.706979+0000 mgr.y (mgr.26054) 1588 : cluster 0 pgmap v1777: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:31.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:31 smithi098 bash[76817]: cluster 2023-09-23T19:10:30.706979+0000 mgr.y (mgr.26054) 1588 : cluster 0 pgmap v1777: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:32.436 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:10:33.246 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T19:10:33.246 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-09-23T19:10:33.246 INFO:teuthology.orchestra.run.smithi173.stdout: "complaint_time": 30, 2023-09-23T19:10:33.246 INFO:teuthology.orchestra.run.smithi173.stdout: "num_blocked_ops": 0 2023-09-23T19:10:33.246 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T19:10:33.918 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:33 smithi098 bash[76817]: cluster 2023-09-23T19:10:32.707865+0000 mgr.y (mgr.26054) 1589 : cluster 0 pgmap v1778: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:33.919 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:33 smithi098 bash[79516]: cluster 2023-09-23T19:10:32.707865+0000 mgr.y (mgr.26054) 1589 : cluster 0 pgmap v1778: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:34.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:33 smithi173 bash[59422]: cluster 2023-09-23T19:10:32.707865+0000 mgr.y (mgr.26054) 1589 : cluster 0 pgmap v1778: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:34.153 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T19:10:36.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:35 smithi173 bash[59422]: cluster 2023-09-23T19:10:34.708509+0000 mgr.y (mgr.26054) 1590 : cluster 0 pgmap v1779: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:36.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:35 smithi098 bash[76817]: cluster 2023-09-23T19:10:34.708509+0000 mgr.y (mgr.26054) 1590 : cluster 0 pgmap v1779: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:36.087 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:35 smithi098 bash[79516]: cluster 2023-09-23T19:10:34.708509+0000 mgr.y (mgr.26054) 1590 : cluster 0 pgmap v1779: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:36.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:10:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:10:36] "GET /metrics HTTP/1.1" 200 34184 "" "Prometheus/2.33.4" 2023-09-23T19:10:36.795 INFO:teuthology.orchestra.run.smithi098.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T19:10:37.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:36 smithi173 bash[59422]: audit 2023-09-23T19:10:36.212916+0000 mon.c (mon.2) 162 : audit 1 from='client.? 172.21.15.98:0/3310862261' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:10:37.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:36 smithi173 bash[59422]: audit 2023-09-23T19:10:36.213533+0000 mon.a (mon.0) 2183 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:10:37.086 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:36 smithi098 bash[79516]: audit 2023-09-23T19:10:36.212916+0000 mon.c (mon.2) 162 : audit 1 from='client.? 172.21.15.98:0/3310862261' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:10:37.086 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:36 smithi098 bash[79516]: audit 2023-09-23T19:10:36.213533+0000 mon.a (mon.0) 2183 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:10:37.086 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:36 smithi098 bash[76817]: audit 2023-09-23T19:10:36.212916+0000 mon.c (mon.2) 162 : audit 1 from='client.? 172.21.15.98:0/3310862261' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:10:37.086 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:36 smithi098 bash[76817]: audit 2023-09-23T19:10:36.213533+0000 mon.a (mon.0) 2183 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:10:37.817 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T19:10:38.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:37 smithi098 bash[76817]: cluster 2023-09-23T19:10:36.709405+0000 mgr.y (mgr.26054) 1591 : cluster 0 pgmap v1780: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:38.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:37 smithi098 bash[76817]: audit 2023-09-23T19:10:36.780244+0000 mon.a (mon.0) 2184 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:10:38.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:37 smithi098 bash[76817]: cluster 2023-09-23T19:10:36.780338+0000 mon.a (mon.0) 2185 : cluster 0 osdmap e394: 8 total, 8 up, 7 in 2023-09-23T19:10:38.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:37 smithi098 bash[76817]: cluster 2023-09-23T19:10:36.784464+0000 osd.4 (osd.4) 123 : cluster 3 failed to encode map e394 with expected crc 2023-09-23T19:10:38.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:37 smithi098 bash[76817]: cluster 2023-09-23T19:10:36.785107+0000 osd.6 (osd.6) 382 : cluster 3 failed to encode map e394 with expected crc 2023-09-23T19:10:38.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:37 smithi098 bash[76817]: cluster 2023-09-23T19:10:36.785416+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e394 with expected crc 2023-09-23T19:10:38.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:37 smithi098 bash[76817]: cluster 2023-09-23T19:10:36.787670+0000 osd.7 (osd.7) 36 : cluster 3 failed to encode map e394 with expected crc 2023-09-23T19:10:38.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:37 smithi098 bash[76817]: cluster 2023-09-23T19:10:36.788238+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e394 with expected crc 2023-09-23T19:10:38.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:37 smithi098 bash[76817]: cluster 2023-09-23T19:10:36.789772+0000 osd.6 (osd.6) 383 : cluster 3 failed to encode map e394 with expected crc 2023-09-23T19:10:38.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:37 smithi098 bash[79516]: cluster 2023-09-23T19:10:36.709405+0000 mgr.y (mgr.26054) 1591 : cluster 0 pgmap v1780: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:38.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:37 smithi098 bash[79516]: audit 2023-09-23T19:10:36.780244+0000 mon.a (mon.0) 2184 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:10:38.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:37 smithi098 bash[79516]: cluster 2023-09-23T19:10:36.780338+0000 mon.a (mon.0) 2185 : cluster 0 osdmap e394: 8 total, 8 up, 7 in 2023-09-23T19:10:38.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:37 smithi098 bash[79516]: cluster 2023-09-23T19:10:36.784464+0000 osd.4 (osd.4) 123 : cluster 3 failed to encode map e394 with expected crc 2023-09-23T19:10:38.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:37 smithi098 bash[79516]: cluster 2023-09-23T19:10:36.785107+0000 osd.6 (osd.6) 382 : cluster 3 failed to encode map e394 with expected crc 2023-09-23T19:10:38.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:37 smithi098 bash[79516]: cluster 2023-09-23T19:10:36.785416+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e394 with expected crc 2023-09-23T19:10:38.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:37 smithi098 bash[79516]: cluster 2023-09-23T19:10:36.787670+0000 osd.7 (osd.7) 36 : cluster 3 failed to encode map e394 with expected crc 2023-09-23T19:10:38.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:37 smithi098 bash[79516]: cluster 2023-09-23T19:10:36.788238+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e394 with expected crc 2023-09-23T19:10:38.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:37 smithi098 bash[79516]: cluster 2023-09-23T19:10:36.789772+0000 osd.6 (osd.6) 383 : cluster 3 failed to encode map e394 with expected crc 2023-09-23T19:10:38.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:37 smithi173 bash[59422]: cluster 2023-09-23T19:10:36.709405+0000 mgr.y (mgr.26054) 1591 : cluster 0 pgmap v1780: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:38.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:37 smithi173 bash[59422]: audit 2023-09-23T19:10:36.780244+0000 mon.a (mon.0) 2184 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:10:38.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:37 smithi173 bash[59422]: cluster 2023-09-23T19:10:36.780338+0000 mon.a (mon.0) 2185 : cluster 0 osdmap e394: 8 total, 8 up, 7 in 2023-09-23T19:10:38.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:37 smithi173 bash[59422]: cluster 2023-09-23T19:10:36.784464+0000 osd.4 (osd.4) 123 : cluster 3 failed to encode map e394 with expected crc 2023-09-23T19:10:38.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:37 smithi173 bash[59422]: cluster 2023-09-23T19:10:36.785107+0000 osd.6 (osd.6) 382 : cluster 3 failed to encode map e394 with expected crc 2023-09-23T19:10:38.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:37 smithi173 bash[59422]: cluster 2023-09-23T19:10:36.785416+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e394 with expected crc 2023-09-23T19:10:38.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:37 smithi173 bash[59422]: cluster 2023-09-23T19:10:36.787670+0000 osd.7 (osd.7) 36 : cluster 3 failed to encode map e394 with expected crc 2023-09-23T19:10:38.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:37 smithi173 bash[59422]: cluster 2023-09-23T19:10:36.788238+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e394 with expected crc 2023-09-23T19:10:38.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:37 smithi173 bash[59422]: cluster 2023-09-23T19:10:36.789772+0000 osd.6 (osd.6) 383 : cluster 3 failed to encode map e394 with expected crc 2023-09-23T19:10:38.925 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:10:39.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:38 smithi098 bash[79516]: cluster 2023-09-23T19:10:37.783041+0000 mon.a (mon.0) 2186 : cluster 0 osdmap e395: 8 total, 8 up, 7 in 2023-09-23T19:10:39.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:38 smithi098 bash[79516]: cluster 2023-09-23T19:10:37.784188+0000 osd.7 (osd.7) 37 : cluster 3 failed to encode map e395 with expected crc 2023-09-23T19:10:39.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:38 smithi098 bash[79516]: cluster 2023-09-23T19:10:37.786771+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e395 with expected crc 2023-09-23T19:10:39.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:38 smithi098 bash[79516]: cluster 2023-09-23T19:10:37.786899+0000 osd.7 (osd.7) 38 : cluster 3 failed to encode map e395 with expected crc 2023-09-23T19:10:39.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:38 smithi098 bash[79516]: cluster 2023-09-23T19:10:37.789806+0000 osd.5 (osd.5) 36 : cluster 3 failed to encode map e395 with expected crc 2023-09-23T19:10:39.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:38 smithi098 bash[79516]: cluster 2023-09-23T19:10:37.792573+0000 osd.5 (osd.5) 37 : cluster 3 failed to encode map e395 with expected crc 2023-09-23T19:10:39.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:38 smithi098 bash[79516]: cluster 2023-09-23T19:10:37.795208+0000 osd.5 (osd.5) 38 : cluster 3 failed to encode map e395 with expected crc 2023-09-23T19:10:39.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:38 smithi098 bash[79516]: cluster 2023-09-23T19:10:37.947763+0000 osd.4 (osd.4) 124 : cluster 3 failed to encode map e395 with expected crc 2023-09-23T19:10:39.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:38 smithi098 bash[76817]: cluster 2023-09-23T19:10:37.783041+0000 mon.a (mon.0) 2186 : cluster 0 osdmap e395: 8 total, 8 up, 7 in 2023-09-23T19:10:39.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:38 smithi098 bash[76817]: cluster 2023-09-23T19:10:37.784188+0000 osd.7 (osd.7) 37 : cluster 3 failed to encode map e395 with expected crc 2023-09-23T19:10:39.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:38 smithi098 bash[76817]: cluster 2023-09-23T19:10:37.786771+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e395 with expected crc 2023-09-23T19:10:39.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:38 smithi098 bash[76817]: cluster 2023-09-23T19:10:37.786899+0000 osd.7 (osd.7) 38 : cluster 3 failed to encode map e395 with expected crc 2023-09-23T19:10:39.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:38 smithi098 bash[76817]: cluster 2023-09-23T19:10:37.789806+0000 osd.5 (osd.5) 36 : cluster 3 failed to encode map e395 with expected crc 2023-09-23T19:10:39.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:38 smithi098 bash[76817]: cluster 2023-09-23T19:10:37.792573+0000 osd.5 (osd.5) 37 : cluster 3 failed to encode map e395 with expected crc 2023-09-23T19:10:39.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:38 smithi098 bash[76817]: cluster 2023-09-23T19:10:37.795208+0000 osd.5 (osd.5) 38 : cluster 3 failed to encode map e395 with expected crc 2023-09-23T19:10:39.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:38 smithi098 bash[76817]: cluster 2023-09-23T19:10:37.947763+0000 osd.4 (osd.4) 124 : cluster 3 failed to encode map e395 with expected crc 2023-09-23T19:10:39.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:38 smithi173 bash[59422]: cluster 2023-09-23T19:10:37.783041+0000 mon.a (mon.0) 2186 : cluster 0 osdmap e395: 8 total, 8 up, 7 in 2023-09-23T19:10:39.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:38 smithi173 bash[59422]: cluster 2023-09-23T19:10:37.784188+0000 osd.7 (osd.7) 37 : cluster 3 failed to encode map e395 with expected crc 2023-09-23T19:10:39.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:38 smithi173 bash[59422]: cluster 2023-09-23T19:10:37.786771+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e395 with expected crc 2023-09-23T19:10:39.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:38 smithi173 bash[59422]: cluster 2023-09-23T19:10:37.786899+0000 osd.7 (osd.7) 38 : cluster 3 failed to encode map e395 with expected crc 2023-09-23T19:10:39.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:38 smithi173 bash[59422]: cluster 2023-09-23T19:10:37.789806+0000 osd.5 (osd.5) 36 : cluster 3 failed to encode map e395 with expected crc 2023-09-23T19:10:39.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:38 smithi173 bash[59422]: cluster 2023-09-23T19:10:37.792573+0000 osd.5 (osd.5) 37 : cluster 3 failed to encode map e395 with expected crc 2023-09-23T19:10:39.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:38 smithi173 bash[59422]: cluster 2023-09-23T19:10:37.795208+0000 osd.5 (osd.5) 38 : cluster 3 failed to encode map e395 with expected crc 2023-09-23T19:10:39.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:38 smithi173 bash[59422]: cluster 2023-09-23T19:10:37.947763+0000 osd.4 (osd.4) 124 : cluster 3 failed to encode map e395 with expected crc 2023-09-23T19:10:40.084 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:39 smithi173 bash[59422]: cluster 2023-09-23T19:10:38.710048+0000 mgr.y (mgr.26054) 1592 : cluster 0 pgmap v1783: 73 pgs: 3 peering, 70 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:40.084 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:39 smithi173 bash[59422]: audit 2023-09-23T19:10:39.653735+0000 mon.a (mon.0) 2187 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:10:40.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:39 smithi098 bash[79516]: cluster 2023-09-23T19:10:38.710048+0000 mgr.y (mgr.26054) 1592 : cluster 0 pgmap v1783: 73 pgs: 3 peering, 70 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:40.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:39 smithi098 bash[79516]: audit 2023-09-23T19:10:39.653735+0000 mon.a (mon.0) 2187 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:10:40.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:39 smithi098 bash[76817]: cluster 2023-09-23T19:10:38.710048+0000 mgr.y (mgr.26054) 1592 : cluster 0 pgmap v1783: 73 pgs: 3 peering, 70 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:40.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:39 smithi098 bash[76817]: audit 2023-09-23T19:10:39.653735+0000 mon.a (mon.0) 2187 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:10:41.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:41 smithi098 bash[76817]: cluster 2023-09-23T19:10:40.710896+0000 mgr.y (mgr.26054) 1593 : cluster 0 pgmap v1784: 73 pgs: 3 peering, 70 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 8 B/s, 0 objects/s recovering 2023-09-23T19:10:41.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:41 smithi098 bash[79516]: cluster 2023-09-23T19:10:40.710896+0000 mgr.y (mgr.26054) 1593 : cluster 0 pgmap v1784: 73 pgs: 3 peering, 70 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 8 B/s, 0 objects/s recovering 2023-09-23T19:10:42.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:41 smithi173 bash[59422]: cluster 2023-09-23T19:10:40.710896+0000 mgr.y (mgr.26054) 1593 : cluster 0 pgmap v1784: 73 pgs: 3 peering, 70 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 8 B/s, 0 objects/s recovering 2023-09-23T19:10:42.317 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T19:10:42.317 INFO:teuthology.orchestra.run.smithi173.stdout: "size": 20, 2023-09-23T19:10:42.317 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 600, 2023-09-23T19:10:42.317 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [] 2023-09-23T19:10:42.317 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T19:10:42.592 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:10:43.192 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T19:10:44.026 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:43 smithi098 bash[76817]: cluster 2023-09-23T19:10:42.711819+0000 mgr.y (mgr.26054) 1594 : cluster 0 pgmap v1785: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 8 B/s, 0 objects/s recovering 2023-09-23T19:10:44.026 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:43 smithi098 bash[79516]: cluster 2023-09-23T19:10:42.711819+0000 mgr.y (mgr.26054) 1594 : cluster 0 pgmap v1785: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 8 B/s, 0 objects/s recovering 2023-09-23T19:10:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:43 smithi173 bash[59422]: cluster 2023-09-23T19:10:42.711819+0000 mgr.y (mgr.26054) 1594 : cluster 0 pgmap v1785: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 8 B/s, 0 objects/s recovering 2023-09-23T19:10:46.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:45 smithi173 bash[59422]: cluster 2023-09-23T19:10:44.712416+0000 mgr.y (mgr.26054) 1595 : cluster 0 pgmap v1786: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 8 B/s, 0 objects/s recovering 2023-09-23T19:10:46.034 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:45 smithi098 bash[79516]: cluster 2023-09-23T19:10:44.712416+0000 mgr.y (mgr.26054) 1595 : cluster 0 pgmap v1786: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 8 B/s, 0 objects/s recovering 2023-09-23T19:10:46.034 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:45 smithi098 bash[76817]: cluster 2023-09-23T19:10:44.712416+0000 mgr.y (mgr.26054) 1595 : cluster 0 pgmap v1786: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 8 B/s, 0 objects/s recovering 2023-09-23T19:10:46.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:10:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:10:46] "GET /metrics HTTP/1.1" 200 34164 "" "Prometheus/2.33.4" 2023-09-23T19:10:46.796 INFO:teuthology.orchestra.run.smithi098.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T19:10:47.060 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:46 smithi098 bash[76817]: audit 2023-09-23T19:10:46.416461+0000 mon.a (mon.0) 2188 : audit 1 from='client.? 172.21.15.98:0/1519450837' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:10:47.060 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:46 smithi098 bash[79516]: audit 2023-09-23T19:10:46.416461+0000 mon.a (mon.0) 2188 : audit 1 from='client.? 172.21.15.98:0/1519450837' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:10:47.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:46 smithi173 bash[59422]: audit 2023-09-23T19:10:46.416461+0000 mon.a (mon.0) 2188 : audit 1 from='client.? 172.21.15.98:0/1519450837' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:10:47.817 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T19:10:47.817 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:10:47.979 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:10:48.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:47 smithi098 bash[76817]: cluster 2023-09-23T19:10:46.713386+0000 mgr.y (mgr.26054) 1596 : cluster 0 pgmap v1787: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 7 B/s, 0 objects/s recovering 2023-09-23T19:10:48.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:47 smithi098 bash[76817]: audit 2023-09-23T19:10:46.786706+0000 mon.a (mon.0) 2189 : audit 1 from='client.? 172.21.15.98:0/1519450837' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:10:48.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:47 smithi098 bash[76817]: cluster 2023-09-23T19:10:46.786832+0000 mon.a (mon.0) 2190 : cluster 0 osdmap e396: 8 total, 8 up, 7 in 2023-09-23T19:10:48.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:47 smithi098 bash[76817]: cluster 2023-09-23T19:10:46.786989+0000 osd.6 (osd.6) 384 : cluster 3 failed to encode map e396 with expected crc 2023-09-23T19:10:48.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:47 smithi098 bash[76817]: cluster 2023-09-23T19:10:46.787013+0000 osd.7 (osd.7) 39 : cluster 3 failed to encode map e396 with expected crc 2023-09-23T19:10:48.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:47 smithi098 bash[76817]: cluster 2023-09-23T19:10:46.790350+0000 osd.4 (osd.4) 125 : cluster 3 failed to encode map e396 with expected crc 2023-09-23T19:10:48.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:47 smithi098 bash[79516]: cluster 2023-09-23T19:10:46.713386+0000 mgr.y (mgr.26054) 1596 : cluster 0 pgmap v1787: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 7 B/s, 0 objects/s recovering 2023-09-23T19:10:48.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:47 smithi098 bash[79516]: audit 2023-09-23T19:10:46.786706+0000 mon.a (mon.0) 2189 : audit 1 from='client.? 172.21.15.98:0/1519450837' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:10:48.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:47 smithi098 bash[79516]: cluster 2023-09-23T19:10:46.786832+0000 mon.a (mon.0) 2190 : cluster 0 osdmap e396: 8 total, 8 up, 7 in 2023-09-23T19:10:48.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:47 smithi098 bash[79516]: cluster 2023-09-23T19:10:46.786989+0000 osd.6 (osd.6) 384 : cluster 3 failed to encode map e396 with expected crc 2023-09-23T19:10:48.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:47 smithi098 bash[79516]: cluster 2023-09-23T19:10:46.787013+0000 osd.7 (osd.7) 39 : cluster 3 failed to encode map e396 with expected crc 2023-09-23T19:10:48.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:47 smithi098 bash[79516]: cluster 2023-09-23T19:10:46.790350+0000 osd.4 (osd.4) 125 : cluster 3 failed to encode map e396 with expected crc 2023-09-23T19:10:48.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:47 smithi173 bash[59422]: cluster 2023-09-23T19:10:46.713386+0000 mgr.y (mgr.26054) 1596 : cluster 0 pgmap v1787: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 7 B/s, 0 objects/s recovering 2023-09-23T19:10:48.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:47 smithi173 bash[59422]: audit 2023-09-23T19:10:46.786706+0000 mon.a (mon.0) 2189 : audit 1 from='client.? 172.21.15.98:0/1519450837' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:10:48.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:47 smithi173 bash[59422]: cluster 2023-09-23T19:10:46.786832+0000 mon.a (mon.0) 2190 : cluster 0 osdmap e396: 8 total, 8 up, 7 in 2023-09-23T19:10:48.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:47 smithi173 bash[59422]: cluster 2023-09-23T19:10:46.786989+0000 osd.6 (osd.6) 384 : cluster 3 failed to encode map e396 with expected crc 2023-09-23T19:10:48.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:47 smithi173 bash[59422]: cluster 2023-09-23T19:10:46.787013+0000 osd.7 (osd.7) 39 : cluster 3 failed to encode map e396 with expected crc 2023-09-23T19:10:48.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:47 smithi173 bash[59422]: cluster 2023-09-23T19:10:46.790350+0000 osd.4 (osd.4) 125 : cluster 3 failed to encode map e396 with expected crc 2023-09-23T19:10:50.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:49 smithi173 bash[59422]: cluster 2023-09-23T19:10:48.714484+0000 mgr.y (mgr.26054) 1597 : cluster 0 pgmap v1789: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 6 B/s, 0 objects/s recovering 2023-09-23T19:10:50.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:49 smithi098 bash[76817]: cluster 2023-09-23T19:10:48.714484+0000 mgr.y (mgr.26054) 1597 : cluster 0 pgmap v1789: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 6 B/s, 0 objects/s recovering 2023-09-23T19:10:50.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:49 smithi098 bash[79516]: cluster 2023-09-23T19:10:48.714484+0000 mgr.y (mgr.26054) 1597 : cluster 0 pgmap v1789: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 6 B/s, 0 objects/s recovering 2023-09-23T19:10:51.354 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T19:10:51.354 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-09-23T19:10:51.354 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 0 2023-09-23T19:10:51.355 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T19:10:51.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:51 smithi173 bash[59422]: audit 2023-09-23T19:10:50.613157+0000 mon.a (mon.0) 2191 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:10:51.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:51 smithi173 bash[59422]: cluster 2023-09-23T19:10:50.715274+0000 mgr.y (mgr.26054) 1598 : cluster 0 pgmap v1790: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:51.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:51 smithi098 bash[76817]: audit 2023-09-23T19:10:50.613157+0000 mon.a (mon.0) 2191 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:10:51.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:51 smithi098 bash[76817]: cluster 2023-09-23T19:10:50.715274+0000 mgr.y (mgr.26054) 1598 : cluster 0 pgmap v1790: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:51.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:51 smithi098 bash[79516]: audit 2023-09-23T19:10:50.613157+0000 mon.a (mon.0) 2191 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:10:51.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:51 smithi098 bash[79516]: cluster 2023-09-23T19:10:50.715274+0000 mgr.y (mgr.26054) 1598 : cluster 0 pgmap v1790: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:52.269 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T19:10:52.594 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:10:54.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:53 smithi173 bash[59422]: cluster 2023-09-23T19:10:52.716238+0000 mgr.y (mgr.26054) 1599 : cluster 0 pgmap v1791: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:54.125 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:53 smithi098 bash[76817]: cluster 2023-09-23T19:10:52.716238+0000 mgr.y (mgr.26054) 1599 : cluster 0 pgmap v1791: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:54.125 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:53 smithi098 bash[79516]: cluster 2023-09-23T19:10:52.716238+0000 mgr.y (mgr.26054) 1599 : cluster 0 pgmap v1791: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:55.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:54 smithi098 bash[76817]: audit 2023-09-23T19:10:54.654157+0000 mon.a (mon.0) 2192 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:10:55.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:54 smithi098 bash[79516]: audit 2023-09-23T19:10:54.654157+0000 mon.a (mon.0) 2192 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:10:55.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:54 smithi173 bash[59422]: audit 2023-09-23T19:10:54.654157+0000 mon.a (mon.0) 2192 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:10:56.086 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:55 smithi098 bash[79516]: cluster 2023-09-23T19:10:54.716778+0000 mgr.y (mgr.26054) 1600 : cluster 0 pgmap v1792: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:56.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:55 smithi098 bash[76817]: cluster 2023-09-23T19:10:54.716778+0000 mgr.y (mgr.26054) 1600 : cluster 0 pgmap v1792: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:56.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:55 smithi173 bash[59422]: cluster 2023-09-23T19:10:54.716778+0000 mgr.y (mgr.26054) 1600 : cluster 0 pgmap v1792: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:56.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:10:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:10:56] "GET /metrics HTTP/1.1" 200 34165 "" "Prometheus/2.33.4" 2023-09-23T19:10:56.486 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T19:10:56.490 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-23T19:10:57.046 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:10:57.079 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:56 smithi173 bash[59422]: audit 2023-09-23T19:10:56.465763+0000 mgr.y (mgr.26054) 1601 : audit 0 from='client.36825 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:10:57.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:56 smithi098 bash[79516]: audit 2023-09-23T19:10:56.465763+0000 mgr.y (mgr.26054) 1601 : audit 0 from='client.36825 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:10:57.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:56 smithi098 bash[76817]: audit 2023-09-23T19:10:56.465763+0000 mgr.y (mgr.26054) 1601 : audit 0 from='client.36825 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:10:57.554 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":1792,"stamp":"2023-09-23T19:10:54.716484+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":12228,"num_read_kb":26865,"num_write":1950,"num_write_kb":27871,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":164,"num_bytes_recovered":12591255,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2144,"ondisk_log_size":2144,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":221,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5889976,"kb_used_data":288000,"kb_used_omap":51,"kb_used_meta":5601868,"kb_avail":650326088,"statfs":{"total":671965249536,"available":665933914112,"internally_reserved":0,"allocated":294912000,"data_stored":284528724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":52652,"internal_metadata":5736313428},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004416"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":107,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.788298+0000","last_change":"2023-09-23T18:59:13.709108+0000","last_active":"2023-09-23T19:10:37.788298+0000","last_peered":"2023-09-23T19:10:37.788298+0000","last_clean":"2023-09-23T19:10:37.788298+0000","last_became_active":"2023-09-23T18:59:13.708511+0000","last_became_peered":"2023-09-23T18:59:13.708511+0000","last_unstale":"2023-09-23T19:10:37.788298+0000","last_undegraded":"2023-09-23T19:10:37.788298+0000","last_fullsized":"2023-09-23T19:10:37.788298+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:59.295137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":121,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.785678+0000","last_change":"2023-09-23T18:59:13.702575+0000","last_active":"2023-09-23T19:10:37.785678+0000","last_peered":"2023-09-23T19:10:37.785678+0000","last_clean":"2023-09-23T19:10:37.785678+0000","last_became_active":"2023-09-23T18:59:13.701967+0000","last_became_peered":"2023-09-23T18:59:13.701967+0000","last_unstale":"2023-09-23T19:10:37.785678+0000","last_undegraded":"2023-09-23T19:10:37.785678+0000","last_fullsized":"2023-09-23T19:10:37.785678+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:30:12.982518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":388,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.787431+0000","last_change":"2023-09-23T18:59:13.707831+0000","last_active":"2023-09-23T19:10:37.787431+0000","last_peered":"2023-09-23T19:10:37.787431+0000","last_clean":"2023-09-23T19:10:37.787431+0000","last_became_active":"2023-09-23T18:59:13.707524+0000","last_became_peered":"2023-09-23T18:59:13.707524+0000","last_unstale":"2023-09-23T19:10:37.787431+0000","last_undegraded":"2023-09-23T19:10:37.787431+0000","last_fullsized":"2023-09-23T19:10:37.787431+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:42.677892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":442,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:45.319707+0000","last_change":"2023-09-23T19:09:03.326411+0000","last_active":"2023-09-23T19:10:45.319707+0000","last_peered":"2023-09-23T19:10:45.319707+0000","last_clean":"2023-09-23T19:10:45.319707+0000","last_became_active":"2023-09-23T19:09:03.325980+0000","last_became_peered":"2023-09-23T19:09:03.325980+0000","last_unstale":"2023-09-23T19:10:45.319707+0000","last_undegraded":"2023-09-23T19:10:45.319707+0000","last_fullsized":"2023-09-23T19:10:45.319707+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:43.638573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":113,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T19:10:47.136613+0000","last_change":"2023-09-23T18:58:38.718387+0000","last_active":"2023-09-23T19:10:47.136613+0000","last_peered":"2023-09-23T19:10:47.136613+0000","last_clean":"2023-09-23T19:10:47.136613+0000","last_became_active":"2023-09-23T18:58:38.717999+0000","last_became_peered":"2023-09-23T18:58:38.717999+0000","last_unstale":"2023-09-23T19:10:47.136613+0000","last_undegraded":"2023-09-23T19:10:47.136613+0000","last_fullsized":"2023-09-23T19:10:47.136613+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:05.073213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":99,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.802467+0000","last_change":"2023-09-23T19:08:37.780028+0000","last_active":"2023-09-23T19:10:37.802467+0000","last_peered":"2023-09-23T19:10:37.802467+0000","last_clean":"2023-09-23T19:10:37.802467+0000","last_became_active":"2023-09-23T19:08:37.779430+0000","last_became_peered":"2023-09-23T19:08:37.779430+0000","last_unstale":"2023-09-23T19:10:37.802467+0000","last_undegraded":"2023-09-23T19:10:37.802467+0000","last_fullsized":"2023-09-23T19:10:37.802467+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":94,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.802529+0000","last_change":"2023-09-23T19:08:37.779252+0000","last_active":"2023-09-23T19:10:37.802529+0000","last_peered":"2023-09-23T19:10:37.802529+0000","last_clean":"2023-09-23T19:10:37.802529+0000","last_became_active":"2023-09-23T19:08:37.778722+0000","last_became_peered":"2023-09-23T19:08:37.778722+0000","last_unstale":"2023-09-23T19:10:37.802529+0000","last_undegraded":"2023-09-23T19:10:37.802529+0000","last_fullsized":"2023-09-23T19:10:37.802529+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":113,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.785712+0000","last_change":"2023-09-23T18:58:38.710920+0000","last_active":"2023-09-23T19:10:37.785712+0000","last_peered":"2023-09-23T19:10:37.785712+0000","last_clean":"2023-09-23T19:10:37.785712+0000","last_became_active":"2023-09-23T18:58:38.710563+0000","last_became_peered":"2023-09-23T18:58:38.710563+0000","last_unstale":"2023-09-23T19:10:37.785712+0000","last_undegraded":"2023-09-23T19:10:37.785712+0000","last_fullsized":"2023-09-23T19:10:37.785712+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:42.386631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":11,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.803136+0000","last_change":"2023-09-23T19:10:37.800599+0000","last_active":"2023-09-23T19:10:37.803136+0000","last_peered":"2023-09-23T19:10:37.803136+0000","last_clean":"2023-09-23T19:10:37.803136+0000","last_became_active":"2023-09-23T19:10:37.800130+0000","last_became_peered":"2023-09-23T19:10:37.800130+0000","last_unstale":"2023-09-23T19:10:37.803136+0000","last_undegraded":"2023-09-23T19:10:37.803136+0000","last_fullsized":"2023-09-23T19:10:37.803136+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":388,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.790397+0000","last_change":"2023-09-23T19:09:42.642530+0000","last_active":"2023-09-23T19:10:37.790397+0000","last_peered":"2023-09-23T19:10:37.790397+0000","last_clean":"2023-09-23T19:10:37.790397+0000","last_became_active":"2023-09-23T19:09:42.641782+0000","last_became_peered":"2023-09-23T19:09:42.641782+0000","last_unstale":"2023-09-23T19:10:37.790397+0000","last_undegraded":"2023-09-23T19:10:37.790397+0000","last_fullsized":"2023-09-23T19:10:37.790397+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":278,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.791274+0000","last_change":"2023-09-23T19:09:03.320766+0000","last_active":"2023-09-23T19:10:37.791274+0000","last_peered":"2023-09-23T19:10:37.791274+0000","last_clean":"2023-09-23T19:10:37.791274+0000","last_became_active":"2023-09-23T19:09:03.320264+0000","last_became_peered":"2023-09-23T19:09:03.320264+0000","last_unstale":"2023-09-23T19:10:37.791274+0000","last_undegraded":"2023-09-23T19:10:37.791274+0000","last_fullsized":"2023-09-23T19:10:37.791274+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":342,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.787766+0000","last_change":"2023-09-23T18:33:53.850096+0000","last_active":"2023-09-23T19:10:37.787766+0000","last_peered":"2023-09-23T19:10:37.787766+0000","last_clean":"2023-09-23T19:10:37.787766+0000","last_became_active":"2023-09-23T18:33:53.849811+0000","last_became_peered":"2023-09-23T18:33:53.849811+0000","last_unstale":"2023-09-23T19:10:37.787766+0000","last_undegraded":"2023-09-23T19:10:37.787766+0000","last_fullsized":"2023-09-23T19:10:37.787766+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:44.938072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":104,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.790458+0000","last_change":"2023-09-23T19:09:03.313359+0000","last_active":"2023-09-23T19:10:37.790458+0000","last_peered":"2023-09-23T19:10:37.790458+0000","last_clean":"2023-09-23T19:10:37.790458+0000","last_became_active":"2023-09-23T19:09:03.312806+0000","last_became_peered":"2023-09-23T19:09:03.312806+0000","last_unstale":"2023-09-23T19:10:37.790458+0000","last_undegraded":"2023-09-23T19:10:37.790458+0000","last_fullsized":"2023-09-23T19:10:37.790458+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":354,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.787808+0000","last_change":"2023-09-23T19:08:37.769682+0000","last_active":"2023-09-23T19:10:37.787808+0000","last_peered":"2023-09-23T19:10:37.787808+0000","last_clean":"2023-09-23T19:10:37.787808+0000","last_became_active":"2023-09-23T19:08:37.768331+0000","last_became_peered":"2023-09-23T19:08:37.768331+0000","last_unstale":"2023-09-23T19:10:37.787808+0000","last_undegraded":"2023-09-23T19:10:37.787808+0000","last_fullsized":"2023-09-23T19:10:37.787808+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:50.441487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":277,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.790629+0000","last_change":"2023-09-23T19:09:03.314758+0000","last_active":"2023-09-23T19:10:37.790629+0000","last_peered":"2023-09-23T19:10:37.790629+0000","last_clean":"2023-09-23T19:10:37.790629+0000","last_became_active":"2023-09-23T19:09:03.314277+0000","last_became_peered":"2023-09-23T19:09:03.314277+0000","last_unstale":"2023-09-23T19:10:37.790629+0000","last_undegraded":"2023-09-23T19:10:37.790629+0000","last_fullsized":"2023-09-23T19:10:37.790629+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":414,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.785374+0000","last_change":"2023-09-23T19:09:03.319623+0000","last_active":"2023-09-23T19:10:37.785374+0000","last_peered":"2023-09-23T19:10:37.785374+0000","last_clean":"2023-09-23T19:10:37.785374+0000","last_became_active":"2023-09-23T19:09:03.319283+0000","last_became_peered":"2023-09-23T19:09:03.319283+0000","last_unstale":"2023-09-23T19:10:37.785374+0000","last_undegraded":"2023-09-23T19:10:37.785374+0000","last_fullsized":"2023-09-23T19:10:37.785374+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:34.646037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":165,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.789986+0000","last_change":"2023-09-23T19:09:03.326943+0000","last_active":"2023-09-23T19:10:37.789986+0000","last_peered":"2023-09-23T19:10:37.789986+0000","last_clean":"2023-09-23T19:10:37.789986+0000","last_became_active":"2023-09-23T19:09:03.326526+0000","last_became_peered":"2023-09-23T19:09:03.326526+0000","last_unstale":"2023-09-23T19:10:37.789986+0000","last_undegraded":"2023-09-23T19:10:37.789986+0000","last_fullsized":"2023-09-23T19:10:37.789986+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.803140+0000","last_change":"2023-09-23T19:10:37.801034+0000","last_active":"2023-09-23T19:10:37.803140+0000","last_peered":"2023-09-23T19:10:37.803140+0000","last_clean":"2023-09-23T19:10:37.803140+0000","last_became_active":"2023-09-23T19:10:37.800310+0000","last_became_peered":"2023-09-23T19:10:37.800310+0000","last_unstale":"2023-09-23T19:10:37.803140+0000","last_undegraded":"2023-09-23T19:10:37.803140+0000","last_fullsized":"2023-09-23T19:10:37.803140+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":260,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.789866+0000","last_change":"2023-09-23T19:09:03.315467+0000","last_active":"2023-09-23T19:10:37.789866+0000","last_peered":"2023-09-23T19:10:37.789866+0000","last_clean":"2023-09-23T19:10:37.789866+0000","last_became_active":"2023-09-23T19:09:03.313654+0000","last_became_peered":"2023-09-23T19:09:03.313654+0000","last_unstale":"2023-09-23T19:10:37.789866+0000","last_undegraded":"2023-09-23T19:10:37.789866+0000","last_fullsized":"2023-09-23T19:10:37.789866+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":133,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T19:10:47.136664+0000","last_change":"2023-09-23T19:09:03.319829+0000","last_active":"2023-09-23T19:10:47.136664+0000","last_peered":"2023-09-23T19:10:47.136664+0000","last_clean":"2023-09-23T19:10:47.136664+0000","last_became_active":"2023-09-23T19:09:03.319522+0000","last_became_peered":"2023-09-23T19:09:03.319522+0000","last_unstale":"2023-09-23T19:10:47.136664+0000","last_undegraded":"2023-09-23T19:10:47.136664+0000","last_fullsized":"2023-09-23T19:10:47.136664+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:57:03.744906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":266,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.852171+0000","last_change":"2023-09-23T19:10:37.851745+0000","last_active":"2023-09-23T19:10:37.852171+0000","last_peered":"2023-09-23T19:10:37.852171+0000","last_clean":"2023-09-23T19:10:37.852171+0000","last_became_active":"2023-09-23T19:10:37.797727+0000","last_became_peered":"2023-09-23T19:10:37.797727+0000","last_unstale":"2023-09-23T19:10:37.852171+0000","last_undegraded":"2023-09-23T19:10:37.852171+0000","last_fullsized":"2023-09-23T19:10:37.852171+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:36.467704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":65,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.786014+0000","last_change":"2023-09-23T19:09:03.318996+0000","last_active":"2023-09-23T19:10:37.786014+0000","last_peered":"2023-09-23T19:10:37.786014+0000","last_clean":"2023-09-23T19:10:37.786014+0000","last_became_active":"2023-09-23T19:09:03.318593+0000","last_became_peered":"2023-09-23T19:09:03.318593+0000","last_unstale":"2023-09-23T19:10:37.786014+0000","last_undegraded":"2023-09-23T19:10:37.786014+0000","last_fullsized":"2023-09-23T19:10:37.786014+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:23.764508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":257,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.791029+0000","last_change":"2023-09-23T19:09:03.321932+0000","last_active":"2023-09-23T19:10:37.791029+0000","last_peered":"2023-09-23T19:10:37.791029+0000","last_clean":"2023-09-23T19:10:37.791029+0000","last_became_active":"2023-09-23T19:09:03.320840+0000","last_became_peered":"2023-09-23T19:09:03.320840+0000","last_unstale":"2023-09-23T19:10:37.791029+0000","last_undegraded":"2023-09-23T19:10:37.791029+0000","last_fullsized":"2023-09-23T19:10:37.791029+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.802967+0000","last_change":"2023-09-23T19:10:37.800494+0000","last_active":"2023-09-23T19:10:37.802967+0000","last_peered":"2023-09-23T19:10:37.802967+0000","last_clean":"2023-09-23T19:10:37.802967+0000","last_became_active":"2023-09-23T19:10:37.800131+0000","last_became_peered":"2023-09-23T19:10:37.800131+0000","last_unstale":"2023-09-23T19:10:37.802967+0000","last_undegraded":"2023-09-23T19:10:37.802967+0000","last_fullsized":"2023-09-23T19:10:37.802967+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":292,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.790363+0000","last_change":"2023-09-23T19:09:03.321485+0000","last_active":"2023-09-23T19:10:37.790363+0000","last_peered":"2023-09-23T19:10:37.790363+0000","last_clean":"2023-09-23T19:10:37.790363+0000","last_became_active":"2023-09-23T19:09:03.320932+0000","last_became_peered":"2023-09-23T19:09:03.320932+0000","last_unstale":"2023-09-23T19:10:37.790363+0000","last_undegraded":"2023-09-23T19:10:37.790363+0000","last_fullsized":"2023-09-23T19:10:37.790363+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":370,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.787556+0000","last_change":"2023-09-23T19:03:25.324518+0000","last_active":"2023-09-23T19:10:37.787556+0000","last_peered":"2023-09-23T19:10:37.787556+0000","last_clean":"2023-09-23T19:10:37.787556+0000","last_became_active":"2023-09-23T19:03:25.324166+0000","last_became_peered":"2023-09-23T19:03:25.324166+0000","last_unstale":"2023-09-23T19:10:37.787556+0000","last_undegraded":"2023-09-23T19:10:37.787556+0000","last_fullsized":"2023-09-23T19:10:37.787556+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:24.379778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":289,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.847203+0000","last_change":"2023-09-23T19:10:37.846671+0000","last_active":"2023-09-23T19:10:37.847203+0000","last_peered":"2023-09-23T19:10:37.847203+0000","last_clean":"2023-09-23T19:10:37.847203+0000","last_became_active":"2023-09-23T19:10:37.800522+0000","last_became_peered":"2023-09-23T19:10:37.800522+0000","last_unstale":"2023-09-23T19:10:37.847203+0000","last_undegraded":"2023-09-23T19:10:37.847203+0000","last_fullsized":"2023-09-23T19:10:37.847203+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":260,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.792215+0000","last_change":"2023-09-23T19:09:02.306080+0000","last_active":"2023-09-23T19:10:37.792215+0000","last_peered":"2023-09-23T19:10:37.792215+0000","last_clean":"2023-09-23T19:10:37.792215+0000","last_became_active":"2023-09-23T19:09:02.305679+0000","last_became_peered":"2023-09-23T19:09:02.305679+0000","last_unstale":"2023-09-23T19:10:37.792215+0000","last_undegraded":"2023-09-23T19:10:37.792215+0000","last_fullsized":"2023-09-23T19:10:37.792215+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:55.893544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":94,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.799345+0000","last_change":"2023-09-23T19:08:37.773654+0000","last_active":"2023-09-23T19:10:37.799345+0000","last_peered":"2023-09-23T19:10:37.799345+0000","last_clean":"2023-09-23T19:10:37.799345+0000","last_became_active":"2023-09-23T19:08:37.773196+0000","last_became_peered":"2023-09-23T19:08:37.773196+0000","last_unstale":"2023-09-23T19:10:37.799345+0000","last_undegraded":"2023-09-23T19:10:37.799345+0000","last_fullsized":"2023-09-23T19:10:37.799345+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":407,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.789415+0000","last_change":"2023-09-23T19:09:42.637333+0000","last_active":"2023-09-23T19:10:37.789415+0000","last_peered":"2023-09-23T19:10:37.789415+0000","last_clean":"2023-09-23T19:10:37.789415+0000","last_became_active":"2023-09-23T19:09:42.637015+0000","last_became_peered":"2023-09-23T19:09:42.637015+0000","last_unstale":"2023-09-23T19:10:37.789415+0000","last_undegraded":"2023-09-23T19:10:37.789415+0000","last_fullsized":"2023-09-23T19:10:37.789415+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:47.410433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":130,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.784872+0000","last_change":"2023-09-23T18:59:13.721533+0000","last_active":"2023-09-23T19:10:37.784872+0000","last_peered":"2023-09-23T19:10:37.784872+0000","last_clean":"2023-09-23T19:10:37.784872+0000","last_became_active":"2023-09-23T18:59:13.711121+0000","last_became_peered":"2023-09-23T18:59:13.711121+0000","last_unstale":"2023-09-23T19:10:37.784872+0000","last_undegraded":"2023-09-23T19:10:37.784872+0000","last_fullsized":"2023-09-23T19:10:37.784872+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:51.950774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":333,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.784876+0000","last_change":"2023-09-23T18:59:13.702825+0000","last_active":"2023-09-23T19:10:37.784876+0000","last_peered":"2023-09-23T19:10:37.784876+0000","last_clean":"2023-09-23T19:10:37.784876+0000","last_became_active":"2023-09-23T18:59:13.702503+0000","last_became_peered":"2023-09-23T18:59:13.702503+0000","last_unstale":"2023-09-23T19:10:37.784876+0000","last_undegraded":"2023-09-23T19:10:37.784876+0000","last_fullsized":"2023-09-23T19:10:37.784876+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.263337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":237,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.788179+0000","last_change":"2023-09-23T18:59:13.708841+0000","last_active":"2023-09-23T19:10:37.788179+0000","last_peered":"2023-09-23T19:10:37.788179+0000","last_clean":"2023-09-23T19:10:37.788179+0000","last_became_active":"2023-09-23T18:59:13.707760+0000","last_became_peered":"2023-09-23T18:59:13.707760+0000","last_unstale":"2023-09-23T19:10:37.788179+0000","last_undegraded":"2023-09-23T19:10:37.788179+0000","last_fullsized":"2023-09-23T19:10:37.788179+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:02.417127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":386,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.793048+0000","last_change":"2023-09-23T18:59:13.708710+0000","last_active":"2023-09-23T19:10:37.793048+0000","last_peered":"2023-09-23T19:10:37.793048+0000","last_clean":"2023-09-23T19:10:37.793048+0000","last_became_active":"2023-09-23T18:59:13.708349+0000","last_became_peered":"2023-09-23T18:59:13.708349+0000","last_unstale":"2023-09-23T19:10:37.793048+0000","last_undegraded":"2023-09-23T19:10:37.793048+0000","last_fullsized":"2023-09-23T19:10:37.793048+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:33.033640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":311,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.791743+0000","last_change":"2023-09-23T19:09:03.310639+0000","last_active":"2023-09-23T19:10:37.791743+0000","last_peered":"2023-09-23T19:10:37.791743+0000","last_clean":"2023-09-23T19:10:37.791743+0000","last_became_active":"2023-09-23T19:09:03.310175+0000","last_became_peered":"2023-09-23T19:09:03.310175+0000","last_unstale":"2023-09-23T19:10:37.791743+0000","last_undegraded":"2023-09-23T19:10:37.791743+0000","last_fullsized":"2023-09-23T19:10:37.791743+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:49.504419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":340,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.789312+0000","last_change":"2023-09-23T18:59:13.708690+0000","last_active":"2023-09-23T19:10:37.789312+0000","last_peered":"2023-09-23T19:10:37.789312+0000","last_clean":"2023-09-23T19:10:37.789312+0000","last_became_active":"2023-09-23T18:59:13.708248+0000","last_became_peered":"2023-09-23T18:59:13.708248+0000","last_unstale":"2023-09-23T19:10:37.789312+0000","last_undegraded":"2023-09-23T19:10:37.789312+0000","last_fullsized":"2023-09-23T19:10:37.789312+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:43:08.360496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":88,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.785012+0000","last_change":"2023-09-23T19:09:03.322253+0000","last_active":"2023-09-23T19:10:37.785012+0000","last_peered":"2023-09-23T19:10:37.785012+0000","last_clean":"2023-09-23T19:10:37.785012+0000","last_became_active":"2023-09-23T19:09:03.320102+0000","last_became_peered":"2023-09-23T19:09:03.320102+0000","last_unstale":"2023-09-23T19:10:37.785012+0000","last_undegraded":"2023-09-23T19:10:37.785012+0000","last_fullsized":"2023-09-23T19:10:37.785012+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:01.268221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":402,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.788809+0000","last_change":"2023-09-23T19:09:03.309621+0000","last_active":"2023-09-23T19:10:37.788809+0000","last_peered":"2023-09-23T19:10:37.788809+0000","last_clean":"2023-09-23T19:10:37.788809+0000","last_became_active":"2023-09-23T19:09:03.309195+0000","last_became_peered":"2023-09-23T19:09:03.309195+0000","last_unstale":"2023-09-23T19:10:37.788809+0000","last_undegraded":"2023-09-23T19:10:37.788809+0000","last_fullsized":"2023-09-23T19:10:37.788809+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:21.448033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"383'306","reported_seq":1451,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.788129+0000","last_change":"2023-09-23T19:09:23.687377+0000","last_active":"2023-09-23T19:10:37.788129+0000","last_peered":"2023-09-23T19:10:37.788129+0000","last_clean":"2023-09-23T19:10:37.788129+0000","last_became_active":"2023-09-23T19:09:02.306435+0000","last_became_peered":"2023-09-23T19:09:02.306435+0000","last_unstale":"2023-09-23T19:10:37.788129+0000","last_undegraded":"2023-09-23T19:10:37.788129+0000","last_fullsized":"2023-09-23T19:10:37.788129+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":306,"log_dups_size":0,"ondisk_log_size":306,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:54:30.990476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086649800000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1624,"num_read_kb":2591,"num_write":297,"num_write_kb":3316,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":94,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.800664+0000","last_change":"2023-09-23T19:08:37.774207+0000","last_active":"2023-09-23T19:10:37.800664+0000","last_peered":"2023-09-23T19:10:37.800664+0000","last_clean":"2023-09-23T19:10:37.800664+0000","last_became_active":"2023-09-23T19:08:37.773764+0000","last_became_peered":"2023-09-23T19:08:37.773764+0000","last_unstale":"2023-09-23T19:10:37.800664+0000","last_undegraded":"2023-09-23T19:10:37.800664+0000","last_fullsized":"2023-09-23T19:10:37.800664+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":392,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.800608+0000","last_change":"2023-09-23T19:10:37.800203+0000","last_active":"2023-09-23T19:10:37.800608+0000","last_peered":"2023-09-23T19:10:37.800608+0000","last_clean":"2023-09-23T19:10:37.800608+0000","last_became_active":"2023-09-23T19:10:37.799806+0000","last_became_peered":"2023-09-23T19:10:37.799806+0000","last_unstale":"2023-09-23T19:10:37.800608+0000","last_undegraded":"2023-09-23T19:10:37.800608+0000","last_fullsized":"2023-09-23T19:10:37.800608+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:49.948374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"383'125","reported_seq":477,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.799456+0000","last_change":"2023-09-23T19:09:23.642827+0000","last_active":"2023-09-23T19:10:37.799456+0000","last_peered":"2023-09-23T19:10:37.799456+0000","last_clean":"2023-09-23T19:10:37.799456+0000","last_became_active":"2023-09-23T19:08:37.774901+0000","last_became_peered":"2023-09-23T19:08:37.774901+0000","last_unstale":"2023-09-23T19:10:37.799456+0000","last_undegraded":"2023-09-23T19:10:37.799456+0000","last_fullsized":"2023-09-23T19:10:37.799456+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":125,"log_dups_size":0,"ondisk_log_size":125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1121,"num_read_kb":2820,"num_write":107,"num_write_kb":1462,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":107,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.788057+0000","last_change":"2023-09-23T18:59:13.703057+0000","last_active":"2023-09-23T19:10:37.788057+0000","last_peered":"2023-09-23T19:10:37.788057+0000","last_clean":"2023-09-23T19:10:37.788057+0000","last_became_active":"2023-09-23T18:59:13.702759+0000","last_became_peered":"2023-09-23T18:59:13.702759+0000","last_unstale":"2023-09-23T19:10:37.788057+0000","last_undegraded":"2023-09-23T19:10:37.788057+0000","last_fullsized":"2023-09-23T19:10:37.788057+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:53.278738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":409,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.788001+0000","last_change":"2023-09-23T18:58:38.726546+0000","last_active":"2023-09-23T19:10:37.788001+0000","last_peered":"2023-09-23T19:10:37.788001+0000","last_clean":"2023-09-23T19:10:37.788001+0000","last_became_active":"2023-09-23T18:58:38.724765+0000","last_became_peered":"2023-09-23T18:58:38.724765+0000","last_unstale":"2023-09-23T19:10:37.788001+0000","last_undegraded":"2023-09-23T19:10:37.788001+0000","last_fullsized":"2023-09-23T19:10:37.788001+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:05.322676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"383'289","reported_seq":1312,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.791585+0000","last_change":"2023-09-23T19:09:23.637617+0000","last_active":"2023-09-23T19:10:37.791585+0000","last_peered":"2023-09-23T19:10:37.791585+0000","last_clean":"2023-09-23T19:10:37.791585+0000","last_became_active":"2023-09-23T18:58:38.707923+0000","last_became_peered":"2023-09-23T18:58:38.707923+0000","last_unstale":"2023-09-23T19:10:37.791585+0000","last_undegraded":"2023-09-23T19:10:37.791585+0000","last_fullsized":"2023-09-23T19:10:37.791585+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":289,"log_dups_size":0,"ondisk_log_size":289,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:59.471903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047940899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1602,"num_read_kb":4202,"num_write":289,"num_write_kb":2667,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":345,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.787317+0000","last_change":"2023-09-23T18:33:53.849084+0000","last_active":"2023-09-23T19:10:37.787317+0000","last_peered":"2023-09-23T19:10:37.787317+0000","last_clean":"2023-09-23T19:10:37.787317+0000","last_became_active":"2023-09-23T18:33:53.848757+0000","last_became_peered":"2023-09-23T18:33:53.848757+0000","last_unstale":"2023-09-23T19:10:37.787317+0000","last_undegraded":"2023-09-23T19:10:37.787317+0000","last_fullsized":"2023-09-23T19:10:37.787317+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:15.267742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":415,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:42.065697+0000","last_change":"2023-09-23T19:08:37.776940+0000","last_active":"2023-09-23T19:10:42.065697+0000","last_peered":"2023-09-23T19:10:42.065697+0000","last_clean":"2023-09-23T19:10:42.065697+0000","last_became_active":"2023-09-23T19:08:37.776564+0000","last_became_peered":"2023-09-23T19:08:37.776564+0000","last_unstale":"2023-09-23T19:10:42.065697+0000","last_undegraded":"2023-09-23T19:10:42.065697+0000","last_fullsized":"2023-09-23T19:10:42.065697+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:44.299192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"383'317","reported_seq":1364,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.799778+0000","last_change":"2023-09-23T19:09:23.643011+0000","last_active":"2023-09-23T19:10:37.799778+0000","last_peered":"2023-09-23T19:10:37.799778+0000","last_clean":"2023-09-23T19:10:37.799778+0000","last_became_active":"2023-09-23T19:09:03.312411+0000","last_became_peered":"2023-09-23T19:09:03.312411+0000","last_unstale":"2023-09-23T19:10:37.799778+0000","last_undegraded":"2023-09-23T19:10:37.799778+0000","last_fullsized":"2023-09-23T19:10:37.799778+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":317,"log_dups_size":0,"ondisk_log_size":317,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1525,"num_read_kb":2676,"num_write":278,"num_write_kb":5451,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":497,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.789669+0000","last_change":"2023-09-23T19:09:02.309826+0000","last_active":"2023-09-23T19:10:37.789669+0000","last_peered":"2023-09-23T19:10:37.789669+0000","last_clean":"2023-09-23T19:10:37.789669+0000","last_became_active":"2023-09-23T19:09:02.309300+0000","last_became_peered":"2023-09-23T19:09:02.309300+0000","last_unstale":"2023-09-23T19:10:37.789669+0000","last_undegraded":"2023-09-23T19:10:37.789669+0000","last_fullsized":"2023-09-23T19:10:37.789669+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_clean_scrub_stamp":"2023-09-23T18:08:35.701374+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":150,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.797624+0000","last_change":"2023-09-23T19:10:37.797301+0000","last_active":"2023-09-23T19:10:37.797624+0000","last_peered":"2023-09-23T19:10:37.797624+0000","last_clean":"2023-09-23T19:10:37.797624+0000","last_became_active":"2023-09-23T19:10:37.796972+0000","last_became_peered":"2023-09-23T19:10:37.796972+0000","last_unstale":"2023-09-23T19:10:37.797624+0000","last_undegraded":"2023-09-23T19:10:37.797624+0000","last_fullsized":"2023-09-23T19:10:37.797624+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:25.458233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":51,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.791160+0000","last_change":"2023-09-23T19:09:03.315025+0000","last_active":"2023-09-23T19:10:37.791160+0000","last_peered":"2023-09-23T19:10:37.791160+0000","last_clean":"2023-09-23T19:10:37.791160+0000","last_became_active":"2023-09-23T19:09:03.313741+0000","last_became_peered":"2023-09-23T19:09:03.313741+0000","last_unstale":"2023-09-23T19:10:37.791160+0000","last_undegraded":"2023-09-23T19:10:37.791160+0000","last_fullsized":"2023-09-23T19:10:37.791160+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"383'163","reported_seq":812,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.789761+0000","last_change":"2023-09-23T19:09:23.643786+0000","last_active":"2023-09-23T19:10:37.789761+0000","last_peered":"2023-09-23T19:10:37.789761+0000","last_clean":"2023-09-23T19:10:37.789761+0000","last_became_active":"2023-09-23T19:09:02.310096+0000","last_became_peered":"2023-09-23T19:09:02.310096+0000","last_unstale":"2023-09-23T19:10:37.789761+0000","last_undegraded":"2023-09-23T19:10:37.789761+0000","last_fullsized":"2023-09-23T19:10:37.789761+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":163,"log_dups_size":0,"ondisk_log_size":163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1010,"num_read_kb":4243,"num_write":132,"num_write_kb":3279,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":452,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.789237+0000","last_change":"2023-09-23T19:08:38.775076+0000","last_active":"2023-09-23T19:10:37.789237+0000","last_peered":"2023-09-23T19:10:37.789237+0000","last_clean":"2023-09-23T19:10:37.789237+0000","last_became_active":"2023-09-23T19:08:38.774626+0000","last_became_peered":"2023-09-23T19:08:38.774626+0000","last_unstale":"2023-09-23T19:10:37.789237+0000","last_undegraded":"2023-09-23T19:10:37.789237+0000","last_fullsized":"2023-09-23T19:10:37.789237+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:20:42.987390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":429,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.789471+0000","last_change":"2023-09-23T18:51:43.087834+0000","last_active":"2023-09-23T19:10:37.789471+0000","last_peered":"2023-09-23T19:10:37.789471+0000","last_clean":"2023-09-23T19:10:37.789471+0000","last_became_active":"2023-09-23T18:51:43.087309+0000","last_became_peered":"2023-09-23T18:51:43.087309+0000","last_unstale":"2023-09-23T19:10:37.789471+0000","last_undegraded":"2023-09-23T19:10:37.789471+0000","last_fullsized":"2023-09-23T19:10:37.789471+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:53.913425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"383'142","reported_seq":754,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.791038+0000","last_change":"2023-09-23T19:09:23.643776+0000","last_active":"2023-09-23T19:10:37.791038+0000","last_peered":"2023-09-23T19:10:37.791038+0000","last_clean":"2023-09-23T19:10:37.791038+0000","last_became_active":"2023-09-23T19:09:03.321173+0000","last_became_peered":"2023-09-23T19:09:03.321173+0000","last_unstale":"2023-09-23T19:10:37.791038+0000","last_undegraded":"2023-09-23T19:10:37.791038+0000","last_fullsized":"2023-09-23T19:10:37.791038+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":142,"log_dups_size":0,"ondisk_log_size":142,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":777,"num_read_kb":2427,"num_write":137,"num_write_kb":5585,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":368,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.800436+0000","last_change":"2023-09-23T19:10:37.800014+0000","last_active":"2023-09-23T19:10:37.800436+0000","last_peered":"2023-09-23T19:10:37.800436+0000","last_clean":"2023-09-23T19:10:37.800436+0000","last_became_active":"2023-09-23T19:10:37.799660+0000","last_became_peered":"2023-09-23T19:10:37.799660+0000","last_unstale":"2023-09-23T19:10:37.800436+0000","last_undegraded":"2023-09-23T19:10:37.800436+0000","last_fullsized":"2023-09-23T19:10:37.800436+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:39.131137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":108,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.799643+0000","last_change":"2023-09-23T19:09:03.310928+0000","last_active":"2023-09-23T19:10:37.799643+0000","last_peered":"2023-09-23T19:10:37.799643+0000","last_clean":"2023-09-23T19:10:37.799643+0000","last_became_active":"2023-09-23T19:09:03.310498+0000","last_became_peered":"2023-09-23T19:09:03.310498+0000","last_unstale":"2023-09-23T19:10:37.799643+0000","last_undegraded":"2023-09-23T19:10:37.799643+0000","last_fullsized":"2023-09-23T19:10:37.799643+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":414,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.786376+0000","last_change":"2023-09-23T19:09:03.319699+0000","last_active":"2023-09-23T19:10:37.786376+0000","last_peered":"2023-09-23T19:10:37.786376+0000","last_clean":"2023-09-23T19:10:37.786376+0000","last_became_active":"2023-09-23T19:09:03.318772+0000","last_became_peered":"2023-09-23T19:09:03.318772+0000","last_unstale":"2023-09-23T19:10:37.786376+0000","last_undegraded":"2023-09-23T19:10:37.786376+0000","last_fullsized":"2023-09-23T19:10:37.786376+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:41.224329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"383'372","reported_seq":1643,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:42.065626+0000","last_change":"2023-09-23T19:09:23.687378+0000","last_active":"2023-09-23T19:10:42.065626+0000","last_peered":"2023-09-23T19:10:42.065626+0000","last_clean":"2023-09-23T19:10:42.065626+0000","last_became_active":"2023-09-23T18:59:16.452030+0000","last_became_peered":"2023-09-23T18:59:16.452030+0000","last_unstale":"2023-09-23T19:10:42.065626+0000","last_undegraded":"2023-09-23T19:10:42.065626+0000","last_fullsized":"2023-09-23T19:10:42.065626+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":372,"log_dups_size":0,"ondisk_log_size":372,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:20.988663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067870799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2145,"num_read_kb":3069,"num_write":293,"num_write_kb":1311,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":90,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.784590+0000","last_change":"2023-09-23T19:09:03.322018+0000","last_active":"2023-09-23T19:10:37.784590+0000","last_peered":"2023-09-23T19:10:37.784590+0000","last_clean":"2023-09-23T19:10:37.784590+0000","last_became_active":"2023-09-23T19:09:03.321316+0000","last_became_peered":"2023-09-23T19:09:03.321316+0000","last_unstale":"2023-09-23T19:10:37.784590+0000","last_undegraded":"2023-09-23T19:10:37.784590+0000","last_fullsized":"2023-09-23T19:10:37.784590+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:46:37.431152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"383'236","reported_seq":1461,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.790873+0000","last_change":"2023-09-23T19:09:23.644949+0000","last_active":"2023-09-23T19:10:37.790873+0000","last_peered":"2023-09-23T19:10:37.790873+0000","last_clean":"2023-09-23T19:10:37.790873+0000","last_became_active":"2023-09-23T19:09:05.321513+0000","last_became_peered":"2023-09-23T19:09:05.321513+0000","last_unstale":"2023-09-23T19:10:37.790873+0000","last_undegraded":"2023-09-23T19:10:37.790873+0000","last_fullsized":"2023-09-23T19:10:37.790873+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":236,"log_dups_size":0,"ondisk_log_size":236,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2130,"num_read_kb":4337,"num_write":157,"num_write_kb":2355,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":433,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.789600+0000","last_change":"2023-09-23T19:09:03.328610+0000","last_active":"2023-09-23T19:10:37.789600+0000","last_peered":"2023-09-23T19:10:37.789600+0000","last_clean":"2023-09-23T19:10:37.789600+0000","last_became_active":"2023-09-23T19:09:03.312603+0000","last_became_peered":"2023-09-23T19:09:03.312603+0000","last_unstale":"2023-09-23T19:10:37.789600+0000","last_undegraded":"2023-09-23T19:10:37.789600+0000","last_fullsized":"2023-09-23T19:10:37.789600+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:29.029824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":296,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.797859+0000","last_change":"2023-09-23T19:08:37.776658+0000","last_active":"2023-09-23T19:10:37.797859+0000","last_peered":"2023-09-23T19:10:37.797859+0000","last_clean":"2023-09-23T19:10:37.797859+0000","last_became_active":"2023-09-23T19:08:37.776268+0000","last_became_peered":"2023-09-23T19:08:37.776268+0000","last_unstale":"2023-09-23T19:10:37.797859+0000","last_undegraded":"2023-09-23T19:10:37.797859+0000","last_fullsized":"2023-09-23T19:10:37.797859+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":385,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.789524+0000","last_change":"2023-09-23T19:09:03.312771+0000","last_active":"2023-09-23T19:10:37.789524+0000","last_peered":"2023-09-23T19:10:37.789524+0000","last_clean":"2023-09-23T19:10:37.789524+0000","last_became_active":"2023-09-23T19:09:03.312349+0000","last_became_peered":"2023-09-23T19:09:03.312349+0000","last_unstale":"2023-09-23T19:10:37.789524+0000","last_undegraded":"2023-09-23T19:10:37.789524+0000","last_fullsized":"2023-09-23T19:10:37.789524+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:57.252470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":186,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.793281+0000","last_change":"2023-09-23T18:58:38.715946+0000","last_active":"2023-09-23T19:10:37.793281+0000","last_peered":"2023-09-23T19:10:37.793281+0000","last_clean":"2023-09-23T19:10:37.793281+0000","last_became_active":"2023-09-23T18:58:38.715584+0000","last_became_peered":"2023-09-23T18:58:38.715584+0000","last_unstale":"2023-09-23T19:10:37.793281+0000","last_undegraded":"2023-09-23T19:10:37.793281+0000","last_fullsized":"2023-09-23T19:10:37.793281+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:57.811713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":110,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.799460+0000","last_change":"2023-09-23T19:08:37.774463+0000","last_active":"2023-09-23T19:10:37.799460+0000","last_peered":"2023-09-23T19:10:37.799460+0000","last_clean":"2023-09-23T19:10:37.799460+0000","last_became_active":"2023-09-23T19:08:37.773983+0000","last_became_peered":"2023-09-23T19:08:37.773983+0000","last_unstale":"2023-09-23T19:10:37.799460+0000","last_undegraded":"2023-09-23T19:10:37.799460+0000","last_fullsized":"2023-09-23T19:10:37.799460+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":384,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.784754+0000","last_change":"2023-09-23T18:59:13.712801+0000","last_active":"2023-09-23T19:10:37.784754+0000","last_peered":"2023-09-23T19:10:37.784754+0000","last_clean":"2023-09-23T19:10:37.784754+0000","last_became_active":"2023-09-23T18:59:13.710748+0000","last_became_peered":"2023-09-23T18:59:13.710748+0000","last_unstale":"2023-09-23T19:10:37.784754+0000","last_undegraded":"2023-09-23T19:10:37.784754+0000","last_fullsized":"2023-09-23T19:10:37.784754+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:26.256173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":110,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.800780+0000","last_change":"2023-09-23T19:08:37.775330+0000","last_active":"2023-09-23T19:10:37.800780+0000","last_peered":"2023-09-23T19:10:37.800780+0000","last_clean":"2023-09-23T19:10:37.800780+0000","last_became_active":"2023-09-23T19:08:37.774581+0000","last_became_peered":"2023-09-23T19:08:37.774581+0000","last_unstale":"2023-09-23T19:10:37.800780+0000","last_undegraded":"2023-09-23T19:10:37.800780+0000","last_fullsized":"2023-09-23T19:10:37.800780+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":349,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.784984+0000","last_change":"2023-09-23T18:59:13.699034+0000","last_active":"2023-09-23T19:10:37.784984+0000","last_peered":"2023-09-23T19:10:37.784984+0000","last_clean":"2023-09-23T19:10:37.784984+0000","last_became_active":"2023-09-23T18:59:13.698718+0000","last_became_peered":"2023-09-23T18:59:13.698718+0000","last_unstale":"2023-09-23T19:10:37.784984+0000","last_undegraded":"2023-09-23T19:10:37.784984+0000","last_fullsized":"2023-09-23T19:10:37.784984+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:57.449621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":114,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:42.647585+0000","last_change":"2023-09-23T18:58:38.711301+0000","last_active":"2023-09-23T19:10:42.647585+0000","last_peered":"2023-09-23T19:10:42.647585+0000","last_clean":"2023-09-23T19:10:42.647585+0000","last_became_active":"2023-09-23T18:58:38.710923+0000","last_became_peered":"2023-09-23T18:58:38.710923+0000","last_unstale":"2023-09-23T19:10:42.647585+0000","last_undegraded":"2023-09-23T19:10:42.647585+0000","last_fullsized":"2023-09-23T19:10:42.647585+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:17.756147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":113,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T19:10:47.136784+0000","last_change":"2023-09-23T18:58:38.711839+0000","last_active":"2023-09-23T19:10:47.136784+0000","last_peered":"2023-09-23T19:10:47.136784+0000","last_clean":"2023-09-23T19:10:47.136784+0000","last_became_active":"2023-09-23T18:58:38.711347+0000","last_became_peered":"2023-09-23T18:58:38.711347+0000","last_unstale":"2023-09-23T19:10:47.136784+0000","last_undegraded":"2023-09-23T19:10:47.136784+0000","last_fullsized":"2023-09-23T19:10:47.136784+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:55.319645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":472,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.790380+0000","last_change":"2023-09-23T19:08:37.776379+0000","last_active":"2023-09-23T19:10:37.790380+0000","last_peered":"2023-09-23T19:10:37.790380+0000","last_clean":"2023-09-23T19:10:37.790380+0000","last_became_active":"2023-09-23T19:08:37.776012+0000","last_became_peered":"2023-09-23T19:08:37.776012+0000","last_unstale":"2023-09-23T19:10:37.790380+0000","last_undegraded":"2023-09-23T19:10:37.790380+0000","last_fullsized":"2023-09-23T19:10:37.790380+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:16.823018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":348,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.787418+0000","last_change":"2023-09-23T19:02:58.680806+0000","last_active":"2023-09-23T19:10:37.787418+0000","last_peered":"2023-09-23T19:10:37.787418+0000","last_clean":"2023-09-23T19:10:37.787418+0000","last_became_active":"2023-09-23T19:02:58.680426+0000","last_became_peered":"2023-09-23T19:02:58.680426+0000","last_unstale":"2023-09-23T19:10:37.787418+0000","last_undegraded":"2023-09-23T19:10:37.787418+0000","last_fullsized":"2023-09-23T19:10:37.787418+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:16:15.128164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":8324,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56162,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11934,"num_read_kb":26365,"num_write":1690,"num_write_kb":25426,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":120,"num_bytes_recovered":12582931,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7733,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":367,"seq":1576252997655,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":347440,"kb_used_data":12584,"kb_used_omap":13,"kb_used_meta":334834,"kb_avail":93397712,"statfs":{"total":95995035648,"available":95639257088,"internally_reserved":0,"allocated":12886016,"data_stored":11915152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14028,"internal_metadata":342870324},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:10:06 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.3169999999999999,"5min":1.3169999999999999,"15min":1.3169999999999999},"last":0.89200000000000002},{"interface":"front","average":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.339,"5min":1.339,"15min":1.339},"last":1.3959999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:10:06 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.4199999999999999,"5min":1.4199999999999999,"15min":1.4199999999999999},"last":1.022},{"interface":"front","average":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":1.4379999999999999,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":1.4239999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:10:06 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.016,"15min":1.016},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":2.4049999999999998,"5min":2.4049999999999998,"15min":2.4049999999999998},"last":1.462},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"min":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"max":{"1min":1.407,"5min":1.407,"15min":1.407},"last":1.1679999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:10:06 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.2969999999999999,"5min":1.2969999999999999,"15min":1.2969999999999999},"last":1.1120000000000001},{"interface":"front","average":{"1min":1.0700000000000001,"5min":1.0700000000000001,"15min":1.0700000000000001},"min":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":2.2749999999999999,"5min":2.2749999999999999,"15min":2.2749999999999999},"last":0.84099999999999997}]},{"osd":4,"last update":"Sat Sep 23 19:10:06 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"min":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":0.79900000000000004},{"interface":"front","average":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"min":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.417,"5min":1.417,"15min":1.417},"last":1.3380000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:10:06 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.038,"15min":1.038},"min":{"1min":0.622,"5min":0.622,"15min":0.622},"max":{"1min":2.3660000000000001,"5min":2.3660000000000001,"15min":2.3660000000000001},"last":1.256},{"interface":"front","average":{"1min":1.083,"5min":1.083,"15min":1.083},"min":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"max":{"1min":1.409,"5min":1.409,"15min":1.409},"last":1.365}]},{"osd":6,"last update":"Sat Sep 23 19:10:06 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":1.121,"15min":1.121},"min":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":2.3300000000000001,"5min":2.3300000000000001,"15min":2.3300000000000001},"last":1.1459999999999999},{"interface":"front","average":{"1min":1.1519999999999999,"5min":1.1519999999999999,"15min":1.1519999999999999},"min":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"max":{"1min":1.496,"5min":1.496,"15min":1.496},"last":1.488}]}]},{"osd":6,"up_from":42,"seq":180388627148,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":899752,"kb_used_data":4384,"kb_used_omap":0,"kb_used_meta":895360,"kb_avail":92845400,"statfs":{"total":95995035648,"available":95073689600,"internally_reserved":0,"allocated":4489216,"data_stored":3526136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916848640},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:10:02 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.80800000000000005,"15min":1.877},"min":{"1min":0.38300000000000001,"5min":0.33000000000000002,"15min":0.26300000000000001},"max":{"1min":1.407,"5min":1.407,"15min":100.648},"last":0.67800000000000005},{"interface":"front","average":{"1min":1.0149999999999999,"5min":0.92300000000000004,"15min":1.9770000000000001},"min":{"1min":0.58799999999999997,"5min":0.42899999999999999,"15min":0.317},"max":{"1min":1.4550000000000001,"5min":1.4550000000000001,"15min":100.70399999999999},"last":0.98799999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:10:02 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.91200000000000003,"15min":1.6499999999999999},"min":{"1min":0.47399999999999998,"5min":0.439,"15min":0.29699999999999999},"max":{"1min":1.2330000000000001,"5min":8.6489999999999991,"15min":46.162999999999997},"last":0.92200000000000004},{"interface":"front","average":{"1min":0.82399999999999995,"5min":0.89200000000000002,"15min":1.4810000000000001},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.27900000000000003},"max":{"1min":1.4079999999999999,"5min":4.4269999999999996,"15min":46.430999999999997},"last":0.59499999999999997}]},{"osd":2,"last update":"Sat Sep 23 19:10:02 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":0.81599999999999995,"15min":1.5820000000000001},"min":{"1min":0.52400000000000002,"5min":0.34000000000000002,"15min":0.29599999999999999},"max":{"1min":1.3120000000000001,"5min":1.4630000000000001,"15min":66.587999999999994},"last":0.83799999999999997},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.88200000000000001,"15min":1.599},"min":{"1min":0.61399999999999999,"5min":0.39400000000000002,"15min":0.35299999999999998},"max":{"1min":1.21,"5min":1.5009999999999999,"15min":66.614000000000004},"last":0.96599999999999997}]},{"osd":3,"last update":"Sat Sep 23 19:10:13 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.88200000000000001,"15min":1.8939999999999999},"min":{"1min":0.66100000000000003,"5min":0.39000000000000001,"15min":0.34000000000000002},"max":{"1min":1.5489999999999999,"5min":1.5489999999999999,"15min":93.456000000000003},"last":0.95099999999999996},{"interface":"front","average":{"1min":0.99099999999999999,"5min":0.88700000000000001,"15min":1.9159999999999999},"min":{"1min":0.62,"5min":0.504,"15min":0.38300000000000001},"max":{"1min":1.399,"5min":1.399,"15min":93.472999999999999},"last":0.89800000000000002}]},{"osd":4,"last update":"Sat Sep 23 19:10:26 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":0.84199999999999997,"15min":0.77200000000000002},"min":{"1min":0.69799999999999995,"5min":0.36099999999999999,"15min":0.32200000000000001},"max":{"1min":1.242,"5min":1.242,"15min":4.1150000000000002},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.1919999999999999,"5min":1.0309999999999999,"15min":0.86299999999999999},"min":{"1min":0.76700000000000002,"5min":0.49099999999999999,"15min":0.35799999999999998},"max":{"1min":1.4890000000000001,"5min":1.4890000000000001,"15min":4.0179999999999998},"last":0.875}]},{"osd":5,"last update":"Sat Sep 23 19:10:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.042,"15min":1.024},"min":{"1min":0.73499999999999999,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.4530000000000001,"5min":1.464,"15min":1.464},"last":0.80000000000000004},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.0109999999999999,"15min":0.98299999999999998},"min":{"1min":0.89800000000000002,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.4670000000000001,"5min":1.4670000000000001,"15min":1.4670000000000001},"last":0.77600000000000002}]},{"osd":7,"last update":"Sat Sep 23 19:10:02 2023","interfaces":[{"interface":"back","average":{"1min":1.081,"5min":1.081,"15min":1.081},"min":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"max":{"1min":1.631,"5min":1.631,"15min":1.631},"last":1.0289999999999999},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.1080000000000001,"15min":1.1080000000000001},"min":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"max":{"1min":1.8819999999999999,"5min":1.8819999999999999,"15min":1.8819999999999999},"last":0.75900000000000001}]}]},{"osd":1,"up_from":123,"seq":528280977851,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1001192,"kb_used_data":9056,"kb_used_omap":7,"kb_used_meta":992120,"kb_avail":92743960,"statfs":{"total":95995035648,"available":94969815040,"internally_reserved":0,"allocated":9273344,"data_stored":7276801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":1015931662},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:10:05 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.72499999999999998,"15min":2.3100000000000001},"min":{"1min":0.41499999999999998,"5min":0.24299999999999999,"15min":0.23499999999999999},"max":{"1min":0.79400000000000004,"5min":4.4420000000000002,"15min":90.492000000000004},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.68700000000000006,"5min":0.77900000000000003,"15min":2.101},"min":{"1min":0.40300000000000002,"5min":0.188,"15min":0.17399999999999999},"max":{"1min":0.94899999999999995,"5min":7.1699999999999999,"15min":90.552999999999997},"last":0.73499999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:10:05 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.66800000000000004,"15min":1.76},"min":{"1min":0.437,"5min":0.245,"15min":0.219},"max":{"1min":1.1180000000000001,"5min":4.5279999999999996,"15min":87.832999999999998},"last":1.1679999999999999},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.66400000000000003,"15min":1.9430000000000001},"min":{"1min":0.35999999999999999,"5min":0.24299999999999999,"15min":0.22800000000000001},"max":{"1min":1.0369999999999999,"5min":4.5620000000000003,"15min":87.846999999999994},"last":1.0580000000000001}]},{"osd":3,"last update":"Sat Sep 23 19:10:05 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.67000000000000004,"15min":1.7549999999999999},"min":{"1min":0.371,"5min":0.32200000000000001,"15min":0.24099999999999999},"max":{"1min":1.135,"5min":4.694,"15min":99.126000000000005},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.69999999999999996,"15min":1.718},"min":{"1min":0.36499999999999999,"5min":0.34100000000000003,"15min":0.21299999999999999},"max":{"1min":1.089,"5min":4.5759999999999996,"15min":99.165999999999997},"last":0.90000000000000002}]},{"osd":4,"last update":"Sat Sep 23 19:10:24 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.92500000000000004,"15min":1.2909999999999999},"min":{"1min":0.496,"5min":0.48899999999999999,"15min":0.314},"max":{"1min":1.4199999999999999,"5min":7.3070000000000004,"15min":8.3559999999999999},"last":0.93999999999999995},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.89800000000000002,"15min":1.2709999999999999},"min":{"1min":0.75,"5min":0.56499999999999995,"15min":0.34200000000000003},"max":{"1min":1.5820000000000001,"5min":2.6859999999999999,"15min":9.0549999999999997},"last":1.0329999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:10:40 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":1.012,"15min":1.016},"min":{"1min":0.67700000000000005,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.476,"5min":3.6539999999999999,"15min":3.6539999999999999},"last":1.0820000000000001},{"interface":"front","average":{"1min":0.94899999999999995,"5min":1.109,"15min":1.135},"min":{"1min":0.625,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.2789999999999999,"5min":4.6159999999999997,"15min":4.6159999999999997},"last":1.0069999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:10:05 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.90700000000000003,"15min":1.2649999999999999},"min":{"1min":0.69699999999999995,"5min":0.42399999999999999,"15min":0.38100000000000001},"max":{"1min":1.413,"5min":4.6059999999999999,"15min":9.1029999999999998},"last":0.98399999999999999},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.85899999999999999,"15min":1.3620000000000001},"min":{"1min":0.503,"5min":0.40500000000000003,"15min":0.373},"max":{"1min":1.444,"5min":2.75,"15min":12.887},"last":1.131}]},{"osd":7,"last update":"Sat Sep 23 19:10:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"min":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"max":{"1min":1.4139999999999999,"5min":1.4139999999999999,"15min":1.4139999999999999},"last":1.242},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.0149999999999999,"15min":1.0149999999999999},"min":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.472,"5min":1.472,"15min":1.472},"last":1.214}]}]},{"osd":0,"up_from":88,"seq":377957122581,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1129000,"kb_used_data":9440,"kb_used_omap":8,"kb_used_meta":1119543,"kb_avail":92616152,"statfs":{"total":95995035648,"available":94838939648,"internally_reserved":0,"allocated":9666560,"data_stored":7844043,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":1146412926},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:10:21 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.59299999999999997,"15min":2.3690000000000002},"min":{"1min":0.40000000000000002,"5min":0.247,"15min":0.19800000000000001},"max":{"1min":1.069,"5min":3.4359999999999999,"15min":49.168999999999997},"last":0.86699999999999999},{"interface":"front","average":{"1min":0.747,"5min":0.67400000000000004,"15min":2.5110000000000001},"min":{"1min":0.51000000000000001,"5min":0.249,"15min":0.20499999999999999},"max":{"1min":1.208,"5min":3.4670000000000001,"15min":50.673000000000002},"last":0.998}]},{"osd":2,"last update":"Sat Sep 23 19:09:55 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":1.1850000000000001,"15min":2.0710000000000002},"min":{"1min":0.432,"5min":0.28699999999999998,"15min":0.215},"max":{"1min":1.327,"5min":11.497,"15min":59.381999999999998},"last":0.748},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.995,"15min":1.7150000000000001},"min":{"1min":0.42599999999999999,"5min":0.26800000000000002,"15min":0.22900000000000001},"max":{"1min":0.93600000000000005,"5min":9.5190000000000001,"15min":59.783000000000001},"last":1.242}]},{"osd":3,"last update":"Sat Sep 23 19:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.72099999999999997,"5min":0.65000000000000002,"15min":2.1139999999999999},"min":{"1min":0.42699999999999999,"5min":0.27400000000000002,"15min":0.23200000000000001},"max":{"1min":1.4390000000000001,"5min":2.641,"15min":113.375},"last":1.042},{"interface":"front","average":{"1min":0.81799999999999995,"5min":0.754,"15min":2.1669999999999998},"min":{"1min":0.45200000000000001,"5min":0.32900000000000001,"15min":0.217},"max":{"1min":1.22,"5min":2.98,"15min":113.327},"last":1.4670000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":0.84399999999999997,"15min":1.349},"min":{"1min":0.61299999999999999,"5min":0.40500000000000003,"15min":0.33200000000000002},"max":{"1min":1.333,"5min":1.375,"15min":55.710999999999999},"last":1.147},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.85799999999999998,"15min":1.3400000000000001},"min":{"1min":0.65600000000000003,"5min":0.51200000000000001,"15min":0.30299999999999999},"max":{"1min":1.3640000000000001,"5min":1.548,"15min":46.509},"last":1.3400000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":0.998,"15min":0.998},"min":{"1min":0.65800000000000003,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.363,"5min":1.472,"15min":1.472},"last":1.538},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.046,"15min":1.038},"min":{"1min":0.88300000000000001,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.4279999999999999,"5min":1.4630000000000001,"15min":1.4630000000000001},"last":1.2809999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:10:21 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":0.94799999999999995,"15min":1.3260000000000001},"min":{"1min":0.71299999999999997,"5min":0.47899999999999998,"15min":0.36799999999999999},"max":{"1min":1.4650000000000001,"5min":5.3789999999999996,"15min":55.796999999999997},"last":1.379},{"interface":"front","average":{"1min":1.069,"5min":0.93799999999999994,"15min":1.3779999999999999},"min":{"1min":0.73399999999999999,"5min":0.51600000000000001,"15min":0.38600000000000001},"max":{"1min":1.4430000000000001,"5min":2.9020000000000001,"15min":46.411999999999999},"last":1.6200000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:10:06 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.085,"15min":1.085},"min":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"max":{"1min":1.3280000000000001,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":1.5820000000000001},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.0680000000000001,"15min":1.0680000000000001},"min":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"max":{"1min":1.504,"5min":1.504,"15min":1.504},"last":1.3129999999999999}]}]},{"osd":2,"up_from":98,"seq":420906795529,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1000288,"kb_used_data":8856,"kb_used_omap":11,"kb_used_meta":991412,"kb_avail":92744864,"statfs":{"total":95995035648,"available":94970740736,"internally_reserved":0,"allocated":9068544,"data_stored":7256874,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11949,"internal_metadata":1015206227},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:09:52 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.749,"15min":2.742},"min":{"1min":0.31900000000000001,"5min":0.20799999999999999,"15min":0.16600000000000001},"max":{"1min":1.5089999999999999,"5min":7.452,"15min":143.048},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.84499999999999997,"15min":2.9470000000000001},"min":{"1min":0.32200000000000001,"5min":0.27400000000000002,"15min":0.187},"max":{"1min":0.96499999999999997,"5min":7.2329999999999997,"15min":143.25200000000001},"last":0.56999999999999995}]},{"osd":1,"last update":"Sat Sep 23 19:09:52 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.84799999999999998,"15min":2.0979999999999999},"min":{"1min":0.501,"5min":0.24399999999999999,"15min":0.20599999999999999},"max":{"1min":1.0149999999999999,"5min":7.524,"15min":104.43899999999999},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.755,"5min":0.88400000000000001,"15min":2.323},"min":{"1min":0.35499999999999998,"5min":0.27900000000000003,"15min":0.23799999999999999},"max":{"1min":1.121,"5min":6.4729999999999999,"15min":99.867999999999995},"last":0.77700000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:10:35 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.59499999999999997,"15min":1.8919999999999999},"min":{"1min":0.443,"5min":0.252,"15min":0.222},"max":{"1min":1.347,"5min":1.347,"15min":85.102999999999994},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.86199999999999999,"5min":0.64500000000000002,"15min":1.883},"min":{"1min":0.59799999999999998,"5min":0.33300000000000002,"15min":0.23200000000000001},"max":{"1min":1.2130000000000001,"5min":1.2130000000000001,"15min":84.531999999999996},"last":0.84699999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:10:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":0.878,"15min":1.0629999999999999},"min":{"1min":0.83999999999999997,"5min":0.40300000000000002,"15min":0.31900000000000001},"max":{"1min":1.3799999999999999,"5min":1.3799999999999999,"15min":8.9969999999999999},"last":0.92900000000000005},{"interface":"front","average":{"1min":1.0429999999999999,"5min":0.89200000000000002,"15min":1.028},"min":{"1min":0.66500000000000004,"5min":0.45300000000000001,"15min":0.32000000000000001},"max":{"1min":1.536,"5min":1.536,"15min":6.0049999999999999},"last":0.81899999999999995}]},{"osd":5,"last update":"Sat Sep 23 19:10:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":0.997,"15min":0.98399999999999999},"min":{"1min":0.69799999999999995,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.494,"5min":1.6020000000000001,"15min":1.6020000000000001},"last":1.022},{"interface":"front","average":{"1min":1.109,"5min":1.0609999999999999,"15min":1.0529999999999999},"min":{"1min":0.82599999999999996,"5min":0.72499999999999998,"15min":0.72499999999999998},"max":{"1min":1.327,"5min":1.6659999999999999,"15min":1.6659999999999999},"last":1.2290000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:09:52 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":0.92400000000000004,"15min":1.012},"min":{"1min":0.70999999999999996,"5min":0.371,"15min":0.32800000000000001},"max":{"1min":2.0489999999999999,"5min":5.8140000000000001,"15min":6.6369999999999996},"last":1.1020000000000001},{"interface":"front","average":{"1min":1.107,"5min":1.0960000000000001,"15min":1.121},"min":{"1min":0.66400000000000003,"5min":0.45600000000000002,"15min":0.36699999999999999},"max":{"1min":1.7969999999999999,"5min":9.048,"15min":9.048},"last":1.0469999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:10:02 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":1.117,"15min":1.117},"min":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"max":{"1min":1.744,"5min":1.744,"15min":1.744},"last":1.28},{"interface":"front","average":{"1min":1.1519999999999999,"5min":1.1519999999999999,"15min":1.1519999999999999},"min":{"1min":0.623,"5min":0.623,"15min":0.623},"max":{"1min":1.6879999999999999,"5min":1.6879999999999999,"15min":1.6879999999999999},"last":1.1910000000000001}]}]},{"osd":3,"up_from":103,"seq":442381632004,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":935644,"kb_used_data":4820,"kb_used_omap":4,"kb_used_meta":930811,"kb_avail":92809508,"statfs":{"total":95995035648,"available":95036936192,"internally_reserved":0,"allocated":4935680,"data_stored":3064751,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4826,"internal_metadata":953150758},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.55600000000000005,"15min":2.7320000000000002},"min":{"1min":0.34599999999999997,"5min":0.252,"15min":0.191},"max":{"1min":1.0089999999999999,"5min":1.034,"15min":296.55399999999997},"last":0.871},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.57899999999999996,"15min":2.8250000000000002},"min":{"1min":0.38900000000000001,"5min":0.27900000000000003,"15min":0.18099999999999999},"max":{"1min":1.222,"5min":1.2430000000000001,"15min":296.36900000000003},"last":0.626}]},{"osd":1,"last update":"Sat Sep 23 19:10:18 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":0.66900000000000004,"15min":3.4860000000000002},"min":{"1min":0.374,"5min":0.252,"15min":0.22500000000000001},"max":{"1min":1.278,"5min":4.3940000000000001,"15min":343.79300000000001},"last":0.93799999999999994},{"interface":"front","average":{"1min":0.76800000000000002,"5min":0.76600000000000001,"15min":3.484},"min":{"1min":0.41199999999999998,"5min":0.32400000000000001,"15min":0.214},"max":{"1min":1.1899999999999999,"5min":7.8410000000000002,"15min":343.73000000000002},"last":0.97699999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.71799999999999997,"5min":0.63100000000000001,"15min":3.0049999999999999},"min":{"1min":0.39500000000000002,"5min":0.317,"15min":0.23699999999999999},"max":{"1min":1.385,"5min":1.385,"15min":178.06200000000001},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.76000000000000001,"5min":0.65900000000000003,"15min":2.8290000000000002},"min":{"1min":0.45900000000000002,"5min":0.29799999999999999,"15min":0.26300000000000001},"max":{"1min":1.4159999999999999,"5min":2.1429999999999998,"15min":178.267},"last":1.1439999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":0.82599999999999996,"15min":1.149},"min":{"1min":0.67800000000000005,"5min":0.438,"15min":0.33100000000000002},"max":{"1min":1.4390000000000001,"5min":1.4390000000000001,"15min":12.028},"last":1.3600000000000001},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.85999999999999999,"15min":1.2110000000000001},"min":{"1min":0.63800000000000001,"5min":0.47699999999999998,"15min":0.34300000000000003},"max":{"1min":1.363,"5min":1.7090000000000001,"15min":11.369},"last":1.0669999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:10:42 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":0.97399999999999998,"15min":0.95399999999999996},"min":{"1min":0.82899999999999996,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.7190000000000001,"5min":1.7190000000000001,"15min":1.7190000000000001},"last":0.91400000000000003},{"interface":"front","average":{"1min":1.135,"5min":0.98699999999999999,"15min":0.96299999999999997},"min":{"1min":0.69499999999999995,"5min":0.624,"15min":0.624},"max":{"1min":1.5289999999999999,"5min":1.633,"15min":1.633},"last":1.226}]},{"osd":6,"last update":"Sat Sep 23 19:10:30 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":0.89700000000000002,"15min":1.292},"min":{"1min":0.79200000000000004,"5min":0.42499999999999999,"15min":0.33700000000000002},"max":{"1min":1.792,"5min":1.792,"15min":12.926},"last":1.042},{"interface":"front","average":{"1min":1.131,"5min":0.94199999999999995,"15min":1.242},"min":{"1min":0.752,"5min":0.51900000000000002,"15min":0.38500000000000001},"max":{"1min":1.474,"5min":1.839,"15min":11.300000000000001},"last":1.2929999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:10:05 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.0720000000000001,"15min":1.0720000000000001},"min":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":1.792,"5min":1.792,"15min":1.792},"last":1.4119999999999999},{"interface":"front","average":{"1min":0.97399999999999998,"5min":0.97399999999999998,"15min":0.97399999999999998},"min":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"max":{"1min":1.329,"5min":1.329,"15min":1.329},"last":1.448}]}]},{"osd":4,"up_from":0,"seq":1228360646834,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":360,"seq":1546188226588,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":576660,"kb_used_data":238860,"kb_used_omap":5,"kb_used_meta":337786,"kb_avail":93168492,"statfs":{"total":95995035648,"available":95404535808,"internally_reserved":0,"allocated":244592640,"data_stored":243644967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6117,"internal_metadata":345892891},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.81799999999999995,"15min":0.79800000000000004},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.3100000000000001,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":0.96999999999999997},{"interface":"front","average":{"1min":0.91900000000000004,"5min":0.89700000000000002,"15min":0.89300000000000002},"min":{"1min":0.56499999999999995,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":1.1699999999999999,"5min":1.5529999999999999,"15min":1.5529999999999999},"last":0.93400000000000005}]},{"osd":1,"last update":"Sat Sep 23 19:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":0.83299999999999996,"15min":0.82099999999999995},"min":{"1min":0.57099999999999995,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.216,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.84399999999999997},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.86499999999999999,"15min":0.85099999999999998},"min":{"1min":0.55300000000000005,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.228,"5min":1.24,"15min":1.24},"last":1.1970000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":0.89200000000000002,"15min":0.88},"min":{"1min":0.63200000000000001,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.1910000000000001,"5min":1.242,"15min":1.242},"last":1.1160000000000001},{"interface":"front","average":{"1min":0.874,"5min":0.85799999999999998,"15min":0.85499999999999998},"min":{"1min":0.55800000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.242,"5min":1.242,"15min":1.242},"last":1.139}]},{"osd":3,"last update":"Sat Sep 23 19:10:38 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":0.95899999999999996,"15min":0.94999999999999996},"min":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.27,"5min":1.5249999999999999,"15min":1.5249999999999999},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"min":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.1910000000000001,"5min":2.121,"15min":2.121},"last":1.056}]},{"osd":4,"last update":"Sat Sep 23 19:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":0.84599999999999997,"15min":0.84599999999999997},"min":{"1min":0.59499999999999997,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.173,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":0.749},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.93899999999999995,"15min":0.93400000000000005},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.2929999999999999,"5min":1.399,"15min":1.399},"last":1.0229999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.96199999999999997,"15min":0.95899999999999996},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.2749999999999999,"5min":1.2889999999999999,"15min":1.2889999999999999},"last":1.1579999999999999},{"interface":"front","average":{"1min":0.96699999999999997,"5min":0.98699999999999999,"15min":0.99099999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":1.325,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":0.89800000000000002}]},{"osd":7,"last update":"Sat Sep 23 19:10:03 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.109,"15min":1.109},"min":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"max":{"1min":1.3520000000000001,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":1.087},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"min":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"max":{"1min":1.4670000000000001,"5min":1.4670000000000001,"15min":1.4670000000000001},"last":1.25}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4224,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8388608,"data_stored":8388608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2112,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7725,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3429,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11243,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6117,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11916,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:10:57.560 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:10:57.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:57 smithi098 bash[79516]: cluster 2023-09-23T19:10:56.717635+0000 mgr.y (mgr.26054) 1602 : cluster 0 pgmap v1793: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:57.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:57 smithi098 bash[76817]: cluster 2023-09-23T19:10:56.717635+0000 mgr.y (mgr.26054) 1602 : cluster 0 pgmap v1793: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:10:58.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:57 smithi173 bash[59422]: cluster 2023-09-23T19:10:56.717635+0000 mgr.y (mgr.26054) 1602 : cluster 0 pgmap v1793: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:00.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:10:59 smithi098 bash[76817]: cluster 2023-09-23T19:10:58.718753+0000 mgr.y (mgr.26054) 1603 : cluster 0 pgmap v1794: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:00.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:10:59 smithi098 bash[79516]: cluster 2023-09-23T19:10:58.718753+0000 mgr.y (mgr.26054) 1603 : cluster 0 pgmap v1794: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:00.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:10:59 smithi173 bash[59422]: cluster 2023-09-23T19:10:58.718753+0000 mgr.y (mgr.26054) 1603 : cluster 0 pgmap v1794: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:00.569 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T19:11:00.569 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-09-23T19:11:00.569 INFO:teuthology.orchestra.run.smithi173.stdout: "complaint_time": 30, 2023-09-23T19:11:00.569 INFO:teuthology.orchestra.run.smithi173.stdout: "num_blocked_ops": 0 2023-09-23T19:11:00.570 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T19:11:01.580 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T19:11:01.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:01 smithi098 bash[76817]: cluster 2023-09-23T19:11:00.719428+0000 mgr.y (mgr.26054) 1604 : cluster 0 pgmap v1795: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:01.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:01 smithi098 bash[79516]: cluster 2023-09-23T19:11:00.719428+0000 mgr.y (mgr.26054) 1604 : cluster 0 pgmap v1795: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:02.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:01 smithi173 bash[59422]: cluster 2023-09-23T19:11:00.719428+0000 mgr.y (mgr.26054) 1604 : cluster 0 pgmap v1795: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:02.375 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:11:03.923 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:03 smithi098 bash[79516]: cluster 2023-09-23T19:11:02.719939+0000 mgr.y (mgr.26054) 1605 : cluster 0 pgmap v1796: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:03.923 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:03 smithi098 bash[76817]: cluster 2023-09-23T19:11:02.719939+0000 mgr.y (mgr.26054) 1605 : cluster 0 pgmap v1796: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:04.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:03 smithi173 bash[59422]: cluster 2023-09-23T19:11:02.719939+0000 mgr.y (mgr.26054) 1605 : cluster 0 pgmap v1796: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:05.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:05 smithi098 bash[79516]: cluster 2023-09-23T19:11:04.720564+0000 mgr.y (mgr.26054) 1606 : cluster 0 pgmap v1797: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:05.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:05 smithi098 bash[76817]: cluster 2023-09-23T19:11:04.720564+0000 mgr.y (mgr.26054) 1606 : cluster 0 pgmap v1797: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:06.068 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T19:11:06.069 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-23T19:11:06.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:05 smithi173 bash[59422]: cluster 2023-09-23T19:11:04.720564+0000 mgr.y (mgr.26054) 1606 : cluster 0 pgmap v1797: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:06.356 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:11:06.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:11:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:11:06] "GET /metrics HTTP/1.1" 200 34165 "" "Prometheus/2.33.4" 2023-09-23T19:11:07.056 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:06 smithi098 bash[76817]: audit 2023-09-23T19:11:06.044955+0000 mgr.y (mgr.26054) 1607 : audit 0 from='client.36831 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:11:07.057 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:06 smithi098 bash[79516]: audit 2023-09-23T19:11:06.044955+0000 mgr.y (mgr.26054) 1607 : audit 0 from='client.36831 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:11:07.058 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":1797,"stamp":"2023-09-23T19:11:04.720206+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":12228,"num_read_kb":26865,"num_write":1950,"num_write_kb":27871,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":164,"num_bytes_recovered":12591255,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2144,"ondisk_log_size":2144,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":221,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5851576,"kb_used_data":249600,"kb_used_omap":51,"kb_used_meta":5601868,"kb_avail":650364488,"statfs":{"total":671965249536,"available":665973235712,"internally_reserved":0,"allocated":255590400,"data_stored":245207124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":52652,"internal_metadata":5736313428},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004628"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":107,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.788298+0000","last_change":"2023-09-23T18:59:13.709108+0000","last_active":"2023-09-23T19:10:37.788298+0000","last_peered":"2023-09-23T19:10:37.788298+0000","last_clean":"2023-09-23T19:10:37.788298+0000","last_became_active":"2023-09-23T18:59:13.708511+0000","last_became_peered":"2023-09-23T18:59:13.708511+0000","last_unstale":"2023-09-23T19:10:37.788298+0000","last_undegraded":"2023-09-23T19:10:37.788298+0000","last_fullsized":"2023-09-23T19:10:37.788298+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:59.295137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":121,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.785678+0000","last_change":"2023-09-23T18:59:13.702575+0000","last_active":"2023-09-23T19:10:37.785678+0000","last_peered":"2023-09-23T19:10:37.785678+0000","last_clean":"2023-09-23T19:10:37.785678+0000","last_became_active":"2023-09-23T18:59:13.701967+0000","last_became_peered":"2023-09-23T18:59:13.701967+0000","last_unstale":"2023-09-23T19:10:37.785678+0000","last_undegraded":"2023-09-23T19:10:37.785678+0000","last_fullsized":"2023-09-23T19:10:37.785678+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:30:12.982518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":388,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.787431+0000","last_change":"2023-09-23T18:59:13.707831+0000","last_active":"2023-09-23T19:10:37.787431+0000","last_peered":"2023-09-23T19:10:37.787431+0000","last_clean":"2023-09-23T19:10:37.787431+0000","last_became_active":"2023-09-23T18:59:13.707524+0000","last_became_peered":"2023-09-23T18:59:13.707524+0000","last_unstale":"2023-09-23T19:10:37.787431+0000","last_undegraded":"2023-09-23T19:10:37.787431+0000","last_fullsized":"2023-09-23T19:10:37.787431+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:42.677892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":442,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:45.319707+0000","last_change":"2023-09-23T19:09:03.326411+0000","last_active":"2023-09-23T19:10:45.319707+0000","last_peered":"2023-09-23T19:10:45.319707+0000","last_clean":"2023-09-23T19:10:45.319707+0000","last_became_active":"2023-09-23T19:09:03.325980+0000","last_became_peered":"2023-09-23T19:09:03.325980+0000","last_unstale":"2023-09-23T19:10:45.319707+0000","last_undegraded":"2023-09-23T19:10:45.319707+0000","last_fullsized":"2023-09-23T19:10:45.319707+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:43.638573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":113,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T19:10:47.136613+0000","last_change":"2023-09-23T18:58:38.718387+0000","last_active":"2023-09-23T19:10:47.136613+0000","last_peered":"2023-09-23T19:10:47.136613+0000","last_clean":"2023-09-23T19:10:47.136613+0000","last_became_active":"2023-09-23T18:58:38.717999+0000","last_became_peered":"2023-09-23T18:58:38.717999+0000","last_unstale":"2023-09-23T19:10:47.136613+0000","last_undegraded":"2023-09-23T19:10:47.136613+0000","last_fullsized":"2023-09-23T19:10:47.136613+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:05.073213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":99,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.802467+0000","last_change":"2023-09-23T19:08:37.780028+0000","last_active":"2023-09-23T19:10:37.802467+0000","last_peered":"2023-09-23T19:10:37.802467+0000","last_clean":"2023-09-23T19:10:37.802467+0000","last_became_active":"2023-09-23T19:08:37.779430+0000","last_became_peered":"2023-09-23T19:08:37.779430+0000","last_unstale":"2023-09-23T19:10:37.802467+0000","last_undegraded":"2023-09-23T19:10:37.802467+0000","last_fullsized":"2023-09-23T19:10:37.802467+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":94,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.802529+0000","last_change":"2023-09-23T19:08:37.779252+0000","last_active":"2023-09-23T19:10:37.802529+0000","last_peered":"2023-09-23T19:10:37.802529+0000","last_clean":"2023-09-23T19:10:37.802529+0000","last_became_active":"2023-09-23T19:08:37.778722+0000","last_became_peered":"2023-09-23T19:08:37.778722+0000","last_unstale":"2023-09-23T19:10:37.802529+0000","last_undegraded":"2023-09-23T19:10:37.802529+0000","last_fullsized":"2023-09-23T19:10:37.802529+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":113,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.785712+0000","last_change":"2023-09-23T18:58:38.710920+0000","last_active":"2023-09-23T19:10:37.785712+0000","last_peered":"2023-09-23T19:10:37.785712+0000","last_clean":"2023-09-23T19:10:37.785712+0000","last_became_active":"2023-09-23T18:58:38.710563+0000","last_became_peered":"2023-09-23T18:58:38.710563+0000","last_unstale":"2023-09-23T19:10:37.785712+0000","last_undegraded":"2023-09-23T19:10:37.785712+0000","last_fullsized":"2023-09-23T19:10:37.785712+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:42.386631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":11,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.803136+0000","last_change":"2023-09-23T19:10:37.800599+0000","last_active":"2023-09-23T19:10:37.803136+0000","last_peered":"2023-09-23T19:10:37.803136+0000","last_clean":"2023-09-23T19:10:37.803136+0000","last_became_active":"2023-09-23T19:10:37.800130+0000","last_became_peered":"2023-09-23T19:10:37.800130+0000","last_unstale":"2023-09-23T19:10:37.803136+0000","last_undegraded":"2023-09-23T19:10:37.803136+0000","last_fullsized":"2023-09-23T19:10:37.803136+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":388,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.790397+0000","last_change":"2023-09-23T19:09:42.642530+0000","last_active":"2023-09-23T19:10:37.790397+0000","last_peered":"2023-09-23T19:10:37.790397+0000","last_clean":"2023-09-23T19:10:37.790397+0000","last_became_active":"2023-09-23T19:09:42.641782+0000","last_became_peered":"2023-09-23T19:09:42.641782+0000","last_unstale":"2023-09-23T19:10:37.790397+0000","last_undegraded":"2023-09-23T19:10:37.790397+0000","last_fullsized":"2023-09-23T19:10:37.790397+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":278,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.791274+0000","last_change":"2023-09-23T19:09:03.320766+0000","last_active":"2023-09-23T19:10:37.791274+0000","last_peered":"2023-09-23T19:10:37.791274+0000","last_clean":"2023-09-23T19:10:37.791274+0000","last_became_active":"2023-09-23T19:09:03.320264+0000","last_became_peered":"2023-09-23T19:09:03.320264+0000","last_unstale":"2023-09-23T19:10:37.791274+0000","last_undegraded":"2023-09-23T19:10:37.791274+0000","last_fullsized":"2023-09-23T19:10:37.791274+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":342,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.787766+0000","last_change":"2023-09-23T18:33:53.850096+0000","last_active":"2023-09-23T19:10:37.787766+0000","last_peered":"2023-09-23T19:10:37.787766+0000","last_clean":"2023-09-23T19:10:37.787766+0000","last_became_active":"2023-09-23T18:33:53.849811+0000","last_became_peered":"2023-09-23T18:33:53.849811+0000","last_unstale":"2023-09-23T19:10:37.787766+0000","last_undegraded":"2023-09-23T19:10:37.787766+0000","last_fullsized":"2023-09-23T19:10:37.787766+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:44.938072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":104,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.790458+0000","last_change":"2023-09-23T19:09:03.313359+0000","last_active":"2023-09-23T19:10:37.790458+0000","last_peered":"2023-09-23T19:10:37.790458+0000","last_clean":"2023-09-23T19:10:37.790458+0000","last_became_active":"2023-09-23T19:09:03.312806+0000","last_became_peered":"2023-09-23T19:09:03.312806+0000","last_unstale":"2023-09-23T19:10:37.790458+0000","last_undegraded":"2023-09-23T19:10:37.790458+0000","last_fullsized":"2023-09-23T19:10:37.790458+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":355,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T19:10:47.154372+0000","last_change":"2023-09-23T19:08:37.769682+0000","last_active":"2023-09-23T19:10:47.154372+0000","last_peered":"2023-09-23T19:10:47.154372+0000","last_clean":"2023-09-23T19:10:47.154372+0000","last_became_active":"2023-09-23T19:08:37.768331+0000","last_became_peered":"2023-09-23T19:08:37.768331+0000","last_unstale":"2023-09-23T19:10:47.154372+0000","last_undegraded":"2023-09-23T19:10:47.154372+0000","last_fullsized":"2023-09-23T19:10:47.154372+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:50.441487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":277,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.790629+0000","last_change":"2023-09-23T19:09:03.314758+0000","last_active":"2023-09-23T19:10:37.790629+0000","last_peered":"2023-09-23T19:10:37.790629+0000","last_clean":"2023-09-23T19:10:37.790629+0000","last_became_active":"2023-09-23T19:09:03.314277+0000","last_became_peered":"2023-09-23T19:09:03.314277+0000","last_unstale":"2023-09-23T19:10:37.790629+0000","last_undegraded":"2023-09-23T19:10:37.790629+0000","last_fullsized":"2023-09-23T19:10:37.790629+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":414,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.785374+0000","last_change":"2023-09-23T19:09:03.319623+0000","last_active":"2023-09-23T19:10:37.785374+0000","last_peered":"2023-09-23T19:10:37.785374+0000","last_clean":"2023-09-23T19:10:37.785374+0000","last_became_active":"2023-09-23T19:09:03.319283+0000","last_became_peered":"2023-09-23T19:09:03.319283+0000","last_unstale":"2023-09-23T19:10:37.785374+0000","last_undegraded":"2023-09-23T19:10:37.785374+0000","last_fullsized":"2023-09-23T19:10:37.785374+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:34.646037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":165,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.789986+0000","last_change":"2023-09-23T19:09:03.326943+0000","last_active":"2023-09-23T19:10:37.789986+0000","last_peered":"2023-09-23T19:10:37.789986+0000","last_clean":"2023-09-23T19:10:37.789986+0000","last_became_active":"2023-09-23T19:09:03.326526+0000","last_became_peered":"2023-09-23T19:09:03.326526+0000","last_unstale":"2023-09-23T19:10:37.789986+0000","last_undegraded":"2023-09-23T19:10:37.789986+0000","last_fullsized":"2023-09-23T19:10:37.789986+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.803140+0000","last_change":"2023-09-23T19:10:37.801034+0000","last_active":"2023-09-23T19:10:37.803140+0000","last_peered":"2023-09-23T19:10:37.803140+0000","last_clean":"2023-09-23T19:10:37.803140+0000","last_became_active":"2023-09-23T19:10:37.800310+0000","last_became_peered":"2023-09-23T19:10:37.800310+0000","last_unstale":"2023-09-23T19:10:37.803140+0000","last_undegraded":"2023-09-23T19:10:37.803140+0000","last_fullsized":"2023-09-23T19:10:37.803140+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":260,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.789866+0000","last_change":"2023-09-23T19:09:03.315467+0000","last_active":"2023-09-23T19:10:37.789866+0000","last_peered":"2023-09-23T19:10:37.789866+0000","last_clean":"2023-09-23T19:10:37.789866+0000","last_became_active":"2023-09-23T19:09:03.313654+0000","last_became_peered":"2023-09-23T19:09:03.313654+0000","last_unstale":"2023-09-23T19:10:37.789866+0000","last_undegraded":"2023-09-23T19:10:37.789866+0000","last_fullsized":"2023-09-23T19:10:37.789866+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":133,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T19:10:47.136664+0000","last_change":"2023-09-23T19:09:03.319829+0000","last_active":"2023-09-23T19:10:47.136664+0000","last_peered":"2023-09-23T19:10:47.136664+0000","last_clean":"2023-09-23T19:10:47.136664+0000","last_became_active":"2023-09-23T19:09:03.319522+0000","last_became_peered":"2023-09-23T19:09:03.319522+0000","last_unstale":"2023-09-23T19:10:47.136664+0000","last_undegraded":"2023-09-23T19:10:47.136664+0000","last_fullsized":"2023-09-23T19:10:47.136664+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:57:03.744906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":266,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.852171+0000","last_change":"2023-09-23T19:10:37.851745+0000","last_active":"2023-09-23T19:10:37.852171+0000","last_peered":"2023-09-23T19:10:37.852171+0000","last_clean":"2023-09-23T19:10:37.852171+0000","last_became_active":"2023-09-23T19:10:37.797727+0000","last_became_peered":"2023-09-23T19:10:37.797727+0000","last_unstale":"2023-09-23T19:10:37.852171+0000","last_undegraded":"2023-09-23T19:10:37.852171+0000","last_fullsized":"2023-09-23T19:10:37.852171+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:36.467704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":65,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.786014+0000","last_change":"2023-09-23T19:09:03.318996+0000","last_active":"2023-09-23T19:10:37.786014+0000","last_peered":"2023-09-23T19:10:37.786014+0000","last_clean":"2023-09-23T19:10:37.786014+0000","last_became_active":"2023-09-23T19:09:03.318593+0000","last_became_peered":"2023-09-23T19:09:03.318593+0000","last_unstale":"2023-09-23T19:10:37.786014+0000","last_undegraded":"2023-09-23T19:10:37.786014+0000","last_fullsized":"2023-09-23T19:10:37.786014+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:23.764508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":257,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.791029+0000","last_change":"2023-09-23T19:09:03.321932+0000","last_active":"2023-09-23T19:10:37.791029+0000","last_peered":"2023-09-23T19:10:37.791029+0000","last_clean":"2023-09-23T19:10:37.791029+0000","last_became_active":"2023-09-23T19:09:03.320840+0000","last_became_peered":"2023-09-23T19:09:03.320840+0000","last_unstale":"2023-09-23T19:10:37.791029+0000","last_undegraded":"2023-09-23T19:10:37.791029+0000","last_fullsized":"2023-09-23T19:10:37.791029+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.802967+0000","last_change":"2023-09-23T19:10:37.800494+0000","last_active":"2023-09-23T19:10:37.802967+0000","last_peered":"2023-09-23T19:10:37.802967+0000","last_clean":"2023-09-23T19:10:37.802967+0000","last_became_active":"2023-09-23T19:10:37.800131+0000","last_became_peered":"2023-09-23T19:10:37.800131+0000","last_unstale":"2023-09-23T19:10:37.802967+0000","last_undegraded":"2023-09-23T19:10:37.802967+0000","last_fullsized":"2023-09-23T19:10:37.802967+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":292,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.790363+0000","last_change":"2023-09-23T19:09:03.321485+0000","last_active":"2023-09-23T19:10:37.790363+0000","last_peered":"2023-09-23T19:10:37.790363+0000","last_clean":"2023-09-23T19:10:37.790363+0000","last_became_active":"2023-09-23T19:09:03.320932+0000","last_became_peered":"2023-09-23T19:09:03.320932+0000","last_unstale":"2023-09-23T19:10:37.790363+0000","last_undegraded":"2023-09-23T19:10:37.790363+0000","last_fullsized":"2023-09-23T19:10:37.790363+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":370,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.787556+0000","last_change":"2023-09-23T19:03:25.324518+0000","last_active":"2023-09-23T19:10:37.787556+0000","last_peered":"2023-09-23T19:10:37.787556+0000","last_clean":"2023-09-23T19:10:37.787556+0000","last_became_active":"2023-09-23T19:03:25.324166+0000","last_became_peered":"2023-09-23T19:03:25.324166+0000","last_unstale":"2023-09-23T19:10:37.787556+0000","last_undegraded":"2023-09-23T19:10:37.787556+0000","last_fullsized":"2023-09-23T19:10:37.787556+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:24.379778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":289,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.847203+0000","last_change":"2023-09-23T19:10:37.846671+0000","last_active":"2023-09-23T19:10:37.847203+0000","last_peered":"2023-09-23T19:10:37.847203+0000","last_clean":"2023-09-23T19:10:37.847203+0000","last_became_active":"2023-09-23T19:10:37.800522+0000","last_became_peered":"2023-09-23T19:10:37.800522+0000","last_unstale":"2023-09-23T19:10:37.847203+0000","last_undegraded":"2023-09-23T19:10:37.847203+0000","last_fullsized":"2023-09-23T19:10:37.847203+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":260,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.792215+0000","last_change":"2023-09-23T19:09:02.306080+0000","last_active":"2023-09-23T19:10:37.792215+0000","last_peered":"2023-09-23T19:10:37.792215+0000","last_clean":"2023-09-23T19:10:37.792215+0000","last_became_active":"2023-09-23T19:09:02.305679+0000","last_became_peered":"2023-09-23T19:09:02.305679+0000","last_unstale":"2023-09-23T19:10:37.792215+0000","last_undegraded":"2023-09-23T19:10:37.792215+0000","last_fullsized":"2023-09-23T19:10:37.792215+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:55.893544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":94,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.799345+0000","last_change":"2023-09-23T19:08:37.773654+0000","last_active":"2023-09-23T19:10:37.799345+0000","last_peered":"2023-09-23T19:10:37.799345+0000","last_clean":"2023-09-23T19:10:37.799345+0000","last_became_active":"2023-09-23T19:08:37.773196+0000","last_became_peered":"2023-09-23T19:08:37.773196+0000","last_unstale":"2023-09-23T19:10:37.799345+0000","last_undegraded":"2023-09-23T19:10:37.799345+0000","last_fullsized":"2023-09-23T19:10:37.799345+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":407,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.789415+0000","last_change":"2023-09-23T19:09:42.637333+0000","last_active":"2023-09-23T19:10:37.789415+0000","last_peered":"2023-09-23T19:10:37.789415+0000","last_clean":"2023-09-23T19:10:37.789415+0000","last_became_active":"2023-09-23T19:09:42.637015+0000","last_became_peered":"2023-09-23T19:09:42.637015+0000","last_unstale":"2023-09-23T19:10:37.789415+0000","last_undegraded":"2023-09-23T19:10:37.789415+0000","last_fullsized":"2023-09-23T19:10:37.789415+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:47.410433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":130,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.784872+0000","last_change":"2023-09-23T18:59:13.721533+0000","last_active":"2023-09-23T19:10:37.784872+0000","last_peered":"2023-09-23T19:10:37.784872+0000","last_clean":"2023-09-23T19:10:37.784872+0000","last_became_active":"2023-09-23T18:59:13.711121+0000","last_became_peered":"2023-09-23T18:59:13.711121+0000","last_unstale":"2023-09-23T19:10:37.784872+0000","last_undegraded":"2023-09-23T19:10:37.784872+0000","last_fullsized":"2023-09-23T19:10:37.784872+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:51.950774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":333,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.784876+0000","last_change":"2023-09-23T18:59:13.702825+0000","last_active":"2023-09-23T19:10:37.784876+0000","last_peered":"2023-09-23T19:10:37.784876+0000","last_clean":"2023-09-23T19:10:37.784876+0000","last_became_active":"2023-09-23T18:59:13.702503+0000","last_became_peered":"2023-09-23T18:59:13.702503+0000","last_unstale":"2023-09-23T19:10:37.784876+0000","last_undegraded":"2023-09-23T19:10:37.784876+0000","last_fullsized":"2023-09-23T19:10:37.784876+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.263337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":237,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.788179+0000","last_change":"2023-09-23T18:59:13.708841+0000","last_active":"2023-09-23T19:10:37.788179+0000","last_peered":"2023-09-23T19:10:37.788179+0000","last_clean":"2023-09-23T19:10:37.788179+0000","last_became_active":"2023-09-23T18:59:13.707760+0000","last_became_peered":"2023-09-23T18:59:13.707760+0000","last_unstale":"2023-09-23T19:10:37.788179+0000","last_undegraded":"2023-09-23T19:10:37.788179+0000","last_fullsized":"2023-09-23T19:10:37.788179+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:02.417127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":386,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.793048+0000","last_change":"2023-09-23T18:59:13.708710+0000","last_active":"2023-09-23T19:10:37.793048+0000","last_peered":"2023-09-23T19:10:37.793048+0000","last_clean":"2023-09-23T19:10:37.793048+0000","last_became_active":"2023-09-23T18:59:13.708349+0000","last_became_peered":"2023-09-23T18:59:13.708349+0000","last_unstale":"2023-09-23T19:10:37.793048+0000","last_undegraded":"2023-09-23T19:10:37.793048+0000","last_fullsized":"2023-09-23T19:10:37.793048+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:33.033640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":311,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.791743+0000","last_change":"2023-09-23T19:09:03.310639+0000","last_active":"2023-09-23T19:10:37.791743+0000","last_peered":"2023-09-23T19:10:37.791743+0000","last_clean":"2023-09-23T19:10:37.791743+0000","last_became_active":"2023-09-23T19:09:03.310175+0000","last_became_peered":"2023-09-23T19:09:03.310175+0000","last_unstale":"2023-09-23T19:10:37.791743+0000","last_undegraded":"2023-09-23T19:10:37.791743+0000","last_fullsized":"2023-09-23T19:10:37.791743+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:49.504419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":340,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.789312+0000","last_change":"2023-09-23T18:59:13.708690+0000","last_active":"2023-09-23T19:10:37.789312+0000","last_peered":"2023-09-23T19:10:37.789312+0000","last_clean":"2023-09-23T19:10:37.789312+0000","last_became_active":"2023-09-23T18:59:13.708248+0000","last_became_peered":"2023-09-23T18:59:13.708248+0000","last_unstale":"2023-09-23T19:10:37.789312+0000","last_undegraded":"2023-09-23T19:10:37.789312+0000","last_fullsized":"2023-09-23T19:10:37.789312+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:43:08.360496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":88,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.785012+0000","last_change":"2023-09-23T19:09:03.322253+0000","last_active":"2023-09-23T19:10:37.785012+0000","last_peered":"2023-09-23T19:10:37.785012+0000","last_clean":"2023-09-23T19:10:37.785012+0000","last_became_active":"2023-09-23T19:09:03.320102+0000","last_became_peered":"2023-09-23T19:09:03.320102+0000","last_unstale":"2023-09-23T19:10:37.785012+0000","last_undegraded":"2023-09-23T19:10:37.785012+0000","last_fullsized":"2023-09-23T19:10:37.785012+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:01.268221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":402,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.788809+0000","last_change":"2023-09-23T19:09:03.309621+0000","last_active":"2023-09-23T19:10:37.788809+0000","last_peered":"2023-09-23T19:10:37.788809+0000","last_clean":"2023-09-23T19:10:37.788809+0000","last_became_active":"2023-09-23T19:09:03.309195+0000","last_became_peered":"2023-09-23T19:09:03.309195+0000","last_unstale":"2023-09-23T19:10:37.788809+0000","last_undegraded":"2023-09-23T19:10:37.788809+0000","last_fullsized":"2023-09-23T19:10:37.788809+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:21.448033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"383'306","reported_seq":1451,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.788129+0000","last_change":"2023-09-23T19:09:23.687377+0000","last_active":"2023-09-23T19:10:37.788129+0000","last_peered":"2023-09-23T19:10:37.788129+0000","last_clean":"2023-09-23T19:10:37.788129+0000","last_became_active":"2023-09-23T19:09:02.306435+0000","last_became_peered":"2023-09-23T19:09:02.306435+0000","last_unstale":"2023-09-23T19:10:37.788129+0000","last_undegraded":"2023-09-23T19:10:37.788129+0000","last_fullsized":"2023-09-23T19:10:37.788129+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":306,"log_dups_size":0,"ondisk_log_size":306,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:54:30.990476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086649800000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1624,"num_read_kb":2591,"num_write":297,"num_write_kb":3316,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":94,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.800664+0000","last_change":"2023-09-23T19:08:37.774207+0000","last_active":"2023-09-23T19:10:37.800664+0000","last_peered":"2023-09-23T19:10:37.800664+0000","last_clean":"2023-09-23T19:10:37.800664+0000","last_became_active":"2023-09-23T19:08:37.773764+0000","last_became_peered":"2023-09-23T19:08:37.773764+0000","last_unstale":"2023-09-23T19:10:37.800664+0000","last_undegraded":"2023-09-23T19:10:37.800664+0000","last_fullsized":"2023-09-23T19:10:37.800664+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":392,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.800608+0000","last_change":"2023-09-23T19:10:37.800203+0000","last_active":"2023-09-23T19:10:37.800608+0000","last_peered":"2023-09-23T19:10:37.800608+0000","last_clean":"2023-09-23T19:10:37.800608+0000","last_became_active":"2023-09-23T19:10:37.799806+0000","last_became_peered":"2023-09-23T19:10:37.799806+0000","last_unstale":"2023-09-23T19:10:37.800608+0000","last_undegraded":"2023-09-23T19:10:37.800608+0000","last_fullsized":"2023-09-23T19:10:37.800608+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:49.948374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"383'125","reported_seq":477,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.799456+0000","last_change":"2023-09-23T19:09:23.642827+0000","last_active":"2023-09-23T19:10:37.799456+0000","last_peered":"2023-09-23T19:10:37.799456+0000","last_clean":"2023-09-23T19:10:37.799456+0000","last_became_active":"2023-09-23T19:08:37.774901+0000","last_became_peered":"2023-09-23T19:08:37.774901+0000","last_unstale":"2023-09-23T19:10:37.799456+0000","last_undegraded":"2023-09-23T19:10:37.799456+0000","last_fullsized":"2023-09-23T19:10:37.799456+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":125,"log_dups_size":0,"ondisk_log_size":125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1121,"num_read_kb":2820,"num_write":107,"num_write_kb":1462,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":107,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.788057+0000","last_change":"2023-09-23T18:59:13.703057+0000","last_active":"2023-09-23T19:10:37.788057+0000","last_peered":"2023-09-23T19:10:37.788057+0000","last_clean":"2023-09-23T19:10:37.788057+0000","last_became_active":"2023-09-23T18:59:13.702759+0000","last_became_peered":"2023-09-23T18:59:13.702759+0000","last_unstale":"2023-09-23T19:10:37.788057+0000","last_undegraded":"2023-09-23T19:10:37.788057+0000","last_fullsized":"2023-09-23T19:10:37.788057+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:53.278738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":409,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.788001+0000","last_change":"2023-09-23T18:58:38.726546+0000","last_active":"2023-09-23T19:10:37.788001+0000","last_peered":"2023-09-23T19:10:37.788001+0000","last_clean":"2023-09-23T19:10:37.788001+0000","last_became_active":"2023-09-23T18:58:38.724765+0000","last_became_peered":"2023-09-23T18:58:38.724765+0000","last_unstale":"2023-09-23T19:10:37.788001+0000","last_undegraded":"2023-09-23T19:10:37.788001+0000","last_fullsized":"2023-09-23T19:10:37.788001+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:05.322676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"383'289","reported_seq":1312,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.791585+0000","last_change":"2023-09-23T19:09:23.637617+0000","last_active":"2023-09-23T19:10:37.791585+0000","last_peered":"2023-09-23T19:10:37.791585+0000","last_clean":"2023-09-23T19:10:37.791585+0000","last_became_active":"2023-09-23T18:58:38.707923+0000","last_became_peered":"2023-09-23T18:58:38.707923+0000","last_unstale":"2023-09-23T19:10:37.791585+0000","last_undegraded":"2023-09-23T19:10:37.791585+0000","last_fullsized":"2023-09-23T19:10:37.791585+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":289,"log_dups_size":0,"ondisk_log_size":289,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:59.471903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047940899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1602,"num_read_kb":4202,"num_write":289,"num_write_kb":2667,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":345,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.787317+0000","last_change":"2023-09-23T18:33:53.849084+0000","last_active":"2023-09-23T19:10:37.787317+0000","last_peered":"2023-09-23T19:10:37.787317+0000","last_clean":"2023-09-23T19:10:37.787317+0000","last_became_active":"2023-09-23T18:33:53.848757+0000","last_became_peered":"2023-09-23T18:33:53.848757+0000","last_unstale":"2023-09-23T19:10:37.787317+0000","last_undegraded":"2023-09-23T19:10:37.787317+0000","last_fullsized":"2023-09-23T19:10:37.787317+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:15.267742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":415,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:42.065697+0000","last_change":"2023-09-23T19:08:37.776940+0000","last_active":"2023-09-23T19:10:42.065697+0000","last_peered":"2023-09-23T19:10:42.065697+0000","last_clean":"2023-09-23T19:10:42.065697+0000","last_became_active":"2023-09-23T19:08:37.776564+0000","last_became_peered":"2023-09-23T19:08:37.776564+0000","last_unstale":"2023-09-23T19:10:42.065697+0000","last_undegraded":"2023-09-23T19:10:42.065697+0000","last_fullsized":"2023-09-23T19:10:42.065697+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:44.299192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"383'317","reported_seq":1364,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.799778+0000","last_change":"2023-09-23T19:09:23.643011+0000","last_active":"2023-09-23T19:10:37.799778+0000","last_peered":"2023-09-23T19:10:37.799778+0000","last_clean":"2023-09-23T19:10:37.799778+0000","last_became_active":"2023-09-23T19:09:03.312411+0000","last_became_peered":"2023-09-23T19:09:03.312411+0000","last_unstale":"2023-09-23T19:10:37.799778+0000","last_undegraded":"2023-09-23T19:10:37.799778+0000","last_fullsized":"2023-09-23T19:10:37.799778+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":317,"log_dups_size":0,"ondisk_log_size":317,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1525,"num_read_kb":2676,"num_write":278,"num_write_kb":5451,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":497,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.789669+0000","last_change":"2023-09-23T19:09:02.309826+0000","last_active":"2023-09-23T19:10:37.789669+0000","last_peered":"2023-09-23T19:10:37.789669+0000","last_clean":"2023-09-23T19:10:37.789669+0000","last_became_active":"2023-09-23T19:09:02.309300+0000","last_became_peered":"2023-09-23T19:09:02.309300+0000","last_unstale":"2023-09-23T19:10:37.789669+0000","last_undegraded":"2023-09-23T19:10:37.789669+0000","last_fullsized":"2023-09-23T19:10:37.789669+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_clean_scrub_stamp":"2023-09-23T18:08:35.701374+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":150,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.797624+0000","last_change":"2023-09-23T19:10:37.797301+0000","last_active":"2023-09-23T19:10:37.797624+0000","last_peered":"2023-09-23T19:10:37.797624+0000","last_clean":"2023-09-23T19:10:37.797624+0000","last_became_active":"2023-09-23T19:10:37.796972+0000","last_became_peered":"2023-09-23T19:10:37.796972+0000","last_unstale":"2023-09-23T19:10:37.797624+0000","last_undegraded":"2023-09-23T19:10:37.797624+0000","last_fullsized":"2023-09-23T19:10:37.797624+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:25.458233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":51,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.791160+0000","last_change":"2023-09-23T19:09:03.315025+0000","last_active":"2023-09-23T19:10:37.791160+0000","last_peered":"2023-09-23T19:10:37.791160+0000","last_clean":"2023-09-23T19:10:37.791160+0000","last_became_active":"2023-09-23T19:09:03.313741+0000","last_became_peered":"2023-09-23T19:09:03.313741+0000","last_unstale":"2023-09-23T19:10:37.791160+0000","last_undegraded":"2023-09-23T19:10:37.791160+0000","last_fullsized":"2023-09-23T19:10:37.791160+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"383'163","reported_seq":812,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.789761+0000","last_change":"2023-09-23T19:09:23.643786+0000","last_active":"2023-09-23T19:10:37.789761+0000","last_peered":"2023-09-23T19:10:37.789761+0000","last_clean":"2023-09-23T19:10:37.789761+0000","last_became_active":"2023-09-23T19:09:02.310096+0000","last_became_peered":"2023-09-23T19:09:02.310096+0000","last_unstale":"2023-09-23T19:10:37.789761+0000","last_undegraded":"2023-09-23T19:10:37.789761+0000","last_fullsized":"2023-09-23T19:10:37.789761+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":163,"log_dups_size":0,"ondisk_log_size":163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1010,"num_read_kb":4243,"num_write":132,"num_write_kb":3279,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":452,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.789237+0000","last_change":"2023-09-23T19:08:38.775076+0000","last_active":"2023-09-23T19:10:37.789237+0000","last_peered":"2023-09-23T19:10:37.789237+0000","last_clean":"2023-09-23T19:10:37.789237+0000","last_became_active":"2023-09-23T19:08:38.774626+0000","last_became_peered":"2023-09-23T19:08:38.774626+0000","last_unstale":"2023-09-23T19:10:37.789237+0000","last_undegraded":"2023-09-23T19:10:37.789237+0000","last_fullsized":"2023-09-23T19:10:37.789237+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:20:42.987390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":429,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.789471+0000","last_change":"2023-09-23T18:51:43.087834+0000","last_active":"2023-09-23T19:10:37.789471+0000","last_peered":"2023-09-23T19:10:37.789471+0000","last_clean":"2023-09-23T19:10:37.789471+0000","last_became_active":"2023-09-23T18:51:43.087309+0000","last_became_peered":"2023-09-23T18:51:43.087309+0000","last_unstale":"2023-09-23T19:10:37.789471+0000","last_undegraded":"2023-09-23T19:10:37.789471+0000","last_fullsized":"2023-09-23T19:10:37.789471+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:53.913425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"383'142","reported_seq":754,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.791038+0000","last_change":"2023-09-23T19:09:23.643776+0000","last_active":"2023-09-23T19:10:37.791038+0000","last_peered":"2023-09-23T19:10:37.791038+0000","last_clean":"2023-09-23T19:10:37.791038+0000","last_became_active":"2023-09-23T19:09:03.321173+0000","last_became_peered":"2023-09-23T19:09:03.321173+0000","last_unstale":"2023-09-23T19:10:37.791038+0000","last_undegraded":"2023-09-23T19:10:37.791038+0000","last_fullsized":"2023-09-23T19:10:37.791038+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":142,"log_dups_size":0,"ondisk_log_size":142,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":777,"num_read_kb":2427,"num_write":137,"num_write_kb":5585,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":368,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.800436+0000","last_change":"2023-09-23T19:10:37.800014+0000","last_active":"2023-09-23T19:10:37.800436+0000","last_peered":"2023-09-23T19:10:37.800436+0000","last_clean":"2023-09-23T19:10:37.800436+0000","last_became_active":"2023-09-23T19:10:37.799660+0000","last_became_peered":"2023-09-23T19:10:37.799660+0000","last_unstale":"2023-09-23T19:10:37.800436+0000","last_undegraded":"2023-09-23T19:10:37.800436+0000","last_fullsized":"2023-09-23T19:10:37.800436+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:39.131137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":108,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.799643+0000","last_change":"2023-09-23T19:09:03.310928+0000","last_active":"2023-09-23T19:10:37.799643+0000","last_peered":"2023-09-23T19:10:37.799643+0000","last_clean":"2023-09-23T19:10:37.799643+0000","last_became_active":"2023-09-23T19:09:03.310498+0000","last_became_peered":"2023-09-23T19:09:03.310498+0000","last_unstale":"2023-09-23T19:10:37.799643+0000","last_undegraded":"2023-09-23T19:10:37.799643+0000","last_fullsized":"2023-09-23T19:10:37.799643+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":414,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.786376+0000","last_change":"2023-09-23T19:09:03.319699+0000","last_active":"2023-09-23T19:10:37.786376+0000","last_peered":"2023-09-23T19:10:37.786376+0000","last_clean":"2023-09-23T19:10:37.786376+0000","last_became_active":"2023-09-23T19:09:03.318772+0000","last_became_peered":"2023-09-23T19:09:03.318772+0000","last_unstale":"2023-09-23T19:10:37.786376+0000","last_undegraded":"2023-09-23T19:10:37.786376+0000","last_fullsized":"2023-09-23T19:10:37.786376+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:41.224329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"383'372","reported_seq":1643,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:42.065626+0000","last_change":"2023-09-23T19:09:23.687378+0000","last_active":"2023-09-23T19:10:42.065626+0000","last_peered":"2023-09-23T19:10:42.065626+0000","last_clean":"2023-09-23T19:10:42.065626+0000","last_became_active":"2023-09-23T18:59:16.452030+0000","last_became_peered":"2023-09-23T18:59:16.452030+0000","last_unstale":"2023-09-23T19:10:42.065626+0000","last_undegraded":"2023-09-23T19:10:42.065626+0000","last_fullsized":"2023-09-23T19:10:42.065626+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":372,"log_dups_size":0,"ondisk_log_size":372,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:20.988663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067870799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2145,"num_read_kb":3069,"num_write":293,"num_write_kb":1311,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":90,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.784590+0000","last_change":"2023-09-23T19:09:03.322018+0000","last_active":"2023-09-23T19:10:37.784590+0000","last_peered":"2023-09-23T19:10:37.784590+0000","last_clean":"2023-09-23T19:10:37.784590+0000","last_became_active":"2023-09-23T19:09:03.321316+0000","last_became_peered":"2023-09-23T19:09:03.321316+0000","last_unstale":"2023-09-23T19:10:37.784590+0000","last_undegraded":"2023-09-23T19:10:37.784590+0000","last_fullsized":"2023-09-23T19:10:37.784590+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:46:37.431152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"383'236","reported_seq":1461,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.790873+0000","last_change":"2023-09-23T19:09:23.644949+0000","last_active":"2023-09-23T19:10:37.790873+0000","last_peered":"2023-09-23T19:10:37.790873+0000","last_clean":"2023-09-23T19:10:37.790873+0000","last_became_active":"2023-09-23T19:09:05.321513+0000","last_became_peered":"2023-09-23T19:09:05.321513+0000","last_unstale":"2023-09-23T19:10:37.790873+0000","last_undegraded":"2023-09-23T19:10:37.790873+0000","last_fullsized":"2023-09-23T19:10:37.790873+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":236,"log_dups_size":0,"ondisk_log_size":236,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2130,"num_read_kb":4337,"num_write":157,"num_write_kb":2355,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":433,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.789600+0000","last_change":"2023-09-23T19:09:03.328610+0000","last_active":"2023-09-23T19:10:37.789600+0000","last_peered":"2023-09-23T19:10:37.789600+0000","last_clean":"2023-09-23T19:10:37.789600+0000","last_became_active":"2023-09-23T19:09:03.312603+0000","last_became_peered":"2023-09-23T19:09:03.312603+0000","last_unstale":"2023-09-23T19:10:37.789600+0000","last_undegraded":"2023-09-23T19:10:37.789600+0000","last_fullsized":"2023-09-23T19:10:37.789600+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:29.029824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":296,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.797859+0000","last_change":"2023-09-23T19:08:37.776658+0000","last_active":"2023-09-23T19:10:37.797859+0000","last_peered":"2023-09-23T19:10:37.797859+0000","last_clean":"2023-09-23T19:10:37.797859+0000","last_became_active":"2023-09-23T19:08:37.776268+0000","last_became_peered":"2023-09-23T19:08:37.776268+0000","last_unstale":"2023-09-23T19:10:37.797859+0000","last_undegraded":"2023-09-23T19:10:37.797859+0000","last_fullsized":"2023-09-23T19:10:37.797859+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":385,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.789524+0000","last_change":"2023-09-23T19:09:03.312771+0000","last_active":"2023-09-23T19:10:37.789524+0000","last_peered":"2023-09-23T19:10:37.789524+0000","last_clean":"2023-09-23T19:10:37.789524+0000","last_became_active":"2023-09-23T19:09:03.312349+0000","last_became_peered":"2023-09-23T19:09:03.312349+0000","last_unstale":"2023-09-23T19:10:37.789524+0000","last_undegraded":"2023-09-23T19:10:37.789524+0000","last_fullsized":"2023-09-23T19:10:37.789524+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:57.252470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":186,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.793281+0000","last_change":"2023-09-23T18:58:38.715946+0000","last_active":"2023-09-23T19:10:37.793281+0000","last_peered":"2023-09-23T19:10:37.793281+0000","last_clean":"2023-09-23T19:10:37.793281+0000","last_became_active":"2023-09-23T18:58:38.715584+0000","last_became_peered":"2023-09-23T18:58:38.715584+0000","last_unstale":"2023-09-23T19:10:37.793281+0000","last_undegraded":"2023-09-23T19:10:37.793281+0000","last_fullsized":"2023-09-23T19:10:37.793281+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:57.811713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":110,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.799460+0000","last_change":"2023-09-23T19:08:37.774463+0000","last_active":"2023-09-23T19:10:37.799460+0000","last_peered":"2023-09-23T19:10:37.799460+0000","last_clean":"2023-09-23T19:10:37.799460+0000","last_became_active":"2023-09-23T19:08:37.773983+0000","last_became_peered":"2023-09-23T19:08:37.773983+0000","last_unstale":"2023-09-23T19:10:37.799460+0000","last_undegraded":"2023-09-23T19:10:37.799460+0000","last_fullsized":"2023-09-23T19:10:37.799460+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":384,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.784754+0000","last_change":"2023-09-23T18:59:13.712801+0000","last_active":"2023-09-23T19:10:37.784754+0000","last_peered":"2023-09-23T19:10:37.784754+0000","last_clean":"2023-09-23T19:10:37.784754+0000","last_became_active":"2023-09-23T18:59:13.710748+0000","last_became_peered":"2023-09-23T18:59:13.710748+0000","last_unstale":"2023-09-23T19:10:37.784754+0000","last_undegraded":"2023-09-23T19:10:37.784754+0000","last_fullsized":"2023-09-23T19:10:37.784754+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:26.256173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":110,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.800780+0000","last_change":"2023-09-23T19:08:37.775330+0000","last_active":"2023-09-23T19:10:37.800780+0000","last_peered":"2023-09-23T19:10:37.800780+0000","last_clean":"2023-09-23T19:10:37.800780+0000","last_became_active":"2023-09-23T19:08:37.774581+0000","last_became_peered":"2023-09-23T19:08:37.774581+0000","last_unstale":"2023-09-23T19:10:37.800780+0000","last_undegraded":"2023-09-23T19:10:37.800780+0000","last_fullsized":"2023-09-23T19:10:37.800780+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":349,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.784984+0000","last_change":"2023-09-23T18:59:13.699034+0000","last_active":"2023-09-23T19:10:37.784984+0000","last_peered":"2023-09-23T19:10:37.784984+0000","last_clean":"2023-09-23T19:10:37.784984+0000","last_became_active":"2023-09-23T18:59:13.698718+0000","last_became_peered":"2023-09-23T18:59:13.698718+0000","last_unstale":"2023-09-23T19:10:37.784984+0000","last_undegraded":"2023-09-23T19:10:37.784984+0000","last_fullsized":"2023-09-23T19:10:37.784984+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:57.449621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":114,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:42.647585+0000","last_change":"2023-09-23T18:58:38.711301+0000","last_active":"2023-09-23T19:10:42.647585+0000","last_peered":"2023-09-23T19:10:42.647585+0000","last_clean":"2023-09-23T19:10:42.647585+0000","last_became_active":"2023-09-23T18:58:38.710923+0000","last_became_peered":"2023-09-23T18:58:38.710923+0000","last_unstale":"2023-09-23T19:10:42.647585+0000","last_undegraded":"2023-09-23T19:10:42.647585+0000","last_fullsized":"2023-09-23T19:10:42.647585+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:17.756147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":113,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T19:10:47.136784+0000","last_change":"2023-09-23T18:58:38.711839+0000","last_active":"2023-09-23T19:10:47.136784+0000","last_peered":"2023-09-23T19:10:47.136784+0000","last_clean":"2023-09-23T19:10:47.136784+0000","last_became_active":"2023-09-23T18:58:38.711347+0000","last_became_peered":"2023-09-23T18:58:38.711347+0000","last_unstale":"2023-09-23T19:10:47.136784+0000","last_undegraded":"2023-09-23T19:10:47.136784+0000","last_fullsized":"2023-09-23T19:10:47.136784+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:55.319645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":472,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.790380+0000","last_change":"2023-09-23T19:08:37.776379+0000","last_active":"2023-09-23T19:10:37.790380+0000","last_peered":"2023-09-23T19:10:37.790380+0000","last_clean":"2023-09-23T19:10:37.790380+0000","last_became_active":"2023-09-23T19:08:37.776012+0000","last_became_peered":"2023-09-23T19:08:37.776012+0000","last_unstale":"2023-09-23T19:10:37.790380+0000","last_undegraded":"2023-09-23T19:10:37.790380+0000","last_fullsized":"2023-09-23T19:10:37.790380+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:16.823018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":348,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T19:10:37.787418+0000","last_change":"2023-09-23T19:02:58.680806+0000","last_active":"2023-09-23T19:10:37.787418+0000","last_peered":"2023-09-23T19:10:37.787418+0000","last_clean":"2023-09-23T19:10:37.787418+0000","last_became_active":"2023-09-23T19:02:58.680426+0000","last_became_peered":"2023-09-23T19:02:58.680426+0000","last_unstale":"2023-09-23T19:10:37.787418+0000","last_undegraded":"2023-09-23T19:10:37.787418+0000","last_fullsized":"2023-09-23T19:10:37.787418+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:16:15.128164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":8324,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56162,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11934,"num_read_kb":26365,"num_write":1690,"num_write_kb":25426,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":120,"num_bytes_recovered":12582931,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7733,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":367,"seq":1576252997657,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":347440,"kb_used_data":12584,"kb_used_omap":13,"kb_used_meta":334834,"kb_avail":93397712,"statfs":{"total":95995035648,"available":95639257088,"internally_reserved":0,"allocated":12886016,"data_stored":11915152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14028,"internal_metadata":342870324},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:10:06 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.3169999999999999,"5min":1.3169999999999999,"15min":1.3169999999999999},"last":0.90000000000000002},{"interface":"front","average":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.339,"5min":1.339,"15min":1.339},"last":0.68999999999999995}]},{"osd":1,"last update":"Sat Sep 23 19:10:06 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.4199999999999999,"5min":1.4199999999999999,"15min":1.4199999999999999},"last":1.0009999999999999},{"interface":"front","average":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":1.4379999999999999,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":0.78000000000000003}]},{"osd":2,"last update":"Sat Sep 23 19:10:06 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.016,"15min":1.016},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":2.4049999999999998,"5min":2.4049999999999998,"15min":2.4049999999999998},"last":0.95999999999999996},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"min":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"max":{"1min":1.407,"5min":1.407,"15min":1.407},"last":0.71199999999999997}]},{"osd":3,"last update":"Sat Sep 23 19:10:06 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.2969999999999999,"5min":1.2969999999999999,"15min":1.2969999999999999},"last":0.97499999999999998},{"interface":"front","average":{"1min":1.0700000000000001,"5min":1.0700000000000001,"15min":1.0700000000000001},"min":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":2.2749999999999999,"5min":2.2749999999999999,"15min":2.2749999999999999},"last":0.92400000000000004}]},{"osd":4,"last update":"Sat Sep 23 19:10:06 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"min":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"min":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.417,"5min":1.417,"15min":1.417},"last":0.60199999999999998}]},{"osd":5,"last update":"Sat Sep 23 19:10:06 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.038,"15min":1.038},"min":{"1min":0.622,"5min":0.622,"15min":0.622},"max":{"1min":2.3660000000000001,"5min":2.3660000000000001,"15min":2.3660000000000001},"last":0.73899999999999999},{"interface":"front","average":{"1min":1.083,"5min":1.083,"15min":1.083},"min":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"max":{"1min":1.409,"5min":1.409,"15min":1.409},"last":0.65500000000000003}]},{"osd":6,"last update":"Sat Sep 23 19:10:06 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":1.121,"15min":1.121},"min":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":2.3300000000000001,"5min":2.3300000000000001,"15min":2.3300000000000001},"last":0.86499999999999999},{"interface":"front","average":{"1min":1.1519999999999999,"5min":1.1519999999999999,"15min":1.1519999999999999},"min":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"max":{"1min":1.496,"5min":1.496,"15min":1.496},"last":0.82999999999999996}]}]},{"osd":6,"up_from":42,"seq":180388627150,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":899752,"kb_used_data":4384,"kb_used_omap":0,"kb_used_meta":895360,"kb_avail":92845400,"statfs":{"total":95995035648,"available":95073689600,"internally_reserved":0,"allocated":4489216,"data_stored":3526136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916848640},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:10:02 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.80800000000000005,"15min":1.877},"min":{"1min":0.38300000000000001,"5min":0.33000000000000002,"15min":0.26300000000000001},"max":{"1min":1.407,"5min":1.407,"15min":100.648},"last":0.81399999999999995},{"interface":"front","average":{"1min":1.0149999999999999,"5min":0.92300000000000004,"15min":1.9770000000000001},"min":{"1min":0.58799999999999997,"5min":0.42899999999999999,"15min":0.317},"max":{"1min":1.4550000000000001,"5min":1.4550000000000001,"15min":100.70399999999999},"last":0.86799999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:10:02 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.91200000000000003,"15min":1.6499999999999999},"min":{"1min":0.47399999999999998,"5min":0.439,"15min":0.29699999999999999},"max":{"1min":1.2330000000000001,"5min":8.6489999999999991,"15min":46.162999999999997},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.82399999999999995,"5min":0.89200000000000002,"15min":1.4810000000000001},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.27900000000000003},"max":{"1min":1.4079999999999999,"5min":4.4269999999999996,"15min":46.430999999999997},"last":0.84299999999999997}]},{"osd":2,"last update":"Sat Sep 23 19:10:02 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":0.81599999999999995,"15min":1.5820000000000001},"min":{"1min":0.52400000000000002,"5min":0.34000000000000002,"15min":0.29599999999999999},"max":{"1min":1.3120000000000001,"5min":1.4630000000000001,"15min":66.587999999999994},"last":0.88700000000000001},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.88200000000000001,"15min":1.599},"min":{"1min":0.61399999999999999,"5min":0.39400000000000002,"15min":0.35299999999999998},"max":{"1min":1.21,"5min":1.5009999999999999,"15min":66.614000000000004},"last":0.91300000000000003}]},{"osd":3,"last update":"Sat Sep 23 19:10:13 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.88200000000000001,"15min":1.8939999999999999},"min":{"1min":0.66100000000000003,"5min":0.39000000000000001,"15min":0.34000000000000002},"max":{"1min":1.5489999999999999,"5min":1.5489999999999999,"15min":93.456000000000003},"last":1.0429999999999999},{"interface":"front","average":{"1min":0.99099999999999999,"5min":0.88700000000000001,"15min":1.9159999999999999},"min":{"1min":0.62,"5min":0.504,"15min":0.38300000000000001},"max":{"1min":1.399,"5min":1.399,"15min":93.472999999999999},"last":1.0129999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:10:26 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":0.84199999999999997,"15min":0.77200000000000002},"min":{"1min":0.69799999999999995,"5min":0.36099999999999999,"15min":0.32200000000000001},"max":{"1min":1.242,"5min":1.242,"15min":4.1150000000000002},"last":0.94799999999999995},{"interface":"front","average":{"1min":1.1919999999999999,"5min":1.0309999999999999,"15min":0.86299999999999999},"min":{"1min":0.76700000000000002,"5min":0.49099999999999999,"15min":0.35799999999999998},"max":{"1min":1.4890000000000001,"5min":1.4890000000000001,"15min":4.0179999999999998},"last":0.93200000000000005}]},{"osd":5,"last update":"Sat Sep 23 19:10:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.042,"15min":1.024},"min":{"1min":0.73499999999999999,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.4530000000000001,"5min":1.464,"15min":1.464},"last":0.99399999999999999},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.0109999999999999,"15min":0.98299999999999998},"min":{"1min":0.89800000000000002,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.4670000000000001,"5min":1.4670000000000001,"15min":1.4670000000000001},"last":0.98399999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:10:02 2023","interfaces":[{"interface":"back","average":{"1min":1.081,"5min":1.081,"15min":1.081},"min":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"max":{"1min":1.631,"5min":1.631,"15min":1.631},"last":0.71199999999999997},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.1080000000000001,"15min":1.1080000000000001},"min":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"max":{"1min":1.8819999999999999,"5min":1.8819999999999999,"15min":1.8819999999999999},"last":0.73599999999999999}]}]},{"osd":1,"up_from":123,"seq":528280977853,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1001192,"kb_used_data":9056,"kb_used_omap":7,"kb_used_meta":992120,"kb_avail":92743960,"statfs":{"total":95995035648,"available":94969815040,"internally_reserved":0,"allocated":9273344,"data_stored":7276801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":1015931662},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:10:05 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.72499999999999998,"15min":2.3100000000000001},"min":{"1min":0.41499999999999998,"5min":0.24299999999999999,"15min":0.23499999999999999},"max":{"1min":0.79400000000000004,"5min":4.4420000000000002,"15min":90.492000000000004},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.68700000000000006,"5min":0.77900000000000003,"15min":2.101},"min":{"1min":0.40300000000000002,"5min":0.188,"15min":0.17399999999999999},"max":{"1min":0.94899999999999995,"5min":7.1699999999999999,"15min":90.552999999999997},"last":1.0289999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:10:05 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.66800000000000004,"15min":1.76},"min":{"1min":0.437,"5min":0.245,"15min":0.219},"max":{"1min":1.1180000000000001,"5min":4.5279999999999996,"15min":87.832999999999998},"last":0.80000000000000004},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.66400000000000003,"15min":1.9430000000000001},"min":{"1min":0.35999999999999999,"5min":0.24299999999999999,"15min":0.22800000000000001},"max":{"1min":1.0369999999999999,"5min":4.5620000000000003,"15min":87.846999999999994},"last":0.64700000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:10:05 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.67000000000000004,"15min":1.7549999999999999},"min":{"1min":0.371,"5min":0.32200000000000001,"15min":0.24099999999999999},"max":{"1min":1.135,"5min":4.694,"15min":99.126000000000005},"last":0.90100000000000002},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.69999999999999996,"15min":1.718},"min":{"1min":0.36499999999999999,"5min":0.34100000000000003,"15min":0.21299999999999999},"max":{"1min":1.089,"5min":4.5759999999999996,"15min":99.165999999999997},"last":1.216}]},{"osd":4,"last update":"Sat Sep 23 19:10:24 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.92500000000000004,"15min":1.2909999999999999},"min":{"1min":0.496,"5min":0.48899999999999999,"15min":0.314},"max":{"1min":1.4199999999999999,"5min":7.3070000000000004,"15min":8.3559999999999999},"last":1.093},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.89800000000000002,"15min":1.2709999999999999},"min":{"1min":0.75,"5min":0.56499999999999995,"15min":0.34200000000000003},"max":{"1min":1.5820000000000001,"5min":2.6859999999999999,"15min":9.0549999999999997},"last":1.145}]},{"osd":5,"last update":"Sat Sep 23 19:10:40 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":1.012,"15min":1.016},"min":{"1min":0.67700000000000005,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.476,"5min":3.6539999999999999,"15min":3.6539999999999999},"last":1.0069999999999999},{"interface":"front","average":{"1min":0.94899999999999995,"5min":1.109,"15min":1.135},"min":{"1min":0.625,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.2789999999999999,"5min":4.6159999999999997,"15min":4.6159999999999997},"last":1.268}]},{"osd":6,"last update":"Sat Sep 23 19:10:05 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.90700000000000003,"15min":1.2649999999999999},"min":{"1min":0.69699999999999995,"5min":0.42399999999999999,"15min":0.38100000000000001},"max":{"1min":1.413,"5min":4.6059999999999999,"15min":9.1029999999999998},"last":1.056},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.85899999999999999,"15min":1.3620000000000001},"min":{"1min":0.503,"5min":0.40500000000000003,"15min":0.373},"max":{"1min":1.444,"5min":2.75,"15min":12.887},"last":1.181}]},{"osd":7,"last update":"Sat Sep 23 19:10:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"min":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"max":{"1min":1.4139999999999999,"5min":1.4139999999999999,"15min":1.4139999999999999},"last":1.3220000000000001},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.0149999999999999,"15min":1.0149999999999999},"min":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.472,"5min":1.472,"15min":1.472},"last":1.2470000000000001}]}]},{"osd":0,"up_from":88,"seq":377957122583,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1129000,"kb_used_data":9440,"kb_used_omap":8,"kb_used_meta":1119543,"kb_avail":92616152,"statfs":{"total":95995035648,"available":94838939648,"internally_reserved":0,"allocated":9666560,"data_stored":7844043,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":1146412926},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:10:21 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.59299999999999997,"15min":2.3690000000000002},"min":{"1min":0.40000000000000002,"5min":0.247,"15min":0.19800000000000001},"max":{"1min":1.069,"5min":3.4359999999999999,"15min":49.168999999999997},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.747,"5min":0.67400000000000004,"15min":2.5110000000000001},"min":{"1min":0.51000000000000001,"5min":0.249,"15min":0.20499999999999999},"max":{"1min":1.208,"5min":3.4670000000000001,"15min":50.673000000000002},"last":0.82499999999999996}]},{"osd":2,"last update":"Sat Sep 23 19:10:58 2023","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":0.68700000000000006,"15min":2.0339999999999998},"min":{"1min":0.44900000000000001,"5min":0.28699999999999998,"15min":0.215},"max":{"1min":1.0780000000000001,"5min":1.327,"15min":59.381999999999998},"last":0.90400000000000003},{"interface":"front","average":{"1min":0.71899999999999997,"5min":0.63,"15min":1.6970000000000001},"min":{"1min":0.379,"5min":0.317,"15min":0.22900000000000001},"max":{"1min":1.242,"5min":1.242,"15min":59.783000000000001},"last":1.004}]},{"osd":3,"last update":"Sat Sep 23 19:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.72099999999999997,"5min":0.65000000000000002,"15min":2.1139999999999999},"min":{"1min":0.42699999999999999,"5min":0.27400000000000002,"15min":0.23200000000000001},"max":{"1min":1.4390000000000001,"5min":2.641,"15min":113.375},"last":0.96399999999999997},{"interface":"front","average":{"1min":0.81799999999999995,"5min":0.754,"15min":2.1669999999999998},"min":{"1min":0.45200000000000001,"5min":0.32900000000000001,"15min":0.217},"max":{"1min":1.22,"5min":2.98,"15min":113.327},"last":1.056}]},{"osd":4,"last update":"Sat Sep 23 19:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":0.84399999999999997,"15min":1.349},"min":{"1min":0.61299999999999999,"5min":0.40500000000000003,"15min":0.33200000000000002},"max":{"1min":1.333,"5min":1.375,"15min":55.710999999999999},"last":0.86299999999999999},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.85799999999999998,"15min":1.3400000000000001},"min":{"1min":0.65600000000000003,"5min":0.51200000000000001,"15min":0.30299999999999999},"max":{"1min":1.3640000000000001,"5min":1.548,"15min":46.509},"last":1.464}]},{"osd":5,"last update":"Sat Sep 23 19:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":0.998,"15min":0.998},"min":{"1min":0.65800000000000003,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.363,"5min":1.472,"15min":1.472},"last":1.5669999999999999},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.046,"15min":1.038},"min":{"1min":0.88300000000000001,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.4279999999999999,"5min":1.4630000000000001,"15min":1.4630000000000001},"last":1.1359999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:10:21 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":0.94799999999999995,"15min":1.3260000000000001},"min":{"1min":0.71299999999999997,"5min":0.47899999999999998,"15min":0.36799999999999999},"max":{"1min":1.4650000000000001,"5min":5.3789999999999996,"15min":55.796999999999997},"last":1.0780000000000001},{"interface":"front","average":{"1min":1.069,"5min":0.93799999999999994,"15min":1.3779999999999999},"min":{"1min":0.73399999999999999,"5min":0.51600000000000001,"15min":0.38600000000000001},"max":{"1min":1.4430000000000001,"5min":2.9020000000000001,"15min":46.411999999999999},"last":1.522}]},{"osd":7,"last update":"Sat Sep 23 19:10:06 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.085,"15min":1.085},"min":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"max":{"1min":1.3280000000000001,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":1.6160000000000001},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.0680000000000001,"15min":1.0680000000000001},"min":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"max":{"1min":1.504,"5min":1.504,"15min":1.504},"last":1.034}]}]},{"osd":2,"up_from":98,"seq":420906795531,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1000288,"kb_used_data":8856,"kb_used_omap":11,"kb_used_meta":991412,"kb_avail":92744864,"statfs":{"total":95995035648,"available":94970740736,"internally_reserved":0,"allocated":9068544,"data_stored":7256874,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11949,"internal_metadata":1015206227},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:10:52 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.58799999999999997,"15min":2.7490000000000001},"min":{"1min":0.436,"5min":0.20799999999999999,"15min":0.16600000000000001},"max":{"1min":0.96999999999999997,"5min":1.5089999999999999,"15min":143.048},"last":1.0680000000000001},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.58199999999999996,"15min":2.899},"min":{"1min":0.53400000000000003,"5min":0.32200000000000001,"15min":0.187},"max":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":143.25200000000001},"last":0.88700000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:10:52 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":0.63800000000000001,"15min":2.0720000000000001},"min":{"1min":0.53200000000000003,"5min":0.34200000000000003,"15min":0.20599999999999999},"max":{"1min":1.1419999999999999,"5min":1.1419999999999999,"15min":104.43899999999999},"last":0.83899999999999997},{"interface":"front","average":{"1min":0.78500000000000003,"5min":0.67100000000000004,"15min":2.2949999999999999},"min":{"1min":0.36699999999999999,"5min":0.314,"15min":0.23799999999999999},"max":{"1min":1.1120000000000001,"5min":1.121,"15min":99.867999999999995},"last":1.038}]},{"osd":3,"last update":"Sat Sep 23 19:10:35 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.59499999999999997,"15min":1.8919999999999999},"min":{"1min":0.443,"5min":0.252,"15min":0.222},"max":{"1min":1.347,"5min":1.347,"15min":85.102999999999994},"last":0.94499999999999995},{"interface":"front","average":{"1min":0.86199999999999999,"5min":0.64500000000000002,"15min":1.883},"min":{"1min":0.59799999999999998,"5min":0.33300000000000002,"15min":0.23200000000000001},"max":{"1min":1.2130000000000001,"5min":1.2130000000000001,"15min":84.531999999999996},"last":1.2589999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:10:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":0.878,"15min":1.0629999999999999},"min":{"1min":0.83999999999999997,"5min":0.40300000000000002,"15min":0.31900000000000001},"max":{"1min":1.3799999999999999,"5min":1.3799999999999999,"15min":8.9969999999999999},"last":1.2},{"interface":"front","average":{"1min":1.0429999999999999,"5min":0.89200000000000002,"15min":1.028},"min":{"1min":0.66500000000000004,"5min":0.45300000000000001,"15min":0.32000000000000001},"max":{"1min":1.536,"5min":1.536,"15min":6.0049999999999999},"last":1.302}]},{"osd":5,"last update":"Sat Sep 23 19:10:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":0.997,"15min":0.98399999999999999},"min":{"1min":0.69799999999999995,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.494,"5min":1.6020000000000001,"15min":1.6020000000000001},"last":1.143},{"interface":"front","average":{"1min":1.109,"5min":1.0609999999999999,"15min":1.0529999999999999},"min":{"1min":0.82599999999999996,"5min":0.72499999999999998,"15min":0.72499999999999998},"max":{"1min":1.327,"5min":1.6659999999999999,"15min":1.6659999999999999},"last":1.3360000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:10:52 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":0.91500000000000004,"15min":1.0029999999999999},"min":{"1min":0.69299999999999995,"5min":0.56499999999999995,"15min":0.32800000000000001},"max":{"1min":1.4419999999999999,"5min":2.0489999999999999,"15min":5.9649999999999999},"last":1.4219999999999999},{"interface":"front","average":{"1min":1.099,"5min":0.95799999999999996,"15min":1.1299999999999999},"min":{"1min":0.77400000000000002,"5min":0.58799999999999997,"15min":0.36699999999999999},"max":{"1min":1.5940000000000001,"5min":1.7969999999999999,"15min":9.048},"last":1.492}]},{"osd":7,"last update":"Sat Sep 23 19:10:02 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":1.117,"15min":1.117},"min":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"max":{"1min":1.744,"5min":1.744,"15min":1.744},"last":1.4370000000000001},{"interface":"front","average":{"1min":1.1519999999999999,"5min":1.1519999999999999,"15min":1.1519999999999999},"min":{"1min":0.623,"5min":0.623,"15min":0.623},"max":{"1min":1.6879999999999999,"5min":1.6879999999999999,"15min":1.6879999999999999},"last":1.3919999999999999}]}]},{"osd":3,"up_from":103,"seq":442381632006,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":935644,"kb_used_data":4820,"kb_used_omap":4,"kb_used_meta":930811,"kb_avail":92809508,"statfs":{"total":95995035648,"available":95036936192,"internally_reserved":0,"allocated":4935680,"data_stored":3064751,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4826,"internal_metadata":953150758},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.55600000000000005,"15min":2.7320000000000002},"min":{"1min":0.34599999999999997,"5min":0.252,"15min":0.191},"max":{"1min":1.0089999999999999,"5min":1.034,"15min":296.55399999999997},"last":0.80400000000000005},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.57899999999999996,"15min":2.8250000000000002},"min":{"1min":0.38900000000000001,"5min":0.27900000000000003,"15min":0.18099999999999999},"max":{"1min":1.222,"5min":1.2430000000000001,"15min":296.36900000000003},"last":0.71699999999999997}]},{"osd":1,"last update":"Sat Sep 23 19:10:18 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":0.66900000000000004,"15min":3.4860000000000002},"min":{"1min":0.374,"5min":0.252,"15min":0.22500000000000001},"max":{"1min":1.278,"5min":4.3940000000000001,"15min":343.79300000000001},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.76800000000000002,"5min":0.76600000000000001,"15min":3.484},"min":{"1min":0.41199999999999998,"5min":0.32400000000000001,"15min":0.214},"max":{"1min":1.1899999999999999,"5min":7.8410000000000002,"15min":343.73000000000002},"last":1.0589999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.71799999999999997,"5min":0.63100000000000001,"15min":3.0049999999999999},"min":{"1min":0.39500000000000002,"5min":0.317,"15min":0.23699999999999999},"max":{"1min":1.385,"5min":1.385,"15min":178.06200000000001},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.76000000000000001,"5min":0.65900000000000003,"15min":2.8290000000000002},"min":{"1min":0.45900000000000002,"5min":0.29799999999999999,"15min":0.26300000000000001},"max":{"1min":1.4159999999999999,"5min":2.1429999999999998,"15min":178.267},"last":0.85899999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":0.82599999999999996,"15min":1.149},"min":{"1min":0.67800000000000005,"5min":0.438,"15min":0.33100000000000002},"max":{"1min":1.4390000000000001,"5min":1.4390000000000001,"15min":12.028},"last":1.0169999999999999},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.85999999999999999,"15min":1.2110000000000001},"min":{"1min":0.63800000000000001,"5min":0.47699999999999998,"15min":0.34300000000000003},"max":{"1min":1.363,"5min":1.7090000000000001,"15min":11.369},"last":0.99299999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:10:42 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":0.97399999999999998,"15min":0.95399999999999996},"min":{"1min":0.82899999999999996,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.7190000000000001,"5min":1.7190000000000001,"15min":1.7190000000000001},"last":0.96499999999999997},{"interface":"front","average":{"1min":1.135,"5min":0.98699999999999999,"15min":0.96299999999999997},"min":{"1min":0.69499999999999995,"5min":0.624,"15min":0.624},"max":{"1min":1.5289999999999999,"5min":1.633,"15min":1.633},"last":1.083}]},{"osd":6,"last update":"Sat Sep 23 19:10:30 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":0.89700000000000002,"15min":1.292},"min":{"1min":0.79200000000000004,"5min":0.42499999999999999,"15min":0.33700000000000002},"max":{"1min":1.792,"5min":1.792,"15min":12.926},"last":0.83199999999999996},{"interface":"front","average":{"1min":1.131,"5min":0.94199999999999995,"15min":1.242},"min":{"1min":0.752,"5min":0.51900000000000002,"15min":0.38500000000000001},"max":{"1min":1.474,"5min":1.839,"15min":11.300000000000001},"last":1.169}]},{"osd":7,"last update":"Sat Sep 23 19:10:05 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.0720000000000001,"15min":1.0720000000000001},"min":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":1.792,"5min":1.792,"15min":1.792},"last":1.232},{"interface":"front","average":{"1min":0.97399999999999998,"5min":0.97399999999999998,"15min":0.97399999999999998},"min":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"max":{"1min":1.329,"5min":1.329,"15min":1.329},"last":0.89800000000000002}]}]},{"osd":4,"up_from":0,"seq":1228360646837,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":360,"seq":1546188226590,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":538260,"kb_used_data":200460,"kb_used_omap":5,"kb_used_meta":337786,"kb_avail":93206892,"statfs":{"total":95995035648,"available":95443857408,"internally_reserved":0,"allocated":205271040,"data_stored":204323367,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6117,"internal_metadata":345892891},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.81799999999999995,"15min":0.79800000000000004},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.3100000000000001,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":1.0640000000000001},{"interface":"front","average":{"1min":0.91900000000000004,"5min":0.89700000000000002,"15min":0.89300000000000002},"min":{"1min":0.56499999999999995,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":1.1699999999999999,"5min":1.5529999999999999,"15min":1.5529999999999999},"last":0.91000000000000003}]},{"osd":1,"last update":"Sat Sep 23 19:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":0.83299999999999996,"15min":0.82099999999999995},"min":{"1min":0.57099999999999995,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.216,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.748},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.86499999999999999,"15min":0.85099999999999998},"min":{"1min":0.55300000000000005,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.228,"5min":1.24,"15min":1.24},"last":0.67400000000000004}]},{"osd":2,"last update":"Sat Sep 23 19:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":0.89200000000000002,"15min":0.88},"min":{"1min":0.63200000000000001,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.1910000000000001,"5min":1.242,"15min":1.242},"last":0.98199999999999998},{"interface":"front","average":{"1min":0.874,"5min":0.85799999999999998,"15min":0.85499999999999998},"min":{"1min":0.55800000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.242,"5min":1.242,"15min":1.242},"last":0.94299999999999995}]},{"osd":3,"last update":"Sat Sep 23 19:10:38 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":0.95899999999999996,"15min":0.94999999999999996},"min":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.27,"5min":1.5249999999999999,"15min":1.5249999999999999},"last":1.2170000000000001},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"min":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.1910000000000001,"5min":2.121,"15min":2.121},"last":0.81200000000000006}]},{"osd":4,"last update":"Sat Sep 23 19:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":0.84599999999999997,"15min":0.84599999999999997},"min":{"1min":0.59499999999999997,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.173,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":0.85499999999999998},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.93899999999999995,"15min":0.93400000000000005},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.2929999999999999,"5min":1.399,"15min":1.399},"last":0.627}]},{"osd":6,"last update":"Sat Sep 23 19:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.96199999999999997,"15min":0.95899999999999996},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.2749999999999999,"5min":1.2889999999999999,"15min":1.2889999999999999},"last":1.022},{"interface":"front","average":{"1min":0.96699999999999997,"5min":0.98699999999999999,"15min":0.99099999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":1.325,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":1.151}]},{"osd":7,"last update":"Sat Sep 23 19:10:03 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.109,"15min":1.109},"min":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"max":{"1min":1.3520000000000001,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":1.28},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"min":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"max":{"1min":1.4670000000000001,"5min":1.4670000000000001,"15min":1.4670000000000001},"last":0.88600000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4224,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8388608,"data_stored":8388608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2112,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7725,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3429,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11243,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6117,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11916,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:11:07.065 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T19:11:07.065 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T19:11:07.065 INFO:tasks.thrashosds.thrasher:Adding osd 4 2023-09-23T19:11:07.066 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd in 4 2023-09-23T19:11:07.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:06 smithi173 bash[59422]: audit 2023-09-23T19:11:06.044955+0000 mgr.y (mgr.26054) 1607 : audit 0 from='client.36831 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:11:07.873 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:07 smithi173 bash[59422]: cluster 2023-09-23T19:11:06.721174+0000 mgr.y (mgr.26054) 1608 : cluster 0 pgmap v1798: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:08.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:07 smithi098 bash[76817]: cluster 2023-09-23T19:11:06.721174+0000 mgr.y (mgr.26054) 1608 : cluster 0 pgmap v1798: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:08.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:07 smithi098 bash[79516]: cluster 2023-09-23T19:11:06.721174+0000 mgr.y (mgr.26054) 1608 : cluster 0 pgmap v1798: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:09.937 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T19:11:09.937 INFO:teuthology.orchestra.run.smithi173.stdout: "size": 20, 2023-09-23T19:11:09.937 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 600, 2023-09-23T19:11:09.937 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-09-23T19:11:09.937 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.937 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45944.0:155 11.17 11:e9c6360c:::image20:head [call rbd.create in=9b] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e360)", 2023-09-23T19:11:09.938 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:08:36.779727+0000", 2023-09-23T19:11:09.938 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 153.151498551, 2023-09-23T19:11:09.938 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.001362839, 2023-09-23T19:11:09.938 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:09.938 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:11:09.938 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:09.938 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45944", 2023-09-23T19:11:09.938 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/1050517665", 2023-09-23T19:11:09.938 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 155 2023-09-23T19:11:09.938 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.938 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:09.938 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.939 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:09.939 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:36.779727+0000", 2023-09-23T19:11:09.939 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:09.939 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.939 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.939 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:09.939 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:36.779727+0000", 2023-09-23T19:11:09.939 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T19:11:09.939 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.939 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.939 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:09.939 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:36.779726+0000", 2023-09-23T19:11:09.939 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0022e-05 2023-09-23T19:11:09.940 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.940 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.940 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:09.940 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:36.779736+0000", 2023-09-23T19:11:09.940 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2564e-05 2023-09-23T19:11:09.940 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.940 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.940 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:09.940 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:36.779748+0000", 2023-09-23T19:11:09.940 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.112e-05 2023-09-23T19:11:09.940 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.940 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.940 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:09.941 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:36.779759+0000", 2023-09-23T19:11:09.941 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001086664 2023-09-23T19:11:09.941 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.941 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.941 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:09.941 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:36.780846+0000", 2023-09-23T19:11:09.941 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.0327e-05 2023-09-23T19:11:09.941 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.941 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.941 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T19:11:09.941 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:36.780866+0000", 2023-09-23T19:11:09.941 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.99903336300000001 2023-09-23T19:11:09.942 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.942 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.942 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:09.942 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.779900+0000", 2023-09-23T19:11:09.942 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000143795 2023-09-23T19:11:09.942 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.942 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.942 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:11:09.942 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.780044+0000", 2023-09-23T19:11:09.942 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00026820199999999999 2023-09-23T19:11:09.942 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.942 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.942 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 6", 2023-09-23T19:11:09.943 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.780312+0000", 2023-09-23T19:11:09.943 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00048094700000000002 2023-09-23T19:11:09.943 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.943 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.943 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T19:11:09.943 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.780793+0000", 2023-09-23T19:11:09.943 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00019771799999999999 2023-09-23T19:11:09.943 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.943 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.943 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:09.943 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.780990+0000", 2023-09-23T19:11:09.943 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.7234000000000003e-05 2023-09-23T19:11:09.944 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.944 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.944 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:11:09.944 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.781028+0000", 2023-09-23T19:11:09.944 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.2619999999999993e-05 2023-09-23T19:11:09.944 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.944 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.944 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:09.944 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.781090+0000", 2023-09-23T19:11:09.944 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.001362839 2023-09-23T19:11:09.944 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:09.944 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:09.944 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:09.945 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.945 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.945 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.45944.0:162 11.a e361/360 11:57642022:::image19:head v 361'5, mlcod=361'3)", 2023-09-23T19:11:09.945 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:08:37.786239+0000", 2023-09-23T19:11:09.945 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 152.14498727200001, 2023-09-23T19:11:09.945 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0079432749999999996, 2023-09-23T19:11:09.945 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:09.945 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:11:09.945 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:09.945 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.945 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:09.945 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.786239+0000", 2023-09-23T19:11:09.945 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:09.946 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.946 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.946 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:09.946 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.786239+0000", 2023-09-23T19:11:09.946 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T19:11:09.946 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.946 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.946 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:09.946 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.786238+0000", 2023-09-23T19:11:09.946 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.2409999999999995e-06 2023-09-23T19:11:09.946 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.946 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.947 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:09.947 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.786246+0000", 2023-09-23T19:11:09.947 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0740999999999999e-05 2023-09-23T19:11:09.947 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.947 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.947 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:09.947 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.786257+0000", 2023-09-23T19:11:09.947 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.0489999999999998e-06 2023-09-23T19:11:09.947 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.947 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.947 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:09.947 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.786264+0000", 2023-09-23T19:11:09.947 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.1375000000000003e-05 2023-09-23T19:11:09.948 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.948 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.948 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:09.948 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.786305+0000", 2023-09-23T19:11:09.948 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4984e-05 2023-09-23T19:11:09.948 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.948 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.948 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:11:09.948 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.786330+0000", 2023-09-23T19:11:09.948 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0077528889999999998 2023-09-23T19:11:09.948 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.948 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.948 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:11:09.949 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.794083+0000", 2023-09-23T19:11:09.949 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.8848e-05 2023-09-23T19:11:09.949 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.949 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.949 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:09.949 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:37.794182+0000", 2023-09-23T19:11:09.949 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0079432749999999996 2023-09-23T19:11:09.949 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:09.949 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:09.949 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:09.949 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.949 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.949 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46061.0:30 2.2 2:4fd71197:::rbd_data.b3ed9942bcee.0000000000000000:head [set-alloc-hint object_size 4194304 write_size 4194304,write 8192~4096 [fadvise_nocache]] snapc 0=[] ondisk+write+known_if_redirected e365)", 2023-09-23T19:11:09.950 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:08:53.651534+0000", 2023-09-23T19:11:09.950 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 136.27969159700001, 2023-09-23T19:11:09.950 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0031576769999999998, 2023-09-23T19:11:09.950 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:09.950 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:11:09.950 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:09.950 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46061", 2023-09-23T19:11:09.950 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/4074385900", 2023-09-23T19:11:09.950 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 30 2023-09-23T19:11:09.950 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.950 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:09.950 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.950 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:09.951 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.651534+0000", 2023-09-23T19:11:09.951 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:09.951 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.951 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.951 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:09.951 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.651534+0000", 2023-09-23T19:11:09.951 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999971 2023-09-23T19:11:09.951 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.951 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.951 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:09.951 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.651532+0000", 2023-09-23T19:11:09.951 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.495e-05 2023-09-23T19:11:09.951 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.952 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.952 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:09.952 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.651557+0000", 2023-09-23T19:11:09.952 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.6165000000000001e-05 2023-09-23T19:11:09.952 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.952 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.952 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:09.952 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.651583+0000", 2023-09-23T19:11:09.952 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1457000000000001e-05 2023-09-23T19:11:09.953 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.953 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.953 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:09.953 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.651594+0000", 2023-09-23T19:11:09.953 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.4074999999999996e-05 2023-09-23T19:11:09.953 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.953 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.953 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:09.953 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.651688+0000", 2023-09-23T19:11:09.953 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018676500000000001 2023-09-23T19:11:09.953 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.953 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.953 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:11:09.954 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.651875+0000", 2023-09-23T19:11:09.954 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000227822 2023-09-23T19:11:09.954 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.954 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.954 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,6", 2023-09-23T19:11:09.954 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.652103+0000", 2023-09-23T19:11:09.954 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0023892589999999999 2023-09-23T19:11:09.954 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.954 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.954 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:09.954 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.654492+0000", 2023-09-23T19:11:09.955 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1105e-05 2023-09-23T19:11:09.955 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.955 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.955 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T19:11:09.955 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.654503+0000", 2023-09-23T19:11:09.955 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.2741999999999997e-05 2023-09-23T19:11:09.955 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.955 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.955 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:09.955 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.654556+0000", 2023-09-23T19:11:09.955 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.9110999999999995e-05 2023-09-23T19:11:09.955 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.956 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.956 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:11:09.956 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.654635+0000", 2023-09-23T19:11:09.956 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6960999999999999e-05 2023-09-23T19:11:09.956 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.956 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.956 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:09.956 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.654692+0000", 2023-09-23T19:11:09.956 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0031576769999999998 2023-09-23T19:11:09.956 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:09.956 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:09.956 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:09.957 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.957 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.957 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46061.0:31 2.2 2:4fd71197:::rbd_data.b3ed9942bcee.0000000000000000:head [set-alloc-hint object_size 4194304 write_size 4194304,write 102400~12288 [fadvise_nocache]] snapc 0=[] ondisk+write+known_if_redirected e365)", 2023-09-23T19:11:09.957 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:08:53.651612+0000", 2023-09-23T19:11:09.957 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 136.279613509, 2023-09-23T19:11:09.957 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.003291169, 2023-09-23T19:11:09.957 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:09.957 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:11:09.957 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:09.957 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46061", 2023-09-23T19:11:09.957 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/4074385900", 2023-09-23T19:11:09.957 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 31 2023-09-23T19:11:09.958 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.958 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:09.958 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.958 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:09.958 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.651612+0000", 2023-09-23T19:11:09.958 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:09.958 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.958 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.958 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:09.958 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.651612+0000", 2023-09-23T19:11:09.958 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T19:11:09.959 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.959 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.959 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:09.959 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.651612+0000", 2023-09-23T19:11:09.959 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.3335999999999999e-05 2023-09-23T19:11:09.959 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.959 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.959 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:09.959 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.651665+0000", 2023-09-23T19:11:09.959 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0046e-05 2023-09-23T19:11:09.959 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.959 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.959 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:09.960 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.651675+0000", 2023-09-23T19:11:09.960 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0346000000000001e-05 2023-09-23T19:11:09.960 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.960 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.960 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:09.960 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.651685+0000", 2023-09-23T19:11:09.960 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00092686299999999999 2023-09-23T19:11:09.960 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.960 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.960 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:09.960 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.652612+0000", 2023-09-23T19:11:09.960 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00017017700000000001 2023-09-23T19:11:09.961 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.961 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.961 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:11:09.961 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.652782+0000", 2023-09-23T19:11:09.961 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00021937399999999999 2023-09-23T19:11:09.961 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.961 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.961 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,6", 2023-09-23T19:11:09.961 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.653002+0000", 2023-09-23T19:11:09.961 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001751613 2023-09-23T19:11:09.961 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.962 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.962 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:09.962 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.654753+0000", 2023-09-23T19:11:09.962 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7249e-05 2023-09-23T19:11:09.962 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.962 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.962 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:09.962 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.654811+0000", 2023-09-23T19:11:09.962 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3757999999999999e-05 2023-09-23T19:11:09.962 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.962 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.962 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T19:11:09.963 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.654834+0000", 2023-09-23T19:11:09.963 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.7577999999999998e-05 2023-09-23T19:11:09.963 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.963 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.963 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:11:09.963 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.654872+0000", 2023-09-23T19:11:09.963 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.1643999999999997e-05 2023-09-23T19:11:09.964 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.964 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.964 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:09.964 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.654904+0000", 2023-09-23T19:11:09.964 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.003291169 2023-09-23T19:11:09.964 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:09.964 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:09.964 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:09.964 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.965 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.965 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46061.0:32 2.2 2:4fd71197:::rbd_data.b3ed9942bcee.0000000000000000:head [set-alloc-hint object_size 4194304 write_size 4194304,write 1024000~73728 [fadvise_nocache]] snapc 0=[] ondisk+write+known_if_redirected e365)", 2023-09-23T19:11:09.965 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:08:53.651695+0000", 2023-09-23T19:11:09.965 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 136.27953108200001, 2023-09-23T19:11:09.965 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0034564079999999998, 2023-09-23T19:11:09.965 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:09.965 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:11:09.965 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:09.965 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46061", 2023-09-23T19:11:09.965 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/4074385900", 2023-09-23T19:11:09.966 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 32 2023-09-23T19:11:09.966 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.966 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:09.966 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.966 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:09.966 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.651695+0000", 2023-09-23T19:11:09.966 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:09.966 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.966 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.966 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:09.967 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.651695+0000", 2023-09-23T19:11:09.967 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T19:11:09.967 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.967 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.967 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:09.967 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.651694+0000", 2023-09-23T19:11:09.967 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.7115999999999996e-05 2023-09-23T19:11:09.967 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.967 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.967 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:09.968 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.651791+0000", 2023-09-23T19:11:09.968 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.153e-05 2023-09-23T19:11:09.968 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.968 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.968 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:09.968 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.651803+0000", 2023-09-23T19:11:09.968 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7019999999999998e-06 2023-09-23T19:11:09.968 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.968 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.968 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:09.969 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.651808+0000", 2023-09-23T19:11:09.969 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0016929829999999999 2023-09-23T19:11:09.969 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.969 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.969 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:09.969 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.653501+0000", 2023-09-23T19:11:09.969 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00010601400000000001 2023-09-23T19:11:09.969 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.969 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.969 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:11:09.969 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.653607+0000", 2023-09-23T19:11:09.970 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00022062000000000001 2023-09-23T19:11:09.970 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.970 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.970 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,6", 2023-09-23T19:11:09.970 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.653828+0000", 2023-09-23T19:11:09.970 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001088592 2023-09-23T19:11:09.970 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.970 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.970 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T19:11:09.970 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.654917+0000", 2023-09-23T19:11:09.971 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000114703 2023-09-23T19:11:09.971 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.971 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.971 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:09.971 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.655031+0000", 2023-09-23T19:11:09.971 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.2233000000000001e-05 2023-09-23T19:11:09.971 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.971 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.971 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:09.971 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.655064+0000", 2023-09-23T19:11:09.972 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.2607e-05 2023-09-23T19:11:09.972 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.972 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.972 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:11:09.972 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.655116+0000", 2023-09-23T19:11:09.972 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.5139999999999999e-05 2023-09-23T19:11:09.972 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.972 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.972 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:09.972 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:53.655151+0000", 2023-09-23T19:11:09.972 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0034564079999999998 2023-09-23T19:11:09.973 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:09.973 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:09.973 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:09.973 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.973 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.973 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46079.0:51 2.3 2:c4c92e5a:::rbd_trash:head [call rbd.trash_add in=12b] snapc 0=[] ondisk+write+known_if_redirected+full_try e365)", 2023-09-23T19:11:09.973 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:08:55.865195+0000", 2023-09-23T19:11:09.973 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 134.066030512, 2023-09-23T19:11:09.973 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0021033029999999999, 2023-09-23T19:11:09.973 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:09.973 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:11:09.974 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:09.974 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46079", 2023-09-23T19:11:09.974 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/811283827", 2023-09-23T19:11:09.974 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 51 2023-09-23T19:11:09.974 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.974 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:09.974 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.974 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:09.974 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:55.865195+0000", 2023-09-23T19:11:09.974 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:09.975 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.975 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.975 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:09.975 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:55.865195+0000", 2023-09-23T19:11:09.975 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T19:11:09.975 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.975 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.975 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:09.975 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:55.865194+0000", 2023-09-23T19:11:09.975 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.4329999999999993e-06 2023-09-23T19:11:09.975 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.976 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.976 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:09.976 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:55.865203+0000", 2023-09-23T19:11:09.976 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7289999999999999e-05 2023-09-23T19:11:09.976 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.976 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.976 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:09.976 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:55.865221+0000", 2023-09-23T19:11:09.976 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.365e-06 2023-09-23T19:11:09.976 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.977 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.977 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:09.977 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:55.865227+0000", 2023-09-23T19:11:09.977 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00011986599999999999 2023-09-23T19:11:09.977 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.977 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.977 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:09.977 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:55.865347+0000", 2023-09-23T19:11:09.977 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00017502199999999999 2023-09-23T19:11:09.977 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.977 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.978 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:11:09.978 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:55.865522+0000", 2023-09-23T19:11:09.978 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00038375300000000002 2023-09-23T19:11:09.978 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.978 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.978 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,3", 2023-09-23T19:11:09.978 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:55.865906+0000", 2023-09-23T19:11:09.978 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00085068199999999996 2023-09-23T19:11:09.978 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.978 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.978 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T19:11:09.979 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:55.866756+0000", 2023-09-23T19:11:09.979 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000107507 2023-09-23T19:11:09.979 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.979 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.979 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:09.979 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:55.866864+0000", 2023-09-23T19:11:09.979 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00017493599999999999 2023-09-23T19:11:09.979 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.979 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.979 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:09.980 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:55.867039+0000", 2023-09-23T19:11:09.980 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.9887000000000002e-05 2023-09-23T19:11:09.980 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.980 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.980 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:11:09.980 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:55.867129+0000", 2023-09-23T19:11:09.980 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000170209 2023-09-23T19:11:09.980 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.980 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.980 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:09.980 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:08:55.867299+0000", 2023-09-23T19:11:09.981 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0021033029999999999 2023-09-23T19:11:09.981 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:09.981 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:09.981 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:09.981 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.981 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.981 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46088.0:8 2.3 2:d04613b6:::rbd_header.b408e23a9739:head [create,call rbd.create in=9b] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-09-23T19:11:09.981 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:02.316100+0000", 2023-09-23T19:11:09.981 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 127.61512591499999, 2023-09-23T19:11:09.981 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.004362311, 2023-09-23T19:11:09.981 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:09.982 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:11:09.982 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:09.982 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46088", 2023-09-23T19:11:09.982 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/1258423516", 2023-09-23T19:11:09.982 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 8 2023-09-23T19:11:09.982 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.982 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:09.982 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.982 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:09.982 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:02.316100+0000", 2023-09-23T19:11:09.983 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:09.983 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.983 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.983 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:09.983 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:02.316100+0000", 2023-09-23T19:11:09.983 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T19:11:09.983 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.983 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.983 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:09.983 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:02.316098+0000", 2023-09-23T19:11:09.983 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.307e-05 2023-09-23T19:11:09.984 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.984 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.984 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:09.984 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:02.316111+0000", 2023-09-23T19:11:09.984 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9293999999999999e-05 2023-09-23T19:11:09.984 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.984 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.984 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:09.984 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:02.316130+0000", 2023-09-23T19:11:09.984 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.1649999999999995e-06 2023-09-23T19:11:09.985 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.985 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.985 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:09.985 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:02.316140+0000", 2023-09-23T19:11:09.985 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.2551000000000005e-05 2023-09-23T19:11:09.985 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.985 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.985 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:09.985 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:02.316202+0000", 2023-09-23T19:11:09.985 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.4059999999999994e-06 2023-09-23T19:11:09.986 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.986 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.986 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T19:11:09.986 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:02.316212+0000", 2023-09-23T19:11:09.986 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.99696817400000004 2023-09-23T19:11:09.986 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.986 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.986 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:09.986 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.313180+0000", 2023-09-23T19:11:09.986 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000200457 2023-09-23T19:11:09.987 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.987 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.987 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:11:09.987 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.313380+0000", 2023-09-23T19:11:09.987 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00038843699999999999 2023-09-23T19:11:09.987 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.987 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.987 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:11:09.987 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.313769+0000", 2023-09-23T19:11:09.987 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0011377920000000001 2023-09-23T19:11:09.987 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.988 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.988 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:09.988 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.314906+0000", 2023-09-23T19:11:09.988 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0043851510000000003 2023-09-23T19:11:09.988 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.988 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.988 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T19:11:09.989 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.319292+0000", 2023-09-23T19:11:09.989 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00093948800000000004 2023-09-23T19:11:09.989 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.989 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.989 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:09.989 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.320231+0000", 2023-09-23T19:11:09.990 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.2216000000000001e-05 2023-09-23T19:11:09.990 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.990 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.990 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:11:09.990 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.320303+0000", 2023-09-23T19:11:09.991 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015907599999999999 2023-09-23T19:11:09.991 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.991 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.991 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:09.991 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.320462+0000", 2023-09-23T19:11:09.991 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.004362311 2023-09-23T19:11:09.992 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:09.992 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:09.992 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:09.992 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.992 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.992 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46088.0:71 2.3 2:d04613b6:::rbd_header.b408e23a9739:head [call rbd.set_size in=11b] snapc 0=[] ondisk+write+known_if_redirected e369)", 2023-09-23T19:11:09.993 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:03.567253+0000", 2023-09-23T19:11:09.993 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 126.36397347400001, 2023-09-23T19:11:09.993 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0023903150000000001, 2023-09-23T19:11:09.993 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:09.993 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:11:09.993 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:09.993 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46088", 2023-09-23T19:11:09.994 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/1258423516", 2023-09-23T19:11:09.994 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 71 2023-09-23T19:11:09.994 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.994 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:09.994 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.994 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:09.994 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.567253+0000", 2023-09-23T19:11:09.995 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:09.995 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.995 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.995 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:09.995 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.567253+0000", 2023-09-23T19:11:09.995 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999971 2023-09-23T19:11:09.996 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.996 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.996 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:09.996 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.567250+0000", 2023-09-23T19:11:09.996 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6869000000000001e-05 2023-09-23T19:11:09.996 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.997 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.997 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:09.997 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.567267+0000", 2023-09-23T19:11:09.997 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.0340000000000001e-05 2023-09-23T19:11:09.997 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.997 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.997 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:09.997 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.567297+0000", 2023-09-23T19:11:09.997 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.448e-05 2023-09-23T19:11:09.997 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.998 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.998 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:09.998 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.567312+0000", 2023-09-23T19:11:09.998 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016964700000000001 2023-09-23T19:11:09.998 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.998 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.998 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:09.998 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.567481+0000", 2023-09-23T19:11:09.998 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0001641 2023-09-23T19:11:09.999 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.999 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.999 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:11:09.999 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.567645+0000", 2023-09-23T19:11:09.999 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00032627399999999999 2023-09-23T19:11:09.999 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:09.999 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:09.999 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:11:09.999 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.567972+0000", 2023-09-23T19:11:09.999 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00077871899999999998 2023-09-23T19:11:10.000 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.000 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.000 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T19:11:10.000 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.568750+0000", 2023-09-23T19:11:10.000 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00029542099999999998 2023-09-23T19:11:10.000 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.000 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.000 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:10.000 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.569046+0000", 2023-09-23T19:11:10.000 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000387969 2023-09-23T19:11:10.000 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.001 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.001 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:10.001 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.569434+0000", 2023-09-23T19:11:10.001 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000115198 2023-09-23T19:11:10.001 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.001 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.001 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:11:10.001 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.569549+0000", 2023-09-23T19:11:10.001 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.4021e-05 2023-09-23T19:11:10.001 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.002 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.002 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:10.002 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.569643+0000", 2023-09-23T19:11:10.002 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0023903150000000001 2023-09-23T19:11:10.002 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.002 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:10.002 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.002 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.002 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.002 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46088.0:91 2.3 2:d04613b6:::rbd_header.b408e23a9739:head [call rbd.set_size in=11b] snapc 0=[] ondisk+write+known_if_redirected e369)", 2023-09-23T19:11:10.003 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:03.658573+0000", 2023-09-23T19:11:10.003 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 126.27265278199999, 2023-09-23T19:11:10.003 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0021722949999999999, 2023-09-23T19:11:10.003 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:10.003 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:11:10.003 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:10.003 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46088", 2023-09-23T19:11:10.003 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/1258423516", 2023-09-23T19:11:10.003 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 91 2023-09-23T19:11:10.003 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.004 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:10.004 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.004 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:10.004 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.658573+0000", 2023-09-23T19:11:10.004 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:10.004 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.004 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.004 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:10.004 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.658573+0000", 2023-09-23T19:11:10.004 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999971 2023-09-23T19:11:10.004 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.005 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.005 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:10.005 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.658571+0000", 2023-09-23T19:11:10.005 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.471e-05 2023-09-23T19:11:10.005 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.005 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.005 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:10.005 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.658585+0000", 2023-09-23T19:11:10.005 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.1563000000000003e-05 2023-09-23T19:11:10.005 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.006 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.006 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:10.006 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.658617+0000", 2023-09-23T19:11:10.006 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.1495999999999998e-05 2023-09-23T19:11:10.006 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.006 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.006 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:10.006 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.658648+0000", 2023-09-23T19:11:10.006 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00010507500000000001 2023-09-23T19:11:10.006 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.006 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.007 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:10.007 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.658753+0000", 2023-09-23T19:11:10.007 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014124 2023-09-23T19:11:10.007 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.007 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.007 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:11:10.007 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.658895+0000", 2023-09-23T19:11:10.007 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000340708 2023-09-23T19:11:10.007 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.007 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.008 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:11:10.008 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.659235+0000", 2023-09-23T19:11:10.008 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00087173700000000003 2023-09-23T19:11:10.008 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.008 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.008 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T19:11:10.008 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.660107+0000", 2023-09-23T19:11:10.008 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014943399999999999 2023-09-23T19:11:10.008 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.008 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.008 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:10.009 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.660257+0000", 2023-09-23T19:11:10.009 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00040551500000000001 2023-09-23T19:11:10.009 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.009 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.009 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:10.009 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.660662+0000", 2023-09-23T19:11:10.009 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6940999999999997e-05 2023-09-23T19:11:10.009 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.009 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.009 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:11:10.009 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.660709+0000", 2023-09-23T19:11:10.010 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.6526000000000002e-05 2023-09-23T19:11:10.010 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.010 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.010 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:10.010 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.660746+0000", 2023-09-23T19:11:10.010 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0021722949999999999 2023-09-23T19:11:10.010 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.010 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:10.010 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.010 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.011 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.011 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46088.0:100 2.3 2:d04613b6:::rbd_header.b408e23a9739:head [call rbd.set_size in=11b] snapc 0=[] ondisk+write+known_if_redirected e369)", 2023-09-23T19:11:10.011 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:03.764379+0000", 2023-09-23T19:11:10.011 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 126.16684710600001, 2023-09-23T19:11:10.011 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0021229389999999999, 2023-09-23T19:11:10.011 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:10.011 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:11:10.011 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:10.011 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46088", 2023-09-23T19:11:10.011 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/1258423516", 2023-09-23T19:11:10.012 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 100 2023-09-23T19:11:10.012 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.012 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:10.012 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.012 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:10.012 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.764379+0000", 2023-09-23T19:11:10.012 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:10.012 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.012 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.012 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:10.012 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.764379+0000", 2023-09-23T19:11:10.013 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T19:11:10.013 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.013 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.013 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:10.013 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.764377+0000", 2023-09-23T19:11:10.013 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.1640000000000001e-06 2023-09-23T19:11:10.013 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.013 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.013 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:10.013 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.764386+0000", 2023-09-23T19:11:10.013 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8583000000000001e-05 2023-09-23T19:11:10.014 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.014 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.014 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:10.014 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.764404+0000", 2023-09-23T19:11:10.014 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.2586000000000003e-05 2023-09-23T19:11:10.014 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.014 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.014 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:10.014 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.764437+0000", 2023-09-23T19:11:10.015 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.6068999999999994e-05 2023-09-23T19:11:10.015 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.015 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.015 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:10.015 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.764503+0000", 2023-09-23T19:11:10.015 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.0062999999999995e-05 2023-09-23T19:11:10.015 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.015 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.015 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:11:10.015 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.764593+0000", 2023-09-23T19:11:10.015 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00024738700000000001 2023-09-23T19:11:10.016 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.016 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.016 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:11:10.016 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.764840+0000", 2023-09-23T19:11:10.016 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00076204799999999998 2023-09-23T19:11:10.016 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.016 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.016 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T19:11:10.016 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.765602+0000", 2023-09-23T19:11:10.016 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00021626100000000001 2023-09-23T19:11:10.016 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.017 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.017 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:10.017 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.765819+0000", 2023-09-23T19:11:10.017 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00058792999999999996 2023-09-23T19:11:10.017 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.017 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.017 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:10.017 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.766406+0000", 2023-09-23T19:11:10.017 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.3003000000000001e-05 2023-09-23T19:11:10.017 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.017 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.018 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:11:10.018 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.766459+0000", 2023-09-23T19:11:10.018 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.2391999999999997e-05 2023-09-23T19:11:10.018 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.018 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.018 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:10.018 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.766502+0000", 2023-09-23T19:11:10.018 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0021229389999999999 2023-09-23T19:11:10.018 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.018 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:10.019 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.019 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.019 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.019 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46088.0:111 2.3 2:d04613b6:::rbd_header.b408e23a9739:head [call rbd.set_size in=11b] snapc 0=[] ondisk+write+known_if_redirected e369)", 2023-09-23T19:11:10.019 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:03.952375+0000", 2023-09-23T19:11:10.019 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 125.978850689, 2023-09-23T19:11:10.019 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002115591, 2023-09-23T19:11:10.019 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:10.019 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:11:10.019 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:10.020 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46088", 2023-09-23T19:11:10.020 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/1258423516", 2023-09-23T19:11:10.020 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 111 2023-09-23T19:11:10.020 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.020 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:10.020 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.020 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:10.020 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.952375+0000", 2023-09-23T19:11:10.020 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:10.020 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.020 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.021 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:10.021 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.952375+0000", 2023-09-23T19:11:10.021 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-09-23T19:11:10.021 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.021 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.021 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:10.021 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.952373+0000", 2023-09-23T19:11:10.021 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4497e-05 2023-09-23T19:11:10.021 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.021 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.021 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:10.022 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.952387+0000", 2023-09-23T19:11:10.022 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.048e-05 2023-09-23T19:11:10.022 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.022 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.022 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:10.022 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.952418+0000", 2023-09-23T19:11:10.022 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8258999999999999e-05 2023-09-23T19:11:10.022 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.022 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.022 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:10.023 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.952436+0000", 2023-09-23T19:11:10.023 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000125401 2023-09-23T19:11:10.023 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.023 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.023 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:10.023 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.952561+0000", 2023-09-23T19:11:10.023 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000138978 2023-09-23T19:11:10.023 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.023 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.023 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:11:10.023 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.952700+0000", 2023-09-23T19:11:10.024 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00033711899999999998 2023-09-23T19:11:10.024 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.024 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.024 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:11:10.024 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.953038+0000", 2023-09-23T19:11:10.024 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00071063400000000003 2023-09-23T19:11:10.024 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.024 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.024 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T19:11:10.024 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.953748+0000", 2023-09-23T19:11:10.024 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000111162 2023-09-23T19:11:10.025 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.025 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.025 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:10.025 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.953859+0000", 2023-09-23T19:11:10.025 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00049690499999999996 2023-09-23T19:11:10.025 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.025 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.025 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:10.025 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.954356+0000", 2023-09-23T19:11:10.025 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.4097999999999994e-05 2023-09-23T19:11:10.026 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.026 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.026 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:11:10.026 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.954430+0000", 2023-09-23T19:11:10.026 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.0605e-05 2023-09-23T19:11:10.026 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.026 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.026 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:10.026 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.954491+0000", 2023-09-23T19:11:10.026 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002115591 2023-09-23T19:11:10.027 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.027 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:10.027 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.027 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.027 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.027 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46088.0:118 2.3 2:d04613b6:::rbd_header.b408e23a9739:head [watch unwatch cookie 140275242574224] snapc 0=[] ondisk+write+known_if_redirected e369)", 2023-09-23T19:11:10.027 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:03.960765+0000", 2023-09-23T19:11:10.027 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 125.9704607, 2023-09-23T19:11:10.027 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0023221299999999999, 2023-09-23T19:11:10.027 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:10.027 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:11:10.028 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:10.028 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46088", 2023-09-23T19:11:10.028 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/1258423516", 2023-09-23T19:11:10.028 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 118 2023-09-23T19:11:10.028 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.028 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:10.028 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.028 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:10.028 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.960765+0000", 2023-09-23T19:11:10.028 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:10.029 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.029 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.029 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:10.029 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.960765+0000", 2023-09-23T19:11:10.029 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T19:11:10.029 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.029 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.029 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:10.029 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.960764+0000", 2023-09-23T19:11:10.029 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.4549999999999997e-06 2023-09-23T19:11:10.030 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.030 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.030 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:10.030 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.960769+0000", 2023-09-23T19:11:10.030 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.055e-06 2023-09-23T19:11:10.030 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.030 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.030 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:10.030 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.960777+0000", 2023-09-23T19:11:10.031 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.057e-06 2023-09-23T19:11:10.031 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.031 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.031 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:10.031 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.960782+0000", 2023-09-23T19:11:10.031 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00039993500000000003 2023-09-23T19:11:10.031 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.032 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.032 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:10.032 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.961182+0000", 2023-09-23T19:11:10.032 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000132001 2023-09-23T19:11:10.032 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.032 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.032 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:11:10.032 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.961314+0000", 2023-09-23T19:11:10.032 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000300208 2023-09-23T19:11:10.032 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.033 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.033 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:11:10.033 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.961614+0000", 2023-09-23T19:11:10.033 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00063202699999999996 2023-09-23T19:11:10.033 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.033 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.033 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T19:11:10.033 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.962246+0000", 2023-09-23T19:11:10.033 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00043891399999999999 2023-09-23T19:11:10.033 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.033 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.034 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:10.034 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.962685+0000", 2023-09-23T19:11:10.034 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000102742 2023-09-23T19:11:10.034 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.034 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.034 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:10.034 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.962788+0000", 2023-09-23T19:11:10.034 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.8658999999999993e-05 2023-09-23T19:11:10.034 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.034 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.035 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:11:10.035 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.962866+0000", 2023-09-23T19:11:10.035 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00022104199999999999 2023-09-23T19:11:10.035 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.035 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.035 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:10.035 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.963087+0000", 2023-09-23T19:11:10.035 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0023221299999999999 2023-09-23T19:11:10.035 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.036 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:10.036 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.036 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.036 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.036 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46088.0:119 2.3 2:d04613b6:::rbd_header.b408e23a9739:head [notify-ack] snapc 0=[] ondisk+read+known_if_redirected e369)", 2023-09-23T19:11:10.036 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:03.961190+0000", 2023-09-23T19:11:10.036 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 125.97003626599999, 2023-09-23T19:11:10.036 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0021385409999999999, 2023-09-23T19:11:10.036 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:10.036 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "started", 2023-09-23T19:11:10.037 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:10.037 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46088", 2023-09-23T19:11:10.037 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/1258423516", 2023-09-23T19:11:10.037 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 119 2023-09-23T19:11:10.037 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.037 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:10.037 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.037 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:10.037 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.961190+0000", 2023-09-23T19:11:10.038 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:10.038 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.038 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.038 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:10.038 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.961190+0000", 2023-09-23T19:11:10.039 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T19:11:10.039 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.039 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.039 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:10.039 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.961188+0000", 2023-09-23T19:11:10.039 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0549e-05 2023-09-23T19:11:10.039 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.039 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.039 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:10.039 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.961199+0000", 2023-09-23T19:11:10.040 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5922000000000001e-05 2023-09-23T19:11:10.040 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.040 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.040 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:10.040 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.961215+0000", 2023-09-23T19:11:10.040 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.5729999999999996e-06 2023-09-23T19:11:10.040 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.040 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.040 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:10.040 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.961221+0000", 2023-09-23T19:11:10.040 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00080077499999999999 2023-09-23T19:11:10.041 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.041 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.041 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:10.041 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.962022+0000", 2023-09-23T19:11:10.041 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00010739300000000001 2023-09-23T19:11:10.041 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.041 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.041 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for rw locks", 2023-09-23T19:11:10.041 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.962129+0000", 2023-09-23T19:11:10.041 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001005534 2023-09-23T19:11:10.041 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.042 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.042 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:10.042 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.963135+0000", 2023-09-23T19:11:10.042 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.9591000000000003e-05 2023-09-23T19:11:10.042 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.042 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.042 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:11:10.042 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.963224+0000", 2023-09-23T19:11:10.042 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000103828 2023-09-23T19:11:10.042 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.045 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.045 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:10.045 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.963328+0000", 2023-09-23T19:11:10.045 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0021385409999999999 2023-09-23T19:11:10.045 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.045 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:10.045 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.045 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.046 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.046 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46100.0:36 2.3 2:dc539d31:::rbd_data.b408e23a9739.0000000000000017:head [sparse-read 0~4194304 [fadvise_nocache]] snapc 0=[] ondisk+read+known_if_redirected e371)", 2023-09-23T19:11:10.046 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:05.360144+0000", 2023-09-23T19:11:10.046 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 124.571082092, 2023-09-23T19:11:10.046 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0061211470000000004, 2023-09-23T19:11:10.046 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:10.046 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T19:11:10.046 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:10.047 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46100", 2023-09-23T19:11:10.047 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2818002861", 2023-09-23T19:11:10.047 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 36 2023-09-23T19:11:10.047 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.047 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:10.047 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.047 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:10.047 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360144+0000", 2023-09-23T19:11:10.047 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:10.047 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.048 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.048 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:10.048 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360144+0000", 2023-09-23T19:11:10.048 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999962 2023-09-23T19:11:10.048 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.048 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.048 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:10.048 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360140+0000", 2023-09-23T19:11:10.048 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8751999999999999e-05 2023-09-23T19:11:10.048 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.048 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.049 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:10.049 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360159+0000", 2023-09-23T19:11:10.049 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.9342999999999999e-05 2023-09-23T19:11:10.049 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.049 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.049 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:10.049 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360198+0000", 2023-09-23T19:11:10.049 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.0159e-05 2023-09-23T19:11:10.049 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.049 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.050 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:10.050 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360229+0000", 2023-09-23T19:11:10.050 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00019987999999999999 2023-09-23T19:11:10.050 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.050 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.050 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:10.050 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360428+0000", 2023-09-23T19:11:10.050 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.1622999999999999e-05 2023-09-23T19:11:10.050 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.051 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.051 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op must wait for map", 2023-09-23T19:11:10.051 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360450+0000", 2023-09-23T19:11:10.051 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00566119 2023-09-23T19:11:10.051 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.052 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.052 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:10.052 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.366111+0000", 2023-09-23T19:11:10.052 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000153836 2023-09-23T19:11:10.052 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.052 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.053 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:10.053 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.366265+0000", 2023-09-23T19:11:10.053 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0061211470000000004 2023-09-23T19:11:10.053 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.053 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:10.053 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.053 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.053 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.053 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46100.0:40 2.3 2:de691159:::rbd_data.b408e23a9739.000000000000001c:head [sparse-read 0~4194304 [fadvise_nocache]] snapc 0=[] ondisk+read+known_if_redirected e371)", 2023-09-23T19:11:10.054 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:05.360259+0000", 2023-09-23T19:11:10.054 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 124.57096726100001, 2023-09-23T19:11:10.054 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0061112060000000001, 2023-09-23T19:11:10.054 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:10.054 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T19:11:10.054 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:10.054 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46100", 2023-09-23T19:11:10.054 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2818002861", 2023-09-23T19:11:10.054 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 40 2023-09-23T19:11:10.054 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.055 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:10.055 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.055 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:10.055 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360259+0000", 2023-09-23T19:11:10.055 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:10.055 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.055 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.055 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:10.055 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360259+0000", 2023-09-23T19:11:10.055 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T19:11:10.056 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.056 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.056 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:10.056 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360257+0000", 2023-09-23T19:11:10.056 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2092e-05 2023-09-23T19:11:10.056 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.056 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.056 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:10.056 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360269+0000", 2023-09-23T19:11:10.056 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3907e-05 2023-09-23T19:11:10.057 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.057 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.057 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:10.057 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360293+0000", 2023-09-23T19:11:10.057 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6022999999999999e-05 2023-09-23T19:11:10.057 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.057 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.057 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:10.057 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360309+0000", 2023-09-23T19:11:10.057 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00042279699999999998 2023-09-23T19:11:10.058 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.058 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.058 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:10.058 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360732+0000", 2023-09-23T19:11:10.058 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6739e-05 2023-09-23T19:11:10.058 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.058 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.058 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting_for_map not empty", 2023-09-23T19:11:10.058 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360748+0000", 2023-09-23T19:11:10.058 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0055417280000000001 2023-09-23T19:11:10.059 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.059 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.059 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:10.059 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.366290+0000", 2023-09-23T19:11:10.059 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.9888999999999994e-05 2023-09-23T19:11:10.059 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.059 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.059 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:10.059 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.366370+0000", 2023-09-23T19:11:10.060 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0061112060000000001 2023-09-23T19:11:10.060 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.060 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:10.060 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.060 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.060 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.060 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.36759.0:16 2.3 2:d97abd30:::rbd_object_map.b6661fc69d7d:head [call lock.lock in=8b] snapc 0=[] ondisk+write+known_if_redirected+full_try e383)", 2023-09-23T19:11:10.060 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:28.118216+0000", 2023-09-23T19:11:10.060 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 101.813009835, 2023-09-23T19:11:10.060 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002381523, 2023-09-23T19:11:10.061 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:10.061 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:11:10.061 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:10.061 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.36759", 2023-09-23T19:11:10.061 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2637551032", 2023-09-23T19:11:10.061 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 16 2023-09-23T19:11:10.061 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.061 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:10.061 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.061 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:10.062 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.118216+0000", 2023-09-23T19:11:10.062 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:10.062 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.062 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.062 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:10.062 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.118216+0000", 2023-09-23T19:11:10.062 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T19:11:10.062 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.062 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.062 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:10.063 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.118214+0000", 2023-09-23T19:11:10.063 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4e-05 2023-09-23T19:11:10.063 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.063 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.063 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:10.063 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.118228+0000", 2023-09-23T19:11:10.063 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4196999999999999e-05 2023-09-23T19:11:10.063 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.063 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.063 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:10.064 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.118252+0000", 2023-09-23T19:11:10.064 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0473e-05 2023-09-23T19:11:10.064 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.064 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.064 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:10.064 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.118263+0000", 2023-09-23T19:11:10.064 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000137305 2023-09-23T19:11:10.064 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.064 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.064 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:10.064 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.118400+0000", 2023-09-23T19:11:10.065 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000159455 2023-09-23T19:11:10.065 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.065 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.065 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:11:10.065 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.118560+0000", 2023-09-23T19:11:10.065 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00034891300000000001 2023-09-23T19:11:10.065 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.065 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.065 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:11:10.065 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.118908+0000", 2023-09-23T19:11:10.066 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00088387100000000005 2023-09-23T19:11:10.066 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.066 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.066 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T19:11:10.066 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.119792+0000", 2023-09-23T19:11:10.066 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00033074599999999998 2023-09-23T19:11:10.066 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.066 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.066 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:10.066 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.120123+0000", 2023-09-23T19:11:10.067 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00032956800000000003 2023-09-23T19:11:10.067 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.067 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.067 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:10.067 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.120453+0000", 2023-09-23T19:11:10.067 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.2359000000000002e-05 2023-09-23T19:11:10.067 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.067 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.067 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:11:10.067 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.120535+0000", 2023-09-23T19:11:10.068 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.2712999999999994e-05 2023-09-23T19:11:10.068 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.068 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.068 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:10.068 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.120598+0000", 2023-09-23T19:11:10.068 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002381523 2023-09-23T19:11:10.068 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.068 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:10.068 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.068 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.068 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.069 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.36759.0:42 2.3 2:d97abd30:::rbd_object_map.b6661fc69d7d:head [call lock.lock in=8b] snapc 0=[] ondisk+write+known_if_redirected+full_try e383)", 2023-09-23T19:11:10.069 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:28.143471+0000", 2023-09-23T19:11:10.069 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 101.78775479799999, 2023-09-23T19:11:10.069 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0021612200000000002, 2023-09-23T19:11:10.069 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:10.069 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:11:10.069 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:10.069 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.36759", 2023-09-23T19:11:10.069 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2637551032", 2023-09-23T19:11:10.069 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 42 2023-09-23T19:11:10.070 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.070 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:10.070 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.070 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:10.070 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.143471+0000", 2023-09-23T19:11:10.070 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:10.070 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.070 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.070 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:10.070 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.143471+0000", 2023-09-23T19:11:10.071 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-09-23T19:11:10.071 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.071 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.071 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:10.071 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.143469+0000", 2023-09-23T19:11:10.071 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4959e-05 2023-09-23T19:11:10.071 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.071 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.071 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:10.071 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.143484+0000", 2023-09-23T19:11:10.072 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.9468e-05 2023-09-23T19:11:10.072 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.072 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.072 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:10.072 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.143513+0000", 2023-09-23T19:11:10.072 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2619999999999999e-05 2023-09-23T19:11:10.072 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.072 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.072 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:10.072 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.143526+0000", 2023-09-23T19:11:10.073 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.8781000000000002e-05 2023-09-23T19:11:10.073 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.073 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.073 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:10.073 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.143625+0000", 2023-09-23T19:11:10.073 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014421700000000001 2023-09-23T19:11:10.073 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.073 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.073 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:11:10.074 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.143769+0000", 2023-09-23T19:11:10.074 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00030605200000000002 2023-09-23T19:11:10.074 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.074 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.074 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:11:10.074 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.144075+0000", 2023-09-23T19:11:10.074 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00081743500000000004 2023-09-23T19:11:10.074 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.074 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.075 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T19:11:10.075 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.144892+0000", 2023-09-23T19:11:10.075 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018915199999999999 2023-09-23T19:11:10.075 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.075 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.076 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:10.076 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.145081+0000", 2023-09-23T19:11:10.076 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00041149200000000001 2023-09-23T19:11:10.076 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.076 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.076 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:10.077 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.145493+0000", 2023-09-23T19:11:10.077 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.4383999999999997e-05 2023-09-23T19:11:10.077 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.077 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.077 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:11:10.077 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.145577+0000", 2023-09-23T19:11:10.078 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5102000000000002e-05 2023-09-23T19:11:10.078 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.078 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.078 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:10.078 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.145632+0000", 2023-09-23T19:11:10.078 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0021612200000000002 2023-09-23T19:11:10.078 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.079 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:10.079 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.079 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.079 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.079 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.36759.0:46 2.3 2:d97abd30:::rbd_object_map.b6661fc69d7d:head [call lock.unlock in=10b] snapc 0=[] ondisk+write+known_if_redirected+full_try e383)", 2023-09-23T19:11:10.079 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:28.147383+0000", 2023-09-23T19:11:10.079 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 101.78384313799999, 2023-09-23T19:11:10.080 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0021092429999999998, 2023-09-23T19:11:10.080 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:10.080 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:11:10.080 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:10.080 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.36759", 2023-09-23T19:11:10.080 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2637551032", 2023-09-23T19:11:10.080 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 46 2023-09-23T19:11:10.081 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.081 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:10.081 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.081 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:10.081 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.147383+0000", 2023-09-23T19:11:10.081 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:10.082 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.082 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.082 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:10.082 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.147383+0000", 2023-09-23T19:11:10.082 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999971 2023-09-23T19:11:10.082 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.082 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.082 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:10.082 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.147380+0000", 2023-09-23T19:11:10.083 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7822999999999999e-05 2023-09-23T19:11:10.083 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.083 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.083 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:10.083 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.147398+0000", 2023-09-23T19:11:10.083 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.1905e-05 2023-09-23T19:11:10.083 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.083 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.083 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:10.084 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.147430+0000", 2023-09-23T19:11:10.084 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5574e-05 2023-09-23T19:11:10.084 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.084 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.084 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:10.084 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.147445+0000", 2023-09-23T19:11:10.084 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.5291999999999997e-05 2023-09-23T19:11:10.084 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.084 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.084 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:10.084 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.147521+0000", 2023-09-23T19:11:10.085 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013511099999999999 2023-09-23T19:11:10.085 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.085 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.085 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:11:10.085 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.147656+0000", 2023-09-23T19:11:10.085 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00028151199999999998 2023-09-23T19:11:10.085 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.085 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.086 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:11:10.086 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.147937+0000", 2023-09-23T19:11:10.086 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00084891299999999997 2023-09-23T19:11:10.086 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.086 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.086 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:10.086 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.148786+0000", 2023-09-23T19:11:10.086 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.926e-05 2023-09-23T19:11:10.086 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.086 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.087 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T19:11:10.087 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.148825+0000", 2023-09-23T19:11:10.087 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00054545400000000001 2023-09-23T19:11:10.087 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.087 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.087 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:10.087 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.149371+0000", 2023-09-23T19:11:10.087 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.8386e-05 2023-09-23T19:11:10.087 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.087 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.088 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:11:10.088 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.149429+0000", 2023-09-23T19:11:10.088 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.2918999999999998e-05 2023-09-23T19:11:10.088 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.088 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.088 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:10.088 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.149492+0000", 2023-09-23T19:11:10.088 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0021092429999999998 2023-09-23T19:11:10.088 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.088 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:10.089 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.089 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.089 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.089 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.36759.0:51 2.3 2:c4c92e5a:::rbd_trash:head [call rbd.trash_add in=12b] snapc 0=[] ondisk+write+known_if_redirected+full_try e383)", 2023-09-23T19:11:10.089 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:28.153238+0000", 2023-09-23T19:11:10.089 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 101.77798779, 2023-09-23T19:11:10.089 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002138937, 2023-09-23T19:11:10.089 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:10.089 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:11:10.089 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:10.089 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.36759", 2023-09-23T19:11:10.090 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2637551032", 2023-09-23T19:11:10.090 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 51 2023-09-23T19:11:10.090 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.090 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:10.090 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.090 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:10.090 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.153238+0000", 2023-09-23T19:11:10.090 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:10.090 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.091 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.091 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:10.091 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.153238+0000", 2023-09-23T19:11:10.091 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T19:11:10.091 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.091 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.091 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:10.091 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.153236+0000", 2023-09-23T19:11:10.091 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3025e-05 2023-09-23T19:11:10.091 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.091 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.092 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:10.092 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.153249+0000", 2023-09-23T19:11:10.092 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4584999999999999e-05 2023-09-23T19:11:10.092 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.092 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.092 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:10.092 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.153274+0000", 2023-09-23T19:11:10.092 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0832000000000001e-05 2023-09-23T19:11:10.092 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.092 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.092 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:10.093 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.153285+0000", 2023-09-23T19:11:10.093 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.9272999999999997e-05 2023-09-23T19:11:10.093 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.093 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.093 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:10.093 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.153374+0000", 2023-09-23T19:11:10.093 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018765399999999999 2023-09-23T19:11:10.093 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.093 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.093 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:11:10.094 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.153562+0000", 2023-09-23T19:11:10.094 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00045828200000000001 2023-09-23T19:11:10.094 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.094 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.094 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:11:10.094 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.154020+0000", 2023-09-23T19:11:10.094 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00068849099999999995 2023-09-23T19:11:10.094 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.094 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.094 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T19:11:10.095 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.154708+0000", 2023-09-23T19:11:10.095 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00048684899999999997 2023-09-23T19:11:10.095 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.095 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.095 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:10.095 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.155195+0000", 2023-09-23T19:11:10.095 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.4747999999999997e-05 2023-09-23T19:11:10.095 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.095 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.095 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:10.096 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.155260+0000", 2023-09-23T19:11:10.096 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.8441000000000004e-05 2023-09-23T19:11:10.096 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.096 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.096 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:11:10.096 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.155328+0000", 2023-09-23T19:11:10.096 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.8838999999999998e-05 2023-09-23T19:11:10.096 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.096 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.096 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:10.096 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.155377+0000", 2023-09-23T19:11:10.097 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002138937 2023-09-23T19:11:10.097 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.097 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:10.097 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.097 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.097 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.097 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.36759.0:72 2.3 2:d97abd30:::rbd_object_map.b6661fc69d7d:head [call lock.lock in=8b] snapc 0=[] ondisk+write+known_if_redirected+full_try e383)", 2023-09-23T19:11:10.097 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:28.177545+0000", 2023-09-23T19:11:10.097 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 101.753681496, 2023-09-23T19:11:10.097 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0021414839999999999, 2023-09-23T19:11:10.098 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:10.098 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:11:10.098 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:10.098 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.36759", 2023-09-23T19:11:10.098 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2637551032", 2023-09-23T19:11:10.098 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 72 2023-09-23T19:11:10.098 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.098 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:10.098 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.098 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:10.099 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.177545+0000", 2023-09-23T19:11:10.099 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:10.099 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.099 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.099 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:10.099 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.177545+0000", 2023-09-23T19:11:10.099 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-09-23T19:11:10.099 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.099 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.099 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:10.100 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.177542+0000", 2023-09-23T19:11:10.100 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3857e-05 2023-09-23T19:11:10.100 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.100 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.100 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:10.100 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.177556+0000", 2023-09-23T19:11:10.100 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.7345e-05 2023-09-23T19:11:10.100 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.100 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.100 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:10.100 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.177583+0000", 2023-09-23T19:11:10.101 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2206e-05 2023-09-23T19:11:10.101 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.101 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.101 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:10.101 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.177596+0000", 2023-09-23T19:11:10.101 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012601800000000001 2023-09-23T19:11:10.101 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.101 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.101 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:10.101 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.177722+0000", 2023-09-23T19:11:10.102 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000137897 2023-09-23T19:11:10.102 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.102 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.102 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:11:10.102 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.177860+0000", 2023-09-23T19:11:10.102 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00028948899999999998 2023-09-23T19:11:10.102 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.102 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.102 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,7", 2023-09-23T19:11:10.103 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.178149+0000", 2023-09-23T19:11:10.103 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00070110099999999998 2023-09-23T19:11:10.103 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.103 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.103 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T19:11:10.103 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.178850+0000", 2023-09-23T19:11:10.103 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000453158 2023-09-23T19:11:10.103 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.103 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.103 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:10.103 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.179303+0000", 2023-09-23T19:11:10.104 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00021622000000000001 2023-09-23T19:11:10.104 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.104 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.104 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:10.104 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.179520+0000", 2023-09-23T19:11:10.104 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.4010000000000005e-05 2023-09-23T19:11:10.104 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.104 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.104 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:11:10.104 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.179594+0000", 2023-09-23T19:11:10.105 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.2458000000000001e-05 2023-09-23T19:11:10.105 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:10.105 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:10.105 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:10.105 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.179686+0000", 2023-09-23T19:11:10.105 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0021414839999999999 2023-09-23T19:11:10.105 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.105 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:10.105 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.105 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:10.105 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:10.106 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T19:11:10.106 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:09 smithi173 bash[59422]: cluster 2023-09-23T19:11:08.722004+0000 mgr.y (mgr.26054) 1609 : cluster 0 pgmap v1799: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:10.106 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:09 smithi173 bash[59422]: audit 2023-09-23T19:11:09.654498+0000 mon.a (mon.0) 2193 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:11:10.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:09 smithi098 bash[76817]: cluster 2023-09-23T19:11:08.722004+0000 mgr.y (mgr.26054) 1609 : cluster 0 pgmap v1799: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:10.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:09 smithi098 bash[76817]: audit 2023-09-23T19:11:09.654498+0000 mon.a (mon.0) 2193 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:11:10.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:09 smithi098 bash[79516]: cluster 2023-09-23T19:11:08.722004+0000 mgr.y (mgr.26054) 1609 : cluster 0 pgmap v1799: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:10.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:09 smithi098 bash[79516]: audit 2023-09-23T19:11:09.654498+0000 mon.a (mon.0) 2193 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:11:10.823 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T19:11:11.847 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:11:11.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:11 smithi098 bash[76817]: cluster 2023-09-23T19:11:10.722819+0000 mgr.y (mgr.26054) 1610 : cluster 0 pgmap v1800: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:11.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:11 smithi098 bash[79516]: cluster 2023-09-23T19:11:10.722819+0000 mgr.y (mgr.26054) 1610 : cluster 0 pgmap v1800: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:12.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:11 smithi173 bash[59422]: cluster 2023-09-23T19:11:10.722819+0000 mgr.y (mgr.26054) 1610 : cluster 0 pgmap v1800: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:13.786 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:13 smithi098 bash[79516]: cluster 2023-09-23T19:11:12.723717+0000 mgr.y (mgr.26054) 1611 : cluster 0 pgmap v1801: 73 pgs: 73 active+clean; 8.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:14.095 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:13 smithi098 bash[76817]: cluster 2023-09-23T19:11:12.723717+0000 mgr.y (mgr.26054) 1611 : cluster 0 pgmap v1801: 73 pgs: 73 active+clean; 8.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:14.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:13 smithi173 bash[59422]: cluster 2023-09-23T19:11:12.723717+0000 mgr.y (mgr.26054) 1611 : cluster 0 pgmap v1801: 73 pgs: 73 active+clean; 8.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:15.596 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:11:16.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:15 smithi173 bash[59422]: cluster 2023-09-23T19:11:14.724314+0000 mgr.y (mgr.26054) 1612 : cluster 0 pgmap v1802: 73 pgs: 73 active+clean; 8.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:16.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:15 smithi173 bash[59422]: audit 2023-09-23T19:11:15.707748+0000 mon.a (mon.0) 2194 : audit 0 from='client.? 172.21.15.98:0/1386965552' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T19:11:16.087 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:15 smithi098 bash[79516]: cluster 2023-09-23T19:11:14.724314+0000 mgr.y (mgr.26054) 1612 : cluster 0 pgmap v1802: 73 pgs: 73 active+clean; 8.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:16.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:15 smithi098 bash[79516]: audit 2023-09-23T19:11:15.707748+0000 mon.a (mon.0) 2194 : audit 0 from='client.? 172.21.15.98:0/1386965552' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T19:11:16.088 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:15 smithi098 bash[76817]: debug 2023-09-23T19:11:15.704+0000 7fc8a2a01700 -1 mon.a@0(leader).osd e396 definitely_dead 0 2023-09-23T19:11:16.088 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:15 smithi098 bash[76817]: cluster 2023-09-23T19:11:14.724314+0000 mgr.y (mgr.26054) 1612 : cluster 0 pgmap v1802: 73 pgs: 73 active+clean; 8.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:16.088 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:15 smithi098 bash[76817]: audit 2023-09-23T19:11:15.707748+0000 mon.a (mon.0) 2194 : audit 0 from='client.? 172.21.15.98:0/1386965552' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T19:11:16.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:11:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:11:16] "GET /metrics HTTP/1.1" 200 34167 "" "Prometheus/2.33.4" 2023-09-23T19:11:16.671 INFO:teuthology.orchestra.run.smithi098.stderr:marked in osd.4. 2023-09-23T19:11:17.589 INFO:tasks.thrashosds.thrasher:Added osd 4 2023-09-23T19:11:17.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:17 smithi098 bash[79516]: cluster 2023-09-23T19:11:16.651142+0000 osd.4 (osd.4) 126 : cluster 3 failed to encode map e397 with expected crc 2023-09-23T19:11:17.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:17 smithi098 bash[79516]: cluster 2023-09-23T19:11:16.654765+0000 osd.7 (osd.7) 40 : cluster 3 failed to encode map e397 with expected crc 2023-09-23T19:11:17.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:17 smithi098 bash[79516]: audit 2023-09-23T19:11:16.656330+0000 mon.a (mon.0) 2195 : audit 1 from='client.? 172.21.15.98:0/1386965552' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T19:11:17.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:17 smithi098 bash[79516]: cluster 2023-09-23T19:11:16.656422+0000 mon.a (mon.0) 2196 : cluster 0 osdmap e397: 8 total, 8 up, 8 in 2023-09-23T19:11:17.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:17 smithi098 bash[79516]: cluster 2023-09-23T19:11:16.659112+0000 osd.4 (osd.4) 127 : cluster 3 failed to encode map e397 with expected crc 2023-09-23T19:11:17.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:17 smithi098 bash[79516]: cluster 2023-09-23T19:11:16.659134+0000 osd.7 (osd.7) 41 : cluster 3 failed to encode map e397 with expected crc 2023-09-23T19:11:17.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:17 smithi098 bash[79516]: cluster 2023-09-23T19:11:16.660542+0000 osd.4 (osd.4) 128 : cluster 3 failed to encode map e397 with expected crc 2023-09-23T19:11:17.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:17 smithi098 bash[79516]: cluster 2023-09-23T19:11:16.665170+0000 osd.5 (osd.5) 39 : cluster 3 failed to encode map e397 with expected crc 2023-09-23T19:11:17.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:17 smithi098 bash[79516]: cluster 2023-09-23T19:11:16.724964+0000 mgr.y (mgr.26054) 1613 : cluster 0 pgmap v1804: 73 pgs: 73 active+clean; 8.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:17.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:17 smithi098 bash[76817]: cluster 2023-09-23T19:11:16.651142+0000 osd.4 (osd.4) 126 : cluster 3 failed to encode map e397 with expected crc 2023-09-23T19:11:17.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:17 smithi098 bash[76817]: cluster 2023-09-23T19:11:16.654765+0000 osd.7 (osd.7) 40 : cluster 3 failed to encode map e397 with expected crc 2023-09-23T19:11:17.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:17 smithi098 bash[76817]: audit 2023-09-23T19:11:16.656330+0000 mon.a (mon.0) 2195 : audit 1 from='client.? 172.21.15.98:0/1386965552' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T19:11:17.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:17 smithi098 bash[76817]: cluster 2023-09-23T19:11:16.656422+0000 mon.a (mon.0) 2196 : cluster 0 osdmap e397: 8 total, 8 up, 8 in 2023-09-23T19:11:17.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:17 smithi098 bash[76817]: cluster 2023-09-23T19:11:16.659112+0000 osd.4 (osd.4) 127 : cluster 3 failed to encode map e397 with expected crc 2023-09-23T19:11:17.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:17 smithi098 bash[76817]: cluster 2023-09-23T19:11:16.659134+0000 osd.7 (osd.7) 41 : cluster 3 failed to encode map e397 with expected crc 2023-09-23T19:11:17.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:17 smithi098 bash[76817]: cluster 2023-09-23T19:11:16.660542+0000 osd.4 (osd.4) 128 : cluster 3 failed to encode map e397 with expected crc 2023-09-23T19:11:17.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:17 smithi098 bash[76817]: cluster 2023-09-23T19:11:16.665170+0000 osd.5 (osd.5) 39 : cluster 3 failed to encode map e397 with expected crc 2023-09-23T19:11:17.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:17 smithi098 bash[76817]: cluster 2023-09-23T19:11:16.724964+0000 mgr.y (mgr.26054) 1613 : cluster 0 pgmap v1804: 73 pgs: 73 active+clean; 8.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:18.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:17 smithi173 bash[59422]: cluster 2023-09-23T19:11:16.651142+0000 osd.4 (osd.4) 126 : cluster 3 failed to encode map e397 with expected crc 2023-09-23T19:11:18.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:17 smithi173 bash[59422]: cluster 2023-09-23T19:11:16.654765+0000 osd.7 (osd.7) 40 : cluster 3 failed to encode map e397 with expected crc 2023-09-23T19:11:18.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:17 smithi173 bash[59422]: audit 2023-09-23T19:11:16.656330+0000 mon.a (mon.0) 2195 : audit 1 from='client.? 172.21.15.98:0/1386965552' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T19:11:18.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:17 smithi173 bash[59422]: cluster 2023-09-23T19:11:16.656422+0000 mon.a (mon.0) 2196 : cluster 0 osdmap e397: 8 total, 8 up, 8 in 2023-09-23T19:11:18.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:17 smithi173 bash[59422]: cluster 2023-09-23T19:11:16.659112+0000 osd.4 (osd.4) 127 : cluster 3 failed to encode map e397 with expected crc 2023-09-23T19:11:18.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:17 smithi173 bash[59422]: cluster 2023-09-23T19:11:16.659134+0000 osd.7 (osd.7) 41 : cluster 3 failed to encode map e397 with expected crc 2023-09-23T19:11:18.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:17 smithi173 bash[59422]: cluster 2023-09-23T19:11:16.660542+0000 osd.4 (osd.4) 128 : cluster 3 failed to encode map e397 with expected crc 2023-09-23T19:11:18.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:17 smithi173 bash[59422]: cluster 2023-09-23T19:11:16.665170+0000 osd.5 (osd.5) 39 : cluster 3 failed to encode map e397 with expected crc 2023-09-23T19:11:18.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:17 smithi173 bash[59422]: cluster 2023-09-23T19:11:16.724964+0000 mgr.y (mgr.26054) 1613 : cluster 0 pgmap v1804: 73 pgs: 73 active+clean; 8.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:18.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:18 smithi098 bash[76817]: cluster 2023-09-23T19:11:17.658773+0000 osd.4 (osd.4) 129 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:18.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:18 smithi098 bash[76817]: cluster 2023-09-23T19:11:17.659783+0000 osd.7 (osd.7) 42 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:18.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:18 smithi098 bash[76817]: cluster 2023-09-23T19:11:17.659831+0000 mon.a (mon.0) 2197 : cluster 0 osdmap e398: 8 total, 8 up, 8 in 2023-09-23T19:11:18.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:18 smithi098 bash[76817]: cluster 2023-09-23T19:11:17.661963+0000 osd.4 (osd.4) 130 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:18.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:18 smithi098 bash[76817]: cluster 2023-09-23T19:11:17.662710+0000 osd.7 (osd.7) 43 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:18.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:18 smithi098 bash[76817]: cluster 2023-09-23T19:11:17.663207+0000 osd.4 (osd.4) 131 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:18.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:18 smithi098 bash[76817]: cluster 2023-09-23T19:11:17.663965+0000 osd.7 (osd.7) 44 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:18.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:18 smithi098 bash[76817]: cluster 2023-09-23T19:11:17.664190+0000 osd.5 (osd.5) 40 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:18.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:18 smithi098 bash[76817]: cluster 2023-09-23T19:11:17.667568+0000 osd.7 (osd.7) 45 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:18.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:18 smithi098 bash[76817]: cluster 2023-09-23T19:11:17.669566+0000 osd.7 (osd.7) 46 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:18.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:18 smithi098 bash[76817]: cluster 2023-09-23T19:11:17.671689+0000 osd.7 (osd.7) 47 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:18.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:18 smithi098 bash[76817]: cluster 2023-09-23T19:11:17.673593+0000 osd.7 (osd.7) 48 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:18.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:18 smithi098 bash[79516]: cluster 2023-09-23T19:11:17.658773+0000 osd.4 (osd.4) 129 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:18.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:18 smithi098 bash[79516]: cluster 2023-09-23T19:11:17.659783+0000 osd.7 (osd.7) 42 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:18.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:18 smithi098 bash[79516]: cluster 2023-09-23T19:11:17.659831+0000 mon.a (mon.0) 2197 : cluster 0 osdmap e398: 8 total, 8 up, 8 in 2023-09-23T19:11:18.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:18 smithi098 bash[79516]: cluster 2023-09-23T19:11:17.661963+0000 osd.4 (osd.4) 130 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:18.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:18 smithi098 bash[79516]: cluster 2023-09-23T19:11:17.662710+0000 osd.7 (osd.7) 43 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:18.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:18 smithi098 bash[79516]: cluster 2023-09-23T19:11:17.663207+0000 osd.4 (osd.4) 131 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:18.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:18 smithi098 bash[79516]: cluster 2023-09-23T19:11:17.663965+0000 osd.7 (osd.7) 44 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:18.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:18 smithi098 bash[79516]: cluster 2023-09-23T19:11:17.664190+0000 osd.5 (osd.5) 40 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:18.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:18 smithi098 bash[79516]: cluster 2023-09-23T19:11:17.667568+0000 osd.7 (osd.7) 45 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:18.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:18 smithi098 bash[79516]: cluster 2023-09-23T19:11:17.669566+0000 osd.7 (osd.7) 46 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:18.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:18 smithi098 bash[79516]: cluster 2023-09-23T19:11:17.671689+0000 osd.7 (osd.7) 47 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:18.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:18 smithi098 bash[79516]: cluster 2023-09-23T19:11:17.673593+0000 osd.7 (osd.7) 48 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:19.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:18 smithi173 bash[59422]: cluster 2023-09-23T19:11:17.658773+0000 osd.4 (osd.4) 129 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:19.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:18 smithi173 bash[59422]: cluster 2023-09-23T19:11:17.659783+0000 osd.7 (osd.7) 42 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:19.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:18 smithi173 bash[59422]: cluster 2023-09-23T19:11:17.659831+0000 mon.a (mon.0) 2197 : cluster 0 osdmap e398: 8 total, 8 up, 8 in 2023-09-23T19:11:19.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:18 smithi173 bash[59422]: cluster 2023-09-23T19:11:17.661963+0000 osd.4 (osd.4) 130 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:19.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:18 smithi173 bash[59422]: cluster 2023-09-23T19:11:17.662710+0000 osd.7 (osd.7) 43 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:19.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:18 smithi173 bash[59422]: cluster 2023-09-23T19:11:17.663207+0000 osd.4 (osd.4) 131 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:19.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:18 smithi173 bash[59422]: cluster 2023-09-23T19:11:17.663965+0000 osd.7 (osd.7) 44 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:19.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:18 smithi173 bash[59422]: cluster 2023-09-23T19:11:17.664190+0000 osd.5 (osd.5) 40 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:19.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:18 smithi173 bash[59422]: cluster 2023-09-23T19:11:17.667568+0000 osd.7 (osd.7) 45 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:19.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:18 smithi173 bash[59422]: cluster 2023-09-23T19:11:17.669566+0000 osd.7 (osd.7) 46 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:19.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:18 smithi173 bash[59422]: cluster 2023-09-23T19:11:17.671689+0000 osd.7 (osd.7) 47 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:19.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:18 smithi173 bash[59422]: cluster 2023-09-23T19:11:17.673593+0000 osd.7 (osd.7) 48 : cluster 3 failed to encode map e398 with expected crc 2023-09-23T19:11:19.302 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T19:11:19.303 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-09-23T19:11:19.303 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 0 2023-09-23T19:11:19.303 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T19:11:19.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:19 smithi173 bash[59422]: cluster 2023-09-23T19:11:18.667270+0000 mon.a (mon.0) 2198 : cluster 0 osdmap e399: 8 total, 8 up, 8 in 2023-09-23T19:11:19.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:19 smithi173 bash[59422]: cluster 2023-09-23T19:11:18.669484+0000 osd.5 (osd.5) 41 : cluster 3 failed to encode map e399 with expected crc 2023-09-23T19:11:19.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:19 smithi173 bash[59422]: cluster 2023-09-23T19:11:18.671478+0000 osd.7 (osd.7) 49 : cluster 3 failed to encode map e399 with expected crc 2023-09-23T19:11:19.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:19 smithi173 bash[59422]: cluster 2023-09-23T19:11:18.674251+0000 osd.7 (osd.7) 50 : cluster 3 failed to encode map e399 with expected crc 2023-09-23T19:11:19.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:19 smithi173 bash[59422]: cluster 2023-09-23T19:11:18.676402+0000 osd.7 (osd.7) 51 : cluster 3 failed to encode map e399 with expected crc 2023-09-23T19:11:19.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:19 smithi173 bash[59422]: cluster 2023-09-23T19:11:18.676862+0000 osd.6 (osd.6) 385 : cluster 3 failed to encode map e399 with expected crc 2023-09-23T19:11:19.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:19 smithi173 bash[59422]: cluster 2023-09-23T19:11:18.725651+0000 mgr.y (mgr.26054) 1614 : cluster 0 pgmap v1807: 73 pgs: 6 peering, 67 active+clean; 8.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:19.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:19 smithi098 bash[76817]: cluster 2023-09-23T19:11:18.667270+0000 mon.a (mon.0) 2198 : cluster 0 osdmap e399: 8 total, 8 up, 8 in 2023-09-23T19:11:19.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:19 smithi098 bash[76817]: cluster 2023-09-23T19:11:18.669484+0000 osd.5 (osd.5) 41 : cluster 3 failed to encode map e399 with expected crc 2023-09-23T19:11:19.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:19 smithi098 bash[76817]: cluster 2023-09-23T19:11:18.671478+0000 osd.7 (osd.7) 49 : cluster 3 failed to encode map e399 with expected crc 2023-09-23T19:11:19.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:19 smithi098 bash[76817]: cluster 2023-09-23T19:11:18.674251+0000 osd.7 (osd.7) 50 : cluster 3 failed to encode map e399 with expected crc 2023-09-23T19:11:19.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:19 smithi098 bash[76817]: cluster 2023-09-23T19:11:18.676402+0000 osd.7 (osd.7) 51 : cluster 3 failed to encode map e399 with expected crc 2023-09-23T19:11:19.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:19 smithi098 bash[76817]: cluster 2023-09-23T19:11:18.676862+0000 osd.6 (osd.6) 385 : cluster 3 failed to encode map e399 with expected crc 2023-09-23T19:11:19.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:19 smithi098 bash[76817]: cluster 2023-09-23T19:11:18.725651+0000 mgr.y (mgr.26054) 1614 : cluster 0 pgmap v1807: 73 pgs: 6 peering, 67 active+clean; 8.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:19.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:19 smithi098 bash[79516]: cluster 2023-09-23T19:11:18.667270+0000 mon.a (mon.0) 2198 : cluster 0 osdmap e399: 8 total, 8 up, 8 in 2023-09-23T19:11:19.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:19 smithi098 bash[79516]: cluster 2023-09-23T19:11:18.669484+0000 osd.5 (osd.5) 41 : cluster 3 failed to encode map e399 with expected crc 2023-09-23T19:11:19.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:19 smithi098 bash[79516]: cluster 2023-09-23T19:11:18.671478+0000 osd.7 (osd.7) 49 : cluster 3 failed to encode map e399 with expected crc 2023-09-23T19:11:19.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:19 smithi098 bash[79516]: cluster 2023-09-23T19:11:18.674251+0000 osd.7 (osd.7) 50 : cluster 3 failed to encode map e399 with expected crc 2023-09-23T19:11:19.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:19 smithi098 bash[79516]: cluster 2023-09-23T19:11:18.676402+0000 osd.7 (osd.7) 51 : cluster 3 failed to encode map e399 with expected crc 2023-09-23T19:11:19.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:19 smithi098 bash[79516]: cluster 2023-09-23T19:11:18.676862+0000 osd.6 (osd.6) 385 : cluster 3 failed to encode map e399 with expected crc 2023-09-23T19:11:19.946 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:19 smithi098 bash[79516]: cluster 2023-09-23T19:11:18.725651+0000 mgr.y (mgr.26054) 1614 : cluster 0 pgmap v1807: 73 pgs: 6 peering, 67 active+clean; 8.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-09-23T19:11:20.237 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T19:11:20.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:20 smithi098 bash[76817]: cluster 2023-09-23T19:11:19.676823+0000 osd.6 (osd.6) 386 : cluster 3 failed to encode map e400 with expected crc 2023-09-23T19:11:20.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:20 smithi098 bash[76817]: cluster 2023-09-23T19:11:19.678450+0000 mon.a (mon.0) 2199 : cluster 0 osdmap e400: 8 total, 8 up, 8 in 2023-09-23T19:11:20.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:20 smithi098 bash[76817]: cluster 2023-09-23T19:11:19.679645+0000 osd.4 (osd.4) 132 : cluster 3 failed to encode map e400 with expected crc 2023-09-23T19:11:20.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:20 smithi098 bash[76817]: cluster 2023-09-23T19:11:19.682552+0000 osd.7 (osd.7) 52 : cluster 3 failed to encode map e400 with expected crc 2023-09-23T19:11:20.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:20 smithi098 bash[76817]: cluster 2023-09-23T19:11:19.684161+0000 osd.7 (osd.7) 53 : cluster 3 failed to encode map e400 with expected crc 2023-09-23T19:11:20.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:20 smithi098 bash[76817]: cluster 2023-09-23T19:11:19.807289+0000 osd.5 (osd.5) 42 : cluster 3 failed to encode map e400 with expected crc 2023-09-23T19:11:20.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:20 smithi098 bash[79516]: cluster 2023-09-23T19:11:19.676823+0000 osd.6 (osd.6) 386 : cluster 3 failed to encode map e400 with expected crc 2023-09-23T19:11:20.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:20 smithi098 bash[79516]: cluster 2023-09-23T19:11:19.678450+0000 mon.a (mon.0) 2199 : cluster 0 osdmap e400: 8 total, 8 up, 8 in 2023-09-23T19:11:20.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:20 smithi098 bash[79516]: cluster 2023-09-23T19:11:19.679645+0000 osd.4 (osd.4) 132 : cluster 3 failed to encode map e400 with expected crc 2023-09-23T19:11:20.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:20 smithi098 bash[79516]: cluster 2023-09-23T19:11:19.682552+0000 osd.7 (osd.7) 52 : cluster 3 failed to encode map e400 with expected crc 2023-09-23T19:11:20.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:20 smithi098 bash[79516]: cluster 2023-09-23T19:11:19.684161+0000 osd.7 (osd.7) 53 : cluster 3 failed to encode map e400 with expected crc 2023-09-23T19:11:20.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:20 smithi098 bash[79516]: cluster 2023-09-23T19:11:19.807289+0000 osd.5 (osd.5) 42 : cluster 3 failed to encode map e400 with expected crc 2023-09-23T19:11:21.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:20 smithi173 bash[59422]: cluster 2023-09-23T19:11:19.676823+0000 osd.6 (osd.6) 386 : cluster 3 failed to encode map e400 with expected crc 2023-09-23T19:11:21.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:20 smithi173 bash[59422]: cluster 2023-09-23T19:11:19.678450+0000 mon.a (mon.0) 2199 : cluster 0 osdmap e400: 8 total, 8 up, 8 in 2023-09-23T19:11:21.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:20 smithi173 bash[59422]: cluster 2023-09-23T19:11:19.679645+0000 osd.4 (osd.4) 132 : cluster 3 failed to encode map e400 with expected crc 2023-09-23T19:11:21.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:20 smithi173 bash[59422]: cluster 2023-09-23T19:11:19.682552+0000 osd.7 (osd.7) 52 : cluster 3 failed to encode map e400 with expected crc 2023-09-23T19:11:21.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:20 smithi173 bash[59422]: cluster 2023-09-23T19:11:19.684161+0000 osd.7 (osd.7) 53 : cluster 3 failed to encode map e400 with expected crc 2023-09-23T19:11:21.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:20 smithi173 bash[59422]: cluster 2023-09-23T19:11:19.807289+0000 osd.5 (osd.5) 42 : cluster 3 failed to encode map e400 with expected crc 2023-09-23T19:11:22.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:21 smithi173 bash[59422]: cluster 2023-09-23T19:11:20.681632+0000 osd.6 (osd.6) 387 : cluster 3 failed to encode map e401 with expected crc 2023-09-23T19:11:22.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:21 smithi173 bash[59422]: cluster 2023-09-23T19:11:20.681681+0000 osd.5 (osd.5) 43 : cluster 3 failed to encode map e401 with expected crc 2023-09-23T19:11:22.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:21 smithi173 bash[59422]: cluster 2023-09-23T19:11:20.682167+0000 mon.a (mon.0) 2200 : cluster 0 osdmap e401: 8 total, 8 up, 8 in 2023-09-23T19:11:22.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:21 smithi173 bash[59422]: cluster 2023-09-23T19:11:20.686334+0000 osd.7 (osd.7) 54 : cluster 3 failed to encode map e401 with expected crc 2023-09-23T19:11:22.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:21 smithi173 bash[59422]: cluster 2023-09-23T19:11:20.688417+0000 osd.4 (osd.4) 133 : cluster 3 failed to encode map e401 with expected crc 2023-09-23T19:11:22.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:21 smithi173 bash[59422]: cluster 2023-09-23T19:11:20.690846+0000 osd.7 (osd.7) 55 : cluster 3 failed to encode map e401 with expected crc 2023-09-23T19:11:22.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:21 smithi173 bash[59422]: cluster 2023-09-23T19:11:20.726263+0000 mgr.y (mgr.26054) 1615 : cluster 0 pgmap v1810: 73 pgs: 7 peering, 66 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 1.0 MiB/s, 2 keys/s, 6 objects/s recovering 2023-09-23T19:11:22.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:21 smithi098 bash[79516]: cluster 2023-09-23T19:11:20.681632+0000 osd.6 (osd.6) 387 : cluster 3 failed to encode map e401 with expected crc 2023-09-23T19:11:22.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:21 smithi098 bash[79516]: cluster 2023-09-23T19:11:20.681681+0000 osd.5 (osd.5) 43 : cluster 3 failed to encode map e401 with expected crc 2023-09-23T19:11:22.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:21 smithi098 bash[79516]: cluster 2023-09-23T19:11:20.682167+0000 mon.a (mon.0) 2200 : cluster 0 osdmap e401: 8 total, 8 up, 8 in 2023-09-23T19:11:22.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:21 smithi098 bash[79516]: cluster 2023-09-23T19:11:20.686334+0000 osd.7 (osd.7) 54 : cluster 3 failed to encode map e401 with expected crc 2023-09-23T19:11:22.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:21 smithi098 bash[79516]: cluster 2023-09-23T19:11:20.688417+0000 osd.4 (osd.4) 133 : cluster 3 failed to encode map e401 with expected crc 2023-09-23T19:11:22.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:21 smithi098 bash[79516]: cluster 2023-09-23T19:11:20.690846+0000 osd.7 (osd.7) 55 : cluster 3 failed to encode map e401 with expected crc 2023-09-23T19:11:22.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:21 smithi098 bash[79516]: cluster 2023-09-23T19:11:20.726263+0000 mgr.y (mgr.26054) 1615 : cluster 0 pgmap v1810: 73 pgs: 7 peering, 66 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 1.0 MiB/s, 2 keys/s, 6 objects/s recovering 2023-09-23T19:11:22.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:21 smithi098 bash[76817]: cluster 2023-09-23T19:11:20.681632+0000 osd.6 (osd.6) 387 : cluster 3 failed to encode map e401 with expected crc 2023-09-23T19:11:22.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:21 smithi098 bash[76817]: cluster 2023-09-23T19:11:20.681681+0000 osd.5 (osd.5) 43 : cluster 3 failed to encode map e401 with expected crc 2023-09-23T19:11:22.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:21 smithi098 bash[76817]: cluster 2023-09-23T19:11:20.682167+0000 mon.a (mon.0) 2200 : cluster 0 osdmap e401: 8 total, 8 up, 8 in 2023-09-23T19:11:22.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:21 smithi098 bash[76817]: cluster 2023-09-23T19:11:20.686334+0000 osd.7 (osd.7) 54 : cluster 3 failed to encode map e401 with expected crc 2023-09-23T19:11:22.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:21 smithi098 bash[76817]: cluster 2023-09-23T19:11:20.688417+0000 osd.4 (osd.4) 133 : cluster 3 failed to encode map e401 with expected crc 2023-09-23T19:11:22.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:21 smithi098 bash[76817]: cluster 2023-09-23T19:11:20.690846+0000 osd.7 (osd.7) 55 : cluster 3 failed to encode map e401 with expected crc 2023-09-23T19:11:22.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:21 smithi098 bash[76817]: cluster 2023-09-23T19:11:20.726263+0000 mgr.y (mgr.26054) 1615 : cluster 0 pgmap v1810: 73 pgs: 7 peering, 66 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 1.0 MiB/s, 2 keys/s, 6 objects/s recovering 2023-09-23T19:11:22.590 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-09-23T19:11:22.590 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 51 2023-09-23T19:11:22.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:22 smithi098 bash[76817]: audit 2023-09-23T19:11:22.502018+0000 mon.a (mon.0) 2201 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:11:22.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:22 smithi098 bash[79516]: audit 2023-09-23T19:11:22.502018+0000 mon.a (mon.0) 2201 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:11:23.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:22 smithi173 bash[59422]: audit 2023-09-23T19:11:22.502018+0000 mon.a (mon.0) 2201 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:11:24.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:23 smithi173 bash[59422]: cluster 2023-09-23T19:11:22.727169+0000 mgr.y (mgr.26054) 1616 : cluster 0 pgmap v1811: 73 pgs: 1 peering, 72 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 809 KiB/s, 1 keys/s, 10 objects/s recovering 2023-09-23T19:11:24.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:23 smithi098 bash[76817]: cluster 2023-09-23T19:11:22.727169+0000 mgr.y (mgr.26054) 1616 : cluster 0 pgmap v1811: 73 pgs: 1 peering, 72 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 809 KiB/s, 1 keys/s, 10 objects/s recovering 2023-09-23T19:11:24.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:23 smithi098 bash[79516]: cluster 2023-09-23T19:11:22.727169+0000 mgr.y (mgr.26054) 1616 : cluster 0 pgmap v1811: 73 pgs: 1 peering, 72 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 809 KiB/s, 1 keys/s, 10 objects/s recovering 2023-09-23T19:11:25.013 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:11:25.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:24 smithi173 bash[59422]: audit 2023-09-23T19:11:24.654860+0000 mon.a (mon.0) 2202 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:11:25.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:24 smithi098 bash[76817]: audit 2023-09-23T19:11:24.654860+0000 mon.a (mon.0) 2202 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:11:25.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:24 smithi098 bash[79516]: audit 2023-09-23T19:11:24.654860+0000 mon.a (mon.0) 2202 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:11:26.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:25 smithi173 bash[59422]: cluster 2023-09-23T19:11:24.727938+0000 mgr.y (mgr.26054) 1617 : cluster 0 pgmap v1812: 73 pgs: 1 peering, 72 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 676 KiB/s, 1 keys/s, 8 objects/s recovering 2023-09-23T19:11:26.088 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:25 smithi098 bash[76817]: cluster 2023-09-23T19:11:24.727938+0000 mgr.y (mgr.26054) 1617 : cluster 0 pgmap v1812: 73 pgs: 1 peering, 72 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 676 KiB/s, 1 keys/s, 8 objects/s recovering 2023-09-23T19:11:26.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:25 smithi098 bash[79516]: cluster 2023-09-23T19:11:24.727938+0000 mgr.y (mgr.26054) 1617 : cluster 0 pgmap v1812: 73 pgs: 1 peering, 72 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 676 KiB/s, 1 keys/s, 8 objects/s recovering 2023-09-23T19:11:26.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:11:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:11:26] "GET /metrics HTTP/1.1" 200 34184 "" "Prometheus/2.33.4" 2023-09-23T19:11:27.371 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:11:28.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:27 smithi098 bash[79516]: cluster 2023-09-23T19:11:26.728878+0000 mgr.y (mgr.26054) 1618 : cluster 0 pgmap v1813: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 512 KiB/s, 1 keys/s, 6 objects/s recovering 2023-09-23T19:11:28.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:27 smithi098 bash[76817]: cluster 2023-09-23T19:11:26.728878+0000 mgr.y (mgr.26054) 1618 : cluster 0 pgmap v1813: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 512 KiB/s, 1 keys/s, 6 objects/s recovering 2023-09-23T19:11:28.259 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:27 smithi173 bash[59422]: cluster 2023-09-23T19:11:26.728878+0000 mgr.y (mgr.26054) 1618 : cluster 0 pgmap v1813: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 512 KiB/s, 1 keys/s, 6 objects/s recovering 2023-09-23T19:11:28.448 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T19:11:28.449 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-09-23T19:11:28.449 INFO:teuthology.orchestra.run.smithi173.stdout: "complaint_time": 30, 2023-09-23T19:11:28.449 INFO:teuthology.orchestra.run.smithi173.stdout: "num_blocked_ops": 0 2023-09-23T19:11:28.449 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T19:11:29.407 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T19:11:29.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:29 smithi173 bash[59422]: audit 2023-09-23T19:11:28.541454+0000 mon.a (mon.0) 2203 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:11:29.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:29 smithi173 bash[59422]: audit 2023-09-23T19:11:28.549196+0000 mon.a (mon.0) 2204 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:11:29.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:29 smithi173 bash[59422]: cluster 2023-09-23T19:11:28.729835+0000 mgr.y (mgr.26054) 1619 : cluster 0 pgmap v1814: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 452 KiB/s, 1 keys/s, 5 objects/s recovering 2023-09-23T19:11:29.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:29 smithi173 bash[59422]: audit 2023-09-23T19:11:29.056093+0000 mon.a (mon.0) 2205 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:11:29.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:29 smithi173 bash[59422]: audit 2023-09-23T19:11:29.065064+0000 mon.a (mon.0) 2206 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:11:29.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:29 smithi173 bash[59422]: audit 2023-09-23T19:11:29.537164+0000 mon.a (mon.0) 2207 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:11:29.897 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:29 smithi098 bash[79516]: audit 2023-09-23T19:11:28.541454+0000 mon.a (mon.0) 2203 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:11:29.898 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:29 smithi098 bash[79516]: audit 2023-09-23T19:11:28.549196+0000 mon.a (mon.0) 2204 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:11:29.898 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:29 smithi098 bash[79516]: cluster 2023-09-23T19:11:28.729835+0000 mgr.y (mgr.26054) 1619 : cluster 0 pgmap v1814: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 452 KiB/s, 1 keys/s, 5 objects/s recovering 2023-09-23T19:11:29.898 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:29 smithi098 bash[79516]: audit 2023-09-23T19:11:29.056093+0000 mon.a (mon.0) 2205 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:11:29.898 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:29 smithi098 bash[79516]: audit 2023-09-23T19:11:29.065064+0000 mon.a (mon.0) 2206 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:11:29.898 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:29 smithi098 bash[79516]: audit 2023-09-23T19:11:29.537164+0000 mon.a (mon.0) 2207 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:11:29.898 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:29 smithi098 bash[76817]: audit 2023-09-23T19:11:28.541454+0000 mon.a (mon.0) 2203 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:11:29.898 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:29 smithi098 bash[76817]: audit 2023-09-23T19:11:28.549196+0000 mon.a (mon.0) 2204 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:11:29.898 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:29 smithi098 bash[76817]: cluster 2023-09-23T19:11:28.729835+0000 mgr.y (mgr.26054) 1619 : cluster 0 pgmap v1814: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 452 KiB/s, 1 keys/s, 5 objects/s recovering 2023-09-23T19:11:29.899 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:29 smithi098 bash[76817]: audit 2023-09-23T19:11:29.056093+0000 mon.a (mon.0) 2205 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:11:29.899 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:29 smithi098 bash[76817]: audit 2023-09-23T19:11:29.065064+0000 mon.a (mon.0) 2206 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:11:29.899 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:29 smithi098 bash[76817]: audit 2023-09-23T19:11:29.537164+0000 mon.a (mon.0) 2207 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:11:30.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:30 smithi098 bash[76817]: audit 2023-09-23T19:11:29.538698+0000 mon.a (mon.0) 2208 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:11:30.941 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:30 smithi098 bash[76817]: audit 2023-09-23T19:11:29.549102+0000 mon.a (mon.0) 2209 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:11:30.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:30 smithi098 bash[79516]: audit 2023-09-23T19:11:29.538698+0000 mon.a (mon.0) 2208 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:11:30.942 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:30 smithi098 bash[79516]: audit 2023-09-23T19:11:29.549102+0000 mon.a (mon.0) 2209 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:11:31.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:30 smithi173 bash[59422]: audit 2023-09-23T19:11:29.538698+0000 mon.a (mon.0) 2208 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:11:31.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:30 smithi173 bash[59422]: audit 2023-09-23T19:11:29.549102+0000 mon.a (mon.0) 2209 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:11:31.380 INFO:teuthology.orchestra.run.smithi098.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-09-23T19:11:31.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:31 smithi098 bash[76817]: audit 2023-09-23T19:11:30.636399+0000 mon.a (mon.0) 2210 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:11:31.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:31 smithi098 bash[76817]: cluster 2023-09-23T19:11:30.730792+0000 mgr.y (mgr.26054) 1620 : cluster 0 pgmap v1815: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-09-23T19:11:31.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:31 smithi098 bash[76817]: audit 2023-09-23T19:11:31.373152+0000 mon.c (mon.2) 163 : audit 1 from='client.? 172.21.15.98:0/2460058112' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "51"}]: dispatch 2023-09-23T19:11:31.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:31 smithi098 bash[76817]: audit 2023-09-23T19:11:31.373727+0000 mon.a (mon.0) 2211 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "51"}]: dispatch 2023-09-23T19:11:31.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:31 smithi098 bash[79516]: audit 2023-09-23T19:11:30.636399+0000 mon.a (mon.0) 2210 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:11:31.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:31 smithi098 bash[79516]: cluster 2023-09-23T19:11:30.730792+0000 mgr.y (mgr.26054) 1620 : cluster 0 pgmap v1815: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-09-23T19:11:31.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:31 smithi098 bash[79516]: audit 2023-09-23T19:11:31.373152+0000 mon.c (mon.2) 163 : audit 1 from='client.? 172.21.15.98:0/2460058112' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "51"}]: dispatch 2023-09-23T19:11:31.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:31 smithi098 bash[79516]: audit 2023-09-23T19:11:31.373727+0000 mon.a (mon.0) 2211 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "51"}]: dispatch 2023-09-23T19:11:32.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:31 smithi173 bash[59422]: audit 2023-09-23T19:11:30.636399+0000 mon.a (mon.0) 2210 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:11:32.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:31 smithi173 bash[59422]: cluster 2023-09-23T19:11:30.730792+0000 mgr.y (mgr.26054) 1620 : cluster 0 pgmap v1815: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-09-23T19:11:32.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:31 smithi173 bash[59422]: audit 2023-09-23T19:11:31.373152+0000 mon.c (mon.2) 163 : audit 1 from='client.? 172.21.15.98:0/2460058112' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "51"}]: dispatch 2023-09-23T19:11:32.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:31 smithi173 bash[59422]: audit 2023-09-23T19:11:31.373727+0000 mon.a (mon.0) 2211 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "51"}]: dispatch 2023-09-23T19:11:32.329 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-09-23T19:11:32.330 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-09-23T19:11:34.182 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T19:11:34.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:33 smithi098 bash[76817]: cluster 2023-09-23T19:11:32.731637+0000 mgr.y (mgr.26054) 1621 : cluster 0 pgmap v1816: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-09-23T19:11:34.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:33 smithi098 bash[79516]: cluster 2023-09-23T19:11:32.731637+0000 mgr.y (mgr.26054) 1621 : cluster 0 pgmap v1816: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-09-23T19:11:34.219 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:33 smithi173 bash[59422]: cluster 2023-09-23T19:11:32.731637+0000 mgr.y (mgr.26054) 1621 : cluster 0 pgmap v1816: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-09-23T19:11:36.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:35 smithi173 bash[59422]: cluster 2023-09-23T19:11:34.732457+0000 mgr.y (mgr.26054) 1622 : cluster 0 pgmap v1817: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:36.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:35 smithi098 bash[79516]: cluster 2023-09-23T19:11:34.732457+0000 mgr.y (mgr.26054) 1622 : cluster 0 pgmap v1817: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:36.089 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:35 smithi098 bash[76817]: cluster 2023-09-23T19:11:34.732457+0000 mgr.y (mgr.26054) 1622 : cluster 0 pgmap v1817: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:36.445 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:11:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:11:36] "GET /metrics HTTP/1.1" 200 34184 "" "Prometheus/2.33.4" 2023-09-23T19:11:37.106 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:11:37.694 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-09-23T19:11:37.694 INFO:teuthology.orchestra.run.smithi173.stdout: "size": 20, 2023-09-23T19:11:37.694 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 600, 2023-09-23T19:11:37.695 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-09-23T19:11:37.695 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.695 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46088.0:51 2.0 2:01db4400:::rbd_data.b408e23a9739.0000000000000002:head [set-alloc-hint object_size 4194304 write_size 4194304,write 1851392~143360 [fadvise_nocache]] snapc 0=[] ondisk+write+known_if_redirected e369)", 2023-09-23T19:11:37.695 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:03.386322+0000", 2023-09-23T19:11:37.695 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 154.30168943999999, 2023-09-23T19:11:37.695 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.041417773999999997, 2023-09-23T19:11:37.695 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:37.696 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:11:37.696 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:37.696 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46088", 2023-09-23T19:11:37.696 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/1258423516", 2023-09-23T19:11:37.696 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 51 2023-09-23T19:11:37.696 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.696 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:37.696 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.696 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:37.696 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.386322+0000", 2023-09-23T19:11:37.697 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:37.697 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.697 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.697 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:37.697 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.386322+0000", 2023-09-23T19:11:37.697 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T19:11:37.697 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.698 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.698 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:37.698 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.386320+0000", 2023-09-23T19:11:37.698 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00030942399999999998 2023-09-23T19:11:37.698 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.698 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.698 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:37.698 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.386629+0000", 2023-09-23T19:11:37.699 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6452000000000001e-05 2023-09-23T19:11:37.699 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.699 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.699 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:37.699 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.386645+0000", 2023-09-23T19:11:37.699 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.9990000000000003e-06 2023-09-23T19:11:37.699 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.700 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.700 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:37.700 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.386655+0000", 2023-09-23T19:11:37.700 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.4145000000000002e-05 2023-09-23T19:11:37.700 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.700 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.700 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:37.700 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.386740+0000", 2023-09-23T19:11:37.701 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000137892 2023-09-23T19:11:37.701 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.701 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.701 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:11:37.701 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.386877+0000", 2023-09-23T19:11:37.701 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018325700000000001 2023-09-23T19:11:37.701 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.701 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.702 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,1", 2023-09-23T19:11:37.702 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.387061+0000", 2023-09-23T19:11:37.702 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000735022 2023-09-23T19:11:37.702 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.702 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.702 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T19:11:37.702 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.387796+0000", 2023-09-23T19:11:37.703 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00080011600000000002 2023-09-23T19:11:37.703 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.703 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.703 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:37.703 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.388596+0000", 2023-09-23T19:11:37.703 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.038936871999999997 2023-09-23T19:11:37.703 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.703 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.704 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:37.704 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.427533+0000", 2023-09-23T19:11:37.704 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.2798000000000002e-05 2023-09-23T19:11:37.704 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.704 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.704 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:11:37.704 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.427626+0000", 2023-09-23T19:11:37.704 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000113903 2023-09-23T19:11:37.705 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.705 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.705 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:37.705 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.427739+0000", 2023-09-23T19:11:37.705 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.041417773999999997 2023-09-23T19:11:37.705 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.705 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:37.705 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.705 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.706 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.706 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46088.0:80 2.7 e369/368 2:e1b06f48:::rbd_data.b408e23a9739.0000000000000018:head v 369'113, mlcod=0'0)", 2023-09-23T19:11:37.706 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:03.591465+0000", 2023-09-23T19:11:37.706 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 154.09654634500001, 2023-09-23T19:11:37.706 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0012538600000000001, 2023-09-23T19:11:37.706 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:37.706 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:11:37.706 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:37.707 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.707 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:37.707 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.591465+0000", 2023-09-23T19:11:37.707 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:37.707 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.707 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.707 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:37.708 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.591465+0000", 2023-09-23T19:11:37.708 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T19:11:37.708 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.708 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.708 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:37.708 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.591463+0000", 2023-09-23T19:11:37.708 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016640200000000001 2023-09-23T19:11:37.709 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.709 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.709 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:37.709 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.591629+0000", 2023-09-23T19:11:37.709 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.6291e-05 2023-09-23T19:11:37.709 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.709 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.710 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:37.710 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.591655+0000", 2023-09-23T19:11:37.710 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0693000000000001e-05 2023-09-23T19:11:37.710 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.710 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.710 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:37.710 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.591666+0000", 2023-09-23T19:11:37.710 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.9637999999999997e-05 2023-09-23T19:11:37.710 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.710 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.711 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:37.711 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.591766+0000", 2023-09-23T19:11:37.711 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.0255e-05 2023-09-23T19:11:37.711 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.711 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.711 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:11:37.711 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.591806+0000", 2023-09-23T19:11:37.711 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000849232 2023-09-23T19:11:37.711 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.711 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.712 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:11:37.712 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.592655+0000", 2023-09-23T19:11:37.712 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.3325999999999996e-05 2023-09-23T19:11:37.712 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.712 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.712 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:37.712 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.592719+0000", 2023-09-23T19:11:37.712 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0012538600000000001 2023-09-23T19:11:37.713 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.713 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:37.713 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.713 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.713 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.713 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46088.0:100 2.3 e369/368 2:d04613b6:::rbd_header.b408e23a9739:head v 369'163, mlcod=369'161)", 2023-09-23T19:11:37.713 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:03.765144+0000", 2023-09-23T19:11:37.713 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 153.92286671799999, 2023-09-23T19:11:37.714 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001056698, 2023-09-23T19:11:37.714 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:37.714 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:11:37.714 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:37.714 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.714 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:37.714 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.765144+0000", 2023-09-23T19:11:37.714 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:37.715 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.715 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.715 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:37.715 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.765144+0000", 2023-09-23T19:11:37.715 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-09-23T19:11:37.715 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.715 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.715 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:37.715 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.765142+0000", 2023-09-23T19:11:37.716 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8406e-05 2023-09-23T19:11:37.716 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.716 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.716 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:37.716 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.765161+0000", 2023-09-23T19:11:37.716 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.3166000000000001e-05 2023-09-23T19:11:37.716 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.716 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.716 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:37.716 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.765194+0000", 2023-09-23T19:11:37.717 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3573e-05 2023-09-23T19:11:37.717 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.717 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.717 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:37.717 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.765207+0000", 2023-09-23T19:11:37.717 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.7647999999999997e-05 2023-09-23T19:11:37.717 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.717 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.717 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:37.717 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.765305+0000", 2023-09-23T19:11:37.718 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.2886000000000002e-05 2023-09-23T19:11:37.718 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.718 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.718 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:11:37.718 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.765348+0000", 2023-09-23T19:11:37.718 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00077830300000000005 2023-09-23T19:11:37.718 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.718 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.718 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:11:37.719 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.766126+0000", 2023-09-23T19:11:37.719 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.4942000000000002e-05 2023-09-23T19:11:37.719 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.719 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.719 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:37.719 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:03.766201+0000", 2023-09-23T19:11:37.719 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001056698 2023-09-23T19:11:37.719 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.719 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:37.719 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.720 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.720 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.720 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46094.0:15 2.0 2:01db4400:::rbd_data.b408e23a9739.0000000000000002:head [sparse-read 0~143360 [fadvise_nocache]] snapc 0=[] ondisk+read+known_if_redirected e369)", 2023-09-23T19:11:37.720 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:04.014147+0000", 2023-09-23T19:11:37.720 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 153.673863939, 2023-09-23T19:11:37.720 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001039379, 2023-09-23T19:11:37.720 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:37.720 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "started", 2023-09-23T19:11:37.721 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:37.721 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46094", 2023-09-23T19:11:37.721 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/1435943678", 2023-09-23T19:11:37.721 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 15 2023-09-23T19:11:37.721 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.721 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:37.721 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.721 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:37.721 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.014147+0000", 2023-09-23T19:11:37.722 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:37.722 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.722 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.722 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:37.722 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.014147+0000", 2023-09-23T19:11:37.722 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T19:11:37.722 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.722 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.722 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:37.723 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.014146+0000", 2023-09-23T19:11:37.723 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.0259999999999998e-06 2023-09-23T19:11:37.723 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.723 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.723 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:37.723 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.014151+0000", 2023-09-23T19:11:37.723 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2724999999999999e-05 2023-09-23T19:11:37.723 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.723 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.723 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:37.724 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.014164+0000", 2023-09-23T19:11:37.724 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.1140000000000006e-06 2023-09-23T19:11:37.724 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.724 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.724 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:37.724 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.014172+0000", 2023-09-23T19:11:37.724 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00010242999999999999 2023-09-23T19:11:37.724 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.724 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.725 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:37.725 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.014274+0000", 2023-09-23T19:11:37.725 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00017635900000000001 2023-09-23T19:11:37.725 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.725 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.725 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:11:37.725 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.014451+0000", 2023-09-23T19:11:37.725 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00073571400000000005 2023-09-23T19:11:37.725 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.726 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.726 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:37.726 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.015187+0000", 2023-09-23T19:11:37.726 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001039379 2023-09-23T19:11:37.726 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.726 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:37.726 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.726 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.726 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.726 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46094.0:20 2.0 2:00431823:::rbd_data.b408e23a9739.0000000000000007:head [sparse-read 0~4194304 [fadvise_nocache]] snapc 0=[] ondisk+read+known_if_redirected e369)", 2023-09-23T19:11:37.726 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:04.014307+0000", 2023-09-23T19:11:37.727 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 153.67370377099999, 2023-09-23T19:11:37.727 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0012231919999999999, 2023-09-23T19:11:37.727 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:37.727 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T19:11:37.727 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:37.727 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46094", 2023-09-23T19:11:37.727 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/1435943678", 2023-09-23T19:11:37.727 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 20 2023-09-23T19:11:37.727 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.727 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:37.728 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.728 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:37.728 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.014307+0000", 2023-09-23T19:11:37.728 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:37.729 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.729 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.729 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:37.729 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.014307+0000", 2023-09-23T19:11:37.729 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T19:11:37.729 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.729 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.729 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:37.730 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.014306+0000", 2023-09-23T19:11:37.730 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.003e-06 2023-09-23T19:11:37.730 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.730 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.730 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:37.731 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.014312+0000", 2023-09-23T19:11:37.731 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4421e-05 2023-09-23T19:11:37.731 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.731 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.731 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:37.731 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.014327+0000", 2023-09-23T19:11:37.731 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3502999999999999e-05 2023-09-23T19:11:37.731 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.731 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.731 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:37.731 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.014340+0000", 2023-09-23T19:11:37.732 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00093884100000000005 2023-09-23T19:11:37.732 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.732 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.732 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:37.732 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.015279+0000", 2023-09-23T19:11:37.732 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00025159200000000003 2023-09-23T19:11:37.732 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.733 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.733 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:37.733 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.015531+0000", 2023-09-23T19:11:37.733 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0012231919999999999 2023-09-23T19:11:37.733 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.733 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:37.733 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.733 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.733 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.734 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46094.0:31 2.0 2:02fc9cee:::rbd_data.b408e23a9739.0000000000000013:head [sparse-read 0~4194304 [fadvise_nocache]] snapc 0=[] ondisk+read+known_if_redirected e369)", 2023-09-23T19:11:37.734 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:04.034460+0000", 2023-09-23T19:11:37.734 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 153.653550905, 2023-09-23T19:11:37.734 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0012191960000000001, 2023-09-23T19:11:37.734 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:37.734 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T19:11:37.734 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:37.734 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46094", 2023-09-23T19:11:37.735 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/1435943678", 2023-09-23T19:11:37.735 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 31 2023-09-23T19:11:37.735 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.735 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:37.735 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.735 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:37.735 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.034460+0000", 2023-09-23T19:11:37.736 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:37.736 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.736 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.736 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:37.736 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.034460+0000", 2023-09-23T19:11:37.736 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T19:11:37.736 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.737 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.737 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:37.737 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.034459+0000", 2023-09-23T19:11:37.737 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7819999999999999e-06 2023-09-23T19:11:37.737 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.737 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.738 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:37.738 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.034465+0000", 2023-09-23T19:11:37.738 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2992e-05 2023-09-23T19:11:37.738 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.738 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.738 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:37.739 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.034478+0000", 2023-09-23T19:11:37.739 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.5199999999999997e-06 2023-09-23T19:11:37.739 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.739 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.739 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:37.739 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.034486+0000", 2023-09-23T19:11:37.740 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0010965770000000001 2023-09-23T19:11:37.740 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.740 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.740 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:37.740 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.035583+0000", 2023-09-23T19:11:37.740 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.6347000000000004e-05 2023-09-23T19:11:37.740 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.740 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.740 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:37.740 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.035679+0000", 2023-09-23T19:11:37.741 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0012191960000000001 2023-09-23T19:11:37.741 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.741 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:37.741 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.741 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.741 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.741 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46094.0:49 2.5 2:bfb9a74b:::rbd_data.b408e23a9739.0000000000000025:head [sparse-read 0~4194304 [fadvise_nocache]] snapc 0=[] ondisk+read+known_if_redirected e369)", 2023-09-23T19:11:37.741 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:04.057514+0000", 2023-09-23T19:11:37.742 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 153.63049703799999, 2023-09-23T19:11:37.742 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001037247, 2023-09-23T19:11:37.742 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:37.742 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T19:11:37.742 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:37.742 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46094", 2023-09-23T19:11:37.742 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/1435943678", 2023-09-23T19:11:37.742 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 49 2023-09-23T19:11:37.743 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.743 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:37.743 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.743 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:37.743 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.057514+0000", 2023-09-23T19:11:37.743 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:37.743 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.743 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.744 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:37.744 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.057514+0000", 2023-09-23T19:11:37.744 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T19:11:37.744 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.744 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.744 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:37.744 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.057513+0000", 2023-09-23T19:11:37.744 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.6660000000000003e-06 2023-09-23T19:11:37.744 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.744 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.744 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:37.745 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.057522+0000", 2023-09-23T19:11:37.745 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3703999999999999e-05 2023-09-23T19:11:37.745 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.745 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.745 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:37.745 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.057546+0000", 2023-09-23T19:11:37.745 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2337999999999999e-05 2023-09-23T19:11:37.745 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.745 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.745 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:37.746 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.057558+0000", 2023-09-23T19:11:37.746 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00077695399999999995 2023-09-23T19:11:37.746 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.746 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.746 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:37.746 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.058335+0000", 2023-09-23T19:11:37.746 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00021613200000000001 2023-09-23T19:11:37.746 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.747 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.747 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:37.747 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.058551+0000", 2023-09-23T19:11:37.747 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001037247 2023-09-23T19:11:37.747 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.747 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:37.747 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.747 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.748 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.748 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46094.0:188 2.0 2:0c17b639:::rbd_data.b408e23a9739.00000000000000b0:head [sparse-read 0~4194304 [fadvise_nocache]] snapc 0=[] ondisk+read+known_if_redirected e369)", 2023-09-23T19:11:37.748 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:04.244543+0000", 2023-09-23T19:11:37.748 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 153.44346769200001, 2023-09-23T19:11:37.748 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001071217, 2023-09-23T19:11:37.748 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:37.748 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T19:11:37.748 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:37.749 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46094", 2023-09-23T19:11:37.749 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/1435943678", 2023-09-23T19:11:37.749 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 188 2023-09-23T19:11:37.749 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.749 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:37.749 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.749 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:37.749 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.244543+0000", 2023-09-23T19:11:37.749 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:37.749 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.749 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.750 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:37.750 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.244543+0000", 2023-09-23T19:11:37.750 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T19:11:37.750 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.750 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.750 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:37.750 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.244542+0000", 2023-09-23T19:11:37.750 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.6930000000000007e-06 2023-09-23T19:11:37.750 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.750 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.751 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:37.751 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.244550+0000", 2023-09-23T19:11:37.751 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2629999999999998e-05 2023-09-23T19:11:37.751 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.751 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.751 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:37.751 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.244572+0000", 2023-09-23T19:11:37.751 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1843e-05 2023-09-23T19:11:37.752 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.752 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.752 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:37.752 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.244584+0000", 2023-09-23T19:11:37.752 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00088064399999999998 2023-09-23T19:11:37.752 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.752 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.752 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:37.752 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.245465+0000", 2023-09-23T19:11:37.752 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014977400000000001 2023-09-23T19:11:37.752 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.753 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.753 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:37.753 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.245615+0000", 2023-09-23T19:11:37.753 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001071217 2023-09-23T19:11:37.753 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.753 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:37.753 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.753 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.753 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.753 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46100.0:18 2.5 2:b79ecccc:::rbd_data.b408e23a9739.0000000000000006:head [sparse-read 0~4194304 [fadvise_nocache]] snapc 0=[] ondisk+read+known_if_redirected e370)", 2023-09-23T19:11:37.753 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:04.357381+0000", 2023-09-23T19:11:37.753 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 153.33062991700001, 2023-09-23T19:11:37.754 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0010365369999999999, 2023-09-23T19:11:37.754 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:37.754 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T19:11:37.754 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:37.754 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46100", 2023-09-23T19:11:37.754 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2818002861", 2023-09-23T19:11:37.754 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 18 2023-09-23T19:11:37.754 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.754 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:37.755 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.755 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:37.755 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.357381+0000", 2023-09-23T19:11:37.755 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:37.755 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.755 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.755 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:37.755 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.357381+0000", 2023-09-23T19:11:37.755 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-09-23T19:11:37.755 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.755 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.758 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:37.758 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.357381+0000", 2023-09-23T19:11:37.758 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.8739999999999997e-06 2023-09-23T19:11:37.758 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.758 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.758 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:37.758 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.357384+0000", 2023-09-23T19:11:37.758 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0900999999999999e-05 2023-09-23T19:11:37.759 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.759 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.759 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:37.759 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.357395+0000", 2023-09-23T19:11:37.759 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.1539999999999996e-06 2023-09-23T19:11:37.759 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.759 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.759 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:37.759 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.357403+0000", 2023-09-23T19:11:37.759 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00088733200000000001 2023-09-23T19:11:37.759 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.759 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.760 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:37.760 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.358290+0000", 2023-09-23T19:11:37.760 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000127892 2023-09-23T19:11:37.760 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.760 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.760 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:37.760 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.358418+0000", 2023-09-23T19:11:37.760 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0010365369999999999 2023-09-23T19:11:37.760 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.760 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:37.760 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.761 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.761 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.761 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46100.0:21 2.7 2:fec40bef:::rbd_data.b408e23a9739.0000000000000009:head [sparse-read 0~4194304 [fadvise_nocache]] snapc 0=[] ondisk+read+known_if_redirected e370)", 2023-09-23T19:11:37.761 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:04.357506+0000", 2023-09-23T19:11:37.761 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 153.330504752, 2023-09-23T19:11:37.761 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.96461242899999999, 2023-09-23T19:11:37.761 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:37.761 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T19:11:37.761 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:37.761 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46100", 2023-09-23T19:11:37.761 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2818002861", 2023-09-23T19:11:37.762 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 21 2023-09-23T19:11:37.762 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.762 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:37.762 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.762 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:37.762 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.357506+0000", 2023-09-23T19:11:37.762 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:37.762 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.762 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.762 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:37.763 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.357506+0000", 2023-09-23T19:11:37.763 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T19:11:37.763 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.763 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.763 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:37.763 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.357505+0000", 2023-09-23T19:11:37.763 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.8680000000000001e-06 2023-09-23T19:11:37.764 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.764 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.764 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:37.764 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.357512+0000", 2023-09-23T19:11:37.764 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.5040000000000003e-05 2023-09-23T19:11:37.764 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.764 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.764 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:37.764 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.357547+0000", 2023-09-23T19:11:37.765 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.3080000000000004e-06 2023-09-23T19:11:37.765 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.765 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.765 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:37.765 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.357553+0000", 2023-09-23T19:11:37.765 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.3411999999999994e-05 2023-09-23T19:11:37.766 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.766 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.766 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:37.766 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.357637+0000", 2023-09-23T19:11:37.766 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.1821999999999999e-05 2023-09-23T19:11:37.766 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.766 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.766 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-09-23T19:11:37.766 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:04.357659+0000", 2023-09-23T19:11:37.767 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.96436264999999999 2023-09-23T19:11:37.767 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.767 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.767 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:37.767 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.322021+0000", 2023-09-23T19:11:37.767 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.7607000000000002e-05 2023-09-23T19:11:37.767 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.768 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.768 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:37.768 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.322119+0000", 2023-09-23T19:11:37.768 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.96461242899999999 2023-09-23T19:11:37.768 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.768 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:37.768 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.768 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.768 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.769 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46100.0:34 2.7 2:efdf2fa9:::rbd_data.b408e23a9739.0000000000000016:head [sparse-read 0~4194304 [fadvise_nocache]] snapc 0=[] ondisk+read+known_if_redirected e371)", 2023-09-23T19:11:37.769 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:05.360322+0000", 2023-09-23T19:11:37.769 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 152.32768908599999, 2023-09-23T19:11:37.769 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0011777070000000001, 2023-09-23T19:11:37.769 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:37.769 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T19:11:37.769 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:37.770 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46100", 2023-09-23T19:11:37.770 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2818002861", 2023-09-23T19:11:37.770 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 34 2023-09-23T19:11:37.770 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.770 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:37.770 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.770 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:37.770 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360322+0000", 2023-09-23T19:11:37.771 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:37.771 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.771 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.771 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:37.771 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360322+0000", 2023-09-23T19:11:37.771 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T19:11:37.772 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.772 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.772 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:37.772 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360321+0000", 2023-09-23T19:11:37.772 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.3300000000000001e-06 2023-09-23T19:11:37.772 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.773 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.773 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:37.773 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360328+0000", 2023-09-23T19:11:37.773 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.0815999999999999e-05 2023-09-23T19:11:37.773 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.773 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.773 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:37.773 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360349+0000", 2023-09-23T19:11:37.774 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2743e-05 2023-09-23T19:11:37.774 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.774 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.774 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:37.774 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360362+0000", 2023-09-23T19:11:37.774 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00096706400000000001 2023-09-23T19:11:37.775 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.775 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.775 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:37.775 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.361329+0000", 2023-09-23T19:11:37.775 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000171094 2023-09-23T19:11:37.775 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.776 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.776 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:37.776 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.361500+0000", 2023-09-23T19:11:37.776 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0011777070000000001 2023-09-23T19:11:37.776 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.776 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:37.776 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.776 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.776 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.776 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46100.0:35 2.7 2:e1b06f48:::rbd_data.b408e23a9739.0000000000000018:head [sparse-read 0~77824 [fadvise_nocache]] snapc 0=[] ondisk+read+known_if_redirected e371)", 2023-09-23T19:11:37.777 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:05.360424+0000", 2023-09-23T19:11:37.777 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 152.32758725799999, 2023-09-23T19:11:37.777 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0019015550000000001, 2023-09-23T19:11:37.777 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:37.777 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "started", 2023-09-23T19:11:37.777 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:37.777 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46100", 2023-09-23T19:11:37.777 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2818002861", 2023-09-23T19:11:37.777 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 35 2023-09-23T19:11:37.778 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.778 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:37.778 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.778 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:37.778 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360424+0000", 2023-09-23T19:11:37.778 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:37.778 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.778 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.779 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:37.779 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360424+0000", 2023-09-23T19:11:37.779 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T19:11:37.779 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.779 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.779 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:37.779 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360422+0000", 2023-09-23T19:11:37.779 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.0960000000000001e-06 2023-09-23T19:11:37.780 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.780 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.780 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:37.780 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360430+0000", 2023-09-23T19:11:37.780 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.7675000000000001e-05 2023-09-23T19:11:37.780 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.780 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.780 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:37.781 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360468+0000", 2023-09-23T19:11:37.781 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2379e-05 2023-09-23T19:11:37.781 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.781 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.781 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:37.781 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.360480+0000", 2023-09-23T19:11:37.781 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0010704499999999999 2023-09-23T19:11:37.781 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.781 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.781 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:37.782 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.361551+0000", 2023-09-23T19:11:37.782 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016117099999999999 2023-09-23T19:11:37.782 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.782 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.782 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:11:37.782 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.361712+0000", 2023-09-23T19:11:37.782 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00061342000000000005 2023-09-23T19:11:37.782 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.782 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.782 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:37.782 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.362325+0000", 2023-09-23T19:11:37.783 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0019015550000000001 2023-09-23T19:11:37.783 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.783 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:37.783 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.783 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.783 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.783 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46100.0:49 2.5 2:bfb9a74b:::rbd_data.b408e23a9739.0000000000000025:head [sparse-read 0~4194304 [fadvise_nocache]] snapc 0=[] ondisk+read+known_if_redirected e371)", 2023-09-23T19:11:37.783 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:05.393896+0000", 2023-09-23T19:11:37.783 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 152.294114863, 2023-09-23T19:11:37.783 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0010375339999999999, 2023-09-23T19:11:37.784 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:37.784 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T19:11:37.784 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:37.784 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46100", 2023-09-23T19:11:37.784 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2818002861", 2023-09-23T19:11:37.784 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 49 2023-09-23T19:11:37.784 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.784 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:37.784 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.784 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:37.785 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.393896+0000", 2023-09-23T19:11:37.785 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:37.785 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.785 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.785 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:37.785 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.393896+0000", 2023-09-23T19:11:37.785 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T19:11:37.785 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.786 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.786 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:37.786 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.393895+0000", 2023-09-23T19:11:37.786 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.9569999999999999e-06 2023-09-23T19:11:37.786 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.786 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.786 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:37.787 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.393901+0000", 2023-09-23T19:11:37.787 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3967e-05 2023-09-23T19:11:37.787 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.788 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.788 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:37.788 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.393915+0000", 2023-09-23T19:11:37.788 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2628e-05 2023-09-23T19:11:37.788 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.788 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.788 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:37.788 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.393928+0000", 2023-09-23T19:11:37.789 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00084529600000000005 2023-09-23T19:11:37.789 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.789 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.789 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:37.789 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.394773+0000", 2023-09-23T19:11:37.789 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016064399999999999 2023-09-23T19:11:37.789 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.789 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.789 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:37.790 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.394934+0000", 2023-09-23T19:11:37.790 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0010375339999999999 2023-09-23T19:11:37.790 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.790 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:37.790 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.790 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.790 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.790 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46100.0:55 2.5 2:b2c2c343:::rbd_data.b408e23a9739.000000000000002b:head [sparse-read 0~4194304 [fadvise_nocache]] snapc 0=[] ondisk+read+known_if_redirected e371)", 2023-09-23T19:11:37.790 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:05.429961+0000", 2023-09-23T19:11:37.791 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 152.25805031799999, 2023-09-23T19:11:37.791 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0011576760000000001, 2023-09-23T19:11:37.791 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:37.791 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T19:11:37.791 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:37.791 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46100", 2023-09-23T19:11:37.791 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2818002861", 2023-09-23T19:11:37.791 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 55 2023-09-23T19:11:37.791 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.792 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:37.792 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.792 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:37.792 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.429961+0000", 2023-09-23T19:11:37.792 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:37.792 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.792 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.792 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:37.792 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.429961+0000", 2023-09-23T19:11:37.793 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T19:11:37.793 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.793 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.793 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:37.793 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.429959+0000", 2023-09-23T19:11:37.793 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.01e-05 2023-09-23T19:11:37.793 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.793 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.793 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:37.793 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.429969+0000", 2023-09-23T19:11:37.793 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.0143e-05 2023-09-23T19:11:37.794 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.794 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.794 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:37.794 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.429989+0000", 2023-09-23T19:11:37.794 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3027000000000001e-05 2023-09-23T19:11:37.794 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.794 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.794 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:37.794 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.430002+0000", 2023-09-23T19:11:37.795 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000876934 2023-09-23T19:11:37.795 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.795 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.795 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:37.795 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.430879+0000", 2023-09-23T19:11:37.795 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00023927900000000001 2023-09-23T19:11:37.795 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.795 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.796 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:37.796 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.431118+0000", 2023-09-23T19:11:37.796 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0011576760000000001 2023-09-23T19:11:37.796 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.796 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:37.796 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.796 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.796 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.796 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46100.0:60 2.5 2:a848dfaf:::rbd_data.b408e23a9739.0000000000000030:head [sparse-read 0~4194304 [fadvise_nocache]] snapc 0=[] ondisk+read+known_if_redirected e371)", 2023-09-23T19:11:37.797 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:05.430110+0000", 2023-09-23T19:11:37.797 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 152.257901541, 2023-09-23T19:11:37.797 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00130327, 2023-09-23T19:11:37.797 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:37.797 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T19:11:37.797 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:37.797 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46100", 2023-09-23T19:11:37.797 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2818002861", 2023-09-23T19:11:37.797 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 60 2023-09-23T19:11:37.798 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.798 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:37.798 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.798 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:37.798 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.430110+0000", 2023-09-23T19:11:37.798 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:37.798 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.798 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.798 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:37.799 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.430110+0000", 2023-09-23T19:11:37.799 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T19:11:37.799 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.799 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.799 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:37.799 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.430108+0000", 2023-09-23T19:11:37.799 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.5099999999999998e-06 2023-09-23T19:11:37.799 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.800 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.800 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:37.800 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.430117+0000", 2023-09-23T19:11:37.800 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.1985999999999999e-05 2023-09-23T19:11:37.800 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.801 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.801 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:37.801 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.430139+0000", 2023-09-23T19:11:37.801 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3404000000000001e-05 2023-09-23T19:11:37.801 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.801 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.801 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:37.801 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.430152+0000", 2023-09-23T19:11:37.801 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001048771 2023-09-23T19:11:37.802 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.802 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.802 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:37.802 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.431201+0000", 2023-09-23T19:11:37.802 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00021189900000000001 2023-09-23T19:11:37.802 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.803 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.803 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:37.803 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:05.431413+0000", 2023-09-23T19:11:37.803 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00130327 2023-09-23T19:11:37.803 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.803 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:37.803 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.803 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.804 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.804 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.36726.0:1 2.5 2:bcfccb67:::rbd_data.b6661fc69d7d.000000000000:head [writefull 0~4194304] snapc 0=[] ondisk+write+known_if_redirected e383)", 2023-09-23T19:11:37.804 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:27.868302+0000", 2023-09-23T19:11:37.804 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 129.81970959700001, 2023-09-23T19:11:37.804 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.027313181999999998, 2023-09-23T19:11:37.804 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:37.804 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:11:37.805 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-09-23T19:11:37.805 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.36726", 2023-09-23T19:11:37.805 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.98:0/2108052158", 2023-09-23T19:11:37.805 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 1 2023-09-23T19:11:37.805 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.805 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:37.805 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.805 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:37.805 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:27.868302+0000", 2023-09-23T19:11:37.805 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:37.806 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.806 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.806 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:37.806 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:27.868302+0000", 2023-09-23T19:11:37.806 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999989 2023-09-23T19:11:37.807 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.807 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.807 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:37.807 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:27.868291+0000", 2023-09-23T19:11:37.807 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0082981970000000002 2023-09-23T19:11:37.807 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.807 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.807 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:37.808 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:27.876589+0000", 2023-09-23T19:11:37.808 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.5952999999999999e-05 2023-09-23T19:11:37.808 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.808 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.808 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:37.808 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:27.876635+0000", 2023-09-23T19:11:37.809 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9771000000000001e-05 2023-09-23T19:11:37.809 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.809 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.809 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:37.809 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:27.876655+0000", 2023-09-23T19:11:37.809 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000165441 2023-09-23T19:11:37.809 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.810 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.810 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:37.810 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:27.876820+0000", 2023-09-23T19:11:37.810 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00040618300000000001 2023-09-23T19:11:37.810 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.810 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.810 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:11:37.810 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:27.877226+0000", 2023-09-23T19:11:37.811 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00050782500000000005 2023-09-23T19:11:37.811 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.811 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.811 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,1", 2023-09-23T19:11:37.811 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:27.877734+0000", 2023-09-23T19:11:37.811 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0090798560000000007 2023-09-23T19:11:37.811 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.811 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.811 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-09-23T19:11:37.812 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:27.886814+0000", 2023-09-23T19:11:37.812 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0052678250000000003 2023-09-23T19:11:37.812 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.812 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.812 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:37.812 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:27.892082+0000", 2023-09-23T19:11:37.812 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0034149990000000002 2023-09-23T19:11:37.812 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.812 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.813 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-09-23T19:11:37.813 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:27.895497+0000", 2023-09-23T19:11:37.813 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.5347999999999998e-05 2023-09-23T19:11:37.813 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.813 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.813 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:11:37.813 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:27.895562+0000", 2023-09-23T19:11:37.813 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.2561e-05 2023-09-23T19:11:37.813 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.813 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.814 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:37.814 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:27.895615+0000", 2023-09-23T19:11:37.814 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.027313181999999998 2023-09-23T19:11:37.814 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.814 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:37.814 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.814 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.814 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.814 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.36759.0:16 2.3 e383/368 2:d97abd30:::rbd_object_map.b6661fc69d7d:head v 383'307, mlcod=383'305)", 2023-09-23T19:11:37.815 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:09:28.119109+0000", 2023-09-23T19:11:37.815 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 129.56890182500001, 2023-09-23T19:11:37.815 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0011041569999999999, 2023-09-23T19:11:37.815 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:37.815 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T19:11:37.815 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:37.815 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.815 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:37.815 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.119109+0000", 2023-09-23T19:11:37.815 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:37.816 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.816 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.816 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:37.816 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.119109+0000", 2023-09-23T19:11:37.816 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-09-23T19:11:37.816 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.816 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.816 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:37.816 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.119107+0000", 2023-09-23T19:11:37.816 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.1100000000000001e-05 2023-09-23T19:11:37.817 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.817 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.817 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:37.817 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.119128+0000", 2023-09-23T19:11:37.817 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.4413999999999999e-05 2023-09-23T19:11:37.817 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.817 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.817 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:37.818 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.119163+0000", 2023-09-23T19:11:37.818 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6436e-05 2023-09-23T19:11:37.818 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.818 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.818 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:37.818 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.119179+0000", 2023-09-23T19:11:37.818 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012832800000000001 2023-09-23T19:11:37.819 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.819 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.819 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:37.819 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.119307+0000", 2023-09-23T19:11:37.819 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4815999999999998e-05 2023-09-23T19:11:37.819 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.819 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.819 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-09-23T19:11:37.819 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.119362+0000", 2023-09-23T19:11:37.820 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00079210399999999996 2023-09-23T19:11:37.820 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.820 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.820 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-09-23T19:11:37.820 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.120154+0000", 2023-09-23T19:11:37.820 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.9082000000000001e-05 2023-09-23T19:11:37.820 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.820 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.820 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:37.820 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:09:28.120213+0000", 2023-09-23T19:11:37.821 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0011041569999999999 2023-09-23T19:11:37.821 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.821 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:37.821 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.821 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.821 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.821 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "MOSDPGPushReply(2.5 399/398 [PushReplyOp(2:bcfccb67:::rbd_data.b6661fc69d7d.000000000000:head)])", 2023-09-23T19:11:37.821 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:11:19.584627+0000", 2023-09-23T19:11:37.821 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 18.103383848, 2023-09-23T19:11:37.821 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0027635910000000001, 2023-09-23T19:11:37.822 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:37.822 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T19:11:37.822 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:37.822 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.822 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:37.822 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:11:19.584627+0000", 2023-09-23T19:11:37.822 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:37.822 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.822 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.822 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:37.822 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:11:19.584627+0000", 2023-09-23T19:11:37.823 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T19:11:37.823 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.823 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.823 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:37.823 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:11:19.584626+0000", 2023-09-23T19:11:37.823 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.4800000000000001e-06 2023-09-23T19:11:37.823 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.823 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.823 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:37.823 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:11:19.584635+0000", 2023-09-23T19:11:37.824 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2811e-05 2023-09-23T19:11:37.824 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.824 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.824 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:37.824 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:11:19.584648+0000", 2023-09-23T19:11:37.824 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6010000000000002e-06 2023-09-23T19:11:37.824 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.824 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.824 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:37.824 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:11:19.584653+0000", 2023-09-23T19:11:37.824 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.3940999999999998e-05 2023-09-23T19:11:37.824 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.825 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.825 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:37.825 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:11:19.584697+0000", 2023-09-23T19:11:37.825 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0026936490000000002 2023-09-23T19:11:37.825 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.825 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.825 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:37.825 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:11:19.587391+0000", 2023-09-23T19:11:37.825 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0027635910000000001 2023-09-23T19:11:37.826 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.826 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:37.826 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.826 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.826 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.826 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "MOSDPGPushReply(2.5 399/398 [PushReplyOp(2:bcfccb67:::rbd_data.b6661fc69d7d.000000000000:head)])", 2023-09-23T19:11:37.826 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:11:19.589770+0000", 2023-09-23T19:11:37.826 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 18.098241415, 2023-09-23T19:11:37.826 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002923101, 2023-09-23T19:11:37.826 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:37.826 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T19:11:37.827 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:37.827 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.827 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:37.827 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:11:19.589770+0000", 2023-09-23T19:11:37.827 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:37.827 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.827 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.827 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:37.827 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:11:19.589770+0000", 2023-09-23T19:11:37.828 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-09-23T19:11:37.828 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.828 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.828 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:37.828 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:11:19.589769+0000", 2023-09-23T19:11:37.828 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.9739999999999999e-06 2023-09-23T19:11:37.828 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.828 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.828 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:37.829 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:11:19.589775+0000", 2023-09-23T19:11:37.829 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1267000000000001e-05 2023-09-23T19:11:37.829 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.829 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.829 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:37.829 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:11:19.589786+0000", 2023-09-23T19:11:37.829 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.3260000000000003e-06 2023-09-23T19:11:37.829 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.829 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.829 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:37.829 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:11:19.589791+0000", 2023-09-23T19:11:37.830 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.1951000000000001e-05 2023-09-23T19:11:37.830 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.830 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.830 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:37.830 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:11:19.589883+0000", 2023-09-23T19:11:37.830 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0028095849999999999 2023-09-23T19:11:37.830 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.831 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.831 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:37.831 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:11:19.592693+0000", 2023-09-23T19:11:37.831 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002923101 2023-09-23T19:11:37.831 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.831 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:37.831 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.831 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.831 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.832 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "MOSDPGPushReply(2.5 399/398 [PushReplyOp(2:bcfccb67:::rbd_data.b6661fc69d7d.000000000000:head)])", 2023-09-23T19:11:37.832 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-09-23T19:11:19.595143+0000", 2023-09-23T19:11:37.832 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 18.092868519, 2023-09-23T19:11:37.832 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0026250219999999999, 2023-09-23T19:11:37.832 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-09-23T19:11:37.832 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-09-23T19:11:37.832 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-09-23T19:11:37.832 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.832 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-09-23T19:11:37.833 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:11:19.595143+0000", 2023-09-23T19:11:37.833 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-09-23T19:11:37.833 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.833 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.833 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-09-23T19:11:37.834 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:11:19.595143+0000", 2023-09-23T19:11:37.834 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-09-23T19:11:37.834 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.834 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.834 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-09-23T19:11:37.834 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:11:19.595141+0000", 2023-09-23T19:11:37.834 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2758999999999999e-05 2023-09-23T19:11:37.834 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.834 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.835 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-09-23T19:11:37.835 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:11:19.595154+0000", 2023-09-23T19:11:37.835 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.7090999999999999e-05 2023-09-23T19:11:37.835 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.835 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.835 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-09-23T19:11:37.835 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:11:19.595181+0000", 2023-09-23T19:11:37.836 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.256e-05 2023-09-23T19:11:37.836 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.836 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.836 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-09-23T19:11:37.836 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:11:19.595193+0000", 2023-09-23T19:11:37.836 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000109849 2023-09-23T19:11:37.837 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.837 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.837 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-09-23T19:11:37.837 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:11:19.595303+0000", 2023-09-23T19:11:37.837 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0024643709999999999 2023-09-23T19:11:37.837 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-09-23T19:11:37.837 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-09-23T19:11:37.837 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-09-23T19:11:37.837 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-09-23T19:11:19.597768+0000", 2023-09-23T19:11:37.837 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0026250219999999999 2023-09-23T19:11:37.838 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.838 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:37.838 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.838 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-09-23T19:11:37.838 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-09-23T19:11:37.838 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-09-23T19:11:38.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:37 smithi173 bash[59422]: cluster 2023-09-23T19:11:36.733309+0000 mgr.y (mgr.26054) 1623 : cluster 0 pgmap v1818: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:38.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:37 smithi098 bash[79516]: cluster 2023-09-23T19:11:36.733309+0000 mgr.y (mgr.26054) 1623 : cluster 0 pgmap v1818: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:38.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:37 smithi098 bash[76817]: cluster 2023-09-23T19:11:36.733309+0000 mgr.y (mgr.26054) 1623 : cluster 0 pgmap v1818: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:40.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:39 smithi098 bash[79516]: cluster 2023-09-23T19:11:38.734343+0000 mgr.y (mgr.26054) 1624 : cluster 0 pgmap v1819: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:40.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:39 smithi098 bash[79516]: audit 2023-09-23T19:11:39.654494+0000 mon.a (mon.0) 2212 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:11:40.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:39 smithi098 bash[76817]: cluster 2023-09-23T19:11:38.734343+0000 mgr.y (mgr.26054) 1624 : cluster 0 pgmap v1819: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:40.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:39 smithi098 bash[76817]: audit 2023-09-23T19:11:39.654494+0000 mon.a (mon.0) 2212 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:11:40.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:39 smithi173 bash[59422]: cluster 2023-09-23T19:11:38.734343+0000 mgr.y (mgr.26054) 1624 : cluster 0 pgmap v1819: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:40.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:39 smithi173 bash[59422]: audit 2023-09-23T19:11:39.654494+0000 mon.a (mon.0) 2212 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:11:40.694 INFO:teuthology.orchestra.run.smithi098.stdout:mon.a: {} 2023-09-23T19:11:40.714 INFO:teuthology.orchestra.run.smithi098.stdout:mon.b: {} 2023-09-23T19:11:40.732 INFO:teuthology.orchestra.run.smithi098.stdout:mon.c: {} 2023-09-23T19:11:40.734 INFO:teuthology.orchestra.run.smithi098.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T19:11:40.734 INFO:teuthology.orchestra.run.smithi098.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T19:11:40.734 INFO:teuthology.orchestra.run.smithi098.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T19:11:41.669 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T19:11:41.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:41 smithi098 bash[76817]: audit 2023-09-23T19:11:40.670199+0000 mon.a (mon.0) 2213 : audit 0 from='client.? 172.21.15.98:0/4050027631' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T19:11:41.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:41 smithi098 bash[76817]: cluster 2023-09-23T19:11:40.735047+0000 mgr.y (mgr.26054) 1625 : cluster 0 pgmap v1820: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:41.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:41 smithi098 bash[79516]: audit 2023-09-23T19:11:40.670199+0000 mon.a (mon.0) 2213 : audit 0 from='client.? 172.21.15.98:0/4050027631' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T19:11:41.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:41 smithi098 bash[79516]: cluster 2023-09-23T19:11:40.735047+0000 mgr.y (mgr.26054) 1625 : cluster 0 pgmap v1820: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:42.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:41 smithi173 bash[59422]: audit 2023-09-23T19:11:40.670199+0000 mon.a (mon.0) 2213 : audit 0 from='client.? 172.21.15.98:0/4050027631' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T19:11:42.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:41 smithi173 bash[59422]: cluster 2023-09-23T19:11:40.735047+0000 mgr.y (mgr.26054) 1625 : cluster 0 pgmap v1820: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:44.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:43 smithi098 bash[76817]: cluster 2023-09-23T19:11:42.735886+0000 mgr.y (mgr.26054) 1626 : cluster 0 pgmap v1821: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:44.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:43 smithi098 bash[79516]: cluster 2023-09-23T19:11:42.735886+0000 mgr.y (mgr.26054) 1626 : cluster 0 pgmap v1821: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:44.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:43 smithi173 bash[59422]: cluster 2023-09-23T19:11:42.735886+0000 mgr.y (mgr.26054) 1626 : cluster 0 pgmap v1821: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:46.088 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:45 smithi098 bash[76817]: cluster 2023-09-23T19:11:44.736758+0000 mgr.y (mgr.26054) 1627 : cluster 0 pgmap v1822: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:46.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:45 smithi098 bash[79516]: cluster 2023-09-23T19:11:44.736758+0000 mgr.y (mgr.26054) 1627 : cluster 0 pgmap v1822: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:46.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:45 smithi173 bash[59422]: cluster 2023-09-23T19:11:44.736758+0000 mgr.y (mgr.26054) 1627 : cluster 0 pgmap v1822: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:46.441 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:11:46.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:11:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:11:46] "GET /metrics HTTP/1.1" 200 34166 "" "Prometheus/2.33.4" 2023-09-23T19:11:47.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:47 smithi098 bash[79516]: cluster 2023-09-23T19:11:46.737608+0000 mgr.y (mgr.26054) 1628 : cluster 0 pgmap v1823: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:47.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:47 smithi098 bash[76817]: cluster 2023-09-23T19:11:46.737608+0000 mgr.y (mgr.26054) 1628 : cluster 0 pgmap v1823: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:48.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:47 smithi173 bash[59422]: cluster 2023-09-23T19:11:46.737608+0000 mgr.y (mgr.26054) 1628 : cluster 0 pgmap v1823: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:49.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:49 smithi098 bash[76817]: cluster 2023-09-23T19:11:48.738592+0000 mgr.y (mgr.26054) 1629 : cluster 0 pgmap v1824: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:49.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:49 smithi098 bash[79516]: cluster 2023-09-23T19:11:48.738592+0000 mgr.y (mgr.26054) 1629 : cluster 0 pgmap v1824: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:50.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:49 smithi173 bash[59422]: cluster 2023-09-23T19:11:48.738592+0000 mgr.y (mgr.26054) 1629 : cluster 0 pgmap v1824: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:50.836 INFO:teuthology.orchestra.run.smithi098.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T19:11:51.143 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:50 smithi098 bash[76817]: audit 2023-09-23T19:11:50.093347+0000 mon.a (mon.0) 2214 : audit 1 from='client.? 172.21.15.98:0/2394015011' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:11:51.143 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:50 smithi098 bash[79516]: audit 2023-09-23T19:11:50.093347+0000 mon.a (mon.0) 2214 : audit 1 from='client.? 172.21.15.98:0/2394015011' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:11:51.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:50 smithi173 bash[59422]: audit 2023-09-23T19:11:50.093347+0000 mon.a (mon.0) 2214 : audit 1 from='client.? 172.21.15.98:0/2394015011' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:11:51.834 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-09-23T19:11:52.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:51 smithi098 bash[76817]: cluster 2023-09-23T19:11:50.739388+0000 mgr.y (mgr.26054) 1630 : cluster 0 pgmap v1825: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:52.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:51 smithi098 bash[76817]: audit 2023-09-23T19:11:50.824610+0000 mon.a (mon.0) 2215 : audit 1 from='client.? 172.21.15.98:0/2394015011' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:11:52.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:51 smithi098 bash[76817]: cluster 2023-09-23T19:11:50.824712+0000 mon.a (mon.0) 2216 : cluster 0 osdmap e402: 8 total, 8 up, 8 in 2023-09-23T19:11:52.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:51 smithi098 bash[76817]: cluster 2023-09-23T19:11:50.828967+0000 osd.4 (osd.4) 134 : cluster 3 failed to encode map e402 with expected crc 2023-09-23T19:11:52.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:51 smithi098 bash[79516]: cluster 2023-09-23T19:11:50.739388+0000 mgr.y (mgr.26054) 1630 : cluster 0 pgmap v1825: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:52.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:51 smithi098 bash[79516]: audit 2023-09-23T19:11:50.824610+0000 mon.a (mon.0) 2215 : audit 1 from='client.? 172.21.15.98:0/2394015011' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:11:52.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:51 smithi098 bash[79516]: cluster 2023-09-23T19:11:50.824712+0000 mon.a (mon.0) 2216 : cluster 0 osdmap e402: 8 total, 8 up, 8 in 2023-09-23T19:11:52.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:51 smithi098 bash[79516]: cluster 2023-09-23T19:11:50.828967+0000 osd.4 (osd.4) 134 : cluster 3 failed to encode map e402 with expected crc 2023-09-23T19:11:52.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:51 smithi173 bash[59422]: cluster 2023-09-23T19:11:50.739388+0000 mgr.y (mgr.26054) 1630 : cluster 0 pgmap v1825: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:52.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:51 smithi173 bash[59422]: audit 2023-09-23T19:11:50.824610+0000 mon.a (mon.0) 2215 : audit 1 from='client.? 172.21.15.98:0/2394015011' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:11:52.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:51 smithi173 bash[59422]: cluster 2023-09-23T19:11:50.824712+0000 mon.a (mon.0) 2216 : cluster 0 osdmap e402: 8 total, 8 up, 8 in 2023-09-23T19:11:52.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:51 smithi173 bash[59422]: cluster 2023-09-23T19:11:50.828967+0000 osd.4 (osd.4) 134 : cluster 3 failed to encode map e402 with expected crc 2023-09-23T19:11:54.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:53 smithi098 bash[76817]: cluster 2023-09-23T19:11:52.740264+0000 mgr.y (mgr.26054) 1631 : cluster 0 pgmap v1827: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:54.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:53 smithi098 bash[79516]: cluster 2023-09-23T19:11:52.740264+0000 mgr.y (mgr.26054) 1631 : cluster 0 pgmap v1827: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:54.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:53 smithi173 bash[59422]: cluster 2023-09-23T19:11:52.740264+0000 mgr.y (mgr.26054) 1631 : cluster 0 pgmap v1827: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:55.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:54 smithi098 bash[76817]: audit 2023-09-23T19:11:54.655961+0000 mon.a (mon.0) 2217 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:11:55.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:54 smithi098 bash[79516]: audit 2023-09-23T19:11:54.655961+0000 mon.a (mon.0) 2217 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:11:55.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:54 smithi173 bash[59422]: audit 2023-09-23T19:11:54.655961+0000 mon.a (mon.0) 2217 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:11:56.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:55 smithi098 bash[76817]: cluster 2023-09-23T19:11:54.741096+0000 mgr.y (mgr.26054) 1632 : cluster 0 pgmap v1828: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:56.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:55 smithi098 bash[79516]: cluster 2023-09-23T19:11:54.741096+0000 mgr.y (mgr.26054) 1632 : cluster 0 pgmap v1828: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:56.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:11:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:11:56] "GET /metrics HTTP/1.1" 200 34166 "" "Prometheus/2.33.4" 2023-09-23T19:11:56.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:55 smithi173 bash[59422]: cluster 2023-09-23T19:11:54.741096+0000 mgr.y (mgr.26054) 1632 : cluster 0 pgmap v1828: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:56.612 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:11:58.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:57 smithi098 bash[76817]: cluster 2023-09-23T19:11:56.741958+0000 mgr.y (mgr.26054) 1633 : cluster 0 pgmap v1829: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:58.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:57 smithi098 bash[79516]: cluster 2023-09-23T19:11:56.741958+0000 mgr.y (mgr.26054) 1633 : cluster 0 pgmap v1829: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:11:58.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:57 smithi173 bash[59422]: cluster 2023-09-23T19:11:56.741958+0000 mgr.y (mgr.26054) 1633 : cluster 0 pgmap v1829: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:00.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:11:59 smithi098 bash[76817]: cluster 2023-09-23T19:11:58.742660+0000 mgr.y (mgr.26054) 1634 : cluster 0 pgmap v1830: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:00.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:11:59 smithi098 bash[79516]: cluster 2023-09-23T19:11:58.742660+0000 mgr.y (mgr.26054) 1634 : cluster 0 pgmap v1830: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:00.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:11:59 smithi173 bash[59422]: cluster 2023-09-23T19:11:58.742660+0000 mgr.y (mgr.26054) 1634 : cluster 0 pgmap v1830: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:00.875 INFO:teuthology.orchestra.run.smithi098.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-09-23T19:12:01.157 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:00 smithi098 bash[79516]: audit 2023-09-23T19:12:00.208110+0000 mon.a (mon.0) 2218 : audit 1 from='client.? 172.21.15.98:0/3976685182' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:12:01.157 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:00 smithi098 bash[76817]: audit 2023-09-23T19:12:00.208110+0000 mon.a (mon.0) 2218 : audit 1 from='client.? 172.21.15.98:0/3976685182' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:12:01.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:00 smithi173 bash[59422]: audit 2023-09-23T19:12:00.208110+0000 mon.a (mon.0) 2218 : audit 1 from='client.? 172.21.15.98:0/3976685182' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T19:12:01.837 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T19:12:02.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:01 smithi098 bash[79516]: cluster 2023-09-23T19:12:00.743524+0000 mgr.y (mgr.26054) 1635 : cluster 0 pgmap v1831: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:02.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:01 smithi098 bash[79516]: audit 2023-09-23T19:12:00.865477+0000 mon.a (mon.0) 2219 : audit 1 from='client.? 172.21.15.98:0/3976685182' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:12:02.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:01 smithi098 bash[79516]: cluster 2023-09-23T19:12:00.865565+0000 mon.a (mon.0) 2220 : cluster 0 osdmap e403: 8 total, 8 up, 8 in 2023-09-23T19:12:02.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:01 smithi098 bash[79516]: cluster 2023-09-23T19:12:00.867593+0000 osd.7 (osd.7) 56 : cluster 3 failed to encode map e403 with expected crc 2023-09-23T19:12:02.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:01 smithi098 bash[79516]: cluster 2023-09-23T19:12:00.871296+0000 osd.4 (osd.4) 135 : cluster 3 failed to encode map e403 with expected crc 2023-09-23T19:12:02.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:01 smithi098 bash[76817]: cluster 2023-09-23T19:12:00.743524+0000 mgr.y (mgr.26054) 1635 : cluster 0 pgmap v1831: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:02.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:01 smithi098 bash[76817]: audit 2023-09-23T19:12:00.865477+0000 mon.a (mon.0) 2219 : audit 1 from='client.? 172.21.15.98:0/3976685182' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:12:02.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:01 smithi098 bash[76817]: cluster 2023-09-23T19:12:00.865565+0000 mon.a (mon.0) 2220 : cluster 0 osdmap e403: 8 total, 8 up, 8 in 2023-09-23T19:12:02.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:01 smithi098 bash[76817]: cluster 2023-09-23T19:12:00.867593+0000 osd.7 (osd.7) 56 : cluster 3 failed to encode map e403 with expected crc 2023-09-23T19:12:02.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:01 smithi098 bash[76817]: cluster 2023-09-23T19:12:00.871296+0000 osd.4 (osd.4) 135 : cluster 3 failed to encode map e403 with expected crc 2023-09-23T19:12:02.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:01 smithi173 bash[59422]: cluster 2023-09-23T19:12:00.743524+0000 mgr.y (mgr.26054) 1635 : cluster 0 pgmap v1831: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:02.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:01 smithi173 bash[59422]: audit 2023-09-23T19:12:00.865477+0000 mon.a (mon.0) 2219 : audit 1 from='client.? 172.21.15.98:0/3976685182' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T19:12:02.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:01 smithi173 bash[59422]: cluster 2023-09-23T19:12:00.865565+0000 mon.a (mon.0) 2220 : cluster 0 osdmap e403: 8 total, 8 up, 8 in 2023-09-23T19:12:02.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:01 smithi173 bash[59422]: cluster 2023-09-23T19:12:00.867593+0000 osd.7 (osd.7) 56 : cluster 3 failed to encode map e403 with expected crc 2023-09-23T19:12:02.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:01 smithi173 bash[59422]: cluster 2023-09-23T19:12:00.871296+0000 osd.4 (osd.4) 135 : cluster 3 failed to encode map e403 with expected crc 2023-09-23T19:12:03.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:02 smithi098 bash[79516]: cluster 2023-09-23T19:12:01.877732+0000 mon.a (mon.0) 2221 : cluster 0 osdmap e404: 8 total, 8 up, 8 in 2023-09-23T19:12:03.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:02 smithi098 bash[79516]: cluster 2023-09-23T19:12:02.201473+0000 osd.7 (osd.7) 57 : cluster 3 failed to encode map e404 with expected crc 2023-09-23T19:12:03.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:02 smithi098 bash[79516]: cluster 2023-09-23T19:12:02.413903+0000 osd.5 (osd.5) 44 : cluster 3 failed to encode map e404 with expected crc 2023-09-23T19:12:03.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:02 smithi098 bash[76817]: cluster 2023-09-23T19:12:01.877732+0000 mon.a (mon.0) 2221 : cluster 0 osdmap e404: 8 total, 8 up, 8 in 2023-09-23T19:12:03.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:02 smithi098 bash[76817]: cluster 2023-09-23T19:12:02.201473+0000 osd.7 (osd.7) 57 : cluster 3 failed to encode map e404 with expected crc 2023-09-23T19:12:03.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:02 smithi098 bash[76817]: cluster 2023-09-23T19:12:02.413903+0000 osd.5 (osd.5) 44 : cluster 3 failed to encode map e404 with expected crc 2023-09-23T19:12:03.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:02 smithi173 bash[59422]: cluster 2023-09-23T19:12:01.877732+0000 mon.a (mon.0) 2221 : cluster 0 osdmap e404: 8 total, 8 up, 8 in 2023-09-23T19:12:03.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:02 smithi173 bash[59422]: cluster 2023-09-23T19:12:02.201473+0000 osd.7 (osd.7) 57 : cluster 3 failed to encode map e404 with expected crc 2023-09-23T19:12:03.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:02 smithi173 bash[59422]: cluster 2023-09-23T19:12:02.413903+0000 osd.5 (osd.5) 44 : cluster 3 failed to encode map e404 with expected crc 2023-09-23T19:12:04.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:03 smithi098 bash[76817]: cluster 2023-09-23T19:12:02.281252+0000 osd.6 (osd.6) 388 : cluster 3 failed to encode map e404 with expected crc 2023-09-23T19:12:04.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:03 smithi098 bash[76817]: cluster 2023-09-23T19:12:02.744420+0000 mgr.y (mgr.26054) 1636 : cluster 0 pgmap v1834: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:04.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:03 smithi098 bash[79516]: cluster 2023-09-23T19:12:02.281252+0000 osd.6 (osd.6) 388 : cluster 3 failed to encode map e404 with expected crc 2023-09-23T19:12:04.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:03 smithi098 bash[79516]: cluster 2023-09-23T19:12:02.744420+0000 mgr.y (mgr.26054) 1636 : cluster 0 pgmap v1834: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:04.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:03 smithi173 bash[59422]: cluster 2023-09-23T19:12:02.281252+0000 osd.6 (osd.6) 388 : cluster 3 failed to encode map e404 with expected crc 2023-09-23T19:12:04.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:03 smithi173 bash[59422]: cluster 2023-09-23T19:12:02.744420+0000 mgr.y (mgr.26054) 1636 : cluster 0 pgmap v1834: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:06.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:05 smithi098 bash[76817]: cluster 2023-09-23T19:12:04.745323+0000 mgr.y (mgr.26054) 1637 : cluster 0 pgmap v1835: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:06.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:12:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:12:06] "GET /metrics HTTP/1.1" 200 34166 "" "Prometheus/2.33.4" 2023-09-23T19:12:06.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:05 smithi098 bash[79516]: cluster 2023-09-23T19:12:04.745323+0000 mgr.y (mgr.26054) 1637 : cluster 0 pgmap v1835: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:06.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:05 smithi173 bash[59422]: cluster 2023-09-23T19:12:04.745323+0000 mgr.y (mgr.26054) 1637 : cluster 0 pgmap v1835: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:06.621 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:12:08.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:07 smithi098 bash[76817]: cluster 2023-09-23T19:12:06.746163+0000 mgr.y (mgr.26054) 1638 : cluster 0 pgmap v1836: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:08.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:07 smithi098 bash[79516]: cluster 2023-09-23T19:12:06.746163+0000 mgr.y (mgr.26054) 1638 : cluster 0 pgmap v1836: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:08.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:07 smithi173 bash[59422]: cluster 2023-09-23T19:12:06.746163+0000 mgr.y (mgr.26054) 1638 : cluster 0 pgmap v1836: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:10.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:09 smithi098 bash[76817]: cluster 2023-09-23T19:12:08.747104+0000 mgr.y (mgr.26054) 1639 : cluster 0 pgmap v1837: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:10.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:09 smithi098 bash[76817]: audit 2023-09-23T19:12:09.656246+0000 mon.a (mon.0) 2222 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:12:10.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:09 smithi098 bash[79516]: cluster 2023-09-23T19:12:08.747104+0000 mgr.y (mgr.26054) 1639 : cluster 0 pgmap v1837: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:10.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:09 smithi098 bash[79516]: audit 2023-09-23T19:12:09.656246+0000 mon.a (mon.0) 2222 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:12:10.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:09 smithi173 bash[59422]: cluster 2023-09-23T19:12:08.747104+0000 mgr.y (mgr.26054) 1639 : cluster 0 pgmap v1837: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:10.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:09 smithi173 bash[59422]: audit 2023-09-23T19:12:09.656246+0000 mon.a (mon.0) 2222 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:12:10.927 INFO:teuthology.orchestra.run.smithi098.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T19:12:11.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:10 smithi098 bash[79516]: audit 2023-09-23T19:12:10.334549+0000 mon.c (mon.2) 164 : audit 1 from='client.? 172.21.15.98:0/334662695' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:12:11.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:10 smithi098 bash[79516]: audit 2023-09-23T19:12:10.335168+0000 mon.a (mon.0) 2223 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:12:11.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:10 smithi098 bash[76817]: audit 2023-09-23T19:12:10.334549+0000 mon.c (mon.2) 164 : audit 1 from='client.? 172.21.15.98:0/334662695' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:12:11.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:10 smithi098 bash[76817]: audit 2023-09-23T19:12:10.335168+0000 mon.a (mon.0) 2223 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:12:11.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:10 smithi173 bash[59422]: audit 2023-09-23T19:12:10.334549+0000 mon.c (mon.2) 164 : audit 1 from='client.? 172.21.15.98:0/334662695' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:12:11.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:10 smithi173 bash[59422]: audit 2023-09-23T19:12:10.335168+0000 mon.a (mon.0) 2223 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:12:11.930 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-09-23T19:12:12.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:11 smithi098 bash[76817]: cluster 2023-09-23T19:12:10.747937+0000 mgr.y (mgr.26054) 1640 : cluster 0 pgmap v1838: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:12.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:11 smithi098 bash[76817]: audit 2023-09-23T19:12:10.914522+0000 mon.a (mon.0) 2224 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:12:12.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:11 smithi098 bash[76817]: cluster 2023-09-23T19:12:10.914577+0000 mon.a (mon.0) 2225 : cluster 0 osdmap e405: 8 total, 8 up, 8 in 2023-09-23T19:12:12.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:11 smithi098 bash[76817]: cluster 2023-09-23T19:12:10.916936+0000 osd.7 (osd.7) 58 : cluster 3 failed to encode map e405 with expected crc 2023-09-23T19:12:12.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:11 smithi098 bash[76817]: cluster 2023-09-23T19:12:10.917958+0000 osd.4 (osd.4) 136 : cluster 3 failed to encode map e405 with expected crc 2023-09-23T19:12:12.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:11 smithi098 bash[76817]: cluster 2023-09-23T19:12:11.369619+0000 osd.5 (osd.5) 45 : cluster 3 failed to encode map e405 with expected crc 2023-09-23T19:12:12.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:11 smithi098 bash[79516]: cluster 2023-09-23T19:12:10.747937+0000 mgr.y (mgr.26054) 1640 : cluster 0 pgmap v1838: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:12.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:11 smithi098 bash[79516]: audit 2023-09-23T19:12:10.914522+0000 mon.a (mon.0) 2224 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:12:12.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:11 smithi098 bash[79516]: cluster 2023-09-23T19:12:10.914577+0000 mon.a (mon.0) 2225 : cluster 0 osdmap e405: 8 total, 8 up, 8 in 2023-09-23T19:12:12.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:11 smithi098 bash[79516]: cluster 2023-09-23T19:12:10.916936+0000 osd.7 (osd.7) 58 : cluster 3 failed to encode map e405 with expected crc 2023-09-23T19:12:12.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:11 smithi098 bash[79516]: cluster 2023-09-23T19:12:10.917958+0000 osd.4 (osd.4) 136 : cluster 3 failed to encode map e405 with expected crc 2023-09-23T19:12:12.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:11 smithi098 bash[79516]: cluster 2023-09-23T19:12:11.369619+0000 osd.5 (osd.5) 45 : cluster 3 failed to encode map e405 with expected crc 2023-09-23T19:12:12.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:11 smithi173 bash[59422]: cluster 2023-09-23T19:12:10.747937+0000 mgr.y (mgr.26054) 1640 : cluster 0 pgmap v1838: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:12.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:11 smithi173 bash[59422]: audit 2023-09-23T19:12:10.914522+0000 mon.a (mon.0) 2224 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:12:12.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:11 smithi173 bash[59422]: cluster 2023-09-23T19:12:10.914577+0000 mon.a (mon.0) 2225 : cluster 0 osdmap e405: 8 total, 8 up, 8 in 2023-09-23T19:12:12.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:11 smithi173 bash[59422]: cluster 2023-09-23T19:12:10.916936+0000 osd.7 (osd.7) 58 : cluster 3 failed to encode map e405 with expected crc 2023-09-23T19:12:12.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:11 smithi173 bash[59422]: cluster 2023-09-23T19:12:10.917958+0000 osd.4 (osd.4) 136 : cluster 3 failed to encode map e405 with expected crc 2023-09-23T19:12:12.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:11 smithi173 bash[59422]: cluster 2023-09-23T19:12:11.369619+0000 osd.5 (osd.5) 45 : cluster 3 failed to encode map e405 with expected crc 2023-09-23T19:12:13.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:12 smithi098 bash[76817]: cluster 2023-09-23T19:12:11.369499+0000 osd.6 (osd.6) 389 : cluster 3 failed to encode map e405 with expected crc 2023-09-23T19:12:13.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:12 smithi098 bash[79516]: cluster 2023-09-23T19:12:11.369499+0000 osd.6 (osd.6) 389 : cluster 3 failed to encode map e405 with expected crc 2023-09-23T19:12:13.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:12 smithi173 bash[59422]: cluster 2023-09-23T19:12:11.369499+0000 osd.6 (osd.6) 389 : cluster 3 failed to encode map e405 with expected crc 2023-09-23T19:12:14.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:13 smithi098 bash[76817]: cluster 2023-09-23T19:12:12.748753+0000 mgr.y (mgr.26054) 1641 : cluster 0 pgmap v1840: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:14.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:13 smithi098 bash[79516]: cluster 2023-09-23T19:12:12.748753+0000 mgr.y (mgr.26054) 1641 : cluster 0 pgmap v1840: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:14.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:13 smithi173 bash[59422]: cluster 2023-09-23T19:12:12.748753+0000 mgr.y (mgr.26054) 1641 : cluster 0 pgmap v1840: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:16.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:15 smithi098 bash[79516]: cluster 2023-09-23T19:12:14.749692+0000 mgr.y (mgr.26054) 1642 : cluster 0 pgmap v1841: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:16.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:15 smithi098 bash[76817]: cluster 2023-09-23T19:12:14.749692+0000 mgr.y (mgr.26054) 1642 : cluster 0 pgmap v1841: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:16.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:12:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:12:16] "GET /metrics HTTP/1.1" 200 34166 "" "Prometheus/2.33.4" 2023-09-23T19:12:16.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:15 smithi173 bash[59422]: cluster 2023-09-23T19:12:14.749692+0000 mgr.y (mgr.26054) 1642 : cluster 0 pgmap v1841: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:16.704 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:12:18.190 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:17 smithi098 bash[76817]: cluster 2023-09-23T19:12:16.750388+0000 mgr.y (mgr.26054) 1643 : cluster 0 pgmap v1842: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:18.190 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:17 smithi098 bash[79516]: cluster 2023-09-23T19:12:16.750388+0000 mgr.y (mgr.26054) 1643 : cluster 0 pgmap v1842: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:18.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:17 smithi173 bash[59422]: cluster 2023-09-23T19:12:16.750388+0000 mgr.y (mgr.26054) 1643 : cluster 0 pgmap v1842: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:20.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:19 smithi098 bash[79516]: cluster 2023-09-23T19:12:18.751274+0000 mgr.y (mgr.26054) 1644 : cluster 0 pgmap v1843: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:20.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:19 smithi098 bash[76817]: cluster 2023-09-23T19:12:18.751274+0000 mgr.y (mgr.26054) 1644 : cluster 0 pgmap v1843: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:20.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:19 smithi173 bash[59422]: cluster 2023-09-23T19:12:18.751274+0000 mgr.y (mgr.26054) 1644 : cluster 0 pgmap v1843: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:20.966 INFO:teuthology.orchestra.run.smithi098.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-09-23T19:12:21.255 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:20 smithi098 bash[79516]: audit 2023-09-23T19:12:20.500387+0000 mon.a (mon.0) 2226 : audit 1 from='client.? 172.21.15.98:0/3999466983' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:12:21.255 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:20 smithi098 bash[76817]: audit 2023-09-23T19:12:20.500387+0000 mon.a (mon.0) 2226 : audit 1 from='client.? 172.21.15.98:0/3999466983' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:12:21.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:20 smithi173 bash[59422]: audit 2023-09-23T19:12:20.500387+0000 mon.a (mon.0) 2226 : audit 1 from='client.? 172.21.15.98:0/3999466983' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T19:12:21.964 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T19:12:22.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:21 smithi173 bash[59422]: cluster 2023-09-23T19:12:20.752120+0000 mgr.y (mgr.26054) 1645 : cluster 0 pgmap v1844: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:22.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:21 smithi173 bash[59422]: audit 2023-09-23T19:12:20.955525+0000 mon.a (mon.0) 2227 : audit 1 from='client.? 172.21.15.98:0/3999466983' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:12:22.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:21 smithi173 bash[59422]: cluster 2023-09-23T19:12:20.955585+0000 mon.a (mon.0) 2228 : cluster 0 osdmap e406: 8 total, 8 up, 8 in 2023-09-23T19:12:22.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:21 smithi173 bash[59422]: cluster 2023-09-23T19:12:20.957572+0000 osd.7 (osd.7) 59 : cluster 3 failed to encode map e406 with expected crc 2023-09-23T19:12:22.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:21 smithi173 bash[59422]: cluster 2023-09-23T19:12:20.958142+0000 osd.4 (osd.4) 137 : cluster 3 failed to encode map e406 with expected crc 2023-09-23T19:12:22.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:21 smithi173 bash[59422]: cluster 2023-09-23T19:12:20.958295+0000 osd.6 (osd.6) 390 : cluster 3 failed to encode map e406 with expected crc 2023-09-23T19:12:22.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:21 smithi098 bash[79516]: cluster 2023-09-23T19:12:20.752120+0000 mgr.y (mgr.26054) 1645 : cluster 0 pgmap v1844: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:22.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:21 smithi098 bash[79516]: audit 2023-09-23T19:12:20.955525+0000 mon.a (mon.0) 2227 : audit 1 from='client.? 172.21.15.98:0/3999466983' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:12:22.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:21 smithi098 bash[79516]: cluster 2023-09-23T19:12:20.955585+0000 mon.a (mon.0) 2228 : cluster 0 osdmap e406: 8 total, 8 up, 8 in 2023-09-23T19:12:22.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:21 smithi098 bash[79516]: cluster 2023-09-23T19:12:20.957572+0000 osd.7 (osd.7) 59 : cluster 3 failed to encode map e406 with expected crc 2023-09-23T19:12:22.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:21 smithi098 bash[79516]: cluster 2023-09-23T19:12:20.958142+0000 osd.4 (osd.4) 137 : cluster 3 failed to encode map e406 with expected crc 2023-09-23T19:12:22.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:21 smithi098 bash[79516]: cluster 2023-09-23T19:12:20.958295+0000 osd.6 (osd.6) 390 : cluster 3 failed to encode map e406 with expected crc 2023-09-23T19:12:22.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:21 smithi098 bash[76817]: cluster 2023-09-23T19:12:20.752120+0000 mgr.y (mgr.26054) 1645 : cluster 0 pgmap v1844: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:22.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:21 smithi098 bash[76817]: audit 2023-09-23T19:12:20.955525+0000 mon.a (mon.0) 2227 : audit 1 from='client.? 172.21.15.98:0/3999466983' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T19:12:22.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:21 smithi098 bash[76817]: cluster 2023-09-23T19:12:20.955585+0000 mon.a (mon.0) 2228 : cluster 0 osdmap e406: 8 total, 8 up, 8 in 2023-09-23T19:12:22.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:21 smithi098 bash[76817]: cluster 2023-09-23T19:12:20.957572+0000 osd.7 (osd.7) 59 : cluster 3 failed to encode map e406 with expected crc 2023-09-23T19:12:22.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:21 smithi098 bash[76817]: cluster 2023-09-23T19:12:20.958142+0000 osd.4 (osd.4) 137 : cluster 3 failed to encode map e406 with expected crc 2023-09-23T19:12:22.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:21 smithi098 bash[76817]: cluster 2023-09-23T19:12:20.958295+0000 osd.6 (osd.6) 390 : cluster 3 failed to encode map e406 with expected crc 2023-09-23T19:12:24.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:23 smithi173 bash[59422]: cluster 2023-09-23T19:12:22.752894+0000 mgr.y (mgr.26054) 1646 : cluster 0 pgmap v1846: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:24.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:23 smithi098 bash[79516]: cluster 2023-09-23T19:12:22.752894+0000 mgr.y (mgr.26054) 1646 : cluster 0 pgmap v1846: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:24.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:23 smithi098 bash[76817]: cluster 2023-09-23T19:12:22.752894+0000 mgr.y (mgr.26054) 1646 : cluster 0 pgmap v1846: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:25.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:24 smithi173 bash[59422]: audit 2023-09-23T19:12:24.656440+0000 mon.a (mon.0) 2229 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:12:25.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:24 smithi098 bash[79516]: audit 2023-09-23T19:12:24.656440+0000 mon.a (mon.0) 2229 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:12:25.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:24 smithi098 bash[76817]: audit 2023-09-23T19:12:24.656440+0000 mon.a (mon.0) 2229 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:12:26.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:25 smithi173 bash[59422]: cluster 2023-09-23T19:12:24.753754+0000 mgr.y (mgr.26054) 1647 : cluster 0 pgmap v1847: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:26.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:25 smithi098 bash[76817]: cluster 2023-09-23T19:12:24.753754+0000 mgr.y (mgr.26054) 1647 : cluster 0 pgmap v1847: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:26.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:12:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:12:26] "GET /metrics HTTP/1.1" 200 34164 "" "Prometheus/2.33.4" 2023-09-23T19:12:26.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:25 smithi098 bash[79516]: cluster 2023-09-23T19:12:24.753754+0000 mgr.y (mgr.26054) 1647 : cluster 0 pgmap v1847: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:26.735 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:12:28.229 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:27 smithi098 bash[79516]: cluster 2023-09-23T19:12:26.754646+0000 mgr.y (mgr.26054) 1648 : cluster 0 pgmap v1848: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:28.230 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:27 smithi098 bash[76817]: cluster 2023-09-23T19:12:26.754646+0000 mgr.y (mgr.26054) 1648 : cluster 0 pgmap v1848: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:28.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:27 smithi173 bash[59422]: cluster 2023-09-23T19:12:26.754646+0000 mgr.y (mgr.26054) 1648 : cluster 0 pgmap v1848: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:30.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:29 smithi173 bash[59422]: cluster 2023-09-23T19:12:28.755726+0000 mgr.y (mgr.26054) 1649 : cluster 0 pgmap v1849: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:30.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:29 smithi173 bash[59422]: audit 2023-09-23T19:12:29.629897+0000 mon.a (mon.0) 2230 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:12:30.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:29 smithi098 bash[76817]: cluster 2023-09-23T19:12:28.755726+0000 mgr.y (mgr.26054) 1649 : cluster 0 pgmap v1849: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:30.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:29 smithi098 bash[76817]: audit 2023-09-23T19:12:29.629897+0000 mon.a (mon.0) 2230 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:12:30.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:29 smithi098 bash[79516]: cluster 2023-09-23T19:12:28.755726+0000 mgr.y (mgr.26054) 1649 : cluster 0 pgmap v1849: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:30.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:29 smithi098 bash[79516]: audit 2023-09-23T19:12:29.629897+0000 mon.a (mon.0) 2230 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:12:31.000 INFO:teuthology.orchestra.run.smithi098.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T19:12:31.263 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:30 smithi098 bash[76817]: audit 2023-09-23T19:12:30.451737+0000 mon.a (mon.0) 2231 : audit 1 from='client.? 172.21.15.98:0/1451626' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:12:31.264 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:30 smithi098 bash[79516]: audit 2023-09-23T19:12:30.451737+0000 mon.a (mon.0) 2231 : audit 1 from='client.? 172.21.15.98:0/1451626' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:12:31.281 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:30 smithi173 bash[59422]: audit 2023-09-23T19:12:30.451737+0000 mon.a (mon.0) 2231 : audit 1 from='client.? 172.21.15.98:0/1451626' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:12:31.981 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-09-23T19:12:32.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:31 smithi173 bash[59422]: cluster 2023-09-23T19:12:30.756564+0000 mgr.y (mgr.26054) 1650 : cluster 0 pgmap v1850: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:32.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:31 smithi173 bash[59422]: audit 2023-09-23T19:12:30.991005+0000 mon.a (mon.0) 2232 : audit 1 from='client.? 172.21.15.98:0/1451626' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:12:32.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:31 smithi173 bash[59422]: cluster 2023-09-23T19:12:30.991074+0000 mon.a (mon.0) 2233 : cluster 0 osdmap e407: 8 total, 8 up, 8 in 2023-09-23T19:12:32.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:31 smithi173 bash[59422]: cluster 2023-09-23T19:12:30.991320+0000 osd.4 (osd.4) 138 : cluster 3 failed to encode map e407 with expected crc 2023-09-23T19:12:32.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:31 smithi173 bash[59422]: cluster 2023-09-23T19:12:30.991668+0000 osd.6 (osd.6) 391 : cluster 3 failed to encode map e407 with expected crc 2023-09-23T19:12:32.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:31 smithi173 bash[59422]: cluster 2023-09-23T19:12:31.074476+0000 osd.7 (osd.7) 60 : cluster 3 failed to encode map e407 with expected crc 2023-09-23T19:12:32.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:31 smithi173 bash[59422]: cluster 2023-09-23T19:12:31.338520+0000 osd.5 (osd.5) 46 : cluster 3 failed to encode map e407 with expected crc 2023-09-23T19:12:32.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:31 smithi098 bash[76817]: cluster 2023-09-23T19:12:30.756564+0000 mgr.y (mgr.26054) 1650 : cluster 0 pgmap v1850: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:32.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:31 smithi098 bash[76817]: audit 2023-09-23T19:12:30.991005+0000 mon.a (mon.0) 2232 : audit 1 from='client.? 172.21.15.98:0/1451626' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:12:32.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:31 smithi098 bash[76817]: cluster 2023-09-23T19:12:30.991074+0000 mon.a (mon.0) 2233 : cluster 0 osdmap e407: 8 total, 8 up, 8 in 2023-09-23T19:12:32.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:31 smithi098 bash[76817]: cluster 2023-09-23T19:12:30.991320+0000 osd.4 (osd.4) 138 : cluster 3 failed to encode map e407 with expected crc 2023-09-23T19:12:32.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:31 smithi098 bash[76817]: cluster 2023-09-23T19:12:30.991668+0000 osd.6 (osd.6) 391 : cluster 3 failed to encode map e407 with expected crc 2023-09-23T19:12:32.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:31 smithi098 bash[76817]: cluster 2023-09-23T19:12:31.074476+0000 osd.7 (osd.7) 60 : cluster 3 failed to encode map e407 with expected crc 2023-09-23T19:12:32.446 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:31 smithi098 bash[76817]: cluster 2023-09-23T19:12:31.338520+0000 osd.5 (osd.5) 46 : cluster 3 failed to encode map e407 with expected crc 2023-09-23T19:12:32.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:31 smithi098 bash[79516]: cluster 2023-09-23T19:12:30.756564+0000 mgr.y (mgr.26054) 1650 : cluster 0 pgmap v1850: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:32.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:31 smithi098 bash[79516]: audit 2023-09-23T19:12:30.991005+0000 mon.a (mon.0) 2232 : audit 1 from='client.? 172.21.15.98:0/1451626' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:12:32.446 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:31 smithi098 bash[79516]: cluster 2023-09-23T19:12:30.991074+0000 mon.a (mon.0) 2233 : cluster 0 osdmap e407: 8 total, 8 up, 8 in 2023-09-23T19:12:32.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:31 smithi098 bash[79516]: cluster 2023-09-23T19:12:30.991320+0000 osd.4 (osd.4) 138 : cluster 3 failed to encode map e407 with expected crc 2023-09-23T19:12:32.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:31 smithi098 bash[79516]: cluster 2023-09-23T19:12:30.991668+0000 osd.6 (osd.6) 391 : cluster 3 failed to encode map e407 with expected crc 2023-09-23T19:12:32.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:31 smithi098 bash[79516]: cluster 2023-09-23T19:12:31.074476+0000 osd.7 (osd.7) 60 : cluster 3 failed to encode map e407 with expected crc 2023-09-23T19:12:32.447 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:31 smithi098 bash[79516]: cluster 2023-09-23T19:12:31.338520+0000 osd.5 (osd.5) 46 : cluster 3 failed to encode map e407 with expected crc 2023-09-23T19:12:34.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:33 smithi173 bash[59422]: cluster 2023-09-23T19:12:32.757497+0000 mgr.y (mgr.26054) 1651 : cluster 0 pgmap v1852: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:34.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:33 smithi098 bash[76817]: cluster 2023-09-23T19:12:32.757497+0000 mgr.y (mgr.26054) 1651 : cluster 0 pgmap v1852: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:34.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:33 smithi098 bash[79516]: cluster 2023-09-23T19:12:32.757497+0000 mgr.y (mgr.26054) 1651 : cluster 0 pgmap v1852: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:36.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:36 smithi173 bash[59422]: cluster 2023-09-23T19:12:34.758479+0000 mgr.y (mgr.26054) 1652 : cluster 0 pgmap v1853: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:36.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:36 smithi173 bash[59422]: audit 2023-09-23T19:12:35.776858+0000 mon.a (mon.0) 2234 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:12:36.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:36 smithi173 bash[59422]: audit 2023-09-23T19:12:35.783939+0000 mon.a (mon.0) 2235 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:12:36.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:36 smithi173 bash[59422]: audit 2023-09-23T19:12:35.926686+0000 mon.a (mon.0) 2236 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:12:36.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:36 smithi173 bash[59422]: audit 2023-09-23T19:12:35.935999+0000 mon.a (mon.0) 2237 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:12:36.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:35 smithi098 bash[76817]: cluster 2023-09-23T19:12:34.758479+0000 mgr.y (mgr.26054) 1652 : cluster 0 pgmap v1853: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:36.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:35 smithi098 bash[76817]: audit 2023-09-23T19:12:35.776858+0000 mon.a (mon.0) 2234 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:12:36.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:35 smithi098 bash[76817]: audit 2023-09-23T19:12:35.783939+0000 mon.a (mon.0) 2235 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:12:36.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:35 smithi098 bash[76817]: audit 2023-09-23T19:12:35.926686+0000 mon.a (mon.0) 2236 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:12:36.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:35 smithi098 bash[76817]: audit 2023-09-23T19:12:35.935999+0000 mon.a (mon.0) 2237 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:12:36.445 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:12:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:12:36] "GET /metrics HTTP/1.1" 200 34164 "" "Prometheus/2.33.4" 2023-09-23T19:12:36.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:35 smithi098 bash[79516]: cluster 2023-09-23T19:12:34.758479+0000 mgr.y (mgr.26054) 1652 : cluster 0 pgmap v1853: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:36.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:35 smithi098 bash[79516]: audit 2023-09-23T19:12:35.776858+0000 mon.a (mon.0) 2234 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:12:36.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:35 smithi098 bash[79516]: audit 2023-09-23T19:12:35.783939+0000 mon.a (mon.0) 2235 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:12:36.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:35 smithi098 bash[79516]: audit 2023-09-23T19:12:35.926686+0000 mon.a (mon.0) 2236 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:12:36.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:35 smithi098 bash[79516]: audit 2023-09-23T19:12:35.935999+0000 mon.a (mon.0) 2237 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:12:36.536 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:12:37.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:37 smithi173 bash[59422]: audit 2023-09-23T19:12:36.407251+0000 mon.a (mon.0) 2238 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:12:37.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:37 smithi173 bash[59422]: audit 2023-09-23T19:12:36.408844+0000 mon.a (mon.0) 2239 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:12:37.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:37 smithi173 bash[59422]: audit 2023-09-23T19:12:36.417044+0000 mon.a (mon.0) 2240 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:12:37.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:37 smithi098 bash[76817]: audit 2023-09-23T19:12:36.407251+0000 mon.a (mon.0) 2238 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:12:37.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:37 smithi098 bash[76817]: audit 2023-09-23T19:12:36.408844+0000 mon.a (mon.0) 2239 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:12:37.444 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:37 smithi098 bash[76817]: audit 2023-09-23T19:12:36.417044+0000 mon.a (mon.0) 2240 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:12:37.444 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:37 smithi098 bash[79516]: audit 2023-09-23T19:12:36.407251+0000 mon.a (mon.0) 2238 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:12:37.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:37 smithi098 bash[79516]: audit 2023-09-23T19:12:36.408844+0000 mon.a (mon.0) 2239 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:12:37.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:37 smithi098 bash[79516]: audit 2023-09-23T19:12:36.417044+0000 mon.a (mon.0) 2240 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:12:38.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:38 smithi098 bash[79516]: cluster 2023-09-23T19:12:36.759446+0000 mgr.y (mgr.26054) 1653 : cluster 0 pgmap v1854: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:38.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:38 smithi098 bash[76817]: cluster 2023-09-23T19:12:36.759446+0000 mgr.y (mgr.26054) 1653 : cluster 0 pgmap v1854: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:38.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:38 smithi173 bash[59422]: cluster 2023-09-23T19:12:36.759446+0000 mgr.y (mgr.26054) 1653 : cluster 0 pgmap v1854: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:40.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:40 smithi098 bash[79516]: cluster 2023-09-23T19:12:38.760390+0000 mgr.y (mgr.26054) 1654 : cluster 0 pgmap v1855: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:40.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:40 smithi098 bash[79516]: audit 2023-09-23T19:12:39.656764+0000 mon.a (mon.0) 2241 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:12:40.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:40 smithi098 bash[76817]: cluster 2023-09-23T19:12:38.760390+0000 mgr.y (mgr.26054) 1654 : cluster 0 pgmap v1855: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:40.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:40 smithi098 bash[76817]: audit 2023-09-23T19:12:39.656764+0000 mon.a (mon.0) 2241 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:12:40.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:40 smithi173 bash[59422]: cluster 2023-09-23T19:12:38.760390+0000 mgr.y (mgr.26054) 1654 : cluster 0 pgmap v1855: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:40.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:40 smithi173 bash[59422]: audit 2023-09-23T19:12:39.656764+0000 mon.a (mon.0) 2241 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:12:41.446 INFO:teuthology.orchestra.run.smithi098.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-09-23T19:12:41.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:41 smithi098 bash[76817]: audit 2023-09-23T19:12:40.504455+0000 mon.c (mon.2) 165 : audit 1 from='client.? 172.21.15.98:0/3979410515' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:12:41.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:41 smithi098 bash[76817]: audit 2023-09-23T19:12:40.505015+0000 mon.a (mon.0) 2242 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:12:41.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:41 smithi098 bash[79516]: audit 2023-09-23T19:12:40.504455+0000 mon.c (mon.2) 165 : audit 1 from='client.? 172.21.15.98:0/3979410515' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:12:41.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:41 smithi098 bash[79516]: audit 2023-09-23T19:12:40.505015+0000 mon.a (mon.0) 2242 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:12:41.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:41 smithi173 bash[59422]: audit 2023-09-23T19:12:40.504455+0000 mon.c (mon.2) 165 : audit 1 from='client.? 172.21.15.98:0/3979410515' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:12:41.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:41 smithi173 bash[59422]: audit 2023-09-23T19:12:40.505015+0000 mon.a (mon.0) 2242 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T19:12:42.476 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T19:12:42.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:42 smithi098 bash[79516]: cluster 2023-09-23T19:12:40.761208+0000 mgr.y (mgr.26054) 1655 : cluster 0 pgmap v1856: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:42.694 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:42 smithi098 bash[79516]: audit 2023-09-23T19:12:41.433942+0000 mon.a (mon.0) 2243 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:12:42.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:42 smithi098 bash[79516]: cluster 2023-09-23T19:12:41.434057+0000 mon.a (mon.0) 2244 : cluster 0 osdmap e408: 8 total, 8 up, 8 in 2023-09-23T19:12:42.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:42 smithi098 bash[79516]: cluster 2023-09-23T19:12:41.436321+0000 osd.7 (osd.7) 61 : cluster 3 failed to encode map e408 with expected crc 2023-09-23T19:12:42.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:42 smithi098 bash[79516]: cluster 2023-09-23T19:12:41.437658+0000 osd.6 (osd.6) 392 : cluster 3 failed to encode map e408 with expected crc 2023-09-23T19:12:42.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:42 smithi098 bash[76817]: cluster 2023-09-23T19:12:40.761208+0000 mgr.y (mgr.26054) 1655 : cluster 0 pgmap v1856: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:42.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:42 smithi098 bash[76817]: audit 2023-09-23T19:12:41.433942+0000 mon.a (mon.0) 2243 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:12:42.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:42 smithi098 bash[76817]: cluster 2023-09-23T19:12:41.434057+0000 mon.a (mon.0) 2244 : cluster 0 osdmap e408: 8 total, 8 up, 8 in 2023-09-23T19:12:42.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:42 smithi098 bash[76817]: cluster 2023-09-23T19:12:41.436321+0000 osd.7 (osd.7) 61 : cluster 3 failed to encode map e408 with expected crc 2023-09-23T19:12:42.695 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:42 smithi098 bash[76817]: cluster 2023-09-23T19:12:41.437658+0000 osd.6 (osd.6) 392 : cluster 3 failed to encode map e408 with expected crc 2023-09-23T19:12:42.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:42 smithi173 bash[59422]: cluster 2023-09-23T19:12:40.761208+0000 mgr.y (mgr.26054) 1655 : cluster 0 pgmap v1856: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:42.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:42 smithi173 bash[59422]: audit 2023-09-23T19:12:41.433942+0000 mon.a (mon.0) 2243 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T19:12:42.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:42 smithi173 bash[59422]: cluster 2023-09-23T19:12:41.434057+0000 mon.a (mon.0) 2244 : cluster 0 osdmap e408: 8 total, 8 up, 8 in 2023-09-23T19:12:42.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:42 smithi173 bash[59422]: cluster 2023-09-23T19:12:41.436321+0000 osd.7 (osd.7) 61 : cluster 3 failed to encode map e408 with expected crc 2023-09-23T19:12:42.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:42 smithi173 bash[59422]: cluster 2023-09-23T19:12:41.437658+0000 osd.6 (osd.6) 392 : cluster 3 failed to encode map e408 with expected crc 2023-09-23T19:12:44.694 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:44 smithi098 bash[76817]: cluster 2023-09-23T19:12:42.762015+0000 mgr.y (mgr.26054) 1656 : cluster 0 pgmap v1858: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:44.695 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:44 smithi098 bash[79516]: cluster 2023-09-23T19:12:42.762015+0000 mgr.y (mgr.26054) 1656 : cluster 0 pgmap v1858: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:44.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:44 smithi173 bash[59422]: cluster 2023-09-23T19:12:42.762015+0000 mgr.y (mgr.26054) 1656 : cluster 0 pgmap v1858: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:46.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:12:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:12:46] "GET /metrics HTTP/1.1" 200 34166 "" "Prometheus/2.33.4" 2023-09-23T19:12:46.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:46 smithi173 bash[59422]: cluster 2023-09-23T19:12:44.762930+0000 mgr.y (mgr.26054) 1657 : cluster 0 pgmap v1859: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:46.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:46 smithi098 bash[76817]: cluster 2023-09-23T19:12:44.762930+0000 mgr.y (mgr.26054) 1657 : cluster 0 pgmap v1859: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:46.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:46 smithi098 bash[79516]: cluster 2023-09-23T19:12:44.762930+0000 mgr.y (mgr.26054) 1657 : cluster 0 pgmap v1859: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:47.249 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:12:48.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:48 smithi173 bash[59422]: cluster 2023-09-23T19:12:46.763720+0000 mgr.y (mgr.26054) 1658 : cluster 0 pgmap v1860: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:48.876 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:48 smithi098 bash[79516]: cluster 2023-09-23T19:12:46.763720+0000 mgr.y (mgr.26054) 1658 : cluster 0 pgmap v1860: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:48.877 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:48 smithi098 bash[76817]: cluster 2023-09-23T19:12:46.763720+0000 mgr.y (mgr.26054) 1658 : cluster 0 pgmap v1860: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:50.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:50 smithi173 bash[59422]: cluster 2023-09-23T19:12:48.764645+0000 mgr.y (mgr.26054) 1659 : cluster 0 pgmap v1861: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:50.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:50 smithi098 bash[76817]: cluster 2023-09-23T19:12:48.764645+0000 mgr.y (mgr.26054) 1659 : cluster 0 pgmap v1861: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:50.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:50 smithi098 bash[79516]: cluster 2023-09-23T19:12:48.764645+0000 mgr.y (mgr.26054) 1659 : cluster 0 pgmap v1861: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:51.700 INFO:teuthology.orchestra.run.smithi098.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T19:12:51.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:51 smithi098 bash[76817]: cluster 2023-09-23T19:12:50.765432+0000 mgr.y (mgr.26054) 1660 : cluster 0 pgmap v1862: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:51.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:51 smithi098 bash[76817]: audit 2023-09-23T19:12:51.033788+0000 mon.a (mon.0) 2245 : audit 1 from='client.? 172.21.15.98:0/2187965310' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:12:51.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:51 smithi098 bash[79516]: cluster 2023-09-23T19:12:50.765432+0000 mgr.y (mgr.26054) 1660 : cluster 0 pgmap v1862: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:51.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:51 smithi098 bash[79516]: audit 2023-09-23T19:12:51.033788+0000 mon.a (mon.0) 2245 : audit 1 from='client.? 172.21.15.98:0/2187965310' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:12:52.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:51 smithi173 bash[59422]: cluster 2023-09-23T19:12:50.765432+0000 mgr.y (mgr.26054) 1660 : cluster 0 pgmap v1862: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:52.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:51 smithi173 bash[59422]: audit 2023-09-23T19:12:51.033788+0000 mon.a (mon.0) 2245 : audit 1 from='client.? 172.21.15.98:0/2187965310' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:12:52.693 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-09-23T19:12:52.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:52 smithi098 bash[79516]: audit 2023-09-23T19:12:51.690797+0000 mon.a (mon.0) 2246 : audit 1 from='client.? 172.21.15.98:0/2187965310' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:12:52.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:52 smithi098 bash[79516]: cluster 2023-09-23T19:12:51.690863+0000 mon.a (mon.0) 2247 : cluster 0 osdmap e409: 8 total, 8 up, 8 in 2023-09-23T19:12:52.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:52 smithi098 bash[79516]: cluster 2023-09-23T19:12:51.693325+0000 osd.5 (osd.5) 47 : cluster 3 failed to encode map e409 with expected crc 2023-09-23T19:12:52.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:52 smithi098 bash[79516]: cluster 2023-09-23T19:12:51.694230+0000 osd.6 (osd.6) 393 : cluster 3 failed to encode map e409 with expected crc 2023-09-23T19:12:52.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:52 smithi098 bash[76817]: audit 2023-09-23T19:12:51.690797+0000 mon.a (mon.0) 2246 : audit 1 from='client.? 172.21.15.98:0/2187965310' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:12:52.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:52 smithi098 bash[76817]: cluster 2023-09-23T19:12:51.690863+0000 mon.a (mon.0) 2247 : cluster 0 osdmap e409: 8 total, 8 up, 8 in 2023-09-23T19:12:52.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:52 smithi098 bash[76817]: cluster 2023-09-23T19:12:51.693325+0000 osd.5 (osd.5) 47 : cluster 3 failed to encode map e409 with expected crc 2023-09-23T19:12:52.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:52 smithi098 bash[76817]: cluster 2023-09-23T19:12:51.694230+0000 osd.6 (osd.6) 393 : cluster 3 failed to encode map e409 with expected crc 2023-09-23T19:12:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:52 smithi173 bash[59422]: audit 2023-09-23T19:12:51.690797+0000 mon.a (mon.0) 2246 : audit 1 from='client.? 172.21.15.98:0/2187965310' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:12:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:52 smithi173 bash[59422]: cluster 2023-09-23T19:12:51.690863+0000 mon.a (mon.0) 2247 : cluster 0 osdmap e409: 8 total, 8 up, 8 in 2023-09-23T19:12:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:52 smithi173 bash[59422]: cluster 2023-09-23T19:12:51.693325+0000 osd.5 (osd.5) 47 : cluster 3 failed to encode map e409 with expected crc 2023-09-23T19:12:53.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:52 smithi173 bash[59422]: cluster 2023-09-23T19:12:51.694230+0000 osd.6 (osd.6) 393 : cluster 3 failed to encode map e409 with expected crc 2023-09-23T19:12:54.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:53 smithi173 bash[59422]: cluster 2023-09-23T19:12:52.766065+0000 mgr.y (mgr.26054) 1661 : cluster 0 pgmap v1864: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:54.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:53 smithi173 bash[59422]: cluster 2023-09-23T19:12:53.463960+0000 osd.7 (osd.7) 62 : cluster 3 failed to encode map e409 with expected crc 2023-09-23T19:12:54.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:53 smithi098 bash[79516]: cluster 2023-09-23T19:12:52.766065+0000 mgr.y (mgr.26054) 1661 : cluster 0 pgmap v1864: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:54.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:53 smithi098 bash[79516]: cluster 2023-09-23T19:12:53.463960+0000 osd.7 (osd.7) 62 : cluster 3 failed to encode map e409 with expected crc 2023-09-23T19:12:54.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:53 smithi098 bash[76817]: cluster 2023-09-23T19:12:52.766065+0000 mgr.y (mgr.26054) 1661 : cluster 0 pgmap v1864: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:54.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:53 smithi098 bash[76817]: cluster 2023-09-23T19:12:53.463960+0000 osd.7 (osd.7) 62 : cluster 3 failed to encode map e409 with expected crc 2023-09-23T19:12:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:54 smithi173 bash[59422]: cluster 2023-09-23T19:12:53.463489+0000 osd.4 (osd.4) 139 : cluster 3 failed to encode map e409 with expected crc 2023-09-23T19:12:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:54 smithi173 bash[59422]: audit 2023-09-23T19:12:54.657159+0000 mon.a (mon.0) 2248 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:12:55.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:54 smithi098 bash[76817]: cluster 2023-09-23T19:12:53.463489+0000 osd.4 (osd.4) 139 : cluster 3 failed to encode map e409 with expected crc 2023-09-23T19:12:55.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:54 smithi098 bash[76817]: audit 2023-09-23T19:12:54.657159+0000 mon.a (mon.0) 2248 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:12:55.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:54 smithi098 bash[79516]: cluster 2023-09-23T19:12:53.463489+0000 osd.4 (osd.4) 139 : cluster 3 failed to encode map e409 with expected crc 2023-09-23T19:12:55.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:54 smithi098 bash[79516]: audit 2023-09-23T19:12:54.657159+0000 mon.a (mon.0) 2248 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:12:56.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:55 smithi173 bash[59422]: cluster 2023-09-23T19:12:54.766909+0000 mgr.y (mgr.26054) 1662 : cluster 0 pgmap v1865: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:56.087 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:55 smithi098 bash[79516]: cluster 2023-09-23T19:12:54.766909+0000 mgr.y (mgr.26054) 1662 : cluster 0 pgmap v1865: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:56.088 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:55 smithi098 bash[76817]: cluster 2023-09-23T19:12:54.766909+0000 mgr.y (mgr.26054) 1662 : cluster 0 pgmap v1865: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:56.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:12:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:12:56] "GET /metrics HTTP/1.1" 200 34166 "" "Prometheus/2.33.4" 2023-09-23T19:12:57.498 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:12:58.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:57 smithi098 bash[76817]: cluster 2023-09-23T19:12:56.767721+0000 mgr.y (mgr.26054) 1663 : cluster 0 pgmap v1866: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:58.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:57 smithi098 bash[79516]: cluster 2023-09-23T19:12:56.767721+0000 mgr.y (mgr.26054) 1663 : cluster 0 pgmap v1866: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:12:58.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:57 smithi173 bash[59422]: cluster 2023-09-23T19:12:56.767721+0000 mgr.y (mgr.26054) 1663 : cluster 0 pgmap v1866: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:00.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:12:59 smithi098 bash[79516]: cluster 2023-09-23T19:12:58.768663+0000 mgr.y (mgr.26054) 1664 : cluster 0 pgmap v1867: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:00.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:12:59 smithi098 bash[76817]: cluster 2023-09-23T19:12:58.768663+0000 mgr.y (mgr.26054) 1664 : cluster 0 pgmap v1867: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:00.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:12:59 smithi173 bash[59422]: cluster 2023-09-23T19:12:58.768663+0000 mgr.y (mgr.26054) 1664 : cluster 0 pgmap v1867: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:01.710 INFO:teuthology.orchestra.run.smithi098.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-09-23T19:13:01.966 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:01 smithi098 bash[79516]: cluster 2023-09-23T19:13:00.769455+0000 mgr.y (mgr.26054) 1665 : cluster 0 pgmap v1868: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:01.967 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:01 smithi098 bash[79516]: audit 2023-09-23T19:13:01.328719+0000 mon.c (mon.2) 166 : audit 1 from='client.? 172.21.15.98:0/906043716' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:13:01.967 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:01 smithi098 bash[79516]: audit 2023-09-23T19:13:01.329490+0000 mon.a (mon.0) 2249 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:13:01.967 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:01 smithi098 bash[76817]: cluster 2023-09-23T19:13:00.769455+0000 mgr.y (mgr.26054) 1665 : cluster 0 pgmap v1868: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:01.967 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:01 smithi098 bash[76817]: audit 2023-09-23T19:13:01.328719+0000 mon.c (mon.2) 166 : audit 1 from='client.? 172.21.15.98:0/906043716' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:13:01.967 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:01 smithi098 bash[76817]: audit 2023-09-23T19:13:01.329490+0000 mon.a (mon.0) 2249 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:13:02.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:01 smithi173 bash[59422]: cluster 2023-09-23T19:13:00.769455+0000 mgr.y (mgr.26054) 1665 : cluster 0 pgmap v1868: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:02.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:01 smithi173 bash[59422]: audit 2023-09-23T19:13:01.328719+0000 mon.c (mon.2) 166 : audit 1 from='client.? 172.21.15.98:0/906043716' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:13:02.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:01 smithi173 bash[59422]: audit 2023-09-23T19:13:01.329490+0000 mon.a (mon.0) 2249 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T19:13:02.688 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T19:13:02.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:02 smithi098 bash[79516]: cluster 2023-09-23T19:13:01.698286+0000 osd.5 (osd.5) 48 : cluster 3 failed to encode map e410 with expected crc 2023-09-23T19:13:02.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:02 smithi098 bash[79516]: audit 2023-09-23T19:13:01.698832+0000 mon.a (mon.0) 2250 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:13:02.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:02 smithi098 bash[79516]: cluster 2023-09-23T19:13:01.698839+0000 osd.7 (osd.7) 63 : cluster 3 failed to encode map e410 with expected crc 2023-09-23T19:13:02.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:02 smithi098 bash[79516]: cluster 2023-09-23T19:13:01.698940+0000 mon.a (mon.0) 2251 : cluster 0 osdmap e410: 8 total, 8 up, 8 in 2023-09-23T19:13:02.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:02 smithi098 bash[79516]: cluster 2023-09-23T19:13:01.701412+0000 osd.6 (osd.6) 394 : cluster 3 failed to encode map e410 with expected crc 2023-09-23T19:13:02.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:02 smithi098 bash[79516]: cluster 2023-09-23T19:13:01.703081+0000 osd.6 (osd.6) 395 : cluster 3 failed to encode map e410 with expected crc 2023-09-23T19:13:02.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:02 smithi098 bash[76817]: cluster 2023-09-23T19:13:01.698286+0000 osd.5 (osd.5) 48 : cluster 3 failed to encode map e410 with expected crc 2023-09-23T19:13:02.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:02 smithi098 bash[76817]: audit 2023-09-23T19:13:01.698832+0000 mon.a (mon.0) 2250 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:13:02.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:02 smithi098 bash[76817]: cluster 2023-09-23T19:13:01.698839+0000 osd.7 (osd.7) 63 : cluster 3 failed to encode map e410 with expected crc 2023-09-23T19:13:02.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:02 smithi098 bash[76817]: cluster 2023-09-23T19:13:01.698940+0000 mon.a (mon.0) 2251 : cluster 0 osdmap e410: 8 total, 8 up, 8 in 2023-09-23T19:13:02.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:02 smithi098 bash[76817]: cluster 2023-09-23T19:13:01.701412+0000 osd.6 (osd.6) 394 : cluster 3 failed to encode map e410 with expected crc 2023-09-23T19:13:02.946 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:02 smithi098 bash[76817]: cluster 2023-09-23T19:13:01.703081+0000 osd.6 (osd.6) 395 : cluster 3 failed to encode map e410 with expected crc 2023-09-23T19:13:03.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:02 smithi173 bash[59422]: cluster 2023-09-23T19:13:01.698286+0000 osd.5 (osd.5) 48 : cluster 3 failed to encode map e410 with expected crc 2023-09-23T19:13:03.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:02 smithi173 bash[59422]: audit 2023-09-23T19:13:01.698832+0000 mon.a (mon.0) 2250 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T19:13:03.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:02 smithi173 bash[59422]: cluster 2023-09-23T19:13:01.698839+0000 osd.7 (osd.7) 63 : cluster 3 failed to encode map e410 with expected crc 2023-09-23T19:13:03.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:02 smithi173 bash[59422]: cluster 2023-09-23T19:13:01.698940+0000 mon.a (mon.0) 2251 : cluster 0 osdmap e410: 8 total, 8 up, 8 in 2023-09-23T19:13:03.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:02 smithi173 bash[59422]: cluster 2023-09-23T19:13:01.701412+0000 osd.6 (osd.6) 394 : cluster 3 failed to encode map e410 with expected crc 2023-09-23T19:13:03.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:02 smithi173 bash[59422]: cluster 2023-09-23T19:13:01.703081+0000 osd.6 (osd.6) 395 : cluster 3 failed to encode map e410 with expected crc 2023-09-23T19:13:04.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:03 smithi173 bash[59422]: cluster 2023-09-23T19:13:02.709067+0000 mon.a (mon.0) 2252 : cluster 0 osdmap e411: 8 total, 8 up, 8 in 2023-09-23T19:13:04.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:03 smithi173 bash[59422]: cluster 2023-09-23T19:13:02.709128+0000 osd.6 (osd.6) 396 : cluster 3 failed to encode map e411 with expected crc 2023-09-23T19:13:04.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:03 smithi173 bash[59422]: cluster 2023-09-23T19:13:02.710564+0000 osd.4 (osd.4) 140 : cluster 3 failed to encode map e411 with expected crc 2023-09-23T19:13:04.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:03 smithi173 bash[59422]: cluster 2023-09-23T19:13:02.770095+0000 mgr.y (mgr.26054) 1666 : cluster 0 pgmap v1871: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:04.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:03 smithi098 bash[79516]: cluster 2023-09-23T19:13:02.709067+0000 mon.a (mon.0) 2252 : cluster 0 osdmap e411: 8 total, 8 up, 8 in 2023-09-23T19:13:04.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:03 smithi098 bash[79516]: cluster 2023-09-23T19:13:02.709128+0000 osd.6 (osd.6) 396 : cluster 3 failed to encode map e411 with expected crc 2023-09-23T19:13:04.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:03 smithi098 bash[79516]: cluster 2023-09-23T19:13:02.710564+0000 osd.4 (osd.4) 140 : cluster 3 failed to encode map e411 with expected crc 2023-09-23T19:13:04.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:03 smithi098 bash[79516]: cluster 2023-09-23T19:13:02.770095+0000 mgr.y (mgr.26054) 1666 : cluster 0 pgmap v1871: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:04.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:03 smithi098 bash[76817]: cluster 2023-09-23T19:13:02.709067+0000 mon.a (mon.0) 2252 : cluster 0 osdmap e411: 8 total, 8 up, 8 in 2023-09-23T19:13:04.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:03 smithi098 bash[76817]: cluster 2023-09-23T19:13:02.709128+0000 osd.6 (osd.6) 396 : cluster 3 failed to encode map e411 with expected crc 2023-09-23T19:13:04.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:03 smithi098 bash[76817]: cluster 2023-09-23T19:13:02.710564+0000 osd.4 (osd.4) 140 : cluster 3 failed to encode map e411 with expected crc 2023-09-23T19:13:04.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:03 smithi098 bash[76817]: cluster 2023-09-23T19:13:02.770095+0000 mgr.y (mgr.26054) 1666 : cluster 0 pgmap v1871: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:06.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:05 smithi098 bash[76817]: cluster 2023-09-23T19:13:04.770930+0000 mgr.y (mgr.26054) 1667 : cluster 0 pgmap v1872: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:06.087 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:05 smithi098 bash[79516]: cluster 2023-09-23T19:13:04.770930+0000 mgr.y (mgr.26054) 1667 : cluster 0 pgmap v1872: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:06.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:05 smithi173 bash[59422]: cluster 2023-09-23T19:13:04.770930+0000 mgr.y (mgr.26054) 1667 : cluster 0 pgmap v1872: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:06.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:13:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:13:06] "GET /metrics HTTP/1.1" 200 34166 "" "Prometheus/2.33.4" 2023-09-23T19:13:07.474 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:13:08.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:07 smithi098 bash[79516]: cluster 2023-09-23T19:13:06.771814+0000 mgr.y (mgr.26054) 1668 : cluster 0 pgmap v1873: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:08.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:07 smithi098 bash[76817]: cluster 2023-09-23T19:13:06.771814+0000 mgr.y (mgr.26054) 1668 : cluster 0 pgmap v1873: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:08.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:07 smithi173 bash[59422]: cluster 2023-09-23T19:13:06.771814+0000 mgr.y (mgr.26054) 1668 : cluster 0 pgmap v1873: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:10.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:09 smithi098 bash[76817]: cluster 2023-09-23T19:13:08.772652+0000 mgr.y (mgr.26054) 1669 : cluster 0 pgmap v1874: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:10.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:09 smithi098 bash[76817]: audit 2023-09-23T19:13:09.657656+0000 mon.a (mon.0) 2253 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:13:10.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:09 smithi098 bash[79516]: cluster 2023-09-23T19:13:08.772652+0000 mgr.y (mgr.26054) 1669 : cluster 0 pgmap v1874: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:10.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:09 smithi098 bash[79516]: audit 2023-09-23T19:13:09.657656+0000 mon.a (mon.0) 2253 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:13:10.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:09 smithi173 bash[59422]: cluster 2023-09-23T19:13:08.772652+0000 mgr.y (mgr.26054) 1669 : cluster 0 pgmap v1874: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:10.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:09 smithi173 bash[59422]: audit 2023-09-23T19:13:09.657656+0000 mon.a (mon.0) 2253 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:13:11.709 INFO:teuthology.orchestra.run.smithi098.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T19:13:11.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:11 smithi098 bash[79516]: cluster 2023-09-23T19:13:10.773360+0000 mgr.y (mgr.26054) 1670 : cluster 0 pgmap v1875: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:11.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:11 smithi098 bash[79516]: audit 2023-09-23T19:13:11.494073+0000 mon.a (mon.0) 2254 : audit 1 from='client.? 172.21.15.98:0/617152144' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:13:11.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:11 smithi098 bash[76817]: cluster 2023-09-23T19:13:10.773360+0000 mgr.y (mgr.26054) 1670 : cluster 0 pgmap v1875: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:11.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:11 smithi098 bash[76817]: audit 2023-09-23T19:13:11.494073+0000 mon.a (mon.0) 2254 : audit 1 from='client.? 172.21.15.98:0/617152144' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:13:12.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:11 smithi173 bash[59422]: cluster 2023-09-23T19:13:10.773360+0000 mgr.y (mgr.26054) 1670 : cluster 0 pgmap v1875: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:12.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:11 smithi173 bash[59422]: audit 2023-09-23T19:13:11.494073+0000 mon.a (mon.0) 2254 : audit 1 from='client.? 172.21.15.98:0/617152144' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:13:12.602 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-09-23T19:13:12.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:12 smithi098 bash[79516]: audit 2023-09-23T19:13:11.699446+0000 mon.a (mon.0) 2255 : audit 1 from='client.? 172.21.15.98:0/617152144' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:13:12.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:12 smithi098 bash[79516]: cluster 2023-09-23T19:13:11.699532+0000 mon.a (mon.0) 2256 : cluster 0 osdmap e412: 8 total, 8 up, 8 in 2023-09-23T19:13:12.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:12 smithi098 bash[79516]: cluster 2023-09-23T19:13:11.701261+0000 osd.7 (osd.7) 64 : cluster 3 failed to encode map e412 with expected crc 2023-09-23T19:13:12.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:12 smithi098 bash[79516]: cluster 2023-09-23T19:13:11.701538+0000 osd.6 (osd.6) 397 : cluster 3 failed to encode map e412 with expected crc 2023-09-23T19:13:12.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:12 smithi098 bash[79516]: cluster 2023-09-23T19:13:11.704162+0000 osd.4 (osd.4) 141 : cluster 3 failed to encode map e412 with expected crc 2023-09-23T19:13:12.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:12 smithi098 bash[76817]: audit 2023-09-23T19:13:11.699446+0000 mon.a (mon.0) 2255 : audit 1 from='client.? 172.21.15.98:0/617152144' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:13:12.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:12 smithi098 bash[76817]: cluster 2023-09-23T19:13:11.699532+0000 mon.a (mon.0) 2256 : cluster 0 osdmap e412: 8 total, 8 up, 8 in 2023-09-23T19:13:12.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:12 smithi098 bash[76817]: cluster 2023-09-23T19:13:11.701261+0000 osd.7 (osd.7) 64 : cluster 3 failed to encode map e412 with expected crc 2023-09-23T19:13:12.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:12 smithi098 bash[76817]: cluster 2023-09-23T19:13:11.701538+0000 osd.6 (osd.6) 397 : cluster 3 failed to encode map e412 with expected crc 2023-09-23T19:13:12.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:12 smithi098 bash[76817]: cluster 2023-09-23T19:13:11.704162+0000 osd.4 (osd.4) 141 : cluster 3 failed to encode map e412 with expected crc 2023-09-23T19:13:13.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:12 smithi173 bash[59422]: audit 2023-09-23T19:13:11.699446+0000 mon.a (mon.0) 2255 : audit 1 from='client.? 172.21.15.98:0/617152144' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:13:13.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:12 smithi173 bash[59422]: cluster 2023-09-23T19:13:11.699532+0000 mon.a (mon.0) 2256 : cluster 0 osdmap e412: 8 total, 8 up, 8 in 2023-09-23T19:13:13.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:12 smithi173 bash[59422]: cluster 2023-09-23T19:13:11.701261+0000 osd.7 (osd.7) 64 : cluster 3 failed to encode map e412 with expected crc 2023-09-23T19:13:13.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:12 smithi173 bash[59422]: cluster 2023-09-23T19:13:11.701538+0000 osd.6 (osd.6) 397 : cluster 3 failed to encode map e412 with expected crc 2023-09-23T19:13:13.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:12 smithi173 bash[59422]: cluster 2023-09-23T19:13:11.704162+0000 osd.4 (osd.4) 141 : cluster 3 failed to encode map e412 with expected crc 2023-09-23T19:13:14.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:13 smithi173 bash[59422]: cluster 2023-09-23T19:13:12.774105+0000 mgr.y (mgr.26054) 1671 : cluster 0 pgmap v1877: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:14.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:13 smithi098 bash[76817]: cluster 2023-09-23T19:13:12.774105+0000 mgr.y (mgr.26054) 1671 : cluster 0 pgmap v1877: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:14.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:13 smithi098 bash[79516]: cluster 2023-09-23T19:13:12.774105+0000 mgr.y (mgr.26054) 1671 : cluster 0 pgmap v1877: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:16.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:15 smithi098 bash[76817]: cluster 2023-09-23T19:13:14.774927+0000 mgr.y (mgr.26054) 1672 : cluster 0 pgmap v1878: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:16.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:15 smithi098 bash[79516]: cluster 2023-09-23T19:13:14.774927+0000 mgr.y (mgr.26054) 1672 : cluster 0 pgmap v1878: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:16.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:15 smithi173 bash[59422]: cluster 2023-09-23T19:13:14.774927+0000 mgr.y (mgr.26054) 1672 : cluster 0 pgmap v1878: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:16.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:13:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:13:16] "GET /metrics HTTP/1.1" 200 34168 "" "Prometheus/2.33.4" 2023-09-23T19:13:17.384 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:13:18.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:17 smithi098 bash[76817]: cluster 2023-09-23T19:13:16.775848+0000 mgr.y (mgr.26054) 1673 : cluster 0 pgmap v1879: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:18.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:17 smithi098 bash[79516]: cluster 2023-09-23T19:13:16.775848+0000 mgr.y (mgr.26054) 1673 : cluster 0 pgmap v1879: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:18.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:17 smithi173 bash[59422]: cluster 2023-09-23T19:13:16.775848+0000 mgr.y (mgr.26054) 1673 : cluster 0 pgmap v1879: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:20.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:19 smithi098 bash[76817]: cluster 2023-09-23T19:13:18.776897+0000 mgr.y (mgr.26054) 1674 : cluster 0 pgmap v1880: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:20.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:19 smithi098 bash[79516]: cluster 2023-09-23T19:13:18.776897+0000 mgr.y (mgr.26054) 1674 : cluster 0 pgmap v1880: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:20.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:19 smithi173 bash[59422]: cluster 2023-09-23T19:13:18.776897+0000 mgr.y (mgr.26054) 1674 : cluster 0 pgmap v1880: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:21.713 INFO:teuthology.orchestra.run.smithi098.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-09-23T19:13:22.014 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:21 smithi098 bash[76817]: cluster 2023-09-23T19:13:20.777579+0000 mgr.y (mgr.26054) 1675 : cluster 0 pgmap v1881: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:22.014 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:21 smithi098 bash[76817]: audit 2023-09-23T19:13:21.217639+0000 mon.a (mon.0) 2257 : audit 1 from='client.? 172.21.15.98:0/3983519524' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:13:22.014 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:21 smithi098 bash[79516]: cluster 2023-09-23T19:13:20.777579+0000 mgr.y (mgr.26054) 1675 : cluster 0 pgmap v1881: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:22.014 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:21 smithi098 bash[79516]: audit 2023-09-23T19:13:21.217639+0000 mon.a (mon.0) 2257 : audit 1 from='client.? 172.21.15.98:0/3983519524' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:13:22.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:21 smithi173 bash[59422]: cluster 2023-09-23T19:13:20.777579+0000 mgr.y (mgr.26054) 1675 : cluster 0 pgmap v1881: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:22.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:21 smithi173 bash[59422]: audit 2023-09-23T19:13:21.217639+0000 mon.a (mon.0) 2257 : audit 1 from='client.? 172.21.15.98:0/3983519524' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T19:13:22.786 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T19:13:23.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:22 smithi173 bash[59422]: audit 2023-09-23T19:13:21.702619+0000 mon.a (mon.0) 2258 : audit 1 from='client.? 172.21.15.98:0/3983519524' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:13:23.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:22 smithi173 bash[59422]: cluster 2023-09-23T19:13:21.702709+0000 mon.a (mon.0) 2259 : cluster 0 osdmap e413: 8 total, 8 up, 8 in 2023-09-23T19:13:23.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:22 smithi173 bash[59422]: cluster 2023-09-23T19:13:21.704957+0000 osd.6 (osd.6) 398 : cluster 3 failed to encode map e413 with expected crc 2023-09-23T19:13:23.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:22 smithi173 bash[59422]: cluster 2023-09-23T19:13:21.705052+0000 osd.4 (osd.4) 142 : cluster 3 failed to encode map e413 with expected crc 2023-09-23T19:13:23.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:22 smithi173 bash[59422]: cluster 2023-09-23T19:13:21.872412+0000 osd.5 (osd.5) 49 : cluster 3 failed to encode map e413 with expected crc 2023-09-23T19:13:23.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:22 smithi098 bash[76817]: audit 2023-09-23T19:13:21.702619+0000 mon.a (mon.0) 2258 : audit 1 from='client.? 172.21.15.98:0/3983519524' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:13:23.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:22 smithi098 bash[76817]: cluster 2023-09-23T19:13:21.702709+0000 mon.a (mon.0) 2259 : cluster 0 osdmap e413: 8 total, 8 up, 8 in 2023-09-23T19:13:23.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:22 smithi098 bash[76817]: cluster 2023-09-23T19:13:21.704957+0000 osd.6 (osd.6) 398 : cluster 3 failed to encode map e413 with expected crc 2023-09-23T19:13:23.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:22 smithi098 bash[76817]: cluster 2023-09-23T19:13:21.705052+0000 osd.4 (osd.4) 142 : cluster 3 failed to encode map e413 with expected crc 2023-09-23T19:13:23.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:22 smithi098 bash[76817]: cluster 2023-09-23T19:13:21.872412+0000 osd.5 (osd.5) 49 : cluster 3 failed to encode map e413 with expected crc 2023-09-23T19:13:23.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:22 smithi098 bash[79516]: audit 2023-09-23T19:13:21.702619+0000 mon.a (mon.0) 2258 : audit 1 from='client.? 172.21.15.98:0/3983519524' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T19:13:23.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:22 smithi098 bash[79516]: cluster 2023-09-23T19:13:21.702709+0000 mon.a (mon.0) 2259 : cluster 0 osdmap e413: 8 total, 8 up, 8 in 2023-09-23T19:13:23.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:22 smithi098 bash[79516]: cluster 2023-09-23T19:13:21.704957+0000 osd.6 (osd.6) 398 : cluster 3 failed to encode map e413 with expected crc 2023-09-23T19:13:23.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:22 smithi098 bash[79516]: cluster 2023-09-23T19:13:21.705052+0000 osd.4 (osd.4) 142 : cluster 3 failed to encode map e413 with expected crc 2023-09-23T19:13:23.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:22 smithi098 bash[79516]: cluster 2023-09-23T19:13:21.872412+0000 osd.5 (osd.5) 49 : cluster 3 failed to encode map e413 with expected crc 2023-09-23T19:13:24.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:23 smithi173 bash[59422]: cluster 2023-09-23T19:13:22.778349+0000 mgr.y (mgr.26054) 1676 : cluster 0 pgmap v1883: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:24.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:23 smithi098 bash[76817]: cluster 2023-09-23T19:13:22.778349+0000 mgr.y (mgr.26054) 1676 : cluster 0 pgmap v1883: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:24.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:23 smithi098 bash[79516]: cluster 2023-09-23T19:13:22.778349+0000 mgr.y (mgr.26054) 1676 : cluster 0 pgmap v1883: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:25.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:24 smithi173 bash[59422]: audit 2023-09-23T19:13:24.658124+0000 mon.a (mon.0) 2260 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:13:25.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:24 smithi098 bash[79516]: audit 2023-09-23T19:13:24.658124+0000 mon.a (mon.0) 2260 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:13:25.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:24 smithi098 bash[76817]: audit 2023-09-23T19:13:24.658124+0000 mon.a (mon.0) 2260 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:13:26.030 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:25 smithi173 bash[59422]: cluster 2023-09-23T19:13:24.779151+0000 mgr.y (mgr.26054) 1677 : cluster 0 pgmap v1884: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:26.088 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:25 smithi098 bash[76817]: cluster 2023-09-23T19:13:24.779151+0000 mgr.y (mgr.26054) 1677 : cluster 0 pgmap v1884: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:26.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:25 smithi098 bash[79516]: cluster 2023-09-23T19:13:24.779151+0000 mgr.y (mgr.26054) 1677 : cluster 0 pgmap v1884: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:26.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:13:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:13:26] "GET /metrics HTTP/1.1" 200 34165 "" "Prometheus/2.33.4" 2023-09-23T19:13:27.558 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:13:27.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:27 smithi098 bash[79516]: cluster 2023-09-23T19:13:26.780043+0000 mgr.y (mgr.26054) 1678 : cluster 0 pgmap v1885: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:27.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:27 smithi098 bash[76817]: cluster 2023-09-23T19:13:26.780043+0000 mgr.y (mgr.26054) 1678 : cluster 0 pgmap v1885: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:28.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:27 smithi173 bash[59422]: cluster 2023-09-23T19:13:26.780043+0000 mgr.y (mgr.26054) 1678 : cluster 0 pgmap v1885: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:30.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:29 smithi098 bash[76817]: cluster 2023-09-23T19:13:28.780732+0000 mgr.y (mgr.26054) 1679 : cluster 0 pgmap v1886: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:30.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:29 smithi098 bash[79516]: cluster 2023-09-23T19:13:28.780732+0000 mgr.y (mgr.26054) 1679 : cluster 0 pgmap v1886: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:30.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:29 smithi173 bash[59422]: cluster 2023-09-23T19:13:28.780732+0000 mgr.y (mgr.26054) 1679 : cluster 0 pgmap v1886: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:31.715 INFO:teuthology.orchestra.run.smithi098.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T19:13:31.768 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:31 smithi098 bash[76817]: cluster 2023-09-23T19:13:30.781563+0000 mgr.y (mgr.26054) 1680 : cluster 0 pgmap v1887: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:31.768 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:31 smithi098 bash[76817]: audit 2023-09-23T19:13:31.452787+0000 mon.a (mon.0) 2261 : audit 1 from='client.? 172.21.15.98:0/615529511' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:13:31.769 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:31 smithi098 bash[79516]: cluster 2023-09-23T19:13:30.781563+0000 mgr.y (mgr.26054) 1680 : cluster 0 pgmap v1887: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:31.769 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:31 smithi098 bash[79516]: audit 2023-09-23T19:13:31.452787+0000 mon.a (mon.0) 2261 : audit 1 from='client.? 172.21.15.98:0/615529511' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:13:32.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:31 smithi173 bash[59422]: cluster 2023-09-23T19:13:30.781563+0000 mgr.y (mgr.26054) 1680 : cluster 0 pgmap v1887: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:32.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:31 smithi173 bash[59422]: audit 2023-09-23T19:13:31.452787+0000 mon.a (mon.0) 2261 : audit 1 from='client.? 172.21.15.98:0/615529511' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:13:32.732 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-09-23T19:13:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:32 smithi173 bash[59422]: audit 2023-09-23T19:13:31.704394+0000 mon.a (mon.0) 2262 : audit 1 from='client.? 172.21.15.98:0/615529511' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:13:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:32 smithi173 bash[59422]: cluster 2023-09-23T19:13:31.704482+0000 mon.a (mon.0) 2263 : cluster 0 osdmap e414: 8 total, 8 up, 8 in 2023-09-23T19:13:33.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:32 smithi173 bash[59422]: cluster 2023-09-23T19:13:32.402654+0000 osd.5 (osd.5) 50 : cluster 3 failed to encode map e414 with expected crc 2023-09-23T19:13:33.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:32 smithi173 bash[59422]: cluster 2023-09-23T19:13:32.405281+0000 osd.5 (osd.5) 51 : cluster 3 failed to encode map e414 with expected crc 2023-09-23T19:13:33.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:32 smithi173 bash[59422]: cluster 2023-09-23T19:13:32.433672+0000 osd.7 (osd.7) 65 : cluster 3 failed to encode map e414 with expected crc 2023-09-23T19:13:33.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:32 smithi098 bash[79516]: audit 2023-09-23T19:13:31.704394+0000 mon.a (mon.0) 2262 : audit 1 from='client.? 172.21.15.98:0/615529511' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:13:33.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:32 smithi098 bash[79516]: cluster 2023-09-23T19:13:31.704482+0000 mon.a (mon.0) 2263 : cluster 0 osdmap e414: 8 total, 8 up, 8 in 2023-09-23T19:13:33.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:32 smithi098 bash[79516]: cluster 2023-09-23T19:13:32.402654+0000 osd.5 (osd.5) 50 : cluster 3 failed to encode map e414 with expected crc 2023-09-23T19:13:33.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:32 smithi098 bash[79516]: cluster 2023-09-23T19:13:32.405281+0000 osd.5 (osd.5) 51 : cluster 3 failed to encode map e414 with expected crc 2023-09-23T19:13:33.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:32 smithi098 bash[79516]: cluster 2023-09-23T19:13:32.433672+0000 osd.7 (osd.7) 65 : cluster 3 failed to encode map e414 with expected crc 2023-09-23T19:13:33.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:32 smithi098 bash[76817]: audit 2023-09-23T19:13:31.704394+0000 mon.a (mon.0) 2262 : audit 1 from='client.? 172.21.15.98:0/615529511' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:13:33.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:32 smithi098 bash[76817]: cluster 2023-09-23T19:13:31.704482+0000 mon.a (mon.0) 2263 : cluster 0 osdmap e414: 8 total, 8 up, 8 in 2023-09-23T19:13:33.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:32 smithi098 bash[76817]: cluster 2023-09-23T19:13:32.402654+0000 osd.5 (osd.5) 50 : cluster 3 failed to encode map e414 with expected crc 2023-09-23T19:13:33.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:32 smithi098 bash[76817]: cluster 2023-09-23T19:13:32.405281+0000 osd.5 (osd.5) 51 : cluster 3 failed to encode map e414 with expected crc 2023-09-23T19:13:33.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:32 smithi098 bash[76817]: cluster 2023-09-23T19:13:32.433672+0000 osd.7 (osd.7) 65 : cluster 3 failed to encode map e414 with expected crc 2023-09-23T19:13:34.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:33 smithi173 bash[59422]: cluster 2023-09-23T19:13:32.284269+0000 osd.4 (osd.4) 143 : cluster 3 failed to encode map e414 with expected crc 2023-09-23T19:13:34.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:33 smithi173 bash[59422]: cluster 2023-09-23T19:13:32.287125+0000 osd.4 (osd.4) 144 : cluster 3 failed to encode map e414 with expected crc 2023-09-23T19:13:34.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:33 smithi173 bash[59422]: cluster 2023-09-23T19:13:32.290079+0000 osd.4 (osd.4) 145 : cluster 3 failed to encode map e414 with expected crc 2023-09-23T19:13:34.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:33 smithi173 bash[59422]: cluster 2023-09-23T19:13:32.434016+0000 osd.6 (osd.6) 399 : cluster 3 failed to encode map e414 with expected crc 2023-09-23T19:13:34.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:33 smithi173 bash[59422]: cluster 2023-09-23T19:13:32.782282+0000 mgr.y (mgr.26054) 1681 : cluster 0 pgmap v1889: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:34.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:33 smithi098 bash[79516]: cluster 2023-09-23T19:13:32.284269+0000 osd.4 (osd.4) 143 : cluster 3 failed to encode map e414 with expected crc 2023-09-23T19:13:34.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:33 smithi098 bash[79516]: cluster 2023-09-23T19:13:32.287125+0000 osd.4 (osd.4) 144 : cluster 3 failed to encode map e414 with expected crc 2023-09-23T19:13:34.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:33 smithi098 bash[79516]: cluster 2023-09-23T19:13:32.290079+0000 osd.4 (osd.4) 145 : cluster 3 failed to encode map e414 with expected crc 2023-09-23T19:13:34.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:33 smithi098 bash[79516]: cluster 2023-09-23T19:13:32.434016+0000 osd.6 (osd.6) 399 : cluster 3 failed to encode map e414 with expected crc 2023-09-23T19:13:34.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:33 smithi098 bash[79516]: cluster 2023-09-23T19:13:32.782282+0000 mgr.y (mgr.26054) 1681 : cluster 0 pgmap v1889: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:34.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:33 smithi098 bash[76817]: cluster 2023-09-23T19:13:32.284269+0000 osd.4 (osd.4) 143 : cluster 3 failed to encode map e414 with expected crc 2023-09-23T19:13:34.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:33 smithi098 bash[76817]: cluster 2023-09-23T19:13:32.287125+0000 osd.4 (osd.4) 144 : cluster 3 failed to encode map e414 with expected crc 2023-09-23T19:13:34.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:33 smithi098 bash[76817]: cluster 2023-09-23T19:13:32.290079+0000 osd.4 (osd.4) 145 : cluster 3 failed to encode map e414 with expected crc 2023-09-23T19:13:34.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:33 smithi098 bash[76817]: cluster 2023-09-23T19:13:32.434016+0000 osd.6 (osd.6) 399 : cluster 3 failed to encode map e414 with expected crc 2023-09-23T19:13:34.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:33 smithi098 bash[76817]: cluster 2023-09-23T19:13:32.782282+0000 mgr.y (mgr.26054) 1681 : cluster 0 pgmap v1889: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:36.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:35 smithi098 bash[76817]: cluster 2023-09-23T19:13:34.783094+0000 mgr.y (mgr.26054) 1682 : cluster 0 pgmap v1890: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:36.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:35 smithi098 bash[79516]: cluster 2023-09-23T19:13:34.783094+0000 mgr.y (mgr.26054) 1682 : cluster 0 pgmap v1890: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:36.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:13:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:13:36] "GET /metrics HTTP/1.1" 200 34165 "" "Prometheus/2.33.4" 2023-09-23T19:13:36.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:35 smithi173 bash[59422]: cluster 2023-09-23T19:13:34.783094+0000 mgr.y (mgr.26054) 1682 : cluster 0 pgmap v1890: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:37.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:36 smithi098 bash[76817]: audit 2023-09-23T19:13:36.501641+0000 mon.a (mon.0) 2264 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:13:37.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:36 smithi098 bash[79516]: audit 2023-09-23T19:13:36.501641+0000 mon.a (mon.0) 2264 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:13:37.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:36 smithi173 bash[59422]: audit 2023-09-23T19:13:36.501641+0000 mon.a (mon.0) 2264 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:13:37.507 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:13:38.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:37 smithi098 bash[76817]: cluster 2023-09-23T19:13:36.783748+0000 mgr.y (mgr.26054) 1683 : cluster 0 pgmap v1891: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:38.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:37 smithi098 bash[79516]: cluster 2023-09-23T19:13:36.783748+0000 mgr.y (mgr.26054) 1683 : cluster 0 pgmap v1891: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:38.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:37 smithi173 bash[59422]: cluster 2023-09-23T19:13:36.783748+0000 mgr.y (mgr.26054) 1683 : cluster 0 pgmap v1891: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:40.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:39 smithi098 bash[76817]: cluster 2023-09-23T19:13:38.784271+0000 mgr.y (mgr.26054) 1684 : cluster 0 pgmap v1892: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:40.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:39 smithi098 bash[76817]: audit 2023-09-23T19:13:39.658450+0000 mon.a (mon.0) 2265 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:13:40.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:39 smithi098 bash[79516]: cluster 2023-09-23T19:13:38.784271+0000 mgr.y (mgr.26054) 1684 : cluster 0 pgmap v1892: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:40.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:39 smithi098 bash[79516]: audit 2023-09-23T19:13:39.658450+0000 mon.a (mon.0) 2265 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:13:40.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:39 smithi173 bash[59422]: cluster 2023-09-23T19:13:38.784271+0000 mgr.y (mgr.26054) 1684 : cluster 0 pgmap v1892: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:40.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:39 smithi173 bash[59422]: audit 2023-09-23T19:13:39.658450+0000 mon.a (mon.0) 2265 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:13:41.715 INFO:teuthology.orchestra.run.smithi098.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-09-23T19:13:41.969 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:41 smithi098 bash[79516]: cluster 2023-09-23T19:13:40.784922+0000 mgr.y (mgr.26054) 1685 : cluster 0 pgmap v1893: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:41.970 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:41 smithi098 bash[79516]: audit 2023-09-23T19:13:41.387047+0000 mon.a (mon.0) 2266 : audit 1 from='client.? 172.21.15.98:0/3168957097' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:13:41.970 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:41 smithi098 bash[79516]: audit 2023-09-23T19:13:41.655344+0000 mon.a (mon.0) 2267 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:13:41.970 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:41 smithi098 bash[79516]: audit 2023-09-23T19:13:41.662557+0000 mon.a (mon.0) 2268 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:13:41.970 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:41 smithi098 bash[76817]: cluster 2023-09-23T19:13:40.784922+0000 mgr.y (mgr.26054) 1685 : cluster 0 pgmap v1893: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:41.970 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:41 smithi098 bash[76817]: audit 2023-09-23T19:13:41.387047+0000 mon.a (mon.0) 2266 : audit 1 from='client.? 172.21.15.98:0/3168957097' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:13:41.970 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:41 smithi098 bash[76817]: audit 2023-09-23T19:13:41.655344+0000 mon.a (mon.0) 2267 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:13:41.970 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:41 smithi098 bash[76817]: audit 2023-09-23T19:13:41.662557+0000 mon.a (mon.0) 2268 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:13:42.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:41 smithi173 bash[59422]: cluster 2023-09-23T19:13:40.784922+0000 mgr.y (mgr.26054) 1685 : cluster 0 pgmap v1893: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:42.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:41 smithi173 bash[59422]: audit 2023-09-23T19:13:41.387047+0000 mon.a (mon.0) 2266 : audit 1 from='client.? 172.21.15.98:0/3168957097' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T19:13:42.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:41 smithi173 bash[59422]: audit 2023-09-23T19:13:41.655344+0000 mon.a (mon.0) 2267 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:13:42.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:41 smithi173 bash[59422]: audit 2023-09-23T19:13:41.662557+0000 mon.a (mon.0) 2268 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:13:42.650 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T19:13:42.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:42 smithi098 bash[76817]: audit 2023-09-23T19:13:41.707099+0000 mon.a (mon.0) 2269 : audit 1 from='client.? 172.21.15.98:0/3168957097' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:13:42.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:42 smithi098 bash[76817]: cluster 2023-09-23T19:13:41.707182+0000 mon.a (mon.0) 2270 : cluster 0 osdmap e415: 8 total, 8 up, 8 in 2023-09-23T19:13:42.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:42 smithi098 bash[76817]: cluster 2023-09-23T19:13:42.387157+0000 osd.7 (osd.7) 66 : cluster 3 failed to encode map e415 with expected crc 2023-09-23T19:13:42.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:42 smithi098 bash[76817]: cluster 2023-09-23T19:13:42.475627+0000 osd.5 (osd.5) 52 : cluster 3 failed to encode map e415 with expected crc 2023-09-23T19:13:42.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:42 smithi098 bash[79516]: audit 2023-09-23T19:13:41.707099+0000 mon.a (mon.0) 2269 : audit 1 from='client.? 172.21.15.98:0/3168957097' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:13:42.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:42 smithi098 bash[79516]: cluster 2023-09-23T19:13:41.707182+0000 mon.a (mon.0) 2270 : cluster 0 osdmap e415: 8 total, 8 up, 8 in 2023-09-23T19:13:42.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:42 smithi098 bash[79516]: cluster 2023-09-23T19:13:42.387157+0000 osd.7 (osd.7) 66 : cluster 3 failed to encode map e415 with expected crc 2023-09-23T19:13:42.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:42 smithi098 bash[79516]: cluster 2023-09-23T19:13:42.475627+0000 osd.5 (osd.5) 52 : cluster 3 failed to encode map e415 with expected crc 2023-09-23T19:13:43.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:42 smithi173 bash[59422]: audit 2023-09-23T19:13:41.707099+0000 mon.a (mon.0) 2269 : audit 1 from='client.? 172.21.15.98:0/3168957097' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T19:13:43.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:42 smithi173 bash[59422]: cluster 2023-09-23T19:13:41.707182+0000 mon.a (mon.0) 2270 : cluster 0 osdmap e415: 8 total, 8 up, 8 in 2023-09-23T19:13:43.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:42 smithi173 bash[59422]: cluster 2023-09-23T19:13:42.387157+0000 osd.7 (osd.7) 66 : cluster 3 failed to encode map e415 with expected crc 2023-09-23T19:13:43.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:42 smithi173 bash[59422]: cluster 2023-09-23T19:13:42.475627+0000 osd.5 (osd.5) 52 : cluster 3 failed to encode map e415 with expected crc 2023-09-23T19:13:44.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:43 smithi173 bash[59422]: cluster 2023-09-23T19:13:42.263224+0000 osd.6 (osd.6) 400 : cluster 3 failed to encode map e415 with expected crc 2023-09-23T19:13:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:43 smithi173 bash[59422]: cluster 2023-09-23T19:13:42.644575+0000 osd.4 (osd.4) 146 : cluster 3 failed to encode map e415 with expected crc 2023-09-23T19:13:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:43 smithi173 bash[59422]: audit 2023-09-23T19:13:42.752380+0000 mon.a (mon.0) 2271 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:13:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:43 smithi173 bash[59422]: audit 2023-09-23T19:13:42.759151+0000 mon.a (mon.0) 2272 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:13:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:43 smithi173 bash[59422]: cluster 2023-09-23T19:13:42.785622+0000 mgr.y (mgr.26054) 1686 : cluster 0 pgmap v1895: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:43 smithi173 bash[59422]: audit 2023-09-23T19:13:43.231539+0000 mon.a (mon.0) 2273 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:13:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:43 smithi173 bash[59422]: audit 2023-09-23T19:13:43.233479+0000 mon.a (mon.0) 2274 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:13:44.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:43 smithi173 bash[59422]: audit 2023-09-23T19:13:43.242761+0000 mon.a (mon.0) 2275 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:13:44.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:43 smithi098 bash[76817]: cluster 2023-09-23T19:13:42.263224+0000 osd.6 (osd.6) 400 : cluster 3 failed to encode map e415 with expected crc 2023-09-23T19:13:44.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:43 smithi098 bash[76817]: cluster 2023-09-23T19:13:42.644575+0000 osd.4 (osd.4) 146 : cluster 3 failed to encode map e415 with expected crc 2023-09-23T19:13:44.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:43 smithi098 bash[76817]: audit 2023-09-23T19:13:42.752380+0000 mon.a (mon.0) 2271 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:13:44.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:43 smithi098 bash[76817]: audit 2023-09-23T19:13:42.759151+0000 mon.a (mon.0) 2272 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:13:44.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:43 smithi098 bash[76817]: cluster 2023-09-23T19:13:42.785622+0000 mgr.y (mgr.26054) 1686 : cluster 0 pgmap v1895: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:44.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:43 smithi098 bash[76817]: audit 2023-09-23T19:13:43.231539+0000 mon.a (mon.0) 2273 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:13:44.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:43 smithi098 bash[76817]: audit 2023-09-23T19:13:43.233479+0000 mon.a (mon.0) 2274 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:13:44.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:43 smithi098 bash[76817]: audit 2023-09-23T19:13:43.242761+0000 mon.a (mon.0) 2275 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:13:44.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:43 smithi098 bash[79516]: cluster 2023-09-23T19:13:42.263224+0000 osd.6 (osd.6) 400 : cluster 3 failed to encode map e415 with expected crc 2023-09-23T19:13:44.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:43 smithi098 bash[79516]: cluster 2023-09-23T19:13:42.644575+0000 osd.4 (osd.4) 146 : cluster 3 failed to encode map e415 with expected crc 2023-09-23T19:13:44.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:43 smithi098 bash[79516]: audit 2023-09-23T19:13:42.752380+0000 mon.a (mon.0) 2271 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:13:44.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:43 smithi098 bash[79516]: audit 2023-09-23T19:13:42.759151+0000 mon.a (mon.0) 2272 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:13:44.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:43 smithi098 bash[79516]: cluster 2023-09-23T19:13:42.785622+0000 mgr.y (mgr.26054) 1686 : cluster 0 pgmap v1895: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:44.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:43 smithi098 bash[79516]: audit 2023-09-23T19:13:43.231539+0000 mon.a (mon.0) 2273 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:13:44.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:43 smithi098 bash[79516]: audit 2023-09-23T19:13:43.233479+0000 mon.a (mon.0) 2274 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:13:44.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:43 smithi098 bash[79516]: audit 2023-09-23T19:13:43.242761+0000 mon.a (mon.0) 2275 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:13:46.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:13:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:13:46] "GET /metrics HTTP/1.1" 200 34166 "" "Prometheus/2.33.4" 2023-09-23T19:13:46.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:45 smithi098 bash[79516]: cluster 2023-09-23T19:13:44.786459+0000 mgr.y (mgr.26054) 1687 : cluster 0 pgmap v1896: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:46.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:45 smithi098 bash[76817]: cluster 2023-09-23T19:13:44.786459+0000 mgr.y (mgr.26054) 1687 : cluster 0 pgmap v1896: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:46.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:45 smithi173 bash[59422]: cluster 2023-09-23T19:13:44.786459+0000 mgr.y (mgr.26054) 1687 : cluster 0 pgmap v1896: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:47.432 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:13:48.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:47 smithi098 bash[79516]: cluster 2023-09-23T19:13:46.787269+0000 mgr.y (mgr.26054) 1688 : cluster 0 pgmap v1897: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:48.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:47 smithi098 bash[76817]: cluster 2023-09-23T19:13:46.787269+0000 mgr.y (mgr.26054) 1688 : cluster 0 pgmap v1897: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:48.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:47 smithi173 bash[59422]: cluster 2023-09-23T19:13:46.787269+0000 mgr.y (mgr.26054) 1688 : cluster 0 pgmap v1897: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:50.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:49 smithi098 bash[79516]: cluster 2023-09-23T19:13:48.787984+0000 mgr.y (mgr.26054) 1689 : cluster 0 pgmap v1898: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:50.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:49 smithi098 bash[76817]: cluster 2023-09-23T19:13:48.787984+0000 mgr.y (mgr.26054) 1689 : cluster 0 pgmap v1898: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:50.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:49 smithi173 bash[59422]: cluster 2023-09-23T19:13:48.787984+0000 mgr.y (mgr.26054) 1689 : cluster 0 pgmap v1898: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:51.720 INFO:teuthology.orchestra.run.smithi098.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T19:13:52.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:51 smithi173 bash[59422]: cluster 2023-09-23T19:13:50.788906+0000 mgr.y (mgr.26054) 1690 : cluster 0 pgmap v1899: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:52.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:51 smithi173 bash[59422]: audit 2023-09-23T19:13:51.369394+0000 mon.c (mon.2) 167 : audit 1 from='client.? 172.21.15.98:0/73085385' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:13:52.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:51 smithi173 bash[59422]: audit 2023-09-23T19:13:51.370063+0000 mon.a (mon.0) 2276 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:13:52.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:51 smithi098 bash[76817]: cluster 2023-09-23T19:13:50.788906+0000 mgr.y (mgr.26054) 1690 : cluster 0 pgmap v1899: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:52.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:51 smithi098 bash[76817]: audit 2023-09-23T19:13:51.369394+0000 mon.c (mon.2) 167 : audit 1 from='client.? 172.21.15.98:0/73085385' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:13:52.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:51 smithi098 bash[76817]: audit 2023-09-23T19:13:51.370063+0000 mon.a (mon.0) 2276 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:13:52.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:51 smithi098 bash[79516]: cluster 2023-09-23T19:13:50.788906+0000 mgr.y (mgr.26054) 1690 : cluster 0 pgmap v1899: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:52.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:51 smithi098 bash[79516]: audit 2023-09-23T19:13:51.369394+0000 mon.c (mon.2) 167 : audit 1 from='client.? 172.21.15.98:0/73085385' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:13:52.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:51 smithi098 bash[79516]: audit 2023-09-23T19:13:51.370063+0000 mon.a (mon.0) 2276 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:13:52.669 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-09-23T19:13:52.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:52 smithi098 bash[76817]: audit 2023-09-23T19:13:51.711207+0000 mon.a (mon.0) 2277 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:13:52.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:52 smithi098 bash[76817]: cluster 2023-09-23T19:13:51.711283+0000 mon.a (mon.0) 2278 : cluster 0 osdmap e416: 8 total, 8 up, 8 in 2023-09-23T19:13:52.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:52 smithi098 bash[76817]: cluster 2023-09-23T19:13:51.712233+0000 osd.6 (osd.6) 401 : cluster 3 failed to encode map e416 with expected crc 2023-09-23T19:13:52.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:52 smithi098 bash[76817]: cluster 2023-09-23T19:13:51.712278+0000 osd.4 (osd.4) 147 : cluster 3 failed to encode map e416 with expected crc 2023-09-23T19:13:52.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:52 smithi098 bash[79516]: audit 2023-09-23T19:13:51.711207+0000 mon.a (mon.0) 2277 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:13:52.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:52 smithi098 bash[79516]: cluster 2023-09-23T19:13:51.711283+0000 mon.a (mon.0) 2278 : cluster 0 osdmap e416: 8 total, 8 up, 8 in 2023-09-23T19:13:52.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:52 smithi098 bash[79516]: cluster 2023-09-23T19:13:51.712233+0000 osd.6 (osd.6) 401 : cluster 3 failed to encode map e416 with expected crc 2023-09-23T19:13:52.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:52 smithi098 bash[79516]: cluster 2023-09-23T19:13:51.712278+0000 osd.4 (osd.4) 147 : cluster 3 failed to encode map e416 with expected crc 2023-09-23T19:13:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:52 smithi173 bash[59422]: audit 2023-09-23T19:13:51.711207+0000 mon.a (mon.0) 2277 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:13:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:52 smithi173 bash[59422]: cluster 2023-09-23T19:13:51.711283+0000 mon.a (mon.0) 2278 : cluster 0 osdmap e416: 8 total, 8 up, 8 in 2023-09-23T19:13:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:52 smithi173 bash[59422]: cluster 2023-09-23T19:13:51.712233+0000 osd.6 (osd.6) 401 : cluster 3 failed to encode map e416 with expected crc 2023-09-23T19:13:53.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:52 smithi173 bash[59422]: cluster 2023-09-23T19:13:51.712278+0000 osd.4 (osd.4) 147 : cluster 3 failed to encode map e416 with expected crc 2023-09-23T19:13:54.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:53 smithi173 bash[59422]: cluster 2023-09-23T19:13:52.789588+0000 mgr.y (mgr.26054) 1691 : cluster 0 pgmap v1901: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:54.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:53 smithi098 bash[79516]: cluster 2023-09-23T19:13:52.789588+0000 mgr.y (mgr.26054) 1691 : cluster 0 pgmap v1901: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:54.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:53 smithi098 bash[76817]: cluster 2023-09-23T19:13:52.789588+0000 mgr.y (mgr.26054) 1691 : cluster 0 pgmap v1901: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:55.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:54 smithi173 bash[59422]: audit 2023-09-23T19:13:54.658774+0000 mon.a (mon.0) 2279 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:13:55.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:54 smithi098 bash[76817]: audit 2023-09-23T19:13:54.658774+0000 mon.a (mon.0) 2279 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:13:55.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:54 smithi098 bash[79516]: audit 2023-09-23T19:13:54.658774+0000 mon.a (mon.0) 2279 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:13:56.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:55 smithi173 bash[59422]: cluster 2023-09-23T19:13:54.790402+0000 mgr.y (mgr.26054) 1692 : cluster 0 pgmap v1902: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:56.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:55 smithi098 bash[76817]: cluster 2023-09-23T19:13:54.790402+0000 mgr.y (mgr.26054) 1692 : cluster 0 pgmap v1902: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:56.087 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:55 smithi098 bash[79516]: cluster 2023-09-23T19:13:54.790402+0000 mgr.y (mgr.26054) 1692 : cluster 0 pgmap v1902: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:56.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:13:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:13:56] "GET /metrics HTTP/1.1" 200 34163 "" "Prometheus/2.33.4" 2023-09-23T19:13:57.443 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:13:58.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:57 smithi098 bash[76817]: cluster 2023-09-23T19:13:56.791274+0000 mgr.y (mgr.26054) 1693 : cluster 0 pgmap v1903: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:58.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:57 smithi098 bash[79516]: cluster 2023-09-23T19:13:56.791274+0000 mgr.y (mgr.26054) 1693 : cluster 0 pgmap v1903: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:13:58.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:57 smithi173 bash[59422]: cluster 2023-09-23T19:13:56.791274+0000 mgr.y (mgr.26054) 1693 : cluster 0 pgmap v1903: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:00.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:13:59 smithi098 bash[79516]: cluster 2023-09-23T19:13:58.791963+0000 mgr.y (mgr.26054) 1694 : cluster 0 pgmap v1904: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:00.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:13:59 smithi098 bash[76817]: cluster 2023-09-23T19:13:58.791963+0000 mgr.y (mgr.26054) 1694 : cluster 0 pgmap v1904: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:00.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:13:59 smithi173 bash[59422]: cluster 2023-09-23T19:13:58.791963+0000 mgr.y (mgr.26054) 1694 : cluster 0 pgmap v1904: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:01.723 INFO:teuthology.orchestra.run.smithi098.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-09-23T19:14:01.975 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:01 smithi098 bash[79516]: cluster 2023-09-23T19:14:00.792881+0000 mgr.y (mgr.26054) 1695 : cluster 0 pgmap v1905: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:01.975 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:01 smithi098 bash[79516]: audit 2023-09-23T19:14:01.217290+0000 mon.a (mon.0) 2280 : audit 1 from='client.? 172.21.15.98:0/3749933927' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:14:01.975 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:01 smithi098 bash[76817]: cluster 2023-09-23T19:14:00.792881+0000 mgr.y (mgr.26054) 1695 : cluster 0 pgmap v1905: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:01.976 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:01 smithi098 bash[76817]: audit 2023-09-23T19:14:01.217290+0000 mon.a (mon.0) 2280 : audit 1 from='client.? 172.21.15.98:0/3749933927' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:14:02.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:01 smithi173 bash[59422]: cluster 2023-09-23T19:14:00.792881+0000 mgr.y (mgr.26054) 1695 : cluster 0 pgmap v1905: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:02.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:01 smithi173 bash[59422]: audit 2023-09-23T19:14:01.217290+0000 mon.a (mon.0) 2280 : audit 1 from='client.? 172.21.15.98:0/3749933927' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T19:14:02.714 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T19:14:03.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:02 smithi173 bash[59422]: audit 2023-09-23T19:14:01.712909+0000 mon.a (mon.0) 2281 : audit 1 from='client.? 172.21.15.98:0/3749933927' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:14:03.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:02 smithi173 bash[59422]: cluster 2023-09-23T19:14:01.712999+0000 mon.a (mon.0) 2282 : cluster 0 osdmap e417: 8 total, 8 up, 8 in 2023-09-23T19:14:03.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:02 smithi173 bash[59422]: cluster 2023-09-23T19:14:01.714828+0000 osd.7 (osd.7) 67 : cluster 3 failed to encode map e417 with expected crc 2023-09-23T19:14:03.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:02 smithi173 bash[59422]: cluster 2023-09-23T19:14:01.715028+0000 osd.6 (osd.6) 402 : cluster 3 failed to encode map e417 with expected crc 2023-09-23T19:14:03.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:02 smithi098 bash[76817]: audit 2023-09-23T19:14:01.712909+0000 mon.a (mon.0) 2281 : audit 1 from='client.? 172.21.15.98:0/3749933927' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:14:03.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:02 smithi098 bash[76817]: cluster 2023-09-23T19:14:01.712999+0000 mon.a (mon.0) 2282 : cluster 0 osdmap e417: 8 total, 8 up, 8 in 2023-09-23T19:14:03.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:02 smithi098 bash[76817]: cluster 2023-09-23T19:14:01.714828+0000 osd.7 (osd.7) 67 : cluster 3 failed to encode map e417 with expected crc 2023-09-23T19:14:03.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:02 smithi098 bash[76817]: cluster 2023-09-23T19:14:01.715028+0000 osd.6 (osd.6) 402 : cluster 3 failed to encode map e417 with expected crc 2023-09-23T19:14:03.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:02 smithi098 bash[79516]: audit 2023-09-23T19:14:01.712909+0000 mon.a (mon.0) 2281 : audit 1 from='client.? 172.21.15.98:0/3749933927' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T19:14:03.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:02 smithi098 bash[79516]: cluster 2023-09-23T19:14:01.712999+0000 mon.a (mon.0) 2282 : cluster 0 osdmap e417: 8 total, 8 up, 8 in 2023-09-23T19:14:03.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:02 smithi098 bash[79516]: cluster 2023-09-23T19:14:01.714828+0000 osd.7 (osd.7) 67 : cluster 3 failed to encode map e417 with expected crc 2023-09-23T19:14:03.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:02 smithi098 bash[79516]: cluster 2023-09-23T19:14:01.715028+0000 osd.6 (osd.6) 402 : cluster 3 failed to encode map e417 with expected crc 2023-09-23T19:14:04.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:03 smithi173 bash[59422]: cluster 2023-09-23T19:14:01.991534+0000 osd.4 (osd.4) 148 : cluster 3 failed to encode map e417 with expected crc 2023-09-23T19:14:04.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:03 smithi173 bash[59422]: cluster 2023-09-23T19:14:02.793524+0000 mgr.y (mgr.26054) 1696 : cluster 0 pgmap v1907: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:04.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:03 smithi098 bash[76817]: cluster 2023-09-23T19:14:01.991534+0000 osd.4 (osd.4) 148 : cluster 3 failed to encode map e417 with expected crc 2023-09-23T19:14:04.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:03 smithi098 bash[76817]: cluster 2023-09-23T19:14:02.793524+0000 mgr.y (mgr.26054) 1696 : cluster 0 pgmap v1907: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:04.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:03 smithi098 bash[79516]: cluster 2023-09-23T19:14:01.991534+0000 osd.4 (osd.4) 148 : cluster 3 failed to encode map e417 with expected crc 2023-09-23T19:14:04.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:03 smithi098 bash[79516]: cluster 2023-09-23T19:14:02.793524+0000 mgr.y (mgr.26054) 1696 : cluster 0 pgmap v1907: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:06.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:14:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:14:06] "GET /metrics HTTP/1.1" 200 34163 "" "Prometheus/2.33.4" 2023-09-23T19:14:06.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:05 smithi098 bash[79516]: cluster 2023-09-23T19:14:04.794277+0000 mgr.y (mgr.26054) 1697 : cluster 0 pgmap v1908: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:06.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:05 smithi098 bash[76817]: cluster 2023-09-23T19:14:04.794277+0000 mgr.y (mgr.26054) 1697 : cluster 0 pgmap v1908: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:06.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:05 smithi173 bash[59422]: cluster 2023-09-23T19:14:04.794277+0000 mgr.y (mgr.26054) 1697 : cluster 0 pgmap v1908: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:07.483 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:14:08.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:07 smithi098 bash[76817]: cluster 2023-09-23T19:14:06.795084+0000 mgr.y (mgr.26054) 1698 : cluster 0 pgmap v1909: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:08.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:07 smithi098 bash[79516]: cluster 2023-09-23T19:14:06.795084+0000 mgr.y (mgr.26054) 1698 : cluster 0 pgmap v1909: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:08.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:07 smithi173 bash[59422]: cluster 2023-09-23T19:14:06.795084+0000 mgr.y (mgr.26054) 1698 : cluster 0 pgmap v1909: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:10.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:09 smithi098 bash[79516]: cluster 2023-09-23T19:14:08.795909+0000 mgr.y (mgr.26054) 1699 : cluster 0 pgmap v1910: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:10.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:09 smithi098 bash[79516]: audit 2023-09-23T19:14:09.659281+0000 mon.a (mon.0) 2283 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:14:10.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:09 smithi098 bash[76817]: cluster 2023-09-23T19:14:08.795909+0000 mgr.y (mgr.26054) 1699 : cluster 0 pgmap v1910: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:10.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:09 smithi098 bash[76817]: audit 2023-09-23T19:14:09.659281+0000 mon.a (mon.0) 2283 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:14:10.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:09 smithi173 bash[59422]: cluster 2023-09-23T19:14:08.795909+0000 mgr.y (mgr.26054) 1699 : cluster 0 pgmap v1910: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:10.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:09 smithi173 bash[59422]: audit 2023-09-23T19:14:09.659281+0000 mon.a (mon.0) 2283 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:14:11.727 INFO:teuthology.orchestra.run.smithi098.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T19:14:12.015 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:11 smithi098 bash[79516]: cluster 2023-09-23T19:14:10.796788+0000 mgr.y (mgr.26054) 1700 : cluster 0 pgmap v1911: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:12.024 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:11 smithi098 bash[79516]: audit 2023-09-23T19:14:11.622661+0000 mon.a (mon.0) 2284 : audit 1 from='client.? 172.21.15.98:0/1985022779' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:14:12.024 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:11 smithi098 bash[76817]: cluster 2023-09-23T19:14:10.796788+0000 mgr.y (mgr.26054) 1700 : cluster 0 pgmap v1911: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:12.024 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:11 smithi098 bash[76817]: audit 2023-09-23T19:14:11.622661+0000 mon.a (mon.0) 2284 : audit 1 from='client.? 172.21.15.98:0/1985022779' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:14:12.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:11 smithi173 bash[59422]: cluster 2023-09-23T19:14:10.796788+0000 mgr.y (mgr.26054) 1700 : cluster 0 pgmap v1911: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:12.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:11 smithi173 bash[59422]: audit 2023-09-23T19:14:11.622661+0000 mon.a (mon.0) 2284 : audit 1 from='client.? 172.21.15.98:0/1985022779' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:14:12.791 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-09-23T19:14:13.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:12 smithi173 bash[59422]: audit 2023-09-23T19:14:11.717879+0000 mon.a (mon.0) 2285 : audit 1 from='client.? 172.21.15.98:0/1985022779' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:14:13.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:12 smithi173 bash[59422]: cluster 2023-09-23T19:14:11.717982+0000 mon.a (mon.0) 2286 : cluster 0 osdmap e418: 8 total, 8 up, 8 in 2023-09-23T19:14:13.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:12 smithi173 bash[59422]: cluster 2023-09-23T19:14:11.718137+0000 osd.6 (osd.6) 403 : cluster 3 failed to encode map e418 with expected crc 2023-09-23T19:14:13.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:12 smithi173 bash[59422]: cluster 2023-09-23T19:14:11.718420+0000 osd.4 (osd.4) 149 : cluster 3 failed to encode map e418 with expected crc 2023-09-23T19:14:13.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:12 smithi098 bash[79516]: audit 2023-09-23T19:14:11.717879+0000 mon.a (mon.0) 2285 : audit 1 from='client.? 172.21.15.98:0/1985022779' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:14:13.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:12 smithi098 bash[79516]: cluster 2023-09-23T19:14:11.717982+0000 mon.a (mon.0) 2286 : cluster 0 osdmap e418: 8 total, 8 up, 8 in 2023-09-23T19:14:13.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:12 smithi098 bash[79516]: cluster 2023-09-23T19:14:11.718137+0000 osd.6 (osd.6) 403 : cluster 3 failed to encode map e418 with expected crc 2023-09-23T19:14:13.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:12 smithi098 bash[79516]: cluster 2023-09-23T19:14:11.718420+0000 osd.4 (osd.4) 149 : cluster 3 failed to encode map e418 with expected crc 2023-09-23T19:14:13.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:12 smithi098 bash[76817]: audit 2023-09-23T19:14:11.717879+0000 mon.a (mon.0) 2285 : audit 1 from='client.? 172.21.15.98:0/1985022779' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:14:13.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:12 smithi098 bash[76817]: cluster 2023-09-23T19:14:11.717982+0000 mon.a (mon.0) 2286 : cluster 0 osdmap e418: 8 total, 8 up, 8 in 2023-09-23T19:14:13.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:12 smithi098 bash[76817]: cluster 2023-09-23T19:14:11.718137+0000 osd.6 (osd.6) 403 : cluster 3 failed to encode map e418 with expected crc 2023-09-23T19:14:13.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:12 smithi098 bash[76817]: cluster 2023-09-23T19:14:11.718420+0000 osd.4 (osd.4) 149 : cluster 3 failed to encode map e418 with expected crc 2023-09-23T19:14:14.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:13 smithi173 bash[59422]: cluster 2023-09-23T19:14:12.797562+0000 mgr.y (mgr.26054) 1701 : cluster 0 pgmap v1913: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:14.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:13 smithi173 bash[59422]: cluster 2023-09-23T19:14:12.846384+0000 osd.7 (osd.7) 68 : cluster 3 failed to encode map e418 with expected crc 2023-09-23T19:14:14.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:13 smithi098 bash[76817]: cluster 2023-09-23T19:14:12.797562+0000 mgr.y (mgr.26054) 1701 : cluster 0 pgmap v1913: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:14.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:13 smithi098 bash[76817]: cluster 2023-09-23T19:14:12.846384+0000 osd.7 (osd.7) 68 : cluster 3 failed to encode map e418 with expected crc 2023-09-23T19:14:14.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:13 smithi098 bash[79516]: cluster 2023-09-23T19:14:12.797562+0000 mgr.y (mgr.26054) 1701 : cluster 0 pgmap v1913: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:14.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:13 smithi098 bash[79516]: cluster 2023-09-23T19:14:12.846384+0000 osd.7 (osd.7) 68 : cluster 3 failed to encode map e418 with expected crc 2023-09-23T19:14:16.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:15 smithi098 bash[76817]: cluster 2023-09-23T19:14:14.798474+0000 mgr.y (mgr.26054) 1702 : cluster 0 pgmap v1914: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:16.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:14:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:14:16] "GET /metrics HTTP/1.1" 200 34167 "" "Prometheus/2.33.4" 2023-09-23T19:14:16.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:15 smithi098 bash[79516]: cluster 2023-09-23T19:14:14.798474+0000 mgr.y (mgr.26054) 1702 : cluster 0 pgmap v1914: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:16.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:15 smithi173 bash[59422]: cluster 2023-09-23T19:14:14.798474+0000 mgr.y (mgr.26054) 1702 : cluster 0 pgmap v1914: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:17.574 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:14:18.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:17 smithi098 bash[79516]: cluster 2023-09-23T19:14:16.799407+0000 mgr.y (mgr.26054) 1703 : cluster 0 pgmap v1915: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:18.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:17 smithi098 bash[76817]: cluster 2023-09-23T19:14:16.799407+0000 mgr.y (mgr.26054) 1703 : cluster 0 pgmap v1915: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:18.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:17 smithi173 bash[59422]: cluster 2023-09-23T19:14:16.799407+0000 mgr.y (mgr.26054) 1703 : cluster 0 pgmap v1915: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:20.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:19 smithi098 bash[79516]: cluster 2023-09-23T19:14:18.800228+0000 mgr.y (mgr.26054) 1704 : cluster 0 pgmap v1916: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:20.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:19 smithi098 bash[76817]: cluster 2023-09-23T19:14:18.800228+0000 mgr.y (mgr.26054) 1704 : cluster 0 pgmap v1916: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:20.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:19 smithi173 bash[59422]: cluster 2023-09-23T19:14:18.800228+0000 mgr.y (mgr.26054) 1704 : cluster 0 pgmap v1916: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:21.734 INFO:teuthology.orchestra.run.smithi098.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-09-23T19:14:22.013 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:21 smithi098 bash[79516]: cluster 2023-09-23T19:14:20.800813+0000 mgr.y (mgr.26054) 1705 : cluster 0 pgmap v1917: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:22.013 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:21 smithi098 bash[79516]: audit 2023-09-23T19:14:21.583841+0000 mon.a (mon.0) 2287 : audit 1 from='client.? 172.21.15.98:0/2305291991' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:14:22.014 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:21 smithi098 bash[76817]: cluster 2023-09-23T19:14:20.800813+0000 mgr.y (mgr.26054) 1705 : cluster 0 pgmap v1917: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:22.014 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:21 smithi098 bash[76817]: audit 2023-09-23T19:14:21.583841+0000 mon.a (mon.0) 2287 : audit 1 from='client.? 172.21.15.98:0/2305291991' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:14:22.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:21 smithi173 bash[59422]: cluster 2023-09-23T19:14:20.800813+0000 mgr.y (mgr.26054) 1705 : cluster 0 pgmap v1917: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:22.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:21 smithi173 bash[59422]: audit 2023-09-23T19:14:21.583841+0000 mon.a (mon.0) 2287 : audit 1 from='client.? 172.21.15.98:0/2305291991' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T19:14:22.761 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-09-23T19:14:22.761 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-09-23T19:14:22.761 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-09-23T19:14:22.761 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-09-23T19:14:22.761 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T19:14:22.761 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T19:14:23.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:22 smithi173 bash[59422]: cluster 2023-09-23T19:14:21.721011+0000 osd.4 (osd.4) 150 : cluster 3 failed to encode map e419 with expected crc 2023-09-23T19:14:23.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:22 smithi173 bash[59422]: cluster 2023-09-23T19:14:21.723191+0000 osd.6 (osd.6) 404 : cluster 3 failed to encode map e419 with expected crc 2023-09-23T19:14:23.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:22 smithi173 bash[59422]: audit 2023-09-23T19:14:21.723862+0000 mon.a (mon.0) 2288 : audit 1 from='client.? 172.21.15.98:0/2305291991' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:14:23.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:22 smithi173 bash[59422]: cluster 2023-09-23T19:14:21.723959+0000 mon.a (mon.0) 2289 : cluster 0 osdmap e419: 8 total, 8 up, 8 in 2023-09-23T19:14:23.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:22 smithi173 bash[59422]: cluster 2023-09-23T19:14:21.896845+0000 osd.7 (osd.7) 69 : cluster 3 failed to encode map e419 with expected crc 2023-09-23T19:14:23.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:22 smithi098 bash[79516]: cluster 2023-09-23T19:14:21.721011+0000 osd.4 (osd.4) 150 : cluster 3 failed to encode map e419 with expected crc 2023-09-23T19:14:23.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:22 smithi098 bash[79516]: cluster 2023-09-23T19:14:21.723191+0000 osd.6 (osd.6) 404 : cluster 3 failed to encode map e419 with expected crc 2023-09-23T19:14:23.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:22 smithi098 bash[79516]: audit 2023-09-23T19:14:21.723862+0000 mon.a (mon.0) 2288 : audit 1 from='client.? 172.21.15.98:0/2305291991' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:14:23.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:22 smithi098 bash[79516]: cluster 2023-09-23T19:14:21.723959+0000 mon.a (mon.0) 2289 : cluster 0 osdmap e419: 8 total, 8 up, 8 in 2023-09-23T19:14:23.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:22 smithi098 bash[79516]: cluster 2023-09-23T19:14:21.896845+0000 osd.7 (osd.7) 69 : cluster 3 failed to encode map e419 with expected crc 2023-09-23T19:14:23.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:22 smithi098 bash[76817]: cluster 2023-09-23T19:14:21.721011+0000 osd.4 (osd.4) 150 : cluster 3 failed to encode map e419 with expected crc 2023-09-23T19:14:23.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:22 smithi098 bash[76817]: cluster 2023-09-23T19:14:21.723191+0000 osd.6 (osd.6) 404 : cluster 3 failed to encode map e419 with expected crc 2023-09-23T19:14:23.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:22 smithi098 bash[76817]: audit 2023-09-23T19:14:21.723862+0000 mon.a (mon.0) 2288 : audit 1 from='client.? 172.21.15.98:0/2305291991' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T19:14:23.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:22 smithi098 bash[76817]: cluster 2023-09-23T19:14:21.723959+0000 mon.a (mon.0) 2289 : cluster 0 osdmap e419: 8 total, 8 up, 8 in 2023-09-23T19:14:23.196 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:22 smithi098 bash[76817]: cluster 2023-09-23T19:14:21.896845+0000 osd.7 (osd.7) 69 : cluster 3 failed to encode map e419 with expected crc 2023-09-23T19:14:24.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:23 smithi173 bash[59422]: cluster 2023-09-23T19:14:22.801508+0000 mgr.y (mgr.26054) 1706 : cluster 0 pgmap v1919: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:24.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:23 smithi098 bash[76817]: cluster 2023-09-23T19:14:22.801508+0000 mgr.y (mgr.26054) 1706 : cluster 0 pgmap v1919: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:24.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:23 smithi098 bash[79516]: cluster 2023-09-23T19:14:22.801508+0000 mgr.y (mgr.26054) 1706 : cluster 0 pgmap v1919: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:25.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:24 smithi173 bash[59422]: audit 2023-09-23T19:14:24.659621+0000 mon.a (mon.0) 2290 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:14:25.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:24 smithi098 bash[76817]: audit 2023-09-23T19:14:24.659621+0000 mon.a (mon.0) 2290 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:14:25.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:24 smithi098 bash[79516]: audit 2023-09-23T19:14:24.659621+0000 mon.a (mon.0) 2290 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:14:26.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:25 smithi173 bash[59422]: cluster 2023-09-23T19:14:24.802336+0000 mgr.y (mgr.26054) 1707 : cluster 0 pgmap v1920: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:26.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:25 smithi098 bash[76817]: cluster 2023-09-23T19:14:24.802336+0000 mgr.y (mgr.26054) 1707 : cluster 0 pgmap v1920: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:26.087 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:25 smithi098 bash[79516]: cluster 2023-09-23T19:14:24.802336+0000 mgr.y (mgr.26054) 1707 : cluster 0 pgmap v1920: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:26.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:14:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:14:26] "GET /metrics HTTP/1.1" 200 34163 "" "Prometheus/2.33.4" 2023-09-23T19:14:27.543 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:14:28.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:27 smithi098 bash[76817]: cluster 2023-09-23T19:14:26.803178+0000 mgr.y (mgr.26054) 1708 : cluster 0 pgmap v1921: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:28.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:27 smithi098 bash[79516]: cluster 2023-09-23T19:14:26.803178+0000 mgr.y (mgr.26054) 1708 : cluster 0 pgmap v1921: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:28.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:27 smithi173 bash[59422]: cluster 2023-09-23T19:14:26.803178+0000 mgr.y (mgr.26054) 1708 : cluster 0 pgmap v1921: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:30.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:29 smithi098 bash[76817]: cluster 2023-09-23T19:14:28.803923+0000 mgr.y (mgr.26054) 1709 : cluster 0 pgmap v1922: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:30.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:29 smithi098 bash[79516]: cluster 2023-09-23T19:14:28.803923+0000 mgr.y (mgr.26054) 1709 : cluster 0 pgmap v1922: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:30.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:29 smithi173 bash[59422]: cluster 2023-09-23T19:14:28.803923+0000 mgr.y (mgr.26054) 1709 : cluster 0 pgmap v1922: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:31.346 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T19:14:31.346 INFO:teuthology.orchestra.run.smithi098.stdout:{"epoch":419,"fsid":"756c96d4-5a3b-11ee-8db4-212e2dc638e7","created":"2023-09-23T18:04:11.580781+0000","modified":"2023-09-23T19:14:21.707124+0000","last_up_change":"2023-09-23T19:09:01.261166+0000","last_in_change":"2023-09-23T19:11:16.642043+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T18:08:35.146924+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"82","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T18:13:12.421802+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"383","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":27,"snap_epoch":383,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T18:15:07.820408+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T18:15:08.333052+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4f35f176-4d07-4157-aeb9-dd8f71da3a63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":84,"up_from":88,"up_thru":403,"down_at":85,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6802","nonce":1320995478},{"type":"v1","addr":"172.21.15.98:6803","nonce":1320995478}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6804","nonce":1320995478},{"type":"v1","addr":"172.21.15.98:6805","nonce":1320995478}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6808","nonce":1320995478},{"type":"v1","addr":"172.21.15.98:6809","nonce":1320995478}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6806","nonce":1320995478},{"type":"v1","addr":"172.21.15.98:6807","nonce":1320995478}]},"public_addr":"172.21.15.98:6803/1320995478","cluster_addr":"172.21.15.98:6805/1320995478","heartbeat_back_addr":"172.21.15.98:6809/1320995478","heartbeat_front_addr":"172.21.15.98:6807/1320995478","state":["exists","up"]},{"osd":1,"uuid":"5d777d1f-3bd3-49ba-b0de-0ac532ee3934","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":93,"last_clean_end":104,"up_from":123,"up_thru":400,"down_at":105,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6810","nonce":1500977610},{"type":"v1","addr":"172.21.15.98:6811","nonce":1500977610}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6812","nonce":1500977610},{"type":"v1","addr":"172.21.15.98:6813","nonce":1500977610}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6816","nonce":1500977610},{"type":"v1","addr":"172.21.15.98:6817","nonce":1500977610}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6814","nonce":1500977610},{"type":"v1","addr":"172.21.15.98:6815","nonce":1500977610}]},"public_addr":"172.21.15.98:6811/1500977610","cluster_addr":"172.21.15.98:6813/1500977610","heartbeat_back_addr":"172.21.15.98:6817/1500977610","heartbeat_front_addr":"172.21.15.98:6815/1500977610","state":["exists","up"]},{"osd":2,"uuid":"63978288-ac36-4cf2-bbb7-46dc828e6ce5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":94,"up_from":98,"up_thru":394,"down_at":95,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6818","nonce":4249816143},{"type":"v1","addr":"172.21.15.98:6819","nonce":4249816143}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6820","nonce":4249816143},{"type":"v1","addr":"172.21.15.98:6821","nonce":4249816143}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6824","nonce":4249816143},{"type":"v1","addr":"172.21.15.98:6825","nonce":4249816143}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6822","nonce":4249816143},{"type":"v1","addr":"172.21.15.98:6823","nonce":4249816143}]},"public_addr":"172.21.15.98:6819/4249816143","cluster_addr":"172.21.15.98:6821/4249816143","heartbeat_back_addr":"172.21.15.98:6825/4249816143","heartbeat_front_addr":"172.21.15.98:6823/4249816143","state":["exists","up"]},{"osd":3,"uuid":"48596d91-46da-4815-b810-d493bbf01dcc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":99,"up_from":103,"up_thru":397,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6826","nonce":2042614518},{"type":"v1","addr":"172.21.15.98:6827","nonce":2042614518}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6828","nonce":2042614518},{"type":"v1","addr":"172.21.15.98:6829","nonce":2042614518}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6832","nonce":2042614518},{"type":"v1","addr":"172.21.15.98:6833","nonce":2042614518}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6830","nonce":2042614518},{"type":"v1","addr":"172.21.15.98:6831","nonce":2042614518}]},"public_addr":"172.21.15.98:6827/2042614518","cluster_addr":"172.21.15.98:6829/2042614518","heartbeat_back_addr":"172.21.15.98:6833/2042614518","heartbeat_front_addr":"172.21.15.98:6831/2042614518","state":["exists","up"]},{"osd":4,"uuid":"a562afc6-ffe6-44bb-91a9-8a04018c3c6f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":262,"up_from":286,"up_thru":397,"down_at":263,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6800","nonce":1430610213},{"type":"v1","addr":"172.21.15.173:6801","nonce":1430610213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6802","nonce":1430610213},{"type":"v1","addr":"172.21.15.173:6803","nonce":1430610213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6806","nonce":1430610213},{"type":"v1","addr":"172.21.15.173:6807","nonce":1430610213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6804","nonce":1430610213},{"type":"v1","addr":"172.21.15.173:6805","nonce":1430610213}]},"public_addr":"172.21.15.173:6801/1430610213","cluster_addr":"172.21.15.173:6803/1430610213","heartbeat_back_addr":"172.21.15.173:6807/1430610213","heartbeat_front_addr":"172.21.15.173:6805/1430610213","state":["exists","up"]},{"osd":5,"uuid":"add28146-8301-4b4c-9616-967e9df3a8d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":36,"last_clean_end":322,"up_from":360,"up_thru":397,"down_at":323,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6808","nonce":1502690846},{"type":"v1","addr":"172.21.15.173:6809","nonce":1502690846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6810","nonce":1502690846},{"type":"v1","addr":"172.21.15.173:6811","nonce":1502690846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6814","nonce":1502690846},{"type":"v1","addr":"172.21.15.173:6815","nonce":1502690846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6812","nonce":1502690846},{"type":"v1","addr":"172.21.15.173:6813","nonce":1502690846}]},"public_addr":"172.21.15.173:6809/1502690846","cluster_addr":"172.21.15.173:6811/1502690846","heartbeat_back_addr":"172.21.15.173:6815/1502690846","heartbeat_front_addr":"172.21.15.173:6813/1502690846","state":["exists","up"]},{"osd":6,"uuid":"f30854de-bc26-465d-8b0e-a9211ae2d9fa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":410,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6816","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6817","nonce":3550013548}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6818","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6819","nonce":3550013548}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6822","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6823","nonce":3550013548}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6820","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6821","nonce":3550013548}]},"public_addr":"172.21.15.173:6817/3550013548","cluster_addr":"172.21.15.173:6819/3550013548","heartbeat_back_addr":"172.21.15.173:6823/3550013548","heartbeat_front_addr":"172.21.15.173:6821/3550013548","state":["exists","up"]},{"osd":7,"uuid":"5f540de3-48f9-4bc8-b14a-9aadf838f16b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":48,"last_clean_end":270,"up_from":367,"up_thru":400,"down_at":271,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6824","nonce":1212718676},{"type":"v1","addr":"172.21.15.173:6825","nonce":1212718676}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6826","nonce":1212718676},{"type":"v1","addr":"172.21.15.173:6827","nonce":1212718676}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6830","nonce":1212718676},{"type":"v1","addr":"172.21.15.173:6831","nonce":1212718676}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6828","nonce":1212718676},{"type":"v1","addr":"172.21.15.173:6829","nonce":1212718676}]},"public_addr":"172.21.15.173:6825/1212718676","cluster_addr":"172.21.15.173:6827/1212718676","heartbeat_back_addr":"172.21.15.173:6831/1212718676","heartbeat_front_addr":"172.21.15.173:6829/1212718676","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:07:04.739243+0000","dead_epoch":84},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:07:48.583198+0000","dead_epoch":104},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:08:33.173348+0000","dead_epoch":94},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:09:18.218502+0000","dead_epoch":99},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:09:59.765019+0000","dead_epoch":262},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:10:42.490689+0000","dead_epoch":322},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:11:24.357920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:12:08.168900+0000","dead_epoch":270}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.4","mappings":[{"from":1,"to":6}]},{"pgid":"3.16","mappings":[{"from":1,"to":2}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.11","mappings":[{"from":3,"to":0}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.98:0/325576677":"2023-09-24T18:21:39.415268+0000","172.21.15.98:0/2288964511":"2023-09-24T18:21:39.415268+0000","172.21.15.98:0/910147733":"2023-09-24T18:21:39.415268+0000","172.21.15.98:6800/3510470804":"2023-09-24T18:21:39.415268+0000","172.21.15.98:0/3410138197":"2023-09-24T18:21:39.415268+0000","172.21.15.98:0/3213365423":"2023-09-24T18:04:40.596823+0000","172.21.15.98:0/335394369":"2023-09-24T18:21:39.415268+0000","172.21.15.98:6801/3368447653":"2023-09-24T18:04:40.596823+0000","172.21.15.98:6800/3706591260":"2023-09-24T18:18:53.712554+0000","172.21.15.98:0/1142988558":"2023-09-24T18:12:42.525837+0000","172.21.15.98:6801/3510470804":"2023-09-24T18:21:39.415268+0000","172.21.15.98:0/2194193535":"2023-09-24T18:12:42.525837+0000","172.21.15.98:0/4085572202":"2023-09-24T18:18:53.712554+0000","172.21.15.173:0/2105184133":"2023-09-24T18:20:34.294726+0000","172.21.15.98:6800/3368447653":"2023-09-24T18:04:40.596823+0000","172.21.15.173:6834/1425795384":"2023-09-24T18:20:34.294726+0000","172.21.15.98:0/3501988932":"2023-09-24T18:21:39.415268+0000","172.21.15.173:0/2782586450":"2023-09-24T18:20:34.294726+0000","172.21.15.98:6800/483469857":"2023-09-24T18:12:42.525837+0000","172.21.15.98:0/3976553993":"2023-09-24T18:04:40.596823+0000","172.21.15.98:0/3472030975":"2023-09-24T18:05:11.060631+0000","172.21.15.173:0/4281585910":"2023-09-24T18:20:34.294726+0000","172.21.15.98:6801/3706591260":"2023-09-24T18:18:53.712554+0000","172.21.15.98:0/1603044865":"2023-09-24T18:05:11.060631+0000","172.21.15.173:6835/1425795384":"2023-09-24T18:20:34.294726+0000","172.21.15.98:6801/483469857":"2023-09-24T18:12:42.525837+0000","172.21.15.173:0/2534791896":"2023-09-24T18:20:34.294726+0000","172.21.15.98:6800/1392949322":"2023-09-24T18:05:11.060631+0000","172.21.15.98:6801/1392949322":"2023-09-24T18:05:11.060631+0000","172.21.15.98:0/2098141849":"2023-09-24T18:18:53.712554+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T19:14:31.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:31 smithi098 bash[79516]: cluster 2023-09-23T19:14:30.804825+0000 mgr.y (mgr.26054) 1710 : cluster 0 pgmap v1923: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:31.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:31 smithi098 bash[79516]: audit 2023-09-23T19:14:31.338431+0000 mon.a (mon.0) 2291 : audit 0 from='client.? 172.21.15.98:0/785880376' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:14:31.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:31 smithi098 bash[76817]: cluster 2023-09-23T19:14:30.804825+0000 mgr.y (mgr.26054) 1710 : cluster 0 pgmap v1923: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:31.945 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:31 smithi098 bash[76817]: audit 2023-09-23T19:14:31.338431+0000 mon.a (mon.0) 2291 : audit 0 from='client.? 172.21.15.98:0/785880376' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:14:32.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:31 smithi173 bash[59422]: cluster 2023-09-23T19:14:30.804825+0000 mgr.y (mgr.26054) 1710 : cluster 0 pgmap v1923: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:32.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:31 smithi173 bash[59422]: audit 2023-09-23T19:14:31.338431+0000 mon.a (mon.0) 2291 : audit 0 from='client.? 172.21.15.98:0/785880376' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:14:32.302 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T19:14:32.302 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T19:14:34.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:33 smithi098 bash[76817]: cluster 2023-09-23T19:14:32.805572+0000 mgr.y (mgr.26054) 1711 : cluster 0 pgmap v1924: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:34.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:33 smithi098 bash[79516]: cluster 2023-09-23T19:14:32.805572+0000 mgr.y (mgr.26054) 1711 : cluster 0 pgmap v1924: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:34.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:33 smithi173 bash[59422]: cluster 2023-09-23T19:14:32.805572+0000 mgr.y (mgr.26054) 1711 : cluster 0 pgmap v1924: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:36.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:35 smithi098 bash[76817]: cluster 2023-09-23T19:14:34.806439+0000 mgr.y (mgr.26054) 1712 : cluster 0 pgmap v1925: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:36.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:35 smithi098 bash[79516]: cluster 2023-09-23T19:14:34.806439+0000 mgr.y (mgr.26054) 1712 : cluster 0 pgmap v1925: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:36.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:14:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:14:36] "GET /metrics HTTP/1.1" 200 34163 "" "Prometheus/2.33.4" 2023-09-23T19:14:36.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:35 smithi173 bash[59422]: cluster 2023-09-23T19:14:34.806439+0000 mgr.y (mgr.26054) 1712 : cluster 0 pgmap v1925: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:37.083 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:14:38.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:37 smithi098 bash[76817]: cluster 2023-09-23T19:14:36.807293+0000 mgr.y (mgr.26054) 1713 : cluster 0 pgmap v1926: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:38.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:37 smithi098 bash[79516]: cluster 2023-09-23T19:14:36.807293+0000 mgr.y (mgr.26054) 1713 : cluster 0 pgmap v1926: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:38.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:37 smithi173 bash[59422]: cluster 2023-09-23T19:14:36.807293+0000 mgr.y (mgr.26054) 1713 : cluster 0 pgmap v1926: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:40.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:39 smithi098 bash[79516]: cluster 2023-09-23T19:14:38.807964+0000 mgr.y (mgr.26054) 1714 : cluster 0 pgmap v1927: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:40.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:39 smithi098 bash[79516]: audit 2023-09-23T19:14:39.660278+0000 mon.a (mon.0) 2292 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:14:40.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:39 smithi098 bash[76817]: cluster 2023-09-23T19:14:38.807964+0000 mgr.y (mgr.26054) 1714 : cluster 0 pgmap v1927: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:40.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:39 smithi098 bash[76817]: audit 2023-09-23T19:14:39.660278+0000 mon.a (mon.0) 2292 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:14:40.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:39 smithi173 bash[59422]: cluster 2023-09-23T19:14:38.807964+0000 mgr.y (mgr.26054) 1714 : cluster 0 pgmap v1927: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:40.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:39 smithi173 bash[59422]: audit 2023-09-23T19:14:39.660278+0000 mon.a (mon.0) 2292 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:14:40.767 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T19:14:40.768 INFO:teuthology.orchestra.run.smithi098.stdout:{"epoch":419,"fsid":"756c96d4-5a3b-11ee-8db4-212e2dc638e7","created":"2023-09-23T18:04:11.580781+0000","modified":"2023-09-23T19:14:21.707124+0000","last_up_change":"2023-09-23T19:09:01.261166+0000","last_in_change":"2023-09-23T19:11:16.642043+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T18:08:35.146924+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"82","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T18:13:12.421802+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"383","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":27,"snap_epoch":383,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T18:15:07.820408+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T18:15:08.333052+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4f35f176-4d07-4157-aeb9-dd8f71da3a63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":84,"up_from":88,"up_thru":403,"down_at":85,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6802","nonce":1320995478},{"type":"v1","addr":"172.21.15.98:6803","nonce":1320995478}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6804","nonce":1320995478},{"type":"v1","addr":"172.21.15.98:6805","nonce":1320995478}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6808","nonce":1320995478},{"type":"v1","addr":"172.21.15.98:6809","nonce":1320995478}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6806","nonce":1320995478},{"type":"v1","addr":"172.21.15.98:6807","nonce":1320995478}]},"public_addr":"172.21.15.98:6803/1320995478","cluster_addr":"172.21.15.98:6805/1320995478","heartbeat_back_addr":"172.21.15.98:6809/1320995478","heartbeat_front_addr":"172.21.15.98:6807/1320995478","state":["exists","up"]},{"osd":1,"uuid":"5d777d1f-3bd3-49ba-b0de-0ac532ee3934","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":93,"last_clean_end":104,"up_from":123,"up_thru":400,"down_at":105,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6810","nonce":1500977610},{"type":"v1","addr":"172.21.15.98:6811","nonce":1500977610}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6812","nonce":1500977610},{"type":"v1","addr":"172.21.15.98:6813","nonce":1500977610}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6816","nonce":1500977610},{"type":"v1","addr":"172.21.15.98:6817","nonce":1500977610}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6814","nonce":1500977610},{"type":"v1","addr":"172.21.15.98:6815","nonce":1500977610}]},"public_addr":"172.21.15.98:6811/1500977610","cluster_addr":"172.21.15.98:6813/1500977610","heartbeat_back_addr":"172.21.15.98:6817/1500977610","heartbeat_front_addr":"172.21.15.98:6815/1500977610","state":["exists","up"]},{"osd":2,"uuid":"63978288-ac36-4cf2-bbb7-46dc828e6ce5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":94,"up_from":98,"up_thru":394,"down_at":95,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6818","nonce":4249816143},{"type":"v1","addr":"172.21.15.98:6819","nonce":4249816143}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6820","nonce":4249816143},{"type":"v1","addr":"172.21.15.98:6821","nonce":4249816143}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6824","nonce":4249816143},{"type":"v1","addr":"172.21.15.98:6825","nonce":4249816143}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6822","nonce":4249816143},{"type":"v1","addr":"172.21.15.98:6823","nonce":4249816143}]},"public_addr":"172.21.15.98:6819/4249816143","cluster_addr":"172.21.15.98:6821/4249816143","heartbeat_back_addr":"172.21.15.98:6825/4249816143","heartbeat_front_addr":"172.21.15.98:6823/4249816143","state":["exists","up"]},{"osd":3,"uuid":"48596d91-46da-4815-b810-d493bbf01dcc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":99,"up_from":103,"up_thru":397,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6826","nonce":2042614518},{"type":"v1","addr":"172.21.15.98:6827","nonce":2042614518}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6828","nonce":2042614518},{"type":"v1","addr":"172.21.15.98:6829","nonce":2042614518}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6832","nonce":2042614518},{"type":"v1","addr":"172.21.15.98:6833","nonce":2042614518}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6830","nonce":2042614518},{"type":"v1","addr":"172.21.15.98:6831","nonce":2042614518}]},"public_addr":"172.21.15.98:6827/2042614518","cluster_addr":"172.21.15.98:6829/2042614518","heartbeat_back_addr":"172.21.15.98:6833/2042614518","heartbeat_front_addr":"172.21.15.98:6831/2042614518","state":["exists","up"]},{"osd":4,"uuid":"a562afc6-ffe6-44bb-91a9-8a04018c3c6f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":262,"up_from":286,"up_thru":397,"down_at":263,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6800","nonce":1430610213},{"type":"v1","addr":"172.21.15.173:6801","nonce":1430610213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6802","nonce":1430610213},{"type":"v1","addr":"172.21.15.173:6803","nonce":1430610213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6806","nonce":1430610213},{"type":"v1","addr":"172.21.15.173:6807","nonce":1430610213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6804","nonce":1430610213},{"type":"v1","addr":"172.21.15.173:6805","nonce":1430610213}]},"public_addr":"172.21.15.173:6801/1430610213","cluster_addr":"172.21.15.173:6803/1430610213","heartbeat_back_addr":"172.21.15.173:6807/1430610213","heartbeat_front_addr":"172.21.15.173:6805/1430610213","state":["exists","up"]},{"osd":5,"uuid":"add28146-8301-4b4c-9616-967e9df3a8d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":36,"last_clean_end":322,"up_from":360,"up_thru":397,"down_at":323,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6808","nonce":1502690846},{"type":"v1","addr":"172.21.15.173:6809","nonce":1502690846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6810","nonce":1502690846},{"type":"v1","addr":"172.21.15.173:6811","nonce":1502690846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6814","nonce":1502690846},{"type":"v1","addr":"172.21.15.173:6815","nonce":1502690846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6812","nonce":1502690846},{"type":"v1","addr":"172.21.15.173:6813","nonce":1502690846}]},"public_addr":"172.21.15.173:6809/1502690846","cluster_addr":"172.21.15.173:6811/1502690846","heartbeat_back_addr":"172.21.15.173:6815/1502690846","heartbeat_front_addr":"172.21.15.173:6813/1502690846","state":["exists","up"]},{"osd":6,"uuid":"f30854de-bc26-465d-8b0e-a9211ae2d9fa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":410,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6816","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6817","nonce":3550013548}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6818","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6819","nonce":3550013548}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6822","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6823","nonce":3550013548}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6820","nonce":3550013548},{"type":"v1","addr":"172.21.15.173:6821","nonce":3550013548}]},"public_addr":"172.21.15.173:6817/3550013548","cluster_addr":"172.21.15.173:6819/3550013548","heartbeat_back_addr":"172.21.15.173:6823/3550013548","heartbeat_front_addr":"172.21.15.173:6821/3550013548","state":["exists","up"]},{"osd":7,"uuid":"5f540de3-48f9-4bc8-b14a-9aadf838f16b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":48,"last_clean_end":270,"up_from":367,"up_thru":400,"down_at":271,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6824","nonce":1212718676},{"type":"v1","addr":"172.21.15.173:6825","nonce":1212718676}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6826","nonce":1212718676},{"type":"v1","addr":"172.21.15.173:6827","nonce":1212718676}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6830","nonce":1212718676},{"type":"v1","addr":"172.21.15.173:6831","nonce":1212718676}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6828","nonce":1212718676},{"type":"v1","addr":"172.21.15.173:6829","nonce":1212718676}]},"public_addr":"172.21.15.173:6825/1212718676","cluster_addr":"172.21.15.173:6827/1212718676","heartbeat_back_addr":"172.21.15.173:6831/1212718676","heartbeat_front_addr":"172.21.15.173:6829/1212718676","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:07:04.739243+0000","dead_epoch":84},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:07:48.583198+0000","dead_epoch":104},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:08:33.173348+0000","dead_epoch":94},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:09:18.218502+0000","dead_epoch":99},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:09:59.765019+0000","dead_epoch":262},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:10:42.490689+0000","dead_epoch":322},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:11:24.357920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T18:12:08.168900+0000","dead_epoch":270}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.4","mappings":[{"from":1,"to":6}]},{"pgid":"3.16","mappings":[{"from":1,"to":2}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.11","mappings":[{"from":3,"to":0}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.98:0/325576677":"2023-09-24T18:21:39.415268+0000","172.21.15.98:0/3213365423":"2023-09-24T18:04:40.596823+0000","172.21.15.98:0/2288964511":"2023-09-24T18:21:39.415268+0000","172.21.15.98:0/2098141849":"2023-09-24T18:18:53.712554+0000","172.21.15.98:0/910147733":"2023-09-24T18:21:39.415268+0000","172.21.15.98:6800/3510470804":"2023-09-24T18:21:39.415268+0000","172.21.15.173:0/2105184133":"2023-09-24T18:20:34.294726+0000","172.21.15.98:6800/3706591260":"2023-09-24T18:18:53.712554+0000","172.21.15.98:0/1142988558":"2023-09-24T18:12:42.525837+0000","172.21.15.98:6801/3510470804":"2023-09-24T18:21:39.415268+0000","172.21.15.98:0/2194193535":"2023-09-24T18:12:42.525837+0000","172.21.15.98:0/4085572202":"2023-09-24T18:18:53.712554+0000","172.21.15.98:6800/3368447653":"2023-09-24T18:04:40.596823+0000","172.21.15.173:6834/1425795384":"2023-09-24T18:20:34.294726+0000","172.21.15.98:6800/483469857":"2023-09-24T18:12:42.525837+0000","172.21.15.98:0/3410138197":"2023-09-24T18:21:39.415268+0000","172.21.15.98:0/3976553993":"2023-09-24T18:04:40.596823+0000","172.21.15.98:0/3472030975":"2023-09-24T18:05:11.060631+0000","172.21.15.173:0/4281585910":"2023-09-24T18:20:34.294726+0000","172.21.15.98:6801/3706591260":"2023-09-24T18:18:53.712554+0000","172.21.15.98:0/1603044865":"2023-09-24T18:05:11.060631+0000","172.21.15.173:6835/1425795384":"2023-09-24T18:20:34.294726+0000","172.21.15.173:0/2782586450":"2023-09-24T18:20:34.294726+0000","172.21.15.98:0/3501988932":"2023-09-24T18:21:39.415268+0000","172.21.15.98:6801/483469857":"2023-09-24T18:12:42.525837+0000","172.21.15.98:6801/3368447653":"2023-09-24T18:04:40.596823+0000","172.21.15.98:0/335394369":"2023-09-24T18:21:39.415268+0000","172.21.15.173:0/2534791896":"2023-09-24T18:20:34.294726+0000","172.21.15.98:6800/1392949322":"2023-09-24T18:05:11.060631+0000","172.21.15.98:6801/1392949322":"2023-09-24T18:05:11.060631+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T19:14:41.766 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T19:14:41.766 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T19:14:41.767 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T19:14:41.767 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T19:14:41.767 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T19:14:41.768 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T19:14:41.768 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T19:14:41.769 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T19:14:42.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:41 smithi173 bash[59422]: audit 2023-09-23T19:14:40.760447+0000 mon.c (mon.2) 168 : audit 0 from='client.? 172.21.15.98:0/1194251602' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:14:42.028 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:41 smithi173 bash[59422]: cluster 2023-09-23T19:14:40.808930+0000 mgr.y (mgr.26054) 1715 : cluster 0 pgmap v1928: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:42.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:41 smithi098 bash[76817]: audit 2023-09-23T19:14:40.760447+0000 mon.c (mon.2) 168 : audit 0 from='client.? 172.21.15.98:0/1194251602' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:14:42.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:41 smithi098 bash[76817]: cluster 2023-09-23T19:14:40.808930+0000 mgr.y (mgr.26054) 1715 : cluster 0 pgmap v1928: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:42.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:41 smithi098 bash[79516]: audit 2023-09-23T19:14:40.760447+0000 mon.c (mon.2) 168 : audit 0 from='client.? 172.21.15.98:0/1194251602' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T19:14:42.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:41 smithi098 bash[79516]: cluster 2023-09-23T19:14:40.808930+0000 mgr.y (mgr.26054) 1715 : cluster 0 pgmap v1928: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:44.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:43 smithi098 bash[76817]: cluster 2023-09-23T19:14:42.809794+0000 mgr.y (mgr.26054) 1716 : cluster 0 pgmap v1929: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:44.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:43 smithi098 bash[76817]: audit 2023-09-23T19:14:43.325412+0000 mon.a (mon.0) 2293 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:14:44.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:43 smithi098 bash[79516]: cluster 2023-09-23T19:14:42.809794+0000 mgr.y (mgr.26054) 1716 : cluster 0 pgmap v1929: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:44.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:43 smithi098 bash[79516]: audit 2023-09-23T19:14:43.325412+0000 mon.a (mon.0) 2293 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:14:44.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:43 smithi173 bash[59422]: cluster 2023-09-23T19:14:42.809794+0000 mgr.y (mgr.26054) 1716 : cluster 0 pgmap v1929: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:44.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:43 smithi173 bash[59422]: audit 2023-09-23T19:14:43.325412+0000 mon.a (mon.0) 2293 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:14:46.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:45 smithi098 bash[76817]: cluster 2023-09-23T19:14:44.810687+0000 mgr.y (mgr.26054) 1717 : cluster 0 pgmap v1930: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:46.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:14:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:14:46] "GET /metrics HTTP/1.1" 200 34161 "" "Prometheus/2.33.4" 2023-09-23T19:14:46.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:45 smithi098 bash[79516]: cluster 2023-09-23T19:14:44.810687+0000 mgr.y (mgr.26054) 1717 : cluster 0 pgmap v1930: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:46.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:45 smithi173 bash[59422]: cluster 2023-09-23T19:14:44.810687+0000 mgr.y (mgr.26054) 1717 : cluster 0 pgmap v1930: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:46.649 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:14:46.649 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:14:46.649 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:14:46.650 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:14:46.650 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:14:46.652 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:14:46.654 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:14:46.654 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:14:48.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:47 smithi098 bash[76817]: cluster 2023-09-23T19:14:46.811618+0000 mgr.y (mgr.26054) 1718 : cluster 0 pgmap v1931: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:48.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:47 smithi098 bash[79516]: cluster 2023-09-23T19:14:46.811618+0000 mgr.y (mgr.26054) 1718 : cluster 0 pgmap v1931: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:48.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:47 smithi173 bash[59422]: cluster 2023-09-23T19:14:46.811618+0000 mgr.y (mgr.26054) 1718 : cluster 0 pgmap v1931: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:50.008 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:49 smithi098 bash[76817]: cluster 2023-09-23T19:14:48.812434+0000 mgr.y (mgr.26054) 1719 : cluster 0 pgmap v1932: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:50.008 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:49 smithi098 bash[76817]: audit 2023-09-23T19:14:48.817568+0000 mon.a (mon.0) 2294 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:14:50.008 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:49 smithi098 bash[76817]: audit 2023-09-23T19:14:48.823252+0000 mon.a (mon.0) 2295 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:14:50.008 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:49 smithi098 bash[76817]: audit 2023-09-23T19:14:49.628496+0000 mon.a (mon.0) 2296 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:14:50.008 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:49 smithi098 bash[76817]: audit 2023-09-23T19:14:49.636833+0000 mon.a (mon.0) 2297 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:14:50.009 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:49 smithi098 bash[79516]: cluster 2023-09-23T19:14:48.812434+0000 mgr.y (mgr.26054) 1719 : cluster 0 pgmap v1932: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:50.009 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:49 smithi098 bash[79516]: audit 2023-09-23T19:14:48.817568+0000 mon.a (mon.0) 2294 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:14:50.009 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:49 smithi098 bash[79516]: audit 2023-09-23T19:14:48.823252+0000 mon.a (mon.0) 2295 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:14:50.009 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:49 smithi098 bash[79516]: audit 2023-09-23T19:14:49.628496+0000 mon.a (mon.0) 2296 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:14:50.009 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:49 smithi098 bash[79516]: audit 2023-09-23T19:14:49.636833+0000 mon.a (mon.0) 2297 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:14:50.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:49 smithi173 bash[59422]: cluster 2023-09-23T19:14:48.812434+0000 mgr.y (mgr.26054) 1719 : cluster 0 pgmap v1932: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:50.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:49 smithi173 bash[59422]: audit 2023-09-23T19:14:48.817568+0000 mon.a (mon.0) 2294 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:14:50.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:49 smithi173 bash[59422]: audit 2023-09-23T19:14:48.823252+0000 mon.a (mon.0) 2295 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:14:50.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:49 smithi173 bash[59422]: audit 2023-09-23T19:14:49.628496+0000 mon.a (mon.0) 2296 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:14:50.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:49 smithi173 bash[59422]: audit 2023-09-23T19:14:49.636833+0000 mon.a (mon.0) 2297 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:14:51.180 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:50 smithi098 bash[76817]: audit 2023-09-23T19:14:50.108098+0000 mon.a (mon.0) 2298 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:14:51.180 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:50 smithi098 bash[76817]: audit 2023-09-23T19:14:50.109801+0000 mon.a (mon.0) 2299 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:14:51.180 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:50 smithi098 bash[76817]: audit 2023-09-23T19:14:50.116506+0000 mon.a (mon.0) 2300 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:14:51.180 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:50 smithi098 bash[79516]: audit 2023-09-23T19:14:50.108098+0000 mon.a (mon.0) 2298 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:14:51.180 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:50 smithi098 bash[79516]: audit 2023-09-23T19:14:50.109801+0000 mon.a (mon.0) 2299 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:14:51.181 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:50 smithi098 bash[79516]: audit 2023-09-23T19:14:50.116506+0000 mon.a (mon.0) 2300 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:14:51.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:50 smithi173 bash[59422]: audit 2023-09-23T19:14:50.108098+0000 mon.a (mon.0) 2298 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T19:14:51.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:50 smithi173 bash[59422]: audit 2023-09-23T19:14:50.109801+0000 mon.a (mon.0) 2299 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T19:14:51.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:50 smithi173 bash[59422]: audit 2023-09-23T19:14:50.116506+0000 mon.a (mon.0) 2300 : audit 1 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' 2023-09-23T19:14:51.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:51 smithi098 bash[76817]: cluster 2023-09-23T19:14:50.813098+0000 mgr.y (mgr.26054) 1720 : cluster 0 pgmap v1933: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:51.944 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:51 smithi098 bash[79516]: cluster 2023-09-23T19:14:50.813098+0000 mgr.y (mgr.26054) 1720 : cluster 0 pgmap v1933: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:52.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:51 smithi173 bash[59422]: cluster 2023-09-23T19:14:50.813098+0000 mgr.y (mgr.26054) 1720 : cluster 0 pgmap v1933: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:54.176 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:53 smithi098 bash[79516]: cluster 2023-09-23T19:14:52.813852+0000 mgr.y (mgr.26054) 1721 : cluster 0 pgmap v1934: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:54.176 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:53 smithi098 bash[76817]: cluster 2023-09-23T19:14:52.813852+0000 mgr.y (mgr.26054) 1721 : cluster 0 pgmap v1934: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:54.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:53 smithi173 bash[59422]: cluster 2023-09-23T19:14:52.813852+0000 mgr.y (mgr.26054) 1721 : cluster 0 pgmap v1934: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:55.186 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:54 smithi098 bash[76817]: audit 2023-09-23T19:14:54.659670+0000 mon.a (mon.0) 2301 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:14:55.186 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:54 smithi098 bash[79516]: audit 2023-09-23T19:14:54.659670+0000 mon.a (mon.0) 2301 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:14:55.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:54 smithi173 bash[59422]: audit 2023-09-23T19:14:54.659670+0000 mon.a (mon.0) 2301 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:14:56.137 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:55 smithi098 bash[79516]: cluster 2023-09-23T19:14:54.814388+0000 mgr.y (mgr.26054) 1722 : cluster 0 pgmap v1935: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:56.138 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:55 smithi098 bash[76817]: cluster 2023-09-23T19:14:54.814388+0000 mgr.y (mgr.26054) 1722 : cluster 0 pgmap v1935: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:56.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:55 smithi173 bash[59422]: cluster 2023-09-23T19:14:54.814388+0000 mgr.y (mgr.26054) 1722 : cluster 0 pgmap v1935: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:56.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:14:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:14:56] "GET /metrics HTTP/1.1" 200 34165 "" "Prometheus/2.33.4" 2023-09-23T19:14:58.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:57 smithi098 bash[76817]: cluster 2023-09-23T19:14:56.815299+0000 mgr.y (mgr.26054) 1723 : cluster 0 pgmap v1936: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:58.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:57 smithi098 bash[79516]: cluster 2023-09-23T19:14:56.815299+0000 mgr.y (mgr.26054) 1723 : cluster 0 pgmap v1936: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:58.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:57 smithi173 bash[59422]: cluster 2023-09-23T19:14:56.815299+0000 mgr.y (mgr.26054) 1723 : cluster 0 pgmap v1936: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:59.877 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:14:59 smithi098 bash[79516]: cluster 2023-09-23T19:14:58.816214+0000 mgr.y (mgr.26054) 1724 : cluster 0 pgmap v1937: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:14:59.878 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:14:59 smithi098 bash[76817]: cluster 2023-09-23T19:14:58.816214+0000 mgr.y (mgr.26054) 1724 : cluster 0 pgmap v1937: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:00.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:14:59 smithi173 bash[59422]: cluster 2023-09-23T19:14:58.816214+0000 mgr.y (mgr.26054) 1724 : cluster 0 pgmap v1937: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:01.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:01 smithi173 bash[59422]: cluster 2023-09-23T19:15:00.816792+0000 mgr.y (mgr.26054) 1725 : cluster 0 pgmap v1938: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:02.095 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:01 smithi098 bash[79516]: cluster 2023-09-23T19:15:00.816792+0000 mgr.y (mgr.26054) 1725 : cluster 0 pgmap v1938: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:02.095 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:01 smithi098 bash[76817]: cluster 2023-09-23T19:15:00.816792+0000 mgr.y (mgr.26054) 1725 : cluster 0 pgmap v1938: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:02.125 INFO:teuthology.orchestra.run.smithi098.stdout:442381632054 2023-09-23T19:15:02.126 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T19:15:03.280 INFO:teuthology.orchestra.run.smithi098.stdout:420906795579 2023-09-23T19:15:03.280 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T19:15:04.151 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:03 smithi098 bash[76817]: cluster 2023-09-23T19:15:02.817493+0000 mgr.y (mgr.26054) 1726 : cluster 0 pgmap v1939: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:04.151 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:03 smithi098 bash[79516]: cluster 2023-09-23T19:15:02.817493+0000 mgr.y (mgr.26054) 1726 : cluster 0 pgmap v1939: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:04.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:03 smithi173 bash[59422]: cluster 2023-09-23T19:15:02.817493+0000 mgr.y (mgr.26054) 1726 : cluster 0 pgmap v1939: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:04.292 INFO:teuthology.orchestra.run.smithi098.stdout:528280977901 2023-09-23T19:15:04.293 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T19:15:05.450 INFO:teuthology.orchestra.run.smithi098.stdout:377957122632 2023-09-23T19:15:05.450 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T19:15:06.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:15:06 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:15:06] "GET /metrics HTTP/1.1" 200 34165 "" "Prometheus/2.33.4" 2023-09-23T19:15:06.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:05 smithi098 bash[76817]: cluster 2023-09-23T19:15:04.818459+0000 mgr.y (mgr.26054) 1727 : cluster 0 pgmap v1940: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:06.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:05 smithi098 bash[79516]: cluster 2023-09-23T19:15:04.818459+0000 mgr.y (mgr.26054) 1727 : cluster 0 pgmap v1940: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:06.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:05 smithi173 bash[59422]: cluster 2023-09-23T19:15:04.818459+0000 mgr.y (mgr.26054) 1727 : cluster 0 pgmap v1940: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:08.053 INFO:teuthology.orchestra.run.smithi098.stdout:1576252997706 2023-09-23T19:15:08.053 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T19:15:08.056 INFO:teuthology.orchestra.run.smithi098.stdout:1228360646886 2023-09-23T19:15:08.056 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T19:15:08.110 INFO:teuthology.orchestra.run.smithi098.stdout:180388627199 2023-09-23T19:15:08.110 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T19:15:08.164 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:07 smithi098 bash[79516]: cluster 2023-09-23T19:15:06.818981+0000 mgr.y (mgr.26054) 1728 : cluster 0 pgmap v1941: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:08.164 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:07 smithi098 bash[76817]: cluster 2023-09-23T19:15:06.818981+0000 mgr.y (mgr.26054) 1728 : cluster 0 pgmap v1941: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:08.257 INFO:teuthology.orchestra.run.smithi098.stdout:1546188226639 2023-09-23T19:15:08.258 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T19:15:08.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:07 smithi173 bash[59422]: cluster 2023-09-23T19:15:06.818981+0000 mgr.y (mgr.26054) 1728 : cluster 0 pgmap v1941: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:10.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:09 smithi098 bash[76817]: cluster 2023-09-23T19:15:08.819779+0000 mgr.y (mgr.26054) 1729 : cluster 0 pgmap v1942: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:10.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:09 smithi098 bash[76817]: audit 2023-09-23T19:15:09.660789+0000 mon.a (mon.0) 2302 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:15:10.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:09 smithi098 bash[79516]: cluster 2023-09-23T19:15:08.819779+0000 mgr.y (mgr.26054) 1729 : cluster 0 pgmap v1942: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:10.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:09 smithi098 bash[79516]: audit 2023-09-23T19:15:09.660789+0000 mon.a (mon.0) 2302 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:15:10.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:09 smithi173 bash[59422]: cluster 2023-09-23T19:15:08.819779+0000 mgr.y (mgr.26054) 1729 : cluster 0 pgmap v1942: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:10.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:09 smithi173 bash[59422]: audit 2023-09-23T19:15:09.660789+0000 mon.a (mon.0) 2302 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:15:10.944 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:15:10.945 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:15:10.945 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:15:10.945 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:15:11.958 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:15:11.958 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:15:11.960 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:15:11.984 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:11 smithi098 bash[79516]: cluster 2023-09-23T19:15:10.820656+0000 mgr.y (mgr.26054) 1730 : cluster 0 pgmap v1943: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:11.985 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:11 smithi098 bash[76817]: cluster 2023-09-23T19:15:10.820656+0000 mgr.y (mgr.26054) 1730 : cluster 0 pgmap v1943: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:12.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:11 smithi173 bash[59422]: cluster 2023-09-23T19:15:10.820656+0000 mgr.y (mgr.26054) 1730 : cluster 0 pgmap v1943: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:12.973 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:15:14.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:13 smithi098 bash[76817]: cluster 2023-09-23T19:15:12.821482+0000 mgr.y (mgr.26054) 1731 : cluster 0 pgmap v1944: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:14.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:13 smithi098 bash[79516]: cluster 2023-09-23T19:15:12.821482+0000 mgr.y (mgr.26054) 1731 : cluster 0 pgmap v1944: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:14.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:13 smithi173 bash[59422]: cluster 2023-09-23T19:15:12.821482+0000 mgr.y (mgr.26054) 1731 : cluster 0 pgmap v1944: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:16.086 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:15 smithi098 bash[79516]: cluster 2023-09-23T19:15:14.822248+0000 mgr.y (mgr.26054) 1732 : cluster 0 pgmap v1945: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:16.087 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:15 smithi098 bash[76817]: cluster 2023-09-23T19:15:14.822248+0000 mgr.y (mgr.26054) 1732 : cluster 0 pgmap v1945: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:16.194 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:15:16 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:15:16] "GET /metrics HTTP/1.1" 200 34169 "" "Prometheus/2.33.4" 2023-09-23T19:15:16.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:15 smithi173 bash[59422]: cluster 2023-09-23T19:15:14.822248+0000 mgr.y (mgr.26054) 1732 : cluster 0 pgmap v1945: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:18.162 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:17 smithi098 bash[79516]: cluster 2023-09-23T19:15:16.822894+0000 mgr.y (mgr.26054) 1733 : cluster 0 pgmap v1946: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:18.162 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:17 smithi098 bash[76817]: cluster 2023-09-23T19:15:16.822894+0000 mgr.y (mgr.26054) 1733 : cluster 0 pgmap v1946: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:18.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:17 smithi173 bash[59422]: cluster 2023-09-23T19:15:16.822894+0000 mgr.y (mgr.26054) 1733 : cluster 0 pgmap v1946: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:20.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:19 smithi098 bash[76817]: cluster 2023-09-23T19:15:18.823357+0000 mgr.y (mgr.26054) 1734 : cluster 0 pgmap v1947: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:20.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:19 smithi098 bash[79516]: cluster 2023-09-23T19:15:18.823357+0000 mgr.y (mgr.26054) 1734 : cluster 0 pgmap v1947: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:20.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:19 smithi173 bash[59422]: cluster 2023-09-23T19:15:18.823357+0000 mgr.y (mgr.26054) 1734 : cluster 0 pgmap v1947: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:21.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:21 smithi098 bash[76817]: cluster 2023-09-23T19:15:20.823824+0000 mgr.y (mgr.26054) 1735 : cluster 0 pgmap v1948: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:21.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:21 smithi098 bash[79516]: cluster 2023-09-23T19:15:20.823824+0000 mgr.y (mgr.26054) 1735 : cluster 0 pgmap v1948: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:22.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:21 smithi173 bash[59422]: cluster 2023-09-23T19:15:20.823824+0000 mgr.y (mgr.26054) 1735 : cluster 0 pgmap v1948: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:24.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:23 smithi098 bash[79516]: cluster 2023-09-23T19:15:22.824612+0000 mgr.y (mgr.26054) 1736 : cluster 0 pgmap v1949: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:24.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:23 smithi098 bash[76817]: cluster 2023-09-23T19:15:22.824612+0000 mgr.y (mgr.26054) 1736 : cluster 0 pgmap v1949: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:24.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:23 smithi173 bash[59422]: cluster 2023-09-23T19:15:22.824612+0000 mgr.y (mgr.26054) 1736 : cluster 0 pgmap v1949: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:25.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:24 smithi098 bash[79516]: audit 2023-09-23T19:15:24.660443+0000 mon.a (mon.0) 2303 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:15:25.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:24 smithi098 bash[76817]: audit 2023-09-23T19:15:24.660443+0000 mon.a (mon.0) 2303 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:15:25.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:24 smithi173 bash[59422]: audit 2023-09-23T19:15:24.660443+0000 mon.a (mon.0) 2303 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:15:25.974 INFO:teuthology.orchestra.run.smithi098.stdout:442381632059 2023-09-23T19:15:26.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:25 smithi173 bash[59422]: cluster 2023-09-23T19:15:24.825455+0000 mgr.y (mgr.26054) 1737 : cluster 0 pgmap v1950: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:26.444 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:15:26 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:15:26] "GET /metrics HTTP/1.1" 200 34166 "" "Prometheus/2.33.4" 2023-09-23T19:15:26.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:25 smithi098 bash[79516]: cluster 2023-09-23T19:15:24.825455+0000 mgr.y (mgr.26054) 1737 : cluster 0 pgmap v1950: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:26.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:25 smithi098 bash[76817]: cluster 2023-09-23T19:15:24.825455+0000 mgr.y (mgr.26054) 1737 : cluster 0 pgmap v1950: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:26.944 INFO:teuthology.orchestra.run.smithi098.stdout:1576252997711 2023-09-23T19:15:27.165 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:26 smithi098 bash[79516]: audit 2023-09-23T19:15:25.969853+0000 mon.a (mon.0) 2304 : audit 0 from='client.? 172.21.15.98:0/3527723304' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T19:15:27.166 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:26 smithi098 bash[76817]: audit 2023-09-23T19:15:25.969853+0000 mon.a (mon.0) 2304 : audit 0 from='client.? 172.21.15.98:0/3527723304' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T19:15:27.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:26 smithi173 bash[59422]: audit 2023-09-23T19:15:25.969853+0000 mon.a (mon.0) 2304 : audit 0 from='client.? 172.21.15.98:0/3527723304' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T19:15:27.493 INFO:teuthology.orchestra.run.smithi098.stdout:528280977906 2023-09-23T19:15:27.542 INFO:teuthology.orchestra.run.smithi098.stdout:420906795584 2023-09-23T19:15:28.184 INFO:teuthology.orchestra.run.smithi098.stdout:1546188226644 2023-09-23T19:15:28.252 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:27 smithi098 bash[79516]: cluster 2023-09-23T19:15:26.826368+0000 mgr.y (mgr.26054) 1738 : cluster 0 pgmap v1951: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:28.252 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:27 smithi098 bash[79516]: audit 2023-09-23T19:15:26.939743+0000 mon.c (mon.2) 169 : audit 0 from='client.? 172.21.15.98:0/3218142596' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T19:15:28.252 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:27 smithi098 bash[79516]: audit 2023-09-23T19:15:27.488406+0000 mon.c (mon.2) 170 : audit 0 from='client.? 172.21.15.98:0/4147135414' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T19:15:28.252 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:27 smithi098 bash[79516]: audit 2023-09-23T19:15:27.537741+0000 mon.a (mon.0) 2305 : audit 0 from='client.? 172.21.15.98:0/60055943' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T19:15:28.252 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:27 smithi098 bash[76817]: cluster 2023-09-23T19:15:26.826368+0000 mgr.y (mgr.26054) 1738 : cluster 0 pgmap v1951: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:28.253 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:27 smithi098 bash[76817]: audit 2023-09-23T19:15:26.939743+0000 mon.c (mon.2) 169 : audit 0 from='client.? 172.21.15.98:0/3218142596' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T19:15:28.253 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:27 smithi098 bash[76817]: audit 2023-09-23T19:15:27.488406+0000 mon.c (mon.2) 170 : audit 0 from='client.? 172.21.15.98:0/4147135414' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T19:15:28.253 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:27 smithi098 bash[76817]: audit 2023-09-23T19:15:27.537741+0000 mon.a (mon.0) 2305 : audit 0 from='client.? 172.21.15.98:0/60055943' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T19:15:28.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:27 smithi173 bash[59422]: cluster 2023-09-23T19:15:26.826368+0000 mgr.y (mgr.26054) 1738 : cluster 0 pgmap v1951: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:28.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:27 smithi173 bash[59422]: audit 2023-09-23T19:15:26.939743+0000 mon.c (mon.2) 169 : audit 0 from='client.? 172.21.15.98:0/3218142596' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T19:15:28.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:27 smithi173 bash[59422]: audit 2023-09-23T19:15:27.488406+0000 mon.c (mon.2) 170 : audit 0 from='client.? 172.21.15.98:0/4147135414' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T19:15:28.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:27 smithi173 bash[59422]: audit 2023-09-23T19:15:27.537741+0000 mon.a (mon.0) 2305 : audit 0 from='client.? 172.21.15.98:0/60055943' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T19:15:29.023 INFO:teuthology.orchestra.run.smithi098.stdout:377957122638 2023-09-23T19:15:29.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:28 smithi098 bash[76817]: audit 2023-09-23T19:15:28.179705+0000 mon.c (mon.2) 171 : audit 0 from='client.? 172.21.15.98:0/1328849783' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T19:15:29.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:28 smithi098 bash[79516]: audit 2023-09-23T19:15:28.179705+0000 mon.c (mon.2) 171 : audit 0 from='client.? 172.21.15.98:0/1328849783' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T19:15:29.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:28 smithi173 bash[59422]: audit 2023-09-23T19:15:28.179705+0000 mon.c (mon.2) 171 : audit 0 from='client.? 172.21.15.98:0/1328849783' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T19:15:29.890 INFO:tasks.cephadm.ceph_manager.ceph:need seq 442381632054 got 442381632059 for osd.3 2023-09-23T19:15:29.891 DEBUG:teuthology.parallel:result is None 2023-09-23T19:15:30.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:29 smithi098 bash[76817]: cluster 2023-09-23T19:15:28.826909+0000 mgr.y (mgr.26054) 1739 : cluster 0 pgmap v1952: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:30.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:29 smithi098 bash[76817]: audit 2023-09-23T19:15:29.018366+0000 mon.a (mon.0) 2306 : audit 0 from='client.? 172.21.15.98:0/3779264024' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T19:15:30.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:29 smithi098 bash[79516]: cluster 2023-09-23T19:15:28.826909+0000 mgr.y (mgr.26054) 1739 : cluster 0 pgmap v1952: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:30.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:29 smithi098 bash[79516]: audit 2023-09-23T19:15:29.018366+0000 mon.a (mon.0) 2306 : audit 0 from='client.? 172.21.15.98:0/3779264024' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T19:15:30.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:29 smithi173 bash[59422]: cluster 2023-09-23T19:15:28.826909+0000 mgr.y (mgr.26054) 1739 : cluster 0 pgmap v1952: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:30.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:29 smithi173 bash[59422]: audit 2023-09-23T19:15:29.018366+0000 mon.a (mon.0) 2306 : audit 0 from='client.? 172.21.15.98:0/3779264024' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T19:15:30.456 INFO:teuthology.orchestra.run.smithi098.stdout:180388627204 2023-09-23T19:15:30.717 INFO:teuthology.orchestra.run.smithi098.stdout:1228360646890 2023-09-23T19:15:31.145 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1576252997706 got 1576252997711 for osd.7 2023-09-23T19:15:31.145 DEBUG:teuthology.parallel:result is None 2023-09-23T19:15:31.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:30 smithi098 bash[76817]: audit 2023-09-23T19:15:30.451142+0000 mon.a (mon.0) 2307 : audit 0 from='client.? 172.21.15.98:0/2179464056' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T19:15:31.191 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:30 smithi098 bash[76817]: audit 2023-09-23T19:15:30.711948+0000 mon.c (mon.2) 172 : audit 0 from='client.? 172.21.15.98:0/2923135404' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T19:15:31.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:30 smithi098 bash[79516]: audit 2023-09-23T19:15:30.451142+0000 mon.a (mon.0) 2307 : audit 0 from='client.? 172.21.15.98:0/2179464056' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T19:15:31.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:30 smithi098 bash[79516]: audit 2023-09-23T19:15:30.711948+0000 mon.c (mon.2) 172 : audit 0 from='client.? 172.21.15.98:0/2923135404' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T19:15:31.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:30 smithi173 bash[59422]: audit 2023-09-23T19:15:30.451142+0000 mon.a (mon.0) 2307 : audit 0 from='client.? 172.21.15.98:0/2179464056' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T19:15:31.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:30 smithi173 bash[59422]: audit 2023-09-23T19:15:30.711948+0000 mon.c (mon.2) 172 : audit 0 from='client.? 172.21.15.98:0/2923135404' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T19:15:32.021 INFO:tasks.cephadm.ceph_manager.ceph:need seq 420906795579 got 420906795584 for osd.2 2023-09-23T19:15:32.021 DEBUG:teuthology.parallel:result is None 2023-09-23T19:15:32.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:31 smithi098 bash[79516]: cluster 2023-09-23T19:15:30.827755+0000 mgr.y (mgr.26054) 1740 : cluster 0 pgmap v1953: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:32.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:31 smithi098 bash[76817]: cluster 2023-09-23T19:15:30.827755+0000 mgr.y (mgr.26054) 1740 : cluster 0 pgmap v1953: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:32.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:31 smithi173 bash[59422]: cluster 2023-09-23T19:15:30.827755+0000 mgr.y (mgr.26054) 1740 : cluster 0 pgmap v1953: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:33.453 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1546188226639 got 1546188226644 for osd.5 2023-09-23T19:15:33.454 DEBUG:teuthology.parallel:result is None 2023-09-23T19:15:33.602 INFO:tasks.cephadm.ceph_manager.ceph:need seq 528280977901 got 528280977906 for osd.1 2023-09-23T19:15:33.602 DEBUG:teuthology.parallel:result is None 2023-09-23T19:15:34.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:33 smithi173 bash[59422]: cluster 2023-09-23T19:15:32.828460+0000 mgr.y (mgr.26054) 1741 : cluster 0 pgmap v1954: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:34.445 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:34 smithi098 bash[76817]: cluster 2023-09-23T19:15:32.828460+0000 mgr.y (mgr.26054) 1741 : cluster 0 pgmap v1954: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:34.445 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:34 smithi098 bash[79516]: cluster 2023-09-23T19:15:32.828460+0000 mgr.y (mgr.26054) 1741 : cluster 0 pgmap v1954: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:34.996 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388627199 got 180388627204 for osd.6 2023-09-23T19:15:34.996 DEBUG:teuthology.parallel:result is None 2023-09-23T19:15:35.015 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1228360646886 got 1228360646890 for osd.4 2023-09-23T19:15:35.015 DEBUG:teuthology.parallel:result is None 2023-09-23T19:15:35.057 INFO:tasks.cephadm.ceph_manager.ceph:need seq 377957122632 got 377957122638 for osd.0 2023-09-23T19:15:35.057 DEBUG:teuthology.parallel:result is None 2023-09-23T19:15:35.057 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T19:15:35.057 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:15:36.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:35 smithi098 bash[76817]: cluster 2023-09-23T19:15:34.829178+0000 mgr.y (mgr.26054) 1742 : cluster 0 pgmap v1955: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:36.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:15:36 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:15:36] "GET /metrics HTTP/1.1" 200 34166 "" "Prometheus/2.33.4" 2023-09-23T19:15:36.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:35 smithi098 bash[79516]: cluster 2023-09-23T19:15:34.829178+0000 mgr.y (mgr.26054) 1742 : cluster 0 pgmap v1955: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:36.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:35 smithi173 bash[59422]: cluster 2023-09-23T19:15:34.829178+0000 mgr.y (mgr.26054) 1742 : cluster 0 pgmap v1955: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:38.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:37 smithi098 bash[76817]: cluster 2023-09-23T19:15:36.830094+0000 mgr.y (mgr.26054) 1743 : cluster 0 pgmap v1956: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:38.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:37 smithi098 bash[79516]: cluster 2023-09-23T19:15:36.830094+0000 mgr.y (mgr.26054) 1743 : cluster 0 pgmap v1956: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:38.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:37 smithi173 bash[59422]: cluster 2023-09-23T19:15:36.830094+0000 mgr.y (mgr.26054) 1743 : cluster 0 pgmap v1956: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:39.877 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:15:40.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:39 smithi098 bash[76817]: cluster 2023-09-23T19:15:38.830919+0000 mgr.y (mgr.26054) 1744 : cluster 0 pgmap v1957: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:40.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:39 smithi098 bash[76817]: audit 2023-09-23T19:15:39.661606+0000 mon.a (mon.0) 2308 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:15:40.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:39 smithi098 bash[79516]: cluster 2023-09-23T19:15:38.830919+0000 mgr.y (mgr.26054) 1744 : cluster 0 pgmap v1957: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:40.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:39 smithi098 bash[79516]: audit 2023-09-23T19:15:39.661606+0000 mon.a (mon.0) 2308 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:15:40.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:39 smithi173 bash[59422]: cluster 2023-09-23T19:15:38.830919+0000 mgr.y (mgr.26054) 1744 : cluster 0 pgmap v1957: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:40.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:39 smithi173 bash[59422]: audit 2023-09-23T19:15:39.661606+0000 mon.a (mon.0) 2308 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T19:15:41.944 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:41 smithi098 bash[76817]: cluster 2023-09-23T19:15:40.831810+0000 mgr.y (mgr.26054) 1745 : cluster 0 pgmap v1958: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:41.945 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:41 smithi098 bash[79516]: cluster 2023-09-23T19:15:40.831810+0000 mgr.y (mgr.26054) 1745 : cluster 0 pgmap v1958: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:42.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:41 smithi173 bash[59422]: cluster 2023-09-23T19:15:40.831810+0000 mgr.y (mgr.26054) 1745 : cluster 0 pgmap v1958: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:43.564 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T19:15:43.567 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-23T19:15:44.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:43 smithi098 bash[76817]: cluster 2023-09-23T19:15:42.832634+0000 mgr.y (mgr.26054) 1746 : cluster 0 pgmap v1959: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:44.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:43 smithi098 bash[79516]: cluster 2023-09-23T19:15:42.832634+0000 mgr.y (mgr.26054) 1746 : cluster 0 pgmap v1959: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:44.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:43 smithi173 bash[59422]: cluster 2023-09-23T19:15:42.832634+0000 mgr.y (mgr.26054) 1746 : cluster 0 pgmap v1959: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:44.526 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":1959,"stamp":"2023-09-23T19:15:42.832125+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":12228,"num_read_kb":26865,"num_write":1950,"num_write_kb":27871,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":213,"num_bytes_recovered":16790439,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2144,"ondisk_log_size":2144,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6026312,"kb_used_data":62536,"kb_used_omap":62,"kb_used_meta":5963649,"kb_avail":743934904,"statfs":{"total":767960285184,"available":761789341696,"internally_reserved":0,"allocated":64036864,"data_stored":52357425,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":63895,"internal_metadata":6106777193},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004928"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":31,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.725569+0000","last_change":"2023-09-23T19:11:17.683481+0000","last_active":"2023-09-23T19:14:11.725569+0000","last_peered":"2023-09-23T19:14:11.725569+0000","last_clean":"2023-09-23T19:14:11.725569+0000","last_became_active":"2023-09-23T19:11:17.681965+0000","last_became_peered":"2023-09-23T19:11:17.681965+0000","last_unstale":"2023-09-23T19:14:11.725569+0000","last_undegraded":"2023-09-23T19:14:11.725569+0000","last_fullsized":"2023-09-23T19:14:11.725569+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":155,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.715944+0000","last_change":"2023-09-23T19:11:17.681338+0000","last_active":"2023-09-23T19:14:11.715944+0000","last_peered":"2023-09-23T19:14:11.715944+0000","last_clean":"2023-09-23T19:14:11.715944+0000","last_became_active":"2023-09-23T19:11:17.680951+0000","last_became_peered":"2023-09-23T19:11:17.680951+0000","last_unstale":"2023-09-23T19:14:11.715944+0000","last_undegraded":"2023-09-23T19:14:11.715944+0000","last_fullsized":"2023-09-23T19:14:11.715944+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:26.573271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":422,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.893589+0000","last_change":"2023-09-23T19:11:17.679652+0000","last_active":"2023-09-23T19:14:21.893589+0000","last_peered":"2023-09-23T19:14:21.893589+0000","last_clean":"2023-09-23T19:14:21.893589+0000","last_became_active":"2023-09-23T19:11:17.679260+0000","last_became_peered":"2023-09-23T19:11:17.679260+0000","last_unstale":"2023-09-23T19:14:21.893589+0000","last_undegraded":"2023-09-23T19:14:21.893589+0000","last_fullsized":"2023-09-23T19:14:21.893589+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:14.084906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":467,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.894540+0000","last_change":"2023-09-23T19:09:03.326411+0000","last_active":"2023-09-23T19:14:21.894540+0000","last_peered":"2023-09-23T19:14:21.894540+0000","last_clean":"2023-09-23T19:14:21.894540+0000","last_became_active":"2023-09-23T19:09:03.325980+0000","last_became_peered":"2023-09-23T19:09:03.325980+0000","last_unstale":"2023-09-23T19:14:21.894540+0000","last_undegraded":"2023-09-23T19:14:21.894540+0000","last_fullsized":"2023-09-23T19:14:21.894540+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:43.638573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":138,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.721156+0000","last_change":"2023-09-23T18:58:38.718387+0000","last_active":"2023-09-23T19:14:21.721156+0000","last_peered":"2023-09-23T19:14:21.721156+0000","last_clean":"2023-09-23T19:14:21.721156+0000","last_became_active":"2023-09-23T18:58:38.717999+0000","last_became_peered":"2023-09-23T18:58:38.717999+0000","last_unstale":"2023-09-23T19:14:21.721156+0000","last_undegraded":"2023-09-23T19:14:21.721156+0000","last_fullsized":"2023-09-23T19:14:21.721156+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:05.073213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":278,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.725408+0000","last_change":"2023-09-23T19:13:02.800175+0000","last_active":"2023-09-23T19:14:11.725408+0000","last_peered":"2023-09-23T19:14:11.725408+0000","last_clean":"2023-09-23T19:14:11.725408+0000","last_became_active":"2023-09-23T19:13:02.799137+0000","last_became_peered":"2023-09-23T19:13:02.799137+0000","last_unstale":"2023-09-23T19:14:11.725408+0000","last_undegraded":"2023-09-23T19:14:11.725408+0000","last_fullsized":"2023-09-23T19:14:11.725408+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":288,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.894983+0000","last_change":"2023-09-23T19:12:02.417052+0000","last_active":"2023-09-23T19:14:21.894983+0000","last_peered":"2023-09-23T19:14:21.894983+0000","last_clean":"2023-09-23T19:14:21.894983+0000","last_became_active":"2023-09-23T19:12:02.416041+0000","last_became_peered":"2023-09-23T19:12:02.416041+0000","last_unstale":"2023-09-23T19:14:21.894983+0000","last_undegraded":"2023-09-23T19:14:21.894983+0000","last_fullsized":"2023-09-23T19:14:21.894983+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:05.013964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":138,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.720928+0000","last_change":"2023-09-23T18:58:38.710920+0000","last_active":"2023-09-23T19:14:21.720928+0000","last_peered":"2023-09-23T19:14:21.720928+0000","last_clean":"2023-09-23T19:14:21.720928+0000","last_became_active":"2023-09-23T18:58:38.710563+0000","last_became_peered":"2023-09-23T18:58:38.710563+0000","last_unstale":"2023-09-23T19:14:21.720928+0000","last_undegraded":"2023-09-23T19:14:21.720928+0000","last_fullsized":"2023-09-23T19:14:21.720928+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:42.386631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":34,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.913207+0000","last_change":"2023-09-23T19:10:37.800599+0000","last_active":"2023-09-23T19:14:11.913207+0000","last_peered":"2023-09-23T19:14:11.913207+0000","last_clean":"2023-09-23T19:14:11.913207+0000","last_became_active":"2023-09-23T19:10:37.800130+0000","last_became_peered":"2023-09-23T19:10:37.800130+0000","last_unstale":"2023-09-23T19:14:11.913207+0000","last_undegraded":"2023-09-23T19:14:11.913207+0000","last_fullsized":"2023-09-23T19:14:11.913207+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":398,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.893671+0000","last_change":"2023-09-23T19:12:02.284586+0000","last_active":"2023-09-23T19:14:21.893671+0000","last_peered":"2023-09-23T19:14:21.893671+0000","last_clean":"2023-09-23T19:14:21.893671+0000","last_became_active":"2023-09-23T19:12:02.284069+0000","last_became_peered":"2023-09-23T19:12:02.284069+0000","last_unstale":"2023-09-23T19:14:21.893671+0000","last_undegraded":"2023-09-23T19:14:21.893671+0000","last_fullsized":"2023-09-23T19:14:21.893671+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:07:35.047940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":316,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.852039+0000","last_change":"2023-09-23T19:11:18.008462+0000","last_active":"2023-09-23T19:14:12.852039+0000","last_peered":"2023-09-23T19:14:12.852039+0000","last_clean":"2023-09-23T19:14:12.852039+0000","last_became_active":"2023-09-23T19:11:17.691174+0000","last_became_peered":"2023-09-23T19:11:17.691174+0000","last_unstale":"2023-09-23T19:14:12.852039+0000","last_undegraded":"2023-09-23T19:14:12.852039+0000","last_fullsized":"2023-09-23T19:14:12.852039+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":367,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.348456+0000","last_change":"2023-09-23T18:33:53.850096+0000","last_active":"2023-09-23T19:14:12.348456+0000","last_peered":"2023-09-23T19:14:12.348456+0000","last_clean":"2023-09-23T19:14:12.348456+0000","last_became_active":"2023-09-23T18:33:53.849811+0000","last_became_peered":"2023-09-23T18:33:53.849811+0000","last_unstale":"2023-09-23T19:14:12.348456+0000","last_undegraded":"2023-09-23T19:14:12.348456+0000","last_fullsized":"2023-09-23T19:14:12.348456+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:44.938072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":41,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.725910+0000","last_change":"2023-09-23T19:11:17.699173+0000","last_active":"2023-09-23T19:14:11.725910+0000","last_peered":"2023-09-23T19:14:11.725910+0000","last_clean":"2023-09-23T19:14:11.725910+0000","last_became_active":"2023-09-23T19:11:17.685016+0000","last_became_peered":"2023-09-23T19:11:17.685016+0000","last_unstale":"2023-09-23T19:14:11.725910+0000","last_undegraded":"2023-09-23T19:14:11.725910+0000","last_fullsized":"2023-09-23T19:14:11.725910+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":31,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.725813+0000","last_change":"2023-09-23T19:11:17.683010+0000","last_active":"2023-09-23T19:14:11.725813+0000","last_peered":"2023-09-23T19:14:11.725813+0000","last_clean":"2023-09-23T19:14:11.725813+0000","last_became_active":"2023-09-23T19:11:17.682613+0000","last_became_peered":"2023-09-23T19:11:17.682613+0000","last_unstale":"2023-09-23T19:14:11.725813+0000","last_undegraded":"2023-09-23T19:14:11.725813+0000","last_fullsized":"2023-09-23T19:14:11.725813+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":315,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.851840+0000","last_change":"2023-09-23T19:11:17.832325+0000","last_active":"2023-09-23T19:14:12.851840+0000","last_peered":"2023-09-23T19:14:12.851840+0000","last_clean":"2023-09-23T19:14:12.851840+0000","last_became_active":"2023-09-23T19:11:17.688749+0000","last_became_peered":"2023-09-23T19:11:17.688749+0000","last_unstale":"2023-09-23T19:14:12.851840+0000","last_undegraded":"2023-09-23T19:14:12.851840+0000","last_fullsized":"2023-09-23T19:14:12.851840+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":439,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.721662+0000","last_change":"2023-09-23T19:09:03.319623+0000","last_active":"2023-09-23T19:14:21.721662+0000","last_peered":"2023-09-23T19:14:21.721662+0000","last_clean":"2023-09-23T19:14:21.721662+0000","last_became_active":"2023-09-23T19:09:03.319283+0000","last_became_peered":"2023-09-23T19:09:03.319283+0000","last_unstale":"2023-09-23T19:14:21.721662+0000","last_undegraded":"2023-09-23T19:14:21.721662+0000","last_fullsized":"2023-09-23T19:14:21.721662+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:34.646037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":252,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.895570+0000","last_change":"2023-09-23T19:12:02.205046+0000","last_active":"2023-09-23T19:14:21.895570+0000","last_peered":"2023-09-23T19:14:21.895570+0000","last_clean":"2023-09-23T19:14:21.895570+0000","last_became_active":"2023-09-23T19:12:02.204322+0000","last_became_peered":"2023-09-23T19:12:02.204322+0000","last_unstale":"2023-09-23T19:14:21.895570+0000","last_undegraded":"2023-09-23T19:14:21.895570+0000","last_fullsized":"2023-09-23T19:14:21.895570+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:35.102992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":34,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.913075+0000","last_change":"2023-09-23T19:10:37.801034+0000","last_active":"2023-09-23T19:14:11.913075+0000","last_peered":"2023-09-23T19:14:11.913075+0000","last_clean":"2023-09-23T19:14:11.913075+0000","last_became_active":"2023-09-23T19:10:37.800310+0000","last_became_peered":"2023-09-23T19:10:37.800310+0000","last_unstale":"2023-09-23T19:14:11.913075+0000","last_undegraded":"2023-09-23T19:14:11.913075+0000","last_fullsized":"2023-09-23T19:14:11.913075+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":292,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.851478+0000","last_change":"2023-09-23T19:11:17.685822+0000","last_active":"2023-09-23T19:14:12.851478+0000","last_peered":"2023-09-23T19:14:12.851478+0000","last_clean":"2023-09-23T19:14:12.851478+0000","last_became_active":"2023-09-23T19:11:17.685456+0000","last_became_peered":"2023-09-23T19:11:17.685456+0000","last_unstale":"2023-09-23T19:14:12.851478+0000","last_undegraded":"2023-09-23T19:14:12.851478+0000","last_fullsized":"2023-09-23T19:14:12.851478+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":253,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.894617+0000","last_change":"2023-09-23T19:12:02.204580+0000","last_active":"2023-09-23T19:14:21.894617+0000","last_peered":"2023-09-23T19:14:21.894617+0000","last_clean":"2023-09-23T19:14:21.894617+0000","last_became_active":"2023-09-23T19:12:02.203730+0000","last_became_peered":"2023-09-23T19:12:02.203730+0000","last_unstale":"2023-09-23T19:14:21.894617+0000","last_undegraded":"2023-09-23T19:14:21.894617+0000","last_fullsized":"2023-09-23T19:14:21.894617+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:54.397927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":285,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.724282+0000","last_change":"2023-09-23T19:13:02.721633+0000","last_active":"2023-09-23T19:14:11.724282+0000","last_peered":"2023-09-23T19:14:11.724282+0000","last_clean":"2023-09-23T19:14:11.724282+0000","last_became_active":"2023-09-23T19:13:02.721035+0000","last_became_peered":"2023-09-23T19:13:02.721035+0000","last_unstale":"2023-09-23T19:14:11.724282+0000","last_undegraded":"2023-09-23T19:14:11.724282+0000","last_fullsized":"2023-09-23T19:14:11.724282+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":90,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.721292+0000","last_change":"2023-09-23T19:09:03.318996+0000","last_active":"2023-09-23T19:14:21.721292+0000","last_peered":"2023-09-23T19:14:21.721292+0000","last_clean":"2023-09-23T19:14:21.721292+0000","last_became_active":"2023-09-23T19:09:03.318593+0000","last_became_peered":"2023-09-23T19:09:03.318593+0000","last_unstale":"2023-09-23T19:14:21.721292+0000","last_undegraded":"2023-09-23T19:14:21.721292+0000","last_fullsized":"2023-09-23T19:14:21.721292+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:23.764508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":293,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.852152+0000","last_change":"2023-09-23T19:11:17.876664+0000","last_active":"2023-09-23T19:14:12.852152+0000","last_peered":"2023-09-23T19:14:12.852152+0000","last_clean":"2023-09-23T19:14:12.852152+0000","last_became_active":"2023-09-23T19:11:17.689646+0000","last_became_peered":"2023-09-23T19:11:17.689646+0000","last_unstale":"2023-09-23T19:14:12.852152+0000","last_undegraded":"2023-09-23T19:14:12.852152+0000","last_fullsized":"2023-09-23T19:14:12.852152+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":34,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.912486+0000","last_change":"2023-09-23T19:10:37.800494+0000","last_active":"2023-09-23T19:14:11.912486+0000","last_peered":"2023-09-23T19:14:11.912486+0000","last_clean":"2023-09-23T19:14:11.912486+0000","last_became_active":"2023-09-23T19:10:37.800131+0000","last_became_peered":"2023-09-23T19:10:37.800131+0000","last_unstale":"2023-09-23T19:14:11.912486+0000","last_undegraded":"2023-09-23T19:14:11.912486+0000","last_fullsized":"2023-09-23T19:14:11.912486+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":328,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.851994+0000","last_change":"2023-09-23T19:11:17.921164+0000","last_active":"2023-09-23T19:14:12.851994+0000","last_peered":"2023-09-23T19:14:12.851994+0000","last_clean":"2023-09-23T19:14:12.851994+0000","last_became_active":"2023-09-23T19:11:17.690216+0000","last_became_peered":"2023-09-23T19:11:17.690216+0000","last_unstale":"2023-09-23T19:14:12.851994+0000","last_undegraded":"2023-09-23T19:14:12.851994+0000","last_fullsized":"2023-09-23T19:14:12.851994+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":31,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.725008+0000","last_change":"2023-09-23T19:11:17.686637+0000","last_active":"2023-09-23T19:14:11.725008+0000","last_peered":"2023-09-23T19:14:11.725008+0000","last_clean":"2023-09-23T19:14:11.725008+0000","last_became_active":"2023-09-23T19:11:17.684587+0000","last_became_peered":"2023-09-23T19:11:17.684587+0000","last_unstale":"2023-09-23T19:14:11.725008+0000","last_undegraded":"2023-09-23T19:14:11.725008+0000","last_fullsized":"2023-09-23T19:14:11.725008+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":312,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.851717+0000","last_change":"2023-09-23T19:10:37.846671+0000","last_active":"2023-09-23T19:14:12.851717+0000","last_peered":"2023-09-23T19:14:12.851717+0000","last_clean":"2023-09-23T19:14:12.851717+0000","last_became_active":"2023-09-23T19:10:37.800522+0000","last_became_peered":"2023-09-23T19:10:37.800522+0000","last_unstale":"2023-09-23T19:14:12.851717+0000","last_undegraded":"2023-09-23T19:14:12.851717+0000","last_fullsized":"2023-09-23T19:14:12.851717+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":242,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.724738+0000","last_change":"2023-09-23T19:13:02.724651+0000","last_active":"2023-09-23T19:14:11.724738+0000","last_peered":"2023-09-23T19:14:11.724738+0000","last_clean":"2023-09-23T19:14:11.724738+0000","last_became_active":"2023-09-23T19:13:02.724250+0000","last_became_peered":"2023-09-23T19:13:02.724250+0000","last_unstale":"2023-09-23T19:14:11.724738+0000","last_undegraded":"2023-09-23T19:14:11.724738+0000","last_fullsized":"2023-09-23T19:14:11.724738+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":117,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.912849+0000","last_change":"2023-09-23T19:08:37.773654+0000","last_active":"2023-09-23T19:14:11.912849+0000","last_peered":"2023-09-23T19:14:11.912849+0000","last_clean":"2023-09-23T19:14:11.912849+0000","last_became_active":"2023-09-23T19:08:37.773196+0000","last_became_peered":"2023-09-23T19:08:37.773196+0000","last_unstale":"2023-09-23T19:14:11.912849+0000","last_undegraded":"2023-09-23T19:14:11.912849+0000","last_fullsized":"2023-09-23T19:14:11.912849+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":441,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.843791+0000","last_change":"2023-09-23T19:11:17.670732+0000","last_active":"2023-09-23T19:14:12.843791+0000","last_peered":"2023-09-23T19:14:12.843791+0000","last_clean":"2023-09-23T19:14:12.843791+0000","last_became_active":"2023-09-23T19:11:17.670431+0000","last_became_peered":"2023-09-23T19:11:17.670431+0000","last_unstale":"2023-09-23T19:14:12.843791+0000","last_undegraded":"2023-09-23T19:14:12.843791+0000","last_fullsized":"2023-09-23T19:14:12.843791+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:13:31.550360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":168,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.721567+0000","last_change":"2023-09-23T19:11:17.689473+0000","last_active":"2023-09-23T19:14:21.721567+0000","last_peered":"2023-09-23T19:14:21.721567+0000","last_clean":"2023-09-23T19:14:21.721567+0000","last_became_active":"2023-09-23T19:11:17.681348+0000","last_became_peered":"2023-09-23T19:11:17.681348+0000","last_unstale":"2023-09-23T19:14:21.721567+0000","last_undegraded":"2023-09-23T19:14:21.721567+0000","last_fullsized":"2023-09-23T19:14:21.721567+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:55:33.916670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":31,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.725955+0000","last_change":"2023-09-23T19:11:17.681567+0000","last_active":"2023-09-23T19:14:11.725955+0000","last_peered":"2023-09-23T19:14:11.725955+0000","last_clean":"2023-09-23T19:14:11.725955+0000","last_became_active":"2023-09-23T19:11:17.681131+0000","last_became_peered":"2023-09-23T19:11:17.681131+0000","last_unstale":"2023-09-23T19:14:11.725955+0000","last_undegraded":"2023-09-23T19:14:11.725955+0000","last_fullsized":"2023-09-23T19:14:11.725955+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":235,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.724503+0000","last_change":"2023-09-23T19:13:02.800517+0000","last_active":"2023-09-23T19:14:11.724503+0000","last_peered":"2023-09-23T19:14:11.724503+0000","last_clean":"2023-09-23T19:14:11.724503+0000","last_became_active":"2023-09-23T19:13:02.799204+0000","last_became_peered":"2023-09-23T19:13:02.799204+0000","last_unstale":"2023-09-23T19:14:11.724503+0000","last_undegraded":"2023-09-23T19:14:11.724503+0000","last_fullsized":"2023-09-23T19:14:11.724503+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":31,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.726044+0000","last_change":"2023-09-23T19:11:17.683123+0000","last_active":"2023-09-23T19:14:11.726044+0000","last_peered":"2023-09-23T19:14:11.726044+0000","last_clean":"2023-09-23T19:14:11.726044+0000","last_became_active":"2023-09-23T19:11:17.682648+0000","last_became_peered":"2023-09-23T19:11:17.682648+0000","last_unstale":"2023-09-23T19:14:11.726044+0000","last_undegraded":"2023-09-23T19:14:11.726044+0000","last_fullsized":"2023-09-23T19:14:11.726044+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":31,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.725117+0000","last_change":"2023-09-23T19:11:17.687047+0000","last_active":"2023-09-23T19:14:11.725117+0000","last_peered":"2023-09-23T19:14:11.725117+0000","last_clean":"2023-09-23T19:14:11.725117+0000","last_became_active":"2023-09-23T19:11:17.684963+0000","last_became_peered":"2023-09-23T19:11:17.684963+0000","last_unstale":"2023-09-23T19:14:11.725117+0000","last_undegraded":"2023-09-23T19:14:11.725117+0000","last_fullsized":"2023-09-23T19:14:11.725117+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":31,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.726156+0000","last_change":"2023-09-23T19:11:17.682563+0000","last_active":"2023-09-23T19:14:11.726156+0000","last_peered":"2023-09-23T19:14:11.726156+0000","last_clean":"2023-09-23T19:14:11.726156+0000","last_became_active":"2023-09-23T19:11:17.682142+0000","last_became_peered":"2023-09-23T19:11:17.682142+0000","last_unstale":"2023-09-23T19:14:11.726156+0000","last_undegraded":"2023-09-23T19:14:11.726156+0000","last_fullsized":"2023-09-23T19:14:11.726156+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":113,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:16.297026+0000","last_change":"2023-09-23T19:09:03.322253+0000","last_active":"2023-09-23T19:14:16.297026+0000","last_peered":"2023-09-23T19:14:16.297026+0000","last_clean":"2023-09-23T19:14:16.297026+0000","last_became_active":"2023-09-23T19:09:03.320102+0000","last_became_peered":"2023-09-23T19:09:03.320102+0000","last_unstale":"2023-09-23T19:14:16.297026+0000","last_undegraded":"2023-09-23T19:14:16.297026+0000","last_fullsized":"2023-09-23T19:14:16.297026+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:01.268221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":436,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.894960+0000","last_change":"2023-09-23T19:11:17.680502+0000","last_active":"2023-09-23T19:14:21.894960+0000","last_peered":"2023-09-23T19:14:21.894960+0000","last_clean":"2023-09-23T19:14:21.894960+0000","last_became_active":"2023-09-23T19:11:17.679955+0000","last_became_peered":"2023-09-23T19:11:17.679955+0000","last_unstale":"2023-09-23T19:14:21.894960+0000","last_undegraded":"2023-09-23T19:14:21.894960+0000","last_fullsized":"2023-09-23T19:14:21.894960+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:01.547326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"383'306","reported_seq":1477,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.894901+0000","last_change":"2023-09-23T19:09:23.687377+0000","last_active":"2023-09-23T19:14:21.894901+0000","last_peered":"2023-09-23T19:14:21.894901+0000","last_clean":"2023-09-23T19:14:21.894901+0000","last_became_active":"2023-09-23T19:09:02.306435+0000","last_became_peered":"2023-09-23T19:09:02.306435+0000","last_unstale":"2023-09-23T19:14:21.894901+0000","last_undegraded":"2023-09-23T19:14:21.894901+0000","last_fullsized":"2023-09-23T19:14:21.894901+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":306,"log_dups_size":0,"ondisk_log_size":306,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:54:30.990476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086649800000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1624,"num_read_kb":2591,"num_write":297,"num_write_kb":3316,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":117,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.913144+0000","last_change":"2023-09-23T19:08:37.774207+0000","last_active":"2023-09-23T19:14:11.913144+0000","last_peered":"2023-09-23T19:14:11.913144+0000","last_clean":"2023-09-23T19:14:11.913144+0000","last_became_active":"2023-09-23T19:08:37.773764+0000","last_became_peered":"2023-09-23T19:08:37.773764+0000","last_unstale":"2023-09-23T19:14:11.913144+0000","last_undegraded":"2023-09-23T19:14:11.913144+0000","last_fullsized":"2023-09-23T19:14:11.913144+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":427,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.894332+0000","last_change":"2023-09-23T19:11:17.680196+0000","last_active":"2023-09-23T19:14:21.894332+0000","last_peered":"2023-09-23T19:14:21.894332+0000","last_clean":"2023-09-23T19:14:21.894332+0000","last_became_active":"2023-09-23T19:11:17.679638+0000","last_became_peered":"2023-09-23T19:11:17.679638+0000","last_unstale":"2023-09-23T19:14:21.894332+0000","last_undegraded":"2023-09-23T19:14:21.894332+0000","last_fullsized":"2023-09-23T19:14:21.894332+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:30.323348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"383'125","reported_seq":500,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.912703+0000","last_change":"2023-09-23T19:09:23.642827+0000","last_active":"2023-09-23T19:14:11.912703+0000","last_peered":"2023-09-23T19:14:11.912703+0000","last_clean":"2023-09-23T19:14:11.912703+0000","last_became_active":"2023-09-23T19:08:37.774901+0000","last_became_peered":"2023-09-23T19:08:37.774901+0000","last_unstale":"2023-09-23T19:14:11.912703+0000","last_undegraded":"2023-09-23T19:14:11.912703+0000","last_fullsized":"2023-09-23T19:14:11.912703+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":125,"log_dups_size":0,"ondisk_log_size":125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1121,"num_read_kb":2820,"num_write":107,"num_write_kb":1462,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":31,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.725875+0000","last_change":"2023-09-23T19:11:17.679851+0000","last_active":"2023-09-23T19:14:11.725875+0000","last_peered":"2023-09-23T19:14:11.725875+0000","last_clean":"2023-09-23T19:14:11.725875+0000","last_became_active":"2023-09-23T19:11:17.679454+0000","last_became_peered":"2023-09-23T19:11:17.679454+0000","last_unstale":"2023-09-23T19:14:11.725875+0000","last_undegraded":"2023-09-23T19:14:11.725875+0000","last_fullsized":"2023-09-23T19:14:11.725875+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":435,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.895136+0000","last_change":"2023-09-23T18:58:38.726546+0000","last_active":"2023-09-23T19:14:21.895136+0000","last_peered":"2023-09-23T19:14:21.895136+0000","last_clean":"2023-09-23T19:14:21.895136+0000","last_became_active":"2023-09-23T18:58:38.724765+0000","last_became_peered":"2023-09-23T18:58:38.724765+0000","last_unstale":"2023-09-23T19:14:21.895136+0000","last_undegraded":"2023-09-23T19:14:21.895136+0000","last_fullsized":"2023-09-23T19:14:21.895136+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:05.322676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"383'289","reported_seq":1337,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.894396+0000","last_change":"2023-09-23T19:09:23.637617+0000","last_active":"2023-09-23T19:14:21.894396+0000","last_peered":"2023-09-23T19:14:21.894396+0000","last_clean":"2023-09-23T19:14:21.894396+0000","last_became_active":"2023-09-23T18:58:38.707923+0000","last_became_peered":"2023-09-23T18:58:38.707923+0000","last_unstale":"2023-09-23T19:14:21.894396+0000","last_undegraded":"2023-09-23T19:14:21.894396+0000","last_fullsized":"2023-09-23T19:14:21.894396+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":289,"log_dups_size":0,"ondisk_log_size":289,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:59.471903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047940899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1602,"num_read_kb":4202,"num_write":289,"num_write_kb":2667,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":370,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.348215+0000","last_change":"2023-09-23T18:33:53.849084+0000","last_active":"2023-09-23T19:14:12.348215+0000","last_peered":"2023-09-23T19:14:12.348215+0000","last_clean":"2023-09-23T19:14:12.348215+0000","last_became_active":"2023-09-23T18:33:53.848757+0000","last_became_peered":"2023-09-23T18:33:53.848757+0000","last_unstale":"2023-09-23T19:14:12.348215+0000","last_undegraded":"2023-09-23T19:14:12.348215+0000","last_fullsized":"2023-09-23T19:14:12.348215+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:15.267742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":440,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.895133+0000","last_change":"2023-09-23T19:08:37.776940+0000","last_active":"2023-09-23T19:14:21.895133+0000","last_peered":"2023-09-23T19:14:21.895133+0000","last_clean":"2023-09-23T19:14:21.895133+0000","last_became_active":"2023-09-23T19:08:37.776564+0000","last_became_peered":"2023-09-23T19:08:37.776564+0000","last_unstale":"2023-09-23T19:14:21.895133+0000","last_undegraded":"2023-09-23T19:14:21.895133+0000","last_fullsized":"2023-09-23T19:14:21.895133+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:44.299192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"383'317","reported_seq":1387,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.912779+0000","last_change":"2023-09-23T19:09:23.643011+0000","last_active":"2023-09-23T19:14:11.912779+0000","last_peered":"2023-09-23T19:14:11.912779+0000","last_clean":"2023-09-23T19:14:11.912779+0000","last_became_active":"2023-09-23T19:09:03.312411+0000","last_became_peered":"2023-09-23T19:09:03.312411+0000","last_unstale":"2023-09-23T19:14:11.912779+0000","last_undegraded":"2023-09-23T19:14:11.912779+0000","last_fullsized":"2023-09-23T19:14:11.912779+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":317,"log_dups_size":0,"ondisk_log_size":317,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1525,"num_read_kb":2676,"num_write":278,"num_write_kb":5451,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":520,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.851120+0000","last_change":"2023-09-23T19:09:02.309826+0000","last_active":"2023-09-23T19:14:12.851120+0000","last_peered":"2023-09-23T19:14:12.851120+0000","last_clean":"2023-09-23T19:14:12.851120+0000","last_became_active":"2023-09-23T19:09:02.309300+0000","last_became_peered":"2023-09-23T19:09:02.309300+0000","last_unstale":"2023-09-23T19:14:12.851120+0000","last_undegraded":"2023-09-23T19:14:12.851120+0000","last_fullsized":"2023-09-23T19:14:12.851120+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_clean_scrub_stamp":"2023-09-23T18:08:35.701374+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":176,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.721038+0000","last_change":"2023-09-23T19:10:37.797301+0000","last_active":"2023-09-23T19:14:21.721038+0000","last_peered":"2023-09-23T19:14:21.721038+0000","last_clean":"2023-09-23T19:14:21.721038+0000","last_became_active":"2023-09-23T19:10:37.796972+0000","last_became_peered":"2023-09-23T19:10:37.796972+0000","last_unstale":"2023-09-23T19:14:21.721038+0000","last_undegraded":"2023-09-23T19:14:21.721038+0000","last_fullsized":"2023-09-23T19:14:21.721038+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:25.458233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":224,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.724926+0000","last_change":"2023-09-23T19:13:02.727195+0000","last_active":"2023-09-23T19:14:11.724926+0000","last_peered":"2023-09-23T19:14:11.724926+0000","last_clean":"2023-09-23T19:14:11.724926+0000","last_became_active":"2023-09-23T19:13:02.726835+0000","last_became_peered":"2023-09-23T19:13:02.726835+0000","last_unstale":"2023-09-23T19:14:11.724926+0000","last_undegraded":"2023-09-23T19:14:11.724926+0000","last_fullsized":"2023-09-23T19:14:11.724926+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"383'163","reported_seq":835,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.851118+0000","last_change":"2023-09-23T19:09:23.643786+0000","last_active":"2023-09-23T19:14:12.851118+0000","last_peered":"2023-09-23T19:14:12.851118+0000","last_clean":"2023-09-23T19:14:12.851118+0000","last_became_active":"2023-09-23T19:09:02.310096+0000","last_became_peered":"2023-09-23T19:09:02.310096+0000","last_unstale":"2023-09-23T19:14:12.851118+0000","last_undegraded":"2023-09-23T19:14:12.851118+0000","last_fullsized":"2023-09-23T19:14:12.851118+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":163,"log_dups_size":0,"ondisk_log_size":163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1010,"num_read_kb":4243,"num_write":132,"num_write_kb":3279,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":486,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.893871+0000","last_change":"2023-09-23T19:11:17.667944+0000","last_active":"2023-09-23T19:14:21.893871+0000","last_peered":"2023-09-23T19:14:21.893871+0000","last_clean":"2023-09-23T19:14:21.893871+0000","last_became_active":"2023-09-23T19:11:17.667587+0000","last_became_peered":"2023-09-23T19:11:17.667587+0000","last_unstale":"2023-09-23T19:14:21.893871+0000","last_undegraded":"2023-09-23T19:14:21.893871+0000","last_fullsized":"2023-09-23T19:14:21.893871+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:12.409946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":454,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.843698+0000","last_change":"2023-09-23T18:51:43.087834+0000","last_active":"2023-09-23T19:14:12.843698+0000","last_peered":"2023-09-23T19:14:12.843698+0000","last_clean":"2023-09-23T19:14:12.843698+0000","last_became_active":"2023-09-23T18:51:43.087309+0000","last_became_peered":"2023-09-23T18:51:43.087309+0000","last_unstale":"2023-09-23T19:14:12.843698+0000","last_undegraded":"2023-09-23T19:14:12.843698+0000","last_fullsized":"2023-09-23T19:14:12.843698+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:53.913425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"383'142","reported_seq":826,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.851849+0000","last_change":"2023-09-23T19:11:20.698564+0000","last_active":"2023-09-23T19:14:12.851849+0000","last_peered":"2023-09-23T19:14:12.851849+0000","last_clean":"2023-09-23T19:14:12.851849+0000","last_became_active":"2023-09-23T19:11:20.697584+0000","last_became_peered":"2023-09-23T19:11:20.697584+0000","last_unstale":"2023-09-23T19:14:12.851849+0000","last_undegraded":"2023-09-23T19:14:12.851849+0000","last_fullsized":"2023-09-23T19:14:12.851849+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":142,"log_dups_size":0,"ondisk_log_size":142,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":777,"num_read_kb":2427,"num_write":137,"num_write_kb":5585,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":378,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.724668+0000","last_change":"2023-09-23T19:13:02.724504+0000","last_active":"2023-09-23T19:14:11.724668+0000","last_peered":"2023-09-23T19:14:11.724668+0000","last_clean":"2023-09-23T19:14:11.724668+0000","last_became_active":"2023-09-23T19:13:02.724129+0000","last_became_peered":"2023-09-23T19:13:02.724129+0000","last_unstale":"2023-09-23T19:14:11.724668+0000","last_undegraded":"2023-09-23T19:14:11.724668+0000","last_fullsized":"2023-09-23T19:14:11.724668+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":315,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.844007+0000","last_change":"2023-09-23T19:12:02.416447+0000","last_active":"2023-09-23T19:14:12.844007+0000","last_peered":"2023-09-23T19:14:12.844007+0000","last_clean":"2023-09-23T19:14:12.844007+0000","last_became_active":"2023-09-23T19:12:02.415736+0000","last_became_peered":"2023-09-23T19:12:02.415736+0000","last_unstale":"2023-09-23T19:14:12.844007+0000","last_undegraded":"2023-09-23T19:14:12.844007+0000","last_fullsized":"2023-09-23T19:14:12.844007+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:02:36.968506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":439,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.715759+0000","last_change":"2023-09-23T19:09:03.319699+0000","last_active":"2023-09-23T19:14:11.715759+0000","last_peered":"2023-09-23T19:14:11.715759+0000","last_clean":"2023-09-23T19:14:11.715759+0000","last_became_active":"2023-09-23T19:09:03.318772+0000","last_became_peered":"2023-09-23T19:09:03.318772+0000","last_unstale":"2023-09-23T19:14:11.715759+0000","last_undegraded":"2023-09-23T19:14:11.715759+0000","last_fullsized":"2023-09-23T19:14:11.715759+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:41.224329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"383'372","reported_seq":1735,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.895077+0000","last_change":"2023-09-23T19:11:20.693753+0000","last_active":"2023-09-23T19:14:21.895077+0000","last_peered":"2023-09-23T19:14:21.895077+0000","last_clean":"2023-09-23T19:14:21.895077+0000","last_became_active":"2023-09-23T19:11:20.692775+0000","last_became_peered":"2023-09-23T19:11:20.692775+0000","last_unstale":"2023-09-23T19:14:21.895077+0000","last_undegraded":"2023-09-23T19:14:21.895077+0000","last_fullsized":"2023-09-23T19:14:21.895077+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":372,"log_dups_size":0,"ondisk_log_size":372,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:13:44.744641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067870799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2145,"num_read_kb":3069,"num_write":293,"num_write_kb":1311,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":116,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.720758+0000","last_change":"2023-09-23T19:09:03.322018+0000","last_active":"2023-09-23T19:14:21.720758+0000","last_peered":"2023-09-23T19:14:21.720758+0000","last_clean":"2023-09-23T19:14:21.720758+0000","last_became_active":"2023-09-23T19:09:03.321316+0000","last_became_peered":"2023-09-23T19:09:03.321316+0000","last_unstale":"2023-09-23T19:14:21.720758+0000","last_undegraded":"2023-09-23T19:14:21.720758+0000","last_fullsized":"2023-09-23T19:14:21.720758+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:46:37.431152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"383'236","reported_seq":1464,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.725411+0000","last_change":"2023-09-23T19:13:02.725131+0000","last_active":"2023-09-23T19:14:11.725411+0000","last_peered":"2023-09-23T19:14:11.725411+0000","last_clean":"2023-09-23T19:14:11.725411+0000","last_became_active":"2023-09-23T19:13:02.724555+0000","last_became_peered":"2023-09-23T19:13:02.724555+0000","last_unstale":"2023-09-23T19:14:11.725411+0000","last_undegraded":"2023-09-23T19:14:11.725411+0000","last_fullsized":"2023-09-23T19:14:11.725411+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":236,"log_dups_size":0,"ondisk_log_size":236,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2130,"num_read_kb":4337,"num_write":157,"num_write_kb":2355,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":471,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.843926+0000","last_change":"2023-09-23T19:11:17.741918+0000","last_active":"2023-09-23T19:14:12.843926+0000","last_peered":"2023-09-23T19:14:12.843926+0000","last_clean":"2023-09-23T19:14:12.843926+0000","last_became_active":"2023-09-23T19:11:17.669939+0000","last_became_peered":"2023-09-23T19:11:17.669939+0000","last_unstale":"2023-09-23T19:14:12.843926+0000","last_undegraded":"2023-09-23T19:14:12.843926+0000","last_fullsized":"2023-09-23T19:14:12.843926+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:49.886612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":31,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.724945+0000","last_change":"2023-09-23T19:11:17.682022+0000","last_active":"2023-09-23T19:14:11.724945+0000","last_peered":"2023-09-23T19:14:11.724945+0000","last_clean":"2023-09-23T19:14:11.724945+0000","last_became_active":"2023-09-23T19:11:17.681383+0000","last_became_peered":"2023-09-23T19:11:17.681383+0000","last_unstale":"2023-09-23T19:14:11.724945+0000","last_undegraded":"2023-09-23T19:14:11.724945+0000","last_fullsized":"2023-09-23T19:14:11.724945+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":419,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.844043+0000","last_change":"2023-09-23T19:11:17.676241+0000","last_active":"2023-09-23T19:14:12.844043+0000","last_peered":"2023-09-23T19:14:12.844043+0000","last_clean":"2023-09-23T19:14:12.844043+0000","last_became_active":"2023-09-23T19:11:17.675971+0000","last_became_peered":"2023-09-23T19:11:17.675971+0000","last_unstale":"2023-09-23T19:14:12.844043+0000","last_undegraded":"2023-09-23T19:14:12.844043+0000","last_fullsized":"2023-09-23T19:14:12.844043+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:48:01.923477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":211,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.894601+0000","last_change":"2023-09-23T18:58:38.715946+0000","last_active":"2023-09-23T19:14:21.894601+0000","last_peered":"2023-09-23T19:14:21.894601+0000","last_clean":"2023-09-23T19:14:21.894601+0000","last_became_active":"2023-09-23T18:58:38.715584+0000","last_became_peered":"2023-09-23T18:58:38.715584+0000","last_unstale":"2023-09-23T19:14:21.894601+0000","last_undegraded":"2023-09-23T19:14:21.894601+0000","last_fullsized":"2023-09-23T19:14:21.894601+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:57.811713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":146,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.912992+0000","last_change":"2023-09-23T19:11:17.690830+0000","last_active":"2023-09-23T19:14:11.912992+0000","last_peered":"2023-09-23T19:14:11.912992+0000","last_clean":"2023-09-23T19:14:11.912992+0000","last_became_active":"2023-09-23T19:11:17.672837+0000","last_became_peered":"2023-09-23T19:11:17.672837+0000","last_unstale":"2023-09-23T19:14:11.912992+0000","last_undegraded":"2023-09-23T19:14:11.912992+0000","last_fullsized":"2023-09-23T19:14:11.912992+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":31,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.726051+0000","last_change":"2023-09-23T19:11:17.682804+0000","last_active":"2023-09-23T19:14:11.726051+0000","last_peered":"2023-09-23T19:14:11.726051+0000","last_clean":"2023-09-23T19:14:11.726051+0000","last_became_active":"2023-09-23T19:11:17.682024+0000","last_became_peered":"2023-09-23T19:11:17.682024+0000","last_unstale":"2023-09-23T19:14:11.726051+0000","last_undegraded":"2023-09-23T19:14:11.726051+0000","last_fullsized":"2023-09-23T19:14:11.726051+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":146,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.913314+0000","last_change":"2023-09-23T19:11:17.742516+0000","last_active":"2023-09-23T19:14:11.913314+0000","last_peered":"2023-09-23T19:14:11.913314+0000","last_clean":"2023-09-23T19:14:11.913314+0000","last_became_active":"2023-09-23T19:11:17.675587+0000","last_became_peered":"2023-09-23T19:11:17.675587+0000","last_unstale":"2023-09-23T19:14:11.913314+0000","last_undegraded":"2023-09-23T19:14:11.913314+0000","last_fullsized":"2023-09-23T19:14:11.913314+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":31,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.725285+0000","last_change":"2023-09-23T19:11:17.684025+0000","last_active":"2023-09-23T19:14:11.725285+0000","last_peered":"2023-09-23T19:14:11.725285+0000","last_clean":"2023-09-23T19:14:11.725285+0000","last_became_active":"2023-09-23T19:11:17.683571+0000","last_became_peered":"2023-09-23T19:11:17.683571+0000","last_unstale":"2023-09-23T19:14:11.725285+0000","last_undegraded":"2023-09-23T19:14:11.725285+0000","last_fullsized":"2023-09-23T19:14:11.725285+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":139,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.721291+0000","last_change":"2023-09-23T18:58:38.711301+0000","last_active":"2023-09-23T19:14:21.721291+0000","last_peered":"2023-09-23T19:14:21.721291+0000","last_clean":"2023-09-23T19:14:21.721291+0000","last_became_active":"2023-09-23T18:58:38.710923+0000","last_became_peered":"2023-09-23T18:58:38.710923+0000","last_unstale":"2023-09-23T19:14:21.721291+0000","last_undegraded":"2023-09-23T19:14:21.721291+0000","last_fullsized":"2023-09-23T19:14:21.721291+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:17.756147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":138,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.721243+0000","last_change":"2023-09-23T18:58:38.711839+0000","last_active":"2023-09-23T19:14:21.721243+0000","last_peered":"2023-09-23T19:14:21.721243+0000","last_clean":"2023-09-23T19:14:21.721243+0000","last_became_active":"2023-09-23T18:58:38.711347+0000","last_became_peered":"2023-09-23T18:58:38.711347+0000","last_unstale":"2023-09-23T19:14:21.721243+0000","last_undegraded":"2023-09-23T19:14:21.721243+0000","last_fullsized":"2023-09-23T19:14:21.721243+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:55.319645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":497,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.895183+0000","last_change":"2023-09-23T19:08:37.776379+0000","last_active":"2023-09-23T19:14:21.895183+0000","last_peered":"2023-09-23T19:14:21.895183+0000","last_clean":"2023-09-23T19:14:21.895183+0000","last_became_active":"2023-09-23T19:08:37.776012+0000","last_became_peered":"2023-09-23T19:08:37.776012+0000","last_unstale":"2023-09-23T19:14:21.895183+0000","last_undegraded":"2023-09-23T19:14:21.895183+0000","last_fullsized":"2023-09-23T19:14:21.895183+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:16.823018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":31,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.724306+0000","last_change":"2023-09-23T19:11:17.681695+0000","last_active":"2023-09-23T19:14:11.724306+0000","last_peered":"2023-09-23T19:14:11.724306+0000","last_clean":"2023-09-23T19:14:11.724306+0000","last_became_active":"2023-09-23T19:11:17.681228+0000","last_became_peered":"2023-09-23T19:11:17.681228+0000","last_unstale":"2023-09-23T19:14:11.724306+0000","last_undegraded":"2023-09-23T19:14:11.724306+0000","last_fullsized":"2023-09-23T19:14:11.724306+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":13204,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56162,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11934,"num_read_kb":26365,"num_write":1690,"num_write_kb":25426,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":159,"num_bytes_recovered":16777235,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7733,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":367,"seq":1576252997714,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":347624,"kb_used_data":12768,"kb_used_omap":13,"kb_used_meta":334834,"kb_avail":93397528,"statfs":{"total":95995035648,"available":95639068672,"internally_reserved":0,"allocated":13074432,"data_stored":12088504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14028,"internal_metadata":342870324},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:15:14 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.94799999999999995,"15min":0.92000000000000004},"min":{"1min":0.52500000000000002,"5min":0.50800000000000001,"15min":0.48099999999999998},"max":{"1min":1.222,"5min":1.738,"15min":1.738},"last":0.871},{"interface":"front","average":{"1min":0.82499999999999996,"5min":0.89000000000000001,"15min":0.83499999999999996},"min":{"1min":0.55300000000000005,"5min":0.46000000000000002,"15min":0.36599999999999999},"max":{"1min":1.0880000000000001,"5min":1.3959999999999999,"15min":1.3959999999999999},"last":0.79100000000000004}]},{"osd":1,"last update":"Sat Sep 23 19:15:14 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":0.94499999999999995,"15min":0.91600000000000004},"min":{"1min":0.61699999999999999,"5min":0.53000000000000003,"15min":0.49299999999999999},"max":{"1min":1.125,"5min":1.478,"15min":1.478},"last":1.075},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.98399999999999999,"15min":0.93200000000000005},"min":{"1min":0.60199999999999998,"5min":0.59099999999999997,"15min":0.46400000000000002},"max":{"1min":1.3620000000000001,"5min":1.4239999999999999,"15min":1.4379999999999999},"last":1.1419999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:15:14 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.97099999999999997,"15min":1.0009999999999999},"min":{"1min":0.63800000000000001,"5min":0.56100000000000005,"15min":0.53600000000000003},"max":{"1min":1.194,"5min":1.462,"15min":2.4049999999999998},"last":0.93500000000000005},{"interface":"front","average":{"1min":0.97399999999999998,"5min":1.0029999999999999,"15min":1.0209999999999999},"min":{"1min":0.69899999999999995,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.2749999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.54400000000000004}]},{"osd":3,"last update":"Sat Sep 23 19:15:14 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.93799999999999994,"15min":0.871},"min":{"1min":0.72299999999999998,"5min":0.65500000000000003,"15min":0.44900000000000001},"max":{"1min":1.0880000000000001,"5min":1.357,"15min":1.357},"last":0.91200000000000003},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.0049999999999999,"15min":1.048},"min":{"1min":0.63500000000000001,"5min":0.628,"15min":0.57899999999999996},"max":{"1min":1.206,"5min":1.6579999999999999,"15min":2.2749999999999999},"last":1.1220000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:15:14 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":0.98399999999999999,"15min":0.96799999999999997},"min":{"1min":0.65900000000000003,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.472,"5min":1.6399999999999999,"15min":1.6399999999999999},"last":0.96399999999999997},{"interface":"front","average":{"1min":1.0049999999999999,"5min":0.97199999999999998,"15min":0.93999999999999995},"min":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.5029999999999999,"5min":1.6299999999999999,"15min":1.6299999999999999},"last":0.98699999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:15:14 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":1.002,"15min":1.026},"min":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"max":{"1min":1.266,"5min":1.615,"15min":2.3660000000000001},"last":0.82899999999999996},{"interface":"front","average":{"1min":1.056,"5min":1.0880000000000001,"15min":1.0840000000000001},"min":{"1min":0.58899999999999997,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.548,"5min":1.6879999999999999,"15min":1.6879999999999999},"last":0.73599999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.0880000000000001,"15min":1.1100000000000001},"min":{"1min":0.79400000000000004,"5min":0.68400000000000005,"15min":0.64900000000000002},"max":{"1min":1.3140000000000001,"5min":1.7450000000000001,"15min":2.3300000000000001},"last":1.01},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.171,"15min":1.1579999999999999},"min":{"1min":0.753,"5min":0.69999999999999996,"15min":0.69999999999999996},"max":{"1min":1.667,"5min":1.8120000000000001,"15min":1.8120000000000001},"last":1.048}]}]},{"osd":6,"up_from":42,"seq":180388627207,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":899932,"kb_used_data":4564,"kb_used_omap":0,"kb_used_meta":895360,"kb_avail":92845220,"statfs":{"total":95995035648,"available":95073505280,"internally_reserved":0,"allocated":4673536,"data_stored":3699454,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916848640},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:15:09 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":1.0229999999999999,"15min":1.609},"min":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.26300000000000001},"max":{"1min":1.419,"5min":4.5730000000000004,"15min":100.648},"last":1.6120000000000001},{"interface":"front","average":{"1min":1.1379999999999999,"5min":1.119,"15min":1.7110000000000001},"min":{"1min":0.91900000000000004,"5min":0.66500000000000004,"15min":0.317},"max":{"1min":1.351,"5min":4.6449999999999996,"15min":100.70399999999999},"last":1.6879999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:15:09 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":0.89800000000000002,"15min":1.21},"min":{"1min":0.55800000000000005,"5min":0.53900000000000003,"15min":0.29699999999999999},"max":{"1min":1.3080000000000001,"5min":1.3380000000000001,"15min":32.149999999999999},"last":1.891},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.89000000000000001,"15min":1.1419999999999999},"min":{"1min":0.58599999999999997,"5min":0.56599999999999995,"15min":0.27900000000000003},"max":{"1min":1.27,"5min":1.4470000000000001,"15min":32.814},"last":0.91200000000000003}]},{"osd":2,"last update":"Sat Sep 23 19:15:09 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":0.92800000000000005,"15min":1.583},"min":{"1min":0.64300000000000002,"5min":0.57299999999999995,"15min":0.33200000000000002},"max":{"1min":1.226,"5min":1.296,"15min":66.587999999999994},"last":0.95099999999999996},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.012,"15min":1.639},"min":{"1min":0.79300000000000004,"5min":0.61199999999999999,"15min":0.372},"max":{"1min":1.3500000000000001,"5min":1.367,"15min":66.614000000000004},"last":1.034}]},{"osd":3,"last update":"Sat Sep 23 19:15:23 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":0.99099999999999999,"15min":1.3380000000000001},"min":{"1min":0.73699999999999999,"5min":0.61599999999999999,"15min":0.38100000000000001},"max":{"1min":1.329,"5min":1.6259999999999999,"15min":93.456000000000003},"last":1.506},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.0740000000000001,"15min":1.369},"min":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.38900000000000001},"max":{"1min":1.448,"5min":4.593,"15min":93.472999999999999},"last":1.389}]},{"osd":4,"last update":"Sat Sep 23 19:15:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":0.98799999999999999,"15min":0.85099999999999998},"min":{"1min":0.73599999999999999,"5min":0.63600000000000001,"15min":0.36099999999999999},"max":{"1min":1.8049999999999999,"5min":1.8049999999999999,"15min":3.1899999999999999},"last":2.0390000000000001},{"interface":"front","average":{"1min":1.1910000000000001,"5min":1.1739999999999999,"15min":0.996},"min":{"1min":0.68500000000000005,"5min":0.48699999999999999,"15min":0.41799999999999998},"max":{"1min":1.659,"5min":4.8179999999999996,"15min":4.8179999999999996},"last":1.966}]},{"osd":5,"last update":"Sat Sep 23 19:14:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":1.1279999999999999,"15min":1.0529999999999999},"min":{"1min":0.69699999999999995,"5min":0.66900000000000004,"15min":0.54400000000000004},"max":{"1min":1.4850000000000001,"5min":4.7309999999999999,"15min":4.7309999999999999},"last":1.7649999999999999},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.1870000000000001,"15min":1.0409999999999999},"min":{"1min":0.96399999999999997,"5min":0.70399999999999996,"15min":0.55000000000000004},"max":{"1min":1.5740000000000001,"5min":4.7859999999999996,"15min":4.7859999999999996},"last":1.48}]},{"osd":7,"last update":"Sat Sep 23 19:15:09 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.107,"15min":1.089},"min":{"1min":0.80200000000000005,"5min":0.67500000000000004,"15min":0.65400000000000003},"max":{"1min":1.5109999999999999,"5min":4.7009999999999996,"15min":4.7009999999999996},"last":1.5600000000000001},{"interface":"front","average":{"1min":1.077,"5min":1.083,"15min":1.099},"min":{"1min":0.74399999999999999,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.4990000000000001,"5min":4.6260000000000003,"15min":4.6260000000000003},"last":1.2230000000000001}]}]},{"osd":1,"up_from":123,"seq":528280977910,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":997340,"kb_used_data":5204,"kb_used_omap":7,"kb_used_meta":992120,"kb_avail":92747812,"statfs":{"total":95995035648,"available":94973759488,"internally_reserved":0,"allocated":5328896,"data_stored":3262807,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":1015931662},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:15:15 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.68100000000000005,"15min":1.7889999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.23499999999999999},"max":{"1min":0.84099999999999997,"5min":1.238,"15min":90.492000000000004},"last":0.78600000000000003},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.77900000000000003,"15min":1.627},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.188},"max":{"1min":0.97199999999999998,"5min":1.359,"15min":90.552999999999997},"last":0.72299999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:15:15 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.72199999999999998,"15min":1.512},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.219},"max":{"1min":0.96299999999999997,"5min":1.1679999999999999,"15min":87.832999999999998},"last":0.503},{"interface":"front","average":{"1min":0.754,"5min":0.80800000000000005,"15min":1.6990000000000001},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.24299999999999999},"max":{"1min":1.0620000000000001,"5min":1.181,"15min":87.846999999999994},"last":0.749}]},{"osd":3,"last update":"Sat Sep 23 19:15:15 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":0.85099999999999998,"15min":1.3420000000000001},"min":{"1min":0.434,"5min":0.434,"15min":0.24099999999999999},"max":{"1min":1.175,"5min":1.486,"15min":99.126000000000005},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.76300000000000001,"5min":0.85199999999999998,"15min":1.3200000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.21299999999999999},"max":{"1min":1.1200000000000001,"5min":1.4419999999999999,"15min":99.165999999999997},"last":0.35999999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:15:34 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":1.036,"15min":1.0449999999999999},"min":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.32900000000000001},"max":{"1min":1.2709999999999999,"5min":1.548,"15min":7.3070000000000004},"last":0.85399999999999998},{"interface":"front","average":{"1min":0.94799999999999995,"5min":1.0700000000000001,"15min":1.0580000000000001},"min":{"1min":0.69699999999999995,"5min":0.65100000000000002,"15min":0.40699999999999997},"max":{"1min":1.2609999999999999,"5min":1.7829999999999999,"15min":7.5640000000000001},"last":0.68300000000000005}]},{"osd":5,"last update":"Sat Sep 23 19:14:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":1.0429999999999999,"15min":1.0269999999999999},"min":{"1min":0.81499999999999995,"5min":0.67700000000000005,"15min":0.59499999999999997},"max":{"1min":1.948,"5min":1.948,"15min":3.6539999999999999},"last":1.026},{"interface":"front","average":{"1min":1.1259999999999999,"5min":1.0820000000000001,"15min":1.1259999999999999},"min":{"1min":0.78500000000000003,"5min":0.625,"15min":0.55400000000000005},"max":{"1min":1.8380000000000001,"5min":1.8380000000000001,"15min":4.6159999999999997},"last":1.036}]},{"osd":6,"last update":"Sat Sep 23 19:15:15 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":1.0580000000000001,"15min":1.0489999999999999},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.38100000000000001},"max":{"1min":1.21,"5min":1.708,"15min":7.6299999999999999},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.98799999999999999,"5min":1.0640000000000001,"15min":1.2070000000000001},"min":{"1min":0.66900000000000004,"5min":0.65200000000000002,"15min":0.38100000000000001},"max":{"1min":1.274,"5min":1.8640000000000001,"15min":12.887},"last":0.88}]},{"osd":7,"last update":"Sat Sep 23 19:15:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":1.1579999999999999,"15min":1.0840000000000001},"min":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":1.3759999999999999,"5min":1.901,"15min":1.901},"last":0.73099999999999998},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.1459999999999999,"15min":1.0580000000000001},"min":{"1min":0.70499999999999996,"5min":0.69399999999999995,"15min":0.66100000000000003},"max":{"1min":1.3799999999999999,"5min":1.931,"15min":1.931},"last":0.93700000000000006}]}]},{"osd":0,"up_from":88,"seq":377957122640,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1129228,"kb_used_data":9668,"kb_used_omap":8,"kb_used_meta":1119543,"kb_avail":92615924,"statfs":{"total":95995035648,"available":94838706176,"internally_reserved":0,"allocated":9900032,"data_stored":8023266,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":1146412926},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:15:31 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.71399999999999997,"15min":1.7729999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.21099999999999999},"max":{"1min":1.9550000000000001,"5min":1.9550000000000001,"15min":49.168999999999997},"last":0.437},{"interface":"front","average":{"1min":0.76600000000000001,"5min":0.81699999999999995,"15min":1.8580000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.249},"max":{"1min":2.0379999999999998,"5min":2.0379999999999998,"15min":50.673000000000002},"last":0.60699999999999998}]},{"osd":2,"last update":"Sat Sep 23 19:15:13 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.82999999999999996,"15min":1.7490000000000001},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.22600000000000001},"max":{"1min":3.198,"5min":3.198,"15min":52.914000000000001},"last":0.505},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.751,"15min":1.4199999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.22900000000000001},"max":{"1min":1.0309999999999999,"5min":1.2589999999999999,"15min":48.963999999999999},"last":0.70699999999999996}]},{"osd":3,"last update":"Sat Sep 23 19:14:40 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":0.80500000000000005,"15min":1.651},"min":{"1min":0.51300000000000001,"5min":0.42699999999999999,"15min":0.23200000000000001},"max":{"1min":1.196,"5min":1.4390000000000001,"15min":113.375},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.96599999999999997,"5min":0.94299999999999995,"15min":1.778},"min":{"1min":0.73099999999999998,"5min":0.45200000000000001,"15min":0.23899999999999999},"max":{"1min":1.3260000000000001,"5min":1.4670000000000001,"15min":113.327},"last":0.81599999999999995}]},{"osd":4,"last update":"Sat Sep 23 19:14:40 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.95399999999999996,"15min":1.26},"min":{"1min":0.63,"5min":0.61299999999999999,"15min":0.33200000000000002},"max":{"1min":1.1439999999999999,"5min":1.6379999999999999,"15min":55.710999999999999},"last":0.75},{"interface":"front","average":{"1min":1.109,"5min":1.0429999999999999,"15min":1.3360000000000001},"min":{"1min":0.78100000000000003,"5min":0.65600000000000003,"15min":0.38},"max":{"1min":1.5349999999999999,"5min":1.895,"15min":46.509},"last":0.92800000000000005}]},{"osd":5,"last update":"Sat Sep 23 19:14:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":1.085,"15min":1.0269999999999999},"min":{"1min":0.81299999999999994,"5min":0.65800000000000003,"15min":0.57199999999999995},"max":{"1min":1.4590000000000001,"5min":1.8300000000000001,"15min":1.8300000000000001},"last":1.141},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.1160000000000001,"15min":1.0609999999999999},"min":{"1min":0.73699999999999999,"5min":0.61299999999999999,"15min":0.48399999999999999},"max":{"1min":1.4850000000000001,"5min":1.79,"15min":1.79},"last":1.2609999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:15:31 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":1.0449999999999999,"15min":1.2729999999999999},"min":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.378},"max":{"1min":2.2930000000000001,"5min":2.2930000000000001,"15min":55.796999999999997},"last":0.79900000000000004},{"interface":"front","average":{"1min":1.0429999999999999,"5min":1.157,"15min":1.4019999999999999},"min":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.38600000000000001},"max":{"1min":1.3660000000000001,"5min":1.8080000000000001,"15min":46.411999999999999},"last":1.044}]},{"osd":7,"last update":"Sat Sep 23 19:15:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.206,"15min":1.125},"min":{"1min":0.78600000000000003,"5min":0.72299999999999998,"15min":0.72299999999999998},"max":{"1min":1.5880000000000001,"5min":1.712,"15min":1.712},"last":0.98999999999999999},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.1220000000000001,"15min":1.0860000000000001},"min":{"1min":0.746,"5min":0.73699999999999999,"15min":0.73699999999999999},"max":{"1min":2.145,"5min":2.145,"15min":2.145},"last":0.84499999999999997}]}]},{"osd":2,"up_from":98,"seq":420906795588,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1000528,"kb_used_data":9096,"kb_used_omap":11,"kb_used_meta":991412,"kb_avail":92744624,"statfs":{"total":95995035648,"available":94970494976,"internally_reserved":0,"allocated":9314304,"data_stored":7433633,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11949,"internal_metadata":1015206227},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:15:05 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.71799999999999997,"15min":2.6970000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.184},"max":{"1min":1.0820000000000001,"5min":1.179,"15min":143.048},"last":1.0580000000000001},{"interface":"front","average":{"1min":0.747,"5min":0.71299999999999997,"15min":2.7909999999999999},"min":{"1min":0.50700000000000001,"5min":0.38600000000000001,"15min":0.189},"max":{"1min":1.038,"5min":1.111,"15min":143.25200000000001},"last":0.997}]},{"osd":1,"last update":"Sat Sep 23 19:15:05 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":0.81899999999999995,"15min":1.2949999999999999},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.24299999999999999},"max":{"1min":1.1819999999999999,"5min":1.272,"15min":46.601999999999997},"last":0.92600000000000005},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.83499999999999996,"15min":1.4650000000000001},"min":{"1min":0.56100000000000005,"5min":0.34399999999999997,"15min":0.23899999999999999},"max":{"1min":1.377,"5min":1.377,"15min":49.411999999999999},"last":1.3069999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:15:41 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.72799999999999998,"15min":0.92400000000000004},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.24399999999999999},"max":{"1min":1.0449999999999999,"5min":1.157,"15min":7.9619999999999997},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.76700000000000002,"5min":0.86399999999999999,"15min":0.98999999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.23200000000000001},"max":{"1min":1.159,"5min":1.2609999999999999,"15min":9.1679999999999993},"last":1.159}]},{"osd":4,"last update":"Sat Sep 23 19:15:41 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.087,"15min":1.129},"min":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.38800000000000001},"max":{"1min":1.526,"5min":1.597,"15min":8.9969999999999999},"last":1.526},{"interface":"front","average":{"1min":0.96199999999999997,"5min":0.98199999999999998,"15min":1.038},"min":{"1min":0.68200000000000005,"5min":0.57499999999999996,"15min":0.40899999999999997},"max":{"1min":1.266,"5min":1.4199999999999999,"15min":6.0049999999999999},"last":1.1970000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:14:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":1.0700000000000001,"15min":1.008},"min":{"1min":0.80800000000000005,"5min":0.624,"15min":0.55400000000000005},"max":{"1min":1.486,"5min":1.5449999999999999,"15min":1.6020000000000001},"last":1.4259999999999999},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.1759999999999999,"15min":1.0920000000000001},"min":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.544,"5min":1.73,"15min":1.73},"last":1.2569999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:15:05 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.0249999999999999,"15min":1.026},"min":{"1min":0.78400000000000003,"5min":0.61399999999999999,"15min":0.35999999999999999},"max":{"1min":1.3009999999999999,"5min":1.4419999999999999,"15min":5.9649999999999999},"last":1.1060000000000001},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.1859999999999999,"15min":1.179},"min":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.36699999999999999},"max":{"1min":1.4279999999999999,"5min":1.8069999999999999,"15min":9.048},"last":1.4610000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":1.196,"15min":1.143},"min":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"max":{"1min":1.4530000000000001,"5min":1.8560000000000001,"15min":1.8560000000000001},"last":1.3620000000000001},{"interface":"front","average":{"1min":1.159,"5min":1.1279999999999999,"15min":1.1439999999999999},"min":{"1min":0.82999999999999996,"5min":0.69499999999999995,"15min":0.623},"max":{"1min":1.528,"5min":1.528,"15min":1.6879999999999999},"last":1.5489999999999999}]}]},{"osd":3,"up_from":103,"seq":442381632063,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":935904,"kb_used_data":5080,"kb_used_omap":4,"kb_used_meta":930811,"kb_avail":92809248,"statfs":{"total":95995035648,"available":95036669952,"internally_reserved":0,"allocated":5201920,"data_stored":3247104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4826,"internal_metadata":953150758},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:15:35 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.65800000000000003,"15min":1.2849999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.19700000000000001},"max":{"1min":0.879,"5min":1.177,"15min":40.652000000000001},"last":1.0780000000000001},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.748,"15min":1.4019999999999999},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.18099999999999999},"max":{"1min":1.1819999999999999,"5min":1.202,"15min":42.091999999999999},"last":0.77300000000000002}]},{"osd":1,"last update":"Sat Sep 23 19:15:25 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.84499999999999997,"15min":3.1190000000000002},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.252},"max":{"1min":0.95599999999999996,"5min":6.1680000000000001,"15min":343.79300000000001},"last":1.0149999999999999},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.86599999999999999,"15min":3.2069999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.22500000000000001},"max":{"1min":1.0740000000000001,"5min":6.1369999999999996,"15min":343.73000000000002},"last":1.2190000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:15:35 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.77200000000000002,"15min":1.526},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.26300000000000001},"max":{"1min":1.204,"5min":1.204,"15min":61.334000000000003},"last":0.84999999999999998},{"interface":"front","average":{"1min":0.68700000000000006,"5min":0.83599999999999997,"15min":1.417},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.26900000000000002},"max":{"1min":1.115,"5min":1.3080000000000001,"15min":52.493000000000002},"last":0.89100000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:15:35 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.97699999999999998,"15min":1.073},"min":{"1min":0.624,"5min":0.624,"15min":0.41399999999999998},"max":{"1min":1.5660000000000001,"5min":1.5660000000000001,"15min":7.9119999999999999},"last":0.80900000000000005},{"interface":"front","average":{"1min":0.88600000000000001,"5min":1.0369999999999999,"15min":1.0349999999999999},"min":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.42399999999999999},"max":{"1min":1.286,"5min":1.621,"15min":5.0949999999999998},"last":1.196}]},{"osd":5,"last update":"Sat Sep 23 19:14:50 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":1.1259999999999999,"15min":1.004},"min":{"1min":0.747,"5min":0.71599999999999997,"15min":0.60599999999999998},"max":{"1min":1.6060000000000001,"5min":3.1360000000000001,"15min":3.1360000000000001},"last":0.96199999999999997},{"interface":"front","average":{"1min":1.2370000000000001,"5min":1.179,"15min":1.0269999999999999},"min":{"1min":0.71299999999999997,"5min":0.68899999999999995,"15min":0.624},"max":{"1min":1.5669999999999999,"5min":1.621,"15min":1.633},"last":1.127}]},{"osd":6,"last update":"Sat Sep 23 19:15:35 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.1259999999999999,"15min":1.123},"min":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.33700000000000002},"max":{"1min":1.623,"5min":3.0430000000000001,"15min":8.4169999999999998},"last":1.054},{"interface":"front","average":{"1min":0.995,"5min":1.157,"15min":1.125},"min":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.40100000000000002},"max":{"1min":1.4910000000000001,"5min":1.7529999999999999,"15min":5.125},"last":1.2869999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:15:15 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":1.21,"15min":1.1180000000000001},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.6699999999999999,"5min":1.849,"15min":1.849},"last":1.248},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.127,"15min":1.0249999999999999},"min":{"1min":0.752,"5min":0.752,"15min":0.70899999999999996},"max":{"1min":1.6140000000000001,"5min":1.6140000000000001,"15min":1.6140000000000001},"last":1.3080000000000001}]}]},{"osd":4,"up_from":286,"seq":1228360646893,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":369888,"kb_used_data":8088,"kb_used_omap":10,"kb_used_meta":361781,"kb_avail":93375264,"statfs":{"total":95995035648,"available":95616270336,"internally_reserved":0,"allocated":8282112,"data_stored":7303762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11243,"internal_metadata":370463765},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:15:28 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.94299999999999995,"15min":1.1759999999999999},"min":{"1min":0.55400000000000005,"5min":0.5,"15min":0.30599999999999999},"max":{"1min":1.718,"5min":1.718,"15min":47.448999999999998},"last":1.0149999999999999},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.998,"15min":1.171},"min":{"1min":0.55900000000000005,"5min":0.55700000000000005,"15min":0.29399999999999998},"max":{"1min":1.4299999999999999,"5min":1.867,"15min":47.487000000000002},"last":1.151}]},{"osd":1,"last update":"Sat Sep 23 19:15:28 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":0.997,"15min":0.98599999999999999},"min":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.313},"max":{"1min":1.8109999999999999,"5min":1.8109999999999999,"15min":4.7670000000000003},"last":1.1180000000000001},{"interface":"front","average":{"1min":0.96099999999999997,"5min":1.004,"15min":0.98999999999999999},"min":{"1min":0.69499999999999995,"5min":0.58299999999999996,"15min":0.35199999999999998},"max":{"1min":1.55,"5min":2.6880000000000002,"15min":5.0099999999999998},"last":0.70899999999999996}]},{"osd":2,"last update":"Sat Sep 23 19:15:28 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":1.0349999999999999,"15min":1.0600000000000001},"min":{"1min":0.69799999999999995,"5min":0.63200000000000001,"15min":0.35099999999999998},"max":{"1min":1.603,"5min":2.4039999999999999,"15min":15.493},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.97399999999999998,"5min":1.002,"15min":1.091},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.34799999999999998},"max":{"1min":1.514,"5min":1.514,"15min":15.416},"last":0.73599999999999999}]},{"osd":3,"last update":"Sat Sep 23 19:15:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":1.0249999999999999,"15min":1.288},"min":{"1min":0.75900000000000001,"5min":0.68200000000000005,"15min":0.39400000000000002},"max":{"1min":1.617,"5min":1.617,"15min":62.235999999999997},"last":1.042},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.0289999999999999,"15min":1.2390000000000001},"min":{"1min":0.69899999999999995,"5min":0.66100000000000003,"15min":0.33900000000000002},"max":{"1min":1.8620000000000001,"5min":1.8620000000000001,"15min":61.423000000000002},"last":0.89000000000000001}]},{"osd":5,"last update":"Sat Sep 23 19:14:46 2023","interfaces":[{"interface":"back","average":{"1min":1.081,"5min":0.997,"15min":1.002},"min":{"1min":0.71099999999999997,"5min":0.56499999999999995,"15min":0.53500000000000003},"max":{"1min":2.609,"5min":2.609,"15min":2.609},"last":0.95999999999999996},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.0489999999999999,"15min":0.94699999999999995},"min":{"1min":0.78200000000000003,"5min":0.59899999999999998,"15min":0.51700000000000002},"max":{"1min":1.484,"5min":1.589,"15min":1.7430000000000001},"last":0.85199999999999998}]},{"osd":6,"last update":"Sat Sep 23 19:15:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1739999999999999,"5min":1.1080000000000001,"15min":0.93400000000000005},"min":{"1min":0.91000000000000003,"5min":0.56799999999999995,"15min":0.40100000000000002},"max":{"1min":1.8420000000000001,"5min":1.8420000000000001,"15min":2.0059999999999998},"last":1.1739999999999999},{"interface":"front","average":{"1min":1.206,"5min":1.1040000000000001,"15min":0.94799999999999995},"min":{"1min":0.86599999999999999,"5min":0.56599999999999995,"15min":0.371},"max":{"1min":1.9179999999999999,"5min":1.9179999999999999,"15min":2.8359999999999999},"last":1.22}]},{"osd":7,"last update":"Sat Sep 23 19:15:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2769999999999999,"5min":1.181,"15min":1.151},"min":{"1min":1.004,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.9319999999999999,"5min":1.9319999999999999,"15min":2.1819999999999999},"last":1.2649999999999999},{"interface":"front","average":{"1min":1.319,"5min":1.1819999999999999,"15min":1.1879999999999999},"min":{"1min":0.86099999999999999,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":2.23,"5min":2.23,"15min":2.23},"last":1.0900000000000001}]}]},{"osd":5,"up_from":360,"seq":1546188226647,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":345868,"kb_used_data":8068,"kb_used_omap":5,"kb_used_meta":337786,"kb_avail":93399284,"statfs":{"total":95995035648,"available":95640866816,"internally_reserved":0,"allocated":8261632,"data_stored":7298895,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6117,"internal_metadata":345892891},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:14:50 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.93600000000000005,"15min":0.83799999999999997},"min":{"1min":0.53700000000000003,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.343,"5min":1.343,"15min":1.343},"last":1.28},{"interface":"front","average":{"1min":0.90500000000000003,"5min":0.94199999999999995,"15min":0.90800000000000003},"min":{"1min":0.58499999999999996,"5min":0.498,"15min":0.46700000000000003},"max":{"1min":1.292,"5min":1.3660000000000001,"15min":1.5529999999999999},"last":0.996}]},{"osd":1,"last update":"Sat Sep 23 19:14:50 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":0.93200000000000005,"15min":0.85399999999999998},"min":{"1min":0.65500000000000003,"5min":0.54200000000000004,"15min":0.42599999999999999},"max":{"1min":1.298,"5min":1.3400000000000001,"15min":1.3400000000000001},"last":0.94599999999999995},{"interface":"front","average":{"1min":1.0129999999999999,"5min":0.94599999999999995,"15min":0.878},"min":{"1min":0.73899999999999999,"5min":0.55300000000000005,"15min":0.51400000000000001},"max":{"1min":1.2749999999999999,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":1.0069999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:14:50 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1,"15min":0.91600000000000004},"min":{"1min":0.627,"5min":0.57899999999999996,"15min":0.56599999999999995},"max":{"1min":1.4319999999999999,"5min":1.4319999999999999,"15min":1.4319999999999999},"last":0.95699999999999996},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.95199999999999996,"15min":0.88600000000000001},"min":{"1min":0.72999999999999998,"5min":0.55800000000000005,"15min":0.55200000000000005},"max":{"1min":1.46,"5min":1.46,"15min":1.46},"last":1.214}]},{"osd":3,"last update":"Sat Sep 23 19:14:50 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.002,"15min":0.96399999999999997},"min":{"1min":0.84399999999999997,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.284,"5min":1.466,"15min":1.5249999999999999},"last":1.1459999999999999},{"interface":"front","average":{"1min":1.038,"5min":1.0029999999999999,"15min":0.96599999999999997},"min":{"1min":0.68400000000000005,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.3939999999999999,"5min":1.415,"15min":2.121},"last":1.0329999999999999}]},{"osd":4,"last update":"Sat Sep 23 19:14:50 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.92000000000000004,"15min":0.87},"min":{"1min":0.72999999999999998,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.242,"5min":1.2549999999999999,"15min":1.2729999999999999},"last":0.84499999999999997},{"interface":"front","average":{"1min":0.99299999999999999,"5min":0.94699999999999995,"15min":0.93700000000000006},"min":{"1min":0.69599999999999995,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.375,"5min":1.375,"15min":1.399},"last":1.0840000000000001}]},{"osd":6,"last update":"Sat Sep 23 19:14:50 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.0149999999999999,"15min":0.97699999999999998},"min":{"1min":0.72199999999999998,"5min":0.372,"15min":0.372},"max":{"1min":1.343,"5min":1.448,"15min":1.448},"last":1.1339999999999999},{"interface":"front","average":{"1min":1.125,"5min":1.0269999999999999,"15min":1.004},"min":{"1min":0.78200000000000003,"5min":0.314,"15min":0.314},"max":{"1min":1.3959999999999999,"5min":1.411,"15min":1.411},"last":1.056}]},{"osd":7,"last update":"Sat Sep 23 19:15:09 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.0329999999999999,"15min":1.083},"min":{"1min":0.71599999999999997,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.319,"5min":1.319,"15min":1.3520000000000001},"last":0.89800000000000002},{"interface":"front","average":{"1min":1.105,"5min":1.0900000000000001,"15min":1.135},"min":{"1min":0.71199999999999997,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.333,"5min":1.796,"15min":1.796},"last":0.88100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4224,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8388608,"data_stored":8388608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2112,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7725,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3429,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11243,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6117,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11916,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:15:44.532 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T19:15:45.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:44 smithi098 bash[76817]: audit 2023-09-23T19:15:43.540288+0000 mgr.y (mgr.26054) 1747 : audit 0 from='client.37056 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:15:45.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:44 smithi098 bash[79516]: audit 2023-09-23T19:15:43.540288+0000 mgr.y (mgr.26054) 1747 : audit 0 from='client.37056 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:15:45.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:44 smithi173 bash[59422]: audit 2023-09-23T19:15:43.540288+0000 mgr.y (mgr.26054) 1747 : audit 0 from='client.37056 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:15:46.194 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:45 smithi098 bash[79516]: cluster 2023-09-23T19:15:44.833476+0000 mgr.y (mgr.26054) 1748 : cluster 0 pgmap v1960: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:46.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:45 smithi098 bash[76817]: cluster 2023-09-23T19:15:44.833476+0000 mgr.y (mgr.26054) 1748 : cluster 0 pgmap v1960: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:46.195 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:15:46 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:15:46] "GET /metrics HTTP/1.1" 200 34165 "" "Prometheus/2.33.4" 2023-09-23T19:15:46.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:45 smithi173 bash[59422]: cluster 2023-09-23T19:15:44.833476+0000 mgr.y (mgr.26054) 1748 : cluster 0 pgmap v1960: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:48.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:47 smithi098 bash[76817]: cluster 2023-09-23T19:15:46.834436+0000 mgr.y (mgr.26054) 1749 : cluster 0 pgmap v1961: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:48.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:47 smithi098 bash[79516]: cluster 2023-09-23T19:15:46.834436+0000 mgr.y (mgr.26054) 1749 : cluster 0 pgmap v1961: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:48.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:47 smithi173 bash[59422]: cluster 2023-09-23T19:15:46.834436+0000 mgr.y (mgr.26054) 1749 : cluster 0 pgmap v1961: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:49.312 INFO:teuthology.orchestra.run.smithi098.stderr:Inferring config /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T19:15:50.194 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:49 smithi098 bash[76817]: cluster 2023-09-23T19:15:48.835138+0000 mgr.y (mgr.26054) 1750 : cluster 0 pgmap v1962: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:50.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:49 smithi098 bash[79516]: cluster 2023-09-23T19:15:48.835138+0000 mgr.y (mgr.26054) 1750 : cluster 0 pgmap v1962: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:50.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:49 smithi173 bash[59422]: cluster 2023-09-23T19:15:48.835138+0000 mgr.y (mgr.26054) 1750 : cluster 0 pgmap v1962: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:51.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:50 smithi098 bash[76817]: audit 2023-09-23T19:15:50.198650+0000 mon.a (mon.0) 2309 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:15:51.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:50 smithi098 bash[79516]: audit 2023-09-23T19:15:50.198650+0000 mon.a (mon.0) 2309 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:15:51.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:50 smithi173 bash[59422]: audit 2023-09-23T19:15:50.198650+0000 mon.a (mon.0) 2309 : audit 0 from='mgr.26054 172.21.15.98:0/3608332770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T19:15:52.195 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:51 smithi098 bash[79516]: cluster 2023-09-23T19:15:50.836032+0000 mgr.y (mgr.26054) 1751 : cluster 0 pgmap v1963: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:52.195 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:51 smithi098 bash[76817]: cluster 2023-09-23T19:15:50.836032+0000 mgr.y (mgr.26054) 1751 : cluster 0 pgmap v1963: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:52.277 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:51 smithi173 bash[59422]: cluster 2023-09-23T19:15:50.836032+0000 mgr.y (mgr.26054) 1751 : cluster 0 pgmap v1963: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:53.176 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-23T19:15:53.179 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-23T19:15:54.068 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":1964,"stamp":"2023-09-23T19:15:52.836336+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":12228,"num_read_kb":26865,"num_write":1950,"num_write_kb":27871,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":213,"num_bytes_recovered":16790439,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2144,"ondisk_log_size":2144,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6026312,"kb_used_data":62536,"kb_used_omap":62,"kb_used_meta":5963649,"kb_avail":743934904,"statfs":{"total":767960285184,"available":761789341696,"internally_reserved":0,"allocated":64036864,"data_stored":52357425,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":63895,"internal_metadata":6106777193},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005087"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":31,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.725569+0000","last_change":"2023-09-23T19:11:17.683481+0000","last_active":"2023-09-23T19:14:11.725569+0000","last_peered":"2023-09-23T19:14:11.725569+0000","last_clean":"2023-09-23T19:14:11.725569+0000","last_became_active":"2023-09-23T19:11:17.681965+0000","last_became_peered":"2023-09-23T19:11:17.681965+0000","last_unstale":"2023-09-23T19:14:11.725569+0000","last_undegraded":"2023-09-23T19:14:11.725569+0000","last_fullsized":"2023-09-23T19:14:11.725569+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":155,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.715944+0000","last_change":"2023-09-23T19:11:17.681338+0000","last_active":"2023-09-23T19:14:11.715944+0000","last_peered":"2023-09-23T19:14:11.715944+0000","last_clean":"2023-09-23T19:14:11.715944+0000","last_became_active":"2023-09-23T19:11:17.680951+0000","last_became_peered":"2023-09-23T19:11:17.680951+0000","last_unstale":"2023-09-23T19:14:11.715944+0000","last_undegraded":"2023-09-23T19:14:11.715944+0000","last_fullsized":"2023-09-23T19:14:11.715944+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:26.573271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":422,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.893589+0000","last_change":"2023-09-23T19:11:17.679652+0000","last_active":"2023-09-23T19:14:21.893589+0000","last_peered":"2023-09-23T19:14:21.893589+0000","last_clean":"2023-09-23T19:14:21.893589+0000","last_became_active":"2023-09-23T19:11:17.679260+0000","last_became_peered":"2023-09-23T19:11:17.679260+0000","last_unstale":"2023-09-23T19:14:21.893589+0000","last_undegraded":"2023-09-23T19:14:21.893589+0000","last_fullsized":"2023-09-23T19:14:21.893589+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:14.084906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":467,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.894540+0000","last_change":"2023-09-23T19:09:03.326411+0000","last_active":"2023-09-23T19:14:21.894540+0000","last_peered":"2023-09-23T19:14:21.894540+0000","last_clean":"2023-09-23T19:14:21.894540+0000","last_became_active":"2023-09-23T19:09:03.325980+0000","last_became_peered":"2023-09-23T19:09:03.325980+0000","last_unstale":"2023-09-23T19:14:21.894540+0000","last_undegraded":"2023-09-23T19:14:21.894540+0000","last_fullsized":"2023-09-23T19:14:21.894540+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:43.638573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":138,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.721156+0000","last_change":"2023-09-23T18:58:38.718387+0000","last_active":"2023-09-23T19:14:21.721156+0000","last_peered":"2023-09-23T19:14:21.721156+0000","last_clean":"2023-09-23T19:14:21.721156+0000","last_became_active":"2023-09-23T18:58:38.717999+0000","last_became_peered":"2023-09-23T18:58:38.717999+0000","last_unstale":"2023-09-23T19:14:21.721156+0000","last_undegraded":"2023-09-23T19:14:21.721156+0000","last_fullsized":"2023-09-23T19:14:21.721156+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:05.073213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":278,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.725408+0000","last_change":"2023-09-23T19:13:02.800175+0000","last_active":"2023-09-23T19:14:11.725408+0000","last_peered":"2023-09-23T19:14:11.725408+0000","last_clean":"2023-09-23T19:14:11.725408+0000","last_became_active":"2023-09-23T19:13:02.799137+0000","last_became_peered":"2023-09-23T19:13:02.799137+0000","last_unstale":"2023-09-23T19:14:11.725408+0000","last_undegraded":"2023-09-23T19:14:11.725408+0000","last_fullsized":"2023-09-23T19:14:11.725408+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":288,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.894983+0000","last_change":"2023-09-23T19:12:02.417052+0000","last_active":"2023-09-23T19:14:21.894983+0000","last_peered":"2023-09-23T19:14:21.894983+0000","last_clean":"2023-09-23T19:14:21.894983+0000","last_became_active":"2023-09-23T19:12:02.416041+0000","last_became_peered":"2023-09-23T19:12:02.416041+0000","last_unstale":"2023-09-23T19:14:21.894983+0000","last_undegraded":"2023-09-23T19:14:21.894983+0000","last_fullsized":"2023-09-23T19:14:21.894983+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:05.013964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":138,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.720928+0000","last_change":"2023-09-23T18:58:38.710920+0000","last_active":"2023-09-23T19:14:21.720928+0000","last_peered":"2023-09-23T19:14:21.720928+0000","last_clean":"2023-09-23T19:14:21.720928+0000","last_became_active":"2023-09-23T18:58:38.710563+0000","last_became_peered":"2023-09-23T18:58:38.710563+0000","last_unstale":"2023-09-23T19:14:21.720928+0000","last_undegraded":"2023-09-23T19:14:21.720928+0000","last_fullsized":"2023-09-23T19:14:21.720928+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:42.386631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":34,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.913207+0000","last_change":"2023-09-23T19:10:37.800599+0000","last_active":"2023-09-23T19:14:11.913207+0000","last_peered":"2023-09-23T19:14:11.913207+0000","last_clean":"2023-09-23T19:14:11.913207+0000","last_became_active":"2023-09-23T19:10:37.800130+0000","last_became_peered":"2023-09-23T19:10:37.800130+0000","last_unstale":"2023-09-23T19:14:11.913207+0000","last_undegraded":"2023-09-23T19:14:11.913207+0000","last_fullsized":"2023-09-23T19:14:11.913207+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":398,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.893671+0000","last_change":"2023-09-23T19:12:02.284586+0000","last_active":"2023-09-23T19:14:21.893671+0000","last_peered":"2023-09-23T19:14:21.893671+0000","last_clean":"2023-09-23T19:14:21.893671+0000","last_became_active":"2023-09-23T19:12:02.284069+0000","last_became_peered":"2023-09-23T19:12:02.284069+0000","last_unstale":"2023-09-23T19:14:21.893671+0000","last_undegraded":"2023-09-23T19:14:21.893671+0000","last_fullsized":"2023-09-23T19:14:21.893671+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:07:35.047940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":316,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.852039+0000","last_change":"2023-09-23T19:11:18.008462+0000","last_active":"2023-09-23T19:14:12.852039+0000","last_peered":"2023-09-23T19:14:12.852039+0000","last_clean":"2023-09-23T19:14:12.852039+0000","last_became_active":"2023-09-23T19:11:17.691174+0000","last_became_peered":"2023-09-23T19:11:17.691174+0000","last_unstale":"2023-09-23T19:14:12.852039+0000","last_undegraded":"2023-09-23T19:14:12.852039+0000","last_fullsized":"2023-09-23T19:14:12.852039+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":368,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.894435+0000","last_change":"2023-09-23T18:33:53.850096+0000","last_active":"2023-09-23T19:14:21.894435+0000","last_peered":"2023-09-23T19:14:21.894435+0000","last_clean":"2023-09-23T19:14:21.894435+0000","last_became_active":"2023-09-23T18:33:53.849811+0000","last_became_peered":"2023-09-23T18:33:53.849811+0000","last_unstale":"2023-09-23T19:14:21.894435+0000","last_undegraded":"2023-09-23T19:14:21.894435+0000","last_fullsized":"2023-09-23T19:14:21.894435+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:44.938072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":41,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.725910+0000","last_change":"2023-09-23T19:11:17.699173+0000","last_active":"2023-09-23T19:14:11.725910+0000","last_peered":"2023-09-23T19:14:11.725910+0000","last_clean":"2023-09-23T19:14:11.725910+0000","last_became_active":"2023-09-23T19:11:17.685016+0000","last_became_peered":"2023-09-23T19:11:17.685016+0000","last_unstale":"2023-09-23T19:14:11.725910+0000","last_undegraded":"2023-09-23T19:14:11.725910+0000","last_fullsized":"2023-09-23T19:14:11.725910+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":31,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.725813+0000","last_change":"2023-09-23T19:11:17.683010+0000","last_active":"2023-09-23T19:14:11.725813+0000","last_peered":"2023-09-23T19:14:11.725813+0000","last_clean":"2023-09-23T19:14:11.725813+0000","last_became_active":"2023-09-23T19:11:17.682613+0000","last_became_peered":"2023-09-23T19:11:17.682613+0000","last_unstale":"2023-09-23T19:14:11.725813+0000","last_undegraded":"2023-09-23T19:14:11.725813+0000","last_fullsized":"2023-09-23T19:14:11.725813+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":315,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.851840+0000","last_change":"2023-09-23T19:11:17.832325+0000","last_active":"2023-09-23T19:14:12.851840+0000","last_peered":"2023-09-23T19:14:12.851840+0000","last_clean":"2023-09-23T19:14:12.851840+0000","last_became_active":"2023-09-23T19:11:17.688749+0000","last_became_peered":"2023-09-23T19:11:17.688749+0000","last_unstale":"2023-09-23T19:14:12.851840+0000","last_undegraded":"2023-09-23T19:14:12.851840+0000","last_fullsized":"2023-09-23T19:14:12.851840+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":439,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.721662+0000","last_change":"2023-09-23T19:09:03.319623+0000","last_active":"2023-09-23T19:14:21.721662+0000","last_peered":"2023-09-23T19:14:21.721662+0000","last_clean":"2023-09-23T19:14:21.721662+0000","last_became_active":"2023-09-23T19:09:03.319283+0000","last_became_peered":"2023-09-23T19:09:03.319283+0000","last_unstale":"2023-09-23T19:14:21.721662+0000","last_undegraded":"2023-09-23T19:14:21.721662+0000","last_fullsized":"2023-09-23T19:14:21.721662+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:34.646037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":252,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.895570+0000","last_change":"2023-09-23T19:12:02.205046+0000","last_active":"2023-09-23T19:14:21.895570+0000","last_peered":"2023-09-23T19:14:21.895570+0000","last_clean":"2023-09-23T19:14:21.895570+0000","last_became_active":"2023-09-23T19:12:02.204322+0000","last_became_peered":"2023-09-23T19:12:02.204322+0000","last_unstale":"2023-09-23T19:14:21.895570+0000","last_undegraded":"2023-09-23T19:14:21.895570+0000","last_fullsized":"2023-09-23T19:14:21.895570+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:35.102992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":34,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.913075+0000","last_change":"2023-09-23T19:10:37.801034+0000","last_active":"2023-09-23T19:14:11.913075+0000","last_peered":"2023-09-23T19:14:11.913075+0000","last_clean":"2023-09-23T19:14:11.913075+0000","last_became_active":"2023-09-23T19:10:37.800310+0000","last_became_peered":"2023-09-23T19:10:37.800310+0000","last_unstale":"2023-09-23T19:14:11.913075+0000","last_undegraded":"2023-09-23T19:14:11.913075+0000","last_fullsized":"2023-09-23T19:14:11.913075+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":292,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.851478+0000","last_change":"2023-09-23T19:11:17.685822+0000","last_active":"2023-09-23T19:14:12.851478+0000","last_peered":"2023-09-23T19:14:12.851478+0000","last_clean":"2023-09-23T19:14:12.851478+0000","last_became_active":"2023-09-23T19:11:17.685456+0000","last_became_peered":"2023-09-23T19:11:17.685456+0000","last_unstale":"2023-09-23T19:14:12.851478+0000","last_undegraded":"2023-09-23T19:14:12.851478+0000","last_fullsized":"2023-09-23T19:14:12.851478+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":253,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.894617+0000","last_change":"2023-09-23T19:12:02.204580+0000","last_active":"2023-09-23T19:14:21.894617+0000","last_peered":"2023-09-23T19:14:21.894617+0000","last_clean":"2023-09-23T19:14:21.894617+0000","last_became_active":"2023-09-23T19:12:02.203730+0000","last_became_peered":"2023-09-23T19:12:02.203730+0000","last_unstale":"2023-09-23T19:14:21.894617+0000","last_undegraded":"2023-09-23T19:14:21.894617+0000","last_fullsized":"2023-09-23T19:14:21.894617+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:54.397927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":285,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.724282+0000","last_change":"2023-09-23T19:13:02.721633+0000","last_active":"2023-09-23T19:14:11.724282+0000","last_peered":"2023-09-23T19:14:11.724282+0000","last_clean":"2023-09-23T19:14:11.724282+0000","last_became_active":"2023-09-23T19:13:02.721035+0000","last_became_peered":"2023-09-23T19:13:02.721035+0000","last_unstale":"2023-09-23T19:14:11.724282+0000","last_undegraded":"2023-09-23T19:14:11.724282+0000","last_fullsized":"2023-09-23T19:14:11.724282+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":90,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.721292+0000","last_change":"2023-09-23T19:09:03.318996+0000","last_active":"2023-09-23T19:14:21.721292+0000","last_peered":"2023-09-23T19:14:21.721292+0000","last_clean":"2023-09-23T19:14:21.721292+0000","last_became_active":"2023-09-23T19:09:03.318593+0000","last_became_peered":"2023-09-23T19:09:03.318593+0000","last_unstale":"2023-09-23T19:14:21.721292+0000","last_undegraded":"2023-09-23T19:14:21.721292+0000","last_fullsized":"2023-09-23T19:14:21.721292+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:23.764508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":293,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.852152+0000","last_change":"2023-09-23T19:11:17.876664+0000","last_active":"2023-09-23T19:14:12.852152+0000","last_peered":"2023-09-23T19:14:12.852152+0000","last_clean":"2023-09-23T19:14:12.852152+0000","last_became_active":"2023-09-23T19:11:17.689646+0000","last_became_peered":"2023-09-23T19:11:17.689646+0000","last_unstale":"2023-09-23T19:14:12.852152+0000","last_undegraded":"2023-09-23T19:14:12.852152+0000","last_fullsized":"2023-09-23T19:14:12.852152+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":34,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.912486+0000","last_change":"2023-09-23T19:10:37.800494+0000","last_active":"2023-09-23T19:14:11.912486+0000","last_peered":"2023-09-23T19:14:11.912486+0000","last_clean":"2023-09-23T19:14:11.912486+0000","last_became_active":"2023-09-23T19:10:37.800131+0000","last_became_peered":"2023-09-23T19:10:37.800131+0000","last_unstale":"2023-09-23T19:14:11.912486+0000","last_undegraded":"2023-09-23T19:14:11.912486+0000","last_fullsized":"2023-09-23T19:14:11.912486+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":328,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.851994+0000","last_change":"2023-09-23T19:11:17.921164+0000","last_active":"2023-09-23T19:14:12.851994+0000","last_peered":"2023-09-23T19:14:12.851994+0000","last_clean":"2023-09-23T19:14:12.851994+0000","last_became_active":"2023-09-23T19:11:17.690216+0000","last_became_peered":"2023-09-23T19:11:17.690216+0000","last_unstale":"2023-09-23T19:14:12.851994+0000","last_undegraded":"2023-09-23T19:14:12.851994+0000","last_fullsized":"2023-09-23T19:14:12.851994+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":31,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.725008+0000","last_change":"2023-09-23T19:11:17.686637+0000","last_active":"2023-09-23T19:14:11.725008+0000","last_peered":"2023-09-23T19:14:11.725008+0000","last_clean":"2023-09-23T19:14:11.725008+0000","last_became_active":"2023-09-23T19:11:17.684587+0000","last_became_peered":"2023-09-23T19:11:17.684587+0000","last_unstale":"2023-09-23T19:14:11.725008+0000","last_undegraded":"2023-09-23T19:14:11.725008+0000","last_fullsized":"2023-09-23T19:14:11.725008+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":312,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.851717+0000","last_change":"2023-09-23T19:10:37.846671+0000","last_active":"2023-09-23T19:14:12.851717+0000","last_peered":"2023-09-23T19:14:12.851717+0000","last_clean":"2023-09-23T19:14:12.851717+0000","last_became_active":"2023-09-23T19:10:37.800522+0000","last_became_peered":"2023-09-23T19:10:37.800522+0000","last_unstale":"2023-09-23T19:14:12.851717+0000","last_undegraded":"2023-09-23T19:14:12.851717+0000","last_fullsized":"2023-09-23T19:14:12.851717+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":242,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.724738+0000","last_change":"2023-09-23T19:13:02.724651+0000","last_active":"2023-09-23T19:14:11.724738+0000","last_peered":"2023-09-23T19:14:11.724738+0000","last_clean":"2023-09-23T19:14:11.724738+0000","last_became_active":"2023-09-23T19:13:02.724250+0000","last_became_peered":"2023-09-23T19:13:02.724250+0000","last_unstale":"2023-09-23T19:14:11.724738+0000","last_undegraded":"2023-09-23T19:14:11.724738+0000","last_fullsized":"2023-09-23T19:14:11.724738+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":117,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.912849+0000","last_change":"2023-09-23T19:08:37.773654+0000","last_active":"2023-09-23T19:14:11.912849+0000","last_peered":"2023-09-23T19:14:11.912849+0000","last_clean":"2023-09-23T19:14:11.912849+0000","last_became_active":"2023-09-23T19:08:37.773196+0000","last_became_peered":"2023-09-23T19:08:37.773196+0000","last_unstale":"2023-09-23T19:14:11.912849+0000","last_undegraded":"2023-09-23T19:14:11.912849+0000","last_fullsized":"2023-09-23T19:14:11.912849+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":441,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.843791+0000","last_change":"2023-09-23T19:11:17.670732+0000","last_active":"2023-09-23T19:14:12.843791+0000","last_peered":"2023-09-23T19:14:12.843791+0000","last_clean":"2023-09-23T19:14:12.843791+0000","last_became_active":"2023-09-23T19:11:17.670431+0000","last_became_peered":"2023-09-23T19:11:17.670431+0000","last_unstale":"2023-09-23T19:14:12.843791+0000","last_undegraded":"2023-09-23T19:14:12.843791+0000","last_fullsized":"2023-09-23T19:14:12.843791+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:13:31.550360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":168,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.721567+0000","last_change":"2023-09-23T19:11:17.689473+0000","last_active":"2023-09-23T19:14:21.721567+0000","last_peered":"2023-09-23T19:14:21.721567+0000","last_clean":"2023-09-23T19:14:21.721567+0000","last_became_active":"2023-09-23T19:11:17.681348+0000","last_became_peered":"2023-09-23T19:11:17.681348+0000","last_unstale":"2023-09-23T19:14:21.721567+0000","last_undegraded":"2023-09-23T19:14:21.721567+0000","last_fullsized":"2023-09-23T19:14:21.721567+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:55:33.916670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":31,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.725955+0000","last_change":"2023-09-23T19:11:17.681567+0000","last_active":"2023-09-23T19:14:11.725955+0000","last_peered":"2023-09-23T19:14:11.725955+0000","last_clean":"2023-09-23T19:14:11.725955+0000","last_became_active":"2023-09-23T19:11:17.681131+0000","last_became_peered":"2023-09-23T19:11:17.681131+0000","last_unstale":"2023-09-23T19:14:11.725955+0000","last_undegraded":"2023-09-23T19:14:11.725955+0000","last_fullsized":"2023-09-23T19:14:11.725955+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":235,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.724503+0000","last_change":"2023-09-23T19:13:02.800517+0000","last_active":"2023-09-23T19:14:11.724503+0000","last_peered":"2023-09-23T19:14:11.724503+0000","last_clean":"2023-09-23T19:14:11.724503+0000","last_became_active":"2023-09-23T19:13:02.799204+0000","last_became_peered":"2023-09-23T19:13:02.799204+0000","last_unstale":"2023-09-23T19:14:11.724503+0000","last_undegraded":"2023-09-23T19:14:11.724503+0000","last_fullsized":"2023-09-23T19:14:11.724503+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":31,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.726044+0000","last_change":"2023-09-23T19:11:17.683123+0000","last_active":"2023-09-23T19:14:11.726044+0000","last_peered":"2023-09-23T19:14:11.726044+0000","last_clean":"2023-09-23T19:14:11.726044+0000","last_became_active":"2023-09-23T19:11:17.682648+0000","last_became_peered":"2023-09-23T19:11:17.682648+0000","last_unstale":"2023-09-23T19:14:11.726044+0000","last_undegraded":"2023-09-23T19:14:11.726044+0000","last_fullsized":"2023-09-23T19:14:11.726044+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":31,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.725117+0000","last_change":"2023-09-23T19:11:17.687047+0000","last_active":"2023-09-23T19:14:11.725117+0000","last_peered":"2023-09-23T19:14:11.725117+0000","last_clean":"2023-09-23T19:14:11.725117+0000","last_became_active":"2023-09-23T19:11:17.684963+0000","last_became_peered":"2023-09-23T19:11:17.684963+0000","last_unstale":"2023-09-23T19:14:11.725117+0000","last_undegraded":"2023-09-23T19:14:11.725117+0000","last_fullsized":"2023-09-23T19:14:11.725117+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":31,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.726156+0000","last_change":"2023-09-23T19:11:17.682563+0000","last_active":"2023-09-23T19:14:11.726156+0000","last_peered":"2023-09-23T19:14:11.726156+0000","last_clean":"2023-09-23T19:14:11.726156+0000","last_became_active":"2023-09-23T19:11:17.682142+0000","last_became_peered":"2023-09-23T19:11:17.682142+0000","last_unstale":"2023-09-23T19:14:11.726156+0000","last_undegraded":"2023-09-23T19:14:11.726156+0000","last_fullsized":"2023-09-23T19:14:11.726156+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":113,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:16.297026+0000","last_change":"2023-09-23T19:09:03.322253+0000","last_active":"2023-09-23T19:14:16.297026+0000","last_peered":"2023-09-23T19:14:16.297026+0000","last_clean":"2023-09-23T19:14:16.297026+0000","last_became_active":"2023-09-23T19:09:03.320102+0000","last_became_peered":"2023-09-23T19:09:03.320102+0000","last_unstale":"2023-09-23T19:14:16.297026+0000","last_undegraded":"2023-09-23T19:14:16.297026+0000","last_fullsized":"2023-09-23T19:14:16.297026+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:01.268221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":436,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.894960+0000","last_change":"2023-09-23T19:11:17.680502+0000","last_active":"2023-09-23T19:14:21.894960+0000","last_peered":"2023-09-23T19:14:21.894960+0000","last_clean":"2023-09-23T19:14:21.894960+0000","last_became_active":"2023-09-23T19:11:17.679955+0000","last_became_peered":"2023-09-23T19:11:17.679955+0000","last_unstale":"2023-09-23T19:14:21.894960+0000","last_undegraded":"2023-09-23T19:14:21.894960+0000","last_fullsized":"2023-09-23T19:14:21.894960+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:01.547326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"383'306","reported_seq":1477,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.894901+0000","last_change":"2023-09-23T19:09:23.687377+0000","last_active":"2023-09-23T19:14:21.894901+0000","last_peered":"2023-09-23T19:14:21.894901+0000","last_clean":"2023-09-23T19:14:21.894901+0000","last_became_active":"2023-09-23T19:09:02.306435+0000","last_became_peered":"2023-09-23T19:09:02.306435+0000","last_unstale":"2023-09-23T19:14:21.894901+0000","last_undegraded":"2023-09-23T19:14:21.894901+0000","last_fullsized":"2023-09-23T19:14:21.894901+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":306,"log_dups_size":0,"ondisk_log_size":306,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:54:30.990476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086649800000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1624,"num_read_kb":2591,"num_write":297,"num_write_kb":3316,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":117,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.913144+0000","last_change":"2023-09-23T19:08:37.774207+0000","last_active":"2023-09-23T19:14:11.913144+0000","last_peered":"2023-09-23T19:14:11.913144+0000","last_clean":"2023-09-23T19:14:11.913144+0000","last_became_active":"2023-09-23T19:08:37.773764+0000","last_became_peered":"2023-09-23T19:08:37.773764+0000","last_unstale":"2023-09-23T19:14:11.913144+0000","last_undegraded":"2023-09-23T19:14:11.913144+0000","last_fullsized":"2023-09-23T19:14:11.913144+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":427,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.894332+0000","last_change":"2023-09-23T19:11:17.680196+0000","last_active":"2023-09-23T19:14:21.894332+0000","last_peered":"2023-09-23T19:14:21.894332+0000","last_clean":"2023-09-23T19:14:21.894332+0000","last_became_active":"2023-09-23T19:11:17.679638+0000","last_became_peered":"2023-09-23T19:11:17.679638+0000","last_unstale":"2023-09-23T19:14:21.894332+0000","last_undegraded":"2023-09-23T19:14:21.894332+0000","last_fullsized":"2023-09-23T19:14:21.894332+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:30.323348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"383'125","reported_seq":500,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.912703+0000","last_change":"2023-09-23T19:09:23.642827+0000","last_active":"2023-09-23T19:14:11.912703+0000","last_peered":"2023-09-23T19:14:11.912703+0000","last_clean":"2023-09-23T19:14:11.912703+0000","last_became_active":"2023-09-23T19:08:37.774901+0000","last_became_peered":"2023-09-23T19:08:37.774901+0000","last_unstale":"2023-09-23T19:14:11.912703+0000","last_undegraded":"2023-09-23T19:14:11.912703+0000","last_fullsized":"2023-09-23T19:14:11.912703+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":125,"log_dups_size":0,"ondisk_log_size":125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1121,"num_read_kb":2820,"num_write":107,"num_write_kb":1462,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":31,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.725875+0000","last_change":"2023-09-23T19:11:17.679851+0000","last_active":"2023-09-23T19:14:11.725875+0000","last_peered":"2023-09-23T19:14:11.725875+0000","last_clean":"2023-09-23T19:14:11.725875+0000","last_became_active":"2023-09-23T19:11:17.679454+0000","last_became_peered":"2023-09-23T19:11:17.679454+0000","last_unstale":"2023-09-23T19:14:11.725875+0000","last_undegraded":"2023-09-23T19:14:11.725875+0000","last_fullsized":"2023-09-23T19:14:11.725875+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":435,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.895136+0000","last_change":"2023-09-23T18:58:38.726546+0000","last_active":"2023-09-23T19:14:21.895136+0000","last_peered":"2023-09-23T19:14:21.895136+0000","last_clean":"2023-09-23T19:14:21.895136+0000","last_became_active":"2023-09-23T18:58:38.724765+0000","last_became_peered":"2023-09-23T18:58:38.724765+0000","last_unstale":"2023-09-23T19:14:21.895136+0000","last_undegraded":"2023-09-23T19:14:21.895136+0000","last_fullsized":"2023-09-23T19:14:21.895136+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:05.322676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"383'289","reported_seq":1337,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.894396+0000","last_change":"2023-09-23T19:09:23.637617+0000","last_active":"2023-09-23T19:14:21.894396+0000","last_peered":"2023-09-23T19:14:21.894396+0000","last_clean":"2023-09-23T19:14:21.894396+0000","last_became_active":"2023-09-23T18:58:38.707923+0000","last_became_peered":"2023-09-23T18:58:38.707923+0000","last_unstale":"2023-09-23T19:14:21.894396+0000","last_undegraded":"2023-09-23T19:14:21.894396+0000","last_fullsized":"2023-09-23T19:14:21.894396+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":289,"log_dups_size":0,"ondisk_log_size":289,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:59.471903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047940899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1602,"num_read_kb":4202,"num_write":289,"num_write_kb":2667,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":370,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.348215+0000","last_change":"2023-09-23T18:33:53.849084+0000","last_active":"2023-09-23T19:14:12.348215+0000","last_peered":"2023-09-23T19:14:12.348215+0000","last_clean":"2023-09-23T19:14:12.348215+0000","last_became_active":"2023-09-23T18:33:53.848757+0000","last_became_peered":"2023-09-23T18:33:53.848757+0000","last_unstale":"2023-09-23T19:14:12.348215+0000","last_undegraded":"2023-09-23T19:14:12.348215+0000","last_fullsized":"2023-09-23T19:14:12.348215+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:15.267742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":440,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.895133+0000","last_change":"2023-09-23T19:08:37.776940+0000","last_active":"2023-09-23T19:14:21.895133+0000","last_peered":"2023-09-23T19:14:21.895133+0000","last_clean":"2023-09-23T19:14:21.895133+0000","last_became_active":"2023-09-23T19:08:37.776564+0000","last_became_peered":"2023-09-23T19:08:37.776564+0000","last_unstale":"2023-09-23T19:14:21.895133+0000","last_undegraded":"2023-09-23T19:14:21.895133+0000","last_fullsized":"2023-09-23T19:14:21.895133+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:44.299192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"383'317","reported_seq":1387,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.912779+0000","last_change":"2023-09-23T19:09:23.643011+0000","last_active":"2023-09-23T19:14:11.912779+0000","last_peered":"2023-09-23T19:14:11.912779+0000","last_clean":"2023-09-23T19:14:11.912779+0000","last_became_active":"2023-09-23T19:09:03.312411+0000","last_became_peered":"2023-09-23T19:09:03.312411+0000","last_unstale":"2023-09-23T19:14:11.912779+0000","last_undegraded":"2023-09-23T19:14:11.912779+0000","last_fullsized":"2023-09-23T19:14:11.912779+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":317,"log_dups_size":0,"ondisk_log_size":317,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1525,"num_read_kb":2676,"num_write":278,"num_write_kb":5451,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":520,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.851120+0000","last_change":"2023-09-23T19:09:02.309826+0000","last_active":"2023-09-23T19:14:12.851120+0000","last_peered":"2023-09-23T19:14:12.851120+0000","last_clean":"2023-09-23T19:14:12.851120+0000","last_became_active":"2023-09-23T19:09:02.309300+0000","last_became_peered":"2023-09-23T19:09:02.309300+0000","last_unstale":"2023-09-23T19:14:12.851120+0000","last_undegraded":"2023-09-23T19:14:12.851120+0000","last_fullsized":"2023-09-23T19:14:12.851120+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:08:35.701374+0000","last_clean_scrub_stamp":"2023-09-23T18:08:35.701374+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":176,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.721038+0000","last_change":"2023-09-23T19:10:37.797301+0000","last_active":"2023-09-23T19:14:21.721038+0000","last_peered":"2023-09-23T19:14:21.721038+0000","last_clean":"2023-09-23T19:14:21.721038+0000","last_became_active":"2023-09-23T19:10:37.796972+0000","last_became_peered":"2023-09-23T19:10:37.796972+0000","last_unstale":"2023-09-23T19:14:21.721038+0000","last_undegraded":"2023-09-23T19:14:21.721038+0000","last_fullsized":"2023-09-23T19:14:21.721038+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:25.458233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":224,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.724926+0000","last_change":"2023-09-23T19:13:02.727195+0000","last_active":"2023-09-23T19:14:11.724926+0000","last_peered":"2023-09-23T19:14:11.724926+0000","last_clean":"2023-09-23T19:14:11.724926+0000","last_became_active":"2023-09-23T19:13:02.726835+0000","last_became_peered":"2023-09-23T19:13:02.726835+0000","last_unstale":"2023-09-23T19:14:11.724926+0000","last_undegraded":"2023-09-23T19:14:11.724926+0000","last_fullsized":"2023-09-23T19:14:11.724926+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"383'163","reported_seq":835,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.851118+0000","last_change":"2023-09-23T19:09:23.643786+0000","last_active":"2023-09-23T19:14:12.851118+0000","last_peered":"2023-09-23T19:14:12.851118+0000","last_clean":"2023-09-23T19:14:12.851118+0000","last_became_active":"2023-09-23T19:09:02.310096+0000","last_became_peered":"2023-09-23T19:09:02.310096+0000","last_unstale":"2023-09-23T19:14:12.851118+0000","last_undegraded":"2023-09-23T19:14:12.851118+0000","last_fullsized":"2023-09-23T19:14:12.851118+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":163,"log_dups_size":0,"ondisk_log_size":163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1010,"num_read_kb":4243,"num_write":132,"num_write_kb":3279,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":486,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.893871+0000","last_change":"2023-09-23T19:11:17.667944+0000","last_active":"2023-09-23T19:14:21.893871+0000","last_peered":"2023-09-23T19:14:21.893871+0000","last_clean":"2023-09-23T19:14:21.893871+0000","last_became_active":"2023-09-23T19:11:17.667587+0000","last_became_peered":"2023-09-23T19:11:17.667587+0000","last_unstale":"2023-09-23T19:14:21.893871+0000","last_undegraded":"2023-09-23T19:14:21.893871+0000","last_fullsized":"2023-09-23T19:14:21.893871+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:12.409946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":454,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.843698+0000","last_change":"2023-09-23T18:51:43.087834+0000","last_active":"2023-09-23T19:14:12.843698+0000","last_peered":"2023-09-23T19:14:12.843698+0000","last_clean":"2023-09-23T19:14:12.843698+0000","last_became_active":"2023-09-23T18:51:43.087309+0000","last_became_peered":"2023-09-23T18:51:43.087309+0000","last_unstale":"2023-09-23T19:14:12.843698+0000","last_undegraded":"2023-09-23T19:14:12.843698+0000","last_fullsized":"2023-09-23T19:14:12.843698+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:53.913425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"383'142","reported_seq":826,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.851849+0000","last_change":"2023-09-23T19:11:20.698564+0000","last_active":"2023-09-23T19:14:12.851849+0000","last_peered":"2023-09-23T19:14:12.851849+0000","last_clean":"2023-09-23T19:14:12.851849+0000","last_became_active":"2023-09-23T19:11:20.697584+0000","last_became_peered":"2023-09-23T19:11:20.697584+0000","last_unstale":"2023-09-23T19:14:12.851849+0000","last_undegraded":"2023-09-23T19:14:12.851849+0000","last_fullsized":"2023-09-23T19:14:12.851849+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":142,"log_dups_size":0,"ondisk_log_size":142,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":777,"num_read_kb":2427,"num_write":137,"num_write_kb":5585,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":378,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.724668+0000","last_change":"2023-09-23T19:13:02.724504+0000","last_active":"2023-09-23T19:14:11.724668+0000","last_peered":"2023-09-23T19:14:11.724668+0000","last_clean":"2023-09-23T19:14:11.724668+0000","last_became_active":"2023-09-23T19:13:02.724129+0000","last_became_peered":"2023-09-23T19:13:02.724129+0000","last_unstale":"2023-09-23T19:14:11.724668+0000","last_undegraded":"2023-09-23T19:14:11.724668+0000","last_fullsized":"2023-09-23T19:14:11.724668+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":315,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.844007+0000","last_change":"2023-09-23T19:12:02.416447+0000","last_active":"2023-09-23T19:14:12.844007+0000","last_peered":"2023-09-23T19:14:12.844007+0000","last_clean":"2023-09-23T19:14:12.844007+0000","last_became_active":"2023-09-23T19:12:02.415736+0000","last_became_peered":"2023-09-23T19:12:02.415736+0000","last_unstale":"2023-09-23T19:14:12.844007+0000","last_undegraded":"2023-09-23T19:14:12.844007+0000","last_fullsized":"2023-09-23T19:14:12.844007+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:02:36.968506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":439,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.715759+0000","last_change":"2023-09-23T19:09:03.319699+0000","last_active":"2023-09-23T19:14:11.715759+0000","last_peered":"2023-09-23T19:14:11.715759+0000","last_clean":"2023-09-23T19:14:11.715759+0000","last_became_active":"2023-09-23T19:09:03.318772+0000","last_became_peered":"2023-09-23T19:09:03.318772+0000","last_unstale":"2023-09-23T19:14:11.715759+0000","last_undegraded":"2023-09-23T19:14:11.715759+0000","last_fullsized":"2023-09-23T19:14:11.715759+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:41.224329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"383'372","reported_seq":1735,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.895077+0000","last_change":"2023-09-23T19:11:20.693753+0000","last_active":"2023-09-23T19:14:21.895077+0000","last_peered":"2023-09-23T19:14:21.895077+0000","last_clean":"2023-09-23T19:14:21.895077+0000","last_became_active":"2023-09-23T19:11:20.692775+0000","last_became_peered":"2023-09-23T19:11:20.692775+0000","last_unstale":"2023-09-23T19:14:21.895077+0000","last_undegraded":"2023-09-23T19:14:21.895077+0000","last_fullsized":"2023-09-23T19:14:21.895077+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":372,"log_dups_size":0,"ondisk_log_size":372,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:13:44.744641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067870799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2145,"num_read_kb":3069,"num_write":293,"num_write_kb":1311,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":116,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.720758+0000","last_change":"2023-09-23T19:09:03.322018+0000","last_active":"2023-09-23T19:14:21.720758+0000","last_peered":"2023-09-23T19:14:21.720758+0000","last_clean":"2023-09-23T19:14:21.720758+0000","last_became_active":"2023-09-23T19:09:03.321316+0000","last_became_peered":"2023-09-23T19:09:03.321316+0000","last_unstale":"2023-09-23T19:14:21.720758+0000","last_undegraded":"2023-09-23T19:14:21.720758+0000","last_fullsized":"2023-09-23T19:14:21.720758+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:46:37.431152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"383'236","reported_seq":1464,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.725411+0000","last_change":"2023-09-23T19:13:02.725131+0000","last_active":"2023-09-23T19:14:11.725411+0000","last_peered":"2023-09-23T19:14:11.725411+0000","last_clean":"2023-09-23T19:14:11.725411+0000","last_became_active":"2023-09-23T19:13:02.724555+0000","last_became_peered":"2023-09-23T19:13:02.724555+0000","last_unstale":"2023-09-23T19:14:11.725411+0000","last_undegraded":"2023-09-23T19:14:11.725411+0000","last_fullsized":"2023-09-23T19:14:11.725411+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:12.780682+0000","last_clean_scrub_stamp":"2023-09-23T18:13:12.780682+0000","objects_scrubbed":0,"log_size":236,"log_dups_size":0,"ondisk_log_size":236,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2130,"num_read_kb":4337,"num_write":157,"num_write_kb":2355,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":471,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.843926+0000","last_change":"2023-09-23T19:11:17.741918+0000","last_active":"2023-09-23T19:14:12.843926+0000","last_peered":"2023-09-23T19:14:12.843926+0000","last_clean":"2023-09-23T19:14:12.843926+0000","last_became_active":"2023-09-23T19:11:17.669939+0000","last_became_peered":"2023-09-23T19:11:17.669939+0000","last_unstale":"2023-09-23T19:14:12.843926+0000","last_undegraded":"2023-09-23T19:14:12.843926+0000","last_fullsized":"2023-09-23T19:14:12.843926+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:49.886612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":31,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.724945+0000","last_change":"2023-09-23T19:11:17.682022+0000","last_active":"2023-09-23T19:14:11.724945+0000","last_peered":"2023-09-23T19:14:11.724945+0000","last_clean":"2023-09-23T19:14:11.724945+0000","last_became_active":"2023-09-23T19:11:17.681383+0000","last_became_peered":"2023-09-23T19:11:17.681383+0000","last_unstale":"2023-09-23T19:14:11.724945+0000","last_undegraded":"2023-09-23T19:14:11.724945+0000","last_fullsized":"2023-09-23T19:14:11.724945+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":419,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:12.844043+0000","last_change":"2023-09-23T19:11:17.676241+0000","last_active":"2023-09-23T19:14:12.844043+0000","last_peered":"2023-09-23T19:14:12.844043+0000","last_clean":"2023-09-23T19:14:12.844043+0000","last_became_active":"2023-09-23T19:11:17.675971+0000","last_became_peered":"2023-09-23T19:11:17.675971+0000","last_unstale":"2023-09-23T19:14:12.844043+0000","last_undegraded":"2023-09-23T19:14:12.844043+0000","last_fullsized":"2023-09-23T19:14:12.844043+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:48:01.923477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":211,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.894601+0000","last_change":"2023-09-23T18:58:38.715946+0000","last_active":"2023-09-23T19:14:21.894601+0000","last_peered":"2023-09-23T19:14:21.894601+0000","last_clean":"2023-09-23T19:14:21.894601+0000","last_became_active":"2023-09-23T18:58:38.715584+0000","last_became_peered":"2023-09-23T18:58:38.715584+0000","last_unstale":"2023-09-23T19:14:21.894601+0000","last_undegraded":"2023-09-23T19:14:21.894601+0000","last_fullsized":"2023-09-23T19:14:21.894601+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:57.811713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":146,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.912992+0000","last_change":"2023-09-23T19:11:17.690830+0000","last_active":"2023-09-23T19:14:11.912992+0000","last_peered":"2023-09-23T19:14:11.912992+0000","last_clean":"2023-09-23T19:14:11.912992+0000","last_became_active":"2023-09-23T19:11:17.672837+0000","last_became_peered":"2023-09-23T19:11:17.672837+0000","last_unstale":"2023-09-23T19:14:11.912992+0000","last_undegraded":"2023-09-23T19:14:11.912992+0000","last_fullsized":"2023-09-23T19:14:11.912992+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":31,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.726051+0000","last_change":"2023-09-23T19:11:17.682804+0000","last_active":"2023-09-23T19:14:11.726051+0000","last_peered":"2023-09-23T19:14:11.726051+0000","last_clean":"2023-09-23T19:14:11.726051+0000","last_became_active":"2023-09-23T19:11:17.682024+0000","last_became_peered":"2023-09-23T19:11:17.682024+0000","last_unstale":"2023-09-23T19:14:11.726051+0000","last_undegraded":"2023-09-23T19:14:11.726051+0000","last_fullsized":"2023-09-23T19:14:11.726051+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":146,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.913314+0000","last_change":"2023-09-23T19:11:17.742516+0000","last_active":"2023-09-23T19:14:11.913314+0000","last_peered":"2023-09-23T19:14:11.913314+0000","last_clean":"2023-09-23T19:14:11.913314+0000","last_became_active":"2023-09-23T19:11:17.675587+0000","last_became_peered":"2023-09-23T19:11:17.675587+0000","last_unstale":"2023-09-23T19:14:11.913314+0000","last_undegraded":"2023-09-23T19:14:11.913314+0000","last_fullsized":"2023-09-23T19:14:11.913314+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":31,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.725285+0000","last_change":"2023-09-23T19:11:17.684025+0000","last_active":"2023-09-23T19:14:11.725285+0000","last_peered":"2023-09-23T19:14:11.725285+0000","last_clean":"2023-09-23T19:14:11.725285+0000","last_became_active":"2023-09-23T19:11:17.683571+0000","last_became_peered":"2023-09-23T19:11:17.683571+0000","last_unstale":"2023-09-23T19:14:11.725285+0000","last_undegraded":"2023-09-23T19:14:11.725285+0000","last_fullsized":"2023-09-23T19:14:11.725285+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":139,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.721291+0000","last_change":"2023-09-23T18:58:38.711301+0000","last_active":"2023-09-23T19:14:21.721291+0000","last_peered":"2023-09-23T19:14:21.721291+0000","last_clean":"2023-09-23T19:14:21.721291+0000","last_became_active":"2023-09-23T18:58:38.710923+0000","last_became_peered":"2023-09-23T18:58:38.710923+0000","last_unstale":"2023-09-23T19:14:21.721291+0000","last_undegraded":"2023-09-23T19:14:21.721291+0000","last_fullsized":"2023-09-23T19:14:21.721291+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:17.756147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":138,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.721243+0000","last_change":"2023-09-23T18:58:38.711839+0000","last_active":"2023-09-23T19:14:21.721243+0000","last_peered":"2023-09-23T19:14:21.721243+0000","last_clean":"2023-09-23T19:14:21.721243+0000","last_became_active":"2023-09-23T18:58:38.711347+0000","last_became_peered":"2023-09-23T18:58:38.711347+0000","last_unstale":"2023-09-23T19:14:21.721243+0000","last_undegraded":"2023-09-23T19:14:21.721243+0000","last_fullsized":"2023-09-23T19:14:21.721243+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:55.319645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":497,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T19:14:21.895183+0000","last_change":"2023-09-23T19:08:37.776379+0000","last_active":"2023-09-23T19:14:21.895183+0000","last_peered":"2023-09-23T19:14:21.895183+0000","last_clean":"2023-09-23T19:14:21.895183+0000","last_became_active":"2023-09-23T19:08:37.776012+0000","last_became_peered":"2023-09-23T19:08:37.776012+0000","last_unstale":"2023-09-23T19:14:21.895183+0000","last_undegraded":"2023-09-23T19:14:21.895183+0000","last_fullsized":"2023-09-23T19:14:21.895183+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:07.870594+0000","last_clean_scrub_stamp":"2023-09-23T18:15:07.870594+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:16.823018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":31,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T19:14:11.724306+0000","last_change":"2023-09-23T19:11:17.681695+0000","last_active":"2023-09-23T19:14:11.724306+0000","last_peered":"2023-09-23T19:14:11.724306+0000","last_clean":"2023-09-23T19:14:11.724306+0000","last_became_active":"2023-09-23T19:11:17.681228+0000","last_became_peered":"2023-09-23T19:11:17.681228+0000","last_unstale":"2023-09-23T19:14:11.724306+0000","last_undegraded":"2023-09-23T19:14:11.724306+0000","last_fullsized":"2023-09-23T19:14:11.724306+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:15:08.873352+0000","last_clean_scrub_stamp":"2023-09-23T18:15:08.873352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":13204,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56162,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11934,"num_read_kb":26365,"num_write":1690,"num_write_kb":25426,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":159,"num_bytes_recovered":16777235,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7733,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":367,"seq":1576252997716,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":347624,"kb_used_data":12768,"kb_used_omap":13,"kb_used_meta":334834,"kb_avail":93397528,"statfs":{"total":95995035648,"available":95639068672,"internally_reserved":0,"allocated":13074432,"data_stored":12088504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14028,"internal_metadata":342870324},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:15:14 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.94799999999999995,"15min":0.92000000000000004},"min":{"1min":0.52500000000000002,"5min":0.50800000000000001,"15min":0.48099999999999998},"max":{"1min":1.222,"5min":1.738,"15min":1.738},"last":1.054},{"interface":"front","average":{"1min":0.82499999999999996,"5min":0.89000000000000001,"15min":0.83499999999999996},"min":{"1min":0.55300000000000005,"5min":0.46000000000000002,"15min":0.36599999999999999},"max":{"1min":1.0880000000000001,"5min":1.3959999999999999,"15min":1.3959999999999999},"last":0.95699999999999996}]},{"osd":1,"last update":"Sat Sep 23 19:15:14 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":0.94499999999999995,"15min":0.91600000000000004},"min":{"1min":0.61699999999999999,"5min":0.53000000000000003,"15min":0.49299999999999999},"max":{"1min":1.125,"5min":1.478,"15min":1.478},"last":1.095},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.98399999999999999,"15min":0.93200000000000005},"min":{"1min":0.60199999999999998,"5min":0.59099999999999997,"15min":0.46400000000000002},"max":{"1min":1.3620000000000001,"5min":1.4239999999999999,"15min":1.4379999999999999},"last":1.2549999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:15:14 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.97099999999999997,"15min":1.0009999999999999},"min":{"1min":0.63800000000000001,"5min":0.56100000000000005,"15min":0.53600000000000003},"max":{"1min":1.194,"5min":1.462,"15min":2.4049999999999998},"last":1.02},{"interface":"front","average":{"1min":0.97399999999999998,"5min":1.0029999999999999,"15min":1.0209999999999999},"min":{"1min":0.69899999999999995,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.2749999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.90100000000000002}]},{"osd":3,"last update":"Sat Sep 23 19:15:14 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.93799999999999994,"15min":0.871},"min":{"1min":0.72299999999999998,"5min":0.65500000000000003,"15min":0.44900000000000001},"max":{"1min":1.0880000000000001,"5min":1.357,"15min":1.357},"last":1.0029999999999999},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.0049999999999999,"15min":1.048},"min":{"1min":0.63500000000000001,"5min":0.628,"15min":0.57899999999999996},"max":{"1min":1.206,"5min":1.6579999999999999,"15min":2.2749999999999999},"last":0.98099999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:15:14 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":0.98399999999999999,"15min":0.96799999999999997},"min":{"1min":0.65900000000000003,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.472,"5min":1.6399999999999999,"15min":1.6399999999999999},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.0049999999999999,"5min":0.97199999999999998,"15min":0.93999999999999995},"min":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.5029999999999999,"5min":1.6299999999999999,"15min":1.6299999999999999},"last":0.80900000000000005}]},{"osd":5,"last update":"Sat Sep 23 19:15:14 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":1.002,"15min":1.026},"min":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"max":{"1min":1.266,"5min":1.615,"15min":2.3660000000000001},"last":0.76400000000000001},{"interface":"front","average":{"1min":1.056,"5min":1.0880000000000001,"15min":1.0840000000000001},"min":{"1min":0.58899999999999997,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.548,"5min":1.6879999999999999,"15min":1.6879999999999999},"last":1.323}]},{"osd":6,"last update":"Sat Sep 23 19:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.0880000000000001,"15min":1.1100000000000001},"min":{"1min":0.79400000000000004,"5min":0.68400000000000005,"15min":0.64900000000000002},"max":{"1min":1.3140000000000001,"5min":1.7450000000000001,"15min":2.3300000000000001},"last":0.93100000000000005},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.171,"15min":1.1579999999999999},"min":{"1min":0.753,"5min":0.69999999999999996,"15min":0.69999999999999996},"max":{"1min":1.667,"5min":1.8120000000000001,"15min":1.8120000000000001},"last":1.153}]}]},{"osd":6,"up_from":42,"seq":180388627209,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":899932,"kb_used_data":4564,"kb_used_omap":0,"kb_used_meta":895360,"kb_avail":92845220,"statfs":{"total":95995035648,"available":95073505280,"internally_reserved":0,"allocated":4673536,"data_stored":3699454,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916848640},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:15:09 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":1.0229999999999999,"15min":1.609},"min":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.26300000000000001},"max":{"1min":1.419,"5min":4.5730000000000004,"15min":100.648},"last":1.524},{"interface":"front","average":{"1min":1.1379999999999999,"5min":1.119,"15min":1.7110000000000001},"min":{"1min":0.91900000000000004,"5min":0.66500000000000004,"15min":0.317},"max":{"1min":1.351,"5min":4.6449999999999996,"15min":100.70399999999999},"last":1.4550000000000001}]},{"osd":1,"last update":"Sat Sep 23 19:15:09 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":0.89800000000000002,"15min":1.21},"min":{"1min":0.55800000000000005,"5min":0.53900000000000003,"15min":0.29699999999999999},"max":{"1min":1.3080000000000001,"5min":1.3380000000000001,"15min":32.149999999999999},"last":1.272},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.89000000000000001,"15min":1.1419999999999999},"min":{"1min":0.58599999999999997,"5min":0.56599999999999995,"15min":0.27900000000000003},"max":{"1min":1.27,"5min":1.4470000000000001,"15min":32.814},"last":1.5760000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:15:09 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":0.92800000000000005,"15min":1.583},"min":{"1min":0.64300000000000002,"5min":0.57299999999999995,"15min":0.33200000000000002},"max":{"1min":1.226,"5min":1.296,"15min":66.587999999999994},"last":0.95099999999999996},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.012,"15min":1.639},"min":{"1min":0.79300000000000004,"5min":0.61199999999999999,"15min":0.372},"max":{"1min":1.3500000000000001,"5min":1.367,"15min":66.614000000000004},"last":1.71}]},{"osd":3,"last update":"Sat Sep 23 19:15:23 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":0.99099999999999999,"15min":1.3380000000000001},"min":{"1min":0.73699999999999999,"5min":0.61599999999999999,"15min":0.38100000000000001},"max":{"1min":1.329,"5min":1.6259999999999999,"15min":93.456000000000003},"last":0.878},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.0740000000000001,"15min":1.369},"min":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.38900000000000001},"max":{"1min":1.448,"5min":4.593,"15min":93.472999999999999},"last":1.405}]},{"osd":4,"last update":"Sat Sep 23 19:15:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":0.98799999999999999,"15min":0.85099999999999998},"min":{"1min":0.73599999999999999,"5min":0.63600000000000001,"15min":0.36099999999999999},"max":{"1min":1.8049999999999999,"5min":1.8049999999999999,"15min":3.1899999999999999},"last":1.859},{"interface":"front","average":{"1min":1.1910000000000001,"5min":1.1739999999999999,"15min":0.996},"min":{"1min":0.68500000000000005,"5min":0.48699999999999999,"15min":0.41799999999999998},"max":{"1min":1.659,"5min":4.8179999999999996,"15min":4.8179999999999996},"last":1.607}]},{"osd":5,"last update":"Sat Sep 23 19:14:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":1.1279999999999999,"15min":1.0529999999999999},"min":{"1min":0.69699999999999995,"5min":0.66900000000000004,"15min":0.54400000000000004},"max":{"1min":1.4850000000000001,"5min":4.7309999999999999,"15min":4.7309999999999999},"last":1.3680000000000001},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.1870000000000001,"15min":1.0409999999999999},"min":{"1min":0.96399999999999997,"5min":0.70399999999999996,"15min":0.55000000000000004},"max":{"1min":1.5740000000000001,"5min":4.7859999999999996,"15min":4.7859999999999996},"last":1.7949999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:15:09 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.107,"15min":1.089},"min":{"1min":0.80200000000000005,"5min":0.67500000000000004,"15min":0.65400000000000003},"max":{"1min":1.5109999999999999,"5min":4.7009999999999996,"15min":4.7009999999999996},"last":1.3220000000000001},{"interface":"front","average":{"1min":1.077,"5min":1.083,"15min":1.099},"min":{"1min":0.74399999999999999,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.4990000000000001,"5min":4.6260000000000003,"15min":4.6260000000000003},"last":1.931}]}]},{"osd":1,"up_from":123,"seq":528280977912,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":997340,"kb_used_data":5204,"kb_used_omap":7,"kb_used_meta":992120,"kb_avail":92747812,"statfs":{"total":95995035648,"available":94973759488,"internally_reserved":0,"allocated":5328896,"data_stored":3262807,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":1015931662},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:15:15 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.68100000000000005,"15min":1.7889999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.23499999999999999},"max":{"1min":0.84099999999999997,"5min":1.238,"15min":90.492000000000004},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.77900000000000003,"15min":1.627},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.188},"max":{"1min":0.97199999999999998,"5min":1.359,"15min":90.552999999999997},"last":0.74199999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:15:15 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.72199999999999998,"15min":1.512},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.219},"max":{"1min":0.96299999999999997,"5min":1.1679999999999999,"15min":87.832999999999998},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.754,"5min":0.80800000000000005,"15min":1.6990000000000001},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.24299999999999999},"max":{"1min":1.0620000000000001,"5min":1.181,"15min":87.846999999999994},"last":0.59599999999999997}]},{"osd":3,"last update":"Sat Sep 23 19:15:15 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":0.85099999999999998,"15min":1.3420000000000001},"min":{"1min":0.434,"5min":0.434,"15min":0.24099999999999999},"max":{"1min":1.175,"5min":1.486,"15min":99.126000000000005},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.76300000000000001,"5min":0.85199999999999998,"15min":1.3200000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.21299999999999999},"max":{"1min":1.1200000000000001,"5min":1.4419999999999999,"15min":99.165999999999997},"last":0.80300000000000005}]},{"osd":4,"last update":"Sat Sep 23 19:15:34 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":1.036,"15min":1.0449999999999999},"min":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.32900000000000001},"max":{"1min":1.2709999999999999,"5min":1.548,"15min":7.3070000000000004},"last":1.1000000000000001},{"interface":"front","average":{"1min":0.94799999999999995,"5min":1.0700000000000001,"15min":1.0580000000000001},"min":{"1min":0.69699999999999995,"5min":0.65100000000000002,"15min":0.40699999999999997},"max":{"1min":1.2609999999999999,"5min":1.7829999999999999,"15min":7.5640000000000001},"last":1.0469999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:15:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.0469999999999999,"15min":1.026},"min":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.59499999999999997},"max":{"1min":1.365,"5min":1.948,"15min":3.6539999999999999},"last":1.1419999999999999},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.0880000000000001,"15min":1.115},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.55400000000000005},"max":{"1min":1.2310000000000001,"5min":1.8380000000000001,"15min":4.6159999999999997},"last":1.002}]},{"osd":6,"last update":"Sat Sep 23 19:15:15 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":1.0580000000000001,"15min":1.0489999999999999},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.38100000000000001},"max":{"1min":1.21,"5min":1.708,"15min":7.6299999999999999},"last":0.96899999999999997},{"interface":"front","average":{"1min":0.98799999999999999,"5min":1.0640000000000001,"15min":1.2070000000000001},"min":{"1min":0.66900000000000004,"5min":0.65200000000000002,"15min":0.38100000000000001},"max":{"1min":1.274,"5min":1.8640000000000001,"15min":12.887},"last":0.86799999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:15:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":1.1579999999999999,"15min":1.0840000000000001},"min":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":1.3759999999999999,"5min":1.901,"15min":1.901},"last":1.2030000000000001},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.1459999999999999,"15min":1.0580000000000001},"min":{"1min":0.70499999999999996,"5min":0.69399999999999995,"15min":0.66100000000000003},"max":{"1min":1.3799999999999999,"5min":1.931,"15min":1.931},"last":0.94299999999999995}]}]},{"osd":0,"up_from":88,"seq":377957122642,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1129228,"kb_used_data":9668,"kb_used_omap":8,"kb_used_meta":1119543,"kb_avail":92615924,"statfs":{"total":95995035648,"available":94838706176,"internally_reserved":0,"allocated":9900032,"data_stored":8023266,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":1146412926},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 19:15:31 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.71399999999999997,"15min":1.7729999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.21099999999999999},"max":{"1min":1.9550000000000001,"5min":1.9550000000000001,"15min":49.168999999999997},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.76600000000000001,"5min":0.81699999999999995,"15min":1.8580000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.249},"max":{"1min":2.0379999999999998,"5min":2.0379999999999998,"15min":50.673000000000002},"last":0.50900000000000001}]},{"osd":2,"last update":"Sat Sep 23 19:15:13 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.82999999999999996,"15min":1.7490000000000001},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.22600000000000001},"max":{"1min":3.198,"5min":3.198,"15min":52.914000000000001},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.751,"15min":1.4199999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.22900000000000001},"max":{"1min":1.0309999999999999,"5min":1.2589999999999999,"15min":48.963999999999999},"last":0.56499999999999995}]},{"osd":3,"last update":"Sat Sep 23 19:15:43 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.79900000000000004,"15min":1.165},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.23200000000000001},"max":{"1min":2.3679999999999999,"5min":2.3679999999999999,"15min":12.904},"last":0.81100000000000005},{"interface":"front","average":{"1min":0.748,"5min":0.92900000000000005,"15min":1.3089999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.23899999999999999},"max":{"1min":1.802,"5min":1.802,"15min":14.91},"last":0.85299999999999998}]},{"osd":4,"last update":"Sat Sep 23 19:15:43 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":0.93700000000000006,"15min":1.2470000000000001},"min":{"1min":0.66700000000000004,"5min":0.629,"15min":0.375},"max":{"1min":2.226,"5min":2.226,"15min":55.710999999999999},"last":0.89000000000000001},{"interface":"front","average":{"1min":0.95299999999999996,"5min":1.0449999999999999,"15min":1.331},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.40400000000000003},"max":{"1min":1.202,"5min":1.895,"15min":46.509},"last":1.018}]},{"osd":5,"last update":"Sat Sep 23 19:14:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":1.085,"15min":1.0269999999999999},"min":{"1min":0.81299999999999994,"5min":0.65800000000000003,"15min":0.57199999999999995},"max":{"1min":1.4590000000000001,"5min":1.8300000000000001,"15min":1.8300000000000001},"last":1.1220000000000001},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.1160000000000001,"15min":1.0609999999999999},"min":{"1min":0.73699999999999999,"5min":0.61299999999999999,"15min":0.48399999999999999},"max":{"1min":1.4850000000000001,"5min":1.79,"15min":1.79},"last":0.996}]},{"osd":6,"last update":"Sat Sep 23 19:15:31 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":1.0449999999999999,"15min":1.2729999999999999},"min":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.378},"max":{"1min":2.2930000000000001,"5min":2.2930000000000001,"15min":55.796999999999997},"last":0.94499999999999995},{"interface":"front","average":{"1min":1.0429999999999999,"5min":1.157,"15min":1.4019999999999999},"min":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.38600000000000001},"max":{"1min":1.3660000000000001,"5min":1.8080000000000001,"15min":46.411999999999999},"last":1.163}]},{"osd":7,"last update":"Sat Sep 23 19:15:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.206,"15min":1.125},"min":{"1min":0.78600000000000003,"5min":0.72299999999999998,"15min":0.72299999999999998},"max":{"1min":1.5880000000000001,"5min":1.712,"15min":1.712},"last":1.079},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.1220000000000001,"15min":1.0860000000000001},"min":{"1min":0.746,"5min":0.73699999999999999,"15min":0.73699999999999999},"max":{"1min":2.145,"5min":2.145,"15min":2.145},"last":1.0449999999999999}]}]},{"osd":2,"up_from":98,"seq":420906795590,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1000528,"kb_used_data":9096,"kb_used_omap":11,"kb_used_meta":991412,"kb_avail":92744624,"statfs":{"total":95995035648,"available":94970494976,"internally_reserved":0,"allocated":9314304,"data_stored":7433633,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11949,"internal_metadata":1015206227},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:15:05 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.71799999999999997,"15min":2.6970000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.184},"max":{"1min":1.0820000000000001,"5min":1.179,"15min":143.048},"last":0.751},{"interface":"front","average":{"1min":0.747,"5min":0.71299999999999997,"15min":2.7909999999999999},"min":{"1min":0.50700000000000001,"5min":0.38600000000000001,"15min":0.189},"max":{"1min":1.038,"5min":1.111,"15min":143.25200000000001},"last":0.70599999999999996}]},{"osd":1,"last update":"Sat Sep 23 19:15:05 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":0.81899999999999995,"15min":1.2949999999999999},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.24299999999999999},"max":{"1min":1.1819999999999999,"5min":1.272,"15min":46.601999999999997},"last":0.626},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.83499999999999996,"15min":1.4650000000000001},"min":{"1min":0.56100000000000005,"5min":0.34399999999999997,"15min":0.23899999999999999},"max":{"1min":1.377,"5min":1.377,"15min":49.411999999999999},"last":0.58899999999999997}]},{"osd":3,"last update":"Sat Sep 23 19:15:41 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.72799999999999998,"15min":0.92400000000000004},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.24399999999999999},"max":{"1min":1.0449999999999999,"5min":1.157,"15min":7.9619999999999997},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.76700000000000002,"5min":0.86399999999999999,"15min":0.98999999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.23200000000000001},"max":{"1min":1.159,"5min":1.2609999999999999,"15min":9.1679999999999993},"last":0.92400000000000004}]},{"osd":4,"last update":"Sat Sep 23 19:15:41 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.087,"15min":1.129},"min":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.38800000000000001},"max":{"1min":1.526,"5min":1.597,"15min":8.9969999999999999},"last":1.0649999999999999},{"interface":"front","average":{"1min":0.96199999999999997,"5min":0.98199999999999998,"15min":1.038},"min":{"1min":0.68200000000000005,"5min":0.57499999999999996,"15min":0.40899999999999997},"max":{"1min":1.266,"5min":1.4199999999999999,"15min":6.0049999999999999},"last":1.0009999999999999}]},{"osd":5,"last update":"Sat Sep 23 19:14:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":1.0700000000000001,"15min":1.008},"min":{"1min":0.80800000000000005,"5min":0.624,"15min":0.55400000000000005},"max":{"1min":1.486,"5min":1.5449999999999999,"15min":1.6020000000000001},"last":1.0269999999999999},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.1759999999999999,"15min":1.0920000000000001},"min":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.544,"5min":1.73,"15min":1.73},"last":1.0129999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:15:05 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.0249999999999999,"15min":1.026},"min":{"1min":0.78400000000000003,"5min":0.61399999999999999,"15min":0.35999999999999999},"max":{"1min":1.3009999999999999,"5min":1.4419999999999999,"15min":5.9649999999999999},"last":0.82299999999999995},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.1859999999999999,"15min":1.179},"min":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.36699999999999999},"max":{"1min":1.4279999999999999,"5min":1.8069999999999999,"15min":9.048},"last":1.2390000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":1.196,"15min":1.143},"min":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"max":{"1min":1.4530000000000001,"5min":1.8560000000000001,"15min":1.8560000000000001},"last":1.125},{"interface":"front","average":{"1min":1.159,"5min":1.1279999999999999,"15min":1.1439999999999999},"min":{"1min":0.82999999999999996,"5min":0.69499999999999995,"15min":0.623},"max":{"1min":1.528,"5min":1.528,"15min":1.6879999999999999},"last":0.98299999999999998}]}]},{"osd":3,"up_from":103,"seq":442381632065,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":935904,"kb_used_data":5080,"kb_used_omap":4,"kb_used_meta":930811,"kb_avail":92809248,"statfs":{"total":95995035648,"available":95036669952,"internally_reserved":0,"allocated":5201920,"data_stored":3247104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4826,"internal_metadata":953150758},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:15:35 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.65800000000000003,"15min":1.2849999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.19700000000000001},"max":{"1min":0.879,"5min":1.177,"15min":40.652000000000001},"last":0.76100000000000001},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.748,"15min":1.4019999999999999},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.18099999999999999},"max":{"1min":1.1819999999999999,"5min":1.202,"15min":42.091999999999999},"last":0.86599999999999999}]},{"osd":1,"last update":"Sat Sep 23 19:15:25 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.84499999999999997,"15min":3.1190000000000002},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.252},"max":{"1min":0.95599999999999996,"5min":6.1680000000000001,"15min":343.79300000000001},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.86599999999999999,"15min":3.2069999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.22500000000000001},"max":{"1min":1.0740000000000001,"5min":6.1369999999999996,"15min":343.73000000000002},"last":0.81799999999999995}]},{"osd":2,"last update":"Sat Sep 23 19:15:35 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.77200000000000002,"15min":1.526},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.26300000000000001},"max":{"1min":1.204,"5min":1.204,"15min":61.334000000000003},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.68700000000000006,"5min":0.83599999999999997,"15min":1.417},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.26900000000000002},"max":{"1min":1.115,"5min":1.3080000000000001,"15min":52.493000000000002},"last":0.51000000000000001}]},{"osd":4,"last update":"Sat Sep 23 19:15:35 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.97699999999999998,"15min":1.073},"min":{"1min":0.624,"5min":0.624,"15min":0.41399999999999998},"max":{"1min":1.5660000000000001,"5min":1.5660000000000001,"15min":7.9119999999999999},"last":0.98099999999999998},{"interface":"front","average":{"1min":0.88600000000000001,"5min":1.0369999999999999,"15min":1.0349999999999999},"min":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.42399999999999999},"max":{"1min":1.286,"5min":1.621,"15min":5.0949999999999998},"last":0.90100000000000002}]},{"osd":5,"last update":"Sat Sep 23 19:14:50 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":1.1259999999999999,"15min":1.004},"min":{"1min":0.747,"5min":0.71599999999999997,"15min":0.60599999999999998},"max":{"1min":1.6060000000000001,"5min":3.1360000000000001,"15min":3.1360000000000001},"last":0.95899999999999996},{"interface":"front","average":{"1min":1.2370000000000001,"5min":1.179,"15min":1.0269999999999999},"min":{"1min":0.71299999999999997,"5min":0.68899999999999995,"15min":0.624},"max":{"1min":1.5669999999999999,"5min":1.621,"15min":1.633},"last":0.80000000000000004}]},{"osd":6,"last update":"Sat Sep 23 19:15:35 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.1259999999999999,"15min":1.123},"min":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.33700000000000002},"max":{"1min":1.623,"5min":3.0430000000000001,"15min":8.4169999999999998},"last":1.04},{"interface":"front","average":{"1min":0.995,"5min":1.157,"15min":1.125},"min":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.40100000000000002},"max":{"1min":1.4910000000000001,"5min":1.7529999999999999,"15min":5.125},"last":1.0820000000000001}]},{"osd":7,"last update":"Sat Sep 23 19:15:15 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":1.21,"15min":1.1180000000000001},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.6699999999999999,"5min":1.849,"15min":1.849},"last":1.0029999999999999},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.127,"15min":1.0249999999999999},"min":{"1min":0.752,"5min":0.752,"15min":0.70899999999999996},"max":{"1min":1.6140000000000001,"5min":1.6140000000000001,"15min":1.6140000000000001},"last":0.92800000000000005}]}]},{"osd":4,"up_from":286,"seq":1228360646895,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":369888,"kb_used_data":8088,"kb_used_omap":10,"kb_used_meta":361781,"kb_avail":93375264,"statfs":{"total":95995035648,"available":95616270336,"internally_reserved":0,"allocated":8282112,"data_stored":7303762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11243,"internal_metadata":370463765},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:15:28 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.94299999999999995,"15min":1.1759999999999999},"min":{"1min":0.55400000000000005,"5min":0.5,"15min":0.30599999999999999},"max":{"1min":1.718,"5min":1.718,"15min":47.448999999999998},"last":1.2050000000000001},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.998,"15min":1.171},"min":{"1min":0.55900000000000005,"5min":0.55700000000000005,"15min":0.29399999999999998},"max":{"1min":1.4299999999999999,"5min":1.867,"15min":47.487000000000002},"last":0.94799999999999995}]},{"osd":1,"last update":"Sat Sep 23 19:15:28 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":0.997,"15min":0.98599999999999999},"min":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.313},"max":{"1min":1.8109999999999999,"5min":1.8109999999999999,"15min":4.7670000000000003},"last":1.0600000000000001},{"interface":"front","average":{"1min":0.96099999999999997,"5min":1.004,"15min":0.98999999999999999},"min":{"1min":0.69499999999999995,"5min":0.58299999999999996,"15min":0.35199999999999998},"max":{"1min":1.55,"5min":2.6880000000000002,"15min":5.0099999999999998},"last":0.99299999999999999}]},{"osd":2,"last update":"Sat Sep 23 19:15:28 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":1.0349999999999999,"15min":1.0600000000000001},"min":{"1min":0.69799999999999995,"5min":0.63200000000000001,"15min":0.35099999999999998},"max":{"1min":1.603,"5min":2.4039999999999999,"15min":15.493},"last":1.0329999999999999},{"interface":"front","average":{"1min":0.97399999999999998,"5min":1.002,"15min":1.091},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.34799999999999998},"max":{"1min":1.514,"5min":1.514,"15min":15.416},"last":0.91200000000000003}]},{"osd":3,"last update":"Sat Sep 23 19:15:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":1.0249999999999999,"15min":1.288},"min":{"1min":0.75900000000000001,"5min":0.68200000000000005,"15min":0.39400000000000002},"max":{"1min":1.617,"5min":1.617,"15min":62.235999999999997},"last":1.0920000000000001},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.0289999999999999,"15min":1.2390000000000001},"min":{"1min":0.69899999999999995,"5min":0.66100000000000003,"15min":0.33900000000000002},"max":{"1min":1.8620000000000001,"5min":1.8620000000000001,"15min":61.423000000000002},"last":0.877}]},{"osd":5,"last update":"Sat Sep 23 19:15:47 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.028,"15min":1.012},"min":{"1min":0.748,"5min":0.56499999999999995,"15min":0.53500000000000003},"max":{"1min":1.9570000000000001,"5min":2.609,"15min":2.609},"last":0.75600000000000001},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.0600000000000001,"15min":0.96599999999999997},"min":{"1min":0.68799999999999994,"5min":0.59899999999999998,"15min":0.51700000000000002},"max":{"1min":1.679,"5min":1.679,"15min":1.7430000000000001},"last":0.77700000000000002}]},{"osd":6,"last update":"Sat Sep 23 19:15:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1739999999999999,"5min":1.1080000000000001,"15min":0.93400000000000005},"min":{"1min":0.91000000000000003,"5min":0.56799999999999995,"15min":0.40100000000000002},"max":{"1min":1.8420000000000001,"5min":1.8420000000000001,"15min":2.0059999999999998},"last":1.1599999999999999},{"interface":"front","average":{"1min":1.206,"5min":1.1040000000000001,"15min":0.94799999999999995},"min":{"1min":0.86599999999999999,"5min":0.56599999999999995,"15min":0.371},"max":{"1min":1.9179999999999999,"5min":1.9179999999999999,"15min":2.8359999999999999},"last":1.145}]},{"osd":7,"last update":"Sat Sep 23 19:15:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2769999999999999,"5min":1.181,"15min":1.151},"min":{"1min":1.004,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.9319999999999999,"5min":1.9319999999999999,"15min":2.1819999999999999},"last":1.006},{"interface":"front","average":{"1min":1.319,"5min":1.1819999999999999,"15min":1.1879999999999999},"min":{"1min":0.86099999999999999,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":2.23,"5min":2.23,"15min":2.23},"last":0.85299999999999998}]}]},{"osd":5,"up_from":360,"seq":1546188226649,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":345868,"kb_used_data":8068,"kb_used_omap":5,"kb_used_meta":337786,"kb_avail":93399284,"statfs":{"total":95995035648,"available":95640866816,"internally_reserved":0,"allocated":8261632,"data_stored":7298895,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6117,"internal_metadata":345892891},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 19:14:50 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.93600000000000005,"15min":0.83799999999999997},"min":{"1min":0.53700000000000003,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.343,"5min":1.343,"15min":1.343},"last":0.90000000000000002},{"interface":"front","average":{"1min":0.90500000000000003,"5min":0.94199999999999995,"15min":0.90800000000000003},"min":{"1min":0.58499999999999996,"5min":0.498,"15min":0.46700000000000003},"max":{"1min":1.292,"5min":1.3660000000000001,"15min":1.5529999999999999},"last":0.96699999999999997}]},{"osd":1,"last update":"Sat Sep 23 19:14:50 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":0.93200000000000005,"15min":0.85399999999999998},"min":{"1min":0.65500000000000003,"5min":0.54200000000000004,"15min":0.42599999999999999},"max":{"1min":1.298,"5min":1.3400000000000001,"15min":1.3400000000000001},"last":0.86799999999999999},{"interface":"front","average":{"1min":1.0129999999999999,"5min":0.94599999999999995,"15min":0.878},"min":{"1min":0.73899999999999999,"5min":0.55300000000000005,"15min":0.51400000000000001},"max":{"1min":1.2749999999999999,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":1.077}]},{"osd":2,"last update":"Sat Sep 23 19:14:50 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1,"15min":0.91600000000000004},"min":{"1min":0.627,"5min":0.57899999999999996,"15min":0.56599999999999995},"max":{"1min":1.4319999999999999,"5min":1.4319999999999999,"15min":1.4319999999999999},"last":0.82699999999999996},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.95199999999999996,"15min":0.88600000000000001},"min":{"1min":0.72999999999999998,"5min":0.55800000000000005,"15min":0.55200000000000005},"max":{"1min":1.46,"5min":1.46,"15min":1.46},"last":1.131}]},{"osd":3,"last update":"Sat Sep 23 19:14:50 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.002,"15min":0.96399999999999997},"min":{"1min":0.84399999999999997,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.284,"5min":1.466,"15min":1.5249999999999999},"last":1.2170000000000001},{"interface":"front","average":{"1min":1.038,"5min":1.0029999999999999,"15min":0.96599999999999997},"min":{"1min":0.68400000000000005,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.3939999999999999,"5min":1.415,"15min":2.121},"last":1.113}]},{"osd":4,"last update":"Sat Sep 23 19:14:50 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.92000000000000004,"15min":0.87},"min":{"1min":0.72999999999999998,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.242,"5min":1.2549999999999999,"15min":1.2729999999999999},"last":1.0569999999999999},{"interface":"front","average":{"1min":0.99299999999999999,"5min":0.94699999999999995,"15min":0.93700000000000006},"min":{"1min":0.69599999999999995,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.375,"5min":1.375,"15min":1.399},"last":1.0009999999999999}]},{"osd":6,"last update":"Sat Sep 23 19:14:50 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.0149999999999999,"15min":0.97699999999999998},"min":{"1min":0.72199999999999998,"5min":0.372,"15min":0.372},"max":{"1min":1.343,"5min":1.448,"15min":1.448},"last":1.1699999999999999},{"interface":"front","average":{"1min":1.125,"5min":1.0269999999999999,"15min":1.004},"min":{"1min":0.78200000000000003,"5min":0.314,"15min":0.314},"max":{"1min":1.3959999999999999,"5min":1.411,"15min":1.411},"last":1.2809999999999999}]},{"osd":7,"last update":"Sat Sep 23 19:15:09 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.0329999999999999,"15min":1.083},"min":{"1min":0.71599999999999997,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.319,"5min":1.319,"15min":1.3520000000000001},"last":1.3280000000000001},{"interface":"front","average":{"1min":1.105,"5min":1.0900000000000001,"15min":1.135},"min":{"1min":0.71199999999999997,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.333,"5min":1.796,"15min":1.796},"last":1.2350000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4224,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8388608,"data_stored":8388608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2112,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8322,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7410,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7725,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3429,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11243,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6117,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11916,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T19:15:54.073 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T19:15:54.073 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-09-23T19:15:54.170 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=99c1727d9a2442d3a6f045f6e895770f Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-09-23T19:15:54.172 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-09-23T19:15:54.182 INFO:tasks.cephadm:Teardown begin 2023-09-23T19:15:54.183 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T19:15:54.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:53 smithi098 bash[79516]: cluster 2023-09-23T19:15:52.836838+0000 mgr.y (mgr.26054) 1752 : cluster 0 pgmap v1964: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:54.196 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:53 smithi098 bash[79516]: audit 2023-09-23T19:15:53.154021+0000 mgr.y (mgr.26054) 1753 : audit 0 from='client.47060 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:15:54.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:53 smithi098 bash[76817]: cluster 2023-09-23T19:15:52.836838+0000 mgr.y (mgr.26054) 1752 : cluster 0 pgmap v1964: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:54.197 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:53 smithi098 bash[76817]: audit 2023-09-23T19:15:53.154021+0000 mgr.y (mgr.26054) 1753 : audit 0 from='client.47060 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:15:54.217 DEBUG:teuthology.orchestra.run.smithi173:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T19:15:54.230 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:53 smithi173 bash[59422]: cluster 2023-09-23T19:15:52.836838+0000 mgr.y (mgr.26054) 1752 : cluster 0 pgmap v1964: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T19:15:54.230 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:53 smithi173 bash[59422]: audit 2023-09-23T19:15:53.154021+0000 mgr.y (mgr.26054) 1753 : audit 0 from='client.47060 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T19:15:54.233 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-09-23T19:15:54.233 DEBUG:teuthology.orchestra.run.smithi098:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-09-23T19:15:54.289 DEBUG:teuthology.orchestra.run.smithi173:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-09-23T19:15:54.297 INFO:tasks.cephadm:Stopping all daemons... 2023-09-23T19:15:54.297 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-09-23T19:15:54.298 DEBUG:teuthology.orchestra.run.smithi098:> sudo systemctl stop ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mon.a 2023-09-23T19:15:54.577 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:54 smithi098 systemd[1]: Stopping Ceph mon.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T19:15:54.901 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:15:54 smithi098 bash[73085]: [23/Sep/2023:19:15:54] ENGINE Bus STOPPING 2023-09-23T19:15:54.902 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:15:54 smithi098 bash[73085]: [23/Sep/2023:19:15:54] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T19:15:54.902 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:15:54 smithi098 bash[73085]: [23/Sep/2023:19:15:54] ENGINE Bus STOPPED 2023-09-23T19:15:54.902 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:15:54 smithi098 bash[73085]: [23/Sep/2023:19:15:54] ENGINE Bus STARTING 2023-09-23T19:15:54.902 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:15:54 smithi098 bash[73085]: [23/Sep/2023:19:15:54] ENGINE Serving on http://:::9283 2023-09-23T19:15:54.902 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:15:54 smithi098 bash[73085]: [23/Sep/2023:19:15:54] ENGINE Bus STARTED 2023-09-23T19:15:54.902 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:54 smithi098 bash[76817]: debug 2023-09-23T19:15:54.568+0000 7fc8a820c700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-09-23T19:15:54.903 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:54 smithi098 bash[76817]: debug 2023-09-23T19:15:54.568+0000 7fc8a820c700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-09-23T19:15:55.825 INFO:journalctl@ceph.mon.a.smithi098.stdout:Sep 23 19:15:55 smithi098 bash[284601]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-mon-a 2023-09-23T19:15:55.830 DEBUG:teuthology.orchestra.run.smithi098:> sudo pkill -f 'journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mon.a.service' 2023-09-23T19:15:55.945 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T19:15:55.945 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-09-23T19:15:55.946 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-09-23T19:15:55.946 DEBUG:teuthology.orchestra.run.smithi098:> sudo systemctl stop ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mon.c 2023-09-23T19:15:56.088 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:55 smithi098 systemd[1]: Stopping Ceph mon.c for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T19:15:56.409 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:56 smithi098 bash[79516]: debug 2023-09-23T19:15:56.092+0000 7f5d20abc700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-09-23T19:15:56.409 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:56 smithi098 bash[79516]: debug 2023-09-23T19:15:56.092+0000 7f5d20abc700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-09-23T19:15:56.409 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:15:56 smithi098 bash[73085]: ::ffff:172.21.15.173 - - [23/Sep/2023:19:15:56] "GET /metrics HTTP/1.1" 200 34161 "" "Prometheus/2.33.4" 2023-09-23T19:15:57.191 INFO:journalctl@ceph.mon.c.smithi098.stdout:Sep 23 19:15:56 smithi098 bash[284718]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-mon-c 2023-09-23T19:15:57.206 DEBUG:teuthology.orchestra.run.smithi098:> sudo pkill -f 'journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mon.c.service' 2023-09-23T19:15:57.298 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T19:15:57.299 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-09-23T19:15:57.299 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-09-23T19:15:57.299 DEBUG:teuthology.orchestra.run.smithi173:> sudo systemctl stop ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mon.b 2023-09-23T19:15:57.640 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:57 smithi173 systemd[1]: Stopping Ceph mon.b for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T19:15:57.640 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:57 smithi173 bash[59422]: debug 2023-09-23T19:15:57.483+0000 7f0772609700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-09-23T19:15:57.640 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:57 smithi173 bash[59422]: debug 2023-09-23T19:15:57.483+0000 7f0772609700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-09-23T19:15:58.808 DEBUG:teuthology.orchestra.run.smithi173:> sudo pkill -f 'journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mon.b.service' 2023-09-23T19:15:58.816 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:58 smithi173 bash[170144]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-mon-b 2023-09-23T19:15:58.817 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:58 smithi173 bash[170208]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-mon-b 2023-09-23T19:15:58.817 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:58 smithi173 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mon.b.service: Succeeded. 2023-09-23T19:15:58.817 INFO:journalctl@ceph.mon.b.smithi173.stdout:Sep 23 19:15:58 smithi173 systemd[1]: Stopped Ceph mon.b for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T19:15:58.837 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T19:15:58.837 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-09-23T19:15:58.838 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-09-23T19:15:58.838 DEBUG:teuthology.orchestra.run.smithi098:> sudo systemctl stop ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mgr.y 2023-09-23T19:15:59.105 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:15:58 smithi098 systemd[1]: Stopping Ceph mgr.y for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T19:15:59.105 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:15:59 smithi098 bash[73085]: debug 2023-09-23T19:15:59.016+0000 7f90e21c2700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T19:16:00.140 INFO:journalctl@ceph.mgr.y.smithi098.stdout:Sep 23 19:15:59 smithi098 bash[285137]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-mgr-y 2023-09-23T19:16:00.147 DEBUG:teuthology.orchestra.run.smithi098:> sudo pkill -f 'journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mgr.y.service' 2023-09-23T19:16:00.226 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T19:16:00.226 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-09-23T19:16:00.226 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-09-23T19:16:00.227 DEBUG:teuthology.orchestra.run.smithi173:> sudo systemctl stop ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mgr.x 2023-09-23T19:16:00.500 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 19:16:00 smithi173 systemd[1]: Stopping Ceph mgr.x for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T19:16:00.500 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 19:16:00 smithi173 bash[54982]: ts=2023-09-23T19:16:00.422Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.98:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.98:8765: connect: connection refused" 2023-09-23T19:16:00.501 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 19:16:00 smithi173 bash[54982]: ts=2023-09-23T19:16:00.423Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.98:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.98:8765: connect: connection refused" 2023-09-23T19:16:00.501 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 19:16:00 smithi173 bash[54982]: ts=2023-09-23T19:16:00.426Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.98:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.98:8765: connect: connection refused" 2023-09-23T19:16:00.501 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 19:16:00 smithi173 bash[54982]: ts=2023-09-23T19:16:00.426Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.98:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.98:8765: connect: connection refused" 2023-09-23T19:16:01.465 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Sep 23 19:16:01 smithi173 bash[170247]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-mgr-x 2023-09-23T19:16:01.472 DEBUG:teuthology.orchestra.run.smithi173:> sudo pkill -f 'journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@mgr.x.service' 2023-09-23T19:16:01.502 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T19:16:01.502 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-09-23T19:16:01.503 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-09-23T19:16:01.503 DEBUG:teuthology.orchestra.run.smithi098:> sudo systemctl stop ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.0 2023-09-23T19:16:01.954 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 19:16:01 smithi098 systemd[1]: Stopping Ceph osd.0 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T19:16:01.955 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 19:16:01 smithi098 bash[85332]: debug 2023-09-23T19:16:01.676+0000 7f76b5e66700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T19:16:01.955 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 19:16:01 smithi098 bash[85332]: debug 2023-09-23T19:16:01.676+0000 7f76b5e66700 -1 osd.0 419 *** Got signal Terminated *** 2023-09-23T19:16:01.955 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 19:16:01 smithi098 bash[85332]: debug 2023-09-23T19:16:01.676+0000 7f76b5e66700 -1 osd.0 419 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T19:16:08.444 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 19:16:08 smithi098 bash[285238]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-0 2023-09-23T19:16:08.445 INFO:journalctl@ceph.osd.0.smithi098.stdout:Sep 23 19:16:08 smithi098 bash[285306]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-0 2023-09-23T19:16:11.178 DEBUG:teuthology.orchestra.run.smithi098:> sudo pkill -f 'journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.0.service' 2023-09-23T19:16:11.260 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T19:16:11.260 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-09-23T19:16:11.261 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-09-23T19:16:11.261 DEBUG:teuthology.orchestra.run.smithi098:> sudo systemctl stop ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T19:16:11.445 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 19:16:11 smithi098 systemd[1]: Stopping Ceph osd.1 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T19:16:11.945 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 19:16:11 smithi098 bash[114693]: debug 2023-09-23T19:16:11.448+0000 7f3a0075c700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T19:16:11.945 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 19:16:11 smithi098 bash[114693]: debug 2023-09-23T19:16:11.448+0000 7f3a0075c700 -1 osd.1 419 *** Got signal Terminated *** 2023-09-23T19:16:11.945 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 19:16:11 smithi098 bash[114693]: debug 2023-09-23T19:16:11.448+0000 7f3a0075c700 -1 osd.1 419 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T19:16:18.188 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 19:16:17 smithi098 bash[285445]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-1 2023-09-23T19:16:18.188 INFO:journalctl@ceph.osd.1.smithi098.stdout:Sep 23 19:16:18 smithi098 bash[285511]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-1 2023-09-23T19:16:20.362 DEBUG:teuthology.orchestra.run.smithi098:> sudo pkill -f 'journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.1.service' 2023-09-23T19:16:20.395 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T19:16:20.396 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-09-23T19:16:20.396 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-09-23T19:16:20.396 DEBUG:teuthology.orchestra.run.smithi098:> sudo systemctl stop ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.2 2023-09-23T19:16:20.609 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 19:16:20 smithi098 systemd[1]: Stopping Ceph osd.2 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T19:16:20.945 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 19:16:20 smithi098 bash[90938]: debug 2023-09-23T19:16:20.600+0000 7fac5359c700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T19:16:20.945 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 19:16:20 smithi098 bash[90938]: debug 2023-09-23T19:16:20.600+0000 7fac5359c700 -1 osd.2 419 *** Got signal Terminated *** 2023-09-23T19:16:20.945 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 19:16:20 smithi098 bash[90938]: debug 2023-09-23T19:16:20.600+0000 7fac5359c700 -1 osd.2 419 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T19:16:27.110 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 19:16:26 smithi098 bash[285653]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-2 2023-09-23T19:16:27.110 INFO:journalctl@ceph.osd.2.smithi098.stdout:Sep 23 19:16:27 smithi098 bash[285718]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-2 2023-09-23T19:16:28.527 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:28 smithi173 bash[162608]: debug 2023-09-23T19:16:28.299+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:28.527 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:28 smithi173 bash[39445]: debug 2023-09-23T19:16:28.059+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:29.194 DEBUG:teuthology.orchestra.run.smithi098:> sudo pkill -f 'journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.2.service' 2023-09-23T19:16:29.227 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T19:16:29.227 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-09-23T19:16:29.227 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-09-23T19:16:29.227 DEBUG:teuthology.orchestra.run.smithi098:> sudo systemctl stop ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.3 2023-09-23T19:16:29.277 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:29 smithi173 bash[39445]: debug 2023-09-23T19:16:29.011+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:29.445 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 19:16:29 smithi098 systemd[1]: Stopping Ceph osd.3 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T19:16:29.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:29 smithi173 bash[162608]: debug 2023-09-23T19:16:29.303+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:29.777 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:29 smithi173 bash[160505]: debug 2023-09-23T19:16:29.347+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:04.735932+0000 front 2023-09-23T19:16:04.736255+0000 (oldest deadline 2023-09-23T19:16:29.336670+0000) 2023-09-23T19:16:29.944 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 19:16:29 smithi098 bash[93735]: debug 2023-09-23T19:16:29.448+0000 7fa99940b700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T19:16:29.945 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 19:16:29 smithi098 bash[93735]: debug 2023-09-23T19:16:29.448+0000 7fa99940b700 -1 osd.3 419 *** Got signal Terminated *** 2023-09-23T19:16:29.945 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 19:16:29 smithi098 bash[93735]: debug 2023-09-23T19:16:29.448+0000 7fa99940b700 -1 osd.3 419 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T19:16:30.277 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:30 smithi173 bash[39445]: debug 2023-09-23T19:16:29.999+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:30.777 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:30 smithi173 bash[160505]: debug 2023-09-23T19:16:30.371+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:04.735932+0000 front 2023-09-23T19:16:04.736255+0000 (oldest deadline 2023-09-23T19:16:29.336670+0000) 2023-09-23T19:16:30.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:30 smithi173 bash[162608]: debug 2023-09-23T19:16:30.307+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:31.277 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:30 smithi173 bash[39445]: debug 2023-09-23T19:16:30.963+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:31.776 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:31 smithi173 bash[162608]: debug 2023-09-23T19:16:31.307+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:31.777 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:31 smithi173 bash[160505]: debug 2023-09-23T19:16:31.375+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:04.735932+0000 front 2023-09-23T19:16:04.736255+0000 (oldest deadline 2023-09-23T19:16:29.336670+0000) 2023-09-23T19:16:32.194 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 19:16:31 smithi098 bash[93735]: debug 2023-09-23T19:16:31.728+0000 7fa9941fe700 -1 osd.3 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.002224+0000 front 2023-09-23T19:16:05.002559+0000 (oldest deadline 2023-09-23T19:16:30.902483+0000) 2023-09-23T19:16:32.277 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:31 smithi173 bash[39445]: debug 2023-09-23T19:16:31.959+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:32.277 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 19:16:31 smithi173 bash[130836]: debug 2023-09-23T19:16:31.795+0000 7fd74ca2c700 -1 osd.4 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.721826+0000 front 2023-09-23T19:16:05.721984+0000 (oldest deadline 2023-09-23T19:16:31.522450+0000) 2023-09-23T19:16:32.777 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:32 smithi173 bash[160505]: debug 2023-09-23T19:16:32.403+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:04.735932+0000 front 2023-09-23T19:16:04.736255+0000 (oldest deadline 2023-09-23T19:16:29.336670+0000) 2023-09-23T19:16:32.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:32 smithi173 bash[162608]: debug 2023-09-23T19:16:32.279+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:33.194 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 19:16:32 smithi098 bash[93735]: debug 2023-09-23T19:16:32.736+0000 7fa9941fe700 -1 osd.3 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.002224+0000 front 2023-09-23T19:16:05.002559+0000 (oldest deadline 2023-09-23T19:16:30.902483+0000) 2023-09-23T19:16:33.277 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 19:16:32 smithi173 bash[130836]: debug 2023-09-23T19:16:32.799+0000 7fd74ca2c700 -1 osd.4 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.721826+0000 front 2023-09-23T19:16:05.721984+0000 (oldest deadline 2023-09-23T19:16:31.522450+0000) 2023-09-23T19:16:33.277 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:32 smithi173 bash[39445]: debug 2023-09-23T19:16:32.955+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:33.759 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:33 smithi173 bash[160505]: debug 2023-09-23T19:16:33.391+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:04.735932+0000 front 2023-09-23T19:16:04.736255+0000 (oldest deadline 2023-09-23T19:16:29.336670+0000) 2023-09-23T19:16:33.759 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:33 smithi173 bash[162608]: debug 2023-09-23T19:16:33.291+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:34.027 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 19:16:33 smithi173 bash[130836]: debug 2023-09-23T19:16:33.755+0000 7fd74ca2c700 -1 osd.4 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.721826+0000 front 2023-09-23T19:16:05.721984+0000 (oldest deadline 2023-09-23T19:16:31.522450+0000) 2023-09-23T19:16:34.027 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:33 smithi173 bash[39445]: debug 2023-09-23T19:16:33.955+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:34.195 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 19:16:33 smithi098 bash[93735]: debug 2023-09-23T19:16:33.732+0000 7fa9941fe700 -1 osd.3 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.002224+0000 front 2023-09-23T19:16:05.002559+0000 (oldest deadline 2023-09-23T19:16:30.902483+0000) 2023-09-23T19:16:34.527 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:34 smithi173 bash[160505]: debug 2023-09-23T19:16:34.439+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:04.735932+0000 front 2023-09-23T19:16:04.736255+0000 (oldest deadline 2023-09-23T19:16:29.336670+0000) 2023-09-23T19:16:34.527 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:34 smithi173 bash[162608]: debug 2023-09-23T19:16:34.263+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:35.027 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 19:16:34 smithi173 bash[130836]: debug 2023-09-23T19:16:34.719+0000 7fd74ca2c700 -1 osd.4 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.721826+0000 front 2023-09-23T19:16:05.721984+0000 (oldest deadline 2023-09-23T19:16:31.522450+0000) 2023-09-23T19:16:35.027 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:34 smithi173 bash[39445]: debug 2023-09-23T19:16:34.923+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:35.690 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:35 smithi173 bash[160505]: debug 2023-09-23T19:16:35.423+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:04.735932+0000 front 2023-09-23T19:16:04.736255+0000 (oldest deadline 2023-09-23T19:16:29.336670+0000) 2023-09-23T19:16:35.691 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:35 smithi173 bash[162608]: debug 2023-09-23T19:16:35.311+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:36.012 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 19:16:35 smithi098 bash[285862]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-3 2023-09-23T19:16:36.012 INFO:journalctl@ceph.osd.3.smithi098.stdout:Sep 23 19:16:35 smithi098 bash[285930]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-3 2023-09-23T19:16:36.027 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 19:16:35 smithi173 bash[130836]: debug 2023-09-23T19:16:35.687+0000 7fd74ca2c700 -1 osd.4 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.721826+0000 front 2023-09-23T19:16:05.721984+0000 (oldest deadline 2023-09-23T19:16:31.522450+0000) 2023-09-23T19:16:36.027 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:35 smithi173 bash[39445]: debug 2023-09-23T19:16:35.911+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:36.711 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:36 smithi173 bash[160505]: debug 2023-09-23T19:16:36.399+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:04.735932+0000 front 2023-09-23T19:16:04.736255+0000 (oldest deadline 2023-09-23T19:16:29.336670+0000) 2023-09-23T19:16:36.712 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:36 smithi173 bash[162608]: debug 2023-09-23T19:16:36.279+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:37.027 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 19:16:36 smithi173 bash[130836]: debug 2023-09-23T19:16:36.707+0000 7fd74ca2c700 -1 osd.4 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.721826+0000 front 2023-09-23T19:16:05.721984+0000 (oldest deadline 2023-09-23T19:16:31.522450+0000) 2023-09-23T19:16:37.027 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:36 smithi173 bash[39445]: debug 2023-09-23T19:16:36.915+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:37.527 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:37 smithi173 bash[160505]: debug 2023-09-23T19:16:37.443+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:04.735932+0000 front 2023-09-23T19:16:04.736255+0000 (oldest deadline 2023-09-23T19:16:29.336670+0000) 2023-09-23T19:16:37.527 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:37 smithi173 bash[162608]: debug 2023-09-23T19:16:37.247+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:38.027 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 19:16:37 smithi173 bash[130836]: debug 2023-09-23T19:16:37.743+0000 7fd74ca2c700 -1 osd.4 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.721826+0000 front 2023-09-23T19:16:05.721984+0000 (oldest deadline 2023-09-23T19:16:31.522450+0000) 2023-09-23T19:16:38.027 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:37 smithi173 bash[39445]: debug 2023-09-23T19:16:37.947+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:38.027 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:37 smithi173 bash[39445]: debug 2023-09-23T19:16:37.947+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.421286+0000 front 2023-09-23T19:16:15.421235+0000 (oldest deadline 2023-09-23T19:16:37.720708+0000) 2023-09-23T19:16:38.028 DEBUG:teuthology.orchestra.run.smithi098:> sudo pkill -f 'journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.3.service' 2023-09-23T19:16:38.059 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T19:16:38.060 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-09-23T19:16:38.060 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-09-23T19:16:38.060 DEBUG:teuthology.orchestra.run.smithi173:> sudo systemctl stop ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.4 2023-09-23T19:16:38.451 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:38 smithi173 bash[162608]: debug 2023-09-23T19:16:38.263+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:38.451 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:38 smithi173 bash[162608]: debug 2023-09-23T19:16:38.263+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:16:38.451 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 19:16:38 smithi173 systemd[1]: Stopping Ceph osd.4 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T19:16:38.451 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 19:16:38 smithi173 bash[130836]: debug 2023-09-23T19:16:38.231+0000 7fd74faaf700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T19:16:38.452 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 19:16:38 smithi173 bash[130836]: debug 2023-09-23T19:16:38.231+0000 7fd74faaf700 -1 osd.4 419 *** Got signal Terminated *** 2023-09-23T19:16:38.452 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 19:16:38 smithi173 bash[130836]: debug 2023-09-23T19:16:38.231+0000 7fd74faaf700 -1 osd.4 419 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T19:16:38.716 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:38 smithi173 bash[160505]: debug 2023-09-23T19:16:38.447+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:04.735932+0000 front 2023-09-23T19:16:04.736255+0000 (oldest deadline 2023-09-23T19:16:29.336670+0000) 2023-09-23T19:16:38.716 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:38 smithi173 bash[160505]: debug 2023-09-23T19:16:38.447+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:14.538792+0000 front 2023-09-23T19:16:14.539076+0000 (oldest deadline 2023-09-23T19:16:38.038675+0000) 2023-09-23T19:16:38.981 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 19:16:38 smithi173 bash[130836]: debug 2023-09-23T19:16:38.711+0000 7fd74ca2c700 -1 osd.4 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.721826+0000 front 2023-09-23T19:16:05.721984+0000 (oldest deadline 2023-09-23T19:16:31.522450+0000) 2023-09-23T19:16:39.277 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:39 smithi173 bash[162608]: debug 2023-09-23T19:16:39.223+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:39.277 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:39 smithi173 bash[162608]: debug 2023-09-23T19:16:39.223+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:16:39.278 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:38 smithi173 bash[39445]: debug 2023-09-23T19:16:38.979+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:39.278 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:38 smithi173 bash[39445]: debug 2023-09-23T19:16:38.979+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.421286+0000 front 2023-09-23T19:16:15.421235+0000 (oldest deadline 2023-09-23T19:16:37.720708+0000) 2023-09-23T19:16:39.777 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 19:16:39 smithi173 bash[130836]: debug 2023-09-23T19:16:39.727+0000 7fd74ca2c700 -1 osd.4 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.721826+0000 front 2023-09-23T19:16:05.721984+0000 (oldest deadline 2023-09-23T19:16:31.522450+0000) 2023-09-23T19:16:39.777 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 19:16:39 smithi173 bash[130836]: debug 2023-09-23T19:16:39.727+0000 7fd74ca2c700 -1 osd.4 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:13.823684+0000 front 2023-09-23T19:16:13.823662+0000 (oldest deadline 2023-09-23T19:16:39.723530+0000) 2023-09-23T19:16:39.777 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:39 smithi173 bash[160505]: debug 2023-09-23T19:16:39.495+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:04.735932+0000 front 2023-09-23T19:16:04.736255+0000 (oldest deadline 2023-09-23T19:16:29.336670+0000) 2023-09-23T19:16:39.777 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:39 smithi173 bash[160505]: debug 2023-09-23T19:16:39.495+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:14.538792+0000 front 2023-09-23T19:16:14.539076+0000 (oldest deadline 2023-09-23T19:16:38.038675+0000) 2023-09-23T19:16:40.265 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:39 smithi173 bash[39445]: debug 2023-09-23T19:16:39.931+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:40.266 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:39 smithi173 bash[39445]: debug 2023-09-23T19:16:39.931+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.421286+0000 front 2023-09-23T19:16:15.421235+0000 (oldest deadline 2023-09-23T19:16:37.720708+0000) 2023-09-23T19:16:40.526 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:40 smithi173 bash[162608]: debug 2023-09-23T19:16:40.263+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:40.527 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:40 smithi173 bash[162608]: debug 2023-09-23T19:16:40.263+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:16:40.934 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 19:16:40 smithi173 bash[130836]: debug 2023-09-23T19:16:40.723+0000 7fd74ca2c700 -1 osd.4 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.721826+0000 front 2023-09-23T19:16:05.721984+0000 (oldest deadline 2023-09-23T19:16:31.522450+0000) 2023-09-23T19:16:40.934 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 19:16:40 smithi173 bash[130836]: debug 2023-09-23T19:16:40.723+0000 7fd74ca2c700 -1 osd.4 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:13.823684+0000 front 2023-09-23T19:16:13.823662+0000 (oldest deadline 2023-09-23T19:16:39.723530+0000) 2023-09-23T19:16:40.934 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:40 smithi173 bash[160505]: debug 2023-09-23T19:16:40.531+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:04.735932+0000 front 2023-09-23T19:16:04.736255+0000 (oldest deadline 2023-09-23T19:16:29.336670+0000) 2023-09-23T19:16:40.935 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:40 smithi173 bash[160505]: debug 2023-09-23T19:16:40.531+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:14.538792+0000 front 2023-09-23T19:16:14.539076+0000 (oldest deadline 2023-09-23T19:16:38.038675+0000) 2023-09-23T19:16:41.277 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:40 smithi173 bash[39445]: debug 2023-09-23T19:16:40.931+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:41.277 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:40 smithi173 bash[39445]: debug 2023-09-23T19:16:40.931+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.421286+0000 front 2023-09-23T19:16:15.421235+0000 (oldest deadline 2023-09-23T19:16:37.720708+0000) 2023-09-23T19:16:41.561 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:41 smithi173 bash[162608]: debug 2023-09-23T19:16:41.291+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:41.561 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:41 smithi173 bash[162608]: debug 2023-09-23T19:16:41.291+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:16:41.918 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 19:16:41 smithi173 bash[130836]: debug 2023-09-23T19:16:41.751+0000 7fd74ca2c700 -1 osd.4 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.721826+0000 front 2023-09-23T19:16:05.721984+0000 (oldest deadline 2023-09-23T19:16:31.522450+0000) 2023-09-23T19:16:41.918 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 19:16:41 smithi173 bash[130836]: debug 2023-09-23T19:16:41.751+0000 7fd74ca2c700 -1 osd.4 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:13.823684+0000 front 2023-09-23T19:16:13.823662+0000 (oldest deadline 2023-09-23T19:16:39.723530+0000) 2023-09-23T19:16:41.918 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:41 smithi173 bash[160505]: debug 2023-09-23T19:16:41.559+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:04.735932+0000 front 2023-09-23T19:16:04.736255+0000 (oldest deadline 2023-09-23T19:16:29.336670+0000) 2023-09-23T19:16:41.919 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:41 smithi173 bash[160505]: debug 2023-09-23T19:16:41.559+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:14.538792+0000 front 2023-09-23T19:16:14.539076+0000 (oldest deadline 2023-09-23T19:16:38.038675+0000) 2023-09-23T19:16:42.277 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:41 smithi173 bash[39445]: debug 2023-09-23T19:16:41.915+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:42.277 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:41 smithi173 bash[39445]: debug 2023-09-23T19:16:41.915+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.421286+0000 front 2023-09-23T19:16:15.421235+0000 (oldest deadline 2023-09-23T19:16:37.720708+0000) 2023-09-23T19:16:42.756 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:42 smithi173 bash[162608]: debug 2023-09-23T19:16:42.303+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:42.757 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:42 smithi173 bash[162608]: debug 2023-09-23T19:16:42.303+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:16:42.757 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:42 smithi173 bash[160505]: debug 2023-09-23T19:16:42.535+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:04.735932+0000 front 2023-09-23T19:16:04.736255+0000 (oldest deadline 2023-09-23T19:16:29.336670+0000) 2023-09-23T19:16:42.757 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:42 smithi173 bash[160505]: debug 2023-09-23T19:16:42.535+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:14.538792+0000 front 2023-09-23T19:16:14.539076+0000 (oldest deadline 2023-09-23T19:16:38.038675+0000) 2023-09-23T19:16:43.027 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 19:16:42 smithi173 bash[130836]: debug 2023-09-23T19:16:42.751+0000 7fd74ca2c700 -1 osd.4 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.721826+0000 front 2023-09-23T19:16:05.721984+0000 (oldest deadline 2023-09-23T19:16:31.522450+0000) 2023-09-23T19:16:43.027 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 19:16:42 smithi173 bash[130836]: debug 2023-09-23T19:16:42.751+0000 7fd74ca2c700 -1 osd.4 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:13.823684+0000 front 2023-09-23T19:16:13.823662+0000 (oldest deadline 2023-09-23T19:16:39.723530+0000) 2023-09-23T19:16:43.027 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:42 smithi173 bash[39445]: debug 2023-09-23T19:16:42.879+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:43.027 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:42 smithi173 bash[39445]: debug 2023-09-23T19:16:42.879+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.421286+0000 front 2023-09-23T19:16:15.421235+0000 (oldest deadline 2023-09-23T19:16:37.720708+0000) 2023-09-23T19:16:43.601 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:43 smithi173 bash[160505]: debug 2023-09-23T19:16:43.571+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:04.735932+0000 front 2023-09-23T19:16:04.736255+0000 (oldest deadline 2023-09-23T19:16:29.336670+0000) 2023-09-23T19:16:43.601 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:43 smithi173 bash[160505]: debug 2023-09-23T19:16:43.571+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:14.538792+0000 front 2023-09-23T19:16:14.539076+0000 (oldest deadline 2023-09-23T19:16:38.038675+0000) 2023-09-23T19:16:43.601 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:43 smithi173 bash[162608]: debug 2023-09-23T19:16:43.323+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:43.601 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:43 smithi173 bash[162608]: debug 2023-09-23T19:16:43.323+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:16:44.027 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:43 smithi173 bash[39445]: debug 2023-09-23T19:16:43.839+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:44.027 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:43 smithi173 bash[39445]: debug 2023-09-23T19:16:43.839+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.421286+0000 front 2023-09-23T19:16:15.421235+0000 (oldest deadline 2023-09-23T19:16:37.720708+0000) 2023-09-23T19:16:44.498 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 19:16:44 smithi173 bash[170354]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-4 2023-09-23T19:16:44.498 INFO:journalctl@ceph.osd.4.smithi173.stdout:Sep 23 19:16:44 smithi173 bash[170420]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-4 2023-09-23T19:16:44.498 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:44 smithi173 bash[162608]: debug 2023-09-23T19:16:44.355+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:44.499 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:44 smithi173 bash[162608]: debug 2023-09-23T19:16:44.355+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:16:44.776 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:44 smithi173 bash[160505]: debug 2023-09-23T19:16:44.523+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:04.735932+0000 front 2023-09-23T19:16:04.736255+0000 (oldest deadline 2023-09-23T19:16:29.336670+0000) 2023-09-23T19:16:44.777 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:44 smithi173 bash[160505]: debug 2023-09-23T19:16:44.523+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:14.538792+0000 front 2023-09-23T19:16:14.539076+0000 (oldest deadline 2023-09-23T19:16:38.038675+0000) 2023-09-23T19:16:45.277 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:44 smithi173 bash[39445]: debug 2023-09-23T19:16:44.871+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:45.277 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:44 smithi173 bash[39445]: debug 2023-09-23T19:16:44.871+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.421286+0000 front 2023-09-23T19:16:15.421235+0000 (oldest deadline 2023-09-23T19:16:37.720708+0000) 2023-09-23T19:16:45.726 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:45 smithi173 bash[160505]: debug 2023-09-23T19:16:45.567+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:04.735932+0000 front 2023-09-23T19:16:04.736255+0000 (oldest deadline 2023-09-23T19:16:29.336670+0000) 2023-09-23T19:16:45.727 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:45 smithi173 bash[160505]: debug 2023-09-23T19:16:45.567+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:14.538792+0000 front 2023-09-23T19:16:14.539076+0000 (oldest deadline 2023-09-23T19:16:38.038675+0000) 2023-09-23T19:16:45.727 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:45 smithi173 bash[162608]: debug 2023-09-23T19:16:45.403+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:45.727 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:45 smithi173 bash[162608]: debug 2023-09-23T19:16:45.403+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:16:46.264 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:45 smithi173 bash[39445]: debug 2023-09-23T19:16:45.915+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:46.264 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:45 smithi173 bash[39445]: debug 2023-09-23T19:16:45.915+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.421286+0000 front 2023-09-23T19:16:15.421235+0000 (oldest deadline 2023-09-23T19:16:37.720708+0000) 2023-09-23T19:16:46.527 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:46 smithi173 bash[162608]: debug 2023-09-23T19:16:46.359+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:46.527 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:46 smithi173 bash[162608]: debug 2023-09-23T19:16:46.359+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:16:46.777 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:46 smithi173 bash[160505]: debug 2023-09-23T19:16:46.591+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:04.735932+0000 front 2023-09-23T19:16:04.736255+0000 (oldest deadline 2023-09-23T19:16:29.336670+0000) 2023-09-23T19:16:46.777 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:46 smithi173 bash[160505]: debug 2023-09-23T19:16:46.591+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:14.538792+0000 front 2023-09-23T19:16:14.539076+0000 (oldest deadline 2023-09-23T19:16:38.038675+0000) 2023-09-23T19:16:46.976 DEBUG:teuthology.orchestra.run.smithi173:> sudo pkill -f 'journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.4.service' 2023-09-23T19:16:47.008 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T19:16:47.008 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-09-23T19:16:47.008 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-09-23T19:16:47.008 DEBUG:teuthology.orchestra.run.smithi173:> sudo systemctl stop ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.5 2023-09-23T19:16:47.238 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:47 smithi173 systemd[1]: Stopping Ceph osd.5 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T19:16:47.239 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:46 smithi173 bash[39445]: debug 2023-09-23T19:16:46.955+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:47.239 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:46 smithi173 bash[39445]: debug 2023-09-23T19:16:46.955+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.421286+0000 front 2023-09-23T19:16:15.421235+0000 (oldest deadline 2023-09-23T19:16:37.720708+0000) 2023-09-23T19:16:47.526 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:47 smithi173 bash[162608]: debug 2023-09-23T19:16:47.327+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:47.527 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:47 smithi173 bash[162608]: debug 2023-09-23T19:16:47.327+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:16:47.527 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:47 smithi173 bash[160505]: debug 2023-09-23T19:16:47.235+0000 7f88d3c1e700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T19:16:47.527 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:47 smithi173 bash[160505]: debug 2023-09-23T19:16:47.235+0000 7f88d3c1e700 -1 osd.5 419 *** Got signal Terminated *** 2023-09-23T19:16:47.527 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:47 smithi173 bash[160505]: debug 2023-09-23T19:16:47.235+0000 7f88d3c1e700 -1 osd.5 419 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T19:16:47.990 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:47 smithi173 bash[160505]: debug 2023-09-23T19:16:47.547+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:04.735932+0000 front 2023-09-23T19:16:04.736255+0000 (oldest deadline 2023-09-23T19:16:29.336670+0000) 2023-09-23T19:16:47.990 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:47 smithi173 bash[160505]: debug 2023-09-23T19:16:47.547+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:14.538792+0000 front 2023-09-23T19:16:14.539076+0000 (oldest deadline 2023-09-23T19:16:38.038675+0000) 2023-09-23T19:16:48.277 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:47 smithi173 bash[39445]: debug 2023-09-23T19:16:47.987+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:48.277 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:47 smithi173 bash[39445]: debug 2023-09-23T19:16:47.987+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.421286+0000 front 2023-09-23T19:16:15.421235+0000 (oldest deadline 2023-09-23T19:16:37.720708+0000) 2023-09-23T19:16:48.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:48 smithi173 bash[162608]: debug 2023-09-23T19:16:48.303+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:48.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:48 smithi173 bash[162608]: debug 2023-09-23T19:16:48.303+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:16:48.777 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:48 smithi173 bash[160505]: debug 2023-09-23T19:16:48.514+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:04.735932+0000 front 2023-09-23T19:16:04.736255+0000 (oldest deadline 2023-09-23T19:16:29.336670+0000) 2023-09-23T19:16:48.777 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:48 smithi173 bash[160505]: debug 2023-09-23T19:16:48.514+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:14.538792+0000 front 2023-09-23T19:16:14.539076+0000 (oldest deadline 2023-09-23T19:16:38.038675+0000) 2023-09-23T19:16:49.276 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:49 smithi173 bash[39445]: debug 2023-09-23T19:16:49.002+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:49.277 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:49 smithi173 bash[39445]: debug 2023-09-23T19:16:49.002+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.421286+0000 front 2023-09-23T19:16:15.421235+0000 (oldest deadline 2023-09-23T19:16:37.720708+0000) 2023-09-23T19:16:49.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:49 smithi173 bash[162608]: debug 2023-09-23T19:16:49.330+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:49.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:49 smithi173 bash[162608]: debug 2023-09-23T19:16:49.330+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:16:49.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:49 smithi173 bash[162608]: debug 2023-09-23T19:16:49.330+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.020808+0000 front 2023-09-23T19:16:23.020919+0000 (oldest deadline 2023-09-23T19:16:48.320446+0000) 2023-09-23T19:16:49.777 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:49 smithi173 bash[160505]: debug 2023-09-23T19:16:49.526+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:04.735932+0000 front 2023-09-23T19:16:04.736255+0000 (oldest deadline 2023-09-23T19:16:29.336670+0000) 2023-09-23T19:16:49.777 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:49 smithi173 bash[160505]: debug 2023-09-23T19:16:49.526+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:14.538792+0000 front 2023-09-23T19:16:14.539076+0000 (oldest deadline 2023-09-23T19:16:38.038675+0000) 2023-09-23T19:16:49.778 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:49 smithi173 bash[160505]: debug 2023-09-23T19:16:49.526+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.340020+0000 front 2023-09-23T19:16:23.339921+0000 (oldest deadline 2023-09-23T19:16:49.239693+0000) 2023-09-23T19:16:50.277 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:49 smithi173 bash[39445]: debug 2023-09-23T19:16:49.990+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:50.277 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:49 smithi173 bash[39445]: debug 2023-09-23T19:16:49.990+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.421286+0000 front 2023-09-23T19:16:15.421235+0000 (oldest deadline 2023-09-23T19:16:37.720708+0000) 2023-09-23T19:16:50.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:50 smithi173 bash[162608]: debug 2023-09-23T19:16:50.342+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:50.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:50 smithi173 bash[162608]: debug 2023-09-23T19:16:50.342+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:16:50.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:50 smithi173 bash[162608]: debug 2023-09-23T19:16:50.342+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.020808+0000 front 2023-09-23T19:16:23.020919+0000 (oldest deadline 2023-09-23T19:16:48.320446+0000) 2023-09-23T19:16:50.777 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:50 smithi173 bash[160505]: debug 2023-09-23T19:16:50.514+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:04.735932+0000 front 2023-09-23T19:16:04.736255+0000 (oldest deadline 2023-09-23T19:16:29.336670+0000) 2023-09-23T19:16:50.778 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:50 smithi173 bash[160505]: debug 2023-09-23T19:16:50.514+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:14.538792+0000 front 2023-09-23T19:16:14.539076+0000 (oldest deadline 2023-09-23T19:16:38.038675+0000) 2023-09-23T19:16:50.778 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:50 smithi173 bash[160505]: debug 2023-09-23T19:16:50.514+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.340020+0000 front 2023-09-23T19:16:23.339921+0000 (oldest deadline 2023-09-23T19:16:49.239693+0000) 2023-09-23T19:16:51.304 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:51 smithi173 bash[39445]: debug 2023-09-23T19:16:51.034+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:51.304 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:51 smithi173 bash[39445]: debug 2023-09-23T19:16:51.034+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.421286+0000 front 2023-09-23T19:16:15.421235+0000 (oldest deadline 2023-09-23T19:16:37.720708+0000) 2023-09-23T19:16:51.304 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:51 smithi173 bash[39445]: debug 2023-09-23T19:16:51.034+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.622289+0000 front 2023-09-23T19:16:23.622185+0000 (oldest deadline 2023-09-23T19:16:50.622602+0000) 2023-09-23T19:16:51.557 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:51 smithi173 bash[162608]: debug 2023-09-23T19:16:51.298+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:51.558 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:51 smithi173 bash[162608]: debug 2023-09-23T19:16:51.298+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:16:51.558 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:51 smithi173 bash[162608]: debug 2023-09-23T19:16:51.298+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.020808+0000 front 2023-09-23T19:16:23.020919+0000 (oldest deadline 2023-09-23T19:16:48.320446+0000) 2023-09-23T19:16:52.027 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:51 smithi173 bash[160505]: debug 2023-09-23T19:16:51.554+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:04.735932+0000 front 2023-09-23T19:16:04.736255+0000 (oldest deadline 2023-09-23T19:16:29.336670+0000) 2023-09-23T19:16:52.027 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:51 smithi173 bash[160505]: debug 2023-09-23T19:16:51.554+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:14.538792+0000 front 2023-09-23T19:16:14.539076+0000 (oldest deadline 2023-09-23T19:16:38.038675+0000) 2023-09-23T19:16:52.027 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:51 smithi173 bash[160505]: debug 2023-09-23T19:16:51.554+0000 7f88ce396700 -1 osd.5 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.340020+0000 front 2023-09-23T19:16:23.339921+0000 (oldest deadline 2023-09-23T19:16:49.239693+0000) 2023-09-23T19:16:52.312 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:52 smithi173 bash[39445]: debug 2023-09-23T19:16:52.026+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:52.313 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:52 smithi173 bash[39445]: debug 2023-09-23T19:16:52.026+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.421286+0000 front 2023-09-23T19:16:15.421235+0000 (oldest deadline 2023-09-23T19:16:37.720708+0000) 2023-09-23T19:16:52.313 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:52 smithi173 bash[39445]: debug 2023-09-23T19:16:52.026+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.622289+0000 front 2023-09-23T19:16:23.622185+0000 (oldest deadline 2023-09-23T19:16:50.622602+0000) 2023-09-23T19:16:52.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:52 smithi173 bash[162608]: debug 2023-09-23T19:16:52.306+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:52.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:52 smithi173 bash[162608]: debug 2023-09-23T19:16:52.306+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:16:52.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:52 smithi173 bash[162608]: debug 2023-09-23T19:16:52.306+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.020808+0000 front 2023-09-23T19:16:23.020919+0000 (oldest deadline 2023-09-23T19:16:48.320446+0000) 2023-09-23T19:16:53.238 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:52 smithi173 bash[170561]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-5 2023-09-23T19:16:53.239 INFO:journalctl@ceph.osd.5.smithi173.stdout:Sep 23 19:16:53 smithi173 bash[170625]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-5 2023-09-23T19:16:53.239 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:53 smithi173 bash[39445]: debug 2023-09-23T19:16:53.054+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:53.239 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:53 smithi173 bash[39445]: debug 2023-09-23T19:16:53.054+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.421286+0000 front 2023-09-23T19:16:15.421235+0000 (oldest deadline 2023-09-23T19:16:37.720708+0000) 2023-09-23T19:16:53.239 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:53 smithi173 bash[39445]: debug 2023-09-23T19:16:53.054+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.622289+0000 front 2023-09-23T19:16:23.622185+0000 (oldest deadline 2023-09-23T19:16:50.622602+0000) 2023-09-23T19:16:53.527 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:53 smithi173 bash[162608]: debug 2023-09-23T19:16:53.354+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:53.527 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:53 smithi173 bash[162608]: debug 2023-09-23T19:16:53.354+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:16:53.527 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:53 smithi173 bash[162608]: debug 2023-09-23T19:16:53.354+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.020808+0000 front 2023-09-23T19:16:23.020919+0000 (oldest deadline 2023-09-23T19:16:48.320446+0000) 2023-09-23T19:16:54.365 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:54 smithi173 bash[39445]: debug 2023-09-23T19:16:54.102+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:54.365 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:54 smithi173 bash[39445]: debug 2023-09-23T19:16:54.102+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.421286+0000 front 2023-09-23T19:16:15.421235+0000 (oldest deadline 2023-09-23T19:16:37.720708+0000) 2023-09-23T19:16:54.366 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:54 smithi173 bash[39445]: debug 2023-09-23T19:16:54.102+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.622289+0000 front 2023-09-23T19:16:23.622185+0000 (oldest deadline 2023-09-23T19:16:50.622602+0000) 2023-09-23T19:16:54.366 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:54 smithi173 bash[162608]: debug 2023-09-23T19:16:54.338+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:54.366 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:54 smithi173 bash[162608]: debug 2023-09-23T19:16:54.338+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:16:54.367 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:54 smithi173 bash[162608]: debug 2023-09-23T19:16:54.338+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.020808+0000 front 2023-09-23T19:16:23.020919+0000 (oldest deadline 2023-09-23T19:16:48.320446+0000) 2023-09-23T19:16:55.277 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:55 smithi173 bash[39445]: debug 2023-09-23T19:16:55.102+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:55.277 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:55 smithi173 bash[39445]: debug 2023-09-23T19:16:55.102+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.421286+0000 front 2023-09-23T19:16:15.421235+0000 (oldest deadline 2023-09-23T19:16:37.720708+0000) 2023-09-23T19:16:55.277 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:55 smithi173 bash[39445]: debug 2023-09-23T19:16:55.102+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.622289+0000 front 2023-09-23T19:16:23.622185+0000 (oldest deadline 2023-09-23T19:16:50.622602+0000) 2023-09-23T19:16:55.497 DEBUG:teuthology.orchestra.run.smithi173:> sudo pkill -f 'journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.5.service' 2023-09-23T19:16:55.528 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T19:16:55.528 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-09-23T19:16:55.528 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-09-23T19:16:55.528 DEBUG:teuthology.orchestra.run.smithi173:> sudo systemctl stop ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.6 2023-09-23T19:16:55.730 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:55 smithi173 systemd[1]: Stopping Ceph osd.6 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T19:16:55.731 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:55 smithi173 bash[162608]: debug 2023-09-23T19:16:55.374+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:55.731 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:55 smithi173 bash[162608]: debug 2023-09-23T19:16:55.374+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:16:55.731 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:55 smithi173 bash[162608]: debug 2023-09-23T19:16:55.374+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.020808+0000 front 2023-09-23T19:16:23.020919+0000 (oldest deadline 2023-09-23T19:16:48.320446+0000) 2023-09-23T19:16:56.027 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:55 smithi173 bash[39445]: debug 2023-09-23T19:16:55.726+0000 7f5ca7de8700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T19:16:56.027 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:55 smithi173 bash[39445]: debug 2023-09-23T19:16:55.726+0000 7f5ca7de8700 -1 osd.6 419 *** Got signal Terminated *** 2023-09-23T19:16:56.027 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:55 smithi173 bash[39445]: debug 2023-09-23T19:16:55.726+0000 7f5ca7de8700 -1 osd.6 419 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T19:16:56.408 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:56 smithi173 bash[39445]: debug 2023-09-23T19:16:56.134+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:56.408 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:56 smithi173 bash[39445]: debug 2023-09-23T19:16:56.134+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.421286+0000 front 2023-09-23T19:16:15.421235+0000 (oldest deadline 2023-09-23T19:16:37.720708+0000) 2023-09-23T19:16:56.408 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:56 smithi173 bash[39445]: debug 2023-09-23T19:16:56.134+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.622289+0000 front 2023-09-23T19:16:23.622185+0000 (oldest deadline 2023-09-23T19:16:50.622602+0000) 2023-09-23T19:16:56.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:56 smithi173 bash[162608]: debug 2023-09-23T19:16:56.402+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:56.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:56 smithi173 bash[162608]: debug 2023-09-23T19:16:56.402+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:16:56.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:56 smithi173 bash[162608]: debug 2023-09-23T19:16:56.402+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.020808+0000 front 2023-09-23T19:16:23.020919+0000 (oldest deadline 2023-09-23T19:16:48.320446+0000) 2023-09-23T19:16:57.527 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:57 smithi173 bash[39445]: debug 2023-09-23T19:16:57.118+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:57.527 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:57 smithi173 bash[39445]: debug 2023-09-23T19:16:57.118+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.421286+0000 front 2023-09-23T19:16:15.421235+0000 (oldest deadline 2023-09-23T19:16:37.720708+0000) 2023-09-23T19:16:57.527 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:57 smithi173 bash[39445]: debug 2023-09-23T19:16:57.118+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.622289+0000 front 2023-09-23T19:16:23.622185+0000 (oldest deadline 2023-09-23T19:16:50.622602+0000) 2023-09-23T19:16:57.527 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:57 smithi173 bash[39445]: debug 2023-09-23T19:16:57.118+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6830 osd.3 since back 2023-09-23T19:16:33.524289+0000 front 2023-09-23T19:16:33.524254+0000 (oldest deadline 2023-09-23T19:16:56.423959+0000) 2023-09-23T19:16:57.527 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:57 smithi173 bash[162608]: debug 2023-09-23T19:16:57.362+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:57.528 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:57 smithi173 bash[162608]: debug 2023-09-23T19:16:57.362+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:16:57.528 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:57 smithi173 bash[162608]: debug 2023-09-23T19:16:57.362+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.020808+0000 front 2023-09-23T19:16:23.020919+0000 (oldest deadline 2023-09-23T19:16:48.320446+0000) 2023-09-23T19:16:58.385 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:58 smithi173 bash[39445]: debug 2023-09-23T19:16:58.098+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:58.385 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:58 smithi173 bash[39445]: debug 2023-09-23T19:16:58.098+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.421286+0000 front 2023-09-23T19:16:15.421235+0000 (oldest deadline 2023-09-23T19:16:37.720708+0000) 2023-09-23T19:16:58.385 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:58 smithi173 bash[39445]: debug 2023-09-23T19:16:58.098+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.622289+0000 front 2023-09-23T19:16:23.622185+0000 (oldest deadline 2023-09-23T19:16:50.622602+0000) 2023-09-23T19:16:58.385 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:58 smithi173 bash[39445]: debug 2023-09-23T19:16:58.098+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6830 osd.3 since back 2023-09-23T19:16:33.524289+0000 front 2023-09-23T19:16:33.524254+0000 (oldest deadline 2023-09-23T19:16:56.423959+0000) 2023-09-23T19:16:58.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:58 smithi173 bash[162608]: debug 2023-09-23T19:16:58.378+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:58.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:58 smithi173 bash[162608]: debug 2023-09-23T19:16:58.378+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:16:58.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:58 smithi173 bash[162608]: debug 2023-09-23T19:16:58.378+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.020808+0000 front 2023-09-23T19:16:23.020919+0000 (oldest deadline 2023-09-23T19:16:48.320446+0000) 2023-09-23T19:16:58.778 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:58 smithi173 bash[162608]: debug 2023-09-23T19:16:58.378+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6830 osd.3 since back 2023-09-23T19:16:32.322738+0000 front 2023-09-23T19:16:32.322542+0000 (oldest deadline 2023-09-23T19:16:57.523218+0000) 2023-09-23T19:16:59.385 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:59 smithi173 bash[39445]: debug 2023-09-23T19:16:59.110+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:16:59.385 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:59 smithi173 bash[39445]: debug 2023-09-23T19:16:59.110+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.421286+0000 front 2023-09-23T19:16:15.421235+0000 (oldest deadline 2023-09-23T19:16:37.720708+0000) 2023-09-23T19:16:59.385 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:59 smithi173 bash[39445]: debug 2023-09-23T19:16:59.110+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.622289+0000 front 2023-09-23T19:16:23.622185+0000 (oldest deadline 2023-09-23T19:16:50.622602+0000) 2023-09-23T19:16:59.386 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:16:59 smithi173 bash[39445]: debug 2023-09-23T19:16:59.110+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6830 osd.3 since back 2023-09-23T19:16:33.524289+0000 front 2023-09-23T19:16:33.524254+0000 (oldest deadline 2023-09-23T19:16:56.423959+0000) 2023-09-23T19:16:59.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:59 smithi173 bash[162608]: debug 2023-09-23T19:16:59.378+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:16:59.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:59 smithi173 bash[162608]: debug 2023-09-23T19:16:59.378+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:16:59.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:59 smithi173 bash[162608]: debug 2023-09-23T19:16:59.378+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.020808+0000 front 2023-09-23T19:16:23.020919+0000 (oldest deadline 2023-09-23T19:16:48.320446+0000) 2023-09-23T19:16:59.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:16:59 smithi173 bash[162608]: debug 2023-09-23T19:16:59.378+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6830 osd.3 since back 2023-09-23T19:16:32.322738+0000 front 2023-09-23T19:16:32.322542+0000 (oldest deadline 2023-09-23T19:16:57.523218+0000) 2023-09-23T19:17:00.421 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:17:00 smithi173 bash[39445]: debug 2023-09-23T19:17:00.134+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:05.618288+0000 front 2023-09-23T19:16:05.618528+0000 (oldest deadline 2023-09-23T19:16:27.918161+0000) 2023-09-23T19:17:00.421 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:17:00 smithi173 bash[39445]: debug 2023-09-23T19:17:00.134+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.421286+0000 front 2023-09-23T19:16:15.421235+0000 (oldest deadline 2023-09-23T19:16:37.720708+0000) 2023-09-23T19:17:00.421 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:17:00 smithi173 bash[39445]: debug 2023-09-23T19:17:00.134+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.622289+0000 front 2023-09-23T19:16:23.622185+0000 (oldest deadline 2023-09-23T19:16:50.622602+0000) 2023-09-23T19:17:00.422 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:17:00 smithi173 bash[39445]: debug 2023-09-23T19:17:00.134+0000 7f5ca155e700 -1 osd.6 419 heartbeat_check: no reply from 172.21.15.98:6830 osd.3 since back 2023-09-23T19:16:33.524289+0000 front 2023-09-23T19:16:33.524254+0000 (oldest deadline 2023-09-23T19:16:56.423959+0000) 2023-09-23T19:17:00.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:00 smithi173 bash[162608]: debug 2023-09-23T19:17:00.418+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:17:00.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:00 smithi173 bash[162608]: debug 2023-09-23T19:17:00.418+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:17:00.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:00 smithi173 bash[162608]: debug 2023-09-23T19:17:00.418+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.020808+0000 front 2023-09-23T19:16:23.020919+0000 (oldest deadline 2023-09-23T19:16:48.320446+0000) 2023-09-23T19:17:00.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:00 smithi173 bash[162608]: debug 2023-09-23T19:17:00.418+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6830 osd.3 since back 2023-09-23T19:16:32.322738+0000 front 2023-09-23T19:16:32.322542+0000 (oldest deadline 2023-09-23T19:16:57.523218+0000) 2023-09-23T19:17:00.778 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 19:17:00 smithi173 bash[54982]: ts=2023-09-23T19:17:00.422Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.98:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.98:8765: connect: connection refused" 2023-09-23T19:17:00.778 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 19:17:00 smithi173 bash[54982]: ts=2023-09-23T19:17:00.423Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.98:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.98:8765: connect: connection refused" 2023-09-23T19:17:00.778 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 19:17:00 smithi173 bash[54982]: ts=2023-09-23T19:17:00.426Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.98:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.98:8765: connect: connection refused" 2023-09-23T19:17:00.778 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 19:17:00 smithi173 bash[54982]: ts=2023-09-23T19:17:00.426Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.98:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.98:8765: connect: connection refused" 2023-09-23T19:17:01.776 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:17:01 smithi173 bash[170768]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-6 2023-09-23T19:17:01.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:01 smithi173 bash[162608]: debug 2023-09-23T19:17:01.406+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:17:01.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:01 smithi173 bash[162608]: debug 2023-09-23T19:17:01.406+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:17:01.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:01 smithi173 bash[162608]: debug 2023-09-23T19:17:01.406+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.020808+0000 front 2023-09-23T19:16:23.020919+0000 (oldest deadline 2023-09-23T19:16:48.320446+0000) 2023-09-23T19:17:01.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:01 smithi173 bash[162608]: debug 2023-09-23T19:17:01.406+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6830 osd.3 since back 2023-09-23T19:16:32.322738+0000 front 2023-09-23T19:16:32.322542+0000 (oldest deadline 2023-09-23T19:16:57.523218+0000) 2023-09-23T19:17:02.277 INFO:journalctl@ceph.osd.6.smithi173.stdout:Sep 23 19:17:01 smithi173 bash[170835]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-6 2023-09-23T19:17:02.733 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:02 smithi173 bash[162608]: debug 2023-09-23T19:17:02.386+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:17:02.733 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:02 smithi173 bash[162608]: debug 2023-09-23T19:17:02.386+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:17:02.733 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:02 smithi173 bash[162608]: debug 2023-09-23T19:17:02.386+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.020808+0000 front 2023-09-23T19:16:23.020919+0000 (oldest deadline 2023-09-23T19:16:48.320446+0000) 2023-09-23T19:17:02.734 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:02 smithi173 bash[162608]: debug 2023-09-23T19:17:02.386+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6830 osd.3 since back 2023-09-23T19:16:32.322738+0000 front 2023-09-23T19:16:32.322542+0000 (oldest deadline 2023-09-23T19:16:57.523218+0000) 2023-09-23T19:17:03.527 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:03 smithi173 bash[162608]: debug 2023-09-23T19:17:03.378+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:17:03.527 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:03 smithi173 bash[162608]: debug 2023-09-23T19:17:03.378+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:17:03.527 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:03 smithi173 bash[162608]: debug 2023-09-23T19:17:03.378+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.020808+0000 front 2023-09-23T19:16:23.020919+0000 (oldest deadline 2023-09-23T19:16:48.320446+0000) 2023-09-23T19:17:03.527 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:03 smithi173 bash[162608]: debug 2023-09-23T19:17:03.378+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6830 osd.3 since back 2023-09-23T19:16:32.322738+0000 front 2023-09-23T19:16:32.322542+0000 (oldest deadline 2023-09-23T19:16:57.523218+0000) 2023-09-23T19:17:03.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo pkill -f 'journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.6.service' 2023-09-23T19:17:03.943 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T19:17:03.943 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-09-23T19:17:03.943 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-09-23T19:17:03.943 DEBUG:teuthology.orchestra.run.smithi173:> sudo systemctl stop ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.7 2023-09-23T19:17:04.277 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:04 smithi173 systemd[1]: Stopping Ceph osd.7 for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T19:17:04.277 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:04 smithi173 bash[162608]: debug 2023-09-23T19:17:04.134+0000 7fee49eab700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T19:17:04.277 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:04 smithi173 bash[162608]: debug 2023-09-23T19:17:04.134+0000 7fee49eab700 -1 osd.7 419 *** Got signal Terminated *** 2023-09-23T19:17:04.277 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:04 smithi173 bash[162608]: debug 2023-09-23T19:17:04.134+0000 7fee49eab700 -1 osd.7 419 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T19:17:04.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:04 smithi173 bash[162608]: debug 2023-09-23T19:17:04.378+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:17:04.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:04 smithi173 bash[162608]: debug 2023-09-23T19:17:04.378+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:17:04.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:04 smithi173 bash[162608]: debug 2023-09-23T19:17:04.378+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.020808+0000 front 2023-09-23T19:16:23.020919+0000 (oldest deadline 2023-09-23T19:16:48.320446+0000) 2023-09-23T19:17:04.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:04 smithi173 bash[162608]: debug 2023-09-23T19:17:04.378+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6830 osd.3 since back 2023-09-23T19:16:32.322738+0000 front 2023-09-23T19:16:32.322542+0000 (oldest deadline 2023-09-23T19:16:57.523218+0000) 2023-09-23T19:17:05.776 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:05 smithi173 bash[162608]: debug 2023-09-23T19:17:05.398+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:17:05.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:05 smithi173 bash[162608]: debug 2023-09-23T19:17:05.398+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:17:05.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:05 smithi173 bash[162608]: debug 2023-09-23T19:17:05.398+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.020808+0000 front 2023-09-23T19:16:23.020919+0000 (oldest deadline 2023-09-23T19:16:48.320446+0000) 2023-09-23T19:17:05.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:05 smithi173 bash[162608]: debug 2023-09-23T19:17:05.398+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6830 osd.3 since back 2023-09-23T19:16:32.322738+0000 front 2023-09-23T19:16:32.322542+0000 (oldest deadline 2023-09-23T19:16:57.523218+0000) 2023-09-23T19:17:06.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:06 smithi173 bash[162608]: debug 2023-09-23T19:17:06.358+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:17:06.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:06 smithi173 bash[162608]: debug 2023-09-23T19:17:06.358+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:17:06.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:06 smithi173 bash[162608]: debug 2023-09-23T19:17:06.358+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.020808+0000 front 2023-09-23T19:16:23.020919+0000 (oldest deadline 2023-09-23T19:16:48.320446+0000) 2023-09-23T19:17:06.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:06 smithi173 bash[162608]: debug 2023-09-23T19:17:06.358+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6830 osd.3 since back 2023-09-23T19:16:32.322738+0000 front 2023-09-23T19:16:32.322542+0000 (oldest deadline 2023-09-23T19:16:57.523218+0000) 2023-09-23T19:17:07.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:07 smithi173 bash[162608]: debug 2023-09-23T19:17:07.322+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:17:07.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:07 smithi173 bash[162608]: debug 2023-09-23T19:17:07.322+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:17:07.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:07 smithi173 bash[162608]: debug 2023-09-23T19:17:07.322+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.020808+0000 front 2023-09-23T19:16:23.020919+0000 (oldest deadline 2023-09-23T19:16:48.320446+0000) 2023-09-23T19:17:07.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:07 smithi173 bash[162608]: debug 2023-09-23T19:17:07.322+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6830 osd.3 since back 2023-09-23T19:16:32.322738+0000 front 2023-09-23T19:16:32.322542+0000 (oldest deadline 2023-09-23T19:16:57.523218+0000) 2023-09-23T19:17:08.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:08 smithi173 bash[162608]: debug 2023-09-23T19:17:08.338+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6806 osd.0 since back 2023-09-23T19:16:03.816674+0000 front 2023-09-23T19:16:03.816815+0000 (oldest deadline 2023-09-23T19:16:27.916754+0000) 2023-09-23T19:17:08.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:08 smithi173 bash[162608]: debug 2023-09-23T19:17:08.338+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6814 osd.1 since back 2023-09-23T19:16:15.518830+0000 front 2023-09-23T19:16:15.519008+0000 (oldest deadline 2023-09-23T19:16:37.719256+0000) 2023-09-23T19:17:08.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:08 smithi173 bash[162608]: debug 2023-09-23T19:17:08.338+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6822 osd.2 since back 2023-09-23T19:16:23.020808+0000 front 2023-09-23T19:16:23.020919+0000 (oldest deadline 2023-09-23T19:16:48.320446+0000) 2023-09-23T19:17:08.777 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:08 smithi173 bash[162608]: debug 2023-09-23T19:17:08.338+0000 7fee42e20700 -1 osd.7 419 heartbeat_check: no reply from 172.21.15.98:6830 osd.3 since back 2023-09-23T19:16:32.322738+0000 front 2023-09-23T19:16:32.322542+0000 (oldest deadline 2023-09-23T19:16:57.523218+0000) 2023-09-23T19:17:10.134 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:09 smithi173 bash[170979]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-7 2023-09-23T19:17:10.135 INFO:journalctl@ceph.osd.7.smithi173.stdout:Sep 23 19:17:10 smithi173 bash[171042]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-osd-7 2023-09-23T19:17:12.073 DEBUG:teuthology.orchestra.run.smithi173:> sudo pkill -f 'journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@osd.7.service' 2023-09-23T19:17:12.101 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T19:17:12.101 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-09-23T19:17:12.101 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-09-23T19:17:12.101 DEBUG:teuthology.orchestra.run.smithi173:> sudo systemctl stop ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@prometheus.a 2023-09-23T19:17:12.336 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 19:17:12 smithi173 systemd[1]: Stopping Ceph prometheus.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T19:17:12.336 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 19:17:12 smithi173 bash[54982]: ts=2023-09-23T19:17:12.306Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T19:17:12.337 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 19:17:12 smithi173 bash[54982]: ts=2023-09-23T19:17:12.306Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-09-23T19:17:12.337 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 19:17:12 smithi173 bash[54982]: ts=2023-09-23T19:17:12.306Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-09-23T19:17:12.337 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 19:17:12 smithi173 bash[54982]: ts=2023-09-23T19:17:12.306Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-09-23T19:17:12.337 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 19:17:12 smithi173 bash[54982]: ts=2023-09-23T19:17:12.306Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-09-23T19:17:12.337 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 19:17:12 smithi173 bash[54982]: ts=2023-09-23T19:17:12.306Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-09-23T19:17:12.337 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 19:17:12 smithi173 bash[54982]: ts=2023-09-23T19:17:12.306Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T19:17:12.337 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 19:17:12 smithi173 bash[54982]: ts=2023-09-23T19:17:12.307Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-09-23T19:17:12.337 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 19:17:12 smithi173 bash[54982]: ts=2023-09-23T19:17:12.307Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T19:17:12.338 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 19:17:12 smithi173 bash[54982]: ts=2023-09-23T19:17:12.309Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T19:17:12.338 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 19:17:12 smithi173 bash[54982]: ts=2023-09-23T19:17:12.309Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-09-23T19:17:12.338 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Sep 23 19:17:12 smithi173 bash[54982]: ts=2023-09-23T19:17:12.309Z caller=main.go:1066 level=info msg="See you next time!" 2023-09-23T19:17:13.187 DEBUG:teuthology.orchestra.run.smithi173:> sudo pkill -f 'journalctl -f -n 0 -u ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@prometheus.a.service' 2023-09-23T19:17:13.216 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T19:17:13.216 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-09-23T19:17:13.216 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 --force --keep-logs 2023-09-23T19:17:20.195 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 19:17:19 smithi098 systemd[1]: Stopping Ceph node-exporter.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T19:17:21.166 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 19:17:20 smithi098 bash[286295]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-node-exporter-a 2023-09-23T19:17:21.166 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 19:17:20 smithi098 bash[286359]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-node-exporter-a 2023-09-23T19:17:21.166 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 19:17:20 smithi098 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T19:17:21.166 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 19:17:20 smithi098 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-09-23T19:17:21.166 INFO:journalctl@ceph.node-exporter.a.smithi098.stdout:Sep 23 19:17:20 smithi098 systemd[1]: Stopped Ceph node-exporter.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T19:17:21.444 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 19:17:21 smithi098 systemd[1]: Stopping Ceph alertmanager.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T19:17:21.445 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 19:17:21 smithi098 bash[72190]: level=info ts=2023-09-23T19:17:21.306Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-09-23T19:17:22.195 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 19:17:21 smithi098 bash[286421]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-alertmanager-a 2023-09-23T19:17:22.195 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 19:17:22 smithi098 bash[286484]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-alertmanager-a 2023-09-23T19:17:22.195 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 19:17:22 smithi098 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T19:17:22.195 INFO:journalctl@ceph.alertmanager.a.smithi098.stdout:Sep 23 19:17:22 smithi098 systemd[1]: Stopped Ceph alertmanager.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T19:17:28.602 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 --force --keep-logs 2023-09-23T19:17:35.277 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 19:17:34 smithi173 systemd[1]: Stopping Ceph node-exporter.b for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T19:17:36.026 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 19:17:35 smithi173 bash[171506]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-node-exporter-b 2023-09-23T19:17:36.027 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 19:17:35 smithi173 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-09-23T19:17:36.027 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 19:17:35 smithi173 bash[171567]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-node-exporter-b 2023-09-23T19:17:36.027 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 19:17:35 smithi173 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-09-23T19:17:36.027 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Sep 23 19:17:35 smithi173 systemd[1]: Stopped Ceph node-exporter.b for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T19:17:36.526 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 19:17:36 smithi173 systemd[1]: Stopping Ceph grafana.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7... 2023-09-23T19:17:36.527 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 19:17:36 smithi173 bash[53604]: t=2023-09-23T19:17:36+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-09-23T19:17:37.276 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 19:17:37 smithi173 bash[171628]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-grafana-a 2023-09-23T19:17:37.277 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 19:17:37 smithi173 bash[171691]: Error response from daemon: No such container: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7-grafana-a 2023-09-23T19:17:37.277 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 19:17:37 smithi173 systemd[1]: ceph-756c96d4-5a3b-11ee-8db4-212e2dc638e7@grafana.a.service: Succeeded. 2023-09-23T19:17:37.277 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Sep 23 19:17:37 smithi173 systemd[1]: Stopped Ceph grafana.a for 756c96d4-5a3b-11ee-8db4-212e2dc638e7. 2023-09-23T19:17:42.917 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T19:17:42.934 DEBUG:teuthology.orchestra.run.smithi173:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T19:17:42.949 INFO:tasks.cephadm:Archiving crash dumps... 2023-09-23T19:17:42.951 DEBUG:teuthology.misc:Transferring archived files from smithi098:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401774/remote/smithi098/crash 2023-09-23T19:17:42.952 DEBUG:teuthology.orchestra.run.smithi098:> sudo tar c -f - -C /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/crash -- . 2023-09-23T19:17:43.037 INFO:teuthology.orchestra.run.smithi098.stderr:tar: /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/crash: Cannot open: No such file or directory 2023-09-23T19:17:43.037 INFO:teuthology.orchestra.run.smithi098.stderr:tar: Error is not recoverable: exiting now 2023-09-23T19:17:43.039 DEBUG:teuthology.misc:Transferring archived files from smithi173:/var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401774/remote/smithi173/crash 2023-09-23T19:17:43.040 DEBUG:teuthology.orchestra.run.smithi173:> sudo tar c -f - -C /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/crash -- . 2023-09-23T19:17:43.076 INFO:teuthology.orchestra.run.smithi173.stderr:tar: /var/lib/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/crash: Cannot open: No such file or directory 2023-09-23T19:17:43.076 INFO:teuthology.orchestra.run.smithi173.stderr:tar: Error is not recoverable: exiting now 2023-09-23T19:17:43.078 INFO:tasks.cephadm:Checking cluster log for badness... 2023-09-23T19:17:43.078 DEBUG:teuthology.orchestra.run.smithi098:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-09-23T19:17:43.093 INFO:teuthology.orchestra.run.smithi098.stderr:grep: /var/log/ceph/756c96d4-5a3b-11ee-8db4-212e2dc638e7/ceph.log: No such file or directory 2023-09-23T19:17:43.095 INFO:tasks.cephadm:Compressing logs... 2023-09-23T19:17:43.095 DEBUG:teuthology.orchestra.run.smithi098:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T19:17:43.140 DEBUG:teuthology.orchestra.run.smithi173:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T19:17:43.156 INFO:teuthology.orchestra.run.smithi173.stderr:find: '/var/log/rbd-target-api': No such file or directory 2023-09-23T19:17:43.170 INFO:teuthology.orchestra.run.smithi098.stderr:find: '/var/log/rbd-target-api': No such file or directory 2023-09-23T19:29:22.044 INFO:tasks.cephadm:Archiving logs... 2023-09-23T19:29:22.045 DEBUG:teuthology.misc:Transferring archived files from smithi098:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401774/remote/smithi098/log 2023-09-23T19:29:22.046 DEBUG:teuthology.orchestra.run.smithi098:> sudo tar c -f - -C /var/log/ceph -- . 2023-09-23T19:30:48.941 DEBUG:teuthology.misc:Transferring archived files from smithi173:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401774/remote/smithi173/log 2023-09-23T19:30:48.942 DEBUG:teuthology.orchestra.run.smithi173:> sudo tar c -f - -C /var/log/ceph -- . 2023-09-23T19:31:27.764 INFO:tasks.cephadm:Removing cluster... 2023-09-23T19:31:27.764 DEBUG:teuthology.orchestra.run.smithi098:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 --force 2023-09-23T19:31:30.229 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 756c96d4-5a3b-11ee-8db4-212e2dc638e7 --force 2023-09-23T19:31:32.085 INFO:tasks.cephadm:Removing cephadm ... 2023-09-23T19:31:32.085 DEBUG:teuthology.orchestra.run.smithi098:> rm -rf /home/ubuntu/cephtest/cephadm 2023-09-23T19:31:32.115 DEBUG:teuthology.orchestra.run.smithi173:> rm -rf /home/ubuntu/cephtest/cephadm 2023-09-23T19:31:32.131 INFO:tasks.cephadm:Teardown complete 2023-09-23T19:31:32.132 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-09-23T19:31:32.141 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-09-23T19:31:32.141 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-09-23T19:31:32.158 DEBUG:teuthology.orchestra.run.smithi173:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-09-23T19:31:32.241 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-09-23T19:31:32.250 INFO:teuthology.task.clock:Checking final clock skew... 2023-09-23T19:31:32.251 DEBUG:teuthology.orchestra.run.smithi098:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T19:31:32.253 DEBUG:teuthology.orchestra.run.smithi173:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T19:31:32.314 INFO:teuthology.orchestra.run.smithi098.stdout: remote refid st t when poll reach delay offset jitter 2023-09-23T19:31:32.315 INFO:teuthology.orchestra.run.smithi098.stdout:============================================================================== 2023-09-23T19:31:32.315 INFO:teuthology.orchestra.run.smithi098.stdout:*hv01.front.sepi 44.4.53.2 2 u 90 128 377 0.164 6.537 1.264 2023-09-23T19:31:32.315 INFO:teuthology.orchestra.run.smithi098.stdout:+hv02.front.sepi 63.231.80.2 3 u 36 128 377 0.147 4.691 0.949 2023-09-23T19:31:32.315 INFO:teuthology.orchestra.run.smithi098.stdout:+hv03.front.sepi 74.6.168.72 3 u 38 128 377 0.155 3.016 1.398 2023-09-23T19:31:32.315 INFO:teuthology.orchestra.run.smithi098.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 0.000 0.000 2023-09-23T19:31:32.317 INFO:teuthology.orchestra.run.smithi173.stdout: remote refid st t when poll reach delay offset jitter 2023-09-23T19:31:32.317 INFO:teuthology.orchestra.run.smithi173.stdout:============================================================================== 2023-09-23T19:31:32.317 INFO:teuthology.orchestra.run.smithi173.stdout:*hv01.front.sepi 44.4.53.2 2 u 5 256 377 0.193 0.771 1.314 2023-09-23T19:31:32.317 INFO:teuthology.orchestra.run.smithi173.stdout:+hv02.front.sepi 63.231.80.2 3 u 240 256 377 0.173 2.692 1.297 2023-09-23T19:31:32.317 INFO:teuthology.orchestra.run.smithi173.stdout:+hv03.front.sepi 74.6.168.72 3 u 222 256 377 0.153 -2.604 1.279 2023-09-23T19:31:32.317 INFO:teuthology.orchestra.run.smithi173.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 0.000 0.000 2023-09-23T19:31:32.318 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-09-23T19:31:32.326 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-09-23T19:31:32.327 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-09-23T19:31:32.353 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-09-23T19:31:32.386 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-09-23T19:31:32.419 INFO:teuthology.task.internal:Duration was 5755.665670 seconds 2023-09-23T19:31:32.419 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-09-23T19:31:32.453 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-09-23T19:31:32.453 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-09-23T19:31:32.455 DEBUG:teuthology.orchestra.run.smithi173:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-09-23T19:31:32.573 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-09-23T19:31:32.573 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi098.front.sepia.ceph.com 2023-09-23T19:31:32.573 DEBUG:teuthology.orchestra.run.smithi098:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-09-23T19:31:32.591 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi173.front.sepia.ceph.com 2023-09-23T19:31:32.591 DEBUG:teuthology.orchestra.run.smithi173:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-09-23T19:31:32.610 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-09-23T19:31:32.610 DEBUG:teuthology.orchestra.run.smithi098:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T19:31:32.638 DEBUG:teuthology.orchestra.run.smithi173:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T19:31:32.865 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-09-23T19:31:32.865 DEBUG:teuthology.orchestra.run.smithi098:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-09-23T19:31:32.867 DEBUG:teuthology.orchestra.run.smithi173:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-09-23T19:31:33.569 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-09-23T19:31:33.580 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-09-23T19:31:33.580 DEBUG:teuthology.orchestra.run.smithi098:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-09-23T19:31:33.630 DEBUG:teuthology.orchestra.run.smithi173:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-09-23T19:31:33.664 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-09-23T19:31:33.675 DEBUG:teuthology.orchestra.run.smithi098:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-09-23T19:31:33.677 DEBUG:teuthology.orchestra.run.smithi173:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-09-23T19:31:33.691 INFO:teuthology.orchestra.run.smithi098.stdout:kernel.core_pattern = core 2023-09-23T19:31:33.721 INFO:teuthology.orchestra.run.smithi173.stdout:kernel.core_pattern = core 2023-09-23T19:31:33.764 DEBUG:teuthology.orchestra.run.smithi098:> test -e /home/ubuntu/cephtest/archive/coredump 2023-09-23T19:31:33.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:33.770 DEBUG:teuthology.orchestra.run.smithi173:> test -e /home/ubuntu/cephtest/archive/coredump 2023-09-23T19:31:33.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:33.813 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-09-23T19:31:33.823 INFO:teuthology.task.internal:Transferring archived files... 2023-09-23T19:31:33.825 DEBUG:teuthology.misc:Transferring archived files from smithi098:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401774/remote/smithi098 2023-09-23T19:31:33.825 DEBUG:teuthology.orchestra.run.smithi098:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-09-23T19:31:33.878 DEBUG:teuthology.misc:Transferring archived files from smithi173:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401774/remote/smithi173 2023-09-23T19:31:33.878 DEBUG:teuthology.orchestra.run.smithi173:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-09-23T19:31:33.916 INFO:teuthology.task.internal:Removing archive directory... 2023-09-23T19:31:33.916 DEBUG:teuthology.orchestra.run.smithi098:> rm -rf -- /home/ubuntu/cephtest/archive 2023-09-23T19:31:33.918 DEBUG:teuthology.orchestra.run.smithi173:> rm -rf -- /home/ubuntu/cephtest/archive 2023-09-23T19:31:33.959 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-09-23T19:31:33.969 INFO:teuthology.task.internal:Not uploading archives. 2023-09-23T19:31:33.969 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-09-23T19:31:33.987 INFO:teuthology.task.internal:Tidying up after the test... 2023-09-23T19:31:33.987 DEBUG:teuthology.orchestra.run.smithi098:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-09-23T19:31:33.989 DEBUG:teuthology.orchestra.run.smithi173:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-09-23T19:31:33.995 INFO:teuthology.orchestra.run.smithi098.stdout: 1048601 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 23 19:31 /home/ubuntu/cephtest 2023-09-23T19:31:33.995 INFO:teuthology.orchestra.run.smithi098.stdout: 1195927 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Sep 23 19:09 /home/ubuntu/cephtest/mnt.0 2023-09-23T19:31:33.995 INFO:teuthology.orchestra.run.smithi098.stdout: 1195931 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Sep 23 19:09 /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T19:31:33.997 INFO:teuthology.orchestra.run.smithi098.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-09-23T19:31:34.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:31:34.001 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi098 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-09-23T19:31:34.007 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-09-23T19:31:34.018 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-09-23T19:31:34.054 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi098 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-09-23T19:31:34.054 INFO:teuthology.nuke:Checking targets against current locks 2023-09-23T19:31:34.069 INFO:teuthology.orchestra.run.smithi173.stdout: 1048601 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 23 19:31 /home/ubuntu/cephtest 2023-09-23T19:31:34.092 DEBUG:teuthology.nuke:shortname: smithi098 2023-09-23T19:31:34.092 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T19:31:34.116 DEBUG:teuthology.nuke:shortname: smithi173 2023-09-23T19:31:34.116 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T19:31:34.119 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi098.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401774', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 17:46:41.669238', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM='} 2023-09-23T19:31:34.137 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi173.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401774', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 17:46:41.671199', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM='} 2023-09-23T19:31:34.171 INFO:teuthology.orchestra.console.smithi098:Power off 2023-09-23T19:31:34.172 DEBUG:teuthology.orchestra.console.smithi098:pexpect command: ipmitool -H smithi098.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-09-23T19:31:34.186 INFO:teuthology.orchestra.console.smithi173:Power off 2023-09-23T19:31:34.186 DEBUG:teuthology.orchestra.console.smithi173:pexpect command: ipmitool -H smithi173.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-09-23T19:31:34.196 DEBUG:teuthology.orchestra.console.smithi098:power off output: Chassis Power Control: Down/Off 2023-09-23T19:31:34.197 DEBUG:teuthology.orchestra.console.smithi098:pexpect command: ipmitool -H smithi098.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T19:31:34.209 DEBUG:teuthology.orchestra.console.smithi173:power off output: Chassis Power Control: Down/Off 2023-09-23T19:31:34.209 DEBUG:teuthology.orchestra.console.smithi173:pexpect command: ipmitool -H smithi173.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T19:31:34.219 DEBUG:teuthology.orchestra.console.smithi098:check power output: Chassis Power is on 2023-09-23T19:31:34.230 DEBUG:teuthology.orchestra.console.smithi173:check power output: Chassis Power is on 2023-09-23T19:31:38.220 DEBUG:teuthology.orchestra.console.smithi098:pexpect command: ipmitool -H smithi098.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T19:31:38.232 DEBUG:teuthology.orchestra.console.smithi173:pexpect command: ipmitool -H smithi173.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T19:31:38.331 DEBUG:teuthology.orchestra.console.smithi098:check power output: Chassis Power is on 2023-09-23T19:31:38.343 DEBUG:teuthology.orchestra.console.smithi173:check power output: Chassis Power is on 2023-09-23T19:31:42.336 DEBUG:teuthology.orchestra.console.smithi098:pexpect command: ipmitool -H smithi098.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T19:31:42.345 DEBUG:teuthology.orchestra.console.smithi173:pexpect command: ipmitool -H smithi173.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T19:31:42.447 DEBUG:teuthology.orchestra.console.smithi098:check power output: Chassis Power is on 2023-09-23T19:31:42.454 DEBUG:teuthology.orchestra.console.smithi173:check power output: Chassis Power is on 2023-09-23T19:31:46.452 DEBUG:teuthology.orchestra.console.smithi098:pexpect command: ipmitool -H smithi098.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T19:31:46.461 DEBUG:teuthology.orchestra.console.smithi173:pexpect command: ipmitool -H smithi173.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T19:31:46.563 DEBUG:teuthology.orchestra.console.smithi098:check power output: Chassis Power is off 2023-09-23T19:31:46.570 DEBUG:teuthology.orchestra.console.smithi173:check power output: Chassis Power is off 2023-09-23T19:31:46.664 INFO:teuthology.orchestra.console.smithi098:Power off completed 2023-09-23T19:31:46.671 INFO:teuthology.orchestra.console.smithi173:Power off completed 2023-09-23T19:31:46.776 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} duration: 5755.665669679642 failure_reason: '' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=99c1727d9a2442d3a6f045f6e895770f status: fail success: false 2023-09-23T19:31:46.776 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-23T19:31:46.852 INFO:teuthology.run:FAIL